Move the "empty search for latest" to collections

There is now a "latest" collection (or "latest_nsfw").
The feature didn't look much used, since it didn't work on many
backends.
Using collections will make it easy to support other things
like most viewed, featured, etc.
As a bonus, I added tests for every backend with the feature.

modules/arte/backend.py

@@ -20,7 +20,8 @@
 
 from __future__ import with_statement
 
-from weboob.capabilities.video import ICapVideo
+from weboob.capabilities.video import ICapVideo, BaseVideo
+from weboob.capabilities.collection import ICapCollection, CollectionNotFound
 from weboob.tools.backend import BaseBackend, BackendConfig
 from weboob.tools.value import Value
 
@@ -31,7 +32,7 @@ from .video import ArteVideo
 __all__ = ['ArteBackend']
 
 
-class ArteBackend(BaseBackend, ICapVideo):
+class ArteBackend(BaseBackend, ICapVideo, ICapCollection):
     NAME = 'arte'
     MAINTAINER = 'Romain Bignon'
     EMAIL = 'romain@weboob.org'
@@ -50,7 +51,7 @@ class ArteBackend(BaseBackend, ICapVideo):
         with self.browser:
             return self.browser.get_video(_id)
 
-    def search_videos(self, pattern=None, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False, max_results=None):
+    def search_videos(self, pattern, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False, max_results=None):
         with self.browser:
             return self.browser.search_videos(pattern)
 
@@ -65,4 +66,21 @@ class ArteBackend(BaseBackend, ICapVideo):
 
         return video
 
+    def iter_resources(self, objs, split_path):
+        if BaseVideo in objs:
+            collection = self.get_collection(objs, split_path)
+            if collection.path_level == 0:
+                yield self.get_collection(objs, [u'latest'])
+            if collection.split_path == [u'latest']:
+                for video in self.browser.latest_videos():
+                    yield video
+
+    def validate_collection(self, objs, collection):
+        if collection.path_level == 0:
+            return
+        if BaseVideo in objs and collection.split_path == [u'latest']:
+            collection.title = u'Latest Arte videos'
+            return
+        raise CollectionNotFound(collection.split_path)
+
     OBJECTS = {ArteVideo: fill_video}

modules/arte/browser.py

@@ -52,9 +52,11 @@ class ArteBrowser(BaseBrowser):
         self.location('http://videos.arte.tv/fr/videos/toutesLesVideos')
 
     def search_videos(self, pattern):
-        if not pattern:
-            self.home()
-        else:
         self.location(self.buildurl('/%s/do_search/videos/%s' % (self.lang, self.SEARCH_LANG[self.lang]), q=pattern.encode('utf-8')))
         assert self.is_on_page(IndexPage)
         return self.page.iter_videos()
+
+    def latest_videos(self):
+        self.home()
+        assert self.is_on_page(IndexPage)
+        return self.page.iter_videos()

modules/arte/test.py

@@ -19,13 +19,22 @@
 
 
 from weboob.tools.test import BackendTest
+from weboob.capabilities.video import BaseVideo
+
 
 class ArteTest(BackendTest):
     BACKEND = 'arte'
 
-    def test_arte(self):
+    def test_search(self):
         l = list(self.backend.search_videos('arte'))
         if len(l) > 0:
             v = l[0]
             self.backend.fillobj(v, ('url',))
             self.assertTrue(v.url and v.url.startswith('rtmp://'), 'URL for video "%s" not found: %s' % (v.id, v.url))
+
+    def test_latest(self):
+        l = list(self.backend.iter_resources([BaseVideo], [u'latest']))
+        assert len(l)
+        v = l[0]
+        self.backend.fillobj(v, ('url',))
+        self.assertTrue(v.url and v.url.startswith('rtmp://'), 'URL for video "%s" not found: %s' % (v.id, v.url))

modules/canalplus/backend.py

@@ -48,7 +48,7 @@ class CanalplusBackend(BaseBackend, ICapVideo, ICapCollection):
     def create_default_browser(self):
         return self.create_browser(quality=self.config['quality'].get())
 
-    def search_videos(self, pattern=None, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False, max_results=None):
+    def search_videos(self, pattern, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False, max_results=None):
         with self.browser:
             return self.browser.search_videos(pattern)
 

modules/cappedtv/backend.py

@@ -10,7 +10,8 @@
 
 from __future__ import with_statement
 
-from weboob.capabilities.video import ICapVideo
+from weboob.capabilities.video import ICapVideo, BaseVideo
+from weboob.capabilities.collection import ICapCollection, CollectionNotFound
 from weboob.tools.backend import BaseBackend
 from .browser import CappedBrowser, CappedVideo
 
@@ -18,7 +19,7 @@ from .browser import CappedBrowser, CappedVideo
 __all__ = ['CappedBackend']
 
 
-class CappedBackend(BaseBackend, ICapVideo):
+class CappedBackend(BaseBackend, ICapVideo, ICapCollection):
     NAME = 'cappedtv'
     MAINTAINER = 'Lord'
     EMAIL = 'lord@lordtoniok.com'
@@ -31,7 +32,7 @@ class CappedBackend(BaseBackend, ICapVideo):
         with self.browser:
             return self.browser.get_video(_id)
 
-    def search_videos(self, pattern=None, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=None, max_results=None):
+    def search_videos(self, pattern, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=None, max_results=None):
         with self.browser:
             return self.browser.search_videos(pattern)
 
@@ -45,4 +46,21 @@ class CappedBackend(BaseBackend, ICapVideo):
 
         return video
 
+    def iter_resources(self, objs, split_path):
+        if BaseVideo in objs:
+            collection = self.get_collection(objs, split_path)
+            if collection.path_level == 0:
+                yield self.get_collection(objs, [u'latest'])
+            if collection.split_path == [u'latest']:
+                for video in self.browser.latest_videos():
+                    yield video
+
+    def validate_collection(self, objs, collection):
+        if collection.path_level == 0:
+            return
+        if BaseVideo in objs and collection.split_path == [u'latest']:
+            collection.title = u'Latest CappedTV videos'
+            return
+        raise CollectionNotFound(collection.split_path)
+
     OBJECTS = {CappedVideo: fill_video}

modules/cappedtv/browser.py

@@ -78,7 +78,6 @@ class IndexPage(BasePage):
             yield video
 
 
-
 # parser for the video page
 class VideoPage(BasePage):
     def get_video(self, video=None):
@@ -119,9 +118,11 @@ class CappedBrowser(BaseBrowser):
         return self.page.get_video(video)
 
     def search_videos(self, pattern):
-        if not pattern:
-            self.home()
-        else:
         self.location('/search?s=%s' % (urllib.quote_plus(pattern.encode('utf-8'))))
         assert self.is_on_page(IndexPage)
         return self.page.iter_videos()
+
+    def latest_videos(self):
+        self.home()
+        assert self.is_on_page(IndexPage)
+        return self.page.iter_videos()

modules/cappedtv/test.py

@@ -10,18 +10,23 @@
 
 
 from weboob.tools.test import BackendTest
-
-
-__all__ = ['CappedTest']
+from weboob.capabilities.video import BaseVideo
 
 
 class CappedTest(BackendTest):
     BACKEND = 'cappedtv'
 
-    def test_capped(self):
+    def test_search(self):
         l = list(self.backend.search_videos('kewlers'))
         self.assertTrue(len(l) > 0)
         v = l[0]
         self.backend.fillobj(v, ('url',))
         self.assertTrue(v.url and v.url.startswith('http://'), 'URL for video "%s" not found: %s' % (v.id, v.url))
         self.backend.browser.openurl(v.url)
+
+    def test_latest(self):
+        l = list(self.backend.iter_resources([BaseVideo], [u'latest']))
+        self.assertTrue(len(l) > 0)
+        v = l[0]
+        self.backend.fillobj(v, ('url',))
+        self.assertTrue(v.url and v.url.startswith('http://'), 'URL for video "%s" not found: %s' % (v.id, v.url))

modules/dailymotion/backend.py

@@ -20,7 +20,8 @@
 
 from __future__ import with_statement
 
-from weboob.capabilities.video import ICapVideo
+from weboob.capabilities.video import ICapVideo, BaseVideo
+from weboob.capabilities.collection import ICapCollection, CollectionNotFound
 from weboob.tools.backend import BaseBackend
 
 from .browser import DailymotionBrowser
@@ -30,7 +31,7 @@ from .video import DailymotionVideo
 __all__ = ['DailymotionBackend']
 
 
-class DailymotionBackend(BaseBackend, ICapVideo):
+class DailymotionBackend(BaseBackend, ICapVideo, ICapCollection):
     NAME = 'dailymotion'
     MAINTAINER = 'Romain Bignon'
     EMAIL = 'romain@weboob.org'
@@ -44,7 +45,7 @@ class DailymotionBackend(BaseBackend, ICapVideo):
             return self.browser.get_video(_id)
 
     SORTBY = ['relevance', 'rated', 'visited', None]
-    def search_videos(self, pattern=None, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False, max_results=None):
+    def search_videos(self, pattern, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False, max_results=None):
         with self.browser:
             return self.browser.search_videos(pattern, self.SORTBY[sortby])
 
@@ -59,4 +60,21 @@ class DailymotionBackend(BaseBackend, ICapVideo):
 
         return video
 
+    def iter_resources(self, objs, split_path):
+        if BaseVideo in objs:
+            collection = self.get_collection(objs, split_path)
+            if collection.path_level == 0:
+                yield self.get_collection(objs, [u'latest'])
+            if collection.split_path == [u'latest']:
+                for video in self.browser.latest_videos():
+                    yield video
+
+    def validate_collection(self, objs, collection):
+        if collection.path_level == 0:
+            return
+        if BaseVideo in objs and collection.split_path == [u'latest']:
+            collection.title = u'Latest Dailymotion videos'
+            return
+        raise CollectionNotFound(collection.split_path)
+
     OBJECTS = {DailymotionVideo: fill_video}

modules/dailymotion/browser.py

@@ -47,9 +47,6 @@ class DailymotionBrowser(BaseBrowser):
         self.location('/1')
 
     def search_videos(self, pattern, sortby):
-        if not pattern:
-            self.home()
-        else:
         pattern = pattern.replace('/', '').encode('utf-8')
         if sortby is None:
             url = '/search/%s/1' % quote_plus(pattern)
@@ -59,3 +56,9 @@ class DailymotionBrowser(BaseBrowser):
 
         assert self.is_on_page(IndexPage)
         return self.page.iter_videos()
+
+    def latest_videos(self):
+        self.home()
+
+        assert self.is_on_page(IndexPage)
+        return self.page.iter_videos()

modules/dailymotion/test.py

@@ -19,14 +19,23 @@
 
 
 from weboob.tools.test import BackendTest
+from weboob.capabilities.video import BaseVideo
+
 
 class DailymotionTest(BackendTest):
     BACKEND = 'dailymotion'
 
-    def test_dailymotion(self):
+    def test_search(self):
         l = list(self.backend.search_videos('chirac'))
         self.assertTrue(len(l) > 0)
         v = l[0]
         self.backend.fillobj(v, ('url',))
         self.assertTrue(v.url and v.url.startswith('http://'), 'URL for video "%s" not found: %s' % (v.id, v.url))
         self.backend.browser.openurl(v.url)
+
+    def test_latest(self):
+        l = list(self.backend.iter_resources([BaseVideo], [u'latest']))
+        assert len(l)
+        v = l[0]
+        self.backend.fillobj(v, ('url',))
+        self.assertTrue(v.url and v.url.startswith('http://'), 'URL for video "%s" not found: %s' % (v.id, v.url))

modules/ehentai/backend.py

@@ -20,7 +20,8 @@
 from __future__ import with_statement
 
 import re
-from weboob.capabilities.gallery import ICapGallery
+from weboob.capabilities.gallery import ICapGallery, BaseGallery
+from weboob.capabilities.collection import ICapCollection, CollectionNotFound
 from weboob.tools.backend import BaseBackend, BackendConfig
 from weboob.tools.misc import ratelimit
 from weboob.tools.value import Value, ValueBackendPassword
@@ -32,7 +33,7 @@ from .gallery import EHentaiGallery, EHentaiImage
 __all__ = ['EHentaiBackend']
 
 
-class EHentaiBackend(BaseBackend, ICapGallery):
+class EHentaiBackend(BaseBackend, ICapGallery, ICapCollection):
     NAME = 'ehentai'
     MAINTAINER = 'Roger Philibert'
     EMAIL = 'roger.philibert@gmail.com'
@@ -53,7 +54,7 @@ class EHentaiBackend(BaseBackend, ICapGallery):
             password = None
         return self.create_browser(self.config['domain'].get(), username, password)
 
-    def search_gallery(self, pattern=None, sortby=None, max_results=None):
+    def search_gallery(self, pattern, sortby=None, max_results=None):
         with self.browser:
             return self.browser.search_gallery(pattern)
 
@@ -94,6 +95,23 @@ class EHentaiBackend(BaseBackend, ICapGallery):
                 ratelimit("ehentai_get", 2)
                 image.data = self.browser.readurl(image.url)
 
+    def iter_resources(self, objs, split_path):
+        if BaseGallery in objs:
+            collection = self.get_collection(objs, split_path)
+            if collection.path_level == 0:
+                yield self.get_collection(objs, [u'latest_nsfw'])
+            if collection.split_path == [u'latest_nsfw']:
+                for gallery in self.browser.latest_gallery():
+                    yield gallery
+
+    def validate_collection(self, objs, collection):
+        if collection.path_level == 0:
+            return
+        if BaseGallery in objs and collection.split_path == [u'latest_nsfw']:
+            collection.title = u'Latest E-Hentai galleries (NSFW)'
+            return
+        raise CollectionNotFound(collection.split_path)
+
     OBJECTS = {
             EHentaiGallery: fill_gallery,
             EHentaiImage: fill_image}

modules/ehentai/browser.py

@@ -56,6 +56,11 @@ class EHentaiBrowser(BaseBrowser):
         assert self.is_on_page(IndexPage)
         return self.page.iter_galleries()
 
+    def latest_gallery(self):
+        self.location('/')
+        assert self.is_on_page(IndexPage)
+        return self.page.iter_galleries()
+
     def iter_gallery_images(self, gallery):
         self.location(gallery.url)
         assert self.is_on_page(GalleryPage)

modules/ehentai/test.py

@@ -19,12 +19,13 @@
 
 
 from weboob.tools.test import BackendTest
+from weboob.capabilities.gallery import BaseGallery
 
 
 class EHentaiTest(BackendTest):
     BACKEND = 'ehentai'
 
-    def test_ehentai(self):
+    def test_search(self):
         l = list(self.backend.search_gallery('lol'))
         self.assertTrue(len(l) > 0)
         v = l[0]
@@ -36,3 +37,7 @@ class EHentaiTest(BackendTest):
         self.backend.fillobj(img, ('url',))
         self.assertTrue(v.url and v.url.startswith('http://'), 'URL for first image in gallery "%s" not found: %s' % (v.id, img.url))
         self.backend.browser.openurl(img.url)
+
+    def test_latest(self):
+        l = list(self.backend.iter_resources([BaseGallery], [u'latest_nsfw']))
+        assert len(l) > 0

modules/francetelevisions/backend.py

@@ -20,7 +20,8 @@
 
 from __future__ import with_statement
 
-from weboob.capabilities.video import ICapVideo
+from weboob.capabilities.video import ICapVideo, BaseVideo
+from weboob.capabilities.collection import ICapCollection, CollectionNotFound
 from weboob.tools.backend import BaseBackend
 
 from .browser import PluzzBrowser
@@ -30,7 +31,7 @@ from .video import PluzzVideo
 __all__ = ['PluzzBackend']
 
 
-class PluzzBackend(BaseBackend, ICapVideo):
+class PluzzBackend(BaseBackend, ICapVideo, ICapCollection):
     NAME = 'francetelevisions'
     MAINTAINER = 'Romain Bignon'
     EMAIL = 'romain@weboob.org'
@@ -43,7 +44,7 @@ class PluzzBackend(BaseBackend, ICapVideo):
         with self.browser:
             return self.browser.get_video(_id)
 
-    def search_videos(self, pattern=None, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False, max_results=None):
+    def search_videos(self, pattern, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False, max_results=None):
         with self.browser:
             return self.browser.search_videos(pattern)
 
@@ -58,4 +59,21 @@ class PluzzBackend(BaseBackend, ICapVideo):
 
         return video
 
+    def iter_resources(self, objs, split_path):
+        if BaseVideo in objs:
+            collection = self.get_collection(objs, split_path)
+            if collection.path_level == 0:
+                yield self.get_collection(objs, [u'latest'])
+            if collection.split_path == [u'latest']:
+                for video in self.browser.latest_videos():
+                    yield video
+
+    def validate_collection(self, objs, collection):
+        if collection.path_level == 0:
+            return
+        if BaseVideo in objs and collection.split_path == [u'latest']:
+            collection.title = u'Latest France Télévisions videos'
+            return
+        raise CollectionNotFound(collection.split_path)
+
     OBJECTS = {PluzzVideo: fill_video}

modules/francetelevisions/browser.py

@@ -59,14 +59,17 @@ class PluzzBrowser(BaseBrowser):
         self.location('/replay/1')
 
     def search_videos(self, pattern):
-        if not pattern:
-            self.home()
-        else:
         self.location(self.buildurl('/recherche.html', q=pattern.encode('utf-8')))
 
         assert self.is_on_page(IndexPage)
         return self.page.iter_videos()
 
+    def latest_videos(self):
+        self.home()
+
+        assert self.is_on_page(IndexPage)
+        return self.page.iter_videos()
+
     def parse_info(self, data, video):
         parser = etree.XMLParser(encoding='utf-8')
         root = etree.XML(data, parser)

modules/francetelevisions/test.py

@@ -19,14 +19,22 @@
 
 
 from weboob.tools.test import BackendTest
+from weboob.capabilities.video import BaseVideo
 
 class PluzzTest(BackendTest):
     BACKEND = 'francetelevisions'
 
-    def test_francetelevisions(self):
+    def test_search(self):
         # If the test fails, it might be good news!
         l = list(self.backend.search_videos('Plus belle la vie'))
         self.assertTrue(len(l) > 0)
         v = l[0]
         self.backend.fillobj(v, ('url',))
         self.assertTrue(v.url and v.url.startswith('mms://'), 'URL for video "%s" not found: %s' % (v.id, v.url))
+
+    def test_latest(self):
+        l = list(self.backend.iter_resources([BaseVideo], [u'latest']))
+        assert len(l)
+        v = l[0]
+        self.backend.fillobj(v, ('url',))
+        assert v.url

modules/ina/backend.py

@@ -42,7 +42,7 @@ class InaBackend(BaseBackend, ICapVideo):
     def get_video(self, _id):
         return self.browser.get_video(_id)
 
-    def search_videos(self, pattern=None, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False, max_results=None):
+    def search_videos(self, pattern, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False, max_results=None):
         with self.browser:
             return self.browser.search_videos(pattern)
 

modules/nolifetv/backend.py

@@ -20,7 +20,8 @@
 
 from __future__ import with_statement
 
-from weboob.capabilities.video import ICapVideo
+from weboob.capabilities.video import ICapVideo, BaseVideo
+from weboob.capabilities.collection import ICapCollection, CollectionNotFound
 from weboob.tools.value import Value, ValueBackendPassword
 from weboob.tools.backend import BaseBackend, BackendConfig
 
@@ -31,7 +32,7 @@ from .video import NolifeTVVideo
 __all__ = ['NolifeTVBackend']
 
 
-class NolifeTVBackend(BaseBackend, ICapVideo):
+class NolifeTVBackend(BaseBackend, ICapVideo, ICapCollection):
     NAME = 'nolifetv'
     MAINTAINER = 'Romain Bignon'
     EMAIL = 'romain@weboob.org'
@@ -55,7 +56,7 @@ class NolifeTVBackend(BaseBackend, ICapVideo):
             video = self.browser.get_video(_id)
         return video
 
-    def search_videos(self, pattern=None, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False, max_results=None):
+    def search_videos(self, pattern, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False, max_results=None):
         with self.browser:
             return self.browser.search_videos(pattern)
 
@@ -70,4 +71,21 @@ class NolifeTVBackend(BaseBackend, ICapVideo):
 
         return video
 
+    def iter_resources(self, objs, split_path):
+        if BaseVideo in objs:
+            collection = self.get_collection(objs, split_path)
+            if collection.path_level == 0:
+                yield self.get_collection(objs, [u'latest'])
+            if collection.split_path == [u'latest']:
+                for video in self.browser.latest_videos():
+                    yield video
+
+    def validate_collection(self, objs, collection):
+        if collection.path_level == 0:
+            return
+        if BaseVideo in objs and collection.split_path == [u'latest']:
+            collection.title = u'Latest NoLiveTV videos'
+            return
+        raise CollectionNotFound(collection.split_path)
+
     OBJECTS = {NolifeTVVideo: fill_video}

modules/nolifetv/browser.py

@@ -70,9 +70,11 @@ class NolifeTVBrowser(BaseBrowser):
         return self.page.get_video(video)
 
     def search_videos(self, pattern):
-        if not pattern:
-            self.home()
-        else:
         self.location('/index.php?', 'search=%s' % urllib.quote_plus(pattern.encode('utf-8')))
         assert self.is_on_page(IndexPage)
         return self.page.iter_videos()
+
+    def latest_videos(self):
+        self.home()
+        assert self.is_on_page(IndexPage)
+        return self.page.iter_videos()

modules/nolifetv/test.py

@@ -19,12 +19,15 @@
 
 
 from weboob.tools.test import BackendTest
+from weboob.capabilities.video import BaseVideo
+
 from .pages.video import ForbiddenVideo
 
+
 class NolifeTVTest(BackendTest):
     BACKEND = 'nolifetv'
 
-    def test_nolife(self):
+    def test_search(self):
         l = list(self.backend.search_videos('nolife'))
         self.assertTrue(len(l) > 0)
         for v in l:
@@ -35,3 +38,15 @@ class NolifeTVTest(BackendTest):
             else:
                 self.assertTrue(v.url and v.url.startswith('http://'), 'URL for video "%s" not found: %s' % (v.id, v.url))
                 break
+
+    def test_latest(self):
+        l = list(self.backend.iter_resources([BaseVideo], [u'latest']))
+        assert len(l) > 0
+        for v in l:
+            try:
+                self.backend.fillobj(v, ('url',))
+            except ForbiddenVideo:
+                continue
+            else:
+                self.assertTrue(v.url and v.url.startswith('http://'), 'URL for video "%s" not found: %s' % (v.id, v.url))
+                break