new API for the fillobj feature and severaaal fixes

weboob/backends/youjizz/backend.py

@@ -20,6 +20,7 @@ from weboob.capabilities.video import ICapVideo
 from weboob.tools.backend import BaseBackend
 
 from .browser import YoujizzBrowser
+from .video import YoujizzVideo
 
 
 __all__ = ['YoujizzBackend']
@@ -46,3 +47,9 @@ class YoujizzBackend(BaseBackend, ICapVideo):
         if not nsfw:
             return
         return self.browser.iter_search_results(pattern)
+
+    def fill_video(self, video, fields):
+        # ignore the fields param: VideoPage.get_video() returns all the information
+        return self.browser.get_video(YoujizzVideo.id2url(video.id), video)
+
+    OBJECTS = {YoujizzVideo: fill_video}

weboob/backends/youjizz/browser.py

@@ -16,13 +16,10 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 
-import logging
-import re
 import urllib
 
-from weboob.tools.browser import BaseBrowser, BrowserUnavailable
+from weboob.tools.browser import BaseBrowser
 from weboob.tools.browser.decorators import check_domain, id2url
-from weboob.tools.misc import iter_fields, to_unicode
 
 from .pages.index import IndexPage
 from .pages.video import VideoPage
@@ -41,15 +38,10 @@ class YoujizzBrowser(BaseBrowser):
              r'http://.*youjizz\.com/videos/(?P<id>.+)\.html': VideoPage,
             }
 
-    def fillobj(self, video, fields):
-        # ignore the fields param: VideoPage.get_video() returns all the information
-        self.location(YoujizzVideo.id2url(video.id))
-        return self.page.get_video(video)
-
     @id2url(YoujizzVideo.id2url)
-    def get_video(self, url):
+    def get_video(self, url, video=None):
         self.location(url)
-        return self.page.get_video()
+        return self.page.get_video(video)
 
     @check_domain
     def iter_page_urls(self, mozaic_url):

weboob/backends/youjizz/pages/index.py

@@ -43,7 +43,14 @@ class IndexPage(BasePage):
             title = title_el.text.strip()
 
             time_span = select(span, 'span.thumbtime span', 1)
-            minutes, seconds = (int(v) for v in time_span.text.strip().split(':'))
+            time_txt = time_span.text.strip()
+            if time_txt == 'N/A':
+                minutes, seconds = 0, 0
+            elif ':' in time_txt:
+                minutes, seconds = (int(v) for v in time_txt.split(':'))
+            else:
+                raise SelectElementException('Unable to parse the video duration: %s' % time_txt)
+
 
             yield YoujizzVideo(_id,
                                title=title,

weboob/backends/youporn/backend.py

@@ -20,6 +20,7 @@ from weboob.capabilities.video import ICapVideo
 from weboob.tools.backend import BaseBackend
 
 from .browser import YoupornBrowser
+from .video import YoupornVideo
 
 
 __all__ = ['YoupornBackend']
@@ -46,3 +47,9 @@ class YoupornBackend(BaseBackend, ICapVideo):
 
     def iter_page_urls(self, mozaic_url):
         raise NotImplementedError()
+
+    def fill_video(self, video, fields):
+        # ignore the fields param: VideoPage.get_video() returns all the information
+        return self.browser.get_video(YoupornVideo.id2url(video.id), video)
+
+    OBJECTS = {YoupornVideo: fill_video}

weboob/backends/youporn/browser.py

@@ -16,11 +16,8 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 
-import logging
-
 from weboob.tools.browser import BaseBrowser
 from weboob.tools.browser.decorators import id2url
-from weboob.tools.misc import iter_fields
 
 from .pages.index import IndexPage
 from .pages.video import VideoPage
@@ -39,15 +36,10 @@ class YoupornBrowser(BaseBrowser):
              r'http://[w\.]*youporngay\.com:80/watch/(?P<id>.+)': VideoPage,
             }
 
-    def fillobj(self, video, fields):
-        # ignore the fields param: VideoPage.get_video() returns all the information
-        self.location(YoupornVideo.id2url(video.id))
-        return self.page.get_video(video)
-
     @id2url(YoupornVideo.id2url)
-    def get_video(self, url):
+    def get_video(self, url, video=None):
         self.location(url)
-        return self.page.get_video()
+        return self.page.get_video(video)
 
     def iter_search_results(self, pattern, sortby):
         if not pattern:

weboob/backends/youporn/pages/video.py

@@ -18,7 +18,6 @@
 
 import re
 import datetime
-from logging import warning
 
 from weboob.tools.parsers.lxmlparser import select
 

weboob/backends/youtube/backend.py

@@ -17,11 +17,9 @@
 
 
 import datetime
-import logging
 
 from weboob.capabilities.video import ICapVideo
 from weboob.tools.backend import BaseBackend
-from weboob.tools.misc import iter_fields
 
 from .browser import YoutubeBrowser
 from .video import YoutubeVideo
@@ -67,3 +65,9 @@ class YoutubeBackend(BaseBackend, ICapVideo):
 
     def iter_page_urls(self, mozaic_url):
         raise NotImplementedError()
+
+    def fill_video(self, video, fields):
+        # ignore the fields param: VideoPage.get_video() returns all the information
+        return self.browser.get_video(YoutubeVideo.id2url(video.id), video)
+
+    OBJECTS = {YoutubeVideo: fill_video}

weboob/backends/youtube/browser.py

@@ -16,8 +16,6 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 
-import logging
-
 from weboob.tools.browser import BaseBrowser
 from weboob.tools.browser.decorators import id2url
 
@@ -36,12 +34,7 @@ class YoutubeBrowser(BaseBrowser):
              r'.*youtube\.com/verify_age\?next_url=(?P<next_url>.+)': VerifyAgePage,
             }
 
-    def fillobj(self, video, fields):
-        # ignore the fields param: VideoPage.get_video() returns all the information
-        self.location(YoutubeVideo.id2url(video.id))
-        return self.page.get_video(video)
-
     @id2url(YoutubeVideo.id2url)
-    def get_video(self, url):
+    def get_video(self, url, video=None):
         self.location(url)
-        return self.page.get_video()
+        return self.page.get_video(video)

weboob/backends/youtube/pages.py

@@ -24,7 +24,7 @@ from weboob.tools.parsers.lxmlparser import select
 from .video import YoutubeVideo
 
 
-__all__ = ['ForbiddenVideoPage', 'VerifyAgePage', 'VideoPage']
+__all__ = ['ForbiddenVideo', 'ForbiddenVideoPage', 'VerifyAgePage', 'VideoPage']
 
 
 class ForbiddenVideo(Exception):

weboob/tools/application/console.py

@@ -25,6 +25,7 @@ import sys
 
 from weboob.core import CallErrors
 from weboob.core.backends import BackendsConfig
+from weboob.tools.backend import NotSupportedObject
 
 from .base import BackendNotFound, BaseApplication
 from .formatters.load import formatters, load_formatter
@@ -277,10 +278,11 @@ class ConsoleApplication(BaseApplication):
         for i, (backend, result) in enumerate(self.weboob.do(function, *args, **kwargs)):
             if self.options.count and i == self.options.count:
                 break
-            fields = set(self.selected_fields) - set('*')
+            if self.selected_fields:
-            if fields:
+                fields = set(self.selected_fields) - set('*')
-                try:
+                if fields:
-                    backend.browser.fillobj(result, fields)
+                    try:
-                except Exception, e:
+                        backend.fillobj(result, fields)
-                    logging.warning(u'Could not retrieve required fields (%s): %s' % (','.join(fields), e))
+                    except NotSupportedObject, e:
+                        logging.warning(u'Could not retrieve required fields (%s): %s' % (','.join(fields), e))
             yield backend, result

weboob/tools/backend.py

@@ -21,9 +21,11 @@ import os
 from threading import RLock
 
 
-__all__ = ['BaseBackend']
+__all__ = ['BaseBackend', 'NotSupportedObject']
 
 
+class NotSupportedObject(Exception): pass
+
 class BackendStorage(object):
     def __init__(self, name, storage):
         self.name = name
@@ -72,6 +74,12 @@ class BaseBackend(object):
     BROWSER = None
     # Test class
     TEST = None
+    # Supported objects to fill
+    # The key is the class and the value the method to call to fill
+    # Method prototype: method(object, fields)
+    # When the method is called, fields are only the one which are
+    # NOT yet filled.
+    OBJECTS = {}
 
     class ConfigField(object):
         def __init__(self, default=None, is_masked=False, regexp=None, description=None):
@@ -165,3 +173,10 @@ class BaseBackend(object):
         if not self.TEST:
             return None
         return self.TEST(self)
+
+    def fillobj(self, obj, fields):
+        for key, value in self.OBJECTS.iteritems():
+            if isinstance(obj, key):
+                return value(self, obj, fields)
+
+        raise NotSupportedObject('The object of type %s is not supported by this backend' % type(obj))