handle required fields

weboob/backends/youporn/backend.py

@@ -39,10 +39,10 @@ class YoupornBackend(BaseBackend, ICapVideo):
         return self.browser.get_video(_id)
 
     SORTBY = ['relevance', 'rating', 'views', 'time']
-    def iter_search_results(self, pattern=None, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False):
+    def iter_search_results(self, pattern=None, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False, required_fields=None):
         if not nsfw:
             return iter(set())
-        return self.browser.iter_search_results(pattern, self.SORTBY[sortby])
+        return self.browser.iter_search_results(pattern, self.SORTBY[sortby], required_fields=required_fields)
 
     def iter_page_urls(self, mozaic_url):
         raise NotImplementedError()

weboob/backends/youporn/browser.py

@@ -16,8 +16,11 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 
+import logging
+
 from weboob.tools.browser import BaseBrowser
 from weboob.tools.browser.decorators import id2url
+from weboob.tools.misc import iter_fields
 
 from .pages.index import IndexPage
 from .pages.video import VideoPage
@@ -36,20 +39,30 @@ class YoupornBrowser(BaseBrowser):
              'http://[w\.]*youporngay\.com:80/watch/.+': VideoPage,
             }
 
-    def __init__(self):
-        # Disallow arguments
-        BaseBrowser.__init__(self)
+    @id2url(YoupornVideo.id2url)
+    def get_video(self, url, video=None):
+        self.location(url)
+        if video is None:
+            return video.page.video
+        else:
+            for k, v in iter_fields(self.page.video):
+                if v and getattr(video, k) != v:
+                    setattr(video, k, v)
+            return video
 
-    def iter_search_results(self, pattern, sortby):
+    def iter_search_results(self, pattern, sortby, required_fields=None):
         if not pattern:
             self.home()
         else:
             self.location(self.buildurl('/search/%s' % sortby, query=pattern))
-
         assert self.is_on_page(IndexPage)
-        return self.page.iter_videos()
-
-    @id2url(YoupornVideo.id2url)
-    def get_video(self, url):
-        self.location(url)
-        return self.page.video
+        for video in self.page.iter_videos():
+            if required_fields is not None:
+                missing_required_fields = set(required_fields) - set(k for k, v in iter_fields(video) if v)
+                if missing_required_fields:
+                    logging.debug(u'Completing missing required fields: %s' % missing_required_fields)
+                    self.get_video(video.id, video=video)
+                    missing_required_fields = set(required_fields) - set(k for k, v in iter_fields(video) if v)
+                    if missing_required_fields:
+                        raise Exception(u'Could not load all required fields. Missing: %s' % missing_required_fields)
+            yield video