diff --git a/weboob/backends/youjizz/backend.py b/weboob/backends/youjizz/backend.py index 07c1e04a..8aa4797c 100644 --- a/weboob/backends/youjizz/backend.py +++ b/weboob/backends/youjizz/backend.py @@ -20,6 +20,7 @@ from weboob.capabilities.video import ICapVideo from weboob.tools.backend import BaseBackend from .browser import YoujizzBrowser +from .video import YoujizzVideo __all__ = ['YoujizzBackend'] @@ -46,3 +47,9 @@ class YoujizzBackend(BaseBackend, ICapVideo): if not nsfw: return return self.browser.iter_search_results(pattern) + + def fill_video(self, video, fields): + # ignore the fields param: VideoPage.get_video() returns all the information + return self.browser.get_video(YoujizzVideo.id2url(video.id), video) + + OBJECTS = {YoujizzVideo: fill_video} diff --git a/weboob/backends/youjizz/browser.py b/weboob/backends/youjizz/browser.py index d3520748..9f3d2c90 100644 --- a/weboob/backends/youjizz/browser.py +++ b/weboob/backends/youjizz/browser.py @@ -16,13 +16,10 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -import logging -import re import urllib -from weboob.tools.browser import BaseBrowser, BrowserUnavailable +from weboob.tools.browser import BaseBrowser from weboob.tools.browser.decorators import check_domain, id2url -from weboob.tools.misc import iter_fields, to_unicode from .pages.index import IndexPage from .pages.video import VideoPage @@ -41,15 +38,10 @@ class YoujizzBrowser(BaseBrowser): r'http://.*youjizz\.com/videos/(?P.+)\.html': VideoPage, } - def fillobj(self, video, fields): - # ignore the fields param: VideoPage.get_video() returns all the information - self.location(YoujizzVideo.id2url(video.id)) - return self.page.get_video(video) - @id2url(YoujizzVideo.id2url) - def get_video(self, url): + def get_video(self, url, video=None): self.location(url) - return self.page.get_video() + return self.page.get_video(video) @check_domain def iter_page_urls(self, mozaic_url): diff --git a/weboob/backends/youjizz/pages/index.py b/weboob/backends/youjizz/pages/index.py index d514acd7..b782f51c 100644 --- a/weboob/backends/youjizz/pages/index.py +++ b/weboob/backends/youjizz/pages/index.py @@ -43,7 +43,14 @@ class IndexPage(BasePage): title = title_el.text.strip() time_span = select(span, 'span.thumbtime span', 1) - minutes, seconds = (int(v) for v in time_span.text.strip().split(':')) + time_txt = time_span.text.strip() + if time_txt == 'N/A': + minutes, seconds = 0, 0 + elif ':' in time_txt: + minutes, seconds = (int(v) for v in time_txt.split(':')) + else: + raise SelectElementException('Unable to parse the video duration: %s' % time_txt) + yield YoujizzVideo(_id, title=title, diff --git a/weboob/backends/youporn/backend.py b/weboob/backends/youporn/backend.py index df462e75..70a94f43 100644 --- a/weboob/backends/youporn/backend.py +++ b/weboob/backends/youporn/backend.py @@ -20,6 +20,7 @@ from weboob.capabilities.video import ICapVideo from weboob.tools.backend import BaseBackend from .browser import YoupornBrowser +from .video import YoupornVideo __all__ = ['YoupornBackend'] @@ -46,3 +47,9 @@ class YoupornBackend(BaseBackend, ICapVideo): def iter_page_urls(self, mozaic_url): raise NotImplementedError() + + def fill_video(self, video, fields): + # ignore the fields param: VideoPage.get_video() returns all the information + return self.browser.get_video(YoupornVideo.id2url(video.id), video) + + OBJECTS = {YoupornVideo: fill_video} diff --git a/weboob/backends/youporn/browser.py b/weboob/backends/youporn/browser.py index 5202b7a8..eabc879e 100644 --- a/weboob/backends/youporn/browser.py +++ b/weboob/backends/youporn/browser.py @@ -16,11 +16,8 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -import logging - from weboob.tools.browser import BaseBrowser from weboob.tools.browser.decorators import id2url -from weboob.tools.misc import iter_fields from .pages.index import IndexPage from .pages.video import VideoPage @@ -39,15 +36,10 @@ class YoupornBrowser(BaseBrowser): r'http://[w\.]*youporngay\.com:80/watch/(?P.+)': VideoPage, } - def fillobj(self, video, fields): - # ignore the fields param: VideoPage.get_video() returns all the information - self.location(YoupornVideo.id2url(video.id)) - return self.page.get_video(video) - @id2url(YoupornVideo.id2url) - def get_video(self, url): + def get_video(self, url, video=None): self.location(url) - return self.page.get_video() + return self.page.get_video(video) def iter_search_results(self, pattern, sortby): if not pattern: diff --git a/weboob/backends/youporn/pages/video.py b/weboob/backends/youporn/pages/video.py index c7df0d5a..d1f131cb 100644 --- a/weboob/backends/youporn/pages/video.py +++ b/weboob/backends/youporn/pages/video.py @@ -18,7 +18,6 @@ import re import datetime -from logging import warning from weboob.tools.parsers.lxmlparser import select diff --git a/weboob/backends/youtube/backend.py b/weboob/backends/youtube/backend.py index 05eebe2f..ce681648 100644 --- a/weboob/backends/youtube/backend.py +++ b/weboob/backends/youtube/backend.py @@ -17,11 +17,9 @@ import datetime -import logging from weboob.capabilities.video import ICapVideo from weboob.tools.backend import BaseBackend -from weboob.tools.misc import iter_fields from .browser import YoutubeBrowser from .video import YoutubeVideo @@ -67,3 +65,9 @@ class YoutubeBackend(BaseBackend, ICapVideo): def iter_page_urls(self, mozaic_url): raise NotImplementedError() + + def fill_video(self, video, fields): + # ignore the fields param: VideoPage.get_video() returns all the information + return self.browser.get_video(YoutubeVideo.id2url(video.id), video) + + OBJECTS = {YoutubeVideo: fill_video} diff --git a/weboob/backends/youtube/browser.py b/weboob/backends/youtube/browser.py index 0ad0bc67..bde0bd3d 100644 --- a/weboob/backends/youtube/browser.py +++ b/weboob/backends/youtube/browser.py @@ -16,8 +16,6 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -import logging - from weboob.tools.browser import BaseBrowser from weboob.tools.browser.decorators import id2url @@ -36,12 +34,7 @@ class YoutubeBrowser(BaseBrowser): r'.*youtube\.com/verify_age\?next_url=(?P.+)': VerifyAgePage, } - def fillobj(self, video, fields): - # ignore the fields param: VideoPage.get_video() returns all the information - self.location(YoutubeVideo.id2url(video.id)) - return self.page.get_video(video) - @id2url(YoutubeVideo.id2url) - def get_video(self, url): + def get_video(self, url, video=None): self.location(url) - return self.page.get_video() + return self.page.get_video(video) diff --git a/weboob/backends/youtube/pages.py b/weboob/backends/youtube/pages.py index 33b18ddc..05c67018 100644 --- a/weboob/backends/youtube/pages.py +++ b/weboob/backends/youtube/pages.py @@ -24,7 +24,7 @@ from weboob.tools.parsers.lxmlparser import select from .video import YoutubeVideo -__all__ = ['ForbiddenVideoPage', 'VerifyAgePage', 'VideoPage'] +__all__ = ['ForbiddenVideo', 'ForbiddenVideoPage', 'VerifyAgePage', 'VideoPage'] class ForbiddenVideo(Exception): diff --git a/weboob/tools/application/console.py b/weboob/tools/application/console.py index 2b296998..7efbf627 100644 --- a/weboob/tools/application/console.py +++ b/weboob/tools/application/console.py @@ -25,6 +25,7 @@ import sys from weboob.core import CallErrors from weboob.core.backends import BackendsConfig +from weboob.tools.backend import NotSupportedObject from .base import BackendNotFound, BaseApplication from .formatters.load import formatters, load_formatter @@ -277,10 +278,11 @@ class ConsoleApplication(BaseApplication): for i, (backend, result) in enumerate(self.weboob.do(function, *args, **kwargs)): if self.options.count and i == self.options.count: break - fields = set(self.selected_fields) - set('*') - if fields: - try: - backend.browser.fillobj(result, fields) - except Exception, e: - logging.warning(u'Could not retrieve required fields (%s): %s' % (','.join(fields), e)) + if self.selected_fields: + fields = set(self.selected_fields) - set('*') + if fields: + try: + backend.fillobj(result, fields) + except NotSupportedObject, e: + logging.warning(u'Could not retrieve required fields (%s): %s' % (','.join(fields), e)) yield backend, result diff --git a/weboob/tools/backend.py b/weboob/tools/backend.py index 8170f80a..14dcc7d4 100644 --- a/weboob/tools/backend.py +++ b/weboob/tools/backend.py @@ -21,9 +21,11 @@ import os from threading import RLock -__all__ = ['BaseBackend'] +__all__ = ['BaseBackend', 'NotSupportedObject'] +class NotSupportedObject(Exception): pass + class BackendStorage(object): def __init__(self, name, storage): self.name = name @@ -72,6 +74,12 @@ class BaseBackend(object): BROWSER = None # Test class TEST = None + # Supported objects to fill + # The key is the class and the value the method to call to fill + # Method prototype: method(object, fields) + # When the method is called, fields are only the one which are + # NOT yet filled. + OBJECTS = {} class ConfigField(object): def __init__(self, default=None, is_masked=False, regexp=None, description=None): @@ -165,3 +173,10 @@ class BaseBackend(object): if not self.TEST: return None return self.TEST(self) + + def fillobj(self, obj, fields): + for key, value in self.OBJECTS.iteritems(): + if isinstance(obj, key): + return value(self, obj, fields) + + raise NotSupportedObject('The object of type %s is not supported by this backend' % type(obj))