From b1f688e6d58473e00280d6b78a2a1b6f46b8ccf1 Mon Sep 17 00:00:00 2001 From: Romain Bignon Date: Sun, 18 Jul 2010 19:31:17 +0200 Subject: [PATCH 1/3] wip (I commit everythings because my computer will die coming soon) --- weboob/backends/youjizz/backend.py | 12 ++++++++---- weboob/backends/youporn/backend.py | 9 ++++++--- weboob/backends/youtube/backend.py | 6 ++++-- weboob/tools/application/console.py | 27 +++++++++++++++++++++------ 4 files changed, 39 insertions(+), 15 deletions(-) diff --git a/weboob/backends/youjizz/backend.py b/weboob/backends/youjizz/backend.py index 8aa4797c..7cec9764 100644 --- a/weboob/backends/youjizz/backend.py +++ b/weboob/backends/youjizz/backend.py @@ -37,19 +37,23 @@ class YoujizzBackend(BaseBackend, ICapVideo): BROWSER = YoujizzBrowser def get_video(self, _id): - video = self.browser.get_video(_id) + with self.browser: + video = self.browser.get_video(_id) return video def iter_page_urls(self, mozaic_url): - return self.browser.iter_page_urls(mozaic_url) + with self.browser: + return self.browser.iter_page_urls(mozaic_url) def iter_search_results(self, pattern=None, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False): if not nsfw: return - return self.browser.iter_search_results(pattern) + with self.browser: + return self.browser.iter_search_results(pattern) def fill_video(self, video, fields): # ignore the fields param: VideoPage.get_video() returns all the information - return self.browser.get_video(YoujizzVideo.id2url(video.id), video) + with self.browser: + return self.browser.get_video(YoujizzVideo.id2url(video.id), video) OBJECTS = {YoujizzVideo: fill_video} diff --git a/weboob/backends/youporn/backend.py b/weboob/backends/youporn/backend.py index 70a94f43..db4019ab 100644 --- a/weboob/backends/youporn/backend.py +++ b/weboob/backends/youporn/backend.py @@ -37,19 +37,22 @@ class YoupornBackend(BaseBackend, ICapVideo): BROWSER = YoupornBrowser def get_video(self, _id): - return self.browser.get_video(_id) + with self.browser: + return self.browser.get_video(_id) SORTBY = ['relevance', 'rating', 'views', 'time'] def iter_search_results(self, pattern=None, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False): if not nsfw: return - return self.browser.iter_search_results(pattern, self.SORTBY[sortby]) + with self.browser: + return self.browser.iter_search_results(pattern, self.SORTBY[sortby]) def iter_page_urls(self, mozaic_url): raise NotImplementedError() def fill_video(self, video, fields): # ignore the fields param: VideoPage.get_video() returns all the information - return self.browser.get_video(YoupornVideo.id2url(video.id), video) + with self.browser: + return self.browser.get_video(YoupornVideo.id2url(video.id), video) OBJECTS = {YoupornVideo: fill_video} diff --git a/weboob/backends/youtube/backend.py b/weboob/backends/youtube/backend.py index ce681648..02626fa8 100644 --- a/weboob/backends/youtube/backend.py +++ b/weboob/backends/youtube/backend.py @@ -39,7 +39,8 @@ class YoutubeBackend(BaseBackend, ICapVideo): BROWSER = YoutubeBrowser def get_video(self, _id): - return self.browser.get_video(_id) + with self.browser: + return self.browser.get_video(_id) def iter_search_results(self, pattern=None, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False): import gdata.youtube.service @@ -68,6 +69,7 @@ class YoutubeBackend(BaseBackend, ICapVideo): def fill_video(self, video, fields): # ignore the fields param: VideoPage.get_video() returns all the information - return self.browser.get_video(YoutubeVideo.id2url(video.id), video) + with self.browser: + return self.browser.get_video(YoutubeVideo.id2url(video.id), video) OBJECTS = {YoutubeVideo: fill_video} diff --git a/weboob/tools/application/console.py b/weboob/tools/application/console.py index 7efbf627..04b4c664 100644 --- a/weboob/tools/application/console.py +++ b/weboob/tools/application/console.py @@ -275,14 +275,29 @@ class ConsoleApplication(BaseApplication): """ Call Weboob.do(), after having filled the yielded object, if selected fields are given by user. """ - for i, (backend, result) in enumerate(self.weboob.do(function, *args, **kwargs)): - if self.options.count and i == self.options.count: - break + + def inner(backend, count, selected_fields, function, *args, **kwargs): + res = getattr(backend, function)(*args, **kwargs) if self.selected_fields: fields = set(self.selected_fields) - set('*') + else: + fields = None + if hasattr(res, '__iter__'): + for i, sub in enumerate(res): + if self.options.count and i == self.options.count: + break + if fields: + try: + backend.fillobj(sub, fields) + except NoSupportedObject, e: + logging.warning(u'Could not retrieve required fields (%s): %s' % (','.join(fields), e)) + yield sub + else: if fields: try: - backend.fillobj(result, fields) - except NotSupportedObject, e: + backend.fillobj(sub, fields) + except NoSupportedObject, e: logging.warning(u'Could not retrieve required fields (%s): %s' % (','.join(fields), e)) - yield backend, result + return res + + return self.weboob.do(inner, function, *args, **kwargs) From ef27837042d2f75dc50d5cc7355754f145d660c8 Mon Sep 17 00:00:00 2001 From: Romain Bignon Date: Mon, 19 Jul 2010 22:00:57 +0200 Subject: [PATCH 2/3] fetching mails works for sluts too --- weboob/backends/aum/backend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weboob/backends/aum/backend.py b/weboob/backends/aum/backend.py index 27302375..5707e725 100644 --- a/weboob/backends/aum/backend.py +++ b/weboob/backends/aum/backend.py @@ -140,7 +140,7 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesReply, ICapDating, ICapC new_baskets = self.browser.nb_new_baskets() if new_baskets: ids = self.browser.get_baskets() - while new_baskets > 0: + while new_baskets > 0 and len(ids) > new_baskets: new_baskets -= 1 profile = self.browser.get_profile(ids[new_baskets]) From 1398c0f7d13795be6bd72c574ebb65679f4c78a3 Mon Sep 17 00:00:00 2001 From: Romain Bignon Date: Tue, 20 Jul 2010 20:37:37 +0200 Subject: [PATCH 3/3] improvement of the fillobj --- weboob/tools/application/base.py | 29 +++++++++++++++++++++++++++++ weboob/tools/application/console.py | 27 +-------------------------- 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/weboob/tools/application/base.py b/weboob/tools/application/base.py index 8478106e..18443bdf 100644 --- a/weboob/tools/application/base.py +++ b/weboob/tools/application/base.py @@ -23,6 +23,7 @@ from optparse import OptionGroup, OptionParser from weboob.core.ouiboube import Weboob from weboob.tools.config.iconfig import ConfigError +from weboob.tools.backend import NotSupportedObject __all__ = ['BackendNotFound', 'BaseApplication', 'ConfigError'] @@ -185,6 +186,34 @@ class BaseApplication(object): version = '%s v%s' % (self.APPNAME, self.VERSION) return version + def _complete_obj(self, backend, obj, fields): + if fields: + try: + backend.fillobj(obj, fields) + except NotSupportedObject, e: + logging.warning(u'Could not retrieve required fields (%s): %s' % (','.join(fields), e)) + return obj + + def _complete_iter(self, backend, count, fields, res): + for i, sub in enumerate(res): + if count and i == count: + break + sub = self._complete_obj(backend, sub, fields) + yield sub + + def complete(self, backend, count, selected_fields, function, *args, **kwargs): + res = getattr(backend, function)(*args, **kwargs) + + if self.selected_fields: + fields = set(self.selected_fields) - set('*') + else: + fields = None + + if hasattr(res, '__iter__'): + return self._complete_iter(backend, count, fields, res) + else: + return self._complete_obj(backend, fields, res) + @classmethod def run(klass, args=None): """ diff --git a/weboob/tools/application/console.py b/weboob/tools/application/console.py index 04b4c664..39949e91 100644 --- a/weboob/tools/application/console.py +++ b/weboob/tools/application/console.py @@ -25,7 +25,6 @@ import sys from weboob.core import CallErrors from weboob.core.backends import BackendsConfig -from weboob.tools.backend import NotSupportedObject from .base import BackendNotFound, BaseApplication from .formatters.load import formatters, load_formatter @@ -276,28 +275,4 @@ class ConsoleApplication(BaseApplication): Call Weboob.do(), after having filled the yielded object, if selected fields are given by user. """ - def inner(backend, count, selected_fields, function, *args, **kwargs): - res = getattr(backend, function)(*args, **kwargs) - if self.selected_fields: - fields = set(self.selected_fields) - set('*') - else: - fields = None - if hasattr(res, '__iter__'): - for i, sub in enumerate(res): - if self.options.count and i == self.options.count: - break - if fields: - try: - backend.fillobj(sub, fields) - except NoSupportedObject, e: - logging.warning(u'Could not retrieve required fields (%s): %s' % (','.join(fields), e)) - yield sub - else: - if fields: - try: - backend.fillobj(sub, fields) - except NoSupportedObject, e: - logging.warning(u'Could not retrieve required fields (%s): %s' % (','.join(fields), e)) - return res - - return self.weboob.do(inner, function, *args, **kwargs) + return self.weboob.do(self.complete, self.options.count, self.selected_fields, function, *args, **kwargs)