From c1279aa49683943f289250dacade9d476ac29713 Mon Sep 17 00:00:00 2001 From: Romain Bignon Date: Thu, 20 May 2010 10:39:49 +0200 Subject: [PATCH] new API in BaseBackend to create browser --- weboob/backend.py | 42 +++++++++++++++++++++++++++ weboob/backends/aum/backend.py | 12 +++----- weboob/backends/bnporc/backend.py | 14 ++------- weboob/backends/canaltp/backend.py | 7 ++--- weboob/backends/cragr/backend.py | 14 ++------- weboob/backends/dlfp/backend.py | 10 ++----- weboob/backends/gazelle/backend.py | 12 +++----- weboob/backends/transilien/backend.py | 4 +-- weboob/backends/youjizz/backend.py | 15 +++------- weboob/backends/youporn/backend.py | 15 +++------- weboob/backends/youtube/backend.py | 15 +++------- weboob/tools/browser.py | 7 ++++- 12 files changed, 82 insertions(+), 85 deletions(-) diff --git a/weboob/backend.py b/weboob/backend.py index dfc22bb5..5b114afc 100644 --- a/weboob/backend.py +++ b/weboob/backend.py @@ -19,6 +19,7 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. """ import re +import os from threading import RLock @@ -66,6 +67,8 @@ class BaseBackend(object): CONFIG = {} # Storage STORAGE = {} + # Browser class + BROWSER = None class ConfigField(object): def __init__(self, default=None, is_masked=False, regexp=None, description=None): @@ -89,6 +92,11 @@ class BaseBackend(object): self.weboob = weboob self.name = name self.lock = RLock() + + # Private fields (which start with '_') + self._private_config = dict([(key, value) for key, value in config.iteritems() if key.startswith('_')]) + + # Configuration of backend self.config = {} for name, field in self.CONFIG.iteritems(): value = config.get(name, field.default) @@ -110,6 +118,40 @@ class BaseBackend(object): self.storage = BackendStorage(self.name, storage) self.storage.load(self.STORAGE) + @property + def browser(self): + """ + Attribute 'browser'. The browser is created at the first call + of this attribute, to avoid useless pages access. + + Note that the 'default_browser' method is called to create it. + """ + if not hasattr(self, '_browser'): + self._browser = self.default_browser() + return self._browser + + def default_browser(self): + """ + Method to overload to build the default browser in + attribute 'browser'. + """ + return self.build_browser() + + def build_browser(self, *args, **kwargs): + """ + Build a browser from the BROWSER class attribute and the + given arguments. + """ + if not self.BROWSER: + return None + + if '_proxy' in self._private_config: + kwargs['proxy'] = self._private_config['_proxy'] + elif 'HTTP_PROXY' in os.environ: + kwargs['proxy'] = os.environ['HTTP_PROXY'] + + return self.BROWSER(*args, **kwargs) + def has_caps(self, *caps): for c in caps: if isinstance(self, c): diff --git a/weboob/backends/aum/backend.py b/weboob/backends/aum/backend.py index 03b50afc..ae9c93a4 100644 --- a/weboob/backends/aum/backend.py +++ b/weboob/backends/aum/backend.py @@ -47,18 +47,14 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesReply, ICapDating, ICapC 'password': BaseBackend.ConfigField(description='Password of account', is_masked=True), } STORAGE = {'profiles_walker': {'viewed': []} } + BROWSER = AdopteUnMec + + def default_browser(self): + return self.build_browser(self.config['username'], self.config['password']) # Private - _browser = None _profiles_walker = None - def __getattr__(self, name): - if name == 'browser': - if not self._browser: - self._browser = AdopteUnMec(self.config['username'], self.config['password']) - return self._browser - raise AttributeError, name - def iter_messages(self, thread=None): for message in self._iter_messages(thread, False): yield message diff --git a/weboob/backends/bnporc/backend.py b/weboob/backends/bnporc/backend.py index 83ec8a27..1cc0542e 100644 --- a/weboob/backends/bnporc/backend.py +++ b/weboob/backends/bnporc/backend.py @@ -34,22 +34,15 @@ class BNPorcBackend(BaseBackend, ICapBank): CONFIG = {'login': BaseBackend.ConfigField(description='Account ID'), 'password': BaseBackend.ConfigField(description='Password of account', is_masked=True) } - browser = None + BROWSER = BNPorc - def need_browser(func): - def inner(self, *args, **kwargs): - if not self.browser: - self.browser = BNPorc(self.config['login'], self.config['password']) + def default_browser(self): + return self.build_browser(self.config['login'], self.config['password']) - return func(self, *args, **kwargs) - return inner - - @need_browser def iter_accounts(self): for account in self.browser.get_accounts_list(): yield account - @need_browser def get_account(self, _id): try: _id = long(_id) @@ -62,7 +55,6 @@ class BNPorcBackend(BaseBackend, ICapBank): else: raise AccountNotFound() - @need_browser def iter_operations(self, account): for coming in self.browser.get_coming_operations(account): yield coming diff --git a/weboob/backends/canaltp/backend.py b/weboob/backends/canaltp/backend.py index 70d1578d..0ede5615 100644 --- a/weboob/backends/canaltp/backend.py +++ b/weboob/backends/canaltp/backend.py @@ -30,15 +30,14 @@ class CanalTPBackend(BaseBackend, ICapTravel): VERSION = '1.0' LICENSE = 'GPLv3' DESCRIPTION = "French trains" + BROWSER = CanalTP def iter_station_search(self, pattern): - canaltp = CanalTP() - for _id, name in canaltp.iter_station_search(pattern): + for _id, name in self.browser.iter_station_search(pattern): yield Station(_id, name) def iter_station_departures(self, station_id, arrival_id=None): - canaltp = CanalTP() - for i, d in enumerate(canaltp.iter_station_departures(station_id, arrival_id)): + for i, d in enumerate(self.browser.iter_station_departures(station_id, arrival_id)): departure = Departure(i, d['type'], d['time']) departure.departure_station = d['departure'] departure.arrival_station = d['arrival'] diff --git a/weboob/backends/cragr/backend.py b/weboob/backends/cragr/backend.py index c14c1eb5..900d0c8b 100644 --- a/weboob/backends/cragr/backend.py +++ b/weboob/backends/cragr/backend.py @@ -35,22 +35,15 @@ class CragrBackend(BaseBackend, ICapBank): 'password': BaseBackend.ConfigField(description='Password of account', is_masked=True), 'website': BaseBackend.ConfigField(description='What website to use', default='m.lefil.com'), } - browser = None + BROWSER = Cragr - def need_browser(func): - def inner(self, *args, **kwargs): - if not self.browser: - self.browser = Cragr(self.config['website'], self.config['login'], self.config['password']) + def default_browser(self): + return self.build_browser(self.config['website'], self.config['login'], self.config['password']) - return func(self, *args, **kwargs) - return inner - - @need_browser def iter_accounts(self): for account in self.browser.get_accounts_list(): yield account - @need_browser def get_account(self, _id): try: _id = long(_id) @@ -63,7 +56,6 @@ class CragrBackend(BaseBackend, ICapBank): else: raise AccountNotFound() - @need_browser def iter_operations(self, account): """ Not supported yet """ return iter([]) diff --git a/weboob/backends/dlfp/backend.py b/weboob/backends/dlfp/backend.py index 850871c6..0dce5fa4 100644 --- a/weboob/backends/dlfp/backend.py +++ b/weboob/backends/dlfp/backend.py @@ -39,14 +39,10 @@ class DLFPBackend(BaseBackend, ICapMessages, ICapMessagesReply): 'get_telegrams': BaseBackend.ConfigField(default=False, description='Get telegrams'), } STORAGE = {'seen': {}} - _browser = None + BROWSER = DLFP - def __getattr__(self, name): - if name == 'browser': - if not self._browser: - self._browser = DLFP(self.config['username'], self.config['password']) - return self._browser - raise AttributeError, name + def default_browser(self): + return self.build_browser(self.config['username'], self.config['password']) def iter_messages(self, thread=None): return self._iter_messages(thread, False) diff --git a/weboob/backends/gazelle/backend.py b/weboob/backends/gazelle/backend.py index 3ebfddb4..f7ff40bc 100644 --- a/weboob/backends/gazelle/backend.py +++ b/weboob/backends/gazelle/backend.py @@ -40,15 +40,11 @@ class GazelleBackend(BaseBackend, ICapTorrent): 'protocol': BaseBackend.ConfigField(description='Protocol to use ("http" or "https")'), 'domain': BaseBackend.ConfigField(description='Domain (example "ssl.what.cd")'), } - _browser = None + BROWSER = GazelleBrowser - def __getattr__(self, name): - if name == 'browser': - if not self._browser: - self._browser = GazelleBrowser(self.config['protocol'], self.config['domain'], - self.config['username'], self.config['password']) - return self._browser - raise AttributeError, name + def default_browser(self): + return self.build_browser(self.config['protocol'], self.config['domain'], + self.config['username'], self.config['password']) def get_torrent(self, id): return self.browser.get_torrent(id) diff --git a/weboob/backends/transilien/backend.py b/weboob/backends/transilien/backend.py index b8fb7d5c..2315bc69 100644 --- a/weboob/backends/transilien/backend.py +++ b/weboob/backends/transilien/backend.py @@ -31,6 +31,7 @@ class TransilienBackend(BaseBackend, ICapTravel): VERSION = '1.0' LICENSE = 'GPLv3' DESCRIPTION = "Transports in Paris" + BROWSER = Transilien def iter_station_search(self, pattern): pattern = pattern.lower() @@ -39,8 +40,7 @@ class TransilienBackend(BaseBackend, ICapTravel): yield Station(_id, name) def iter_station_departures(self, station_id, arrival_id=None): - transilien = Transilien() - for i, d in enumerate(transilien.iter_station_departures(station_id, arrival_id)): + for i, d in enumerate(self.browser.iter_station_departures(station_id, arrival_id)): departure = Departure(i, d['type'], d['time']) departure.departure_station = d['departure'] departure.arrival_station = d['arrival'] diff --git a/weboob/backends/youjizz/backend.py b/weboob/backends/youjizz/backend.py index f43cd792..f5c4e93a 100644 --- a/weboob/backends/youjizz/backend.py +++ b/weboob/backends/youjizz/backend.py @@ -1,16 +1,16 @@ # -*- coding: utf-8 -*- # Copyright(C) 2010 Roger Philibert -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, version 3 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. @@ -35,16 +35,9 @@ class YoujizzBackend(BaseBackend, ICapVideoProvider): LICENSE = 'GPLv3' CONFIG = {} - _browser = None + BROWSER = YoujizzBrowser domain = u'youjizz.com' - def __getattr__(self, name): - if name == 'browser': - if not self._browser: - self._browser = YoujizzBrowser() - return self._browser - raise AttributeError, name - @id2url(domain, YoujizzVideo.id2url) def get_video(self, _id): return self.browser.get_video(_id) diff --git a/weboob/backends/youporn/backend.py b/weboob/backends/youporn/backend.py index 3adf4204..7439422d 100644 --- a/weboob/backends/youporn/backend.py +++ b/weboob/backends/youporn/backend.py @@ -1,16 +1,16 @@ # -*- coding: utf-8 -*- # Copyright(C) 2010 Romain Bignon -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, version 3 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. @@ -35,16 +35,9 @@ class YoupornBackend(BaseBackend, ICapVideoProvider): LICENSE = 'GPLv3' CONFIG = {} - _browser = None + BROWSER = YoupornBrowser domain = u'youporn.com' - def __getattr__(self, name): - if name == 'browser': - if not self._browser: - self._browser = YoupornBrowser() - return self._browser - raise AttributeError, name - @id2url(domain, YoupornVideo.id2url) def get_video(self, _id): return self.browser.get_video(_id) diff --git a/weboob/backends/youtube/backend.py b/weboob/backends/youtube/backend.py index ca48fcc1..1852026c 100644 --- a/weboob/backends/youtube/backend.py +++ b/weboob/backends/youtube/backend.py @@ -1,16 +1,16 @@ # -*- coding: utf-8 -*- # Copyright(C) 2010 Christophe Benz, Romain Bignon -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, version 3 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. @@ -37,16 +37,9 @@ class YoutubeBackend(BaseBackend, ICapVideoProvider): LICENSE = 'GPLv3' CONFIG = {} - _browser = None + BROWSER = YoutubeBrowser domain = u'youtube.com' - def __getattr__(self, name): - if name == 'browser': - if not self._browser: - self._browser = YoutubeBrowser() - return self._browser - raise AttributeError, name - @id2url(domain, YoutubeVideo.id2url) def get_video(self, _id): return self.browser.get_video(_id) diff --git a/weboob/tools/browser.py b/weboob/tools/browser.py index 27262422..1353b159 100644 --- a/weboob/tools/browser.py +++ b/weboob/tools/browser.py @@ -148,7 +148,12 @@ class BaseBrowser(mechanize.Browser): # Use a proxy if proxy: - self.set_proxies({"http": proxy}) + proto = 'http' + if proxy.find('://') >= 0: + proto, domain = proxy.split('://', 1) + else: + domain = proxy + self.set_proxies({proto: domain}) # Share cookies with firefox if firefox_cookies and HAVE_COOKIES: