nojhan/weboob-devel
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Try to separate saved responses by backend

Browse source 
closes #661
This commit is contained in:
Laurent Bachelier 2012-02-01 20:07:55 +01:00
commit 03f65bb121
2 changed files with 10 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

3
weboob/tools/backend.py
View file

@ -227,6 +227,9 @@ class BaseBackend(object):
kwargs['proxy'] = os.environ['HTTP_PROXY'] kwargs['proxy'] = os.environ['HTTP_PROXY']
kwargs['logger'] = self.logger kwargs['logger'] = self.logger
if self.BROWSER.SAVE_RESPONSES and self.BROWSER.responses_dirname:
kwargs.setdefault('responses_dirname', os.path.join(self.BROWSER.responses_dirname, self.name))
return self.BROWSER(*args, **kwargs) return self.BROWSER(*args, **kwargs)
@classmethod @classmethod

11
weboob/tools/browser/browser.py
View file

@ -155,8 +155,7 @@ class StandardBrowser(mechanize.Browser):
default_features.remove('_robots') default_features.remove('_robots')
default_features.remove('_refresh') default_features.remove('_refresh')
def __init__(self, firefox_cookies=None, parser=None, history=NoHistory(), proxy=None, logger=None, def __init__(self, firefox_cookies=None, parser=None, history=NoHistory(), proxy=None, logger=None, factory=None, responses_dirname=None):
factory=None):
""" """
Constructor of Browser. Constructor of Browser.
@ -207,6 +206,8 @@ class StandardBrowser(mechanize.Browser):
# Enable log messages from mechanize.Browser # Enable log messages from mechanize.Browser
self.set_debug_redirects(True) self.set_debug_redirects(True)
self.responses_dirname = responses_dirname
def __enter__(self): def __enter__(self):
self.lock.acquire() self.lock.acquire()
@ -263,6 +264,8 @@ class StandardBrowser(mechanize.Browser):
if self.responses_dirname is None: if self.responses_dirname is None:
self.responses_dirname = tempfile.mkdtemp(prefix='weboob_session_') self.responses_dirname = tempfile.mkdtemp(prefix='weboob_session_')
print >>sys.stderr, 'Debug data will be saved in this directory: %s' % self.responses_dirname print >>sys.stderr, 'Debug data will be saved in this directory: %s' % self.responses_dirname
elif not os.path.isdir(self.responses_dirname):
os.makedirs(self.responses_dirname)
# get the content-type, remove optionnal charset part # get the content-type, remove optionnal charset part
mimetype = result.info().get('Content-Type', '').split(';')[0] mimetype = result.info().get('Content-Type', '').split(';')[0]
# due to http://bugs.python.org/issue1043134 # due to http://bugs.python.org/issue1043134
@ -392,7 +395,7 @@ class BaseBrowser(StandardBrowser):
def __init__(self, username=None, password=None, firefox_cookies=None, def __init__(self, username=None, password=None, firefox_cookies=None,
parser=None, history=NoHistory(), proxy=None, logger=None, parser=None, history=NoHistory(), proxy=None, logger=None,
factory=None, get_home=True): factory=None, get_home=True, responses_dirname=None):
""" """
Constructor of Browser. Constructor of Browser.
@ -407,7 +410,7 @@ class BaseBrowser(StandardBrowser):
@param factory [object] Mechanize factory. None to use Mechanize's default. @param factory [object] Mechanize factory. None to use Mechanize's default.
@param get_home [bool] Try to get the homepage. @param get_home [bool] Try to get the homepage.
""" """
StandardBrowser.__init__(self, firefox_cookies, parser, history, proxy, logger, factory) StandardBrowser.__init__(self, firefox_cookies, parser, history, proxy, logger, factory, responses_dirname)
self.page = None self.page = None
self.last_update = 0.0 self.last_update = 0.0
self.username = username self.username = username