nojhan/weboob-devel
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

enhance save_response option

Browse source
This commit is contained in:
Christophe Benz 2010-11-22 01:07:01 +01:00
commit 0a801006dc
2 changed files with 29 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

46
weboob/tools/application/base.py
View file

@ -16,10 +16,12 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
import sys, os
import logging import logging
import optparse import optparse
from optparse import OptionGroup, OptionParser from optparse import OptionGroup, OptionParser
import os
import sys
import tempfile
from weboob.capabilities.base import NotAvailable, NotLoaded from weboob.capabilities.base import NotAvailable, NotLoaded
from weboob.core import Weboob, CallErrors from weboob.core import Weboob, CallErrors
@ -286,14 +288,7 @@ class BaseApplication(object):
print ' '.join(items) print ' '.join(items)
sys.exit(0) sys.exit(0)
if self.options.save_responses: if self.options.debug or self.options.save_responses:
from weboob.tools.browser import BaseBrowser
BaseBrowser.SAVE_RESPONSES = True
for handler in logging.root.handlers:
logging.root.removeHandler(handler)
if self.options.debug:
level = logging.DEBUG level = logging.DEBUG
elif self.options.verbose: elif self.options.verbose:
level = logging.INFO level = logging.INFO
@ -304,25 +299,22 @@ class BaseApplication(object):
logging.root.setLevel(level) logging.root.setLevel(level)
if self.options.save_responses:
responses_dirname = tempfile.mkdtemp(prefix='weboob_session_')
print 'Debug data will be saved in this directory: %s' % responses_dirname
from weboob.tools.browser import BaseBrowser
BaseBrowser.SAVE_RESPONSES = True
BaseBrowser.responses_dirname = responses_dirname
self.add_logging_file_handler(os.path.join(responses_dirname, 'debug.log'))
# file logger # file logger
if self.options.logging_file: if self.options.logging_file:
try: self.add_logging_file_handler(self.options.logging_file)
stream = open(self.options.logging_file, 'w')
except IOError, e:
self.logger.error('Unable to create the logging file: %s' % e)
sys.exit(1)
else:
format = '%(asctime)s:%(levelname)s:%(name)s:%(pathname)s:%(lineno)d:%(funcName)s %(message)s'
handler = logging.StreamHandler(stream)
handler.setLevel(level)
handler.setFormatter(logging.Formatter(format))
logging.root.addHandler(handler)
else: else:
# stdout logger # stdout logger
format = '%(asctime)s:%(levelname)s:%(name)s:%(filename)s:%(lineno)d:%(funcName)s %(message)s' format = '%(asctime)s:%(levelname)s:%(name)s:%(filename)s:%(lineno)d:%(funcName)s %(message)s'
handler = logging.StreamHandler(sys.stdout) handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(createColoredFormatter(sys.stdout, format)) handler.setFormatter(createColoredFormatter(sys.stdout, format))
handler.setLevel(level)
logging.root.addHandler(handler) logging.root.addHandler(handler)
self._handle_options() self._handle_options()
@ -330,6 +322,18 @@ class BaseApplication(object):
return args return args
def add_logging_file_handler(self, filename):
try:
stream = open(filename, 'w')
except IOError, e:
self.logger.error('Unable to create the logging file: %s' % e)
sys.exit(1)
else:
format = '%(asctime)s:%(levelname)s:%(name)s:%(pathname)s:%(lineno)d:%(funcName)s %(message)s'
handler = logging.StreamHandler(stream)
handler.setFormatter(logging.Formatter(format))
logging.root.addHandler(handler)
@classmethod @classmethod
def run(klass, args=None): def run(klass, args=None):
""" """

9
weboob/tools/browser/browser.py
View file

@ -125,7 +125,7 @@ class BaseBrowser(mechanize.Browser):
} }
USER_AGENT = USER_AGENTS['desktop_firefox'] USER_AGENT = USER_AGENTS['desktop_firefox']
SAVE_RESPONSES = False SAVE_RESPONSES = False
responses_dir = None responses_dirname = None
responses_count = 0 responses_count = 0
# ------ Abstract methods -------------------------------------- # ------ Abstract methods --------------------------------------
@ -283,13 +283,12 @@ class BaseBrowser(mechanize.Browser):
Save a stream to a temporary file, and log its name. Save a stream to a temporary file, and log its name.
The stream is rewinded after saving. The stream is rewinded after saving.
""" """
if self.responses_dir is None: assert self.responses_dirname is not None
self.responses_dir = tempfile.mkdtemp(prefix='weboob/session_') response_filepath = os.path.join(self.responses_dirname, unicode(self.responses_count))
response_filepath = os.path.join(self.responses_dir, unicode(self.responses_count))
with open(response_filepath, 'w') as f: with open(response_filepath, 'w') as f:
f.write(result.read()) f.write(result.read())
result.seek(0) result.seek(0)
match_filepath = os.path.join(self.responses_dir, 'url_response_match.txt') match_filepath = os.path.join(self.responses_dirname, 'url_response_match.txt')
with open(match_filepath, 'a') as f: with open(match_filepath, 'a') as f:
f.write('%s\t%s\n' % (result.geturl(), response_filepath)) f.write('%s\t%s\n' % (result.geturl(), response_filepath))
self.responses_count += 1 self.responses_count += 1