From 42028c390f600a1792d65434ad69aa8a53a0a91d Mon Sep 17 00:00:00 2001 From: Romain Bignon Date: Tue, 25 Mar 2014 22:58:35 +0100 Subject: [PATCH] add a 'settings' attribute on loggers to store user values like 'save_responses', 'responses_dirname', 'ssl_insecure', etc. --- weboob/tools/application/base.py | 32 +++++++++++--------------------- weboob/tools/backend.py | 4 ++-- weboob/tools/browser/browser.py | 19 +++++++++---------- weboob/tools/browser2/browser.py | 7 +++---- weboob/tools/log.py | 11 +++++++++-- 5 files changed, 34 insertions(+), 39 deletions(-) diff --git a/weboob/tools/application/base.py b/weboob/tools/application/base.py index 0635110e..99029ba4 100644 --- a/weboob/tools/application/base.py +++ b/weboob/tools/application/base.py @@ -31,7 +31,7 @@ from weboob.tools.browser.browser import FormFieldConversionWarning from weboob.core import Weboob, CallErrors from weboob.core.backendscfg import BackendsConfig from weboob.tools.config.iconfig import ConfigError -from weboob.tools.log import createColoredFormatter, getLogger +from weboob.tools.log import createColoredFormatter, getLogger, settings as log_settings from weboob.tools.misc import to_unicode from .results import ResultsConditionError @@ -140,6 +140,7 @@ class BaseApplication(object): self.config = None self.options = None self.condition = None + self.storage = None if option_parser is None: self._parser = OptionParser(self.SYNOPSIS, version=self._get_optparse_version()) else: @@ -327,12 +328,6 @@ class BaseApplication(object): if self.options.debug or self.options.save_responses: level = logging.DEBUG - from weboob.tools.browser import StandardBrowser - from weboob.tools.browser2 import BaseBrowser as Browser2 - StandardBrowser.DEBUG_MECHANIZE = True - # required to actually display or save the stuff - logger = logging.getLogger("mechanize") - logger.setLevel(logging.INFO) elif self.options.verbose: level = logging.INFO elif self.options.quiet: @@ -340,10 +335,7 @@ class BaseApplication(object): else: level = logging.WARNING if self.options.insecure: - from weboob.tools.browser import StandardBrowser - from weboob.tools.browser2 import BaseBrowser as Browser2 # NOQA - StandardBrowser.INSECURE = True - Browser2.VERIFY = False + log_settings['ssl_insecure'] = True # this only matters to developers if not self.options.debug and not self.options.save_responses: @@ -355,10 +347,8 @@ class BaseApplication(object): if self.options.save_responses: responses_dirname = tempfile.mkdtemp(prefix='weboob_session_') print >>sys.stderr, 'Debug data will be saved in this directory: %s' % responses_dirname - StandardBrowser.SAVE_RESPONSES = True - Browser2.SAVE_RESPONSES = True - StandardBrowser.responses_dirname = responses_dirname - Browser2.responses_dirname = responses_dirname + log_settings['save_responses'] = True + log_settings['responses_dirname'] = responses_dirname handlers.append(self.create_logging_file_handler(os.path.join(responses_dirname, 'debug.log'))) # file logger @@ -375,16 +365,16 @@ class BaseApplication(object): return args @classmethod - def create_default_logger(klass): + def create_default_logger(cls): # stdout logger - format = '%(asctime)s:%(levelname)s:%(name)s:' + klass.VERSION +\ + format = '%(asctime)s:%(levelname)s:%(name)s:' + cls.VERSION +\ ':%(filename)s:%(lineno)d:%(funcName)s %(message)s' handler = logging.StreamHandler(sys.stdout) handler.setFormatter(createColoredFormatter(sys.stdout, format)) return handler @classmethod - def setup_logging(klass, level, handlers): + def setup_logging(cls, level, handlers): logging.root.handlers = [] logging.root.setLevel(level) @@ -405,7 +395,7 @@ class BaseApplication(object): return handler @classmethod - def run(klass, args=None): + def run(cls, args=None): """ This static method can be called to run the application. @@ -421,13 +411,13 @@ class BaseApplication(object): >>> MyApplication.run() """ - klass.setup_logging(logging.INFO, [klass.create_default_logger()]) + cls.setup_logging(logging.INFO, [cls.create_default_logger()]) if args is None: args = [(sys.stdin.encoding and arg.decode(sys.stdin.encoding) or to_unicode(arg)) for arg in sys.argv] try: - app = klass() + app = cls() except BackendsConfig.WrongPermissions as e: print >>sys.stderr, e sys.exit(1) diff --git a/weboob/tools/backend.py b/weboob/tools/backend.py index dfde884b..1bb5ded6 100644 --- a/weboob/tools/backend.py +++ b/weboob/tools/backend.py @@ -339,8 +339,8 @@ class BaseBackend(object): kwargs['logger'] = self.logger - if hasattr(self.BROWSER, 'SAVE_RESPONSES') and self.BROWSER.SAVE_RESPONSES and self.BROWSER.responses_dirname: - kwargs.setdefault('responses_dirname', os.path.join(self.BROWSER.responses_dirname, + if self.logger.settings['responses_dirname']: + kwargs.setdefault('responses_dirname', os.path.join(self.logger.settings['responses_dirname'], self._private_config.get('_debug_dir', self.name))) return self.BROWSER(*args, **kwargs) diff --git a/weboob/tools/browser/browser.py b/weboob/tools/browser/browser.py index 1483c37d..e061c8e1 100644 --- a/weboob/tools/browser/browser.py +++ b/weboob/tools/browser/browser.py @@ -21,7 +21,6 @@ from copy import copy from httplib import BadStatusLine -from logging import warning try: import mechanize @@ -41,6 +40,7 @@ import time import urllib import urllib2 import mimetypes +import logging from contextlib import closing from gzip import GzipFile import warnings @@ -55,7 +55,7 @@ from weboob.tools.parsers import get_parser try: from .firefox_cookies import FirefoxCookieJar except ImportError as e: - warning("Unable to store cookies: %s" % e) + logging.warning("Unable to store cookies: %s", e) HAVE_COOKIES = False else: HAVE_COOKIES = True @@ -194,15 +194,11 @@ class StandardBrowser(mechanize.Browser): 'wget': 'Wget/1.11.4', } USER_AGENT = USER_AGENTS['desktop_firefox'] - SAVE_RESPONSES = False DEBUG_HTTP = False DEBUG_MECHANIZE = False DEFAULT_TIMEOUT = 15 INSECURE = False # if True, do not validate SSL - responses_dirname = None - responses_count = 0 - logger = None # ------ Browser methods --------------------------------------- @@ -244,11 +240,14 @@ class StandardBrowser(mechanize.Browser): # display messages from httplib self.set_debug_http(True) - if self.DEBUG_MECHANIZE: + if logging.root.level == logging.DEBUG: # Enable log messages from mechanize.Browser self.set_debug_redirects(True) + mech_logger = logging.getLogger("mechanize") + mech_logger.setLevel(logging.INFO) self.responses_dirname = responses_dirname + self.responses_count = 0 def __enter__(self): self.lock.acquire() @@ -304,7 +303,7 @@ class StandardBrowser(mechanize.Browser): result = self.openurl(url, *args, **kwargs) if result: - if self.SAVE_RESPONSES: + if self.logger.settings['save_responses']: self.save_response(result) return result.read() else: @@ -431,7 +430,7 @@ class StandardBrowser(mechanize.Browser): return def lowsslcheck(self, domain, hsh): - if self.INSECURE: + if self.INSECURE or self.logger.settings['ssl_insecure']: return certhash = self._certhash(domain) if self.logger: @@ -698,7 +697,7 @@ class BaseBrowser(StandardBrowser): self.logger.debug('[user_id=%s] Went on %s' % (self.username, result.geturl())) self.last_update = time.time() - if self.SAVE_RESPONSES: + if self.logger.settings['save_responses']: self.save_response(result) document = self.get_document(result, parser, encoding=pageCls.ENCODING) diff --git a/weboob/tools/browser2/browser.py b/weboob/tools/browser2/browser.py index a134b536..17c329de 100644 --- a/weboob/tools/browser2/browser.py +++ b/weboob/tools/browser2/browser.py @@ -124,7 +124,6 @@ class BaseBrowser(object): REFRESH_MAX = 0.0 VERIFY = True - SAVE_RESPONSES = False PROXIES = None @@ -199,7 +198,7 @@ class BaseBrowser(object): session.proxies = self.PROXIES - session.verify = self.VERIFY + session.verify = self.VERIFY and not self.logger.settings['ssl_insecure'] # defines a max_retries. It's mandatory in case a server is not # handling keep alive correctly, like the proxy burp @@ -214,7 +213,7 @@ class BaseBrowser(object): profile.setup_session(session) - if self.SAVE_RESPONSES: + if self.logger.settings['save_responses']: session.hooks['response'].append(self._save) self.session = session @@ -279,7 +278,7 @@ class BaseBrowser(object): proxies = self.PROXIES if verify is None: - verify = self.VERIFY + verify = self.VERIFY and not self.logger.settings['ssl_insecure'] if timeout is None: timeout = self.TIMEOUT diff --git a/weboob/tools/log.py b/weboob/tools/log.py index 57a66918..d7611fb5 100644 --- a/weboob/tools/log.py +++ b/weboob/tools/log.py @@ -17,11 +17,12 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from collections import defaultdict from logging import Formatter, getLogger as _getLogger import sys -__all__ = ['getLogger', 'createColoredFormatter'] +__all__ = ['getLogger', 'createColoredFormatter', 'settings'] RESET_SEQ = "\033[0m" @@ -36,10 +37,16 @@ COLORS = { } +# Global settings f logger. +settings = defaultdict(lambda: None) + + def getLogger(name, parent=None): if parent: name = parent.name + '.' + name - return _getLogger(name) + logger = _getLogger(name) + logger.settings = settings + return logger class ColoredFormatter(Formatter):