use several loggers for parts of weboob

This commit is contained in:
Romain Bignon 2010-10-30 16:09:33 +02:00
commit 84b4003bf4
14 changed files with 74 additions and 67 deletions

View file

@ -18,7 +18,6 @@
from __future__ import with_statement from __future__ import with_statement
import logging
import sys import sys
import weboob import weboob
@ -97,7 +96,7 @@ class HaveSex(ReplApplication):
found = 1 found = 1
if not found: if not found:
logging.error(u'Profile not found') self.logger.error(u'Profile not found')
return True return True
@ -116,7 +115,7 @@ class HaveSex(ReplApplication):
except weboob.core.CallErrors, errors: except weboob.core.CallErrors, errors:
for backend, error, backtrace in errors: for backend, error, backtrace in errors:
if isinstance(error, OptimizationNotFound): if isinstance(error, OptimizationNotFound):
logging.error(u'Optimization "%s" not found' % optim) self.logger.error(u'Optimization "%s" not found' % optim)
def do_start(self, *optims): def do_start(self, *optims):
""" """

View file

@ -19,7 +19,6 @@ from __future__ import with_statement
import datetime import datetime
from dateutil import tz from dateutil import tz
from logging import warning, debug
from weboob.capabilities.base import NotLoaded from weboob.capabilities.base import NotLoaded
from weboob.capabilities.chat import ICapChat from weboob.capabilities.chat import ICapChat
@ -30,6 +29,7 @@ from weboob.capabilities.account import ICapAccount
from weboob.tools.backend import BaseBackend from weboob.tools.backend import BaseBackend
from weboob.tools.browser import BrowserUnavailable from weboob.tools.browser import BrowserUnavailable
from weboob.tools.value import Value, ValuesDict, ValueBool from weboob.tools.value import Value, ValuesDict, ValueBool
from weboob.tools.log import getLogger
from .captcha import CaptchaError from .captcha import CaptchaError
from .antispam import AntiSpam from .antispam import AntiSpam
@ -105,7 +105,7 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesPost, ICapDating, ICapCh
if not contact.get_id(): if not contact.get_id():
continue continue
if self.antispam and not self.antispam.check(contact): if self.antispam and not self.antispam.check(contact):
debug('Skipped a spam-thread from %s' % contact.get_name()) self.logger.debug('Skipped a spam-thread from %s' % contact.get_name())
self.report_spam(contact.get_id(), contact.get_suppr_id()) self.report_spam(contact.get_id(), contact.get_suppr_id())
continue continue
thread = Thread(contact.get_id()) thread = Thread(contact.get_id())
@ -141,7 +141,7 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesPost, ICapDating, ICapCh
for mail in mails: for mail in mails:
flags = 0 flags = 0
if self.antispam and not self.antispam.check(mail): if self.antispam and not self.antispam.check(mail):
debug('Skipped a spam-mail from %s' % mail.sender) self.logger.debug('Skipped a spam-mail from %s' % mail.sender)
self.report_spam(thread.id, contact and contact.get_suppr_id()) self.report_spam(thread.id, contact and contact.get_suppr_id())
break break
@ -152,7 +152,7 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesPost, ICapDating, ICapCh
with self.browser: with self.browser:
profiles[mail.profile_link] = self.browser.get_profile(mail.profile_link) profiles[mail.profile_link] = self.browser.get_profile(mail.profile_link)
if self.antispam and not self.antispam.check(profiles[mail.profile_link]): if self.antispam and not self.antispam.check(profiles[mail.profile_link]):
debug('Skipped a spam-mail-profile from %s' % mail.sender) self.logger.debug('Skipped a spam-mail-profile from %s' % mail.sender)
self.report_spam(thread.id, contact and contact.get_suppr_id()) self.report_spam(thread.id, contact and contact.get_suppr_id())
break break
mail.signature += u'\n%s' % profiles[mail.profile_link].get_profile_text() mail.signature += u'\n%s' % profiles[mail.profile_link].get_profile_text()
@ -203,7 +203,7 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesPost, ICapDating, ICapCh
if not contact.get_id(): if not contact.get_id():
continue continue
if self.antispam and not self.antispam.check(contact): if self.antispam and not self.antispam.check(contact):
debug('Skipped a spam-unread-thread from %s' % contact.get_name()) self.logger.debug('Skipped a spam-unread-thread from %s' % contact.get_name())
self.report_spam(contact.get_id(), contact.get_suppr_id()) self.report_spam(contact.get_id(), contact.get_suppr_id())
continue continue
slut = self._get_slut(contact.get_id()) slut = self._get_slut(contact.get_id())
@ -224,7 +224,7 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesPost, ICapDating, ICapCh
new_baskets -= 1 new_baskets -= 1
profile = self.browser.get_profile(ids[new_baskets]) profile = self.browser.get_profile(ids[new_baskets])
if self.antispam and not self.antispam.check(profile): if self.antispam and not self.antispam.check(profile):
debug('Skipped a spam-basket from %s' % profile.get_name()) self.logger.debug('Skipped a spam-basket from %s' % profile.get_name())
self.report_spam(profile.get_id()) self.report_spam(profile.get_id())
continue continue
@ -242,7 +242,7 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesPost, ICapDating, ICapCh
flags=Message.IS_UNREAD) flags=Message.IS_UNREAD)
yield thread.root yield thread.root
except BrowserUnavailable, e: except BrowserUnavailable, e:
debug('No messages, browser is unavailable: %s' % e) self.logger.debug('No messages, browser is unavailable: %s' % e)
pass # don't care about waiting pass # don't care about waiting
def set_message_read(self, message): def set_message_read(self, message):
@ -345,7 +345,7 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesPost, ICapDating, ICapCh
elif contact['cat'] == 2: elif contact['cat'] == 2:
s = Contact.STATUS_AWAY s = Contact.STATUS_AWAY
else: else:
warning('Unknown AuM contact status: %s' % contact['cat']) self.logger.warning('Unknown AuM contact status: %s' % contact['cat'])
if not status & s or ids and contact['id'] in ids: if not status & s or ids and contact['id'] in ids:
continue continue
@ -405,7 +405,7 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesPost, ICapDating, ICapCh
country= account.properties['country'].value, country= account.properties['country'].value,
godfather= account.properties['godfather'].value) godfather= account.properties['godfather'].value)
except CaptchaError: except CaptchaError:
debug('Unable to resolve captcha. Retrying...') getLogger('aum').debug('Unable to resolve captcha. Retrying...')
browser = None browser = None
def get_account(self): def get_account(self):

View file

@ -18,7 +18,6 @@
import datetime import datetime
import time import time
from logging import warning
import random import random
import simplejson import simplejson
import urllib import urllib
@ -225,20 +224,20 @@ class AuMBrowser(BaseBrowser):
@pageaccess @pageaccess
def send_charm(self, id): def send_charm(self, id):
result = self.openurl('http://www.adopteunmec.com/fajax_addBasket.php?id=%s' % id).read() result = self.openurl('http://www.adopteunmec.com/fajax_addBasket.php?id=%s' % id).read()
warning('Charm: %s' % result) self.logger.warning('Charm: %s' % result)
return result.find('noMoreFlashes') < 0 return result.find('noMoreFlashes') < 0
@pageaccess @pageaccess
def add_basket(self, id): def add_basket(self, id):
result = self.openurl('http://www.adopteunmec.com/fajax_addBasket.php?id=%s' % id).read() result = self.openurl('http://www.adopteunmec.com/fajax_addBasket.php?id=%s' % id).read()
warning('Basket: %s' % result) self.logger.warning('Basket: %s' % result)
# TODO check if it works (but it should) # TODO check if it works (but it should)
return True return True
@pageaccess @pageaccess
def deblock(self, id): def deblock(self, id):
result = self.openurl('http://www.adopteunmec.com/fajax_postMessage.php?action=deblock&to=%s' % id).read() result = self.openurl('http://www.adopteunmec.com/fajax_postMessage.php?action=deblock&to=%s' % id).read()
warning('Deblock: %s' % result) self.logger.warning('Deblock: %s' % result)
return True return True
@pageaccess @pageaccess

View file

@ -18,11 +18,11 @@
from __future__ import with_statement from __future__ import with_statement
from logging import info
from random import randint from random import randint
from weboob.tools.browser import BrowserUnavailable from weboob.tools.browser import BrowserUnavailable
from weboob.capabilities.dating import Optimization from weboob.capabilities.dating import Optimization
from weboob.tools.log import getLogger
__all__ = ['ProfilesWalker'] __all__ = ['ProfilesWalker']
@ -33,9 +33,10 @@ class ProfilesWalker(Optimization):
self.sched = sched self.sched = sched
self.storage = storage self.storage = storage
self.browser = browser self.browser = browser
self.logger = getLogger('walker', browser.logger)
self.visited_profiles = set(storage.get('profiles_walker', 'viewed')) self.visited_profiles = set(storage.get('profiles_walker', 'viewed'))
info(u'Loaded %d already visited profiles from storage.' % len(self.visited_profiles)) self.logger.info(u'Loaded %d already visited profiles from storage.' % len(self.visited_profiles))
self.profiles_queue = set() self.profiles_queue = set()
def save(self): def save(self):
@ -57,7 +58,7 @@ class ProfilesWalker(Optimization):
try: try:
with self.browser: with self.browser:
profiles_to_visit = self.browser.search_profiles().difference(self.visited_profiles) profiles_to_visit = self.browser.search_profiles().difference(self.visited_profiles)
info(u'Enqueuing profiles to visit: %s' % profiles_to_visit) self.logger.info(u'Enqueuing profiles to visit: %s' % profiles_to_visit)
self.profiles_queue = set(profiles_to_visit) self.profiles_queue = set(profiles_to_visit)
self.save() self.save()
except BrowserUnavailable: except BrowserUnavailable:
@ -73,7 +74,7 @@ class ProfilesWalker(Optimization):
try: try:
with self.browser: with self.browser:
profile = self.browser.get_profile(id) profile = self.browser.get_profile(id)
info(u'Visited profile %s (%s)' % (profile.get_name(), id)) self.logger.info(u'Visited profile %s (%s)' % (profile.get_name(), id))
# Get score from the aum_score module # Get score from the aum_score module
#d = self.nucentral_core.callService(context.Context.fromComponent(self), 'aum_score', 'score', profile) #d = self.nucentral_core.callService(context.Context.fromComponent(self), 'aum_score', 'score', profile)

View file

@ -17,7 +17,6 @@
import re import re
from logging import error, warning
from weboob.tools.browser import BasePage, BrowserUnavailable from weboob.tools.browser import BasePage, BrowserUnavailable
class PageBase(BasePage): class PageBase(BasePage):
@ -61,7 +60,7 @@ class PageBase(BasePage):
child = tag.childNodes[0].childNodes[0].childNodes[3] child = tag.childNodes[0].childNodes[0].childNodes[3]
return int(child.childNodes[0].childNodes[1].data.replace(' ', '').strip()) return int(child.childNodes[0].childNodes[1].data.replace(' ', '').strip())
error("Error: I can't find the score :(") self.logger.error("Error: I can't find the score :(")
return '0' return '0'
def __get_indicator(self, elementName): def __get_indicator(self, elementName):
@ -74,14 +73,14 @@ class PageBase(BasePage):
if not hasattr(child, 'data'): if not hasattr(child, 'data'):
if child.tagName != u'blink': if child.tagName != u'blink':
warning("Warning: %s counter isn't a blink and hasn't data" % elementName) self.logger.warning("Warning: %s counter isn't a blink and hasn't data" % elementName)
child = child.childNodes[0] child = child.childNodes[0]
if not hasattr(child, 'data'): if not hasattr(child, 'data'):
break break
return int(child.data) return int(child.data)
error("Error: I can't find the %s counter :(" % elementName) self.logger.error("Error: I can't find the %s counter :(" % elementName)
return 0 return 0
MYNAME_REGEXP = re.compile("Bonjour (.*)") MYNAME_REGEXP = re.compile("Bonjour (.*)")
@ -95,7 +94,7 @@ class PageBase(BasePage):
if m: if m:
return m.group(1) return m.group(1)
warning('Warning: Unable to fetch name') self.logger.warning('Warning: Unable to fetch name')
return '?' return '?'
def nb_new_mails(self): def nb_new_mails(self):

View file

@ -362,7 +362,7 @@ class ProfilePage(PageBase):
try: try:
fields[label1].put_value(d, value2) fields[label1].put_value(d, value2)
except KeyError: except KeyError:
warning('Unable to find "%s" (%s)' % (label1, repr(label1))) self.logger.warning('Unable to find "%s" (%s)' % (label1, repr(label1)))
elif label1 and label2: elif label1 and label2:
# two titles, so there will have a list of value in # two titles, so there will have a list of value in
# next lines on each columns # next lines on each columns

View file

@ -18,8 +18,6 @@
from __future__ import with_statement from __future__ import with_statement
from logging import warning
from weboob.capabilities.messages import ICapMessages, Message, Thread from weboob.capabilities.messages import ICapMessages, Message, Thread
from weboob.tools.backend import BaseBackend from weboob.tools.backend import BaseBackend
from weboob.tools.value import Value, ValuesDict from weboob.tools.value import Value, ValuesDict
@ -52,7 +50,7 @@ class FourChanBackend(BaseBackend, ICapMessages):
id = thread.id id = thread.id
if not '.' in id: if not '.' in id:
warning('Malformated ID (%s)' % id) self.logger.warning('Malformated ID (%s)' % id)
return return
board, thread_id = self._splitid(id) board, thread_id = self._splitid(id)

View file

@ -18,7 +18,6 @@
from datetime import datetime from datetime import datetime
import re import re
from logging import warning
from weboob.tools.browser import BasePage from weboob.tools.browser import BasePage
@ -53,7 +52,7 @@ class BoardPage(BasePage):
if m: if m:
self.board = m.group(1) self.board = m.group(1)
else: else:
warning('Unable to find board') self.logger.warning('Unable to find board')
self.board = 'unknown' self.board = 'unknown'
forms = self.document.getroot().cssselect('form') forms = self.document.getroot().cssselect('form')
@ -65,7 +64,7 @@ class BoardPage(BasePage):
break break
if form is None: if form is None:
warning('No delform :(') self.logger.warning('No delform :(')
article = None article = None
for div in form.getchildren(): for div in form.getchildren():

View file

@ -19,11 +19,11 @@
from __future__ import with_statement from __future__ import with_statement
from copy import copy from copy import copy
from logging import debug
from threading import Thread, Event, RLock, Timer from threading import Thread, Event, RLock, Timer
from weboob.capabilities.base import CapBaseObject from weboob.capabilities.base import CapBaseObject
from weboob.tools.misc import get_backtrace from weboob.tools.misc import get_backtrace
from weboob.tools.log import getLogger
__all__ = ['BackendsCall', 'CallErrors'] __all__ = ['BackendsCall', 'CallErrors']
@ -44,6 +44,7 @@ class BackendsCall(object):
@param function backends' method name, or callable object @param function backends' method name, or callable object
@param args, kwargs arguments given to called functions @param args, kwargs arguments given to called functions
""" """
self.logger = getLogger('bcall')
# Store if a backend is finished # Store if a backend is finished
self.backends = {} self.backends = {}
for backend in backends: for backend in backends:
@ -64,7 +65,7 @@ class BackendsCall(object):
# Create jobs for each backend # Create jobs for each backend
with self.mutex: with self.mutex:
for backend in backends: for backend in backends:
debug('Creating a new thread for %s' % backend) self.logger.debug('Creating a new thread for %s' % backend)
self.threads.append(Timer(0, self._caller, (backend, function, args, kwargs)).start()) self.threads.append(Timer(0, self._caller, (backend, function, args, kwargs)).start())
if not backends: if not backends:
self.finish_event.set() self.finish_event.set()
@ -82,21 +83,21 @@ class BackendsCall(object):
self.response_event.set() self.response_event.set()
def _caller(self, backend, function, args, kwargs): def _caller(self, backend, function, args, kwargs):
debug('%s: Thread created successfully' % backend) self.logger.debug('%s: Thread created successfully' % backend)
with backend: with backend:
try: try:
# Call method on backend # Call method on backend
try: try:
debug('%s: Calling function %s' % (backend, function)) self.logger.debug('%s: Calling function %s' % (backend, function))
if callable(function): if callable(function):
result = function(backend, *args, **kwargs) result = function(backend, *args, **kwargs)
else: else:
result = getattr(backend, function)(*args, **kwargs) result = getattr(backend, function)(*args, **kwargs)
except Exception, error: except Exception, error:
debug('%s: Called function %s raised an error: %r' % (backend, function, error)) self.logger.debug('%s: Called function %s raised an error: %r' % (backend, function, error))
self._store_error(backend, error) self._store_error(backend, error)
else: else:
debug('%s: Called function %s returned: %r' % (backend, function, result)) self.logger.debug('%s: Called function %s returned: %r' % (backend, function, result))
if hasattr(result, '__iter__') and not isinstance(result, basestring): if hasattr(result, '__iter__') and not isinstance(result, basestring):
# Loop on iterator # Loop on iterator

View file

@ -19,12 +19,12 @@
from __future__ import with_statement from __future__ import with_statement
import logging import logging
from logging import debug, error, exception
import os import os
import re import re
from weboob.capabilities.base import IBaseCap from weboob.capabilities.base import IBaseCap
from weboob.tools.backend import BaseBackend from weboob.tools.backend import BaseBackend
from weboob.tools.log import getLogger
__all__ = ['Module', 'ModulesLoader'] __all__ = ['Module', 'ModulesLoader']
@ -32,6 +32,7 @@ __all__ = ['Module', 'ModulesLoader']
class Module(object): class Module(object):
def __init__(self, package): def __init__(self, package):
self.logger = getLogger('backend')
self.package = package self.package = package
self.klass = None self.klass = None
for attrname in dir(self.package): for attrname in dir(self.package):
@ -89,14 +90,15 @@ class Module(object):
return False return False
def create_instance(self, weboob, instance_name, config, storage): def create_instance(self, weboob, instance_name, config, storage):
backend_instance = self.klass(weboob, instance_name, config, storage) backend_instance = self.klass(weboob, instance_name, config, storage, self.logger)
debug(u'Created backend instance "%s" for backend "%s"' % (instance_name, self.name)) self.logger.debug(u'Created backend instance "%s" for backend "%s"' % (instance_name, self.name))
return backend_instance return backend_instance
class ModulesLoader(object): class ModulesLoader(object):
def __init__(self): def __init__(self):
self.loaded = {} self.loaded = {}
self.logger = getLogger('modules')
def get_or_load_module(self, module_name): def get_or_load_module(self, module_name):
if module_name not in self.loaded: if module_name not in self.loaded:
@ -129,13 +131,13 @@ class ModulesLoader(object):
except ImportError, e: except ImportError, e:
msg = u'Unable to load module "%s": %s' % (module_name, e) msg = u'Unable to load module "%s": %s' % (module_name, e)
if logging.root.level == logging.DEBUG: if logging.root.level == logging.DEBUG:
exception(msg) self.logger.exception(msg)
return return
else: else:
error(msg) self.logger.error(msg)
return return
if module.name in self.loaded: if module.name in self.loaded:
debug('Module "%s" is already loaded from %s' % (module_name, module.package.__path__[0])) self.logger.debug('Module "%s" is already loaded from %s' % (module_name, module.package.__path__[0]))
return return
self.loaded[module.name] = module self.loaded[module.name] = module
debug('Loaded module "%s" from %s' % (module_name, module.package.__path__[0])) self.logger.debug('Loaded module "%s" from %s' % (module_name, module.package.__path__[0]))

View file

@ -18,7 +18,6 @@
from __future__ import with_statement from __future__ import with_statement
from logging import warning
import os import os
from weboob.core.bcall import BackendsCall from weboob.core.bcall import BackendsCall
@ -26,6 +25,7 @@ from weboob.core.modules import ModulesLoader
from weboob.core.backendscfg import BackendsConfig from weboob.core.backendscfg import BackendsConfig
from weboob.core.scheduler import Scheduler from weboob.core.scheduler import Scheduler
from weboob.tools.backend import BaseBackend from weboob.tools.backend import BaseBackend
from weboob.tools.log import getLogger
__all__ = ['Weboob'] __all__ = ['Weboob']
@ -36,6 +36,7 @@ class Weboob(object):
BACKENDS_FILENAME = 'backends' BACKENDS_FILENAME = 'backends'
def __init__(self, workdir=WORKDIR, backends_filename=None, scheduler=None, storage=None): def __init__(self, workdir=WORKDIR, backends_filename=None, scheduler=None, storage=None):
self.logger = getLogger('weboob')
self.workdir = workdir self.workdir = workdir
self.backend_instances = {} self.backend_instances = {}
@ -48,7 +49,7 @@ class Weboob(object):
if not os.path.exists(self.workdir): if not os.path.exists(self.workdir):
os.mkdir(self.workdir, 0700) os.mkdir(self.workdir, 0700)
elif not os.path.isdir(self.workdir): elif not os.path.isdir(self.workdir):
warning(u'"%s" is not a directory' % self.workdir) self.logger.warning(u'"%s" is not a directory' % self.workdir)
# Backends loader # Backends loader
self.modules_loader = ModulesLoader() self.modules_loader = ModulesLoader()
@ -96,15 +97,15 @@ class Weboob(object):
continue continue
module = self.modules_loader.get_or_load_module(module_name) module = self.modules_loader.get_or_load_module(module_name)
if module is None: if module is None:
warning(u'Backend "%s" is referenced in ~/.weboob/backends ' self.logger.warning(u'Backend "%s" is referenced in ~/.weboob/backends '
'configuration file, but was not found. ' 'configuration file, but was not found. '
'Hint: is it installed?' % module_name) 'Hint: is it installed?' % module_name)
continue continue
if caps is not None and not module.has_caps(caps): if caps is not None and not module.has_caps(caps):
continue continue
if instance_name in self.backend_instances: if instance_name in self.backend_instances:
warning(u'Oops, the backend "%s" is already loaded. Unload it before reloading...' % instance_name) self.logger.warning(u'Oops, the backend "%s" is already loaded. Unload it before reloading...' % instance_name)
self.unload_backends(instance_name) self.unload_backends(instance_name)
try: try:
@ -186,7 +187,7 @@ class Weboob(object):
else: else:
backends.append(backend) backends.append(backend)
else: else:
warning(u'The "backends" value isn\'t supported: %r' % _backends) self.logger.warning(u'The "backends" value isn\'t supported: %r' % _backends)
if 'caps' in kwargs: if 'caps' in kwargs:
caps = kwargs.pop('caps') caps = kwargs.pop('caps')

View file

@ -18,8 +18,8 @@
from __future__ import with_statement from __future__ import with_statement
import logging
from threading import Timer, Event, RLock from threading import Timer, Event, RLock
from weboob.tools.log import getLogger
__all__ = ['Scheduler'] __all__ = ['Scheduler']
@ -40,6 +40,7 @@ class IScheduler(object):
class Scheduler(IScheduler): class Scheduler(IScheduler):
def __init__(self): def __init__(self):
self.logger = getLogger('scheduler')
self.mutex = RLock() self.mutex = RLock()
self.stop_event = Event() self.stop_event = Event()
self.count = 0 self.count = 0
@ -51,7 +52,7 @@ class Scheduler(IScheduler):
with self.mutex: with self.mutex:
self.count += 1 self.count += 1
logging.debug('function "%s" will be called in %s seconds' % (function.__name__, interval)) self.logger.debug('function "%s" will be called in %s seconds' % (function.__name__, interval))
timer = Timer(interval, self._callback, (self.count, function, args)) timer = Timer(interval, self._callback, (self.count, function, args))
self.queue[self.count] = timer self.queue[self.count] = timer
timer.start() timer.start()

View file

@ -18,10 +18,10 @@
import os import os
from threading import RLock from threading import RLock
from logging import debug
from weboob.capabilities.base import CapBaseObject, FieldNotFound, IBaseCap, NotLoaded from weboob.capabilities.base import CapBaseObject, FieldNotFound, IBaseCap, NotLoaded
from weboob.tools.misc import iter_fields from weboob.tools.misc import iter_fields
from weboob.tools.log import getLogger
__all__ = ['BaseBackend', 'ObjectNotAvailable'] __all__ = ['BaseBackend', 'ObjectNotAvailable']
@ -95,7 +95,8 @@ class BaseBackend(object):
def __repr__(self): def __repr__(self):
return u"<Backend '%s'>" % self.name return u"<Backend '%s'>" % self.name
def __init__(self, weboob, name, config, storage): def __init__(self, weboob, name, config, storage, logger=None):
self.logger = getLogger(self.NAME, parent=logger)
self.weboob = weboob self.weboob = weboob
self.name = name self.name = name
self.lock = RLock() self.lock = RLock()
@ -143,7 +144,7 @@ class BaseBackend(object):
try: try:
import xdg.IconTheme import xdg.IconTheme
except ImportError: except ImportError:
debug(u'Python xdg module was not found. Please install it to read icon files.') self.logger.debug(u'Python xdg module was not found. Please install it to read icon files.')
else: else:
return xdg.IconTheme.getIconPath(self.NAME) return xdg.IconTheme.getIconPath(self.NAME)
@ -178,6 +179,7 @@ class BaseBackend(object):
kwargs['proxy'] = self._private_config['_proxy'] kwargs['proxy'] = self._private_config['_proxy']
elif 'HTTP_PROXY' in os.environ: elif 'HTTP_PROXY' in os.environ:
kwargs['proxy'] = os.environ['HTTP_PROXY'] kwargs['proxy'] = os.environ['HTTP_PROXY']
kwargs['logger'] = self.logger
return self.BROWSER(*args, **kwargs) return self.BROWSER(*args, **kwargs)
@ -235,5 +237,5 @@ class BaseBackend(object):
for key, value in self.OBJECTS.iteritems(): for key, value in self.OBJECTS.iteritems():
if isinstance(obj, key): if isinstance(obj, key):
debug(u'Fill %r with fields: %s' % (obj, missing_fields)) self.logger.debug(u'Fill %r with fields: %s' % (obj, missing_fields))
return value(self, obj, missing_fields) or obj return value(self, obj, missing_fields) or obj

View file

@ -24,7 +24,7 @@ from weboob.tools.mech import ClientForm
ControlNotFoundError = ClientForm.ControlNotFoundError ControlNotFoundError = ClientForm.ControlNotFoundError
import re import re
import time import time
from logging import warning, debug from logging import warning
from copy import copy from copy import copy
from threading import RLock from threading import RLock
import os import os
@ -32,6 +32,7 @@ import tempfile
from weboob.tools.parsers import get_parser from weboob.tools.parsers import get_parser
from weboob.tools.decorators import retry from weboob.tools.decorators import retry
from weboob.tools.log import getLogger
# Try to load cookies # Try to load cookies
try: try:
@ -91,12 +92,13 @@ class BasePage(object):
""" """
Base page Base page
""" """
def __init__(self, browser, document, url='', groups=None, group_dict=None): def __init__(self, browser, document, url='', groups=None, group_dict=None, logger=None):
self.browser = browser self.browser = browser
self.document = document self.document = document
self.url = url self.url = url
self.groups = groups self.groups = groups
self.group_dict = group_dict self.group_dict = group_dict
self.logger = getLogger('page', logger)
def on_loaded(self): def on_loaded(self):
""" """
@ -156,7 +158,7 @@ class BaseBrowser(mechanize.Browser):
default_features.remove('_robots') default_features.remove('_robots')
def __init__(self, username=None, password=None, firefox_cookies=None, def __init__(self, username=None, password=None, firefox_cookies=None,
parser=None, history=NoHistory(), proxy=None): parser=None, history=NoHistory(), proxy=None, logger=None):
""" """
Constructor of Browser. Constructor of Browser.
@ -167,8 +169,11 @@ class BaseBrowser(mechanize.Browser):
@param parser [IParser] parser to use on HTML files. @param parser [IParser] parser to use on HTML files.
@param hisory [object] History manager. Default value is an object @param hisory [object] History manager. Default value is an object
which does not keep history. which does not keep history.
@param proxy [str] proxy URL to use.
""" """
mechanize.Browser.__init__(self, history=history) mechanize.Browser.__init__(self, history=history)
self.logger = getLogger('browser', logger)
self.addheaders = [ self.addheaders = [
['User-agent', self.USER_AGENT] ['User-agent', self.USER_AGENT]
] ]
@ -242,7 +247,7 @@ class BaseBrowser(mechanize.Browser):
Open an URL but do not create a Page object. Open an URL but do not create a Page object.
""" """
if_fail = kwargs.pop('if_fail', 'raise') if_fail = kwargs.pop('if_fail', 'raise')
debug('Opening URL "%s", %s' % (args, kwargs)) self.logger.debug('Opening URL "%s", %s' % (args, kwargs))
try: try:
return mechanize.Browser.open_novisit(self, *args, **kwargs) return mechanize.Browser.open_novisit(self, *args, **kwargs)
except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError, BadStatusLine), e: except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError, BadStatusLine), e:
@ -280,7 +285,7 @@ class BaseBrowser(mechanize.Browser):
fd, path = tempfile.mkstemp(prefix="response", dir=tmpdir) fd, path = tempfile.mkstemp(prefix="response", dir=tmpdir)
with os.fdopen(fd, 'w') as f: with os.fdopen(fd, 'w') as f:
f.write(result.read()) f.write(result.read())
debug("Response saved to %s" % path) self.logger.debug("Response saved to %s" % path)
result.seek(0) result.seek(0)
def submit(self, *args, **kwargs): def submit(self, *args, **kwargs):
@ -366,23 +371,23 @@ class BaseBrowser(mechanize.Browser):
#if isinstance(data, unicode): #if isinstance(data, unicode):
# data = data.encode('utf-8') # data = data.encode('utf-8')
#print data #print data
warning('Oh my fucking god, there isn\'t any page corresponding to URL %s' % result.geturl()) self.logger.warning('Oh my fucking god, there isn\'t any page corresponding to URL %s' % result.geturl())
self.save_response(result) self.save_response(result)
return return
debug('[user_id=%s] Went on %s' % (self.username, result.geturl())) self.logger.debug('[user_id=%s] Went on %s' % (self.username, result.geturl()))
self.last_update = time.time() self.last_update = time.time()
if self.SAVE_RESPONSES: if self.SAVE_RESPONSES:
self.save_response(result) self.save_response(result)
document = self.get_document(result) document = self.get_document(result)
self.page = pageCls(self, document, result.geturl(), groups=page_groups, group_dict=page_group_dict) self.page = pageCls(self, document, result.geturl(), groups=page_groups, group_dict=page_group_dict, logger=self.logger)
self.page.on_loaded() self.page.on_loaded()
if self.password is not None and not self.is_logged(): if self.password is not None and not self.is_logged():
debug('!! Relogin !!') self.logger.debug('!! Relogin !!')
self.login() self.login()
return return