use several loggers for parts of weboob
This commit is contained in:
parent
a1a58c7c32
commit
84b4003bf4
14 changed files with 74 additions and 67 deletions
|
|
@ -18,7 +18,6 @@
|
||||||
|
|
||||||
from __future__ import with_statement
|
from __future__ import with_statement
|
||||||
|
|
||||||
import logging
|
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
import weboob
|
import weboob
|
||||||
|
|
@ -97,7 +96,7 @@ class HaveSex(ReplApplication):
|
||||||
found = 1
|
found = 1
|
||||||
|
|
||||||
if not found:
|
if not found:
|
||||||
logging.error(u'Profile not found')
|
self.logger.error(u'Profile not found')
|
||||||
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
@ -116,7 +115,7 @@ class HaveSex(ReplApplication):
|
||||||
except weboob.core.CallErrors, errors:
|
except weboob.core.CallErrors, errors:
|
||||||
for backend, error, backtrace in errors:
|
for backend, error, backtrace in errors:
|
||||||
if isinstance(error, OptimizationNotFound):
|
if isinstance(error, OptimizationNotFound):
|
||||||
logging.error(u'Optimization "%s" not found' % optim)
|
self.logger.error(u'Optimization "%s" not found' % optim)
|
||||||
|
|
||||||
def do_start(self, *optims):
|
def do_start(self, *optims):
|
||||||
"""
|
"""
|
||||||
|
|
|
||||||
|
|
@ -19,7 +19,6 @@ from __future__ import with_statement
|
||||||
|
|
||||||
import datetime
|
import datetime
|
||||||
from dateutil import tz
|
from dateutil import tz
|
||||||
from logging import warning, debug
|
|
||||||
|
|
||||||
from weboob.capabilities.base import NotLoaded
|
from weboob.capabilities.base import NotLoaded
|
||||||
from weboob.capabilities.chat import ICapChat
|
from weboob.capabilities.chat import ICapChat
|
||||||
|
|
@ -30,6 +29,7 @@ from weboob.capabilities.account import ICapAccount
|
||||||
from weboob.tools.backend import BaseBackend
|
from weboob.tools.backend import BaseBackend
|
||||||
from weboob.tools.browser import BrowserUnavailable
|
from weboob.tools.browser import BrowserUnavailable
|
||||||
from weboob.tools.value import Value, ValuesDict, ValueBool
|
from weboob.tools.value import Value, ValuesDict, ValueBool
|
||||||
|
from weboob.tools.log import getLogger
|
||||||
|
|
||||||
from .captcha import CaptchaError
|
from .captcha import CaptchaError
|
||||||
from .antispam import AntiSpam
|
from .antispam import AntiSpam
|
||||||
|
|
@ -105,7 +105,7 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesPost, ICapDating, ICapCh
|
||||||
if not contact.get_id():
|
if not contact.get_id():
|
||||||
continue
|
continue
|
||||||
if self.antispam and not self.antispam.check(contact):
|
if self.antispam and not self.antispam.check(contact):
|
||||||
debug('Skipped a spam-thread from %s' % contact.get_name())
|
self.logger.debug('Skipped a spam-thread from %s' % contact.get_name())
|
||||||
self.report_spam(contact.get_id(), contact.get_suppr_id())
|
self.report_spam(contact.get_id(), contact.get_suppr_id())
|
||||||
continue
|
continue
|
||||||
thread = Thread(contact.get_id())
|
thread = Thread(contact.get_id())
|
||||||
|
|
@ -141,7 +141,7 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesPost, ICapDating, ICapCh
|
||||||
for mail in mails:
|
for mail in mails:
|
||||||
flags = 0
|
flags = 0
|
||||||
if self.antispam and not self.antispam.check(mail):
|
if self.antispam and not self.antispam.check(mail):
|
||||||
debug('Skipped a spam-mail from %s' % mail.sender)
|
self.logger.debug('Skipped a spam-mail from %s' % mail.sender)
|
||||||
self.report_spam(thread.id, contact and contact.get_suppr_id())
|
self.report_spam(thread.id, contact and contact.get_suppr_id())
|
||||||
break
|
break
|
||||||
|
|
||||||
|
|
@ -152,7 +152,7 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesPost, ICapDating, ICapCh
|
||||||
with self.browser:
|
with self.browser:
|
||||||
profiles[mail.profile_link] = self.browser.get_profile(mail.profile_link)
|
profiles[mail.profile_link] = self.browser.get_profile(mail.profile_link)
|
||||||
if self.antispam and not self.antispam.check(profiles[mail.profile_link]):
|
if self.antispam and not self.antispam.check(profiles[mail.profile_link]):
|
||||||
debug('Skipped a spam-mail-profile from %s' % mail.sender)
|
self.logger.debug('Skipped a spam-mail-profile from %s' % mail.sender)
|
||||||
self.report_spam(thread.id, contact and contact.get_suppr_id())
|
self.report_spam(thread.id, contact and contact.get_suppr_id())
|
||||||
break
|
break
|
||||||
mail.signature += u'\n%s' % profiles[mail.profile_link].get_profile_text()
|
mail.signature += u'\n%s' % profiles[mail.profile_link].get_profile_text()
|
||||||
|
|
@ -203,7 +203,7 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesPost, ICapDating, ICapCh
|
||||||
if not contact.get_id():
|
if not contact.get_id():
|
||||||
continue
|
continue
|
||||||
if self.antispam and not self.antispam.check(contact):
|
if self.antispam and not self.antispam.check(contact):
|
||||||
debug('Skipped a spam-unread-thread from %s' % contact.get_name())
|
self.logger.debug('Skipped a spam-unread-thread from %s' % contact.get_name())
|
||||||
self.report_spam(contact.get_id(), contact.get_suppr_id())
|
self.report_spam(contact.get_id(), contact.get_suppr_id())
|
||||||
continue
|
continue
|
||||||
slut = self._get_slut(contact.get_id())
|
slut = self._get_slut(contact.get_id())
|
||||||
|
|
@ -224,7 +224,7 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesPost, ICapDating, ICapCh
|
||||||
new_baskets -= 1
|
new_baskets -= 1
|
||||||
profile = self.browser.get_profile(ids[new_baskets])
|
profile = self.browser.get_profile(ids[new_baskets])
|
||||||
if self.antispam and not self.antispam.check(profile):
|
if self.antispam and not self.antispam.check(profile):
|
||||||
debug('Skipped a spam-basket from %s' % profile.get_name())
|
self.logger.debug('Skipped a spam-basket from %s' % profile.get_name())
|
||||||
self.report_spam(profile.get_id())
|
self.report_spam(profile.get_id())
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
|
@ -242,7 +242,7 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesPost, ICapDating, ICapCh
|
||||||
flags=Message.IS_UNREAD)
|
flags=Message.IS_UNREAD)
|
||||||
yield thread.root
|
yield thread.root
|
||||||
except BrowserUnavailable, e:
|
except BrowserUnavailable, e:
|
||||||
debug('No messages, browser is unavailable: %s' % e)
|
self.logger.debug('No messages, browser is unavailable: %s' % e)
|
||||||
pass # don't care about waiting
|
pass # don't care about waiting
|
||||||
|
|
||||||
def set_message_read(self, message):
|
def set_message_read(self, message):
|
||||||
|
|
@ -345,7 +345,7 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesPost, ICapDating, ICapCh
|
||||||
elif contact['cat'] == 2:
|
elif contact['cat'] == 2:
|
||||||
s = Contact.STATUS_AWAY
|
s = Contact.STATUS_AWAY
|
||||||
else:
|
else:
|
||||||
warning('Unknown AuM contact status: %s' % contact['cat'])
|
self.logger.warning('Unknown AuM contact status: %s' % contact['cat'])
|
||||||
|
|
||||||
if not status & s or ids and contact['id'] in ids:
|
if not status & s or ids and contact['id'] in ids:
|
||||||
continue
|
continue
|
||||||
|
|
@ -405,7 +405,7 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesPost, ICapDating, ICapCh
|
||||||
country= account.properties['country'].value,
|
country= account.properties['country'].value,
|
||||||
godfather= account.properties['godfather'].value)
|
godfather= account.properties['godfather'].value)
|
||||||
except CaptchaError:
|
except CaptchaError:
|
||||||
debug('Unable to resolve captcha. Retrying...')
|
getLogger('aum').debug('Unable to resolve captcha. Retrying...')
|
||||||
browser = None
|
browser = None
|
||||||
|
|
||||||
def get_account(self):
|
def get_account(self):
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,6 @@
|
||||||
|
|
||||||
import datetime
|
import datetime
|
||||||
import time
|
import time
|
||||||
from logging import warning
|
|
||||||
import random
|
import random
|
||||||
import simplejson
|
import simplejson
|
||||||
import urllib
|
import urllib
|
||||||
|
|
@ -225,20 +224,20 @@ class AuMBrowser(BaseBrowser):
|
||||||
@pageaccess
|
@pageaccess
|
||||||
def send_charm(self, id):
|
def send_charm(self, id):
|
||||||
result = self.openurl('http://www.adopteunmec.com/fajax_addBasket.php?id=%s' % id).read()
|
result = self.openurl('http://www.adopteunmec.com/fajax_addBasket.php?id=%s' % id).read()
|
||||||
warning('Charm: %s' % result)
|
self.logger.warning('Charm: %s' % result)
|
||||||
return result.find('noMoreFlashes') < 0
|
return result.find('noMoreFlashes') < 0
|
||||||
|
|
||||||
@pageaccess
|
@pageaccess
|
||||||
def add_basket(self, id):
|
def add_basket(self, id):
|
||||||
result = self.openurl('http://www.adopteunmec.com/fajax_addBasket.php?id=%s' % id).read()
|
result = self.openurl('http://www.adopteunmec.com/fajax_addBasket.php?id=%s' % id).read()
|
||||||
warning('Basket: %s' % result)
|
self.logger.warning('Basket: %s' % result)
|
||||||
# TODO check if it works (but it should)
|
# TODO check if it works (but it should)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
@pageaccess
|
@pageaccess
|
||||||
def deblock(self, id):
|
def deblock(self, id):
|
||||||
result = self.openurl('http://www.adopteunmec.com/fajax_postMessage.php?action=deblock&to=%s' % id).read()
|
result = self.openurl('http://www.adopteunmec.com/fajax_postMessage.php?action=deblock&to=%s' % id).read()
|
||||||
warning('Deblock: %s' % result)
|
self.logger.warning('Deblock: %s' % result)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
@pageaccess
|
@pageaccess
|
||||||
|
|
|
||||||
|
|
@ -18,11 +18,11 @@
|
||||||
|
|
||||||
from __future__ import with_statement
|
from __future__ import with_statement
|
||||||
|
|
||||||
from logging import info
|
|
||||||
from random import randint
|
from random import randint
|
||||||
|
|
||||||
from weboob.tools.browser import BrowserUnavailable
|
from weboob.tools.browser import BrowserUnavailable
|
||||||
from weboob.capabilities.dating import Optimization
|
from weboob.capabilities.dating import Optimization
|
||||||
|
from weboob.tools.log import getLogger
|
||||||
|
|
||||||
|
|
||||||
__all__ = ['ProfilesWalker']
|
__all__ = ['ProfilesWalker']
|
||||||
|
|
@ -33,9 +33,10 @@ class ProfilesWalker(Optimization):
|
||||||
self.sched = sched
|
self.sched = sched
|
||||||
self.storage = storage
|
self.storage = storage
|
||||||
self.browser = browser
|
self.browser = browser
|
||||||
|
self.logger = getLogger('walker', browser.logger)
|
||||||
|
|
||||||
self.visited_profiles = set(storage.get('profiles_walker', 'viewed'))
|
self.visited_profiles = set(storage.get('profiles_walker', 'viewed'))
|
||||||
info(u'Loaded %d already visited profiles from storage.' % len(self.visited_profiles))
|
self.logger.info(u'Loaded %d already visited profiles from storage.' % len(self.visited_profiles))
|
||||||
self.profiles_queue = set()
|
self.profiles_queue = set()
|
||||||
|
|
||||||
def save(self):
|
def save(self):
|
||||||
|
|
@ -57,7 +58,7 @@ class ProfilesWalker(Optimization):
|
||||||
try:
|
try:
|
||||||
with self.browser:
|
with self.browser:
|
||||||
profiles_to_visit = self.browser.search_profiles().difference(self.visited_profiles)
|
profiles_to_visit = self.browser.search_profiles().difference(self.visited_profiles)
|
||||||
info(u'Enqueuing profiles to visit: %s' % profiles_to_visit)
|
self.logger.info(u'Enqueuing profiles to visit: %s' % profiles_to_visit)
|
||||||
self.profiles_queue = set(profiles_to_visit)
|
self.profiles_queue = set(profiles_to_visit)
|
||||||
self.save()
|
self.save()
|
||||||
except BrowserUnavailable:
|
except BrowserUnavailable:
|
||||||
|
|
@ -73,7 +74,7 @@ class ProfilesWalker(Optimization):
|
||||||
try:
|
try:
|
||||||
with self.browser:
|
with self.browser:
|
||||||
profile = self.browser.get_profile(id)
|
profile = self.browser.get_profile(id)
|
||||||
info(u'Visited profile %s (%s)' % (profile.get_name(), id))
|
self.logger.info(u'Visited profile %s (%s)' % (profile.get_name(), id))
|
||||||
|
|
||||||
# Get score from the aum_score module
|
# Get score from the aum_score module
|
||||||
#d = self.nucentral_core.callService(context.Context.fromComponent(self), 'aum_score', 'score', profile)
|
#d = self.nucentral_core.callService(context.Context.fromComponent(self), 'aum_score', 'score', profile)
|
||||||
|
|
|
||||||
|
|
@ -17,7 +17,6 @@
|
||||||
|
|
||||||
|
|
||||||
import re
|
import re
|
||||||
from logging import error, warning
|
|
||||||
from weboob.tools.browser import BasePage, BrowserUnavailable
|
from weboob.tools.browser import BasePage, BrowserUnavailable
|
||||||
|
|
||||||
class PageBase(BasePage):
|
class PageBase(BasePage):
|
||||||
|
|
@ -61,7 +60,7 @@ class PageBase(BasePage):
|
||||||
child = tag.childNodes[0].childNodes[0].childNodes[3]
|
child = tag.childNodes[0].childNodes[0].childNodes[3]
|
||||||
return int(child.childNodes[0].childNodes[1].data.replace(' ', '').strip())
|
return int(child.childNodes[0].childNodes[1].data.replace(' ', '').strip())
|
||||||
|
|
||||||
error("Error: I can't find the score :(")
|
self.logger.error("Error: I can't find the score :(")
|
||||||
return '0'
|
return '0'
|
||||||
|
|
||||||
def __get_indicator(self, elementName):
|
def __get_indicator(self, elementName):
|
||||||
|
|
@ -74,14 +73,14 @@ class PageBase(BasePage):
|
||||||
|
|
||||||
if not hasattr(child, 'data'):
|
if not hasattr(child, 'data'):
|
||||||
if child.tagName != u'blink':
|
if child.tagName != u'blink':
|
||||||
warning("Warning: %s counter isn't a blink and hasn't data" % elementName)
|
self.logger.warning("Warning: %s counter isn't a blink and hasn't data" % elementName)
|
||||||
child = child.childNodes[0]
|
child = child.childNodes[0]
|
||||||
if not hasattr(child, 'data'):
|
if not hasattr(child, 'data'):
|
||||||
break
|
break
|
||||||
|
|
||||||
return int(child.data)
|
return int(child.data)
|
||||||
|
|
||||||
error("Error: I can't find the %s counter :(" % elementName)
|
self.logger.error("Error: I can't find the %s counter :(" % elementName)
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
MYNAME_REGEXP = re.compile("Bonjour (.*)")
|
MYNAME_REGEXP = re.compile("Bonjour (.*)")
|
||||||
|
|
@ -95,7 +94,7 @@ class PageBase(BasePage):
|
||||||
if m:
|
if m:
|
||||||
return m.group(1)
|
return m.group(1)
|
||||||
|
|
||||||
warning('Warning: Unable to fetch name')
|
self.logger.warning('Warning: Unable to fetch name')
|
||||||
return '?'
|
return '?'
|
||||||
|
|
||||||
def nb_new_mails(self):
|
def nb_new_mails(self):
|
||||||
|
|
|
||||||
|
|
@ -362,7 +362,7 @@ class ProfilePage(PageBase):
|
||||||
try:
|
try:
|
||||||
fields[label1].put_value(d, value2)
|
fields[label1].put_value(d, value2)
|
||||||
except KeyError:
|
except KeyError:
|
||||||
warning('Unable to find "%s" (%s)' % (label1, repr(label1)))
|
self.logger.warning('Unable to find "%s" (%s)' % (label1, repr(label1)))
|
||||||
elif label1 and label2:
|
elif label1 and label2:
|
||||||
# two titles, so there will have a list of value in
|
# two titles, so there will have a list of value in
|
||||||
# next lines on each columns
|
# next lines on each columns
|
||||||
|
|
|
||||||
|
|
@ -18,8 +18,6 @@
|
||||||
|
|
||||||
from __future__ import with_statement
|
from __future__ import with_statement
|
||||||
|
|
||||||
from logging import warning
|
|
||||||
|
|
||||||
from weboob.capabilities.messages import ICapMessages, Message, Thread
|
from weboob.capabilities.messages import ICapMessages, Message, Thread
|
||||||
from weboob.tools.backend import BaseBackend
|
from weboob.tools.backend import BaseBackend
|
||||||
from weboob.tools.value import Value, ValuesDict
|
from weboob.tools.value import Value, ValuesDict
|
||||||
|
|
@ -52,7 +50,7 @@ class FourChanBackend(BaseBackend, ICapMessages):
|
||||||
id = thread.id
|
id = thread.id
|
||||||
|
|
||||||
if not '.' in id:
|
if not '.' in id:
|
||||||
warning('Malformated ID (%s)' % id)
|
self.logger.warning('Malformated ID (%s)' % id)
|
||||||
return
|
return
|
||||||
|
|
||||||
board, thread_id = self._splitid(id)
|
board, thread_id = self._splitid(id)
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,6 @@
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
import re
|
import re
|
||||||
from logging import warning
|
|
||||||
|
|
||||||
from weboob.tools.browser import BasePage
|
from weboob.tools.browser import BasePage
|
||||||
|
|
||||||
|
|
@ -53,7 +52,7 @@ class BoardPage(BasePage):
|
||||||
if m:
|
if m:
|
||||||
self.board = m.group(1)
|
self.board = m.group(1)
|
||||||
else:
|
else:
|
||||||
warning('Unable to find board')
|
self.logger.warning('Unable to find board')
|
||||||
self.board = 'unknown'
|
self.board = 'unknown'
|
||||||
|
|
||||||
forms = self.document.getroot().cssselect('form')
|
forms = self.document.getroot().cssselect('form')
|
||||||
|
|
@ -65,7 +64,7 @@ class BoardPage(BasePage):
|
||||||
break
|
break
|
||||||
|
|
||||||
if form is None:
|
if form is None:
|
||||||
warning('No delform :(')
|
self.logger.warning('No delform :(')
|
||||||
|
|
||||||
article = None
|
article = None
|
||||||
for div in form.getchildren():
|
for div in form.getchildren():
|
||||||
|
|
|
||||||
|
|
@ -19,11 +19,11 @@
|
||||||
from __future__ import with_statement
|
from __future__ import with_statement
|
||||||
|
|
||||||
from copy import copy
|
from copy import copy
|
||||||
from logging import debug
|
|
||||||
from threading import Thread, Event, RLock, Timer
|
from threading import Thread, Event, RLock, Timer
|
||||||
|
|
||||||
from weboob.capabilities.base import CapBaseObject
|
from weboob.capabilities.base import CapBaseObject
|
||||||
from weboob.tools.misc import get_backtrace
|
from weboob.tools.misc import get_backtrace
|
||||||
|
from weboob.tools.log import getLogger
|
||||||
|
|
||||||
|
|
||||||
__all__ = ['BackendsCall', 'CallErrors']
|
__all__ = ['BackendsCall', 'CallErrors']
|
||||||
|
|
@ -44,6 +44,7 @@ class BackendsCall(object):
|
||||||
@param function backends' method name, or callable object
|
@param function backends' method name, or callable object
|
||||||
@param args, kwargs arguments given to called functions
|
@param args, kwargs arguments given to called functions
|
||||||
"""
|
"""
|
||||||
|
self.logger = getLogger('bcall')
|
||||||
# Store if a backend is finished
|
# Store if a backend is finished
|
||||||
self.backends = {}
|
self.backends = {}
|
||||||
for backend in backends:
|
for backend in backends:
|
||||||
|
|
@ -64,7 +65,7 @@ class BackendsCall(object):
|
||||||
# Create jobs for each backend
|
# Create jobs for each backend
|
||||||
with self.mutex:
|
with self.mutex:
|
||||||
for backend in backends:
|
for backend in backends:
|
||||||
debug('Creating a new thread for %s' % backend)
|
self.logger.debug('Creating a new thread for %s' % backend)
|
||||||
self.threads.append(Timer(0, self._caller, (backend, function, args, kwargs)).start())
|
self.threads.append(Timer(0, self._caller, (backend, function, args, kwargs)).start())
|
||||||
if not backends:
|
if not backends:
|
||||||
self.finish_event.set()
|
self.finish_event.set()
|
||||||
|
|
@ -82,21 +83,21 @@ class BackendsCall(object):
|
||||||
self.response_event.set()
|
self.response_event.set()
|
||||||
|
|
||||||
def _caller(self, backend, function, args, kwargs):
|
def _caller(self, backend, function, args, kwargs):
|
||||||
debug('%s: Thread created successfully' % backend)
|
self.logger.debug('%s: Thread created successfully' % backend)
|
||||||
with backend:
|
with backend:
|
||||||
try:
|
try:
|
||||||
# Call method on backend
|
# Call method on backend
|
||||||
try:
|
try:
|
||||||
debug('%s: Calling function %s' % (backend, function))
|
self.logger.debug('%s: Calling function %s' % (backend, function))
|
||||||
if callable(function):
|
if callable(function):
|
||||||
result = function(backend, *args, **kwargs)
|
result = function(backend, *args, **kwargs)
|
||||||
else:
|
else:
|
||||||
result = getattr(backend, function)(*args, **kwargs)
|
result = getattr(backend, function)(*args, **kwargs)
|
||||||
except Exception, error:
|
except Exception, error:
|
||||||
debug('%s: Called function %s raised an error: %r' % (backend, function, error))
|
self.logger.debug('%s: Called function %s raised an error: %r' % (backend, function, error))
|
||||||
self._store_error(backend, error)
|
self._store_error(backend, error)
|
||||||
else:
|
else:
|
||||||
debug('%s: Called function %s returned: %r' % (backend, function, result))
|
self.logger.debug('%s: Called function %s returned: %r' % (backend, function, result))
|
||||||
|
|
||||||
if hasattr(result, '__iter__') and not isinstance(result, basestring):
|
if hasattr(result, '__iter__') and not isinstance(result, basestring):
|
||||||
# Loop on iterator
|
# Loop on iterator
|
||||||
|
|
|
||||||
|
|
@ -19,12 +19,12 @@
|
||||||
from __future__ import with_statement
|
from __future__ import with_statement
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
from logging import debug, error, exception
|
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
|
||||||
from weboob.capabilities.base import IBaseCap
|
from weboob.capabilities.base import IBaseCap
|
||||||
from weboob.tools.backend import BaseBackend
|
from weboob.tools.backend import BaseBackend
|
||||||
|
from weboob.tools.log import getLogger
|
||||||
|
|
||||||
|
|
||||||
__all__ = ['Module', 'ModulesLoader']
|
__all__ = ['Module', 'ModulesLoader']
|
||||||
|
|
@ -32,6 +32,7 @@ __all__ = ['Module', 'ModulesLoader']
|
||||||
|
|
||||||
class Module(object):
|
class Module(object):
|
||||||
def __init__(self, package):
|
def __init__(self, package):
|
||||||
|
self.logger = getLogger('backend')
|
||||||
self.package = package
|
self.package = package
|
||||||
self.klass = None
|
self.klass = None
|
||||||
for attrname in dir(self.package):
|
for attrname in dir(self.package):
|
||||||
|
|
@ -89,14 +90,15 @@ class Module(object):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def create_instance(self, weboob, instance_name, config, storage):
|
def create_instance(self, weboob, instance_name, config, storage):
|
||||||
backend_instance = self.klass(weboob, instance_name, config, storage)
|
backend_instance = self.klass(weboob, instance_name, config, storage, self.logger)
|
||||||
debug(u'Created backend instance "%s" for backend "%s"' % (instance_name, self.name))
|
self.logger.debug(u'Created backend instance "%s" for backend "%s"' % (instance_name, self.name))
|
||||||
return backend_instance
|
return backend_instance
|
||||||
|
|
||||||
|
|
||||||
class ModulesLoader(object):
|
class ModulesLoader(object):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.loaded = {}
|
self.loaded = {}
|
||||||
|
self.logger = getLogger('modules')
|
||||||
|
|
||||||
def get_or_load_module(self, module_name):
|
def get_or_load_module(self, module_name):
|
||||||
if module_name not in self.loaded:
|
if module_name not in self.loaded:
|
||||||
|
|
@ -129,13 +131,13 @@ class ModulesLoader(object):
|
||||||
except ImportError, e:
|
except ImportError, e:
|
||||||
msg = u'Unable to load module "%s": %s' % (module_name, e)
|
msg = u'Unable to load module "%s": %s' % (module_name, e)
|
||||||
if logging.root.level == logging.DEBUG:
|
if logging.root.level == logging.DEBUG:
|
||||||
exception(msg)
|
self.logger.exception(msg)
|
||||||
return
|
return
|
||||||
else:
|
else:
|
||||||
error(msg)
|
self.logger.error(msg)
|
||||||
return
|
return
|
||||||
if module.name in self.loaded:
|
if module.name in self.loaded:
|
||||||
debug('Module "%s" is already loaded from %s' % (module_name, module.package.__path__[0]))
|
self.logger.debug('Module "%s" is already loaded from %s' % (module_name, module.package.__path__[0]))
|
||||||
return
|
return
|
||||||
self.loaded[module.name] = module
|
self.loaded[module.name] = module
|
||||||
debug('Loaded module "%s" from %s' % (module_name, module.package.__path__[0]))
|
self.logger.debug('Loaded module "%s" from %s' % (module_name, module.package.__path__[0]))
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,6 @@
|
||||||
|
|
||||||
from __future__ import with_statement
|
from __future__ import with_statement
|
||||||
|
|
||||||
from logging import warning
|
|
||||||
import os
|
import os
|
||||||
|
|
||||||
from weboob.core.bcall import BackendsCall
|
from weboob.core.bcall import BackendsCall
|
||||||
|
|
@ -26,6 +25,7 @@ from weboob.core.modules import ModulesLoader
|
||||||
from weboob.core.backendscfg import BackendsConfig
|
from weboob.core.backendscfg import BackendsConfig
|
||||||
from weboob.core.scheduler import Scheduler
|
from weboob.core.scheduler import Scheduler
|
||||||
from weboob.tools.backend import BaseBackend
|
from weboob.tools.backend import BaseBackend
|
||||||
|
from weboob.tools.log import getLogger
|
||||||
|
|
||||||
|
|
||||||
__all__ = ['Weboob']
|
__all__ = ['Weboob']
|
||||||
|
|
@ -36,6 +36,7 @@ class Weboob(object):
|
||||||
BACKENDS_FILENAME = 'backends'
|
BACKENDS_FILENAME = 'backends'
|
||||||
|
|
||||||
def __init__(self, workdir=WORKDIR, backends_filename=None, scheduler=None, storage=None):
|
def __init__(self, workdir=WORKDIR, backends_filename=None, scheduler=None, storage=None):
|
||||||
|
self.logger = getLogger('weboob')
|
||||||
self.workdir = workdir
|
self.workdir = workdir
|
||||||
self.backend_instances = {}
|
self.backend_instances = {}
|
||||||
|
|
||||||
|
|
@ -48,7 +49,7 @@ class Weboob(object):
|
||||||
if not os.path.exists(self.workdir):
|
if not os.path.exists(self.workdir):
|
||||||
os.mkdir(self.workdir, 0700)
|
os.mkdir(self.workdir, 0700)
|
||||||
elif not os.path.isdir(self.workdir):
|
elif not os.path.isdir(self.workdir):
|
||||||
warning(u'"%s" is not a directory' % self.workdir)
|
self.logger.warning(u'"%s" is not a directory' % self.workdir)
|
||||||
|
|
||||||
# Backends loader
|
# Backends loader
|
||||||
self.modules_loader = ModulesLoader()
|
self.modules_loader = ModulesLoader()
|
||||||
|
|
@ -96,15 +97,15 @@ class Weboob(object):
|
||||||
continue
|
continue
|
||||||
module = self.modules_loader.get_or_load_module(module_name)
|
module = self.modules_loader.get_or_load_module(module_name)
|
||||||
if module is None:
|
if module is None:
|
||||||
warning(u'Backend "%s" is referenced in ~/.weboob/backends '
|
self.logger.warning(u'Backend "%s" is referenced in ~/.weboob/backends '
|
||||||
'configuration file, but was not found. '
|
'configuration file, but was not found. '
|
||||||
'Hint: is it installed?' % module_name)
|
'Hint: is it installed?' % module_name)
|
||||||
continue
|
continue
|
||||||
if caps is not None and not module.has_caps(caps):
|
if caps is not None and not module.has_caps(caps):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if instance_name in self.backend_instances:
|
if instance_name in self.backend_instances:
|
||||||
warning(u'Oops, the backend "%s" is already loaded. Unload it before reloading...' % instance_name)
|
self.logger.warning(u'Oops, the backend "%s" is already loaded. Unload it before reloading...' % instance_name)
|
||||||
self.unload_backends(instance_name)
|
self.unload_backends(instance_name)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
@ -186,7 +187,7 @@ class Weboob(object):
|
||||||
else:
|
else:
|
||||||
backends.append(backend)
|
backends.append(backend)
|
||||||
else:
|
else:
|
||||||
warning(u'The "backends" value isn\'t supported: %r' % _backends)
|
self.logger.warning(u'The "backends" value isn\'t supported: %r' % _backends)
|
||||||
|
|
||||||
if 'caps' in kwargs:
|
if 'caps' in kwargs:
|
||||||
caps = kwargs.pop('caps')
|
caps = kwargs.pop('caps')
|
||||||
|
|
|
||||||
|
|
@ -18,8 +18,8 @@
|
||||||
|
|
||||||
from __future__ import with_statement
|
from __future__ import with_statement
|
||||||
|
|
||||||
import logging
|
|
||||||
from threading import Timer, Event, RLock
|
from threading import Timer, Event, RLock
|
||||||
|
from weboob.tools.log import getLogger
|
||||||
|
|
||||||
|
|
||||||
__all__ = ['Scheduler']
|
__all__ = ['Scheduler']
|
||||||
|
|
@ -40,6 +40,7 @@ class IScheduler(object):
|
||||||
|
|
||||||
class Scheduler(IScheduler):
|
class Scheduler(IScheduler):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
|
self.logger = getLogger('scheduler')
|
||||||
self.mutex = RLock()
|
self.mutex = RLock()
|
||||||
self.stop_event = Event()
|
self.stop_event = Event()
|
||||||
self.count = 0
|
self.count = 0
|
||||||
|
|
@ -51,7 +52,7 @@ class Scheduler(IScheduler):
|
||||||
|
|
||||||
with self.mutex:
|
with self.mutex:
|
||||||
self.count += 1
|
self.count += 1
|
||||||
logging.debug('function "%s" will be called in %s seconds' % (function.__name__, interval))
|
self.logger.debug('function "%s" will be called in %s seconds' % (function.__name__, interval))
|
||||||
timer = Timer(interval, self._callback, (self.count, function, args))
|
timer = Timer(interval, self._callback, (self.count, function, args))
|
||||||
self.queue[self.count] = timer
|
self.queue[self.count] = timer
|
||||||
timer.start()
|
timer.start()
|
||||||
|
|
|
||||||
|
|
@ -18,10 +18,10 @@
|
||||||
|
|
||||||
import os
|
import os
|
||||||
from threading import RLock
|
from threading import RLock
|
||||||
from logging import debug
|
|
||||||
|
|
||||||
from weboob.capabilities.base import CapBaseObject, FieldNotFound, IBaseCap, NotLoaded
|
from weboob.capabilities.base import CapBaseObject, FieldNotFound, IBaseCap, NotLoaded
|
||||||
from weboob.tools.misc import iter_fields
|
from weboob.tools.misc import iter_fields
|
||||||
|
from weboob.tools.log import getLogger
|
||||||
|
|
||||||
|
|
||||||
__all__ = ['BaseBackend', 'ObjectNotAvailable']
|
__all__ = ['BaseBackend', 'ObjectNotAvailable']
|
||||||
|
|
@ -95,7 +95,8 @@ class BaseBackend(object):
|
||||||
def __repr__(self):
|
def __repr__(self):
|
||||||
return u"<Backend '%s'>" % self.name
|
return u"<Backend '%s'>" % self.name
|
||||||
|
|
||||||
def __init__(self, weboob, name, config, storage):
|
def __init__(self, weboob, name, config, storage, logger=None):
|
||||||
|
self.logger = getLogger(self.NAME, parent=logger)
|
||||||
self.weboob = weboob
|
self.weboob = weboob
|
||||||
self.name = name
|
self.name = name
|
||||||
self.lock = RLock()
|
self.lock = RLock()
|
||||||
|
|
@ -143,7 +144,7 @@ class BaseBackend(object):
|
||||||
try:
|
try:
|
||||||
import xdg.IconTheme
|
import xdg.IconTheme
|
||||||
except ImportError:
|
except ImportError:
|
||||||
debug(u'Python xdg module was not found. Please install it to read icon files.')
|
self.logger.debug(u'Python xdg module was not found. Please install it to read icon files.')
|
||||||
else:
|
else:
|
||||||
return xdg.IconTheme.getIconPath(self.NAME)
|
return xdg.IconTheme.getIconPath(self.NAME)
|
||||||
|
|
||||||
|
|
@ -178,6 +179,7 @@ class BaseBackend(object):
|
||||||
kwargs['proxy'] = self._private_config['_proxy']
|
kwargs['proxy'] = self._private_config['_proxy']
|
||||||
elif 'HTTP_PROXY' in os.environ:
|
elif 'HTTP_PROXY' in os.environ:
|
||||||
kwargs['proxy'] = os.environ['HTTP_PROXY']
|
kwargs['proxy'] = os.environ['HTTP_PROXY']
|
||||||
|
kwargs['logger'] = self.logger
|
||||||
|
|
||||||
return self.BROWSER(*args, **kwargs)
|
return self.BROWSER(*args, **kwargs)
|
||||||
|
|
||||||
|
|
@ -235,5 +237,5 @@ class BaseBackend(object):
|
||||||
|
|
||||||
for key, value in self.OBJECTS.iteritems():
|
for key, value in self.OBJECTS.iteritems():
|
||||||
if isinstance(obj, key):
|
if isinstance(obj, key):
|
||||||
debug(u'Fill %r with fields: %s' % (obj, missing_fields))
|
self.logger.debug(u'Fill %r with fields: %s' % (obj, missing_fields))
|
||||||
return value(self, obj, missing_fields) or obj
|
return value(self, obj, missing_fields) or obj
|
||||||
|
|
|
||||||
|
|
@ -24,7 +24,7 @@ from weboob.tools.mech import ClientForm
|
||||||
ControlNotFoundError = ClientForm.ControlNotFoundError
|
ControlNotFoundError = ClientForm.ControlNotFoundError
|
||||||
import re
|
import re
|
||||||
import time
|
import time
|
||||||
from logging import warning, debug
|
from logging import warning
|
||||||
from copy import copy
|
from copy import copy
|
||||||
from threading import RLock
|
from threading import RLock
|
||||||
import os
|
import os
|
||||||
|
|
@ -32,6 +32,7 @@ import tempfile
|
||||||
|
|
||||||
from weboob.tools.parsers import get_parser
|
from weboob.tools.parsers import get_parser
|
||||||
from weboob.tools.decorators import retry
|
from weboob.tools.decorators import retry
|
||||||
|
from weboob.tools.log import getLogger
|
||||||
|
|
||||||
# Try to load cookies
|
# Try to load cookies
|
||||||
try:
|
try:
|
||||||
|
|
@ -91,12 +92,13 @@ class BasePage(object):
|
||||||
"""
|
"""
|
||||||
Base page
|
Base page
|
||||||
"""
|
"""
|
||||||
def __init__(self, browser, document, url='', groups=None, group_dict=None):
|
def __init__(self, browser, document, url='', groups=None, group_dict=None, logger=None):
|
||||||
self.browser = browser
|
self.browser = browser
|
||||||
self.document = document
|
self.document = document
|
||||||
self.url = url
|
self.url = url
|
||||||
self.groups = groups
|
self.groups = groups
|
||||||
self.group_dict = group_dict
|
self.group_dict = group_dict
|
||||||
|
self.logger = getLogger('page', logger)
|
||||||
|
|
||||||
def on_loaded(self):
|
def on_loaded(self):
|
||||||
"""
|
"""
|
||||||
|
|
@ -156,7 +158,7 @@ class BaseBrowser(mechanize.Browser):
|
||||||
default_features.remove('_robots')
|
default_features.remove('_robots')
|
||||||
|
|
||||||
def __init__(self, username=None, password=None, firefox_cookies=None,
|
def __init__(self, username=None, password=None, firefox_cookies=None,
|
||||||
parser=None, history=NoHistory(), proxy=None):
|
parser=None, history=NoHistory(), proxy=None, logger=None):
|
||||||
"""
|
"""
|
||||||
Constructor of Browser.
|
Constructor of Browser.
|
||||||
|
|
||||||
|
|
@ -167,8 +169,11 @@ class BaseBrowser(mechanize.Browser):
|
||||||
@param parser [IParser] parser to use on HTML files.
|
@param parser [IParser] parser to use on HTML files.
|
||||||
@param hisory [object] History manager. Default value is an object
|
@param hisory [object] History manager. Default value is an object
|
||||||
which does not keep history.
|
which does not keep history.
|
||||||
|
@param proxy [str] proxy URL to use.
|
||||||
"""
|
"""
|
||||||
mechanize.Browser.__init__(self, history=history)
|
mechanize.Browser.__init__(self, history=history)
|
||||||
|
self.logger = getLogger('browser', logger)
|
||||||
|
|
||||||
self.addheaders = [
|
self.addheaders = [
|
||||||
['User-agent', self.USER_AGENT]
|
['User-agent', self.USER_AGENT]
|
||||||
]
|
]
|
||||||
|
|
@ -242,7 +247,7 @@ class BaseBrowser(mechanize.Browser):
|
||||||
Open an URL but do not create a Page object.
|
Open an URL but do not create a Page object.
|
||||||
"""
|
"""
|
||||||
if_fail = kwargs.pop('if_fail', 'raise')
|
if_fail = kwargs.pop('if_fail', 'raise')
|
||||||
debug('Opening URL "%s", %s' % (args, kwargs))
|
self.logger.debug('Opening URL "%s", %s' % (args, kwargs))
|
||||||
try:
|
try:
|
||||||
return mechanize.Browser.open_novisit(self, *args, **kwargs)
|
return mechanize.Browser.open_novisit(self, *args, **kwargs)
|
||||||
except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError, BadStatusLine), e:
|
except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError, BadStatusLine), e:
|
||||||
|
|
@ -280,7 +285,7 @@ class BaseBrowser(mechanize.Browser):
|
||||||
fd, path = tempfile.mkstemp(prefix="response", dir=tmpdir)
|
fd, path = tempfile.mkstemp(prefix="response", dir=tmpdir)
|
||||||
with os.fdopen(fd, 'w') as f:
|
with os.fdopen(fd, 'w') as f:
|
||||||
f.write(result.read())
|
f.write(result.read())
|
||||||
debug("Response saved to %s" % path)
|
self.logger.debug("Response saved to %s" % path)
|
||||||
result.seek(0)
|
result.seek(0)
|
||||||
|
|
||||||
def submit(self, *args, **kwargs):
|
def submit(self, *args, **kwargs):
|
||||||
|
|
@ -366,23 +371,23 @@ class BaseBrowser(mechanize.Browser):
|
||||||
#if isinstance(data, unicode):
|
#if isinstance(data, unicode):
|
||||||
# data = data.encode('utf-8')
|
# data = data.encode('utf-8')
|
||||||
#print data
|
#print data
|
||||||
warning('Oh my fucking god, there isn\'t any page corresponding to URL %s' % result.geturl())
|
self.logger.warning('Oh my fucking god, there isn\'t any page corresponding to URL %s' % result.geturl())
|
||||||
self.save_response(result)
|
self.save_response(result)
|
||||||
|
|
||||||
return
|
return
|
||||||
|
|
||||||
debug('[user_id=%s] Went on %s' % (self.username, result.geturl()))
|
self.logger.debug('[user_id=%s] Went on %s' % (self.username, result.geturl()))
|
||||||
self.last_update = time.time()
|
self.last_update = time.time()
|
||||||
|
|
||||||
if self.SAVE_RESPONSES:
|
if self.SAVE_RESPONSES:
|
||||||
self.save_response(result)
|
self.save_response(result)
|
||||||
|
|
||||||
document = self.get_document(result)
|
document = self.get_document(result)
|
||||||
self.page = pageCls(self, document, result.geturl(), groups=page_groups, group_dict=page_group_dict)
|
self.page = pageCls(self, document, result.geturl(), groups=page_groups, group_dict=page_group_dict, logger=self.logger)
|
||||||
self.page.on_loaded()
|
self.page.on_loaded()
|
||||||
|
|
||||||
if self.password is not None and not self.is_logged():
|
if self.password is not None and not self.is_logged():
|
||||||
debug('!! Relogin !!')
|
self.logger.debug('!! Relogin !!')
|
||||||
self.login()
|
self.login()
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue