move core files to weboob.core
This commit is contained in:
parent
944662c4bf
commit
1955d1be59
25 changed files with 66 additions and 44 deletions
|
|
@ -23,7 +23,7 @@ import os
|
||||||
from logging import warning
|
from logging import warning
|
||||||
from time import sleep
|
from time import sleep
|
||||||
|
|
||||||
from weboob.backend import BaseBackend
|
from weboob.core.backend import BaseBackend
|
||||||
from weboob.capabilities.chat import ICapChat
|
from weboob.capabilities.chat import ICapChat
|
||||||
from weboob.capabilities.messages import ICapMessages, ICapMessagesReply, Message
|
from weboob.capabilities.messages import ICapMessages, ICapMessagesReply, Message
|
||||||
from weboob.capabilities.dating import ICapDating, StatusField
|
from weboob.capabilities.dating import ICapDating, StatusField
|
||||||
|
|
|
||||||
|
|
@ -17,7 +17,7 @@
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
|
||||||
from weboob.backend import BaseBackend
|
from weboob.core.backend import BaseBackend
|
||||||
from weboob.capabilities.bank import ICapBank, AccountNotFound
|
from weboob.capabilities.bank import ICapBank, AccountNotFound
|
||||||
|
|
||||||
from .browser import BNPorc
|
from .browser import BNPorc
|
||||||
|
|
|
||||||
|
|
@ -16,7 +16,7 @@
|
||||||
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||||
|
|
||||||
|
|
||||||
from weboob.backend import BaseBackend
|
from weboob.core.backend import BaseBackend
|
||||||
from weboob.capabilities.travel import ICapTravel, Station, Departure
|
from weboob.capabilities.travel import ICapTravel, Station, Departure
|
||||||
|
|
||||||
from .browser import CanalTP
|
from .browser import CanalTP
|
||||||
|
|
|
||||||
|
|
@ -16,7 +16,7 @@
|
||||||
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||||
|
|
||||||
|
|
||||||
from weboob.backend import BaseBackend
|
from weboob.core.backend import BaseBackend
|
||||||
from weboob.capabilities.bank import ICapBank, AccountNotFound
|
from weboob.capabilities.bank import ICapBank, AccountNotFound
|
||||||
|
|
||||||
from .browser import Cragr
|
from .browser import Cragr
|
||||||
|
|
|
||||||
|
|
@ -17,7 +17,7 @@
|
||||||
|
|
||||||
from __future__ import with_statement
|
from __future__ import with_statement
|
||||||
|
|
||||||
from weboob.backend import BaseBackend
|
from weboob.core.backend import BaseBackend
|
||||||
from weboob.tools.browser import BrowserUnavailable
|
from weboob.tools.browser import BrowserUnavailable
|
||||||
from weboob.capabilities.messages import ICapMessages, ICapMessagesReply, Message
|
from weboob.capabilities.messages import ICapMessages, ICapMessagesReply, Message
|
||||||
|
|
||||||
|
|
@ -28,7 +28,7 @@ class DLFPBackend(BaseBackend, ICapMessages, ICapMessagesReply):
|
||||||
NAME = 'dlfp'
|
NAME = 'dlfp'
|
||||||
MAINTAINER = 'Romain Bignon'
|
MAINTAINER = 'Romain Bignon'
|
||||||
EMAIL = 'romain@peerfuse.org'
|
EMAIL = 'romain@peerfuse.org'
|
||||||
VERSION = '1.0'
|
VERSION = '0.1'
|
||||||
LICENSE = 'GPLv3'
|
LICENSE = 'GPLv3'
|
||||||
DESCRIPTION = "Da Linux French Page"
|
DESCRIPTION = "Da Linux French Page"
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -19,7 +19,7 @@ from __future__ import with_statement
|
||||||
|
|
||||||
from logging import warning
|
from logging import warning
|
||||||
|
|
||||||
from weboob.backend import BaseBackend
|
from weboob.core.backend import BaseBackend
|
||||||
from weboob.capabilities.messages import ICapMessages, Message
|
from weboob.capabilities.messages import ICapMessages, Message
|
||||||
|
|
||||||
from .browser import FourChan
|
from .browser import FourChan
|
||||||
|
|
@ -28,7 +28,7 @@ class FourChanBackend(BaseBackend, ICapMessages):
|
||||||
NAME = 'fourchan'
|
NAME = 'fourchan'
|
||||||
MAINTAINER = 'Romain Bignon'
|
MAINTAINER = 'Romain Bignon'
|
||||||
EMAIL = 'romain@peerfuse.org'
|
EMAIL = 'romain@peerfuse.org'
|
||||||
VERSION = '1.0'
|
VERSION = '0.1'
|
||||||
LICENSE = 'GPLv3'
|
LICENSE = 'GPLv3'
|
||||||
DESCRIPTION = "4chan website"
|
DESCRIPTION = "4chan website"
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -16,7 +16,7 @@
|
||||||
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||||
|
|
||||||
|
|
||||||
from weboob.backend import BaseBackend
|
from weboob.core.backend import BaseBackend
|
||||||
from weboob.capabilities.torrent import ICapTorrent
|
from weboob.capabilities.torrent import ICapTorrent
|
||||||
|
|
||||||
from .browser import GazelleBrowser
|
from .browser import GazelleBrowser
|
||||||
|
|
|
||||||
|
|
@ -16,7 +16,7 @@
|
||||||
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||||
|
|
||||||
|
|
||||||
from weboob.backend import BaseBackend
|
from weboob.core.backend import BaseBackend
|
||||||
from weboob.capabilities.video import ICapVideo
|
from weboob.capabilities.video import ICapVideo
|
||||||
|
|
||||||
from .browser import InaBrowser
|
from .browser import InaBrowser
|
||||||
|
|
|
||||||
|
|
@ -16,7 +16,7 @@
|
||||||
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||||
|
|
||||||
|
|
||||||
from weboob.backend import BaseBackend
|
from weboob.core.backend import BaseBackend
|
||||||
from weboob.capabilities.travel import ICapTravel, Station, Departure
|
from weboob.capabilities.travel import ICapTravel, Station, Departure
|
||||||
|
|
||||||
from .browser import Transilien
|
from .browser import Transilien
|
||||||
|
|
|
||||||
|
|
@ -16,7 +16,7 @@
|
||||||
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||||
|
|
||||||
|
|
||||||
from weboob.backend import BaseBackend
|
from weboob.core.backend import BaseBackend
|
||||||
from weboob.capabilities.video import ICapVideo
|
from weboob.capabilities.video import ICapVideo
|
||||||
|
|
||||||
from .browser import YoujizzBrowser
|
from .browser import YoujizzBrowser
|
||||||
|
|
|
||||||
|
|
@ -16,7 +16,7 @@
|
||||||
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||||
|
|
||||||
|
|
||||||
from weboob.backend import BaseBackend
|
from weboob.core.backend import BaseBackend
|
||||||
from weboob.capabilities.video import ICapVideo
|
from weboob.capabilities.video import ICapVideo
|
||||||
|
|
||||||
from .browser import YoupornBrowser
|
from .browser import YoupornBrowser
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
from weboob.backend import BaseBackend
|
from weboob.core.backend import BaseBackend
|
||||||
from weboob.capabilities.video import ICapVideo
|
from weboob.capabilities.video import ICapVideo
|
||||||
|
|
||||||
from .browser import YoutubeBrowser
|
from .browser import YoutubeBrowser
|
||||||
|
|
|
||||||
|
|
@ -22,7 +22,7 @@ from xml.dom import minidom
|
||||||
# TODO store datetime objects instead of strings
|
# TODO store datetime objects instead of strings
|
||||||
# from datetime import datetime
|
# from datetime import datetime
|
||||||
|
|
||||||
from weboob.backend import BaseBackend
|
from weboob.core.backend import BaseBackend
|
||||||
from weboob.capabilities.weather import ICapWeather, CityNotFound, Current, Forecast
|
from weboob.capabilities.weather import ICapWeather, CityNotFound, Current, Forecast
|
||||||
|
|
||||||
class YWeatherBackend(BaseBackend, ICapWeather):
|
class YWeatherBackend(BaseBackend, ICapWeather):
|
||||||
|
|
|
||||||
19
weboob/core/__init__.py
Normal file
19
weboob/core/__init__.py
Normal file
|
|
@ -0,0 +1,19 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
# Copyright(C) 2010 Christophe Benz
|
||||||
|
#
|
||||||
|
# This program is free software; you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU General Public License as published by
|
||||||
|
# the Free Software Foundation, version 3 of the License.
|
||||||
|
#
|
||||||
|
# This program is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU General Public License
|
||||||
|
# along with this program; if not, write to the Free Software
|
||||||
|
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||||
|
|
||||||
|
|
||||||
|
from .bcall import CallErrors
|
||||||
166
weboob/core/backend.py
Normal file
166
weboob/core/backend.py
Normal file
|
|
@ -0,0 +1,166 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
# Copyright(C) 2010 Romain Bignon
|
||||||
|
#
|
||||||
|
# This program is free software; you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU General Public License as published by
|
||||||
|
# the Free Software Foundation, version 3 of the License.
|
||||||
|
#
|
||||||
|
# This program is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU General Public License
|
||||||
|
# along with this program; if not, write to the Free Software
|
||||||
|
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||||
|
|
||||||
|
|
||||||
|
import re
|
||||||
|
import os
|
||||||
|
from threading import RLock
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ['BackendStorage', 'BaseBackend']
|
||||||
|
|
||||||
|
|
||||||
|
class BackendStorage(object):
|
||||||
|
def __init__(self, name, storage):
|
||||||
|
self.name = name
|
||||||
|
self.storage = storage
|
||||||
|
|
||||||
|
def set(self, *args):
|
||||||
|
if self.storage:
|
||||||
|
return self.storage.set('backends', self.name, *args)
|
||||||
|
|
||||||
|
def get(self, *args, **kwargs):
|
||||||
|
if self.storage:
|
||||||
|
return self.storage.get('backends', self.name, *args, **kwargs)
|
||||||
|
else:
|
||||||
|
return kwargs.get('default', None)
|
||||||
|
|
||||||
|
def load(self, default):
|
||||||
|
if self.storage:
|
||||||
|
return self.storage.load('backends', self.name, default)
|
||||||
|
|
||||||
|
def save(self):
|
||||||
|
if self.storage:
|
||||||
|
return self.storage.save('backends', self.name)
|
||||||
|
|
||||||
|
class BaseBackend(object):
|
||||||
|
# Module name.
|
||||||
|
NAME = None
|
||||||
|
# Name of the maintainer of this module.
|
||||||
|
MAINTAINER = '<unspecifier>'
|
||||||
|
# Email address of the maintainer.
|
||||||
|
EMAIL = '<unspecified>'
|
||||||
|
# Version of module (for information only).
|
||||||
|
VERSION = '<unspecified>'
|
||||||
|
# Description
|
||||||
|
DESCRIPTION = '<unspecified>'
|
||||||
|
# License of this module.
|
||||||
|
LICENSE = '<unspecified>'
|
||||||
|
# Icon file path
|
||||||
|
ICON = ''
|
||||||
|
# Configuration required for this module. # Values must be ConfigField
|
||||||
|
# objects.
|
||||||
|
CONFIG = {}
|
||||||
|
# Storage
|
||||||
|
STORAGE = {}
|
||||||
|
# Browser class
|
||||||
|
BROWSER = None
|
||||||
|
# Test class
|
||||||
|
TEST = None
|
||||||
|
|
||||||
|
class ConfigField(object):
|
||||||
|
def __init__(self, default=None, is_masked=False, regexp=None, description=None):
|
||||||
|
self.default = default
|
||||||
|
self.is_masked = is_masked
|
||||||
|
self.regexp = regexp
|
||||||
|
self.description = description
|
||||||
|
|
||||||
|
class ConfigError(Exception): pass
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
self.lock.acquire()
|
||||||
|
|
||||||
|
def __exit__(self, t, v, tb):
|
||||||
|
self.lock.release()
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
return u"<Backend '%s'>" % self.name
|
||||||
|
|
||||||
|
def __init__(self, weboob, name, config, storage):
|
||||||
|
self.weboob = weboob
|
||||||
|
self.name = name
|
||||||
|
self.lock = RLock()
|
||||||
|
|
||||||
|
# Private fields (which start with '_')
|
||||||
|
self._private_config = dict([(key, value) for key, value in config.iteritems() if key.startswith('_')])
|
||||||
|
|
||||||
|
# Configuration of backend
|
||||||
|
self.config = {}
|
||||||
|
for name, field in self.CONFIG.iteritems():
|
||||||
|
value = config.get(name, field.default)
|
||||||
|
|
||||||
|
if value is None:
|
||||||
|
raise BaseBackend.ConfigError('Missing parameter "%s" (%s)' % (name, field.description))
|
||||||
|
|
||||||
|
if field.regexp and not re.match(field.regexp, str(value)):
|
||||||
|
raise BaseBackend.ConfigError('Value of "%s" does not match regexp "%s"' % (name, field.regexp))
|
||||||
|
|
||||||
|
if not field.default is None:
|
||||||
|
if isinstance(field.default, bool) and not isinstance(value, bool):
|
||||||
|
value = value.lower() in ('1', 'true', 'on', 'yes')
|
||||||
|
elif isinstance(field.default, int) and not isinstance(value, int):
|
||||||
|
value = int(value)
|
||||||
|
elif isinstance(field.default, float) and not isinstance(value, float):
|
||||||
|
value = float(value)
|
||||||
|
self.config[name] = value
|
||||||
|
self.storage = BackendStorage(self.name, storage)
|
||||||
|
self.storage.load(self.STORAGE)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def browser(self):
|
||||||
|
"""
|
||||||
|
Attribute 'browser'. The browser is created at the first call
|
||||||
|
of this attribute, to avoid useless pages access.
|
||||||
|
|
||||||
|
Note that the 'default_browser' method is called to create it.
|
||||||
|
"""
|
||||||
|
if not hasattr(self, '_browser'):
|
||||||
|
self._browser = self.default_browser()
|
||||||
|
return self._browser
|
||||||
|
|
||||||
|
def default_browser(self):
|
||||||
|
"""
|
||||||
|
Method to overload to build the default browser in
|
||||||
|
attribute 'browser'.
|
||||||
|
"""
|
||||||
|
return self.build_browser()
|
||||||
|
|
||||||
|
def build_browser(self, *args, **kwargs):
|
||||||
|
"""
|
||||||
|
Build a browser from the BROWSER class attribute and the
|
||||||
|
given arguments.
|
||||||
|
"""
|
||||||
|
if not self.BROWSER:
|
||||||
|
return None
|
||||||
|
|
||||||
|
if '_proxy' in self._private_config:
|
||||||
|
kwargs['proxy'] = self._private_config['_proxy']
|
||||||
|
elif 'HTTP_PROXY' in os.environ:
|
||||||
|
kwargs['proxy'] = os.environ['HTTP_PROXY']
|
||||||
|
|
||||||
|
return self.BROWSER(*args, **kwargs)
|
||||||
|
|
||||||
|
def has_caps(self, *caps):
|
||||||
|
for c in caps:
|
||||||
|
if isinstance(self, c):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def get_test(self):
|
||||||
|
if not self.TEST:
|
||||||
|
return None
|
||||||
|
return self.TEST(self)
|
||||||
188
weboob/core/bcall.py
Normal file
188
weboob/core/bcall.py
Normal file
|
|
@ -0,0 +1,188 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
# Copyright(C) 2010 Romain Bignon, Christophe Benz
|
||||||
|
#
|
||||||
|
# This program is free software; you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU General Public License as published by
|
||||||
|
# the Free Software Foundation, version 3 of the License.
|
||||||
|
#
|
||||||
|
# This program is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU General Public License
|
||||||
|
# along with this program; if not, write to the Free Software
|
||||||
|
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||||
|
|
||||||
|
|
||||||
|
from __future__ import with_statement
|
||||||
|
|
||||||
|
from copy import copy
|
||||||
|
import logging
|
||||||
|
from logging import debug
|
||||||
|
from threading import Thread, Event, RLock, Timer
|
||||||
|
from weboob.tools.misc import get_backtrace
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ['BackendsCall', 'CallErrors']
|
||||||
|
|
||||||
|
|
||||||
|
class CallErrors(Exception):
|
||||||
|
def __init__(self, errors):
|
||||||
|
Exception.__init__(self, u'These errors have been raised in backend threads '\
|
||||||
|
'(use --debug option to print backtraces):\n%s' % (
|
||||||
|
u'\n'.join((u' * %s: %s%s' % (backend, error, backtrace + '\n'
|
||||||
|
if logging.root.level == logging.DEBUG else ''))
|
||||||
|
for backend, error, backtrace in errors)))
|
||||||
|
self.errors = copy(errors)
|
||||||
|
|
||||||
|
def __iter__(self):
|
||||||
|
return self.errors.__iter__()
|
||||||
|
|
||||||
|
|
||||||
|
class BackendsCall(object):
|
||||||
|
def __init__(self, backends, function, *args, **kwargs):
|
||||||
|
"""
|
||||||
|
@param backends list of backends to call
|
||||||
|
@param function backends' method name, or callable object
|
||||||
|
@param args, kwargs arguments given to called functions
|
||||||
|
"""
|
||||||
|
# Store if a backend is finished
|
||||||
|
self.backends = {}
|
||||||
|
for backend in backends:
|
||||||
|
self.backends[backend.name] = False
|
||||||
|
# Global mutex on object
|
||||||
|
self.mutex = RLock()
|
||||||
|
# Event set when every backends have give their data
|
||||||
|
self.finish_event = Event()
|
||||||
|
# Event set when there are new responses
|
||||||
|
self.response_event = Event()
|
||||||
|
# Waiting responses
|
||||||
|
self.responses = []
|
||||||
|
# Errors
|
||||||
|
self.errors = []
|
||||||
|
# Threads
|
||||||
|
self.threads = []
|
||||||
|
|
||||||
|
# Create jobs for each backend
|
||||||
|
with self.mutex:
|
||||||
|
for backend in backends:
|
||||||
|
debug('Creating a new thread for %s' % backend)
|
||||||
|
self.threads.append(Timer(0, self._caller, (backend, function, args, kwargs)).start())
|
||||||
|
if not backends:
|
||||||
|
self.finish_event.set()
|
||||||
|
|
||||||
|
def _store_error(self, backend, error):
|
||||||
|
with self.mutex:
|
||||||
|
backtrace = get_backtrace(error)
|
||||||
|
self.errors.append((backend, error, backtrace))
|
||||||
|
|
||||||
|
def _store_result(self, backend, result):
|
||||||
|
with self.mutex:
|
||||||
|
self.responses.append((backend, result))
|
||||||
|
self.response_event.set()
|
||||||
|
|
||||||
|
def _caller(self, backend, function, args, kwargs):
|
||||||
|
debug('%s: Thread created successfully' % backend)
|
||||||
|
with backend:
|
||||||
|
try:
|
||||||
|
# Call method on backend
|
||||||
|
try:
|
||||||
|
debug('%s: Calling function %s' % (backend, function))
|
||||||
|
if callable(function):
|
||||||
|
result = function(backend, *args, **kwargs)
|
||||||
|
else:
|
||||||
|
result = getattr(backend, function)(*args, **kwargs)
|
||||||
|
except Exception, error:
|
||||||
|
self._store_error(backend, error)
|
||||||
|
else:
|
||||||
|
debug('%s: Called function %s returned: "%s"' % (backend, function, result))
|
||||||
|
|
||||||
|
if hasattr(result, '__iter__'):
|
||||||
|
# Loop on iterator
|
||||||
|
try:
|
||||||
|
for subresult in result:
|
||||||
|
# Lock mutex only in loop in case the iterator is slow
|
||||||
|
# (for example if backend do some parsing operations)
|
||||||
|
self._store_result(backend, subresult)
|
||||||
|
except Exception, error:
|
||||||
|
self._store_error(backend, error)
|
||||||
|
else:
|
||||||
|
self._store_result(backend, result)
|
||||||
|
finally:
|
||||||
|
with self.mutex:
|
||||||
|
# This backend is now finished
|
||||||
|
self.backends[backend.name] = True
|
||||||
|
for finished in self.backends.itervalues():
|
||||||
|
if not finished:
|
||||||
|
return
|
||||||
|
self.finish_event.set()
|
||||||
|
self.response_event.set()
|
||||||
|
|
||||||
|
def _callback_thread_run(self, callback, errback):
|
||||||
|
responses = []
|
||||||
|
while not self.finish_event.isSet() or self.response_event.isSet():
|
||||||
|
self.response_event.wait()
|
||||||
|
with self.mutex:
|
||||||
|
responses = self.responses
|
||||||
|
self.responses = []
|
||||||
|
|
||||||
|
# Reset event
|
||||||
|
self.response_event.clear()
|
||||||
|
|
||||||
|
# Consume responses
|
||||||
|
while responses:
|
||||||
|
callback(*responses.pop(0))
|
||||||
|
|
||||||
|
if errback:
|
||||||
|
with self.mutex:
|
||||||
|
while self.errors:
|
||||||
|
errback(*self.errors.pop(0))
|
||||||
|
|
||||||
|
callback(None, None)
|
||||||
|
|
||||||
|
def callback_thread(self, callback, errback=None):
|
||||||
|
"""
|
||||||
|
Call this method to create a thread which will callback a
|
||||||
|
specified function everytimes a new result comes.
|
||||||
|
|
||||||
|
When the process is over, the function will be called with
|
||||||
|
both arguments set to None.
|
||||||
|
|
||||||
|
The functions prototypes:
|
||||||
|
def callback(backend, result)
|
||||||
|
def errback(backend, error)
|
||||||
|
|
||||||
|
"""
|
||||||
|
thread = Thread(target=self._callback_thread_run, args=(callback, errback))
|
||||||
|
thread.start()
|
||||||
|
return thread
|
||||||
|
|
||||||
|
def wait(self):
|
||||||
|
self.finish_event.wait()
|
||||||
|
|
||||||
|
with self.mutex:
|
||||||
|
if self.errors:
|
||||||
|
raise CallErrors(self.errors)
|
||||||
|
|
||||||
|
def __iter__(self):
|
||||||
|
# Don't know how to factorize with _callback_thread_run
|
||||||
|
responses = []
|
||||||
|
while not self.finish_event.isSet() or self.response_event.isSet():
|
||||||
|
self.response_event.wait()
|
||||||
|
with self.mutex:
|
||||||
|
responses = self.responses
|
||||||
|
self.responses = []
|
||||||
|
|
||||||
|
# Reset event
|
||||||
|
self.response_event.clear()
|
||||||
|
|
||||||
|
# Consume responses
|
||||||
|
while responses:
|
||||||
|
yield responses.pop(0)
|
||||||
|
|
||||||
|
# Raise errors
|
||||||
|
with self.mutex:
|
||||||
|
if self.errors:
|
||||||
|
raise CallErrors(self.errors)
|
||||||
178
weboob/core/engine.py
Normal file
178
weboob/core/engine.py
Normal file
|
|
@ -0,0 +1,178 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
# Copyright(C) 2010 Romain Bignon
|
||||||
|
#
|
||||||
|
# This program is free software; you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU General Public License as published by
|
||||||
|
# the Free Software Foundation, version 3 of the License.
|
||||||
|
#
|
||||||
|
# This program is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU General Public License
|
||||||
|
# along with this program; if not, write to the Free Software
|
||||||
|
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||||
|
|
||||||
|
|
||||||
|
from __future__ import with_statement
|
||||||
|
|
||||||
|
from logging import warning
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from weboob.core.bcall import BackendsCall
|
||||||
|
from weboob.core.modules import ModulesLoader, BackendsConfig
|
||||||
|
from weboob.core.backend import BaseBackend
|
||||||
|
from weboob.core.scheduler import Scheduler
|
||||||
|
|
||||||
|
if sys.version_info[:2] <= (2, 5):
|
||||||
|
import weboob.tools.property
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ['Weboob']
|
||||||
|
|
||||||
|
|
||||||
|
class Weboob(object):
|
||||||
|
WORKDIR = os.path.join(os.path.expanduser('~'), '.weboob')
|
||||||
|
BACKENDS_FILENAME = 'backends'
|
||||||
|
|
||||||
|
def __init__(self, workdir=WORKDIR, backends_filename=None, scheduler=None):
|
||||||
|
self.workdir = workdir
|
||||||
|
self.backends = {}
|
||||||
|
|
||||||
|
# Scheduler
|
||||||
|
if scheduler is None:
|
||||||
|
scheduler = Scheduler()
|
||||||
|
self.scheduler = scheduler
|
||||||
|
|
||||||
|
# Create WORKDIR
|
||||||
|
if not os.path.exists(self.workdir):
|
||||||
|
os.mkdir(self.workdir, 0700)
|
||||||
|
elif not os.path.isdir(self.workdir):
|
||||||
|
warning('"%s" is not a directory' % self.workdir)
|
||||||
|
|
||||||
|
# Modules loader
|
||||||
|
self.modules_loader = ModulesLoader()
|
||||||
|
|
||||||
|
# Backends config
|
||||||
|
if not backends_filename:
|
||||||
|
backends_filename = os.path.join(self.workdir, self.BACKENDS_FILENAME)
|
||||||
|
elif not backends_filename.startswith('/'):
|
||||||
|
backends_filename = os.path.join(self.workdir, backends_filename)
|
||||||
|
self.backends_config = BackendsConfig(backends_filename)
|
||||||
|
|
||||||
|
def load_backends(self, caps=None, names=None, storage=None):
|
||||||
|
loaded_backends = {}
|
||||||
|
for name, _type, params in self.backends_config.iter_backends():
|
||||||
|
try:
|
||||||
|
module = self.modules_loader.get_or_load_module(_type)
|
||||||
|
except KeyError:
|
||||||
|
warning(u'Unable to find module "%s" for backend "%s"' % (_type, name))
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Check conditions
|
||||||
|
if (not caps is None and not module.has_caps(caps)) or \
|
||||||
|
(not names is None and not name in names):
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
self.backends[name] = module.create_backend(self, name, params, storage)
|
||||||
|
loaded_backends[name] = self.backends[name]
|
||||||
|
except Exception, e:
|
||||||
|
warning(u'Unable to load "%s" backend: %s. filename=%s' % (name, e, self.backends_config.confpath))
|
||||||
|
|
||||||
|
return loaded_backends
|
||||||
|
|
||||||
|
def load_modules(self, caps=None, names=None, storage=None):
|
||||||
|
loaded_backends = {}
|
||||||
|
self.modules_loader.load()
|
||||||
|
for name, module in self.modules_loader.modules.iteritems():
|
||||||
|
if (caps is None or module.has_caps(caps)) and \
|
||||||
|
(names is None or module.get_name() in names):
|
||||||
|
try:
|
||||||
|
name = module.get_name()
|
||||||
|
self.backends[name] = module.create_backend(self, name, {}, storage)
|
||||||
|
loaded_backends[name] = self.backends[name]
|
||||||
|
except Exception, e:
|
||||||
|
warning(u'Unable to load "%s" module as backend with no config: %s' % (name, e))
|
||||||
|
return loaded_backends
|
||||||
|
|
||||||
|
def iter_backends(self, caps=None):
|
||||||
|
"""
|
||||||
|
Iter on each backends.
|
||||||
|
|
||||||
|
Note: each backend is locked when it is returned.
|
||||||
|
|
||||||
|
@param caps Optional list of capabilities to select backends
|
||||||
|
@return iterator on selected backends.
|
||||||
|
"""
|
||||||
|
for name, backend in sorted(self.backends.iteritems()):
|
||||||
|
if caps is None or backend.has_caps(caps):
|
||||||
|
with backend:
|
||||||
|
yield backend
|
||||||
|
|
||||||
|
def do(self, function, *args, **kwargs):
|
||||||
|
"""
|
||||||
|
Do calls on loaded backends with specified arguments, in separated
|
||||||
|
threads.
|
||||||
|
|
||||||
|
This function has two modes:
|
||||||
|
- If 'function' is a string, it calls the method with this name on
|
||||||
|
each backends with the specified arguments;
|
||||||
|
- If 'function' is a callable, it calls it in a separated thread with
|
||||||
|
the locked backend instance at first arguments, and *args and
|
||||||
|
**kwargs.
|
||||||
|
|
||||||
|
@param function backend's method name, or callable object
|
||||||
|
@return an iterator of results
|
||||||
|
"""
|
||||||
|
backends = list(self.iter_backends())
|
||||||
|
return BackendsCall(backends, function, *args, **kwargs)
|
||||||
|
|
||||||
|
def do_caps(self, caps, function, *args, **kwargs):
|
||||||
|
"""
|
||||||
|
Do calls on loaded modules with the specified capabilities, in
|
||||||
|
separated threads.
|
||||||
|
|
||||||
|
See also documentation of the 'do' method.
|
||||||
|
|
||||||
|
@param caps list of caps or cap to select backends
|
||||||
|
@param function backend's method name, or callable object
|
||||||
|
@return an iterator of results
|
||||||
|
"""
|
||||||
|
backends = list(self.iter_backends(caps))
|
||||||
|
return BackendsCall(backends, function, *args, **kwargs)
|
||||||
|
|
||||||
|
def do_backends(self, backends, function, *args, **kwargs):
|
||||||
|
if backends is None:
|
||||||
|
backends = list(self.iter_backends())
|
||||||
|
elif isinstance(backends, BaseBackend):
|
||||||
|
backends = [backends]
|
||||||
|
elif isinstance(backends, (str,unicode)):
|
||||||
|
backends = [backend for backend in self.iter_backends() if backend.name == backends]
|
||||||
|
elif isinstance(backends, (list,tuple)):
|
||||||
|
old_backends = backends
|
||||||
|
backends = []
|
||||||
|
for b in old_backends:
|
||||||
|
if isinstance(b, (str,unicode)):
|
||||||
|
try:
|
||||||
|
backends.append(self.backends[self.backends.index(b)])
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
backends.append(b)
|
||||||
|
return BackendsCall(backends, function, *args, **kwargs)
|
||||||
|
|
||||||
|
def schedule(self, interval, function, *args):
|
||||||
|
return self.scheduler.schedule(interval, function, *args)
|
||||||
|
|
||||||
|
def repeat(self, interval, function, *args):
|
||||||
|
return self.scheduler.repeat(interval, function, *args)
|
||||||
|
|
||||||
|
def want_stop(self):
|
||||||
|
return self.scheduler.want_stop()
|
||||||
|
|
||||||
|
def loop(self):
|
||||||
|
return self.scheduler.run()
|
||||||
179
weboob/core/modules.py
Normal file
179
weboob/core/modules.py
Normal file
|
|
@ -0,0 +1,179 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
# Copyright(C) 2010 Romain Bignon
|
||||||
|
#
|
||||||
|
# This program is free software; you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU General Public License as published by
|
||||||
|
# the Free Software Foundation, version 3 of the License.
|
||||||
|
#
|
||||||
|
# This program is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU General Public License
|
||||||
|
# along with this program; if not, write to the Free Software
|
||||||
|
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||||
|
|
||||||
|
|
||||||
|
from __future__ import with_statement
|
||||||
|
|
||||||
|
from ConfigParser import SafeConfigParser
|
||||||
|
import logging
|
||||||
|
from logging import debug, error, exception, warning
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import stat
|
||||||
|
|
||||||
|
from weboob.core.backend import BaseBackend
|
||||||
|
from weboob.capabilities.cap import ICap
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ['Module']
|
||||||
|
|
||||||
|
|
||||||
|
class Module(object):
|
||||||
|
def __init__(self, name, module):
|
||||||
|
self.name = name
|
||||||
|
self.module = module
|
||||||
|
self.klass = None
|
||||||
|
for attrname in dir(self.module):
|
||||||
|
attr = getattr(self.module, attrname)
|
||||||
|
if isinstance(attr, type) and issubclass(attr, BaseBackend) and attr != BaseBackend:
|
||||||
|
self.klass = attr
|
||||||
|
|
||||||
|
if not self.klass:
|
||||||
|
raise ImportError("This is not a backend module (no BaseBackend class found)")
|
||||||
|
|
||||||
|
def get_name(self):
|
||||||
|
return self.klass.NAME
|
||||||
|
|
||||||
|
def get_maintainer(self):
|
||||||
|
return '%s <%s>' % (self.klass.MAINTAINER, self.klass.EMAIL)
|
||||||
|
|
||||||
|
def get_version(self):
|
||||||
|
return self.klass.VERSION
|
||||||
|
|
||||||
|
def get_description(self):
|
||||||
|
return self.klass.DESCRIPTION
|
||||||
|
|
||||||
|
def get_license(self):
|
||||||
|
return self.klass.LICENSE
|
||||||
|
|
||||||
|
def get_config(self):
|
||||||
|
return self.klass.CONFIG
|
||||||
|
|
||||||
|
def get_icon_path(self):
|
||||||
|
return self.klass.ICON
|
||||||
|
|
||||||
|
def iter_caps(self):
|
||||||
|
for cap in self.klass.__bases__:
|
||||||
|
if issubclass(cap, ICap) and cap != ICap:
|
||||||
|
yield cap
|
||||||
|
|
||||||
|
def has_caps(self, *caps):
|
||||||
|
for c in caps:
|
||||||
|
if issubclass(self.klass, c):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def create_backend(self, weboob, name, config, storage):
|
||||||
|
debug('Created backend "%s"' % name)
|
||||||
|
return self.klass(weboob, name, config, storage)
|
||||||
|
|
||||||
|
class BackendsConfig(object):
|
||||||
|
class WrongPermissions(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def __init__(self, confpath):
|
||||||
|
self.confpath = confpath
|
||||||
|
try:
|
||||||
|
mode = os.stat(confpath).st_mode
|
||||||
|
except OSError:
|
||||||
|
os.mknod(confpath, 0600)
|
||||||
|
else:
|
||||||
|
if mode & stat.S_IRGRP or mode & stat.S_IROTH:
|
||||||
|
raise self.WrongPermissions(
|
||||||
|
u'Weboob will not start until config file %s is readable by group or other users.' % confpath)
|
||||||
|
|
||||||
|
def iter_backends(self):
|
||||||
|
config = SafeConfigParser()
|
||||||
|
config.read(self.confpath)
|
||||||
|
for name in config.sections():
|
||||||
|
params = dict(config.items(name, raw=True))
|
||||||
|
try:
|
||||||
|
yield name, params.pop('_type'), params
|
||||||
|
except KeyError:
|
||||||
|
warning('Missing field "_type" for backend "%s"', name)
|
||||||
|
continue
|
||||||
|
|
||||||
|
def add_backend(self, name, _type, params, edit=False):
|
||||||
|
if not name:
|
||||||
|
raise ValueError(u'Please give a name to the backend.')
|
||||||
|
config = SafeConfigParser()
|
||||||
|
config.read(self.confpath)
|
||||||
|
if not edit:
|
||||||
|
config.add_section(name)
|
||||||
|
config.set(name, '_type', _type)
|
||||||
|
for key, value in params.iteritems():
|
||||||
|
config.set(name, key, value)
|
||||||
|
with open(self.confpath, 'wb') as f:
|
||||||
|
config.write(f)
|
||||||
|
|
||||||
|
def edit_backend(self, name, _type, params):
|
||||||
|
return self.add_backend(name, _type, params, True)
|
||||||
|
|
||||||
|
def get_backend(self, name):
|
||||||
|
config = SafeConfigParser()
|
||||||
|
config.read(self.confpath)
|
||||||
|
if not config.has_section(name):
|
||||||
|
raise KeyError(u'Backend "%s" not found' % name)
|
||||||
|
|
||||||
|
items = dict(config.items(name, raw=True))
|
||||||
|
try:
|
||||||
|
return items.pop('_type'), items
|
||||||
|
except KeyError:
|
||||||
|
warning('Missing field "_type" for backend "%s"', name)
|
||||||
|
raise KeyError(u'Backend "%s" not found' % name)
|
||||||
|
|
||||||
|
def remove_backend(self, name):
|
||||||
|
config = SafeConfigParser()
|
||||||
|
config.read(self.confpath)
|
||||||
|
config.remove_section(name)
|
||||||
|
with open(self.confpath, 'wb') as f:
|
||||||
|
config.write(f)
|
||||||
|
|
||||||
|
class ModulesLoader(object):
|
||||||
|
def __init__(self):
|
||||||
|
self.modules = {}
|
||||||
|
|
||||||
|
def get_or_load_module(self, name):
|
||||||
|
if name not in self.modules:
|
||||||
|
self.load_module('weboob.backends.%s' % name)
|
||||||
|
return self.modules[name]
|
||||||
|
|
||||||
|
def load(self):
|
||||||
|
import weboob.backends
|
||||||
|
for path in weboob.backends.__path__:
|
||||||
|
regexp = re.compile('^%s/([\w\d_]+)$' % path)
|
||||||
|
for root, dirs, files in os.walk(path):
|
||||||
|
m = regexp.match(root)
|
||||||
|
if m and '__init__.py' in files:
|
||||||
|
self.load_module('weboob.backends.%s' % m.group(1))
|
||||||
|
|
||||||
|
def load_module(self, name):
|
||||||
|
try:
|
||||||
|
module = Module(name, __import__(name, fromlist=[str(name)]))
|
||||||
|
except ImportError, e:
|
||||||
|
msg = 'Unable to load module "%s": %s' % (name, e)
|
||||||
|
if logging.root.level == logging.DEBUG:
|
||||||
|
exception(msg)
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
error(msg)
|
||||||
|
return
|
||||||
|
if module.get_name() in self.modules:
|
||||||
|
warning('Module "%s" is already loaded (%s)' % (self.modules[module.get_name()].module, name))
|
||||||
|
return
|
||||||
|
self.modules[module.get_name()] = module
|
||||||
|
debug('Loaded module "%s" from %s' % (name, module.module.__path__))
|
||||||
84
weboob/core/scheduler.py
Normal file
84
weboob/core/scheduler.py
Normal file
|
|
@ -0,0 +1,84 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
# Copyright(C) 2010 Romain Bignon, Christophe Benz
|
||||||
|
#
|
||||||
|
# This program is free software; you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU General Public License as published by
|
||||||
|
# the Free Software Foundation, version 3 of the License.
|
||||||
|
#
|
||||||
|
# This program is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU General Public License
|
||||||
|
# along with this program; if not, write to the Free Software
|
||||||
|
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||||
|
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from threading import Timer, Event
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ['Scheduler']
|
||||||
|
|
||||||
|
|
||||||
|
class IScheduler(object):
|
||||||
|
def schedule(self, interval, function, *args):
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
def repeat(self, interval, function, *args):
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
def want_stop(self):
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
class Scheduler(IScheduler):
|
||||||
|
def __init__(self):
|
||||||
|
self.stop_event = Event()
|
||||||
|
self.count = 0
|
||||||
|
self.queue = {}
|
||||||
|
|
||||||
|
def schedule(self, interval, function, *args):
|
||||||
|
if self.stop_event.isSet():
|
||||||
|
return
|
||||||
|
|
||||||
|
self.count += 1
|
||||||
|
logging.debug('function "%s" will be called in %s seconds' % (function.__name__, interval))
|
||||||
|
timer = Timer(interval, function, args)
|
||||||
|
timer.start()
|
||||||
|
self.queue[self.count] = timer
|
||||||
|
return self.count
|
||||||
|
|
||||||
|
def repeat(self, interval, function, *args):
|
||||||
|
return self._repeat(True, interval, function, *args)
|
||||||
|
|
||||||
|
def _repeat(self, first, interval, function, *args):
|
||||||
|
return self.schedule(0 if first else interval, self._repeated_cb, interval, function, args)
|
||||||
|
|
||||||
|
def _wait_to_stop(self):
|
||||||
|
self.want_stop()
|
||||||
|
for e in self.queue.itervalues():
|
||||||
|
e.cancel()
|
||||||
|
e.join()
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
try:
|
||||||
|
while 1:
|
||||||
|
self.stop_event.wait(0.1)
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
self._wait_to_stop()
|
||||||
|
raise
|
||||||
|
else:
|
||||||
|
self._wait_to_stop()
|
||||||
|
return True
|
||||||
|
|
||||||
|
def want_stop(self):
|
||||||
|
self.stop_event.set()
|
||||||
|
|
||||||
|
def _repeated_cb(self, interval, function, args):
|
||||||
|
function(*args)
|
||||||
|
self._repeat(False, interval, function, *args)
|
||||||
|
|
@ -28,8 +28,8 @@ import sys
|
||||||
import logging
|
import logging
|
||||||
import asyncore
|
import asyncore
|
||||||
|
|
||||||
from weboob import Weboob
|
from weboob.core.engine import Weboob
|
||||||
from weboob.scheduler import Scheduler
|
from weboob.core.scheduler import Scheduler
|
||||||
from weboob.capabilities.messages import ICapMessages, ICapMessagesReply, Message
|
from weboob.capabilities.messages import ICapMessages, ICapMessagesReply, Message
|
||||||
from weboob.tools.application import ConsoleApplication
|
from weboob.tools.application import ConsoleApplication
|
||||||
from weboob.tools.misc import html2text, get_backtrace, utc2local
|
from weboob.tools.misc import html2text, get_backtrace, utc2local
|
||||||
|
|
|
||||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue