move core files to weboob.core

This commit is contained in:
Christophe Benz 2010-07-06 12:12:29 +02:00
commit 1955d1be59
25 changed files with 66 additions and 44 deletions

19
weboob/core/__init__.py Normal file
View file

@ -0,0 +1,19 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010 Christophe Benz
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
from .bcall import CallErrors

166
weboob/core/backend.py Normal file
View file

@ -0,0 +1,166 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010 Romain Bignon
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
import re
import os
from threading import RLock
__all__ = ['BackendStorage', 'BaseBackend']
class BackendStorage(object):
def __init__(self, name, storage):
self.name = name
self.storage = storage
def set(self, *args):
if self.storage:
return self.storage.set('backends', self.name, *args)
def get(self, *args, **kwargs):
if self.storage:
return self.storage.get('backends', self.name, *args, **kwargs)
else:
return kwargs.get('default', None)
def load(self, default):
if self.storage:
return self.storage.load('backends', self.name, default)
def save(self):
if self.storage:
return self.storage.save('backends', self.name)
class BaseBackend(object):
# Module name.
NAME = None
# Name of the maintainer of this module.
MAINTAINER = '<unspecifier>'
# Email address of the maintainer.
EMAIL = '<unspecified>'
# Version of module (for information only).
VERSION = '<unspecified>'
# Description
DESCRIPTION = '<unspecified>'
# License of this module.
LICENSE = '<unspecified>'
# Icon file path
ICON = ''
# Configuration required for this module. # Values must be ConfigField
# objects.
CONFIG = {}
# Storage
STORAGE = {}
# Browser class
BROWSER = None
# Test class
TEST = None
class ConfigField(object):
def __init__(self, default=None, is_masked=False, regexp=None, description=None):
self.default = default
self.is_masked = is_masked
self.regexp = regexp
self.description = description
class ConfigError(Exception): pass
def __enter__(self):
self.lock.acquire()
def __exit__(self, t, v, tb):
self.lock.release()
def __repr__(self):
return u"<Backend '%s'>" % self.name
def __init__(self, weboob, name, config, storage):
self.weboob = weboob
self.name = name
self.lock = RLock()
# Private fields (which start with '_')
self._private_config = dict([(key, value) for key, value in config.iteritems() if key.startswith('_')])
# Configuration of backend
self.config = {}
for name, field in self.CONFIG.iteritems():
value = config.get(name, field.default)
if value is None:
raise BaseBackend.ConfigError('Missing parameter "%s" (%s)' % (name, field.description))
if field.regexp and not re.match(field.regexp, str(value)):
raise BaseBackend.ConfigError('Value of "%s" does not match regexp "%s"' % (name, field.regexp))
if not field.default is None:
if isinstance(field.default, bool) and not isinstance(value, bool):
value = value.lower() in ('1', 'true', 'on', 'yes')
elif isinstance(field.default, int) and not isinstance(value, int):
value = int(value)
elif isinstance(field.default, float) and not isinstance(value, float):
value = float(value)
self.config[name] = value
self.storage = BackendStorage(self.name, storage)
self.storage.load(self.STORAGE)
@property
def browser(self):
"""
Attribute 'browser'. The browser is created at the first call
of this attribute, to avoid useless pages access.
Note that the 'default_browser' method is called to create it.
"""
if not hasattr(self, '_browser'):
self._browser = self.default_browser()
return self._browser
def default_browser(self):
"""
Method to overload to build the default browser in
attribute 'browser'.
"""
return self.build_browser()
def build_browser(self, *args, **kwargs):
"""
Build a browser from the BROWSER class attribute and the
given arguments.
"""
if not self.BROWSER:
return None
if '_proxy' in self._private_config:
kwargs['proxy'] = self._private_config['_proxy']
elif 'HTTP_PROXY' in os.environ:
kwargs['proxy'] = os.environ['HTTP_PROXY']
return self.BROWSER(*args, **kwargs)
def has_caps(self, *caps):
for c in caps:
if isinstance(self, c):
return True
return False
def get_test(self):
if not self.TEST:
return None
return self.TEST(self)

188
weboob/core/bcall.py Normal file
View file

@ -0,0 +1,188 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010 Romain Bignon, Christophe Benz
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
from __future__ import with_statement
from copy import copy
import logging
from logging import debug
from threading import Thread, Event, RLock, Timer
from weboob.tools.misc import get_backtrace
__all__ = ['BackendsCall', 'CallErrors']
class CallErrors(Exception):
def __init__(self, errors):
Exception.__init__(self, u'These errors have been raised in backend threads '\
'(use --debug option to print backtraces):\n%s' % (
u'\n'.join((u' * %s: %s%s' % (backend, error, backtrace + '\n'
if logging.root.level == logging.DEBUG else ''))
for backend, error, backtrace in errors)))
self.errors = copy(errors)
def __iter__(self):
return self.errors.__iter__()
class BackendsCall(object):
def __init__(self, backends, function, *args, **kwargs):
"""
@param backends list of backends to call
@param function backends' method name, or callable object
@param args, kwargs arguments given to called functions
"""
# Store if a backend is finished
self.backends = {}
for backend in backends:
self.backends[backend.name] = False
# Global mutex on object
self.mutex = RLock()
# Event set when every backends have give their data
self.finish_event = Event()
# Event set when there are new responses
self.response_event = Event()
# Waiting responses
self.responses = []
# Errors
self.errors = []
# Threads
self.threads = []
# Create jobs for each backend
with self.mutex:
for backend in backends:
debug('Creating a new thread for %s' % backend)
self.threads.append(Timer(0, self._caller, (backend, function, args, kwargs)).start())
if not backends:
self.finish_event.set()
def _store_error(self, backend, error):
with self.mutex:
backtrace = get_backtrace(error)
self.errors.append((backend, error, backtrace))
def _store_result(self, backend, result):
with self.mutex:
self.responses.append((backend, result))
self.response_event.set()
def _caller(self, backend, function, args, kwargs):
debug('%s: Thread created successfully' % backend)
with backend:
try:
# Call method on backend
try:
debug('%s: Calling function %s' % (backend, function))
if callable(function):
result = function(backend, *args, **kwargs)
else:
result = getattr(backend, function)(*args, **kwargs)
except Exception, error:
self._store_error(backend, error)
else:
debug('%s: Called function %s returned: "%s"' % (backend, function, result))
if hasattr(result, '__iter__'):
# Loop on iterator
try:
for subresult in result:
# Lock mutex only in loop in case the iterator is slow
# (for example if backend do some parsing operations)
self._store_result(backend, subresult)
except Exception, error:
self._store_error(backend, error)
else:
self._store_result(backend, result)
finally:
with self.mutex:
# This backend is now finished
self.backends[backend.name] = True
for finished in self.backends.itervalues():
if not finished:
return
self.finish_event.set()
self.response_event.set()
def _callback_thread_run(self, callback, errback):
responses = []
while not self.finish_event.isSet() or self.response_event.isSet():
self.response_event.wait()
with self.mutex:
responses = self.responses
self.responses = []
# Reset event
self.response_event.clear()
# Consume responses
while responses:
callback(*responses.pop(0))
if errback:
with self.mutex:
while self.errors:
errback(*self.errors.pop(0))
callback(None, None)
def callback_thread(self, callback, errback=None):
"""
Call this method to create a thread which will callback a
specified function everytimes a new result comes.
When the process is over, the function will be called with
both arguments set to None.
The functions prototypes:
def callback(backend, result)
def errback(backend, error)
"""
thread = Thread(target=self._callback_thread_run, args=(callback, errback))
thread.start()
return thread
def wait(self):
self.finish_event.wait()
with self.mutex:
if self.errors:
raise CallErrors(self.errors)
def __iter__(self):
# Don't know how to factorize with _callback_thread_run
responses = []
while not self.finish_event.isSet() or self.response_event.isSet():
self.response_event.wait()
with self.mutex:
responses = self.responses
self.responses = []
# Reset event
self.response_event.clear()
# Consume responses
while responses:
yield responses.pop(0)
# Raise errors
with self.mutex:
if self.errors:
raise CallErrors(self.errors)

178
weboob/core/engine.py Normal file
View file

@ -0,0 +1,178 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010 Romain Bignon
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
from __future__ import with_statement
from logging import warning
import os
import sys
from weboob.core.bcall import BackendsCall
from weboob.core.modules import ModulesLoader, BackendsConfig
from weboob.core.backend import BaseBackend
from weboob.core.scheduler import Scheduler
if sys.version_info[:2] <= (2, 5):
import weboob.tools.property
__all__ = ['Weboob']
class Weboob(object):
WORKDIR = os.path.join(os.path.expanduser('~'), '.weboob')
BACKENDS_FILENAME = 'backends'
def __init__(self, workdir=WORKDIR, backends_filename=None, scheduler=None):
self.workdir = workdir
self.backends = {}
# Scheduler
if scheduler is None:
scheduler = Scheduler()
self.scheduler = scheduler
# Create WORKDIR
if not os.path.exists(self.workdir):
os.mkdir(self.workdir, 0700)
elif not os.path.isdir(self.workdir):
warning('"%s" is not a directory' % self.workdir)
# Modules loader
self.modules_loader = ModulesLoader()
# Backends config
if not backends_filename:
backends_filename = os.path.join(self.workdir, self.BACKENDS_FILENAME)
elif not backends_filename.startswith('/'):
backends_filename = os.path.join(self.workdir, backends_filename)
self.backends_config = BackendsConfig(backends_filename)
def load_backends(self, caps=None, names=None, storage=None):
loaded_backends = {}
for name, _type, params in self.backends_config.iter_backends():
try:
module = self.modules_loader.get_or_load_module(_type)
except KeyError:
warning(u'Unable to find module "%s" for backend "%s"' % (_type, name))
continue
# Check conditions
if (not caps is None and not module.has_caps(caps)) or \
(not names is None and not name in names):
continue
try:
self.backends[name] = module.create_backend(self, name, params, storage)
loaded_backends[name] = self.backends[name]
except Exception, e:
warning(u'Unable to load "%s" backend: %s. filename=%s' % (name, e, self.backends_config.confpath))
return loaded_backends
def load_modules(self, caps=None, names=None, storage=None):
loaded_backends = {}
self.modules_loader.load()
for name, module in self.modules_loader.modules.iteritems():
if (caps is None or module.has_caps(caps)) and \
(names is None or module.get_name() in names):
try:
name = module.get_name()
self.backends[name] = module.create_backend(self, name, {}, storage)
loaded_backends[name] = self.backends[name]
except Exception, e:
warning(u'Unable to load "%s" module as backend with no config: %s' % (name, e))
return loaded_backends
def iter_backends(self, caps=None):
"""
Iter on each backends.
Note: each backend is locked when it is returned.
@param caps Optional list of capabilities to select backends
@return iterator on selected backends.
"""
for name, backend in sorted(self.backends.iteritems()):
if caps is None or backend.has_caps(caps):
with backend:
yield backend
def do(self, function, *args, **kwargs):
"""
Do calls on loaded backends with specified arguments, in separated
threads.
This function has two modes:
- If 'function' is a string, it calls the method with this name on
each backends with the specified arguments;
- If 'function' is a callable, it calls it in a separated thread with
the locked backend instance at first arguments, and *args and
**kwargs.
@param function backend's method name, or callable object
@return an iterator of results
"""
backends = list(self.iter_backends())
return BackendsCall(backends, function, *args, **kwargs)
def do_caps(self, caps, function, *args, **kwargs):
"""
Do calls on loaded modules with the specified capabilities, in
separated threads.
See also documentation of the 'do' method.
@param caps list of caps or cap to select backends
@param function backend's method name, or callable object
@return an iterator of results
"""
backends = list(self.iter_backends(caps))
return BackendsCall(backends, function, *args, **kwargs)
def do_backends(self, backends, function, *args, **kwargs):
if backends is None:
backends = list(self.iter_backends())
elif isinstance(backends, BaseBackend):
backends = [backends]
elif isinstance(backends, (str,unicode)):
backends = [backend for backend in self.iter_backends() if backend.name == backends]
elif isinstance(backends, (list,tuple)):
old_backends = backends
backends = []
for b in old_backends:
if isinstance(b, (str,unicode)):
try:
backends.append(self.backends[self.backends.index(b)])
except ValueError:
pass
else:
backends.append(b)
return BackendsCall(backends, function, *args, **kwargs)
def schedule(self, interval, function, *args):
return self.scheduler.schedule(interval, function, *args)
def repeat(self, interval, function, *args):
return self.scheduler.repeat(interval, function, *args)
def want_stop(self):
return self.scheduler.want_stop()
def loop(self):
return self.scheduler.run()

179
weboob/core/modules.py Normal file
View file

@ -0,0 +1,179 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010 Romain Bignon
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
from __future__ import with_statement
from ConfigParser import SafeConfigParser
import logging
from logging import debug, error, exception, warning
import os
import re
import stat
from weboob.core.backend import BaseBackend
from weboob.capabilities.cap import ICap
__all__ = ['Module']
class Module(object):
def __init__(self, name, module):
self.name = name
self.module = module
self.klass = None
for attrname in dir(self.module):
attr = getattr(self.module, attrname)
if isinstance(attr, type) and issubclass(attr, BaseBackend) and attr != BaseBackend:
self.klass = attr
if not self.klass:
raise ImportError("This is not a backend module (no BaseBackend class found)")
def get_name(self):
return self.klass.NAME
def get_maintainer(self):
return '%s <%s>' % (self.klass.MAINTAINER, self.klass.EMAIL)
def get_version(self):
return self.klass.VERSION
def get_description(self):
return self.klass.DESCRIPTION
def get_license(self):
return self.klass.LICENSE
def get_config(self):
return self.klass.CONFIG
def get_icon_path(self):
return self.klass.ICON
def iter_caps(self):
for cap in self.klass.__bases__:
if issubclass(cap, ICap) and cap != ICap:
yield cap
def has_caps(self, *caps):
for c in caps:
if issubclass(self.klass, c):
return True
return False
def create_backend(self, weboob, name, config, storage):
debug('Created backend "%s"' % name)
return self.klass(weboob, name, config, storage)
class BackendsConfig(object):
class WrongPermissions(Exception):
pass
def __init__(self, confpath):
self.confpath = confpath
try:
mode = os.stat(confpath).st_mode
except OSError:
os.mknod(confpath, 0600)
else:
if mode & stat.S_IRGRP or mode & stat.S_IROTH:
raise self.WrongPermissions(
u'Weboob will not start until config file %s is readable by group or other users.' % confpath)
def iter_backends(self):
config = SafeConfigParser()
config.read(self.confpath)
for name in config.sections():
params = dict(config.items(name, raw=True))
try:
yield name, params.pop('_type'), params
except KeyError:
warning('Missing field "_type" for backend "%s"', name)
continue
def add_backend(self, name, _type, params, edit=False):
if not name:
raise ValueError(u'Please give a name to the backend.')
config = SafeConfigParser()
config.read(self.confpath)
if not edit:
config.add_section(name)
config.set(name, '_type', _type)
for key, value in params.iteritems():
config.set(name, key, value)
with open(self.confpath, 'wb') as f:
config.write(f)
def edit_backend(self, name, _type, params):
return self.add_backend(name, _type, params, True)
def get_backend(self, name):
config = SafeConfigParser()
config.read(self.confpath)
if not config.has_section(name):
raise KeyError(u'Backend "%s" not found' % name)
items = dict(config.items(name, raw=True))
try:
return items.pop('_type'), items
except KeyError:
warning('Missing field "_type" for backend "%s"', name)
raise KeyError(u'Backend "%s" not found' % name)
def remove_backend(self, name):
config = SafeConfigParser()
config.read(self.confpath)
config.remove_section(name)
with open(self.confpath, 'wb') as f:
config.write(f)
class ModulesLoader(object):
def __init__(self):
self.modules = {}
def get_or_load_module(self, name):
if name not in self.modules:
self.load_module('weboob.backends.%s' % name)
return self.modules[name]
def load(self):
import weboob.backends
for path in weboob.backends.__path__:
regexp = re.compile('^%s/([\w\d_]+)$' % path)
for root, dirs, files in os.walk(path):
m = regexp.match(root)
if m and '__init__.py' in files:
self.load_module('weboob.backends.%s' % m.group(1))
def load_module(self, name):
try:
module = Module(name, __import__(name, fromlist=[str(name)]))
except ImportError, e:
msg = 'Unable to load module "%s": %s' % (name, e)
if logging.root.level == logging.DEBUG:
exception(msg)
return
else:
error(msg)
return
if module.get_name() in self.modules:
warning('Module "%s" is already loaded (%s)' % (self.modules[module.get_name()].module, name))
return
self.modules[module.get_name()] = module
debug('Loaded module "%s" from %s' % (name, module.module.__path__))

84
weboob/core/scheduler.py Normal file
View file

@ -0,0 +1,84 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010 Romain Bignon, Christophe Benz
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
import logging
from threading import Timer, Event
__all__ = ['Scheduler']
class IScheduler(object):
def schedule(self, interval, function, *args):
raise NotImplementedError()
def repeat(self, interval, function, *args):
raise NotImplementedError()
def run(self):
raise NotImplementedError()
def want_stop(self):
raise NotImplementedError()
class Scheduler(IScheduler):
def __init__(self):
self.stop_event = Event()
self.count = 0
self.queue = {}
def schedule(self, interval, function, *args):
if self.stop_event.isSet():
return
self.count += 1
logging.debug('function "%s" will be called in %s seconds' % (function.__name__, interval))
timer = Timer(interval, function, args)
timer.start()
self.queue[self.count] = timer
return self.count
def repeat(self, interval, function, *args):
return self._repeat(True, interval, function, *args)
def _repeat(self, first, interval, function, *args):
return self.schedule(0 if first else interval, self._repeated_cb, interval, function, args)
def _wait_to_stop(self):
self.want_stop()
for e in self.queue.itervalues():
e.cancel()
e.join()
def run(self):
try:
while 1:
self.stop_event.wait(0.1)
except KeyboardInterrupt:
self._wait_to_stop()
raise
else:
self._wait_to_stop()
return True
def want_stop(self):
self.stop_event.set()
def _repeated_cb(self, interval, function, args):
function(*args)
self._repeat(False, interval, function, *args)