move core files to weboob.core
This commit is contained in:
parent
944662c4bf
commit
1955d1be59
25 changed files with 66 additions and 44 deletions
19
weboob/core/__init__.py
Normal file
19
weboob/core/__init__.py
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright(C) 2010 Christophe Benz
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, version 3 of the License.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
|
||||
|
||||
from .bcall import CallErrors
|
||||
166
weboob/core/backend.py
Normal file
166
weboob/core/backend.py
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright(C) 2010 Romain Bignon
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, version 3 of the License.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
|
||||
|
||||
import re
|
||||
import os
|
||||
from threading import RLock
|
||||
|
||||
|
||||
__all__ = ['BackendStorage', 'BaseBackend']
|
||||
|
||||
|
||||
class BackendStorage(object):
|
||||
def __init__(self, name, storage):
|
||||
self.name = name
|
||||
self.storage = storage
|
||||
|
||||
def set(self, *args):
|
||||
if self.storage:
|
||||
return self.storage.set('backends', self.name, *args)
|
||||
|
||||
def get(self, *args, **kwargs):
|
||||
if self.storage:
|
||||
return self.storage.get('backends', self.name, *args, **kwargs)
|
||||
else:
|
||||
return kwargs.get('default', None)
|
||||
|
||||
def load(self, default):
|
||||
if self.storage:
|
||||
return self.storage.load('backends', self.name, default)
|
||||
|
||||
def save(self):
|
||||
if self.storage:
|
||||
return self.storage.save('backends', self.name)
|
||||
|
||||
class BaseBackend(object):
|
||||
# Module name.
|
||||
NAME = None
|
||||
# Name of the maintainer of this module.
|
||||
MAINTAINER = '<unspecifier>'
|
||||
# Email address of the maintainer.
|
||||
EMAIL = '<unspecified>'
|
||||
# Version of module (for information only).
|
||||
VERSION = '<unspecified>'
|
||||
# Description
|
||||
DESCRIPTION = '<unspecified>'
|
||||
# License of this module.
|
||||
LICENSE = '<unspecified>'
|
||||
# Icon file path
|
||||
ICON = ''
|
||||
# Configuration required for this module. # Values must be ConfigField
|
||||
# objects.
|
||||
CONFIG = {}
|
||||
# Storage
|
||||
STORAGE = {}
|
||||
# Browser class
|
||||
BROWSER = None
|
||||
# Test class
|
||||
TEST = None
|
||||
|
||||
class ConfigField(object):
|
||||
def __init__(self, default=None, is_masked=False, regexp=None, description=None):
|
||||
self.default = default
|
||||
self.is_masked = is_masked
|
||||
self.regexp = regexp
|
||||
self.description = description
|
||||
|
||||
class ConfigError(Exception): pass
|
||||
|
||||
def __enter__(self):
|
||||
self.lock.acquire()
|
||||
|
||||
def __exit__(self, t, v, tb):
|
||||
self.lock.release()
|
||||
|
||||
def __repr__(self):
|
||||
return u"<Backend '%s'>" % self.name
|
||||
|
||||
def __init__(self, weboob, name, config, storage):
|
||||
self.weboob = weboob
|
||||
self.name = name
|
||||
self.lock = RLock()
|
||||
|
||||
# Private fields (which start with '_')
|
||||
self._private_config = dict([(key, value) for key, value in config.iteritems() if key.startswith('_')])
|
||||
|
||||
# Configuration of backend
|
||||
self.config = {}
|
||||
for name, field in self.CONFIG.iteritems():
|
||||
value = config.get(name, field.default)
|
||||
|
||||
if value is None:
|
||||
raise BaseBackend.ConfigError('Missing parameter "%s" (%s)' % (name, field.description))
|
||||
|
||||
if field.regexp and not re.match(field.regexp, str(value)):
|
||||
raise BaseBackend.ConfigError('Value of "%s" does not match regexp "%s"' % (name, field.regexp))
|
||||
|
||||
if not field.default is None:
|
||||
if isinstance(field.default, bool) and not isinstance(value, bool):
|
||||
value = value.lower() in ('1', 'true', 'on', 'yes')
|
||||
elif isinstance(field.default, int) and not isinstance(value, int):
|
||||
value = int(value)
|
||||
elif isinstance(field.default, float) and not isinstance(value, float):
|
||||
value = float(value)
|
||||
self.config[name] = value
|
||||
self.storage = BackendStorage(self.name, storage)
|
||||
self.storage.load(self.STORAGE)
|
||||
|
||||
@property
|
||||
def browser(self):
|
||||
"""
|
||||
Attribute 'browser'. The browser is created at the first call
|
||||
of this attribute, to avoid useless pages access.
|
||||
|
||||
Note that the 'default_browser' method is called to create it.
|
||||
"""
|
||||
if not hasattr(self, '_browser'):
|
||||
self._browser = self.default_browser()
|
||||
return self._browser
|
||||
|
||||
def default_browser(self):
|
||||
"""
|
||||
Method to overload to build the default browser in
|
||||
attribute 'browser'.
|
||||
"""
|
||||
return self.build_browser()
|
||||
|
||||
def build_browser(self, *args, **kwargs):
|
||||
"""
|
||||
Build a browser from the BROWSER class attribute and the
|
||||
given arguments.
|
||||
"""
|
||||
if not self.BROWSER:
|
||||
return None
|
||||
|
||||
if '_proxy' in self._private_config:
|
||||
kwargs['proxy'] = self._private_config['_proxy']
|
||||
elif 'HTTP_PROXY' in os.environ:
|
||||
kwargs['proxy'] = os.environ['HTTP_PROXY']
|
||||
|
||||
return self.BROWSER(*args, **kwargs)
|
||||
|
||||
def has_caps(self, *caps):
|
||||
for c in caps:
|
||||
if isinstance(self, c):
|
||||
return True
|
||||
return False
|
||||
|
||||
def get_test(self):
|
||||
if not self.TEST:
|
||||
return None
|
||||
return self.TEST(self)
|
||||
188
weboob/core/bcall.py
Normal file
188
weboob/core/bcall.py
Normal file
|
|
@ -0,0 +1,188 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright(C) 2010 Romain Bignon, Christophe Benz
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, version 3 of the License.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
|
||||
|
||||
from __future__ import with_statement
|
||||
|
||||
from copy import copy
|
||||
import logging
|
||||
from logging import debug
|
||||
from threading import Thread, Event, RLock, Timer
|
||||
from weboob.tools.misc import get_backtrace
|
||||
|
||||
|
||||
__all__ = ['BackendsCall', 'CallErrors']
|
||||
|
||||
|
||||
class CallErrors(Exception):
|
||||
def __init__(self, errors):
|
||||
Exception.__init__(self, u'These errors have been raised in backend threads '\
|
||||
'(use --debug option to print backtraces):\n%s' % (
|
||||
u'\n'.join((u' * %s: %s%s' % (backend, error, backtrace + '\n'
|
||||
if logging.root.level == logging.DEBUG else ''))
|
||||
for backend, error, backtrace in errors)))
|
||||
self.errors = copy(errors)
|
||||
|
||||
def __iter__(self):
|
||||
return self.errors.__iter__()
|
||||
|
||||
|
||||
class BackendsCall(object):
|
||||
def __init__(self, backends, function, *args, **kwargs):
|
||||
"""
|
||||
@param backends list of backends to call
|
||||
@param function backends' method name, or callable object
|
||||
@param args, kwargs arguments given to called functions
|
||||
"""
|
||||
# Store if a backend is finished
|
||||
self.backends = {}
|
||||
for backend in backends:
|
||||
self.backends[backend.name] = False
|
||||
# Global mutex on object
|
||||
self.mutex = RLock()
|
||||
# Event set when every backends have give their data
|
||||
self.finish_event = Event()
|
||||
# Event set when there are new responses
|
||||
self.response_event = Event()
|
||||
# Waiting responses
|
||||
self.responses = []
|
||||
# Errors
|
||||
self.errors = []
|
||||
# Threads
|
||||
self.threads = []
|
||||
|
||||
# Create jobs for each backend
|
||||
with self.mutex:
|
||||
for backend in backends:
|
||||
debug('Creating a new thread for %s' % backend)
|
||||
self.threads.append(Timer(0, self._caller, (backend, function, args, kwargs)).start())
|
||||
if not backends:
|
||||
self.finish_event.set()
|
||||
|
||||
def _store_error(self, backend, error):
|
||||
with self.mutex:
|
||||
backtrace = get_backtrace(error)
|
||||
self.errors.append((backend, error, backtrace))
|
||||
|
||||
def _store_result(self, backend, result):
|
||||
with self.mutex:
|
||||
self.responses.append((backend, result))
|
||||
self.response_event.set()
|
||||
|
||||
def _caller(self, backend, function, args, kwargs):
|
||||
debug('%s: Thread created successfully' % backend)
|
||||
with backend:
|
||||
try:
|
||||
# Call method on backend
|
||||
try:
|
||||
debug('%s: Calling function %s' % (backend, function))
|
||||
if callable(function):
|
||||
result = function(backend, *args, **kwargs)
|
||||
else:
|
||||
result = getattr(backend, function)(*args, **kwargs)
|
||||
except Exception, error:
|
||||
self._store_error(backend, error)
|
||||
else:
|
||||
debug('%s: Called function %s returned: "%s"' % (backend, function, result))
|
||||
|
||||
if hasattr(result, '__iter__'):
|
||||
# Loop on iterator
|
||||
try:
|
||||
for subresult in result:
|
||||
# Lock mutex only in loop in case the iterator is slow
|
||||
# (for example if backend do some parsing operations)
|
||||
self._store_result(backend, subresult)
|
||||
except Exception, error:
|
||||
self._store_error(backend, error)
|
||||
else:
|
||||
self._store_result(backend, result)
|
||||
finally:
|
||||
with self.mutex:
|
||||
# This backend is now finished
|
||||
self.backends[backend.name] = True
|
||||
for finished in self.backends.itervalues():
|
||||
if not finished:
|
||||
return
|
||||
self.finish_event.set()
|
||||
self.response_event.set()
|
||||
|
||||
def _callback_thread_run(self, callback, errback):
|
||||
responses = []
|
||||
while not self.finish_event.isSet() or self.response_event.isSet():
|
||||
self.response_event.wait()
|
||||
with self.mutex:
|
||||
responses = self.responses
|
||||
self.responses = []
|
||||
|
||||
# Reset event
|
||||
self.response_event.clear()
|
||||
|
||||
# Consume responses
|
||||
while responses:
|
||||
callback(*responses.pop(0))
|
||||
|
||||
if errback:
|
||||
with self.mutex:
|
||||
while self.errors:
|
||||
errback(*self.errors.pop(0))
|
||||
|
||||
callback(None, None)
|
||||
|
||||
def callback_thread(self, callback, errback=None):
|
||||
"""
|
||||
Call this method to create a thread which will callback a
|
||||
specified function everytimes a new result comes.
|
||||
|
||||
When the process is over, the function will be called with
|
||||
both arguments set to None.
|
||||
|
||||
The functions prototypes:
|
||||
def callback(backend, result)
|
||||
def errback(backend, error)
|
||||
|
||||
"""
|
||||
thread = Thread(target=self._callback_thread_run, args=(callback, errback))
|
||||
thread.start()
|
||||
return thread
|
||||
|
||||
def wait(self):
|
||||
self.finish_event.wait()
|
||||
|
||||
with self.mutex:
|
||||
if self.errors:
|
||||
raise CallErrors(self.errors)
|
||||
|
||||
def __iter__(self):
|
||||
# Don't know how to factorize with _callback_thread_run
|
||||
responses = []
|
||||
while not self.finish_event.isSet() or self.response_event.isSet():
|
||||
self.response_event.wait()
|
||||
with self.mutex:
|
||||
responses = self.responses
|
||||
self.responses = []
|
||||
|
||||
# Reset event
|
||||
self.response_event.clear()
|
||||
|
||||
# Consume responses
|
||||
while responses:
|
||||
yield responses.pop(0)
|
||||
|
||||
# Raise errors
|
||||
with self.mutex:
|
||||
if self.errors:
|
||||
raise CallErrors(self.errors)
|
||||
178
weboob/core/engine.py
Normal file
178
weboob/core/engine.py
Normal file
|
|
@ -0,0 +1,178 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright(C) 2010 Romain Bignon
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, version 3 of the License.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
|
||||
|
||||
from __future__ import with_statement
|
||||
|
||||
from logging import warning
|
||||
import os
|
||||
import sys
|
||||
|
||||
from weboob.core.bcall import BackendsCall
|
||||
from weboob.core.modules import ModulesLoader, BackendsConfig
|
||||
from weboob.core.backend import BaseBackend
|
||||
from weboob.core.scheduler import Scheduler
|
||||
|
||||
if sys.version_info[:2] <= (2, 5):
|
||||
import weboob.tools.property
|
||||
|
||||
|
||||
__all__ = ['Weboob']
|
||||
|
||||
|
||||
class Weboob(object):
|
||||
WORKDIR = os.path.join(os.path.expanduser('~'), '.weboob')
|
||||
BACKENDS_FILENAME = 'backends'
|
||||
|
||||
def __init__(self, workdir=WORKDIR, backends_filename=None, scheduler=None):
|
||||
self.workdir = workdir
|
||||
self.backends = {}
|
||||
|
||||
# Scheduler
|
||||
if scheduler is None:
|
||||
scheduler = Scheduler()
|
||||
self.scheduler = scheduler
|
||||
|
||||
# Create WORKDIR
|
||||
if not os.path.exists(self.workdir):
|
||||
os.mkdir(self.workdir, 0700)
|
||||
elif not os.path.isdir(self.workdir):
|
||||
warning('"%s" is not a directory' % self.workdir)
|
||||
|
||||
# Modules loader
|
||||
self.modules_loader = ModulesLoader()
|
||||
|
||||
# Backends config
|
||||
if not backends_filename:
|
||||
backends_filename = os.path.join(self.workdir, self.BACKENDS_FILENAME)
|
||||
elif not backends_filename.startswith('/'):
|
||||
backends_filename = os.path.join(self.workdir, backends_filename)
|
||||
self.backends_config = BackendsConfig(backends_filename)
|
||||
|
||||
def load_backends(self, caps=None, names=None, storage=None):
|
||||
loaded_backends = {}
|
||||
for name, _type, params in self.backends_config.iter_backends():
|
||||
try:
|
||||
module = self.modules_loader.get_or_load_module(_type)
|
||||
except KeyError:
|
||||
warning(u'Unable to find module "%s" for backend "%s"' % (_type, name))
|
||||
continue
|
||||
|
||||
# Check conditions
|
||||
if (not caps is None and not module.has_caps(caps)) or \
|
||||
(not names is None and not name in names):
|
||||
continue
|
||||
|
||||
try:
|
||||
self.backends[name] = module.create_backend(self, name, params, storage)
|
||||
loaded_backends[name] = self.backends[name]
|
||||
except Exception, e:
|
||||
warning(u'Unable to load "%s" backend: %s. filename=%s' % (name, e, self.backends_config.confpath))
|
||||
|
||||
return loaded_backends
|
||||
|
||||
def load_modules(self, caps=None, names=None, storage=None):
|
||||
loaded_backends = {}
|
||||
self.modules_loader.load()
|
||||
for name, module in self.modules_loader.modules.iteritems():
|
||||
if (caps is None or module.has_caps(caps)) and \
|
||||
(names is None or module.get_name() in names):
|
||||
try:
|
||||
name = module.get_name()
|
||||
self.backends[name] = module.create_backend(self, name, {}, storage)
|
||||
loaded_backends[name] = self.backends[name]
|
||||
except Exception, e:
|
||||
warning(u'Unable to load "%s" module as backend with no config: %s' % (name, e))
|
||||
return loaded_backends
|
||||
|
||||
def iter_backends(self, caps=None):
|
||||
"""
|
||||
Iter on each backends.
|
||||
|
||||
Note: each backend is locked when it is returned.
|
||||
|
||||
@param caps Optional list of capabilities to select backends
|
||||
@return iterator on selected backends.
|
||||
"""
|
||||
for name, backend in sorted(self.backends.iteritems()):
|
||||
if caps is None or backend.has_caps(caps):
|
||||
with backend:
|
||||
yield backend
|
||||
|
||||
def do(self, function, *args, **kwargs):
|
||||
"""
|
||||
Do calls on loaded backends with specified arguments, in separated
|
||||
threads.
|
||||
|
||||
This function has two modes:
|
||||
- If 'function' is a string, it calls the method with this name on
|
||||
each backends with the specified arguments;
|
||||
- If 'function' is a callable, it calls it in a separated thread with
|
||||
the locked backend instance at first arguments, and *args and
|
||||
**kwargs.
|
||||
|
||||
@param function backend's method name, or callable object
|
||||
@return an iterator of results
|
||||
"""
|
||||
backends = list(self.iter_backends())
|
||||
return BackendsCall(backends, function, *args, **kwargs)
|
||||
|
||||
def do_caps(self, caps, function, *args, **kwargs):
|
||||
"""
|
||||
Do calls on loaded modules with the specified capabilities, in
|
||||
separated threads.
|
||||
|
||||
See also documentation of the 'do' method.
|
||||
|
||||
@param caps list of caps or cap to select backends
|
||||
@param function backend's method name, or callable object
|
||||
@return an iterator of results
|
||||
"""
|
||||
backends = list(self.iter_backends(caps))
|
||||
return BackendsCall(backends, function, *args, **kwargs)
|
||||
|
||||
def do_backends(self, backends, function, *args, **kwargs):
|
||||
if backends is None:
|
||||
backends = list(self.iter_backends())
|
||||
elif isinstance(backends, BaseBackend):
|
||||
backends = [backends]
|
||||
elif isinstance(backends, (str,unicode)):
|
||||
backends = [backend for backend in self.iter_backends() if backend.name == backends]
|
||||
elif isinstance(backends, (list,tuple)):
|
||||
old_backends = backends
|
||||
backends = []
|
||||
for b in old_backends:
|
||||
if isinstance(b, (str,unicode)):
|
||||
try:
|
||||
backends.append(self.backends[self.backends.index(b)])
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
backends.append(b)
|
||||
return BackendsCall(backends, function, *args, **kwargs)
|
||||
|
||||
def schedule(self, interval, function, *args):
|
||||
return self.scheduler.schedule(interval, function, *args)
|
||||
|
||||
def repeat(self, interval, function, *args):
|
||||
return self.scheduler.repeat(interval, function, *args)
|
||||
|
||||
def want_stop(self):
|
||||
return self.scheduler.want_stop()
|
||||
|
||||
def loop(self):
|
||||
return self.scheduler.run()
|
||||
179
weboob/core/modules.py
Normal file
179
weboob/core/modules.py
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright(C) 2010 Romain Bignon
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, version 3 of the License.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
|
||||
|
||||
from __future__ import with_statement
|
||||
|
||||
from ConfigParser import SafeConfigParser
|
||||
import logging
|
||||
from logging import debug, error, exception, warning
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
|
||||
from weboob.core.backend import BaseBackend
|
||||
from weboob.capabilities.cap import ICap
|
||||
|
||||
|
||||
__all__ = ['Module']
|
||||
|
||||
|
||||
class Module(object):
|
||||
def __init__(self, name, module):
|
||||
self.name = name
|
||||
self.module = module
|
||||
self.klass = None
|
||||
for attrname in dir(self.module):
|
||||
attr = getattr(self.module, attrname)
|
||||
if isinstance(attr, type) and issubclass(attr, BaseBackend) and attr != BaseBackend:
|
||||
self.klass = attr
|
||||
|
||||
if not self.klass:
|
||||
raise ImportError("This is not a backend module (no BaseBackend class found)")
|
||||
|
||||
def get_name(self):
|
||||
return self.klass.NAME
|
||||
|
||||
def get_maintainer(self):
|
||||
return '%s <%s>' % (self.klass.MAINTAINER, self.klass.EMAIL)
|
||||
|
||||
def get_version(self):
|
||||
return self.klass.VERSION
|
||||
|
||||
def get_description(self):
|
||||
return self.klass.DESCRIPTION
|
||||
|
||||
def get_license(self):
|
||||
return self.klass.LICENSE
|
||||
|
||||
def get_config(self):
|
||||
return self.klass.CONFIG
|
||||
|
||||
def get_icon_path(self):
|
||||
return self.klass.ICON
|
||||
|
||||
def iter_caps(self):
|
||||
for cap in self.klass.__bases__:
|
||||
if issubclass(cap, ICap) and cap != ICap:
|
||||
yield cap
|
||||
|
||||
def has_caps(self, *caps):
|
||||
for c in caps:
|
||||
if issubclass(self.klass, c):
|
||||
return True
|
||||
return False
|
||||
|
||||
def create_backend(self, weboob, name, config, storage):
|
||||
debug('Created backend "%s"' % name)
|
||||
return self.klass(weboob, name, config, storage)
|
||||
|
||||
class BackendsConfig(object):
|
||||
class WrongPermissions(Exception):
|
||||
pass
|
||||
|
||||
def __init__(self, confpath):
|
||||
self.confpath = confpath
|
||||
try:
|
||||
mode = os.stat(confpath).st_mode
|
||||
except OSError:
|
||||
os.mknod(confpath, 0600)
|
||||
else:
|
||||
if mode & stat.S_IRGRP or mode & stat.S_IROTH:
|
||||
raise self.WrongPermissions(
|
||||
u'Weboob will not start until config file %s is readable by group or other users.' % confpath)
|
||||
|
||||
def iter_backends(self):
|
||||
config = SafeConfigParser()
|
||||
config.read(self.confpath)
|
||||
for name in config.sections():
|
||||
params = dict(config.items(name, raw=True))
|
||||
try:
|
||||
yield name, params.pop('_type'), params
|
||||
except KeyError:
|
||||
warning('Missing field "_type" for backend "%s"', name)
|
||||
continue
|
||||
|
||||
def add_backend(self, name, _type, params, edit=False):
|
||||
if not name:
|
||||
raise ValueError(u'Please give a name to the backend.')
|
||||
config = SafeConfigParser()
|
||||
config.read(self.confpath)
|
||||
if not edit:
|
||||
config.add_section(name)
|
||||
config.set(name, '_type', _type)
|
||||
for key, value in params.iteritems():
|
||||
config.set(name, key, value)
|
||||
with open(self.confpath, 'wb') as f:
|
||||
config.write(f)
|
||||
|
||||
def edit_backend(self, name, _type, params):
|
||||
return self.add_backend(name, _type, params, True)
|
||||
|
||||
def get_backend(self, name):
|
||||
config = SafeConfigParser()
|
||||
config.read(self.confpath)
|
||||
if not config.has_section(name):
|
||||
raise KeyError(u'Backend "%s" not found' % name)
|
||||
|
||||
items = dict(config.items(name, raw=True))
|
||||
try:
|
||||
return items.pop('_type'), items
|
||||
except KeyError:
|
||||
warning('Missing field "_type" for backend "%s"', name)
|
||||
raise KeyError(u'Backend "%s" not found' % name)
|
||||
|
||||
def remove_backend(self, name):
|
||||
config = SafeConfigParser()
|
||||
config.read(self.confpath)
|
||||
config.remove_section(name)
|
||||
with open(self.confpath, 'wb') as f:
|
||||
config.write(f)
|
||||
|
||||
class ModulesLoader(object):
|
||||
def __init__(self):
|
||||
self.modules = {}
|
||||
|
||||
def get_or_load_module(self, name):
|
||||
if name not in self.modules:
|
||||
self.load_module('weboob.backends.%s' % name)
|
||||
return self.modules[name]
|
||||
|
||||
def load(self):
|
||||
import weboob.backends
|
||||
for path in weboob.backends.__path__:
|
||||
regexp = re.compile('^%s/([\w\d_]+)$' % path)
|
||||
for root, dirs, files in os.walk(path):
|
||||
m = regexp.match(root)
|
||||
if m and '__init__.py' in files:
|
||||
self.load_module('weboob.backends.%s' % m.group(1))
|
||||
|
||||
def load_module(self, name):
|
||||
try:
|
||||
module = Module(name, __import__(name, fromlist=[str(name)]))
|
||||
except ImportError, e:
|
||||
msg = 'Unable to load module "%s": %s' % (name, e)
|
||||
if logging.root.level == logging.DEBUG:
|
||||
exception(msg)
|
||||
return
|
||||
else:
|
||||
error(msg)
|
||||
return
|
||||
if module.get_name() in self.modules:
|
||||
warning('Module "%s" is already loaded (%s)' % (self.modules[module.get_name()].module, name))
|
||||
return
|
||||
self.modules[module.get_name()] = module
|
||||
debug('Loaded module "%s" from %s' % (name, module.module.__path__))
|
||||
84
weboob/core/scheduler.py
Normal file
84
weboob/core/scheduler.py
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright(C) 2010 Romain Bignon, Christophe Benz
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, version 3 of the License.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
|
||||
|
||||
import logging
|
||||
from threading import Timer, Event
|
||||
|
||||
|
||||
__all__ = ['Scheduler']
|
||||
|
||||
|
||||
class IScheduler(object):
|
||||
def schedule(self, interval, function, *args):
|
||||
raise NotImplementedError()
|
||||
|
||||
def repeat(self, interval, function, *args):
|
||||
raise NotImplementedError()
|
||||
|
||||
def run(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
def want_stop(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
class Scheduler(IScheduler):
|
||||
def __init__(self):
|
||||
self.stop_event = Event()
|
||||
self.count = 0
|
||||
self.queue = {}
|
||||
|
||||
def schedule(self, interval, function, *args):
|
||||
if self.stop_event.isSet():
|
||||
return
|
||||
|
||||
self.count += 1
|
||||
logging.debug('function "%s" will be called in %s seconds' % (function.__name__, interval))
|
||||
timer = Timer(interval, function, args)
|
||||
timer.start()
|
||||
self.queue[self.count] = timer
|
||||
return self.count
|
||||
|
||||
def repeat(self, interval, function, *args):
|
||||
return self._repeat(True, interval, function, *args)
|
||||
|
||||
def _repeat(self, first, interval, function, *args):
|
||||
return self.schedule(0 if first else interval, self._repeated_cb, interval, function, args)
|
||||
|
||||
def _wait_to_stop(self):
|
||||
self.want_stop()
|
||||
for e in self.queue.itervalues():
|
||||
e.cancel()
|
||||
e.join()
|
||||
|
||||
def run(self):
|
||||
try:
|
||||
while 1:
|
||||
self.stop_event.wait(0.1)
|
||||
except KeyboardInterrupt:
|
||||
self._wait_to_stop()
|
||||
raise
|
||||
else:
|
||||
self._wait_to_stop()
|
||||
return True
|
||||
|
||||
def want_stop(self):
|
||||
self.stop_event.set()
|
||||
|
||||
def _repeated_cb(self, interval, function, args):
|
||||
function(*args)
|
||||
self._repeat(False, interval, function, *args)
|
||||
Loading…
Add table
Add a link
Reference in a new issue