move core files to weboob.core

weboob/backends/aum/backend.py

@@ -23,7 +23,7 @@ import os
 from logging import warning
 from time import sleep
 
-from weboob.backend import BaseBackend
+from weboob.core.backend import BaseBackend
 from weboob.capabilities.chat import ICapChat
 from weboob.capabilities.messages import ICapMessages, ICapMessagesReply, Message
 from weboob.capabilities.dating import ICapDating, StatusField

weboob/backends/bnporc/backend.py

@@ -17,7 +17,7 @@
 
 import os
 
-from weboob.backend import BaseBackend
+from weboob.core.backend import BaseBackend
 from weboob.capabilities.bank import ICapBank, AccountNotFound
 
 from .browser import BNPorc

weboob/backends/canaltp/backend.py

@@ -16,7 +16,7 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 
-from weboob.backend import BaseBackend
+from weboob.core.backend import BaseBackend
 from weboob.capabilities.travel import ICapTravel, Station, Departure
 
 from .browser import CanalTP

weboob/backends/cragr/backend.py

@@ -16,7 +16,7 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 
-from weboob.backend import BaseBackend
+from weboob.core.backend import BaseBackend
 from weboob.capabilities.bank import ICapBank, AccountNotFound
 
 from .browser import Cragr

weboob/backends/dlfp/backend.py

@@ -17,7 +17,7 @@
 
 from __future__ import with_statement
 
-from weboob.backend import BaseBackend
+from weboob.core.backend import BaseBackend
 from weboob.tools.browser import BrowserUnavailable
 from weboob.capabilities.messages import ICapMessages, ICapMessagesReply, Message
 
@@ -28,7 +28,7 @@ class DLFPBackend(BaseBackend, ICapMessages, ICapMessagesReply):
     NAME = 'dlfp'
     MAINTAINER = 'Romain Bignon'
     EMAIL = 'romain@peerfuse.org'
-    VERSION = '1.0'
+    VERSION = '0.1'
     LICENSE = 'GPLv3'
     DESCRIPTION = "Da Linux French Page"
 

weboob/backends/fourchan/backend.py

@@ -19,7 +19,7 @@ from __future__ import with_statement
 
 from logging import warning
 
-from weboob.backend import BaseBackend
+from weboob.core.backend import BaseBackend
 from weboob.capabilities.messages import ICapMessages, Message
 
 from .browser import FourChan
@@ -28,7 +28,7 @@ class FourChanBackend(BaseBackend, ICapMessages):
     NAME = 'fourchan'
     MAINTAINER = 'Romain Bignon'
     EMAIL = 'romain@peerfuse.org'
-    VERSION = '1.0'
+    VERSION = '0.1'
     LICENSE = 'GPLv3'
     DESCRIPTION = "4chan website"
 

weboob/backends/gazelle/backend.py

@@ -16,7 +16,7 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 
-from weboob.backend import BaseBackend
+from weboob.core.backend import BaseBackend
 from weboob.capabilities.torrent import ICapTorrent
 
 from .browser import GazelleBrowser

weboob/backends/ina/backend.py

@@ -16,7 +16,7 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 
-from weboob.backend import BaseBackend
+from weboob.core.backend import BaseBackend
 from weboob.capabilities.video import ICapVideo
 
 from .browser import InaBrowser

weboob/backends/transilien/backend.py

@@ -16,7 +16,7 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 
-from weboob.backend import BaseBackend
+from weboob.core.backend import BaseBackend
 from weboob.capabilities.travel import ICapTravel, Station, Departure
 
 from .browser import Transilien

weboob/backends/youjizz/backend.py

@@ -16,7 +16,7 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 
-from weboob.backend import BaseBackend
+from weboob.core.backend import BaseBackend
 from weboob.capabilities.video import ICapVideo
 
 from .browser import YoujizzBrowser

weboob/backends/youporn/backend.py

@@ -16,7 +16,7 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 
-from weboob.backend import BaseBackend
+from weboob.core.backend import BaseBackend
 from weboob.capabilities.video import ICapVideo
 
 from .browser import YoupornBrowser

weboob/backends/youtube/backend.py

@@ -18,7 +18,7 @@
 
 import logging
 
-from weboob.backend import BaseBackend
+from weboob.core.backend import BaseBackend
 from weboob.capabilities.video import ICapVideo
 
 from .browser import YoutubeBrowser

weboob/backends/yweather/backend.py

@@ -22,7 +22,7 @@ from xml.dom import minidom
 # TODO store datetime objects instead of strings
 # from datetime import datetime
 
-from weboob.backend import BaseBackend
+from weboob.core.backend import BaseBackend
 from weboob.capabilities.weather import ICapWeather, CityNotFound, Current, Forecast
 
 class YWeatherBackend(BaseBackend, ICapWeather):

weboob/core/__init__.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2010  Christophe Benz
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+
+from .bcall import CallErrors

weboob/core/backend.py

@@ -0,0 +1,166 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2010  Romain Bignon
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+
+import re
+import os
+from threading import RLock
+
+
+__all__ = ['BackendStorage', 'BaseBackend']
+
+
+class BackendStorage(object):
+    def __init__(self, name, storage):
+        self.name = name
+        self.storage = storage
+
+    def set(self, *args):
+        if self.storage:
+            return self.storage.set('backends', self.name, *args)
+
+    def get(self, *args, **kwargs):
+        if self.storage:
+            return self.storage.get('backends', self.name, *args, **kwargs)
+        else:
+            return kwargs.get('default', None)
+
+    def load(self, default):
+        if self.storage:
+            return self.storage.load('backends', self.name, default)
+
+    def save(self):
+        if self.storage:
+            return self.storage.save('backends', self.name)
+
+class BaseBackend(object):
+    # Module name.
+    NAME = None
+    # Name of the maintainer of this module.
+    MAINTAINER = '<unspecifier>'
+    # Email address of the maintainer.
+    EMAIL = '<unspecified>'
+    # Version of module (for information only).
+    VERSION = '<unspecified>'
+    # Description
+    DESCRIPTION = '<unspecified>'
+    # License of this module.
+    LICENSE = '<unspecified>'
+    # Icon file path
+    ICON = ''
+    # Configuration required for this module.  # Values must be ConfigField
+    # objects.
+    CONFIG = {}
+    # Storage
+    STORAGE = {}
+    # Browser class
+    BROWSER = None
+    # Test class
+    TEST = None
+
+    class ConfigField(object):
+        def __init__(self, default=None, is_masked=False, regexp=None, description=None):
+            self.default = default
+            self.is_masked = is_masked
+            self.regexp = regexp
+            self.description = description
+
+    class ConfigError(Exception): pass
+
+    def __enter__(self):
+        self.lock.acquire()
+
+    def __exit__(self, t, v, tb):
+        self.lock.release()
+
+    def __repr__(self):
+        return u"<Backend '%s'>" % self.name
+
+    def __init__(self, weboob, name, config, storage):
+        self.weboob = weboob
+        self.name = name
+        self.lock = RLock()
+
+        # Private fields (which start with '_')
+        self._private_config = dict([(key, value) for key, value in config.iteritems() if key.startswith('_')])
+
+        # Configuration of backend
+        self.config = {}
+        for name, field in self.CONFIG.iteritems():
+            value = config.get(name, field.default)
+
+            if value is None:
+                raise BaseBackend.ConfigError('Missing parameter "%s" (%s)' % (name, field.description))
+
+            if field.regexp and not re.match(field.regexp, str(value)):
+                raise BaseBackend.ConfigError('Value of "%s" does not match regexp "%s"' % (name, field.regexp))
+
+            if not field.default is None:
+                if isinstance(field.default, bool) and not isinstance(value, bool):
+                    value = value.lower() in ('1', 'true', 'on', 'yes')
+                elif isinstance(field.default, int) and not isinstance(value, int):
+                    value = int(value)
+                elif isinstance(field.default, float) and not isinstance(value, float):
+                    value = float(value)
+            self.config[name] = value
+        self.storage = BackendStorage(self.name, storage)
+        self.storage.load(self.STORAGE)
+
+    @property
+    def browser(self):
+        """
+        Attribute 'browser'. The browser is created at the first call
+        of this attribute, to avoid useless pages access.
+
+        Note that the 'default_browser' method is called to create it.
+        """
+        if not hasattr(self, '_browser'):
+            self._browser = self.default_browser()
+        return self._browser
+
+    def default_browser(self):
+        """
+        Method to overload to build the default browser in
+        attribute 'browser'.
+        """
+        return self.build_browser()
+
+    def build_browser(self, *args, **kwargs):
+        """
+        Build a browser from the BROWSER class attribute and the
+        given arguments.
+        """
+        if not self.BROWSER:
+            return None
+
+        if '_proxy' in self._private_config:
+            kwargs['proxy'] = self._private_config['_proxy']
+        elif 'HTTP_PROXY' in os.environ:
+            kwargs['proxy'] = os.environ['HTTP_PROXY']
+
+        return self.BROWSER(*args, **kwargs)
+
+    def has_caps(self, *caps):
+        for c in caps:
+            if isinstance(self, c):
+                return True
+        return False
+
+    def get_test(self):
+        if not self.TEST:
+            return None
+        return self.TEST(self)

weboob/core/bcall.py

@@ -0,0 +1,188 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2010  Romain Bignon, Christophe Benz
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+
+from __future__ import with_statement
+
+from copy import copy
+import logging
+from logging import debug
+from threading import Thread, Event, RLock, Timer
+from weboob.tools.misc import get_backtrace
+
+
+__all__ = ['BackendsCall', 'CallErrors']
+
+
+class CallErrors(Exception):
+    def __init__(self, errors):
+        Exception.__init__(self, u'These errors have been raised in backend threads '\
+                           '(use --debug option to print backtraces):\n%s' % (
+            u'\n'.join((u' * %s: %s%s' % (backend, error, backtrace + '\n'
+                                          if logging.root.level == logging.DEBUG else ''))
+                       for backend, error, backtrace in errors)))
+        self.errors = copy(errors)
+
+    def __iter__(self):
+        return self.errors.__iter__()
+
+
+class BackendsCall(object):
+    def __init__(self, backends, function, *args, **kwargs):
+        """
+        @param backends  list of backends to call
+        @param function  backends' method name, or callable object
+        @param args, kwargs  arguments given to called functions
+        """
+        # Store if a backend is finished
+        self.backends = {}
+        for backend in backends:
+            self.backends[backend.name] = False
+        # Global mutex on object
+        self.mutex = RLock()
+        # Event set when every backends have give their data
+        self.finish_event = Event()
+        # Event set when there are new responses
+        self.response_event = Event()
+        # Waiting responses
+        self.responses = []
+        # Errors
+        self.errors = []
+        # Threads
+        self.threads = []
+
+        # Create jobs for each backend
+        with self.mutex:
+            for backend in backends:
+                debug('Creating a new thread for %s' % backend)
+                self.threads.append(Timer(0, self._caller, (backend, function, args, kwargs)).start())
+            if not backends:
+                self.finish_event.set()
+
+    def _store_error(self, backend, error):
+        with self.mutex:
+            backtrace = get_backtrace(error)
+            self.errors.append((backend, error, backtrace))
+
+    def _store_result(self, backend, result):
+        with self.mutex:
+            self.responses.append((backend, result))
+            self.response_event.set()
+
+    def _caller(self, backend, function, args, kwargs):
+        debug('%s: Thread created successfully' % backend)
+        with backend:
+            try:
+                # Call method on backend
+                try:
+                    debug('%s: Calling function %s' % (backend, function))
+                    if callable(function):
+                        result = function(backend, *args, **kwargs)
+                    else:
+                        result = getattr(backend, function)(*args, **kwargs)
+                except Exception, error:
+                    self._store_error(backend, error)
+                else:
+                    debug('%s: Called function %s returned: "%s"' % (backend, function, result))
+
+                    if hasattr(result, '__iter__'):
+                        # Loop on iterator
+                        try:
+                            for subresult in result:
+                                # Lock mutex only in loop in case the iterator is slow
+                                # (for example if backend do some parsing operations)
+                                self._store_result(backend, subresult)
+                        except Exception, error:
+                            self._store_error(backend, error)
+                    else:
+                        self._store_result(backend, result)
+            finally:
+                with self.mutex:
+                    # This backend is now finished
+                    self.backends[backend.name] = True
+                    for finished in self.backends.itervalues():
+                        if not finished:
+                            return
+                    self.finish_event.set()
+                    self.response_event.set()
+
+    def _callback_thread_run(self, callback, errback):
+        responses = []
+        while not self.finish_event.isSet() or self.response_event.isSet():
+            self.response_event.wait()
+            with self.mutex:
+                responses = self.responses
+                self.responses = []
+
+                # Reset event
+                self.response_event.clear()
+
+            # Consume responses
+            while responses:
+                callback(*responses.pop(0))
+
+        if errback:
+            with self.mutex:
+                while self.errors:
+                    errback(*self.errors.pop(0))
+
+        callback(None, None)
+
+    def callback_thread(self, callback, errback=None):
+        """
+        Call this method to create a thread which will callback a
+        specified function everytimes a new result comes.
+
+        When the process is over, the function will be called with
+        both arguments set to None.
+
+        The functions prototypes:
+            def callback(backend, result)
+            def errback(backend, error)
+
+        """
+        thread = Thread(target=self._callback_thread_run, args=(callback, errback))
+        thread.start()
+        return thread
+
+    def wait(self):
+        self.finish_event.wait()
+
+        with self.mutex:
+            if self.errors:
+                raise CallErrors(self.errors)
+
+    def __iter__(self):
+        # Don't know how to factorize with _callback_thread_run
+        responses = []
+        while not self.finish_event.isSet() or self.response_event.isSet():
+            self.response_event.wait()
+            with self.mutex:
+                responses = self.responses
+                self.responses = []
+
+                # Reset event
+                self.response_event.clear()
+
+            # Consume responses
+            while responses:
+                yield responses.pop(0)
+
+        # Raise errors
+        with self.mutex:
+            if self.errors:
+                raise CallErrors(self.errors)

weboob/core/engine.py

@@ -0,0 +1,178 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2010  Romain Bignon
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+
+from __future__ import with_statement
+
+from logging import warning
+import os
+import sys
+
+from weboob.core.bcall import BackendsCall
+from weboob.core.modules import ModulesLoader, BackendsConfig
+from weboob.core.backend import BaseBackend
+from weboob.core.scheduler import Scheduler
+
+if sys.version_info[:2] <= (2, 5):
+    import weboob.tools.property
+
+
+__all__ = ['Weboob']
+
+
+class Weboob(object):
+    WORKDIR = os.path.join(os.path.expanduser('~'), '.weboob')
+    BACKENDS_FILENAME = 'backends'
+
+    def __init__(self, workdir=WORKDIR, backends_filename=None, scheduler=None):
+        self.workdir = workdir
+        self.backends = {}
+
+        # Scheduler
+        if scheduler is None:
+            scheduler = Scheduler()
+        self.scheduler = scheduler
+
+        # Create WORKDIR
+        if not os.path.exists(self.workdir):
+            os.mkdir(self.workdir, 0700)
+        elif not os.path.isdir(self.workdir):
+            warning('"%s" is not a directory' % self.workdir)
+
+        # Modules loader
+        self.modules_loader = ModulesLoader()
+
+        # Backends config
+        if not backends_filename:
+            backends_filename = os.path.join(self.workdir, self.BACKENDS_FILENAME)
+        elif not backends_filename.startswith('/'):
+            backends_filename = os.path.join(self.workdir, backends_filename)
+        self.backends_config = BackendsConfig(backends_filename)
+
+    def load_backends(self, caps=None, names=None, storage=None):
+        loaded_backends = {}
+        for name, _type, params in self.backends_config.iter_backends():
+            try:
+                module = self.modules_loader.get_or_load_module(_type)
+            except KeyError:
+                warning(u'Unable to find module "%s" for backend "%s"' % (_type, name))
+                continue
+
+            # Check conditions
+            if (not caps is None and not module.has_caps(caps)) or \
+               (not names is None and not name in names):
+                continue
+
+            try:
+                self.backends[name] = module.create_backend(self, name, params, storage)
+                loaded_backends[name] = self.backends[name]
+            except Exception, e:
+                warning(u'Unable to load "%s" backend: %s. filename=%s' % (name, e, self.backends_config.confpath))
+
+        return loaded_backends
+
+    def load_modules(self, caps=None, names=None, storage=None):
+        loaded_backends = {}
+        self.modules_loader.load()
+        for name, module in self.modules_loader.modules.iteritems():
+            if (caps is None or module.has_caps(caps)) and \
+               (names is None or module.get_name() in names):
+                try:
+                    name = module.get_name()
+                    self.backends[name] = module.create_backend(self, name, {}, storage)
+                    loaded_backends[name] = self.backends[name]
+                except Exception, e:
+                    warning(u'Unable to load "%s" module as backend with no config: %s' % (name, e))
+        return loaded_backends
+
+    def iter_backends(self, caps=None):
+        """
+        Iter on each backends.
+
+        Note: each backend is locked when it is returned.
+
+        @param caps  Optional list of capabilities to select backends
+        @return  iterator on selected backends.
+        """
+        for name, backend in sorted(self.backends.iteritems()):
+            if caps is None or backend.has_caps(caps):
+                with backend:
+                    yield backend
+
+    def do(self, function, *args, **kwargs):
+        """
+        Do calls on loaded backends with specified arguments, in separated
+        threads.
+
+        This function has two modes:
+        - If 'function' is a string, it calls the method with this name on
+          each backends with the specified arguments;
+        - If 'function' is a callable, it calls it in a separated thread with
+          the locked backend instance at first arguments, and *args and
+          **kwargs.
+
+        @param function  backend's method name, or callable object
+        @return  an iterator of results
+        """
+        backends = list(self.iter_backends())
+        return BackendsCall(backends, function, *args, **kwargs)
+
+    def do_caps(self, caps, function, *args, **kwargs):
+        """
+        Do calls on loaded modules with the specified capabilities, in
+        separated threads.
+
+        See also documentation of the 'do' method.
+
+        @param caps  list of caps or cap to select backends
+        @param function  backend's method name, or callable object
+        @return  an iterator of results
+        """
+        backends = list(self.iter_backends(caps))
+        return BackendsCall(backends, function, *args, **kwargs)
+
+    def do_backends(self, backends, function, *args, **kwargs):
+        if backends is None:
+            backends = list(self.iter_backends())
+        elif isinstance(backends, BaseBackend):
+            backends = [backends]
+        elif isinstance(backends, (str,unicode)):
+            backends = [backend for backend in self.iter_backends() if backend.name == backends]
+        elif isinstance(backends, (list,tuple)):
+            old_backends = backends
+            backends = []
+            for b in old_backends:
+                if isinstance(b, (str,unicode)):
+                    try:
+                        backends.append(self.backends[self.backends.index(b)])
+                    except ValueError:
+                        pass
+                else:
+                    backends.append(b)
+        return BackendsCall(backends, function, *args, **kwargs)
+
+    def schedule(self, interval, function, *args):
+        return self.scheduler.schedule(interval, function, *args)
+
+    def repeat(self, interval, function, *args):
+        return self.scheduler.repeat(interval, function, *args)
+
+    def want_stop(self):
+        return self.scheduler.want_stop()
+
+    def loop(self):
+        return self.scheduler.run()

weboob/core/modules.py

@@ -0,0 +1,179 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2010  Romain Bignon
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+
+from __future__ import with_statement
+
+from ConfigParser import SafeConfigParser
+import logging
+from logging import debug, error, exception, warning
+import os
+import re
+import stat
+
+from weboob.core.backend import BaseBackend
+from weboob.capabilities.cap import ICap
+
+
+__all__ = ['Module']
+
+
+class Module(object):
+    def __init__(self, name, module):
+        self.name = name
+        self.module = module
+        self.klass = None
+        for attrname in dir(self.module):
+            attr = getattr(self.module, attrname)
+            if isinstance(attr, type) and issubclass(attr, BaseBackend) and attr != BaseBackend:
+                self.klass = attr
+
+        if not self.klass:
+            raise ImportError("This is not a backend module (no BaseBackend class found)")
+
+    def get_name(self):
+        return self.klass.NAME
+
+    def get_maintainer(self):
+        return '%s <%s>' % (self.klass.MAINTAINER, self.klass.EMAIL)
+
+    def get_version(self):
+        return self.klass.VERSION
+
+    def get_description(self):
+        return self.klass.DESCRIPTION
+
+    def get_license(self):
+        return self.klass.LICENSE
+
+    def get_config(self):
+        return self.klass.CONFIG
+
+    def get_icon_path(self):
+        return self.klass.ICON
+
+    def iter_caps(self):
+        for cap in self.klass.__bases__:
+            if issubclass(cap, ICap) and cap != ICap:
+                yield cap
+
+    def has_caps(self, *caps):
+        for c in caps:
+            if issubclass(self.klass, c):
+                return True
+        return False
+
+    def create_backend(self, weboob, name, config, storage):
+        debug('Created backend "%s"' % name)
+        return self.klass(weboob, name, config, storage)
+
+class BackendsConfig(object):
+    class WrongPermissions(Exception):
+        pass
+
+    def __init__(self, confpath):
+        self.confpath = confpath
+        try:
+            mode = os.stat(confpath).st_mode
+        except OSError:
+            os.mknod(confpath, 0600)
+        else:
+            if mode & stat.S_IRGRP or mode & stat.S_IROTH:
+                raise self.WrongPermissions(
+                    u'Weboob will not start until config file %s is readable by group or other users.' % confpath)
+
+    def iter_backends(self):
+        config = SafeConfigParser()
+        config.read(self.confpath)
+        for name in config.sections():
+            params = dict(config.items(name, raw=True))
+            try:
+                yield name, params.pop('_type'), params
+            except KeyError:
+                warning('Missing field "_type" for backend "%s"', name)
+                continue
+
+    def add_backend(self, name, _type, params, edit=False):
+        if not name:
+            raise ValueError(u'Please give a name to the backend.')
+        config = SafeConfigParser()
+        config.read(self.confpath)
+        if not edit:
+            config.add_section(name)
+        config.set(name, '_type', _type)
+        for key, value in params.iteritems():
+            config.set(name, key, value)
+        with open(self.confpath, 'wb') as f:
+            config.write(f)
+
+    def edit_backend(self, name, _type, params):
+        return self.add_backend(name, _type, params, True)
+
+    def get_backend(self, name):
+        config = SafeConfigParser()
+        config.read(self.confpath)
+        if not config.has_section(name):
+            raise KeyError(u'Backend "%s" not found' % name)
+
+        items = dict(config.items(name, raw=True))
+        try:
+            return items.pop('_type'), items
+        except KeyError:
+            warning('Missing field "_type" for backend "%s"', name)
+            raise KeyError(u'Backend "%s" not found' % name)
+
+    def remove_backend(self, name):
+        config = SafeConfigParser()
+        config.read(self.confpath)
+        config.remove_section(name)
+        with open(self.confpath, 'wb') as f:
+            config.write(f)
+
+class ModulesLoader(object):
+    def __init__(self):
+        self.modules = {}
+
+    def get_or_load_module(self, name):
+        if name not in self.modules:
+            self.load_module('weboob.backends.%s' % name)
+        return self.modules[name]
+
+    def load(self):
+            import weboob.backends
+            for path in weboob.backends.__path__:
+                regexp = re.compile('^%s/([\w\d_]+)$' % path)
+                for root, dirs, files in os.walk(path):
+                    m = regexp.match(root)
+                    if m and '__init__.py' in files:
+                        self.load_module('weboob.backends.%s' % m.group(1))
+
+    def load_module(self, name):
+        try:
+            module = Module(name, __import__(name, fromlist=[str(name)]))
+        except ImportError, e:
+            msg = 'Unable to load module "%s": %s' % (name, e)
+            if logging.root.level == logging.DEBUG:
+                exception(msg)
+                return
+            else:
+                error(msg)
+                return
+        if module.get_name() in self.modules:
+            warning('Module "%s" is already loaded (%s)' % (self.modules[module.get_name()].module, name))
+            return
+        self.modules[module.get_name()] = module
+        debug('Loaded module "%s" from %s' % (name, module.module.__path__))

weboob/core/scheduler.py

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2010  Romain Bignon, Christophe Benz
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+
+import logging
+from threading import Timer, Event
+
+
+__all__ = ['Scheduler']
+
+
+class IScheduler(object):
+    def schedule(self, interval, function, *args):
+        raise NotImplementedError()
+
+    def repeat(self, interval, function, *args):
+        raise NotImplementedError()
+
+    def run(self):
+        raise NotImplementedError()
+
+    def want_stop(self):
+        raise NotImplementedError()
+
+class Scheduler(IScheduler):
+    def __init__(self):
+        self.stop_event = Event()
+        self.count = 0
+        self.queue = {}
+
+    def schedule(self, interval, function, *args):
+        if self.stop_event.isSet():
+            return
+
+        self.count += 1
+        logging.debug('function "%s" will be called in %s seconds' % (function.__name__, interval))
+        timer = Timer(interval, function, args)
+        timer.start()
+        self.queue[self.count] = timer
+        return self.count
+
+    def repeat(self, interval, function, *args):
+        return self._repeat(True, interval, function, *args)
+
+    def _repeat(self, first, interval, function, *args):
+        return self.schedule(0 if first else interval, self._repeated_cb, interval, function, args)
+
+    def _wait_to_stop(self):
+        self.want_stop()
+        for e in self.queue.itervalues():
+            e.cancel()
+            e.join()
+
+    def run(self):
+        try:
+            while 1:
+                self.stop_event.wait(0.1)
+        except KeyboardInterrupt:
+            self._wait_to_stop()
+            raise
+        else:
+            self._wait_to_stop()
+        return True
+
+    def want_stop(self):
+        self.stop_event.set()
+
+    def _repeated_cb(self, interval, function, args):
+        function(*args)
+        self._repeat(False, interval, function, *args)

weboob/frontends/monboob/monboob.py

@@ -28,8 +28,8 @@ import sys
 import logging
 import asyncore
 
-from weboob import Weboob
-from weboob.scheduler import Scheduler
+from weboob.core.engine import Weboob
+from weboob.core.scheduler import Scheduler
 from weboob.capabilities.messages import ICapMessages, ICapMessagesReply, Message
 from weboob.tools.application import ConsoleApplication
 from weboob.tools.misc import html2text, get_backtrace, utc2local