nojhan/weboob-devel
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

change way to describe fields of CapBaseObject, and lot of documentation

Browse source
This commit is contained in:
Romain Bignon 2012-03-25 22:29:18 +02:00
commit c6a141595c
35 changed files with 1630 additions and 638 deletions
Download patch file Download diff file Expand all files Collapse all files

9
weboob/tools/application/repl.py
View file

@ -39,7 +39,7 @@ from .formatters.load import FormattersLoader, FormatterLoadError
from .results import ResultsCondition, ResultsConditionError
__all__ = ['ReplApplication', 'NotEnoughArguments']
__all__ = ['NotEnoughArguments', 'ReplApplication']
class NotEnoughArguments(Exception):
@ -354,9 +354,10 @@ class ReplApplication(Cmd, ConsoleApplication):
def complete(self, text, state):
"""
Override of the Cmd.complete() method to:
- add a space at end of proposals
- display only proposals for words which match the
text already written by user.
* add a space at end of proposals
* display only proposals for words which match the
text already written by user.
"""
super(ReplApplication, self).complete(text, state)

140
weboob/tools/backend.py
View file

@ -28,42 +28,103 @@ from weboob.tools.log import getLogger
from weboob.tools.value import ValuesDict
__all__ = ['BaseBackend', 'ObjectNotAvailable']
__all__ = ['ObjectNotAvailable', 'BackendStorage', 'BackendConfig', 'BaseBackend']
class ObjectNotAvailable(Exception):
pass
"""
Raised when an object is not available.
"""
class BackendStorage(object):
"""
This is an abstract layer to store data in storages (:mod:`weboob.tools.storage`)
easily.
It is instancied automatically in constructor of :class:`BaseBackend`, in the
:attr:`BaseBackend.storage` attribute.
:param name: name of backend
:param storage: storage object
:type storage: :class:`weboob.tools.storage.IStorage`
"""
def __init__(self, name, storage):
self.name = name
self.storage = storage
def set(self, *args):
"""
Set value in the storage.
Example:
>>> backend.storage.set('config', 'nb_of_threads', 10)
>>>
:param args: the path where to store value
"""
if self.storage:
return self.storage.set('backends', self.name, *args)
def delete(self, *args):
"""
Delete a value from the storage.
:param args: path to delete.
"""
if self.storage:
return self.storage.delete('backends', self.name, *args)
def get(self, *args, **kwargs):
"""
Get a value or a dict of values in storage.
Example:
>>> backend.storage.get('config', 'nb_of_threads')
10
>>> backend.storage.get('config', 'unexistant', 'path', default='lol')
'lol'
>>> backend.storage.get('config')
{'nb_of_threads': 10, 'other_things': 'blah'}
:param args: path to get
:param default: if specified, default value when path is not found
"""
if self.storage:
return self.storage.get('backends', self.name, *args, **kwargs)
else:
return kwargs.get('default', None)
def load(self, default):
"""
Load storage.
:param default: this is the default tree if storage is empty
:type default: :class:`dict`
"""
if self.storage:
return self.storage.load('backends', self.name, default)
def save(self):
"""
Save storage.
"""
if self.storage:
return self.storage.save('backends', self.name)
class BackendConfig(ValuesDict):
"""
Configuration of a backend.
This class is firstly instanced as a :class:`weboob.tools.value.ValuesDict`,
containing some :class:`weboob.tools.value.Value` (and derivated) objects.
Then, using the :func:`load` method will load configuration from file and
create a copy of the :class:`BackendConfig` object with the loaded values.
"""
modname = None
instname = None
weboob = None
@ -72,12 +133,17 @@ class BackendConfig(ValuesDict):
"""
Load configuration from dict to create an instance.
@param weboob [Weboob] weboob object
@param modname [str] name of module
@param instname [str] name of instance of this backend
@param params [dict] parameters to load
@param nofail [bool] if true, this call can't fail.
@return [BackendConfig]
:param weboob: weboob object
:type weboob: :class:`weboob.core.ouiboube.Weboob`
:param modname: name of the module
:type modname: :class:`str`
:param instname: name of this backend
:type instname: :class:`str`
:param params: parameters to load
:type params: :class:`dict`
:param nofail: if true, this call can't fail
:type nofail: :class:`bool`
:rtype: :class:`BackendConfig`
"""
cfg = BackendConfig()
cfg.modname = modname
@ -103,12 +169,25 @@ class BackendConfig(ValuesDict):
return cfg
def dump(self):
"""
Dump config in a dictionary.
:rtype: :class:`dict`
"""
settings = {}
for name, value in self.iteritems():
settings[name] = value.dump()
return settings
def save(self, edit=True, params=None):
"""
Save backend config.
:param edit: if true, it changes config of an existing backend
:type edit: :class:`bool`
:param params: if specified, params to merge with the ones of the current object
:type params: :class:`dict`
"""
assert self.modname is not None
assert self.instname is not None
assert self.weboob is not None
@ -121,6 +200,22 @@ class BackendConfig(ValuesDict):
class BaseBackend(object):
"""
Base class for backends.
You may derivate it, and also all capabilities you want to implement.
:param weboob: weboob instance
:type weboob: :class:`weboob.core.ouiboube.Weboob`
:param name: name of backend
:type name: :class:`str`
:param config: configuration of backend
:type config: :class:`dict`
:param storage: storage object
:type storage: :class:`weboob.tools.storage.IStorage`
:param logger: logger
:type logger: :class:`logging.Logger`
"""
# Backend name.
NAME = None
# Name of the maintainer of this backend.
@ -152,7 +247,9 @@ class BaseBackend(object):
OBJECTS = {}
class ConfigError(Exception):
pass
"""
Raised when the config can't be loaded.
"""
def __enter__(self):
self.lock.acquire()
@ -184,16 +281,6 @@ class BaseBackend(object):
"""
pass
class classprop(object):
def __init__(self, fget):
self.fget = fget
def __get__(self, inst, objtype=None):
if inst:
return self.fget(inst)
else:
return self.fget(objtype)
_browser = None
@property
@ -202,7 +289,7 @@ class BaseBackend(object):
Attribute 'browser'. The browser is created at the first call
of this attribute, to avoid useless pages access.
Note that the 'create_default_browser' method is called to create it.
Note that the :func:`create_default_browser` method is called to create it.
"""
if self._browser is None:
self._browser = self.create_default_browser()
@ -238,6 +325,11 @@ class BaseBackend(object):
@classmethod
def iter_caps(klass):
"""
Iter capabilities implemented by this backend.
:rtype: iter[:class:`weboob.capabilities.base.IBaseCap`]
"""
def iter_caps(cls):
for base in cls.__bases__:
if issubclass(base, IBaseCap) and base != IBaseCap:
@ -247,6 +339,9 @@ class BaseBackend(object):
return iter_caps(klass)
def has_caps(self, *caps):
"""
Check if this backend implements at least one of these capabilities.
"""
for c in caps:
if (isinstance(c, basestring) and c in [cap.__name__ for cap in self.iter_caps()]) or \
isinstance(self, c):
@ -255,7 +350,10 @@ class BaseBackend(object):
def fillobj(self, obj, fields=None):
"""
@param fields which fields to fill; if None, all fields are filled (list)
Fill an object with the wanted fields.
:param fields: what fields to fill; if None, all fields are filled
:type fields: :class:`list`
"""
def not_loaded(v):
return (v is NotLoaded or isinstance(v, CapBaseObject) and not v.__iscomplete__())

87
weboob/tools/browser/browser.py
View file

@ -53,7 +53,8 @@ else:
__all__ = ['BrowserIncorrectPassword', 'BrowserBanned', 'BrowserUnavailable', 'BrowserRetry',
'BrowserHTTPNotFound', 'BrowserHTTPError', 'BasePage', 'BaseBrowser', 'StandardBrowser']
'BrowserHTTPNotFound', 'BrowserHTTPError', 'BrokenPageError', 'BasePage',
'StandardBrowser', 'BaseBrowser']
# Exceptions
@ -135,6 +136,21 @@ def check_location(func):
return inner
class StandardBrowser(mechanize.Browser):
"""
Standard Browser.
:param firefox_cookies: path to cookies sqlite file
:type firefox_cookies: str
:param parser: parser to use on HTML files
:type parser: :class:`weboob.tools.parsers.iparser.IParser`
:param history: history manager; default value is an object which
does not keep history
:type history: object
:param proxy: proxy URL to use
:type proxy: str
:param factory: mechanize factory. None to use Mechanize's default
:type factory: object
"""
# ------ Class attributes --------------------------------------
@ -161,16 +177,6 @@ class StandardBrowser(mechanize.Browser):
default_features.remove('_refresh')
def __init__(self, firefox_cookies=None, parser=None, history=NoHistory(), proxy=None, logger=None, factory=None, responses_dirname=None):
"""
Constructor of Browser.
@param filefox_cookies [str] Path to cookies' sqlite file.
@param parser [IParser] parser to use on HTML files.
@param history [object] History manager. Default value is an object
which does not keep history.
@param proxy [str] proxy URL to use.
@param factory [object] Mechanize factory. None to use Mechanize's default.
"""
mechanize.Browser.__init__(self, history=history, factory=factory)
self.logger = getLogger('browser', logger)
@ -309,15 +315,16 @@ class StandardBrowser(mechanize.Browser):
def buildurl(base, *args, **kwargs):
"""
Build an URL and escape arguments.
You can give a serie of tuples in *args (and the order is keept), or
a dict in **kwargs (but the order is lost).
You can give a serie of tuples in args (and the order is keept), or
a dict in kwargs (but the order is lost).
Example:
>>> buildurl('/blah.php', ('a', '&'), ('b', '=')
'/blah.php?a=%26&b=%3D'
>>> buildurl('/blah.php', a='&', 'b'='=')
'/blah.php?b=%3D&a=%26'
"""
if not args:
@ -336,11 +343,16 @@ class StandardBrowser(mechanize.Browser):
"""
Set a value to a form field.
@param args [dict] arguments where to look for value.
@param label [str] label in args.
@param field [str] field name. If None, use label instead.
@param value [str] value to give on field.
@param is_list [bool] the field is a list.
:param args: arguments where to look for value
:type args: dict
:param label: label in args
:type label: str
:param field: field name. If None, use label instead
:type field: str
:param value: value to give on field
:type value: str
:param is_list: the field is a list
:type is_list: bool
"""
try:
if not field:
@ -366,6 +378,29 @@ class StandardBrowser(mechanize.Browser):
class BaseBrowser(StandardBrowser):
"""
Base browser class to navigate on a website.
:param username: username on website
:type username: str
:param password: password on website. If it is None, Browser will
not try to login
:type password: str
:param firefox_cookies: path to cookies sqlite file
:type firefox_cookies: str
:param parser: parser to use on HTML files
:type parser: :class:`weboob.tools.parsers.iparser.IParser`
:param history: history manager; default value is an object which
does not keep history
:type history: object
:param proxy: proxy URL to use
:type proxy: str
:param logger: logger to use for logging
:type logger: :class:`logging.Logger`
:param factory: mechanize factory. None to use Mechanize's default
:type factory: object
:param get_home: try to get the homepage.
:type get_homme: bool
:param responses_dirname: directory to store responses
:type responses_dirname: str
"""
# ------ Class attributes --------------------------------------
@ -406,20 +441,6 @@ class BaseBrowser(StandardBrowser):
def __init__(self, username=None, password=None, firefox_cookies=None,
parser=None, history=NoHistory(), proxy=None, logger=None,
factory=None, get_home=True, responses_dirname=None):
"""
Constructor of Browser.
@param username [str] username on website.
@param password [str] password on website. If it is None, Browser will
not try to login.
@param filefox_cookies [str] Path to cookies' sqlite file.
@param parser [IParser] parser to use on HTML files.
@param hisory [object] History manager. Default value is an object
which does not keep history.
@param proxy [str] proxy URL to use.
@param factory [object] Mechanize factory. None to use Mechanize's default.
@param get_home [bool] Try to get the homepage.
"""
StandardBrowser.__init__(self, firefox_cookies, parser, history, proxy, logger, factory, responses_dirname)
self.page = None
self.last_update = 0.0

10
weboob/tools/capabilities/bank/transactions.py
View file

@ -30,6 +30,9 @@ __all__ = ['FrenchTransaction']
class FrenchTransaction(Transaction):
"""
Transaction with some helpers for french bank websites.
"""
PATTERNS = []
def clean_amount(self, text):
@ -61,7 +64,7 @@ class FrenchTransaction(Transaction):
When calling this method, you should have defined patterns (in the
PATTERN class attribute) with a list containing tuples of regexp
and the associated type, for example:
and the associated type, for example::
PATTERNS = [(re.compile('^VIR(EMENT)? (?P<text>.*)'), FrenchTransaction.TYPE_TRANSFER),
(re.compile('^PRLV (?P<text>.*)'), FrenchTransaction.TYPE_ORDER),
@ -70,8 +73,9 @@ class FrenchTransaction(Transaction):
]
In regexps, you can define this patterns:
- text: part of label to store in simplified label
- yy, mm, dd, HH, MM: date and time parts
* text: part of label to store in simplified label
* yy, mm, dd, HH, MM: date and time parts
"""
if not isinstance(date, (datetime.date, datetime.datetime)):
if date.isdigit() and len(date) == 8:

4
weboob/tools/capabilities/messages/GenericBackend.py
View file

@ -27,7 +27,9 @@ from weboob.tools.newsfeed import Newsfeed
class GenericNewspaperBackend(BaseBackend, ICapMessages):
"GenericNewspaperBackend class"
"""
GenericNewspaperBackend class
"""
MAINTAINER = 'Julien Hebert'
EMAIL = 'juke@free.fr'
VERSION = '0.c'

16
weboob/tools/capabilities/thumbnail.py
View file

@ -17,13 +17,23 @@
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.capabilities.base import CapBaseObject, NotLoaded
from weboob.capabilities.base import CapBaseObject, NotLoaded, StringField, BytesField
__all__ = ['Thumbnail']
class Thumbnail(CapBaseObject):
"""
Thumbnail of an image.
"""
url = StringField('URL to photo thumbnail')
data = BytesField('Data')
def __init__(self, url):
CapBaseObject.__init__(self, url)
self.add_field('url', basestring, url.replace(' ', '%20'))
self.add_field('data', str)
self.url = url.replace(' ', '%20')
def __str__(self):
return self.url

18
weboob/tools/parsers/lxmlparser.py
View file

@ -60,14 +60,18 @@ class LxmlHtmlParser(IParser):
"""
Select one or many elements from an element, using lxml cssselect by default.
Raises BrokenPageError if not found.
Raises :class:`weboob.tools.browser.browser.BrokenPageError` if not found.
@param element [obj] element on which to apply selector
@param selector [str] CSS or XPath expression
@param method [str] (cssselect|xpath)
@param nb [int] number of elements expected to be found.
Use None for undefined number, and 'many' for 1 to infinite.
@return one or many Element
:param element: element on which to apply selector
:type element: object
:param selector: CSS or XPath expression
:type selector: str
:param method: (cssselect|xpath)
:type method: str
:param nb: number of elements expected to be found. Use None for
undefined number, and 'many' for 1 to infinite
:type nb: :class:`int` or :class:`str`
:rtype: Element
"""
if method == 'cssselect':
results = element.cssselect(selector)