support new website of bred

modules/bred/bred/__init__.py

@@ -0,0 +1,3 @@
+from .browser import BredBrowser
+
+__all__ = ['BredBrowser']

modules/bred/bred/browser.py

@@ -0,0 +1,114 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2014 Romain Bignon
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+from datetime import date
+from decimal import Decimal
+
+from weboob.capabilities.bank import Account, Transaction
+from weboob.exceptions import BrowserIncorrectPassword
+from weboob.browser import DomainBrowser
+
+
+__all__ = ['BredBrowser']
+
+
+class BredBrowser(DomainBrowser):
+    BASEURL = 'https://www.bred.fr'
+
+    def __init__(self, accnum, login, password, *args, **kwargs):
+        super(BredBrowser, self).__init__(*args, **kwargs)
+        self.login = login
+        self.password = password
+        self.accnum = accnum
+
+        self.do_login(login, password)
+
+    def do_login(self, login, password):
+        self.location('/transactionnel/Authentication', data={'identifiant': login, 'password': password})
+
+        if 'erreur-pwd' in self.url:
+            raise BrowserIncorrectPassword()
+
+    ACCOUNT_TYPES = {'000': Account.TYPE_CHECKING,
+                     '999': Account.TYPE_MARKET,
+                     '011': Account.TYPE_CARD,
+                     '023': Account.TYPE_SAVINGS,
+                     '078': Account.TYPE_SAVINGS,
+                     '080': Account.TYPE_SAVINGS,
+                     '027': Account.TYPE_SAVINGS,
+                     '037': Account.TYPE_SAVINGS,
+                    }
+    def get_accounts_list(self):
+        r = self.open('/transactionnel/services/rest/Account/accounts')
+
+        for content in r.json()['content']:
+            if self.accnum != '00000000000' and content['numero'] != self.accnum:
+                continue
+            for poste in content['postes']:
+                a = Account()
+                a._number = content['numeroLong']
+                a._nature = poste['codeNature']
+                a._consultable = poste['consultable']
+                a.id = '%s.%s' % (a._number, a._nature)
+                a.type = self.ACCOUNT_TYPES.get(poste['codeNature'], Account.TYPE_UNKNOWN)
+
+                if poste['postePortefeuille']:
+                    a.label = u'Portefeuille Titres'
+                    a.balance = Decimal(poste['montantTitres']['valeur'])
+                    a.currency = poste['montantTitres']['monnaie']['code']
+                    yield a
+
+                if not 'libelle' in poste:
+                    continue
+
+                a.label = ' '.join([content['intitule'].strip(), poste['libelle'].strip()])
+                a.balance = Decimal(poste['solde']['valeur'])
+                a.currency = poste['solde']['monnaie']['code']
+                yield a
+
+    def get_history(self, account):
+        if not account._consultable:
+            raise NotImplementedError()
+
+        offset = 0
+        next_page = True
+        while next_page:
+            r = self.open('/transactionnel/services/applications/operations/get/%(number)s/%(nature)s/00/%(currency)s/%(startDate)s/%(endDate)s/%(offset)s/%(limit)s' %
+                          {'number': account._number,
+                           'nature': account._nature,
+                           'currency': account.currency,
+                           'startDate': '2000-01-01',
+                           'endDate': date.today().strftime('%Y-%m-%d'),
+                           'offset': offset,
+                           'limit': 50
+                          })
+            next_page = False
+            offset += 50
+            for op in r.json()['content']['operations']:
+                next_page = True
+                t = Transaction()
+                t.id = op['id']
+                t.amount = Decimal(str(op['montant']))
+                t.date = date.fromtimestamp(op.get('dateDebit', op.get('dateOperation'))/1000)
+                t.rdate = date.fromtimestamp(op.get('dateOperation', op.get('dateDebit'))/1000)
+                t.vdate = date.fromtimestamp(op.get('dateValeur', op.get('dateDebit', op.get('dateOperation')))/1000)
+                t.category = op['categorie']
+                t.label = op['libelle']
+                t.raw = ' '.join([op['libelle']] + op['details'])
+                yield t

modules/bred/dispobank/__init__.py

@@ -0,0 +1,3 @@
+from .browser import DispoBankBrowser
+
+__all__ = ['DispoBankBrowser']

modules/bred/dispobank/browser.py

@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2012-2014 Romain Bignon
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+
+import urllib
+
+from weboob.deprecated.browser import Browser, BrowserIncorrectPassword
+
+from .pages import LoginPage, LoginResultPage, AccountsPage, EmptyPage, TransactionsPage
+
+
+__all__ = ['DispoBankBrowser']
+
+
+class DispoBankBrowser(Browser):
+    PROTOCOL = 'https'
+    DOMAIN = 'www.dispobank.fr'
+    CERTHASH = ['9b77dab9c84e1dc9e0798de561a6541ff15f038f60b36ca74c29be1def6c19a3', '375f1fed165d34aacaaf71674ab14ca6c1b38404cf748278714fde3c58385ff0', '0853a056453b56aea6a29085ef3f3721b18db2052aa8e84220720d44e0eb22af']
+    ENCODING = 'iso-8859-15'
+    PAGES = {r'https://www.\w+.fr/mylittleform.*':                      LoginPage,
+             r'https://www.\w+.fr/Andromede/MainAuth.*':                LoginResultPage,
+             r'https://www.\w+.fr/Andromede/Main':                      AccountsPage,
+             r'https://www.\w+.fr/Andromede/Ecriture':                  TransactionsPage,
+             r'https://www.\w+.fr/Andromede/applications/index.jsp':    EmptyPage,
+             r'https://www.bred.fr/':                                   EmptyPage,
+             r'https://www.dispobank.fr/?':                             LoginPage,
+            }
+
+    URLS = {'bred': {'home': 'https://www.bred.fr/Andromede/Main',
+                     'login': 'https://www.bred.fr/mylittleform?type=1',
+                    },
+            'dispobank': {'home': 'https://www.dispobank.fr',
+                          'login': 'https://www.dispobank.fr',
+                         }
+           }
+
+    def __init__(self, accnum, *args, **kwargs):
+        self.accnum = accnum
+        self.website = 'dispobank'
+        Browser.__init__(self, *args, **kwargs)
+
+    def is_logged(self):
+        return self.page is not None and not self.is_on_page(LoginPage)
+
+    def home(self):
+        if not self.is_logged():
+            self.login()
+        else:
+            self.location(self.URLS[self.website]['home'])
+
+    def login(self):
+        assert isinstance(self.username, basestring)
+        assert isinstance(self.password, basestring)
+
+        if not self.is_on_page(LoginPage):
+            self.location(self.URLS[self.website]['login'], no_login=True)
+
+        self.page.login(self.username, self.password)
+
+        assert self.is_on_page((LoginResultPage, EmptyPage))
+
+        if self.is_on_page(LoginResultPage):
+            error = self.page.get_error()
+            if error is not None:
+                raise BrowserIncorrectPassword(error)
+
+            self.page.confirm()
+
+    def get_accounts_list(self):
+        if not self.is_on_page(AccountsPage):
+            self.location('https://www.%s.fr/Andromede/Main' % self.website)
+        return self.page.get_list()
+
+    def get_account(self, id):
+        assert isinstance(id, basestring)
+
+        l = self.get_accounts_list()
+        for a in l:
+            if a.id == id:
+                return a
+
+        return None
+
+    def get_history(self, account):
+        numero_compte, numero_poste = account.id.split('.')
+        data = {'typeDemande':      'recherche',
+                'motRecherche':     '',
+                'numero_compte':    numero_compte,
+                'numero_poste':     numero_poste,
+                'detail':           '',
+                'tri':              'date',
+                'sens':             'sort',
+                'monnaie':          'EUR',
+                'index_hist':       4
+               }
+        self.location('https://www.%s.fr/Andromede/Ecriture' % self.website, urllib.urlencode(data))
+
+        assert self.is_on_page(TransactionsPage)
+        return self.page.get_history()

modules/bred/dispobank/pages.py

@@ -0,0 +1,239 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2012 Romain Bignon
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+
+from mechanize import FormNotFoundError
+from weboob.deprecated.mech import ClientForm
+ControlNotFoundError = ClientForm.ControlNotFoundError
+
+from decimal import Decimal, InvalidOperation
+import re
+
+from weboob.deprecated.browser import Page
+from weboob.tools.misc import to_unicode
+from weboob.tools.ordereddict import OrderedDict
+from weboob.capabilities.bank import Account
+from weboob.tools.capabilities.bank.transactions import FrenchTransaction
+
+
+class LoginPage(Page):
+    def login(self, login, passwd):
+        try:
+            length = int(self.document.xpath('//input[@id="pass"]')[0].attrib['maxlength'])
+        except (IndexError,KeyError):
+            pass
+        else:
+            passwd = passwd[:length]
+
+        self.browser.select_form(name='authen')
+        try:
+            self.browser['id'] = login.encode(self.browser.ENCODING)
+            self.browser['pass'] = passwd.encode(self.browser.ENCODING)
+        except ControlNotFoundError:
+            self.browser.controls.append(ClientForm.TextControl('text', 'id', {'value': login.encode(self.browser.ENCODING)}))
+            self.browser.controls.append(ClientForm.TextControl('text', 'pass', {'value': passwd.encode(self.browser.ENCODING)}))
+
+        self.browser.submit(nologin=True)
+
+
+class LoginResultPage(Page):
+    def on_loaded(self):
+        for script in self.document.xpath('//script'):
+            text = script.text
+            if text is None:
+                continue
+            m = re.search("window.location.replace\('([^']+)'\);", text)
+            if m:
+                self.browser.location(m.group(1))
+
+        try:
+            self.browser.select_form(name='banque')
+        except FormNotFoundError:
+            pass
+        else:
+            self.browser.set_all_readonly(False)
+            accounts = OrderedDict()
+            for tr in self.document.getroot().cssselect('table.compteTable > tbody > tr'):
+                if len(tr.findall('td')) == 0:
+                    continue
+                attr = tr.xpath('.//a')[0].attrib.get('onclick', '')
+                m = re.search("value = '(\w+)';(checkAndSubmit\('\w+','(\w+)','(\w+)'\))?", attr)
+                if m:
+                    typeCompte = m.group(1)
+                    tagName = m.group(3)
+                    if tagName is not None:
+                        value = self.document.xpath('//input[@name="%s"]' % m.group(3))[int(m.group(4))].attrib['value']
+                    else:
+                        value = typeCompte
+                    accounts[value] = (typeCompte, tagName)
+
+            try:
+                typeCompte, tagName = accounts[self.browser.accnum]
+                value = self.browser.accnum
+            except KeyError:
+                accnums = ', '.join(accounts.keys())
+                if self.browser.accnum != '00000000000':
+                    self.logger.warning(u'Unable to find account "%s". Available ones: %s' % (self.browser.accnum, accnums))
+                elif len(accounts) > 1:
+                    self.logger.warning('There are several accounts, please use "accnum" backend parameter to force the one to use (%s)' % accnums)
+                value, (typeCompte, tagName) = accounts.popitem(last=False)
+            self.browser['typeCompte'] = typeCompte
+            if tagName is not None:
+                self.browser[tagName] = [value]
+            self.browser.submit()
+
+    def confirm(self):
+        self.browser.location('MainAuth?typeDemande=AC', no_login=True)
+
+    def get_error(self):
+        error = self.document.xpath('//td[@class="txt_norm2"]')
+        if len(error) == 0:
+            return None
+
+        error = error[0]
+        if error.find('b') is not None:
+            error = error.find('b')
+
+        return error.text.strip()
+
+
+class EmptyPage(Page):
+    pass
+
+
+class BredBasePage(Page):
+    def js2args(self, s):
+        cur_arg = None
+        args = {}
+        # For example:
+        # javascript:reloadApplication('nom_application', 'compte_telechargement', 'numero_poste', '000', 'numero_compte', '12345678901','monnaie','EUR');
+        for sub in re.findall("'([^']+)'", s):
+            if cur_arg is None:
+                cur_arg = sub
+            else:
+                args[cur_arg] = sub
+                cur_arg = None
+
+        return args
+
+
+class AccountsPage(BredBasePage):
+    def get_list(self):
+        for tr in self.document.xpath('//table[@class="compteTable"]/tr'):
+            if not tr.attrib.get('class', '').startswith('ligne_'):
+                continue
+
+            cols = tr.findall('td')
+
+            if len(cols) < 2:
+                continue
+
+            try:
+                amount = sum([Decimal(FrenchTransaction.clean_amount(txt)) for txt in cols[-1].itertext() if len(txt.strip()) > 0])
+            except InvalidOperation:
+                continue
+
+            a = cols[0].find('a')
+            if a is None:
+                for a in cols[0].xpath('.//li/a'):
+                    args = self.js2args(a.attrib['href'])
+                    if 'numero_compte' not in args or 'numero_poste' not in args:
+                        self.logger.warning('Card link with strange args: %s' % args)
+                        continue
+
+                    account = Account()
+                    account.id = '%s.%s' % (args['numero_compte'], args['numero_poste'])
+                    account.label = u'Carte %s' % self.parser.tocleanstring(a)
+                    account.balance = amount
+                    account.type = account.TYPE_CARD
+                    account.currency = [account.get_currency(txt) for txt in cols[-1].itertext() if len(txt.strip()) > 0][0]
+                    yield account
+                continue
+
+            args = self.js2args(a.attrib['href'])
+
+            if 'numero_compte' not in args or 'numero_poste' not in args:
+                self.logger.warning('Account link for %r with strange args: %s' % (a.attrib.get('alt', a.text), args))
+                continue
+
+            account = Account()
+            account.id = u'%s.%s' % (args['numero_compte'], args['numero_poste'])
+            account.label = to_unicode(a.attrib.get('alt', a.text.strip()))
+            account.balance = amount
+            account.currency = [account.get_currency(txt) for txt in cols[-1].itertext() if len(txt.strip()) > 0][0]
+            yield account
+
+
+class Transaction(FrenchTransaction):
+    PATTERNS = [(re.compile('^RETRAIT G.A.B. \d+ (?P<text>.*?)( CARTE .*)? LE (?P<dd>\d{2})/(?P<mm>\d{2})/(?P<yy>\d{2}).*'),
+                                                            FrenchTransaction.TYPE_WITHDRAWAL),
+                (re.compile('^VIR(EMENT)? (?P<text>.*)'),   FrenchTransaction.TYPE_TRANSFER),
+                (re.compile('^PRLV (?P<text>.*)'),          FrenchTransaction.TYPE_ORDER),
+                (re.compile('^(?P<text>.*) TRANSACTION( CARTE .*)? LE (?P<dd>\d{2})/(?P<mm>\d{2})/(?P<yy>\d{2}) ?(.*)$'),
+                                                            FrenchTransaction.TYPE_CARD),
+                (re.compile('^CHEQUE.*'),                   FrenchTransaction.TYPE_CHECK),
+                (re.compile('^(CONVENTION \d+ )?COTISATION (?P<text>.*)'),
+                                                            FrenchTransaction.TYPE_BANK),
+                (re.compile('^REMISE (?P<text>.*)'),        FrenchTransaction.TYPE_DEPOSIT),
+                (re.compile('^(?P<text>.*)( \d+)? QUITTANCE .*'),
+                                                            FrenchTransaction.TYPE_ORDER),
+                (re.compile('^CB PAIEM. EN \d+ FOIS \d+ (?P<text>.*?) LE .* LE (?P<dd>\d{2})/(?P<mm>\d{2})/(?P<yy>\d{2})$'),
+                                                            FrenchTransaction.TYPE_CARD),
+                (re.compile('^.* LE (?P<dd>\d{2})/(?P<mm>\d{2})/(?P<yy>\d{2})$'),
+                                                            FrenchTransaction.TYPE_UNKNOWN),
+               ]
+
+
+class TransactionsPage(Page):
+    def get_history(self):
+        for tr in self.document.xpath('//div[@class="scrollTbody"]/table//tr'):
+            cols = tr.findall('td')
+
+            if len(cols) < 4:
+                continue
+
+            col_label = cols[1]
+            if col_label.find('a') is not None:
+                col_label = col_label.find('a')
+
+            date = self.parser.tocleanstring(cols[0])
+            label = self.parser.tocleanstring(col_label)
+
+            t = Transaction(col_label.attrib.get('id', ''))
+
+            # an optional tooltip on page contain the second part of the transaction label.
+            tooltip = self.document.xpath('//div[@id="tooltip%s"]' % t.id)
+            raw = label
+            if len(tooltip) > 0:
+                raw += u' ' + u' '.join([txt.strip() for txt in tooltip[0].itertext()])
+
+            raw = re.sub(r'[ ]+', ' ', raw)
+
+            t.parse(date, raw)
+
+            # as only the first part of label is important to user, if there are no subpart
+            # taken by FrenchTransaction regexps, reset the label as first part.
+            if t.label == t.raw:
+                t.label = label
+
+            debit = self.parser.tocleanstring(cols[-2])
+            credit = self.parser.tocleanstring(cols[-1])
+            t.set_amount(credit, debit)
+
+            yield t

modules/bred/module.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright(C) 2012-2013 Romain Bignon
+# Copyright(C) 2012-2014 Romain Bignon
 #
 # This file is part of weboob.
 #
@@ -19,10 +19,12 @@
 
 
 from weboob.capabilities.bank import CapBank, AccountNotFound
+from weboob.capabilities.base import find_object
 from weboob.tools.backend import Module, BackendConfig
 from weboob.tools.value import ValueBackendPassword, Value
 
-from .browser import BredBrowser
+from .bred import BredBrowser
+from .dispobank import DispoBankBrowser
 
 
 __all__ = ['BredModule']
@@ -41,28 +43,22 @@ class BredModule(Module, CapBank):
                                  choices={'bred': 'BRED', 'dispobank': 'DispoBank'}),
                            Value('accnum', label=u'Numéro du compte bancaire (optionnel)', default='', masked=False)
                           )
-    BROWSER = BredBrowser
+
+    BROWSERS = {'bred': BredBrowser,
+                'dispobank': DispoBankBrowser,
+               }
 
     def create_default_browser(self):
-        return self.create_browser(self.config['website'].get(),
+        self.BROWSER = self.BROWSERS[self.config['website'].get()]
-                                   self.config['accnum'].get(),
+        return self.create_browser(self.config['accnum'].get().replace(' ','').zfill(11),
                                    self.config['login'].get(),
                                    self.config['password'].get())
 
     def iter_accounts(self):
-        with self.browser:
+        return self.browser.get_accounts_list()
-            for account in self.browser.get_accounts_list():
-                yield account
 
     def get_account(self, _id):
-        with self.browser:
+        return find_object(self.browser.get_accounts_list(), id=_id, error=AccountNotFound)
-            account = self.browser.get_account(_id)
-
-        if account:
-            return account
-        else:
-            raise AccountNotFound()
 
     def iter_history(self, account):
-        with self.browser:
+        return self.browser.get_history(account)
-            return self.browser.get_history(account)