add pro browser to cmso bank module

modules/cmso/mobile/browser.py

@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2012 Romain Bignon
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+
+from weboob.deprecated.browser import Browser, BrowserIncorrectPassword
+
+from .pages import LoginPage, AccountsPage, TransactionsPage
+
+
+__all__ = ['CmsoMobileBrowser']
+
+
+class CmsoMobileBrowser(Browser):
+    PROTOCOL = 'https'
+    DOMAIN = 'www.cmso.com'
+    ENCODING = 'iso-8859-1'
+    PAGES = {'https://www.cmso.com/domimobile/m.jsp\?a=signin.*':       LoginPage,
+             'https://www.cmso.com/domimobile/m.jsp\?a=sommaire.*':     AccountsPage,
+             'https://www.cmso.com/domimobile/m.jsp\?a=solde.*':        TransactionsPage,
+             'https://www.cmso.com/domimobile/m.jsp\?rels=.*':          TransactionsPage,
+            }
+
+    def is_logged(self):
+        return self.page is not None and not self.is_on_page(LoginPage)
+
+    def home(self):
+        if self.is_logged():
+            self.location('https://www.cmso.com/domimobile/m.jsp?a=sommaire')
+        else:
+            self.login()
+
+    def login(self):
+        """
+        Attempt to log in.
+        Note: this method does nothing if we are already logged in.
+        """
+        assert isinstance(self.username, basestring)
+        assert isinstance(self.password, basestring)
+
+        if self.is_logged():
+            return
+
+        if not self.is_on_page(LoginPage):
+            self.location('https://www.cmso.com/domimobile/m.jsp?a=signin&b=sommaire', no_login=True)
+
+        self.page.login(self.username, self.password)
+
+        if not self.is_logged():
+            raise BrowserIncorrectPassword()
+
+    def get_accounts_list(self):
+        if not self.is_on_page(AccountsPage):
+            self.location('https://www.cmso.com/domimobile/m.jsp?a=sommaire')
+        return self.page.get_list()
+
+    def get_history(self, account):
+        if not self.is_on_page(AccountsPage):
+            self.location('https://www.cmso.com/domimobile/m.jsp?a=sommaire')
+
+        link = account._link
+
+        while link is not None:
+            self.location(link)
+            assert self.is_on_page(TransactionsPage)
+
+            for tr in self.page.get_history():
+                yield tr
+
+            link = self.page.get_next_link()

modules/cmso/mobile/pages.py

@@ -0,0 +1,112 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2012 Romain Bignon
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+
+import datetime
+from decimal import Decimal
+import re
+
+from weboob.deprecated.browser import Page
+from weboob.capabilities.bank import Account
+from weboob.tools.capabilities.bank.transactions import FrenchTransaction
+
+
+class LoginPage(Page):
+    def login(self, login, passwd):
+        self.browser.select_form(name='formIdentification')
+        self.browser['noPersonne'] = login.encode(self.browser.ENCODING)
+        self.browser['motDePasse'] = passwd.encode(self.browser.ENCODING)
+        self.browser.submit(nologin=True)
+
+
+class AccountsPage(Page):
+    def get_list(self):
+        names = set()
+        for li in self.document.xpath('//div[@class="affichMontant"]/ul/li/a'):
+            account = Account()
+            account.label = unicode(li.cssselect('div.row-lib u')[0].text.strip())
+            account.id = re.sub('[ \.\-]+', '', account.label)
+            while account.id in names:
+                account.id = account.id + '1'
+            names.add(account.id)
+            account.balance = Decimal(li.cssselect('p.row-right')[0].text.strip().replace(' ', '').replace(u'\xa0', '').replace(',', ''))
+            account._link = li.attrib['href']
+            yield account
+
+
+class Transaction(FrenchTransaction):
+    PATTERNS = [(re.compile('^RET DAB (?P<dd>\d{2})/?(?P<mm>\d{2})(/?(?P<yy>\d{2}))? (?P<text>.*)'),
+                                                              FrenchTransaction.TYPE_WITHDRAWAL),
+                (re.compile('CARTE (?P<dd>\d{2})/(?P<mm>\d{2}) (?P<text>.*)'),
+                                                              FrenchTransaction.TYPE_CARD),
+                (re.compile('^(?P<category>VIR(EMEN)?T? (SEPA)?(RECU|FAVEUR)?)( /FRM)?(?P<text>.*)'),
+                                                              FrenchTransaction.TYPE_TRANSFER),
+                (re.compile('^PRLV (?P<text>.*)( \d+)?$'),    FrenchTransaction.TYPE_ORDER),
+                (re.compile('^(CHQ|CHEQUE) .*$'),             FrenchTransaction.TYPE_CHECK),
+                (re.compile('^(AGIOS /|FRAIS) (?P<text>.*)'), FrenchTransaction.TYPE_BANK),
+                (re.compile('^(CONVENTION \d+ |F )?COTIS(ATION)? (?P<text>.*)'),
+                                                              FrenchTransaction.TYPE_BANK),
+                (re.compile('^REMISE (?P<text>.*)'),          FrenchTransaction.TYPE_DEPOSIT),
+                (re.compile('^(?P<text>.*)( \d+)? QUITTANCE .*'),
+                                                              FrenchTransaction.TYPE_ORDER),
+                (re.compile('^.* LE (?P<dd>\d{2})/(?P<mm>\d{2})/(?P<yy>\d{2})$'),
+                                                              FrenchTransaction.TYPE_UNKNOWN),
+               ]
+
+
+class TransactionsPage(Page):
+    months = [u'janvier', u'février', u'mars', u'avril', u'mai', u'juin', u'juillet', u'août', u'septembre', u'octobre', u'novembre', u'décembre']
+
+    def get_next_link(self):
+        a = self.document.getroot().cssselect('div#nav-sub p.row-right a')
+        if len(a) == 0:
+            return None
+
+        return a[0].attrib['href']
+
+    def get_history(self):
+        for div in self.document.xpath('//ol[@class="affichMontant"]/li/div'):
+            t = Transaction(0)
+            raw = div.xpath('.//div[@class="row-lib"]')[0].text
+            date = div.xpath('.//span')[0].text.strip()
+            m = re.match('(\d+)(er)? ([^ ]+)( \d+)?$', date)
+            if m:
+                dd = int(m.group(1))
+                mm = self.months.index(m.group(3)) + 1
+                if m.group(4) is not None:
+                    yy = int(m.group(4))
+                else:
+
+                    d = datetime.date.today()
+                    try:
+                        d = d.replace(month=mm, day=dd)
+                    except ValueError:
+                        d = d.replace(year=d.year-1, month=mm, day=dd)
+
+                    yy = d.year
+
+                date = datetime.date(yy, mm, dd)
+            else:
+                self.logger.error('Unable to parse date %r' % date)
+                continue
+
+            t.parse(date, re.sub(r'[ ]+', ' ', raw))
+            t.amount = Decimal(div.xpath('.//span')[-1].text.strip().replace(' ', '').replace(u'\xa0', '').replace(',', ''))
+
+            yield t

modules/cmso/module.py

@@ -19,10 +19,12 @@
 
 
 from weboob.capabilities.bank import CapBank, AccountNotFound
+from weboob.capabilities.base import find_object
 from weboob.tools.backend import Module, BackendConfig
-from weboob.tools.value import ValueBackendPassword
+from weboob.tools.value import Value, ValueBackendPassword
 
-from .browser import Cmso
+from .mobile.browser import CmsoMobileBrowser
+from .web.browser import CmsoProBrowser
 
 
 __all__ = ['CmsoModule']
@@ -36,26 +38,22 @@ class CmsoModule(Module, CapBank):
     DESCRIPTION = u'Crédit Mutuel Sud-Ouest'
     LICENSE = 'AGPLv3+'
     CONFIG = BackendConfig(ValueBackendPassword('login',    label='Identifiant', masked=False),
-                           ValueBackendPassword('password', label='Mot de passe'))
-    BROWSER = Cmso
+                           ValueBackendPassword('password', label='Mot de passe'),
+                           Value('website', label='Type de compte', default='par',
+                                 choices={'par': 'Particuliers', 'pro': 'Professionnels'}))
+    BROWSER = CmsoMobileBrowser
 
     def create_default_browser(self):
+        b = {'par': CmsoMobileBrowser, 'pro': CmsoProBrowser}
+        self.BROWSER = b[self.config['website'].get()]
         return self.create_browser(self.config['login'].get(),
                                    self.config['password'].get())
 
     def iter_accounts(self):
-        with self.browser:
-            return self.browser.get_accounts_list()
+        return self.browser.get_accounts_list()
 
     def get_account(self, _id):
-        with self.browser:
-            account = self.browser.get_account(_id)
-
-        if account:
-            return account
-        else:
-            raise AccountNotFound()
+        return find_object(self.browser.get_accounts_list(), id=_id, error=AccountNotFound)
 
     def iter_history(self, account):
-        with self.browser:
-            return self.browser.get_history(account)
+        return self.browser.get_history(account)

modules/cmso/web/browser.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2014      smurail
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+
+import datetime
+from dateutil.relativedelta import relativedelta
+from itertools import chain
+
+from weboob.browser import LoginBrowser, URL, need_login
+
+from .pages import LoginPage, AccountsPage, HistoryPage
+
+
+class CmsoProBrowser(LoginBrowser):
+    BASEURL = 'https://www.cmso.com/'
+
+    login = URL('/creditmutuel/index_espace.jsp\\?fede=cmso&espace=pro', LoginPage)
+    accounts = URL('/domiweb/prive/professionnel/situationGlobaleProfessionnel/0-situationGlobaleProfessionnel.act', AccountsPage)
+    history = URL('/domiweb/prive/professionnel/situationGlobaleProfessionnel/1-situationGlobaleProfessionnel.act', HistoryPage)
+
+    def do_login(self):
+        self.login.stay_or_go()
+        self.page.login(self.username, self.password)
+
+    @need_login
+    def get_accounts_list(self):
+        self.accounts.go()
+        return self.page.iter_accounts()
+
+    @need_login
+    def get_history(self, account):
+        # Query history for 6 last months
+        def format_date(d):
+            return datetime.date.strftime(d, '%d/%m/%Y')
+        today = datetime.date.today()
+        period = (today - relativedelta(months=6), today)
+        query = {'date': ''.join(map(format_date, period))}
+
+        # Let's go
+        self.location(account._history_url)
+        first_page = self.page
+        rest_page = self.history.go(data=query)
+
+        return chain(first_page.iter_history(), rest_page.iter_history())

modules/cmso/web/pages.py

@@ -0,0 +1,107 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2014      smurail
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+
+import datetime
+
+from weboob.browser.pages import HTMLPage, LoggedPage
+from weboob.exceptions import BrowserIncorrectPassword
+from weboob.browser.elements import ListElement, ItemElement, method
+from weboob.browser.filters.standard import CleanText, CleanDecimal, Regexp, DateGuesser
+from weboob.browser.filters.html import Link
+from weboob.capabilities.bank import Account
+from weboob.tools.capabilities.bank.transactions import FrenchTransaction
+from weboob.tools.date import LinearDateGuesser
+
+__all__ = ['LoginPage']
+
+
+class LoginPage(HTMLPage):
+    def on_load(self):
+        # Yes, I know... In the Wild Wild Web, nobody respects nothing
+        if self.response.status_code == 500:
+            raise BrowserIncorrectPassword
+
+    def login(self, username, password):
+        form = self.get_form(name='formIdentification')
+
+        form['noPersonne'] = username
+        form['motDePasse'] = password
+
+        form.submit()
+
+
+class CmsoListElement(ListElement):
+    item_xpath = '//table[@class="Tb" and tr[1][@class="LnTit"]]/tr[@class="LnA" or @class="LnB"]'
+
+
+class AccountsPage(LoggedPage, HTMLPage):
+    @method
+    class iter_accounts(CmsoListElement):
+        class item(ItemElement):
+            klass = Account
+
+            obj__history_url = Link('./td[1]/a')
+            obj_id = obj__history_url & Regexp(pattern="indCptSelectionne=(\d+)")
+            obj_label = CleanText('./td[1]')
+            obj_balance = CleanDecimal('./td[2]')
+
+
+class Transaction(FrenchTransaction):
+    pass
+
+
+class CmsoTransactionElement(ItemElement):
+    klass = Transaction
+
+    def condition(self):
+        return len(self.el) >= 5 and not self.el.get('id', '').startswith('libelleLong')
+
+
+class HistoryPage(LoggedPage, HTMLPage):
+    def iter_history(self):
+        if self.doc.xpath('//a[@href="1-situationGlobaleProfessionnel.act"]'):
+            return self.iter_history_rest_page()
+        return self.iter_history_first_page()
+
+    @method
+    class iter_history_first_page(CmsoListElement):
+        class item(CmsoTransactionElement):
+            def validate(self, obj):
+                return obj.date >= datetime.date.today().replace(day=1)
+
+            def date(selector):
+                return DateGuesser(CleanText(selector), LinearDateGuesser()) | Transaction.Date(selector)
+
+            obj_date = date('./td[1]')
+            obj_vdate = date('./td[2]')
+            # Each row is followed by a "long labelled" version
+            obj_raw = Transaction.Raw('./following-sibling::tr[1][starts-with(@id, "libelleLong")]/td[3]')
+            obj_amount = Transaction.Amount('./td[5]', './td[4]')
+
+    @method
+    class iter_history_rest_page(CmsoListElement):
+        def find_elements(self):
+            return reversed(list(super(type(self), self).find_elements()))
+
+        class item(CmsoTransactionElement):
+            obj_date = Transaction.Date('./td[2]')
+            obj_vdate = Transaction.Date('./td[1]')
+            obj_raw = Transaction.Raw('./td[3]')
+            obj_amount = Transaction.Amount('./td[5]', './td[4]', replace_dots=False)