support Credit Mutuel and CIC deferred debit

modules/cic/backend.py

@@ -56,9 +56,17 @@ class CICBackend(BaseBackend, ICapBank):
         else:
             raise AccountNotFound()
 
+    def iter_coming(self, account):
+        with self.browser:
+            for tr in self.browser.get_history(account):
+                if tr._is_coming:
+                    yield tr
+
     def iter_history(self, account):
-        for history in self.browser.get_history(account):
+        with self.browser:
-            yield history
+            for tr in self.browser.get_history(account):
+                if not tr._is_coming:
+                    yield tr
 
     def iter_transfer_recipients(self, ignored):
         for account in self.browser.get_accounts_list().itervalues():

modules/cic/browser.py

@@ -18,14 +18,15 @@
 # along with weboob. If not, see <http://www.gnu.org/licenses/>.
 
 
-from urlparse import urlparse
+from urlparse import urlsplit, parse_qsl, urlparse
+from datetime import datetime, timedelta
 
 from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword
 from weboob.capabilities.bank import Transfer, TransferError
-from datetime import datetime
 
-from .pages import LoginPage, LoginErrorPage, AccountsPage, UserSpacePage, \
+from .pages import LoginPage, LoginErrorPage, AccountsPage, UserSpacePage, EmptyPage, \
-                   OperationsPage, NoOperationsPage, InfoPage, TransfertPage
+                   OperationsPage, CardPage, NoOperationsPage, InfoPage, TransfertPage
+
 
 __all__ = ['CICBrowser']
 
@@ -42,10 +43,11 @@ class CICBrowser(BaseBrowser):
              'https://www.cic.fr/.*/fr/banque/espace_personnel.aspx': UserSpacePage,
              'https://www.cic.fr/.*/fr/banque/mouvements.cgi.*': OperationsPage,
              'https://www.cic.fr/.*/fr/banque/nr/nr_devbooster.aspx.*': OperationsPage,
-             'https://www.cic.fr/.*/fr/banque/operations_carte\.cgi.*': OperationsPage,
+             'https://www.cic.fr/.*/fr/banque/operations_carte\.cgi.*': CardPage,
              'https://www.cic.fr/.*/fr/banque/CR/arrivee\.asp.*': NoOperationsPage,
              'https://www.cic.fr/.*/fr/banque/BAD.*': InfoPage,
-             'https://www.cic.fr/.*/fr/banque/.*Vir.*': TransfertPage
+             'https://www.cic.fr/.*/fr/banque/.*Vir.*': TransfertPage,
+             'https://www.cic.fr/.*/fr/': EmptyPage,
             }
 
     currentSubBank = None
@@ -90,11 +92,9 @@ class CICBrowser(BaseBrowser):
         url = urlparse(self.geturl())
         self.currentSubBank = url.path.lstrip('/').split('/')[0]
 
-    def get_history(self, account):
+    def list_operations(self, page_url):
-        page_url = account._link_id
-        #operations_count = 0
         l_ret = []
-        while (page_url):
+        while page_url:
             if page_url.startswith('/'):
                 self.location(page_url)
             else:
@@ -109,6 +109,32 @@ class CICBrowser(BaseBrowser):
 
         return l_ret
 
+    def get_history(self, account):
+        transactions = []
+        last_debit = None
+        for tr in self.list_operations(account._link_id):
+            if tr.raw == 'RELEVE CARTE' and last_debit is None:
+                last_debit = (tr.date - timedelta(days=10)).month
+            else:
+                transactions.append(tr)
+
+        month = 0
+        for card_link in account._card_links:
+            v = urlsplit(card_link)
+            args = dict(parse_qsl(v.query))
+            # useful with 12 -> 1
+            if int(args['mois']) < month:
+                month = month + 1
+            month = int(args['mois'])
+
+            for tr in self.list_operations(card_link):
+                if month > last_debit:
+                    tr._is_coming = True
+                transactions.append(tr)
+
+        transactions.sort(key=lambda tr: tr.rdate, reverse=True)
+        return transactions
+
     def transfer(self, account, to, amount, reason=None):
         # access the transfer page
         transfert_url = 'WI_VPLV_VirUniSaiCpt.asp?RAZ=ALL&Cat=6&PERM=N&CHX=A'

modules/cic/pages.py

@@ -23,6 +23,7 @@ from decimal import Decimal
 import re
 
 from weboob.tools.browser import BasePage
+from weboob.tools.ordereddict import OrderedDict
 from weboob.capabilities.bank import Account
 from weboob.tools.capabilities.bank.transactions import FrenchTransaction
 
@@ -39,6 +40,9 @@ class LoginErrorPage(BasePage):
 class InfoPage(BasePage):
     pass
 
+class EmptyPage(BasePage):
+    pass
+
 class TransfertPage(BasePage):
     pass
 
@@ -47,45 +51,49 @@ class UserSpacePage(BasePage):
 
 class AccountsPage(BasePage):
     def get_list(self):
-        ids = set()
+        accounts = OrderedDict()
 
         for tr in self.document.getiterator('tr'):
             first_td = tr.getchildren()[0]
             if (first_td.attrib.get('class', '') == 'i g' or first_td.attrib.get('class', '') == 'p g') \
                and first_td.find('a') is not None:
-                account = Account()
+
-                account.label = u"%s"%first_td.find('a').text.strip().lstrip(' 0123456789').title()
+                a = first_td.find('a')
-                account._link_id = first_td.find('a').get('href', '')
+                link = a.get('href', '')
-                if account._link_id.startswith('POR_SyntheseLst'):
+                if link.startswith('POR_SyntheseLst'):
                     continue
 
-                url = urlparse(account._link_id)
+                url = urlparse(link)
                 p = parse_qs(url.query)
                 if not 'rib' in p:
                     continue
 
-                account.id = p['rib'][0]
+                for i in (2,1):
+                    balance = FrenchTransaction.clean_amount(tr.getchildren()[i].text.strip(' EUR'))
+                    if len(balance) > 0:
+                        break
+                balance = Decimal(balance)
 
-                if account.id in ids:
+                id = p['rib'][0]
+                if id in accounts:
+                    account = accounts[id]
+                    if not account.coming:
+                        account.coming = Decimal('0.0')
+                    account.coming += balance
+                    account._card_links.append(link)
                     continue
 
-                ids.add(account.id)
+                account = Account()
+                account.id = id
+                account.label = unicode(a.text).strip().lstrip(' 0123456789').title()
+                account._link_id = link
+                account._card_links = []
 
-                s = tr.getchildren()[2].text
+                account.balance = balance
-                if s.strip() == "":
-                    s = tr.getchildren()[1].text
-                balance = u''
-                for c in s:
-                    if c.isdigit() or c == '-':
-                        balance += c
-                    if c == ',':
-                        balance += '.'
-                account.balance = Decimal(balance)
-                yield account
 
-    def next_page_url(self):
+                accounts[account.id] = account
-        """ TODO pouvoir passer à la page des comptes suivante """
+
-        return 0
+        return accounts.itervalues()
 
 class Transaction(FrenchTransaction):
     PATTERNS = [(re.compile('^VIR(EMENT)? (?P<text>.*)'), FrenchTransaction.TYPE_TRANSFER),
@@ -99,6 +107,7 @@ class Transaction(FrenchTransaction):
                 (re.compile('^REMISE (?P<text>.*)'),      FrenchTransaction.TYPE_DEPOSIT),
                ]
 
+    _is_coming = False
 
 class OperationsPage(BasePage):
     def get_history(self):
@@ -120,14 +129,7 @@ class OperationsPage(BasePage):
                 operation = Transaction(index)
                 index += 1
 
-                # Find different parts of label
+                parts = [txt.strip() for txt in tds[-3].itertext() if len(txt.strip()) > 0]
-                parts = []
-                if len(tds[-3].findall('a')) > 0:
-                    parts = [a.text.strip() for a in tds[-3].findall('a')]
-                else:
-                    parts.append(tds[-3].text.strip())
-                    if tds[-3].find('br') is not None:
-                        parts.append(tds[-3].find('br').tail.strip())
 
                 # To simplify categorization of CB, reverse order of parts to separate
                 # location and institution.
@@ -139,23 +141,36 @@ class OperationsPage(BasePage):
 
                 if tds[-1].text is not None and len(tds[-1].text) > 2:
                     s = tds[-1].text.strip()
-                elif tds[-1].text is not None and len(tds[-2].text) > 2:
+                elif tds[-2].text is not None and len(tds[-2].text) > 2:
                     s = tds[-2].text.strip()
                 else:
                     s = "0"
-                balance = u''
+                operation.set_amount(s.rstrip('EUR'))
-                for c in s:
-                    if c.isdigit() or c == "-":
-                        balance += c
-                    if c == ',':
-                        balance += '.'
-                operation.amount = Decimal(balance)
                 yield operation
 
     def next_page_url(self):
         """ TODO pouvoir passer à la page des opérations suivantes """
         return 0
 
+class CardPage(OperationsPage):
+    def get_history(self):
+        index = 0
+        for tr in self.document.xpath('//table[@class="liste"]/tbody/tr'):
+            tds = tr.findall('td')
+            if len(tds) < 4:
+                continue
+
+            tr = Transaction(index)
+
+            parts = [txt.strip() for txt in list(tds[-3].itertext()) + list(tds[-2].itertext()) if len(txt.strip()) > 0]
+
+            tr.parse(date=tds[0].text.strip(' \xa0'),
+                     raw=u' '.join(parts))
+            tr.type = tr.TYPE_CARD
+
+            tr.set_amount(tds[-1].text.rstrip('EUR'))
+            yield tr
+
 class NoOperationsPage(OperationsPage):
     def get_history(self):
         return iter([])

modules/creditmutuel/backend.py

@@ -57,12 +57,16 @@ class CreditMutuelBackend(BaseBackend, ICapBank):
             raise AccountNotFound()
 
     def iter_coming(self, account):
-        """ TODO Not supported yet """
+        with self.browser:
-        return iter([])
+            for tr in self.browser.get_history(account):
+                if tr._is_coming:
+                    yield tr
 
     def iter_history(self, account):
-        for history in self.browser.get_history(account):
+        with self.browser:
-            yield history
+            for tr in self.browser.get_history(account):
+                if not tr._is_coming:
+                    yield tr
 
     def iter_transfer_recipients(self, ignored):
         for account in self.browser.get_accounts_list().itervalues():

modules/creditmutuel/browser.py

@@ -18,12 +18,15 @@
 # along with weboob. If not, see <http://www.gnu.org/licenses/>.
 
 
+from urlparse import urlsplit, parse_qsl, urlparse
+from datetime import datetime, timedelta
+
 from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword
 from weboob.capabilities.bank import Transfer, TransferError
-from datetime import datetime
 
-from .pages import LoginPage, LoginErrorPage, AccountsPage, UserSpacePage, \
+from .pages import LoginPage, LoginErrorPage, AccountsPage, UserSpacePage, EmptyPage, \
-                   OperationsPage, NoOperationsPage, InfoPage, TransfertPage
+                   OperationsPage, CardPage, NoOperationsPage, InfoPage, TransfertPage
+
 
 __all__ = ['CreditMutuelBrowser']
 
@@ -36,20 +39,18 @@ class CreditMutuelBrowser(BaseBrowser):
     USER_AGENT = BaseBrowser.USER_AGENTS['wget']
     PAGES = {'https://www.creditmutuel.fr/groupe/fr/index.html':   LoginPage,
              'https://www.creditmutuel.fr/.*/fr/identification/default.cgi': LoginErrorPage,
-         'https://www.creditmutuel.fr/.*/fr/banque/situation_financiere.cgi': AccountsPage,
+             'https://www.creditmutuel.fr/.*/fr/banque/situation_financiere.cgi': AccountsPage,
-         'https://www.creditmutuel.fr/.*/fr/banque/espace_personnel.aspx': UserSpacePage,
+             'https://www.creditmutuel.fr/.*/fr/banque/espace_personnel.aspx': UserSpacePage,
-         'https://www.creditmutuel.fr/.*/fr/banque/mouvements.cgi.*': OperationsPage,
+             'https://www.creditmutuel.fr/.*/fr/banque/mouvements.cgi.*': OperationsPage,
-         'https://www.creditmutuel.fr/.*/fr/banque/nr/nr_devbooster.aspx.*': OperationsPage,
+             'https://www.creditmutuel.fr/.*/fr/banque/nr/nr_devbooster.aspx.*': OperationsPage,
-         'https://www.creditmutuel.fr/.*/fr/banque/operations_carte\.cgi.*': OperationsPage,
+             'https://www.creditmutuel.fr/.*/fr/banque/operations_carte\.cgi.*': CardPage,
-         'https://www.creditmutuel.fr/.*/fr/banque/CR/arrivee\.asp.*': NoOperationsPage,
+             'https://www.creditmutuel.fr/.*/fr/banque/CR/arrivee\.asp.*': NoOperationsPage,
-         'https://www.creditmutuel.fr/.*/fr/banque/BAD.*': InfoPage,
+             'https://www.creditmutuel.fr/.*/fr/banque/BAD.*': InfoPage,
-         'https://www.creditmutuel.fr/.*/fr/banque/.*Vir.*': TransfertPage
+             'https://www.creditmutuel.fr/.*/fr/banque/.*Vir.*': TransfertPage,
+             'https://www.creditmutuel.fr/.*/fr/': EmptyPage,
             }
 
-    def __init__(self, *args, **kwargs):
+    currentSubBank = None
-        BaseBrowser.__init__(self, *args, **kwargs)
-        #self.SUB_BANKS = ['cmdv','cmcee','cmse', 'cmidf', 'cmsmb', 'cmma', 'cmmabn', 'cmc', 'cmlaco', 'cmnormandie', 'cmm']
-        #self.currentSubBank = None
 
     def is_logged(self):
         return self.page and not self.is_on_page(LoginPage) and not self.is_on_page(LoginErrorPage)
@@ -69,7 +70,6 @@ class CreditMutuelBrowser(BaseBrowser):
         if not self.is_logged() or self.is_on_page(LoginErrorPage):
             raise BrowserIncorrectPassword()
 
-        self.SUB_BANKS = ['cmdv', 'cmcee', 'cmse', 'cmidf', 'cmsmb', 'cmma', 'cmmabn', 'cmc', 'cmlaco', 'cmnormandie', 'cmm']
         self.getCurrentSubBank()
 
     def get_accounts_list(self):
@@ -89,17 +89,12 @@ class CreditMutuelBrowser(BaseBrowser):
 
     def getCurrentSubBank(self):
         # the account list and history urls depend on the sub bank of the user
-        current_url = self.geturl()
+        url = urlparse(self.geturl())
-        current_url_parts = current_url.split('/')
+        self.currentSubBank = url.path.lstrip('/').split('/')[0]
-        for subbank in self.SUB_BANKS:
-            if subbank in current_url_parts:
-                self.currentSubBank = subbank
 
-    def get_history(self, account):
+    def list_operations(self, page_url):
-        page_url = account._link_id
-        #operations_count = 0
         l_ret = []
-        while (page_url):
+        while page_url:
             if page_url.startswith('/'):
                 self.location(page_url)
             else:
@@ -114,6 +109,32 @@ class CreditMutuelBrowser(BaseBrowser):
 
         return l_ret
 
+    def get_history(self, account):
+        transactions = []
+        last_debit = None
+        for tr in self.list_operations(account._link_id):
+            if tr.raw == 'RELEVE CARTE' and last_debit is None:
+                last_debit = (tr.date - timedelta(days=10)).month
+            else:
+                transactions.append(tr)
+
+        month = 0
+        for card_link in account._card_links:
+            v = urlsplit(card_link)
+            args = dict(parse_qsl(v.query))
+            # useful with 12 -> 1
+            if int(args['mois']) < month:
+                month = month + 1
+            month = int(args['mois'])
+
+            for tr in self.list_operations(card_link):
+                if month > last_debit:
+                    tr._is_coming = True
+                transactions.append(tr)
+
+        transactions.sort(key=lambda tr: tr.rdate, reverse=True)
+        return transactions
+
     def transfer(self, account, to, amount, reason=None):
         # access the transfer page
         transfert_url = 'WI_VPLV_VirUniSaiCpt.asp?RAZ=ALL&Cat=6&PERM=N&CHX=A'
@@ -163,8 +184,3 @@ class CreditMutuelBrowser(BaseBrowser):
         transfer.recipient = to
         transfer.date = submit_date
         return transfer
-
-    #def get_coming_operations(self, account):
-    #    if not self.is_on_page(AccountComing) or self.page.account.id != account.id:
-    #        self.location('/NS_AVEEC?ch4=%s' % account._link_id)
-    #    return self.page.get_operations()

modules/creditmutuel/pages.py

@@ -23,6 +23,7 @@ from decimal import Decimal
 import re
 
 from weboob.tools.browser import BasePage
+from weboob.tools.ordereddict import OrderedDict
 from weboob.capabilities.bank import Account
 from weboob.tools.capabilities.bank.transactions import FrenchTransaction
 
@@ -39,6 +40,9 @@ class LoginErrorPage(BasePage):
 class InfoPage(BasePage):
     pass
 
+class EmptyPage(BasePage):
+    pass
+
 class TransfertPage(BasePage):
     pass
 
@@ -47,45 +51,49 @@ class UserSpacePage(BasePage):
 
 class AccountsPage(BasePage):
     def get_list(self):
-        ids = set()
+        accounts = OrderedDict()
 
         for tr in self.document.getiterator('tr'):
             first_td = tr.getchildren()[0]
             if (first_td.attrib.get('class', '') == 'i g' or first_td.attrib.get('class', '') == 'p g') \
                and first_td.find('a') is not None:
-                account = Account()
+
-                account.label = u"%s"%first_td.find('a').text.strip().lstrip(' 0123456789').title()
+                a = first_td.find('a')
-                account._link_id = first_td.find('a').get('href', '')
+                link = a.get('href', '')
-                if account._link_id.startswith('POR_SyntheseLst'):
+                if link.startswith('POR_SyntheseLst'):
                     continue
 
-                url = urlparse(account._link_id)
+                url = urlparse(link)
                 p = parse_qs(url.query)
                 if not 'rib' in p:
                     continue
 
-                account.id = p['rib'][0]
+                for i in (2,1):
+                    balance = FrenchTransaction.clean_amount(tr.getchildren()[i].text.strip(' EUR'))
+                    if len(balance) > 0:
+                        break
+                balance = Decimal(balance)
 
-                if account.id in ids:
+                id = p['rib'][0]
+                if id in accounts:
+                    account = accounts[id]
+                    if not account.coming:
+                        account.coming = Decimal('0.0')
+                    account.coming += balance
+                    account._card_links.append(link)
                     continue
 
-                ids.add(account.id)
+                account = Account()
+                account.id = id
+                account.label = unicode(a.text).strip().lstrip(' 0123456789').title()
+                account._link_id = link
+                account._card_links = []
 
-                s = tr.getchildren()[2].text
+                account.balance = balance
-                if s.strip() == "":
-                    s = tr.getchildren()[1].text
-                balance = u''
-                for c in s:
-                    if c.isdigit() or c == '-':
-                        balance += c
-                    if c == ',':
-                        balance += '.'
-                account.balance = Decimal(balance)
-                yield account
 
-    def next_page_url(self):
+                accounts[account.id] = account
-        """ TODO pouvoir passer à la page des comptes suivante """
+
-        return 0
+        return accounts.itervalues()
 
 class Transaction(FrenchTransaction):
     PATTERNS = [(re.compile('^VIR(EMENT)? (?P<text>.*)'), FrenchTransaction.TYPE_TRANSFER),
@@ -99,6 +107,7 @@ class Transaction(FrenchTransaction):
                 (re.compile('^REMISE (?P<text>.*)'),      FrenchTransaction.TYPE_DEPOSIT),
                ]
 
+    _is_coming = False
 
 class OperationsPage(BasePage):
     def get_history(self):
@@ -120,14 +129,7 @@ class OperationsPage(BasePage):
                 operation = Transaction(index)
                 index += 1
 
-                # Find different parts of label
+                parts = [txt.strip() for txt in tds[-3].itertext() if len(txt.strip()) > 0]
-                parts = []
-                if len(tds[-3].findall('a')) > 0:
-                    parts = [a.text.strip() for a in tds[-3].findall('a')]
-                else:
-                    parts.append(tds[-3].text.strip())
-                    if tds[-3].find('br') is not None:
-                        parts.append(tds[-3].find('br').tail.strip())
 
                 # To simplify categorization of CB, reverse order of parts to separate
                 # location and institution.
@@ -139,23 +141,36 @@ class OperationsPage(BasePage):
 
                 if tds[-1].text is not None and len(tds[-1].text) > 2:
                     s = tds[-1].text.strip()
-                elif tds[-1].text is not None and len(tds[-2].text) > 2:
+                elif tds[-2].text is not None and len(tds[-2].text) > 2:
                     s = tds[-2].text.strip()
                 else:
                     s = "0"
-                balance = u''
+                operation.set_amount(s.rstrip('EUR'))
-                for c in s:
-                    if c.isdigit() or c == "-":
-                        balance += c
-                    if c == ',':
-                        balance += '.'
-                operation.amount = Decimal(balance)
                 yield operation
 
     def next_page_url(self):
         """ TODO pouvoir passer à la page des opérations suivantes """
         return 0
 
+class CardPage(OperationsPage):
+    def get_history(self):
+        index = 0
+        for tr in self.document.xpath('//table[@class="liste"]/tbody/tr'):
+            tds = tr.findall('td')
+            if len(tds) < 4:
+                continue
+
+            tr = Transaction(index)
+
+            parts = [txt.strip() for txt in list(tds[-3].itertext()) + list(tds[-2].itertext()) if len(txt.strip()) > 0]
+
+            tr.parse(date=tds[0].text.strip(' \xa0'),
+                     raw=u' '.join(parts))
+            tr.type = tr.TYPE_CARD
+
+            tr.set_amount(tds[-1].text.rstrip('EUR'))
+            yield tr
+
 class NoOperationsPage(OperationsPage):
     def get_history(self):
         return iter([])