support CB operations (coming and history)

modules/lcl/backend.py

@@ -60,10 +60,13 @@ class LCLBackend(BaseBackend, ICapBank):
             raise AccountNotFound()
 
     def iter_coming(self, account):
-        """ TODO Not supported yet """
+        with self.browser:
-        return iter([])
+            transactions = list(self.browser.get_cb_operations(account))
+            transactions.sort(key=lambda tr: tr.rdate, reverse=True)
+            return transactions
 
     def iter_history(self, account):
         with self.browser:
-            for history in self.browser.get_history(account):
+            transactions = list(self.browser.get_history(account))
-                yield history
+            transactions.sort(key=lambda tr: tr.rdate, reverse=True)
+            return transactions

modules/lcl/browser.py

@@ -18,9 +18,12 @@
 # along with weboob. If not, see <http://www.gnu.org/licenses/>.
 
 
+from urlparse import urlsplit, parse_qsl
+
 from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword
 
-from .pages import SkipPage, LoginPage, AccountsPage, AccountHistoryPage
+from .pages import SkipPage, LoginPage, AccountsPage, AccountHistoryPage, \
+                   CBListPage, CBHistoryPage
 
 
 __all__ = ['LCLBrowser']
@@ -34,8 +37,11 @@ class LCLBrowser(BaseBrowser):
     USER_AGENT = BaseBrowser.USER_AGENTS['wget']
     PAGES = {
         'https://particuliers.secure.lcl.fr/outil/UAUT/Authentication/authenticate': LoginPage,
+        'https://particuliers.secure.lcl.fr/outil/UAUT\?from=.*': LoginPage,
         'https://particuliers.secure.lcl.fr/outil/UWSP/Synthese': AccountsPage,
         'https://particuliers.secure.lcl.fr/outil/UWLM/ListeMouvements.*/accesListeMouvements.*': AccountHistoryPage,
+        'https://particuliers.secure.lcl.fr/outil/UWCB/UWCBEncours.*/listeCBCompte.*': CBListPage,
+        'https://particuliers.secure.lcl.fr/outil/UWCB/UWCBEncours.*/listeOperations.*': CBHistoryPage,
         'https://particuliers.secure.lcl.fr/outil/UAUT/Contrat/selectionnerContrat.*': SkipPage,
         'https://particuliers.secure.lcl.fr/index.html': SkipPage
         }
@@ -69,7 +75,7 @@ class LCLBrowser(BaseBrowser):
 
     def get_accounts_list(self):
         if not self.is_on_page(AccountsPage):
-            self.login()
+            self.location('https://particuliers.secure.lcl.fr/outil/UWSP/Synthese')
         return self.page.get_list()
 
     def get_account(self, id):
@@ -82,11 +88,32 @@ class LCLBrowser(BaseBrowser):
 
         return None
 
-    def get_history(self,account):
+    def get_history(self, account):
-        self.location('%s://%s%s' % (self.PROTOCOL, self.DOMAIN, account._link_id))
+        self.location(account._link_id)
-        return self.page.get_operations(account)
+        for tr in self.page.get_operations():
+            yield tr
 
-    #def get_coming_operations(self, account):
+        for tr in self.get_cb_operations(account, 1):
-    #    if not self.is_on_page(AccountComing) or self.page.account.id != account.id:
+            yield tr
-    #        self.location('/NS_AVEEC?ch4=%s' % account._link_id)
+
-    #    return self.page.get_operations()
+    def get_cb_operations(self, account, month=0):
+        """
+        Get CB operations.
+
+        * month=0 : current operations (non debited)
+        * month=1 : previous month operations (debited)
+        """
+        for link in account._coming_links:
+            v = urlsplit(self.absurl(link))
+            args = dict(parse_qsl(v.query))
+            args['MOIS'] = month
+
+            self.location(self.buildurl(v.path, **args))
+
+            for tr in self.page.get_operations():
+                yield tr
+
+            for card_link in self.page.get_cards():
+                self.location(card_link)
+                for tr in self.page.get_operations():
+                    yield tr

modules/lcl/pages.py

@@ -135,9 +135,12 @@ class AccountsPage(BasePage):
         l = []
         for a in self.document.getiterator('a'):
             link=a.attrib.get('href')
-            if link is not None and link.startswith("/outil/UWLM/ListeMouvements"):
+            if link is None:
+                continue
+            if link.startswith("/outil/UWLM/ListeMouvements"):
                 account = Account()
                 account._link_id=link+"&mode=45"
+                account._coming_links = []
                 parameters=link.split("?").pop().split("&")
                 for parameter in parameters:
                     list=parameter.split("=")
@@ -161,6 +164,21 @@ class AccountsPage(BasePage):
                 account.balance=Decimal(balance)
                 self.logger.debug('%s Type: %s' % (account.label, account._type))
                 l.append(account)
+            if link.startswith('/outil/UWCB/UWCBEncours'):
+                if len(l) == 0:
+                    self.logger.warning('There is a card account but not any check account')
+                    continue
+
+                account = l[-1]
+
+                coming = a.text.replace(u"\u00A0",'').replace(' ','').replace('.','').replace('+','').replace(',','.').strip()
+                if '-' in coming:
+                    coming = '-'+coming.replace('-', '')
+                if not account.coming:
+                    account.coming = Decimal('0')
+                account.coming += Decimal(coming)
+                account._coming_links.append(link)
+
         return l
 
 
@@ -185,61 +203,107 @@ class Transaction(FrenchTransaction):
                ]
 
 class AccountHistoryPage(BasePage):
-    def get_operations(self,account):
+    def get_table(self):
-        operations = []
         tables=self.document.findall("//table[@class='tagTab pyjama']")
-        table=None
+        for table in tables:
-        for i in range(len(tables)):
             # Look for the relevant table in the Pro version
-            header=tables[i].getprevious()
+            header=table.getprevious()
             while str(header.tag)=="<built-in function Comment>":
                 header=header.getprevious()
             header=header.find("div")
             if header is not None:
                 header=header.find("span")
+
             if header is not None and \
                header.text.strip().startswith("Opérations effectuées".decode('utf-8')):
-                table=tables[i]
+                return table
-                break;
+
             # Look for the relevant table in the Particulier version
-            header=tables[i].find("thead").find("tr").find("th[@class='titleTab titleTableft']")
+            header=table.find("thead").find("tr").find("th[@class='titleTab titleTableft']")
             if header is not None and\
                header.text.strip().startswith("Solde au"):
-                table=tables[i]
+                return table
-                break;
+
+    def strip_label(self, s):
+        return s
+
+    def get_operations(self):
+        table = self.get_table()
+        operations = []
+
+        if table is None:
+            return operations
 
         for tr in table.iter('tr'):
             # skip headers and empty rows
             if len(tr.findall("th"))!=0 or\
                len(tr.findall("td"))<=1:
                 continue
-            mntColumn=0
+            mntColumn = 0
 
             date = None
             raw = None
             credit = ''
             debit = ''
             for td in tr.iter('td'):
-                value=td.attrib.get('id')
+                value = td.attrib.get('id')
                 if value is None:
-                    value=td.attrib.get('class');
+                    # if tag has no id nor class, assume it's a label
-                if value.startswith("date"):
+                    value = td.attrib.get('class', 'opLib')
+
+                if value.startswith("date") or value.endswith('center'):
                     # some transaction are included in a <strong> tag
-                    date=u''.join([txt.strip() for txt in td.itertext()])
+                    date = u''.join([txt.strip() for txt in td.itertext()])
                 elif value.startswith("lib") or value.startswith("opLib"):
                     # misclosed A tag requires to grab text from td
-                    raw=u''.join([txt.strip() for txt in td.itertext()])
+                    raw = self.strip_label(u''.join([txt.strip() for txt in td.itertext()]))
-                elif value.startswith("solde") or value.startswith("mnt"):
+                elif value.startswith("solde") or value.startswith("mnt") or \
-                    mntColumn+=1
+                     value.startswith('debit') or value.startswith('credit'):
-                    amount=u''.join([txt.strip() for txt in td.itertext()])
+                    mntColumn += 1
+                    amount = u''.join([txt.strip() for txt in td.itertext()])
                     if amount != "":
-                        if value.startswith("soldeDeb") or mntColumn==1:
+                        if value.startswith("soldeDeb") or value.startswith('debit') or mntColumn==1:
                             debit = amount
                         else:
                             credit = amount
 
-            operation=Transaction(len(operations))
+            if date is None:
+                # skip non-transaction
+                continue
+
+            operation = Transaction(len(operations))
             operation.parse(date, raw)
             operation.set_amount(credit, debit)
+
+            if operation.category == 'RELEVE CB':
+                # strip that transaction which is detailled in CBListPage.
+                continue
+
             operations.append(operation)
         return operations
+
+class CBHistoryPage(AccountHistoryPage):
+    def get_table(self):
+        # there is only one table on the page
+        try:
+            return self.document.findall("//table[@class='tagTab pyjama']")[0]
+        except IndexError:
+            return None
+
+    def strip_label(self, label):
+        # prevent to be considered as a category if there are two spaces.
+        return re.sub(r'[ ]+', ' ', label).strip()
+
+    def get_operations(self):
+        for tr in AccountHistoryPage.get_operations(self):
+            tr.type = tr.TYPE_CARD
+            yield tr
+
+class CBListPage(CBHistoryPage):
+    def get_cards(self):
+        cards = []
+        for a in self.document.getiterator('a'):
+            link = a.attrib.get('href', '')
+            if link.startswith('/outil/UWCB/UWCBEncours') and 'listeOperations' in link:
+                cards.append(link)
+        return cards