Add history support with Browser2

modules/ing/backend.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright(C) 2010-2013 Romain Bignon, Florent Fourcot
+# Copyright(C) 2010-2014 Romain Bignon, Florent Fourcot
 #
 # This file is part of weboob.
 #
@@ -69,7 +69,6 @@ class INGBackend(BaseBackend, ICapBank, ICapBill):
             yield account
 
     def get_account(self, _id):
-        with self.browser:
         account = self.browser.get_account(_id)
         if account:
             return account
@@ -77,7 +76,6 @@ class INGBackend(BaseBackend, ICapBank, ICapBill):
             raise AccountNotFound()
 
     def iter_history(self, account):
-        with self.browser:
         for history in self.browser.get_history(account.id):
             yield history
 

modules/ing/browser.py

@@ -86,11 +86,7 @@ class IngBrowser(LoginBrowser):
     def get_account(self, id):
         assert isinstance(id, basestring)
 
-        if not self.is_on_page(AccountsList) or self.where != "start":
+        l = self.get_accounts_list()
-            self.location(self.accountspage)
-        self.where = "start"
-
-        l = self.page.get_list()
         for a in l:
             if a.id == id:
                 return a
@@ -102,6 +98,7 @@ class IngBrowser(LoginBrowser):
         # are always on a HTML document.
         return True
 
+    @need_login
     def get_history(self, account):
         if not isinstance(account, Account):
             account = self.get_account(account)
@@ -114,7 +111,7 @@ class IngBrowser(LoginBrowser):
             raise NotImplementedError()
 
         if self.where != "start":
-            self.location(self.accountspage)
+            self.accountspage.go()
         data = {"AJAX:EVENTS_COUNT": 1,
                 "AJAXREQUEST": "_viewRoot",
                 "ajaxSingle": "index:setAccount",
@@ -124,7 +121,7 @@ class IngBrowser(LoginBrowser):
                 "javax.faces.ViewState": account._jid,
                 "cptnbr": account._id
                 }
-        self.location(self.accountspage, urllib.urlencode(data))
+        self.accountspage.go(data=data)
         self.where = "history"
         jid = self.page.get_history_jid()
         if jid is None:
@@ -135,7 +132,7 @@ class IngBrowser(LoginBrowser):
         hashlist = []
         while True:
             i = index
-            for transaction in self.page.get_transactions(index):
+            for transaction in self.page.get_transactions(index=index):
                 while transaction.id in hashlist:
                     transaction.id = hashlib.md5(transaction.id + "1").hexdigest()
                 hashlist.append(transaction.id)
@@ -152,7 +149,7 @@ class IngBrowser(LoginBrowser):
                     "index:%s:moreTransactions" % jid: "index:%s:moreTransactions" % jid,
                     "javax.faces.ViewState": account._jid
                     }
-            self.location(self.accountspage, urllib.urlencode(data))
+            self.accountspage.go(data=data)
 
     def get_recipients(self, account):
         if not self.is_on_page(TransferPage):

modules/ing/pages/accounts_list.py

@@ -25,8 +25,8 @@ import hashlib
 
 from weboob.capabilities.bank import Account
 from weboob.capabilities.base import NotAvailable
-from weboob.tools.browser2.page import HTMLPage, method, ListElement, ItemElement
+from weboob.tools.browser2.page import HTMLPage, LoggedPage, method, ListElement, ItemElement
-from weboob.tools.browser2.filters import Attr, CleanText, CleanDecimal, Filter, Field, MultiFilter
+from weboob.tools.browser2.filters import Attr, CleanText, CleanDecimal, Filter, Field, MultiFilter, Env, Date, Lower
 from weboob.tools.capabilities.bank.transactions import FrenchTransaction
 
 
@@ -67,16 +67,16 @@ class AddType(Filter):
         return Account.TYPE_UNKNOWN
 
 
-class AccountsList(HTMLPage):
+class Hashmd5(MultiFilter):
+    def filter(self, values):
+        concat = ''
+        for value in values:
+            concat += u'%s' % value
+        return hashlib.md5(concat.encode('utf-8')).hexdigest()
 
-    monthvalue = {u'janv.': '01', u'févr.': '02', u'mars': '03', u'avr.': '04',
+class AccountsList(LoggedPage, HTMLPage):
-            u'mai': '05', u'juin': '06', u'juil.': '07', u'août': '08',
+
-            u'sept.': '09', u'oct.': '10', u'nov.': '11', u'déc.': '12',
+    i = 0
-            }
-    catvalue = {u'virt': u"Virement", u'autre': u"Autre",
-            u'plvt': u'Prélèvement', u'cb_ret': u"Carte retrait",
-            u'cb_ach': u'Carte achat', u'chq': u'Chèque',
-            u'frais': u'Frais bancaire', u'sepaplvt': u'Prélèvement'}
 
     @method
     class get_list(ListElement):
@@ -95,17 +95,44 @@ class AccountsList(HTMLPage):
             obj__jid = Attr('//input[@name="javax.faces.ViewState"]', 'value')
 
 
-    def get_transactions(self, index):
+    @method
+    class get_transactions(ListElement):
+        item_xpath = '//table'
         i = 0
-        for table in self.document.xpath('//table'):
+
-            try:
+
+        class item(ItemElement):
+            klass = Transaction
+
+            monthvalue = {u'janv.': '01', u'févr.': '02', u'mars': '03', u'avr.': '04',
+                          u'mai': '05', u'juin': '06', u'juil.': '07', u'août': '08',
+                          u'sept.': '09', u'oct.': '10', u'nov.': '11', u'déc.': '12',
+                         }
+            catvalue = {u'virt': u"Virement", u'autre': u"Autre",
+                        u'plvt': u'Prélèvement', u'cb_ret': u"Carte retrait",
+                        u'cb_ach': u'Carte achat', u'chq': u'Chèque',
+                        u'frais': u'Frais bancaire', u'sepaplvt': u'Prélèvement'}
+
+            # we use lower for compatibility with the old website
+            obj_raw = Lower('.//td[@class="lbl"]')
+            obj_amount = CleanDecimal('.//td[starts-with(@class, "amount")]')
+            obj__textdate = Env('_textdate')
+            obj_date = Date(Field('_textdate'), dayfirst=True)
+            obj_rdate = Field('date')
+            obj_id = Hashmd5(Field('_textdate'), Field('raw'), Field('amount'))
+
+
+            def condition(self):
+                if self.el.find('.//td[@class="date"]') is None:
+                    return False
+                if AccountsList.i < self.env['index']:
+                    AccountsList.i += 1
+                    return False
+                return True
+
+            def parse(self, table):
                 textdate = table.find('.//td[@class="date"]').text_content()
-            except AttributeError:
-                continue
                 # Do not parse transactions already parsed
-            if i < index:
-                i += 1
-                continue
                 if textdate == 'hier':
                     textdate = (date.today() - timedelta(days=1)).strftime('%d/%m/%Y')
                 elif textdate == "aujourd'hui":
@@ -115,42 +142,29 @@ class AccountsList(HTMLPage):
                     month = self.monthvalue[frenchmonth]
                     textdate = textdate.replace(' ', '')
                     textdate = textdate.replace(frenchmonth, '/%s/' %month)
-            # We use lower for compatibility with old website
+                self.env['_textdate'] = textdate
-            textraw = self.parser.tocleanstring(table.find('.//td[@class="lbl"]')).lower()
-            # The id will be rewrite
-            op = Transaction(1)
-            amount = op.clean_amount(table.xpath('.//td[starts-with(@class, "amount")]')[0].text_content())
-            id = hashlib.md5(textdate.encode('utf-8') + textraw.encode('utf-8')
-                    + amount.encode('utf-8')).hexdigest()
-            op.id = id
-            op.parse(date = date(*reversed([int(x) for x in textdate.split('/')])),
-                     raw = textraw)
                 category = table.find('.//td[@class="picto"]/span')
                 category = unicode(category.attrib['class'].split('-')[0].lower())
                 try:
-                op.category = self.catvalue[category]
+                    category = self.catvalue[category]
                 except:
-                op.category = category
+                    pass
-            op.amount = Decimal(amount)
+                self.env['category'] = category
-            yield op
 
     def get_history_jid(self):
-        span = self.document.xpath('//span[@id="index:panelASV"]')
+        span = self.doc.xpath('//span[@id="index:panelASV"]')
         if len(span) > 1:
             # Assurance Vie, we do not support this kind of account.
             return None
 
-        span = self.document.xpath('//span[starts-with(@id, "index:j_id")]')[0]
+        span = Attr('//span[starts-with(@id, "index:j_id")]', 'id')(self.doc)
-        jid = span.attrib['id'].split(':')[1]
+        jid = span.split(':')[1]
         return jid
 
     def islast(self):
-        havemore = self.document.getroot().cssselect('.show-more-transactions')
+        havemore = self.doc.getroot().cssselect('.show-more-transactions')
         if len(havemore) == 0:
             return True
 
-        nomore = self.document.getroot().cssselect('.no-more-transactions')
+        nomore = self.doc.getroot().cssselect('.no-more-transactions')
-        if len(nomore) > 0:
+        return (len(nomore) > 0)
-            return True
-        else:
-            return False