From 267b8bbc751ce3c82eaa573c369d885daa50ce64 Mon Sep 17 00:00:00 2001 From: Florent Date: Fri, 28 Mar 2014 11:31:10 +0100 Subject: [PATCH] Add history support with Browser2 --- modules/ing/backend.py | 10 +-- modules/ing/browser.py | 15 ++-- modules/ing/pages/accounts_list.py | 126 ++++++++++++++++------------- 3 files changed, 80 insertions(+), 71 deletions(-) diff --git a/modules/ing/backend.py b/modules/ing/backend.py index e8f59694..9291066d 100644 --- a/modules/ing/backend.py +++ b/modules/ing/backend.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright(C) 2010-2013 Romain Bignon, Florent Fourcot +# Copyright(C) 2010-2014 Romain Bignon, Florent Fourcot # # This file is part of weboob. # @@ -69,17 +69,15 @@ class INGBackend(BaseBackend, ICapBank, ICapBill): yield account def get_account(self, _id): - with self.browser: - account = self.browser.get_account(_id) + account = self.browser.get_account(_id) if account: return account else: raise AccountNotFound() def iter_history(self, account): - with self.browser: - for history in self.browser.get_history(account.id): - yield history + for history in self.browser.get_history(account.id): + yield history def iter_transfer_recipients(self, account): with self.browser: diff --git a/modules/ing/browser.py b/modules/ing/browser.py index 542ffe3e..b788ac3c 100644 --- a/modules/ing/browser.py +++ b/modules/ing/browser.py @@ -86,11 +86,7 @@ class IngBrowser(LoginBrowser): def get_account(self, id): assert isinstance(id, basestring) - if not self.is_on_page(AccountsList) or self.where != "start": - self.location(self.accountspage) - self.where = "start" - - l = self.page.get_list() + l = self.get_accounts_list() for a in l: if a.id == id: return a @@ -102,6 +98,7 @@ class IngBrowser(LoginBrowser): # are always on a HTML document. return True + @need_login def get_history(self, account): if not isinstance(account, Account): account = self.get_account(account) @@ -114,7 +111,7 @@ class IngBrowser(LoginBrowser): raise NotImplementedError() if self.where != "start": - self.location(self.accountspage) + self.accountspage.go() data = {"AJAX:EVENTS_COUNT": 1, "AJAXREQUEST": "_viewRoot", "ajaxSingle": "index:setAccount", @@ -124,7 +121,7 @@ class IngBrowser(LoginBrowser): "javax.faces.ViewState": account._jid, "cptnbr": account._id } - self.location(self.accountspage, urllib.urlencode(data)) + self.accountspage.go(data=data) self.where = "history" jid = self.page.get_history_jid() if jid is None: @@ -135,7 +132,7 @@ class IngBrowser(LoginBrowser): hashlist = [] while True: i = index - for transaction in self.page.get_transactions(index): + for transaction in self.page.get_transactions(index=index): while transaction.id in hashlist: transaction.id = hashlib.md5(transaction.id + "1").hexdigest() hashlist.append(transaction.id) @@ -152,7 +149,7 @@ class IngBrowser(LoginBrowser): "index:%s:moreTransactions" % jid: "index:%s:moreTransactions" % jid, "javax.faces.ViewState": account._jid } - self.location(self.accountspage, urllib.urlencode(data)) + self.accountspage.go(data=data) def get_recipients(self, account): if not self.is_on_page(TransferPage): diff --git a/modules/ing/pages/accounts_list.py b/modules/ing/pages/accounts_list.py index 018061c6..89085ecf 100644 --- a/modules/ing/pages/accounts_list.py +++ b/modules/ing/pages/accounts_list.py @@ -25,8 +25,8 @@ import hashlib from weboob.capabilities.bank import Account from weboob.capabilities.base import NotAvailable -from weboob.tools.browser2.page import HTMLPage, method, ListElement, ItemElement -from weboob.tools.browser2.filters import Attr, CleanText, CleanDecimal, Filter, Field, MultiFilter +from weboob.tools.browser2.page import HTMLPage, LoggedPage, method, ListElement, ItemElement +from weboob.tools.browser2.filters import Attr, CleanText, CleanDecimal, Filter, Field, MultiFilter, Env, Date, Lower from weboob.tools.capabilities.bank.transactions import FrenchTransaction @@ -67,16 +67,16 @@ class AddType(Filter): return Account.TYPE_UNKNOWN -class AccountsList(HTMLPage): +class Hashmd5(MultiFilter): + def filter(self, values): + concat = '' + for value in values: + concat += u'%s' % value + return hashlib.md5(concat.encode('utf-8')).hexdigest() - monthvalue = {u'janv.': '01', u'févr.': '02', u'mars': '03', u'avr.': '04', - u'mai': '05', u'juin': '06', u'juil.': '07', u'août': '08', - u'sept.': '09', u'oct.': '10', u'nov.': '11', u'déc.': '12', - } - catvalue = {u'virt': u"Virement", u'autre': u"Autre", - u'plvt': u'Prélèvement', u'cb_ret': u"Carte retrait", - u'cb_ach': u'Carte achat', u'chq': u'Chèque', - u'frais': u'Frais bancaire', u'sepaplvt': u'Prélèvement'} +class AccountsList(LoggedPage, HTMLPage): + + i = 0 @method class get_list(ListElement): @@ -95,62 +95,76 @@ class AccountsList(HTMLPage): obj__jid = Attr('//input[@name="javax.faces.ViewState"]', 'value') - def get_transactions(self, index): + @method + class get_transactions(ListElement): + item_xpath = '//table' i = 0 - for table in self.document.xpath('//table'): - try: + + + class item(ItemElement): + klass = Transaction + + monthvalue = {u'janv.': '01', u'févr.': '02', u'mars': '03', u'avr.': '04', + u'mai': '05', u'juin': '06', u'juil.': '07', u'août': '08', + u'sept.': '09', u'oct.': '10', u'nov.': '11', u'déc.': '12', + } + catvalue = {u'virt': u"Virement", u'autre': u"Autre", + u'plvt': u'Prélèvement', u'cb_ret': u"Carte retrait", + u'cb_ach': u'Carte achat', u'chq': u'Chèque', + u'frais': u'Frais bancaire', u'sepaplvt': u'Prélèvement'} + + # we use lower for compatibility with the old website + obj_raw = Lower('.//td[@class="lbl"]') + obj_amount = CleanDecimal('.//td[starts-with(@class, "amount")]') + obj__textdate = Env('_textdate') + obj_date = Date(Field('_textdate'), dayfirst=True) + obj_rdate = Field('date') + obj_id = Hashmd5(Field('_textdate'), Field('raw'), Field('amount')) + + + def condition(self): + if self.el.find('.//td[@class="date"]') is None: + return False + if AccountsList.i < self.env['index']: + AccountsList.i += 1 + return False + return True + + def parse(self, table): textdate = table.find('.//td[@class="date"]').text_content() - except AttributeError: - continue - # Do not parse transactions already parsed - if i < index: - i += 1 - continue - if textdate == 'hier': - textdate = (date.today() - timedelta(days=1)).strftime('%d/%m/%Y') - elif textdate == "aujourd'hui": - textdate = date.today().strftime('%d/%m/%Y') - else: - frenchmonth = textdate.split(' ')[1] - month = self.monthvalue[frenchmonth] - textdate = textdate.replace(' ', '') - textdate = textdate.replace(frenchmonth, '/%s/' %month) - # We use lower for compatibility with old website - textraw = self.parser.tocleanstring(table.find('.//td[@class="lbl"]')).lower() - # The id will be rewrite - op = Transaction(1) - amount = op.clean_amount(table.xpath('.//td[starts-with(@class, "amount")]')[0].text_content()) - id = hashlib.md5(textdate.encode('utf-8') + textraw.encode('utf-8') - + amount.encode('utf-8')).hexdigest() - op.id = id - op.parse(date = date(*reversed([int(x) for x in textdate.split('/')])), - raw = textraw) - category = table.find('.//td[@class="picto"]/span') - category = unicode(category.attrib['class'].split('-')[0].lower()) - try: - op.category = self.catvalue[category] - except: - op.category = category - op.amount = Decimal(amount) - yield op + # Do not parse transactions already parsed + if textdate == 'hier': + textdate = (date.today() - timedelta(days=1)).strftime('%d/%m/%Y') + elif textdate == "aujourd'hui": + textdate = date.today().strftime('%d/%m/%Y') + else: + frenchmonth = textdate.split(' ')[1] + month = self.monthvalue[frenchmonth] + textdate = textdate.replace(' ', '') + textdate = textdate.replace(frenchmonth, '/%s/' %month) + self.env['_textdate'] = textdate + category = table.find('.//td[@class="picto"]/span') + category = unicode(category.attrib['class'].split('-')[0].lower()) + try: + category = self.catvalue[category] + except: + pass + self.env['category'] = category def get_history_jid(self): - span = self.document.xpath('//span[@id="index:panelASV"]') + span = self.doc.xpath('//span[@id="index:panelASV"]') if len(span) > 1: # Assurance Vie, we do not support this kind of account. return None - span = self.document.xpath('//span[starts-with(@id, "index:j_id")]')[0] - jid = span.attrib['id'].split(':')[1] + span = Attr('//span[starts-with(@id, "index:j_id")]', 'id')(self.doc) + jid = span.split(':')[1] return jid def islast(self): - havemore = self.document.getroot().cssselect('.show-more-transactions') + havemore = self.doc.getroot().cssselect('.show-more-transactions') if len(havemore) == 0: return True - nomore = self.document.getroot().cssselect('.no-more-transactions') - if len(nomore) > 0: - return True - else: - return False + nomore = self.doc.getroot().cssselect('.no-more-transactions') + return (len(nomore) > 0)