support new bnp website (refs #769)

modules/bnporc/backend.py

@@ -76,12 +76,12 @@ class BNPorcBackend(BaseBackend, ICapBank):
 
     def iter_history(self, account):
         with self.browser:
-            for history in self.browser.get_history(account):
+            for history in self.browser.get_history(account.id):
                 yield history
 
     def iter_operations(self, account):
         with self.browser:
-            for coming in self.browser.get_coming_operations(account):
+            for coming in self.browser.get_coming_operations(account.id):
                 yield coming
 
     def iter_transfer_recipients(self, ignored):

modules/bnporc/browser.py

@@ -26,7 +26,7 @@ from weboob.capabilities.bank import TransferError, Transfer
 from .pages import AccountsList, AccountHistory, ChangePasswordPage, \
                    AccountComing, AccountPrelevement, TransferPage, \
                    TransferConfirmPage, TransferCompletePage, \
-                   LoginPage, ConfirmPage
+                   LoginPage, ConfirmPage, MessagePage
 from .errors import PasswordExpired
 
 
@@ -36,19 +36,20 @@ __all__ = ['BNPorc']
 class BNPorc(BaseBrowser):
     DOMAIN = 'www.secure.bnpparibas.net'
     PROTOCOL = 'https'
+    DEBUG_HTTP = True
     ENCODING = None # refer to the HTML encoding
-    PAGES = {'.*identifiant=DOSSIER_Releves_D_Operation.*': AccountsList,
+    PAGES = {'.*pageId=unedescomptes.*':                    AccountsList,
-             '.*SAF_ROP.*':                                 AccountHistory,
+             '.*pageId=releveoperations.*':                 AccountHistory,
              '.*Action=SAF_CHM.*':                          ChangePasswordPage,
-             '.*NS_AVEDT.*':                                AccountComing,
+             '.*pageId=mouvementsavenir.*':                 AccountComing,
              '.*NS_AVEDP.*':                                AccountPrelevement,
              '.*NS_VIRDF.*':                                TransferPage,
              '.*NS_VIRDC.*':                                TransferConfirmPage,
              '.*/NS_VIRDA\?stp=(?P<id>\d+).*':              TransferCompletePage,
-             '.*Action=DSP_VGLOBALE.*':                     LoginPage,
              '.*type=homeconnex.*':                         LoginPage,
              '.*layout=HomeConnexion.*':                    ConfirmPage,
              '.*SAF_CHM_VALID.*':                           ConfirmPage,
+             '.*Action=DSP_MSG.*':                          MessagePage,
             }
 
     def __init__(self, *args, **kwargs):
@@ -79,13 +80,16 @@ class BNPorc(BaseBrowser):
     def change_password(self, new_password):
         assert new_password.isdigit() and len(new_password) == 6
 
-        self.location('https://www.secure.bnpparibas.net/SAF_CHM?Action=SAF_CHM')
+        buf = self.readurl('https://www.secure.bnpparibas.net/NSFR?Action=SAF_CHM', if_fail='raise')
+        buf = buf[buf.find('/SAF_CHM?Action=SAF_CHM'):]
+        buf = buf[:buf.find('"')]
+        self.location(buf)
         assert self.is_on_page(ChangePasswordPage)
 
         self.page.change_password(self.password, new_password)
 
-        if not self.is_on_page(ConfirmPage):
+        if not self.is_on_page(ConfirmPage) or self.page.get_error() is not None:
-            self.logger.error('Oops, unable to change password')
+            self.logger.error('Oops, unable to change password (%s)' % (self.page.get_error() if self.is_on_page(ConfirmPage) else 'unknown'))
             return
 
         self.password, self.rotating_password = (new_password, self.password)
@@ -126,16 +130,18 @@ class BNPorc(BaseBrowser):
 
         return None
 
-    def get_history(self, account):
+    def get_history(self, id):
-        if not self.is_on_page(AccountHistory) or self.page.account.id != account.id:
+        self.location('/banque/portail/particulier/FicheA?contractId=%d&pageId=releveoperations&_eventId=changeOperationsPerPage&operationsPerPage=200' % int(id))
-            self.location('/SAF_ROP?ch4=%s' % account.link_id)
         return self.page.get_operations()
 
-    def get_coming_operations(self, account):
+    def get_coming_operations(self, id):
-        if not self.is_on_page(AccountComing) or self.page.account.id != account.id:
+        if not self.is_on_page(AccountsList):
-            self.location('/NS_AVEDT?ch4=%s' % account.link_id)
+            self.location('/NSFR?Action=DSP_VGLOBALE')
+        execution = self.page.get_execution_id()
+        self.location('/banque/portail/particulier/FicheA?externalIAId=IAStatements&contractId=%d&pastOrPendingOperations=2&pageId=mouvementsavenir&execution=%s' % (int(id), execution))
         return self.page.get_operations()
 
+    @check_expired_password
     def get_transfer_accounts(self):
         if not self.is_on_page(TransferPage):
             self.location('/NS_VIRDF')
@@ -143,6 +149,7 @@ class BNPorc(BaseBrowser):
         assert self.is_on_page(TransferPage)
         return self.page.get_accounts()
 
+    @check_expired_password
     def transfer(self, from_id, to_id, amount, reason=None):
         if not self.is_on_page(TransferPage):
             self.location('/NS_VIRDF')

modules/bnporc/pages/__init__.py

@@ -22,10 +22,10 @@ from .accounts_list import AccountsList
 from .account_coming import AccountComing
 from .account_history import AccountHistory
 from .transfer import TransferPage, TransferConfirmPage, TransferCompletePage
-from .login import LoginPage, ConfirmPage, ChangePasswordPage
+from .login import LoginPage, ConfirmPage, ChangePasswordPage, MessagePage
 
 class AccountPrelevement(AccountsList): pass
 
 __all__ = ['AccountsList', 'AccountComing', 'AccountHistory', 'LoginPage',
-           'ConfirmPage', 'AccountPrelevement', 'ChangePasswordPage',
+           'ConfirmPage', 'MessagePage', 'AccountPrelevement', 'ChangePasswordPage',
            'TransferPage', 'TransferConfirmPage', 'TransferCompletePage']

modules/bnporc/pages/account_coming.py

@@ -29,7 +29,7 @@ __all__ = ['AccountComing']
 
 
 class AccountComing(BasePage):
-    LABEL_PATTERNS = [('^FACTURECARTEDU(?P<dd>\d{2})(?P<mm>\d{2})(?P<yy>\d{2})(?P<text>.*)',
+    LABEL_PATTERNS = [('^FACTURE CARTE DU (?P<dd>\d{2})(?P<mm>\d{2})(?P<yy>\d{2}) (?P<text>.*)',
                           u'CB %(yy)s-%(mm)s-%(dd)s: %(text)s'),
                       ('^PRELEVEMENT(?P<text>.*)', 'Order: %(text)s'),
                       ('^ECHEANCEPRET(?P<text>.*)', u'Loan payment n°%(text)s'),
@@ -38,20 +38,18 @@ class AccountComing(BasePage):
     def on_loaded(self):
         self.operations = []
 
-        for tr in self.document.getiterator('tr'):
+        for tr in self.document.xpath('//table[@id="tableauOperations"]//tr'):
-            if tr.attrib.get('class', '') == 'hdoc1' or tr.attrib.get('class', '') == 'hdotc1':
+            if 'typeop' in tr.attrib:
                 tds = tr.findall('td')
                 if len(tds) != 3:
                     continue
-                d = tds[0].getchildren()[0].attrib.get('name', '')
+                d = tr.attrib['dateop']
-                d = date(int(d[0:4]), int(d[4:6]), int(d[6:8]))
+                d = date(int(d[4:8]), int(d[2:4]), int(d[0:2]))
-                label = u''
+                label = tds[1].text or u''
-                label += tds[1].text or u''
                 label = label.replace(u'\xa0', u'')
                 for child in tds[1].getchildren():
                     if child.text: label += child.text
                     if child.tail: label += child.tail
-                if tds[1].tail: label += tds[1].tail
                 label = label.strip()
 
                 for pattern, text in self.LABEL_PATTERNS:
@@ -59,7 +57,7 @@ class AccountComing(BasePage):
                     if m:
                         label = text % m.groupdict()
 
-                amount = tds[2].text.replace('.', '').replace(',', '.')
+                amount = tds[2].text.replace('.','').replace(',','.').strip(u' \t\u20ac\xa0€\n\r')
 
                 operation = Operation(len(self.operations))
                 operation.date = d

modules/bnporc/pages/account_history.py

@@ -35,46 +35,34 @@ class AccountHistory(BasePage):
     def on_loaded(self):
         self.operations = []
 
-        for tr in self.document.getiterator('tr'):
+        for tr in self.document.xpath('//table[@id="tableCompte"]//tr'):
-            if tr.attrib.get('class', '') == 'hdoc1' or tr.attrib.get('class', '') == 'hdotc1':
+            if len(tr.xpath('td[@class="debit"]')) == 0:
-                tds = tr.findall('td')
+                continue
-                if len(tds) != 4:
-                    continue
-                d = date(*reversed([int(x) for x in tds[0].text.split('/')]))
-                label = u''
-                label += tds[1].text
-                label = label.replace(u'\xa0', u'')
-                for child in tds[1].getchildren():
-                    if child.text: label += child.text
-                    if child.tail: label += child.tail
-                if tds[1].tail: label += tds[1].tail
 
-                label = label.strip()
+            id = tr.find('td').find('input').attrib['value']
-                category = NotAvailable
+            op = Operation(id)
-                for pattern, _cat, _lab in self.LABEL_PATTERNS:
+            op.label = tr.findall('td')[2].text.replace(u'\xa0', u'').strip()
-                    m = re.match(pattern, label)
+            op.date = date(*reversed([int(x) for x in tr.findall('td')[1].text.split('/')]))
-                    if m:
-                        category = _cat % m.groupdict()
-                        label = _lab % m.groupdict()
-                        break
-                else:
-                    if '  ' in label:
-                        category, useless, label = [part.strip() for part in label.partition('  ')]
 
-                amount = tds[2].text.replace('.', '').replace(',', '.')
+            op.category = NotAvailable
+            for pattern, _cat, _lab in self.LABEL_PATTERNS:
+                m = re.match(pattern, op.label)
+                if m:
+                    op.category = _cat % m.groupdict()
+                    op.label = _lab % m.groupdict()
+                    break
+            else:
+                if '  ' in op.label:
+                    op.category, useless, op.label = [part.strip() for part in op.label.partition('  ')]
 
-                # if we don't have exactly one '.', this is not a floatm try the next
+            debit = tr.xpath('.//td[@class="debit"]')[0].text.replace('.','').replace(',','.').strip(u' \t\u20ac\xa0€\n\r')
-                operation = Operation(len(self.operations))
+            credit = tr.xpath('.//td[@class="credit"]')[0].text.replace('.','').replace(',','.').strip(u' \t\u20ac\xa0€\n\r')
-                if amount.count('.') != 1:
+            if len(debit) > 0:
-                    amount = tds[3].text.replace('.', '').replace(',', '.')
+                op.amount = - float(debit)
-                    operation.amount = float(amount)
+            else:
-                else:
+                op.amount = float(credit)
-                    operation.amount = - float(amount)
 
-                operation.date = d
+            self.operations.append(op)
-                operation.label = label
-                operation.category = category
-                self.operations.append(operation)
 
     def get_operations(self):
         return self.operations

modules/bnporc/pages/accounts_list.py

@@ -39,40 +39,26 @@ class AccountsList(BasePage):
     def get_list(self):
         l = []
         for tr in self.document.getiterator('tr'):
-            if tr.attrib.get('class', '') == 'comptes':
+            if not 'class' in tr.attrib and tr.find('td') is not None and tr.find('td').attrib.get('class', '') == 'typeTitulaire':
                 account = Account()
-                for td in tr.getiterator('td'):
+                account.id = tr.xpath('.//td[@class="libelleCompte"]/input')[0].attrib['id'][len('libelleCompte'):]
-                    if td.attrib.get('headers', '').startswith('Numero_'):
+                account.label = tr.xpath('.//td[@class="libelleCompte"]/a')[0].text.strip()
-                        id = td.text
-                        account.id = ''.join(id.split(' ')).strip()
-                    elif td.attrib.get('headers', '').startswith('Libelle_'):
-                        a = td.findall('a')
-                        label = unicode(a[0].text)
-                        account.label = label.strip()
-                        m = self.LINKID_REGEXP.match(a[0].attrib.get('href', ''))
-                        if m:
-                            account.link_id = m.group(1)
-                    elif td.attrib.get('headers', '').startswith('Solde'):
-                        a = td.findall('a')
-                        balance = a[0].text
-                        balance = balance.replace('.','').replace(',','.')
-                        account.balance = float(balance)
-                    elif td.attrib.get('headers', '').startswith('Avenir'):
-                        a = td.findall('a')
-                        # Some accounts don't have a "coming"
-                        if len(a):
-                            coming = a[0].text
-                            coming = coming.replace('.','').replace(',','.')
-                            account.coming = float(coming)
-                        else:
-                            account.coming = NotAvailable
 
+                tds = tr.findall('td')
+                account.balance = float(tds[3].find('a').text.replace('.','').replace(',','.').strip(u' \t\u20ac\xa0€\n\r'))
+                if tds[4].find('a') is not None:
+                    account.coming = float(tds[4].find('a').text.replace('.','').replace(',','.').strip(u' \t\u20ac\xa0€\n\r'))
+                else:
+                    account.coming = NotAvailable
                 l.append(account)
 
         if len(l) == 0:
             # oops, no accounts? check if we have not exhausted the allowed use
             # of this password
-            for div in self.document.getroot().cssselect('div.Style_texte_gras'):
+            for img in self.document.getroot().cssselect('img[align="middle"]'):
-                if div.text.strip() == 'Vous avez atteint la date de fin de vie de votre code secret.':
+                if img.attrib.get('alt', '') == 'Changez votre code secret':
-                    raise PasswordExpired(div.text.strip())
+                    raise PasswordExpired('Your password has expired')
         return l
+
+    def get_execution_id(self):
+        return self.document.xpath('//input[@name="execution"]')[0].attrib['value']

modules/bnporc/pages/login.py

@@ -18,9 +18,9 @@
 # along with weboob. If not, see <http://www.gnu.org/licenses/>.
 
 
+import re
 from weboob.tools.mech import ClientForm
 import urllib
-from logging import error
 
 from weboob.tools.browser import BasePage, BrowserUnavailable
 from weboob.tools.captcha.virtkeyboard import MappedVirtKeyboard,VirtKeyboardError
@@ -78,7 +78,7 @@ class LoginPage(BasePage):
         try:
             vk=BNPVirtKeyboard(self)
         except VirtKeyboardError,err:
-            error("Error: %s"%err)
+            self.logger.error("Error: %s"%err)
             return False
 
         self.browser.select_form('logincanalnet')
@@ -92,23 +92,60 @@ class LoginPage(BasePage):
 
 
 class ConfirmPage(BasePage):
-    pass
+    def get_error(self):
+        for td in self.document.xpath('//td[@class="hdvon1"]'):
+            if td.text:
+                return td.text.strip()
+        return None
 
+    def get_relocate_url(self):
+        script = self.document.xpath('//script')[0]
+        m = re.match('document.location.replace\("(.*)"\)', script.text[script.text.find('document.location.replace'):])
+        if m:
+            return m.group(1)
+
+class MessagePage(BasePage):
+    def on_loaded(self):
+        pass
 
 class ChangePasswordPage(BasePage):
     def change_password(self, current, new):
         try:
             vk=BNPVirtKeyboard(self)
         except VirtKeyboardError,err:
-            error("Error: %s"%err)
+            self.logger.error("Error: %s"%err)
             return False
 
+        from mechanize import Cookie
+        c = Cookie(0, 'wbo_segment_369721', 'AA%7CAB%7C%7C%7C',
+                      None, False,
+                      '.secure.bnpparibas.net', True, True,
+                      '/', False,
+                      False,
+                      None,
+                      False,
+                      None,
+                      None,
+                      {})
+        cookiejar = self.browser._ua_handlers["_cookies"].cookiejar
+
+        cookiejar.set_cookie(c)
+
+
         code_current=vk.get_string_code(current)
         code_new=vk.get_string_code(new)
 
-        data = {'ch1': code_current,
+        data = (('ch1', code_current.replace('4', '3')),
-                'ch2': code_new,
+                ('ch2', code_new),
-                'ch3': code_new
+                ('radiobutton3', 'radiobutton'),
-               }
+                ('ch3', code_new),
+                ('x', 23),
+                ('y', 13),
+               )
 
-        self.browser.location('/SAF_CHM_VALID', urllib.urlencode(data))
+        headers = {'Referer': self.url}
+        #headers = {'Referer': "https://www.secure.bnpparibas.net/SAF_CHM?Action=SAF_CHM&Origine=SAF_CHM&stp=%s" % (int(datetime.now().strftime('%Y%m%d%H%M%S')))}
+        #import time
+        #time.sleep(10)
+        request = self.browser.request_class('https://www.secure.bnpparibas.net/SAF_CHM_VALID', urllib.urlencode(data), headers)
+        self.browser.location(request)

modules/bnporc/pages/transfer.py

@@ -23,6 +23,7 @@ import re
 from weboob.tools.browser import BasePage
 from weboob.tools.ordereddict import OrderedDict
 from weboob.capabilities.bank import TransferError
+from ..errors import PasswordExpired
 
 
 __all__ = ['TransferPage', 'TransferConfirmPage', 'TransferCompletePage']
@@ -36,6 +37,11 @@ class Account(object):
         self.receive_checkbox = receive_checkbox
 
 class TransferPage(BasePage):
+    def on_loaded(self):
+        for td in self.document.xpath('//td[@class="hdvon1"]'):
+            if td.text and 'Vous avez atteint le seuil de' in td.text:
+                raise PasswordExpired(td.text.strip())
+
     def get_accounts(self):
         accounts = OrderedDict()
         for table in self.document.getiterator('table'):
@@ -55,6 +61,12 @@ class TransferPage(BasePage):
     def transfer(self, from_id, to_id, amount, reason):
         accounts = self.get_accounts()
 
+        # Transform RIBs to short IDs
+        if len(str(from_id)) == 23:
+            from_id = str(from_id)[5:21]
+        if len(str(to_id)) == 23:
+            to_id = str(to_id)[5:21]
+
         try:
             sender = accounts[from_id]
         except KeyError: