From 99cfae2a66b57add6ae648ed2526ae1ec2ad887d Mon Sep 17 00:00:00 2001 From: Romain Bignon Date: Sat, 23 Mar 2013 12:30:32 +0100 Subject: [PATCH] new module for american express --- modules/americanexpress/__init__.py | 23 +++++ modules/americanexpress/backend.py | 71 +++++++++++++++ modules/americanexpress/browser.py | 110 ++++++++++++++++++++++++ modules/americanexpress/favicon.png | Bin 0 -> 3730 bytes modules/americanexpress/pages.py | 129 ++++++++++++++++++++++++++++ modules/americanexpress/test.py | 32 +++++++ 6 files changed, 365 insertions(+) create mode 100644 modules/americanexpress/__init__.py create mode 100644 modules/americanexpress/backend.py create mode 100644 modules/americanexpress/browser.py create mode 100644 modules/americanexpress/favicon.png create mode 100644 modules/americanexpress/pages.py create mode 100644 modules/americanexpress/test.py diff --git a/modules/americanexpress/__init__.py b/modules/americanexpress/__init__.py new file mode 100644 index 00000000..7302887f --- /dev/null +++ b/modules/americanexpress/__init__.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2013 Romain Bignon +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from .backend import AmericanExpressBackend + +__all__ = ['AmericanExpressBackend'] diff --git a/modules/americanexpress/backend.py b/modules/americanexpress/backend.py new file mode 100644 index 00000000..b647ad03 --- /dev/null +++ b/modules/americanexpress/backend.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2013 Romain Bignon +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import with_statement + +from weboob.capabilities.bank import ICapBank, AccountNotFound +from weboob.tools.backend import BaseBackend, BackendConfig +from weboob.tools.value import ValueBackendPassword + +from .browser import AmericanExpressBrowser + + +__all__ = ['AmericanExpressBackend'] + + +class AmericanExpressBackend(BaseBackend, ICapBank): + NAME = 'americanexpress' + MAINTAINER = u'Romain Bignon' + EMAIL = 'romain@weboob.org' + VERSION = '0.f' + DESCRIPTION = u'American Express French bank website' + LICENSE = 'AGPLv3+' + CONFIG = BackendConfig(ValueBackendPassword('login', label='Account ID', masked=False), + ValueBackendPassword('password', label='Password of account')) + BROWSER = AmericanExpressBrowser + + def create_default_browser(self): + return self.create_browser(self.config['login'].get(), + self.config['password'].get()) + + def iter_accounts(self): + with self.browser: + for account in self.browser.get_accounts_list(): + yield account + + def get_account(self, _id): + with self.browser: + account = self.browser.get_account(_id) + + if account: + return account + else: + raise AccountNotFound() + + def iter_history(self, account): + with self.browser: + transactions = list(self.browser.get_history(account)) + transactions.sort(key=lambda tr: tr.rdate, reverse=True) + return [tr for tr in transactions if not tr._is_coming] + + def iter_coming(self, account): + with self.browser: + transactions = list(self.browser.get_history(account)) + transactions.sort(key=lambda tr: tr.rdate, reverse=True) + return [tr for tr in transactions if tr._is_coming] diff --git a/modules/americanexpress/browser.py b/modules/americanexpress/browser.py new file mode 100644 index 00000000..8943a10a --- /dev/null +++ b/modules/americanexpress/browser.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2013 Romain Bignon +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from urlparse import urlsplit, parse_qsl + +from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword +from weboob.tools.date import LinearDateGuesser + +from .pages import LoginPage, AccountsPage, TransactionsPage + + +__all__ = ['AmericanExpressBrowser'] + + +class AmericanExpressBrowser(BaseBrowser): + DOMAIN = 'global.americanexpress.com' + PROTOCOL = 'https' + ENCODING = 'ISO-8859-1' + PAGES = {'https://global.americanexpress.com/myca/logon/.*': LoginPage, + 'https://global.americanexpress.com/myca/intl/acctsumm/.*': AccountsPage, + 'https://global.americanexpress.com/myca/intl/estatement/.*': TransactionsPage, + } + + def is_logged(self): + return self.page is not None and not self.is_on_page(LoginPage) + + def home(self): + if self.is_logged(): + self.location(self.buildurl('/myca/intl/acctsumm/emea/accountSummary.do')) + else: + self.login() + + def login(self): + assert isinstance(self.username, basestring) + assert isinstance(self.password, basestring) + + if not self.is_on_page(LoginPage): + self.location(self.absurl('/myca/logon/emea/action?request_type=LogonHandler&DestPage=https%3A%2F%2Fglobal.americanexpress.com%2Fmyca%2Fintl%2Facctsumm%2Femea%2FaccountSummary.do%3Frequest_type%3D%26Face%3Dfr_FR%26intlink%3Dtopnavvotrecompteneligne-HPmyca&Face=fr_FR&Info=CUExpired'), no_login=True) + + self.page.login(self.username, self.password) + + if not self.is_logged(): + raise BrowserIncorrectPassword() + + def go_on_accounts_list(self): + self.select_form(name='leftnav') + self.form.action = self.absurl('/myca/intl/acctsumm/emea/accountSummary.do') + self.submit() + + def get_accounts_list(self): + if not self.is_on_page(AccountsPage): + self.go_on_accounts_list() + return self.page.get_list() + + def get_account(self, id): + assert isinstance(id, basestring) + + l = self.get_accounts_list() + for a in l: + if a.id == id: + return a + + return None + + def get_history(self, account): + if not self.is_on_page(AccountsPage): + self.go_on_accounts_list() + + url = account._link + + coming = True + date_guesser = LinearDateGuesser() + + while url is not None: + self.select_form(name='leftnav') + self.form.action = self.absurl(url) + self.submit() + + assert self.is_on_page(TransactionsPage) + + for tr in self.page.get_history(date_guesser): + if tr.amount > 0: + coming = False + tr._is_coming = coming + yield tr + + if self.page.is_last(): + url = None + else: + v = urlsplit(url) + args = dict(parse_qsl(v.query)) + args['BPIndex'] = int(args['BPIndex']) + 1 + url = self.buildurl(v.path, **args) diff --git a/modules/americanexpress/favicon.png b/modules/americanexpress/favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..9409b4bdeece46abaa9d419a6615288e0fec794f GIT binary patch literal 3730 zcmV;D4sG#?P)3S4_M=yx1tgZQ5zG`P*I36 zK{N(KG=fT$_+a9TR3i}+8{;KhrSt}Z)&PldI_)r>j#Jv1|Lo<%IeYK5)?Vk#_`l5j zXPU z(po&9+xsvB0nPOxw7(TXdmpstF&KNAvp3jFk6I}NN@;-^ z0r4JittXxn0;lJwT=Vq=^b{4Z<&HIRkHF18b!ZVJx|BdY!Y`9L zxi#VF-zfmLsVQ1Ud@!~#AXyHpV{BZ8#owTOUt5DGe3J$KU$aKXIpW;7lTm;WsPRES z3xy!8q!lb=qf*uoMEnRj;-_<@!i!WX?%N)#nL$8D5VQfWEA2T}5O+BXL(MKjq$TmN z9FT&Yb_*7!F9dqHqlkjbc)OjD9OD zq=t+G+4r+$WhJq*`(;WEZD0ksm8I(RZ=2)2EFkl|dO#cKI78HS2kT`1CXz>`ffZIr z8AHSd9SlpCsa2qCpl{qfLZgJL*Tg+jchX`CbmAen4SXqzqs8pWmklm3*QOw-(q+yn zLAn8pAe*cQ235AAk@nikO<~+$%qV~Z7Q*M9m!=-C{r!-C z(^@$TOQ<@{-1B$Hau#J(#@s+nun{N+v6^%6d#B4Ghg@dsq$ zHfcF3FQE0MMy9rqG1XUJxf*O*5`nEKbp;g80Z_O|5UDVw*R13^WaOBPI!1jP(^W07 z0`!;Cn1Q&#L|azoE{Kj)2x(=hU}k+inSU#wPeY|yx)FzuUtq=KD1?zU`WacMX}QO zs_=P3eB?3}j#aP%K%oe%ZC*E))D_do$U`g@z&?ZH3m-YScsyr-;F1>5_&^0I81@GW z{(3+uUSBI{++Ir=uy*1W6AP;4_&r%`BmFQLe?`H;shx!J?X-hlDGRr@XBbE{<30O86DfDc~Uoc^0vmvHd) zW{WS5ZT%pv zU$ag*X*Z-67@-AB zfG%WgWKX+|T(HR+fE;j8u_!&`%AT!xx9>Z+gzGjC7M22jd29*Q0Ql_N#;{{kh2Orm zlzcWRpbe0m=&BqKZ1+Yh%5jB|QDIx!d17fd_A?VsAS|M;1K|5IkMI(^{doL zm22X`u9+l^R|C8O4~HeDxyo<2SV0PMV|%5^t#4^XA8AaZwz zWJRW?8 z2Q_<^Vb)AX$8vZOb$oIKlLlb|Ussw)wLmSYWkyrhN^((APo|uegeA&WwdMHO02t zv}un~Qrle}{!VNlZdB<;0~8$19T!rT;8EGh3ZMx|`8}xgC?IsJbSaF7@YKUoNq4z` zuLA+bV^w;XAgebCQYg{Smh8CyJ!^~m-nxGlpWeC(H*P);-+E>af1e#<%eoPKb>}4h z`tP&&{`2$r#fLVf^Uo~RceJap(Zj-Ltm)lHk#+H0qWc=+%<{&{K$5Y&47?0R$x zmrab|=Bp?0=E(}@jS$`#GN#YexZ_W+z(VdX9r>LUuLsV`=+pc z(-^+~uJ!oBjsf=UpHcq4Yu}Xe^E+;s#KmhzvG>m>@zU%PE}t01N8h>{4B+(X5 z8z^8!UrBVv20GvJhgoDrHtTP{a1>kCjp69YMQq(ThU+gH#Zyy@s_K07rp-9BRO8_E zSv+*;w9;x6aORC6esZ{3li6U`zA2Rzo||68f&DYsFh0QhE*ZxswoYKr_6zW-$4}tE z^ddgLe+C=J2Y9bo+x_?n3=-~U7VWKeSWyuoVsfKBR~A6BbX`?3c3-sy$7UDt%_nDZ z?9?J|xn=_Ak7$JV;eAv1*kdPf&l9KcpM{_-VAsAWd};p)jE@lZ-nb6qBW41{)c<>T zPU4Em3a8H1ICNqW0I)C&AOLsnoWvFDD$Jg%ap+{Tc4inN4&b6>wG1rVbwF9b!yj0m z{{Q&zPGa{}Yq4gGaPQOe7&7pqL-V-z-RtnVZ4>xT>wC*(sm3j8{NVZ1`0~zmxOLkE z9yoAXSx-{rqf;0m;Px9PF)=p4kQpz|oWsu!&x3(x?fP~tG;2RS(yY;4Pc3lm5|0uM zW?)mZ9zSIPm+K;1#B2QNEAAUem7s__An^*6m>TaFpr4 zrYY+P(J364gosxfSGK)vh4KcKn-({_#HLKAimd3_akqFe4_k~RNhmC@l^kHlSd*+d zs-mkCoM|gN)?&)m+L_nua!j@r8|$u)<@8eRZ+0}+0EL2`d2ba1SU*@hRW8TKp2|l}PU5Pp--PjaS6?^Sy)1%KLCqay%m>#~EzawwL zq_4j3dB4EBNVdNuN#a>g5|~VbBoQKtoe@KhcIrb;c?Mm}+u=jyucm3q9)-;g$4}Z< zyUJOYA|uDr;6F#Df|+VikTb#}sqWVQgBdC_8i=MwHnNM68+XkL$;MrJB0oBUUJ)O- zjRAR>!4^H71s1dYc6v+6ZG4fv8ALMHVFIMv%cK7BAaz7uuW%tJkV=SK6iRleMF1no5K~7%L^cMnnAMFh zHPzyUr}#JRtVC5#AC7x|5G(2FJ3YQ$j=$$QysnarQ&l;*6hx~FDLBfAsw}@}N4tu4 z>H|0(^NQ3nw@fH2NOgaSA&jyH`Yo$w_)w3+a*o_4TJv)6Q&GpdNrx(kvhi8j*rfg! z*Vq}2vyq?8b5*uQmEOGx9e~5x1u{Ebz}mi@gRat;I(C2gCXZV_Cm_U5JwU)9r$0Fk zS87o09t)zvXq2hv9ZWW0kc=N2vn@apdfs5^YK2>f%=(!Jc@l$FZ(pVla#9t@pQtA< w=$JHEaDU!px-;Ow@GC@SpDFfAZVMm(2T2f~eKk). + + +import datetime +from decimal import Decimal +import re + +from weboob.tools.browser import BasePage, BrokenPageError +from weboob.capabilities.bank import Account +from weboob.tools.capabilities.bank.transactions import FrenchTransaction as Transaction + + +__all__ = ['LoginPage', 'AccountsPage', 'TransactionsPage'] + + +class LoginPage(BasePage): + def login(self, username, password): + self.browser.select_form(name='ssoform') + self.browser.set_all_readonly(False) + self.browser['UserID'] = username.encode(self.browser.ENCODING) + self.browser['USERID'] = username.encode(self.browser.ENCODING) + self.browser['Password'] = password.encode(self.browser.ENCODING) + self.browser['PWD'] = password.encode(self.browser.ENCODING) + self.browser.submit(nologin=True) + + + +class AccountsPage(BasePage): + def get_list(self): + for box in self.document.getroot().cssselect('div.roundedBox div.contentBox'): + a = Account() + a.id = self.parser.tocleanstring(box.xpath('.//tr[@id="summaryImageHeaderRow"]//div[@class="summaryTitles"]')[0]) + a.label = self.parser.tocleanstring(box.xpath('.//span[@class="cardTitle"]')[0]) + a.balance = Decimal('0.0') + coming = self.parser.tocleanstring(self.parser.select(box, 'td#colOSBalance div.summaryValues', 1)) + a.coming = Decimal(Transaction.clean_amount(coming)) + a.currency = a.get_currency(coming) + a._link = self.parser.select(box, 'div.summaryTitles a.summaryLink', 1).attrib['href'] + + yield a + +class TransactionsPage(BasePage): + COL_ID = 0 + COL_DATE = 1 + COL_DEBIT_DATE = 2 + COL_LABEL = 3 + COL_VALUE = -1 + + + def is_last(self): + current = False + for option in self.document.xpath('//select[@id="viewPeriod"]/option'): + if 'selected' in option.attrib: + current = True + elif current: + return False + + return True + + def get_debit_date(self): + for option in self.document.xpath('//select[@id="viewPeriod"]/option'): + if 'selected' in option.attrib: + m = re.search('(\d+) ([\w\.]+) (\d{4})$', option.text.strip()) + if m: + return datetime.date(int(m.group(3)), + self.MONTHS.index(m.group(2).rstrip('.')) + 1, + int(m.group(1))) + + COL_DATE = 0 + COL_TEXT = 1 + COL_CREDIT = -2 + COL_DEBIT = -1 + + MONTHS = ['janv', u'févr', u'mars', u'avri', u'mai', u'juin', u'juil', u'août', u'sept', u'oct', u'nov', u'déc'] + + def get_history(self, guesser): + debit_date = self.get_debit_date() + if debit_date is not None: + guesser.current_date = debit_date + + for tr in reversed(self.document.xpath('//div[@id="txnsSection"]//tr[@class="tableStandardText"]')): + cols = tr.findall('td') + + t = Transaction(tr.attrib['id']) + + day, month = self.parser.tocleanstring(cols[self.COL_DATE]).split(' ', 1) + day = int(day) + month = self.MONTHS.index(month.rstrip('.')) + 1 + date = guesser.guess_date(day, month) + + try: + detail = self.parser.select(cols[self.COL_TEXT], 'div.hiddenROC', 1) + except BrokenPageError: + pass + else: + detail.drop_tree() + + raw = (' '.join([txt.strip() for txt in cols[self.COL_TEXT].itertext()])).strip() + credit = self.parser.tocleanstring(cols[self.COL_CREDIT]) + debit = self.parser.tocleanstring(cols[self.COL_DEBIT]) + + t.date = date + t.rdate = date + t.raw = re.sub(r'[ ]+', ' ', raw) + t.label = re.sub('(.*?)( \d+)? .*', r'\1', raw).strip() + t.set_amount(credit, debit) + if t.amount > 0: + t.type = t.TYPE_ORDER + else: + t.type = t.TYPE_CARD + + yield t diff --git a/modules/americanexpress/test.py b/modules/americanexpress/test.py new file mode 100644 index 00000000..1bf927e0 --- /dev/null +++ b/modules/americanexpress/test.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2013 Romain Bignon +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.tools.test import BackendTest + + +class AmericanExpressTest(BackendTest): + BACKEND = 'americanexpress' + + def test_americanexpress(self): + l = list(self.backend.iter_accounts()) + + a = l[0] + list(self.backend.iter_history(a)) + list(self.backend.iter_coming(a))