From aa693d61060ec641f6bde784b1569858a5f0fb7a Mon Sep 17 00:00:00 2001 From: Florent Date: Wed, 29 Aug 2012 13:30:12 +0200 Subject: [PATCH] First implementation of leclercmobile Support login and history operations --- modules/leclercmobile/__init__.py | 23 +++++ modules/leclercmobile/backend.py | 98 ++++++++++++++++++ modules/leclercmobile/browser.py | 130 ++++++++++++++++++++++++ modules/leclercmobile/pages/__init__.py | 25 +++++ modules/leclercmobile/pages/history.py | 105 +++++++++++++++++++ modules/leclercmobile/pages/homepage.py | 43 ++++++++ modules/leclercmobile/pages/login.py | 71 +++++++++++++ modules/leclercmobile/test.py | 34 +++++++ 8 files changed, 529 insertions(+) create mode 100644 modules/leclercmobile/__init__.py create mode 100644 modules/leclercmobile/backend.py create mode 100644 modules/leclercmobile/browser.py create mode 100644 modules/leclercmobile/pages/__init__.py create mode 100644 modules/leclercmobile/pages/history.py create mode 100644 modules/leclercmobile/pages/homepage.py create mode 100644 modules/leclercmobile/pages/login.py create mode 100644 modules/leclercmobile/test.py diff --git a/modules/leclercmobile/__init__.py b/modules/leclercmobile/__init__.py new file mode 100644 index 00000000..1138462d --- /dev/null +++ b/modules/leclercmobile/__init__.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012 Florent Fourcot +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from .backend import LeclercMobileBackend + +__all__ = ['LeclercMobileBackend'] diff --git a/modules/leclercmobile/backend.py b/modules/leclercmobile/backend.py new file mode 100644 index 00000000..dafe364b --- /dev/null +++ b/modules/leclercmobile/backend.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012 Florent Fourcot +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import with_statement + +from weboob.capabilities.bill import ICapBill, SubscriptionNotFound, BillNotFound, Subscription, Bill +from weboob.tools.backend import BaseBackend, BackendConfig +from weboob.tools.value import ValueBackendPassword + +from .browser import Leclercmobile + + +__all__ = ['LeclercMobileBackend'] + + +class LeclercMobileBackend(BaseBackend, ICapBill): + NAME = 'leclercmobile' + MAINTAINER = 'Florent Fourcot' + EMAIL = 'weboob@flo.fourcot.fr' + VERSION = '0.d' + LICENSE = 'AGPLv3+' + DESCRIPTION = 'Leclerc Mobile website' + CONFIG = BackendConfig(ValueBackendPassword('login', + label='Account ID', + masked=False, + regexp='^(\d{10}|)$'), + ValueBackendPassword('password', + label='Password') + ) + BROWSER = Leclercmobile + + def create_default_browser(self): + return self.create_browser(self.config['login'].get(), + self.config['password'].get()) + + def iter_subscription(self): + for subscription in self.browser.get_subscription_list(): + yield subscription + + def get_subscription(self, _id): + if not _id.isdigit(): + raise SubscriptionNotFound() + with self.browser: + subscription = self.browser.get_subscription(_id) + if subscription: + return subscription + else: + raise SubscriptionNotFound() + + def iter_history(self, subscription): + with self.browser: + for history in self.browser.get_history(): + yield history + + def get_bill(self, id): + with self.browser: + bill = self.browser.get_bill(id) + if bill: + return bill + else: + raise BillNotFound() + + def iter_bills(self, subscription): + if not isinstance(subscription, Subscription): + subscription = self.get_subscription(subscription) + + with self.browser: + for bill in self.browser.iter_bills(subscription.id): + yield bill + + # The subscription is actually useless, but maybe for the futur... + def get_details(self, subscription): + with self.browser: + for detail in self.browser.get_details(): + yield detail + + def download_bill(self, bill): + if not isinstance(bill, Bill): + bill = self.get_bill(bill) + + with self.browser: + return self.browser.readurl(bill._url) diff --git a/modules/leclercmobile/browser.py b/modules/leclercmobile/browser.py new file mode 100644 index 00000000..b43f4a13 --- /dev/null +++ b/modules/leclercmobile/browser.py @@ -0,0 +1,130 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012 Fourcot Florent +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +import time +import StringIO + +from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword +from .pages import HomePage, LoginPage, HistoryPage, PdfPage + +__all__ = ['Leclercmobile'] + + +class Leclercmobile(BaseBrowser): + DOMAIN = 'www.securelmobile.fr' + PROTOCOL = 'https' + ENCODING = 'utf-8' + PAGES = {'.*pgeWERL008_Login.aspx.*': LoginPage, + '.*EspaceClient/pgeWERL013_Accueil.aspx': HomePage, + '.*pgeWERL009_ReleveConso.aspx.*': HistoryPage, + '.*ReleveConso.ashx.*': PdfPage + } + accueil = "/EspaceClient/pgeWERL013_Accueil.aspx" + login = "/EspaceClient/pgeWERL008_Login.aspx" + conso = "/EspaceClient/pgeWERL009_ReleveConso.aspx" + + def __init__(self, *args, **kwargs): + BaseBrowser.__init__(self, *args, **kwargs) + + def home(self): + self.location(self.accueil) + + def is_logged(self): + return not self.is_on_page(LoginPage) + + def login(self): + assert isinstance(self.username, basestring) + assert isinstance(self.password, basestring) + assert self.username.isdigit() + + if not self.is_on_page(LoginPage): + self.location(self.login) + + form = self.page.login(self.username, self.password) + + # Site display a javascript popup to wait + while self.page.iswait(): + # In this popup can be an error displayed + if self.page.iserror(): + raise BrowserIncorrectPassword() + time.sleep(1) + self.page.next(self.username, form) + + # The last document contain a redirect url in the javascript + self.location(self.page.getredirect()) + + if self.is_on_page(LoginPage): + raise BrowserIncorrectPassword() + + def viewing_html(self): + # To prevent unknown mimetypes sent by server, we assume we + # are always on a HTML document. + return True + + def get_subscription_list(self): + if not self.is_on_page(HomePage): + self.location(self.acceuil) + + return self.page.get_list() + + def get_subscription(self, id): + assert isinstance(id, basestring) + + if not self.is_on_page(HomePage): + self.location(self.accueil) + + l = self.page.get_list() + for a in l: + if a.id == id: + return a + + return None + + def get_history(self): + if not self.is_on_page(HistoryPage): + self.location(self.conso) + maxid = self.page.getmaxid() + + for i in range(maxid + 1): + response = self.openurl('/EspaceClient/pgeWERL015_RecupReleveConso.aspx?m=-' + str(i)) + mimetype = response.info().get('Content-Type', '').split(';')[0] + if mimetype == "application/pdf": + pdf = PdfPage(StringIO.StringIO(response.read())) + for call in pdf.get_calls(): + yield call + + def get_details(self): + if not self.is_on_page(HistoryPage): + self.location(self.conso) + return self.page.get_details() + + def iter_bills(self, parentid): + if not self.is_on_page(HistoryPage): + self.location(self.conso) + return self.page.date_bills() + + def get_bill(self, id): + assert isinstance(id, basestring) + + if not self.is_on_page(HistoryPage): + self.location(self.conso) + l = self.page.date_bills() + for a in l: + if a.id == id: + return a diff --git a/modules/leclercmobile/pages/__init__.py b/modules/leclercmobile/pages/__init__.py new file mode 100644 index 00000000..05e6c00e --- /dev/null +++ b/modules/leclercmobile/pages/__init__.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012 Florent Fourcot +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from .homepage import HomePage +from .history import HistoryPage, PdfPage +from .login import LoginPage + +__all__ = ['LoginPage', 'HomePage', 'HistoryPage', 'PdfPage'] diff --git a/modules/leclercmobile/pages/history.py b/modules/leclercmobile/pages/history.py new file mode 100644 index 00000000..ad237130 --- /dev/null +++ b/modules/leclercmobile/pages/history.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012 Florent Fourcot +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +import subprocess +import tempfile +import shutil + +from datetime import datetime, date, time +from decimal import Decimal + +from weboob.tools.browser import BasePage +from weboob.capabilities.bill import Detail + + +__all__ = ['HistoryPage', 'PdfPage'] + + +def _get_date(detail): + return detail.datetime + + +class PdfPage(): + def __init__(self, file): + self.pdf = file + + # Standard pdf text extractor take text line by line + # But the position in the file is not always the "real" position to display... + # It produce some unsorted and unparsable data + # Example of bad software: pdfminer and others python tools + # This is why we have to use "ebook-convert" from calibre software, + # it is the only one to 'reflow" text and give some relevant results + # The bad new is that ebook-convert doesn't support simple use with stdin/stdout + def get_calls(self): + pdffile = tempfile.NamedTemporaryFile(bufsize=100000, mode='w', suffix='.pdf') + temptxt = pdffile.name.replace('.pdf', '.txt') + cmd = "ebook-convert" + stdout = open("/dev/null", "w") + shutil.copyfileobj(self.pdf, pdffile) + pdffile.flush() + subprocess.call([cmd, pdffile.name, temptxt], stdout=stdout) + pdffile.close() + txtfile = open(temptxt, 'r') + txt = txtfile.read() + pages = txt.split("DEBIT (€)") + pages.pop(0) # remove headers + details = [] + for page in pages: + page = page.split('RÉGLO MOBILE')[0].split('N.B. Prévoir')[0] # remove footers + lines = page.split('\n') + lines = [x for x in lines if len(x) > 0] # Remove empty lines + numitems = (len(lines) + 1) / 5 # Each line has five columns + for i in range(numitems): + nature = i * 5 + dateop = nature + 1 + corres = dateop + 1 + duree = corres + 1 + price = duree + 1 + + detail = Detail() + mydate = date(*reversed([int(x) for x in lines[dateop].split(' ')[0].split("/")])) + mytime = time(*[int(x) for x in lines[dateop].split(' ')[1].split(":")]) + detail.datetime = datetime.combine(mydate, mytime) + if lines[corres] == '-': + lines[corres] = "" + if lines[duree] == '-': + lines[duree] = '' + detail.label = unicode(lines[nature], encoding='utf-8', errors='replace') + u" " + lines[corres] + u" " + lines[duree] + # Special case with only 4 columns, we insert a price + if "Activation de votre ligne" in detail.label: + lines.insert(price, '0') + try: + detail.price = Decimal(lines[price].replace(',', '.')) + except: + detail.price = Decimal(0) + + details.append(detail) + return sorted(details, key=_get_date, reverse=True) + + +class HistoryPage(BasePage): + def on_loaded(self): + pass + + def getmaxid(self): + max = 1 + while len(self.document.xpath('//li[@id="liMois%s"]' % max)) > 0: + max += 1 + return max - 1 diff --git a/modules/leclercmobile/pages/homepage.py b/modules/leclercmobile/pages/homepage.py new file mode 100644 index 00000000..5662ddc8 --- /dev/null +++ b/modules/leclercmobile/pages/homepage.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012 Florent Fourcot +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from weboob.capabilities.bill import Subscription +from weboob.tools.browser import BasePage + + +__all__ = ['HomePage'] + + +class HomePage(BasePage): + def on_loaded(self): + pass + + def get_list(self): + l = [] + phone = unicode(self.document.xpath('//span[@id="ctl00_ctl00_cMain_cEspCli_lblMsIsdn"]')[0].text.replace(' ', '')) + self.browser.logger.debug('Found ' + phone + ' has phone number') + phoneplan = unicode(self.document.xpath('//span[@id="ctl00_ctl00_cMain_cEspCli_lblOffre"]')[0].text) + self.browser.logger.debug('Found ' + phoneplan + ' has subscription type') + + subscription = Subscription(phone) + subscription.label = phone + ' - ' + phoneplan + + l.append(subscription) + + return l diff --git a/modules/leclercmobile/pages/login.py b/modules/leclercmobile/pages/login.py new file mode 100644 index 00000000..bb694041 --- /dev/null +++ b/modules/leclercmobile/pages/login.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012 Florent Fourcot +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +import StringIO +from weboob.tools.browser import BasePage +from weboob.tools.mech import ClientForm + +__all__ = ['LoginPage'] + + +class LoginPage(BasePage): + def on_loaded(self): + pass + + def login(self, login, password): + form = list(self.browser.forms())[0] + self.browser.select_form("aspnetForm") + self.browser.set_all_readonly(False) + self.browser.controls.append(ClientForm.TextControl('text', '__ASYNCPOST', {'value': "true"})) + self.browser['__EVENTTARGET'] = "ctl00$cMain$lnkValider" + self.browser['ctl00$cMain$ascSaisieMsIsdn$txtMsIsdn'] = login + self.browser['ctl00$cMain$txtMdp'] = password + self.browser.submit(nologin=True) + return form + + def iswait(self): + spanwait = self.document.xpath('//span[@id="ctl00_ascAttente_timerAttente"]') + return len(spanwait) > 0 + + def iserror(self): + error = self.document.xpath('//span[@id="ctl00_cMain_ascLibErreur_lblErreur"]') + return len(error) > 0 + + def getredirect(self): + string = StringIO.StringIO() + self.document.write(string) + try: + redirect = string.getvalue().split('pageRedirect')[1].split('|')[2] + except: + redirect = '' + return redirect + + def next(self, login, form): + self.browser.form = form + string = StringIO.StringIO() + self.document.write(string) + controlvalue = string.getvalue().split('__EVENTVALIDATION')[1].split('|')[1] + state = string.getvalue().split('__VIEWSTATE')[1].split('|')[1] + self.browser.controls.append(ClientForm.TextControl('text', 'ctl00$objScriptManager', {'value': "ctl00$ascAttente$panelAttente|ctl00$ascAttente$timerAttente"})) + self.browser['__VIEWSTATE'] = state + self.browser['__EVENTTARGET'] = "ctl00$ascAttente$timerAttente" + self.browser['__EVENTVALIDATION'] = controlvalue + self.browser['ctl00$cMain$ascSaisieMsIsdn$txtMsIsdn'] = login + self.browser['ctl00$cMain$txtMdp'] = "" + self.browser.submit(nologin=True) diff --git a/modules/leclercmobile/test.py b/modules/leclercmobile/test.py new file mode 100644 index 00000000..cd8af041 --- /dev/null +++ b/modules/leclercmobile/test.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012 Fourcot Florent +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.tools.test import BackendTest + + +__all__ = ['LeclercMobileTest'] + + +class LeclercMobileTest(BackendTest): + BACKEND = 'leclercmobile' + + def test_leclercmobile(self): + for subscription in self.backend.iter_subscription(): + list(self.backend.iter_history(subscription.id)) + for bill in self.backend.iter_bills(subscription.id): + self.backend.download_bill(bill.id)