First implementation of leclercmobile

Support login and history operations

modules/leclercmobile/__init__.py

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2012 Florent Fourcot
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+
+from .backend import LeclercMobileBackend
+
+__all__ = ['LeclercMobileBackend']

modules/leclercmobile/backend.py

@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2012 Florent Fourcot
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import with_statement
+
+from weboob.capabilities.bill import ICapBill, SubscriptionNotFound, BillNotFound, Subscription, Bill
+from weboob.tools.backend import BaseBackend, BackendConfig
+from weboob.tools.value import ValueBackendPassword
+
+from .browser import Leclercmobile
+
+
+__all__ = ['LeclercMobileBackend']
+
+
+class LeclercMobileBackend(BaseBackend, ICapBill):
+    NAME = 'leclercmobile'
+    MAINTAINER = 'Florent Fourcot'
+    EMAIL = 'weboob@flo.fourcot.fr'
+    VERSION = '0.d'
+    LICENSE = 'AGPLv3+'
+    DESCRIPTION = 'Leclerc Mobile website'
+    CONFIG = BackendConfig(ValueBackendPassword('login',
+                                                label='Account ID',
+                                                masked=False,
+                                                regexp='^(\d{10}|)$'),
+                           ValueBackendPassword('password',
+                                                label='Password')
+                          )
+    BROWSER = Leclercmobile
+
+    def create_default_browser(self):
+        return self.create_browser(self.config['login'].get(),
+                                   self.config['password'].get())
+
+    def iter_subscription(self):
+        for subscription in self.browser.get_subscription_list():
+            yield subscription
+
+    def get_subscription(self, _id):
+        if not _id.isdigit():
+            raise SubscriptionNotFound()
+        with self.browser:
+            subscription = self.browser.get_subscription(_id)
+        if subscription:
+            return subscription
+        else:
+            raise SubscriptionNotFound()
+
+    def iter_history(self, subscription):
+        with self.browser:
+            for history in self.browser.get_history():
+                yield history
+
+    def get_bill(self, id):
+        with self.browser:
+            bill = self.browser.get_bill(id)
+        if bill:
+            return bill
+        else:
+            raise BillNotFound()
+
+    def iter_bills(self, subscription):
+        if not isinstance(subscription, Subscription):
+            subscription = self.get_subscription(subscription)
+
+        with self.browser:
+            for bill in self.browser.iter_bills(subscription.id):
+                yield bill
+
+    # The subscription is actually useless, but maybe for the futur...
+    def get_details(self, subscription):
+        with self.browser:
+            for detail in self.browser.get_details():
+                yield detail
+
+    def download_bill(self, bill):
+        if not isinstance(bill, Bill):
+            bill = self.get_bill(bill)
+
+        with self.browser:
+            return self.browser.readurl(bill._url)

modules/leclercmobile/browser.py

@@ -0,0 +1,130 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2012  Fourcot Florent
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+import time
+import StringIO
+
+from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword
+from .pages import HomePage, LoginPage, HistoryPage, PdfPage
+
+__all__ = ['Leclercmobile']
+
+
+class Leclercmobile(BaseBrowser):
+    DOMAIN = 'www.securelmobile.fr'
+    PROTOCOL = 'https'
+    ENCODING = 'utf-8'
+    PAGES = {'.*pgeWERL008_Login.aspx.*':       LoginPage,
+             '.*EspaceClient/pgeWERL013_Accueil.aspx':     HomePage,
+             '.*pgeWERL009_ReleveConso.aspx.*': HistoryPage,
+             '.*ReleveConso.ashx.*': PdfPage
+            }
+    accueil = "/EspaceClient/pgeWERL013_Accueil.aspx"
+    login = "/EspaceClient/pgeWERL008_Login.aspx"
+    conso = "/EspaceClient/pgeWERL009_ReleveConso.aspx"
+
+    def __init__(self, *args, **kwargs):
+        BaseBrowser.__init__(self, *args, **kwargs)
+
+    def home(self):
+        self.location(self.accueil)
+
+    def is_logged(self):
+        return not self.is_on_page(LoginPage)
+
+    def login(self):
+        assert isinstance(self.username, basestring)
+        assert isinstance(self.password, basestring)
+        assert self.username.isdigit()
+
+        if not self.is_on_page(LoginPage):
+            self.location(self.login)
+
+        form = self.page.login(self.username, self.password)
+
+        # Site display a javascript popup to wait
+        while self.page.iswait():
+            # In this popup can be an error displayed
+            if self.page.iserror():
+                raise BrowserIncorrectPassword()
+            time.sleep(1)
+            self.page.next(self.username, form)
+
+        # The last document contain a redirect url in the javascript
+        self.location(self.page.getredirect())
+
+        if self.is_on_page(LoginPage):
+            raise BrowserIncorrectPassword()
+
+    def viewing_html(self):
+        # To prevent unknown mimetypes sent by server, we assume we
+        # are always on a HTML document.
+        return True
+
+    def get_subscription_list(self):
+        if not self.is_on_page(HomePage):
+            self.location(self.acceuil)
+
+        return self.page.get_list()
+
+    def get_subscription(self, id):
+        assert isinstance(id, basestring)
+
+        if not self.is_on_page(HomePage):
+            self.location(self.accueil)
+
+        l = self.page.get_list()
+        for a in l:
+            if a.id == id:
+                return a
+
+        return None
+
+    def get_history(self):
+        if not self.is_on_page(HistoryPage):
+            self.location(self.conso)
+        maxid = self.page.getmaxid()
+
+        for i in range(maxid + 1):
+            response = self.openurl('/EspaceClient/pgeWERL015_RecupReleveConso.aspx?m=-' + str(i))
+            mimetype = response.info().get('Content-Type', '').split(';')[0]
+            if mimetype == "application/pdf":
+                pdf = PdfPage(StringIO.StringIO(response.read()))
+                for call in pdf.get_calls():
+                    yield call
+
+    def get_details(self):
+        if not self.is_on_page(HistoryPage):
+            self.location(self.conso)
+        return self.page.get_details()
+
+    def iter_bills(self, parentid):
+        if not self.is_on_page(HistoryPage):
+            self.location(self.conso)
+        return self.page.date_bills()
+
+    def get_bill(self, id):
+        assert isinstance(id, basestring)
+
+        if not self.is_on_page(HistoryPage):
+            self.location(self.conso)
+        l = self.page.date_bills()
+        for a in l:
+            if a.id == id:
+                return a

modules/leclercmobile/pages/__init__.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2012  Florent Fourcot
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+
+from .homepage import HomePage
+from .history import HistoryPage, PdfPage
+from .login import LoginPage
+
+__all__ = ['LoginPage', 'HomePage', 'HistoryPage', 'PdfPage']

modules/leclercmobile/pages/history.py

@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2012 Florent Fourcot
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+
+import subprocess
+import tempfile
+import shutil
+
+from datetime import datetime, date, time
+from decimal import Decimal
+
+from weboob.tools.browser import BasePage
+from weboob.capabilities.bill import Detail
+
+
+__all__ = ['HistoryPage', 'PdfPage']
+
+
+def _get_date(detail):
+    return detail.datetime
+
+
+class PdfPage():
+    def __init__(self, file):
+        self.pdf = file
+
+    # Standard pdf text extractor take text line by line
+    # But the position in the file is not always the "real" position to display...
+    # It produce some unsorted and unparsable data
+    # Example of bad software: pdfminer and others python tools
+    # This is why we have to use "ebook-convert" from calibre software,
+    # it is the only one to 'reflow" text and give some relevant results
+    # The bad new is that ebook-convert doesn't support simple use with stdin/stdout
+    def get_calls(self):
+        pdffile = tempfile.NamedTemporaryFile(bufsize=100000, mode='w', suffix='.pdf')
+        temptxt = pdffile.name.replace('.pdf', '.txt')
+        cmd = "ebook-convert"
+        stdout = open("/dev/null", "w")
+        shutil.copyfileobj(self.pdf, pdffile)
+        pdffile.flush()
+        subprocess.call([cmd, pdffile.name, temptxt], stdout=stdout)
+        pdffile.close()
+        txtfile = open(temptxt, 'r')
+        txt = txtfile.read()
+        pages = txt.split("DEBIT (€)")
+        pages.pop(0)  # remove headers
+        details = []
+        for page in pages:
+            page = page.split('RÉGLO MOBILE')[0].split('N.B. Prévoir')[0]  # remove footers
+            lines = page.split('\n')
+            lines = [x for x in lines if len(x) > 0]  # Remove empty lines
+            numitems = (len(lines) + 1) / 5  # Each line has five columns
+            for i in range(numitems):
+                nature = i * 5
+                dateop = nature + 1
+                corres = dateop + 1
+                duree = corres + 1
+                price = duree + 1
+
+                detail = Detail()
+                mydate = date(*reversed([int(x) for x in lines[dateop].split(' ')[0].split("/")]))
+                mytime = time(*[int(x) for x in lines[dateop].split(' ')[1].split(":")])
+                detail.datetime = datetime.combine(mydate, mytime)
+                if lines[corres] == '-':
+                    lines[corres] = ""
+                if lines[duree] == '-':
+                    lines[duree] = ''
+                detail.label = unicode(lines[nature], encoding='utf-8', errors='replace') + u" " + lines[corres] + u" " + lines[duree]
+                # Special case with only 4 columns, we insert a price
+                if "Activation de votre ligne" in detail.label:
+                    lines.insert(price, '0')
+                try:
+                    detail.price = Decimal(lines[price].replace(',', '.'))
+                except:
+                    detail.price = Decimal(0)
+
+                details.append(detail)
+        return sorted(details, key=_get_date, reverse=True)
+
+
+class HistoryPage(BasePage):
+    def on_loaded(self):
+        pass
+
+    def getmaxid(self):
+        max = 1
+        while len(self.document.xpath('//li[@id="liMois%s"]' % max)) > 0:
+            max += 1
+        return max - 1

modules/leclercmobile/pages/homepage.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2012  Florent Fourcot
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+from weboob.capabilities.bill import Subscription
+from weboob.tools.browser import BasePage
+
+
+__all__ = ['HomePage']
+
+
+class HomePage(BasePage):
+    def on_loaded(self):
+        pass
+
+    def get_list(self):
+        l = []
+        phone = unicode(self.document.xpath('//span[@id="ctl00_ctl00_cMain_cEspCli_lblMsIsdn"]')[0].text.replace(' ', ''))
+        self.browser.logger.debug('Found ' + phone + ' has phone number')
+        phoneplan = unicode(self.document.xpath('//span[@id="ctl00_ctl00_cMain_cEspCli_lblOffre"]')[0].text)
+        self.browser.logger.debug('Found ' + phoneplan + ' has subscription type')
+
+        subscription = Subscription(phone)
+        subscription.label = phone + ' - ' + phoneplan
+
+        l.append(subscription)
+
+        return l

modules/leclercmobile/pages/login.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2012  Florent Fourcot
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+import StringIO
+from weboob.tools.browser import BasePage
+from weboob.tools.mech import ClientForm
+
+__all__ = ['LoginPage']
+
+
+class LoginPage(BasePage):
+    def on_loaded(self):
+        pass
+
+    def login(self, login, password):
+        form = list(self.browser.forms())[0]
+        self.browser.select_form("aspnetForm")
+        self.browser.set_all_readonly(False)
+        self.browser.controls.append(ClientForm.TextControl('text', '__ASYNCPOST', {'value': "true"}))
+        self.browser['__EVENTTARGET'] = "ctl00$cMain$lnkValider"
+        self.browser['ctl00$cMain$ascSaisieMsIsdn$txtMsIsdn'] = login
+        self.browser['ctl00$cMain$txtMdp'] = password
+        self.browser.submit(nologin=True)
+        return form
+
+    def iswait(self):
+        spanwait = self.document.xpath('//span[@id="ctl00_ascAttente_timerAttente"]')
+        return len(spanwait) > 0
+
+    def iserror(self):
+        error = self.document.xpath('//span[@id="ctl00_cMain_ascLibErreur_lblErreur"]')
+        return len(error) > 0
+
+    def getredirect(self):
+        string = StringIO.StringIO()
+        self.document.write(string)
+        try:
+            redirect = string.getvalue().split('pageRedirect')[1].split('|')[2]
+        except:
+            redirect = ''
+        return redirect
+
+    def next(self, login, form):
+        self.browser.form = form
+        string = StringIO.StringIO()
+        self.document.write(string)
+        controlvalue = string.getvalue().split('__EVENTVALIDATION')[1].split('|')[1]
+        state = string.getvalue().split('__VIEWSTATE')[1].split('|')[1]
+        self.browser.controls.append(ClientForm.TextControl('text', 'ctl00$objScriptManager', {'value': "ctl00$ascAttente$panelAttente|ctl00$ascAttente$timerAttente"}))
+        self.browser['__VIEWSTATE'] = state
+        self.browser['__EVENTTARGET'] = "ctl00$ascAttente$timerAttente"
+        self.browser['__EVENTVALIDATION'] = controlvalue
+        self.browser['ctl00$cMain$ascSaisieMsIsdn$txtMsIsdn'] = login
+        self.browser['ctl00$cMain$txtMdp'] = ""
+        self.browser.submit(nologin=True)

modules/leclercmobile/test.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2012  Fourcot Florent
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+
+from weboob.tools.test import BackendTest
+
+
+__all__ = ['LeclercMobileTest']
+
+
+class LeclercMobileTest(BackendTest):
+    BACKEND = 'leclercmobile'
+
+    def test_leclercmobile(self):
+        for subscription in self.backend.iter_subscription():
+            list(self.backend.iter_history(subscription.id))
+            for bill in self.backend.iter_bills(subscription.id):
+                self.backend.download_bill(bill.id)