diff --git a/modules/prixcarburants/__init__.py b/modules/prixcarburants/__init__.py new file mode 100644 index 00000000..196accd3 --- /dev/null +++ b/modules/prixcarburants/__init__.py @@ -0,0 +1,3 @@ +from .backend import PrixCarburantsBackend + +__all__ = ['PrixCarburantsBackend'] diff --git a/modules/prixcarburants/backend.py b/modules/prixcarburants/backend.py new file mode 100644 index 00000000..392855e3 --- /dev/null +++ b/modules/prixcarburants/backend.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012 Romain Bignon +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from weboob.tools.backend import BaseBackend, BackendConfig +from weboob.tools.value import Value +from weboob.capabilities.pricecomparison import ICapPriceComparison, Price, Product + +from .browser import PrixCarburantsBrowser + + +__all__ = ['PrixCarburantsBackend'] + + +class PrixCarburantsBackend(BaseBackend, ICapPriceComparison): + NAME = 'prixcarburants' + MAINTAINER = u'Romain Bignon' + EMAIL = 'romain@weboob.org' + VERSION = '0.c' + DESCRIPTION = 'French governement website to compare fuel prices' + LICENSE = 'AGPLv3+' + CONFIG = BackendConfig(Value('zipcode', label='Zipcode', regexp='\d+')) + BROWSER = PrixCarburantsBrowser + + def search_products(self, pattern=None): + with self.browser: + for product in self.browser.iter_products(): + if pattern is None or pattern.lower() in product.name.lower(): + yield product + + def iter_prices(self, product): + with self.browser: + return self.browser.iter_prices(self.config['zipcode'].get(), product) + + def get_price(self, id): + with self.browser: + if isinstance(id, Price): + price = id + else: + p_id, s_id = id.split('.', 2) + product = Product(p_id) + for price in self.iter_prices(product): + if price.id == id: + break + else: + return None + + price.shop.info = self.browser.get_shop_info(price.id.split('.', 2)[-1]) + return price + + def fill_price(self, price, fields): + return self.get_price(price) + + OBJECTS = {Price: fill_price, + } diff --git a/modules/prixcarburants/browser.py b/modules/prixcarburants/browser.py new file mode 100644 index 00000000..5025bc23 --- /dev/null +++ b/modules/prixcarburants/browser.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012 Romain Bignon +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +import urllib + +from weboob.tools.browser import BaseBrowser + +from .pages import IndexPage, ComparisonResultsPage, ShopInfoPage + + +__all__ = ['PrixCarburantsBrowser'] + + +class PrixCarburantsBrowser(BaseBrowser): + PROTOCOL = 'http' + DOMAIN = 'www.prix-carburants.economie.gouv.fr' + ENCODING = 'iso-8859-15' + PAGES = { + 'http://www\.prix-carburants\.economie\.gouv\.fr/index\.php': IndexPage, + 'http://www\.prix-carburants\.economie\.gouv\.fr/index\.php\?module=dbgestion\&action=search': ComparisonResultsPage, + 'http://www\.prix-carburants\.economie\.gouv\.fr/index\.php\?module=dbgestion\&action=getPopupInfo': ShopInfoPage, + } + + def iter_products(self): + if not self.is_on_page(IndexPage): + self.location('/index.php') + + assert self.is_on_page(IndexPage) + return self.page.iter_products() + + def iter_prices(self, zipcode, product): + data = {'aff_param_0_0': '', + 'aff_param_0_1': 'les points de vente', + 'aff_param_0_2': '', + 'aff_param_0_3': zipcode, + 'changeNbPerPage': 'off', + 'col*param*pdv_brand': 'Marque', + 'col*param*pdv_city': 'Commune', + 'col*param*pdv_name': 'Nom du point de vente', + 'col*param*pdv_pop': '', + 'col*param*price_fuel_%s' % product.id: 'GPL', + 'col*param*price_lmdate_%s' % product.id: 'Mise a jour GPL', + 'critere_contrainte': 'letters', + 'critere_info': 'pdv_city*0', + 'critere_txt': '', + 'flag_contrainte': 'off', + 'index_contrainte': 0, + 'modeaffichage': 'list', + 'nb_search_per_page': 100, + 'orderBy': 'price_fuel_%s' % product.id, + 'orderType': 'ASC', + 'req_param_0_0': '', + 'req_param_0_1': 'pdv_zipcode', + 'req_param_0_2': 'ILIKE', + 'req_param_0_3': '%s%%' % zipcode, + 'seeFuel': product.id, + 'thisPageLetter': 'Tous', + 'thisPageNumber': 1, + 'toDelete': -1, + } + self.location('/index.php?module=dbgestion&action=search', urllib.urlencode(data)) + + assert self.is_on_page(ComparisonResultsPage) + return self.page.iter_results(product) + + def get_shop_info(self, id): + data = {'pdv_id': id, + 'module': 'dbgestion', + 'action': 'getPopupInfo'} + self.location('/index.php?module=dbgestion&action=getPopupInfo', urllib.urlencode(data)) + + assert self.is_on_page(ShopInfoPage) + return self.page.get_info() diff --git a/modules/prixcarburants/pages.py b/modules/prixcarburants/pages.py new file mode 100644 index 00000000..b6f96d4b --- /dev/null +++ b/modules/prixcarburants/pages.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012 Romain Bignon +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +import re + +from weboob.tools.browser import BasePage +from weboob.capabilities import NotAvailable +from weboob.capabilities.pricecomparison import Product, Shop, Price + + +__all__ = ['IndexPage', 'ComparisonResultsPage', 'ShopInfoPage'] + + +class IndexPage(BasePage): + def iter_products(self): + for li in self.parser.select(self.document.getroot(), 'div#choix_carbu ul li'): + input = li.find('input') + label = li.find('label') + + product = Product(input.attrib['value']) + product.name = label.text.strip() + + if '&' in product.name: + # "E10 & SP95" produces a non-supported table. + continue + + yield product + +class ComparisonResultsPage(BasePage): + def get_product_name(self): + div = self.parser.select(self.document.getroot(), 'div#moins_plus_ariane', 1) + m = re.match('Carburant : (\w+) | .*', div.text) + return m.group(1) + + def iter_results(self, product=None): + price = None + product.name = self.get_product_name() + for tr in self.document.getroot().cssselect('table#tab_resultat tr'): + if tr.attrib.get('id', '').startswith('pdv'): + price = Price('%s.%s' % (product.id, tr.attrib['id'][3:])) + + price.product = product + + tds = tr.findall('td') + price.cost = float(tds[4].text.replace(',', '.')) + price.currency = u'€' + + shop = Shop(price.id) + shop.name = tds[2].text.strip() + shop.location = tds[0].text.strip() + + price.shop = shop + price.set_empty_fields(NotAvailable) + yield price + +class ShopInfoPage(BasePage): + def get_info(self): + return self.parser.tostring(self.parser.select(self.document.getroot(), 'div.colg', 1)) diff --git a/modules/prixcarburants/test.py b/modules/prixcarburants/test.py new file mode 100644 index 00000000..9151b7a4 --- /dev/null +++ b/modules/prixcarburants/test.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012 Romain Bignon +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.tools.test import BackendTest + + +class PrixCarburantsTest(BackendTest): + BACKEND = 'prixcarburants' + + def test_prixcarburants(self): + products = list(self.backend.search_products('gpl')) + self.assertTrue(len(products) == 1) + + prices = list(self.backend.iter_prices(products[0])) + self.backend.fillobj(prices[0])