From a39dff77a5907c63352d4d61525949ed1f882557 Mon Sep 17 00:00:00 2001 From: Bezleputh Date: Wed, 4 Mar 2015 14:21:04 +0100 Subject: [PATCH] [Element] create new element : DictElement use new DictElement in modules --- modules/explorimmo/pages.py | 10 ++-------- modules/feedly/pages.py | 11 +---------- modules/francetelevisions/pages.py | 15 +++------------ modules/ilmatieteenlaitos/pages.py | 11 +---------- modules/logicimmo/pages.py | 10 +--------- modules/meteofrance/pages.py | 12 +----------- modules/pap/pages.py | 12 +----------- modules/seloger/pages.py | 16 ++++++---------- modules/transilien/pages.py | 11 +---------- weboob/browser/elements.py | 21 +++++++++++++++++++++ 10 files changed, 38 insertions(+), 91 deletions(-) diff --git a/modules/explorimmo/pages.py b/modules/explorimmo/pages.py index 9f875268..f6e18ae3 100644 --- a/modules/explorimmo/pages.py +++ b/modules/explorimmo/pages.py @@ -21,7 +21,7 @@ import re from decimal import Decimal from datetime import datetime from weboob.browser.filters.json import Dict -from weboob.browser.elements import ItemElement, ListElement, method +from weboob.browser.elements import ItemElement, ListElement, DictElement, method from weboob.browser.pages import JsonPage, HTMLPage, pagination from weboob.browser.filters.standard import CleanText, CleanDecimal, Regexp, Env, BrowserURL, Filter, Format from weboob.browser.filters.html import CleanHTML, XPath @@ -29,16 +29,10 @@ from weboob.capabilities.base import NotAvailable, NotLoaded from weboob.capabilities.housing import Housing, HousingPhoto, City -class DictElement(ListElement): - def find_elements(self): - for el in self.el[0].get(self.item_xpath): - yield el - - class CitiesPage(JsonPage): @method class get_cities(DictElement): - item_xpath = 'locations' + item_xpath = '0/locations' class item(ItemElement): klass = City diff --git a/modules/feedly/pages.py b/modules/feedly/pages.py index f7d05296..45a14dbd 100644 --- a/modules/feedly/pages.py +++ b/modules/feedly/pages.py @@ -22,21 +22,12 @@ from datetime import datetime from weboob.capabilities.messages import Message from weboob.capabilities.collection import Collection from weboob.browser.pages import JsonPage -from weboob.browser.elements import ListElement, ItemElement, method +from weboob.browser.elements import ItemElement, DictElement, method from weboob.browser.filters.standard import CleanText, Format from weboob.browser.filters.json import Dict from weboob.browser.filters.html import CleanHTML -class DictElement(ListElement): - def find_elements(self): - if self.item_xpath is not None: - for el in self.el.get(self.item_xpath): - yield el - else: - yield self.el - - class ContentsPage(JsonPage): @method diff --git a/modules/francetelevisions/pages.py b/modules/francetelevisions/pages.py index b23d5c1e..7e9b059b 100644 --- a/modules/francetelevisions/pages.py +++ b/modules/francetelevisions/pages.py @@ -23,21 +23,12 @@ from weboob.capabilities.base import BaseObject from datetime import timedelta from weboob.browser.pages import HTMLPage, JsonPage -from weboob.browser.elements import ItemElement, ListElement, method +from weboob.browser.elements import ItemElement, ListElement, DictElement, method from weboob.browser.filters.standard import Filter, CleanText, Regexp, Format, DateTime, Env, Duration from weboob.browser.filters.html import Link, Attr from weboob.browser.filters.json import Dict -class DictElement(ListElement): - def find_elements(self): - if self.item_xpath is not None: - for el in self.el.get('reponse').get(self.item_xpath): - yield el - else: - yield self.el - - class DurationPluzz(Filter): def filter(self, el): duration = Regexp(CleanText('.'), r'.+\|(.+)')(el[0]) @@ -135,7 +126,7 @@ class VideoPage(JsonPage): class Programs(JsonPage): @method class iter_programs(DictElement): - item_xpath = 'programme' + item_xpath = 'reponse/programme' class item(ItemElement): klass = BaseObject @@ -147,7 +138,7 @@ class Programs(JsonPage): class LatestPage(JsonPage): @method class iter_videos(DictElement): - item_xpath = 'emissions' + item_xpath = 'reponse/emissions' class Item(ItemElement): klass = BaseVideo diff --git a/modules/ilmatieteenlaitos/pages.py b/modules/ilmatieteenlaitos/pages.py index bec4ff58..249162e6 100644 --- a/modules/ilmatieteenlaitos/pages.py +++ b/modules/ilmatieteenlaitos/pages.py @@ -21,21 +21,12 @@ from datetime import date from itertools import imap, ifilter from weboob.browser.pages import JsonPage, HTMLPage -from weboob.browser.elements import ItemElement, ListElement, method +from weboob.browser.elements import ItemElement, ListElement, DictElement, method from weboob.capabilities.weather import Forecast, Current, City, Temperature from weboob.browser.filters.json import Dict from weboob.browser.filters.standard import Filter, CleanText, CleanDecimal, Regexp, Format, Date -class DictElement(ListElement): - def find_elements(self): - if self.item_xpath is not None: - for el in self.el: - yield el - else: - yield self.el - - class Id(Filter): def filter(self, txt): return txt.split(", ")[0] diff --git a/modules/logicimmo/pages.py b/modules/logicimmo/pages.py index 76e6173c..7341a1ec 100644 --- a/modules/logicimmo/pages.py +++ b/modules/logicimmo/pages.py @@ -20,7 +20,7 @@ from decimal import Decimal from weboob.browser.pages import HTMLPage, JsonPage -from weboob.browser.elements import ItemElement, ListElement, method +from weboob.browser.elements import ItemElement, ListElement, DictElement, method from weboob.browser.filters.json import Dict from weboob.browser.filters.standard import Format, CleanText, Regexp, CleanDecimal, Date, Env, BrowserURL from weboob.browser.filters.html import XPath @@ -28,17 +28,9 @@ from weboob.capabilities.housing import Housing, HousingPhoto, City from weboob.capabilities.base import NotAvailable -class DictElement(ListElement): - def find_elements(self): - for el in self.el: - yield el - - class CitiesPage(JsonPage): @method class get_cities(DictElement): - item_xpath = '' - class item(ItemElement): klass = City diff --git a/modules/meteofrance/pages.py b/modules/meteofrance/pages.py index 7fad84e6..8c766f88 100644 --- a/modules/meteofrance/pages.py +++ b/modules/meteofrance/pages.py @@ -21,26 +21,16 @@ from datetime import date from weboob.browser.pages import JsonPage, HTMLPage -from weboob.browser.elements import ItemElement, ListElement, method +from weboob.browser.elements import ItemElement, ListElement, DictElement, method from weboob.capabilities.weather import Forecast, Current, City, Temperature from weboob.browser.filters.json import Dict from weboob.browser.filters.html import CleanHTML from weboob.browser.filters.standard import CleanText, CleanDecimal, Regexp, Format -class DictElement(ListElement): - def find_elements(self): - if self.item_xpath is not None: - for el in self.el: - yield el - else: - yield self.el - - class SearchCitiesPage(JsonPage): @method class iter_cities(DictElement): - item_xpath = '.' ignore_duplicate = True class item(ItemElement): diff --git a/modules/pap/pages.py b/modules/pap/pages.py index 943be750..497b7e61 100644 --- a/modules/pap/pages.py +++ b/modules/pap/pages.py @@ -22,7 +22,7 @@ from decimal import Decimal from weboob.tools.date import parse_french_date from weboob.browser.pages import HTMLPage, JsonPage, pagination -from weboob.browser.elements import ItemElement, ListElement, method +from weboob.browser.elements import ItemElement, ListElement, DictElement, method from weboob.browser.filters.standard import CleanText, CleanDecimal, Regexp, Env, BrowserURL, Format from weboob.browser.filters.html import Link, XPath, CleanHTML from weboob.browser.filters.json import Dict @@ -30,19 +30,9 @@ from weboob.capabilities.base import NotAvailable from weboob.capabilities.housing import Housing, City, HousingPhoto -class DictElement(ListElement): - def find_elements(self): - if self.item_xpath is not None: - for el in self.el: - yield el - else: - yield self.el - - class CitiesPage(JsonPage): @method class iter_cities(DictElement): - item_xpath = '.' class item(ItemElement): klass = City diff --git a/modules/seloger/pages.py b/modules/seloger/pages.py index f8141d15..664577ad 100644 --- a/modules/seloger/pages.py +++ b/modules/seloger/pages.py @@ -19,7 +19,7 @@ from weboob.browser.pages import XMLPage, JsonPage, pagination -from weboob.browser.elements import ItemElement, ListElement, method +from weboob.browser.elements import ItemElement, ListElement, DictElement, method from weboob.browser.filters.json import Dict from weboob.browser.filters.html import XPath from weboob.browser.filters.standard import CleanText, CleanDecimal, DateTime @@ -27,21 +27,17 @@ from weboob.capabilities.base import NotAvailable from weboob.capabilities.housing import Housing, HousingPhoto, City -class DictElement(ListElement): - def find_elements(self): - for el in self.el: - if el.get('label') == 'Villes': - for item in el.get('values'): - if 'value' in item: - yield item - - class CitiesPage(JsonPage): @method class iter_cities(DictElement): + item_xpath = '2/values' + class item(ItemElement): klass = City + def condition(self): + return Dict('value', default=None)(self) + obj_id = Dict('value') obj_name = Dict('label') diff --git a/modules/transilien/pages.py b/modules/transilien/pages.py index 6d4533b8..0697db45 100644 --- a/modules/transilien/pages.py +++ b/modules/transilien/pages.py @@ -21,7 +21,7 @@ import re from weboob.browser.pages import JsonPage, HTMLPage -from weboob.browser.elements import TableElement, ItemElement, ListElement, method +from weboob.browser.elements import TableElement, ItemElement, DictElement, method from weboob.capabilities.travel import Station, Departure, RoadStep from weboob.capabilities import NotAvailable from weboob.browser.filters.standard import CleanText, TableCell, Filter, DateTime, Env, Regexp, Duration @@ -30,15 +30,6 @@ from weboob.browser.filters.html import Link from weboob.tools.date import LinearDateGuesser -class DictElement(ListElement): - def find_elements(self): - if self.item_xpath is not None: - for el in self.el.get(self.item_xpath): - yield el - else: - yield self.el - - class RoadMapDuration(Duration): _regexp = re.compile(r'(?P\d?)') kwargs = {'minutes': 'mn'} diff --git a/weboob/browser/elements.py b/weboob/browser/elements.py index 2ce2e8dd..7d3f0fb9 100644 --- a/weboob/browser/elements.py +++ b/weboob/browser/elements.py @@ -307,3 +307,24 @@ class TableElement(ListElement): def get_colnum(self, name): return self._cols.get(name, None) + + +class DictElement(ListElement): + def find_elements(self): + if self.item_xpath is None: + selector = [] + + elif isinstance(self.item_xpath, basestring): + selector = self.item_xpath.split('/') + + else: + selector = self.item_xpath + + for el in selector: + if isinstance(self.el, list): + el = int(el) + + self.el = self.el[el] + + for el in self.el: + yield el