From 89584ee16875ac5f492c07850465876027ed8dbd Mon Sep 17 00:00:00 2001 From: Florent Date: Wed, 19 Mar 2014 11:22:14 +0100 Subject: [PATCH] Convert sachsen module to browser2 --- modules/sachsen/__init__.py | 2 +- modules/sachsen/backend.py | 4 +- modules/sachsen/browser.py | 35 ++---- modules/sachsen/pages.py | 224 +++++++++++++++++++----------------- 4 files changed, 135 insertions(+), 130 deletions(-) diff --git a/modules/sachsen/__init__.py b/modules/sachsen/__init__.py index ffa163ab..3699d60c 100644 --- a/modules/sachsen/__init__.py +++ b/modules/sachsen/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright(C) 2010-2012 Romain Bignon, Florent Fourcot +# Copyright(C) 2010-2014 Florent Fourcot # # This file is part of weboob. # diff --git a/modules/sachsen/backend.py b/modules/sachsen/backend.py index 8459056e..98c5e294 100644 --- a/modules/sachsen/backend.py +++ b/modules/sachsen/backend.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright(C) 2010-2012 Romain Bignon, Florent Fourcot +# Copyright(C) 2010-2014 Florent Fourcot # # This file is part of weboob. # @@ -65,6 +65,8 @@ class SachsenLevelBackend(BaseBackend, ICapGauge): def iter_sensors(self, gauge, pattern=None): if not isinstance(gauge, Gauge): gauge = self._get_gauge_by_id(gauge) + if gauge is None: + raise SensorNotFound() if pattern is None: for sensor in gauge.sensors: yield sensor diff --git a/modules/sachsen/browser.py b/modules/sachsen/browser.py index 11f59870..b9cb9bb2 100644 --- a/modules/sachsen/browser.py +++ b/modules/sachsen/browser.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright(C) 2010-2012 Romain Bignon, Florent Fourcot +# Copyright(C) 2010-2014 Florent Fourcot # # This file is part of weboob. # @@ -18,36 +18,23 @@ # along with weboob. If not, see . -from weboob.tools.browser import BaseBrowser - - +from weboob.tools.browser2 import PagesBrowser, URL from .pages import ListPage, HistoryPage __all__ = ['SachsenBrowser'] -class SachsenBrowser(BaseBrowser): - DOMAIN = u'www.umwelt.sachsen.de' - ENCODING = None - PAGES = {'.*inhalt_re.html.*': ListPage, - '.*hwz/MP/.*': HistoryPage - } +class SachsenBrowser(PagesBrowser): + BASEURL = 'http://www.umwelt.sachsen.de' - homepage = '/de/wu/umwelt/lfug/lfug-internet/hwz/inhalt_re.html' - - def __init__(self, *args, **kwargs): - BaseBrowser.__init__(self, *args, **kwargs) - - def home(self): - self.location(self.homepage) + homepage = URL('/de/wu/umwelt/lfug/lfug-internet/hwz/inhalt_re.html.*', ListPage) + history = URL('/de/wu/umwelt/lfug/lfug-internet/hwz/MP/(?P.*)/index.html', HistoryPage) def get_rivers_list(self): - if not self.is_on_page(ListPage): - self.location(self.homepage) - return self.page.get_rivers_list() + return self.homepage.stay_or_go().get_rivers_list() - def iter_history(self, sensor): - self.location('/de/wu/umwelt/lfug/lfug-internet/hwz/MP/%d/index.html' - % int(sensor.gaugeid)) - return self.page.iter_history(sensor) + def iter_history(self, sensor, **kwargs): + self.history.go(params=int(sensor.gaugeid)) + kwargs['sensor'] = sensor + return self.page.iter_history(**kwargs) diff --git a/modules/sachsen/pages.py b/modules/sachsen/pages.py index 478a625f..9b269d74 100644 --- a/modules/sachsen/pages.py +++ b/modules/sachsen/pages.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright(C) 2010-2012 Romain Bignon, Florent Fourcot +# Copyright(C) 2010-2014 Florent Fourcot # # This file is part of weboob. # @@ -18,121 +18,137 @@ # along with weboob. If not, see . from datetime import datetime, date, time -from weboob.tools.browser import BasePage +from weboob.tools.browser2.page import HTMLPage, method, ListElement, ItemElement +from weboob.tools.browser2.filters import Env from weboob.capabilities.gauge import Gauge, GaugeMeasure, GaugeSensor from weboob.capabilities.base import NotAvailable, NotLoaded +import re __all__ = ['ListPage', 'HistoryPage'] -class ListPage(BasePage): - alarmlevel = {"as1.gif": u"Alarmstufe 1", "as2.gif": u"Alarmstufe 2", - "as3.gif": u"Alarmstufe 3", "as4.gig": u"Alarmstufe 4", - "qua_grau.gif": u"No alarm function", "p_gruen.gif": u"", - "qua_weiss.gif": u"no data", "as0.gif": u"", - "MNW.gif": u""} +class ListPage(HTMLPage): - def get_rivers_list(self): - for pegel in self.document.getroot().xpath(".//a[@onmouseout='pegelaus()']"): - div = pegel.getparent() - img = div.find('.//img').attrib['src'].split('/')[1] - data = pegel.attrib['onmouseover'] \ - .strip('pegelein(').strip(')').replace(",'", ",").split("',") - gauge = Gauge(int(data[7])) - gauge.name = unicode(data[0].strip("'")) - gauge.city = gauge.name.split(' ')[0] - gauge.object = unicode(data[1]) + @method + class get_rivers_list(ListElement): + item_xpath = ".//a[@onmouseout='pegelaus()']" - sensors = [] - try: - lastdate = date(*reversed([int(x) - for x in data[2].split(' ')[0].split(".")])) - lasttime = time(*[int(x) - for x in data[2].split(' ')[1].split(":")]) - lastdate = datetime.combine(lastdate, lasttime) - except: - lastdate = NotAvailable + class item(ItemElement): + klass = Gauge - bildforecast = data[5] - if bildforecast == "pf_gerade.png": - forecast = u"stable" - elif bildforecast == "pf_unten.png": - forecast = u"Go down" - elif bildforecast == "pf_oben.png": - forecast = u"Go up" - else: - forecast = NotAvailable + alarmlevel = {"as1.gif": u"Alarmstufe 1", "as2.gif": u"Alarmstufe 2", + "as3.gif": u"Alarmstufe 3", "as4.gig": u"Alarmstufe 4", + "qua_grau.gif": u"No alarm function", "p_gruen.gif": u"", + "qua_weiss.gif": u"no data", "as0.gif": u"", + "MNW.gif": u""} - try: - level = float(data[3]) - except: - level = NotAvailable - levelsensor = GaugeSensor(gauge.id + "-level") - levelsensor.name = u"Level" - levelsensor.unit = u"cm" - levelsensor.forecast = forecast - lastvalue = GaugeMeasure() - lastvalue.level = level - lastvalue.date = lastdate - try: - lastvalue.alarm = self.alarmlevel[img] - except KeyError: - lastvalue.alarm = u"" - levelsensor.lastvalue = lastvalue - levelsensor.history = NotLoaded - levelsensor.gaugeid = gauge.id - sensors.append(levelsensor) + obj_id = Env('id') + obj_name = Env('name') + obj_city = Env('city') + obj_object = Env('object') + obj_sensors = Env('sensors') - try: - flow = float(data[4]) - except: - flow = NotAvailable - flowsensor = GaugeSensor(gauge.id + "-flow") - flowsensor.name = u"Flow" - flowsensor.unit = u"m3/s" - flowsensor.forecast = forecast - lastvalue = GaugeMeasure() - lastvalue.level = flow - lastvalue.date = lastdate - try: - lastvalue.alarm = self.alarmlevel[img] - except KeyError: - lastvalue.alarm = u"" - flowsensor.lastvalue = lastvalue - flowsensor.history = NotLoaded - flowsensor.gaugeid = gauge.id - sensors.append(flowsensor) - - gauge.sensors = sensors - - yield gauge - - -class HistoryPage(BasePage): - def iter_history(self, sensor): - table = self.document.getroot().cssselect('table[width="215"]') - lines = table[0].cssselect("tr") - lines.pop(0) # remove header - lines.pop(0) # remove first value (already in lastvalue) - for line in lines: - history = GaugeMeasure() - leveldate = date(*reversed([int(x) - for x in line[0].text_content().split(' ')[0].split(".")])) - leveltime = time(*[int(x) - for x in line[0].text_content().split(' ')[1].split(":")]) - history.date = datetime.combine(leveldate, leveltime) - - if sensor.name == u"Level": + def init_sensor(self, _id, name, unit, value, forecast, alarm, date): + sensor = GaugeSensor("%s-%s" % (_id, name.lower())) + sensor.name = name + sensor.unit = unit + sensor.forecast = forecast + lastvalue = GaugeMeasure() + lastvalue.alarm = alarm try: - history.level = float(line[1].text_content()) + lastvalue.level = float(value) except: - history.level = NotAvailable - elif sensor.name == u"Flow": - try: - history.level = float(line[2].text_content()) - except: - history.level = NotAvailable + lastvalue.level = NotAvailable + lastvalue.date = date + sensor.lastvalue = lastvalue + sensor.history = NotLoaded + sensor.gaugeid = unicode(_id) - # TODO: history.alarm - yield history + return sensor + + def parse(self, el): + div = el.getparent() + img = div.find('.//img').attrib['src'].split('/')[1] + data = el.attrib['onmouseover'] \ + .strip('pegelein(').strip(')').replace(",'", ",").split("',") + + self.env['id'] = data[7].strip() + self.env['name'] = unicode(data[0].strip("'")) + self.env['city'] = self.env['name'].split(' ')[0] + self.env['object'] = unicode(data[1]) + + sensors = [] + try: + datenumbers = data[2].split(' ')[0].split(".") + timenumbers = data[2].split(' ')[1].split(":") + lastdate = date(*reversed([int(x) for x in datenumbers])) + lasttime = time(*[int(x) for x in timenumbers]) + lastdate = datetime.combine(lastdate, lasttime) + except: + lastdate = NotAvailable + + bildforecast = data[5] + if bildforecast == "pf_gerade.png": + forecast = u"stable" + elif bildforecast == "pf_unten.png": + forecast = u"Go down" + elif bildforecast == "pf_oben.png": + forecast = u"Go up" + else: + forecast = NotAvailable + + try: + alarm = self.alarmlevel[img] + except KeyError: + alarm = u"" + + levelsensor = self.init_sensor(self.env['id'], u"Level", + u"cm", data[3], forecast, + alarm, lastdate) + sensors.append(levelsensor) + + flowsensor = self.init_sensor(self.env['id'], u"Flow", + u"m3/s", data[4], forecast, + alarm, lastdate) + sensors.append(flowsensor) + + self.env['sensors'] = sensors + + +class HistoryPage(HTMLPage): + @method + class iter_history(ListElement): + item_xpath = '//table[@width="215"]/tr' + + class item(ItemElement): + klass = GaugeMeasure + verif = re.compile("\d\d.\d\d.\d+ \d\d:\d\d") + + obj_date = Env('date') + obj_level = Env('level') + obj_id = None + + def condition(self): + if self.verif.match(self.el[0].text_content()): + return True + return False + + def parse(self, line): + leveldate = date(*reversed([int(x) + for x in line[0].text_content().split(' ')[0].split(".")])) + leveltime = time(*[int(x) + for x in line[0].text_content().split(' ')[1].split(":")]) + self.env['date'] = datetime.combine(leveldate, leveltime) + + if self.env['sensor'].name == u"Level": + try: + self.env['level'] = float(line[1].text_content()) + except: + self.env['level'] = NotAvailable + elif self.env['sensor'].name == u"Flow": + try: + self.env['level'] = float(line[2].text_content()) + except: + self.env['level'] = NotAvailable + # TODO: history.alarm