From 72d10e4695de10bf1f1227b3924e7b7b7f7be3e7 Mon Sep 17 00:00:00 2001 From: Matthieu Weber Date: Fri, 6 Feb 2015 21:16:46 +0200 Subject: [PATCH] wetboobs: added module ilmatieteenlaitos Signed-off-by: Matthieu Weber Signed-off-by: Romain Bignon --- modules/ilmatieteenlaitos/__init__.py | 23 +++++ modules/ilmatieteenlaitos/browser.py | 43 +++++++++ modules/ilmatieteenlaitos/favicon.png | Bin 0 -> 2213 bytes modules/ilmatieteenlaitos/module.py | 49 ++++++++++ modules/ilmatieteenlaitos/pages.py | 130 ++++++++++++++++++++++++++ modules/ilmatieteenlaitos/test.py | 36 +++++++ 6 files changed, 281 insertions(+) create mode 100644 modules/ilmatieteenlaitos/__init__.py create mode 100644 modules/ilmatieteenlaitos/browser.py create mode 100644 modules/ilmatieteenlaitos/favicon.png create mode 100644 modules/ilmatieteenlaitos/module.py create mode 100644 modules/ilmatieteenlaitos/pages.py create mode 100644 modules/ilmatieteenlaitos/test.py diff --git a/modules/ilmatieteenlaitos/__init__.py b/modules/ilmatieteenlaitos/__init__.py new file mode 100644 index 00000000..427fe8e6 --- /dev/null +++ b/modules/ilmatieteenlaitos/__init__.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2015 Matthieu Weber +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from .module import IlmatieteenlaitosModule + +__all__ = ['IlmatieteenlaitosModule'] diff --git a/modules/ilmatieteenlaitos/browser.py b/modules/ilmatieteenlaitos/browser.py new file mode 100644 index 00000000..55446235 --- /dev/null +++ b/modules/ilmatieteenlaitos/browser.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2015 Matthieu Weber +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from weboob.browser.browsers import PagesBrowser +from weboob.browser.url import URL +from .pages import WeatherPage, SearchCitiesPage + +__all__ = ['IlmatieteenlaitosBrowser'] + + +class IlmatieteenlaitosBrowser(PagesBrowser): + BASEURL = 'http://ilmatieteenlaitos.fi' + cities = URL('/etusivu\?p_p_id=locationmenuportlet_WAR_fmiwwwweatherportlets&p_p_lifecycle=2&p_p_state=normal&' + 'p_p_mode=view&p_p_cacheability=cacheLevelFull&term=(?P.*)', SearchCitiesPage) + weather_query = URL('/paikallissaa\?p_p_id=locationmenuportlet_WAR_fmiwwwweatherportlets&p_p_lifecycle=1&' + 'p_p_state=normal&p_p_mode=view&_locationmenuportlet_WAR_fmiwwwweatherportlets_action=' + 'changelocation') + weather = URL('/saa/(?P.*)', WeatherPage) + + def iter_city_search(self, pattern): + return self.cities.go(pattern=pattern).iter_cities() + + def iter_forecast(self, city): + return self.weather_query.go(data={"place": city.name, "forecast": "short"}).iter_forecast() + + def get_current(self, city): + return self.weather_query.go(data={"place": city.name, "forecast": "short"}).get_current() diff --git a/modules/ilmatieteenlaitos/favicon.png b/modules/ilmatieteenlaitos/favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..1265787208d58cce4b0b878856a93f76e503f8e9 GIT binary patch literal 2213 zcmV;W2wL}vP)00009a7bBm000XU z000XU0RWnu7ytkO2XskIMF-yk1QjeBbEgBe0000PbVXQnLvL+uWo~o;Lvm$dbY)~9 zcWHEJAV*0}P*;Ht7XSbVA4x<(RA}DqT7PU)*A@Q!18_`Y8!!xebs}P5wi_>oBr6K0 zSvS&do}6s*2#U0_RYXV)X*J0SN;Hd&DQyW^)TxA2q-1NWDO+KdFb2j#G-fre=uFm^ z8tfG6AcxK#;53e%9}czi_Q#8y5FE#TFE(1Yqx_TS_s;#kbI*P6-g7<{Ns?F&i$rT5d=XHgv;f+bLUPV5C9+&iDWXFQmNEvG#ZUYsZ>fN5)Oy6EaQS?W`RJ!VzKD; z`c%i&>-83kB@hTO1i+#<10+e>?e^y8W~%*1rP4KP)?{U6 z0hpSa8XX<=dc8wKLr$l&zrVk)uP+!3#+aB)rjn8p7K@dM8JL-w>FMbai^bIAx#}I? z`_&nn&6cVko6S~VU!RwkNBI$p#XUVeGcz-(`bmudK@f7eoEoxpU(@cBH^r}yh+iMs z-ZlE3Gn}lK==7B9_{g8c)Zf{&XLULq!5Z}T^o<+jsdo`O3JzPP^ZpxYm$hpdRvgV`HPolwiMqNv0J=9m`bTtg^6!S)BHoc*{8{kP>5#1|a{t5RAAj$QU)R>w0%&hGY>*nDpNuaVDH+)mo8lr2n2O?b*-(f zbY_5x{nzqU@}|o-$9Z(#(B*%V3D~)C{qDox()-AMw&_AeMaA8_cN5G*d;(P73)V{u zUbWLiU-C~V2V?(zdie-#4&~+L!C-K)^}vq%&Y@Oq@hg2a(dP}%N6*AI-~Per)29Jc zR#pxU4lYd~91g41YHHPfG9jchCMSyGj~86~M)@ykb0`V?q@$w)fLg5%hr>${Fquq| zNMv{Y$y8VEoNfCAt0x}4%!I?j3pMGoj z(Op`yww*wMbIH4yB+_+>xvZ>AEEeCpcaJ(h%+7$_Zuk5BrTd!bqX&?G8)>!zpFyY7 z`Tc&o-TqJtH8(c{;O~5azV9cz;7=uUBa@W?;J|?cF~G9~#>U31R%@<$2Qj&h0fEnv z5!)56$;->LTCHPaW3vQoHX8ty!N|z5^xw#o?b)*@2Ad7w)TvVdCcpj^eU~{QWRE9* zJHsrZiXaqhb~GBbTCIZhQbx;HJc0~bfzKci2&`6XG#X|5d_DjL-~TBimumyatZB7c z06w3O?Q*#Qh!i^+`4a>)OrWT!2!PAwViN=bAi8D~BY*LBZ)r7^N(F!*h*<(5u85Jp ztZ;@2Zh?-|x>o zK0^Nx1K{~Tf|YWap5Xy)?Trl1+zX)l#}7uPZak8kjZ3hrBVq%$eCv3 z@5>dbJ|UYLWip`G>jj`xD%l#120%`P*N)-y2K@92yz|pwVb%32;W49i}E$ zXE=qNPUkEEDun`rA29M46l9n{e}DgM3P~gq0N4NW4kLeGh%!u|udfe)L?U5xI2^rR zAH45XWi!~2`PXF<{dI{s7!2z5dJc!f22fK|1AzapkLbG$PefX0v&|uH~T?Q(Xg^6_{)hr%BcbM+X_!_BpA%ZT1C`U539a5x-~N?79bi4_Wk#bN<)?HAgu*|Z8io?N&T%Yy#1 zO&6-FssLClmbl`T_*#jIii-C3_TYW5`?YeK9PHcS+sJU?Qc{CpFxcMSzL;z!ky@p- zwe`@ULxT0v;#d0UG`)G@8V%1_!dZD`Z)BvOM&?viRRx2=W5vf+swz7Xlui;EA&2C1;eyQEUN7kK~M7Z?$Vxr_gmG~)zsH26bdDSKG_+F*}%Yn!C-K^-2h7W zHSu@8Fu$>xjA{`8W*&rZc1{i)iS6$@J3A@8qFgRFo6QTpwM*Et2w$ImYd8hn=&}l5 n|M#TFUlH+loE^|(lHdOa(4MKQ4lv*=00000NkvXXu0mjfXX7h} literal 0 HcmV?d00001 diff --git a/modules/ilmatieteenlaitos/module.py b/modules/ilmatieteenlaitos/module.py new file mode 100644 index 00000000..9edb9224 --- /dev/null +++ b/modules/ilmatieteenlaitos/module.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2015 Matthieu Weber +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.capabilities.weather import CapWeather, CityNotFound +from weboob.tools.backend import Module +from weboob.capabilities.base import find_object +from .browser import IlmatieteenlaitosBrowser + + +__all__ = ['IlmatieteenlaitosModule'] + + +class IlmatieteenlaitosModule(Module, CapWeather): + NAME = 'ilmatieteenlaitos' + MAINTAINER = u'Matthieu Weber' + EMAIL = 'mweber+weboob@free.fr' + VERSION = '1.1' + DESCRIPTION = 'Get forecasts from the Ilmatieteenlaitos.fi website' + LICENSE = 'AGPLv3+' + BROWSER = IlmatieteenlaitosBrowser + + def get_current(self, city_id): + return self.browser.get_current(self.get_city(city_id)) + + def iter_forecast(self, city_id): + return self.browser.iter_forecast(self.get_city(city_id)) + + def iter_city_search(self, pattern): + return self.browser.iter_city_search(pattern) + + def get_city(self, _id): + return find_object(self.iter_city_search(_id), id=_id, error=CityNotFound) diff --git a/modules/ilmatieteenlaitos/pages.py b/modules/ilmatieteenlaitos/pages.py new file mode 100644 index 00000000..bec4ff58 --- /dev/null +++ b/modules/ilmatieteenlaitos/pages.py @@ -0,0 +1,130 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2015 Matthieu Weber +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from datetime import date +from itertools import imap, ifilter + +from weboob.browser.pages import JsonPage, HTMLPage +from weboob.browser.elements import ItemElement, ListElement, method +from weboob.capabilities.weather import Forecast, Current, City, Temperature +from weboob.browser.filters.json import Dict +from weboob.browser.filters.standard import Filter, CleanText, CleanDecimal, Regexp, Format, Date + + +class DictElement(ListElement): + def find_elements(self): + if self.item_xpath is not None: + for el in self.el: + yield el + else: + yield self.el + + +class Id(Filter): + def filter(self, txt): + return txt.split(", ")[0] + + +class SearchCitiesPage(JsonPage): + @method + class iter_cities(DictElement): + item_xpath = '.' + ignore_duplicate = True + + class item(ItemElement): + klass = City + + obj_id = Id(Dict('id')) + obj_name = Dict('value') + + +class WeatherPage(HTMLPage): + @method + class iter_forecast(ListElement): + item_xpath = ('//div[contains(@class, "mid") and contains(@class, "local-weather-forecast")]//' + 'tr[@class="meteogram-dates"]/td') + + class item(ItemElement): + klass = Forecast + + obj_id = CleanText('.//span/@title') + + def obj_date(self): + months = [u'tammikuuta', u'helmikuuta', u'maaliskuuta', u'huhtikuuta', u'toukokuuta', u'kesäkuuta', + u'heinäkuuta', u'elokuuta', u'syyskuuta', u'lokakuuta', u'marraskuuta', u'joulukuuta'] + d = CleanText('.//span/@title')(self).split() + return date(int(d[2]), months.index(d[1])+1, int(d[0].strip("."))) + + def temperatures(self): + offset = int(CleanText('string(sum(./preceding-sibling::td/@colspan))')(self)) + length = int(CleanText('@colspan')(self)) + temps = CleanText('../../../tbody/tr[@class="meteogram-temperatures"]/td[position() > %d ' + 'and position() <= %d]/span' % (offset, offset+length))(self) + return [float(_.strip(u'\xb0')) for _ in temps.split()] + + def obj_low(self): + return Temperature(min(self.temperatures()), u'C') + + def obj_high(self): + return Temperature(max(self.temperatures()), u'C') + + def obj_text(self): + offset = int(CleanText('string(sum(./preceding-sibling::td/@colspan))')(self)) + length = int(CleanText('@colspan')(self)) + hour_test = ('../../tr[@class="meteogram-times"]/td[position() > %d and position() <= %d ' + 'and .//text() = "%%s"]' % (offset, offset+length)) + hour_offset = 'string(count(%s/preceding-sibling::td)+1)' % (hour_test) + values = [ + '../../../tbody/tr[@class="meteogram-weather-symbols"]/td[position() = %d]/div/@title', + '../../../tbody/tr[@class="meteogram-apparent-temperatures"]/td[position() = %d]/div/@title', + '../../../tbody/tr[@class="meteogram-wind-symbols"]/td[position() = %d]/div/@title', + '../../../tbody/tr[@class="meteogram-probabilities-of-precipitation"]/td[position() = %d]' + + '/div/@title', + '../../../tbody/tr[@class="meteogram-hourly-precipitation-values"]/td[position() = %d]/span/@title', + ] + + def descriptive_text_for_hour(hour): + hour_exists = CleanText(hour_test % hour)(self) == hour + if hour_exists: + offset = int(CleanText(hour_offset % hour)(self)) + + def info_for_value(value): + return CleanText(value % offset)(self).replace(u'edeltävän tunnin ', u'') + return ("klo %s: " % hour) + ", ".join(ifilter(bool, imap(info_for_value, values))) + + return u'\n' + u'\n'.join(ifilter(bool, imap(descriptive_text_for_hour, ["02", "14"]))) + + @method + class get_current(ItemElement): + klass = Current + + obj_id = date.today() + obj_date = Date(Regexp(CleanText('//table[@class="observation-text"]//span[@class="time-stamp"]'), + r'^(\d+\.\d+.\d+)')) + obj_text = Format(u'%s, %s, %s', + CleanText(u'(//table[@class="observation-text"])//tr[2]/td[2]'), + CleanText(u'(//table[@class="observation-text"])//tr[5]/td[1]'), + CleanText(u'(//table[@class="observation-text"])//tr[4]/td[2]')) + + def obj_temp(self): + path = u'//table[@class="observation-text"]//span[@class="parameter-name" and text() = "Lämpötila"]' + \ + u'/../span[@class="parameter-value"]' + temp = CleanDecimal(Regexp(CleanText(path), r'^([^ \xa0]+)'), replace_dots=True)(self) + unit = Regexp(CleanText(path), r'\xb0(\w)')(self) + return Temperature(float(temp), unit) diff --git a/modules/ilmatieteenlaitos/test.py b/modules/ilmatieteenlaitos/test.py new file mode 100644 index 00000000..1fc10252 --- /dev/null +++ b/modules/ilmatieteenlaitos/test.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2015 Matthieu Weber +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.tools.test import BackendTest + + +class IlmatieteenlaitosTest(BackendTest): + MODULE = 'ilmatieteenlaitos' + + def test_ilmatieteenlaitos(self): + l = list(self.backend.iter_city_search('helsinki')) + self.assertTrue(len(l) > 0) + + city = l[0] + current = self.backend.get_current(city.id) + self.assertTrue(current.temp.value > -50 and current.temp.value < 50) + + forecasts = list(self.backend.iter_forecast(city.id)) + self.assertTrue(len(forecasts) > 0)