From 25433036360debe432a76c37433dc99e5b539fd7 Mon Sep 17 00:00:00 2001 From: Arno Renevier Date: Thu, 10 May 2012 13:26:37 +0200 Subject: [PATCH] First implementation of weather.com module --- modules/weather/__init__.py | 23 +++++++++++++ modules/weather/backend.py | 47 +++++++++++++++++++++++++ modules/weather/browser.py | 60 ++++++++++++++++++++++++++++++++ modules/weather/pages.py | 68 +++++++++++++++++++++++++++++++++++++ modules/weather/test.py | 41 ++++++++++++++++++++++ 5 files changed, 239 insertions(+) create mode 100644 modules/weather/__init__.py create mode 100644 modules/weather/backend.py create mode 100644 modules/weather/browser.py create mode 100644 modules/weather/pages.py create mode 100644 modules/weather/test.py diff --git a/modules/weather/__init__.py b/modules/weather/__init__.py new file mode 100644 index 00000000..2fdf784c --- /dev/null +++ b/modules/weather/__init__.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012 Arno Renevier +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from .backend import WeatherBackend + +__all__ = ['WeatherBackend'] diff --git a/modules/weather/backend.py b/modules/weather/backend.py new file mode 100644 index 00000000..60f9057f --- /dev/null +++ b/modules/weather/backend.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012 Arno Renevier +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.capabilities.weather import ICapWeather +from weboob.tools.backend import BaseBackend + +from .browser import WeatherBrowser + +__all__ = ['WeatherBackend'] + +class WeatherBackend(BaseBackend, ICapWeather): + NAME = 'weather' + MAINTAINER = 'Arno Renevier' + EMAIL = 'arno@renevier.net' + VERSION = '0.d' + DESCRIPTION = 'Get forecasts from weather.com' + LICENSE = 'AGPLv3+' + BROWSER = WeatherBrowser + + def create_default_browser(self): + return self.create_browser() + + def iter_city_search(self, pattern): + return self.browser.iter_city_search(pattern) + + def get_current(self, city_id): + return self.browser.get_current(city_id) + + def iter_forecast(self, city_id): + return self.browser.iter_forecast(city_id) diff --git a/modules/weather/browser.py b/modules/weather/browser.py new file mode 100644 index 00000000..2fcd0027 --- /dev/null +++ b/modules/weather/browser.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012 Arno Renevier +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +import urllib + +from weboob.tools.browser import BaseBrowser + +from .pages import ForecastPage, WeatherPage, CityPage + +__all__ = ['WeatherBrowser'] + +class WeatherBrowser(BaseBrowser): + DOMAIN = 'www.weather.com' + PROTOCOL = 'http' + ENCODING = 'utf-8' + PAGES = {} + + SEARCH_URL = 'http://www.weather.com/search/enhancedlocalsearch?where=%s' + WEATHER_URL = 'http://www.weather.com/weather/today/%s' + FORECAST_URL = 'http://www.weather.com/weather/tenday/%s' + USER_AGENT = BaseBrowser.USER_AGENTS['desktop_firefox'] + + PAGES = { + (SEARCH_URL.replace('.', '\\.').replace('?', '\\?') % '.*'): CityPage, + (WEATHER_URL.replace('.', '\\.').replace('?', '\\?') % '.*'): WeatherPage, + (FORECAST_URL.replace('.', '\\.').replace('?', '\\?') % '.*'): ForecastPage, + } + + def iter_city_search(self, pattern): + self.location(self.SEARCH_URL % urllib.quote_plus(pattern.encode('utf-8'))) + if self.is_on_page(CityPage): + return self.page.iter_city_search() + elif self.is_on_page(WeatherPage): + return [self.page.get_city()] + + def get_current(self, city_id): + self.location(self.WEATHER_URL % urllib.quote_plus(city_id.encode('utf-8'))) + return self.page.get_current() + + def iter_forecast(self, city_id): + self.location(self.FORECAST_URL % urllib.quote_plus(city_id.encode('utf-8'))) + assert self.is_on_page(ForecastPage) + return self.page.iter_forecast() diff --git a/modules/weather/pages.py b/modules/weather/pages.py new file mode 100644 index 00000000..1af1a9c7 --- /dev/null +++ b/modules/weather/pages.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012 Arno Renevier +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.tools.browser import BasePage +from weboob.capabilities.weather import Forecast, Current, City + +import datetime + + +__all__ = ['CityPage', 'WeatherPage', 'ForecastPage'] + + +class CityPage(BasePage): + def iter_city_search(self): + for item in self.document.findall('//div[@class="searchResultsList"]/ul/li'): + if item.attrib.get('class', '') == 'searchResultsMoreLink': + continue + city_name = unicode(item.text_content().strip()) + city_id = item.find('a').attrib.get("href", "").split("+")[-1] + yield City(city_id, city_name) + +class WeatherPage(BasePage): + def get_city(self): + parts = self.url.split('/')[-1].split('+') + return City(parts[-1], u', '.join(parts[:-1])) + + def get_current(self): + date = datetime.datetime.now() + text = unicode(self.document.findall('//table[@class="twc-forecast-table twc-second"]//tr')[2].find('td').text_content().strip()) + temp = float(self.document.find('//*[@class="twc-col-1 twc-forecast-temperature"]').text_content().strip().split(u'°')[0]) + return Current(date, temp, text, u'F') + +class ForecastPage(BasePage): + def iter_forecast(self): + trs = self.document.findall('//table[@class="twc-forecast-table twc-second"]//tr') + + for day in range (0, 10): + text = unicode(trs[1].findall('td')[day].text_content().strip()) + try: + tlow = float(trs[2].findall('td')[day].text_content().strip().split(u'°')[0]) + except: + tlow = None + try: + thigh = float(trs[3].findall('td')[day].text_content().strip().split(u'°')[0]) + except: + thigh = None + + date = self.document.findall('//table[@class="twc-forecast-table twc-first"]//th')[day].text + if len (date.split(' ')) > 3: + date = " ".join(date.split(' ', 3)[:3]) + yield Forecast(date, tlow, thigh, text, u'F') diff --git a/modules/weather/test.py b/modules/weather/test.py new file mode 100644 index 00000000..c8d44c64 --- /dev/null +++ b/modules/weather/test.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012 Arno Renevier +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.tools.test import BackendTest + +class WeatherTest(BackendTest): + BACKEND = 'weather' + + + def test_cities(self): + paris = self.backend.iter_city_search('crappything¶m=;drop database') + self.assertTrue(len(list(paris)) == 0) + + paris = self.backend.iter_city_search('paris') + self.assertTrue(len(list(paris)) > 1) + + paris = self.backend.iter_city_search('paris france') + self.assertTrue(len(list(paris)) == 1) + + current = self.backend.get_current(paris[0].id) + self.assertTrue(current.temp is float(current.temp)) + + forecasts = list(self.backend.iter_forecast(paris[0].id)) + self.assertTrue(len(forecasts) == 10)