From 12a9aa95088116b21940e781a47b959a66f04053 Mon Sep 17 00:00:00 2001 From: Alexandre Morignot Date: Mon, 17 Nov 2014 15:53:20 +0100 Subject: [PATCH] New module residentadivsor Add the support of www.residentadvisor.net, a calendar of clubbing event. Inherits of CapCalendarEvent and offers the following : - search event by date and city - (un)attend to event if credentials are supplied A function search_events_by_title is implemented in ResidentadvisorBrowser, although not used, in hope that this functionality becomes possible in a near futur. --- modules/residentadvisor/__init__.py | 24 +++++ modules/residentadvisor/browser.py | 125 ++++++++++++++++++++++ modules/residentadvisor/module.py | 156 ++++++++++++++++++++++++++++ modules/residentadvisor/pages.py | 112 ++++++++++++++++++++ modules/residentadvisor/test.py | 48 +++++++++ 5 files changed, 465 insertions(+) create mode 100644 modules/residentadvisor/__init__.py create mode 100644 modules/residentadvisor/browser.py create mode 100644 modules/residentadvisor/module.py create mode 100644 modules/residentadvisor/pages.py create mode 100644 modules/residentadvisor/test.py diff --git a/modules/residentadvisor/__init__.py b/modules/residentadvisor/__init__.py new file mode 100644 index 00000000..12ba20de --- /dev/null +++ b/modules/residentadvisor/__init__.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2014 Alexandre Morignot +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from .module import ResidentadvisorModule + + +__all__ = ['ResidentadvisorModule'] diff --git a/modules/residentadvisor/browser.py b/modules/residentadvisor/browser.py new file mode 100644 index 00000000..e7ba01c6 --- /dev/null +++ b/modules/residentadvisor/browser.py @@ -0,0 +1,125 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2014 Alexandre Morignot +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.browser import LoginBrowser, URL, need_login +from weboob.exceptions import BrowserIncorrectPassword + +from .pages import LoginPage, EventPage, ListPage, SearchPage + +from datetime import datetime + + +class ResidentadvisorBrowser(LoginBrowser): + BASEURL = 'http://www.residentadvisor.net' + + # this ID is used by Resident Advisor + ALBANIA_ID = 223 + + login = URL('https://www.residentadvisor.net/login', LoginPage) + event = URL('/event.aspx\?(?P\d+)', EventPage) + list_events = URL('/events.aspx\?ai=(?P\d+)&v=(?P.+)&yr=(?P\d{4})&mn=(?P\d\d?)&dy=(?P\d\d?)', ListPage) + search_page = URL('/search.aspx?searchstr=(?P)§ion=events&titles=1', SearchPage) + attends = URL('/Output/addhandler.ashx') + + def do_login(self): + self.login.stay_or_go() + self.page.login(self.username, self.password) + + # in case of successful connection, we are redirected to the home page + if self.login.is_here(): + raise BrowserIncorrectPassword() + + def get_events(self, city, v = 'week', date = datetime.now()): + self.list_events.go(v = v, year = date.year, month = date.month, day = date.day, city = city) + assert self.list_events.is_here() + + for event in self.page.get_events(): + yield event + + def get_event(self, _id): + self.event.go(id = _id) + + if not self.event.is_here(): + return None + + event = self.page.get_event() + event.id = _id + event.url = self.event.build(id = _id) + + return event + + def search_events_by_title(self, pattern): + self.search_page.go(query = pattern) + assert self.search_page.is_here() + + for event in self.page.get_events(): + yield event + + def get_country_city_id(self, country, city): + now = datetime.now() + + self.list_events.go(v = 'day', year = now.year, month = now.month, day = now.day, city = self.ALBANIA_ID) + assert self.list_events.is_here() + + country_id = self.page.get_country_id(country) + + if country_id is None: + return None + + self.list_events.go(v = 'day', year = now.year, month = now.month, day = now.day, city = country_id) + assert self.list_events.is_here() + + city_id = self.page.get_city_id(city) + + if city_id is None: + return None + + return city_id + + def get_city_id(self, city): + now = datetime.now() + + country_id = self.ALBANIA_ID + city_id = None + + while True: + self.list_events.go(v = 'day', year = now.year, month = now.month, day = now.day, city = country_id) + assert self.list_events.is_here() + + city_id = self.page.get_city_id(city) + country_id = self.page.get_country_id_next_to(country_id) + + # city_id != None => city found + # country_id = None => no more country, city not found + if city_id is not None or country_id is None: + break + + return city_id + + @need_login + def attends_event(self, id, is_attending): + data = {'type': 'saveFavourite', + 'action':'attending', + 'id': id} + + if not is_attending: + data['type'] = 'deleteFavourite' + + self.attends.open(data = data) diff --git a/modules/residentadvisor/module.py b/modules/residentadvisor/module.py new file mode 100644 index 00000000..d7ae13c9 --- /dev/null +++ b/modules/residentadvisor/module.py @@ -0,0 +1,156 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2014 Alexandre Morignot +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.tools.backend import Module +from weboob.tools.value import Value, ValueBackendPassword +from weboob.tools.backend import BackendConfig +from weboob.capabilities.calendar import CapCalendarEvent, BaseCalendarEvent, CATEGORIES, STATUS +from weboob.capabilities.base import NotAvailable + +from .browser import ResidentadvisorBrowser + +from datetime import datetime, timedelta, time + + +__all__ = ['ResidentadvisorModule'] + + +class ResidentadvisorModule(Module, CapCalendarEvent): + NAME = 'residentadvisor' + DESCRIPTION = u'residentadvisor website' + MAINTAINER = u'Alexandre Morignot' + EMAIL = 'erdnaxeli@cervoi.se' + LICENSE = 'AGPLv3+' + VERSION = '1.1' + + BROWSER = ResidentadvisorBrowser + + CONFIG = BackendConfig(Value('username', label='Username or email', default=''), + ValueBackendPassword('password', label='Password', default=''), + Value('country', required=True), + Value('city', required=True)) + + ASSOCIATED_CATEGORIES = [CATEGORIES.CONCERT] + + _city_id = None + + @property + def city_id(self): + if not self._city_id: + self._city_id = self.browser.get_country_city_id( + country = self.config['country'].get(), + city = self.config['city'].get()) + + return self._city_id + + def create_default_browser(self): + password = None + username = self.config['username'].get() + + if len(username) > 0: + password = self.config['password'].get() + + return self.create_browser(username, password) + + def attends_event(self, event, is_attending): + """ + Attends or not to an event + :param event : the event + :type event : BaseCalendarEvent + :param is_attending : is attending to the event or not + :type is_attending : bool + """ + self.browser.attends_event(event.id, is_attending) + + def get_event(self, _id): + """ + Get an event from an ID. + + :param _id: id of the event + :type _id: str + :rtype: :class:`BaseCalendarEvent` or None is fot found. + """ + return self.browser.get_event(_id) + + def list_events(self, date_from, date_to): + """ + list coming event. + + :param date_from: date of beguinning of the events list + :type date_from: date + :param date_to: date of ending of the events list + :type date_to: date + :rtype: iter[:class:`BaseCalendarEvent`] + """ + # we check if date_to is defined + try: + date_to.date() + except: + # default is week + date_to = date_from + timedelta(days = 7) + + delta = date_to - date_from + + while delta.days >= 0 : + v = 'week' + + if delta.days > 7: + v = 'month' + + for event in self.browser.get_events(v = v, date = date_from, city = self.city_id): + if event.start_date <= date_to: + yield event + + if v == 'week': + date_from += timedelta(days = 7) + else: + date_from += timedelta(days = 30) + + delta = date_to - date_from + + def search_events(self, query): + """ + Search event + + :param query: search query + :type query: :class:`Query` + :rtype: iter[:class:`BaseCalendarEvent`] + """ + if not self.has_matching_categories(query): + raise StopIteration() + + if query.city: + # FIXME + # we need the country to search the city_id in an efficient way + city_id = self.browser.get_city_id(query.city) + + for event in self.browser.get_events(city = city_id): + yield event + else: + for event in self.list_events(query.start_date, query.end_date): + yield event + + def fill_event(self, event, fields): + if set(fields) & set(('end_date', 'price', 'description')): + return self.get_event(event.id) + + return event + + OBJECTS = {BaseCalendarEvent: fill_event} diff --git a/modules/residentadvisor/pages.py b/modules/residentadvisor/pages.py new file mode 100644 index 00000000..04a41782 --- /dev/null +++ b/modules/residentadvisor/pages.py @@ -0,0 +1,112 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2014 Alexandre Morignot +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.capabilities.calendar import CATEGORIES, STATUS +from weboob.browser.elements import ItemElement, ListElement, method +from weboob.browser.filters.html import Attr, CleanHTML, Link +from weboob.browser.filters.standard import CleanDecimal, CleanText, Date, CombineDate, DateTime, Regexp, Time, Type +from weboob.browser.pages import HTMLPage +from weboob.capabilities.calendar import BaseCalendarEvent + +from datetime import timedelta + + +class BasePage(HTMLPage): + @property + def logged(self): + return bool(self.doc.xpath('//li[@id="profile"]/span[contains(text(), "Welcome")]')) + + +class LoginPage(BasePage): + def login(self, username, password): + form = self.get_form() + form['UsernameOrEmailAddress'] = username + form['Password'] = password + form.submit() + + +class ListPage(BasePage): + @method + class get_events(ListElement): + item_xpath = '//ul[@id="items"]/li/article' + + class item(ItemElement): + klass = BaseCalendarEvent + + obj_url = Link('./div[@class="bbox"]/h1/a') + obj_id = Regexp(Link('./div[@class="bbox"]/h1/a'), r'aspx\?(.+)') + obj_location = CleanText('./div[@class="bbox"]/span/a') + obj_start_date = DateTime(Attr('.//time', 'datetime')) + obj_summary = Regexp(Attr('./div[@class="bbox"]/h1/a', 'title'), r'details of (.+)') + obj_category = CATEGORIES.CONCERT + obj_status = STATUS.CONFIRMED + + def get_country_id(self, country): + return Regexp(Link('//li[@id="liCountry"]/ul/li/a[./text()="%s"]' % country, default=''), r'ai=([^&]+)&?', default=None)(self.doc) + + def get_city_id(self, city): + return Regexp(Link('//li[@id="liArea"]/ul/li/a[./text()="%s"]' % city, default=''), r'ai=([^&]+)&?', default=None)(self.doc) + + def get_country_id_next_to(self, country_id): + return Regexp(Link('//li[@id="liCountry"]/ul/li[./a[contains(@href, "ai=%s&")]]/following-sibling::li/a' % country_id, default=''), r'ai=([^&]+)&?', default=None)(self.doc) + + +class EventPage(BasePage): + @method + class get_event(ItemElement): + klass = BaseCalendarEvent + + obj_summary = CleanText('//div[@id="sectionHead"]/h1') + obj_description = CleanHTML('//div[@id="event-item"]/div[3]/p[2]') + obj_price = CleanDecimal(Regexp(CleanText('//aside[@id="detail"]/ul/li[3]'), r'Cost / [^\d]+([\d ,.]+).', default=''), default=None) + obj_location = Regexp(CleanText('//aside[@id="detail"]/ul/li[2]'), r'Venue / (.+)') + obj_booked_entries = Type(CleanText('//h1[@id="MembersFavouriteCount"]'), type=float) + obj_status = STATUS.CONFIRMED + obj_category = CATEGORIES.CONCERT + + _date = Date(CleanText('//aside[@id="detail"]/ul/li[1]/a[1]')) + + def obj_start_date(self): + start_time = Time(Regexp(CleanText('//aside[@id="detail"]/ul/li[1]'), r'(\d{2}:\d{2}) -'))(self) + return CombineDate(self._date, start_time)(self) + + def obj_end_date(self): + end_time = Time(Regexp(CleanText('//aside[@id="detail"]/ul/li[1]'), r'- (\d{2}:\d{2})'))(self) + + end_date = CombineDate(self._date, end_time)(self) + if end_date > self.obj_start_date(): + end_date += timedelta(days = 1) + + return end_date + + +class SearchPage(BasePage): + @method + class get_events(ListElement): + item_xpath = '//main/ul/li/section/div/div/ul/li' + + class item(ItemElement): + klass = BaseCalendarEvent + + obj_url = Link('./a[1]') + obj_id = Regexp(Link('./a[1]'), r'\?(\d+)') + obj_summary = CleanText('./a[1]') + obj_start_date = Date(CleanText('./span[1]')) + obj_booked_entries = Type(CleanText('.//p[@class="attending"]/span'), type=float) diff --git a/modules/residentadvisor/test.py b/modules/residentadvisor/test.py new file mode 100644 index 00000000..8ce7e0e1 --- /dev/null +++ b/modules/residentadvisor/test.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2014 Alexandre Morignot +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.tools.test import BackendTest +from weboob.capabilities.calendar import Query + +from datetime import datetime, timedelta + + +class ResidentadvisorTest(BackendTest): + MODULE = 'residentadvisor' + + def test_searchcity(self): + query = Query() + query.city = u'Melbourne' + + self.assertTrue(len(list(self.backend.search_events(query))) > 0) + + event = self.backend.search_events(query).next() + self.assertTrue(self.backend.get_event(event.id)) + + def test_datefrom(self): + query = Query() + later = (datetime.now() + timedelta(days=31)) + query.start_date = later + + event = self.backend.search_events(query).next() + self.assertTrue(later.date() <= event.start_date.date()) + + event = self.backend.get_event(event.id) + self.assertTrue(later.date() <= event.start_date.date())