From 2413c4bce91ff2bd0a64d224467803e0122b45be Mon Sep 17 00:00:00 2001 From: Bezleputh Date: Thu, 4 Sep 2014 14:33:03 +0200 Subject: [PATCH] [razibus] new CapCalendarEvent module that manages razibus.net --- modules/razibus/__init__.py | 24 ++++++++++ modules/razibus/backend.py | 90 +++++++++++++++++++++++++++++++++++++ modules/razibus/browser.py | 47 +++++++++++++++++++ modules/razibus/calendar.py | 30 +++++++++++++ modules/razibus/pages.py | 84 ++++++++++++++++++++++++++++++++++ modules/razibus/test.py | 32 +++++++++++++ 6 files changed, 307 insertions(+) create mode 100644 modules/razibus/__init__.py create mode 100644 modules/razibus/backend.py create mode 100644 modules/razibus/browser.py create mode 100644 modules/razibus/calendar.py create mode 100644 modules/razibus/pages.py create mode 100644 modules/razibus/test.py diff --git a/modules/razibus/__init__.py b/modules/razibus/__init__.py new file mode 100644 index 00000000..d3c9b354 --- /dev/null +++ b/modules/razibus/__init__.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2014 Bezleputh +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from .backend import RazibusBackend + + +__all__ = ['RazibusBackend'] diff --git a/modules/razibus/backend.py b/modules/razibus/backend.py new file mode 100644 index 00000000..e6bf170f --- /dev/null +++ b/modules/razibus/backend.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2014 Bezleputh +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.tools.backend import BaseBackend, BackendConfig +from weboob.capabilities.calendar import CapCalendarEvent, CATEGORIES +from weboob.tools.ordereddict import OrderedDict +from weboob.tools.value import Value + +from .browser import RazibusBrowser +from .calendar import RazibusCalendarEvent + +__all__ = ['RazibusBackend'] + + +class RazibusBackend(BaseBackend, CapCalendarEvent): + NAME = 'razibus' + DESCRIPTION = u'site annonçant les évènements attendus par les punks a chiens' + MAINTAINER = u'Bezleputh' + EMAIL = 'carton_ben@yahoo.fr' + LICENSE = 'AGPLv3+' + VERSION = '1.0' + ASSOCIATED_CATEGORIES = [CATEGORIES.CONCERT] + BROWSER = RazibusBrowser + + region_choices = OrderedDict([(k, u'%s' % (v)) for k, v in sorted({ + '': u'-- Indifférent --', + 'Alsace': u'Alsace', + 'Aquitaine': u'Aquitaine', + 'Auvergne': u'Auvergne', + 'Basse-Normandie': u'Basse-Normandie', + 'Bourgogne': u'Bourgogne', + 'Bretagne': u'Bretagne', + 'Centre': u'Centre', + 'Champagne-Ardenne': u'Champagne-Ardenne', + 'Franche-Comte': u'Franche-Comté', + 'Haute-Normandie': u'Haute-Normandie', + 'Ile-de-France': u'Île-de-France', + 'Languedoc-Roussillon': u'Languedoc-Roussillon', + 'Limousin': u'Limousin', + 'Lorraine': u'Lorraine', + 'Midi-Pyrenees': u'Midi-Pyrénées', + 'Nord-Pas-de-Calais': u'Nord-Pas-de-Calais', + 'Pays-de-la-Loire': u'Pays de la Loire', + 'Picardie': u'Picardie', + 'Poitou-Charentes': u'Poitou-Charentes', + 'PACA': u'PACA', + 'Rhone-Alpes': u'Rhône-Alpes', + 'Belgique': u'Belgique', + 'Suisse': u'Suisse', + }.iteritems())]) + + CONFIG = BackendConfig(Value('region', label=u'Region', choices=region_choices, default='')) + + def create_default_browser(self): + region = self.config['region'].get() + return self.create_browser(region) + + def search_events(self, query): + return self.browser.list_events(query.start_date, + query.end_date, + query.city, + query.categories) + + def get_event(self, _id): + return self.browser.get_event(_id) + + def list_events(self, date_from, date_to=None): + return self.browser.list_events(date_from, date_to) + + def fill_obj(self, event, fields): + return self.browser.get_event(event.id, event) + + OBJECTS = {RazibusCalendarEvent: fill_obj} diff --git a/modules/razibus/browser.py b/modules/razibus/browser.py new file mode 100644 index 00000000..404b9296 --- /dev/null +++ b/modules/razibus/browser.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2014 Bezleputh +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.tools.browser2 import PagesBrowser, URL + +from .pages import EventListPage, EventPage + + +__all__ = ['RazibusBrowser'] + + +class RazibusBrowser(PagesBrowser): + BASEURL = 'http://razibus.net/' + TIMEOUT = 15 + event_list_page = URL('evenements-a-venir.php\?region=(?P.*)', EventListPage) + event_page = URL('(?P<_id>.*).html', EventPage) + region = None + + def __init__(self, region, *args, **kwargs): + super(RazibusBrowser, self).__init__(*args, **kwargs) + self.region = region + + def get_event(self, _id, event=None): + return self.event_page.go(_id=_id).get_event(obj=event) + + def list_events(self, date_from, date_to, city=None, categories=None): + return self.event_list_page.go(region=self.region).list_events(date_from=date_from, + date_to=date_to, + city=city, + categories=categories) diff --git a/modules/razibus/calendar.py b/modules/razibus/calendar.py new file mode 100644 index 00000000..fd839f3c --- /dev/null +++ b/modules/razibus/calendar.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2013 Bezleputh +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from weboob.capabilities.calendar import BaseCalendarEvent, TRANSP, STATUS, CATEGORIES + + +class RazibusCalendarEvent(BaseCalendarEvent): + + def __init__(self): + BaseCalendarEvent.__init__(self) + self.sequence = 1 + self.transp = TRANSP.TRANSPARENT + self.status = STATUS.CONFIRMED + self.category = CATEGORIES.CONCERT diff --git a/modules/razibus/pages.py b/modules/razibus/pages.py new file mode 100644 index 00000000..a0121b6a --- /dev/null +++ b/modules/razibus/pages.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2014 Bezleputh +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from .calendar import RazibusCalendarEvent + +from datetime import time + +from weboob.tools.browser2.page import HTMLPage, method +from weboob.tools.browser2.elements import ItemElement, ListElement +from weboob.tools.browser2.filters import Regexp, Link, CleanText, DateTime, CombineDate, Filter, Env, CleanHTML + +__all__ = ['EventListPage', 'EventPage'] + + +class EndTime(Filter): + def filter(self, el): + return time.max + + +class EventListPage(HTMLPage): + @method + class list_events(ListElement): + item_xpath = '//div[@class="item"]' + + class item(ItemElement): + klass = RazibusCalendarEvent + + def validate(self, obj): + return (self.is_valid_event(obj, self.env['city'], self.env['categories']) and + self.is_event_in_valid_period(obj.start_date, self.env['date_from'], self.env['date_to'])) + + def is_valid_event(self, event, city, categories): + if city and city != '' and city.upper() != event.city.upper(): + return False + if categories and len(categories) > 0 and event.category not in categories: + return False + return True + + def is_event_in_valid_period(self, event_date, date_from, date_to): + if event_date >= date_from: + if not date_to: + return True + else: + if event_date <= date_to: + return True + return False + + obj_id = Regexp(Link('./p/strong/a[@itemprop="url"]'), 'http://razibus.net/(.*).html') + obj_summary = CleanText('./p/strong/a[@itemprop="url"]') + obj_start_date = DateTime(CleanText('./p/span[@itemprop="startDate"]/@content')) + obj_end_date = CombineDate(DateTime(CleanText('./p/span[@itemprop="startDate"]/@content')), EndTime('.')) + obj_location = CleanText('./p/span[@itemprop="location"]/@content') + obj_city = CleanText('./p/span[@itemprop="location"]') + + +class EventPage(HTMLPage): + @method + class get_event(ItemElement): + klass = RazibusCalendarEvent + + obj_id = Env('_id') + obj_summary = CleanText('//h2[@itemprop="name"]') + obj_start_date = DateTime(CleanText('//span[@itemprop="startDate"]/time/@datetime')) + obj_end_date = CombineDate(DateTime(CleanText('//span[@itemprop="startDate"]/time/@datetime')), EndTime('.')) + obj_location = CleanText('//meta[@property="og:street-address"]/@content') + obj_city = CleanText('//meta[@property="og:locality"]/@content') + obj_url = CleanText('//meta[@property="og:url"]/@content') + obj_description = CleanHTML('//div[@itemprop="description"]') diff --git a/modules/razibus/test.py b/modules/razibus/test.py new file mode 100644 index 00000000..1dc29f3a --- /dev/null +++ b/modules/razibus/test.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2014 Bezleputh +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.tools.test import BackendTest +from datetime import datetime + + +class RazibusTest(BackendTest): + BACKEND = 'razibus' + + def test_razibus(self): + l = list(self.backend.list_events(datetime.now())) + assert len(l) + event = self.backend.get_event(l[0].id) + self.assertTrue(event.url, 'URL for event "%s" not found: %s' % (event.id, event.url))