support repositories to manage backends (closes #747)

modules/ina/__init__.py

@@ -0,0 +1,3 @@
+from .backend import InaBackend
+
+__all__ = ['InaBackend']

modules/ina/backend.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2010-2011 Christophe Benz
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+
+from __future__ import with_statement
+
+from weboob.capabilities.video import ICapVideo
+from weboob.tools.backend import BaseBackend
+
+from .browser import InaBrowser
+from .video import InaVideo
+
+
+__all__ = ['InaBackend']
+
+
+class InaBackend(BaseBackend, ICapVideo):
+    NAME = 'ina'
+    MAINTAINER = 'Christophe Benz'
+    EMAIL = 'christophe.benz@gmail.com'
+    VERSION = '0.a'
+    DESCRIPTION = 'INA french video archives'
+    LICENSE = 'AGPLv3+'
+    BROWSER = InaBrowser
+
+    def get_video(self, _id):
+        return self.browser.get_video(_id)
+
+    def iter_search_results(self, pattern=None, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False, max_results=None):
+        with self.browser:
+            return self.browser.iter_search_results(pattern)
+
+    def fill_video(self, video, fields):
+        if fields != ['thumbnail']:
+            # if we don't want only the thumbnail, we probably want also every fields
+            with self.browser:
+                video = self.browser.get_video(video.id, video)
+        if 'thumbnail' in fields and video.thumbnail:
+            with self.browser:
+                video.thumbnail.data = self.browser.readurl(video.thumbnail.url)
+
+        return video
+
+    OBJECTS = {InaVideo: fill_video}

modules/ina/browser.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2010-2011 Christophe Benz
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+
+from weboob.tools.browser import BaseBrowser
+from weboob.tools.browser.decorators import id2url
+
+from .pages.video import VideoPage, BoutiqueVideoPage
+from .pages.search import SearchPage
+from .video import InaVideo
+
+
+__all__ = ['InaBrowser']
+
+
+class InaBrowser(BaseBrowser):
+    DOMAIN = 'ina.fr'
+    PAGES = {'http://boutique\.ina\.fr/video/.+\.html': BoutiqueVideoPage,
+             'http://www\.ina\.fr/.+\.html': VideoPage,
+             'http://boutique\.ina\.fr/recherche/.+': SearchPage,
+            }
+
+    @id2url(InaVideo.id2url)
+    def get_video(self, url, video=None):
+        self.location(url)
+        return self.page.get_video(video)
+
+    def iter_search_results(self, pattern):
+        self.location(self.buildurl('http://boutique.ina.fr/recherche/recherche', search=pattern.encode('utf-8')))
+        assert self.is_on_page(SearchPage)
+        return self.page.iter_videos()

modules/ina/pages/search.py

@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2010-2011 Romain Bignon
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+
+import datetime
+import re
+
+from weboob.tools.browser import BasePage
+from weboob.tools.browser import BrokenPageError
+
+from ..video import InaVideo
+
+
+__all__ = ['SearchPage']
+
+
+class SearchPage(BasePage):
+    URL_REGEXP = re.compile(r'/video/(.+)\.html')
+
+    def iter_videos(self):
+        try:
+            ul = self.parser.select(self.document.getroot(), 'div.container-videos ul', 1)
+        except BrokenPageError:
+            # It means there are no results.
+            return
+        for li in ul.findall('li'):
+            id = re.sub(self.URL_REGEXP, r'\1', li.find('a').attrib['href'])
+
+            thumbnail = 'http://boutique.ina.fr%s' % li.find('a').find('img').attrib['src']
+
+            title = self.parser.select(li, 'p.titre', 1).text
+
+            date = self.parser.select(li, 'p.date', 1).text
+            day, month, year = [int(s) for s in date.split('/')]
+            date = datetime.datetime(year, month, day)
+
+            duration = self.parser.select(li, 'p.duree', 1).text
+            m = re.match(r'((\d+)h)?((\d+)min)?(\d+)s', duration)
+            if m:
+                duration = datetime.timedelta(hours=int(m.group(2) or 0), minutes=int(m.group(4) or 0), seconds=int(m.group(5)))
+            else:
+                raise BrokenPageError('Unable to match duration (%r)' % duration)
+
+            yield InaVideo('boutique.%s' % id,
+                           title=title,
+                           date=date,
+                           duration=duration,
+                           thumbnail_url=thumbnail,
+                          )

modules/ina/pages/video.py

@@ -0,0 +1,128 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2010-2011 Christophe Benz
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+
+import datetime
+import re
+try:
+    from urlparse import parse_qs
+except ImportError:
+    from cgi import parse_qs
+
+from weboob.tools.browser import BasePage
+from weboob.tools.browser import BrokenPageError
+
+from ..video import InaVideo
+
+
+__all__ = ['VideoPage', 'BoutiqueVideoPage']
+
+
+class BaseVideoPage(BasePage):
+    def get_video(self, video):
+        date, duration = self.get_date_and_duration()
+        if not video:
+            video = InaVideo(self.get_id())
+
+        video.title = self.get_title()
+        video.url = self.get_url()
+        video.date = date
+        video.duration = duration
+        video.description = self.get_description()
+        return video
+
+    def get_id(self):
+        m = self.URL_REGEXP.match(self.url)
+        if m:
+            return self.create_id(m.group(1))
+        self.logger.warning('Unable to parse ID')
+        return 0
+
+    def get_url(self):
+        qs = parse_qs(self.document.getroot().cssselect('param[name="flashvars"]')[0].attrib['value'])
+        url = 'http://mp4.ina.fr/lecture/lire/id_notice/%s/token_notice/%s' % (qs['id_notice'][0], qs['token_notice'][0])
+        return url
+
+    def parse_date_and_duration(self, text):
+        duration_regexp = re.compile('(.* - )?(.+) - ((.+)h)?((.+)min)?(.+)s')
+        m = duration_regexp.match(text)
+        if m:
+            day, month, year = [int(s) for s in m.group(2).split('/')]
+            date = datetime.datetime(year, month, day)
+            duration = datetime.timedelta(hours=int(m.group(4) if m.group(4) is not None else 0),
+                                          minutes=int(m.group(6) if m.group(6) is not None else 0),
+                                          seconds=int(m.group(7)))
+            return date, duration
+        else:
+            raise BrokenPageError('Unable to parse date and duration')
+
+    def create_id(self, id):
+        raise NotImplementedError()
+
+    def get_date_and_duration(self):
+        raise NotImplementedError()
+
+    def get_title(self):
+        raise NotImplementedError()
+
+    def get_description(self):
+        raise NotImplementedError()
+
+class VideoPage(BaseVideoPage):
+    URL_REGEXP = re.compile('http://www.ina.fr/(.+)\.html')
+
+    def create_id(self, id):
+        return u'www.%s' % id
+
+    def get_date_and_duration(self):
+        qr = self.parser.select(self.document.getroot(), 'div.container-global-qr')[0].find('div').findall('div')[1]
+        return self.parse_date_and_duration(qr.find('h2').tail.strip())
+
+    def get_title(self):
+        qr = self.parser.select(self.document.getroot(), 'div.container-global-qr')[0].find('div').findall('div')[1]
+        return qr.find('h2').text.strip()
+
+    def get_description(self):
+        return self.parser.select(self.document.getroot(), 'div.container-global-qr')[1].find('div').find('p').text.strip()
+
+
+class BoutiqueVideoPage(BaseVideoPage):
+    URL_REGEXP = re.compile('http://boutique.ina.fr/video/(.+).html')
+
+    def create_id(self, id):
+        return u'boutique.%s' % id
+
+    def get_description(self):
+        el = self.document.getroot().cssselect('div.bloc-produit-haut div.contenu p')[0]
+        if el is not None:
+            return el.text.strip()
+
+    def get_date_and_duration(self):
+        el = self.document.getroot().cssselect('div.bloc-produit-haut p.date')[0]
+        if el is not None:
+            return self.parse_date_and_duration(el.text.strip())
+        else:
+            raise BrokenPageError('Unable to find date and duration element')
+
+    def get_title(self):
+        el = self.document.getroot().cssselect('div.bloc-produit-haut h1')[0]
+        if el is not None:
+            return unicode(el.text.strip())
+        else:
+            return None

modules/ina/test.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2010-2011 Romain Bignon
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+
+from weboob.tools.test import BackendTest
+
+class INATest(BackendTest):
+    BACKEND = 'ina'
+
+    def test_ina(self):
+        l = list(self.backend.iter_search_results('chirac'))
+        self.assertTrue(len(l) > 0)
+        v = l[0]
+        self.backend.fillobj(v, ('url',))
+        self.assertTrue(v.url and v.url.startswith('http://'), 'URL for video "%s" not found: %s' % (v.id, v.url))
+        self.backend.browser.openurl(v.url)

modules/ina/video.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2010-2011 Christophe Benz
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+
+from weboob.capabilities.video import BaseVideo
+
+
+__all__ = ['InaVideo']
+
+
+class InaVideo(BaseVideo):
+    @classmethod
+    def id2url(cls, _id):
+        if not '.' in _id:
+            return None
+        site, _id = _id.split('.', 1)
+        if site == 'boutique':
+            return 'http://boutique.ina.fr/video/%s.html' % _id
+        if site == 'www':
+            return 'http://www.ina.fr/%s.html' % _id