imdb and cineoob in progress

modules/imdb/__init__.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2013 Julien Veyssier
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+from .backend import ImdbBackend
+
+__all__ = ['ImdbBackend']

modules/imdb/backend.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2013 Julien Veyssier
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+from weboob.capabilities.cinema import ICapCinema
+from weboob.tools.backend import BaseBackend
+
+from .browser import ImdbBrowser
+
+from urllib import quote_plus
+
+__all__ = ['ImdbBackend']
+
+
+class ImdbBackend(BaseBackend, ICapCinema):
+    NAME = 'imdb'
+    MAINTAINER = u'Julien Veyssier'
+    EMAIL = 'julien.veyssier@aiur.fr'
+    VERSION = '0.f'
+    DESCRIPTION = 'Internet Movie Database service'
+    LICENSE = 'AGPLv3+'
+    BROWSER = ImdbBrowser
+
+    def create_default_browser(self):
+        return self.create_browser()
+
+    def get_movie(self, id):
+        return self.browser.get_movie(id)
+
+    def get_person(self, id):
+        return self.browser.get_person(id)
+
+    def iter_movies(self, pattern):
+        return self.browser.iter_movies(quote_plus(pattern.encode('utf-8')))
+
+    def iter_persons(self, pattern):
+        return self.browser.iter_persons(quote_plus(pattern.encode('utf-8')))
+
+    def iter_movie_persons(self, id):
+        return self.browser.iter_movie_persons(id)
+
+    def iter_person_movies(self, id):
+        return self.browser.iter_person_movies(id)

modules/imdb/browser.py

@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2013 Julien Veyssier
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+
+from weboob.tools.browser import BaseBrowser
+from weboob.tools.json import json
+
+from .pages import MoviePage, PersonPage, MovieCrewPage
+
+
+__all__ = ['ImdbBrowser']
+
+
+class ImdbBrowser(BaseBrowser):
+    DOMAIN = 'www.imdb.com'
+    PROTOCOL = 'http'
+    ENCODING = 'utf-8'
+    USER_AGENT = BaseBrowser.USER_AGENTS['wget']
+    PAGES = {
+        'http://www.imdb.com/title/tt[0-9]*/*': MoviePage,
+        'http://www.imdb.com/title/tt[0-9]*/fullcredits.*': MovieCrewPage,
+        'http://www.imdb.com/name/nm.*': PersonPage,
+        }
+
+    def iter_movies(self, pattern):
+        # the api leads to a json result or the html movie page if there is only one result
+        self.location('http://www.imdb.com/xml/find?json=1&tt=on&q=%s' % pattern.encode('utf-8'))
+        if self.is_on_page(MoviePage):
+            yield self.page.get_movie()
+        else:
+            res = self.readurl('http://www.imdb.com/xml/find?json=1&tt=on&q=%s' % pattern.encode('utf-8'))
+            jres = json.loads(res)
+            for restype,mlist in jres.items():
+                for m in mlist:
+                    yield self.get_movie(m['id'])
+
+    def iter_persons(self, pattern):
+        # the api leads to a json result or the html movie page if there is only one result
+        self.location('http://www.imdb.com/xml/find?json=1&nm=on&q=%s' % pattern.encode('utf-8'))
+        if self.is_on_page(PersonPage):
+            yield self.page.get_person()
+        else:
+            res = self.readurl('http://www.imdb.com/xml/find?json=1&nm=on&q=%s' % pattern.encode('utf-8'))
+            jres = json.loads(res)
+            for restype,plist in jres.items():
+                for p in plist:
+                    yield self.get_person(p['id'])
+
+    def get_movie(self, id):
+        self.location('http://www.imdb.com/title/%s' % id)
+        assert self.is_on_page(MoviePage)
+        return self.page.get_movie(id)
+
+    def get_person(self, id):
+        self.location('http://www.imdb.com/name/%s' % id)
+        assert self.is_on_page(PersonPage)
+        return self.page.get_person(id)
+
+    def iter_movie_persons(self, movie_id):
+        self.location('http://www.imdb.com/title/%s' % movie_id)
+        assert self.is_on_page(MoviePage)
+        return self.page.iter_persons(movie_id)
+
+    def iter_person_movies(self, person_id):
+        self.location('http://www.imdb.com/name/%s' % person_id)
+        assert self.is_on_page(PersonPage)
+        return self.page.iter_movies(person_id)

modules/imdb/pages.py

@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2013 Julien Veyssier
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+
+from weboob.capabilities.cinema import Movie, Person
+from weboob.capabilities.base import NotAvailable
+from weboob.tools.browser import BasePage
+
+import string
+from datetime import datetime
+
+
+__all__ = ['MoviePage','PersonPage','MovieCrewPage']
+
+
+class MoviePage(BasePage):
+    def get_movie(self,id):
+        title = NotAvailable
+        duration = NotAvailable
+        description = NotAvailable.__unicode__()
+        td_overview = self.parser.select(self.document.getroot(),'td#overview-top',1)
+        for span in self.parser.select(td_overview,'h1.header span[itemprop=name]'):
+            if span.attrib.get('class','') == 'itemprop':
+                other_titles = span.text
+                if title == NotAvailable:
+                    title = other_titles
+            elif span.attrib.get('class','') == 'title-extra':
+                title = span.text
+        meta = self.parser.select(td_overview,'meta[itemprop=datePublished]',1)
+        datestrings = meta.attrib.get('content','').split('-')
+        if len(datestrings) == 2:
+            datestrings.append('1')
+        time = self.parser.select(td_overview,'time[itemprop=duration]')
+        if len(time) > 0:
+            duration = int(time[0].attrib.get('datetime','').strip(string.letters))
+        desc = self.parser.select(td_overview,'p[itemprop=description]')
+        if len(desc) > 0:
+            description = desc[0].text
+        movie = Movie(id,title.strip())
+        movie.other_titles    = other_titles.strip()
+        movie.release_date    = datetime(int(datestrings[0]),int(datestrings[1]),int(datestrings[2]))
+        movie.duration        = duration
+        movie.description     = description
+        movie.note            = "10/10"
+        movie.awards          = ["aw1","aw2"]
+        movie.roles           = {}
+        return movie
+
+    def iter_persons(self,id):
+        self.browser.location('http://www.imdb.com/title/%s/fullcredits'%id)
+        assert self.browser.is_on_page(MovieCrewPage)
+        for p in self.browser.page.iter_persons():
+            yield p
+
+class MovieCrewPage(BasePage):
+    def iter_persons(self):
+        tables = self.parser.select(self.document.getroot(),'table.cast')
+        if len(tables) > 0:
+            table = tables[0]
+            tds = self.parser.select(table,'td.nm')
+            for td in tds:
+                name = td.text_content()
+                id = td.find('a').attrib.get('href','').strip('/').split('/')[-1]
+                person = Person(id,name)
+                person.real_name = NotAvailable
+                person.birth_date = NotAvailable
+                person.nationality = NotAvailable
+                person.gender = NotAvailable
+                yield person
+
+
+class PersonPage(BasePage):
+    def get_person(self,id):
+        person = Person(id,'nameplop')
+        person.real_name       = 'rn'
+        person.birth_date      = datetime.now()
+        person.birth_place     = "place"
+        person.gender          = "M"
+        person.nationality     = "nn"
+        person.biography       = 'bio'
+        person.awards          = ["aw1","aw2"]
+        person.roles           = {}
+        return person
+
+    def iter_movies(self,person_id):
+        pass

modules/imdb/test.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2013 Julien Veyssier
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+from weboob.tools.test import BackendTest
+
+#from random import choice
+
+class ImdbTest(BackendTest):
+    BACKEND = 'imdb'
+
+    def test_movie(self):
+        movies = list(self.backend.iter_movies('spiderman'))
+        for movie in movies:
+            assert movie.id
+            assert movie.original_title
+            assert movie.release_date
+    
+    def test_persons(self):
+        persons = list(self.backend.iter_persons('robert'))
+        for person in persons:
+            assert person.id
+            assert person.name
+            assert person.birth_date