new backend allocine, in development
This commit is contained in:
parent
e6fe9ed69e
commit
41cb3f7317
5 changed files with 678 additions and 0 deletions
22
modules/allocine/__init__.py
Normal file
22
modules/allocine/__init__.py
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright(C) 2013 Julien Veyssier
|
||||
#
|
||||
# This file is part of weboob.
|
||||
#
|
||||
# weboob is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# weboob is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
from .backend import AllocineBackend
|
||||
|
||||
__all__ = ['AllocineBackend']
|
||||
120
modules/allocine/backend.py
Normal file
120
modules/allocine/backend.py
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright(C) 2013 Julien Veyssier
|
||||
#
|
||||
# This file is part of weboob.
|
||||
#
|
||||
# weboob is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# weboob is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
from weboob.capabilities.cinema import ICapCinema, Person, Movie
|
||||
from weboob.tools.backend import BaseBackend
|
||||
|
||||
from .browser import AllocineBrowser
|
||||
|
||||
from urllib import quote_plus
|
||||
|
||||
__all__ = ['AllocineBackend']
|
||||
|
||||
|
||||
class AllocineBackend(BaseBackend, ICapCinema):
|
||||
NAME = 'allocine'
|
||||
MAINTAINER = u'Julien Veyssier'
|
||||
EMAIL = 'julien.veyssier@aiur.fr'
|
||||
VERSION = '0.f'
|
||||
DESCRIPTION = 'Allocine french cinema database service'
|
||||
LICENSE = 'AGPLv3+'
|
||||
BROWSER = AllocineBrowser
|
||||
|
||||
def create_default_browser(self):
|
||||
return self.create_browser()
|
||||
|
||||
def get_movie(self, id):
|
||||
return self.browser.get_movie(id)
|
||||
|
||||
def get_person(self, id):
|
||||
return self.browser.get_person(id)
|
||||
|
||||
def iter_movies(self, pattern):
|
||||
return self.browser.iter_movies(quote_plus(pattern.encode('utf-8')))
|
||||
|
||||
def iter_persons(self, pattern):
|
||||
return self.browser.iter_persons(quote_plus(pattern.encode('utf-8')))
|
||||
|
||||
def iter_movie_persons(self, id, role=None):
|
||||
return self.browser.iter_movie_persons(id, role)
|
||||
|
||||
def iter_person_movies(self, id, role=None):
|
||||
return self.browser.iter_person_movies(id, role)
|
||||
|
||||
def iter_person_movies_ids(self, id):
|
||||
return self.browser.iter_person_movies_ids(id)
|
||||
|
||||
def iter_movie_persons_ids(self, id):
|
||||
return self.browser.iter_movie_persons_ids(id)
|
||||
|
||||
def get_person_biography(self, id):
|
||||
return self.browser.get_person_biography(id)
|
||||
|
||||
def get_movie_releases(self, id, country=None):
|
||||
return self.browser.get_movie_releases(id, country)
|
||||
|
||||
def fill_person(self, person, fields):
|
||||
if 'real_name' in fields or 'birth_place' in fields\
|
||||
or 'death_date' in fields or 'nationality' in fields\
|
||||
or 'short_biography' in fields or 'roles' in fields\
|
||||
or 'birth_date' in fields or 'thumbnail_url' in fields\
|
||||
or 'gender' in fields or fields is None:
|
||||
per = self.get_person(person.id)
|
||||
person.real_name = per.real_name
|
||||
person.birth_date = per.birth_date
|
||||
person.death_date = per.death_date
|
||||
person.birth_place = per.birth_place
|
||||
person.gender = per.gender
|
||||
person.nationality = per.nationality
|
||||
person.short_biography = per.short_biography
|
||||
person.short_description = per.short_description
|
||||
person.roles = per.roles
|
||||
person.thumbnail_url = per.thumbnail_url
|
||||
|
||||
if 'biography' in fields:
|
||||
person.biography = self.get_person_biography(person.id)
|
||||
|
||||
return person
|
||||
|
||||
def fill_movie(self, movie, fields):
|
||||
if 'other_titles' in fields or 'release_date' in fields\
|
||||
or 'duration' in fields or 'country' in fields\
|
||||
or 'roles' in fields or 'note' in fields\
|
||||
or 'thumbnail_url' in fields:
|
||||
mov = self.get_movie(movie.id)
|
||||
movie.other_titles = mov.other_titles
|
||||
movie.release_date = mov.release_date
|
||||
movie.duration = mov.duration
|
||||
movie.pitch = mov.pitch
|
||||
movie.country = mov.country
|
||||
movie.note = mov.note
|
||||
movie.roles = mov.roles
|
||||
movie.genres = mov.genres
|
||||
movie.short_description = mov.short_description
|
||||
movie.thumbnail_url = mov.thumbnail_url
|
||||
|
||||
if 'all_release_dates' in fields:
|
||||
movie.all_release_dates = self.get_movie_releases(movie.id)
|
||||
|
||||
return movie
|
||||
|
||||
OBJECTS = {
|
||||
Person: fill_person,
|
||||
Movie: fill_movie
|
||||
}
|
||||
238
modules/allocine/browser.py
Normal file
238
modules/allocine/browser.py
Normal file
|
|
@ -0,0 +1,238 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright(C) 2013 Julien Veyssier
|
||||
#
|
||||
# This file is part of weboob.
|
||||
#
|
||||
# weboob is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# weboob is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
import HTMLParser
|
||||
from weboob.tools.browser import BaseBrowser, BrowserHTTPNotFound
|
||||
from weboob.capabilities.base import NotAvailable, NotLoaded
|
||||
from weboob.capabilities.cinema import Movie, Person
|
||||
from weboob.tools.json import json
|
||||
|
||||
from .pages import PersonPage, MovieCrewPage, BiographyPage, FilmographyPage, ReleasePage
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
__all__ = ['AllocineBrowser']
|
||||
|
||||
|
||||
class AllocineBrowser(BaseBrowser):
|
||||
DOMAIN = 'api.allocine.fr'
|
||||
PROTOCOL = 'http'
|
||||
ENCODING = 'utf-8'
|
||||
USER_AGENT = BaseBrowser.USER_AGENTS['wget']
|
||||
#PAGES = {
|
||||
# 'http://www.imdb.com/title/tt[0-9]*/fullcredits.*': MovieCrewPage,
|
||||
# 'http://www.imdb.com/title/tt[0-9]*/releaseinfo.*': ReleasePage,
|
||||
# 'http://www.imdb.com/name/nm[0-9]*/*': PersonPage,
|
||||
# 'http://www.imdb.com/name/nm[0-9]*/bio.*': BiographyPage,
|
||||
# 'http://www.imdb.com/name/nm[0-9]*/filmo.*': FilmographyPage,
|
||||
#}
|
||||
|
||||
def iter_movies(self, pattern):
|
||||
res = self.readurl('http://api.allocine.fr/rest/v3/search?partner=YW5kcm9pZC12M3M&filter=movie&q=%s&format=json' % pattern.encode('utf-8'))
|
||||
jres = json.loads(res)
|
||||
for m in jres['feed']['movie']:
|
||||
tdesc = u''
|
||||
if 'title' in m:
|
||||
tdesc += '%s' % m['title']
|
||||
if 'productionYear' in m:
|
||||
tdesc += ' ; %s' % m['productionYear']
|
||||
elif 'release' in m:
|
||||
tdesc += ' ; %s' % m['release']['releaseDate']
|
||||
short_description = tdesc.strip('; ')
|
||||
movie = Movie(m['code'], unicode(m['originalTitle']))
|
||||
movie.other_titles = NotLoaded
|
||||
movie.release_date = NotLoaded
|
||||
movie.duration = NotLoaded
|
||||
movie.short_description = short_description
|
||||
movie.pitch = NotLoaded
|
||||
movie.country = NotLoaded
|
||||
movie.note = NotLoaded
|
||||
movie.roles = NotLoaded
|
||||
movie.all_release_dates = NotLoaded
|
||||
movie.thumbnail_url = NotLoaded
|
||||
yield movie
|
||||
|
||||
def iter_persons(self, pattern):
|
||||
res = self.readurl('http://api.allocine.fr/rest/v3/search?partner=YW5kcm9pZC12M3M&filter=person&q=%s&format=json' % pattern.encode('utf-8'))
|
||||
jres = json.loads(res)
|
||||
for p in jres['feed']['person']:
|
||||
thumbnail_url = NotAvailable
|
||||
if 'picture' in p:
|
||||
thumbnail_url = unicode(p['picture']['href'])
|
||||
person = Person(p['code'], unicode(p['name']))
|
||||
desc = u''
|
||||
if 'birthDate' in p:
|
||||
desc += '(%s), ' % p['birthDate']
|
||||
if 'activity' in p:
|
||||
for a in p['activity']:
|
||||
desc += '%s, ' % a['$']
|
||||
person.real_name = NotLoaded
|
||||
person.birth_place = NotLoaded
|
||||
person.birth_date = NotLoaded
|
||||
person.death_date = NotLoaded
|
||||
person.gender = NotLoaded
|
||||
person.nationality = NotLoaded
|
||||
person.short_biography = NotLoaded
|
||||
person.short_description = desc.strip(', ')
|
||||
person.roles = NotLoaded
|
||||
person.thumbnail_url = thumbnail_url
|
||||
yield person
|
||||
|
||||
def get_movie(self, id):
|
||||
res = self.readurl(
|
||||
'http://api.allocine.fr/rest/v3/movie?partner=YW5kcm9pZC12M3M&code=%s&profile=large&mediafmt=mp4-lc&format=json&filter=movie&striptags=synopsis,synopsisshort' % id)
|
||||
if res is not None:
|
||||
jres = json.loads(res)['movie']
|
||||
else:
|
||||
return None
|
||||
title = NotAvailable
|
||||
duration = NotAvailable
|
||||
release_date = NotAvailable
|
||||
pitch = NotAvailable
|
||||
country = NotAvailable
|
||||
note = NotAvailable
|
||||
short_description = NotAvailable
|
||||
thumbnail_url = NotAvailable
|
||||
other_titles = []
|
||||
genres = []
|
||||
roles = {}
|
||||
|
||||
if 'originalTitle' not in jres:
|
||||
return
|
||||
title = unicode(jres['originalTitle'].strip())
|
||||
if 'picture' in jres:
|
||||
thumbnail_url = unicode(jres['picture']['href'])
|
||||
if 'genre' in jres:
|
||||
for g in jres['genre']:
|
||||
genres.append(g['$'])
|
||||
if 'runtime' in jres:
|
||||
nbsecs = jres['runtime']
|
||||
duration = nbsecs / 60
|
||||
#if 'also_known_as' in jres:
|
||||
# for other_t in jres['also_known_as']:
|
||||
# if 'country' in other_t and 'title' in other_t:
|
||||
# other_titles.append('%s : %s' % (other_t['country'], htmlparser.unescape(other_t['title'])))
|
||||
if 'release' in jres:
|
||||
dstr = str(jres['release']['releaseDate'])
|
||||
tdate = dstr.split('-')
|
||||
day = 1
|
||||
month = 1
|
||||
year = 1901
|
||||
if len(tdate) > 2:
|
||||
year = int(tdate[0])
|
||||
month = int(tdate[1])
|
||||
day = int(tdate[2])
|
||||
release_date = datetime(year, month, day)
|
||||
if 'nationality' in jres:
|
||||
country = u''
|
||||
for c in jres['nationality']:
|
||||
country += '%s, ' % c['$']
|
||||
country = country.strip(', ')
|
||||
if 'synopsis' in jres:
|
||||
pitch = unicode(jres['synopsis'])
|
||||
if 'statistics' in jres and 'userRating' in jres['statistics']:
|
||||
note = u'%s/10 (%s votes)' % (jres['statistics']['userRating'], jres['statistics']['userReviewCount'])
|
||||
if 'castMember' in jres:
|
||||
for cast in jres['castMember']:
|
||||
if cast['activity']['$'] not in roles:
|
||||
roles[cast['activity']['$']] = []
|
||||
roles[cast['activity']['$']].append(cast['person']['name'])
|
||||
|
||||
movie = Movie(id, title)
|
||||
movie.other_titles = other_titles
|
||||
movie.release_date = release_date
|
||||
movie.duration = duration
|
||||
movie.genres = genres
|
||||
movie.pitch = pitch
|
||||
movie.country = country
|
||||
movie.note = note
|
||||
movie.roles = roles
|
||||
movie.short_description = short_description
|
||||
movie.all_release_dates = NotLoaded
|
||||
movie.thumbnail_url = thumbnail_url
|
||||
return movie
|
||||
|
||||
def get_person(self, id):
|
||||
try:
|
||||
self.location('http://www.imdb.com/name/%s' % id)
|
||||
except BrowserHTTPNotFound:
|
||||
return
|
||||
assert self.is_on_page(PersonPage)
|
||||
return self.page.get_person(id)
|
||||
|
||||
def get_person_biography(self, id):
|
||||
self.location('http://www.imdb.com/name/%s/bio' % id)
|
||||
assert self.is_on_page(BiographyPage)
|
||||
return self.page.get_biography()
|
||||
|
||||
def iter_movie_persons(self, movie_id, role):
|
||||
self.location('http://www.imdb.com/title/%s/fullcredits' % movie_id)
|
||||
assert self.is_on_page(MovieCrewPage)
|
||||
for p in self.page.iter_persons(role):
|
||||
yield p
|
||||
|
||||
def iter_person_movies(self, person_id, role):
|
||||
self.location('http://www.imdb.com/name/%s/filmotype' % person_id)
|
||||
assert self.is_on_page(FilmographyPage)
|
||||
return self.page.iter_movies(role)
|
||||
|
||||
def iter_person_movies_ids(self, person_id):
|
||||
self.location('http://www.imdb.com/name/%s/filmotype' % person_id)
|
||||
assert self.is_on_page(FilmographyPage)
|
||||
for movie in self.page.iter_movies_ids():
|
||||
yield movie
|
||||
|
||||
def iter_movie_persons_ids(self, movie_id):
|
||||
self.location('http://www.imdb.com/title/%s/fullcredits' % movie_id)
|
||||
assert self.is_on_page(MovieCrewPage)
|
||||
for person in self.page.iter_persons_ids():
|
||||
yield person
|
||||
|
||||
def get_movie_releases(self, id, country):
|
||||
return
|
||||
self.location('http://www.imdb.com/title/%s/releaseinfo' % id)
|
||||
assert self.is_on_page(ReleasePage)
|
||||
return self.page.get_movie_releases(country)
|
||||
|
||||
|
||||
dict_hex = {'á': u'á',
|
||||
'é': u'é',
|
||||
'è': u'è',
|
||||
'í': u'í',
|
||||
'ñ': u'ñ',
|
||||
'ó': u'ó',
|
||||
'ú': u'ú',
|
||||
'ü': u'ü',
|
||||
'&': u'&',
|
||||
''': u"'",
|
||||
'à': u'à',
|
||||
'À': u'À',
|
||||
'â': u'â',
|
||||
'É': u'É',
|
||||
'ë': u'ë',
|
||||
'ô': u'ô',
|
||||
'ç': u'ç'
|
||||
}
|
||||
|
||||
|
||||
def latin2unicode(word):
|
||||
for key in dict_hex.keys():
|
||||
word = word.replace(key, dict_hex[key])
|
||||
return unicode(word)
|
||||
231
modules/allocine/pages.py
Normal file
231
modules/allocine/pages.py
Normal file
|
|
@ -0,0 +1,231 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright(C) 2013 Julien Veyssier
|
||||
#
|
||||
# This file is part of weboob.
|
||||
#
|
||||
# weboob is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# weboob is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
from weboob.capabilities.cinema import Person, Movie
|
||||
from weboob.capabilities.base import NotAvailable, NotLoaded
|
||||
from weboob.tools.browser import BasePage
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
__all__ = ['PersonPage', 'MovieCrewPage', 'BiographyPage', 'FilmographyPage', 'ReleasePage']
|
||||
|
||||
|
||||
class ReleasePage(BasePage):
|
||||
''' Page containing releases of a movie
|
||||
'''
|
||||
def get_movie_releases(self, country_filter):
|
||||
result = unicode()
|
||||
links = self.parser.select(self.document.getroot(), 'b a')
|
||||
for a in links:
|
||||
href = a.attrib.get('href', '')
|
||||
if href.strip('/').split('/')[0] == 'calendar' and\
|
||||
(country_filter is None or href.split('region=')[-1].lower() == country_filter):
|
||||
country = a.text
|
||||
td_date = self.parser.select(a.getparent().getparent().getparent(), 'td')[1]
|
||||
date_links = self.parser.select(td_date, 'a')
|
||||
if len(date_links) > 1:
|
||||
date = date_links[1].attrib.get('href', '').strip('/').split('/')[-1]
|
||||
date += '-'+date_links[0].attrib.get('href', '').strip('/').split('/')[-1]
|
||||
else:
|
||||
date = unicode(self.parser.select(a.getparent().getparent().getparent(), 'td')[1].text_content())
|
||||
result += '%s : %s\n' % (country, date)
|
||||
if result == u'':
|
||||
result = NotAvailable
|
||||
else:
|
||||
result = result.strip()
|
||||
return result
|
||||
|
||||
|
||||
class BiographyPage(BasePage):
|
||||
''' Page containing biography of a person
|
||||
'''
|
||||
def get_biography(self):
|
||||
bio = unicode()
|
||||
tn = self.parser.select(self.document.getroot(), 'div#tn15content', 1)
|
||||
# we only read paragraphs, titles and links
|
||||
for ch in tn.getchildren():
|
||||
if ch.tag in ['p', 'h5', 'a']:
|
||||
bio += '%s\n\n' % ch.text_content().strip()
|
||||
if bio == u'':
|
||||
bio = NotAvailable
|
||||
return bio
|
||||
|
||||
|
||||
class MovieCrewPage(BasePage):
|
||||
''' Page listing all the persons related to a movie
|
||||
'''
|
||||
def iter_persons(self, role_filter=None):
|
||||
if (role_filter is None or (role_filter is not None and role_filter == 'actor')):
|
||||
tables = self.parser.select(self.document.getroot(), 'table.cast')
|
||||
if len(tables) > 0:
|
||||
table = tables[0]
|
||||
tds = self.parser.select(table, 'td.nm')
|
||||
for td in tds:
|
||||
id = td.find('a').attrib.get('href', '').strip('/').split('/')[-1]
|
||||
name = unicode(td.find('a').text)
|
||||
char_name = unicode(self.parser.select(td.getparent(), 'td.char', 1).text_content())
|
||||
person = Person(id, name)
|
||||
person.short_description = char_name
|
||||
person.real_name = NotLoaded
|
||||
person.birth_place = NotLoaded
|
||||
person.birth_date = NotLoaded
|
||||
person.death_date = NotLoaded
|
||||
person.gender = NotLoaded
|
||||
person.nationality = NotLoaded
|
||||
person.short_biography = NotLoaded
|
||||
person.roles = NotLoaded
|
||||
person.thumbnail_url = NotLoaded
|
||||
yield person
|
||||
|
||||
for gloss_link in self.parser.select(self.document.getroot(), 'table[cellspacing=1] h5 a'):
|
||||
role = gloss_link.attrib.get('name', '').rstrip('s')
|
||||
if (role_filter is None or (role_filter is not None and role == role_filter)):
|
||||
tbody = gloss_link.getparent().getparent().getparent().getparent()
|
||||
for line in self.parser.select(tbody, 'tr')[1:]:
|
||||
for a in self.parser.select(line, 'a'):
|
||||
role_detail = NotAvailable
|
||||
href = a.attrib.get('href', '')
|
||||
if '/name/nm' in href:
|
||||
id = href.strip('/').split('/')[-1]
|
||||
name = unicode(a.text)
|
||||
if 'glossary' in href:
|
||||
role_detail = unicode(a.text)
|
||||
person = Person(id, name)
|
||||
person.short_description = role_detail
|
||||
yield person
|
||||
# yield self.browser.get_person(id)
|
||||
|
||||
def iter_persons_ids(self):
|
||||
tables = self.parser.select(self.document.getroot(), 'table.cast')
|
||||
if len(tables) > 0:
|
||||
table = tables[0]
|
||||
tds = self.parser.select(table, 'td.nm')
|
||||
for td in tds:
|
||||
id = td.find('a').attrib.get('href', '').strip('/').split('/')[-1]
|
||||
yield id
|
||||
|
||||
|
||||
class PersonPage(BasePage):
|
||||
''' Page giving informations about a person
|
||||
It is used to build a Person instance and to get the movie list related to a person
|
||||
'''
|
||||
def get_person(self, id):
|
||||
name = NotAvailable
|
||||
short_biography = NotAvailable
|
||||
short_description = NotAvailable
|
||||
birth_place = NotAvailable
|
||||
birth_date = NotAvailable
|
||||
death_date = NotAvailable
|
||||
real_name = NotAvailable
|
||||
gender = NotAvailable
|
||||
thumbnail_url = NotAvailable
|
||||
roles = {}
|
||||
nationality = NotAvailable
|
||||
td_overview = self.parser.select(self.document.getroot(), 'td#overview-top', 1)
|
||||
descs = self.parser.select(td_overview, 'span[itemprop=description]')
|
||||
if len(descs) > 0:
|
||||
short_biography = unicode(descs[0].text)
|
||||
rname_block = self.parser.select(td_overview, 'div.txt-block h4.inline')
|
||||
if len(rname_block) > 0 and "born" in rname_block[0].text.lower():
|
||||
links = self.parser.select(rname_block[0].getparent(), 'a')
|
||||
for a in links:
|
||||
href = a.attrib.get('href', '').strip()
|
||||
if href == 'bio':
|
||||
real_name = unicode(a.text.strip())
|
||||
elif 'birth_place' in href:
|
||||
birth_place = unicode(a.text.lower().strip())
|
||||
names = self.parser.select(td_overview, 'h1[itemprop=name]')
|
||||
if len(names) > 0:
|
||||
name = unicode(names[0].text.strip())
|
||||
times = self.parser.select(td_overview, 'time[itemprop=birthDate]')
|
||||
if len(times) > 0:
|
||||
time = times[0].attrib.get('datetime', '').split('-')
|
||||
if len(time) == 3 and int(time[0]) >= 1900:
|
||||
birth_date = datetime(int(time[0]), int(time[1]), int(time[2]))
|
||||
dtimes = self.parser.select(td_overview, 'time[itemprop=deathDate]')
|
||||
if len(dtimes) > 0:
|
||||
dtime = dtimes[0].attrib.get('datetime', '').split('-')
|
||||
if len(dtime) == 3 and int(dtime[0]) >= 1900:
|
||||
death_date = datetime(int(dtime[0]), int(dtime[1]), int(dtime[2]))
|
||||
img_thumbnail = self.parser.select(self.document.getroot(), 'td#img_primary img')
|
||||
if len(img_thumbnail) > 0:
|
||||
thumbnail_url = unicode(img_thumbnail[0].attrib.get('src', ''))
|
||||
|
||||
# go to the filmography page
|
||||
self.browser.location('http://www.imdb.com/name/%s/filmotype' % id)
|
||||
assert self.browser.is_on_page(FilmographyPage)
|
||||
roles = self.browser.page.get_roles()
|
||||
|
||||
person = Person(id, name)
|
||||
person.real_name = real_name
|
||||
person.birth_date = birth_date
|
||||
person.death_date = death_date
|
||||
person.birth_place = birth_place
|
||||
person.gender = gender
|
||||
person.nationality = nationality
|
||||
person.short_biography = short_biography
|
||||
person.short_description = short_description
|
||||
person.roles = roles
|
||||
person.thumbnail_url = thumbnail_url
|
||||
return person
|
||||
|
||||
|
||||
class FilmographyPage(BasePage):
|
||||
''' Page of detailed filmography of a person, sorted by type of role
|
||||
This page is easier to parse than the main person page filmography
|
||||
'''
|
||||
def iter_movies_ids(self):
|
||||
for role_div in self.parser.select(self.document.getroot(), 'div.filmo'):
|
||||
for a in self.parser.select(role_div, 'ol > li > a'):
|
||||
id = a.attrib.get('href', '').strip('/').split('/')[-1]
|
||||
if id.startswith('tt'):
|
||||
yield id
|
||||
|
||||
def get_roles(self):
|
||||
roles = {}
|
||||
for role_div in self.parser.select(self.document.getroot(), 'div.filmo'):
|
||||
role = self.parser.select(role_div, 'h5 a', 1).text.replace(':', '')
|
||||
roles[role] = []
|
||||
for a in self.parser.select(role_div, 'ol > li > a'):
|
||||
id = a.attrib.get('href', '').strip('/').split('/')[-1]
|
||||
if id.startswith('tt'):
|
||||
if '(' in a.tail and ')' in a.tail:
|
||||
between_p = a.tail.split(')')[0].split('(')[1]
|
||||
else:
|
||||
between_p = '????'
|
||||
roles[role].append('(%s) %s' % (between_p, a.text))
|
||||
return roles
|
||||
|
||||
def iter_movies(self, role_filter=None):
|
||||
for role_div in self.parser.select(self.document.getroot(), 'div.filmo'):
|
||||
role = self.parser.select(role_div, 'h5 a', 1).text.replace(':', '')
|
||||
if (role_filter is None or (role_filter is not None and role.lower().strip() == role_filter))\
|
||||
and role != 'In Development':
|
||||
for a in self.parser.select(role_div, 'ol > li > a'):
|
||||
id = a.attrib.get('href', '').strip('/').split('/')[-1]
|
||||
if id.startswith('tt'):
|
||||
title = unicode(a.text)
|
||||
role_detail = NotAvailable
|
||||
if len(a.tail) > 0:
|
||||
role_detail = unicode(' '.join(a.tail.replace('..', '').split()))
|
||||
movie = Movie(id, title)
|
||||
movie.short_description = role_detail
|
||||
yield movie
|
||||
67
modules/allocine/test.py
Normal file
67
modules/allocine/test.py
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright(C) 2013 Julien Veyssier
|
||||
#
|
||||
# This file is part of weboob.
|
||||
#
|
||||
# weboob is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# weboob is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
from weboob.tools.test import BackendTest
|
||||
|
||||
|
||||
class ImdbTest(BackendTest):
|
||||
BACKEND = 'imdb'
|
||||
|
||||
def test_search_movie(self):
|
||||
movies = list(self.backend.iter_movies('spiderman'))
|
||||
for movie in movies:
|
||||
assert movie.id
|
||||
|
||||
def test_get_movie(self):
|
||||
movie = self.backend.get_movie('tt0079980')
|
||||
assert movie.id
|
||||
assert movie.original_title
|
||||
|
||||
def test_search_person(self):
|
||||
persons = list(self.backend.iter_persons('dewaere'))
|
||||
for person in persons:
|
||||
assert person.id
|
||||
|
||||
def test_get_person(self):
|
||||
person = self.backend.get_person('nm0223033')
|
||||
assert person.id
|
||||
assert person.name
|
||||
assert person.birth_date
|
||||
|
||||
def test_movie_persons(self):
|
||||
persons = list(self.backend.iter_movie_persons('tt0079980'))
|
||||
for person in persons:
|
||||
assert person.id
|
||||
assert person.name
|
||||
|
||||
def test_person_movies(self):
|
||||
movies = list(self.backend.iter_person_movies('nm0223033'))
|
||||
for movie in movies:
|
||||
assert movie.id
|
||||
assert movie.original_title
|
||||
|
||||
def test_get_person_biography(self):
|
||||
bio = self.backend.get_person_biography('nm0223033')
|
||||
assert bio != ''
|
||||
assert bio is not None
|
||||
|
||||
def test_get_movie_releases(self):
|
||||
rel = self.backend.get_movie_releases('tt0079980')
|
||||
assert rel != ''
|
||||
assert rel is not None
|
||||
Loading…
Add table
Add a link
Reference in a new issue