nojhan/weboob-devel
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

autopep8 with 120 chars line length on my modules

Browse source
This commit is contained in:
Julien Veyssier 2013-03-16 01:39:35 +01:00
commit 5d923bc73b
39 changed files with 434 additions and 426 deletions
Download patch file Download diff file Expand all files Collapse all files

56
modules/imdb/backend.py
View file

@ -67,25 +67,25 @@ class ImdbBackend(BaseBackend, ICapCinema):
return self.browser.get_person_biography(id)
def get_movie_releases(self, id, country=None):
return self.browser.get_movie_releases(id,country)
return self.browser.get_movie_releases(id, country)
def fill_person(self, person, fields):
if 'real_name' in fields or 'birth_place' in fields\
or 'death_date' in fields or 'nationality' in fields\
or 'short_biography' in fields or 'roles' in fields\
or 'birth_date' in fields or 'thumbnail_url' in fields\
or 'gender' in fields or fields is None:
or 'death_date' in fields or 'nationality' in fields\
or 'short_biography' in fields or 'roles' in fields\
or 'birth_date' in fields or 'thumbnail_url' in fields\
or 'gender' in fields or fields is None:
per = self.get_person(person.id)
person.real_name = per.real_name
person.birth_date = per.birth_date
person.death_date = per.death_date
person.birth_place = per.birth_place
person.gender = per.gender
person.nationality = per.nationality
person.real_name = per.real_name
person.birth_date = per.birth_date
person.death_date = per.death_date
person.birth_place = per.birth_place
person.gender = per.gender
person.nationality = per.nationality
person.short_biography = per.short_biography
person.short_description = per.short_description
person.roles = per.roles
person.thumbnail_url = per.thumbnail_url
person.roles = per.roles
person.thumbnail_url = per.thumbnail_url
if 'biography' in fields:
person.biography = self.get_person_biography(person.id)
@ -94,19 +94,19 @@ class ImdbBackend(BaseBackend, ICapCinema):
def fill_movie(self, movie, fields):
if 'other_titles' in fields or 'release_date' in fields\
or 'duration' in fields or 'country' in fields\
or 'roles' in fields or 'note' in fields\
or 'thumbnail_url' in fields:
or 'duration' in fields or 'country' in fields\
or 'roles' in fields or 'note' in fields\
or 'thumbnail_url' in fields:
mov = self.get_movie(movie.id)
movie.other_titles = mov.other_titles
movie.release_date = mov.release_date
movie.duration = mov.duration
movie.pitch = mov.pitch
movie.country = mov.country
movie.note = mov.note
movie.roles = mov.roles
movie.short_description= mov.short_description
movie.thumbnail_url = mov.thumbnail_url
movie.other_titles = mov.other_titles
movie.release_date = mov.release_date
movie.duration = mov.duration
movie.pitch = mov.pitch
movie.country = mov.country
movie.note = mov.note
movie.roles = mov.roles
movie.short_description = mov.short_description
movie.thumbnail_url = mov.thumbnail_url
if 'all_release_dates' in fields:
movie.all_release_dates = self.get_movie_releases(movie.id)
@ -114,6 +114,6 @@ class ImdbBackend(BaseBackend, ICapCinema):
return movie
OBJECTS = {
Person:fill_person,
Movie:fill_movie
}
Person: fill_person,
Movie: fill_movie
}

100
modules/imdb/browser.py
View file

@ -42,53 +42,55 @@ class ImdbBrowser(BaseBrowser):
'http://www.imdb.com/name/nm[0-9]*/*': PersonPage,
'http://www.imdb.com/name/nm[0-9]*/bio.*': BiographyPage,
'http://www.imdb.com/name/nm[0-9]*/filmo.*': FilmographyPage,
}
}
def iter_movies(self, pattern):
res = self.readurl('http://www.imdb.com/xml/find?json=1&nr=1&tt=on&q=%s' % pattern.encode('utf-8'))
jres = json.loads(res)
for cat in ['title_popular','title_exact','title_approx']:
for cat in ['title_popular', 'title_exact', 'title_approx']:
if cat in jres:
for m in jres[cat]:
tdesc = unicode(m['title_description'])
if '<a' in tdesc and '>' in tdesc:
short_description = u'%s %s'%(tdesc.split('<')[0].strip(', '), tdesc.split('>')[1].split('<')[0])
short_description = u'%s %s' % (tdesc.split('<')[
0].strip(', '), tdesc.split('>')[1].split('<')[0])
else:
short_description = tdesc.strip(', ')
movie = Movie(m['id'],latin2unicode(m['title']))
movie.other_titles = NotLoaded
movie.release_date = NotLoaded
movie.duration = NotLoaded
movie = Movie(m['id'], latin2unicode(m['title']))
movie.other_titles = NotLoaded
movie.release_date = NotLoaded
movie.duration = NotLoaded
movie.short_description = latin2unicode(short_description)
movie.pitch = NotLoaded
movie.country = NotLoaded
movie.note = NotLoaded
movie.roles = NotLoaded
movie.all_release_dates= NotLoaded
movie.thumbnail_url = NotLoaded
movie.pitch = NotLoaded
movie.country = NotLoaded
movie.note = NotLoaded
movie.roles = NotLoaded
movie.all_release_dates = NotLoaded
movie.thumbnail_url = NotLoaded
yield movie
def iter_persons(self, pattern):
res = self.readurl('http://www.imdb.com/xml/find?json=1&nr=1&nm=on&q=%s' % pattern.encode('utf-8'))
jres = json.loads(res)
for cat in ['name_popular','name_exact','name_approx']:
for cat in ['name_popular', 'name_exact', 'name_approx']:
if cat in jres:
for p in jres[cat]:
person = Person(p['id'],latin2unicode(p['name']))
person.real_name = NotLoaded
person.birth_place = NotLoaded
person.birth_date = NotLoaded
person.death_date = NotLoaded
person.gender = NotLoaded
person.nationality = NotLoaded
person.short_biography= NotLoaded
person.short_description= latin2unicode(p['description'])
person.roles = NotLoaded
person.thumbnail_url = NotLoaded
person = Person(p['id'], latin2unicode(p['name']))
person.real_name = NotLoaded
person.birth_place = NotLoaded
person.birth_date = NotLoaded
person.death_date = NotLoaded
person.gender = NotLoaded
person.nationality = NotLoaded
person.short_biography = NotLoaded
person.short_description = latin2unicode(p['description'])
person.roles = NotLoaded
person.thumbnail_url = NotLoaded
yield person
def get_movie(self, id):
res = self.readurl('http://imdbapi.org/?id=%s&type=json&plot=simple&episode=1&lang=en-US&aka=full&release=simple&business=0&tech=0' % id )
res = self.readurl(
'http://imdbapi.org/?id=%s&type=json&plot=simple&episode=1&lang=en-US&aka=full&release=simple&business=0&tech=0' % id)
if res is not None:
jres = json.loads(res)
else:
@ -122,7 +124,7 @@ class ImdbBrowser(BaseBrowser):
if 'also_known_as' in jres:
for other_t in jres['also_known_as']:
if 'country' in other_t and 'title' in other_t:
other_titles.append('%s : %s' % (other_t['country'],htmlparser.unescape(other_t['title'])))
other_titles.append('%s : %s' % (other_t['country'], htmlparser.unescape(other_t['title'])))
if 'release_date' in jres:
dstr = str(jres['release_date'])
year = int(dstr[:4])
@ -134,31 +136,31 @@ class ImdbBrowser(BaseBrowser):
day = int(dstr[-2:])
if day == 0:
day = 1
release_date = datetime(year,month,day)
release_date = datetime(year, month, day)
if 'country' in jres:
country = u''
for c in jres['country']:
country += '%s, '%c
country += '%s, ' % c
country = country[:-2]
if 'plot_simple' in jres:
pitch = unicode(jres['plot_simple'])
if 'rating' in jres and 'rating_count' in jres:
note = u'%s/10 (%s votes)'%(jres['rating'],jres['rating_count'])
for r in ['actor','director','writer']:
if '%ss'%r in jres:
roles['%s'%r] = list(jres['%ss'%r])
note = u'%s/10 (%s votes)' % (jres['rating'], jres['rating_count'])
for r in ['actor', 'director', 'writer']:
if '%ss' % r in jres:
roles['%s' % r] = list(jres['%ss' % r])
movie = Movie(id,title)
movie.other_titles = other_titles
movie.release_date = release_date
movie.duration = duration
movie.pitch = pitch
movie.country = country
movie.note = note
movie.roles = roles
movie.short_description= short_description
movie.all_release_dates= NotLoaded
movie.thumbnail_url = thumbnail_url
movie = Movie(id, title)
movie.other_titles = other_titles
movie.release_date = release_date
movie.duration = duration
movie.pitch = pitch
movie.country = country
movie.note = note
movie.roles = roles
movie.short_description = short_description
movie.all_release_dates = NotLoaded
movie.thumbnail_url = thumbnail_url
return movie
def get_person(self, id):
@ -175,7 +177,7 @@ class ImdbBrowser(BaseBrowser):
return self.page.get_biography()
def iter_movie_persons(self, movie_id, role):
self.location('http://www.imdb.com/title/%s/fullcredits'%movie_id)
self.location('http://www.imdb.com/title/%s/fullcredits' % movie_id)
assert self.is_on_page(MovieCrewPage)
for p in self.page.iter_persons(role):
yield p
@ -192,13 +194,13 @@ class ImdbBrowser(BaseBrowser):
yield movie
def iter_movie_persons_ids(self, movie_id):
self.location('http://www.imdb.com/title/%s/fullcredits'%movie_id)
self.location('http://www.imdb.com/title/%s/fullcredits' % movie_id)
assert self.is_on_page(MovieCrewPage)
for person in self.page.iter_persons_ids():
yield person
def get_movie_releases(self,id, country):
self.location('http://www.imdb.com/title/%s/releaseinfo'%id)
def get_movie_releases(self, id, country):
self.location('http://www.imdb.com/title/%s/releaseinfo' % id)
assert self.is_on_page(ReleasePage)
return self.page.get_movie_releases(country)
@ -222,5 +224,5 @@ dict_hex = {'&#xE1;': u'á',
def latin2unicode(word):
for key in dict_hex.keys():
word = word.replace(key,dict_hex[key])
word = word.replace(key, dict_hex[key])
return unicode(word)

156
modules/imdb/pages.py
View file

@ -25,28 +25,28 @@ from weboob.tools.browser import BasePage
from datetime import datetime
__all__ = ['PersonPage','MovieCrewPage','BiographyPage','FilmographyPage','ReleasePage']
__all__ = ['PersonPage', 'MovieCrewPage', 'BiographyPage', 'FilmographyPage', 'ReleasePage']
class ReleasePage(BasePage):
''' Page containing releases of a movie
'''
def get_movie_releases(self,country_filter):
def get_movie_releases(self, country_filter):
result = unicode()
links = self.parser.select(self.document.getroot(),'b a')
links = self.parser.select(self.document.getroot(), 'b a')
for a in links:
href = a.attrib.get('href','')
href = a.attrib.get('href', '')
if href.strip('/').split('/')[0] == 'calendar' and\
(country_filter is None or href.split('region=')[-1].lower() == country_filter):
(country_filter is None or href.split('region=')[-1].lower() == country_filter):
country = a.text
td_date = self.parser.select(a.getparent().getparent().getparent(),'td')[1]
date_links = self.parser.select(td_date,'a')
td_date = self.parser.select(a.getparent().getparent().getparent(), 'td')[1]
date_links = self.parser.select(td_date, 'a')
if len(date_links) > 1:
date = date_links[1].attrib.get('href','').strip('/').split('/')[-1]
date += '-'+date_links[0].attrib.get('href','').strip('/').split('/')[-1]
date = date_links[1].attrib.get('href', '').strip('/').split('/')[-1]
date += '-'+date_links[0].attrib.get('href', '').strip('/').split('/')[-1]
else:
date = unicode(self.parser.select(a.getparent().getparent().getparent(),'td')[1].text_content())
result += '%s : %s\n' % (country,date)
date = unicode(self.parser.select(a.getparent().getparent().getparent(), 'td')[1].text_content())
result += '%s : %s\n' % (country, date)
if result == u'':
result = NotAvailable
else:
@ -59,11 +59,11 @@ class BiographyPage(BasePage):
'''
def get_biography(self):
bio = unicode()
tn = self.parser.select(self.document.getroot(),'div#tn15content',1)
tn = self.parser.select(self.document.getroot(), 'div#tn15content', 1)
# we only read paragraphs, titles and links
for ch in tn.getchildren():
if ch.tag in ['p','h5','a']:
bio += '%s\n\n'%ch.text_content().strip()
if ch.tag in ['p', 'h5', 'a']:
bio += '%s\n\n' % ch.text_content().strip()
if bio == u'':
bio = NotAvailable
return bio
@ -74,52 +74,52 @@ class MovieCrewPage(BasePage):
'''
def iter_persons(self, role_filter=None):
if (role_filter is None or (role_filter is not None and role_filter == 'actor')):
tables = self.parser.select(self.document.getroot(),'table.cast')
tables = self.parser.select(self.document.getroot(), 'table.cast')
if len(tables) > 0:
table = tables[0]
tds = self.parser.select(table,'td.nm')
tds = self.parser.select(table, 'td.nm')
for td in tds:
id = td.find('a').attrib.get('href','').strip('/').split('/')[-1]
id = td.find('a').attrib.get('href', '').strip('/').split('/')[-1]
name = unicode(td.find('a').text)
char_name = unicode(self.parser.select(td.getparent(),'td.char',1).text_content())
person = Person(id,name)
char_name = unicode(self.parser.select(td.getparent(), 'td.char', 1).text_content())
person = Person(id, name)
person.short_description = char_name
person.real_name = NotLoaded
person.birth_place = NotLoaded
person.birth_date = NotLoaded
person.death_date = NotLoaded
person.gender = NotLoaded
person.nationality = NotLoaded
person.short_biography= NotLoaded
person.roles = NotLoaded
person.thumbnail_url = NotLoaded
person.real_name = NotLoaded
person.birth_place = NotLoaded
person.birth_date = NotLoaded
person.death_date = NotLoaded
person.gender = NotLoaded
person.nationality = NotLoaded
person.short_biography = NotLoaded
person.roles = NotLoaded
person.thumbnail_url = NotLoaded
yield person
for gloss_link in self.parser.select(self.document.getroot(),'table[cellspacing=1] h5 a'):
role = gloss_link.attrib.get('name','').rstrip('s')
for gloss_link in self.parser.select(self.document.getroot(), 'table[cellspacing=1] h5 a'):
role = gloss_link.attrib.get('name', '').rstrip('s')
if (role_filter is None or (role_filter is not None and role == role_filter)):
tbody = gloss_link.getparent().getparent().getparent().getparent()
for line in self.parser.select(tbody,'tr')[1:]:
for a in self.parser.select(line,'a'):
for line in self.parser.select(tbody, 'tr')[1:]:
for a in self.parser.select(line, 'a'):
role_detail = NotAvailable
href = a.attrib.get('href','')
href = a.attrib.get('href', '')
if '/name/nm' in href:
id = href.strip('/').split('/')[-1]
name = unicode(a.text)
if 'glossary' in href:
role_detail = unicode(a.text)
person = Person(id,name)
person = Person(id, name)
person.short_description = role_detail
yield person
#yield self.browser.get_person(id)
# yield self.browser.get_person(id)
def iter_persons_ids(self):
tables = self.parser.select(self.document.getroot(),'table.cast')
tables = self.parser.select(self.document.getroot(), 'table.cast')
if len(tables) > 0:
table = tables[0]
tds = self.parser.select(table,'td.nm')
tds = self.parser.select(table, 'td.nm')
for td in tds:
id = td.find('a').attrib.get('href','').strip('/').split('/')[-1]
id = td.find('a').attrib.get('href', '').strip('/').split('/')[-1]
yield id
@ -127,7 +127,7 @@ class PersonPage(BasePage):
''' Page giving informations about a person
It is used to build a Person instance and to get the movie list related to a person
'''
def get_person(self,id):
def get_person(self, id):
name = NotAvailable
short_biography = NotAvailable
short_description = NotAvailable
@ -139,52 +139,52 @@ class PersonPage(BasePage):
thumbnail_url = NotAvailable
roles = {}
nationality = NotAvailable
td_overview = self.parser.select(self.document.getroot(),'td#overview-top',1)
descs = self.parser.select(td_overview,'span[itemprop=description]')
td_overview = self.parser.select(self.document.getroot(), 'td#overview-top', 1)
descs = self.parser.select(td_overview, 'span[itemprop=description]')
if len(descs) > 0:
short_biography = unicode(descs[0].text)
rname_block = self.parser.select(td_overview,'div.txt-block h4.inline')
rname_block = self.parser.select(td_overview, 'div.txt-block h4.inline')
if len(rname_block) > 0 and "born" in rname_block[0].text.lower():
links = self.parser.select(rname_block[0].getparent(),'a')
links = self.parser.select(rname_block[0].getparent(), 'a')
for a in links:
href = a.attrib.get('href','').strip()
href = a.attrib.get('href', '').strip()
if href == 'bio':
real_name = unicode(a.text.strip())
elif 'birth_place' in href:
birth_place = unicode(a.text.lower().strip())
names = self.parser.select(td_overview,'h1[itemprop=name]')
names = self.parser.select(td_overview, 'h1[itemprop=name]')
if len(names) > 0:
name = unicode(names[0].text.strip())
times = self.parser.select(td_overview,'time[itemprop=birthDate]')
times = self.parser.select(td_overview, 'time[itemprop=birthDate]')
if len(times) > 0:
time = times[0].attrib.get('datetime','').split('-')
time = times[0].attrib.get('datetime', '').split('-')
if len(time) == 3 and int(time[0]) >= 1900:
birth_date = datetime(int(time[0]),int(time[1]),int(time[2]))
dtimes = self.parser.select(td_overview,'time[itemprop=deathDate]')
birth_date = datetime(int(time[0]), int(time[1]), int(time[2]))
dtimes = self.parser.select(td_overview, 'time[itemprop=deathDate]')
if len(dtimes) > 0:
dtime = dtimes[0].attrib.get('datetime','').split('-')
dtime = dtimes[0].attrib.get('datetime', '').split('-')
if len(dtime) == 3 and int(dtime[0]) >= 1900:
death_date = datetime(int(dtime[0]),int(dtime[1]),int(dtime[2]))
img_thumbnail = self.parser.select(self.document.getroot(),'td#img_primary img')
death_date = datetime(int(dtime[0]), int(dtime[1]), int(dtime[2]))
img_thumbnail = self.parser.select(self.document.getroot(), 'td#img_primary img')
if len(img_thumbnail) > 0:
thumbnail_url = unicode(img_thumbnail[0].attrib.get('src',''))
thumbnail_url = unicode(img_thumbnail[0].attrib.get('src', ''))
# go to the filmography page
self.browser.location('http://www.imdb.com/name/%s/filmotype'%id)
self.browser.location('http://www.imdb.com/name/%s/filmotype' % id)
assert self.browser.is_on_page(FilmographyPage)
roles = self.browser.page.get_roles()
person = Person(id,name)
person.real_name = real_name
person.birth_date = birth_date
person.death_date = death_date
person.birth_place = birth_place
person.gender = gender
person.nationality = nationality
person = Person(id, name)
person.real_name = real_name
person.birth_date = birth_date
person.death_date = death_date
person.birth_place = birth_place
person.gender = gender
person.nationality = nationality
person.short_biography = short_biography
person.short_description = short_description
person.roles = roles
person.thumbnail_url = thumbnail_url
person.roles = roles
person.thumbnail_url = thumbnail_url
return person
@ -193,39 +193,39 @@ class FilmographyPage(BasePage):
This page is easier to parse than the main person page filmography
'''
def iter_movies_ids(self):
for role_div in self.parser.select(self.document.getroot(),'div.filmo'):
for a in self.parser.select(role_div,'ol > li > a'):
id = a.attrib.get('href','').strip('/').split('/')[-1]
for role_div in self.parser.select(self.document.getroot(), 'div.filmo'):
for a in self.parser.select(role_div, 'ol > li > a'):
id = a.attrib.get('href', '').strip('/').split('/')[-1]
if id.startswith('tt'):
yield id
def get_roles(self):
roles = {}
for role_div in self.parser.select(self.document.getroot(),'div.filmo'):
role = self.parser.select(role_div,'h5 a',1).text.replace(':','')
for role_div in self.parser.select(self.document.getroot(), 'div.filmo'):
role = self.parser.select(role_div, 'h5 a', 1).text.replace(':', '')
roles[role] = []
for a in self.parser.select(role_div,'ol > li > a'):
id = a.attrib.get('href','').strip('/').split('/')[-1]
for a in self.parser.select(role_div, 'ol > li > a'):
id = a.attrib.get('href', '').strip('/').split('/')[-1]
if id.startswith('tt'):
if '(' in a.tail and ')' in a.tail:
between_p = a.tail.split(')')[0].split('(')[1]
else:
between_p = '????'
roles[role].append('(%s) %s'%(between_p,a.text))
roles[role].append('(%s) %s' % (between_p, a.text))
return roles
def iter_movies(self, role_filter=None):
for role_div in self.parser.select(self.document.getroot(),'div.filmo'):
role = self.parser.select(role_div,'h5 a',1).text.replace(':','')
for role_div in self.parser.select(self.document.getroot(), 'div.filmo'):
role = self.parser.select(role_div, 'h5 a', 1).text.replace(':', '')
if (role_filter is None or (role_filter is not None and role.lower().strip() == role_filter))\
and role != 'In Development':
for a in self.parser.select(role_div,'ol > li > a'):
id = a.attrib.get('href','').strip('/').split('/')[-1]
and role != 'In Development':
for a in self.parser.select(role_div, 'ol > li > a'):
id = a.attrib.get('href', '').strip('/').split('/')[-1]
if id.startswith('tt'):
title = unicode(a.text)
role_detail = NotAvailable
if len(a.tail) > 0:
role_detail = unicode(' '.join(a.tail.replace('..','').split()))
movie = Movie(id,title)
role_detail = unicode(' '.join(a.tail.replace('..', '').split()))
movie = Movie(id, title)
movie.short_description = role_detail
yield movie