nojhan/weboob-devel
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

autopep8 with 120 chars line length on my modules

Browse source
This commit is contained in:
Julien Veyssier 2013-03-16 01:39:35 +01:00
commit 5d923bc73b
39 changed files with 434 additions and 426 deletions
Download patch file Download diff file Expand all files Collapse all files

4
modules/750g/backend.py
View file

@ -17,7 +17,7 @@
# You should have received a copy of the GNU Affero General Public License # You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>. # along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.capabilities.recipe import ICapRecipe,Recipe from weboob.capabilities.recipe import ICapRecipe, Recipe
from weboob.tools.backend import BaseBackend from weboob.tools.backend import BaseBackend
from .browser import SevenFiftyGramsBrowser from .browser import SevenFiftyGramsBrowser
@ -57,5 +57,5 @@ class SevenFiftyGramsBackend(BaseBackend, ICapRecipe):
return recipe return recipe
OBJECTS = { OBJECTS = {
Recipe:fill_recipe, Recipe: fill_recipe,
} }

2
modules/750g/browser.py
View file

@ -37,7 +37,7 @@ class SevenFiftyGramsBrowser(BaseBrowser):
} }
def iter_recipes(self, pattern): def iter_recipes(self, pattern):
self.location('http://www.750g.com/recettes_%s.htm' % (pattern.replace(' ','_'))) self.location('http://www.750g.com/recettes_%s.htm' % (pattern.replace(' ', '_')))
assert self.is_on_page(ResultsPage) assert self.is_on_page(ResultsPage)
return self.page.iter_recipes() return self.page.iter_recipes()

47
modules/750g/pages.py
View file

@ -30,31 +30,32 @@ class ResultsPage(BasePage):
""" Page which contains results as a list of recipies """ Page which contains results as a list of recipies
""" """
def iter_recipes(self): def iter_recipes(self):
for div in self.parser.select(self.document.getroot(),'div.recette_description > div.data'): for div in self.parser.select(self.document.getroot(), 'div.recette_description > div.data'):
links = self.parser.select(div,'div.info > p.title > a.fn') links = self.parser.select(div, 'div.info > p.title > a.fn')
if len(links) > 0: if len(links) > 0:
link = links[0] link = links[0]
title = unicode(link.text) title = unicode(link.text)
#id = unicode(link.attrib.get('href','').strip('/').replace('.htm','htm')) # id = unicode(link.attrib.get('href','').strip('/').replace('.htm','htm'))
id = unicode(self.parser.select(div,'div.carnet-add a',1).attrib.get('href','').split('=')[-1]) id = unicode(self.parser.select(div, 'div.carnet-add a', 1).attrib.get('href', '').split('=')[-1])
thumbnail_url = NotAvailable thumbnail_url = NotAvailable
short_description = NotAvailable short_description = NotAvailable
imgs = self.parser.select(div,'img.recipe-image') imgs = self.parser.select(div, 'img.recipe-image')
if len(imgs) > 0: if len(imgs) > 0:
thumbnail_url = unicode(imgs[0].attrib.get('src','')) thumbnail_url = unicode(imgs[0].attrib.get('src', ''))
short_description = unicode(' '.join(self.parser.select(div,'div.infos_column',1).text_content().split()).strip()) short_description = unicode(' '.join(self.parser.select(
imgs_cost = self.parser.select(div,'div.infos_column img') div, 'div.infos_column', 1).text_content().split()).strip())
imgs_cost = self.parser.select(div, 'div.infos_column img')
cost_tot = len(imgs_cost) cost_tot = len(imgs_cost)
cost_on = 0 cost_on = 0
for img in imgs_cost: for img in imgs_cost:
if img.attrib.get('src','').endswith('euro_on.png'): if img.attrib.get('src', '').endswith('euro_on.png'):
cost_on += 1 cost_on += 1
short_description += u' %s/%s'%(cost_on,cost_tot) short_description += u' %s/%s' % (cost_on, cost_tot)
recipe = Recipe(id,title) recipe = Recipe(id, title)
recipe.thumbnail_url = thumbnail_url recipe.thumbnail_url = thumbnail_url
recipe.short_description= short_description recipe.short_description = short_description
recipe.instructions = NotLoaded recipe.instructions = NotLoaded
recipe.ingredients = NotLoaded recipe.ingredients = NotLoaded
recipe.nb_person = NotLoaded recipe.nb_person = NotLoaded
@ -76,10 +77,10 @@ class RecipePage(BasePage):
instructions = NotAvailable instructions = NotAvailable
comments = [] comments = []
title = unicode(self.parser.select(self.document.getroot(),'head > title',1).text.split(' - ')[1]) title = unicode(self.parser.select(self.document.getroot(), 'head > title', 1).text.split(' - ')[1])
main = self.parser.select(self.document.getroot(),'div.recette_description',1) main = self.parser.select(self.document.getroot(), 'div.recette_description', 1)
rec_infos = self.parser.select(self.document.getroot(),'div.recette_infos div.infos_column strong') rec_infos = self.parser.select(self.document.getroot(), 'div.recette_infos div.infos_column strong')
for info_title in rec_infos: for info_title in rec_infos:
if u'Temps de préparation' in unicode(info_title.text): if u'Temps de préparation' in unicode(info_title.text):
if info_title.tail.strip() != '': if info_title.tail.strip() != '':
@ -96,31 +97,31 @@ class RecipePage(BasePage):
nb_person = int(info_title.tail) nb_person = int(info_title.tail)
ingredients = [] ingredients = []
p_ing = self.parser.select(main,'div.data.top.left > div.content p') p_ing = self.parser.select(main, 'div.data.top.left > div.content p')
for ing in p_ing: for ing in p_ing:
ingtxt = unicode(ing.text_content().strip()) ingtxt = unicode(ing.text_content().strip())
if ingtxt != '': if ingtxt != '':
ingredients.append(ingtxt) ingredients.append(ingtxt)
lines_instr = self.parser.select(main,'div.data.top.right div.content li') lines_instr = self.parser.select(main, 'div.data.top.right div.content li')
if len(lines_instr) > 0: if len(lines_instr) > 0:
instructions = u'' instructions = u''
for line in lines_instr: for line in lines_instr:
inst = ' '.join(line.text_content().strip().split()) inst = ' '.join(line.text_content().strip().split())
instructions += '%s\n'% inst instructions += '%s\n' % inst
instructions = instructions.strip('\n') instructions = instructions.strip('\n')
imgillu = self.parser.select(self.document.getroot(),'div.resume_recette_illustree img.photo') imgillu = self.parser.select(self.document.getroot(), 'div.resume_recette_illustree img.photo')
if len(imgillu) > 0: if len(imgillu) > 0:
picture_url = unicode(imgillu[0].attrib.get('src','')) picture_url = unicode(imgillu[0].attrib.get('src', ''))
for divcom in self.parser.select(self.document.getroot(),'div.comment-outer'): for divcom in self.parser.select(self.document.getroot(), 'div.comment-outer'):
comtxt = unicode(' '.join(divcom.text_content().strip().split())) comtxt = unicode(' '.join(divcom.text_content().strip().split()))
if u'| Répondre' in comtxt: if u'| Répondre' in comtxt:
comtxt = comtxt.strip('0123456789').replace(u' | Répondre','') comtxt = comtxt.strip('0123456789').replace(u' | Répondre', '')
comments.append(comtxt) comments.append(comtxt)
recipe = Recipe(id,title) recipe = Recipe(id, title)
recipe.preparation_time = preparation_time recipe.preparation_time = preparation_time
recipe.cooking_time = cooking_time recipe.cooking_time = cooking_time
recipe.nb_person = nb_person recipe.nb_person = nb_person

4
modules/attilasub/backend.py
View file

@ -17,7 +17,7 @@
# You should have received a copy of the GNU Affero General Public License # You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>. # along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.capabilities.subtitle import ICapSubtitle,LanguageNotSupported from weboob.capabilities.subtitle import ICapSubtitle, LanguageNotSupported
from weboob.tools.backend import BaseBackend from weboob.tools.backend import BaseBackend
from .browser import AttilasubBrowser from .browser import AttilasubBrowser
@ -53,4 +53,4 @@ class AttilasubBackend(BaseBackend, ICapSubtitle):
def iter_subtitles(self, language, pattern): def iter_subtitles(self, language, pattern):
if language not in self.LANGUAGE_LIST: if language not in self.LANGUAGE_LIST:
raise LanguageNotSupported() raise LanguageNotSupported()
return self.browser.iter_subtitles(language,quote_plus(pattern.encode('utf-8'))) return self.browser.iter_subtitles(language, quote_plus(pattern.encode('utf-8')))

5
modules/attilasub/browser.py
View file

@ -37,9 +37,10 @@ class AttilasubBrowser(BaseBrowser):
} }
def iter_subtitles(self, language, pattern): def iter_subtitles(self, language, pattern):
self.location('http://search.freefind.com/find.html?id=81131980&_charset_=&bcd=%%F7&scs=1&pageid=r&query=%s&mode=Find%%20pages%%20matching%%20ALL%%20words' % pattern.encode('utf-8')) self.location('http://search.freefind.com/find.html?id=81131980&_charset_=&bcd=%%F7&scs=1&pageid=r&query=%s&mode=Find%%20pages%%20matching%%20ALL%%20words' %
pattern.encode('utf-8'))
assert self.is_on_page(SearchPage) assert self.is_on_page(SearchPage)
return self.page.iter_subtitles(language,pattern) return self.page.iter_subtitles(language, pattern)
def get_subtitle(self, id): def get_subtitle(self, id):
url_end = id.split('|')[0] url_end = id.split('|')[0]

54
modules/attilasub/pages.py
View file

@ -23,16 +23,16 @@ from weboob.capabilities.base import NotAvailable
from weboob.tools.browser import BasePage from weboob.tools.browser import BasePage
__all__ = ['SubtitlesPage','SearchPage'] __all__ = ['SubtitlesPage', 'SearchPage']
class SearchPage(BasePage): class SearchPage(BasePage):
def iter_subtitles(self, language, pattern): def iter_subtitles(self, language, pattern):
fontresult = self.parser.select(self.document.getroot(),'div.search-results font.search-results') fontresult = self.parser.select(self.document.getroot(), 'div.search-results font.search-results')
# for each result in freefind, explore the subtitle list page to iter subtitles # for each result in freefind, explore the subtitle list page to iter subtitles
for res in fontresult: for res in fontresult:
a = self.parser.select(res,'a',1) a = self.parser.select(res, 'a', 1)
url = a.attrib.get('href','') url = a.attrib.get('href', '')
self.browser.location(url) self.browser.location(url)
assert self.browser.is_on_page(SubtitlesPage) assert self.browser.is_on_page(SubtitlesPage)
# subtitles page does the job # subtitles page does the job
@ -41,15 +41,15 @@ class SearchPage(BasePage):
class SubtitlesPage(BasePage): class SubtitlesPage(BasePage):
def get_subtitle(self,id): def get_subtitle(self, id):
href = id.split('|')[1] href = id.split('|')[1]
# we have to find the 'tr' which contains the link to this address # we have to find the 'tr' which contains the link to this address
a = self.parser.select(self.document.getroot(),'a[href="%s"]'%href,1) a = self.parser.select(self.document.getroot(), 'a[href="%s"]' % href, 1)
line = a.getparent().getparent().getparent().getparent().getparent() line = a.getparent().getparent().getparent().getparent().getparent()
cols = self.parser.select(line,'td') cols = self.parser.select(line, 'td')
traduced_title = self.parser.select(cols[0],'font',1).text.lower() traduced_title = self.parser.select(cols[0], 'font', 1).text.lower()
original_title = self.parser.select(cols[1],'font',1).text.lower() original_title = self.parser.select(cols[1], 'font', 1).text.lower()
nb_cd = self.parser.select(cols[2],'font',1).text.strip() nb_cd = self.parser.select(cols[2], 'font', 1).text.strip()
nb_cd = int(nb_cd.split()[0]) nb_cd = int(nb_cd.split()[0])
traduced_title_words = traduced_title.split() traduced_title_words = traduced_title.split()
@ -59,30 +59,30 @@ class SubtitlesPage(BasePage):
traduced_title = " ".join(traduced_title_words) traduced_title = " ".join(traduced_title_words)
original_title = " ".join(original_title_words) original_title = " ".join(original_title_words)
name = unicode('%s (%s)'%(original_title,traduced_title)) name = unicode('%s (%s)' % (original_title, traduced_title))
url = unicode('http://davidbillemont3.free.fr/%s'%href) url = unicode('http://davidbillemont3.free.fr/%s' % href)
subtitle = Subtitle(id,name) subtitle = Subtitle(id, name)
subtitle.url = url subtitle.url = url
subtitle.language = unicode('fr') subtitle.language = unicode('fr')
subtitle.nb_cd = nb_cd subtitle.nb_cd = nb_cd
subtitle.description = NotAvailable subtitle.description = NotAvailable
return subtitle return subtitle
def iter_subtitles(self,language, pattern): def iter_subtitles(self, language, pattern):
pattern = pattern.strip().replace('+',' ').lower() pattern = pattern.strip().replace('+', ' ').lower()
pattern_words = pattern.split() pattern_words = pattern.split()
tab = self.parser.select(self.document.getroot(),'table[bordercolor="#B8C0B2"]') tab = self.parser.select(self.document.getroot(), 'table[bordercolor="#B8C0B2"]')
if len(tab) == 0: if len(tab) == 0:
tab = self.parser.select(self.document.getroot(),'table[bordercolordark="#B8C0B2"]') tab = self.parser.select(self.document.getroot(), 'table[bordercolordark="#B8C0B2"]')
if len(tab) == 0: if len(tab) == 0:
return return
# some results of freefind point on useless pages # some results of freefind point on useless pages
if tab[0].attrib.get('width','') != '100%': if tab[0].attrib.get('width', '') != '100%':
return return
for line in tab[0].getiterator('tr'): for line in tab[0].getiterator('tr'):
cols = self.parser.select(line,'td') cols = self.parser.select(line, 'td')
traduced_title = self.parser.select(cols[0],'font',1).text.lower() traduced_title = self.parser.select(cols[0], 'font', 1).text.lower()
original_title = self.parser.select(cols[1],'font',1).text.lower() original_title = self.parser.select(cols[1], 'font', 1).text.lower()
traduced_title_words = traduced_title.split() traduced_title_words = traduced_title.split()
original_title_words = original_title.split() original_title_words = original_title.split()
@ -98,13 +98,13 @@ class SubtitlesPage(BasePage):
traduced_title = " ".join(traduced_title_words) traduced_title = " ".join(traduced_title_words)
original_title = " ".join(original_title_words) original_title = " ".join(original_title_words)
nb_cd = self.parser.select(cols[2],'font',1).text.strip() nb_cd = self.parser.select(cols[2], 'font', 1).text.strip()
nb_cd = int(nb_cd.strip(' CD')) nb_cd = int(nb_cd.strip(' CD'))
name = unicode('%s (%s)'%(original_title,traduced_title)) name = unicode('%s (%s)' % (original_title, traduced_title))
href = self.parser.select(cols[3],'a',1).attrib.get('href','') href = self.parser.select(cols[3], 'a', 1).attrib.get('href', '')
url = unicode('http://davidbillemont3.free.fr/%s'%href) url = unicode('http://davidbillemont3.free.fr/%s' % href)
id = unicode('%s|%s'%(self.browser.geturl().split('/')[-1],href)) id = unicode('%s|%s' % (self.browser.geturl().split('/')[-1], href))
subtitle = Subtitle(id,name) subtitle = Subtitle(id, name)
subtitle.url = url subtitle.url = url
subtitle.language = unicode('fr') subtitle.language = unicode('fr')
subtitle.nb_cd = nb_cd subtitle.nb_cd = nb_cd

2
modules/attilasub/test.py
View file

@ -27,7 +27,7 @@ class AttilasubTest(BackendTest):
BACKEND = 'attilasub' BACKEND = 'attilasub'
def test_subtitle(self): def test_subtitle(self):
subtitles = list(self.backend.iter_subtitles('fr','spiderman')) subtitles = list(self.backend.iter_subtitles('fr', 'spiderman'))
assert (len(subtitles) > 0) assert (len(subtitles) > 0)
for subtitle in subtitles: for subtitle in subtitles:
path, qs = urllib.splitquery(subtitle.url) path, qs = urllib.splitquery(subtitle.url)

8
modules/imdb/backend.py
View file

@ -67,7 +67,7 @@ class ImdbBackend(BaseBackend, ICapCinema):
return self.browser.get_person_biography(id) return self.browser.get_person_biography(id)
def get_movie_releases(self, id, country=None): def get_movie_releases(self, id, country=None):
return self.browser.get_movie_releases(id,country) return self.browser.get_movie_releases(id, country)
def fill_person(self, person, fields): def fill_person(self, person, fields):
if 'real_name' in fields or 'birth_place' in fields\ if 'real_name' in fields or 'birth_place' in fields\
@ -105,7 +105,7 @@ class ImdbBackend(BaseBackend, ICapCinema):
movie.country = mov.country movie.country = mov.country
movie.note = mov.note movie.note = mov.note
movie.roles = mov.roles movie.roles = mov.roles
movie.short_description= mov.short_description movie.short_description = mov.short_description
movie.thumbnail_url = mov.thumbnail_url movie.thumbnail_url = mov.thumbnail_url
if 'all_release_dates' in fields: if 'all_release_dates' in fields:
@ -114,6 +114,6 @@ class ImdbBackend(BaseBackend, ICapCinema):
return movie return movie
OBJECTS = { OBJECTS = {
Person:fill_person, Person: fill_person,
Movie:fill_movie Movie: fill_movie
} }

50
modules/imdb/browser.py
View file

@ -47,15 +47,16 @@ class ImdbBrowser(BaseBrowser):
def iter_movies(self, pattern): def iter_movies(self, pattern):
res = self.readurl('http://www.imdb.com/xml/find?json=1&nr=1&tt=on&q=%s' % pattern.encode('utf-8')) res = self.readurl('http://www.imdb.com/xml/find?json=1&nr=1&tt=on&q=%s' % pattern.encode('utf-8'))
jres = json.loads(res) jres = json.loads(res)
for cat in ['title_popular','title_exact','title_approx']: for cat in ['title_popular', 'title_exact', 'title_approx']:
if cat in jres: if cat in jres:
for m in jres[cat]: for m in jres[cat]:
tdesc = unicode(m['title_description']) tdesc = unicode(m['title_description'])
if '<a' in tdesc and '>' in tdesc: if '<a' in tdesc and '>' in tdesc:
short_description = u'%s %s'%(tdesc.split('<')[0].strip(', '), tdesc.split('>')[1].split('<')[0]) short_description = u'%s %s' % (tdesc.split('<')[
0].strip(', '), tdesc.split('>')[1].split('<')[0])
else: else:
short_description = tdesc.strip(', ') short_description = tdesc.strip(', ')
movie = Movie(m['id'],latin2unicode(m['title'])) movie = Movie(m['id'], latin2unicode(m['title']))
movie.other_titles = NotLoaded movie.other_titles = NotLoaded
movie.release_date = NotLoaded movie.release_date = NotLoaded
movie.duration = NotLoaded movie.duration = NotLoaded
@ -64,31 +65,32 @@ class ImdbBrowser(BaseBrowser):
movie.country = NotLoaded movie.country = NotLoaded
movie.note = NotLoaded movie.note = NotLoaded
movie.roles = NotLoaded movie.roles = NotLoaded
movie.all_release_dates= NotLoaded movie.all_release_dates = NotLoaded
movie.thumbnail_url = NotLoaded movie.thumbnail_url = NotLoaded
yield movie yield movie
def iter_persons(self, pattern): def iter_persons(self, pattern):
res = self.readurl('http://www.imdb.com/xml/find?json=1&nr=1&nm=on&q=%s' % pattern.encode('utf-8')) res = self.readurl('http://www.imdb.com/xml/find?json=1&nr=1&nm=on&q=%s' % pattern.encode('utf-8'))
jres = json.loads(res) jres = json.loads(res)
for cat in ['name_popular','name_exact','name_approx']: for cat in ['name_popular', 'name_exact', 'name_approx']:
if cat in jres: if cat in jres:
for p in jres[cat]: for p in jres[cat]:
person = Person(p['id'],latin2unicode(p['name'])) person = Person(p['id'], latin2unicode(p['name']))
person.real_name = NotLoaded person.real_name = NotLoaded
person.birth_place = NotLoaded person.birth_place = NotLoaded
person.birth_date = NotLoaded person.birth_date = NotLoaded
person.death_date = NotLoaded person.death_date = NotLoaded
person.gender = NotLoaded person.gender = NotLoaded
person.nationality = NotLoaded person.nationality = NotLoaded
person.short_biography= NotLoaded person.short_biography = NotLoaded
person.short_description= latin2unicode(p['description']) person.short_description = latin2unicode(p['description'])
person.roles = NotLoaded person.roles = NotLoaded
person.thumbnail_url = NotLoaded person.thumbnail_url = NotLoaded
yield person yield person
def get_movie(self, id): def get_movie(self, id):
res = self.readurl('http://imdbapi.org/?id=%s&type=json&plot=simple&episode=1&lang=en-US&aka=full&release=simple&business=0&tech=0' % id ) res = self.readurl(
'http://imdbapi.org/?id=%s&type=json&plot=simple&episode=1&lang=en-US&aka=full&release=simple&business=0&tech=0' % id)
if res is not None: if res is not None:
jres = json.loads(res) jres = json.loads(res)
else: else:
@ -122,7 +124,7 @@ class ImdbBrowser(BaseBrowser):
if 'also_known_as' in jres: if 'also_known_as' in jres:
for other_t in jres['also_known_as']: for other_t in jres['also_known_as']:
if 'country' in other_t and 'title' in other_t: if 'country' in other_t and 'title' in other_t:
other_titles.append('%s : %s' % (other_t['country'],htmlparser.unescape(other_t['title']))) other_titles.append('%s : %s' % (other_t['country'], htmlparser.unescape(other_t['title'])))
if 'release_date' in jres: if 'release_date' in jres:
dstr = str(jres['release_date']) dstr = str(jres['release_date'])
year = int(dstr[:4]) year = int(dstr[:4])
@ -134,21 +136,21 @@ class ImdbBrowser(BaseBrowser):
day = int(dstr[-2:]) day = int(dstr[-2:])
if day == 0: if day == 0:
day = 1 day = 1
release_date = datetime(year,month,day) release_date = datetime(year, month, day)
if 'country' in jres: if 'country' in jres:
country = u'' country = u''
for c in jres['country']: for c in jres['country']:
country += '%s, '%c country += '%s, ' % c
country = country[:-2] country = country[:-2]
if 'plot_simple' in jres: if 'plot_simple' in jres:
pitch = unicode(jres['plot_simple']) pitch = unicode(jres['plot_simple'])
if 'rating' in jres and 'rating_count' in jres: if 'rating' in jres and 'rating_count' in jres:
note = u'%s/10 (%s votes)'%(jres['rating'],jres['rating_count']) note = u'%s/10 (%s votes)' % (jres['rating'], jres['rating_count'])
for r in ['actor','director','writer']: for r in ['actor', 'director', 'writer']:
if '%ss'%r in jres: if '%ss' % r in jres:
roles['%s'%r] = list(jres['%ss'%r]) roles['%s' % r] = list(jres['%ss' % r])
movie = Movie(id,title) movie = Movie(id, title)
movie.other_titles = other_titles movie.other_titles = other_titles
movie.release_date = release_date movie.release_date = release_date
movie.duration = duration movie.duration = duration
@ -156,8 +158,8 @@ class ImdbBrowser(BaseBrowser):
movie.country = country movie.country = country
movie.note = note movie.note = note
movie.roles = roles movie.roles = roles
movie.short_description= short_description movie.short_description = short_description
movie.all_release_dates= NotLoaded movie.all_release_dates = NotLoaded
movie.thumbnail_url = thumbnail_url movie.thumbnail_url = thumbnail_url
return movie return movie
@ -175,7 +177,7 @@ class ImdbBrowser(BaseBrowser):
return self.page.get_biography() return self.page.get_biography()
def iter_movie_persons(self, movie_id, role): def iter_movie_persons(self, movie_id, role):
self.location('http://www.imdb.com/title/%s/fullcredits'%movie_id) self.location('http://www.imdb.com/title/%s/fullcredits' % movie_id)
assert self.is_on_page(MovieCrewPage) assert self.is_on_page(MovieCrewPage)
for p in self.page.iter_persons(role): for p in self.page.iter_persons(role):
yield p yield p
@ -192,13 +194,13 @@ class ImdbBrowser(BaseBrowser):
yield movie yield movie
def iter_movie_persons_ids(self, movie_id): def iter_movie_persons_ids(self, movie_id):
self.location('http://www.imdb.com/title/%s/fullcredits'%movie_id) self.location('http://www.imdb.com/title/%s/fullcredits' % movie_id)
assert self.is_on_page(MovieCrewPage) assert self.is_on_page(MovieCrewPage)
for person in self.page.iter_persons_ids(): for person in self.page.iter_persons_ids():
yield person yield person
def get_movie_releases(self,id, country): def get_movie_releases(self, id, country):
self.location('http://www.imdb.com/title/%s/releaseinfo'%id) self.location('http://www.imdb.com/title/%s/releaseinfo' % id)
assert self.is_on_page(ReleasePage) assert self.is_on_page(ReleasePage)
return self.page.get_movie_releases(country) return self.page.get_movie_releases(country)
@ -222,5 +224,5 @@ dict_hex = {'&#xE1;': u'á',
def latin2unicode(word): def latin2unicode(word):
for key in dict_hex.keys(): for key in dict_hex.keys():
word = word.replace(key,dict_hex[key]) word = word.replace(key, dict_hex[key])
return unicode(word) return unicode(word)

120
modules/imdb/pages.py
View file

@ -25,28 +25,28 @@ from weboob.tools.browser import BasePage
from datetime import datetime from datetime import datetime
__all__ = ['PersonPage','MovieCrewPage','BiographyPage','FilmographyPage','ReleasePage'] __all__ = ['PersonPage', 'MovieCrewPage', 'BiographyPage', 'FilmographyPage', 'ReleasePage']
class ReleasePage(BasePage): class ReleasePage(BasePage):
''' Page containing releases of a movie ''' Page containing releases of a movie
''' '''
def get_movie_releases(self,country_filter): def get_movie_releases(self, country_filter):
result = unicode() result = unicode()
links = self.parser.select(self.document.getroot(),'b a') links = self.parser.select(self.document.getroot(), 'b a')
for a in links: for a in links:
href = a.attrib.get('href','') href = a.attrib.get('href', '')
if href.strip('/').split('/')[0] == 'calendar' and\ if href.strip('/').split('/')[0] == 'calendar' and\
(country_filter is None or href.split('region=')[-1].lower() == country_filter): (country_filter is None or href.split('region=')[-1].lower() == country_filter):
country = a.text country = a.text
td_date = self.parser.select(a.getparent().getparent().getparent(),'td')[1] td_date = self.parser.select(a.getparent().getparent().getparent(), 'td')[1]
date_links = self.parser.select(td_date,'a') date_links = self.parser.select(td_date, 'a')
if len(date_links) > 1: if len(date_links) > 1:
date = date_links[1].attrib.get('href','').strip('/').split('/')[-1] date = date_links[1].attrib.get('href', '').strip('/').split('/')[-1]
date += '-'+date_links[0].attrib.get('href','').strip('/').split('/')[-1] date += '-'+date_links[0].attrib.get('href', '').strip('/').split('/')[-1]
else: else:
date = unicode(self.parser.select(a.getparent().getparent().getparent(),'td')[1].text_content()) date = unicode(self.parser.select(a.getparent().getparent().getparent(), 'td')[1].text_content())
result += '%s : %s\n' % (country,date) result += '%s : %s\n' % (country, date)
if result == u'': if result == u'':
result = NotAvailable result = NotAvailable
else: else:
@ -59,11 +59,11 @@ class BiographyPage(BasePage):
''' '''
def get_biography(self): def get_biography(self):
bio = unicode() bio = unicode()
tn = self.parser.select(self.document.getroot(),'div#tn15content',1) tn = self.parser.select(self.document.getroot(), 'div#tn15content', 1)
# we only read paragraphs, titles and links # we only read paragraphs, titles and links
for ch in tn.getchildren(): for ch in tn.getchildren():
if ch.tag in ['p','h5','a']: if ch.tag in ['p', 'h5', 'a']:
bio += '%s\n\n'%ch.text_content().strip() bio += '%s\n\n' % ch.text_content().strip()
if bio == u'': if bio == u'':
bio = NotAvailable bio = NotAvailable
return bio return bio
@ -74,15 +74,15 @@ class MovieCrewPage(BasePage):
''' '''
def iter_persons(self, role_filter=None): def iter_persons(self, role_filter=None):
if (role_filter is None or (role_filter is not None and role_filter == 'actor')): if (role_filter is None or (role_filter is not None and role_filter == 'actor')):
tables = self.parser.select(self.document.getroot(),'table.cast') tables = self.parser.select(self.document.getroot(), 'table.cast')
if len(tables) > 0: if len(tables) > 0:
table = tables[0] table = tables[0]
tds = self.parser.select(table,'td.nm') tds = self.parser.select(table, 'td.nm')
for td in tds: for td in tds:
id = td.find('a').attrib.get('href','').strip('/').split('/')[-1] id = td.find('a').attrib.get('href', '').strip('/').split('/')[-1]
name = unicode(td.find('a').text) name = unicode(td.find('a').text)
char_name = unicode(self.parser.select(td.getparent(),'td.char',1).text_content()) char_name = unicode(self.parser.select(td.getparent(), 'td.char', 1).text_content())
person = Person(id,name) person = Person(id, name)
person.short_description = char_name person.short_description = char_name
person.real_name = NotLoaded person.real_name = NotLoaded
person.birth_place = NotLoaded person.birth_place = NotLoaded
@ -90,36 +90,36 @@ class MovieCrewPage(BasePage):
person.death_date = NotLoaded person.death_date = NotLoaded
person.gender = NotLoaded person.gender = NotLoaded
person.nationality = NotLoaded person.nationality = NotLoaded
person.short_biography= NotLoaded person.short_biography = NotLoaded
person.roles = NotLoaded person.roles = NotLoaded
person.thumbnail_url = NotLoaded person.thumbnail_url = NotLoaded
yield person yield person
for gloss_link in self.parser.select(self.document.getroot(),'table[cellspacing=1] h5 a'): for gloss_link in self.parser.select(self.document.getroot(), 'table[cellspacing=1] h5 a'):
role = gloss_link.attrib.get('name','').rstrip('s') role = gloss_link.attrib.get('name', '').rstrip('s')
if (role_filter is None or (role_filter is not None and role == role_filter)): if (role_filter is None or (role_filter is not None and role == role_filter)):
tbody = gloss_link.getparent().getparent().getparent().getparent() tbody = gloss_link.getparent().getparent().getparent().getparent()
for line in self.parser.select(tbody,'tr')[1:]: for line in self.parser.select(tbody, 'tr')[1:]:
for a in self.parser.select(line,'a'): for a in self.parser.select(line, 'a'):
role_detail = NotAvailable role_detail = NotAvailable
href = a.attrib.get('href','') href = a.attrib.get('href', '')
if '/name/nm' in href: if '/name/nm' in href:
id = href.strip('/').split('/')[-1] id = href.strip('/').split('/')[-1]
name = unicode(a.text) name = unicode(a.text)
if 'glossary' in href: if 'glossary' in href:
role_detail = unicode(a.text) role_detail = unicode(a.text)
person = Person(id,name) person = Person(id, name)
person.short_description = role_detail person.short_description = role_detail
yield person yield person
#yield self.browser.get_person(id) # yield self.browser.get_person(id)
def iter_persons_ids(self): def iter_persons_ids(self):
tables = self.parser.select(self.document.getroot(),'table.cast') tables = self.parser.select(self.document.getroot(), 'table.cast')
if len(tables) > 0: if len(tables) > 0:
table = tables[0] table = tables[0]
tds = self.parser.select(table,'td.nm') tds = self.parser.select(table, 'td.nm')
for td in tds: for td in tds:
id = td.find('a').attrib.get('href','').strip('/').split('/')[-1] id = td.find('a').attrib.get('href', '').strip('/').split('/')[-1]
yield id yield id
@ -127,7 +127,7 @@ class PersonPage(BasePage):
''' Page giving informations about a person ''' Page giving informations about a person
It is used to build a Person instance and to get the movie list related to a person It is used to build a Person instance and to get the movie list related to a person
''' '''
def get_person(self,id): def get_person(self, id):
name = NotAvailable name = NotAvailable
short_biography = NotAvailable short_biography = NotAvailable
short_description = NotAvailable short_description = NotAvailable
@ -139,42 +139,42 @@ class PersonPage(BasePage):
thumbnail_url = NotAvailable thumbnail_url = NotAvailable
roles = {} roles = {}
nationality = NotAvailable nationality = NotAvailable
td_overview = self.parser.select(self.document.getroot(),'td#overview-top',1) td_overview = self.parser.select(self.document.getroot(), 'td#overview-top', 1)
descs = self.parser.select(td_overview,'span[itemprop=description]') descs = self.parser.select(td_overview, 'span[itemprop=description]')
if len(descs) > 0: if len(descs) > 0:
short_biography = unicode(descs[0].text) short_biography = unicode(descs[0].text)
rname_block = self.parser.select(td_overview,'div.txt-block h4.inline') rname_block = self.parser.select(td_overview, 'div.txt-block h4.inline')
if len(rname_block) > 0 and "born" in rname_block[0].text.lower(): if len(rname_block) > 0 and "born" in rname_block[0].text.lower():
links = self.parser.select(rname_block[0].getparent(),'a') links = self.parser.select(rname_block[0].getparent(), 'a')
for a in links: for a in links:
href = a.attrib.get('href','').strip() href = a.attrib.get('href', '').strip()
if href == 'bio': if href == 'bio':
real_name = unicode(a.text.strip()) real_name = unicode(a.text.strip())
elif 'birth_place' in href: elif 'birth_place' in href:
birth_place = unicode(a.text.lower().strip()) birth_place = unicode(a.text.lower().strip())
names = self.parser.select(td_overview,'h1[itemprop=name]') names = self.parser.select(td_overview, 'h1[itemprop=name]')
if len(names) > 0: if len(names) > 0:
name = unicode(names[0].text.strip()) name = unicode(names[0].text.strip())
times = self.parser.select(td_overview,'time[itemprop=birthDate]') times = self.parser.select(td_overview, 'time[itemprop=birthDate]')
if len(times) > 0: if len(times) > 0:
time = times[0].attrib.get('datetime','').split('-') time = times[0].attrib.get('datetime', '').split('-')
if len(time) == 3 and int(time[0]) >= 1900: if len(time) == 3 and int(time[0]) >= 1900:
birth_date = datetime(int(time[0]),int(time[1]),int(time[2])) birth_date = datetime(int(time[0]), int(time[1]), int(time[2]))
dtimes = self.parser.select(td_overview,'time[itemprop=deathDate]') dtimes = self.parser.select(td_overview, 'time[itemprop=deathDate]')
if len(dtimes) > 0: if len(dtimes) > 0:
dtime = dtimes[0].attrib.get('datetime','').split('-') dtime = dtimes[0].attrib.get('datetime', '').split('-')
if len(dtime) == 3 and int(dtime[0]) >= 1900: if len(dtime) == 3 and int(dtime[0]) >= 1900:
death_date = datetime(int(dtime[0]),int(dtime[1]),int(dtime[2])) death_date = datetime(int(dtime[0]), int(dtime[1]), int(dtime[2]))
img_thumbnail = self.parser.select(self.document.getroot(),'td#img_primary img') img_thumbnail = self.parser.select(self.document.getroot(), 'td#img_primary img')
if len(img_thumbnail) > 0: if len(img_thumbnail) > 0:
thumbnail_url = unicode(img_thumbnail[0].attrib.get('src','')) thumbnail_url = unicode(img_thumbnail[0].attrib.get('src', ''))
# go to the filmography page # go to the filmography page
self.browser.location('http://www.imdb.com/name/%s/filmotype'%id) self.browser.location('http://www.imdb.com/name/%s/filmotype' % id)
assert self.browser.is_on_page(FilmographyPage) assert self.browser.is_on_page(FilmographyPage)
roles = self.browser.page.get_roles() roles = self.browser.page.get_roles()
person = Person(id,name) person = Person(id, name)
person.real_name = real_name person.real_name = real_name
person.birth_date = birth_date person.birth_date = birth_date
person.death_date = death_date person.death_date = death_date
@ -193,39 +193,39 @@ class FilmographyPage(BasePage):
This page is easier to parse than the main person page filmography This page is easier to parse than the main person page filmography
''' '''
def iter_movies_ids(self): def iter_movies_ids(self):
for role_div in self.parser.select(self.document.getroot(),'div.filmo'): for role_div in self.parser.select(self.document.getroot(), 'div.filmo'):
for a in self.parser.select(role_div,'ol > li > a'): for a in self.parser.select(role_div, 'ol > li > a'):
id = a.attrib.get('href','').strip('/').split('/')[-1] id = a.attrib.get('href', '').strip('/').split('/')[-1]
if id.startswith('tt'): if id.startswith('tt'):
yield id yield id
def get_roles(self): def get_roles(self):
roles = {} roles = {}
for role_div in self.parser.select(self.document.getroot(),'div.filmo'): for role_div in self.parser.select(self.document.getroot(), 'div.filmo'):
role = self.parser.select(role_div,'h5 a',1).text.replace(':','') role = self.parser.select(role_div, 'h5 a', 1).text.replace(':', '')
roles[role] = [] roles[role] = []
for a in self.parser.select(role_div,'ol > li > a'): for a in self.parser.select(role_div, 'ol > li > a'):
id = a.attrib.get('href','').strip('/').split('/')[-1] id = a.attrib.get('href', '').strip('/').split('/')[-1]
if id.startswith('tt'): if id.startswith('tt'):
if '(' in a.tail and ')' in a.tail: if '(' in a.tail and ')' in a.tail:
between_p = a.tail.split(')')[0].split('(')[1] between_p = a.tail.split(')')[0].split('(')[1]
else: else:
between_p = '????' between_p = '????'
roles[role].append('(%s) %s'%(between_p,a.text)) roles[role].append('(%s) %s' % (between_p, a.text))
return roles return roles
def iter_movies(self, role_filter=None): def iter_movies(self, role_filter=None):
for role_div in self.parser.select(self.document.getroot(),'div.filmo'): for role_div in self.parser.select(self.document.getroot(), 'div.filmo'):
role = self.parser.select(role_div,'h5 a',1).text.replace(':','') role = self.parser.select(role_div, 'h5 a', 1).text.replace(':', '')
if (role_filter is None or (role_filter is not None and role.lower().strip() == role_filter))\ if (role_filter is None or (role_filter is not None and role.lower().strip() == role_filter))\
and role != 'In Development': and role != 'In Development':
for a in self.parser.select(role_div,'ol > li > a'): for a in self.parser.select(role_div, 'ol > li > a'):
id = a.attrib.get('href','').strip('/').split('/')[-1] id = a.attrib.get('href', '').strip('/').split('/')[-1]
if id.startswith('tt'): if id.startswith('tt'):
title = unicode(a.text) title = unicode(a.text)
role_detail = NotAvailable role_detail = NotAvailable
if len(a.tail) > 0: if len(a.tail) > 0:
role_detail = unicode(' '.join(a.tail.replace('..','').split())) role_detail = unicode(' '.join(a.tail.replace('..', '').split()))
movie = Movie(id,title) movie = Movie(id, title)
movie.short_description = role_detail movie.short_description = role_detail
yield movie yield movie

4
modules/isohunt/backend.py
View file

@ -49,7 +49,7 @@ class IsohuntBackend(BaseBackend, ICapTorrent):
return self.browser.openurl(torrent.url.encode('utf-8')).read() return self.browser.openurl(torrent.url.encode('utf-8')).read()
def iter_torrents(self, pattern): def iter_torrents(self, pattern):
return self.browser.iter_torrents(pattern.replace(' ','+')) return self.browser.iter_torrents(pattern.replace(' ', '+'))
def fill_torrent(self, torrent, fields): def fill_torrent(self, torrent, fields):
if 'description' in fields or 'files' in fields: if 'description' in fields or 'files' in fields:
@ -61,5 +61,5 @@ class IsohuntBackend(BaseBackend, ICapTorrent):
return torrent return torrent
OBJECTS = { OBJECTS = {
Torrent:fill_torrent Torrent: fill_torrent
} }

4
modules/isohunt/browser.py
View file

@ -32,8 +32,8 @@ class IsohuntBrowser(BaseBrowser):
ENCODING = 'utf-8' ENCODING = 'utf-8'
USER_AGENT = BaseBrowser.USER_AGENTS['wget'] USER_AGENT = BaseBrowser.USER_AGENTS['wget']
PAGES = { PAGES = {
'https://isohunt.com/torrents/.*iht=-1&ihp=1&ihs1=1&iho1=d' : TorrentsPage, 'https://isohunt.com/torrents/.*iht=-1&ihp=1&ihs1=1&iho1=d': TorrentsPage,
'https://isohunt.com/torrent_details.*tab=summary' : TorrentPage, 'https://isohunt.com/torrent_details.*tab=summary': TorrentPage,
} }
def home(self): def home(self):

3
modules/isohunt/pages/torrents.py
View file

@ -73,7 +73,8 @@ class TorrentPage(BasePage):
title = NotAvailable title = NotAvailable
size = NotAvailable size = NotAvailable
url = 'https://isohunt.com/download/%s/%s.torrent' % (id, id) url = 'https://isohunt.com/download/%s/%s.torrent' % (id, id)
title = unicode(self.parser.select(self.document.getroot(),'head > meta[name=title]',1).attrib.get('content','')) title = unicode(self.parser.select(
self.document.getroot(), 'head > meta[name=title]', 1).attrib.get('content', ''))
seed = NotAvailable seed = NotAvailable
leech = NotAvailable leech = NotAvailable
tip_id = "none" tip_id = "none"

2
modules/kickass/backend.py
View file

@ -72,5 +72,5 @@ class KickassBackend(BaseBackend, ICapTorrent):
return torrent return torrent
OBJECTS = { OBJECTS = {
Torrent:fill_torrent Torrent: fill_torrent
} }

8
modules/kickass/pages.py
View file

@ -31,7 +31,7 @@ from weboob.tools.browser import BasePage
from weboob.tools.misc import get_bytes_size from weboob.tools.misc import get_bytes_size
__all__ = ['TorrentsPage','TorrentPage'] __all__ = ['TorrentsPage', 'TorrentPage']
class TorrentsPage(BasePage): class TorrentsPage(BasePage):
@ -53,14 +53,14 @@ class TorrentsPage(BasePage):
.replace('.html', '') .replace('.html', '')
# look for url # look for url
for a in self.parser.select(tr,'div.iaconbox a'): for a in self.parser.select(tr, 'div.iaconbox a'):
href = a.attrib.get('href', '') href = a.attrib.get('href', '')
if href.startswith('magnet'): if href.startswith('magnet'):
magnet = unicode(href) magnet = unicode(href)
elif href.startswith('http'): elif href.startswith('http'):
url = unicode(href) url = unicode(href)
elif href.startswith('//'): elif href.startswith('//'):
url = u'http:%s'%href url = u'http:%s' % href
size = tr.getchildren()[1].text size = tr.getchildren()[1].text
u = tr.getchildren()[1].getchildren()[0].text u = tr.getchildren()[1].getchildren()[0].text
@ -116,7 +116,7 @@ class TorrentPage(BasePage):
if href.startswith('magnet'): if href.startswith('magnet'):
magnet = unicode(href) magnet = unicode(href)
elif href.startswith('//'): elif href.startswith('//'):
url = u'http:%s'%href url = u'http:%s' % href
elif href.startswith('http'): elif href.startswith('http'):
url = unicode(href) url = unicode(href)

4
modules/marmiton/backend.py
View file

@ -17,7 +17,7 @@
# You should have received a copy of the GNU Affero General Public License # You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>. # along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.capabilities.recipe import ICapRecipe,Recipe from weboob.capabilities.recipe import ICapRecipe, Recipe
from weboob.tools.backend import BaseBackend from weboob.tools.backend import BaseBackend
from .browser import MarmitonBrowser from .browser import MarmitonBrowser
@ -59,5 +59,5 @@ class MarmitonBackend(BaseBackend, ICapRecipe):
return recipe return recipe
OBJECTS = { OBJECTS = {
Recipe:fill_recipe, Recipe: fill_recipe,
} }

53
modules/marmiton/pages.py
View file

@ -30,23 +30,24 @@ class ResultsPage(BasePage):
""" Page which contains results as a list of recipies """ Page which contains results as a list of recipies
""" """
def iter_recipes(self): def iter_recipes(self):
for div in self.parser.select(self.document.getroot(),'div.m_search_result'): for div in self.parser.select(self.document.getroot(), 'div.m_search_result'):
tds = self.parser.select(div,'td') tds = self.parser.select(div, 'td')
if len(tds) == 2: if len(tds) == 2:
title = NotAvailable title = NotAvailable
thumbnail_url = NotAvailable thumbnail_url = NotAvailable
short_description = NotAvailable short_description = NotAvailable
imgs = self.parser.select(tds[0],'img') imgs = self.parser.select(tds[0], 'img')
if len(imgs) > 0: if len(imgs) > 0:
thumbnail_url = unicode(imgs[0].attrib.get('src','')) thumbnail_url = unicode(imgs[0].attrib.get('src', ''))
link = self.parser.select(tds[1],'div.m_search_titre_recette a',1) link = self.parser.select(tds[1], 'div.m_search_titre_recette a', 1)
title = unicode(link.text) title = unicode(link.text)
id = link.attrib.get('href','').replace('.aspx','').replace('/recettes/recette_','') id = link.attrib.get('href', '').replace('.aspx', '').replace('/recettes/recette_', '')
short_description = unicode(' '.join(self.parser.select(tds[1],'div.m_search_result_part4',1).text.strip().split('\n'))) short_description = unicode(' '.join(self.parser.select(tds[
1], 'div.m_search_result_part4', 1).text.strip().split('\n')))
recipe = Recipe(id,title) recipe = Recipe(id, title)
recipe.thumbnail_url = thumbnail_url recipe.thumbnail_url = thumbnail_url
recipe.short_description= short_description recipe.short_description = short_description
recipe.instructions = NotLoaded recipe.instructions = NotLoaded
recipe.ingredients = NotLoaded recipe.ingredients = NotLoaded
recipe.nb_person = NotLoaded recipe.nb_person = NotLoaded
@ -68,30 +69,30 @@ class RecipePage(BasePage):
instructions = NotAvailable instructions = NotAvailable
comments = [] comments = []
title = unicode(self.parser.select(self.document.getroot(),'h1.m_title',1).text_content().strip()) title = unicode(self.parser.select(self.document.getroot(), 'h1.m_title', 1).text_content().strip())
main = self.parser.select(self.document.getroot(),'div.m_content_recette_main',1) main = self.parser.select(self.document.getroot(), 'div.m_content_recette_main', 1)
preparation_time = int(self.parser.select(main,'p.m_content_recette_info span.preptime',1).text_content()) preparation_time = int(self.parser.select(main, 'p.m_content_recette_info span.preptime', 1).text_content())
cooking_time = int(self.parser.select(main,'p.m_content_recette_info span.cooktime',1).text_content()) cooking_time = int(self.parser.select(main, 'p.m_content_recette_info span.cooktime', 1).text_content())
ing_header_line = self.parser.select(main,'p.m_content_recette_ingredients span',1).text_content() ing_header_line = self.parser.select(main, 'p.m_content_recette_ingredients span', 1).text_content()
if '(pour' in ing_header_line and ')' in ing_header_line: if '(pour' in ing_header_line and ')' in ing_header_line:
nb_person = int(ing_header_line.split('pour ')[-1].split('personnes)')[0].split()[0]) nb_person = int(ing_header_line.split('pour ')[-1].split('personnes)')[0].split()[0])
ingredients = self.parser.select(main,'p.m_content_recette_ingredients',1).text_content().strip().split('- ') ingredients = self.parser.select(main, 'p.m_content_recette_ingredients', 1).text_content().strip().split('- ')
ingredients=ingredients[1:] ingredients = ingredients[1:]
rinstructions = self.parser.select(main,'div.m_content_recette_todo',1).text_content().strip() rinstructions = self.parser.select(main, 'div.m_content_recette_todo', 1).text_content().strip()
instructions = u'' instructions = u''
for line in rinstructions.split('\n'): for line in rinstructions.split('\n'):
instructions += '%s\n'%line.strip() instructions += '%s\n' % line.strip()
instructions = instructions.strip('\n') instructions = instructions.strip('\n')
imgillu = self.parser.select(self.document.getroot(),'a.m_content_recette_illu img') imgillu = self.parser.select(self.document.getroot(), 'a.m_content_recette_illu img')
if len(imgillu) > 0: if len(imgillu) > 0:
picture_url = unicode(imgillu[0].attrib.get('src','')) picture_url = unicode(imgillu[0].attrib.get('src', ''))
for divcom in self.parser.select(self.document.getroot(),'div.m_commentaire_row'): for divcom in self.parser.select(self.document.getroot(), 'div.m_commentaire_row'):
note = self.parser.select(divcom,'div.m_commentaire_note span',1).text.strip() note = self.parser.select(divcom, 'div.m_commentaire_note span', 1).text.strip()
user = self.parser.select(divcom,'div.m_commentaire_content span',1).text.strip() user = self.parser.select(divcom, 'div.m_commentaire_content span', 1).text.strip()
content = self.parser.select(divcom,'div.m_commentaire_content p',1).text.strip() content = self.parser.select(divcom, 'div.m_commentaire_content p', 1).text.strip()
comments.append(u'user: %s, note: %s, comment: %s'%(user,note,content)) comments.append(u'user: %s, note: %s, comment: %s' % (user, note, content))
recipe = Recipe(id,title) recipe = Recipe(id, title)
recipe.preparation_time = preparation_time recipe.preparation_time = preparation_time
recipe.cooking_time = cooking_time recipe.cooking_time = cooking_time
recipe.nb_person = nb_person recipe.nb_person = nb_person

6
modules/opensubtitles/backend.py
View file

@ -17,7 +17,7 @@
# You should have received a copy of the GNU Affero General Public License # You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>. # along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.capabilities.subtitle import ICapSubtitle,LanguageNotSupported,Subtitle from weboob.capabilities.subtitle import ICapSubtitle, LanguageNotSupported, Subtitle
from weboob.applications.suboob.suboob import LANGUAGE_CONV from weboob.applications.suboob.suboob import LANGUAGE_CONV
from weboob.tools.backend import BaseBackend from weboob.tools.backend import BaseBackend
@ -53,7 +53,7 @@ class OpensubtitlesBackend(BaseBackend, ICapSubtitle):
def iter_subtitles(self, language, pattern): def iter_subtitles(self, language, pattern):
if language not in LANGUAGE_CONV.keys(): if language not in LANGUAGE_CONV.keys():
raise LanguageNotSupported() raise LanguageNotSupported()
return self.browser.iter_subtitles(language,quote_plus(pattern.encode('utf-8'))) return self.browser.iter_subtitles(language, quote_plus(pattern.encode('utf-8')))
def fill_subtitle(self, subtitle, fields): def fill_subtitle(self, subtitle, fields):
if 'description' in fields: if 'description' in fields:
@ -63,5 +63,5 @@ class OpensubtitlesBackend(BaseBackend, ICapSubtitle):
return subtitle return subtitle
OBJECTS = { OBJECTS = {
Subtitle:fill_subtitle, Subtitle: fill_subtitle,
} }

7
modules/opensubtitles/browser.py
View file

@ -35,13 +35,14 @@ class OpensubtitlesBrowser(BaseBrowser):
PAGES = { PAGES = {
'http://www.opensubtitles.org.*search2/sublanguageid.*moviename.*': SearchPage, 'http://www.opensubtitles.org.*search2/sublanguageid.*moviename.*': SearchPage,
'http://www.opensubtitles.org.*search/sublanguageid.*idmovie.*': SubtitlesPage, 'http://www.opensubtitles.org.*search/sublanguageid.*idmovie.*': SubtitlesPage,
'http://www.opensubtitles.org.*search/imdbid.*/sublanguageid.*/moviename.*' : SubtitlesPage, 'http://www.opensubtitles.org.*search/imdbid.*/sublanguageid.*/moviename.*': SubtitlesPage,
'http://www.opensubtitles.org.*subtitles/[0-9]*/.*' : SubtitlePage 'http://www.opensubtitles.org.*subtitles/[0-9]*/.*': SubtitlePage
} }
def iter_subtitles(self, language, pattern): def iter_subtitles(self, language, pattern):
lang = LANGUAGE_CONV[language] lang = LANGUAGE_CONV[language]
self.location('http://www.opensubtitles.org/search2/sublanguageid-%s/moviename-%s' % (lang,pattern.encode('utf-8'))) self.location('http://www.opensubtitles.org/search2/sublanguageid-%s/moviename-%s' % (
lang, pattern.encode('utf-8')))
assert self.is_on_page(SearchPage) or self.is_on_page(SubtitlesPage) or self.is_on_page(SubtitlePage) assert self.is_on_page(SearchPage) or self.is_on_page(SubtitlesPage) or self.is_on_page(SubtitlePage)
return self.page.iter_subtitles() return self.page.iter_subtitles()

64
modules/opensubtitles/pages.py
View file

@ -24,24 +24,24 @@ from weboob.tools.browser import BasePage
from weboob.applications.suboob.suboob import LANGUAGE_CONV from weboob.applications.suboob.suboob import LANGUAGE_CONV
__all__ = ['SubtitlesPage','SubtitlePage','SearchPage'] __all__ = ['SubtitlesPage', 'SubtitlePage', 'SearchPage']
class SearchPage(BasePage): class SearchPage(BasePage):
""" Page which contains results as a list of movies """ Page which contains results as a list of movies
""" """
def iter_subtitles(self): def iter_subtitles(self):
tabresults = self.parser.select(self.document.getroot(),'table#search_results') tabresults = self.parser.select(self.document.getroot(), 'table#search_results')
if len(tabresults) > 0: if len(tabresults) > 0:
table = tabresults[0] table = tabresults[0]
# for each result line, explore the subtitle list page to iter subtitles # for each result line, explore the subtitle list page to iter subtitles
for line in self.parser.select(table,'tr'): for line in self.parser.select(table, 'tr'):
links = self.parser.select(line,'a') links = self.parser.select(line, 'a')
if len(links) > 0: if len(links) > 0:
a = links[0] a = links[0]
url = a.attrib.get('href','') url = a.attrib.get('href', '')
if "ads.opensubtitles" not in url: if "ads.opensubtitles" not in url:
self.browser.location("http://www.opensubtitles.org%s"%url) self.browser.location("http://www.opensubtitles.org%s" % url)
assert self.browser.is_on_page(SubtitlesPage) or self.browser.is_on_page(SubtitlePage) assert self.browser.is_on_page(SubtitlesPage) or self.browser.is_on_page(SubtitlePage)
# subtitles page does the job # subtitles page does the job
for subtitle in self.browser.page.iter_subtitles(): for subtitle in self.browser.page.iter_subtitles():
@ -52,48 +52,48 @@ class SubtitlesPage(BasePage):
""" Page which contains several subtitles for a single movie """ Page which contains several subtitles for a single movie
""" """
def iter_subtitles(self): def iter_subtitles(self):
tabresults = self.parser.select(self.document.getroot(),'table#search_results') tabresults = self.parser.select(self.document.getroot(), 'table#search_results')
if len(tabresults) > 0: if len(tabresults) > 0:
table = tabresults[0] table = tabresults[0]
# for each result line, get informations # for each result line, get informations
# why following line doesn't work all the time (for example 'search fr sopranos guy walks' ? # why following line doesn't work all the time (for example 'search fr sopranos guy walks' ?
#for line in self.parser.select(table,'tr'): # for line in self.parser.select(table,'tr'):
for line in table.getiterator('tr'): for line in table.getiterator('tr'):
# some tr are useless, specially ads # some tr are useless, specially ads
if line.attrib.get('id','').startswith('name'): if line.attrib.get('id', '').startswith('name'):
yield self.get_subtitle_from_line(line) yield self.get_subtitle_from_line(line)
def get_subtitle_from_line(self,line): def get_subtitle_from_line(self, line):
cells = self.parser.select(line,'td') cells = self.parser.select(line, 'td')
if len(cells) > 0: if len(cells) > 0:
links = self.parser.select(line,'a') links = self.parser.select(line, 'a')
a = links[0] a = links[0]
name = u" ".join(a.text.strip().split()) name = u" ".join(a.text.strip().split())
first_cell = cells[0] first_cell = cells[0]
spanlist = self.parser.select(first_cell,'span') spanlist = self.parser.select(first_cell, 'span')
if len(spanlist) > 0: if len(spanlist) > 0:
long_name = spanlist[0].attrib.get('title','') long_name = spanlist[0].attrib.get('title', '')
else: else:
texts = first_cell.itertext() texts = first_cell.itertext()
long_name = texts.next() long_name = texts.next()
long_name = texts.next() long_name = texts.next()
if "Download at 25" in long_name: if "Download at 25" in long_name:
long_name = "---" long_name = "---"
name = "%s (%s)"%(name,long_name) name = "%s (%s)" % (name, long_name)
second_cell = cells[1] second_cell = cells[1]
link = self.parser.select(second_cell,'a',1) link = self.parser.select(second_cell, 'a', 1)
lang = link.attrib.get('href','').split('/')[-1].split('-')[-1] lang = link.attrib.get('href', '').split('/')[-1].split('-')[-1]
for lshort,llong in LANGUAGE_CONV.items(): for lshort, llong in LANGUAGE_CONV.items():
if lang == llong: if lang == llong:
lang = unicode(lshort) lang = unicode(lshort)
break break
nb_cd = int(cells[2].text.strip().lower().replace('cd','')) nb_cd = int(cells[2].text.strip().lower().replace('cd', ''))
cell_dl = cells[4] cell_dl = cells[4]
href = self.parser.select(cell_dl,'a',1).attrib.get('href','') href = self.parser.select(cell_dl, 'a', 1).attrib.get('href', '')
url = unicode('http://www.opensubtitles.org%s'%href) url = unicode('http://www.opensubtitles.org%s' % href)
id = href.split('/')[-1] id = href.split('/')[-1]
subtitle = Subtitle(id,name) subtitle = Subtitle(id, name)
subtitle.url = url subtitle.url = url
subtitle.language = lang subtitle.language = lang
subtitle.nb_cd = nb_cd subtitle.nb_cd = nb_cd
@ -106,15 +106,15 @@ class SubtitlePage(BasePage):
""" """
def get_subtitle(self): def get_subtitle(self):
desc = NotAvailable desc = NotAvailable
father = self.parser.select(self.document.getroot(),'a#app_link',1).getparent() father = self.parser.select(self.document.getroot(), 'a#app_link', 1).getparent()
a = self.parser.select(father,'a')[1] a = self.parser.select(father, 'a')[1]
id = a.attrib.get('href','').split('/')[-1] id = a.attrib.get('href', '').split('/')[-1]
url = unicode('http://www.opensubtitles.org/subtitleserve/sub/%s'%id) url = unicode('http://www.opensubtitles.org/subtitleserve/sub/%s' % id)
link = self.parser.select(self.document.getroot(),'link[rel=bookmark]',1) link = self.parser.select(self.document.getroot(), 'link[rel=bookmark]', 1)
title = unicode(link.attrib.get('title','')) title = unicode(link.attrib.get('title', ''))
nb_cd = int(title.lower().split('cd')[0].split()[-1]) nb_cd = int(title.lower().split('cd')[0].split()[-1])
lang = unicode(title.split('(')[1].split(')')[0]) lang = unicode(title.split('(')[1].split(')')[0])
file_names = self.parser.select(self.document.getroot(),"img[title~=filename]") file_names = self.parser.select(self.document.getroot(), "img[title~=filename]")
if len(file_names) > 0: if len(file_names) > 0:
file_name = file_names[0].getparent().text_content() file_name = file_names[0].getparent().text_content()
file_name = ' '.join(file_name.split()) file_name = ' '.join(file_name.split())
@ -122,11 +122,11 @@ class SubtitlePage(BasePage):
for f in file_names: for f in file_names:
desc_line = f.getparent().text_content() desc_line = f.getparent().text_content()
desc += '\n'+' '.join(desc_line.split()) desc += '\n'+' '.join(desc_line.split())
name = unicode('%s (%s)'%(title,file_name)) name = unicode('%s (%s)' % (title, file_name))
subtitle = Subtitle(id,name) subtitle = Subtitle(id, name)
subtitle.url = url subtitle.url = url
for lshort,llong in LANGUAGE_CONV.items(): for lshort, llong in LANGUAGE_CONV.items():
if lang == llong: if lang == llong:
lang = unicode(lshort) lang = unicode(lshort)
break break

2
modules/opensubtitles/test.py
View file

@ -27,7 +27,7 @@ class OpensubtitlesTest(BackendTest):
def test_subtitle(self): def test_subtitle(self):
lsub = [] lsub = []
subtitles = self.backend.iter_subtitles('fr','spiderman') subtitles = self.backend.iter_subtitles('fr', 'spiderman')
for i in range(5): for i in range(5):
subtitle = subtitles.next() subtitle = subtitles.next()
lsub.append(subtitle) lsub.append(subtitle)

4
modules/parolesmania/backend.py
View file

@ -43,7 +43,7 @@ class ParolesmaniaBackend(BaseBackend, ICapLyrics):
return self.browser.get_lyrics(id) return self.browser.get_lyrics(id)
def iter_lyrics(self, criteria, pattern): def iter_lyrics(self, criteria, pattern):
return self.browser.iter_lyrics(criteria,quote_plus(pattern.encode('utf-8'))) return self.browser.iter_lyrics(criteria, quote_plus(pattern.encode('utf-8')))
def fill_songlyrics(self, songlyrics, fields): def fill_songlyrics(self, songlyrics, fields):
if 'content' in fields: if 'content' in fields:
@ -52,5 +52,5 @@ class ParolesmaniaBackend(BaseBackend, ICapLyrics):
return songlyrics return songlyrics
OBJECTS = { OBJECTS = {
SongLyrics:fill_songlyrics SongLyrics: fill_songlyrics
} }

4
modules/parolesmania/browser.py
View file

@ -42,7 +42,7 @@ class ParolesmaniaBrowser(BaseBrowser):
crit = 'artist' crit = 'artist'
if criteria != 'artist': if criteria != 'artist':
crit = 'title' crit = 'title'
self.location('http://www.parolesmania.com/recherche.php?c=%s&k=%s'%(crit,pattern)) self.location('http://www.parolesmania.com/recherche.php?c=%s&k=%s' % (crit, pattern))
assert self.is_on_page(SongResultsPage) or self.is_on_page(ArtistResultsPage)\ assert self.is_on_page(SongResultsPage) or self.is_on_page(ArtistResultsPage)\
or self.is_on_page(ArtistSongsPage) or self.is_on_page(ArtistSongsPage)
for lyr in self.page.iter_lyrics(): for lyr in self.page.iter_lyrics():
@ -50,6 +50,6 @@ class ParolesmaniaBrowser(BaseBrowser):
def get_lyrics(self, id): def get_lyrics(self, id):
ids = id.split('|') ids = id.split('|')
self.location('http://www.parolesmania.com/paroles_%s/paroles_%s.html' % (ids[0],ids[1])) self.location('http://www.parolesmania.com/paroles_%s/paroles_%s.html' % (ids[0], ids[1]))
assert self.is_on_page(SonglyricsPage) assert self.is_on_page(SonglyricsPage)
return self.page.get_lyrics(id) return self.page.get_lyrics(id)

36
modules/parolesmania/pages.py
View file

@ -23,32 +23,32 @@ from weboob.capabilities.base import NotAvailable, NotLoaded
from weboob.tools.browser import BasePage from weboob.tools.browser import BasePage
__all__ = ['SongResultsPage','SonglyricsPage', 'ArtistResultsPage', 'ArtistSongsPage'] __all__ = ['SongResultsPage', 'SonglyricsPage', 'ArtistResultsPage', 'ArtistSongsPage']
class ArtistResultsPage(BasePage): class ArtistResultsPage(BasePage):
def iter_lyrics(self): def iter_lyrics(self):
for link in self.parser.select(self.document.getroot(),'div#albums > h1 a'): for link in self.parser.select(self.document.getroot(), 'div#albums > h1 a'):
artist = unicode(link.text_content()) artist = unicode(link.text_content())
href = link.attrib.get('href','') href = link.attrib.get('href', '')
if href.startswith('/paroles'): if href.startswith('/paroles'):
self.browser.location('http://www.parolesmania.com%s'%href) self.browser.location('http://www.parolesmania.com%s' % href)
assert self.browser.is_on_page(ArtistSongsPage) assert self.browser.is_on_page(ArtistSongsPage)
for lyr in self.browser.page.iter_lyrics(artist): for lyr in self.browser.page.iter_lyrics(artist):
yield lyr yield lyr
class ArtistSongsPage(BasePage): class ArtistSongsPage(BasePage):
def iter_lyrics(self,artist=None): def iter_lyrics(self, artist=None):
if artist is None: if artist is None:
artist = self.parser.select(self.document.getroot(),'head > title',1).text.replace('Paroles ','') artist = self.parser.select(self.document.getroot(), 'head > title', 1).text.replace('Paroles ', '')
for link in self.parser.select(self.document.getroot(),'div#albums a'): for link in self.parser.select(self.document.getroot(), 'div#albums a'):
href = link.attrib.get('href','') href = link.attrib.get('href', '')
titleattrib = link.attrib.get('title','') titleattrib = link.attrib.get('title', '')
if href.startswith('/paroles') and not href.endswith('alpha.html') and titleattrib.startswith('Paroles '): if href.startswith('/paroles') and not href.endswith('alpha.html') and titleattrib.startswith('Paroles '):
title = unicode(link.text) title = unicode(link.text)
ids = href.replace('/','').replace('.html','').split('paroles_') ids = href.replace('/', '').replace('.html', '').split('paroles_')
id = '%s|%s'%(ids[1],ids[2]) id = '%s|%s' % (ids[1], ids[2])
songlyrics = SongLyrics(id, title) songlyrics = SongLyrics(id, title)
songlyrics.artist = artist songlyrics.artist = artist
songlyrics.content = NotLoaded songlyrics.content = NotLoaded
@ -57,13 +57,13 @@ class ArtistSongsPage(BasePage):
class SongResultsPage(BasePage): class SongResultsPage(BasePage):
def iter_lyrics(self): def iter_lyrics(self):
for link in self.parser.select(self.document.getroot(),'div#albums a'): for link in self.parser.select(self.document.getroot(), 'div#albums a'):
artist = NotAvailable artist = NotAvailable
title = unicode(link.text.split(' - ')[0]) title = unicode(link.text.split(' - ')[0])
href = link.attrib.get('href','') href = link.attrib.get('href', '')
if href.startswith('/paroles') and not href.endswith('alpha.html'): if href.startswith('/paroles') and not href.endswith('alpha.html'):
ids = href.replace('/','').replace('.html','').split('paroles_') ids = href.replace('/', '').replace('.html', '').split('paroles_')
id = '%s|%s'%(ids[1],ids[2]) id = '%s|%s' % (ids[1], ids[2])
artist = unicode(link.text.split(' - ')[1]) artist = unicode(link.text.split(' - ')[1])
songlyrics = SongLyrics(id, title) songlyrics = SongLyrics(id, title)
songlyrics.artist = artist songlyrics.artist = artist
@ -76,12 +76,12 @@ class SonglyricsPage(BasePage):
content = NotAvailable content = NotAvailable
artist = NotAvailable artist = NotAvailable
title = NotAvailable title = NotAvailable
lyrdiv = self.parser.select(self.document.getroot(),'div#songlyrics_h') lyrdiv = self.parser.select(self.document.getroot(), 'div#songlyrics_h')
if len(lyrdiv) > 0: if len(lyrdiv) > 0:
content = unicode(lyrdiv[0].text_content().strip()) content = unicode(lyrdiv[0].text_content().strip())
infos = self.parser.select(self.document.getroot(),'head > title',1).text infos = self.parser.select(self.document.getroot(), 'head > title', 1).text
artist = unicode(infos.split(' - ')[1]) artist = unicode(infos.split(' - ')[1])
title = unicode(infos.split(' - ')[0].replace('Paroles ','')) title = unicode(infos.split(' - ')[0].replace('Paroles ', ''))
songlyrics = SongLyrics(id, title) songlyrics = SongLyrics(id, title)
songlyrics.artist = artist songlyrics.artist = artist
songlyrics.content = content songlyrics.content = content

4
modules/parolesmania/test.py
View file

@ -25,7 +25,7 @@ class ParolesmaniaTest(BackendTest):
BACKEND = 'parolesmania' BACKEND = 'parolesmania'
def test_search_song_n_get(self): def test_search_song_n_get(self):
l_lyrics = list(self.backend.iter_lyrics('song','chien')) l_lyrics = list(self.backend.iter_lyrics('song', 'chien'))
for songlyrics in l_lyrics: for songlyrics in l_lyrics:
assert songlyrics.id assert songlyrics.id
assert songlyrics.title assert songlyrics.title
@ -38,7 +38,7 @@ class ParolesmaniaTest(BackendTest):
assert full_lyr.content is not NotLoaded assert full_lyr.content is not NotLoaded
def test_search_artist(self): def test_search_artist(self):
l_lyrics = list(self.backend.iter_lyrics('artist','boris')) l_lyrics = list(self.backend.iter_lyrics('artist', 'boris'))
for songlyrics in l_lyrics: for songlyrics in l_lyrics:
assert songlyrics.id assert songlyrics.id
assert songlyrics.title assert songlyrics.title

4
modules/parolesmusique/backend.py
View file

@ -41,7 +41,7 @@ class ParolesmusiqueBackend(BaseBackend, ICapLyrics):
return self.browser.get_lyrics(id) return self.browser.get_lyrics(id)
def iter_lyrics(self, criteria, pattern): def iter_lyrics(self, criteria, pattern):
return self.browser.iter_lyrics(criteria,pattern.encode('utf-8')) return self.browser.iter_lyrics(criteria, pattern.encode('utf-8'))
def fill_songlyrics(self, songlyrics, fields): def fill_songlyrics(self, songlyrics, fields):
if 'content' in fields: if 'content' in fields:
@ -50,5 +50,5 @@ class ParolesmusiqueBackend(BaseBackend, ICapLyrics):
return songlyrics return songlyrics
OBJECTS = { OBJECTS = {
SongLyrics:fill_songlyrics SongLyrics: fill_songlyrics
} }

2
modules/parolesmusique/browser.py
View file

@ -42,7 +42,7 @@ class ParolesmusiqueBrowser(BaseBrowser):
def iter_lyrics(self, criteria, pattern): def iter_lyrics(self, criteria, pattern):
self.location('http://www.paroles-musique.com') self.location('http://www.paroles-musique.com')
assert self.is_on_page(HomePage) assert self.is_on_page(HomePage)
return self.page.iter_lyrics(criteria,pattern) return self.page.iter_lyrics(criteria, pattern)
def get_lyrics(self, id): def get_lyrics(self, id):
self.location('http://www.paroles-musique.com/paroles-%s' % id) self.location('http://www.paroles-musique.com/paroles-%s' % id)

26
modules/parolesmusique/pages.py
View file

@ -23,11 +23,11 @@ from weboob.capabilities.base import NotAvailable, NotLoaded
from weboob.tools.browser import BasePage from weboob.tools.browser import BasePage
__all__ = ['SongResultsPage','SonglyricsPage', 'ArtistResultsPage', 'ArtistSongsPage', 'HomePage'] __all__ = ['SongResultsPage', 'SonglyricsPage', 'ArtistResultsPage', 'ArtistSongsPage', 'HomePage']
class HomePage(BasePage): class HomePage(BasePage):
def iter_lyrics(self,criteria,pattern): def iter_lyrics(self, criteria, pattern):
self.browser.select_form(name='rechercher') self.browser.select_form(name='rechercher')
if criteria == 'artist': if criteria == 'artist':
self.browser['termes_a'] = pattern self.browser['termes_a'] = pattern
@ -41,21 +41,21 @@ class HomePage(BasePage):
class ArtistResultsPage(BasePage): class ArtistResultsPage(BasePage):
def iter_lyrics(self): def iter_lyrics(self):
for link in self.parser.select(self.document.getroot(),'div.cont_cat table a.std'): for link in self.parser.select(self.document.getroot(), 'div.cont_cat table a.std'):
artist = unicode(link.text_content()) artist = unicode(link.text_content())
self.browser.location('http://www.paroles-musique.com%s'%link.attrib.get('href','')) self.browser.location('http://www.paroles-musique.com%s' % link.attrib.get('href', ''))
assert self.browser.is_on_page(ArtistSongsPage) assert self.browser.is_on_page(ArtistSongsPage)
for lyr in self.browser.page.iter_lyrics(artist): for lyr in self.browser.page.iter_lyrics(artist):
yield lyr yield lyr
class ArtistSongsPage(BasePage): class ArtistSongsPage(BasePage):
def iter_lyrics(self,artist): def iter_lyrics(self, artist):
for link in self.parser.select(self.document.getroot(),'div.cont_catA div.art_scroll a'): for link in self.parser.select(self.document.getroot(), 'div.cont_catA div.art_scroll a'):
href = link.attrib.get('href','') href = link.attrib.get('href', '')
if href.startswith('./paroles'): if href.startswith('./paroles'):
title = unicode(link.text) title = unicode(link.text)
id = href.replace('./paroles-','') id = href.replace('./paroles-', '')
songlyrics = SongLyrics(id, title) songlyrics = SongLyrics(id, title)
songlyrics.artist = artist songlyrics.artist = artist
songlyrics.content = NotLoaded songlyrics.content = NotLoaded
@ -65,14 +65,14 @@ class ArtistSongsPage(BasePage):
class SongResultsPage(BasePage): class SongResultsPage(BasePage):
def iter_lyrics(self): def iter_lyrics(self):
first = True first = True
for tr in self.parser.select(self.document.getroot(),'div.cont_cat table tr'): for tr in self.parser.select(self.document.getroot(), 'div.cont_cat table tr'):
if first: if first:
first = False first = False
continue continue
artist = NotAvailable artist = NotAvailable
links = self.parser.select(tr,'a.std') links = self.parser.select(tr, 'a.std')
title = unicode(links[0].text) title = unicode(links[0].text)
id = links[0].attrib.get('href','').replace('/paroles-','') id = links[0].attrib.get('href', '').replace('/paroles-', '')
artist = unicode(links[1].text) artist = unicode(links[1].text)
songlyrics = SongLyrics(id, title) songlyrics = SongLyrics(id, title)
songlyrics.artist = artist songlyrics.artist = artist
@ -84,8 +84,8 @@ class SonglyricsPage(BasePage):
def get_lyrics(self, id): def get_lyrics(self, id):
artist = NotAvailable artist = NotAvailable
title = NotAvailable title = NotAvailable
content = unicode(self.parser.select(self.document.getroot(),'div#lyr_scroll',1).text_content().strip()) content = unicode(self.parser.select(self.document.getroot(), 'div#lyr_scroll', 1).text_content().strip())
infos = self.parser.select(self.document.getroot(),'h2.lyrics > font') infos = self.parser.select(self.document.getroot(), 'h2.lyrics > font')
artist = unicode(infos[0].text) artist = unicode(infos[0].text)
title = unicode(infos[1].text) title = unicode(infos[1].text)
songlyrics = SongLyrics(id, title) songlyrics = SongLyrics(id, title)

4
modules/parolesmusique/test.py
View file

@ -25,7 +25,7 @@ class ParolesmusiqueTest(BackendTest):
BACKEND = 'parolesmusique' BACKEND = 'parolesmusique'
def test_search_song_n_get(self): def test_search_song_n_get(self):
l_lyrics = list(self.backend.iter_lyrics('song','chien')) l_lyrics = list(self.backend.iter_lyrics('song', 'chien'))
for songlyrics in l_lyrics: for songlyrics in l_lyrics:
assert songlyrics.id assert songlyrics.id
assert songlyrics.title assert songlyrics.title
@ -38,7 +38,7 @@ class ParolesmusiqueTest(BackendTest):
assert full_lyr.content is not NotLoaded assert full_lyr.content is not NotLoaded
def test_search_artist(self): def test_search_artist(self):
l_lyrics = list(self.backend.iter_lyrics('artist','boris')) l_lyrics = list(self.backend.iter_lyrics('artist', 'boris'))
for songlyrics in l_lyrics: for songlyrics in l_lyrics:
assert songlyrics.id assert songlyrics.id
assert songlyrics.title assert songlyrics.title

4
modules/seeklyrics/backend.py
View file

@ -43,7 +43,7 @@ class SeeklyricsBackend(BaseBackend, ICapLyrics):
return self.browser.get_lyrics(id) return self.browser.get_lyrics(id)
def iter_lyrics(self, criteria, pattern): def iter_lyrics(self, criteria, pattern):
return self.browser.iter_lyrics(criteria,quote_plus(pattern.encode('iso-8859-1'))) return self.browser.iter_lyrics(criteria, quote_plus(pattern.encode('iso-8859-1')))
def fill_songlyrics(self, songlyrics, fields): def fill_songlyrics(self, songlyrics, fields):
if 'content' in fields: if 'content' in fields:
@ -52,5 +52,5 @@ class SeeklyricsBackend(BaseBackend, ICapLyrics):
return songlyrics return songlyrics
OBJECTS = { OBJECTS = {
SongLyrics:fill_songlyrics SongLyrics: fill_songlyrics
} }

2
modules/seeklyrics/browser.py
View file

@ -43,7 +43,7 @@ class SeeklyricsBrowser(BaseBrowser):
type = 2 type = 2
else: else:
type = 1 type = 1
self.location('http://www.seeklyrics.com/search.php?q=%s&t=%s' % (pattern,type)) self.location('http://www.seeklyrics.com/search.php?q=%s&t=%s' % (pattern, type))
assert self.is_on_page(ArtistResultsPage) or self.is_on_page(SongResultsPage) assert self.is_on_page(ArtistResultsPage) or self.is_on_page(SongResultsPage)
return self.page.iter_lyrics() return self.page.iter_lyrics()

28
modules/seeklyrics/pages.py
View file

@ -23,27 +23,27 @@ from weboob.capabilities.base import NotAvailable, NotLoaded
from weboob.tools.browser import BasePage from weboob.tools.browser import BasePage
__all__ = ['SongResultsPage','SonglyricsPage', 'ArtistResultsPage', 'ArtistSongsPage'] __all__ = ['SongResultsPage', 'SonglyricsPage', 'ArtistResultsPage', 'ArtistSongsPage']
class ArtistResultsPage(BasePage): class ArtistResultsPage(BasePage):
def iter_lyrics(self): def iter_lyrics(self):
for link in self.parser.select(self.document.getroot(),'table[title~=Results] a.tlink'): for link in self.parser.select(self.document.getroot(), 'table[title~=Results] a.tlink'):
artist = unicode(link.text_content()) artist = unicode(link.text_content())
self.browser.location('http://www.seeklyrics.com%s'%link.attrib.get('href','')) self.browser.location('http://www.seeklyrics.com%s' % link.attrib.get('href', ''))
assert self.browser.is_on_page(ArtistSongsPage) assert self.browser.is_on_page(ArtistSongsPage)
for lyr in self.browser.page.iter_lyrics(artist): for lyr in self.browser.page.iter_lyrics(artist):
yield lyr yield lyr
class ArtistSongsPage(BasePage): class ArtistSongsPage(BasePage):
def iter_lyrics(self,artist): def iter_lyrics(self, artist):
for th in self.parser.select(self.document.getroot(),'th.text'): for th in self.parser.select(self.document.getroot(), 'th.text'):
txt = th.text_content() txt = th.text_content()
if txt.startswith('Top') and txt.endswith('Lyrics'): if txt.startswith('Top') and txt.endswith('Lyrics'):
for link in self.parser.select(th.getparent().getparent(),'a.tlink'): for link in self.parser.select(th.getparent().getparent(), 'a.tlink'):
title = unicode(link.attrib.get('title','').replace(' Lyrics','')) title = unicode(link.attrib.get('title', '').replace(' Lyrics', ''))
id = link.attrib.get('href','').replace('/lyrics/','').replace('.html','') id = link.attrib.get('href', '').replace('/lyrics/', '').replace('.html', '')
songlyrics = SongLyrics(id, title) songlyrics = SongLyrics(id, title)
songlyrics.artist = artist songlyrics.artist = artist
songlyrics.content = NotLoaded songlyrics.content = NotLoaded
@ -53,15 +53,15 @@ class ArtistSongsPage(BasePage):
class SongResultsPage(BasePage): class SongResultsPage(BasePage):
def iter_lyrics(self): def iter_lyrics(self):
first = True first = True
for tr in self.parser.select(self.document.getroot(),'table[title~=Results] tr'): for tr in self.parser.select(self.document.getroot(), 'table[title~=Results] tr'):
if first: if first:
first = False first = False
continue continue
artist = NotAvailable artist = NotAvailable
ftitle = self.parser.select(tr,'a > font > font',1) ftitle = self.parser.select(tr, 'a > font > font', 1)
title = unicode(ftitle.getparent().getparent().text_content()) title = unicode(ftitle.getparent().getparent().text_content())
id = ftitle.getparent().getparent().attrib.get('href','').replace('/lyrics/','').replace('.html','') id = ftitle.getparent().getparent().attrib.get('href', '').replace('/lyrics/', '').replace('.html', '')
aartist = self.parser.select(tr,'a')[-1] aartist = self.parser.select(tr, 'a')[-1]
artist = unicode(aartist.text) artist = unicode(aartist.text)
songlyrics = SongLyrics(id, title) songlyrics = SongLyrics(id, title)
songlyrics.artist = artist songlyrics.artist = artist
@ -73,12 +73,12 @@ class SonglyricsPage(BasePage):
def get_lyrics(self, id): def get_lyrics(self, id):
artist = NotAvailable artist = NotAvailable
title = NotAvailable title = NotAvailable
l_artitle = self.parser.select(self.document.getroot(),'table.text td > b > h2') l_artitle = self.parser.select(self.document.getroot(), 'table.text td > b > h2')
if len(l_artitle) > 0: if len(l_artitle) > 0:
artitle = l_artitle[0].text.split(' Lyrics by ') artitle = l_artitle[0].text.split(' Lyrics by ')
artist = unicode(artitle[1]) artist = unicode(artitle[1])
title = unicode(artitle[0]) title = unicode(artitle[0])
content = unicode(self.parser.select(self.document.getroot(),'div#songlyrics',1).text_content().strip()) content = unicode(self.parser.select(self.document.getroot(), 'div#songlyrics', 1).text_content().strip())
songlyrics = SongLyrics(id, title) songlyrics = SongLyrics(id, title)
songlyrics.artist = artist songlyrics.artist = artist
songlyrics.content = content songlyrics.content = content

4
modules/seeklyrics/test.py
View file

@ -25,7 +25,7 @@ class SeeklyricsTest(BackendTest):
BACKEND = 'seeklyrics' BACKEND = 'seeklyrics'
def test_search_song_n_get(self): def test_search_song_n_get(self):
l_lyrics = list(self.backend.iter_lyrics('song','Complainte')) l_lyrics = list(self.backend.iter_lyrics('song', 'Complainte'))
for songlyrics in l_lyrics: for songlyrics in l_lyrics:
assert songlyrics.id assert songlyrics.id
assert songlyrics.title assert songlyrics.title
@ -38,7 +38,7 @@ class SeeklyricsTest(BackendTest):
assert full_lyr.content is not NotLoaded assert full_lyr.content is not NotLoaded
def test_search_artist(self): def test_search_artist(self):
l_lyrics = list(self.backend.iter_lyrics('artist','boris vian')) l_lyrics = list(self.backend.iter_lyrics('artist', 'boris vian'))
for songlyrics in l_lyrics: for songlyrics in l_lyrics:
assert songlyrics.id assert songlyrics.id
assert songlyrics.title assert songlyrics.title

6
modules/tvsubtitles/backend.py
View file

@ -17,10 +17,10 @@
# You should have received a copy of the GNU Affero General Public License # You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>. # along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.capabilities.subtitle import ICapSubtitle,LanguageNotSupported from weboob.capabilities.subtitle import ICapSubtitle, LanguageNotSupported
from weboob.tools.backend import BaseBackend from weboob.tools.backend import BaseBackend
from .browser import TvsubtitlesBrowser,LANGUAGE_LIST from .browser import TvsubtitlesBrowser, LANGUAGE_LIST
from urllib import quote_plus from urllib import quote_plus
@ -52,4 +52,4 @@ class TvsubtitlesBackend(BaseBackend, ICapSubtitle):
def iter_subtitles(self, language, pattern): def iter_subtitles(self, language, pattern):
if language not in LANGUAGE_LIST: if language not in LANGUAGE_LIST:
raise LanguageNotSupported() raise LanguageNotSupported()
return self.browser.iter_subtitles(language,quote_plus(pattern.encode('utf-8'))) return self.browser.iter_subtitles(language, quote_plus(pattern.encode('utf-8')))

12
modules/tvsubtitles/browser.py
View file

@ -20,14 +20,14 @@
from weboob.tools.browser import BaseBrowser from weboob.tools.browser import BaseBrowser
from .pages import SeriePage, SearchPage, SeasonPage,HomePage from .pages import SeriePage, SearchPage, SeasonPage, HomePage
__all__ = ['TvsubtitlesBrowser'] __all__ = ['TvsubtitlesBrowser']
LANGUAGE_LIST = ['en','es','fr','de','br','ru','ua','it','gr', LANGUAGE_LIST = ['en', 'es', 'fr', 'de', 'br', 'ru', 'ua', 'it', 'gr',
'ar','hu','pl','tr','nl','pt','sv','da','fi', 'ar', 'hu', 'pl', 'tr', 'nl', 'pt', 'sv', 'da', 'fi',
'ko','cn','jp','bg','cz','ro'] 'ko', 'cn', 'jp', 'bg', 'cz', 'ro']
class TvsubtitlesBrowser(BaseBrowser): class TvsubtitlesBrowser(BaseBrowser):
@ -39,13 +39,13 @@ class TvsubtitlesBrowser(BaseBrowser):
'http://www.tvsubtitles.net': HomePage, 'http://www.tvsubtitles.net': HomePage,
'http://www.tvsubtitles.net/search.php': SearchPage, 'http://www.tvsubtitles.net/search.php': SearchPage,
'http://www.tvsubtitles.net/tvshow-.*.html': SeriePage, 'http://www.tvsubtitles.net/tvshow-.*.html': SeriePage,
'http://www.tvsubtitles.net/subtitle-[0-9]*-[0-9]*-.*.html' : SeasonPage 'http://www.tvsubtitles.net/subtitle-[0-9]*-[0-9]*-.*.html': SeasonPage
} }
def iter_subtitles(self, language, pattern): def iter_subtitles(self, language, pattern):
self.location('http://www.tvsubtitles.net') self.location('http://www.tvsubtitles.net')
assert self.is_on_page(HomePage) assert self.is_on_page(HomePage)
return self.page.iter_subtitles(language,pattern) return self.page.iter_subtitles(language, pattern)
def get_subtitle(self, id): def get_subtitle(self, id):
self.location('http://www.tvsubtitles.net/subtitle-%s.html' % id) self.location('http://www.tvsubtitles.net/subtitle-%s.html' % id)

55
modules/tvsubtitles/pages.py
View file

@ -22,11 +22,11 @@ from weboob.capabilities.subtitle import Subtitle
from weboob.tools.browser import BasePage from weboob.tools.browser import BasePage
__all__ = ['HomePage','SearchPage','SeriePage','SeasonPage'] __all__ = ['HomePage', 'SearchPage', 'SeriePage', 'SeasonPage']
class HomePage(BasePage): class HomePage(BasePage):
def iter_subtitles(self,language,pattern): def iter_subtitles(self, language, pattern):
self.browser.select_form(nr=0) self.browser.select_form(nr=0)
self.browser['q'] = pattern.encode('utf-8') self.browser['q'] = pattern.encode('utf-8')
self.browser.submit() self.browser.submit()
@ -38,15 +38,15 @@ class HomePage(BasePage):
class SearchPage(BasePage): class SearchPage(BasePage):
""" Page which contains results as a list of series """ Page which contains results as a list of series
""" """
def iter_subtitles(self,language): def iter_subtitles(self, language):
list_result = self.parser.select(self.document.getroot(),'div.left_articles ul') list_result = self.parser.select(self.document.getroot(), 'div.left_articles ul')
if len(list_result) > 0: if len(list_result) > 0:
li_result = self.parser.select(list_result[0],'li') li_result = self.parser.select(list_result[0], 'li')
for line in li_result: for line in li_result:
if len(self.parser.select(line,'img[alt=%s]'%language)) > 0: if len(self.parser.select(line, 'img[alt=%s]' % language)) > 0:
link = self.parser.select(line,'a',1) link = self.parser.select(line, 'a', 1)
href = link.attrib.get('href','') href = link.attrib.get('href', '')
self.browser.location("http://%s%s"%(self.browser.DOMAIN,href)) self.browser.location("http://%s%s" % (self.browser.DOMAIN, href))
assert self.browser.is_on_page(SeriePage) assert self.browser.is_on_page(SeriePage)
for subtitle in self.browser.page.iter_subtitles(language): for subtitle in self.browser.page.iter_subtitles(language):
yield subtitle yield subtitle
@ -55,26 +55,27 @@ class SearchPage(BasePage):
class SeriePage(BasePage): class SeriePage(BasePage):
""" Page of all seasons """ Page of all seasons
""" """
def iter_subtitles(self,language,only_one_season=False): def iter_subtitles(self, language, only_one_season=False):
# handle the current season # handle the current season
last_table_line = self.parser.select(self.document.getroot(),'table#table5 tr')[-1] last_table_line = self.parser.select(self.document.getroot(), 'table#table5 tr')[-1]
amount = int(self.parser.select(last_table_line,'td')[2].text_content()) amount = int(self.parser.select(last_table_line, 'td')[2].text_content())
if amount > 0: if amount > 0:
my_lang_img = self.parser.select(last_table_line,'img[alt=%s]'%language) my_lang_img = self.parser.select(last_table_line, 'img[alt=%s]' % language)
if len(my_lang_img) > 0: if len(my_lang_img) > 0:
url_current_season = self.browser.geturl().split('/')[-1].replace('tvshow','subtitle').replace('.html','-%s.html'%language) url_current_season = self.browser.geturl().split('/')[-1].replace(
'tvshow', 'subtitle').replace('.html', '-%s.html' % language)
self.browser.location(url_current_season) self.browser.location(url_current_season)
assert self.browser.is_on_page(SeasonPage) assert self.browser.is_on_page(SeasonPage)
yield self.browser.page.iter_subtitles() yield self.browser.page.iter_subtitles()
if not only_one_season: if not only_one_season:
# handle the other seasons by following top links # handle the other seasons by following top links
other_seasons_links = self.parser.select(self.document.getroot(),'p.description a') other_seasons_links = self.parser.select(self.document.getroot(), 'p.description a')
for link in other_seasons_links: for link in other_seasons_links:
href = link.attrib.get('href','') href = link.attrib.get('href', '')
self.browser.location("http://%s/%s"%(self.browser.DOMAIN,href)) self.browser.location("http://%s/%s" % (self.browser.DOMAIN, href))
assert self.browser.is_on_page(SeriePage) assert self.browser.is_on_page(SeriePage)
for subtitle in self.browser.page.iter_subtitles(language,True): for subtitle in self.browser.page.iter_subtitles(language, True):
yield subtitle yield subtitle
@ -82,19 +83,19 @@ class SeasonPage(BasePage):
""" Page of a season with the right language """ Page of a season with the right language
""" """
def get_subtitle(self): def get_subtitle(self):
filename_line = self.parser.select(self.document.getroot(),'img[alt=filename]',1).getparent().getparent() filename_line = self.parser.select(self.document.getroot(), 'img[alt=filename]', 1).getparent().getparent()
name = unicode(self.parser.select(filename_line,'td')[2].text) name = unicode(self.parser.select(filename_line, 'td')[2].text)
id = self.browser.geturl().split('/')[-1].replace('.html','').replace('subtitle-','') id = self.browser.geturl().split('/')[-1].replace('.html', '').replace('subtitle-', '')
url = unicode('http://%s/download-%s.html'%(self.browser.DOMAIN,id)) url = unicode('http://%s/download-%s.html' % (self.browser.DOMAIN, id))
amount_line = self.parser.select(self.document.getroot(),'tr[title~=amount]',1) amount_line = self.parser.select(self.document.getroot(), 'tr[title~=amount]', 1)
nb_cd = int(self.parser.select(amount_line,'td')[2].text) nb_cd = int(self.parser.select(amount_line, 'td')[2].text)
lang = unicode(url.split('-')[-1].split('.html')[0]) lang = unicode(url.split('-')[-1].split('.html')[0])
filenames_line = self.parser.select(self.document.getroot(),'tr[title~=list]',1) filenames_line = self.parser.select(self.document.getroot(), 'tr[title~=list]', 1)
file_names = self.parser.select(filenames_line,'td')[2].text_content().strip().replace('.srt','.srt\n') file_names = self.parser.select(filenames_line, 'td')[2].text_content().strip().replace('.srt', '.srt\n')
desc = u"files :\n" desc = u"files :\n"
desc += file_names desc += file_names
subtitle = Subtitle(id,name) subtitle = Subtitle(id, name)
subtitle.url = url subtitle.url = url
subtitle.language = lang subtitle.language = lang
subtitle.nb_cd = nb_cd subtitle.nb_cd = nb_cd

2
modules/tvsubtitles/test.py
View file

@ -26,7 +26,7 @@ class TvsubtitlesTest(BackendTest):
BACKEND = 'tvsubtitles' BACKEND = 'tvsubtitles'
def test_subtitle(self): def test_subtitle(self):
subtitles = list(self.backend.iter_subtitles('fr','sopranos')) subtitles = list(self.backend.iter_subtitles('fr', 'sopranos'))
assert (len(subtitles) > 0) assert (len(subtitles) > 0)
for subtitle in subtitles: for subtitle in subtitles:
assert subtitle.url.startswith('http') assert subtitle.url.startswith('http')