fix #1706 marmiton and 750g are now working using browser2
This commit is contained in:
Bezleputh
Romain Bignon
parent
e0cb4118f8
commit
8250846448
9 changed files with 151 additions and 260 deletions
|
|
@ -18,33 +18,25 @@
|
|||
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
from weboob.deprecated.browser import Browser, BrowserHTTPNotFound
|
||||
|
||||
from weboob.browser import PagesBrowser, URL
|
||||
from .pages import RecipePage, ResultsPage
|
||||
|
||||
|
||||
__all__ = ['SevenFiftyGramsBrowser']
|
||||
|
||||
|
||||
class SevenFiftyGramsBrowser(Browser):
|
||||
DOMAIN = 'www.750g.com'
|
||||
PROTOCOL = 'http'
|
||||
ENCODING = 'windows-1252'
|
||||
USER_AGENT = Browser.USER_AGENTS['wget']
|
||||
PAGES = {
|
||||
'http://www.750g.com/recettes_.*.htm': ResultsPage,
|
||||
'http://www.750g.com/fiche_de_cuisine_complete.htm\?recettes_id=[0-9]*': RecipePage,
|
||||
}
|
||||
class SevenFiftyGramsBrowser(PagesBrowser):
|
||||
BASEURL = 'http://www.750g.com/'
|
||||
|
||||
search = URL('recettes_(?P<pattern>.*).htm', ResultsPage)
|
||||
recipe = URL('(?P<id>.*).htm', RecipePage)
|
||||
|
||||
def iter_recipes(self, pattern):
|
||||
self.location('http://www.750g.com/recettes_%s.htm' % (pattern.replace(' ', '_')))
|
||||
assert self.is_on_page(ResultsPage)
|
||||
return self.page.iter_recipes()
|
||||
return self.search.go(pattern=pattern.replace(' ', '_')).iter_recipes()
|
||||
|
||||
def get_recipe(self, id):
|
||||
try:
|
||||
self.location('http://www.750g.com/fiche_de_cuisine_complete.htm?recettes_id=%s' % id)
|
||||
except BrowserHTTPNotFound:
|
||||
return
|
||||
if self.is_on_page(RecipePage):
|
||||
return self.page.get_recipe(id)
|
||||
def get_recipe(self, id, recipe=None):
|
||||
recipe = self.recipe.go(id=id).get_recipe(obj=recipe)
|
||||
comments = list(self.page.get_comments())
|
||||
if comments:
|
||||
recipe.comments = comments
|
||||
return recipe
|
||||
|
|
|
|||
|
|
@ -48,16 +48,7 @@ class SevenFiftyGramsModule(Module, CapRecipe):
|
|||
|
||||
def fill_recipe(self, recipe, fields):
|
||||
if 'nb_person' in fields or 'instructions' in fields:
|
||||
rec = self.get_recipe(recipe.id)
|
||||
recipe.picture_url = rec.picture_url
|
||||
recipe.instructions = rec.instructions
|
||||
recipe.ingredients = rec.ingredients
|
||||
recipe.comments = rec.comments
|
||||
recipe.author = rec.author
|
||||
recipe.nb_person = rec.nb_person
|
||||
recipe.cooking_time = rec.cooking_time
|
||||
recipe.preparation_time = rec.preparation_time
|
||||
|
||||
recipe = self.browser.get_recipe(recipe.id, recipe)
|
||||
return recipe
|
||||
|
||||
OBJECTS = {
|
||||
|
|
|
|||
|
|
@ -19,128 +19,75 @@
|
|||
|
||||
|
||||
from weboob.capabilities.recipe import Recipe, Comment
|
||||
from weboob.capabilities.base import NotAvailable, NotLoaded
|
||||
from weboob.deprecated.browser import Page
|
||||
from weboob.capabilities.base import NotAvailable
|
||||
from weboob.browser.pages import HTMLPage, pagination
|
||||
from weboob.browser.elements import ItemElement, ListElement, method
|
||||
from weboob.browser.filters.standard import CleanText, Regexp, Env, Type, Filter
|
||||
from weboob.browser.filters.html import CleanHTML
|
||||
|
||||
|
||||
class ResultsPage(Page):
|
||||
class Time(Filter):
|
||||
def filter(self, el):
|
||||
if el:
|
||||
if 'h' in el:
|
||||
return 60*int(el.split()[0])
|
||||
return int(el.split()[0])
|
||||
|
||||
|
||||
class ResultsPage(HTMLPage):
|
||||
""" Page which contains results as a list of recipies
|
||||
"""
|
||||
@pagination
|
||||
@method
|
||||
class iter_recipes(ListElement):
|
||||
item_xpath = '//li[@data-type="recette"]'
|
||||
|
||||
def iter_recipes(self):
|
||||
for div in self.parser.select(self.document.getroot(), 'div.recette_description > div.data'):
|
||||
links = self.parser.select(div, 'div.info > p.title > a.fn')
|
||||
if len(links) > 0:
|
||||
link = links[0]
|
||||
title = unicode(link.text)
|
||||
# id = unicode(link.attrib.get('href','').strip('/').replace('.htm','htm'))
|
||||
id = unicode(self.parser.select(div, 'div.carnet-add a', 1).attrib.get('href', '').split('=')[-1])
|
||||
thumbnail_url = NotAvailable
|
||||
short_description = NotAvailable
|
||||
def next_page(self):
|
||||
return CleanText('//li[@class="suivante"]/a/@href')(self)
|
||||
|
||||
imgs = self.parser.select(div, 'img.recipe-image')
|
||||
if len(imgs) > 0:
|
||||
thumbnail_url = unicode(imgs[0].attrib.get('src', ''))
|
||||
short_description = unicode(' '.join(self.parser.select(
|
||||
div, 'div.infos_column', 1).text_content().split()).strip())
|
||||
imgs_cost = self.parser.select(div, 'div.infos_column img')
|
||||
cost_tot = len(imgs_cost)
|
||||
cost_on = 0
|
||||
for img in imgs_cost:
|
||||
if img.attrib.get('src', '').endswith('euro_on.png'):
|
||||
cost_on += 1
|
||||
short_description += u' %s/%s' % (cost_on, cost_tot)
|
||||
|
||||
recipe = Recipe(id, title)
|
||||
recipe.thumbnail_url = thumbnail_url
|
||||
recipe.short_description = short_description
|
||||
recipe.instructions = NotLoaded
|
||||
recipe.ingredients = NotLoaded
|
||||
recipe.nb_person = NotLoaded
|
||||
recipe.cooking_time = NotLoaded
|
||||
recipe.preparation_time = NotLoaded
|
||||
recipe.author = NotLoaded
|
||||
yield recipe
|
||||
class item(ItemElement):
|
||||
klass = Recipe
|
||||
obj_id = Regexp(CleanText('./div[has-class("text")]/h2/a/@href'),
|
||||
'(.*).htm')
|
||||
obj_title = CleanText('./div[has-class("text")]/h2/a')
|
||||
obj_thumbnail_url = CleanText('./div[has-class("image")]/a/img/@src')
|
||||
obj_short_description = CleanText('./div[has-class("text")]/p')
|
||||
obj_author = CleanText('./div[has-class("text")]/h3[@class="auteur"]/a', default=NotAvailable)
|
||||
|
||||
|
||||
class RecipePage(Page):
|
||||
class RecipePage(HTMLPage):
|
||||
""" Page which contains a recipe
|
||||
"""
|
||||
@method
|
||||
class get_comments(ListElement):
|
||||
item_xpath = '//section[@class="commentaires_liste"]/article'
|
||||
|
||||
def get_recipe(self, id):
|
||||
title = NotAvailable
|
||||
preparation_time = NotAvailable
|
||||
cooking_time = NotAvailable
|
||||
nb_person = NotAvailable
|
||||
ingredients = NotAvailable
|
||||
picture_url = NotAvailable
|
||||
instructions = NotAvailable
|
||||
author = NotAvailable
|
||||
comments = NotAvailable
|
||||
class item(ItemElement):
|
||||
klass = Comment
|
||||
|
||||
title = unicode(self.parser.select(self.document.getroot(), 'head > title', 1).text.split(' - ')[1])
|
||||
main = self.parser.select(self.document.getroot(), 'div.recette_description', 1)
|
||||
obj_id = CleanText('./@data-id')
|
||||
obj_author = CleanText('./div[@class="column"]/p[@class="commentaire_info"]/span')
|
||||
obj_text = CleanText('./div[@class="column"]/p[1]')
|
||||
|
||||
rec_infos = self.parser.select(self.document.getroot(), 'div.recette_infos div.infos_column strong')
|
||||
for info_title in rec_infos:
|
||||
if u'Temps de préparation' in unicode(info_title.text):
|
||||
if info_title.tail.strip() != '':
|
||||
preparation_time = int(info_title.tail.split()[0])
|
||||
if 'h' in info_title.tail:
|
||||
preparation_time = 60*preparation_time
|
||||
if 'Temps de cuisson' in info_title.text:
|
||||
if info_title.tail.strip() != '':
|
||||
cooking_time = int(info_title.tail.split()[0])
|
||||
if 'h' in info_title.tail:
|
||||
cooking_time = 60*cooking_time
|
||||
if 'Nombre de personnes' in info_title.text:
|
||||
if info_title.tail.strip() != '':
|
||||
nb_person = [int(info_title.tail)]
|
||||
@method
|
||||
class get_recipe(ItemElement):
|
||||
klass = Recipe
|
||||
|
||||
ingredients = []
|
||||
p_ing = self.parser.select(main, 'div.data.top.left > div.content p')
|
||||
for ing in p_ing:
|
||||
ingtxt = unicode(ing.text_content().strip())
|
||||
if ingtxt != '':
|
||||
ingredients.append(ingtxt)
|
||||
obj_id = Env('id')
|
||||
obj_title = CleanText('//h1[@class="fn"]')
|
||||
|
||||
lines_instr = self.parser.select(main, 'div.data.top.right div.content li')
|
||||
if len(lines_instr) > 0:
|
||||
instructions = u''
|
||||
for line in lines_instr:
|
||||
inst = ' '.join(line.text_content().strip().split())
|
||||
instructions += '%s\n' % inst
|
||||
instructions = instructions.strip('\n')
|
||||
def obj_ingredients(self):
|
||||
ingredients = []
|
||||
for el in self.page.doc.xpath('//section[has-class("recette_ingredients")]/ul/li'):
|
||||
ingredients.append(CleanText('.')(el))
|
||||
return ingredients
|
||||
|
||||
imgillu = self.parser.select(self.document.getroot(), 'div.resume_recette_illustree img.photo')
|
||||
if len(imgillu) > 0:
|
||||
picture_url = unicode(imgillu[0].attrib.get('src', ''))
|
||||
obj_cooking_time = Time(CleanText('//span[@class="cooktime"]'))
|
||||
obj_preparation_time = Time(CleanText('//span[@class="preptime"]'))
|
||||
|
||||
divcoms = self.parser.select(self.document.getroot(), 'div.comment-outer')
|
||||
if len(divcoms) > 0:
|
||||
comments = []
|
||||
for divcom in divcoms:
|
||||
comtxt = unicode(' '.join(divcom.text_content().strip().split()))
|
||||
if u'| Répondre' in comtxt:
|
||||
comtxt = comtxt.strip('0123456789').replace(u' | Répondre', '')
|
||||
author = None
|
||||
if 'par ' in comtxt:
|
||||
author = comtxt.split('par ')[-1].split('|')[0]
|
||||
comtxt = comtxt.replace('par %s' % author, '')
|
||||
comments.append(Comment(text=comtxt, author=author))
|
||||
def obj_nb_person(self):
|
||||
return [Type(CleanText('//span[@class="yield"]'), type=int)(self)]
|
||||
|
||||
links_author = self.parser.select(self.document.getroot(), 'p.auteur a.couleur_membre')
|
||||
if len(links_author) > 0:
|
||||
author = unicode(links_author[0].text.strip())
|
||||
|
||||
recipe = Recipe(id, title)
|
||||
recipe.preparation_time = preparation_time
|
||||
recipe.cooking_time = cooking_time
|
||||
recipe.nb_person = nb_person
|
||||
recipe.ingredients = ingredients
|
||||
recipe.instructions = instructions
|
||||
recipe.picture_url = picture_url
|
||||
recipe.comments = comments
|
||||
recipe.author = author
|
||||
recipe.thumbnail_url = NotLoaded
|
||||
return recipe
|
||||
obj_instructions = CleanHTML('//div[@class="recette_etapes"]')
|
||||
obj_picture_url = CleanText('//section[has-class("recette_infos")]/div/img[@class="photo"]/@src')
|
||||
obj_author = CleanText('//span[@class="author"]', default=NotAvailable)
|
||||
|
|
|
|||
|
|
@ -18,13 +18,13 @@
|
|||
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
from weboob.tools.test import BackendTest
|
||||
|
||||
import itertools
|
||||
|
||||
class SevenFiftyGramsTest(BackendTest):
|
||||
MODULE = '750g'
|
||||
|
||||
def test_recipe(self):
|
||||
recipes = self.backend.iter_recipes('fondue')
|
||||
recipes = list(itertools.islice(self.backend.iter_recipes('fondue'), 0, 20))
|
||||
for recipe in recipes:
|
||||
full_recipe = self.backend.get_recipe(recipe.id)
|
||||
assert full_recipe.instructions
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue