diff --git a/modules/750g/browser.py b/modules/750g/browser.py
index 400db737..333037b0 100644
--- a/modules/750g/browser.py
+++ b/modules/750g/browser.py
@@ -18,33 +18,25 @@
# along with weboob. If not, see .
-from weboob.deprecated.browser import Browser, BrowserHTTPNotFound
-
+from weboob.browser import PagesBrowser, URL
from .pages import RecipePage, ResultsPage
__all__ = ['SevenFiftyGramsBrowser']
-class SevenFiftyGramsBrowser(Browser):
- DOMAIN = 'www.750g.com'
- PROTOCOL = 'http'
- ENCODING = 'windows-1252'
- USER_AGENT = Browser.USER_AGENTS['wget']
- PAGES = {
- 'http://www.750g.com/recettes_.*.htm': ResultsPage,
- 'http://www.750g.com/fiche_de_cuisine_complete.htm\?recettes_id=[0-9]*': RecipePage,
- }
+class SevenFiftyGramsBrowser(PagesBrowser):
+ BASEURL = 'http://www.750g.com/'
+
+ search = URL('recettes_(?P.*).htm', ResultsPage)
+ recipe = URL('(?P.*).htm', RecipePage)
def iter_recipes(self, pattern):
- self.location('http://www.750g.com/recettes_%s.htm' % (pattern.replace(' ', '_')))
- assert self.is_on_page(ResultsPage)
- return self.page.iter_recipes()
+ return self.search.go(pattern=pattern.replace(' ', '_')).iter_recipes()
- def get_recipe(self, id):
- try:
- self.location('http://www.750g.com/fiche_de_cuisine_complete.htm?recettes_id=%s' % id)
- except BrowserHTTPNotFound:
- return
- if self.is_on_page(RecipePage):
- return self.page.get_recipe(id)
+ def get_recipe(self, id, recipe=None):
+ recipe = self.recipe.go(id=id).get_recipe(obj=recipe)
+ comments = list(self.page.get_comments())
+ if comments:
+ recipe.comments = comments
+ return recipe
diff --git a/modules/750g/module.py b/modules/750g/module.py
index 03c24165..cd8b72d4 100644
--- a/modules/750g/module.py
+++ b/modules/750g/module.py
@@ -48,16 +48,7 @@ class SevenFiftyGramsModule(Module, CapRecipe):
def fill_recipe(self, recipe, fields):
if 'nb_person' in fields or 'instructions' in fields:
- rec = self.get_recipe(recipe.id)
- recipe.picture_url = rec.picture_url
- recipe.instructions = rec.instructions
- recipe.ingredients = rec.ingredients
- recipe.comments = rec.comments
- recipe.author = rec.author
- recipe.nb_person = rec.nb_person
- recipe.cooking_time = rec.cooking_time
- recipe.preparation_time = rec.preparation_time
-
+ recipe = self.browser.get_recipe(recipe.id, recipe)
return recipe
OBJECTS = {
diff --git a/modules/750g/pages.py b/modules/750g/pages.py
index ed4e2a9a..640d8d65 100644
--- a/modules/750g/pages.py
+++ b/modules/750g/pages.py
@@ -19,128 +19,75 @@
from weboob.capabilities.recipe import Recipe, Comment
-from weboob.capabilities.base import NotAvailable, NotLoaded
-from weboob.deprecated.browser import Page
+from weboob.capabilities.base import NotAvailable
+from weboob.browser.pages import HTMLPage, pagination
+from weboob.browser.elements import ItemElement, ListElement, method
+from weboob.browser.filters.standard import CleanText, Regexp, Env, Type, Filter
+from weboob.browser.filters.html import CleanHTML
-class ResultsPage(Page):
+class Time(Filter):
+ def filter(self, el):
+ if el:
+ if 'h' in el:
+ return 60*int(el.split()[0])
+ return int(el.split()[0])
+
+
+class ResultsPage(HTMLPage):
""" Page which contains results as a list of recipies
"""
+ @pagination
+ @method
+ class iter_recipes(ListElement):
+ item_xpath = '//li[@data-type="recette"]'
- def iter_recipes(self):
- for div in self.parser.select(self.document.getroot(), 'div.recette_description > div.data'):
- links = self.parser.select(div, 'div.info > p.title > a.fn')
- if len(links) > 0:
- link = links[0]
- title = unicode(link.text)
- # id = unicode(link.attrib.get('href','').strip('/').replace('.htm','htm'))
- id = unicode(self.parser.select(div, 'div.carnet-add a', 1).attrib.get('href', '').split('=')[-1])
- thumbnail_url = NotAvailable
- short_description = NotAvailable
+ def next_page(self):
+ return CleanText('//li[@class="suivante"]/a/@href')(self)
- imgs = self.parser.select(div, 'img.recipe-image')
- if len(imgs) > 0:
- thumbnail_url = unicode(imgs[0].attrib.get('src', ''))
- short_description = unicode(' '.join(self.parser.select(
- div, 'div.infos_column', 1).text_content().split()).strip())
- imgs_cost = self.parser.select(div, 'div.infos_column img')
- cost_tot = len(imgs_cost)
- cost_on = 0
- for img in imgs_cost:
- if img.attrib.get('src', '').endswith('euro_on.png'):
- cost_on += 1
- short_description += u' %s/%s' % (cost_on, cost_tot)
-
- recipe = Recipe(id, title)
- recipe.thumbnail_url = thumbnail_url
- recipe.short_description = short_description
- recipe.instructions = NotLoaded
- recipe.ingredients = NotLoaded
- recipe.nb_person = NotLoaded
- recipe.cooking_time = NotLoaded
- recipe.preparation_time = NotLoaded
- recipe.author = NotLoaded
- yield recipe
+ class item(ItemElement):
+ klass = Recipe
+ obj_id = Regexp(CleanText('./div[has-class("text")]/h2/a/@href'),
+ '(.*).htm')
+ obj_title = CleanText('./div[has-class("text")]/h2/a')
+ obj_thumbnail_url = CleanText('./div[has-class("image")]/a/img/@src')
+ obj_short_description = CleanText('./div[has-class("text")]/p')
+ obj_author = CleanText('./div[has-class("text")]/h3[@class="auteur"]/a', default=NotAvailable)
-class RecipePage(Page):
+class RecipePage(HTMLPage):
""" Page which contains a recipe
"""
+ @method
+ class get_comments(ListElement):
+ item_xpath = '//section[@class="commentaires_liste"]/article'
- def get_recipe(self, id):
- title = NotAvailable
- preparation_time = NotAvailable
- cooking_time = NotAvailable
- nb_person = NotAvailable
- ingredients = NotAvailable
- picture_url = NotAvailable
- instructions = NotAvailable
- author = NotAvailable
- comments = NotAvailable
+ class item(ItemElement):
+ klass = Comment
- title = unicode(self.parser.select(self.document.getroot(), 'head > title', 1).text.split(' - ')[1])
- main = self.parser.select(self.document.getroot(), 'div.recette_description', 1)
+ obj_id = CleanText('./@data-id')
+ obj_author = CleanText('./div[@class="column"]/p[@class="commentaire_info"]/span')
+ obj_text = CleanText('./div[@class="column"]/p[1]')
- rec_infos = self.parser.select(self.document.getroot(), 'div.recette_infos div.infos_column strong')
- for info_title in rec_infos:
- if u'Temps de préparation' in unicode(info_title.text):
- if info_title.tail.strip() != '':
- preparation_time = int(info_title.tail.split()[0])
- if 'h' in info_title.tail:
- preparation_time = 60*preparation_time
- if 'Temps de cuisson' in info_title.text:
- if info_title.tail.strip() != '':
- cooking_time = int(info_title.tail.split()[0])
- if 'h' in info_title.tail:
- cooking_time = 60*cooking_time
- if 'Nombre de personnes' in info_title.text:
- if info_title.tail.strip() != '':
- nb_person = [int(info_title.tail)]
+ @method
+ class get_recipe(ItemElement):
+ klass = Recipe
- ingredients = []
- p_ing = self.parser.select(main, 'div.data.top.left > div.content p')
- for ing in p_ing:
- ingtxt = unicode(ing.text_content().strip())
- if ingtxt != '':
- ingredients.append(ingtxt)
+ obj_id = Env('id')
+ obj_title = CleanText('//h1[@class="fn"]')
- lines_instr = self.parser.select(main, 'div.data.top.right div.content li')
- if len(lines_instr) > 0:
- instructions = u''
- for line in lines_instr:
- inst = ' '.join(line.text_content().strip().split())
- instructions += '%s\n' % inst
- instructions = instructions.strip('\n')
+ def obj_ingredients(self):
+ ingredients = []
+ for el in self.page.doc.xpath('//section[has-class("recette_ingredients")]/ul/li'):
+ ingredients.append(CleanText('.')(el))
+ return ingredients
- imgillu = self.parser.select(self.document.getroot(), 'div.resume_recette_illustree img.photo')
- if len(imgillu) > 0:
- picture_url = unicode(imgillu[0].attrib.get('src', ''))
+ obj_cooking_time = Time(CleanText('//span[@class="cooktime"]'))
+ obj_preparation_time = Time(CleanText('//span[@class="preptime"]'))
- divcoms = self.parser.select(self.document.getroot(), 'div.comment-outer')
- if len(divcoms) > 0:
- comments = []
- for divcom in divcoms:
- comtxt = unicode(' '.join(divcom.text_content().strip().split()))
- if u'| Répondre' in comtxt:
- comtxt = comtxt.strip('0123456789').replace(u' | Répondre', '')
- author = None
- if 'par ' in comtxt:
- author = comtxt.split('par ')[-1].split('|')[0]
- comtxt = comtxt.replace('par %s' % author, '')
- comments.append(Comment(text=comtxt, author=author))
+ def obj_nb_person(self):
+ return [Type(CleanText('//span[@class="yield"]'), type=int)(self)]
- links_author = self.parser.select(self.document.getroot(), 'p.auteur a.couleur_membre')
- if len(links_author) > 0:
- author = unicode(links_author[0].text.strip())
-
- recipe = Recipe(id, title)
- recipe.preparation_time = preparation_time
- recipe.cooking_time = cooking_time
- recipe.nb_person = nb_person
- recipe.ingredients = ingredients
- recipe.instructions = instructions
- recipe.picture_url = picture_url
- recipe.comments = comments
- recipe.author = author
- recipe.thumbnail_url = NotLoaded
- return recipe
+ obj_instructions = CleanHTML('//div[@class="recette_etapes"]')
+ obj_picture_url = CleanText('//section[has-class("recette_infos")]/div/img[@class="photo"]/@src')
+ obj_author = CleanText('//span[@class="author"]', default=NotAvailable)
diff --git a/modules/750g/test.py b/modules/750g/test.py
index a37f8b11..2f0e32bb 100644
--- a/modules/750g/test.py
+++ b/modules/750g/test.py
@@ -18,13 +18,13 @@
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
-
+import itertools
class SevenFiftyGramsTest(BackendTest):
MODULE = '750g'
def test_recipe(self):
- recipes = self.backend.iter_recipes('fondue')
+ recipes = list(itertools.islice(self.backend.iter_recipes('fondue'), 0, 20))
for recipe in recipes:
full_recipe = self.backend.get_recipe(recipe.id)
assert full_recipe.instructions
diff --git a/modules/marmiton/browser.py b/modules/marmiton/browser.py
index 10a1637a..03c11966 100644
--- a/modules/marmiton/browser.py
+++ b/modules/marmiton/browser.py
@@ -18,7 +18,7 @@
# along with weboob. If not, see .
-from weboob.deprecated.browser import Browser, BrowserHTTPNotFound
+from weboob.browser import PagesBrowser, URL
from .pages import RecipePage, ResultsPage
@@ -26,25 +26,17 @@ from .pages import RecipePage, ResultsPage
__all__ = ['MarmitonBrowser']
-class MarmitonBrowser(Browser):
- DOMAIN = 'www.marmiton.org'
- PROTOCOL = 'http'
- ENCODING = 'utf-8'
- USER_AGENT = Browser.USER_AGENTS['wget']
- PAGES = {
- 'http://www.marmiton.org/recettes/recherche.aspx.*': ResultsPage,
- 'http://www.marmiton.org/recettes/recette_.*': RecipePage,
- }
+class MarmitonBrowser(PagesBrowser):
+ BASEURL = 'http://www.marmiton.org/'
+ search = URL('recettes/recherche.aspx\?aqt=(?P.*)', ResultsPage)
+ recipe = URL('recettes/recette_(?P.*).aspx', RecipePage)
def iter_recipes(self, pattern):
- self.location('http://www.marmiton.org/recettes/recherche.aspx?st=5&cli=1&aqt=%s' % (pattern))
- assert self.is_on_page(ResultsPage)
- return self.page.iter_recipes()
+ return self.search.go(pattern=pattern).iter_recipes()
- def get_recipe(self, id):
- try:
- self.location('http://www.marmiton.org/recettes/recette_%s.aspx' % id)
- except BrowserHTTPNotFound:
- return
- if self.is_on_page(RecipePage):
- return self.page.get_recipe(id)
+ def get_recipe(self, id, recipe=None):
+ recipe = self.recipe.go(id=id).get_recipe(obj=recipe)
+ comments = list(self.page.get_comments())
+ if comments:
+ recipe.comments = comments
+ return recipe
diff --git a/modules/marmiton/module.py b/modules/marmiton/module.py
index 5c40e9ce..42b389b1 100644
--- a/modules/marmiton/module.py
+++ b/modules/marmiton/module.py
@@ -44,16 +44,7 @@ class MarmitonModule(Module, CapRecipe):
def fill_recipe(self, recipe, fields):
if 'nb_person' in fields or 'instructions' in fields:
- rec = self.get_recipe(recipe.id)
- recipe.picture_url = rec.picture_url
- recipe.instructions = rec.instructions
- recipe.ingredients = rec.ingredients
- recipe.comments = rec.comments
- recipe.author = rec.author
- recipe.nb_person = rec.nb_person
- recipe.cooking_time = rec.cooking_time
- recipe.preparation_time = rec.preparation_time
-
+ recipe = self.browser.get_recipe(recipe.id, recipe)
return recipe
OBJECTS = {
diff --git a/modules/marmiton/pages.py b/modules/marmiton/pages.py
index db8f1f16..010dbb69 100644
--- a/modules/marmiton/pages.py
+++ b/modules/marmiton/pages.py
@@ -17,93 +17,71 @@
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
-
+from weboob.browser.pages import HTMLPage, pagination
+from weboob.browser.elements import ItemElement, ListElement, method
+from weboob.browser.filters.standard import Regexp, CleanText, Format, Env, Type
+from weboob.browser.filters.html import CleanHTML
from weboob.capabilities.recipe import Recipe, Comment
-from weboob.capabilities.base import NotAvailable, NotLoaded
-from weboob.deprecated.browser import Page
+from weboob.capabilities.base import NotAvailable
-class ResultsPage(Page):
+class ResultsPage(HTMLPage):
""" Page which contains results as a list of recipies
"""
+ @pagination
+ @method
+ class iter_recipes(ListElement):
+ item_xpath = '//div[has-class("recette_classique")]'
- def iter_recipes(self):
- for div in self.parser.select(self.document.getroot(), 'div.m_search_result'):
- tds = self.parser.select(div, 'td')
- if len(tds) == 2:
- title = NotAvailable
- thumbnail_url = NotAvailable
- short_description = NotAvailable
- imgs = self.parser.select(tds[0], 'img')
- if len(imgs) > 0:
- thumbnail_url = unicode(imgs[0].attrib.get('src', ''))
- link = self.parser.select(tds[1], 'div.m_search_titre_recette a', 1)
- title = unicode(link.text)
- id = link.attrib.get('href', '').replace('.aspx', '').replace('/recettes/recette_', '')
- short_description = unicode(' '.join(self.parser.select(tds[
- 1], 'div.m_search_result_part4', 1).text.strip().split('\n')))
+ def next_page(self):
+ return CleanText('//a[@id="ctl00_cphMainContent_m_ctrlSearchEngine_m_ctrlSearchListDisplay_m_ctrlSearchPagination_m_linkNextPage"]/@href',
+ default=None)(self)
- recipe = Recipe(id, title)
- recipe.thumbnail_url = thumbnail_url
- recipe.short_description = short_description
- recipe.instructions = NotLoaded
- recipe.author = NotLoaded
- recipe.ingredients = NotLoaded
- recipe.nb_person = NotLoaded
- recipe.cooking_time = NotLoaded
- recipe.preparation_time = NotLoaded
- yield recipe
+ class item(ItemElement):
+ klass = Recipe
+ obj_id = Regexp(CleanText('./div/div[@class="m_titre_resultat"]/a/@href'),
+ '/recettes/recette_(.*).aspx')
+ obj_title = CleanText('./div/div[@class="m_titre_resultat"]/a')
+ obj_thumbnail_url = CleanText('./a[@class="m_resultat_lien_image"]', default='')
+ obj_short_description = Format('%s. %s',
+ CleanText('./div/div[@class="m_detail_recette"]'),
+ CleanText('./div/div[@class="m_texte_resultat"]'))
-class RecipePage(Page):
+class RecipePage(HTMLPage):
""" Page which contains a recipe
"""
+ @method
+ class get_recipe(ItemElement):
+ klass = Recipe
- def get_recipe(self, id):
- title = NotAvailable
- preparation_time = NotAvailable
- cooking_time = NotAvailable
- nb_person = NotAvailable
- ingredients = NotAvailable
- picture_url = NotAvailable
- instructions = NotAvailable
- comments = NotAvailable
+ obj_id = Env('id')
+ obj_title = CleanText('//h1[@class="m_title"]')
+ obj_preparation_time = Type(CleanText('//span[@class="preptime"]'), type=int)
+ obj_cooking_time = Type(CleanText('//span[@class="cooktime"]'), type=int)
- title = unicode(self.parser.select(self.document.getroot(), 'h1.m_title', 1).text_content().strip())
- main = self.parser.select(self.document.getroot(), 'div.m_content_recette_main', 1)
- preparation_time = int(self.parser.select(main, 'p.m_content_recette_info span.preptime', 1).text_content())
- cooking_time = int(self.parser.select(main, 'p.m_content_recette_info span.cooktime', 1).text_content())
- ing_header_line = self.parser.select(main, 'p.m_content_recette_ingredients span', 1).text_content()
- if '(pour' in ing_header_line and ')' in ing_header_line:
- nb_person = [int(ing_header_line.split('pour ')[-1].split('personnes)')[0].split()[0])]
- ingredients = self.parser.select(main, 'p.m_content_recette_ingredients', 1).text_content().strip().split('- ')
- ingredients = ingredients[1:]
- rinstructions = self.parser.select(main, 'div.m_content_recette_todo', 1).text_content().strip()
- instructions = u''
- for line in rinstructions.split('\n'):
- instructions += '%s\n' % line.strip()
- instructions = instructions.strip('\n')
- imgillu = self.parser.select(self.document.getroot(), 'a.m_content_recette_illu img')
- if len(imgillu) > 0:
- picture_url = unicode(imgillu[0].attrib.get('src', ''))
+ def obj_nb_person(self):
+ nb_pers = Regexp(CleanText('//p[@class="m_content_recette_ingredients"]/span[1]'),
+ '.*\(pour (\d+) personnes\)', default=0)(self)
+ return [nb_pers] if nb_pers else NotAvailable
- divcoms = self.parser.select(self.document.getroot(), 'div.m_commentaire_row')
- if len(divcoms) > 0:
- comments = []
- for divcom in divcoms:
- note = self.parser.select(divcom, 'div.m_commentaire_note span', 1).text.strip()
- user = self.parser.select(divcom, 'div.m_commentaire_content span', 1).text.strip()
- content = self.parser.select(divcom, 'div.m_commentaire_content p', 1).text.strip()
- comments.append(Comment(author=user, rate=note, text=content))
+ def obj_ingredients(self):
+ ingredients = CleanText('//p[@class="m_content_recette_ingredients"]', default='')(self).split('-')
+ if len(ingredients) > 1:
+ return ingredients[1:]
- recipe = Recipe(id, title)
- recipe.preparation_time = preparation_time
- recipe.cooking_time = cooking_time
- recipe.nb_person = nb_person
- recipe.ingredients = ingredients
- recipe.instructions = instructions
- recipe.picture_url = picture_url
- recipe.comments = comments
- recipe.thumbnail_url = NotLoaded
- recipe.author = NotAvailable
- return recipe
+ obj_instructions = CleanHTML('//div[@class="m_content_recette_todo"]')
+ obj_picture_url = CleanText('//a[@class="m_content_recette_illu"]/@href', default=NotAvailable)
+
+ @method
+ class get_comments(ListElement):
+ item_xpath = '//div[@class="m_commentaire_row"]'
+ ignore_duplicate = True
+
+ class item(ItemElement):
+ klass = Comment
+
+ obj_author = CleanText('./div[@class="m_commentaire_content"]/span[1]')
+ obj_rate = CleanText('./div[@class="m_commentaire_note"]/span')
+ obj_text = CleanText('./div[@class="m_commentaire_content"]/p[1]')
+ obj_id = CleanText('./div[@class="m_commentaire_content"]/span[1]')
diff --git a/modules/marmiton/test.py b/modules/marmiton/test.py
index 401fa5d6..c7964b14 100644
--- a/modules/marmiton/test.py
+++ b/modules/marmiton/test.py
@@ -18,13 +18,13 @@
# along with weboob. If not, see .
from weboob.tools.test import BackendTest
-
+import itertools
class MarmitonTest(BackendTest):
MODULE = 'marmiton'
def test_recipe(self):
- recipes = self.backend.iter_recipes('fondue')
+ recipes = list(itertools.islice(self.backend.iter_recipes('fondue'), 0, 20))
for recipe in recipes:
full_recipe = self.backend.get_recipe(recipe.id)
assert full_recipe.instructions
diff --git a/weboob/capabilities/recipe.py b/weboob/capabilities/recipe.py
index e55f5fda..803781a1 100644
--- a/weboob/capabilities/recipe.py
+++ b/weboob/capabilities/recipe.py
@@ -63,7 +63,7 @@ class Recipe(BaseObject):
instructions = StringField('Instruction step list of the recipe')
comments = Field('User comments about the recipe', list)
- def __init__(self, id, title):
+ def __init__(self, id='', title=u''):
BaseObject.__init__(self, id)
self.title = title