nojhan/weboob-devel
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

fix empty article

Browse source
This commit is contained in:
Juke 2011-03-10 10:37:13 +01:00 • committed by Romain Bignon
commit 4951603c73
2 changed files with 52 additions and 36 deletions
Download patch file Download diff file Expand all files Collapse all files

70
weboob/backends/inrocks/pages/article.py
View file

@ -19,7 +19,7 @@
from weboob.tools.parsers.lxmlparser import select, SelectElementException from weboob.tools.parsers.lxmlparser import select, SelectElementException
from weboob.tools.genericArticle import GenericNewsPage, try_remove, \ from weboob.tools.genericArticle import GenericNewsPage, try_remove, \
try_remove_from_selector_list, \ try_remove_from_selector_list, \
drop_comments drop_comments, NoneMainDiv
class ArticlePage(GenericNewsPage): class ArticlePage(GenericNewsPage):
"ArticlePage object for inrocks" "ArticlePage object for inrocks"
@ -30,45 +30,49 @@ class ArticlePage(GenericNewsPage):
self.element_body_selector = "div.maincol" self.element_body_selector = "div.maincol"
def get_body(self): def get_body(self):
element_body = self.get_element_body()
div_header_element = select(element_body, "div.header", 1)
element_detail = select(element_body, "div.details", 1)
div_content_element = select(element_body, "div.content", 1)
drop_comments(element_body)
try_remove(element_body, "div.sidebar")
try_remove(element_detail, "div.footer")
try_remove_from_selector_list(div_header_element,
["h1", "div.picture", "div.date",
"div.news-single-img",
"div.metas_img", "strong"])
try_remove_from_selector_list(div_content_element,
["div.tw_button", "div.wpfblike"])
try : try :
description_element = select(div_header_element, element_body = self.get_element_body()
"div.description", 1) except NoneMainDiv:
except SelectElementException: return None
pass
else: else:
text_content = description_element.text_content() div_header_element = select(element_body, "div.header", 1)
if len(text_content.strip()) == 0 : element_detail = select(element_body, "div.details", 1)
description_element.drop_tree() div_content_element = select(element_body, "div.content", 1)
drop_comments(element_body)
try_remove(element_body, "div.sidebar")
try_remove(element_detail, "div.footer")
try_remove_from_selector_list(div_header_element,
["h1", "div.picture", "div.date",
"div.news-single-img",
"div.metas_img", "strong"])
try_remove_from_selector_list(div_content_element,
["div.tw_button", "div.wpfblike"])
try :
description_element = select(div_header_element,
"div.description", 1)
except SelectElementException:
pass
else: else:
if len(description_element) == 1: text_content = description_element.text_content()
description_element.drop_tag() if len(text_content.strip()) == 0 :
description_element.drop_tree()
else:
if len(description_element) == 1:
description_element.drop_tag()
if len(div_header_element.text_content().strip()) == 0: if len(div_header_element.text_content().strip()) == 0:
div_header_element.drop_tree() div_header_element.drop_tree()
if len(div_header_element) == 1: if len(div_header_element) == 1:
div_header_element.drop_tag() div_header_element.drop_tag()
if len(element_detail) == 1: if len(element_detail) == 1:
element_detail.drop_tag() element_detail.drop_tag()
div_content_element.drop_tag() div_content_element.drop_tag()
return self.browser.parser.tostring(element_body) return self.browser.parser.tostring(element_body)

18
weboob/tools/genericArticle.py
View file

@ -61,11 +61,11 @@ class NoneMainDiv(AttributeError):
class Article(object): class Article(object):
author = u'' author = u''
title = u''
def __init__(self, browser, _id): def __init__(self, browser, _id):
self.browser = browser self.browser = browser
self.id = _id self.id = _id
self.title = u''
self.body = u'' self.body = u''
self.url = u'' self.url = u''
self.date = None self.date = None
@ -84,7 +84,8 @@ class GenericNewsPage(BasePage):
def get_author(self): def get_author(self):
try: try:
return self.get_element_author().text_content().strip() return self.get_element_author().text_content().strip()
except NoAuthorElement: except (NoAuthorElement, NoneMainDiv):
#TODO: Mettre un warning
return self.__article.author return self.__article.author
def get_title(self): def get_title(self):
@ -93,6 +94,12 @@ class GenericNewsPage(BasePage):
self.main_div, self.main_div,
self.element_title_selector, self.element_title_selector,
1).text_content().strip() 1).text_content().strip()
except AttributeError:
if self.main_div == None:
#TODO: Mettre un warning
return self.__article.title
else:
raise
except SelectElementException: except SelectElementException:
raise NoTitleException("no title on %s" % (self.browser)) raise NoTitleException("no title on %s" % (self.browser))
@ -100,7 +107,12 @@ class GenericNewsPage(BasePage):
try : try :
return select(self.main_div, self.element_body_selector, 1) return select(self.main_div, self.element_body_selector, 1)
except SelectElementException: except SelectElementException:
raise NoBodyElement("no body on %s" % (self.browser)) raise NoBodyElement("no body on %s" % (self.browser))
except AttributeError:
if self.main_div == None:
raise NoneMainDiv("main_div is none on %s" % (self.browser))
else:
raise
def get_element_author(self): def get_element_author(self):
try: try: