From 397b5875f1594c94c619a47986bf98bfc3f03d16 Mon Sep 17 00:00:00 2001 From: Juke Date: Mon, 7 Mar 2011 09:44:02 +0100 Subject: [PATCH] factorise --- weboob/applications/boobmsg/boobmsg.py | 6 ++-- weboob/backends/inrocks/pages/article.py | 16 +++++------ weboob/backends/lefigaro/browser.py | 2 -- weboob/backends/lefigaro/pages/article.py | 29 +++++-------------- weboob/backends/lefigaro/pages/flashactu.py | 2 +- weboob/backends/minutes20/browser.py | 12 +++----- weboob/backends/minutes20/pages/article.py | 11 ++++---- weboob/backends/minutes20/pages/simple.py | 4 +-- weboob/tools/genericArticle.py | 31 ++++++++++++++++++++- 9 files changed, 59 insertions(+), 54 deletions(-) diff --git a/weboob/applications/boobmsg/boobmsg.py b/weboob/applications/boobmsg/boobmsg.py index f547a929..51ef3c10 100644 --- a/weboob/applications/boobmsg/boobmsg.py +++ b/weboob/applications/boobmsg/boobmsg.py @@ -298,9 +298,9 @@ class Boobmsg(ReplApplication): _id, backend_name = self.parse_id(arg) cmd = self.do('get_thread', _id, backends=backend_name) for backend, thread in cmd: - for msg in thread.iter_all_messages(): - self.format(msg) - + if thread is not None : + for msg in thread.iter_all_messages(): + self.format(msg) def do_show(self, arg): """ diff --git a/weboob/backends/inrocks/pages/article.py b/weboob/backends/inrocks/pages/article.py index c37e9300..9e1ff518 100644 --- a/weboob/backends/inrocks/pages/article.py +++ b/weboob/backends/inrocks/pages/article.py @@ -17,25 +17,25 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. from weboob.tools.parsers.lxmlparser import select -from weboob.tools.genericArticle import GenericNewsPage, try_remove +from weboob.tools.genericArticle import GenericNewsPage, try_remove, \ + try_remove_from_selector_list class ArticlePage(GenericNewsPage): "ArticlePage object for inrocks" def on_loaded(self): self.main_div = self.document.getroot() + self.element_title_selector = "h1" self.element_author_selector = "div.name>span" - self.element_title_selector = "h1" self.element_body_selector = "div.maincol" def get_body(self): element_body = self.get_element_body() try_remove(element_body, "div.sidebar") - details = select(element_body, "div.details", 1) - try_remove(details, "div.footer") - header = select(element_body, "div.header", 1) - for selector in ["h1", "div.picture", "div.date", "div.news-single-img", - "div.metas_img", "strong"]: - try_remove(header, selector) + try_remove(select(element_body, "div.details", 1), "div.footer") + try_remove_from_selector_list(select(element_body, "div.header", 1), + ["h1", "div.picture", "div.date", + "div.news-single-img", + "div.metas_img", "strong"]) return self.browser.parser.tostring(element_body) diff --git a/weboob/backends/lefigaro/browser.py b/weboob/backends/lefigaro/browser.py index 0e7b5284..8d8906a2 100644 --- a/weboob/backends/lefigaro/browser.py +++ b/weboob/backends/lefigaro/browser.py @@ -49,7 +49,6 @@ class NewspaperFigaroBrowser(BaseBrowser): "http://www.lefigaro.fr/sortir-paris/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage, "http://www.lefigaro.fr/vie-entreprise/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage, "http://www.lefigaro.fr/autres-sports/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage, - "http://www.lefigaro.fr/immobilier/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage, "http://www.lefigaro.fr/environnement/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage, "http://www.lefigaro.fr/rugby/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage, "http://www.lefigaro.fr/societes/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage, @@ -59,7 +58,6 @@ class NewspaperFigaroBrowser(BaseBrowser): "http://www.lefigaro.fr/theatre/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage, "http://www.lefigaro.fr/football-coupes-d-europe/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage, "http://www.lefigaro.fr/sante/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage, - "http://www.lefigaro.fr/sciences/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage, "http://www.lefigaro.fr/assurance/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage, "http://www.lefigaro.fr/retraite/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage, "http://www.lefigaro.fr/tennis/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage, diff --git a/weboob/backends/lefigaro/pages/article.py b/weboob/backends/lefigaro/pages/article.py index a47fb6d7..a0754b00 100644 --- a/weboob/backends/lefigaro/pages/article.py +++ b/weboob/backends/lefigaro/pages/article.py @@ -16,40 +16,25 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -from weboob.tools.parsers.lxmlparser import select, SelectElementException -from weboob.tools.genericArticle import GenericNewsPage, try_remove +from weboob.tools.genericArticle import GenericNewsPage, remove_from_selector_list, drop_comments, try_drop_tree, try_remove_from_selector_list class ArticlePage(GenericNewsPage): "ArticlePage object for inrocks" def on_loaded(self): self.main_div = self.document.getroot() + self.element_title_selector = "h1" self.element_author_selector = "div.name>span" - self.element_title_selector = "h1" self.element_body_selector = "#article" def get_body(self): - element_body = self.get_element_body() - h1_title = select(element_body, self.element_title_selector, 1) + element_body = self.get_element_body() + remove_from_selector_list(element_body, [self.element_title_selector, "link"]) + drop_comments(element_body) + try_drop_tree(element_body, "script") - try: - el_script = select(element_body, "script", 1) - except SelectElementException: - pass - else: - el_script.drop_tree() - - - element_body.remove(h1_title) - - try_remove(element_body, "div.infos") - try_remove(element_body, "div.photo") - try_remove(element_body, "div.art_bandeau_bottom") - try_remove(element_body, "div.view") - try_remove(element_body, "span.auteur_long") - try_remove(element_body, "#toolsbar") + try_remove_from_selector_list(element_body, ["div.infos", "div.photo", "div.art_bandeau_bottom", "div.view", "span.auteur_long", "#toolsbar"]) element_body.find_class("texte")[0].drop_tag() element_body.tag = "div" return self.browser.parser.tostring(element_body) - diff --git a/weboob/backends/lefigaro/pages/flashactu.py b/weboob/backends/lefigaro/pages/flashactu.py index 705f0a68..f218b567 100644 --- a/weboob/backends/lefigaro/pages/flashactu.py +++ b/weboob/backends/lefigaro/pages/flashactu.py @@ -22,9 +22,9 @@ class FlashActuPage(GenericNewsPage): "ArticlePage object for inrocks" def on_loaded(self): self.main_div = self.document.getroot() + self.element_title_selector = "h1" self.element_author_selector = "div.name>span" self.element_body_selector = "h2" - self.element_title_selector = "h1" def get_body(self): element_body = self.get_element_body() diff --git a/weboob/backends/minutes20/browser.py b/weboob/backends/minutes20/browser.py index 76027a06..c986a5c0 100644 --- a/weboob/backends/minutes20/browser.py +++ b/weboob/backends/minutes20/browser.py @@ -45,11 +45,7 @@ class Newspaper20minutesBrowser(BaseBrowser): url = id2url(_id) except ValueError: url = _id - try: - self.location(url) - except IndexError: - if _id == '': - raise ValueError("thread id is empty") - else: - raise - return self.page.get_article(_id) + self.location(url) + + if self.page is not None: + return self.page.get_article(_id) diff --git a/weboob/backends/minutes20/pages/article.py b/weboob/backends/minutes20/pages/article.py index 9c6c8409..9da96c7f 100644 --- a/weboob/backends/minutes20/pages/article.py +++ b/weboob/backends/minutes20/pages/article.py @@ -21,6 +21,11 @@ from .simple import SimplePage class ArticlePage(SimplePage): "ArticlePage object for minutes20" + def on_loaded(self): + self.main_div = self.document.getroot() + self.element_title_selector = "h1" + self.element_author_selector = "div.mna-signature" + self.element_body_selector = "div.mna-body" def get_body(self): element_body = self.get_element_body() @@ -32,9 +37,3 @@ class ArticlePage(SimplePage): pass return self.browser.parser.tostring(element_body) - def on_loaded(self): - self.main_div = self.document.getroot() - self.element_author_selector = "div.mna-signature" - self.element_body_selector = "div.mna-body" - self.element_title_selector = "h1" - diff --git a/weboob/backends/minutes20/pages/simple.py b/weboob/backends/minutes20/pages/simple.py index 43b58cb3..e5015b49 100644 --- a/weboob/backends/minutes20/pages/simple.py +++ b/weboob/backends/minutes20/pages/simple.py @@ -18,13 +18,11 @@ from weboob.tools.genericArticle import GenericNewsPage - class SimplePage(GenericNewsPage): "ArticlePage object for minutes20" def on_loaded(self): self.main_div = self.document.getroot() + self.element_title_selector = "h1" self.element_author_selector = "div.mna-signature" self.element_body_selector = "div.mna-body" - self.element_title_selector = "h1" - diff --git a/weboob/tools/genericArticle.py b/weboob/tools/genericArticle.py index ae4ead8b..929d3ffb 100644 --- a/weboob/tools/genericArticle.py +++ b/weboob/tools/genericArticle.py @@ -16,6 +16,8 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. from weboob.tools.browser import BasePage from weboob.tools.parsers.lxmlparser import select, SelectElementException +from lxml.etree import Comment + def try_remove(base_element, selector): try : @@ -24,9 +26,33 @@ def try_remove(base_element, selector): pass +def try_drop_tree(base_element, selector): + try: + select(base_element, selector, 1).drop_tree() + except SelectElementException: + pass + +def remove_from_selector_list(base_element, selector_list): + for selector in selector_list: + base_element.remove(select(base_element, selector, 1)) + + +def try_remove_from_selector_list(base_element, selector_list): + for selector in selector_list: + try_remove(base_element, selector) + +def drop_comments(base_element): + for comment in base_element.getiterator(Comment): + comment.drop_tree() + + + class NoAuthorElement(SelectElementException): pass +class NoBodyElement(SelectElementException): + pass + class NoTitleException(SelectElementException): pass @@ -71,7 +97,10 @@ class GenericNewsPage(BasePage): raise NoTitleException("no title on %s" % (self.browser)) def get_element_body(self): - return select(self.main_div, self.element_body_selector, 1) + try : + return select(self.main_div, self.element_body_selector, 1) + except SelectElementException: + raise NoBodyElement("no body on %s" % (self.browser)) def get_element_author(self): try: