factorise

weboob/applications/boobmsg/boobmsg.py

@@ -298,9 +298,9 @@ class Boobmsg(ReplApplication):
         _id, backend_name = self.parse_id(arg)
         cmd = self.do('get_thread', _id, backends=backend_name)
         for backend, thread in cmd:
-            for msg in thread.iter_all_messages():
+            if thread is not None :
-                self.format(msg)
+                for msg in thread.iter_all_messages():
-
+                    self.format(msg)
 
     def do_show(self, arg):
         """

weboob/backends/inrocks/pages/article.py

@@ -17,25 +17,25 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 from weboob.tools.parsers.lxmlparser import select
-from weboob.tools.genericArticle import GenericNewsPage, try_remove
+from weboob.tools.genericArticle import GenericNewsPage, try_remove, \
+                                        try_remove_from_selector_list
 
 class ArticlePage(GenericNewsPage):
     "ArticlePage object for inrocks"
     def on_loaded(self):
         self.main_div = self.document.getroot()
+        self.element_title_selector = "h1"
         self.element_author_selector    = "div.name>span"
-        self.element_title_selector     = "h1"
         self.element_body_selector      = "div.maincol"
 
     def get_body(self):
         element_body = self.get_element_body()
         try_remove(element_body, "div.sidebar")
-        details = select(element_body, "div.details", 1)
+        try_remove(select(element_body, "div.details", 1), "div.footer")
-        try_remove(details, "div.footer")
+        try_remove_from_selector_list(select(element_body, "div.header", 1), 
-        header = select(element_body, "div.header", 1)
+                                      ["h1", "div.picture", "div.date", 
-        for selector in ["h1", "div.picture", "div.date", "div.news-single-img",
+                                       "div.news-single-img", 
-                         "div.metas_img", "strong"]:
+                                       "div.metas_img", "strong"])
-            try_remove(header, selector)
         return self.browser.parser.tostring(element_body)
 
 

weboob/backends/lefigaro/browser.py

@@ -49,7 +49,6 @@ class NewspaperFigaroBrowser(BaseBrowser):
              "http://www.lefigaro.fr/sortir-paris/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage,
              "http://www.lefigaro.fr/vie-entreprise/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage,
              "http://www.lefigaro.fr/autres-sports/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage,
-             "http://www.lefigaro.fr/immobilier/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage,
              "http://www.lefigaro.fr/environnement/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage,
              "http://www.lefigaro.fr/rugby/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage,
              "http://www.lefigaro.fr/societes/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage,
@@ -59,7 +58,6 @@ class NewspaperFigaroBrowser(BaseBrowser):
              "http://www.lefigaro.fr/theatre/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage,
              "http://www.lefigaro.fr/football-coupes-d-europe/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage,
              "http://www.lefigaro.fr/sante/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage,
-             "http://www.lefigaro.fr/sciences/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage,
              "http://www.lefigaro.fr/assurance/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage,
              "http://www.lefigaro.fr/retraite/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage,
              "http://www.lefigaro.fr/tennis/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage,

weboob/backends/lefigaro/pages/article.py

@@ -16,40 +16,25 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
-from weboob.tools.parsers.lxmlparser import select, SelectElementException
+from weboob.tools.genericArticle import GenericNewsPage, remove_from_selector_list, drop_comments, try_drop_tree, try_remove_from_selector_list
-from weboob.tools.genericArticle import GenericNewsPage, try_remove
 
 class ArticlePage(GenericNewsPage):
     "ArticlePage object for inrocks"
     def on_loaded(self):
         self.main_div = self.document.getroot()
+        self.element_title_selector = "h1"
         self.element_author_selector    = "div.name>span"
-        self.element_title_selector     = "h1"
         self.element_body_selector      = "#article"
 
     def get_body(self):
-        element_body    = self.get_element_body()
+        element_body = self.get_element_body()
-        h1_title        = select(element_body, self.element_title_selector, 1)
+        remove_from_selector_list(element_body, [self.element_title_selector, "link"])
+        drop_comments(element_body)
+        try_drop_tree(element_body, "script")
 
-        try:
+        try_remove_from_selector_list(element_body, ["div.infos", "div.photo", "div.art_bandeau_bottom", "div.view", "span.auteur_long", "#toolsbar"])
-            el_script       = select(element_body, "script", 1)
-        except SelectElementException:
-            pass
-        else:
-            el_script.drop_tree()
-
-
-        element_body.remove(h1_title)
-
-        try_remove(element_body, "div.infos")
-        try_remove(element_body, "div.photo")
-        try_remove(element_body, "div.art_bandeau_bottom")
-        try_remove(element_body, "div.view")
-        try_remove(element_body, "span.auteur_long")
-        try_remove(element_body, "#toolsbar")
 
         element_body.find_class("texte")[0].drop_tag()
         element_body.tag = "div"
         return self.browser.parser.tostring(element_body)
 
-

weboob/backends/lefigaro/pages/flashactu.py

@@ -22,9 +22,9 @@ class FlashActuPage(GenericNewsPage):
     "ArticlePage object for inrocks"
     def on_loaded(self):
         self.main_div = self.document.getroot()
+        self.element_title_selector = "h1"
         self.element_author_selector    = "div.name>span"
         self.element_body_selector      = "h2"
-        self.element_title_selector     = "h1"
 
     def get_body(self):
         element_body = self.get_element_body()

weboob/backends/minutes20/browser.py

@@ -45,11 +45,7 @@ class Newspaper20minutesBrowser(BaseBrowser):
             url = id2url(_id)
         except ValueError:
             url = _id
-        try:
+        self.location(url)
-            self.location(url)
+
-        except IndexError:
+        if self.page is not None:
-            if _id == '':
+            return self.page.get_article(_id)
-                raise ValueError("thread id is empty")
-            else:
-                raise
-        return self.page.get_article(_id)

weboob/backends/minutes20/pages/article.py

@@ -21,6 +21,11 @@ from .simple import SimplePage
 
 class ArticlePage(SimplePage):
     "ArticlePage object for minutes20"
+    def on_loaded(self):
+        self.main_div = self.document.getroot()
+        self.element_title_selector = "h1"
+        self.element_author_selector = "div.mna-signature"
+        self.element_body_selector = "div.mna-body"
 
     def get_body(self):
         element_body = self.get_element_body()
@@ -32,9 +37,3 @@ class ArticlePage(SimplePage):
             pass
         return self.browser.parser.tostring(element_body)
 
-    def on_loaded(self):
-        self.main_div = self.document.getroot()
-        self.element_author_selector = "div.mna-signature"
-        self.element_body_selector = "div.mna-body"
-        self.element_title_selector = "h1"
-    

weboob/backends/minutes20/pages/simple.py

@@ -18,13 +18,11 @@
 
 from weboob.tools.genericArticle import GenericNewsPage
 
-
 class SimplePage(GenericNewsPage):
     "ArticlePage object for minutes20"
     def on_loaded(self):
         self.main_div = self.document.getroot()
+        self.element_title_selector = "h1"
         self.element_author_selector = "div.mna-signature"
         self.element_body_selector = "div.mna-body"
-        self.element_title_selector = "h1"
 
-    

weboob/tools/genericArticle.py

@@ -16,6 +16,8 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 from weboob.tools.browser import BasePage
 from weboob.tools.parsers.lxmlparser import select, SelectElementException
+from lxml.etree import Comment
+
 
 def try_remove(base_element, selector):
     try :
@@ -24,9 +26,33 @@ def try_remove(base_element, selector):
         pass
 
 
+def try_drop_tree(base_element, selector):
+    try:
+        select(base_element, selector, 1).drop_tree()
+    except SelectElementException:
+        pass
+
+def remove_from_selector_list(base_element, selector_list):
+    for selector in selector_list:
+        base_element.remove(select(base_element, selector, 1))
+
+
+def try_remove_from_selector_list(base_element, selector_list):
+    for selector in selector_list:
+        try_remove(base_element, selector)
+
+def drop_comments(base_element):
+    for comment in base_element.getiterator(Comment):
+        comment.drop_tree()
+
+
+
 class NoAuthorElement(SelectElementException):
     pass
 
+class NoBodyElement(SelectElementException):
+    pass
+
 class NoTitleException(SelectElementException):
     pass
 
@@ -71,7 +97,10 @@ class GenericNewsPage(BasePage):
             raise NoTitleException("no title on %s" % (self.browser)) 
 
     def get_element_body(self):
-        return select(self.main_div, self.element_body_selector, 1)
+        try :
+            return select(self.main_div, self.element_body_selector, 1)
+        except SelectElementException:
+            raise NoBodyElement("no body on %s" % (self.browser)) 
 
     def get_element_author(self):
         try: