Better cleanup of generic articles and lefigaro

try_drop_tree() was only removing the first result.
closes #725

modules/lefigaro/pages/article.py

@@ -33,10 +33,36 @@ class ArticlePage(GenericNewsPage):
         remove_from_selector_list(self.parser, element_body, [self.element_title_selector])
         drop_comments(element_body)
         try_drop_tree(self.parser, element_body, "script")
+        try_drop_tree(self.parser, element_body, "liste")
 
-        try_remove_from_selector_list(self.parser, element_body, ["div.infos", "div.photo", "div.art_bandeau_bottom", "div.view", "span.auteur_long", "#toolsbar", 'link'])
+        try_remove_from_selector_list(self.parser, element_body, ["div#article-comments", "div.infos", "div.photo", "div.art_bandeau_bottom", "div.view", "span.auteur_long", "#toolsbar", 'link'])
+
+        for image in self.parser.select(element_body, 'img'):
+            if image.attrib['src'].endswith('coeur-.gif'):
+                image.drop_tree()
+
+        for div in self.parser.select(element_body, 'div'):
+            if div.text == ' Player Figaro BFM ':
+                obj = div.getnext()
+                a = obj.getnext()
+                if obj.tag == 'object':
+                    obj.drop_tree()
+                if a.tag == 'a' and 'BFM' in a.text:
+                    a.drop_tree()
+                div.drop_tree()
+
+
+        # This part of the article seems manually generated.
+        for crappy_title in self.parser.select(element_body, 'p strong'):
+            if crappy_title.text == 'LIRE AUSSI :' or crappy_title.text == 'LIRE AUSSI:':
+                # Remove if it has only links
+                for related in crappy_title.getparent().itersiblings(tag='p'):
+                    if len(related) == len(list(related.iterchildren(tag='a'))):
+                        related.drop_tree()
+                    else:
+                        break
+                crappy_title.drop_tree()
 
         element_body.find_class("texte")[0].drop_tag()
         element_body.tag = "div"
         return self.parser.tostring(element_body)
-

modules/lefigaro/test.py

@@ -19,6 +19,7 @@
 
 
 from weboob.tools.test import BackendTest
+from weboob.tools.misc import html2text
 
 
 __all__ = ['LeFigaroTest']
@@ -30,3 +31,19 @@ class LeFigaroTest(BackendTest):
     def test_new_messages(self):
         for message in self.backend.iter_unread_messages():
             pass
+
+    def test_content(self):
+        urls = ['http://www.lefigaro.fr/international/2011/10/24/01003-20111024ARTFIG00704-les-islamo-conservateurs-maitres-du-jeu-tunisien.php',
+                'http://www.lefigaro.fr/international/2012/01/29/01003-20120129ARTFIG00191-floride-la-primaire-suspendue-a-l-humeur-des-hispaniques.php']
+
+        for url in urls:
+            thread = self.backend.get_thread(url)
+            assert len(thread.root.content)
+            assert '<script' not in thread.root.content
+            assert 'object' not in thread.root.content
+            assert 'BFM' not in thread.root.content
+
+            assert 'AUSSI' not in thread.root.content
+
+            # no funny tags means html2text does not crash
+            assert len(html2text(thread.root.content))

weboob/tools/capabilities/messages/genericArticle.py

@@ -30,10 +30,8 @@ def try_remove(parser, base_element, selector):
 
 
 def try_drop_tree(parser, base_element, selector):
-    try:
+    for el in parser.select(base_element, selector):
-        parser.select(base_element, selector, 1).drop_tree()
+        el.drop_tree()
-    except BrokenPageError:
-        pass
 
 def remove_from_selector_list(parser, base_element, selector_list):
     for selector in selector_list: