correctly implement iter_messages() to return every messages

weboob/backends/dlfp/backend.py

@@ -39,26 +39,32 @@ class DLFPBackend(Backend, ICapMessages, ICapMessagesReply, ICapUpdatable):
               'get_telegrams': Backend.ConfigField(default=False, description='Get telegrams'),
              }
     STORAGE = {'seen': {}}
-    browser = None
+    _browser = None
 
-    def need_browser(func):
+    def __getattr__(self, name):
-        def inner(self, *args, **kwargs):
+        if name == 'browser':
-            if not self.browser:
+            if not self._browser:
-                self.browser = DLFP(self.config['username'], self.config['password'])
+                self._browser = DLFP(self.config['username'], self.config['password'])
+            return self._browser
+        raise AttributeError, name
 
-            return func(self, *args, **kwargs)
-        return inner
-
-    @need_browser
     def iter_messages(self):
+        for message in self._iter_messages(False):
+            yield message
+
+    def iter_new_messages(self):
+        for message in self._iter_messages(True):
+            yield message
+
+    def _iter_messages(self, only_new):
         if self.config['get_news']:
-            for message in self._iter_messages('newspaper'):
+            for message in self._iter_messages_of('newspaper', only_new):
                 yield message
         if self.config['get_telegrams']:
-            for message in self._iter_messages('telegram'):
+            for message in self._iter_messages_of('telegram', only_new):
                 yield message
 
-    def _iter_messages(self, what):
+    def _iter_messages_of(self, what, only_new):
         if not what in self.storage.get(self.name, 'seen'):
             self.storage.set(self.name, 'seen', what, {})
 
@@ -68,6 +74,11 @@ class DLFPBackend(Backend, ICapMessages, ICapMessagesReply, ICapUpdatable):
 
             if not article.id in self.storage.get(self.name, 'seen', what):
                 seen[article.id] = {'comments': []}
+                new = True
+            else:
+                seen[article.id] = self.storage.get(self.name, 'seen', what, article.id)
+                new = False
+            if not only_new or new:
                 yield Message(thread.id,
                               0,
                               thread.title,
@@ -75,12 +86,14 @@ class DLFPBackend(Backend, ICapMessages, ICapMessagesReply, ICapUpdatable):
                               article.datetime,
                               content=''.join([thread.body, thread.part2]),
                               signature='URL: %s' % article.url)
-            else:
-                seen[article.id] = self.storage.get(self.name, 'seen', what, article.id)
 
             for comment in thread.iter_all_comments():
                 if not comment.id in seen[article.id]['comments']:
                     seen[article.id]['comments'].append(comment.id)
+                    new = True
+                else:
+                    new = False
+                if not only_new or new:
                     yield Message(thread.id,
                                   comment.id,
                                   comment.title,
@@ -91,6 +104,3 @@ class DLFPBackend(Backend, ICapMessages, ICapMessagesReply, ICapUpdatable):
                                   'Score: %d' % comment.score)
         self.storage.set(self.name, 'seen', what, seen)
         self.storage.save(self.name)
-
-    def iter_new_messages(self):
-        return self.iter_messages()

weboob/tools/parser.py

@@ -62,7 +62,10 @@ except ImportError:
             self._target.data(data)
 
         def handle_endtag(self, tag):
-            self._target.end(tag)
+            try:
+                self._target.end(tag)
+            except:
+                pass
 
 class StandardParser(object):
     def parse(self, data, encoding=None):