adapt to new ICapMessages API

weboob/applications/monboob/monboob.py

@@ -30,7 +30,7 @@ import asyncore
 
 from weboob.core.ouiboube import Weboob
 from weboob.core.scheduler import Scheduler
-from weboob.capabilities.messages import ICapMessages, ICapMessagesReply, Message
+from weboob.capabilities.messages import ICapMessages, ICapMessagesPost, Thread, Message
 from weboob.tools.application.console import ConsoleApplication
 from weboob.tools.misc import html2text, get_backtrace, utc2local
 
@@ -158,23 +158,31 @@ class Monboob(ConsoleApplication):
             print >>sys.stderr, 'Backend %s not found' % bname
             return 1
 
-        if not backend.has_caps(ICapMessagesReply):
+        if not backend.has_caps(ICapMessagesPost):
-            print >>sys.stderr, 'The backend %s does not implement ICapMessagesReply' % bname
+            print >>sys.stderr, 'The backend %s does not implement ICapMessagesPost' % bname
             return 1
 
         thread_id, msg_id = id.rsplit('.', 1)
+        thread = Thread(thread_id)
+        message = Message(thread,
+                          0,
+                          title=title,
+                          sender=None,
+                          receiver=None,
+                          parent=Message(thread, msg_id),
+                          content=content)
         try:
-            backend.post_reply(thread_id, msg_id, title, content)
+            backend.post_message(message)
         except Exception, e:
             content = u'Unable to send message to %s:\n' % thread_id
             content += '\n\t%s\n' % e
             if logging.root.level == logging.DEBUG:
                 content += '\n%s\n' % get_backtrace(e)
-            self.send_email(backend, Message(thread_id,
+            self.send_email(backend, Message(thread,
                                              0,
                                              title='Unable to send message',
                                              sender='Monboob',
-                                             parent_message_id=msg_id,
+                                             parent=Message(thread, msg_id),
                                              content=content))
 
     @ConsoleApplication.command("run daemon")
@@ -183,22 +191,23 @@ class Monboob(ConsoleApplication):
         self.weboob.loop()
 
     def process(self):
-        for backend, message in self.weboob.do('iter_new_messages'):
+        for backend, message in self.weboob.do('iter_unread_messages'):
             self.send_email(backend, message)
+            backend.set_message_read(message)
 
     def send_email(self, backend, mail):
         domain = self.config.get('domain')
         recipient = self.config.get('recipient')
 
         reply_id = ''
-        if mail.parent_id:
+        if mail.parent:
-            reply_id = u'<%s.%s@%s>' % (backend.name, mail.parent_id, domain)
+            reply_id = u'<%s.%s@%s>' % (backend.name, mail.parent.full_id, domain)
         subject = mail.title
         sender = u'"%s" <%s@%s>' % (mail.sender.replace('"', '""'), backend.name, domain)
 
         # assume that .date is an UTC datetime
         date = formatdate(time.mktime(utc2local(mail.date).timetuple()), localtime=True)
-        msg_id = u'<%s.%s@%s>' % (backend.name, mail.id, domain)
+        msg_id = u'<%s.%s@%s>' % (backend.name, mail.full_id, domain)
 
         if int(self.config.get('html')) and mail.flags & mail.IS_HTML:
             body = mail.content

weboob/backends/aum/backend.py

@@ -19,16 +19,16 @@ from __future__ import with_statement
 
 from datetime import datetime
 from dateutil import tz
-from logging import warning
+from logging import warning, debug
-from time import sleep
 
 from weboob.capabilities.chat import ICapChat
-from weboob.capabilities.messages import ICapMessages, ICapMessagesReply, Message
+from weboob.capabilities.messages import ICapMessages, ICapMessagesPost, Message, Thread
 from weboob.capabilities.dating import ICapDating, StatusField
 from weboob.capabilities.contact import ICapContact, Contact, ProfileNode
 from weboob.tools.backend import BaseBackend
 from weboob.tools.browser import BrowserUnavailable
 
+from .captcha import CaptchaError
 from .browser import AuMBrowser
 from .exceptions import AdopteWait
 from .optim.profiles_walker import ProfilesWalker
@@ -38,7 +38,7 @@ from .optim.visibility import Visibility
 __all__ = ['AuMBackend']
 
 
-class AuMBackend(BaseBackend, ICapMessages, ICapMessagesReply, ICapDating, ICapChat, ICapContact):
+class AuMBackend(BaseBackend, ICapMessages, ICapMessagesPost, ICapDating, ICapChat, ICapContact):
     NAME = 'aum'
     MAINTAINER = 'Romain Bignon'
     EMAIL = 'romain@peerfuse.org'
@@ -54,17 +54,27 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesReply, ICapDating, ICapC
               }
     BROWSER = AuMBrowser
 
+    MAGIC_ID_BASKET = 1
+
     def create_default_browser(self):
         if self.config['register']:
-            browser = self.create_browser(self.config['username'])
+            browser = None
-            browser.register(password=   self.config['password'],
+            while not browser:
-                             sex=        0,
+                try:
-                             birthday_d= 1,
+                    browser = self.create_browser(self.config['username'])
-                             birthday_m= 1,
+                    browser.register(password=   self.config['password'],
-                             birthday_y= 1970,
+                                     sex=        0,
-                             zipcode=    75001,
+                                     birthday_d= 1,
-                             country=    'fr',
+                                     birthday_m= 1,
-                             godfather=  '')
+                                     birthday_y= 1970,
+                                     zipcode=    75001,
+                                     country=    'fr',
+                                     godfather=  '')
+                except CaptchaError:
+                    debug('Unable to resolve captcha. Retrying...')
+                    browser = None
+            browser.password = self.config['password']
+            return browser
         else:
             return self.create_browser(self.config['username'], self.config['password'])
 
@@ -79,16 +89,117 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesReply, ICapDating, ICapC
             except AdopteWait:
                 return (StatusField('notice', '', u'<h3>You are currently waiting 1am to be able to connect with this account</h3>', StatusField.FIELD_HTML|StatusField.FIELD_TEXT))
 
-    def iter_messages(self, thread=None):
+    def iter_threads(self):
-        for message in self._iter_messages(thread, False):
+        with self.browser:
-            yield message
+            contacts = self.browser.get_threads_list()
 
-    def iter_new_messages(self, thread=None):
+        for contact in contacts:
-        for message in self._iter_messages(thread, True):
+            thread = Thread(contact.get_id())
-            yield message
+            yield thread
+
+    def get_thread(self, id):
+        thread = None
+        if isinstance(id, Thread):
+            thread = id
+            id = thread.id
+
+        if not thread:
+            thread = Thread(id)
+            full = False
+        else:
+            full = True
+
+        with self.browser:
+            mails = self.browser.get_thread_mails(id, full)
+            my_name = self.browser.get_my_name()
+
+        child = None
+        msg = None
+        slut = self._get_slut(id)
+        for mail in mails:
+            flags = 0
+            if mail.date > slut['lastmsg']:
+                flags |= Message.IS_UNREAD
+            if mail.sender != my_name:
+                if mail.new:
+                    flags |= Message.IS_NOT_ACCUSED
+                else:
+                    flags |= Message.IS_ACCUSED
+
+            msg = Message(thread=thread,
+                          id=mail.message_id,
+                          title=mail.title,
+                          sender=mail.sender,
+                          receiver=mail.name if mail.sender == my_name else my_name, # TODO: me
+                          date=mail.date,
+                          content=mail.content,
+                          signature=mail.signature,
+                          children=[],
+                          flags=flags)
+            if child:
+                msg.children.append(child)
+                child.parent = msg
+
+            child = msg
+
+        if full:
+            # If we have get all the messages, replace NotLoaded with None as
+            # parent.
+            msg.parent = None
+        thread.root = msg
+
+        return thread
+
+    def iter_unread_messages(self, thread=None):
+        with self.browser:
+            contacts = self.browser.get_threads_list()
+        for contact in contacts:
+            slut = self._get_slut(contact.get_id())
+            if contact.get_lastmsg_date() > slut['lastmsg']:
+                thread = self.get_thread(contact.get_id())
+                for m in thread.iter_all_messages():
+                    if m.flags & m.IS_UNREAD:
+                        yield m
+
+        # Send mail when someone added me in her basket.
+        # XXX possibly race condition if a slut adds me in her basket
+        #     between the aum.nb_new_baskets() and aum.get_baskets().
+        with self.browser:
+            new_baskets = self.browser.nb_new_baskets()
+            if new_baskets:
+                ids = self.browser.get_baskets()
+                while new_baskets > 0 and len(ids) > new_baskets:
+                    new_baskets -= 1
+                    profile = self.browser.get_profile(ids[new_baskets])
+
+                    thread = Thread(profile.get_id())
+                    thread.root = Message(thread=thread,
+                                          id=self.MAGIC_ID_BASKET,
+                                          title='Basket of %s' % profile.get_name(),
+                                          sender=profile.get_name(),
+                                          receiver=self.browser.get_my_name(),
+                                          date=None, # now
+                                          content='You are taken in her basket!',
+                                          signature=profile.get_profile_text(),
+                                          children=[],
+                                          flags=Message.IS_UNREAD)
+                    yield thread.root
+
+    def set_message_read(self, message):
+        if message.id == self.MAGIC_ID_BASKET:
+            # We don't save baskets.
+            return
+
+        slut = self._get_slut(message.thread.id)
+        if slut['lastmsg'] < message.date:
+            slut['lastmsg'] = message.date
+            #slut['msgstatus'] = contact.get_status()
+            self.storage.set('sluts', message.thread.id, slut)
+            self.storage.save()
 
     def _get_slut(self, id):
-        if not id in self.storage.get('sluts'):
+        sluts = self.storage.get('sluts')
+        if not sluts or not id in sluts:
             slut = {'lastmsg': datetime(1970,1,1),
                     'msgstatus': ''}
         else:
@@ -97,81 +208,9 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesReply, ICapDating, ICapC
         slut['lastmsg'] = slut['lastmsg'].replace(tzinfo=tz.tzutc())
         return slut
 
-    def _iter_messages(self, thread, only_new):
+    def post_message(self, message):
         with self.browser:
-            try:
+            self.browser.post_mail(message.thread.id, message.content)
-                profiles = {}
-
-                if thread:
-                    slut = self._get_slut(int(thread))
-                    for mail in self._iter_thread_messages(thread, only_new, slut['lastmsg'], {}):
-                        if slut['lastmsg'] < mail.date:
-                            slut['lastmsg'] = mail.date
-                        yield mail
-
-                    self.storage.set('sluts', int(thread), slut)
-                    self.storage.save()
-                else:
-                    contacts = self.browser.get_threads_list()
-                    for contact in contacts:
-                        slut = self._get_slut(contact.get_id())
-                        last_msg = slut['lastmsg']
-
-                        if only_new and contact.get_lastmsg_date() < last_msg and contact.get_status() == slut['msgstatus'] or \
-                           not thread is None and int(thread) != contact.get_id():
-                            continue
-
-                        for mail in self._iter_thread_messages(contact.get_id(), only_new, last_msg, profiles):
-                            if last_msg < mail.date:
-                                last_msg = mail.date
-
-                            yield mail
-
-                        slut['lastmsg'] = last_msg
-                        slut['msgstatus'] = contact.get_status()
-                        self.storage.set('sluts', contact.get_id(), slut)
-                        self.storage.save()
-
-                    # Send mail when someone added me in her basket.
-                    # XXX possibly race condition if a slut adds me in her basket
-                    #     between the aum.nbNewBaskets() and aum.getBaskets().
-                    new_baskets = self.browser.nb_new_baskets()
-                    if new_baskets:
-                        ids = self.browser.get_baskets()
-                        while new_baskets > 0 and len(ids) > new_baskets:
-                            new_baskets -= 1
-                            profile = self.browser.get_profile(ids[new_baskets])
-
-                            yield Message(profile.get_id(), 1,
-                                          title='Basket of %s' % profile.get_name(),
-                                          sender=profile.get_name(),
-                                          content='You are taken in her basket!',
-                                          signature=profile.get_profile_text())
-            except BrowserUnavailable:
-                pass
-
-    def _iter_thread_messages(self, id, only_new, last_msg, profiles):
-        mails = self.browser.get_thread_mails(id)
-        for mail in mails:
-            if only_new and mail.date <= last_msg:
-                continue
-
-            if not mail.profile_link in profiles:
-                profiles[mail.profile_link] = self.browser.get_profile(mail.profile_link)
-            mail.signature += u'\n%s' % profiles[mail.profile_link].get_profile_text()
-
-            yield mail
-
-    def post_reply(self, thread_id, reply_id, title, message):
-        while 1:
-            try:
-                with self.browser:
-                    self.browser.post_mail(thread_id, message)
-            except AdopteWait:
-                # If we are on a waiting state, retry every 30 minutes until it is posted.
-                sleep(60*30)
-            else:
-                return
 
     def get_contact(self, contact):
         try:
@@ -268,4 +307,8 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesReply, ICapDating, ICapC
                         data = self.browser.openurl(photo.thumbnail_url).read()
                         contact.set_photo(name, thumbnail_data=data)
 
-    OBJECTS = {Contact: fill_contact}
+    def fill_thread(self, thread, fields):
+        return self.get_thread(thread)
+
+    OBJECTS = {Thread: fill_thread,
+               Contact: fill_contact}

weboob/backends/aum/browser.py

@@ -108,7 +108,7 @@ class AuMBrowser(BaseBrowser):
         if not self.is_on_page(RegisterPage):
             self.location('http://www.adopteunmec.com/register2.php')
 
-        return self.page.register(password, sex, birthday_d, birthday_m, birthday_y, zipcode, country, godfather)
+        return self.page.register(password, sex, birthday_d, birthday_m, birthday_y, zipcode, country)
 
     @pageaccess
     def add_photo(self, name, f):
@@ -198,9 +198,9 @@ class AuMBrowser(BaseBrowser):
         return self.page.get_contact_list()
 
     @pageaccess
-    def get_thread_mails(self, id):
+    def get_thread_mails(self, id, full=False):
         if not self.is_on_page(ContactThreadPage) or self.page.id != int(id):
-            self.page.open_thread_page(id)
+            self.page.open_thread_page(id, full)
         return self.page.mails
 
     @pageaccess

weboob/backends/aum/captcha.py

@@ -103,6 +103,8 @@ class Tile:
         try:
             return self.hash[self.checksum()]
         except KeyError:
+            print 'Unable te resolve:'
+            self.display()
             raise CaptchaError()
 
 class Captcha:

weboob/backends/aum/pages/contact_thread.py

@@ -16,6 +16,7 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 import re
+import time
 from datetime import datetime
 from dateutil import tz
 from logging import error, warning
@@ -23,9 +24,8 @@ from mechanize import FormNotFoundError
 
 from weboob.backends.aum.pages.base import PageBase
 from weboob.backends.aum.exceptions import AdopteCantPostMail
-from weboob.capabilities.messages import Message
 
-class MailParser(Message):
+class MailParser(object):
 
     """
     <td>
@@ -93,8 +93,14 @@ class MailParser(Message):
 
     def __init__(self, thread_id, name, tr):
         #             <td>             <table>  implicit<tbody>  <tr>
-        Message.__init__(self, thread_id, 0, 'Discussion with %s' % name, name)
+        self.thread_id = thread_id
+        self.date = None
+        self.message_id = 0
+        self.title = 'Discussion with %s' % name
+        self.sender = name
+        self.name = name
         self.tr = tr.childNodes[0].childNodes[1].childNodes[0].childNodes[0]
+        self.new = False
 
         tds = self.tr.childNodes
 
@@ -131,6 +137,10 @@ class MailParser(Message):
         self.parse_profile_link()
         self.parse_from()
 
+    @property
+    def date_int(self):
+        return int(time.strftime('%Y%m%d%H%M%S', self.date.timetuple()))
+
     def set_parent_message_id(self, date):
         self.parent_message_id = date
 
@@ -157,7 +167,7 @@ class MailParser(Message):
             self.id = '%s.%s' % (self.thread_id, self.message_id)
 
             if m.group(7).find('nouveau') >= 0:
-                self.flags |= self.IS_UNREAD
+                self.new = True
         else:
             error('Error: unable to parse the datetime string "%s"' % date_str)
 

weboob/backends/aum/pages/login.py

@@ -16,6 +16,8 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 
+import re
+
 from weboob.tools.browser import BrowserIncorrectPassword
 
 from .base import PageBase
@@ -29,7 +31,29 @@ class LoginPage(PageBase):
 
         self.browser.submit()  # submit current form
 
+class RegisterError(Exception):
+    pass
+
 class RegisterPage(PageBase):
+    def on_loaded(self):
+        display_errors = False
+        for div in self.document.getElementsByTagName('div'):
+            if div.getAttribute('class') == 'balloon':
+                display_errors = True
+                break
+
+        if not display_errors:
+            return
+
+        errors = []
+        for script in self.document.getElementsByTagName('script'):
+            for child in script.childNodes:
+                if child and child.data.find('dispErrors') >= 0:
+                    for m in re.finditer('"(\w+)": "(.*)",', child.data):
+                        errors.append(m.group(2))
+
+        raise RegisterError(u'Unable to register account: %s' % ', '.join(errors))
+
     def register(self, password, sex, birthday_d, birthday_m, birthday_y, zipcode, country):
         """
 Form name=register (#1)
@@ -53,7 +77,7 @@ Form name=register (#1)
         self.browser.select_form(name='register')
         self.browser.set_all_readonly(False)
 
-        self.browser['sex'] = str(sex)
+        self.browser['sex'] = [str(sex)]
         self.browser['birthday0'] = [str(birthday_d)]
         self.browser['birthday1'] = [str(birthday_m)]
         self.browser['birthday2'] = [str(birthday_y)]

weboob/backends/dlfp/backend.py

@@ -18,8 +18,7 @@
 from __future__ import with_statement
 
 from weboob.tools.backend import BaseBackend
-from weboob.tools.browser import BrowserUnavailable
+from weboob.capabilities.messages import ICapMessages, ICapMessagesPost, Message, Thread
-from weboob.capabilities.messages import ICapMessages, ICapMessagesReply, Message
 
 from .feeds import ArticlesList
 from .browser import DLFP
@@ -28,7 +27,7 @@ from .browser import DLFP
 __all__ = ['DLFPBackend']
 
 
-class DLFPBackend(BaseBackend, ICapMessages, ICapMessagesReply):
+class DLFPBackend(BaseBackend, ICapMessages, ICapMessagesPost):
     NAME = 'dlfp'
     MAINTAINER = 'Romain Bignon'
     EMAIL = 'romain@peerfuse.org'
@@ -46,75 +45,95 @@ class DLFPBackend(BaseBackend, ICapMessages, ICapMessagesReply):
     def create_default_browser(self):
         return self.create_browser(self.config['username'], self.config['password'])
 
-    def iter_messages(self, thread=None):
+    def iter_threads(self):
-        return self._iter_messages(thread, False)
+        whats = set()
-
-    def iter_new_messages(self, thread=None):
-        return self._iter_messages(thread, True)
-
-    def _iter_messages(self, thread, only_new):
         if self.config['get_news']:
-            for message in self._iter_messages_of('newspaper', thread, only_new):
+            whats.add('newspaper')
-                yield message
         if self.config['get_telegrams']:
-            for message in self._iter_messages_of('telegram', thread, only_new):
+            whats.add('telegram')
-                yield message
 
-    def _iter_messages_of(self, what, thread_wanted, only_new):
+        for what in whats:
-        if not what in self.storage.get('seen', default={}):
+            for article in ArticlesList(what).iter_articles():
-            self.storage.set('seen', what, {})
+                thread = Thread(article.id)
+                thread.title = article.title
+                yield thread
 
-        seen = {}
+    def get_thread(self, id):
-        for article in ArticlesList(what).iter_articles():
+        if isinstance(id, Thread):
-            if thread_wanted and thread_wanted != article.id:
+            thread = id
-                continue
+            id = thread.id
 
-            flags = Message.IS_HTML
+        with self.browser:
-            if not article.id in self.storage.get('seen', what, default={}):
+            content = self.browser.get_content(id)
-                seen[article.id] = {'comments': []}
-                flags |= Message.IS_NEW
-            else:
-                seen[article.id] = self.storage.get('seen', what, article.id, default={})
 
-            try:
+        if not thread:
-                with self.browser:
+            thread = Thread(id)
-                    thread = self.browser.get_content(article.id)
-            except BrowserUnavailable:
-                continue
 
-            if not only_new or flags & Message.IS_NEW:
+        flags = Message.IS_HTML
-                yield Message(thread.id,
+        if not thread.id in self.storage.get('seen', default={}):
-                              0,
+            flags |= Message.IS_UNREAD
-                              thread.title,
+
-                              thread.author,
+        thread.title = content.title
-                              article.datetime,
+        if not thread.date:
-                              content=''.join([thread.body, thread.part2]),
+            thread.date = content.date
-                              signature='URL: %s' % article.url,
+
+        thread.root = Message(thread=thread,
+                              id=0, # root message
+                              title=content.title,
+                              sender=content.author,
+                              receiver=None,
+                              date=thread.date, #TODO XXX WTF this is None
+                              parent=None,
+                              content=''.join([content.body, content.part2]),
+                              signature='URL: %s' % content.url,
+                              children=[],
                               flags=flags)
 
-            for comment in thread.iter_all_comments():
+        for com in content.comments:
-                flags = Message.IS_HTML
+            self._insert_comment(com, thread.root)
-                if not comment.id in seen[article.id]['comments']:
-                    seen[article.id]['comments'].append(comment.id)
-                    flags |= Message.IS_NEW
 
-                if not only_new or flags & Message.IS_NEW:
+        return thread
-                    yield Message(thread.id,
-                                  comment.id,
-                                  comment.title,
-                                  comment.author,
-                                  comment.date,
-                                  comment.reply_id,
-                                  comment.body,
-                                  'Score: %d' % comment.score,
-                                  flags)
 
-        # If there is no articles seen, it's suspicious, probably I can't
+    def _insert_comment(self, com, parent):
-        # fetch the feed.
+        """"
-        if seen:
+        Insert 'com' comment and its children in the parent message.
-            self.storage.set('seen', what, seen)
+        """
-            self.storage.save()
+        flags = Message.IS_HTML
+        if not com.id in self.storage.get('seen', parent.thread.id, 'comments', default=[]):
+            flags |= Message.IS_UNREAD
 
-    def post_reply(self, thread_id, reply_id, title, message):
+        message = Message(thread=parent.thread,
+                          id=com.id,
+                          title=com.title,
+                          sender=com.author,
+                          receiver=None,
+                          date=com.date,
+                          parent=parent,
+                          content=com.body,
+                          signature='Score: %d' % com.score,
+                          children=[],
+                          flags=flags)
+
+        parent.children.append(message)
+        for sub in com.comments:
+            self._insert_comment(sub, message)
+
+    def iter_unread_messages(self, thread=None):
+        for thread in self.iter_threads():
+            self.fill_thread(thread, 'root')
+            for m in thread.iter_all_messages():
+                if m.flags & m.IS_UNREAD:
+                    yield m
+
+    def set_message_read(self, message):
+        self.storage.set('seen', message.thread.id, 'comments', self.storage.get('seen', message.thread.id, 'comments', default=[]) + [message.id])
+        self.storage.save()
+
+    def post_mesage(self, message):
         with self.browser:
-            return self.browser.post_reply(thread_id, reply_id, title, message)
+            return self.browser.post_reply(message.thread.id, message.parent.id, message.title, message.content)
+
+    def fill_thread(self, thread, fields):
+        return self.get_thread(thread)
+
+    OBJECTS = {Thread: fill_thread}

weboob/backends/dlfp/browser.py

@@ -86,3 +86,4 @@ class DLFP(BaseBrowser):
 
     def is_logged(self):
         return (self.page and self.page.is_logged())
+

weboob/backends/dlfp/pages/news.py

@@ -31,7 +31,7 @@ class Comment(object):
         self.reply_id = reply_id
         self.title = u''
         self.author = u''
-        self.date = u''
+        self.date = None
         self.body = u''
         self.score = 0
         self.comments = []
@@ -73,7 +73,8 @@ class Article(object):
         self.author = u''
         self.body = u''
         self.part2 = u''
-        self.date = u''
+        self.date = None
+        self.url = u''
         self.comments = []
 
         for div in tree.findall('div'):
@@ -116,6 +117,7 @@ class ContentPage(DLFPPage):
     def parse_div(self, div):
         if div.attrib.get('class', '') in ('newsdiv', 'centraldiv'):
             self.article = Article(self.browser, url2id(self.url), div)
+            self.article.url = self.url
         if div.attrib.get('class', '') == 'articlediv':
             self.article.parse_part2(div)
         if div.attrib.get('class', '') == 'comments':

weboob/backends/dlfp/test.py

@@ -18,9 +18,13 @@
 
 from weboob.tools.test import BackendTest
 
+
+__all__ = ['DLFPTest']
+
+
 class DLFPTest(BackendTest):
     BACKEND = 'dlfp'
 
     def test_new_messages(self):
-        for message in self.backend.iter_new_messages():
+        for message in self.backend.iter_unread_messages():
             pass

weboob/backends/fourchan/backend.py

@@ -20,7 +20,7 @@ from __future__ import with_statement
 
 from logging import warning
 
-from weboob.capabilities.messages import ICapMessages, Message
+from weboob.capabilities.messages import ICapMessages, Message, Thread
 from weboob.tools.backend import BaseBackend
 
 from .browser import FourChan
@@ -41,71 +41,89 @@ class FourChanBackend(BaseBackend, ICapMessages):
     STORAGE = {'boards': {}}
     BROWSER = FourChan
 
-    def iter_messages(self, thread=None):
+    def _splitid(self, id):
-        return self._iter_messages(thread, False)
+        return id.split('.', 1)
 
-    def iter_new_messages(self, thread=None):
+    def get_thread(self, id):
-        return self._iter_messages(thread, True)
+        thread = None
 
-    def _iter_messages(self, thread, only_new):
+        if isinstance(id, Thread):
-        if thread:
+            thread = id
-            if '.' in thread:
+            id = thread.id
-                board, thread = thread.split('.', 2)
-                return self._iter_messages_of(board, thread, only_new)
-            else:
-                warning('"%s" is not a valid ID' % thread)
-        else:
-            for board in self.config['boards'].split(' '):
-                return self._iter_messages_of(board, None, only_new)
 
-    def _iter_messages_of(self, board, thread_wanted, only_new):
+        if not '.' in id:
-        if not board in self.storage.get('boards', default={}):
+            warning('Malformated ID (%s)' % id)
-            self.storage.set('boards', board, {})
+            return
 
-        if thread_wanted:
+        board, thread_id = self._splitid(id)
-            for message in self._iter_thread_messages(board, thread_wanted, only_new):
+
-                yield message
+        with self.browser:
-        else:
+            _thread = self.browser.get_thread(board, thread_id)
+
+        flags = 0
+        if not _thread.id in self.storage.get('boards', board, default={}):
+            flags |= Message.IS_UNREAD
+
+        if not thread:
+            thread = Thread(id)
+        thread.title = _thread.filename
+        thread.root = Message(thread=thread,
+                              id=0, # root message
+                              title=_thread.filename,
+                              sender=_thread.author,
+                              receiver=None,
+                              date=_thread.datetime,
+                              parent=None,
+                              content=_thread.text,
+                              signature=None,
+                              children=None,
+                              flags=flags|Message.IS_HTML)
+
+        parent = thread.root
+        for comment in _thread.comments:
+            flags = 0
+            if not comment.id in self.storage.get('boards', board, _thread.id, default=[]):
+                flags |= Message.IS_UNREAD
+
+            m = Message(thread=thread,
+                        id=comment.id,
+                        title=_thread.filename,
+                        sender=comment.author,
+                        receiver=None,
+                        date=comment.datetime,
+                        parent=parent,
+                        content=comment.text,
+                        signature=None,
+                        children=None,
+                        flags=flags|Message.IS_HTML)
+            parent.children = [m]
+            parent = m
+
+        return thread
+
+    def iter_threads(self):
+        for board in self.config['boards'].split(' '):
             with self.browser:
                 threads = self.browser.get_threads(board)
             for thread in threads:
-                for message in self._iter_thread_messages(board, thread.id, only_new):
+                t = Thread('%s.%s' % (board, thread.id))
-                    yield message
+                t.title = thread.filename
+                yield t
 
-    def _iter_thread_messages(self, board, thread, only_new):
+    def iter_unread_messages(self):
-        thread = self.browser.get_thread(board, thread)
+        for thread in self.iter_threads():
+            self.fill_thread(thread, 'root')
 
-        flags = Message.IS_HTML
+            for m in thread.iter_all_messages():
-        if thread.id in self.storage.get('boards', board, default={}):
+                if m.flags & Message.IS_UNREAD:
-            self.storage.set('boards', board, thread.id, [])
+                    yield m
-            flags |= Message.IS_NEW
-
-        if not only_new or flags & Message.IS_NEW:
-            yield Message('%s.%s' % (board, thread.id),
-                          0,
-                          thread.filename,
-                          thread.author,
-                          thread.datetime,
-                          content=thread.text,
-                          flags=flags)
-
-        for comment in thread.comments:
-            flags = Message.IS_HTML
-            if not comment.id in self.storage.get('boards', board, thread.id, default=[]):
-                self.storage.set('boards', board, thread.id, self.storage.get('boards', board, thread.id, default=[]) + [comment.id])
-                flags |= Message.IS_NEW
-
-            if not only_new or flags & Message.IS_NEW:
-                yield Message('%s.%s' % (board, thread.id),
-                              comment.id,
-                              thread.filename,
-                              comment.author,
-                              comment.datetime,
-                              0,
-                              comment.text,
-                              flags)
 
+    def set_message_read(self, message):
+        board, thread_id = self._splitid(message.thread.id)
+        self.storage.set('boards', board, thread_id, self.storage.get('boards', board, thread_id, default=[]) + [message.id])
         self.storage.save()
 
-    #def post_reply(self, thread_id, reply_id, title, message):
+    def fill_thread(self, thread, fields):
-    #    return self.browser.post_reply(thread_id, reply_id, title, message)
+        return self.get_thread(thread)
+
+    OBJECTS = {Thread: fill_thread}

weboob/backends/fourchan/browser.py

@@ -34,7 +34,7 @@ class FourChan(BaseBrowser):
         return self.page.articles
 
     def get_thread(self, board, id):
-        self.location('http://boards.4chan.org/%s/res/%d' % (board, id))
+        self.location('http://boards.4chan.org/%s/res/%d' % (board, long(id)))
 
         assert len(self.page.articles) == 1
         return self.page.articles[0]