From e7c24c13c82323805086ae2d10bc7086127369b8 Mon Sep 17 00:00:00 2001 From: Romain Bignon Date: Mon, 30 Aug 2010 16:38:34 +0200 Subject: [PATCH] adapt to new ICapMessages API --- weboob/applications/monboob/monboob.py | 29 ++- weboob/backends/aum/backend.py | 233 ++++++++++++-------- weboob/backends/aum/browser.py | 6 +- weboob/backends/aum/captcha.py | 2 + weboob/backends/aum/pages/contact_thread.py | 18 +- weboob/backends/aum/pages/login.py | 26 ++- weboob/backends/dlfp/backend.py | 143 ++++++------ weboob/backends/dlfp/browser.py | 1 + weboob/backends/dlfp/pages/news.py | 6 +- weboob/backends/dlfp/test.py | 6 +- weboob/backends/fourchan/backend.py | 132 ++++++----- weboob/backends/fourchan/browser.py | 2 +- 12 files changed, 368 insertions(+), 236 deletions(-) diff --git a/weboob/applications/monboob/monboob.py b/weboob/applications/monboob/monboob.py index 9c64e91f..5bcfe5e4 100644 --- a/weboob/applications/monboob/monboob.py +++ b/weboob/applications/monboob/monboob.py @@ -30,7 +30,7 @@ import asyncore from weboob.core.ouiboube import Weboob from weboob.core.scheduler import Scheduler -from weboob.capabilities.messages import ICapMessages, ICapMessagesReply, Message +from weboob.capabilities.messages import ICapMessages, ICapMessagesPost, Thread, Message from weboob.tools.application.console import ConsoleApplication from weboob.tools.misc import html2text, get_backtrace, utc2local @@ -158,23 +158,31 @@ class Monboob(ConsoleApplication): print >>sys.stderr, 'Backend %s not found' % bname return 1 - if not backend.has_caps(ICapMessagesReply): - print >>sys.stderr, 'The backend %s does not implement ICapMessagesReply' % bname + if not backend.has_caps(ICapMessagesPost): + print >>sys.stderr, 'The backend %s does not implement ICapMessagesPost' % bname return 1 thread_id, msg_id = id.rsplit('.', 1) + thread = Thread(thread_id) + message = Message(thread, + 0, + title=title, + sender=None, + receiver=None, + parent=Message(thread, msg_id), + content=content) try: - backend.post_reply(thread_id, msg_id, title, content) + backend.post_message(message) except Exception, e: content = u'Unable to send message to %s:\n' % thread_id content += '\n\t%s\n' % e if logging.root.level == logging.DEBUG: content += '\n%s\n' % get_backtrace(e) - self.send_email(backend, Message(thread_id, + self.send_email(backend, Message(thread, 0, title='Unable to send message', sender='Monboob', - parent_message_id=msg_id, + parent=Message(thread, msg_id), content=content)) @ConsoleApplication.command("run daemon") @@ -183,22 +191,23 @@ class Monboob(ConsoleApplication): self.weboob.loop() def process(self): - for backend, message in self.weboob.do('iter_new_messages'): + for backend, message in self.weboob.do('iter_unread_messages'): self.send_email(backend, message) + backend.set_message_read(message) def send_email(self, backend, mail): domain = self.config.get('domain') recipient = self.config.get('recipient') reply_id = '' - if mail.parent_id: - reply_id = u'<%s.%s@%s>' % (backend.name, mail.parent_id, domain) + if mail.parent: + reply_id = u'<%s.%s@%s>' % (backend.name, mail.parent.full_id, domain) subject = mail.title sender = u'"%s" <%s@%s>' % (mail.sender.replace('"', '""'), backend.name, domain) # assume that .date is an UTC datetime date = formatdate(time.mktime(utc2local(mail.date).timetuple()), localtime=True) - msg_id = u'<%s.%s@%s>' % (backend.name, mail.id, domain) + msg_id = u'<%s.%s@%s>' % (backend.name, mail.full_id, domain) if int(self.config.get('html')) and mail.flags & mail.IS_HTML: body = mail.content diff --git a/weboob/backends/aum/backend.py b/weboob/backends/aum/backend.py index 0339cb45..029f8f34 100644 --- a/weboob/backends/aum/backend.py +++ b/weboob/backends/aum/backend.py @@ -19,16 +19,16 @@ from __future__ import with_statement from datetime import datetime from dateutil import tz -from logging import warning -from time import sleep +from logging import warning, debug from weboob.capabilities.chat import ICapChat -from weboob.capabilities.messages import ICapMessages, ICapMessagesReply, Message +from weboob.capabilities.messages import ICapMessages, ICapMessagesPost, Message, Thread from weboob.capabilities.dating import ICapDating, StatusField from weboob.capabilities.contact import ICapContact, Contact, ProfileNode from weboob.tools.backend import BaseBackend from weboob.tools.browser import BrowserUnavailable +from .captcha import CaptchaError from .browser import AuMBrowser from .exceptions import AdopteWait from .optim.profiles_walker import ProfilesWalker @@ -38,7 +38,7 @@ from .optim.visibility import Visibility __all__ = ['AuMBackend'] -class AuMBackend(BaseBackend, ICapMessages, ICapMessagesReply, ICapDating, ICapChat, ICapContact): +class AuMBackend(BaseBackend, ICapMessages, ICapMessagesPost, ICapDating, ICapChat, ICapContact): NAME = 'aum' MAINTAINER = 'Romain Bignon' EMAIL = 'romain@peerfuse.org' @@ -54,17 +54,27 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesReply, ICapDating, ICapC } BROWSER = AuMBrowser + MAGIC_ID_BASKET = 1 + def create_default_browser(self): if self.config['register']: - browser = self.create_browser(self.config['username']) - browser.register(password= self.config['password'], - sex= 0, - birthday_d= 1, - birthday_m= 1, - birthday_y= 1970, - zipcode= 75001, - country= 'fr', - godfather= '') + browser = None + while not browser: + try: + browser = self.create_browser(self.config['username']) + browser.register(password= self.config['password'], + sex= 0, + birthday_d= 1, + birthday_m= 1, + birthday_y= 1970, + zipcode= 75001, + country= 'fr', + godfather= '') + except CaptchaError: + debug('Unable to resolve captcha. Retrying...') + browser = None + browser.password = self.config['password'] + return browser else: return self.create_browser(self.config['username'], self.config['password']) @@ -79,16 +89,117 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesReply, ICapDating, ICapC except AdopteWait: return (StatusField('notice', '', u'

You are currently waiting 1am to be able to connect with this account

', StatusField.FIELD_HTML|StatusField.FIELD_TEXT)) - def iter_messages(self, thread=None): - for message in self._iter_messages(thread, False): - yield message + def iter_threads(self): + with self.browser: + contacts = self.browser.get_threads_list() - def iter_new_messages(self, thread=None): - for message in self._iter_messages(thread, True): - yield message + for contact in contacts: + thread = Thread(contact.get_id()) + yield thread + + def get_thread(self, id): + thread = None + if isinstance(id, Thread): + thread = id + id = thread.id + + if not thread: + thread = Thread(id) + full = False + else: + full = True + + with self.browser: + mails = self.browser.get_thread_mails(id, full) + my_name = self.browser.get_my_name() + + child = None + msg = None + slut = self._get_slut(id) + for mail in mails: + flags = 0 + if mail.date > slut['lastmsg']: + flags |= Message.IS_UNREAD + if mail.sender != my_name: + if mail.new: + flags |= Message.IS_NOT_ACCUSED + else: + flags |= Message.IS_ACCUSED + + msg = Message(thread=thread, + id=mail.message_id, + title=mail.title, + sender=mail.sender, + receiver=mail.name if mail.sender == my_name else my_name, # TODO: me + date=mail.date, + content=mail.content, + signature=mail.signature, + children=[], + flags=flags) + if child: + msg.children.append(child) + child.parent = msg + + child = msg + + if full: + # If we have get all the messages, replace NotLoaded with None as + # parent. + msg.parent = None + thread.root = msg + + return thread + + def iter_unread_messages(self, thread=None): + with self.browser: + contacts = self.browser.get_threads_list() + for contact in contacts: + slut = self._get_slut(contact.get_id()) + if contact.get_lastmsg_date() > slut['lastmsg']: + thread = self.get_thread(contact.get_id()) + for m in thread.iter_all_messages(): + if m.flags & m.IS_UNREAD: + yield m + + # Send mail when someone added me in her basket. + # XXX possibly race condition if a slut adds me in her basket + # between the aum.nb_new_baskets() and aum.get_baskets(). + with self.browser: + new_baskets = self.browser.nb_new_baskets() + if new_baskets: + ids = self.browser.get_baskets() + while new_baskets > 0 and len(ids) > new_baskets: + new_baskets -= 1 + profile = self.browser.get_profile(ids[new_baskets]) + + thread = Thread(profile.get_id()) + thread.root = Message(thread=thread, + id=self.MAGIC_ID_BASKET, + title='Basket of %s' % profile.get_name(), + sender=profile.get_name(), + receiver=self.browser.get_my_name(), + date=None, # now + content='You are taken in her basket!', + signature=profile.get_profile_text(), + children=[], + flags=Message.IS_UNREAD) + yield thread.root + + def set_message_read(self, message): + if message.id == self.MAGIC_ID_BASKET: + # We don't save baskets. + return + + slut = self._get_slut(message.thread.id) + if slut['lastmsg'] < message.date: + slut['lastmsg'] = message.date + #slut['msgstatus'] = contact.get_status() + self.storage.set('sluts', message.thread.id, slut) + self.storage.save() def _get_slut(self, id): - if not id in self.storage.get('sluts'): + sluts = self.storage.get('sluts') + if not sluts or not id in sluts: slut = {'lastmsg': datetime(1970,1,1), 'msgstatus': ''} else: @@ -97,81 +208,9 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesReply, ICapDating, ICapC slut['lastmsg'] = slut['lastmsg'].replace(tzinfo=tz.tzutc()) return slut - def _iter_messages(self, thread, only_new): + def post_message(self, message): with self.browser: - try: - profiles = {} - - if thread: - slut = self._get_slut(int(thread)) - for mail in self._iter_thread_messages(thread, only_new, slut['lastmsg'], {}): - if slut['lastmsg'] < mail.date: - slut['lastmsg'] = mail.date - yield mail - - self.storage.set('sluts', int(thread), slut) - self.storage.save() - else: - contacts = self.browser.get_threads_list() - for contact in contacts: - slut = self._get_slut(contact.get_id()) - last_msg = slut['lastmsg'] - - if only_new and contact.get_lastmsg_date() < last_msg and contact.get_status() == slut['msgstatus'] or \ - not thread is None and int(thread) != contact.get_id(): - continue - - for mail in self._iter_thread_messages(contact.get_id(), only_new, last_msg, profiles): - if last_msg < mail.date: - last_msg = mail.date - - yield mail - - slut['lastmsg'] = last_msg - slut['msgstatus'] = contact.get_status() - self.storage.set('sluts', contact.get_id(), slut) - self.storage.save() - - # Send mail when someone added me in her basket. - # XXX possibly race condition if a slut adds me in her basket - # between the aum.nbNewBaskets() and aum.getBaskets(). - new_baskets = self.browser.nb_new_baskets() - if new_baskets: - ids = self.browser.get_baskets() - while new_baskets > 0 and len(ids) > new_baskets: - new_baskets -= 1 - profile = self.browser.get_profile(ids[new_baskets]) - - yield Message(profile.get_id(), 1, - title='Basket of %s' % profile.get_name(), - sender=profile.get_name(), - content='You are taken in her basket!', - signature=profile.get_profile_text()) - except BrowserUnavailable: - pass - - def _iter_thread_messages(self, id, only_new, last_msg, profiles): - mails = self.browser.get_thread_mails(id) - for mail in mails: - if only_new and mail.date <= last_msg: - continue - - if not mail.profile_link in profiles: - profiles[mail.profile_link] = self.browser.get_profile(mail.profile_link) - mail.signature += u'\n%s' % profiles[mail.profile_link].get_profile_text() - - yield mail - - def post_reply(self, thread_id, reply_id, title, message): - while 1: - try: - with self.browser: - self.browser.post_mail(thread_id, message) - except AdopteWait: - # If we are on a waiting state, retry every 30 minutes until it is posted. - sleep(60*30) - else: - return + self.browser.post_mail(message.thread.id, message.content) def get_contact(self, contact): try: @@ -268,4 +307,8 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesReply, ICapDating, ICapC data = self.browser.openurl(photo.thumbnail_url).read() contact.set_photo(name, thumbnail_data=data) - OBJECTS = {Contact: fill_contact} + def fill_thread(self, thread, fields): + return self.get_thread(thread) + + OBJECTS = {Thread: fill_thread, + Contact: fill_contact} diff --git a/weboob/backends/aum/browser.py b/weboob/backends/aum/browser.py index 2f348cdd..8b486c42 100644 --- a/weboob/backends/aum/browser.py +++ b/weboob/backends/aum/browser.py @@ -108,7 +108,7 @@ class AuMBrowser(BaseBrowser): if not self.is_on_page(RegisterPage): self.location('http://www.adopteunmec.com/register2.php') - return self.page.register(password, sex, birthday_d, birthday_m, birthday_y, zipcode, country, godfather) + return self.page.register(password, sex, birthday_d, birthday_m, birthday_y, zipcode, country) @pageaccess def add_photo(self, name, f): @@ -198,9 +198,9 @@ class AuMBrowser(BaseBrowser): return self.page.get_contact_list() @pageaccess - def get_thread_mails(self, id): + def get_thread_mails(self, id, full=False): if not self.is_on_page(ContactThreadPage) or self.page.id != int(id): - self.page.open_thread_page(id) + self.page.open_thread_page(id, full) return self.page.mails @pageaccess diff --git a/weboob/backends/aum/captcha.py b/weboob/backends/aum/captcha.py index 7c1bf900..f9a8aa9e 100644 --- a/weboob/backends/aum/captcha.py +++ b/weboob/backends/aum/captcha.py @@ -103,6 +103,8 @@ class Tile: try: return self.hash[self.checksum()] except KeyError: + print 'Unable te resolve:' + self.display() raise CaptchaError() class Captcha: diff --git a/weboob/backends/aum/pages/contact_thread.py b/weboob/backends/aum/pages/contact_thread.py index 3ffc9071..4273edcb 100644 --- a/weboob/backends/aum/pages/contact_thread.py +++ b/weboob/backends/aum/pages/contact_thread.py @@ -16,6 +16,7 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. import re +import time from datetime import datetime from dateutil import tz from logging import error, warning @@ -23,9 +24,8 @@ from mechanize import FormNotFoundError from weboob.backends.aum.pages.base import PageBase from weboob.backends.aum.exceptions import AdopteCantPostMail -from weboob.capabilities.messages import Message -class MailParser(Message): +class MailParser(object): """ @@ -93,8 +93,14 @@ class MailParser(Message): def __init__(self, thread_id, name, tr): # implicit - Message.__init__(self, thread_id, 0, 'Discussion with %s' % name, name) + self.thread_id = thread_id + self.date = None + self.message_id = 0 + self.title = 'Discussion with %s' % name + self.sender = name + self.name = name self.tr = tr.childNodes[0].childNodes[1].childNodes[0].childNodes[0] + self.new = False tds = self.tr.childNodes @@ -131,6 +137,10 @@ class MailParser(Message): self.parse_profile_link() self.parse_from() + @property + def date_int(self): + return int(time.strftime('%Y%m%d%H%M%S', self.date.timetuple())) + def set_parent_message_id(self, date): self.parent_message_id = date @@ -157,7 +167,7 @@ class MailParser(Message): self.id = '%s.%s' % (self.thread_id, self.message_id) if m.group(7).find('nouveau') >= 0: - self.flags |= self.IS_UNREAD + self.new = True else: error('Error: unable to parse the datetime string "%s"' % date_str) diff --git a/weboob/backends/aum/pages/login.py b/weboob/backends/aum/pages/login.py index 68a6d3b2..dc39e20c 100644 --- a/weboob/backends/aum/pages/login.py +++ b/weboob/backends/aum/pages/login.py @@ -16,6 +16,8 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +import re + from weboob.tools.browser import BrowserIncorrectPassword from .base import PageBase @@ -29,7 +31,29 @@ class LoginPage(PageBase): self.browser.submit() # submit current form +class RegisterError(Exception): + pass + class RegisterPage(PageBase): + def on_loaded(self): + display_errors = False + for div in self.document.getElementsByTagName('div'): + if div.getAttribute('class') == 'balloon': + display_errors = True + break + + if not display_errors: + return + + errors = [] + for script in self.document.getElementsByTagName('script'): + for child in script.childNodes: + if child and child.data.find('dispErrors') >= 0: + for m in re.finditer('"(\w+)": "(.*)",', child.data): + errors.append(m.group(2)) + + raise RegisterError(u'Unable to register account: %s' % ', '.join(errors)) + def register(self, password, sex, birthday_d, birthday_m, birthday_y, zipcode, country): """ Form name=register (#1) @@ -53,7 +77,7 @@ Form name=register (#1) self.browser.select_form(name='register') self.browser.set_all_readonly(False) - self.browser['sex'] = str(sex) + self.browser['sex'] = [str(sex)] self.browser['birthday0'] = [str(birthday_d)] self.browser['birthday1'] = [str(birthday_m)] self.browser['birthday2'] = [str(birthday_y)] diff --git a/weboob/backends/dlfp/backend.py b/weboob/backends/dlfp/backend.py index 760efcd2..ecc92fe6 100644 --- a/weboob/backends/dlfp/backend.py +++ b/weboob/backends/dlfp/backend.py @@ -18,8 +18,7 @@ from __future__ import with_statement from weboob.tools.backend import BaseBackend -from weboob.tools.browser import BrowserUnavailable -from weboob.capabilities.messages import ICapMessages, ICapMessagesReply, Message +from weboob.capabilities.messages import ICapMessages, ICapMessagesPost, Message, Thread from .feeds import ArticlesList from .browser import DLFP @@ -28,7 +27,7 @@ from .browser import DLFP __all__ = ['DLFPBackend'] -class DLFPBackend(BaseBackend, ICapMessages, ICapMessagesReply): +class DLFPBackend(BaseBackend, ICapMessages, ICapMessagesPost): NAME = 'dlfp' MAINTAINER = 'Romain Bignon' EMAIL = 'romain@peerfuse.org' @@ -46,75 +45,95 @@ class DLFPBackend(BaseBackend, ICapMessages, ICapMessagesReply): def create_default_browser(self): return self.create_browser(self.config['username'], self.config['password']) - def iter_messages(self, thread=None): - return self._iter_messages(thread, False) - - def iter_new_messages(self, thread=None): - return self._iter_messages(thread, True) - - def _iter_messages(self, thread, only_new): + def iter_threads(self): + whats = set() if self.config['get_news']: - for message in self._iter_messages_of('newspaper', thread, only_new): - yield message + whats.add('newspaper') if self.config['get_telegrams']: - for message in self._iter_messages_of('telegram', thread, only_new): - yield message + whats.add('telegram') - def _iter_messages_of(self, what, thread_wanted, only_new): - if not what in self.storage.get('seen', default={}): - self.storage.set('seen', what, {}) + for what in whats: + for article in ArticlesList(what).iter_articles(): + thread = Thread(article.id) + thread.title = article.title + yield thread - seen = {} - for article in ArticlesList(what).iter_articles(): - if thread_wanted and thread_wanted != article.id: - continue + def get_thread(self, id): + if isinstance(id, Thread): + thread = id + id = thread.id - flags = Message.IS_HTML - if not article.id in self.storage.get('seen', what, default={}): - seen[article.id] = {'comments': []} - flags |= Message.IS_NEW - else: - seen[article.id] = self.storage.get('seen', what, article.id, default={}) + with self.browser: + content = self.browser.get_content(id) - try: - with self.browser: - thread = self.browser.get_content(article.id) - except BrowserUnavailable: - continue + if not thread: + thread = Thread(id) - if not only_new or flags & Message.IS_NEW: - yield Message(thread.id, - 0, - thread.title, - thread.author, - article.datetime, - content=''.join([thread.body, thread.part2]), - signature='URL: %s' % article.url, + flags = Message.IS_HTML + if not thread.id in self.storage.get('seen', default={}): + flags |= Message.IS_UNREAD + + thread.title = content.title + if not thread.date: + thread.date = content.date + + thread.root = Message(thread=thread, + id=0, # root message + title=content.title, + sender=content.author, + receiver=None, + date=thread.date, #TODO XXX WTF this is None + parent=None, + content=''.join([content.body, content.part2]), + signature='URL: %s' % content.url, + children=[], flags=flags) - for comment in thread.iter_all_comments(): - flags = Message.IS_HTML - if not comment.id in seen[article.id]['comments']: - seen[article.id]['comments'].append(comment.id) - flags |= Message.IS_NEW + for com in content.comments: + self._insert_comment(com, thread.root) - if not only_new or flags & Message.IS_NEW: - yield Message(thread.id, - comment.id, - comment.title, - comment.author, - comment.date, - comment.reply_id, - comment.body, - 'Score: %d' % comment.score, - flags) + return thread - # If there is no articles seen, it's suspicious, probably I can't - # fetch the feed. - if seen: - self.storage.set('seen', what, seen) - self.storage.save() + def _insert_comment(self, com, parent): + """" + Insert 'com' comment and its children in the parent message. + """ + flags = Message.IS_HTML + if not com.id in self.storage.get('seen', parent.thread.id, 'comments', default=[]): + flags |= Message.IS_UNREAD - def post_reply(self, thread_id, reply_id, title, message): + message = Message(thread=parent.thread, + id=com.id, + title=com.title, + sender=com.author, + receiver=None, + date=com.date, + parent=parent, + content=com.body, + signature='Score: %d' % com.score, + children=[], + flags=flags) + + parent.children.append(message) + for sub in com.comments: + self._insert_comment(sub, message) + + def iter_unread_messages(self, thread=None): + for thread in self.iter_threads(): + self.fill_thread(thread, 'root') + for m in thread.iter_all_messages(): + if m.flags & m.IS_UNREAD: + yield m + + def set_message_read(self, message): + self.storage.set('seen', message.thread.id, 'comments', self.storage.get('seen', message.thread.id, 'comments', default=[]) + [message.id]) + self.storage.save() + + def post_mesage(self, message): with self.browser: - return self.browser.post_reply(thread_id, reply_id, title, message) + return self.browser.post_reply(message.thread.id, message.parent.id, message.title, message.content) + + def fill_thread(self, thread, fields): + return self.get_thread(thread) + + OBJECTS = {Thread: fill_thread} diff --git a/weboob/backends/dlfp/browser.py b/weboob/backends/dlfp/browser.py index 6c436f98..e114d3f1 100644 --- a/weboob/backends/dlfp/browser.py +++ b/weboob/backends/dlfp/browser.py @@ -86,3 +86,4 @@ class DLFP(BaseBrowser): def is_logged(self): return (self.page and self.page.is_logged()) + diff --git a/weboob/backends/dlfp/pages/news.py b/weboob/backends/dlfp/pages/news.py index 57161554..51b0fd94 100644 --- a/weboob/backends/dlfp/pages/news.py +++ b/weboob/backends/dlfp/pages/news.py @@ -31,7 +31,7 @@ class Comment(object): self.reply_id = reply_id self.title = u'' self.author = u'' - self.date = u'' + self.date = None self.body = u'' self.score = 0 self.comments = [] @@ -73,7 +73,8 @@ class Article(object): self.author = u'' self.body = u'' self.part2 = u'' - self.date = u'' + self.date = None + self.url = u'' self.comments = [] for div in tree.findall('div'): @@ -116,6 +117,7 @@ class ContentPage(DLFPPage): def parse_div(self, div): if div.attrib.get('class', '') in ('newsdiv', 'centraldiv'): self.article = Article(self.browser, url2id(self.url), div) + self.article.url = self.url if div.attrib.get('class', '') == 'articlediv': self.article.parse_part2(div) if div.attrib.get('class', '') == 'comments': diff --git a/weboob/backends/dlfp/test.py b/weboob/backends/dlfp/test.py index c0840823..b9e11915 100644 --- a/weboob/backends/dlfp/test.py +++ b/weboob/backends/dlfp/test.py @@ -18,9 +18,13 @@ from weboob.tools.test import BackendTest + +__all__ = ['DLFPTest'] + + class DLFPTest(BackendTest): BACKEND = 'dlfp' def test_new_messages(self): - for message in self.backend.iter_new_messages(): + for message in self.backend.iter_unread_messages(): pass diff --git a/weboob/backends/fourchan/backend.py b/weboob/backends/fourchan/backend.py index bf9da255..f39bf07a 100644 --- a/weboob/backends/fourchan/backend.py +++ b/weboob/backends/fourchan/backend.py @@ -20,7 +20,7 @@ from __future__ import with_statement from logging import warning -from weboob.capabilities.messages import ICapMessages, Message +from weboob.capabilities.messages import ICapMessages, Message, Thread from weboob.tools.backend import BaseBackend from .browser import FourChan @@ -41,71 +41,89 @@ class FourChanBackend(BaseBackend, ICapMessages): STORAGE = {'boards': {}} BROWSER = FourChan - def iter_messages(self, thread=None): - return self._iter_messages(thread, False) + def _splitid(self, id): + return id.split('.', 1) - def iter_new_messages(self, thread=None): - return self._iter_messages(thread, True) + def get_thread(self, id): + thread = None - def _iter_messages(self, thread, only_new): - if thread: - if '.' in thread: - board, thread = thread.split('.', 2) - return self._iter_messages_of(board, thread, only_new) - else: - warning('"%s" is not a valid ID' % thread) - else: - for board in self.config['boards'].split(' '): - return self._iter_messages_of(board, None, only_new) + if isinstance(id, Thread): + thread = id + id = thread.id - def _iter_messages_of(self, board, thread_wanted, only_new): - if not board in self.storage.get('boards', default={}): - self.storage.set('boards', board, {}) + if not '.' in id: + warning('Malformated ID (%s)' % id) + return - if thread_wanted: - for message in self._iter_thread_messages(board, thread_wanted, only_new): - yield message - else: + board, thread_id = self._splitid(id) + + with self.browser: + _thread = self.browser.get_thread(board, thread_id) + + flags = 0 + if not _thread.id in self.storage.get('boards', board, default={}): + flags |= Message.IS_UNREAD + + if not thread: + thread = Thread(id) + thread.title = _thread.filename + thread.root = Message(thread=thread, + id=0, # root message + title=_thread.filename, + sender=_thread.author, + receiver=None, + date=_thread.datetime, + parent=None, + content=_thread.text, + signature=None, + children=None, + flags=flags|Message.IS_HTML) + + parent = thread.root + for comment in _thread.comments: + flags = 0 + if not comment.id in self.storage.get('boards', board, _thread.id, default=[]): + flags |= Message.IS_UNREAD + + m = Message(thread=thread, + id=comment.id, + title=_thread.filename, + sender=comment.author, + receiver=None, + date=comment.datetime, + parent=parent, + content=comment.text, + signature=None, + children=None, + flags=flags|Message.IS_HTML) + parent.children = [m] + parent = m + + return thread + + def iter_threads(self): + for board in self.config['boards'].split(' '): with self.browser: threads = self.browser.get_threads(board) for thread in threads: - for message in self._iter_thread_messages(board, thread.id, only_new): - yield message + t = Thread('%s.%s' % (board, thread.id)) + t.title = thread.filename + yield t - def _iter_thread_messages(self, board, thread, only_new): - thread = self.browser.get_thread(board, thread) + def iter_unread_messages(self): + for thread in self.iter_threads(): + self.fill_thread(thread, 'root') - flags = Message.IS_HTML - if thread.id in self.storage.get('boards', board, default={}): - self.storage.set('boards', board, thread.id, []) - flags |= Message.IS_NEW - - if not only_new or flags & Message.IS_NEW: - yield Message('%s.%s' % (board, thread.id), - 0, - thread.filename, - thread.author, - thread.datetime, - content=thread.text, - flags=flags) - - for comment in thread.comments: - flags = Message.IS_HTML - if not comment.id in self.storage.get('boards', board, thread.id, default=[]): - self.storage.set('boards', board, thread.id, self.storage.get('boards', board, thread.id, default=[]) + [comment.id]) - flags |= Message.IS_NEW - - if not only_new or flags & Message.IS_NEW: - yield Message('%s.%s' % (board, thread.id), - comment.id, - thread.filename, - comment.author, - comment.datetime, - 0, - comment.text, - flags) + for m in thread.iter_all_messages(): + if m.flags & Message.IS_UNREAD: + yield m + def set_message_read(self, message): + board, thread_id = self._splitid(message.thread.id) + self.storage.set('boards', board, thread_id, self.storage.get('boards', board, thread_id, default=[]) + [message.id]) self.storage.save() - #def post_reply(self, thread_id, reply_id, title, message): - # return self.browser.post_reply(thread_id, reply_id, title, message) + def fill_thread(self, thread, fields): + return self.get_thread(thread) + + OBJECTS = {Thread: fill_thread} diff --git a/weboob/backends/fourchan/browser.py b/weboob/backends/fourchan/browser.py index 3722f123..5e6c86b3 100644 --- a/weboob/backends/fourchan/browser.py +++ b/weboob/backends/fourchan/browser.py @@ -34,7 +34,7 @@ class FourChan(BaseBrowser): return self.page.articles def get_thread(self, board, id): - self.location('http://boards.4chan.org/%s/res/%d' % (board, id)) + self.location('http://boards.4chan.org/%s/res/%d' % (board, long(id))) assert len(self.page.articles) == 1 return self.page.articles[0]