diff --git a/weboob/applications/monboob/monboob.py b/weboob/applications/monboob/monboob.py
index 9c64e91f..5bcfe5e4 100644
--- a/weboob/applications/monboob/monboob.py
+++ b/weboob/applications/monboob/monboob.py
@@ -30,7 +30,7 @@ import asyncore
from weboob.core.ouiboube import Weboob
from weboob.core.scheduler import Scheduler
-from weboob.capabilities.messages import ICapMessages, ICapMessagesReply, Message
+from weboob.capabilities.messages import ICapMessages, ICapMessagesPost, Thread, Message
from weboob.tools.application.console import ConsoleApplication
from weboob.tools.misc import html2text, get_backtrace, utc2local
@@ -158,23 +158,31 @@ class Monboob(ConsoleApplication):
print >>sys.stderr, 'Backend %s not found' % bname
return 1
- if not backend.has_caps(ICapMessagesReply):
- print >>sys.stderr, 'The backend %s does not implement ICapMessagesReply' % bname
+ if not backend.has_caps(ICapMessagesPost):
+ print >>sys.stderr, 'The backend %s does not implement ICapMessagesPost' % bname
return 1
thread_id, msg_id = id.rsplit('.', 1)
+ thread = Thread(thread_id)
+ message = Message(thread,
+ 0,
+ title=title,
+ sender=None,
+ receiver=None,
+ parent=Message(thread, msg_id),
+ content=content)
try:
- backend.post_reply(thread_id, msg_id, title, content)
+ backend.post_message(message)
except Exception, e:
content = u'Unable to send message to %s:\n' % thread_id
content += '\n\t%s\n' % e
if logging.root.level == logging.DEBUG:
content += '\n%s\n' % get_backtrace(e)
- self.send_email(backend, Message(thread_id,
+ self.send_email(backend, Message(thread,
0,
title='Unable to send message',
sender='Monboob',
- parent_message_id=msg_id,
+ parent=Message(thread, msg_id),
content=content))
@ConsoleApplication.command("run daemon")
@@ -183,22 +191,23 @@ class Monboob(ConsoleApplication):
self.weboob.loop()
def process(self):
- for backend, message in self.weboob.do('iter_new_messages'):
+ for backend, message in self.weboob.do('iter_unread_messages'):
self.send_email(backend, message)
+ backend.set_message_read(message)
def send_email(self, backend, mail):
domain = self.config.get('domain')
recipient = self.config.get('recipient')
reply_id = ''
- if mail.parent_id:
- reply_id = u'<%s.%s@%s>' % (backend.name, mail.parent_id, domain)
+ if mail.parent:
+ reply_id = u'<%s.%s@%s>' % (backend.name, mail.parent.full_id, domain)
subject = mail.title
sender = u'"%s" <%s@%s>' % (mail.sender.replace('"', '""'), backend.name, domain)
# assume that .date is an UTC datetime
date = formatdate(time.mktime(utc2local(mail.date).timetuple()), localtime=True)
- msg_id = u'<%s.%s@%s>' % (backend.name, mail.id, domain)
+ msg_id = u'<%s.%s@%s>' % (backend.name, mail.full_id, domain)
if int(self.config.get('html')) and mail.flags & mail.IS_HTML:
body = mail.content
diff --git a/weboob/backends/aum/backend.py b/weboob/backends/aum/backend.py
index 0339cb45..029f8f34 100644
--- a/weboob/backends/aum/backend.py
+++ b/weboob/backends/aum/backend.py
@@ -19,16 +19,16 @@ from __future__ import with_statement
from datetime import datetime
from dateutil import tz
-from logging import warning
-from time import sleep
+from logging import warning, debug
from weboob.capabilities.chat import ICapChat
-from weboob.capabilities.messages import ICapMessages, ICapMessagesReply, Message
+from weboob.capabilities.messages import ICapMessages, ICapMessagesPost, Message, Thread
from weboob.capabilities.dating import ICapDating, StatusField
from weboob.capabilities.contact import ICapContact, Contact, ProfileNode
from weboob.tools.backend import BaseBackend
from weboob.tools.browser import BrowserUnavailable
+from .captcha import CaptchaError
from .browser import AuMBrowser
from .exceptions import AdopteWait
from .optim.profiles_walker import ProfilesWalker
@@ -38,7 +38,7 @@ from .optim.visibility import Visibility
__all__ = ['AuMBackend']
-class AuMBackend(BaseBackend, ICapMessages, ICapMessagesReply, ICapDating, ICapChat, ICapContact):
+class AuMBackend(BaseBackend, ICapMessages, ICapMessagesPost, ICapDating, ICapChat, ICapContact):
NAME = 'aum'
MAINTAINER = 'Romain Bignon'
EMAIL = 'romain@peerfuse.org'
@@ -54,17 +54,27 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesReply, ICapDating, ICapC
}
BROWSER = AuMBrowser
+ MAGIC_ID_BASKET = 1
+
def create_default_browser(self):
if self.config['register']:
- browser = self.create_browser(self.config['username'])
- browser.register(password= self.config['password'],
- sex= 0,
- birthday_d= 1,
- birthday_m= 1,
- birthday_y= 1970,
- zipcode= 75001,
- country= 'fr',
- godfather= '')
+ browser = None
+ while not browser:
+ try:
+ browser = self.create_browser(self.config['username'])
+ browser.register(password= self.config['password'],
+ sex= 0,
+ birthday_d= 1,
+ birthday_m= 1,
+ birthday_y= 1970,
+ zipcode= 75001,
+ country= 'fr',
+ godfather= '')
+ except CaptchaError:
+ debug('Unable to resolve captcha. Retrying...')
+ browser = None
+ browser.password = self.config['password']
+ return browser
else:
return self.create_browser(self.config['username'], self.config['password'])
@@ -79,16 +89,117 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesReply, ICapDating, ICapC
except AdopteWait:
return (StatusField('notice', '', u'
You are currently waiting 1am to be able to connect with this account
', StatusField.FIELD_HTML|StatusField.FIELD_TEXT))
- def iter_messages(self, thread=None):
- for message in self._iter_messages(thread, False):
- yield message
+ def iter_threads(self):
+ with self.browser:
+ contacts = self.browser.get_threads_list()
- def iter_new_messages(self, thread=None):
- for message in self._iter_messages(thread, True):
- yield message
+ for contact in contacts:
+ thread = Thread(contact.get_id())
+ yield thread
+
+ def get_thread(self, id):
+ thread = None
+ if isinstance(id, Thread):
+ thread = id
+ id = thread.id
+
+ if not thread:
+ thread = Thread(id)
+ full = False
+ else:
+ full = True
+
+ with self.browser:
+ mails = self.browser.get_thread_mails(id, full)
+ my_name = self.browser.get_my_name()
+
+ child = None
+ msg = None
+ slut = self._get_slut(id)
+ for mail in mails:
+ flags = 0
+ if mail.date > slut['lastmsg']:
+ flags |= Message.IS_UNREAD
+ if mail.sender != my_name:
+ if mail.new:
+ flags |= Message.IS_NOT_ACCUSED
+ else:
+ flags |= Message.IS_ACCUSED
+
+ msg = Message(thread=thread,
+ id=mail.message_id,
+ title=mail.title,
+ sender=mail.sender,
+ receiver=mail.name if mail.sender == my_name else my_name, # TODO: me
+ date=mail.date,
+ content=mail.content,
+ signature=mail.signature,
+ children=[],
+ flags=flags)
+ if child:
+ msg.children.append(child)
+ child.parent = msg
+
+ child = msg
+
+ if full:
+ # If we have get all the messages, replace NotLoaded with None as
+ # parent.
+ msg.parent = None
+ thread.root = msg
+
+ return thread
+
+ def iter_unread_messages(self, thread=None):
+ with self.browser:
+ contacts = self.browser.get_threads_list()
+ for contact in contacts:
+ slut = self._get_slut(contact.get_id())
+ if contact.get_lastmsg_date() > slut['lastmsg']:
+ thread = self.get_thread(contact.get_id())
+ for m in thread.iter_all_messages():
+ if m.flags & m.IS_UNREAD:
+ yield m
+
+ # Send mail when someone added me in her basket.
+ # XXX possibly race condition if a slut adds me in her basket
+ # between the aum.nb_new_baskets() and aum.get_baskets().
+ with self.browser:
+ new_baskets = self.browser.nb_new_baskets()
+ if new_baskets:
+ ids = self.browser.get_baskets()
+ while new_baskets > 0 and len(ids) > new_baskets:
+ new_baskets -= 1
+ profile = self.browser.get_profile(ids[new_baskets])
+
+ thread = Thread(profile.get_id())
+ thread.root = Message(thread=thread,
+ id=self.MAGIC_ID_BASKET,
+ title='Basket of %s' % profile.get_name(),
+ sender=profile.get_name(),
+ receiver=self.browser.get_my_name(),
+ date=None, # now
+ content='You are taken in her basket!',
+ signature=profile.get_profile_text(),
+ children=[],
+ flags=Message.IS_UNREAD)
+ yield thread.root
+
+ def set_message_read(self, message):
+ if message.id == self.MAGIC_ID_BASKET:
+ # We don't save baskets.
+ return
+
+ slut = self._get_slut(message.thread.id)
+ if slut['lastmsg'] < message.date:
+ slut['lastmsg'] = message.date
+ #slut['msgstatus'] = contact.get_status()
+ self.storage.set('sluts', message.thread.id, slut)
+ self.storage.save()
def _get_slut(self, id):
- if not id in self.storage.get('sluts'):
+ sluts = self.storage.get('sluts')
+ if not sluts or not id in sluts:
slut = {'lastmsg': datetime(1970,1,1),
'msgstatus': ''}
else:
@@ -97,81 +208,9 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesReply, ICapDating, ICapC
slut['lastmsg'] = slut['lastmsg'].replace(tzinfo=tz.tzutc())
return slut
- def _iter_messages(self, thread, only_new):
+ def post_message(self, message):
with self.browser:
- try:
- profiles = {}
-
- if thread:
- slut = self._get_slut(int(thread))
- for mail in self._iter_thread_messages(thread, only_new, slut['lastmsg'], {}):
- if slut['lastmsg'] < mail.date:
- slut['lastmsg'] = mail.date
- yield mail
-
- self.storage.set('sluts', int(thread), slut)
- self.storage.save()
- else:
- contacts = self.browser.get_threads_list()
- for contact in contacts:
- slut = self._get_slut(contact.get_id())
- last_msg = slut['lastmsg']
-
- if only_new and contact.get_lastmsg_date() < last_msg and contact.get_status() == slut['msgstatus'] or \
- not thread is None and int(thread) != contact.get_id():
- continue
-
- for mail in self._iter_thread_messages(contact.get_id(), only_new, last_msg, profiles):
- if last_msg < mail.date:
- last_msg = mail.date
-
- yield mail
-
- slut['lastmsg'] = last_msg
- slut['msgstatus'] = contact.get_status()
- self.storage.set('sluts', contact.get_id(), slut)
- self.storage.save()
-
- # Send mail when someone added me in her basket.
- # XXX possibly race condition if a slut adds me in her basket
- # between the aum.nbNewBaskets() and aum.getBaskets().
- new_baskets = self.browser.nb_new_baskets()
- if new_baskets:
- ids = self.browser.get_baskets()
- while new_baskets > 0 and len(ids) > new_baskets:
- new_baskets -= 1
- profile = self.browser.get_profile(ids[new_baskets])
-
- yield Message(profile.get_id(), 1,
- title='Basket of %s' % profile.get_name(),
- sender=profile.get_name(),
- content='You are taken in her basket!',
- signature=profile.get_profile_text())
- except BrowserUnavailable:
- pass
-
- def _iter_thread_messages(self, id, only_new, last_msg, profiles):
- mails = self.browser.get_thread_mails(id)
- for mail in mails:
- if only_new and mail.date <= last_msg:
- continue
-
- if not mail.profile_link in profiles:
- profiles[mail.profile_link] = self.browser.get_profile(mail.profile_link)
- mail.signature += u'\n%s' % profiles[mail.profile_link].get_profile_text()
-
- yield mail
-
- def post_reply(self, thread_id, reply_id, title, message):
- while 1:
- try:
- with self.browser:
- self.browser.post_mail(thread_id, message)
- except AdopteWait:
- # If we are on a waiting state, retry every 30 minutes until it is posted.
- sleep(60*30)
- else:
- return
+ self.browser.post_mail(message.thread.id, message.content)
def get_contact(self, contact):
try:
@@ -268,4 +307,8 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesReply, ICapDating, ICapC
data = self.browser.openurl(photo.thumbnail_url).read()
contact.set_photo(name, thumbnail_data=data)
- OBJECTS = {Contact: fill_contact}
+ def fill_thread(self, thread, fields):
+ return self.get_thread(thread)
+
+ OBJECTS = {Thread: fill_thread,
+ Contact: fill_contact}
diff --git a/weboob/backends/aum/browser.py b/weboob/backends/aum/browser.py
index 2f348cdd..8b486c42 100644
--- a/weboob/backends/aum/browser.py
+++ b/weboob/backends/aum/browser.py
@@ -108,7 +108,7 @@ class AuMBrowser(BaseBrowser):
if not self.is_on_page(RegisterPage):
self.location('http://www.adopteunmec.com/register2.php')
- return self.page.register(password, sex, birthday_d, birthday_m, birthday_y, zipcode, country, godfather)
+ return self.page.register(password, sex, birthday_d, birthday_m, birthday_y, zipcode, country)
@pageaccess
def add_photo(self, name, f):
@@ -198,9 +198,9 @@ class AuMBrowser(BaseBrowser):
return self.page.get_contact_list()
@pageaccess
- def get_thread_mails(self, id):
+ def get_thread_mails(self, id, full=False):
if not self.is_on_page(ContactThreadPage) or self.page.id != int(id):
- self.page.open_thread_page(id)
+ self.page.open_thread_page(id, full)
return self.page.mails
@pageaccess
diff --git a/weboob/backends/aum/captcha.py b/weboob/backends/aum/captcha.py
index 7c1bf900..f9a8aa9e 100644
--- a/weboob/backends/aum/captcha.py
+++ b/weboob/backends/aum/captcha.py
@@ -103,6 +103,8 @@ class Tile:
try:
return self.hash[self.checksum()]
except KeyError:
+ print 'Unable te resolve:'
+ self.display()
raise CaptchaError()
class Captcha:
diff --git a/weboob/backends/aum/pages/contact_thread.py b/weboob/backends/aum/pages/contact_thread.py
index 3ffc9071..4273edcb 100644
--- a/weboob/backends/aum/pages/contact_thread.py
+++ b/weboob/backends/aum/pages/contact_thread.py
@@ -16,6 +16,7 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
import re
+import time
from datetime import datetime
from dateutil import tz
from logging import error, warning
@@ -23,9 +24,8 @@ from mechanize import FormNotFoundError
from weboob.backends.aum.pages.base import PageBase
from weboob.backends.aum.exceptions import AdopteCantPostMail
-from weboob.capabilities.messages import Message
-class MailParser(Message):
+class MailParser(object):
"""
@@ -93,8 +93,14 @@ class MailParser(Message):
def __init__(self, thread_id, name, tr):
# | implicit
- Message.__init__(self, thread_id, 0, 'Discussion with %s' % name, name)
+ self.thread_id = thread_id
+ self.date = None
+ self.message_id = 0
+ self.title = 'Discussion with %s' % name
+ self.sender = name
+ self.name = name
self.tr = tr.childNodes[0].childNodes[1].childNodes[0].childNodes[0]
+ self.new = False
tds = self.tr.childNodes
@@ -131,6 +137,10 @@ class MailParser(Message):
self.parse_profile_link()
self.parse_from()
+ @property
+ def date_int(self):
+ return int(time.strftime('%Y%m%d%H%M%S', self.date.timetuple()))
+
def set_parent_message_id(self, date):
self.parent_message_id = date
@@ -157,7 +167,7 @@ class MailParser(Message):
self.id = '%s.%s' % (self.thread_id, self.message_id)
if m.group(7).find('nouveau') >= 0:
- self.flags |= self.IS_UNREAD
+ self.new = True
else:
error('Error: unable to parse the datetime string "%s"' % date_str)
diff --git a/weboob/backends/aum/pages/login.py b/weboob/backends/aum/pages/login.py
index 68a6d3b2..dc39e20c 100644
--- a/weboob/backends/aum/pages/login.py
+++ b/weboob/backends/aum/pages/login.py
@@ -16,6 +16,8 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+import re
+
from weboob.tools.browser import BrowserIncorrectPassword
from .base import PageBase
@@ -29,7 +31,29 @@ class LoginPage(PageBase):
self.browser.submit() # submit current form
+class RegisterError(Exception):
+ pass
+
class RegisterPage(PageBase):
+ def on_loaded(self):
+ display_errors = False
+ for div in self.document.getElementsByTagName('div'):
+ if div.getAttribute('class') == 'balloon':
+ display_errors = True
+ break
+
+ if not display_errors:
+ return
+
+ errors = []
+ for script in self.document.getElementsByTagName('script'):
+ for child in script.childNodes:
+ if child and child.data.find('dispErrors') >= 0:
+ for m in re.finditer('"(\w+)": "(.*)",', child.data):
+ errors.append(m.group(2))
+
+ raise RegisterError(u'Unable to register account: %s' % ', '.join(errors))
+
def register(self, password, sex, birthday_d, birthday_m, birthday_y, zipcode, country):
"""
Form name=register (#1)
@@ -53,7 +77,7 @@ Form name=register (#1)
self.browser.select_form(name='register')
self.browser.set_all_readonly(False)
- self.browser['sex'] = str(sex)
+ self.browser['sex'] = [str(sex)]
self.browser['birthday0'] = [str(birthday_d)]
self.browser['birthday1'] = [str(birthday_m)]
self.browser['birthday2'] = [str(birthday_y)]
diff --git a/weboob/backends/dlfp/backend.py b/weboob/backends/dlfp/backend.py
index 760efcd2..ecc92fe6 100644
--- a/weboob/backends/dlfp/backend.py
+++ b/weboob/backends/dlfp/backend.py
@@ -18,8 +18,7 @@
from __future__ import with_statement
from weboob.tools.backend import BaseBackend
-from weboob.tools.browser import BrowserUnavailable
-from weboob.capabilities.messages import ICapMessages, ICapMessagesReply, Message
+from weboob.capabilities.messages import ICapMessages, ICapMessagesPost, Message, Thread
from .feeds import ArticlesList
from .browser import DLFP
@@ -28,7 +27,7 @@ from .browser import DLFP
__all__ = ['DLFPBackend']
-class DLFPBackend(BaseBackend, ICapMessages, ICapMessagesReply):
+class DLFPBackend(BaseBackend, ICapMessages, ICapMessagesPost):
NAME = 'dlfp'
MAINTAINER = 'Romain Bignon'
EMAIL = 'romain@peerfuse.org'
@@ -46,75 +45,95 @@ class DLFPBackend(BaseBackend, ICapMessages, ICapMessagesReply):
def create_default_browser(self):
return self.create_browser(self.config['username'], self.config['password'])
- def iter_messages(self, thread=None):
- return self._iter_messages(thread, False)
-
- def iter_new_messages(self, thread=None):
- return self._iter_messages(thread, True)
-
- def _iter_messages(self, thread, only_new):
+ def iter_threads(self):
+ whats = set()
if self.config['get_news']:
- for message in self._iter_messages_of('newspaper', thread, only_new):
- yield message
+ whats.add('newspaper')
if self.config['get_telegrams']:
- for message in self._iter_messages_of('telegram', thread, only_new):
- yield message
+ whats.add('telegram')
- def _iter_messages_of(self, what, thread_wanted, only_new):
- if not what in self.storage.get('seen', default={}):
- self.storage.set('seen', what, {})
+ for what in whats:
+ for article in ArticlesList(what).iter_articles():
+ thread = Thread(article.id)
+ thread.title = article.title
+ yield thread
- seen = {}
- for article in ArticlesList(what).iter_articles():
- if thread_wanted and thread_wanted != article.id:
- continue
+ def get_thread(self, id):
+ if isinstance(id, Thread):
+ thread = id
+ id = thread.id
- flags = Message.IS_HTML
- if not article.id in self.storage.get('seen', what, default={}):
- seen[article.id] = {'comments': []}
- flags |= Message.IS_NEW
- else:
- seen[article.id] = self.storage.get('seen', what, article.id, default={})
+ with self.browser:
+ content = self.browser.get_content(id)
- try:
- with self.browser:
- thread = self.browser.get_content(article.id)
- except BrowserUnavailable:
- continue
+ if not thread:
+ thread = Thread(id)
- if not only_new or flags & Message.IS_NEW:
- yield Message(thread.id,
- 0,
- thread.title,
- thread.author,
- article.datetime,
- content=''.join([thread.body, thread.part2]),
- signature='URL: %s' % article.url,
+ flags = Message.IS_HTML
+ if not thread.id in self.storage.get('seen', default={}):
+ flags |= Message.IS_UNREAD
+
+ thread.title = content.title
+ if not thread.date:
+ thread.date = content.date
+
+ thread.root = Message(thread=thread,
+ id=0, # root message
+ title=content.title,
+ sender=content.author,
+ receiver=None,
+ date=thread.date, #TODO XXX WTF this is None
+ parent=None,
+ content=''.join([content.body, content.part2]),
+ signature='URL: %s' % content.url,
+ children=[],
flags=flags)
- for comment in thread.iter_all_comments():
- flags = Message.IS_HTML
- if not comment.id in seen[article.id]['comments']:
- seen[article.id]['comments'].append(comment.id)
- flags |= Message.IS_NEW
+ for com in content.comments:
+ self._insert_comment(com, thread.root)
- if not only_new or flags & Message.IS_NEW:
- yield Message(thread.id,
- comment.id,
- comment.title,
- comment.author,
- comment.date,
- comment.reply_id,
- comment.body,
- 'Score: %d' % comment.score,
- flags)
+ return thread
- # If there is no articles seen, it's suspicious, probably I can't
- # fetch the feed.
- if seen:
- self.storage.set('seen', what, seen)
- self.storage.save()
+ def _insert_comment(self, com, parent):
+ """"
+ Insert 'com' comment and its children in the parent message.
+ """
+ flags = Message.IS_HTML
+ if not com.id in self.storage.get('seen', parent.thread.id, 'comments', default=[]):
+ flags |= Message.IS_UNREAD
- def post_reply(self, thread_id, reply_id, title, message):
+ message = Message(thread=parent.thread,
+ id=com.id,
+ title=com.title,
+ sender=com.author,
+ receiver=None,
+ date=com.date,
+ parent=parent,
+ content=com.body,
+ signature='Score: %d' % com.score,
+ children=[],
+ flags=flags)
+
+ parent.children.append(message)
+ for sub in com.comments:
+ self._insert_comment(sub, message)
+
+ def iter_unread_messages(self, thread=None):
+ for thread in self.iter_threads():
+ self.fill_thread(thread, 'root')
+ for m in thread.iter_all_messages():
+ if m.flags & m.IS_UNREAD:
+ yield m
+
+ def set_message_read(self, message):
+ self.storage.set('seen', message.thread.id, 'comments', self.storage.get('seen', message.thread.id, 'comments', default=[]) + [message.id])
+ self.storage.save()
+
+ def post_mesage(self, message):
with self.browser:
- return self.browser.post_reply(thread_id, reply_id, title, message)
+ return self.browser.post_reply(message.thread.id, message.parent.id, message.title, message.content)
+
+ def fill_thread(self, thread, fields):
+ return self.get_thread(thread)
+
+ OBJECTS = {Thread: fill_thread}
diff --git a/weboob/backends/dlfp/browser.py b/weboob/backends/dlfp/browser.py
index 6c436f98..e114d3f1 100644
--- a/weboob/backends/dlfp/browser.py
+++ b/weboob/backends/dlfp/browser.py
@@ -86,3 +86,4 @@ class DLFP(BaseBrowser):
def is_logged(self):
return (self.page and self.page.is_logged())
+
diff --git a/weboob/backends/dlfp/pages/news.py b/weboob/backends/dlfp/pages/news.py
index 57161554..51b0fd94 100644
--- a/weboob/backends/dlfp/pages/news.py
+++ b/weboob/backends/dlfp/pages/news.py
@@ -31,7 +31,7 @@ class Comment(object):
self.reply_id = reply_id
self.title = u''
self.author = u''
- self.date = u''
+ self.date = None
self.body = u''
self.score = 0
self.comments = []
@@ -73,7 +73,8 @@ class Article(object):
self.author = u''
self.body = u''
self.part2 = u''
- self.date = u''
+ self.date = None
+ self.url = u''
self.comments = []
for div in tree.findall('div'):
@@ -116,6 +117,7 @@ class ContentPage(DLFPPage):
def parse_div(self, div):
if div.attrib.get('class', '') in ('newsdiv', 'centraldiv'):
self.article = Article(self.browser, url2id(self.url), div)
+ self.article.url = self.url
if div.attrib.get('class', '') == 'articlediv':
self.article.parse_part2(div)
if div.attrib.get('class', '') == 'comments':
diff --git a/weboob/backends/dlfp/test.py b/weboob/backends/dlfp/test.py
index c0840823..b9e11915 100644
--- a/weboob/backends/dlfp/test.py
+++ b/weboob/backends/dlfp/test.py
@@ -18,9 +18,13 @@
from weboob.tools.test import BackendTest
+
+__all__ = ['DLFPTest']
+
+
class DLFPTest(BackendTest):
BACKEND = 'dlfp'
def test_new_messages(self):
- for message in self.backend.iter_new_messages():
+ for message in self.backend.iter_unread_messages():
pass
diff --git a/weboob/backends/fourchan/backend.py b/weboob/backends/fourchan/backend.py
index bf9da255..f39bf07a 100644
--- a/weboob/backends/fourchan/backend.py
+++ b/weboob/backends/fourchan/backend.py
@@ -20,7 +20,7 @@ from __future__ import with_statement
from logging import warning
-from weboob.capabilities.messages import ICapMessages, Message
+from weboob.capabilities.messages import ICapMessages, Message, Thread
from weboob.tools.backend import BaseBackend
from .browser import FourChan
@@ -41,71 +41,89 @@ class FourChanBackend(BaseBackend, ICapMessages):
STORAGE = {'boards': {}}
BROWSER = FourChan
- def iter_messages(self, thread=None):
- return self._iter_messages(thread, False)
+ def _splitid(self, id):
+ return id.split('.', 1)
- def iter_new_messages(self, thread=None):
- return self._iter_messages(thread, True)
+ def get_thread(self, id):
+ thread = None
- def _iter_messages(self, thread, only_new):
- if thread:
- if '.' in thread:
- board, thread = thread.split('.', 2)
- return self._iter_messages_of(board, thread, only_new)
- else:
- warning('"%s" is not a valid ID' % thread)
- else:
- for board in self.config['boards'].split(' '):
- return self._iter_messages_of(board, None, only_new)
+ if isinstance(id, Thread):
+ thread = id
+ id = thread.id
- def _iter_messages_of(self, board, thread_wanted, only_new):
- if not board in self.storage.get('boards', default={}):
- self.storage.set('boards', board, {})
+ if not '.' in id:
+ warning('Malformated ID (%s)' % id)
+ return
- if thread_wanted:
- for message in self._iter_thread_messages(board, thread_wanted, only_new):
- yield message
- else:
+ board, thread_id = self._splitid(id)
+
+ with self.browser:
+ _thread = self.browser.get_thread(board, thread_id)
+
+ flags = 0
+ if not _thread.id in self.storage.get('boards', board, default={}):
+ flags |= Message.IS_UNREAD
+
+ if not thread:
+ thread = Thread(id)
+ thread.title = _thread.filename
+ thread.root = Message(thread=thread,
+ id=0, # root message
+ title=_thread.filename,
+ sender=_thread.author,
+ receiver=None,
+ date=_thread.datetime,
+ parent=None,
+ content=_thread.text,
+ signature=None,
+ children=None,
+ flags=flags|Message.IS_HTML)
+
+ parent = thread.root
+ for comment in _thread.comments:
+ flags = 0
+ if not comment.id in self.storage.get('boards', board, _thread.id, default=[]):
+ flags |= Message.IS_UNREAD
+
+ m = Message(thread=thread,
+ id=comment.id,
+ title=_thread.filename,
+ sender=comment.author,
+ receiver=None,
+ date=comment.datetime,
+ parent=parent,
+ content=comment.text,
+ signature=None,
+ children=None,
+ flags=flags|Message.IS_HTML)
+ parent.children = [m]
+ parent = m
+
+ return thread
+
+ def iter_threads(self):
+ for board in self.config['boards'].split(' '):
with self.browser:
threads = self.browser.get_threads(board)
for thread in threads:
- for message in self._iter_thread_messages(board, thread.id, only_new):
- yield message
+ t = Thread('%s.%s' % (board, thread.id))
+ t.title = thread.filename
+ yield t
- def _iter_thread_messages(self, board, thread, only_new):
- thread = self.browser.get_thread(board, thread)
+ def iter_unread_messages(self):
+ for thread in self.iter_threads():
+ self.fill_thread(thread, 'root')
- flags = Message.IS_HTML
- if thread.id in self.storage.get('boards', board, default={}):
- self.storage.set('boards', board, thread.id, [])
- flags |= Message.IS_NEW
-
- if not only_new or flags & Message.IS_NEW:
- yield Message('%s.%s' % (board, thread.id),
- 0,
- thread.filename,
- thread.author,
- thread.datetime,
- content=thread.text,
- flags=flags)
-
- for comment in thread.comments:
- flags = Message.IS_HTML
- if not comment.id in self.storage.get('boards', board, thread.id, default=[]):
- self.storage.set('boards', board, thread.id, self.storage.get('boards', board, thread.id, default=[]) + [comment.id])
- flags |= Message.IS_NEW
-
- if not only_new or flags & Message.IS_NEW:
- yield Message('%s.%s' % (board, thread.id),
- comment.id,
- thread.filename,
- comment.author,
- comment.datetime,
- 0,
- comment.text,
- flags)
+ for m in thread.iter_all_messages():
+ if m.flags & Message.IS_UNREAD:
+ yield m
+ def set_message_read(self, message):
+ board, thread_id = self._splitid(message.thread.id)
+ self.storage.set('boards', board, thread_id, self.storage.get('boards', board, thread_id, default=[]) + [message.id])
self.storage.save()
- #def post_reply(self, thread_id, reply_id, title, message):
- # return self.browser.post_reply(thread_id, reply_id, title, message)
+ def fill_thread(self, thread, fields):
+ return self.get_thread(thread)
+
+ OBJECTS = {Thread: fill_thread}
diff --git a/weboob/backends/fourchan/browser.py b/weboob/backends/fourchan/browser.py
index 3722f123..5e6c86b3 100644
--- a/weboob/backends/fourchan/browser.py
+++ b/weboob/backends/fourchan/browser.py
@@ -34,7 +34,7 @@ class FourChan(BaseBrowser):
return self.page.articles
def get_thread(self, board, id):
- self.location('http://boards.4chan.org/%s/res/%d' % (board, id))
+ self.location('http://boards.4chan.org/%s/res/%d' % (board, long(id)))
assert len(self.page.articles) == 1
return self.page.articles[0]
|