From 50d0c9a728d12b4e19ec5876d76b6bde3e145ae0 Mon Sep 17 00:00:00 2001 From: Romain Bignon Date: Fri, 13 Aug 2010 14:38:42 +0200 Subject: [PATCH] enhancement of the ICapMessages.Message object --- weboob/applications/monboob/monboob.py | 38 ++++++------ weboob/backends/aum/backend.py | 10 ++-- weboob/backends/aum/browser.py | 6 +- weboob/backends/aum/pages/contact_thread.py | 31 ++++------ weboob/backends/dlfp/backend.py | 21 ++++--- weboob/backends/fourchan/backend.py | 20 +++---- weboob/capabilities/messages.py | 66 ++++++++------------- weboob/tools/backend.py | 1 - 8 files changed, 82 insertions(+), 111 deletions(-) diff --git a/weboob/applications/monboob/monboob.py b/weboob/applications/monboob/monboob.py index 4c89fae9..b0a382d6 100644 --- a/weboob/applications/monboob/monboob.py +++ b/weboob/applications/monboob/monboob.py @@ -174,7 +174,7 @@ class Monboob(ConsoleApplication): 0, title='Unable to send message', sender='Monboob', - reply_id=msg_id, + parent_message_id=msg_id, content=content)) @ConsoleApplication.command("run daemon") @@ -191,34 +191,34 @@ class Monboob(ConsoleApplication): recipient = self.config.get('recipient') reply_id = '' - if mail.get_reply_id(): - reply_id = u'<%s.%s@%s>' % (backend.name, mail.get_full_reply_id(), domain) - subject = mail.get_title() - sender = u'"%s" <%s@%s>' % (mail.get_from().replace('"', '""'), backend.name, domain) + if mail.parent_id: + reply_id = u'<%s.%s@%s>' % (backend.name, mail.parent_id, domain) + subject = mail.title + sender = u'"%s" <%s@%s>' % (mail.sender.replace('"', '""'), backend.name, domain) - # assume that get_date() returns an UTC datetime - date = formatdate(time.mktime(utc2local(mail.get_date()).timetuple()), localtime=True) - msg_id = u'<%s.%s@%s>' % (backend.name, mail.get_full_id(), domain) + # assume that .date is an UTC datetime + date = formatdate(time.mktime(utc2local(mail.date).timetuple()), localtime=True) + msg_id = u'<%s.%s@%s>' % (backend.name, mail.id, domain) - if int(self.config.get('html')) and mail.is_html: - body = mail.get_content() + if int(self.config.get('html')) and mail.flags & mail.IS_HTML: + body = mail.content content_type = 'html' else: - if mail.is_html: - body = html2text(mail.get_content()) + if mail.flags & mail.IS_HTML: + body = html2text(mail.content) else: - body = mail.get_content() + body = mail.content content_type = 'plain' - if mail.get_signature(): - if int(self.config.get('html')) and mail.is_html: - body += u'

--
%s

' % mail.get_signature() + if mail.signature: + if int(self.config.get('html')) and mail.flags & mail.IS_HTML: + body += u'

--
%s

' % mail.signature else: body += u'\n\n-- \n' - if mail.is_html: - body += html2text(mail.get_signature()) + if mail.flags & mail.IS_HTML: + body += html2text(mail.signature) else: - body += mail.get_signature() + body += mail.signature # Header class is smart enough to try US-ASCII, then the charset we # provide, then fall back to UTF-8. diff --git a/weboob/backends/aum/backend.py b/weboob/backends/aum/backend.py index d7717e40..0339cb45 100644 --- a/weboob/backends/aum/backend.py +++ b/weboob/backends/aum/backend.py @@ -105,8 +105,8 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesReply, ICapDating, ICapC if thread: slut = self._get_slut(int(thread)) for mail in self._iter_thread_messages(thread, only_new, slut['lastmsg'], {}): - if slut['lastmsg'] < mail.get_date(): - slut['lastmsg'] = mail.get_date() + if slut['lastmsg'] < mail.date: + slut['lastmsg'] = mail.date yield mail self.storage.set('sluts', int(thread), slut) @@ -122,8 +122,8 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesReply, ICapDating, ICapC continue for mail in self._iter_thread_messages(contact.get_id(), only_new, last_msg, profiles): - if last_msg < mail.get_date(): - last_msg = mail.get_date() + if last_msg < mail.date: + last_msg = mail.date yield mail @@ -153,7 +153,7 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesReply, ICapDating, ICapC def _iter_thread_messages(self, id, only_new, last_msg, profiles): mails = self.browser.get_thread_mails(id) for mail in mails: - if only_new and mail.get_date() <= last_msg: + if only_new and mail.date <= last_msg: continue if not mail.profile_link in profiles: diff --git a/weboob/backends/aum/browser.py b/weboob/backends/aum/browser.py index fca349cb..f7ce8033 100644 --- a/weboob/backends/aum/browser.py +++ b/weboob/backends/aum/browser.py @@ -199,13 +199,13 @@ class AuMBrowser(BaseBrowser): @pageaccess def get_thread_mails(self, id): - if not self.is_on_page(ContactThreadPage) or self.page.get_id() != int(id): + if not self.is_on_page(ContactThreadPage) or self.page.id != int(id): self.page.open_thread_page(id) - return self.page.get_mails() + return self.page.mails @pageaccess def post_mail(self, id, content): - if not self.is_on_page(ContactThreadPage) or self.page.get_id() != int(id): + if not self.is_on_page(ContactThreadPage) or self.page.id != int(id): self.page.open_thread_page(id) self.page.post(content) diff --git a/weboob/backends/aum/pages/contact_thread.py b/weboob/backends/aum/pages/contact_thread.py index d19d1612..3ffc9071 100644 --- a/weboob/backends/aum/pages/contact_thread.py +++ b/weboob/backends/aum/pages/contact_thread.py @@ -91,9 +91,9 @@ class MailParser(Message): 10: '(L)', } - def __init__(self, id, name, tr): + def __init__(self, thread_id, name, tr): # implicit - Message.__init__(self, id, 0, 'Discussion with %s' % name, name) + Message.__init__(self, thread_id, 0, 'Discussion with %s' % name, name) self.tr = tr.childNodes[0].childNodes[1].childNodes[0].childNodes[0] tds = self.tr.childNodes @@ -131,8 +131,8 @@ class MailParser(Message): self.parse_profile_link() self.parse_from() - def set_reply_id(self, date): - self.reply_id = date + def set_parent_message_id(self, date): + self.parent_message_id = date def parse_date(self, date_str): # To match regexp, we have to remove any return chars in string @@ -153,10 +153,11 @@ class MailParser(Message): d = d.astimezone(tz.tzutc()) # and get timestamp self.date = d - self.id = self.get_date_int() + self.message_id = self.date_int + self.id = '%s.%s' % (self.thread_id, self.message_id) if m.group(7).find('nouveau') >= 0: - self.is_new = True + self.flags |= self.IS_UNREAD else: error('Error: unable to parse the datetime string "%s"' % date_str) @@ -176,7 +177,7 @@ class MailParser(Message): self.signature = u'Profile: http://www.adopteunmec.com%s' % self.profile_link return - warning('Unable to find the profile URL in the message %s@%s' % (self.get_from(), self.get_id())) + warning('Unable to find the profile URL in the message %s@%s' % (self.sender, self.id)) def parse_from(self): tds = self.tr.getElementsByTagName('div') @@ -190,7 +191,7 @@ class MailParser(Message): return - warning('Warning: unable to find from in the mail %s' % self.get_id()) + warning('Warning: unable to find from in the mail %s' % self.id) class ContactThreadPage(PageBase): @@ -274,7 +275,7 @@ class ContactThreadPage(PageBase): id_regexp = re.compile("/thread.php\?id=([0-9]+)") def on_loaded(self): - self.items = [] + self.mails = [] a_list = self.document.getElementsByTagName('a') self.id = 0 @@ -313,12 +314,6 @@ class ContactThreadPage(PageBase): mail = MailParser(self.id, self.name, tag) - if self.items: - self.items[-1].set_reply_id(mail.get_date_int()) - self.items += [mail] - - def get_id(self): - return self.id - - def get_mails(self): - return self.items + if self.mails: + self.mails[-1].set_parent_message_id(mail.date_int) + self.mails += [mail] diff --git a/weboob/backends/dlfp/backend.py b/weboob/backends/dlfp/backend.py index b2c4b287..760efcd2 100644 --- a/weboob/backends/dlfp/backend.py +++ b/weboob/backends/dlfp/backend.py @@ -69,12 +69,12 @@ class DLFPBackend(BaseBackend, ICapMessages, ICapMessagesReply): if thread_wanted and thread_wanted != article.id: continue + flags = Message.IS_HTML if not article.id in self.storage.get('seen', what, default={}): seen[article.id] = {'comments': []} - new = True + flags |= Message.IS_NEW else: seen[article.id] = self.storage.get('seen', what, article.id, default={}) - new = False try: with self.browser: @@ -82,7 +82,7 @@ class DLFPBackend(BaseBackend, ICapMessages, ICapMessagesReply): except BrowserUnavailable: continue - if not only_new or new: + if not only_new or flags & Message.IS_NEW: yield Message(thread.id, 0, thread.title, @@ -90,16 +90,15 @@ class DLFPBackend(BaseBackend, ICapMessages, ICapMessagesReply): article.datetime, content=''.join([thread.body, thread.part2]), signature='URL: %s' % article.url, - is_html=True, - is_new=new) + flags=flags) for comment in thread.iter_all_comments(): + flags = Message.IS_HTML if not comment.id in seen[article.id]['comments']: seen[article.id]['comments'].append(comment.id) - new = True - else: - new = False - if not only_new or new: + flags |= Message.IS_NEW + + if not only_new or flags & Message.IS_NEW: yield Message(thread.id, comment.id, comment.title, @@ -108,8 +107,8 @@ class DLFPBackend(BaseBackend, ICapMessages, ICapMessagesReply): comment.reply_id, comment.body, 'Score: %d' % comment.score, - is_html=True, - is_new=new) + flags) + # If there is no articles seen, it's suspicious, probably I can't # fetch the feed. if seen: diff --git a/weboob/backends/fourchan/backend.py b/weboob/backends/fourchan/backend.py index f09e91c7..bf9da255 100644 --- a/weboob/backends/fourchan/backend.py +++ b/weboob/backends/fourchan/backend.py @@ -75,30 +75,27 @@ class FourChanBackend(BaseBackend, ICapMessages): def _iter_thread_messages(self, board, thread, only_new): thread = self.browser.get_thread(board, thread) + flags = Message.IS_HTML if thread.id in self.storage.get('boards', board, default={}): self.storage.set('boards', board, thread.id, []) - new = True - else: - new = False + flags |= Message.IS_NEW - if not only_new or new: + if not only_new or flags & Message.IS_NEW: yield Message('%s.%s' % (board, thread.id), 0, thread.filename, thread.author, thread.datetime, content=thread.text, - is_html=True, - is_new=new) + flags=flags) for comment in thread.comments: + flags = Message.IS_HTML if not comment.id in self.storage.get('boards', board, thread.id, default=[]): self.storage.set('boards', board, thread.id, self.storage.get('boards', board, thread.id, default=[]) + [comment.id]) - new = True - else: - new = False + flags |= Message.IS_NEW - if not only_new or new: + if not only_new or flags & Message.IS_NEW: yield Message('%s.%s' % (board, thread.id), comment.id, thread.filename, @@ -106,8 +103,7 @@ class FourChanBackend(BaseBackend, ICapMessages): comment.datetime, 0, comment.text, - is_html=True, - is_new=new) + flags) self.storage.save() diff --git a/weboob/capabilities/messages.py b/weboob/capabilities/messages.py index d77c22c0..2dd5def6 100644 --- a/weboob/capabilities/messages.py +++ b/weboob/capabilities/messages.py @@ -19,18 +19,24 @@ import datetime import time -from .base import IBaseCap +from .base import IBaseCap, CapBaseObject, NotLoaded __all__ = ['ICapMessages', 'ICapMessagesReply', 'Message'] -class Message(object): - def __init__(self, thread_id, _id, title, sender, date=None, reply_id=u'', - content=u'', signature=u'', is_html=False, is_new=False): +class Message(CapBaseObject): + IS_HTML = 0x001 + IS_NEW = 0x002 + IS_UNREAD = 0x004 + + def __init__(self, thread_id, message_id, title, sender, date=None, parent_message_id=u'', + content=u'', signature=u'', flags=0): + CapBaseObject.__init__(self, '%s.%s' % (thread_id, message_id)) + self.thread_id = unicode(thread_id) - self.id = unicode(_id) - self.reply_id = unicode(reply_id) + self.message_id = unicode(message_id) + self.parent_message_id = unicode(parent_message_id) self.title = unicode(title) self.sender = unicode(sender) self.signature = unicode(signature) @@ -39,48 +45,24 @@ class Message(object): if date is None: date = datetime.datetime.utcnow() self.date = date - self.is_new = is_new - self.is_html = is_html + self.flags = flags - def get_date_int(self): - return int(time.strftime('%Y%m%d%H%M%S', self.get_date().timetuple())) + @property + def date_int(self): + return int(time.strftime('%Y%m%d%H%M%S', self.date.timetuple())) - def get_full_id(self): - return '%s.%s' % (self.thread_id, self.id) - - def get_full_reply_id(self): - return '%s.%s' % (self.thread_id, self.reply_id) - - def get_id(self): - return self.id - - def get_thread_id(self): - return self.thread_id - - def get_reply_id(self): - return self.reply_id - - def get_title(self): - return self.title - - def get_date(self): - return self.date - - def get_from(self): - return self.sender - - def get_content(self): - return self.content - - def get_signature(self): - return self.signature + @property + def parent_id(self): + if not self.parent_message_id: + return '' + return '%s.%s' % (self.thread_id, self.parent_message_id) def __eq__(self, msg): - return self.id == msg.id and self.thread_id == msg.thread_id + return self.id == msg.id def __repr__(self): - result = '' % ( - self.thread_id, self.id, self.title, self.date, self.sender) + result = '' % ( + self.id, self.title, self.date, self.sender) return result.encode('utf-8') class ICapMessages(IBaseCap): diff --git a/weboob/tools/backend.py b/weboob/tools/backend.py index f9df23f5..236db2bc 100644 --- a/weboob/tools/backend.py +++ b/weboob/tools/backend.py @@ -132,7 +132,6 @@ class BaseBackend(object): elif isinstance(field.default, float) and not isinstance(value, float): value = float(value) - print value, field.choices if field.choices: if (isinstance(field.choices, (tuple,list)) and not value in field.choices) or \ (isinstance(field.choices, dict) and not value in field.choices.iterkeys()):