nojhan/weboob-devel
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

enhancement of the ICapMessages.Message object

Browse source
This commit is contained in:
Romain Bignon 2010-08-13 14:38:42 +02:00
commit 50d0c9a728
8 changed files with 82 additions and 111 deletions
Download patch file Download diff file Expand all files Collapse all files

38
weboob/applications/monboob/monboob.py
View file

@ -174,7 +174,7 @@ class Monboob(ConsoleApplication):
0, 0,
title='Unable to send message', title='Unable to send message',
sender='Monboob', sender='Monboob',
reply_id=msg_id, parent_message_id=msg_id,
content=content)) content=content))
@ConsoleApplication.command("run daemon") @ConsoleApplication.command("run daemon")
@ -191,34 +191,34 @@ class Monboob(ConsoleApplication):
recipient = self.config.get('recipient') recipient = self.config.get('recipient')
reply_id = '' reply_id = ''
if mail.get_reply_id(): if mail.parent_id:
reply_id = u'<%s.%s@%s>' % (backend.name, mail.get_full_reply_id(), domain) reply_id = u'<%s.%s@%s>' % (backend.name, mail.parent_id, domain)
subject = mail.get_title() subject = mail.title
sender = u'"%s" <%s@%s>' % (mail.get_from().replace('"', '""'), backend.name, domain) sender = u'"%s" <%s@%s>' % (mail.sender.replace('"', '""'), backend.name, domain)
# assume that get_date() returns an UTC datetime # assume that .date is an UTC datetime
date = formatdate(time.mktime(utc2local(mail.get_date()).timetuple()), localtime=True) date = formatdate(time.mktime(utc2local(mail.date).timetuple()), localtime=True)
msg_id = u'<%s.%s@%s>' % (backend.name, mail.get_full_id(), domain) msg_id = u'<%s.%s@%s>' % (backend.name, mail.id, domain)
if int(self.config.get('html')) and mail.is_html: if int(self.config.get('html')) and mail.flags & mail.IS_HTML:
body = mail.get_content() body = mail.content
content_type = 'html' content_type = 'html'
else: else:
if mail.is_html: if mail.flags & mail.IS_HTML:
body = html2text(mail.get_content()) body = html2text(mail.content)
else: else:
body = mail.get_content() body = mail.content
content_type = 'plain' content_type = 'plain'
if mail.get_signature(): if mail.signature:
if int(self.config.get('html')) and mail.is_html: if int(self.config.get('html')) and mail.flags & mail.IS_HTML:
body += u'<p>-- <br />%s</p>' % mail.get_signature() body += u'<p>-- <br />%s</p>' % mail.signature
else: else:
body += u'\n\n-- \n' body += u'\n\n-- \n'
if mail.is_html: if mail.flags & mail.IS_HTML:
body += html2text(mail.get_signature()) body += html2text(mail.signature)
else: else:
body += mail.get_signature() body += mail.signature
# Header class is smart enough to try US-ASCII, then the charset we # Header class is smart enough to try US-ASCII, then the charset we
# provide, then fall back to UTF-8. # provide, then fall back to UTF-8.

10
weboob/backends/aum/backend.py
View file

@ -105,8 +105,8 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesReply, ICapDating, ICapC
if thread: if thread:
slut = self._get_slut(int(thread)) slut = self._get_slut(int(thread))
for mail in self._iter_thread_messages(thread, only_new, slut['lastmsg'], {}): for mail in self._iter_thread_messages(thread, only_new, slut['lastmsg'], {}):
if slut['lastmsg'] < mail.get_date(): if slut['lastmsg'] < mail.date:
slut['lastmsg'] = mail.get_date() slut['lastmsg'] = mail.date
yield mail yield mail
self.storage.set('sluts', int(thread), slut) self.storage.set('sluts', int(thread), slut)
@ -122,8 +122,8 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesReply, ICapDating, ICapC
continue continue
for mail in self._iter_thread_messages(contact.get_id(), only_new, last_msg, profiles): for mail in self._iter_thread_messages(contact.get_id(), only_new, last_msg, profiles):
if last_msg < mail.get_date(): if last_msg < mail.date:
last_msg = mail.get_date() last_msg = mail.date
yield mail yield mail
@ -153,7 +153,7 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesReply, ICapDating, ICapC
def _iter_thread_messages(self, id, only_new, last_msg, profiles): def _iter_thread_messages(self, id, only_new, last_msg, profiles):
mails = self.browser.get_thread_mails(id) mails = self.browser.get_thread_mails(id)
for mail in mails: for mail in mails:
if only_new and mail.get_date() <= last_msg: if only_new and mail.date <= last_msg:
continue continue
if not mail.profile_link in profiles: if not mail.profile_link in profiles:

6
weboob/backends/aum/browser.py
View file

@ -199,13 +199,13 @@ class AuMBrowser(BaseBrowser):
@pageaccess @pageaccess
def get_thread_mails(self, id): def get_thread_mails(self, id):
if not self.is_on_page(ContactThreadPage) or self.page.get_id() != int(id): if not self.is_on_page(ContactThreadPage) or self.page.id != int(id):
self.page.open_thread_page(id) self.page.open_thread_page(id)
return self.page.get_mails() return self.page.mails
@pageaccess @pageaccess
def post_mail(self, id, content): def post_mail(self, id, content):
if not self.is_on_page(ContactThreadPage) or self.page.get_id() != int(id): if not self.is_on_page(ContactThreadPage) or self.page.id != int(id):
self.page.open_thread_page(id) self.page.open_thread_page(id)
self.page.post(content) self.page.post(content)

31
weboob/backends/aum/pages/contact_thread.py
View file

@ -91,9 +91,9 @@ class MailParser(Message):
10: '(L)', 10: '(L)',
} }
def __init__(self, id, name, tr): def __init__(self, thread_id, name, tr):
# <td> <table> implicit<tbody> <tr> # <td> <table> implicit<tbody> <tr>
Message.__init__(self, id, 0, 'Discussion with %s' % name, name) Message.__init__(self, thread_id, 0, 'Discussion with %s' % name, name)
self.tr = tr.childNodes[0].childNodes[1].childNodes[0].childNodes[0] self.tr = tr.childNodes[0].childNodes[1].childNodes[0].childNodes[0]
tds = self.tr.childNodes tds = self.tr.childNodes
@ -131,8 +131,8 @@ class MailParser(Message):
self.parse_profile_link() self.parse_profile_link()
self.parse_from() self.parse_from()
def set_reply_id(self, date): def set_parent_message_id(self, date):
self.reply_id = date self.parent_message_id = date
def parse_date(self, date_str): def parse_date(self, date_str):
# To match regexp, we have to remove any return chars in string # To match regexp, we have to remove any return chars in string
@ -153,10 +153,11 @@ class MailParser(Message):
d = d.astimezone(tz.tzutc()) d = d.astimezone(tz.tzutc())
# and get timestamp # and get timestamp
self.date = d self.date = d
self.id = self.get_date_int() self.message_id = self.date_int
self.id = '%s.%s' % (self.thread_id, self.message_id)
if m.group(7).find('nouveau') >= 0: if m.group(7).find('nouveau') >= 0:
self.is_new = True self.flags |= self.IS_UNREAD
else: else:
error('Error: unable to parse the datetime string "%s"' % date_str) error('Error: unable to parse the datetime string "%s"' % date_str)
@ -176,7 +177,7 @@ class MailParser(Message):
self.signature = u'Profile: http://www.adopteunmec.com%s' % self.profile_link self.signature = u'Profile: http://www.adopteunmec.com%s' % self.profile_link
return return
warning('Unable to find the profile URL in the message %s@%s' % (self.get_from(), self.get_id())) warning('Unable to find the profile URL in the message %s@%s' % (self.sender, self.id))
def parse_from(self): def parse_from(self):
tds = self.tr.getElementsByTagName('div') tds = self.tr.getElementsByTagName('div')
@ -190,7 +191,7 @@ class MailParser(Message):
return return
warning('Warning: unable to find from in the mail %s' % self.get_id()) warning('Warning: unable to find from in the mail %s' % self.id)
class ContactThreadPage(PageBase): class ContactThreadPage(PageBase):
@ -274,7 +275,7 @@ class ContactThreadPage(PageBase):
id_regexp = re.compile("/thread.php\?id=([0-9]+)") id_regexp = re.compile("/thread.php\?id=([0-9]+)")
def on_loaded(self): def on_loaded(self):
self.items = [] self.mails = []
a_list = self.document.getElementsByTagName('a') a_list = self.document.getElementsByTagName('a')
self.id = 0 self.id = 0
@ -313,12 +314,6 @@ class ContactThreadPage(PageBase):
mail = MailParser(self.id, self.name, tag) mail = MailParser(self.id, self.name, tag)
if self.items: if self.mails:
self.items[-1].set_reply_id(mail.get_date_int()) self.mails[-1].set_parent_message_id(mail.date_int)
self.items += [mail] self.mails += [mail]
def get_id(self):
return self.id
def get_mails(self):
return self.items

21
weboob/backends/dlfp/backend.py
View file

@ -69,12 +69,12 @@ class DLFPBackend(BaseBackend, ICapMessages, ICapMessagesReply):
if thread_wanted and thread_wanted != article.id: if thread_wanted and thread_wanted != article.id:
continue continue
flags = Message.IS_HTML
if not article.id in self.storage.get('seen', what, default={}): if not article.id in self.storage.get('seen', what, default={}):
seen[article.id] = {'comments': []} seen[article.id] = {'comments': []}
new = True flags |= Message.IS_NEW
else: else:
seen[article.id] = self.storage.get('seen', what, article.id, default={}) seen[article.id] = self.storage.get('seen', what, article.id, default={})
new = False
try: try:
with self.browser: with self.browser:
@ -82,7 +82,7 @@ class DLFPBackend(BaseBackend, ICapMessages, ICapMessagesReply):
except BrowserUnavailable: except BrowserUnavailable:
continue continue
if not only_new or new: if not only_new or flags & Message.IS_NEW:
yield Message(thread.id, yield Message(thread.id,
0, 0,
thread.title, thread.title,
@ -90,16 +90,15 @@ class DLFPBackend(BaseBackend, ICapMessages, ICapMessagesReply):
article.datetime, article.datetime,
content=''.join([thread.body, thread.part2]), content=''.join([thread.body, thread.part2]),
signature='URL: %s' % article.url, signature='URL: %s' % article.url,
is_html=True, flags=flags)
is_new=new)
for comment in thread.iter_all_comments(): for comment in thread.iter_all_comments():
flags = Message.IS_HTML
if not comment.id in seen[article.id]['comments']: if not comment.id in seen[article.id]['comments']:
seen[article.id]['comments'].append(comment.id) seen[article.id]['comments'].append(comment.id)
new = True flags |= Message.IS_NEW
else:
new = False if not only_new or flags & Message.IS_NEW:
if not only_new or new:
yield Message(thread.id, yield Message(thread.id,
comment.id, comment.id,
comment.title, comment.title,
@ -108,8 +107,8 @@ class DLFPBackend(BaseBackend, ICapMessages, ICapMessagesReply):
comment.reply_id, comment.reply_id,
comment.body, comment.body,
'Score: %d' % comment.score, 'Score: %d' % comment.score,
is_html=True, flags)
is_new=new)
# If there is no articles seen, it's suspicious, probably I can't # If there is no articles seen, it's suspicious, probably I can't
# fetch the feed. # fetch the feed.
if seen: if seen:

20
weboob/backends/fourchan/backend.py
View file

@ -75,30 +75,27 @@ class FourChanBackend(BaseBackend, ICapMessages):
def _iter_thread_messages(self, board, thread, only_new): def _iter_thread_messages(self, board, thread, only_new):
thread = self.browser.get_thread(board, thread) thread = self.browser.get_thread(board, thread)
flags = Message.IS_HTML
if thread.id in self.storage.get('boards', board, default={}): if thread.id in self.storage.get('boards', board, default={}):
self.storage.set('boards', board, thread.id, []) self.storage.set('boards', board, thread.id, [])
new = True flags |= Message.IS_NEW
else:
new = False
if not only_new or new: if not only_new or flags & Message.IS_NEW:
yield Message('%s.%s' % (board, thread.id), yield Message('%s.%s' % (board, thread.id),
0, 0,
thread.filename, thread.filename,
thread.author, thread.author,
thread.datetime, thread.datetime,
content=thread.text, content=thread.text,
is_html=True, flags=flags)
is_new=new)
for comment in thread.comments: for comment in thread.comments:
flags = Message.IS_HTML
if not comment.id in self.storage.get('boards', board, thread.id, default=[]): if not comment.id in self.storage.get('boards', board, thread.id, default=[]):
self.storage.set('boards', board, thread.id, self.storage.get('boards', board, thread.id, default=[]) + [comment.id]) self.storage.set('boards', board, thread.id, self.storage.get('boards', board, thread.id, default=[]) + [comment.id])
new = True flags |= Message.IS_NEW
else:
new = False
if not only_new or new: if not only_new or flags & Message.IS_NEW:
yield Message('%s.%s' % (board, thread.id), yield Message('%s.%s' % (board, thread.id),
comment.id, comment.id,
thread.filename, thread.filename,
@ -106,8 +103,7 @@ class FourChanBackend(BaseBackend, ICapMessages):
comment.datetime, comment.datetime,
0, 0,
comment.text, comment.text,
is_html=True, flags)
is_new=new)
self.storage.save() self.storage.save()

66
weboob/capabilities/messages.py
View file

@ -19,18 +19,24 @@
import datetime import datetime
import time import time
from .base import IBaseCap from .base import IBaseCap, CapBaseObject, NotLoaded
__all__ = ['ICapMessages', 'ICapMessagesReply', 'Message'] __all__ = ['ICapMessages', 'ICapMessagesReply', 'Message']
class Message(object): class Message(CapBaseObject):
def __init__(self, thread_id, _id, title, sender, date=None, reply_id=u'', IS_HTML = 0x001
content=u'', signature=u'', is_html=False, is_new=False): IS_NEW = 0x002
IS_UNREAD = 0x004
def __init__(self, thread_id, message_id, title, sender, date=None, parent_message_id=u'',
content=u'', signature=u'', flags=0):
CapBaseObject.__init__(self, '%s.%s' % (thread_id, message_id))
self.thread_id = unicode(thread_id) self.thread_id = unicode(thread_id)
self.id = unicode(_id) self.message_id = unicode(message_id)
self.reply_id = unicode(reply_id) self.parent_message_id = unicode(parent_message_id)
self.title = unicode(title) self.title = unicode(title)
self.sender = unicode(sender) self.sender = unicode(sender)
self.signature = unicode(signature) self.signature = unicode(signature)
@ -39,48 +45,24 @@ class Message(object):
if date is None: if date is None:
date = datetime.datetime.utcnow() date = datetime.datetime.utcnow()
self.date = date self.date = date
self.is_new = is_new self.flags = flags
self.is_html = is_html
def get_date_int(self): @property
return int(time.strftime('%Y%m%d%H%M%S', self.get_date().timetuple())) def date_int(self):
return int(time.strftime('%Y%m%d%H%M%S', self.date.timetuple()))
def get_full_id(self): @property
return '%s.%s' % (self.thread_id, self.id) def parent_id(self):
if not self.parent_message_id:
def get_full_reply_id(self): return ''
return '%s.%s' % (self.thread_id, self.reply_id) return '%s.%s' % (self.thread_id, self.parent_message_id)
def get_id(self):
return self.id
def get_thread_id(self):
return self.thread_id
def get_reply_id(self):
return self.reply_id
def get_title(self):
return self.title
def get_date(self):
return self.date
def get_from(self):
return self.sender
def get_content(self):
return self.content
def get_signature(self):
return self.signature
def __eq__(self, msg): def __eq__(self, msg):
return self.id == msg.id and self.thread_id == msg.thread_id return self.id == msg.id
def __repr__(self): def __repr__(self):
result = '<Message id="%s.%s" title="%s" date="%s" from="%s">' % ( result = '<Message id="%s" title="%s" date="%s" from="%s">' % (
self.thread_id, self.id, self.title, self.date, self.sender) self.id, self.title, self.date, self.sender)
return result.encode('utf-8') return result.encode('utf-8')
class ICapMessages(IBaseCap): class ICapMessages(IBaseCap):

1
weboob/tools/backend.py
View file

@ -132,7 +132,6 @@ class BaseBackend(object):
elif isinstance(field.default, float) and not isinstance(value, float): elif isinstance(field.default, float) and not isinstance(value, float):
value = float(value) value = float(value)
print value, field.choices
if field.choices: if field.choices:
if (isinstance(field.choices, (tuple,list)) and not value in field.choices) or \ if (isinstance(field.choices, (tuple,list)) and not value in field.choices) or \
(isinstance(field.choices, dict) and not value in field.choices.iterkeys()): (isinstance(field.choices, dict) and not value in field.choices.iterkeys()):