nojhan/weboob-devel
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

adapt to new ICapMessages API

Browse source
This commit is contained in:
Romain Bignon 2010-08-30 16:38:34 +02:00
commit e7c24c13c8
12 changed files with 381 additions and 249 deletions
Download patch file Download diff file Expand all files Collapse all files

29
weboob/applications/monboob/monboob.py
View file

@ -30,7 +30,7 @@ import asyncore
from weboob.core.ouiboube import Weboob
from weboob.core.scheduler import Scheduler
from weboob.capabilities.messages import ICapMessages, ICapMessagesReply, Message
from weboob.capabilities.messages import ICapMessages, ICapMessagesPost, Thread, Message
from weboob.tools.application.console import ConsoleApplication
from weboob.tools.misc import html2text, get_backtrace, utc2local
@ -158,23 +158,31 @@ class Monboob(ConsoleApplication):
print >>sys.stderr, 'Backend %s not found' % bname
return 1
if not backend.has_caps(ICapMessagesReply):
print >>sys.stderr, 'The backend %s does not implement ICapMessagesReply' % bname
if not backend.has_caps(ICapMessagesPost):
print >>sys.stderr, 'The backend %s does not implement ICapMessagesPost' % bname
return 1
thread_id, msg_id = id.rsplit('.', 1)
thread = Thread(thread_id)
message = Message(thread,
0,
title=title,
sender=None,
receiver=None,
parent=Message(thread, msg_id),
content=content)
try:
backend.post_reply(thread_id, msg_id, title, content)
backend.post_message(message)
except Exception, e:
content = u'Unable to send message to %s:\n' % thread_id
content += '\n\t%s\n' % e
if logging.root.level == logging.DEBUG:
content += '\n%s\n' % get_backtrace(e)
self.send_email(backend, Message(thread_id,
self.send_email(backend, Message(thread,
0,
title='Unable to send message',
sender='Monboob',
parent_message_id=msg_id,
parent=Message(thread, msg_id),
content=content))
@ConsoleApplication.command("run daemon")
@ -183,22 +191,23 @@ class Monboob(ConsoleApplication):
self.weboob.loop()
def process(self):
for backend, message in self.weboob.do('iter_new_messages'):
for backend, message in self.weboob.do('iter_unread_messages'):
self.send_email(backend, message)
backend.set_message_read(message)
def send_email(self, backend, mail):
domain = self.config.get('domain')
recipient = self.config.get('recipient')
reply_id = ''
if mail.parent_id:
reply_id = u'<%s.%s@%s>' % (backend.name, mail.parent_id, domain)
if mail.parent:
reply_id = u'<%s.%s@%s>' % (backend.name, mail.parent.full_id, domain)
subject = mail.title
sender = u'"%s" <%s@%s>' % (mail.sender.replace('"', '""'), backend.name, domain)
# assume that .date is an UTC datetime
date = formatdate(time.mktime(utc2local(mail.date).timetuple()), localtime=True)
msg_id = u'<%s.%s@%s>' % (backend.name, mail.id, domain)
msg_id = u'<%s.%s@%s>' % (backend.name, mail.full_id, domain)
if int(self.config.get('html')) and mail.flags & mail.IS_HTML:
body = mail.content

233
weboob/backends/aum/backend.py
View file

@ -19,16 +19,16 @@ from __future__ import with_statement
from datetime import datetime
from dateutil import tz
from logging import warning
from time import sleep
from logging import warning, debug
from weboob.capabilities.chat import ICapChat
from weboob.capabilities.messages import ICapMessages, ICapMessagesReply, Message
from weboob.capabilities.messages import ICapMessages, ICapMessagesPost, Message, Thread
from weboob.capabilities.dating import ICapDating, StatusField
from weboob.capabilities.contact import ICapContact, Contact, ProfileNode
from weboob.tools.backend import BaseBackend
from weboob.tools.browser import BrowserUnavailable
from .captcha import CaptchaError
from .browser import AuMBrowser
from .exceptions import AdopteWait
from .optim.profiles_walker import ProfilesWalker
@ -38,7 +38,7 @@ from .optim.visibility import Visibility
__all__ = ['AuMBackend']
class AuMBackend(BaseBackend, ICapMessages, ICapMessagesReply, ICapDating, ICapChat, ICapContact):
class AuMBackend(BaseBackend, ICapMessages, ICapMessagesPost, ICapDating, ICapChat, ICapContact):
NAME = 'aum'
MAINTAINER = 'Romain Bignon'
EMAIL = 'romain@peerfuse.org'
@ -54,17 +54,27 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesReply, ICapDating, ICapC
}
BROWSER = AuMBrowser
MAGIC_ID_BASKET = 1
def create_default_browser(self):
if self.config['register']:
browser = self.create_browser(self.config['username'])
browser.register(password= self.config['password'],
sex= 0,
birthday_d= 1,
birthday_m= 1,
birthday_y= 1970,
zipcode= 75001,
country= 'fr',
godfather= '')
browser = None
while not browser:
try:
browser = self.create_browser(self.config['username'])
browser.register(password= self.config['password'],
sex= 0,
birthday_d= 1,
birthday_m= 1,
birthday_y= 1970,
zipcode= 75001,
country= 'fr',
godfather= '')
except CaptchaError:
debug('Unable to resolve captcha. Retrying...')
browser = None
browser.password = self.config['password']
return browser
else:
return self.create_browser(self.config['username'], self.config['password'])
@ -79,16 +89,117 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesReply, ICapDating, ICapC
except AdopteWait:
return (StatusField('notice', '', u'<h3>You are currently waiting 1am to be able to connect with this account</h3>', StatusField.FIELD_HTML|StatusField.FIELD_TEXT))
def iter_messages(self, thread=None):
for message in self._iter_messages(thread, False):
yield message
def iter_threads(self):
with self.browser:
contacts = self.browser.get_threads_list()
def iter_new_messages(self, thread=None):
for message in self._iter_messages(thread, True):
yield message
for contact in contacts:
thread = Thread(contact.get_id())
yield thread
def get_thread(self, id):
thread = None
if isinstance(id, Thread):
thread = id
id = thread.id
if not thread:
thread = Thread(id)
full = False
else:
full = True
with self.browser:
mails = self.browser.get_thread_mails(id, full)
my_name = self.browser.get_my_name()
child = None
msg = None
slut = self._get_slut(id)
for mail in mails:
flags = 0
if mail.date > slut['lastmsg']:
flags |= Message.IS_UNREAD
if mail.sender != my_name:
if mail.new:
flags |= Message.IS_NOT_ACCUSED
else:
flags |= Message.IS_ACCUSED
msg = Message(thread=thread,
id=mail.message_id,
title=mail.title,
sender=mail.sender,
receiver=mail.name if mail.sender == my_name else my_name, # TODO: me
date=mail.date,
content=mail.content,
signature=mail.signature,
children=[],
flags=flags)
if child:
msg.children.append(child)
child.parent = msg
child = msg
if full:
# If we have get all the messages, replace NotLoaded with None as
# parent.
msg.parent = None
thread.root = msg
return thread
def iter_unread_messages(self, thread=None):
with self.browser:
contacts = self.browser.get_threads_list()
for contact in contacts:
slut = self._get_slut(contact.get_id())
if contact.get_lastmsg_date() > slut['lastmsg']:
thread = self.get_thread(contact.get_id())
for m in thread.iter_all_messages():
if m.flags & m.IS_UNREAD:
yield m
# Send mail when someone added me in her basket.
# XXX possibly race condition if a slut adds me in her basket
# between the aum.nb_new_baskets() and aum.get_baskets().
with self.browser:
new_baskets = self.browser.nb_new_baskets()
if new_baskets:
ids = self.browser.get_baskets()
while new_baskets > 0 and len(ids) > new_baskets:
new_baskets -= 1
profile = self.browser.get_profile(ids[new_baskets])
thread = Thread(profile.get_id())
thread.root = Message(thread=thread,
id=self.MAGIC_ID_BASKET,
title='Basket of %s' % profile.get_name(),
sender=profile.get_name(),
receiver=self.browser.get_my_name(),
date=None, # now
content='You are taken in her basket!',
signature=profile.get_profile_text(),
children=[],
flags=Message.IS_UNREAD)
yield thread.root
def set_message_read(self, message):
if message.id == self.MAGIC_ID_BASKET:
# We don't save baskets.
return
slut = self._get_slut(message.thread.id)
if slut['lastmsg'] < message.date:
slut['lastmsg'] = message.date
#slut['msgstatus'] = contact.get_status()
self.storage.set('sluts', message.thread.id, slut)
self.storage.save()
def _get_slut(self, id):
if not id in self.storage.get('sluts'):
sluts = self.storage.get('sluts')
if not sluts or not id in sluts:
slut = {'lastmsg': datetime(1970,1,1),
'msgstatus': ''}
else:
@ -97,81 +208,9 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesReply, ICapDating, ICapC
slut['lastmsg'] = slut['lastmsg'].replace(tzinfo=tz.tzutc())
return slut
def _iter_messages(self, thread, only_new):
def post_message(self, message):
with self.browser:
try:
profiles = {}
if thread:
slut = self._get_slut(int(thread))
for mail in self._iter_thread_messages(thread, only_new, slut['lastmsg'], {}):
if slut['lastmsg'] < mail.date:
slut['lastmsg'] = mail.date
yield mail
self.storage.set('sluts', int(thread), slut)
self.storage.save()
else:
contacts = self.browser.get_threads_list()
for contact in contacts:
slut = self._get_slut(contact.get_id())
last_msg = slut['lastmsg']
if only_new and contact.get_lastmsg_date() < last_msg and contact.get_status() == slut['msgstatus'] or \
not thread is None and int(thread) != contact.get_id():
continue
for mail in self._iter_thread_messages(contact.get_id(), only_new, last_msg, profiles):
if last_msg < mail.date:
last_msg = mail.date
yield mail
slut['lastmsg'] = last_msg
slut['msgstatus'] = contact.get_status()
self.storage.set('sluts', contact.get_id(), slut)
self.storage.save()
# Send mail when someone added me in her basket.
# XXX possibly race condition if a slut adds me in her basket
# between the aum.nbNewBaskets() and aum.getBaskets().
new_baskets = self.browser.nb_new_baskets()
if new_baskets:
ids = self.browser.get_baskets()
while new_baskets > 0 and len(ids) > new_baskets:
new_baskets -= 1
profile = self.browser.get_profile(ids[new_baskets])
yield Message(profile.get_id(), 1,
title='Basket of %s' % profile.get_name(),
sender=profile.get_name(),
content='You are taken in her basket!',
signature=profile.get_profile_text())
except BrowserUnavailable:
pass
def _iter_thread_messages(self, id, only_new, last_msg, profiles):
mails = self.browser.get_thread_mails(id)
for mail in mails:
if only_new and mail.date <= last_msg:
continue
if not mail.profile_link in profiles:
profiles[mail.profile_link] = self.browser.get_profile(mail.profile_link)
mail.signature += u'\n%s' % profiles[mail.profile_link].get_profile_text()
yield mail
def post_reply(self, thread_id, reply_id, title, message):
while 1:
try:
with self.browser:
self.browser.post_mail(thread_id, message)
except AdopteWait:
# If we are on a waiting state, retry every 30 minutes until it is posted.
sleep(60*30)
else:
return
self.browser.post_mail(message.thread.id, message.content)
def get_contact(self, contact):
try:
@ -268,4 +307,8 @@ class AuMBackend(BaseBackend, ICapMessages, ICapMessagesReply, ICapDating, ICapC
data = self.browser.openurl(photo.thumbnail_url).read()
contact.set_photo(name, thumbnail_data=data)
OBJECTS = {Contact: fill_contact}
def fill_thread(self, thread, fields):
return self.get_thread(thread)
OBJECTS = {Thread: fill_thread,
Contact: fill_contact}

6
weboob/backends/aum/browser.py
View file

@ -108,7 +108,7 @@ class AuMBrowser(BaseBrowser):
if not self.is_on_page(RegisterPage):
self.location('http://www.adopteunmec.com/register2.php')
return self.page.register(password, sex, birthday_d, birthday_m, birthday_y, zipcode, country, godfather)
return self.page.register(password, sex, birthday_d, birthday_m, birthday_y, zipcode, country)
@pageaccess
def add_photo(self, name, f):
@ -198,9 +198,9 @@ class AuMBrowser(BaseBrowser):
return self.page.get_contact_list()
@pageaccess
def get_thread_mails(self, id):
def get_thread_mails(self, id, full=False):
if not self.is_on_page(ContactThreadPage) or self.page.id != int(id):
self.page.open_thread_page(id)
self.page.open_thread_page(id, full)
return self.page.mails
@pageaccess

2
weboob/backends/aum/captcha.py
View file

@ -103,6 +103,8 @@ class Tile:
try:
return self.hash[self.checksum()]
except KeyError:
print 'Unable te resolve:'
self.display()
raise CaptchaError()
class Captcha:

18
weboob/backends/aum/pages/contact_thread.py
View file

@ -16,6 +16,7 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
import re
import time
from datetime import datetime
from dateutil import tz
from logging import error, warning
@ -23,9 +24,8 @@ from mechanize import FormNotFoundError
from weboob.backends.aum.pages.base import PageBase
from weboob.backends.aum.exceptions import AdopteCantPostMail
from weboob.capabilities.messages import Message
class MailParser(Message):
class MailParser(object):
"""
<td>
@ -93,8 +93,14 @@ class MailParser(Message):
def __init__(self, thread_id, name, tr):
# <td> <table> implicit<tbody> <tr>
Message.__init__(self, thread_id, 0, 'Discussion with %s' % name, name)
self.thread_id = thread_id
self.date = None
self.message_id = 0
self.title = 'Discussion with %s' % name
self.sender = name
self.name = name
self.tr = tr.childNodes[0].childNodes[1].childNodes[0].childNodes[0]
self.new = False
tds = self.tr.childNodes
@ -131,6 +137,10 @@ class MailParser(Message):
self.parse_profile_link()
self.parse_from()
@property
def date_int(self):
return int(time.strftime('%Y%m%d%H%M%S', self.date.timetuple()))
def set_parent_message_id(self, date):
self.parent_message_id = date
@ -157,7 +167,7 @@ class MailParser(Message):
self.id = '%s.%s' % (self.thread_id, self.message_id)
if m.group(7).find('nouveau') >= 0:
self.flags |= self.IS_UNREAD
self.new = True
else:
error('Error: unable to parse the datetime string "%s"' % date_str)

26
weboob/backends/aum/pages/login.py
View file

@ -16,6 +16,8 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
import re
from weboob.tools.browser import BrowserIncorrectPassword
from .base import PageBase
@ -29,7 +31,29 @@ class LoginPage(PageBase):
self.browser.submit() # submit current form
class RegisterError(Exception):
pass
class RegisterPage(PageBase):
def on_loaded(self):
display_errors = False
for div in self.document.getElementsByTagName('div'):
if div.getAttribute('class') == 'balloon':
display_errors = True
break
if not display_errors:
return
errors = []
for script in self.document.getElementsByTagName('script'):
for child in script.childNodes:
if child and child.data.find('dispErrors') >= 0:
for m in re.finditer('"(\w+)": "(.*)",', child.data):
errors.append(m.group(2))
raise RegisterError(u'Unable to register account: %s' % ', '.join(errors))
def register(self, password, sex, birthday_d, birthday_m, birthday_y, zipcode, country):
"""
Form name=register (#1)
@ -53,7 +77,7 @@ Form name=register (#1)
self.browser.select_form(name='register')
self.browser.set_all_readonly(False)
self.browser['sex'] = str(sex)
self.browser['sex'] = [str(sex)]
self.browser['birthday0'] = [str(birthday_d)]
self.browser['birthday1'] = [str(birthday_m)]
self.browser['birthday2'] = [str(birthday_y)]

143
weboob/backends/dlfp/backend.py
View file

@ -18,8 +18,7 @@
from __future__ import with_statement
from weboob.tools.backend import BaseBackend
from weboob.tools.browser import BrowserUnavailable
from weboob.capabilities.messages import ICapMessages, ICapMessagesReply, Message
from weboob.capabilities.messages import ICapMessages, ICapMessagesPost, Message, Thread
from .feeds import ArticlesList
from .browser import DLFP
@ -28,7 +27,7 @@ from .browser import DLFP
__all__ = ['DLFPBackend']
class DLFPBackend(BaseBackend, ICapMessages, ICapMessagesReply):
class DLFPBackend(BaseBackend, ICapMessages, ICapMessagesPost):
NAME = 'dlfp'
MAINTAINER = 'Romain Bignon'
EMAIL = 'romain@peerfuse.org'
@ -46,75 +45,95 @@ class DLFPBackend(BaseBackend, ICapMessages, ICapMessagesReply):
def create_default_browser(self):
return self.create_browser(self.config['username'], self.config['password'])
def iter_messages(self, thread=None):
return self._iter_messages(thread, False)
def iter_new_messages(self, thread=None):
return self._iter_messages(thread, True)
def _iter_messages(self, thread, only_new):
def iter_threads(self):
whats = set()
if self.config['get_news']:
for message in self._iter_messages_of('newspaper', thread, only_new):
yield message
whats.add('newspaper')
if self.config['get_telegrams']:
for message in self._iter_messages_of('telegram', thread, only_new):
yield message
whats.add('telegram')
def _iter_messages_of(self, what, thread_wanted, only_new):
if not what in self.storage.get('seen', default={}):
self.storage.set('seen', what, {})
for what in whats:
for article in ArticlesList(what).iter_articles():
thread = Thread(article.id)
thread.title = article.title
yield thread
seen = {}
for article in ArticlesList(what).iter_articles():
if thread_wanted and thread_wanted != article.id:
continue
def get_thread(self, id):
if isinstance(id, Thread):
thread = id
id = thread.id
flags = Message.IS_HTML
if not article.id in self.storage.get('seen', what, default={}):
seen[article.id] = {'comments': []}
flags |= Message.IS_NEW
else:
seen[article.id] = self.storage.get('seen', what, article.id, default={})
with self.browser:
content = self.browser.get_content(id)
try:
with self.browser:
thread = self.browser.get_content(article.id)
except BrowserUnavailable:
continue
if not thread:
thread = Thread(id)
if not only_new or flags & Message.IS_NEW:
yield Message(thread.id,
0,
thread.title,
thread.author,
article.datetime,
content=''.join([thread.body, thread.part2]),
signature='URL: %s' % article.url,
flags = Message.IS_HTML
if not thread.id in self.storage.get('seen', default={}):
flags |= Message.IS_UNREAD
thread.title = content.title
if not thread.date:
thread.date = content.date
thread.root = Message(thread=thread,
id=0, # root message
title=content.title,
sender=content.author,
receiver=None,
date=thread.date, #TODO XXX WTF this is None
parent=None,
content=''.join([content.body, content.part2]),
signature='URL: %s' % content.url,
children=[],
flags=flags)
for comment in thread.iter_all_comments():
flags = Message.IS_HTML
if not comment.id in seen[article.id]['comments']:
seen[article.id]['comments'].append(comment.id)
flags |= Message.IS_NEW
for com in content.comments:
self._insert_comment(com, thread.root)
if not only_new or flags & Message.IS_NEW:
yield Message(thread.id,
comment.id,
comment.title,
comment.author,
comment.date,
comment.reply_id,
comment.body,
'Score: %d' % comment.score,
flags)
return thread
# If there is no articles seen, it's suspicious, probably I can't
# fetch the feed.
if seen:
self.storage.set('seen', what, seen)
self.storage.save()
def _insert_comment(self, com, parent):
""""
Insert 'com' comment and its children in the parent message.
"""
flags = Message.IS_HTML
if not com.id in self.storage.get('seen', parent.thread.id, 'comments', default=[]):
flags |= Message.IS_UNREAD
def post_reply(self, thread_id, reply_id, title, message):
message = Message(thread=parent.thread,
id=com.id,
title=com.title,
sender=com.author,
receiver=None,
date=com.date,
parent=parent,
content=com.body,
signature='Score: %d' % com.score,
children=[],
flags=flags)
parent.children.append(message)
for sub in com.comments:
self._insert_comment(sub, message)
def iter_unread_messages(self, thread=None):
for thread in self.iter_threads():
self.fill_thread(thread, 'root')
for m in thread.iter_all_messages():
if m.flags & m.IS_UNREAD:
yield m
def set_message_read(self, message):
self.storage.set('seen', message.thread.id, 'comments', self.storage.get('seen', message.thread.id, 'comments', default=[]) + [message.id])
self.storage.save()
def post_mesage(self, message):
with self.browser:
return self.browser.post_reply(thread_id, reply_id, title, message)
return self.browser.post_reply(message.thread.id, message.parent.id, message.title, message.content)
def fill_thread(self, thread, fields):
return self.get_thread(thread)
OBJECTS = {Thread: fill_thread}

1
weboob/backends/dlfp/browser.py
View file

@ -86,3 +86,4 @@ class DLFP(BaseBrowser):
def is_logged(self):
return (self.page and self.page.is_logged())

6
weboob/backends/dlfp/pages/news.py
View file

@ -31,7 +31,7 @@ class Comment(object):
self.reply_id = reply_id
self.title = u''
self.author = u''
self.date = u''
self.date = None
self.body = u''
self.score = 0
self.comments = []
@ -73,7 +73,8 @@ class Article(object):
self.author = u''
self.body = u''
self.part2 = u''
self.date = u''
self.date = None
self.url = u''
self.comments = []
for div in tree.findall('div'):
@ -116,6 +117,7 @@ class ContentPage(DLFPPage):
def parse_div(self, div):
if div.attrib.get('class', '') in ('newsdiv', 'centraldiv'):
self.article = Article(self.browser, url2id(self.url), div)
self.article.url = self.url
if div.attrib.get('class', '') == 'articlediv':
self.article.parse_part2(div)
if div.attrib.get('class', '') == 'comments':

6
weboob/backends/dlfp/test.py
View file

@ -18,9 +18,13 @@
from weboob.tools.test import BackendTest
__all__ = ['DLFPTest']
class DLFPTest(BackendTest):
BACKEND = 'dlfp'
def test_new_messages(self):
for message in self.backend.iter_new_messages():
for message in self.backend.iter_unread_messages():
pass

132
weboob/backends/fourchan/backend.py
View file

@ -20,7 +20,7 @@ from __future__ import with_statement
from logging import warning
from weboob.capabilities.messages import ICapMessages, Message
from weboob.capabilities.messages import ICapMessages, Message, Thread
from weboob.tools.backend import BaseBackend
from .browser import FourChan
@ -41,71 +41,89 @@ class FourChanBackend(BaseBackend, ICapMessages):
STORAGE = {'boards': {}}
BROWSER = FourChan
def iter_messages(self, thread=None):
return self._iter_messages(thread, False)
def _splitid(self, id):
return id.split('.', 1)
def iter_new_messages(self, thread=None):
return self._iter_messages(thread, True)
def get_thread(self, id):
thread = None
def _iter_messages(self, thread, only_new):
if thread:
if '.' in thread:
board, thread = thread.split('.', 2)
return self._iter_messages_of(board, thread, only_new)
else:
warning('"%s" is not a valid ID' % thread)
else:
for board in self.config['boards'].split(' '):
return self._iter_messages_of(board, None, only_new)
if isinstance(id, Thread):
thread = id
id = thread.id
def _iter_messages_of(self, board, thread_wanted, only_new):
if not board in self.storage.get('boards', default={}):
self.storage.set('boards', board, {})
if not '.' in id:
warning('Malformated ID (%s)' % id)
return
if thread_wanted:
for message in self._iter_thread_messages(board, thread_wanted, only_new):
yield message
else:
board, thread_id = self._splitid(id)
with self.browser:
_thread = self.browser.get_thread(board, thread_id)
flags = 0
if not _thread.id in self.storage.get('boards', board, default={}):
flags |= Message.IS_UNREAD
if not thread:
thread = Thread(id)
thread.title = _thread.filename
thread.root = Message(thread=thread,
id=0, # root message
title=_thread.filename,
sender=_thread.author,
receiver=None,
date=_thread.datetime,
parent=None,
content=_thread.text,
signature=None,
children=None,
flags=flags|Message.IS_HTML)
parent = thread.root
for comment in _thread.comments:
flags = 0
if not comment.id in self.storage.get('boards', board, _thread.id, default=[]):
flags |= Message.IS_UNREAD
m = Message(thread=thread,
id=comment.id,
title=_thread.filename,
sender=comment.author,
receiver=None,
date=comment.datetime,
parent=parent,
content=comment.text,
signature=None,
children=None,
flags=flags|Message.IS_HTML)
parent.children = [m]
parent = m
return thread
def iter_threads(self):
for board in self.config['boards'].split(' '):
with self.browser:
threads = self.browser.get_threads(board)
for thread in threads:
for message in self._iter_thread_messages(board, thread.id, only_new):
yield message
t = Thread('%s.%s' % (board, thread.id))
t.title = thread.filename
yield t
def _iter_thread_messages(self, board, thread, only_new):
thread = self.browser.get_thread(board, thread)
def iter_unread_messages(self):
for thread in self.iter_threads():
self.fill_thread(thread, 'root')
flags = Message.IS_HTML
if thread.id in self.storage.get('boards', board, default={}):
self.storage.set('boards', board, thread.id, [])
flags |= Message.IS_NEW
if not only_new or flags & Message.IS_NEW:
yield Message('%s.%s' % (board, thread.id),
0,
thread.filename,
thread.author,
thread.datetime,
content=thread.text,
flags=flags)
for comment in thread.comments:
flags = Message.IS_HTML
if not comment.id in self.storage.get('boards', board, thread.id, default=[]):
self.storage.set('boards', board, thread.id, self.storage.get('boards', board, thread.id, default=[]) + [comment.id])
flags |= Message.IS_NEW
if not only_new or flags & Message.IS_NEW:
yield Message('%s.%s' % (board, thread.id),
comment.id,
thread.filename,
comment.author,
comment.datetime,
0,
comment.text,
flags)
for m in thread.iter_all_messages():
if m.flags & Message.IS_UNREAD:
yield m
def set_message_read(self, message):
board, thread_id = self._splitid(message.thread.id)
self.storage.set('boards', board, thread_id, self.storage.get('boards', board, thread_id, default=[]) + [message.id])
self.storage.save()
#def post_reply(self, thread_id, reply_id, title, message):
# return self.browser.post_reply(thread_id, reply_id, title, message)
def fill_thread(self, thread, fields):
return self.get_thread(thread)
OBJECTS = {Thread: fill_thread}

2
weboob/backends/fourchan/browser.py
View file

@ -34,7 +34,7 @@ class FourChan(BaseBrowser):
return self.page.articles
def get_thread(self, board, id):
self.location('http://boards.4chan.org/%s/res/%d' % (board, id))
self.location('http://boards.4chan.org/%s/res/%d' % (board, long(id)))
assert len(self.page.articles) == 1
return self.page.articles[0]