nojhan/weboob-devel
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

better organisation in storage

Browse source
This commit is contained in:
Romain Bignon 2010-04-09 00:15:25 +02:00
commit bdf34a2176
2 changed files with 16 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

21
weboob/backends/dlfp/backend.py
View file

@ -38,7 +38,7 @@ class DLFPBackend(Backend, ICapMessages, ICapMessagesReply, ICapUpdatable):
'get_news': Backend.ConfigField(default=True, description='Get newspapers'), 'get_news': Backend.ConfigField(default=True, description='Get newspapers'),
'get_telegrams': Backend.ConfigField(default=False, description='Get telegrams'), 'get_telegrams': Backend.ConfigField(default=False, description='Get telegrams'),
} }
STORAGE = {'seen': {'contents': [], 'comments': []}} STORAGE = {'seen': {}}
browser = None browser = None
def need_browser(func): def need_browser(func):
@ -59,10 +59,15 @@ class DLFPBackend(Backend, ICapMessages, ICapMessagesReply, ICapUpdatable):
yield message yield message
def _iter_messages(self, what): def _iter_messages(self, what):
if not what in self.storage.get(self.name, 'seen'):
self.storage.set(self.name, 'seen', what, {})
seen = {}
for article in ArticlesList(what).iter_articles(): for article in ArticlesList(what).iter_articles():
thread = self.browser.get_content(article.id) thread = self.browser.get_content(article.id)
if not thread.id in self.storage.get(self.name, 'seen', 'contents'):
self.storage.get(self.name, 'seen', 'contents').append(thread.id) if not article.id in self.storage.get(self.name, 'seen', what):
seen[article.id] = {'comments': []}
yield Message(thread.id, yield Message(thread.id,
0, 0,
thread.title, thread.title,
@ -70,9 +75,12 @@ class DLFPBackend(Backend, ICapMessages, ICapMessagesReply, ICapUpdatable):
article.datetime, article.datetime,
content=''.join([thread.body, thread.part2]), content=''.join([thread.body, thread.part2]),
signature='URL: %s' % article.url) signature='URL: %s' % article.url)
else:
seen[article.id] = self.storage.get(self.name, 'seen', what, article.id)
for comment in thread.iter_all_comments(): for comment in thread.iter_all_comments():
if not comment.id in self.storage.get(self.name, 'seen', 'comments'): if not comment.id in seen[article.id]['comments']:
self.storage.get(self.name, 'seen', 'comments').append(comment.id) seen[article.id]['comments'].append(comment.id)
yield Message(thread.id, yield Message(thread.id,
comment.id, comment.id,
comment.title, comment.title,
@ -81,7 +89,8 @@ class DLFPBackend(Backend, ICapMessages, ICapMessagesReply, ICapUpdatable):
comment.reply_id, comment.reply_id,
comment.body, comment.body,
'Score: %d' % comment.score) 'Score: %d' % comment.score)
self.storage.save(self.name) self.storage.set(self.name, 'seen', what, seen)
self.storage.save(self.name)
def iter_new_messages(self): def iter_new_messages(self):
return self.iter_messages() return self.iter_messages()

2
weboob/tools/storage.py
View file

@ -34,7 +34,7 @@ class IStorage:
raise NotImplementedError() raise NotImplementedError()
try: try:
from .config.yamlconfig import YamlConfig, ConfigError from .config.yamlconfig import YamlConfig
except ImportError, e: except ImportError, e:
debug('Import error for weboob.tools.config.yamlconfig: %s' % e) debug('Import error for weboob.tools.config.yamlconfig: %s' % e)
else: else: