nojhan/weboob-devel
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

[boobot/twitter] fix purge system

Browse source
This commit is contained in:
Bezleputh 2014-09-07 23:13:55 +02:00 • committed by Florent
commit bc0a9b89db
2 changed files with 26 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

20
contrib/boobot.py
View file

@ -19,7 +19,7 @@
# along with weboob. If not, see <http://www.gnu.org/licenses/>. # along with weboob. If not, see <http://www.gnu.org/licenses/>.
from datetime import datetime from datetime import datetime, timedelta
import logging import logging
import re import re
import os import os
@ -30,7 +30,7 @@ from math import log
import urlparse import urlparse
import urllib import urllib
from random import randint, choice from random import randint, choice
import itertools
from irc.bot import SingleServerIRCBot from irc.bot import SingleServerIRCBot
import mechanize import mechanize
from mechanize import _headersutil as headersutil from mechanize import _headersutil as headersutil
@ -204,9 +204,19 @@ class MyThread(Thread):
return None return None
def check_twitter(self): def check_twitter(self):
for backend, thread in self.weboob.do('iter_resources', objs=None, nb_tweets = 10
split_path=['search', 'weboob'], backends=['twitter']):
if not thread.id in backend.storage.get('seen', default={}): for _, backend in self.weboob.load_backends(modules=['twitter']).iteritems():
for thread in list(itertools.islice(backend.iter_resources(None, ['search', 'weboob']),
0,
nb_tweets)):
if not backend.storage.get('lastpurge'):
backend.storage.set('lastpurge', datetime.now() - timedelta(days=60))
backend.storage.save()
if thread.id not in backend.storage.get('seen', default={}) and\
thread.date > backend.storage.get('lastpurge'):
_item = thread.id.split('#') _item = thread.id.split('#')
url = 'https://twitter.com/%s/status/%s' % (_item[0], _item[1]) url = 'https://twitter.com/%s/status/%s' % (_item[0], _item[1])
for msg in self.bot.on_url(url): for msg in self.bot.on_url(url):

10
modules/twitter/backend.py
View file

@ -18,7 +18,6 @@
# along with weboob. If not, see <http://www.gnu.org/licenses/>. # along with weboob. If not, see <http://www.gnu.org/licenses/>.
from datetime import datetime, timedelta from datetime import datetime, timedelta
import time
from weboob.tools.value import Value, ValueBackendPassword from weboob.tools.value import Value, ValueBackendPassword
from weboob.tools.backend import BaseBackend, BackendConfig from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.capabilities.messages import CapMessages, Thread, CapMessagesPost from weboob.capabilities.messages import CapMessages, Thread, CapMessagesPost
@ -97,11 +96,10 @@ class TwitterBackend(BaseBackend, CapMessages, CapMessagesPost, CapCollection):
self._purge_message_read() self._purge_message_read()
def _purge_message_read(self): def _purge_message_read(self):
lastpurge = self.storage.get('lastpurge', default=datetime.now() - timedelta(days=60))
lastpurge = self.storage.get('lastpurge', default=0) if datetime.now() - lastpurge > timedelta(days=60):
self.storage.set('lastpurge', datetime.now() - timedelta(days=60))
if time.time() - lastpurge > 86400:
self.storage.set('lastpurge', time.time())
self.storage.save() self.storage.save()
# we can't directly delete without a "RuntimeError: dictionary changed size during iteration" # we can't directly delete without a "RuntimeError: dictionary changed size during iteration"
@ -111,7 +109,7 @@ class TwitterBackend(BaseBackend, CapMessages, CapMessagesPost, CapCollection):
# if no date available, create a new one (compatibility with "old" storage) # if no date available, create a new one (compatibility with "old" storage)
if not date: if not date:
self.storage.set('seen', id, datetime.now()) self.storage.set('seen', id, datetime.now())
elif datetime.now() - date > timedelta(days=60): elif lastpurge > date:
todelete.append(id) todelete.append(id)
for id in todelete: for id in todelete: