75 lines
2.6 KiB
Python
75 lines
2.6 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
"""
|
|
Copyright(C) 2010 Romain Bignon
|
|
|
|
This program is free software; you can redistribute it and/or modify
|
|
it under the terms of the GNU General Public License as published by
|
|
the Free Software Foundation, version 3 of the License.
|
|
|
|
This program is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
GNU General Public License for more details.
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
along with this program; if not, write to the Free Software
|
|
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
|
|
|
"""
|
|
|
|
from logging import debug
|
|
from random import randint
|
|
from weboob.tools.browser import BrowserUnavailable
|
|
|
|
class ProfilesWalker(object):
|
|
def __init__(self, sched, storage, browser):
|
|
self.sched = sched
|
|
self.storage = storage
|
|
self.browser = browser
|
|
|
|
self.visited_profiles = set(storage.get('profiles_walker', 'viewed'))
|
|
self.profiles_queue = set()
|
|
self.walk_cron = sched.repeat(60, self.walk)
|
|
self.view_cron = sched.schedule(randint(10,40), self.view_profile)
|
|
|
|
def save(self):
|
|
self.storage.set('profiles_walker', 'viewed', self.visited_profiles)
|
|
self.storage.save()
|
|
|
|
def stop(self):
|
|
self.event.cancel(self.event)
|
|
self.event = None
|
|
|
|
def walk(self):
|
|
self.profiles_queue = self.profiles_queue.union(self.browser.search_profiles()).difference(self.visited_profiles)
|
|
self.save()
|
|
|
|
def view_profile(self):
|
|
try:
|
|
try:
|
|
id = self.profiles_queue.pop()
|
|
except KeyError:
|
|
return # empty queue
|
|
|
|
try:
|
|
profile = self.browser.get_profile(id)
|
|
debug(u'Visited %s (%s)' % (profile.get_name(), id))
|
|
|
|
# Get score from the aum_score module
|
|
# d = self.nucentral_core.callService(context.Context.fromComponent(self), 'aum_score', 'score', profile)
|
|
# d.addCallback(self.score_cb, profile.getID())
|
|
# deferredlist.append(d)
|
|
|
|
# do not forget that we visited this profile, to avoid re-visiting it.
|
|
self.visited_profiles.add(id)
|
|
self.save()
|
|
|
|
except BrowserUnavailable:
|
|
# We consider this profil hasn't been [correctly] analysed
|
|
self.profiles_queue.add(id)
|
|
return
|
|
except Exception, e:
|
|
print e
|
|
finally:
|
|
self.sched.schedule(randint(10,40), self.view_profile)
|