[aum] new 'profiles walker' optimization feature

weboob/backends/aum/optim/profiles_walker.py

@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+
+"""
+Copyright(C) 2010  Romain Bignon
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, version 3 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+"""
+
+from logging import debug
+from random import randint
+from weboob.tools.browser import BrowserUnavailable
+
+class ProfilesWalker(object):
+    def __init__(self, sched, storage, browser):
+        self.sched = sched
+        self.storage = storage
+        self.browser = browser
+
+        self.visited_profiles = set(storage.get('profiles_walker', 'viewed'))
+        self.profiles_queue = set()
+        self.walk_cron = sched.repeat(60, self.walk)
+        self.view_cron = sched.schedule(randint(10,40), self.view_profile)
+
+    def save(self):
+        self.storage.set('profiles_walker', 'viewed', self.visited_profiles)
+        self.storage.save()
+
+    def stop(self):
+        self.event.cancel(self.event)
+        self.event = None
+
+    def walk(self):
+        self.profiles_queue = self.profiles_queue.union(self.browser.search_profiles()).difference(self.visited_profiles)
+        self.save()
+
+    def view_profile(self):
+        try:
+            try:
+                id = self.profiles_queue.pop()
+            except KeyError:
+                return # empty queue
+
+            try:
+                profile = self.browser.get_profile(id)
+                debug(u'Visited %s (%s)' % (profile.get_name(), id))
+
+                # Get score from the aum_score module
+                # d = self.nucentral_core.callService(context.Context.fromComponent(self), 'aum_score', 'score', profile)
+                # d.addCallback(self.score_cb, profile.getID())
+                # deferredlist.append(d)
+
+                # do not forget that we visited this profile, to avoid re-visiting it.
+                self.visited_profiles.add(id)
+                self.save()
+
+            except BrowserUnavailable:
+                # We consider this profil hasn't been [correctly] analysed
+                self.profiles_queue.add(id)
+                return
+            except Exception, e:
+                print e
+        finally:
+            self.sched.schedule(randint(10,40), self.view_profile)