Improve EHentai backend

weboob/backends/ehentai/backend.py

@@ -19,8 +19,10 @@
 
 from __future__ import with_statement
 
+import time
 from weboob.capabilities.gallery import ICapGallery
 from weboob.tools.backend import BaseBackend
+from weboob.tools.misc import to_unicode, ratelimit
 from weboob.tools.value import Value, ValuesDict
 
 from .browser import EHentaiBrowser
@@ -43,6 +45,10 @@ class EHentaiBackend(BaseBackend, ICapGallery):
         Value('username', label='Username', default=''),
         Value('password', label='Password', default='', masked=True))
 
+    def __init__(self, *args, **kwargs):
+        BaseBackend.__init__(self, *args, **kwargs)
+        self.time_last_retreived = 0
+
     def create_default_browser(self):
         return self.create_browser(
                 self.config['domain'],
@@ -68,6 +74,15 @@ class EHentaiBackend(BaseBackend, ICapGallery):
     def fill_image(self, image, fields):
         with self.browser:
             image.url = self.browser.get_image_url(image)
+            if 'data' in fields:
+                #offset = time.time() - self.time_last_retreived
+                #if offset < 2:
+                #    time.sleep(2 - offset)
+                #self.time_last_retreived = time.time()
+
+                def get():
+                    image.data = self.browser.readurl(image.url)
+                ratelimit(get, "ehentai_get", 2)
 
     OBJECTS = {
             EHentaiGallery: fill_gallery,

weboob/capabilities/gallery.py

@@ -78,6 +78,16 @@ class BaseImage(CapBaseObject):
         self.add_field('thumbnail', Thumbnail, thumbnail)
         self.add_field('url', basestring, url)
         self.add_field('ext', basestring, ext)
+        self.add_field('data', str)
+
+    def __str__(self):
+        return self.url
+
+    def __repr__(self):
+        return '<Image url="%s">' % self.url
+
+    def __iscomplete__(self):
+        return self.data is not NotLoaded
 
 class ICapGallery(IBaseCap):
     """

weboob/tools/misc.py

@@ -20,6 +20,9 @@
 
 from dateutil import tz
 from logging import warning
+from random import random
+from time import time, sleep
+from os import stat, utime
 import sys
 import traceback
 import types
@@ -117,3 +120,22 @@ def limit(iterator, lim):
         yield iterator.next()
         count += 1
     raise StopIteration()
+
+def ratelimit(function, group, delay):
+    path = '/tmp/weboob_ratelimit.%s' % group
+    while True:
+        try:
+            offset = time() - stat(path).st_mtime
+        except OSError:
+            with open(path, 'w'):
+                pass
+            print 'creating %s' % path
+            offset = 0
+
+        if delay < offset:
+            break
+
+        sleep(delay - offset)
+
+    utime(path, None)
+    function()