diff --git a/weboob/backends/ehentai/backend.py b/weboob/backends/ehentai/backend.py index 5cb0b95e..077978c9 100644 --- a/weboob/backends/ehentai/backend.py +++ b/weboob/backends/ehentai/backend.py @@ -19,8 +19,10 @@ from __future__ import with_statement +import time from weboob.capabilities.gallery import ICapGallery from weboob.tools.backend import BaseBackend +from weboob.tools.misc import to_unicode, ratelimit from weboob.tools.value import Value, ValuesDict from .browser import EHentaiBrowser @@ -43,6 +45,10 @@ class EHentaiBackend(BaseBackend, ICapGallery): Value('username', label='Username', default=''), Value('password', label='Password', default='', masked=True)) + def __init__(self, *args, **kwargs): + BaseBackend.__init__(self, *args, **kwargs) + self.time_last_retreived = 0 + def create_default_browser(self): return self.create_browser( self.config['domain'], @@ -68,6 +74,15 @@ class EHentaiBackend(BaseBackend, ICapGallery): def fill_image(self, image, fields): with self.browser: image.url = self.browser.get_image_url(image) + if 'data' in fields: + #offset = time.time() - self.time_last_retreived + #if offset < 2: + # time.sleep(2 - offset) + #self.time_last_retreived = time.time() + + def get(): + image.data = self.browser.readurl(image.url) + ratelimit(get, "ehentai_get", 2) OBJECTS = { EHentaiGallery: fill_gallery, diff --git a/weboob/capabilities/gallery.py b/weboob/capabilities/gallery.py index 45c59c1f..05352332 100644 --- a/weboob/capabilities/gallery.py +++ b/weboob/capabilities/gallery.py @@ -78,6 +78,16 @@ class BaseImage(CapBaseObject): self.add_field('thumbnail', Thumbnail, thumbnail) self.add_field('url', basestring, url) self.add_field('ext', basestring, ext) + self.add_field('data', str) + + def __str__(self): + return self.url + + def __repr__(self): + return '' % self.url + + def __iscomplete__(self): + return self.data is not NotLoaded class ICapGallery(IBaseCap): """ diff --git a/weboob/tools/misc.py b/weboob/tools/misc.py index df09ac09..95bd496f 100644 --- a/weboob/tools/misc.py +++ b/weboob/tools/misc.py @@ -20,6 +20,9 @@ from dateutil import tz from logging import warning +from random import random +from time import time, sleep +from os import stat, utime import sys import traceback import types @@ -117,3 +120,22 @@ def limit(iterator, lim): yield iterator.next() count += 1 raise StopIteration() + +def ratelimit(function, group, delay): + path = '/tmp/weboob_ratelimit.%s' % group + while True: + try: + offset = time() - stat(path).st_mtime + except OSError: + with open(path, 'w'): + pass + print 'creating %s' % path + offset = 0 + + if delay < offset: + break + + sleep(delay - offset) + + utime(path, None) + function()