From 09e2e74d34fdfd4be5d7d715149ed09580bc5d5f Mon Sep 17 00:00:00 2001 From: Christophe Benz Date: Sun, 16 May 2010 17:42:39 +0200 Subject: [PATCH] add decorator to retry failed requests --- weboob/tools/browser.py | 28 ++++++++++++---------- weboob/tools/decorators.py | 49 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 13 deletions(-) create mode 100644 weboob/tools/decorators.py diff --git a/weboob/tools/browser.py b/weboob/tools/browser.py index b397fed1..e27b8c30 100644 --- a/weboob/tools/browser.py +++ b/weboob/tools/browser.py @@ -24,11 +24,12 @@ import urllib2 import ClientForm import re import time -from logging import warning, error, debug +from logging import warning, debug from copy import copy from threading import RLock from weboob.tools.parsers import get_parser +from weboob.tools.decorators import retry # Try to load cookies try: @@ -197,19 +198,21 @@ class BaseBrowser(mechanize.Browser): return inner @check_location + @retry(BrowserUnavailable, tries=3) def openurl(self, *args, **kwargs): """ Open an URL but do not create a Page object. """ + debug('Opening URL "%s", %s' % (args, kwargs)) try: return mechanize.Browser.open(self, *args, **kwargs) except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError), e: - error('Error opening URL "%s": %s' % (args and args[0] or 'None', e)) - raise BrowserUnavailable() + raise BrowserUnavailable('%s (url="%s")' % (e, args and args[0] or 'None')) except mechanize.BrowserStateError: self.home() return mechanize.Browser.open(self, *args, **kwargs) + @retry(BrowserUnavailable, tries=3) def submit(self, *args, **kwargs): """ Submit the selected form. @@ -217,28 +220,28 @@ class BaseBrowser(mechanize.Browser): try: self._change_location(mechanize.Browser.submit(self, *args, **kwargs)) except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError), e: - error('Error submitting FORM: %s' % e) self.page = None - raise BrowserUnavailable() - except (mechanize.BrowserStateError, BrowserRetry): + raise BrowserUnavailable(e) + except (mechanize.BrowserStateError, BrowserRetry), e: self.home() - raise BrowserUnavailable() + raise BrowserUnavailable(e) def is_on_page(self, pageCls): return isinstance(self.page, pageCls) + @retry(BrowserUnavailable, tries=3) def follow_link(self, *args, **kwargs): try: self._change_location(mechanize.Browser.follow_link(self, *args, **kwargs)) except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError), e: - error('Error following link "%s": %s' % (args and args[0] or "None", e)) self.page = None - raise BrowserUnavailable() - except (mechanize.BrowserStateError, BrowserRetry): + raise BrowserUnavailable('%s (url="%s")' % (e, args and args[0] or 'None')) + except (mechanize.BrowserStateError, BrowserRetry), e: self.home() - raise BrowserUnavailable() + raise BrowserUnavailable(e) @check_location + @retry(BrowserUnavailable, tries=3) def location(self, *args, **kwargs): """ Change location of browser on an URL. @@ -259,9 +262,8 @@ class BaseBrowser(mechanize.Browser): if not self.page or not args or self.page.url != args[0]: self.location(keep_args, keep_kwargs) except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError), e: - error('Error changing location to "%s": %s' % (args and args[0] or 'None', e)) self.page = None - raise BrowserUnavailable() + raise BrowserUnavailable('%s (url="%s")' % (e, args and args[0] or 'None')) except mechanize.BrowserStateError: self.home() self.location(*keep_args, **keep_kwargs) diff --git a/weboob/tools/decorators.py b/weboob/tools/decorators.py new file mode 100644 index 00000000..e78b93cf --- /dev/null +++ b/weboob/tools/decorators.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2010 Christophe Benz +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +import logging +import time + + +__all__ = ['retry'] + + +def retry(ExceptionToCheck, tries=4, delay=3, backoff=2): + """ + Retry decorator + from http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/ + original from http://wiki.python.org/moin/PythonDecoratorLibrary#Retry + """ + def deco_retry(f): + def f_retry(*args, **kwargs): + mtries, mdelay = tries, delay + try_one_last_time = True + while mtries > 1: + try: + return f(*args, **kwargs) + try_one_last_time = False + break + except ExceptionToCheck, e: + logging.debug(u'%s, Retrying in %d seconds...' % (e, mdelay)) + time.sleep(mdelay) + mtries -= 1 + mdelay *= backoff + if try_one_last_time: + return f(*args, **kwargs) + return + return f_retry # true decorator + return deco_retry