add decorator to retry failed requests

weboob/tools/browser.py

@@ -24,11 +24,12 @@ import urllib2
 import ClientForm
 import re
 import time
-from logging import warning, error, debug
+from logging import warning, debug
 from copy import copy
 from threading import RLock
 
 from weboob.tools.parsers import get_parser
+from weboob.tools.decorators import retry
 
 # Try to load cookies
 try:
@@ -197,19 +198,21 @@ class BaseBrowser(mechanize.Browser):
         return inner
 
     @check_location
+    @retry(BrowserUnavailable, tries=3)
     def openurl(self, *args, **kwargs):
         """
         Open an URL but do not create a Page object.
         """
+        debug('Opening URL "%s", %s' % (args, kwargs))
         try:
             return mechanize.Browser.open(self, *args, **kwargs)
         except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError), e:
-            error('Error opening URL "%s": %s' % (args and args[0] or 'None', e))
+            raise BrowserUnavailable('%s (url="%s")' % (e, args and args[0] or 'None'))
-            raise BrowserUnavailable()
         except mechanize.BrowserStateError:
             self.home()
             return mechanize.Browser.open(self, *args, **kwargs)
 
+    @retry(BrowserUnavailable, tries=3)
     def submit(self, *args, **kwargs):
         """
         Submit the selected form.
@@ -217,28 +220,28 @@ class BaseBrowser(mechanize.Browser):
         try:
             self._change_location(mechanize.Browser.submit(self, *args, **kwargs))
         except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError), e:
-            error('Error submitting FORM: %s' % e)
             self.page = None
-            raise BrowserUnavailable()
+            raise BrowserUnavailable(e)
-        except (mechanize.BrowserStateError, BrowserRetry):
+        except (mechanize.BrowserStateError, BrowserRetry), e:
             self.home()
-            raise BrowserUnavailable()
+            raise BrowserUnavailable(e)
 
     def is_on_page(self, pageCls):
         return isinstance(self.page, pageCls)
 
+    @retry(BrowserUnavailable, tries=3)
     def follow_link(self, *args, **kwargs):
         try:
             self._change_location(mechanize.Browser.follow_link(self, *args, **kwargs))
         except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError), e:
-            error('Error following link "%s": %s' % (args and args[0] or "None", e))
             self.page = None
-            raise BrowserUnavailable()
+            raise BrowserUnavailable('%s (url="%s")' % (e, args and args[0] or 'None'))
-        except (mechanize.BrowserStateError, BrowserRetry):
+        except (mechanize.BrowserStateError, BrowserRetry), e:
             self.home()
-            raise BrowserUnavailable()
+            raise BrowserUnavailable(e)
 
     @check_location
+    @retry(BrowserUnavailable, tries=3)
     def location(self, *args, **kwargs):
         """
         Change location of browser on an URL.
@@ -259,9 +262,8 @@ class BaseBrowser(mechanize.Browser):
             if not self.page or not args or self.page.url != args[0]:
                 self.location(keep_args, keep_kwargs)
         except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError), e:
-            error('Error changing location to "%s": %s' % (args and args[0] or 'None', e))
             self.page = None
-            raise BrowserUnavailable()
+            raise BrowserUnavailable('%s (url="%s")' % (e, args and args[0] or 'None'))
         except mechanize.BrowserStateError:
             self.home()
             self.location(*keep_args, **keep_kwargs)

weboob/tools/decorators.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2010  Christophe Benz
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+import logging
+import time
+
+
+__all__ = ['retry']
+
+
+def retry(ExceptionToCheck, tries=4, delay=3, backoff=2):
+    """
+    Retry decorator
+    from http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
+    original from http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
+    """
+    def deco_retry(f):
+        def f_retry(*args, **kwargs):
+            mtries, mdelay = tries, delay
+            try_one_last_time = True
+            while mtries > 1:
+                try:
+                    return f(*args, **kwargs)
+                    try_one_last_time = False
+                    break
+                except ExceptionToCheck, e:
+                    logging.debug(u'%s, Retrying in %d seconds...' % (e, mdelay))
+                    time.sleep(mdelay)
+                    mtries -= 1
+                    mdelay *= backoff
+            if try_one_last_time:
+                return f(*args, **kwargs)
+            return
+        return f_retry # true decorator
+    return deco_retry