From 7b61871f51e57754a87f2d6233755e6f362ae7fb Mon Sep 17 00:00:00 2001 From: Romain Bignon Date: Sat, 19 Mar 2011 11:24:52 +0100 Subject: [PATCH] Do not use retry() with 404 errors --- weboob/tools/browser/__init__.py | 5 +++-- weboob/tools/browser/browser.py | 19 ++++++++++++++----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/weboob/tools/browser/__init__.py b/weboob/tools/browser/__init__.py index a65d9c84..a75c8174 100644 --- a/weboob/tools/browser/__init__.py +++ b/weboob/tools/browser/__init__.py @@ -18,8 +18,9 @@ from weboob.tools.browser.browser import BrowserIncorrectPassword, BrowserBanned, \ BrowserUnavailable, BrowserRetry, \ - BrowserHTTPError, BasePage, BaseBrowser + BrowserHTTPNotFound, BrowserHTTPError, \ + BasePage, BaseBrowser __all__ = ['BrowserIncorrectPassword', 'BrowserBanned', 'BrowserUnavailable', 'BrowserRetry', - 'BrowserHTTPError', 'BasePage', 'BaseBrowser'] + 'BrowserHTTPNotFound', 'BrowserHTTPError', 'BasePage', 'BaseBrowser'] diff --git a/weboob/tools/browser/browser.py b/weboob/tools/browser/browser.py index ea4b84fc..6a9a8225 100644 --- a/weboob/tools/browser/browser.py +++ b/weboob/tools/browser/browser.py @@ -46,7 +46,7 @@ else: __all__ = ['BrowserIncorrectPassword', 'BrowserBanned', 'BrowserUnavailable', 'BrowserRetry', - 'BrowserHTTPError', 'BasePage', 'BaseBrowser'] + 'BrowserHTTPNotFound', 'BrowserHTTPError', 'BasePage', 'BaseBrowser'] # Exceptions @@ -61,6 +61,9 @@ class BrowserBanned(BrowserIncorrectPassword): class BrowserUnavailable(Exception): pass +class BrowserHTTPNotFound(BrowserUnavailable): + pass + class BrowserHTTPError(BrowserUnavailable): pass @@ -272,13 +275,19 @@ class BaseBrowser(mechanize.Browser): return mechanize.Browser.open_novisit(self, *args, **kwargs) except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError, BadStatusLine), e: if if_fail == 'raise': - raise BrowserHTTPError('%s (url="%s")' % (e, args and args[0] or 'None')) + raise self.get_exception(e)('%s (url="%s")' % (e, args and args[0] or 'None')) else: return None except (mechanize.BrowserStateError, BrowserRetry): self.home() return mechanize.Browser.open(self, *args, **kwargs) + def get_exception(self, e): + if isinstance(e, urllib2.HTTPError) and e.getcode() == 404: + return BrowserHTTPNotFound + else: + return BrowserHTTPError + def readurl(self, url, *args, **kwargs): """ Download URL data specifying what to do on failure (nothing by default). @@ -325,7 +334,7 @@ class BaseBrowser(mechanize.Browser): self._change_location(mechanize.Browser.submit(self, *args, **kwargs)) except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError, BadStatusLine), e: self.page = None - raise BrowserHTTPError(e) + raise self.get_exception(e)(e) except (mechanize.BrowserStateError, BrowserRetry), e: self.home() raise BrowserUnavailable(e) @@ -345,7 +354,7 @@ class BaseBrowser(mechanize.Browser): self._change_location(mechanize.Browser.follow_link(self, *args, **kwargs)) except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError, BadStatusLine), e: self.page = None - raise BrowserHTTPError('%s (url="%s")' % (e, args and args[0] or 'None')) + raise self.get_exception(e)('%s (url="%s")' % (e, args and args[0] or 'None')) except (mechanize.BrowserStateError, BrowserRetry), e: self.home() raise BrowserUnavailable(e) @@ -375,7 +384,7 @@ class BaseBrowser(mechanize.Browser): self.location(keep_args, keep_kwargs) except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError, BadStatusLine), e: self.page = None - raise BrowserHTTPError('%s (url="%s")' % (e, args and args[0] or 'None')) + raise self.get_exception(e)('%s (url="%s")' % (e, args and args[0] or 'None')) except mechanize.BrowserStateError: self.home() self.location(*keep_args, **keep_kwargs)