nojhan/weboob-devel
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Do not use retry() with 404 errors

Browse source
This commit is contained in:
Romain Bignon 2011-03-19 11:24:52 +01:00
commit 7b61871f51
2 changed files with 17 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

5
weboob/tools/browser/__init__.py
View file

@ -18,8 +18,9 @@
from weboob.tools.browser.browser import BrowserIncorrectPassword, BrowserBanned, \ from weboob.tools.browser.browser import BrowserIncorrectPassword, BrowserBanned, \
BrowserUnavailable, BrowserRetry, \ BrowserUnavailable, BrowserRetry, \
BrowserHTTPError, BasePage, BaseBrowser BrowserHTTPNotFound, BrowserHTTPError, \
BasePage, BaseBrowser
__all__ = ['BrowserIncorrectPassword', 'BrowserBanned', 'BrowserUnavailable', 'BrowserRetry', __all__ = ['BrowserIncorrectPassword', 'BrowserBanned', 'BrowserUnavailable', 'BrowserRetry',
'BrowserHTTPError', 'BasePage', 'BaseBrowser'] 'BrowserHTTPNotFound', 'BrowserHTTPError', 'BasePage', 'BaseBrowser']

19
weboob/tools/browser/browser.py
View file

@ -46,7 +46,7 @@ else:
__all__ = ['BrowserIncorrectPassword', 'BrowserBanned', 'BrowserUnavailable', 'BrowserRetry', __all__ = ['BrowserIncorrectPassword', 'BrowserBanned', 'BrowserUnavailable', 'BrowserRetry',
'BrowserHTTPError', 'BasePage', 'BaseBrowser'] 'BrowserHTTPNotFound', 'BrowserHTTPError', 'BasePage', 'BaseBrowser']
# Exceptions # Exceptions
@ -61,6 +61,9 @@ class BrowserBanned(BrowserIncorrectPassword):
class BrowserUnavailable(Exception): class BrowserUnavailable(Exception):
pass pass
class BrowserHTTPNotFound(BrowserUnavailable):
pass
class BrowserHTTPError(BrowserUnavailable): class BrowserHTTPError(BrowserUnavailable):
pass pass
@ -272,13 +275,19 @@ class BaseBrowser(mechanize.Browser):
return mechanize.Browser.open_novisit(self, *args, **kwargs) return mechanize.Browser.open_novisit(self, *args, **kwargs)
except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError, BadStatusLine), e: except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError, BadStatusLine), e:
if if_fail == 'raise': if if_fail == 'raise':
raise BrowserHTTPError('%s (url="%s")' % (e, args and args[0] or 'None')) raise self.get_exception(e)('%s (url="%s")' % (e, args and args[0] or 'None'))
else: else:
return None return None
except (mechanize.BrowserStateError, BrowserRetry): except (mechanize.BrowserStateError, BrowserRetry):
self.home() self.home()
return mechanize.Browser.open(self, *args, **kwargs) return mechanize.Browser.open(self, *args, **kwargs)
def get_exception(self, e):
if isinstance(e, urllib2.HTTPError) and e.getcode() == 404:
return BrowserHTTPNotFound
else:
return BrowserHTTPError
def readurl(self, url, *args, **kwargs): def readurl(self, url, *args, **kwargs):
""" """
Download URL data specifying what to do on failure (nothing by default). Download URL data specifying what to do on failure (nothing by default).
@ -325,7 +334,7 @@ class BaseBrowser(mechanize.Browser):
self._change_location(mechanize.Browser.submit(self, *args, **kwargs)) self._change_location(mechanize.Browser.submit(self, *args, **kwargs))
except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError, BadStatusLine), e: except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError, BadStatusLine), e:
self.page = None self.page = None
raise BrowserHTTPError(e) raise self.get_exception(e)(e)
except (mechanize.BrowserStateError, BrowserRetry), e: except (mechanize.BrowserStateError, BrowserRetry), e:
self.home() self.home()
raise BrowserUnavailable(e) raise BrowserUnavailable(e)
@ -345,7 +354,7 @@ class BaseBrowser(mechanize.Browser):
self._change_location(mechanize.Browser.follow_link(self, *args, **kwargs)) self._change_location(mechanize.Browser.follow_link(self, *args, **kwargs))
except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError, BadStatusLine), e: except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError, BadStatusLine), e:
self.page = None self.page = None
raise BrowserHTTPError('%s (url="%s")' % (e, args and args[0] or 'None')) raise self.get_exception(e)('%s (url="%s")' % (e, args and args[0] or 'None'))
except (mechanize.BrowserStateError, BrowserRetry), e: except (mechanize.BrowserStateError, BrowserRetry), e:
self.home() self.home()
raise BrowserUnavailable(e) raise BrowserUnavailable(e)
@ -375,7 +384,7 @@ class BaseBrowser(mechanize.Browser):
self.location(keep_args, keep_kwargs) self.location(keep_args, keep_kwargs)
except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError, BadStatusLine), e: except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError, BadStatusLine), e:
self.page = None self.page = None
raise BrowserHTTPError('%s (url="%s")' % (e, args and args[0] or 'None')) raise self.get_exception(e)('%s (url="%s")' % (e, args and args[0] or 'None'))
except mechanize.BrowserStateError: except mechanize.BrowserStateError:
self.home() self.home()
self.location(*keep_args, **keep_kwargs) self.location(*keep_args, **keep_kwargs)