nojhan/weboob-devel
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

boobot: Faster timeouts

Browse source 
This requires running the development version of Weboob.
This commit is contained in:
Laurent Bachelier 2013-05-09 20:46:03 +02:00 • committed by Romain Bignon
commit 2c633a43c8
1 changed files with 4 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

7
contrib/boobot.py
View file

@ -87,14 +87,15 @@ class HeadRequest(mechanize.Request):
class BoobotBrowser(StandardBrowser):
ENCODING = None
DEFAULT_TIMEOUT = 3
def urlinfo(self, url):
try:
r = self.openurl(HeadRequest(url))
r = self.openurl(HeadRequest(url), _tries=2, _delay=0.2)
body = False
except BrowserHTTPError as e:
if 'HTTP Error 501' in unicode(e) or 'HTTP Error 405' in unicode(e):
r = self.openurl(url)
r = self.openurl(url, _tries=2, _delay=0.2)
body = True
else:
raise e
@ -110,7 +111,7 @@ class BoobotBrowser(StandardBrowser):
title = None
if is_html:
if not body:
r = self.openurl(url)
r = self.openurl(url, _tries=2, _delay=0.2)
encoding = EncodingFinder('windows-1252').encoding(r).lower()
if encoding == 'iso-8859-1':
encoding = 'windows-1252'