nojhan/weboob-devel
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

boobot: Faster timeouts

Browse source 
This requires running the development version of Weboob.
This commit is contained in:
Laurent Bachelier 2013-05-09 20:46:03 +02:00 • committed by Romain Bignon
commit 2c633a43c8
1 changed files with 4 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

7
contrib/boobot.py
View file

@ -87,14 +87,15 @@ class HeadRequest(mechanize.Request):
class BoobotBrowser(StandardBrowser): class BoobotBrowser(StandardBrowser):
ENCODING = None ENCODING = None
DEFAULT_TIMEOUT = 3
def urlinfo(self, url): def urlinfo(self, url):
try: try:
r = self.openurl(HeadRequest(url)) r = self.openurl(HeadRequest(url), _tries=2, _delay=0.2)
body = False body = False
except BrowserHTTPError as e: except BrowserHTTPError as e:
if 'HTTP Error 501' in unicode(e) or 'HTTP Error 405' in unicode(e): if 'HTTP Error 501' in unicode(e) or 'HTTP Error 405' in unicode(e):
r = self.openurl(url) r = self.openurl(url, _tries=2, _delay=0.2)
body = True body = True
else: else:
raise e raise e
@ -110,7 +111,7 @@ class BoobotBrowser(StandardBrowser):
title = None title = None
if is_html: if is_html:
if not body: if not body:
r = self.openurl(url) r = self.openurl(url, _tries=2, _delay=0.2)
encoding = EncodingFinder('windows-1252').encoding(r).lower() encoding = EncodingFinder('windows-1252').encoding(r).lower()
if encoding == 'iso-8859-1': if encoding == 'iso-8859-1':
encoding = 'windows-1252' encoding = 'windows-1252'