nojhan/weboob-devel
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

boobot: Support servers with no HEAD

Browse source
This commit is contained in:
Laurent Bachelier 2013-05-05 21:23:30 +02:00
commit 3710bb87ae
1 changed files with 10 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

13
contrib/boobot.py
View file

@ -34,7 +34,7 @@ from mechanize import _headersutil as headersutil
from mechanize._html import EncodingFinder from mechanize._html import EncodingFinder
from weboob.core import Weboob from weboob.core import Weboob
from weboob.tools.browser import StandardBrowser, BrowserUnavailable from weboob.tools.browser import StandardBrowser, BrowserUnavailable, BrowserHTTPError
from weboob.tools.misc import get_backtrace from weboob.tools.misc import get_backtrace
from weboob.tools.misc import to_unicode from weboob.tools.misc import to_unicode
from weboob.tools.storage import StandardStorage from weboob.tools.storage import StandardStorage
@ -88,7 +88,13 @@ class BoobotBrowser(StandardBrowser):
ENCODING = None ENCODING = None
def urlinfo(self, url): def urlinfo(self, url):
r = self.openurl(HeadRequest(url)) try:
r = self.openurl(HeadRequest(url))
body = False
except BrowserHTTPError as e:
if 'HTTP Error 501' in unicode(e):
r = self.openurl(url)
body = True
headers = r.info() headers = r.info()
content_type = headers.get('Content-Type') content_type = headers.get('Content-Type')
try: try:
@ -100,7 +106,8 @@ class BoobotBrowser(StandardBrowser):
is_html = headersutil.is_html([content_type], url, True) is_html = headersutil.is_html([content_type], url, True)
title = None title = None
if is_html: if is_html:
r = self.openurl(url) if not body:
r = self.openurl(url)
encoding = EncodingFinder('windows-1252').encoding(r).lower().replace('iso-8859-1', 'windows-1252') encoding = EncodingFinder('windows-1252').encoding(r).lower().replace('iso-8859-1', 'windows-1252')
h = self.get_document(r, parser='lxml', encoding=encoding) h = self.get_document(r, parser='lxml', encoding=encoding)
for title in h.xpath('//head/title'): for title in h.xpath('//head/title'):