nojhan/weboob-devel
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

boobot: Guess page encoding

Browse source 
This seems to work with most pages.
This commit is contained in:
Laurent Bachelier 2013-05-05 20:40:51 +02:00
commit ab7287747a
1 changed files with 7 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

11
contrib/boobot.py
View file

@ -31,7 +31,7 @@ import urllib
from irc.bot import SingleServerIRCBot from irc.bot import SingleServerIRCBot
import mechanize import mechanize
from mechanize import _headersutil as headersutil from mechanize import _headersutil as headersutil
import lxml.html from mechanize._html import EncodingFinder
from weboob.core import Weboob from weboob.core import Weboob
from weboob.tools.browser import StandardBrowser, BrowserUnavailable from weboob.tools.browser import StandardBrowser, BrowserUnavailable
@ -85,9 +85,10 @@ class HeadRequest(mechanize.Request):
class BoobotBrowser(StandardBrowser): class BoobotBrowser(StandardBrowser):
ENCODING = None
def urlinfo(self, url): def urlinfo(self, url):
b = StandardBrowser() r = self.openurl(HeadRequest(url))
r = b.openurl(HeadRequest(url))
headers = r.info() headers = r.info()
content_type = headers.get('Content-Type') content_type = headers.get('Content-Type')
try: try:
@ -99,7 +100,9 @@ class BoobotBrowser(StandardBrowser):
is_html = headersutil.is_html([content_type], url, True) is_html = headersutil.is_html([content_type], url, True)
title = None title = None
if is_html: if is_html:
h = lxml.html.fromstring(self.readurl(url)) r = self.openurl(url)
encoding = EncodingFinder('windows-1252').encoding(r)
h = self.get_document(r, parser='lxml', encoding=encoding)
for title in h.xpath('//head/title'): for title in h.xpath('//head/title'):
title = to_unicode(title.text_content()) title = to_unicode(title.text_content())
return content_type, hsize, title return content_type, hsize, title