nojhan/weboob-devel
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

aibility to backends to define a specific pages charset

Browse source
This commit is contained in:
Romain Bignon 2010-04-04 16:51:05 +02:00
commit 655bc10e3e
3 changed files with 11 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

5
weboob/backends/aum/adopte.py
View file

@ -39,12 +39,13 @@ class AdopteParser(HTMLParser):
def __init__(self): def __init__(self):
HTMLParser.__init__(self, tree=treebuilders.getTreeBuilder("dom")) HTMLParser.__init__(self, tree=treebuilders.getTreeBuilder("dom"))
def parse(self, data): def parse(self, data, encoding):
return HTMLParser.parse(self, data, encoding='iso-8859-1') return HTMLParser.parse(self, data, encoding=encoding)
class AdopteUnMec(Browser): class AdopteUnMec(Browser):
DOMAIN = 'www.adopteunmec.com' DOMAIN = 'www.adopteunmec.com'
PROTOCOL = 'http' PROTOCOL = 'http'
ENCODING = 'iso-8859-1'
PAGES = {'http://www.adopteunmec.com/': LoginPage, PAGES = {'http://www.adopteunmec.com/': LoginPage,
'http://www.adopteunmec.com/index.html': LoginPage, 'http://www.adopteunmec.com/index.html': LoginPage,
'http://www.adopteunmec.com/index.php': LoginPage, 'http://www.adopteunmec.com/index.php': LoginPage,

7
weboob/tools/browser.py
View file

@ -84,6 +84,7 @@ class Browser(mechanize.Browser):
DOMAIN = None DOMAIN = None
PROTOCOL = 'http' PROTOCOL = 'http'
ENCODING = 'utf-8'
PAGES = {} PAGES = {}
USER_AGENT = 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.4) Gecko/2008111318 Ubuntu/8.10 (intrepid) Firefox/3.0.3' USER_AGENT = 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.4) Gecko/2008111318 Ubuntu/8.10 (intrepid) Firefox/3.0.3'
@ -110,7 +111,7 @@ class Browser(mechanize.Browser):
# ------ Browser methods --------------------------------------- # ------ Browser methods ---------------------------------------
def __init__(self, username=None, password=None, firefox_cookies=None, parser=StandardParser): def __init__(self, username=None, password=None, firefox_cookies=None, parser=StandardParser):
mechanize.Browser.__init__(self, history=NoHistory()) mechanize.Browser.__init__(self)#, history=NoHistory())
self.addheaders = [ self.addheaders = [
['User-agent', self.USER_AGENT] ['User-agent', self.USER_AGENT]
] ]
@ -235,12 +236,12 @@ class Browser(mechanize.Browser):
debug('[%s] Gone on %s' % (self.username, result.geturl())) debug('[%s] Gone on %s' % (self.username, result.geturl()))
self.last_update = time.time() self.last_update = time.time()
document = self.__parser.parse(result) document = self.__parser.parse(result, self.ENCODING)
self.page = pageCls(self, document, result.geturl()) self.page = pageCls(self, document, result.geturl())
self.page.loaded() self.page.loaded()
if self.password and not self.is_logged(): if self.password and not self.is_logged():
print '!! Relogin !!' debug('!! Relogin !!')
self.login() self.login()
return return

11
weboob/tools/parser.py
View file

@ -26,15 +26,12 @@ except ImportError:
try: try:
from elementtidy import TidyHTMLTreeBuilder from elementtidy import TidyHTMLTreeBuilder
TidyHTMLTreeBuilder.ElementTree = ElementTree # force cElementTree if using it. TidyHTMLTreeBuilder.ElementTree = ElementTree # force cElementTree if using it.
class HTMLTreeBuilder(TidyHTMLTreeBuilder.TidyHTMLTreeBuilder): HTMLTreeBuilder = TidyHTMLTreeBuilder.TidyHTMLTreeBuilder
def __init__(self):
TidyHTMLTreeBuilder.TidyHTMLTreeBuilder.__init__(self, 'utf-8')
except ImportError: except ImportError:
from HTMLParser import HTMLParser from HTMLParser import HTMLParser
class HTMLTreeBuilder(HTMLParser): class HTMLTreeBuilder(HTMLParser):
def __init__(self, html=0, target=None): def __init__(self, encoding=None):
HTMLParser.__init__(self) HTMLParser.__init__(self)
if target is None: if target is None:
target = ElementTree.TreeBuilder() target = ElementTree.TreeBuilder()
@ -61,8 +58,8 @@ except ImportError:
self._target.end(tag) self._target.end(tag)
class StandardParser(object): class StandardParser(object):
def parse(self, data): def parse(self, data, encoding=None):
parser = HTMLTreeBuilder() parser = HTMLTreeBuilder(encoding)
tree = ElementTree.parse(data, parser) tree = ElementTree.parse(data, parser)
for elem in tree.getiterator(): for elem in tree.getiterator():