rename parser/parsers module, add get_parsers() with preference_order

This commit is contained in:
Christophe Benz 2010-04-16 18:00:44 +02:00
commit 8638024756
13 changed files with 104 additions and 70 deletions

View file

@ -22,7 +22,7 @@ import time
from logging import warning from logging import warning
from weboob.tools.browser import BaseBrowser from weboob.tools.browser import BaseBrowser
from weboob.tools.parser import Html5libParser from weboob.tools.parsers.html5libparser import Html5libParser
from weboob.backends.aum.exceptions import AdopteWait from weboob.backends.aum.exceptions import AdopteWait

View file

@ -21,7 +21,7 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
from cStringIO import StringIO from cStringIO import StringIO
from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword
from weboob.tools.parser import ElementTidyParser from weboob.tools.parsers.elementtidyparser import ElementTidyParser
from weboob.backends.bnporc import pages from weboob.backends.bnporc import pages
# Parser # Parser

View file

@ -26,15 +26,15 @@ from .pages.index import IndexPage, LoginPage
from .pages.news import ContentPage from .pages.news import ContentPage
from .tools import id2url, id2threadid, id2contenttype from .tools import id2url, id2threadid, id2contenttype
from weboob.tools.parser import StandardParser from weboob.tools.parsers.htmlparser import HTMLParser
# Parser # Parser
class DLFParser(StandardParser): class DLFParser(HTMLParser):
def parse(self, data, encoding): def parse(self, data, encoding):
s = data.read() s = data.read()
s = s.replace('<<', '<') s = s.replace('<<', '<')
data = StringIO(s) data = StringIO(s)
return StandardParser.parse(self, data, encoding) return HTMLParser.parse(self, data, encoding)
# Browser # Browser
class DLFP(BaseBrowser): class DLFP(BaseBrowser):

View file

@ -22,15 +22,10 @@ from logging import error
import re import re
from weboob.tools.browser import BaseBrowser from weboob.tools.browser import BaseBrowser
from weboob.tools.parser import LxmlHtmlParser
class YoujizzBrowser(BaseBrowser): class YoujizzBrowser(BaseBrowser):
video_file_regex = re.compile(r'"(http://media[^ ,]+\.flv)"') video_file_regex = re.compile(r'"(http://media[^ ,]+\.flv)"')
def __init__(self, *args, **kwargs):
kwargs['parser'] = LxmlHtmlParser()
Browser.__init__(self, *args, **kwargs)
def iter_page_urls(self, mozaic_url): def iter_page_urls(self, mozaic_url):
raise NotImplementedError() raise NotImplementedError()

View file

@ -21,7 +21,7 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
import re import re
from weboob.tools.browser import BaseBrowser from weboob.tools.browser import BaseBrowser
from weboob.tools.parser import LxmlHtmlParser from weboob.tools.parsers.lxmlhtmlparser import LxmlHtmlParser
from .pages import VideoPage from .pages import VideoPage

View file

@ -26,7 +26,7 @@ import time
from logging import warning, error, debug from logging import warning, error, debug
from copy import copy from copy import copy
from weboob.tools.parser import StandardParser from weboob.tools.parsers import get_parser
# Try to load cookies # Try to load cookies
try: try:
@ -110,7 +110,7 @@ class BaseBrowser(mechanize.Browser):
# ------ Browser methods --------------------------------------- # ------ Browser methods ---------------------------------------
def __init__(self, username=None, password=None, firefox_cookies=None, parser=StandardParser(), history=NoHistory()): def __init__(self, username=None, password=None, firefox_cookies=None, parser=get_parser(), history=NoHistory()):
mechanize.Browser.__init__(self, history=history) mechanize.Browser.__init__(self, history=history)
self.addheaders = [ self.addheaders = [
['User-agent', self.USER_AGENT] ['User-agent', self.USER_AGENT]

View file

@ -0,0 +1,63 @@
# -*- coding: utf-8 -*-
"""
Copyright(C) 2010 Christophe Benz, Romain Bignon
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, version 3 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
"""
import logging
__all__ = ['get_parser', 'NoParserFound']
class NoParserFound(Exception): pass
def get_parser(preference_order=['lxml', 'html5lib', 'elementtidy', 'builtin'], *args, **kwargs):
"""
Get a parser from a preference order list.
This allows Weboob to run on systems without lxml, which is the default parser.
Return a parser implementing IParser.
"""
if not isinstance(preference_order, (tuple, list)):
preference_order = [preference_order]
for kind in preference_order:
if kind == 'lxml':
try:
from .lxmlparser import LxmlHtmlParser
return LxmlHtmlParser()
except ImportError:
logging.debug('%s is not installed.' % kind)
elif kind == 'html5lib':
try:
from .html5libparser import Html5libParser
return Html5libParser(*args, **kwargs)
except ImportError:
logging.debug('%s is not installed.' % kind)
elif kind == 'elementtidy':
try:
from .elementtidyparser import ElementTidyParser
return ElementTidyParser()
except ImportError:
logging.debug('%s is not installed.' % kind)
elif kind == 'builtin':
try:
from .htmlparser import HTMLParser
return HTMLParser()
except ImportError:
logging.debug('%s is not installed.' % kind)
raise NoParserFound()

View file

@ -0,0 +1,67 @@
# -*- coding: utf-8 -*-
"""
Copyright(C) 2010 Romain Bignon
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, version 3 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
"""
# XXX Currently, elementtidy segfaults when there are no error, because of
# the behavior of libtidy.
# A patch has been sent to Debian:
# http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=576343
#
# As it is not integrated in Debian yet, and as this problem persists on other
# systems, using elementtidy is for now to avoid.
from elementtidy import TidyHTMLTreeBuilder
try:
from xml.etree import cElementTree as ElementTree
except ImportError:
from xml.etree import ElementTree
from .iparser import IParser
__all__ = ['ElementTidyParser']
class ElementTidyParser(IParser):
def parse(self, data, encoding=None):
TidyHTMLTreeBuilder.ElementTree = ElementTree
HTMLTreeBuilder = TidyHTMLTreeBuilder.TidyHTMLTreeBuilder
parser = HTMLTreeBuilder(encoding)
tree = ElementTree.parse(data, parser)
for elem in tree.getiterator():
if elem.tag.startswith('{'):
elem.tag = elem.tag[elem.tag.find('}')+1:]
return tree
def tostring(self, element):
e = ElementTree.Element('body')
e.text = element.text
e.tail = element.tail
for sub in element.getchildren():
e.append(sub)
s = ''
# XXX OK if it doesn't work with utf-8, the result will be fucking ugly.
for encoding in ('utf-8', 'ISO-8859-1'):
try:
s = ElementTree.tostring(e, encoding)
except UnicodeError:
continue
else:
break
return unicode(s)

View file

@ -0,0 +1,54 @@
# -*- coding: utf-8 -*-
"""
Copyright(C) 2010 Romain Bignon, Christophe Benz
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, version 3 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
"""
from html5lib import treebuilders, HTMLParser
try:
from xml.etree import cElementTree as ElementTree
except ImportError:
from xml.etree import ElementTree
from .iparser import IParser
__all__ = ['Html5libParser']
class Html5libParser(HTMLParser, IParser):
"""
Parser using html5lib.
Note that it is not available on every systems.
"""
# Default implementation for each type of API.
defaults = {'etree': ElementTree,
}
def __init__(self, api='etree'):
# if no default implementation is defined for this api, set it to None
# to let getTreeBuilder() using the corresponding implementation.
implementation = self.defaults.get(api, None)
HTMLParser.__init__(self, tree=treebuilders.getTreeBuilder(api, implementation))
def parse(self, data, encoding):
return HTMLParser.parse(self, data, encoding=encoding)
def tostring(self, elem):
# TODO
raise NotImplementedError()

View file

@ -0,0 +1,95 @@
# -*- coding: utf-8 -*-
"""
Copyright(C) 2010 Romain Bignon
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, version 3 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
"""
from HTMLParser import HTMLParser as _HTMLParser
import htmlentitydefs
try:
from xml.etree import cElementTree as ElementTree
except ImportError:
from xml.etree import ElementTree
from .iparser import IParser
__all__ = ['HTMLParser']
class HTMLTreeBuilder(_HTMLParser):
def __init__(self, encoding=None):
_HTMLParser.__init__(self)
self._target = ElementTree.TreeBuilder()
def doctype(self, name, pubid, system):
pass
def close(self):
tree = self._target.close()
return tree
def handle_starttag(self, tag, attrs):
self._target.start(tag, dict(attrs))
def handle_startendtag(self, tag, attrs):
self._target.start(tag, dict(attrs))
self._target.end(tag)
def handle_charref(self, name):
self._target.data(unichr(int(name)))
def handle_entityref(self, name):
try:
self._target.data(unichr(htmlentitydefs.name2codepoint[name]))
except KeyError:
self._target.data('&' + name)
def handle_data(self, data):
self._target.data(data)
def handle_endtag(self, tag):
try:
self._target.end(tag)
except:
pass
class HTMLParser(IParser):
def parse(self, data, encoding=None):
parser = HTMLTreeBuilder(encoding)
tree = ElementTree.parse(data, parser)
for elem in tree.getiterator():
if elem.tag.startswith('{'):
elem.tag = elem.tag[elem.tag.find('}')+1:]
return tree
def tostring(self, element):
e = ElementTree.Element('body')
e.text = element.text
e.tail = element.tail
for sub in element.getchildren():
e.append(sub)
s = ''
# XXX OK if it doesn't work with utf-8, the result will be fucking ugly.
for encoding in ('utf-8', 'ISO-8859-1'):
try:
s = ElementTree.tostring(e, encoding)
except UnicodeError:
continue
else:
break
return unicode(s)

View file

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
""" """
Copyright(C) 2010 Christophe Benz, Romain Bignon Copyright(C) 2010 Romain Bignon
This program is free software; you can redistribute it and/or modify This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
@ -18,28 +18,19 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
""" """
# Low performances class IParser(object):
# v def parse(self, data, encoding=None):
# v """
try: Parse a HTML document with a specific encoding to get a tree.
from .elementtidyparser import ElementTidyParser, ElementTidyParser as StandardParser
except ImportError: @param data [str] HTML document
pass @param encoding [str] encoding to use
# v @return an object with the structured document
try: """
from .htmlparser import HTMLParser, HTMLParser as StandardParser raise NotImplementedError()
except ImportError:
pass def tostring(self, elem):
# v """
try: Get HTML string from an element.
from .html5libparser import Html5libParser, Html5libParser as StandardParser """
except ImportError: raise NotImplementedError()
pass
# v
try:
from .lxmlparser import LxmlHtmlParser, LxmlHtmlParser as StandardParser
except ImportError:
pass
# v
# v
# High performances

View file

@ -0,0 +1,41 @@
# -*- coding: utf-8 -*-
"""
Copyright(C) 2010 Christophe Benz
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, version 3 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
"""
import lxml.html
from .iparser import IParser
__all__ = ['LxmlHtmlParser']
class LxmlHtmlParser(IParser):
"""
Parser using lxml.
Note that it is not available on every systems.
"""
def parse(self, data, encoding=None):
parser = lxml.html.HTMLParser(encoding=encoding)
return lxml.html.parse(data, parser)
def tostring(self, element):
return lxml.html.tostring(element, encoding=unicode)