nojhan/weboob-devel
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

move select() in parser

Browse source
This commit is contained in:
Romain Bignon 2011-04-08 19:39:57 +02:00
commit 9afb301ebe
30 changed files with 197 additions and 197 deletions
Download patch file Download diff file Expand all files Collapse all files

2
weboob/tools/application/formatters/iformatter.py
View file

@ -25,7 +25,7 @@ import sys
import subprocess
if sys.platform == 'win32':
import WConio
try:
import tty, termios
except ImportError:

4
weboob/tools/browser/__init__.py
View file

@ -21,8 +21,8 @@
from weboob.tools.browser.browser import BrowserIncorrectPassword, BrowserBanned, \
BrowserUnavailable, BrowserRetry, \
BrowserHTTPNotFound, BrowserHTTPError, \
BasePage, BaseBrowser
BasePage, BaseBrowser, BrokenPageError
__all__ = ['BrowserIncorrectPassword', 'BrowserBanned', 'BrowserUnavailable', 'BrowserRetry',
'BrowserHTTPNotFound', 'BrowserHTTPError', 'BasePage', 'BaseBrowser']
'BrowserHTTPNotFound', 'BrowserHTTPError', 'BasePage', 'BaseBrowser', 'BrokenPageError']

3
weboob/tools/browser/browser.py
View file

@ -93,6 +93,8 @@ class NoHistory(object):
def close(self):
pass
class BrokenPageError(Exception):
pass
class BasePage(object):
"""
@ -100,6 +102,7 @@ class BasePage(object):
"""
def __init__(self, browser, document, url='', groups=None, group_dict=None, logger=None):
self.browser = browser
self.parser = browser.parser
self.document = document
self.url = url
self.groups = groups

47
weboob/tools/genericArticle.py
View file

@ -16,32 +16,33 @@
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.tools.browser import BasePage
from weboob.tools.parsers.lxmlparser import select, SelectElementException
from weboob.tools.browser import BrokenPageError
from lxml.etree import Comment
def try_remove(base_element, selector):
def try_remove(parser, base_element, selector):
try :
base_element.remove(select(base_element, selector, 1 ))
except (SelectElementException, ValueError):
base_element.remove(parser.select(base_element, selector, 1 ))
except (BrokenPageError, ValueError):
pass
def try_drop_tree(base_element, selector):
def try_drop_tree(parser, base_element, selector):
try:
select(base_element, selector, 1).drop_tree()
except SelectElementException:
parser.select(base_element, selector, 1).drop_tree()
except BrokenPageError:
pass
def remove_from_selector_list(base_element, selector_list):
def remove_from_selector_list(parser, base_element, selector_list):
for selector in selector_list:
base_element.remove(select(base_element, selector, 1))
base_element.remove(parser.select(base_element, selector, 1))
def try_remove_from_selector_list(base_element, selector_list):
def try_remove_from_selector_list(parser, base_element, selector_list):
for selector in selector_list:
try_remove(base_element, selector)
try_remove(parser, base_element, selector)
def drop_comments(base_element):
for comment in base_element.getiterator(Comment):
@ -49,13 +50,13 @@ def drop_comments(base_element):
class NoAuthorElement(SelectElementException):
class NoAuthorElement(BrokenPageError):
pass
class NoBodyElement(SelectElementException):
class NoBodyElement(BrokenPageError):
pass
class NoTitleException(SelectElementException):
class NoTitleException(BrokenPageError):
pass
class NoneMainDiv(AttributeError):
@ -75,13 +76,13 @@ class Article(object):
class GenericNewsPage(BasePage):
__element_body = NotImplementedError
__article = Article
element_title_selector = NotImplementedError
element_title_selector = NotImplementedError
main_div = NotImplementedError
element_body_selector = NotImplementedError
element_author_selector = NotImplementedError
def get_body(self):
return self.browser.parser.tostring(self.get_element_body())
return self.parser.tostring(self.get_element_body())
def get_author(self):
try:
@ -92,7 +93,7 @@ class GenericNewsPage(BasePage):
def get_title(self):
try :
return select(
return self.parser.select(
self.main_div,
self.element_title_selector,
1).text_content().strip()
@ -102,17 +103,17 @@ class GenericNewsPage(BasePage):
return self.__article.title
else:
raise
except SelectElementException:
except BrokenPageError:
try :
self.element_title_selector = "h1"
return self.get_title()
except SelectElementException:
except BrokenPageError:
raise NoTitleException("no title on %s" % (self.browser))
def get_element_body(self):
try :
return select(self.main_div, self.element_body_selector, 1)
except SelectElementException:
return self.parser.select(self.main_div, self.element_body_selector, 1)
except BrokenPageError:
raise NoBodyElement("no body on %s" % (self.browser))
except AttributeError:
if self.main_div == None:
@ -122,8 +123,8 @@ class GenericNewsPage(BasePage):
def get_element_author(self):
try:
return select(self.main_div, self.element_author_selector, 1)
except SelectElementException:
return self.parser.select(self.main_div, self.element_author_selector, 1)
except BrokenPageError:
raise NoAuthorElement()
except AttributeError:
if self.main_div == None:

81
weboob/tools/parsers/lxmlparser.py
View file

@ -21,50 +21,10 @@
import lxml.html
from .iparser import IParser
from ..browser import BrokenPageError
__all__ = ['LxmlHtmlParser', 'select', 'SelectElementException']
class SelectElementException(Exception):
pass
def select(element, selector, nb=None, method='cssselect'):
"""
Select one or many elements from an element, using lxml cssselect by default.
Raises SelectElementException if not found.
@param element [obj] element on which to apply selector
@param selector [str] CSS or XPath expression
@param method [str] (cssselect|xpath)
@param nb [int] number of elements expected to be found.
Use None for undefined number, and 'many' for 1 to infinite.
@return one or many Element
"""
if method == 'cssselect':
results = element.cssselect(selector)
if nb is None:
return results
elif isinstance(nb, basestring) and nb == 'many':
if results is None or len(results) == 0:
raise SelectElementException('Element not found with selector "%s"' % selector)
elif len(results) == 1:
raise SelectElementException('Only one element found with selector "%s"' % selector)
else:
return results
elif isinstance(nb, int) and nb > 0:
if results is None:
raise SelectElementException('Element not found with selector "%s"' % selector)
elif len(results) < nb:
raise SelectElementException('Not enough elements found (%d expected) with selector "%s"' % (nb, selector))
else:
return results[0] if nb == 1 else results
else:
raise Exception('Unhandled value for kwarg "nb": %s' % nb)
else:
raise NotImplementedError('Only cssselect method is implemented for the moment')
__all__ = ['LxmlHtmlParser']
class LxmlHtmlParser(IParser):
@ -83,3 +43,40 @@ class LxmlHtmlParser(IParser):
def tostring(self, element):
return lxml.html.tostring(element, encoding=unicode)
@classmethod
def select(cls, element, selector, nb=None, method='cssselect'):
"""
Select one or many elements from an element, using lxml cssselect by default.
Raises BrokenPageError if not found.
@param element [obj] element on which to apply selector
@param selector [str] CSS or XPath expression
@param method [str] (cssselect|xpath)
@param nb [int] number of elements expected to be found.
Use None for undefined number, and 'many' for 1 to infinite.
@return one or many Element
"""
if method == 'cssselect':
results = element.cssselect(selector)
if nb is None:
return results
elif isinstance(nb, basestring) and nb == 'many':
if results is None or len(results) == 0:
raise BrokenPageError('Element not found with selector "%s"' % selector)
elif len(results) == 1:
raise BrokenPageError('Only one element found with selector "%s"' % selector)
else:
return results
elif isinstance(nb, int) and nb > 0:
if results is None:
raise BrokenPageError('Element not found with selector "%s"' % selector)
elif len(results) < nb:
raise BrokenPageError('Not enough elements found (%d expected) with selector "%s"' % (nb, selector))
else:
return results[0] if nb == 1 else results
else:
raise Exception('Unhandled value for kwarg "nb": %s' % nb)
else:
raise NotImplementedError('Only cssselect method is implemented for the moment')