nojhan/weboob-devel
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

factorise

Browse source
This commit is contained in:
Juke 2011-03-07 09:44:02 +01:00 • committed by Romain Bignon
commit 397b5875f1
9 changed files with 59 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

6
weboob/applications/boobmsg/boobmsg.py
View file

@ -298,9 +298,9 @@ class Boobmsg(ReplApplication):
_id, backend_name = self.parse_id(arg)
cmd = self.do('get_thread', _id, backends=backend_name)
for backend, thread in cmd:
for msg in thread.iter_all_messages():
self.format(msg)
if thread is not None :
for msg in thread.iter_all_messages():
self.format(msg)
def do_show(self, arg):
"""

16
weboob/backends/inrocks/pages/article.py
View file

@ -17,25 +17,25 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
from weboob.tools.parsers.lxmlparser import select
from weboob.tools.genericArticle import GenericNewsPage, try_remove
from weboob.tools.genericArticle import GenericNewsPage, try_remove, \
try_remove_from_selector_list
class ArticlePage(GenericNewsPage):
"ArticlePage object for inrocks"
def on_loaded(self):
self.main_div = self.document.getroot()
self.element_title_selector = "h1"
self.element_author_selector = "div.name>span"
self.element_title_selector = "h1"
self.element_body_selector = "div.maincol"
def get_body(self):
element_body = self.get_element_body()
try_remove(element_body, "div.sidebar")
details = select(element_body, "div.details", 1)
try_remove(details, "div.footer")
header = select(element_body, "div.header", 1)
for selector in ["h1", "div.picture", "div.date", "div.news-single-img",
"div.metas_img", "strong"]:
try_remove(header, selector)
try_remove(select(element_body, "div.details", 1), "div.footer")
try_remove_from_selector_list(select(element_body, "div.header", 1),
["h1", "div.picture", "div.date",
"div.news-single-img",
"div.metas_img", "strong"])
return self.browser.parser.tostring(element_body)

2
weboob/backends/lefigaro/browser.py
View file

@ -49,7 +49,6 @@ class NewspaperFigaroBrowser(BaseBrowser):
"http://www.lefigaro.fr/sortir-paris/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage,
"http://www.lefigaro.fr/vie-entreprise/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage,
"http://www.lefigaro.fr/autres-sports/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage,
"http://www.lefigaro.fr/immobilier/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage,
"http://www.lefigaro.fr/environnement/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage,
"http://www.lefigaro.fr/rugby/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage,
"http://www.lefigaro.fr/societes/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage,
@ -59,7 +58,6 @@ class NewspaperFigaroBrowser(BaseBrowser):
"http://www.lefigaro.fr/theatre/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage,
"http://www.lefigaro.fr/football-coupes-d-europe/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage,
"http://www.lefigaro.fr/sante/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage,
"http://www.lefigaro.fr/sciences/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage,
"http://www.lefigaro.fr/assurance/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage,
"http://www.lefigaro.fr/retraite/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage,
"http://www.lefigaro.fr/tennis/(\d{4})/(\d{2})/(\d{2})/(.*$)": ArticlePage,

29
weboob/backends/lefigaro/pages/article.py
View file

@ -16,40 +16,25 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
from weboob.tools.parsers.lxmlparser import select, SelectElementException
from weboob.tools.genericArticle import GenericNewsPage, try_remove
from weboob.tools.genericArticle import GenericNewsPage, remove_from_selector_list, drop_comments, try_drop_tree, try_remove_from_selector_list
class ArticlePage(GenericNewsPage):
"ArticlePage object for inrocks"
def on_loaded(self):
self.main_div = self.document.getroot()
self.element_title_selector = "h1"
self.element_author_selector = "div.name>span"
self.element_title_selector = "h1"
self.element_body_selector = "#article"
def get_body(self):
element_body = self.get_element_body()
h1_title = select(element_body, self.element_title_selector, 1)
element_body = self.get_element_body()
remove_from_selector_list(element_body, [self.element_title_selector, "link"])
drop_comments(element_body)
try_drop_tree(element_body, "script")
try:
el_script = select(element_body, "script", 1)
except SelectElementException:
pass
else:
el_script.drop_tree()
element_body.remove(h1_title)
try_remove(element_body, "div.infos")
try_remove(element_body, "div.photo")
try_remove(element_body, "div.art_bandeau_bottom")
try_remove(element_body, "div.view")
try_remove(element_body, "span.auteur_long")
try_remove(element_body, "#toolsbar")
try_remove_from_selector_list(element_body, ["div.infos", "div.photo", "div.art_bandeau_bottom", "div.view", "span.auteur_long", "#toolsbar"])
element_body.find_class("texte")[0].drop_tag()
element_body.tag = "div"
return self.browser.parser.tostring(element_body)

2
weboob/backends/lefigaro/pages/flashactu.py
View file

@ -22,9 +22,9 @@ class FlashActuPage(GenericNewsPage):
"ArticlePage object for inrocks"
def on_loaded(self):
self.main_div = self.document.getroot()
self.element_title_selector = "h1"
self.element_author_selector = "div.name>span"
self.element_body_selector = "h2"
self.element_title_selector = "h1"
def get_body(self):
element_body = self.get_element_body()

12
weboob/backends/minutes20/browser.py
View file

@ -45,11 +45,7 @@ class Newspaper20minutesBrowser(BaseBrowser):
url = id2url(_id)
except ValueError:
url = _id
try:
self.location(url)
except IndexError:
if _id == '':
raise ValueError("thread id is empty")
else:
raise
return self.page.get_article(_id)
self.location(url)
if self.page is not None:
return self.page.get_article(_id)

11
weboob/backends/minutes20/pages/article.py
View file

@ -21,6 +21,11 @@ from .simple import SimplePage
class ArticlePage(SimplePage):
"ArticlePage object for minutes20"
def on_loaded(self):
self.main_div = self.document.getroot()
self.element_title_selector = "h1"
self.element_author_selector = "div.mna-signature"
self.element_body_selector = "div.mna-body"
def get_body(self):
element_body = self.get_element_body()
@ -32,9 +37,3 @@ class ArticlePage(SimplePage):
pass
return self.browser.parser.tostring(element_body)
def on_loaded(self):
self.main_div = self.document.getroot()
self.element_author_selector = "div.mna-signature"
self.element_body_selector = "div.mna-body"
self.element_title_selector = "h1"

4
weboob/backends/minutes20/pages/simple.py
View file

@ -18,13 +18,11 @@
from weboob.tools.genericArticle import GenericNewsPage
class SimplePage(GenericNewsPage):
"ArticlePage object for minutes20"
def on_loaded(self):
self.main_div = self.document.getroot()
self.element_title_selector = "h1"
self.element_author_selector = "div.mna-signature"
self.element_body_selector = "div.mna-body"
self.element_title_selector = "h1"

31
weboob/tools/genericArticle.py
View file

@ -16,6 +16,8 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
from weboob.tools.browser import BasePage
from weboob.tools.parsers.lxmlparser import select, SelectElementException
from lxml.etree import Comment
def try_remove(base_element, selector):
try :
@ -24,9 +26,33 @@ def try_remove(base_element, selector):
pass
def try_drop_tree(base_element, selector):
try:
select(base_element, selector, 1).drop_tree()
except SelectElementException:
pass
def remove_from_selector_list(base_element, selector_list):
for selector in selector_list:
base_element.remove(select(base_element, selector, 1))
def try_remove_from_selector_list(base_element, selector_list):
for selector in selector_list:
try_remove(base_element, selector)
def drop_comments(base_element):
for comment in base_element.getiterator(Comment):
comment.drop_tree()
class NoAuthorElement(SelectElementException):
pass
class NoBodyElement(SelectElementException):
pass
class NoTitleException(SelectElementException):
pass
@ -71,7 +97,10 @@ class GenericNewsPage(BasePage):
raise NoTitleException("no title on %s" % (self.browser))
def get_element_body(self):
return select(self.main_div, self.element_body_selector, 1)
try :
return select(self.main_div, self.element_body_selector, 1)
except SelectElementException:
raise NoBodyElement("no body on %s" % (self.browser))
def get_element_author(self):
try: