move GenericBackend and GenericArticle modules into weboob.tools.capabilities.messages
This commit is contained in:
parent
e0cb6d6dfe
commit
72850a0a79
18 changed files with 22 additions and 312 deletions
|
|
@ -22,8 +22,9 @@
|
||||||
from __future__ import with_statement
|
from __future__ import with_statement
|
||||||
|
|
||||||
from weboob.capabilities.messages import ICapMessages
|
from weboob.capabilities.messages import ICapMessages
|
||||||
|
from weboob.tools.capabilities.messages.GenericBackend import GenericNewspaperBackend
|
||||||
from .browser import NewspaperEcransBrowser
|
from .browser import NewspaperEcransBrowser
|
||||||
from .GenericBackend import GenericNewspaperBackend
|
from .tools import rssid
|
||||||
|
|
||||||
class NewspaperEcransBackend(GenericNewspaperBackend, ICapMessages):
|
class NewspaperEcransBackend(GenericNewspaperBackend, ICapMessages):
|
||||||
"NewspaperEcransBackend class"
|
"NewspaperEcransBackend class"
|
||||||
|
|
@ -36,5 +37,6 @@ class NewspaperEcransBackend(GenericNewspaperBackend, ICapMessages):
|
||||||
DESCRIPTION = u'Ecrans French news website'
|
DESCRIPTION = u'Ecrans French news website'
|
||||||
BROWSER = NewspaperEcransBrowser
|
BROWSER = NewspaperEcransBrowser
|
||||||
RSS_FEED = 'http://www.ecrans.fr/spip.php?page=backend'
|
RSS_FEED = 'http://www.ecrans.fr/spip.php?page=backend'
|
||||||
|
RSSID = rssid
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@
|
||||||
# You should have received a copy of the GNU Affero General Public License
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
from weboob.tools.genericArticle import GenericNewsPage, remove_from_selector_list, try_remove_from_selector_list, try_drop_tree
|
from weboob.tools.capabilities.messages.genericArticle import GenericNewsPage, remove_from_selector_list, try_remove_from_selector_list, try_drop_tree
|
||||||
class ArticlePage(GenericNewsPage):
|
class ArticlePage(GenericNewsPage):
|
||||||
"ArticlePage object for inrocks"
|
"ArticlePage object for inrocks"
|
||||||
def on_loaded(self):
|
def on_loaded(self):
|
||||||
|
|
|
||||||
|
|
@ -22,8 +22,9 @@
|
||||||
from __future__ import with_statement
|
from __future__ import with_statement
|
||||||
|
|
||||||
from weboob.capabilities.messages import ICapMessages
|
from weboob.capabilities.messages import ICapMessages
|
||||||
|
from weboob.tools.capabilities.messages.GenericBackend import GenericNewspaperBackend
|
||||||
from .browser import NewspaperInrocksBrowser
|
from .browser import NewspaperInrocksBrowser
|
||||||
from .GenericBackend import GenericNewspaperBackend
|
from .tools import rssid
|
||||||
|
|
||||||
class NewspaperInrocksBackend(GenericNewspaperBackend, ICapMessages):
|
class NewspaperInrocksBackend(GenericNewspaperBackend, ICapMessages):
|
||||||
"NewspaperInrocksBackend class"
|
"NewspaperInrocksBackend class"
|
||||||
|
|
@ -36,4 +37,4 @@ class NewspaperInrocksBackend(GenericNewspaperBackend, ICapMessages):
|
||||||
DESCRIPTION = u'Inrock French news website'
|
DESCRIPTION = u'Inrock French news website'
|
||||||
BROWSER = NewspaperInrocksBrowser
|
BROWSER = NewspaperInrocksBrowser
|
||||||
RSS_FEED = 'http://www.lesinrocks.com/fileadmin/rss/actus.xml'
|
RSS_FEED = 'http://www.lesinrocks.com/fileadmin/rss/actus.xml'
|
||||||
|
RSSID = rssid
|
||||||
|
|
|
||||||
|
|
@ -19,7 +19,7 @@
|
||||||
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
from weboob.tools.browser import BrokenPageError
|
from weboob.tools.browser import BrokenPageError
|
||||||
from weboob.tools.genericArticle import GenericNewsPage, try_remove, \
|
from weboob.tools.capabilities.messages.genericArticle import GenericNewsPage, try_remove, \
|
||||||
try_remove_from_selector_list, \
|
try_remove_from_selector_list, \
|
||||||
drop_comments, NoneMainDiv
|
drop_comments, NoneMainDiv
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@
|
||||||
# You should have received a copy of the GNU Affero General Public License
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
from weboob.tools.genericArticle import GenericNewsPage
|
from weboob.tools.capabilities.messages.genericArticle import GenericNewsPage
|
||||||
|
|
||||||
class InrocksTvPage(GenericNewsPage):
|
class InrocksTvPage(GenericNewsPage):
|
||||||
"ArticlePage object for inrocks"
|
"ArticlePage object for inrocks"
|
||||||
|
|
|
||||||
|
|
@ -1,99 +0,0 @@
|
||||||
# -*- coding: utf-8 -*-
|
|
||||||
|
|
||||||
# Copyright(C) 2011 Julien Hebert
|
|
||||||
#
|
|
||||||
# This file is part of weboob.
|
|
||||||
#
|
|
||||||
# weboob is free software: you can redistribute it and/or modify
|
|
||||||
# it under the terms of the GNU Affero General Public License as published by
|
|
||||||
# the Free Software Foundation, either version 3 of the License, or
|
|
||||||
# (at your option) any later version.
|
|
||||||
#
|
|
||||||
# weboob is distributed in the hope that it will be useful,
|
|
||||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
# GNU Affero General Public License for more details.
|
|
||||||
#
|
|
||||||
# You should have received a copy of the GNU Affero General Public License
|
|
||||||
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
# python2.5 compatibility
|
|
||||||
from __future__ import with_statement
|
|
||||||
|
|
||||||
from weboob.capabilities.messages import ICapMessages, Message, Thread
|
|
||||||
from weboob.tools.backend import BaseBackend
|
|
||||||
from weboob.tools.newsfeed import Newsfeed
|
|
||||||
from .tools import rssid
|
|
||||||
|
|
||||||
class GenericNewspaperBackend(BaseBackend, ICapMessages):
|
|
||||||
"GenericNewspaperBackend class"
|
|
||||||
MAINTAINER = 'Julien Hebert'
|
|
||||||
EMAIL = 'juke@free.fr'
|
|
||||||
VERSION = '0.9'
|
|
||||||
LICENSE = 'AGPLv3+'
|
|
||||||
STORAGE = {'seen': {}}
|
|
||||||
RSS_FEED = None
|
|
||||||
|
|
||||||
def get_thread(self, _id):
|
|
||||||
if isinstance(_id, Thread):
|
|
||||||
thread = _id
|
|
||||||
_id = thread.id
|
|
||||||
else:
|
|
||||||
thread = None
|
|
||||||
|
|
||||||
with self.browser:
|
|
||||||
content = self.browser.get_content(_id)
|
|
||||||
|
|
||||||
if not thread:
|
|
||||||
thread = Thread(_id)
|
|
||||||
|
|
||||||
flags = Message.IS_HTML
|
|
||||||
if not thread.id in self.storage.get('seen', default={}):
|
|
||||||
flags |= Message.IS_UNREAD
|
|
||||||
thread.title = content.title
|
|
||||||
if not thread.date:
|
|
||||||
thread.date = content.date
|
|
||||||
|
|
||||||
thread.root = Message(
|
|
||||||
thread=thread,
|
|
||||||
id=0,
|
|
||||||
title=content.title,
|
|
||||||
sender=content.author,
|
|
||||||
receivers=None,
|
|
||||||
date=thread.date,
|
|
||||||
parent=None,
|
|
||||||
content=content.body,
|
|
||||||
signature='URL: %s' % content.url,
|
|
||||||
flags=flags,
|
|
||||||
children= [])
|
|
||||||
return thread
|
|
||||||
|
|
||||||
def iter_threads(self):
|
|
||||||
for article in Newsfeed(self.RSS_FEED, rssid).iter_entries():
|
|
||||||
thread = Thread(article.id)
|
|
||||||
thread.title = article.title
|
|
||||||
thread.date = article.datetime
|
|
||||||
yield(thread)
|
|
||||||
|
|
||||||
def fill_thread(self, thread):
|
|
||||||
"fill the thread"
|
|
||||||
return self.get_thread(thread)
|
|
||||||
|
|
||||||
def iter_unread_messages(self, thread=None):
|
|
||||||
for thread in self.iter_threads():
|
|
||||||
self.fill_thread(thread)
|
|
||||||
for msg in thread.iter_all_messages():
|
|
||||||
if msg.flags & msg.IS_UNREAD:
|
|
||||||
yield msg
|
|
||||||
|
|
||||||
def set_message_read(self, message):
|
|
||||||
self.storage.set(
|
|
||||||
'seen',
|
|
||||||
message.thread.id,
|
|
||||||
'comments',
|
|
||||||
self.storage.get(
|
|
||||||
'seen',
|
|
||||||
message.thread.id,
|
|
||||||
'comments',
|
|
||||||
default=[]) + [message.id])
|
|
||||||
self.storage.save()
|
|
||||||
|
|
@ -22,8 +22,9 @@
|
||||||
from __future__ import with_statement
|
from __future__ import with_statement
|
||||||
|
|
||||||
from weboob.capabilities.messages import ICapMessages
|
from weboob.capabilities.messages import ICapMessages
|
||||||
|
from weboob.tools.capabilities.messages.GenericBackend import GenericNewspaperBackend
|
||||||
from .browser import NewspaperFigaroBrowser
|
from .browser import NewspaperFigaroBrowser
|
||||||
from .GenericBackend import GenericNewspaperBackend
|
from .tools import rssid
|
||||||
|
|
||||||
class NewspaperFigaroBackend(GenericNewspaperBackend, ICapMessages):
|
class NewspaperFigaroBackend(GenericNewspaperBackend, ICapMessages):
|
||||||
"NewspaperFigaroBackend class"
|
"NewspaperFigaroBackend class"
|
||||||
|
|
@ -36,5 +37,6 @@ class NewspaperFigaroBackend(GenericNewspaperBackend, ICapMessages):
|
||||||
DESCRIPTION = u'Lefigaro French news website'
|
DESCRIPTION = u'Lefigaro French news website'
|
||||||
BROWSER = NewspaperFigaroBrowser
|
BROWSER = NewspaperFigaroBrowser
|
||||||
RSS_FEED = 'http://rss.lefigaro.fr/lefigaro/laune?format=xml'
|
RSS_FEED = 'http://rss.lefigaro.fr/lefigaro/laune?format=xml'
|
||||||
|
RSSID = rssid
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@
|
||||||
# You should have received a copy of the GNU Affero General Public License
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
from weboob.tools.genericArticle import GenericNewsPage, remove_from_selector_list, drop_comments, try_drop_tree, try_remove_from_selector_list
|
from weboob.tools.capabilities.messages.genericArticle import GenericNewsPage, remove_from_selector_list, drop_comments, try_drop_tree, try_remove_from_selector_list
|
||||||
|
|
||||||
class ArticlePage(GenericNewsPage):
|
class ArticlePage(GenericNewsPage):
|
||||||
"ArticlePage object for inrocks"
|
"ArticlePage object for inrocks"
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@
|
||||||
# You should have received a copy of the GNU Affero General Public License
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
from weboob.tools.genericArticle import GenericNewsPage
|
from weboob.tools.capabilities.messages.genericArticle import GenericNewsPage
|
||||||
|
|
||||||
class FlashActuPage(GenericNewsPage):
|
class FlashActuPage(GenericNewsPage):
|
||||||
"ArticlePage object for inrocks"
|
"ArticlePage object for inrocks"
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@
|
||||||
# You should have received a copy of the GNU Affero General Public License
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
from weboob.tools.genericArticle import GenericNewsPage
|
from weboob.tools.capabilities.messages.genericArticle import GenericNewsPage
|
||||||
|
|
||||||
class SimplePage(GenericNewsPage):
|
class SimplePage(GenericNewsPage):
|
||||||
"ArticlePage object for minutes20"
|
"ArticlePage object for minutes20"
|
||||||
|
|
|
||||||
|
|
@ -1,99 +0,0 @@
|
||||||
# -*- coding: utf-8 -*-
|
|
||||||
|
|
||||||
# Copyright(C) 2011 Julien Hebert
|
|
||||||
#
|
|
||||||
# This file is part of weboob.
|
|
||||||
#
|
|
||||||
# weboob is free software: you can redistribute it and/or modify
|
|
||||||
# it under the terms of the GNU Affero General Public License as published by
|
|
||||||
# the Free Software Foundation, either version 3 of the License, or
|
|
||||||
# (at your option) any later version.
|
|
||||||
#
|
|
||||||
# weboob is distributed in the hope that it will be useful,
|
|
||||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
# GNU Affero General Public License for more details.
|
|
||||||
#
|
|
||||||
# You should have received a copy of the GNU Affero General Public License
|
|
||||||
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
# python2.5 compatibility
|
|
||||||
from __future__ import with_statement
|
|
||||||
|
|
||||||
from weboob.capabilities.messages import ICapMessages, Message, Thread
|
|
||||||
from weboob.tools.backend import BaseBackend
|
|
||||||
from weboob.tools.newsfeed import Newsfeed
|
|
||||||
from .tools import rssid
|
|
||||||
|
|
||||||
class GenericNewspaperBackend(BaseBackend, ICapMessages):
|
|
||||||
"GenericNewspaperBackend class"
|
|
||||||
MAINTAINER = 'Julien Hebert'
|
|
||||||
EMAIL = 'juke@free.fr'
|
|
||||||
VERSION = '0.9'
|
|
||||||
LICENSE = 'AGPLv3+'
|
|
||||||
STORAGE = {'seen': {}}
|
|
||||||
RSS_FEED = None
|
|
||||||
|
|
||||||
def get_thread(self, _id):
|
|
||||||
if isinstance(_id, Thread):
|
|
||||||
thread = _id
|
|
||||||
_id = thread.id
|
|
||||||
else:
|
|
||||||
thread = None
|
|
||||||
|
|
||||||
with self.browser:
|
|
||||||
content = self.browser.get_content(_id)
|
|
||||||
|
|
||||||
if not thread:
|
|
||||||
thread = Thread(_id)
|
|
||||||
|
|
||||||
flags = Message.IS_HTML
|
|
||||||
if not thread.id in self.storage.get('seen', default={}):
|
|
||||||
flags |= Message.IS_UNREAD
|
|
||||||
thread.title = content.title
|
|
||||||
if not thread.date:
|
|
||||||
thread.date = content.date
|
|
||||||
|
|
||||||
thread.root = Message(
|
|
||||||
thread=thread,
|
|
||||||
id=0,
|
|
||||||
title=content.title,
|
|
||||||
sender=content.author,
|
|
||||||
receivers=None,
|
|
||||||
date=thread.date,
|
|
||||||
parent=None,
|
|
||||||
content=content.body,
|
|
||||||
signature='URL: %s' % content.url,
|
|
||||||
flags=flags,
|
|
||||||
children= [])
|
|
||||||
return thread
|
|
||||||
|
|
||||||
def iter_threads(self):
|
|
||||||
for article in Newsfeed(self.RSS_FEED, rssid).iter_entries():
|
|
||||||
thread = Thread(article.id)
|
|
||||||
thread.title = article.title
|
|
||||||
thread.date = article.datetime
|
|
||||||
yield(thread)
|
|
||||||
|
|
||||||
def fill_thread(self, thread):
|
|
||||||
"fill the thread"
|
|
||||||
return self.get_thread(thread)
|
|
||||||
|
|
||||||
def iter_unread_messages(self, thread=None):
|
|
||||||
for thread in self.iter_threads():
|
|
||||||
self.fill_thread(thread)
|
|
||||||
for msg in thread.iter_all_messages():
|
|
||||||
if msg.flags & msg.IS_UNREAD:
|
|
||||||
yield msg
|
|
||||||
|
|
||||||
def set_message_read(self, message):
|
|
||||||
self.storage.set(
|
|
||||||
'seen',
|
|
||||||
message.thread.id,
|
|
||||||
'comments',
|
|
||||||
self.storage.get(
|
|
||||||
'seen',
|
|
||||||
message.thread.id,
|
|
||||||
'comments',
|
|
||||||
default=[]) + [message.id])
|
|
||||||
self.storage.save()
|
|
||||||
|
|
@ -19,8 +19,9 @@
|
||||||
"backend for http://20minutes.fr"
|
"backend for http://20minutes.fr"
|
||||||
|
|
||||||
from weboob.capabilities.messages import ICapMessages
|
from weboob.capabilities.messages import ICapMessages
|
||||||
|
from weboob.tools.capabilities.messages.GenericBackend import GenericNewspaperBackend
|
||||||
from .browser import Newspaper20minutesBrowser
|
from .browser import Newspaper20minutesBrowser
|
||||||
from .GenericBackend import GenericNewspaperBackend
|
from .tools import rssid
|
||||||
|
|
||||||
class Newspaper20minutesBackend(GenericNewspaperBackend, ICapMessages):
|
class Newspaper20minutesBackend(GenericNewspaperBackend, ICapMessages):
|
||||||
"Newspaper20minutesBackend class"
|
"Newspaper20minutesBackend class"
|
||||||
|
|
@ -33,4 +34,5 @@ class Newspaper20minutesBackend(GenericNewspaperBackend, ICapMessages):
|
||||||
DESCRIPTION = u'20minutes French news website'
|
DESCRIPTION = u'20minutes French news website'
|
||||||
BROWSER = Newspaper20minutesBrowser
|
BROWSER = Newspaper20minutesBrowser
|
||||||
RSS_FEED = 'http://www.20minutes.fr/rss/20minutes.xml'
|
RSS_FEED = 'http://www.20minutes.fr/rss/20minutes.xml'
|
||||||
|
RSSID = rssid
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@
|
||||||
# You should have received a copy of the GNU Affero General Public License
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
from weboob.tools.genericArticle import NoAuthorElement, try_remove, NoneMainDiv
|
from weboob.tools.capabilities.messages.genericArticle import NoAuthorElement, try_remove, NoneMainDiv
|
||||||
from .simple import SimplePage
|
from .simple import SimplePage
|
||||||
|
|
||||||
class ArticlePage(SimplePage):
|
class ArticlePage(SimplePage):
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@
|
||||||
# You should have received a copy of the GNU Affero General Public License
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
from weboob.tools.genericArticle import GenericNewsPage
|
from weboob.tools.capabilities.messages.genericArticle import GenericNewsPage
|
||||||
|
|
||||||
class SimplePage(GenericNewsPage):
|
class SimplePage(GenericNewsPage):
|
||||||
"ArticlePage object for minutes20"
|
"ArticlePage object for minutes20"
|
||||||
|
|
|
||||||
|
|
@ -23,7 +23,6 @@ from __future__ import with_statement
|
||||||
from weboob.capabilities.messages import ICapMessages, Message, Thread
|
from weboob.capabilities.messages import ICapMessages, Message, Thread
|
||||||
from weboob.tools.backend import BaseBackend
|
from weboob.tools.backend import BaseBackend
|
||||||
from weboob.tools.newsfeed import Newsfeed
|
from weboob.tools.newsfeed import Newsfeed
|
||||||
from .tools import rssid
|
|
||||||
|
|
||||||
class GenericNewspaperBackend(BaseBackend, ICapMessages):
|
class GenericNewspaperBackend(BaseBackend, ICapMessages):
|
||||||
"GenericNewspaperBackend class"
|
"GenericNewspaperBackend class"
|
||||||
|
|
@ -33,6 +32,7 @@ class GenericNewspaperBackend(BaseBackend, ICapMessages):
|
||||||
LICENSE = 'AGPLv3+'
|
LICENSE = 'AGPLv3+'
|
||||||
STORAGE = {'seen': {}}
|
STORAGE = {'seen': {}}
|
||||||
RSS_FEED = None
|
RSS_FEED = None
|
||||||
|
RSSID = None
|
||||||
|
|
||||||
def get_thread(self, _id):
|
def get_thread(self, _id):
|
||||||
if isinstance(_id, Thread):
|
if isinstance(_id, Thread):
|
||||||
|
|
@ -69,7 +69,7 @@ class GenericNewspaperBackend(BaseBackend, ICapMessages):
|
||||||
return thread
|
return thread
|
||||||
|
|
||||||
def iter_threads(self):
|
def iter_threads(self):
|
||||||
for article in Newsfeed(self.RSS_FEED, rssid).iter_entries():
|
for article in Newsfeed(self.RSS_FEED, GenericNewspaperBackend.RSSID).iter_entries():
|
||||||
thread = Thread(article.id)
|
thread = Thread(article.id)
|
||||||
thread.title = article.title
|
thread.title = article.title
|
||||||
thread.date = article.datetime
|
thread.date = article.datetime
|
||||||
0
weboob/tools/capabilities/messages/__init__.py
Normal file
0
weboob/tools/capabilities/messages/__init__.py
Normal file
142
weboob/tools/capabilities/messages/genericArticle.py
Normal file
142
weboob/tools/capabilities/messages/genericArticle.py
Normal file
|
|
@ -0,0 +1,142 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
# Copyright(C) 2011 Julien Hebert
|
||||||
|
#
|
||||||
|
# This file is part of weboob.
|
||||||
|
#
|
||||||
|
# weboob is free software: you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU Affero General Public License as published by
|
||||||
|
# the Free Software Foundation, either version 3 of the License, or
|
||||||
|
# (at your option) any later version.
|
||||||
|
#
|
||||||
|
# weboob is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU Affero General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
|
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
from weboob.tools.browser import BasePage
|
||||||
|
from weboob.tools.browser import BrokenPageError
|
||||||
|
from lxml.etree import Comment
|
||||||
|
|
||||||
|
|
||||||
|
def try_remove(parser, base_element, selector):
|
||||||
|
try :
|
||||||
|
base_element.remove(parser.select(base_element, selector, 1 ))
|
||||||
|
except (BrokenPageError, ValueError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def try_drop_tree(parser, base_element, selector):
|
||||||
|
try:
|
||||||
|
parser.select(base_element, selector, 1).drop_tree()
|
||||||
|
except BrokenPageError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def remove_from_selector_list(parser, base_element, selector_list):
|
||||||
|
for selector in selector_list:
|
||||||
|
base_element.remove(parser.select(base_element, selector, 1))
|
||||||
|
|
||||||
|
|
||||||
|
def try_remove_from_selector_list(parser, base_element, selector_list):
|
||||||
|
for selector in selector_list:
|
||||||
|
try_remove(parser, base_element, selector)
|
||||||
|
|
||||||
|
def drop_comments(base_element):
|
||||||
|
for comment in base_element.getiterator(Comment):
|
||||||
|
comment.drop_tree()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
class NoAuthorElement(BrokenPageError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
class NoBodyElement(BrokenPageError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
class NoTitleException(BrokenPageError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
class NoneMainDiv(AttributeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
class Article(object):
|
||||||
|
author = u''
|
||||||
|
title = u''
|
||||||
|
|
||||||
|
def __init__(self, browser, _id):
|
||||||
|
self.browser = browser
|
||||||
|
self.id = _id
|
||||||
|
self.body = u''
|
||||||
|
self.url = u''
|
||||||
|
self.date = None
|
||||||
|
|
||||||
|
class GenericNewsPage(BasePage):
|
||||||
|
__element_body = NotImplementedError
|
||||||
|
__article = Article
|
||||||
|
element_title_selector = NotImplementedError
|
||||||
|
main_div = NotImplementedError
|
||||||
|
element_body_selector = NotImplementedError
|
||||||
|
element_author_selector = NotImplementedError
|
||||||
|
|
||||||
|
def get_body(self):
|
||||||
|
return self.parser.tostring(self.get_element_body())
|
||||||
|
|
||||||
|
def get_author(self):
|
||||||
|
try:
|
||||||
|
return self.get_element_author().text_content().strip()
|
||||||
|
except (NoAuthorElement, NoneMainDiv):
|
||||||
|
#TODO: Mettre un warning
|
||||||
|
return self.__article.author
|
||||||
|
|
||||||
|
def get_title(self):
|
||||||
|
try :
|
||||||
|
return self.parser.select(
|
||||||
|
self.main_div,
|
||||||
|
self.element_title_selector,
|
||||||
|
1).text_content().strip()
|
||||||
|
except AttributeError:
|
||||||
|
if self.main_div == None:
|
||||||
|
#TODO: Mettre un warning
|
||||||
|
return self.__article.title
|
||||||
|
else:
|
||||||
|
raise
|
||||||
|
except BrokenPageError:
|
||||||
|
try :
|
||||||
|
self.element_title_selector = "h1"
|
||||||
|
return self.get_title()
|
||||||
|
except BrokenPageError:
|
||||||
|
raise NoTitleException("no title on %s" % (self.browser))
|
||||||
|
|
||||||
|
def get_element_body(self):
|
||||||
|
try :
|
||||||
|
return self.parser.select(self.main_div, self.element_body_selector, 1)
|
||||||
|
except BrokenPageError:
|
||||||
|
raise NoBodyElement("no body on %s" % (self.browser))
|
||||||
|
except AttributeError:
|
||||||
|
if self.main_div == None:
|
||||||
|
raise NoneMainDiv("main_div is none on %s" % (self.browser))
|
||||||
|
else:
|
||||||
|
raise
|
||||||
|
|
||||||
|
def get_element_author(self):
|
||||||
|
try:
|
||||||
|
return self.parser.select(self.main_div, self.element_author_selector, 1)
|
||||||
|
except BrokenPageError:
|
||||||
|
raise NoAuthorElement()
|
||||||
|
except AttributeError:
|
||||||
|
if self.main_div == None:
|
||||||
|
raise NoneMainDiv("main_div is none on %s" % (self.browser))
|
||||||
|
else:
|
||||||
|
raise
|
||||||
|
|
||||||
|
def get_article(self, _id):
|
||||||
|
__article = Article(self.browser, _id)
|
||||||
|
__article.author = self.get_author()
|
||||||
|
__article.title = self.get_title()
|
||||||
|
__article.url = self.url
|
||||||
|
__article.body = self.get_body()
|
||||||
|
|
||||||
|
return __article
|
||||||
Loading…
Reference in a new issue