nojhan/weboob-devel
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add clean_relativ_urls in genericArticle.py

Browse source 
and use it for ecrans.fr
This commit is contained in:
Florent 2012-11-07 18:13:03 +01:00
commit 3cb78d1729
2 changed files with 12 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

10
modules/ecrans/pages/article.py
View file

@ -18,7 +18,7 @@
# You should have received a copy of the GNU Affero General Public License # You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>. # along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.tools.capabilities.messages.genericArticle import GenericNewsPage, remove_from_selector_list, try_remove_from_selector_list, try_drop_tree from weboob.tools.capabilities.messages.genericArticle import GenericNewsPage, remove_from_selector_list, try_remove_from_selector_list, try_drop_tree, clean_relativ_urls
class ArticlePage(GenericNewsPage): class ArticlePage(GenericNewsPage):
@ -34,12 +34,6 @@ class ArticlePage(GenericNewsPage):
remove_from_selector_list(self.parser, element_body, ["p.auteur", "h4"]) remove_from_selector_list(self.parser, element_body, ["p.auteur", "h4"])
try_remove_from_selector_list(self.parser, element_body, ["p.tag", "div.alire", self.element_title_selector, "h4"]) try_remove_from_selector_list(self.parser, element_body, ["p.tag", "div.alire", self.element_title_selector, "h4"])
try_drop_tree(self.parser, element_body, "script") try_drop_tree(self.parser, element_body, "script")
for a in element_body.findall('.//a'): clean_relativ_urls(element_body, "http://ecrans.fr")
if a.attrib["href"][0:7] != "http://":
a.attrib["href"] = "http://ecrans.fr/" + a.attrib["href"]
for img in element_body.xpath('.//img'):
if img.attrib["src"][0:7] != "http://":
img.attrib["src"] = "http://ecrans.fr/" + img.attrib["src"]
return self.parser.tostring(element_body) return self.parser.tostring(element_body)

10
weboob/tools/capabilities/messages/genericArticle.py
View file

@ -48,6 +48,16 @@ def drop_comments(base_element):
for comment in base_element.getiterator(Comment): for comment in base_element.getiterator(Comment):
comment.drop_tree() comment.drop_tree()
# Replace relative url in link and image with a complete url
# Arguments: the html element to clean, and the domain name (with http:// prefix)
def clean_relativ_urls(base_element, domain):
for a in base_element.findall('.//a'):
if a.attrib["href"][0:7] != "http://" and a.attrib["href"][0:7] != "https://":
a.attrib["href"] = domain + a.attrib["href"]
for img in base_element.findall('.//img'):
if img.attrib["src"][0:7] != "http://" and img.attrib["src"][0:7] != "https://":
img.attrib["src"] = domain + img.attrib["src"]
class NoAuthorElement(BrokenPageError): class NoAuthorElement(BrokenPageError):
pass pass