From 715ed17ad411d7698c44b0d92aade01a23c3667c Mon Sep 17 00:00:00 2001 From: Julien Veyssier Date: Tue, 9 Apr 2013 19:29:47 +0200 Subject: [PATCH] new lyrics backend: parolesnet --- modules/parolesnet/__init__.py | 3 ++ modules/parolesnet/backend.py | 54 +++++++++++++++++++++ modules/parolesnet/browser.py | 52 ++++++++++++++++++++ modules/parolesnet/pages.py | 86 ++++++++++++++++++++++++++++++++++ modules/parolesnet/test.py | 46 ++++++++++++++++++ 5 files changed, 241 insertions(+) create mode 100644 modules/parolesnet/__init__.py create mode 100644 modules/parolesnet/backend.py create mode 100644 modules/parolesnet/browser.py create mode 100644 modules/parolesnet/pages.py create mode 100644 modules/parolesnet/test.py diff --git a/modules/parolesnet/__init__.py b/modules/parolesnet/__init__.py new file mode 100644 index 00000000..ae4dd6ff --- /dev/null +++ b/modules/parolesnet/__init__.py @@ -0,0 +1,3 @@ +from .backend import ParolesnetBackend + +__all__ = ['ParolesnetBackend'] diff --git a/modules/parolesnet/backend.py b/modules/parolesnet/backend.py new file mode 100644 index 00000000..d6a373ce --- /dev/null +++ b/modules/parolesnet/backend.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2013 Julien Veyssier +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from weboob.capabilities.lyrics import ICapLyrics, SongLyrics +from weboob.tools.backend import BaseBackend + +from .browser import ParolesnetBrowser + +__all__ = ['ParolesnetBackend'] + + +class ParolesnetBackend(BaseBackend, ICapLyrics): + NAME = 'parolesnet' + MAINTAINER = u'Julien Veyssier' + EMAIL = 'julien.veyssier@aiur.fr' + VERSION = '0.g' + DESCRIPTION = 'paroles.net lyrics website' + LICENSE = 'AGPLv3+' + BROWSER = ParolesnetBrowser + + def create_default_browser(self): + return self.create_browser() + + def get_lyrics(self, id): + return self.browser.get_lyrics(id) + + def iter_lyrics(self, criteria, pattern): + return self.browser.iter_lyrics(criteria, pattern.encode('utf-8')) + + def fill_songlyrics(self, songlyrics, fields): + if 'content' in fields: + sl = self.get_lyrics(songlyrics.id) + songlyrics.content = sl.content + return songlyrics + + OBJECTS = { + SongLyrics: fill_songlyrics + } diff --git a/modules/parolesnet/browser.py b/modules/parolesnet/browser.py new file mode 100644 index 00000000..44012200 --- /dev/null +++ b/modules/parolesnet/browser.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2013 Julien Veyssier +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.tools.browser import BaseBrowser, BrowserHTTPNotFound + +from .pages import ResultsPage, SonglyricsPage, ArtistSongsPage, HomePage + + +__all__ = ['ParolesnetBrowser'] + + +class ParolesnetBrowser(BaseBrowser): + DOMAIN = 'www.paroles.net' + PROTOCOL = 'http' + ENCODING = 'utf-8' + USER_AGENT = BaseBrowser.USER_AGENTS['wget'] + PAGES = { + 'http://www.paroles.net': HomePage, + 'http://www.paroles.net/search': ResultsPage, + 'http://www.paroles.net/.*/paroles-.*': SonglyricsPage, + 'http://www.paroles.net/[a-z\-]*': ArtistSongsPage + } + + def iter_lyrics(self, criteria, pattern): + self.location('http://www.paroles.net') + assert self.is_on_page(HomePage) + return self.page.iter_lyrics(criteria, pattern) + + def get_lyrics(self, id): + try: + self.location('http://www.paroles.net/%s' % id) + except BrowserHTTPNotFound: + return + if self.is_on_page(SonglyricsPage): + return self.page.get_lyrics(id) diff --git a/modules/parolesnet/pages.py b/modules/parolesnet/pages.py new file mode 100644 index 00000000..1f38e14e --- /dev/null +++ b/modules/parolesnet/pages.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2013 Julien Veyssier +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.capabilities.lyrics import SongLyrics +from weboob.capabilities.base import NotAvailable, NotLoaded +from weboob.tools.browser import BasePage + + +__all__ = ['ResultsPage', 'SonglyricsPage', 'ArtistSongsPage', 'HomePage'] + + +class HomePage(BasePage): + def iter_lyrics(self, criteria, pattern): + self.browser.select_form(nr=0) + self.browser['search'] = pattern + self.browser.submit() + assert self.browser.is_on_page(ResultsPage) + for lyr in self.browser.page.iter_lyrics(criteria): + yield lyr + + +class ResultsPage(BasePage): + def iter_lyrics(self, criteria): + for link in self.parser.select(self.document.getroot(), 'div.box-content td.song-name a'): + href = link.attrib.get('href','') + if criteria == 'artist': + if len(href.split('/')) != 4: + continue + else: + self.browser.location('%s' % href) + assert self.browser.is_on_page(ArtistSongsPage) or self.browser.is_on_page(SonglyricsPage) + for lyr in self.browser.page.iter_lyrics(): + yield lyr + else: + if len(href.split('/')) != 5: + continue + else: + artist = unicode(self.parser.select(link.getparent().getparent().getparent(), 'td.song-artist > p', 1).text.strip()) + title = unicode(link.text) + id = unicode(link.attrib.get('href', '').replace('http://www.paroles.net/','')) + lyr = SongLyrics(id, title) + lyr.artist = artist + yield lyr + + +class ArtistSongsPage(BasePage): + def iter_lyrics(self): + artist = unicode(self.parser.select(self.document.getroot(), 'span[itemprop=name]', 1).text) + for link in self.parser.select(self.document.getroot(), 'td.song-name > p[itemprop=name] > a[itemprop=url]'): + href = unicode(link.attrib.get('href', '')) + title = unicode(link.text) + id = href.replace('http://www.paroles.net/', '') + songlyrics = SongLyrics(id, title) + songlyrics.artist = artist + songlyrics.content = NotLoaded + yield songlyrics + + +class SonglyricsPage(BasePage): + def get_lyrics(self, id): + artist = NotAvailable + title = NotAvailable + content = unicode(self.parser.select(self.document.getroot(), 'div.song-text', 1).text_content().strip()) + artist = unicode(self.parser.select(self.document.getroot(), 'span[property$=artist]', 1).text) + title = unicode(self.parser.select(self.document.getroot(), 'span[property$=name]', 1).text) + songlyrics = SongLyrics(id, title) + songlyrics.artist = artist + songlyrics.content = content + return songlyrics diff --git a/modules/parolesnet/test.py b/modules/parolesnet/test.py new file mode 100644 index 00000000..9a556306 --- /dev/null +++ b/modules/parolesnet/test.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2013 Julien Veyssier +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from weboob.tools.test import BackendTest +from weboob.capabilities.base import NotLoaded + + +class ParolesnetTest(BackendTest): + BACKEND = 'parolesnet' + + def test_search_song_n_get(self): + l_lyrics = list(self.backend.iter_lyrics('song', 'chien')) + for songlyrics in l_lyrics: + assert songlyrics.id + assert songlyrics.title + assert songlyrics.artist + assert songlyrics.content is NotLoaded + full_lyr = self.backend.get_lyrics(songlyrics.id) + assert full_lyr.id + assert full_lyr.title + assert full_lyr.artist + assert full_lyr.content is not NotLoaded + + def test_search_artist(self): + l_lyrics = list(self.backend.iter_lyrics('artist', 'boris')) + for songlyrics in l_lyrics: + assert songlyrics.id + assert songlyrics.title + assert songlyrics.artist + assert songlyrics.content is NotLoaded