From dc2004aad346ecdffdb407f1b98b447546a280be Mon Sep 17 00:00:00 2001 From: Julien Veyssier Date: Thu, 14 Mar 2013 15:40:54 +0100 Subject: [PATCH] new module parolesmania for booblyrics --- modules/parolesmania/__init__.py | 23 +++++++++ modules/parolesmania/backend.py | 56 ++++++++++++++++++++ modules/parolesmania/browser.py | 55 ++++++++++++++++++++ modules/parolesmania/pages.py | 88 ++++++++++++++++++++++++++++++++ modules/parolesmania/test.py | 46 +++++++++++++++++ modules/parolesmusique/test.py | 2 +- 6 files changed, 269 insertions(+), 1 deletion(-) create mode 100644 modules/parolesmania/__init__.py create mode 100644 modules/parolesmania/backend.py create mode 100644 modules/parolesmania/browser.py create mode 100644 modules/parolesmania/pages.py create mode 100644 modules/parolesmania/test.py diff --git a/modules/parolesmania/__init__.py b/modules/parolesmania/__init__.py new file mode 100644 index 00000000..46c9622e --- /dev/null +++ b/modules/parolesmania/__init__.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2013 Julien Veyssier +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from .backend import ParolesmaniaBackend + +__all__ = ['ParolesmaniaBackend'] diff --git a/modules/parolesmania/backend.py b/modules/parolesmania/backend.py new file mode 100644 index 00000000..9723237e --- /dev/null +++ b/modules/parolesmania/backend.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2013 Julien Veyssier +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from weboob.capabilities.lyrics import ICapLyrics, SongLyrics +from weboob.tools.backend import BaseBackend + +from .browser import ParolesmaniaBrowser + +from urllib import quote_plus + +__all__ = ['ParolesmaniaBackend'] + + +class ParolesmaniaBackend(BaseBackend, ICapLyrics): + NAME = 'parolesmania' + MAINTAINER = u'Julien Veyssier' + EMAIL = 'julien.veyssier@aiur.fr' + VERSION = '0.f' + DESCRIPTION = 'parolesmania lyrics website' + LICENSE = 'AGPLv3+' + BROWSER = ParolesmaniaBrowser + + def create_default_browser(self): + return self.create_browser() + + def get_lyrics(self, id): + return self.browser.get_lyrics(id) + + def iter_lyrics(self, criteria, pattern): + return self.browser.iter_lyrics(criteria,quote_plus(pattern.encode('utf-8'))) + + def fill_songlyrics(self, songlyrics, fields): + if 'content' in fields: + sl = self.get_lyrics(songlyrics.id) + songlyrics.content = sl.content + return songlyrics + + OBJECTS = { + SongLyrics:fill_songlyrics + } diff --git a/modules/parolesmania/browser.py b/modules/parolesmania/browser.py new file mode 100644 index 00000000..f01f6e9d --- /dev/null +++ b/modules/parolesmania/browser.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2013 Julien Veyssier +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.tools.browser import BaseBrowser + +from .pages import SongResultsPage, SonglyricsPage, ArtistResultsPage, ArtistSongsPage + + +__all__ = ['ParolesmaniaBrowser'] + + +class ParolesmaniaBrowser(BaseBrowser): + DOMAIN = 'www.parolesmania.com' + PROTOCOL = 'http' + ENCODING = 'utf-8' + USER_AGENT = BaseBrowser.USER_AGENTS['wget'] + PAGES = { + 'http://www.parolesmania.com/recherche.php\?c=title.*': SongResultsPage, + 'http://www.parolesmania.com/recherche.php\?c=artist.*': ArtistResultsPage, + 'http://www.parolesmania.com/paroles.*[0-9]*/paroles.*': SonglyricsPage, + 'http://www.parolesmania.com/paroles[^/]*.html': ArtistSongsPage, + } + + def iter_lyrics(self, criteria, pattern): + crit = 'artist' + if criteria != 'artist': + crit = 'title' + self.location('http://www.parolesmania.com/recherche.php?c=%s&k=%s'%(crit,pattern)) + assert self.is_on_page(SongResultsPage) or self.is_on_page(ArtistResultsPage)\ + or self.is_on_page(ArtistSongsPage) + for lyr in self.page.iter_lyrics(): + yield lyr + + def get_lyrics(self, id): + ids = id.split('|') + self.location('http://www.parolesmania.com/paroles_%s/paroles_%s.html' % (ids[0],ids[1])) + assert self.is_on_page(SonglyricsPage) + return self.page.get_lyrics(id) diff --git a/modules/parolesmania/pages.py b/modules/parolesmania/pages.py new file mode 100644 index 00000000..18a85be0 --- /dev/null +++ b/modules/parolesmania/pages.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2013 Julien Veyssier +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.capabilities.lyrics import SongLyrics +from weboob.capabilities.base import NotAvailable, NotLoaded +from weboob.tools.browser import BasePage + + +__all__ = ['SongResultsPage','SonglyricsPage', 'ArtistResultsPage', 'ArtistSongsPage'] + + +class ArtistResultsPage(BasePage): + def iter_lyrics(self): + for link in self.parser.select(self.document.getroot(),'div#albums > h1 a'): + artist = unicode(link.text_content()) + href = link.attrib.get('href','') + if href.startswith('/paroles'): + self.browser.location('http://www.parolesmania.com%s'%href) + assert self.browser.is_on_page(ArtistSongsPage) + for lyr in self.browser.page.iter_lyrics(artist): + yield lyr + + +class ArtistSongsPage(BasePage): + def iter_lyrics(self,artist=None): + if artist == None: + artist = self.parser.select(self.document.getroot(),'head > title',1).text.replace('Paroles ','') + for link in self.parser.select(self.document.getroot(),'div#albums a'): + href = link.attrib.get('href','') + titleattrib = link.attrib.get('title','') + if href.startswith('/paroles') and not href.endswith('alpha.html') and titleattrib.startswith('Paroles '): + title = unicode(link.text) + ids = href.replace('/','').replace('.html','').split('paroles_') + id = '%s|%s'%(ids[1],ids[2]) + songlyrics = SongLyrics(id, title) + songlyrics.artist = artist + songlyrics.content = NotLoaded + yield songlyrics + + +class SongResultsPage(BasePage): + def iter_lyrics(self): + for link in self.parser.select(self.document.getroot(),'div#albums a'): + artist = NotAvailable + title = unicode(link.text.split(' - ')[0]) + href = link.attrib.get('href','') + if href.startswith('/paroles') and not href.endswith('alpha.html'): + ids = href.replace('/','').replace('.html','').split('paroles_') + id = '%s|%s'%(ids[1],ids[2]) + artist = unicode(link.text.split(' - ')[1]) + songlyrics = SongLyrics(id, title) + songlyrics.artist = artist + songlyrics.content = NotLoaded + yield songlyrics + + +class SonglyricsPage(BasePage): + def get_lyrics(self, id): + content = NotAvailable + artist = NotAvailable + title = NotAvailable + lyrdiv = self.parser.select(self.document.getroot(),'div#songlyrics_h') + if len(lyrdiv) > 0: + content = unicode(lyrdiv[0].text_content().strip()) + infos = self.parser.select(self.document.getroot(),'head > title',1).text + artist = unicode(infos.split(' - ')[1]) + title = unicode(infos.split(' - ')[0].replace('Paroles ','')) + songlyrics = SongLyrics(id, title) + songlyrics.artist = artist + songlyrics.content = content + return songlyrics diff --git a/modules/parolesmania/test.py b/modules/parolesmania/test.py new file mode 100644 index 00000000..65cc832a --- /dev/null +++ b/modules/parolesmania/test.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2013 Julien Veyssier +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from weboob.tools.test import BackendTest +from weboob.capabilities.base import NotLoaded + +class ParolesmaniaTest(BackendTest): + BACKEND = 'parolesmania' + + def test_search_song_n_get(self): + l_lyrics = list(self.backend.iter_lyrics('song','chien')) + for songlyrics in l_lyrics: + assert songlyrics.id + assert songlyrics.title + assert songlyrics.artist + assert songlyrics.content is NotLoaded + full_lyr = self.backend.get_lyrics(songlyrics.id) + assert full_lyr.id + assert full_lyr.title + assert full_lyr.artist + assert full_lyr.content is not NotLoaded + + + def test_search_artist(self): + l_lyrics = list(self.backend.iter_lyrics('artist','boris')) + for songlyrics in l_lyrics: + assert songlyrics.id + assert songlyrics.title + assert songlyrics.artist + assert songlyrics.content is NotLoaded diff --git a/modules/parolesmusique/test.py b/modules/parolesmusique/test.py index 88b89bc5..96e56927 100644 --- a/modules/parolesmusique/test.py +++ b/modules/parolesmusique/test.py @@ -20,7 +20,7 @@ from weboob.tools.test import BackendTest from weboob.capabilities.base import NotLoaded -class SeeklyricsTest(BackendTest): +class ParolesmusiqueTest(BackendTest): BACKEND = 'parolesmusique' def test_search_song_n_get(self):