From 591db5c1bef084a4581b33df5c7fe9ca7c85dd1f Mon Sep 17 00:00:00 2001 From: Julien Veyssier Date: Fri, 22 Feb 2013 00:38:12 +0100 Subject: [PATCH] attilasub backend created, search works --- modules/attilasub/__init__.py | 3 ++ modules/attilasub/backend.py | 53 ++++++++++++++++++++++ modules/attilasub/browser.py | 47 ++++++++++++++++++++ modules/attilasub/pages.py | 83 +++++++++++++++++++++++++++++++++++ modules/attilasub/test.py | 38 ++++++++++++++++ 5 files changed, 224 insertions(+) create mode 100644 modules/attilasub/__init__.py create mode 100644 modules/attilasub/backend.py create mode 100644 modules/attilasub/browser.py create mode 100644 modules/attilasub/pages.py create mode 100644 modules/attilasub/test.py diff --git a/modules/attilasub/__init__.py b/modules/attilasub/__init__.py new file mode 100644 index 00000000..053d923e --- /dev/null +++ b/modules/attilasub/__init__.py @@ -0,0 +1,3 @@ +from .backend import AttilasubBackend + +__all__ = ['AttilasubBackend'] diff --git a/modules/attilasub/backend.py b/modules/attilasub/backend.py new file mode 100644 index 00000000..8ff109ed --- /dev/null +++ b/modules/attilasub/backend.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2010-2011 Julien Veyssier +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from weboob.capabilities.subtitle import ICapSubtitle +from weboob.tools.backend import BaseBackend + +from .browser import AttilasubBrowser + +from urllib import quote_plus + +__all__ = ['AttilasubBackend'] + + +class AttilasubBackend(BaseBackend, ICapSubtitle): + NAME = 'attilasub' + MAINTAINER = u'Julien Veyssier' + EMAIL = 'julien.veyssier@aiur.fr' + VERSION = '0.f' + DESCRIPTION = '"Attila'' s Website 2.0" french subtitles' + LICENSE = 'AGPLv3+' + BROWSER = AttilasubBrowser + + def create_default_browser(self): + return self.create_browser() + + #def get_subtitle(self, id): + # return self.browser.get_subtitle(id) + + def get_subtitle_file(self, id): + subtitle = self.browser.get_subtitle(id) + if not subtitle: + return None + + return self.browser.openurl(subtitle.url.encode('utf-8')).read() + + def iter_subtitles(self, pattern): + return self.browser.iter_subtitles(quote_plus(pattern.encode('utf-8'))) diff --git a/modules/attilasub/browser.py b/modules/attilasub/browser.py new file mode 100644 index 00000000..00db1489 --- /dev/null +++ b/modules/attilasub/browser.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2010-2011 Julien Veyssier +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.tools.browser import BaseBrowser + +from .pages import SubtitlesPage, SearchPage + + +__all__ = ['AttilasubBrowser'] + + +class AttilasubBrowser(BaseBrowser): + DOMAIN = 'davidbillemont3.free.fr' + PROTOCOL = 'http' + ENCODING = 'utf-8' + USER_AGENT = BaseBrowser.USER_AGENTS['wget'] + PAGES = { + 'http://search.freefind.com/find.html.*': SearchPage, + 'http://davidbillemont3.free.fr/.*.htm': SubtitlesPage, + } + + def iter_subtitles(self, pattern): + self.location('http://search.freefind.com/find.html?id=81131980&_charset_=&bcd=%%F7&scs=1&pageid=r&query=%s&mode=Find%%20pages%%20matching%%20ALL%%20words' % pattern.encode('utf-8')) + assert self.is_on_page(SearchPage) + return self.page.iter_subtitles(pattern) + + #def get_torrent(self, id): + # self.location('http://kat.ph/%s.html' % id) + # assert self.is_on_page(TorrentPage) + # return self.page.get_torrent(id) diff --git a/modules/attilasub/pages.py b/modules/attilasub/pages.py new file mode 100644 index 00000000..ff1c28d4 --- /dev/null +++ b/modules/attilasub/pages.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2010-2012 Julien Veyssier +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +try: + from urlparse import parse_qs +except ImportError: + from cgi import parse_qs # NOQA + +from urlparse import urlsplit + +from weboob.capabilities.subtitle import Subtitle +from weboob.capabilities.base import NotAvailable +from weboob.tools.browser import BasePage +from weboob.tools.misc import get_bytes_size + + +__all__ = ['SubtitlesPage','SearchPage'] + + +class SearchPage(BasePage): + def iter_subtitles(self,pattern): + fontresult = self.parser.select(self.document.getroot(),'div.search-results font.search-results') + # for each result in freefind, explore the page to iter subtitles + for res in fontresult: + a = self.parser.select(res,'a',1) + url = a.attrib.get('href','') + self.browser.location(url) + assert self.browser.is_on_page(SubtitlesPage) + # subtitles page does the job + for subtitle in self.browser.page.iter_subtitles(pattern): + yield subtitle + + +class SubtitlesPage(BasePage): + def iter_subtitles(self,pattern): + pattern = pattern.strip().replace('+',' ') + pattern_words = pattern.split() + tab = self.parser.select(self.document.getroot(),'table[bordercolor="#B8C0B2"]') + if len(tab) == 0: + tab = self.parser.select(self.document.getroot(),'table[bordercolordark="#B8C0B2"]') + if len(tab) == 0: + return + for line in tab[0].getiterator('tr'): + cols = self.parser.select(line,'td') + traduced_title = self.parser.select(cols[0],'font',1).text.lower() + original_title = self.parser.select(cols[1],'font',1).text.lower() + + traduced_title_words = traduced_title.split() + original_title_words = original_title.split() + + # if the pattern is one word and in the title OR if the + # intersection between pattern and the title is at least 2 words + if (len(pattern_words) == 1 and pattern in traduced_title_words) or\ + (len(pattern_words) == 1 and pattern in original_title_words) or\ + (len(list(set(pattern_words) & set(traduced_title_words))) > 1) or\ + (len(list(set(pattern_words) & set(original_title_words))) > 1): + + # this is to trash special spacing chars + traduced_title = " ".join(traduced_title_words) + original_title = " ".join(original_title_words) + + title = "%s (%s)"%(original_title,traduced_title) + subtitle = Subtitle(title,title) + subtitle.url = self.parser.select(cols[3],'a',1).attrib.get('href','') + subtitle.fps = 0 + yield subtitle diff --git a/modules/attilasub/test.py b/modules/attilasub/test.py new file mode 100644 index 00000000..36fb4fdb --- /dev/null +++ b/modules/attilasub/test.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2010-2011 Julien Veyssier, Laurent Bachelier +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from weboob.tools.test import BackendTest +from weboob.capabilities.base import NotLoaded + +import urllib +from random import choice + +class AttilasubTest(BackendTest): + BACKEND = 'attilasub' + + def test_subtitle(self): + subtitles = list(self.backend.iter_subtitles('spiderman')) + for subtitle in subtitles: + path, qs = urllib.splitquery(subtitle.url) + #assert path.endswith('.torrent') + + # get the file of a random sub + if len(subtitles): + subtitle = choice(subtitles) + self.backend.get_subtitle_file(subtitle.id)