new subtitle backend : tvsubtitles
This commit is contained in:
Julien Veyssier
parent
968df91c4d
commit
a6298871f2
8 changed files with 278 additions and 6 deletions
|
|
@ -62,11 +62,6 @@ class OpensubtitlesBrowser(BaseBrowser):
|
|||
return self.page.iter_subtitles()
|
||||
|
||||
def get_subtitle(self, id):
|
||||
""" the id is formed this way : id_movie|id_file
|
||||
the id_movie helps to find the page
|
||||
the id_file help to find the file into the page
|
||||
if NO id_movie set, using id_file to form the URL
|
||||
"""
|
||||
self.location('http://www.opensubtitles.org/subtitles/%s' % id)
|
||||
assert self.is_on_page(SubtitlePage)
|
||||
return self.page.get_subtitle()
|
||||
|
|
|
|||
|
|
@ -29,7 +29,6 @@ from weboob.capabilities.subtitle import Subtitle
|
|||
from weboob.capabilities.base import NotAvailable
|
||||
from weboob.tools.browser import BasePage
|
||||
from weboob.tools.misc import get_bytes_size
|
||||
import time
|
||||
|
||||
|
||||
__all__ = ['SubtitlesPage','SearchPage']
|
||||
|
|
|
|||
3
modules/tvsubtitles/__init__.py
Normal file
3
modules/tvsubtitles/__init__.py
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
from .backend import TvsubtitlesBackend
|
||||
|
||||
__all__ = ['TvsubtitlesBackend']
|
||||
55
modules/tvsubtitles/backend.py
Normal file
55
modules/tvsubtitles/backend.py
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright(C) 2010-2011 Julien Veyssier
|
||||
#
|
||||
# This file is part of weboob.
|
||||
#
|
||||
# weboob is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# weboob is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
from weboob.capabilities.subtitle import ICapSubtitle
|
||||
from weboob.tools.backend import BaseBackend
|
||||
|
||||
from .browser import TvsubtitlesBrowser,LANGUAGE_CONV
|
||||
|
||||
from urllib import quote_plus
|
||||
|
||||
__all__ = ['TvsubtitlesBackend']
|
||||
|
||||
|
||||
class TvsubtitlesBackend(BaseBackend, ICapSubtitle):
|
||||
NAME = 'tvsubtitles'
|
||||
MAINTAINER = u'Julien Veyssier'
|
||||
EMAIL = 'julien.veyssier@aiur.fr'
|
||||
VERSION = '0.f'
|
||||
DESCRIPTION = 'Tvsubtitles subtitle website'
|
||||
LICENSE = 'AGPLv3+'
|
||||
BROWSER = TvsubtitlesBrowser
|
||||
|
||||
def create_default_browser(self):
|
||||
return self.create_browser()
|
||||
|
||||
def get_subtitle(self, id):
|
||||
return self.browser.get_subtitle(id)
|
||||
|
||||
def get_subtitle_file(self, id):
|
||||
subtitle = self.browser.get_subtitle(id)
|
||||
if not subtitle:
|
||||
return None
|
||||
|
||||
return self.browser.openurl(subtitle.url.encode('utf-8')).read()
|
||||
|
||||
def iter_subtitles(self, language, pattern):
|
||||
if language not in LANGUAGE_CONV.keys():
|
||||
return []
|
||||
return self.browser.iter_subtitles(language,quote_plus(pattern.encode('utf-8')))
|
||||
66
modules/tvsubtitles/browser.py
Normal file
66
modules/tvsubtitles/browser.py
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright(C) 2010-2011 Julien Veyssier
|
||||
#
|
||||
# This file is part of weboob.
|
||||
#
|
||||
# weboob is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# weboob is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
from weboob.tools.browser import BaseBrowser
|
||||
|
||||
from .pages import SeriePage, SearchPage, SeasonPage,HomePage
|
||||
|
||||
|
||||
__all__ = ['OpensubtitlesBrowser']
|
||||
|
||||
LANGUAGE_CONV = {
|
||||
'ar':'ara', 'eo':'epo', 'ga':'', 'ru':'rus',
|
||||
'af':'' , 'et':'est', 'it':'ita', 'sr':'scc',
|
||||
'sq':'alb', 'tl':'' , 'ja':'jpn', 'sk':'slo',
|
||||
'hy':'arm', 'fi':'fin', 'kn':'', 'sl':'slv',
|
||||
'az':'' , 'fr':'fre', 'ko':'kor', 'es':'spa',
|
||||
'eu':'baq', 'gl':'glg', 'la':'', 'sw':'swa',
|
||||
'be':'' , 'ka':'geo', 'lv':'lav', 'sv':'swe',
|
||||
'bn':'ben', 'de':'ger', 'lt':'lit', 'ta':'',
|
||||
'bg':'bul', 'gr':'ell', 'mk':'mac', 'te':'tel',
|
||||
'ca':'cat', 'gu':'' , 'ms':'may', 'th':'tha',
|
||||
'zh':'chi', 'ht':'' , 'mt':'', 'tr':'tur',
|
||||
'hr':'hrv', 'iw':'heb', 'no':'nor', 'uk':'ukr',
|
||||
'cz':'cze', 'hi':'hin', 'fa':'per', 'ur':'urd',
|
||||
'da':'dan', 'hu':'hun', 'pl':'pol', 'vi':'vie',
|
||||
'nl':'dut', 'is':'ice', 'pt':'por', 'cy':'',
|
||||
'en':'eng', 'id':'ind', 'ro':'rum', 'yi':''}
|
||||
|
||||
class TvsubtitlesBrowser(BaseBrowser):
|
||||
DOMAIN = 'www.tvsubtitles.net'
|
||||
PROTOCOL = 'http'
|
||||
ENCODING = 'utf-8'
|
||||
USER_AGENT = BaseBrowser.USER_AGENTS['wget']
|
||||
PAGES = {
|
||||
'http://www.tvsubtitles.net': HomePage,
|
||||
'http://www.tvsubtitles.net/search.php': SearchPage,
|
||||
'http://www.tvsubtitles.net/tvshow-.*.html': SeriePage,
|
||||
'http://www.tvsubtitles.net/subtitle-[0-9]*-[0-9]*-.*.html' : SeasonPage
|
||||
}
|
||||
|
||||
def iter_subtitles(self, language, pattern):
|
||||
self.location('http://www.tvsubtitles.net')
|
||||
assert self.is_on_page(HomePage)
|
||||
return self.page.iter_subtitles(language,pattern)
|
||||
|
||||
def get_subtitle(self, id):
|
||||
self.location('http://www.tvsubtitles.net/subtitle-%s.html' % id)
|
||||
assert self.is_on_page(SeasonPage)
|
||||
return self.page.get_subtitle()
|
||||
BIN
modules/tvsubtitles/favicon.png
Normal file
BIN
modules/tvsubtitles/favicon.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 6.3 KiB |
115
modules/tvsubtitles/pages.py
Normal file
115
modules/tvsubtitles/pages.py
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright(C) 2010-2012 Julien Veyssier
|
||||
#
|
||||
# This file is part of weboob.
|
||||
#
|
||||
# weboob is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# weboob is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
try:
|
||||
from urlparse import parse_qs
|
||||
except ImportError:
|
||||
from cgi import parse_qs # NOQA
|
||||
|
||||
from urlparse import urlsplit
|
||||
|
||||
from weboob.capabilities.subtitle import Subtitle
|
||||
from weboob.capabilities.base import NotAvailable
|
||||
from weboob.tools.browser import BasePage
|
||||
from weboob.tools.misc import get_bytes_size
|
||||
|
||||
|
||||
__all__ = ['HomePage','SearchPage','SeriePage','SeasonPage']
|
||||
|
||||
|
||||
class HomePage(BasePage):
|
||||
def iter_subtitles(self,language,pattern):
|
||||
self.browser.select_form(nr=0)
|
||||
self.browser['q'] = pattern.encode('utf-8')
|
||||
self.browser.submit()
|
||||
assert self.browser.is_on_page(SearchPage)
|
||||
for subtitle in self.browser.page.iter_subtitles(language):
|
||||
yield subtitle
|
||||
|
||||
|
||||
class SearchPage(BasePage):
|
||||
""" Page which contains results as a list of series
|
||||
"""
|
||||
def iter_subtitles(self,language):
|
||||
list_result = self.parser.select(self.document.getroot(),'div.left_articles ul',1)
|
||||
li_result = self.parser.select(list_result,'li')
|
||||
for line in li_result:
|
||||
if len(self.parser.select(line,'img[alt=%s]'%language)) > 0:
|
||||
link = self.parser.select(line,'a',1)
|
||||
href = link.attrib.get('href','')
|
||||
self.browser.location("http://%s%s"%(self.browser.DOMAIN,href))
|
||||
assert self.browser.is_on_page(SeriePage)
|
||||
for subtitle in self.browser.page.iter_subtitles(language):
|
||||
yield subtitle
|
||||
|
||||
|
||||
class SeriePage(BasePage):
|
||||
""" Page of all seasons
|
||||
"""
|
||||
def iter_subtitles(self,language,only_one_season=False):
|
||||
# handle the current season
|
||||
last_table_line = self.parser.select(self.document.getroot(),'table#table5 tr')[-1]
|
||||
amount = int(self.parser.select(last_table_line,'td')[2].text_content())
|
||||
if amount > 0:
|
||||
my_lang_img = self.parser.select(last_table_line,'img[alt=%s]'%language)
|
||||
if len(my_lang_img) > 0:
|
||||
url_current_season = self.browser.geturl().split('/')[-1].replace('tvshow','subtitle').replace('.html','-%s.html'%language)
|
||||
self.browser.location(url_current_season)
|
||||
assert self.browser.is_on_page(SeasonPage)
|
||||
yield self.browser.page.iter_subtitles()
|
||||
|
||||
if not only_one_season:
|
||||
# handle the other seasons by following top links
|
||||
other_seasons_links = self.parser.select(self.document.getroot(),'p.description a')
|
||||
for link in other_seasons_links:
|
||||
href = link.attrib.get('href','')
|
||||
self.browser.location("http://%s/%s"%(self.browser.DOMAIN,href))
|
||||
assert self.browser.is_on_page(SeriePage)
|
||||
for subtitle in self.browser.page.iter_subtitles(language,True):
|
||||
yield subtitle
|
||||
|
||||
|
||||
class SeasonPage(BasePage):
|
||||
""" Page of a season with the right language
|
||||
"""
|
||||
def get_subtitle(self):
|
||||
filename_line = self.parser.select(self.document.getroot(),'img[alt=filename]',1).getparent().getparent()
|
||||
name = self.parser.select(filename_line,'td')[2].text
|
||||
id = self.browser.geturl().split('/')[-1].replace('.html','').replace('subtitle-','')
|
||||
url = "http://%s/download-%s.html"%(self.browser.DOMAIN,id)
|
||||
amount_line = self.parser.select(self.document.getroot(),'tr[title~=amount]',1)
|
||||
nb_cd = int(self.parser.select(amount_line,'td')[2].text)
|
||||
lang = url.split('-')[-1].split('.html')[0]
|
||||
filenames_line = self.parser.select(self.document.getroot(),'tr[title~=list]',1)
|
||||
file_names = self.parser.select(filenames_line,'td')[2].text_content().strip().replace('.srt','.srt\n')
|
||||
desc = u"files :\n"
|
||||
desc += file_names
|
||||
fps = 0
|
||||
|
||||
subtitle = Subtitle(id,name)
|
||||
subtitle.url = url
|
||||
subtitle.fps = fps
|
||||
subtitle.language = lang
|
||||
subtitle.nb_cd = nb_cd
|
||||
subtitle.description = desc
|
||||
return subtitle
|
||||
|
||||
def iter_subtitles(self):
|
||||
return self.get_subtitle()
|
||||
39
modules/tvsubtitles/test.py
Normal file
39
modules/tvsubtitles/test.py
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright(C) 2010-2011 Julien Veyssier
|
||||
#
|
||||
# This file is part of weboob.
|
||||
#
|
||||
# weboob is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# weboob is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
from weboob.tools.test import BackendTest
|
||||
from weboob.capabilities.base import NotLoaded
|
||||
|
||||
import urllib
|
||||
from random import choice
|
||||
|
||||
class TvsubtitlesTest(BackendTest):
|
||||
BACKEND = 'tvsubtitles'
|
||||
|
||||
def test_subtitle(self):
|
||||
subtitles = list(self.backend.iter_subtitles('fr','sopranos'))
|
||||
assert (len(subtitles) > 0)
|
||||
for subtitle in subtitles:
|
||||
path, qs = urllib.splitquery(subtitle.url)
|
||||
assert path.endswith('.zip')
|
||||
|
||||
# get the file of a random sub
|
||||
if len(subtitles):
|
||||
subtitle = choice(subtitles)
|
||||
self.backend.get_subtitle_file(subtitle.id)
|
||||
Loading…
Add table
Add a link
Reference in a new issue