nojhan/weboob-devel
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

rewrite video decorators to be in browser

Browse source
This commit is contained in:
Christophe Benz 2010-05-20 11:46:00 +02:00 • committed by Romain Bignon
commit 0de0f2a768
19 changed files with 167 additions and 119 deletions
Download patch file Download diff file Expand all files Collapse all files

33
weboob/backend.py
View file

@ -157,36 +157,3 @@ class BaseBackend(object):
if isinstance(self, c):
return True
return False
def check_domain(domain):
def wrapper(func):
def inner(self, *args, **kwargs):
url = args[0]
if isinstance(url, (str,unicode)) and not url.isdigit() and u'youjizz.com' not in url:
return None
return func(self, *args, **kwargs)
return inner
return wrapper
def id2url(domain, id2url):
def wrapper(func):
def inner(self, *args, **kwargs):
arg = unicode(args[0])
if arg.startswith('http://'):
if domain in arg:
url = arg
else:
return None
else:
provider, _id = arg.split(':')
if provider == self.name:
url = id2url(_id)
else:
return None
new_args = [url]
new_args.extend(args[1:])
return func(self, *new_args, **kwargs)
return inner
return wrapper

31
weboob/backends/youjizz/__init__.py
View file

@ -1,21 +1,18 @@
# -*- coding: utf-8 -*-
"""
Copyright(C) 2010 Roger Philibert
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, version 3 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
"""
# Copyright(C) 2010 Roger Philibert
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
from .backend import YoujizzBackend

5
weboob/backends/youjizz/backend.py
View file

@ -16,7 +16,7 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
from weboob.backend import check_domain, id2url, BaseBackend
from weboob.backend import BaseBackend
from weboob.capabilities.video import ICapVideoProvider
from .browser import YoujizzBrowser
@ -36,13 +36,10 @@ class YoujizzBackend(BaseBackend, ICapVideoProvider):
CONFIG = {}
BROWSER = YoujizzBrowser
domain = u'youjizz.com'
@id2url(domain, YoujizzVideo.id2url)
def get_video(self, _id):
return self.browser.get_video(_id)
@check_domain(domain)
def iter_page_urls(self, mozaic_url):
return self.browser.iter_page_urls(mozaic_url)

5
weboob/backends/youjizz/browser.py
View file

@ -21,6 +21,7 @@ import urllib
from logging import warning
from weboob.tools.browser import BaseBrowser
from weboob.tools.browser.decorators import check_domain, id2url
from .pages.index import IndexPage
from .video import YoujizzVideo
@ -36,6 +37,7 @@ class YoujizzBrowser(BaseBrowser):
r'http://.*youjizz\.com/search/.+\.html': IndexPage,
}
@id2url(YoujizzVideo.id2url)
def get_video(self, url):
data = self.openurl(url).read()
def _get_url():
@ -56,8 +58,9 @@ class YoujizzBrowser(BaseBrowser):
duration = minutes * 60 + seconds
else:
duration = 0
return YoujizzVideo(_id=u'youjizz:%s' % _id, title=title, url=_get_url(), duration=duration, nsfw=True)
return YoujizzVideo(_id=_id, title=title, url=_get_url(), duration=duration, nsfw=True)
@check_domain
def iter_page_urls(self, mozaic_url):
raise NotImplementedError()

2
weboob/backends/youjizz/pages/index.py
View file

@ -54,7 +54,7 @@ class IndexPage(BasePage):
minutes, seconds = time_span.text.strip().split(':')
duration = 60 * int(minutes) + int(seconds)
yield YoujizzVideo(u'youjizz:%s' % _id,
yield YoujizzVideo(_id,
title=title,
duration=duration,
thumbnail_url=thumbnail_url,

4
weboob/backends/youjizz/video.py
View file

@ -23,6 +23,10 @@ __all__ = ['YoujizzVideo']
class YoujizzVideo(BaseVideo):
def __init__(self, *args, **kwargs):
BaseVideo.__init__(self, *args, **kwargs)
self.id = u'%s@youjizz.com' % self.id
@classmethod
def id2url(cls, _id):
return 'http://www.youjizz.com/videos/%s.html' % _id

5
weboob/backends/youporn/backend.py
View file

@ -16,11 +16,10 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
from weboob.backend import check_domain, id2url, BaseBackend
from weboob.backend import BaseBackend
from weboob.capabilities.video import ICapVideoProvider
from .browser import YoupornBrowser
from .video import YoupornVideo
__all__ = ['YoupornBackend']
@ -38,7 +37,6 @@ class YoupornBackend(BaseBackend, ICapVideoProvider):
BROWSER = YoupornBrowser
domain = u'youporn.com'
@id2url(domain, YoupornVideo.id2url)
def get_video(self, _id):
return self.browser.get_video(_id)
@ -48,6 +46,5 @@ class YoupornBackend(BaseBackend, ICapVideoProvider):
return iter(set())
return self.browser.iter_search_results(pattern, self.SORTBY[sortby])
@check_domain(domain)
def iter_page_urls(self, mozaic_url):
raise NotImplementedError()

33
weboob/backends/youporn/browser.py
View file

@ -1,27 +1,27 @@
# -*- coding: utf-8 -*-
"""
Copyright(C) 2010 Romain Bignon
# Copyright(C) 2010 Romain Bignon
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, version 3 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
"""
from weboob.tools.browser import BaseBrowser
from weboob.tools.browser.decorators import id2url
from .pages.index import IndexPage
from .pages.video import VideoPage
from .video import YoupornVideo
__all__ = ['YoupornBrowser']
@ -49,6 +49,7 @@ class YoupornBrowser(BaseBrowser):
assert self.is_on_page(IndexPage)
return self.page.iter_videos()
@id2url(YoupornVideo.id2url)
def get_video(self, url):
self.location(url)
return self.page.video

2
weboob/backends/youporn/pages/index.py
View file

@ -64,7 +64,7 @@ class IndexPage(PornPage):
rating = float(p.text.strip())
rating_max = float(p.find('span').text.strip()[2:])
yield YoupornVideo('youporn:%d' % int(_id),
yield YoupornVideo(int(_id),
title=title,
rating=rating,
rating_max=rating_max,

2
weboob/backends/youporn/pages/video.py
View file

@ -32,7 +32,7 @@ class VideoPage(PornPage):
if not PornPage.on_loaded(self):
return
self.video = YoupornVideo('youporn:%d' % self.get_id(),
self.video = YoupornVideo(self.get_id(),
self.get_title(),
self.get_url(),
nsfw=True)

4
weboob/backends/youporn/video.py
View file

@ -23,6 +23,10 @@ __all__ = ['YoupornVideo']
class YoupornVideo(BaseVideo):
def __init__(self, *args, **kwargs):
BaseVideo.__init__(self, *args, **kwargs)
self.id = u'%s@youporn.com' % self.id
@classmethod
def id2url(cls, _id):
return 'http://www.youporn.com/watch/%d' % int(_id)

9
weboob/backends/youtube/backend.py
View file

@ -18,7 +18,7 @@
import logging
from weboob.backend import check_domain, id2url, BaseBackend
from weboob.backend import BaseBackend
from weboob.capabilities.video import ICapVideoProvider
from .browser import YoutubeBrowser
@ -38,9 +38,7 @@ class YoutubeBackend(BaseBackend, ICapVideoProvider):
CONFIG = {}
BROWSER = YoutubeBrowser
domain = u'youtube.com'
@id2url(domain, YoutubeVideo.id2url)
def get_video(self, _id):
return self.browser.get_video(_id)
@ -48,7 +46,7 @@ class YoutubeBackend(BaseBackend, ICapVideoProvider):
try:
import gdata.youtube.service
except ImportError:
logging.warning('Youtube backend search feature requires python-gdata package.')
logging.error('Youtube backend search feature requires python-gdata package.')
return
yt_service = gdata.youtube.service.YouTubeService()
query = gdata.youtube.service.YouTubeVideoQuery()
@ -62,12 +60,11 @@ class YoutubeBackend(BaseBackend, ICapVideoProvider):
author = entry.media.name.text.decode('utf-8').strip()
else:
author = None
yield YoutubeVideo(u'youtube:%s' % entry.id.text.split('/')[-1].decode('utf-8'),
yield YoutubeVideo(entry.id.text.split('/')[-1].decode('utf-8'),
title=entry.media.title.text.decode('utf-8').strip(),
author=author,
duration=int(entry.media.duration.seconds.decode('utf-8').strip()),
thumbnail_url=entry.media.thumbnail[0].url.decode('utf-8').strip())
@check_domain(domain)
def iter_page_urls(self, mozaic_url):
raise NotImplementedError()

4
weboob/backends/youtube/browser.py
View file

@ -17,17 +17,21 @@
from weboob.tools.browser import BaseBrowser
from weboob.tools.browser.decorators import check_domain, id2url
from .pages import VideoPage
from .video import YoutubeVideo
__all__ = ['YoutubeBrowser']
class YoutubeBrowser(BaseBrowser):
DOMAIN = u'youtube.com'
PAGES = {'.*youtube\.com/watch\?v=(.+)': VideoPage,
}
@id2url(YoutubeVideo.id2url)
def get_video(self, url):
self.location(url)
return self.page.video

4
weboob/backends/youtube/pages/video.py
View file

@ -32,7 +32,7 @@ class VideoPage(BasePage):
VIDEO_SIGNATURE_REGEX = re.compile(r'&t=([^ ,&]*)')
def on_loaded(self):
self.video = YoutubeVideo(u'youtube:%s' % self.get_id())
self.video = YoutubeVideo(self.get_id())
self.video.title = self.get_title()
self.video.url = self.get_url()
self.set_details(self.video)
@ -51,7 +51,7 @@ class VideoPage(BasePage):
continue
for m in re.finditer(self.VIDEO_SIGNATURE_REGEX, data.text):
video_signature = m.group(1)
return 'http://www.youtube.com/get_video?video_id=%s&t=%s&fmt=18' % (self.video.id, video_signature)
return 'http://www.youtube.com/get_video?video_id=%s&t=%s&fmt=18' % (self.video.provider_id, video_signature)
def get_title(self):
found = self.document.getroot().cssselect('meta[name=title]')

4
weboob/backends/youtube/video.py
View file

@ -23,6 +23,10 @@ __all__ = ['YoutubeVideo']
class YoutubeVideo(BaseVideo):
def __init__(self, *args, **kwargs):
BaseVideo.__init__(self, *args, **kwargs)
self.id = u'%s@youtube.com' % self.id
@classmethod
def id2url(cls, _id):
return 'http://www.youtube.com/watch?v=%s' % _id

36
weboob/capabilities/video.py
View file

@ -1,22 +1,20 @@
# -*- coding: utf-8 -*-
"""
Copyright(C) 2010 Romain Bignon, Christophe Benz
# Copyright(C) 2010 Romain Bignon, Christophe Benz
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, version 3 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
"""
from .cap import ICap
@ -49,7 +47,11 @@ class BaseVideo(object):
@property
def page_url(self):
return self.id2url(self.id.split(':')[1])
return self.id2url(self.provider_id)
@property
def provider_id(self):
return self.id.split('@')[0]
class ICapVideoProvider(ICap):

19
weboob/tools/browser/__init__.py Normal file
View file

@ -0,0 +1,19 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010 Christophe Benz
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
from weboob.tools.browser.browser import *

30
weboob/tools/browser.py → weboob/tools/browser/browser.py
View file

@ -1,22 +1,20 @@
# -*- coding: utf-8 -*-
"""
Copyright(C) 2010 Romain Bignon
# Copyright(C) 2010 Romain Bignon
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, version 3 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
"""
import mechanize
import urllib

54
weboob/tools/browser/decorators.py Normal file
View file

@ -0,0 +1,54 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010 Christophe Benz
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
__all__ = ['check_domain', 'id2url']
def check_domain(domain):
def wrapper(func):
def inner(self, *args, **kwargs):
if self.DOMAIN not in args[0]:
return None
return func(self, *args, **kwargs)
return inner
return wrapper
def id2url(id2url):
def wrapper(func):
def inner(self, *args, **kwargs):
arg = unicode(args[0])
if arg.startswith('http://'):
if self.DOMAIN in arg:
url = arg
else:
return None
else:
if '@' in arg:
_id, provider = arg.split('@')
if provider in self.DOMAIN:
url = id2url(_id)
else:
return None
else:
url = id2url(arg)
new_args = [url]
new_args.extend(args[1:])
return func(self, *new_args, **kwargs)
return inner
return wrapper