diff --git a/weboob/backends/dailymotion/__init__.py b/weboob/backends/dailymotion/__init__.py new file mode 100644 index 00000000..b54c8a2c --- /dev/null +++ b/weboob/backends/dailymotion/__init__.py @@ -0,0 +1,3 @@ +from .backend import DailymotionBackend + +__all__ = ['DailymotionBackend'] diff --git a/weboob/backends/dailymotion/backend.py b/weboob/backends/dailymotion/backend.py new file mode 100644 index 00000000..08e60830 --- /dev/null +++ b/weboob/backends/dailymotion/backend.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2011 Romain Bignon +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +from __future__ import with_statement + +from weboob.capabilities.video import ICapVideo +from weboob.tools.backend import BaseBackend + +from .browser import DailymotionBrowser +from .video import DailymotionVideo + + +__all__ = ['DailymotionBackend'] + + +class DailymotionBackend(BaseBackend, ICapVideo): + NAME = 'dailymotion' + MAINTAINER = 'Romain Bignon' + EMAIL = 'romain@weboob.org' + VERSION = '0.7' + DESCRIPTION = 'Dailymotion videos website' + LICENSE = 'GPLv3' + BROWSER = DailymotionBrowser + + def get_video(self, _id): + with self.browser: + return self.browser.get_video(_id) + + SORTBY = ['relevance', 'rated', 'visited', None] + def iter_search_results(self, pattern=None, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False, max_results=None): + with self.browser: + return self.browser.iter_search_results(pattern, self.SORTBY[sortby]) + + def fill_video(self, video, fields): + if fields != ['thumbnail']: + # if we don't want only the thumbnail, we probably want also every fields + with self.browser: + video = self.browser.get_video(DailymotionVideo.id2url(video.id), video) + if 'thumbnail' in fields: + with self.browser: + video.thumbnail.data = self.browser.readurl(video.thumbnail.url) + + return video + + OBJECTS = {DailymotionVideo: fill_video} diff --git a/weboob/backends/dailymotion/browser.py b/weboob/backends/dailymotion/browser.py new file mode 100644 index 00000000..4463215a --- /dev/null +++ b/weboob/backends/dailymotion/browser.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2011 Romain Bignon +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +from urllib import quote_plus + +from weboob.tools.browser import BaseBrowser +from weboob.tools.browser.decorators import id2url + +from .pages import IndexPage, VideoPage +from .video import DailymotionVideo + + +__all__ = ['DailymotionBrowser'] + + +class DailymotionBrowser(BaseBrowser): + DOMAIN = 'dailymotion.com' + ENCODING = None + PAGES = {r'http://[w\.]*dailymotion\.com/?': IndexPage, + r'http://[w\.]*dailymotion\.com/(\w+/)?search/.*': IndexPage, + r'http://[w\.]*dailymotion\.com/video/(?P.+)': VideoPage, + } + + @id2url(DailymotionVideo.id2url) + def get_video(self, url, video=None): + self.location(url) + return self.page.get_video(video) + + def iter_search_results(self, pattern, sortby): + if not pattern: + self.home() + else: + if sortby is None: + url = '/search/%s/1' % quote_plus(pattern) + else: + url = '/%s/search/%s/1' % (sortby, quote_plus(pattern)) + self.location(url) + + assert self.is_on_page(IndexPage) + return self.page.iter_videos() diff --git a/weboob/backends/dailymotion/pages.py b/weboob/backends/dailymotion/pages.py new file mode 100644 index 00000000..fdb7559c --- /dev/null +++ b/weboob/backends/dailymotion/pages.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2011 Romain Bignon +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +import datetime +import urllib +import re + +from weboob.capabilities.video import VideoThumbnail +from weboob.tools.misc import html2text +from weboob.tools.browser import BasePage +from weboob.tools.parsers.lxmlparser import select + +from .video import DailymotionVideo + + +__all__ = ['IndexPage', 'VideoPage'] + + +class IndexPage(BasePage): + def iter_videos(self): + for div in select(self.document.getroot(), 'div.dmpi_video_item'): + _id = 0 + for cls in div.attrib['class'].split(): + if cls.startswith('id_'): + _id = int(cls[3:]) + break + + if _id == 0: + self.browser.logger.warning('Unable to find the ID of a video') + continue + + video = DailymotionVideo(int(_id)) + video.title = select(div, 'h3 a', 1).text + video.author = select(div, 'div.dmpi_user_login', 1).find('a').text + video.description = html2text(self.browser.parser.tostring(select(div, 'div.dmpi_video_description', 1))).strip() + minutes, seconds = select(div, 'div.duration', 1).text.split(':') + video.duration = datetime.timedelta(minutes=int(minutes), seconds=int(seconds)) + url = select(div, 'img.dmco_image', 1).attrib['src'] + video.thumbnail = VideoThumbnail(url) + + rating_div = select(div, 'div.small_stars', 1) + video.rating_max = self.get_rate(rating_div) + video.rating = self.get_rate(rating_div.find('div')) + # XXX missing date + yield video + + def get_rate(self, div): + m = re.match('width: *(\d+)px', div.attrib['style']) + if m: + return int(m.group(1)) + else: + self.browser.logger.warning('Unable to parse rating: %s' % div.attrib['style']) + return 0 + +class VideoPage(BasePage): + def get_video(self, video=None): + if video is None: + video = DailymotionVideo(self.group_dict['id']) + + div = select(self.document.getroot(), 'div#content', 1) + + video.title = select(div, 'span.title', 1).text + video.author = select(div, 'a.name', 1).text + video.description = select(div, 'div#video_description', 1).text + for script in select(self.document.getroot(), 'div.dmco_html'): + if 'id' in script.attrib and script.attrib['id'].startswith('container_player_'): + text = script.find('script').text + mobj = re.search(r'(?i)addVariable\(\"video\"\s*,\s*\"([^\"]*)\"\)', text) + mediaURL = urllib.unquote(mobj.group(1)) + video.url = mediaURL + + return video diff --git a/weboob/backends/dailymotion/test.py b/weboob/backends/dailymotion/test.py new file mode 100644 index 00000000..5dc6132e --- /dev/null +++ b/weboob/backends/dailymotion/test.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2010-2011 Romain Bignon +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +from weboob.tools.test import BackendTest + +class DailymotionTest(BackendTest): + BACKEND = 'dailymotion' + + def test_dailymotion(self): + l = list(self.backend.iter_search_results('chirac')) + self.assertTrue(len(l) > 0) + v = l[0] + self.backend.fillobj(v, ('url',)) + self.assertTrue(v.url and v.url.startswith('http://'), 'URL for video "%s" not found: %s' % (v.id, v.url)) diff --git a/weboob/backends/dailymotion/video.py b/weboob/backends/dailymotion/video.py new file mode 100644 index 00000000..d9a96e5c --- /dev/null +++ b/weboob/backends/dailymotion/video.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2011 Romain Bignon +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + + +from weboob.capabilities.video import BaseVideo + + +__all__ = ['DailymotionVideo'] + + +class DailymotionVideo(BaseVideo): + @classmethod + def id2url(cls, _id): + if _id.isdigit(): + return 'http://www.dailymotion.com/video/%d' % int(_id) + else: + return None