bugfix #264: use gdata API for youtube

setup.py

@@ -39,6 +39,6 @@ setup(
     packages=find_packages(exclude=['ez_setup']),
     scripts=[os.path.join('scripts', script) for script in os.listdir('scripts')],
     install_requires=[
-                # 'pyyaml',
+        'gdata',
     ]
 )

weboob/backends/youtube/backend.py

@@ -21,7 +21,7 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 import re
 
 from weboob.backend import BaseBackend
-from weboob.capabilities.video import ICapVideoProvider
+from weboob.capabilities.video import ICapVideoProvider, Video
 
 from .browser import YoutubeBrowser
 
@@ -43,22 +43,28 @@ class YoutubeBackend(BaseBackend, ICapVideoProvider):
             return self._browser
         raise AttributeError, name
 
-    def need_url(func):
-        def inner(self, *args, **kwargs):
-            url = args[0]
-            if (u'youtube.com' not in url) and not re.match('^\w+$', url):
-                return None
-            return func(self, *args, **kwargs)
-        return inner
-
-    @need_url
     def get_video(self, _id):
         return self.browser.get_video(_id)
 
-    SORTBY = ['', 'video_avg_rating', 'video_view_count', 'video_date_uploaded']
     def iter_search_results(self, pattern=None, sortby=ICapVideoProvider.SEARCH_RELEVANCE):
-        return self.browser.iter_search_results(pattern, self.SORTBY[sortby])
+        import gdata.youtube.service
+        yt_service = gdata.youtube.service.YouTubeService()
+        query = gdata.youtube.service.YouTubeVideoQuery()
+        query.orderby = ('relevance', 'rating', 'viewCount', 'published')[sortby]
+        query.racy = 'include'
+        if pattern:
+            query.categories.extend('/%s' % search_term.lower().encode('utf-8') for search_term in pattern.split())
+        feed = yt_service.YouTubeQuery(query)
+        for entry in feed.entry:
+            if entry.media.name:
+                author = entry.media.name.text.decode('utf-8').strip()
+            else:
+                author = None
+            yield Video(entry.id.text.split('/')[-1].decode('utf-8'),
+                        title=entry.media.title.text.decode('utf-8').strip(),
+                        author=author,
+                        duration=int(entry.media.duration.seconds.decode('utf-8').strip()),
+                        preview_url=entry.media.thumbnail[0].url.decode('utf-8').strip())
 
-    @need_url
     def iter_page_urls(self, mozaic_url):
         raise NotImplementedError()

weboob/backends/youtube/browser.py

@@ -19,35 +19,20 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 """
 
 import urllib
-import re
 
 from weboob.tools.browser import BaseBrowser
 
-from .pages import VideoPage, ResultsPage
+from .pages import VideoPage
 
 __all__ = ['YoutubeBrowser']
 
 class YoutubeBrowser(BaseBrowser):
     PAGES = {'.*youtube\.com/watch\?v=(.+)': VideoPage,
-             '.*youtube\.com/results\?.*': ResultsPage,
             }
 
-    def iter_search_results(self, pattern, sortby):
+    def id2url(self, _id):
-        if not pattern:
+        return _id if 'youtube.com' in _id else 'http://www.youtube.com/watch?v=%s' % _id
-            self.home()
-        else:
-            if sortby:
-                sortby = '&search_sort=%s' % sortby
-            self.location('http://www.youtube.com/results?search_type=videos&search_query=%s%s' % (urllib.quote_plus(pattern), sortby))
-
-        assert self.is_on_page(ResultsPage)
-        return self.page.iter_videos()
 
     def get_video(self, _id):
-        if re.match('^\w+$', _id):
+        self.location(self.id2url(_id))
-            url = 'http://www.youtube.com/watch?v=%s' % _id
-        else:
-            url = _id
-
-        self.location(url)
         return self.page.video

weboob/backends/youtube/pages/results.py

@@ -1,66 +0,0 @@
-# -*- coding: utf-8 -*-
-
-"""
-Copyright(C) 2010  Romain Bignon
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, version 3 of the License.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-
-"""
-
-import re
-
-from weboob.tools.browser import BasePage
-from weboob.capabilities.video import Video
-
-class ResultsPage(BasePage):
-    WATCH_RE = re.compile('/watch\?v=(\w+)')
-    def iter_videos(self):
-        for div in self.document.getroot().cssselect("div[class^=video-entry]"):
-            a = div.find('a')
-            if a is None:
-                print 'wtf'
-                continue
-
-            _id = ''
-            m = self.WATCH_RE.match(a.attrib['href'])
-            if m:
-                _id = m.group(1)
-
-            title = a.find('span').find('img').attrib['alt']
-            preview_url = a.find('span').find('img').attrib['src']
-            if preview_url.endswith('.gif'):
-                preview_url = a.find('span').find('img').attrib['thumb']
-
-            vtime = a.find('span').find('span')
-            duration = 0
-            if not vtime is None:
-                vtime = vtime.find('span').text.split(':')
-                if len(vtime) > 0:
-                    duration += int(vtime[-1])
-                if len(vtime) > 1:
-                    duration += 60 * int(vtime[-2])
-                if len(vtime) > 3:
-                    duration += 3600 * int(vtime[-3])
-                if len(vtime) > 4:
-                    print 'WTF'
-
-            author = ''
-            author_div = div.cssselect('span[class=video-username]')
-            if author_div:
-                author = author_div[0].find('a').text.strip()
-            yield Video(_id,
-                        title,
-                        author=author,
-                        duration=duration,
-                        preview_url=preview_url)

weboob/backends/youtube/pages/video.py

@@ -25,7 +25,7 @@ from weboob.tools.browser import BasePage
 from weboob.capabilities.video import Video
 
 class VideoPage(BasePage):
-    URL_REGEX = re.compile(r"https?://[w\.]*youtube.com/watch\?v=(\w+)")
+    URL_REGEX = re.compile(r"https?://[w\.]*youtube.com/watch\?v=(.+)")
     VIDEO_SIGNATURE_REGEX = re.compile(r'&t=([^ ,&]*)')
 
     def on_loaded(self):