support authentication on youtube

weboob/backends/youtube/backend.py

@@ -26,8 +26,9 @@ import re
 import urllib
 
 from weboob.capabilities.video import ICapVideo
-from weboob.tools.backend import BaseBackend
+from weboob.tools.backend import BaseBackend, BackendConfig
 from weboob.tools.misc import to_unicode
+from weboob.tools.value import ValueBackendPassword, Value
 
 from .browser import YoutubeBrowser
 from .video import YoutubeVideo
@@ -44,9 +45,18 @@ class YoutubeBackend(BaseBackend, ICapVideo):
     DESCRIPTION = 'Youtube videos website'
     LICENSE = 'AGPLv3+'
     BROWSER = YoutubeBrowser
+    CONFIG = BackendConfig(Value('username', label='Email address', default=''),
+                           ValueBackendPassword('password', label='Password', default=''))
 
     URL_RE = re.compile(r'^https?://(?:\w*\.?youtube\.com/(?:watch\?v=|v/)|youtu\.be\/|\w*\.?youtube\.com\/user\/\w+#p\/u\/\d+\/)([^\?&]+)')
 
+    def create_default_browser(self):
+        password = None
+        username = self.config['username'].get()
+        if len(username) > 0:
+            password = self.config['password'].get()
+        return self.create_browser(username, password)
+
     def _entry2video(self, entry):
         """
         Parse an entry returned by gdata and return a Video object.

weboob/backends/youtube/browser.py

@@ -20,7 +20,9 @@
 
 from weboob.tools.browser import BaseBrowser
 
-from .pages import ForbiddenVideoPage, VerifyAgePage, VideoPage
+from .pages import BaseYoutubePage, VideoPage, ForbiddenVideoPage, \
+                   VerifyAgePage, VerifyControversyPage, \
+                   LoginPage, LoginRedirectPage
 
 
 __all__ = ['YoutubeBrowser']
@@ -29,11 +31,23 @@ __all__ = ['YoutubeBrowser']
 class YoutubeBrowser(BaseBrowser):
     DOMAIN = u'youtube.com'
     ENCODING = None
-    PAGES = {r'.*youtube\.com/watch\?v=(?P<id>.+)': VideoPage,
+    PAGES = {r'https?://.*youtube\.com/': BaseYoutubePage,
-             r'.*youtube\.com/index\?ytsession=.+': ForbiddenVideoPage,
+             r'https?://.*youtube\.com/watch\?v=(?P<id>.+)': VideoPage,
-             r'.*youtube\.com/verify_age\?next_url=(?P<next_url>.+)': VerifyAgePage,
+             r'https?://.*youtube\.com/index\?ytsession=.+': ForbiddenVideoPage,
+             r'https?://.*youtube\.com/verify_age\?next_url=(?P<next_url>.+)': VerifyAgePage,
+             r'https?://.*youtube\.com/verify_controversy\?next_url(?P<next_url>.+)': VerifyControversyPage,
+             r'https?://accounts\.youtube\.com/accounts/SetSID.*': LoginRedirectPage,
+             r'https?://www.google.com/accounts/ServiceLogin.*': LoginPage,
             }
 
+    def is_logged(self):
+        logged = not self.is_on_page(BaseYoutubePage) or self.page.is_logged()
+        return logged
+
+    def login(self):
+        self.location('https://www.google.com/accounts/ServiceLogin?uilel=3&service=youtube&passive=true&continue=http%3A%2F%2Fwww.youtube.com%2Fsignin%3Faction_handle_signin%3Dtrue%26nomobiletemp%3D1%26hl%3Den_US%26next%3D%252F&hl=en_US&ltmpl=sso')
+        self.page.login(self.username, self.password)
+
     def get_video_url(self, player_url):
         self.location(player_url)
 

weboob/backends/youtube/pages.py

@@ -20,28 +20,65 @@
 
 import urllib
 
-from weboob.tools.browser import BasePage
+from weboob.tools.browser import BasePage, BrokenPageError, BrowserIncorrectPassword
 
 
-__all__ = ['ForbiddenVideo', 'ForbiddenVideoPage', 'VerifyAgePage', 'VideoPage']
+__all__ = ['LoginPage', 'LoginRedirectPage', 'ForbiddenVideo', 'ForbiddenVideoPage', \
+           'VerifyAgePage', 'VerifyControversyPage', 'VideoPage']
+
+
+class LoginPage(BasePage):
+    def on_loaded(self):
+        errors = []
+        for errdiv in self.parser.select(self.document.getroot(), 'div.errormsg'):
+            errors.append(errdiv.text.encode('utf-8').strip())
+
+        if len(errors) > 0:
+            raise BrowserIncorrectPassword(', '.join(errors))
+
+    def login(self, username, password):
+        self.browser.select_form(predicate=lambda form: form.attrs.get('id', '') == 'gaia_loginform')
+        self.browser['Email'] = username
+        self.browser['Passwd'] = password
+        self.browser.submit()
+
+class LoginRedirectPage(BasePage):
+    pass
 
 
 class ForbiddenVideo(Exception):
     pass
 
 
-class ForbiddenVideoPage(BasePage):
+class BaseYoutubePage(BasePage):
+    def is_logged(self):
+        try:
+            self.parser.select(self.document.getroot(), 'span#masthead-user-expander', 1)
+        except BrokenPageError:
+            return False
+        else:
+            return True
+
+class ForbiddenVideoPage(BaseYoutubePage):
     def on_loaded(self):
         element = self.parser.select(self.document.getroot(), '.yt-alert-content', 1)
         raise ForbiddenVideo(element.text.strip())
 
 
-class VerifyAgePage(BasePage):
+class VerifyAgePage(BaseYoutubePage):
     def on_loaded(self):
-        raise ForbiddenVideo('This video or group may contain content that is inappropriate for some users')
+        if not self.is_logged():
+            raise ForbiddenVideo('This video or group may contain content that is inappropriate for some users')
 
+        self.browser.select_form(predicate=lambda form: form.attrs.get('id', '') == 'confirm-age-form')
+        self.browser.submit()
 
-class VideoPage(BasePage):
+class VerifyControversyPage(BaseYoutubePage):
+    def on_loaded(self):
+        self.browser.select_form(predicate=lambda form: 'verify_controversy' in form.attrs.get('action', ''))
+        self.browser.submit()
+
+class VideoPage(BaseYoutubePage):
     AVAILABLE_FORMATS = [38, 37, 22, 45, 35, 34, 43, 18, 6, 5, 17, 13]
     FORMAT_EXTENSIONS = {
         13: '3gp',