rewrite video decorators to be in browser

weboob/backend.py

@@ -157,36 +157,3 @@ class BaseBackend(object):
             if isinstance(self, c):
                 return True
         return False
-
-
-def check_domain(domain):
-    def wrapper(func):
-        def inner(self, *args, **kwargs):
-            url = args[0]
-            if isinstance(url, (str,unicode)) and not url.isdigit() and u'youjizz.com' not in url:
-                return None
-            return func(self, *args, **kwargs)
-        return inner
-    return wrapper
-
-
-def id2url(domain, id2url):
-    def wrapper(func):
-        def inner(self, *args, **kwargs):
-            arg = unicode(args[0])
-            if arg.startswith('http://'):
-                if domain in arg:
-                    url = arg
-                else:
-                    return None
-            else:
-                provider, _id = arg.split(':')
-                if provider == self.name:
-                    url = id2url(_id)
-                else:
-                    return None
-            new_args = [url]
-            new_args.extend(args[1:])
-            return func(self, *new_args, **kwargs)
-        return inner
-    return wrapper

weboob/backends/youjizz/__init__.py

@@ -1,21 +1,18 @@
 # -*- coding: utf-8 -*-
 
-"""
+# Copyright(C) 2010  Roger Philibert
-Copyright(C) 2010  Roger Philibert
+# 
-
+# This program is free software; you can redistribute it and/or modify
-This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
-it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 of the License.
-the Free Software Foundation, version 3 of the License.
+# 
-
+# This program is distributed in the hope that it will be useful,
-This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
-but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
-GNU General Public License for more details.
+# 
-
+# You should have received a copy of the GNU General Public License
-You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
-along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-
-"""
 
 from .backend import YoujizzBackend

weboob/backends/youjizz/backend.py

@@ -16,7 +16,7 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 
-from weboob.backend import check_domain, id2url, BaseBackend
+from weboob.backend import BaseBackend
 from weboob.capabilities.video import ICapVideoProvider
 
 from .browser import YoujizzBrowser
@@ -36,13 +36,10 @@ class YoujizzBackend(BaseBackend, ICapVideoProvider):
 
     CONFIG = {}
     BROWSER = YoujizzBrowser
-    domain = u'youjizz.com'
 
-    @id2url(domain, YoujizzVideo.id2url)
     def get_video(self, _id):
         return self.browser.get_video(_id)
 
-    @check_domain(domain)
     def iter_page_urls(self, mozaic_url):
         return self.browser.iter_page_urls(mozaic_url)
 

weboob/backends/youjizz/browser.py

@@ -21,6 +21,7 @@ import urllib
 from logging import warning
 
 from weboob.tools.browser import BaseBrowser
+from weboob.tools.browser.decorators import check_domain, id2url
 
 from .pages.index import IndexPage
 from .video import YoujizzVideo
@@ -36,6 +37,7 @@ class YoujizzBrowser(BaseBrowser):
              r'http://.*youjizz\.com/search/.+\.html': IndexPage,
             }
 
+    @id2url(YoujizzVideo.id2url)
     def get_video(self, url):
         data = self.openurl(url).read()
         def _get_url():
@@ -56,8 +58,9 @@ class YoujizzBrowser(BaseBrowser):
             duration = minutes * 60 + seconds
         else:
             duration = 0
-        return YoujizzVideo(_id=u'youjizz:%s' % _id, title=title, url=_get_url(), duration=duration, nsfw=True)
+        return YoujizzVideo(_id=_id, title=title, url=_get_url(), duration=duration, nsfw=True)
 
+    @check_domain
     def iter_page_urls(self, mozaic_url):
         raise NotImplementedError()
 

weboob/backends/youjizz/pages/index.py

@@ -54,7 +54,7 @@ class IndexPage(BasePage):
                 minutes, seconds = time_span.text.strip().split(':')
                 duration = 60 * int(minutes) + int(seconds)
 
-            yield YoujizzVideo(u'youjizz:%s' % _id,
+            yield YoujizzVideo(_id,
                                title=title,
                                duration=duration,
                                thumbnail_url=thumbnail_url,

weboob/backends/youjizz/video.py

@@ -23,6 +23,10 @@ __all__ = ['YoujizzVideo']
 
 
 class YoujizzVideo(BaseVideo):
+    def __init__(self, *args, **kwargs):
+        BaseVideo.__init__(self, *args, **kwargs)
+        self.id = u'%s@youjizz.com' % self.id
+
     @classmethod
     def id2url(cls, _id):
         return 'http://www.youjizz.com/videos/%s.html' % _id

weboob/backends/youporn/backend.py

@@ -16,11 +16,10 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 
-from weboob.backend import check_domain, id2url, BaseBackend
+from weboob.backend import BaseBackend
 from weboob.capabilities.video import ICapVideoProvider
 
 from .browser import YoupornBrowser
-from .video import YoupornVideo
 
 
 __all__ = ['YoupornBackend']
@@ -38,7 +37,6 @@ class YoupornBackend(BaseBackend, ICapVideoProvider):
     BROWSER = YoupornBrowser
     domain = u'youporn.com'
 
-    @id2url(domain, YoupornVideo.id2url)
     def get_video(self, _id):
         return self.browser.get_video(_id)
 
@@ -48,6 +46,5 @@ class YoupornBackend(BaseBackend, ICapVideoProvider):
             return iter(set())
         return self.browser.iter_search_results(pattern, self.SORTBY[sortby])
 
-    @check_domain(domain)
     def iter_page_urls(self, mozaic_url):
         raise NotImplementedError()

weboob/backends/youporn/browser.py

@@ -1,27 +1,27 @@
 # -*- coding: utf-8 -*-
 
-"""
+# Copyright(C) 2010  Romain Bignon
-Copyright(C) 2010  Romain Bignon
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, version 3 of the License.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-
-"""
 
 from weboob.tools.browser import BaseBrowser
+from weboob.tools.browser.decorators import id2url
 
 from .pages.index import IndexPage
 from .pages.video import VideoPage
+from .video import YoupornVideo
 
 
 __all__ = ['YoupornBrowser']
@@ -49,6 +49,7 @@ class YoupornBrowser(BaseBrowser):
         assert self.is_on_page(IndexPage)
         return self.page.iter_videos()
 
+    @id2url(YoupornVideo.id2url)
     def get_video(self, url):
         self.location(url)
         return self.page.video

weboob/backends/youporn/pages/index.py

@@ -64,7 +64,7 @@ class IndexPage(PornPage):
                     rating = float(p.text.strip())
                     rating_max = float(p.find('span').text.strip()[2:])
 
-                yield YoupornVideo('youporn:%d' % int(_id),
+                yield YoupornVideo(int(_id),
                                    title=title,
                                    rating=rating,
                                    rating_max=rating_max,

weboob/backends/youporn/pages/video.py

@@ -32,7 +32,7 @@ class VideoPage(PornPage):
         if not PornPage.on_loaded(self):
             return
 
-        self.video = YoupornVideo('youporn:%d' % self.get_id(),
+        self.video = YoupornVideo(self.get_id(),
                                   self.get_title(),
                                   self.get_url(),
                                   nsfw=True)

weboob/backends/youporn/video.py

@@ -23,6 +23,10 @@ __all__ = ['YoupornVideo']
 
 
 class YoupornVideo(BaseVideo):
+    def __init__(self, *args, **kwargs):
+        BaseVideo.__init__(self, *args, **kwargs)
+        self.id = u'%s@youporn.com' % self.id
+
     @classmethod
     def id2url(cls, _id):
         return 'http://www.youporn.com/watch/%d' % int(_id)

weboob/backends/youtube/backend.py

@@ -18,7 +18,7 @@
 
 import logging
 
-from weboob.backend import check_domain, id2url, BaseBackend
+from weboob.backend import BaseBackend
 from weboob.capabilities.video import ICapVideoProvider
 
 from .browser import YoutubeBrowser
@@ -38,9 +38,7 @@ class YoutubeBackend(BaseBackend, ICapVideoProvider):
 
     CONFIG = {}
     BROWSER = YoutubeBrowser
-    domain = u'youtube.com'
 
-    @id2url(domain, YoutubeVideo.id2url)
     def get_video(self, _id):
         return self.browser.get_video(_id)
 
@@ -48,7 +46,7 @@ class YoutubeBackend(BaseBackend, ICapVideoProvider):
         try:
             import gdata.youtube.service
         except ImportError:
-            logging.warning('Youtube backend search feature requires python-gdata package.')
+            logging.error('Youtube backend search feature requires python-gdata package.')
             return
         yt_service = gdata.youtube.service.YouTubeService()
         query = gdata.youtube.service.YouTubeVideoQuery()
@@ -62,12 +60,11 @@ class YoutubeBackend(BaseBackend, ICapVideoProvider):
                 author = entry.media.name.text.decode('utf-8').strip()
             else:
                 author = None
-            yield YoutubeVideo(u'youtube:%s' % entry.id.text.split('/')[-1].decode('utf-8'),
+            yield YoutubeVideo(entry.id.text.split('/')[-1].decode('utf-8'),
                                title=entry.media.title.text.decode('utf-8').strip(),
                                author=author,
                                duration=int(entry.media.duration.seconds.decode('utf-8').strip()),
                                thumbnail_url=entry.media.thumbnail[0].url.decode('utf-8').strip())
 
-    @check_domain(domain)
     def iter_page_urls(self, mozaic_url):
         raise NotImplementedError()

weboob/backends/youtube/browser.py

@@ -17,17 +17,21 @@
 
 
 from weboob.tools.browser import BaseBrowser
+from weboob.tools.browser.decorators import check_domain, id2url
 
 from .pages import VideoPage
+from .video import YoutubeVideo
 
 
 __all__ = ['YoutubeBrowser']
 
 
 class YoutubeBrowser(BaseBrowser):
+    DOMAIN = u'youtube.com'
     PAGES = {'.*youtube\.com/watch\?v=(.+)': VideoPage,
             }
 
+    @id2url(YoutubeVideo.id2url)
     def get_video(self, url):
         self.location(url)
         return self.page.video

weboob/backends/youtube/pages/video.py

@@ -32,7 +32,7 @@ class VideoPage(BasePage):
     VIDEO_SIGNATURE_REGEX = re.compile(r'&t=([^ ,&]*)')
 
     def on_loaded(self):
-        self.video = YoutubeVideo(u'youtube:%s' % self.get_id())
+        self.video = YoutubeVideo(self.get_id())
         self.video.title = self.get_title()
         self.video.url = self.get_url()
         self.set_details(self.video)
@@ -51,7 +51,7 @@ class VideoPage(BasePage):
                 continue
             for m in re.finditer(self.VIDEO_SIGNATURE_REGEX, data.text):
                 video_signature = m.group(1)
-        return 'http://www.youtube.com/get_video?video_id=%s&t=%s&fmt=18' % (self.video.id, video_signature)
+        return 'http://www.youtube.com/get_video?video_id=%s&t=%s&fmt=18' % (self.video.provider_id, video_signature)
 
     def get_title(self):
         found = self.document.getroot().cssselect('meta[name=title]')

weboob/backends/youtube/video.py

@@ -23,6 +23,10 @@ __all__ = ['YoutubeVideo']
 
 
 class YoutubeVideo(BaseVideo):
+    def __init__(self, *args, **kwargs):
+        BaseVideo.__init__(self, *args, **kwargs)
+        self.id = u'%s@youtube.com' % self.id
+
     @classmethod
     def id2url(cls, _id):
         return 'http://www.youtube.com/watch?v=%s' % _id

weboob/capabilities/video.py

@@ -1,22 +1,20 @@
 # -*- coding: utf-8 -*-
 
-"""
+# Copyright(C) 2010  Romain Bignon, Christophe Benz
-Copyright(C) 2010  Romain Bignon, Christophe Benz
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, version 3 of the License.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-
-"""
 
 from .cap import ICap
 
@@ -49,7 +47,11 @@ class BaseVideo(object):
 
     @property
     def page_url(self):
-        return self.id2url(self.id.split(':')[1])
+        return self.id2url(self.provider_id)
+
+    @property
+    def provider_id(self):
+        return self.id.split('@')[0]
 
 
 class ICapVideoProvider(ICap):

weboob/tools/browser/__init__.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2010  Christophe Benz
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+
+from weboob.tools.browser.browser import *

weboob/tools/browser/browser.py

@@ -0,0 +1,371 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2010  Romain Bignon
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+
+import mechanize
+import urllib
+import urllib2
+import ClientForm
+import re
+import time
+from logging import warning, debug
+from copy import copy
+from threading import RLock
+
+from weboob.tools.parsers import get_parser
+from weboob.tools.decorators import retry
+
+# Try to load cookies
+try:
+    from weboob.tools.firefox_cookies import FirefoxCookieJar
+except ImportError, e:
+    warning("Unable to store cookies: %s" % e)
+    HAVE_COOKIES = False
+else:
+    HAVE_COOKIES = True
+
+
+__all__ = ['BrowserIncorrectPassword', 'BrowserBanned', 'BrowserUnavailable', 'BrowserRetry',
+           'BasePage', 'BaseBrowser', ]
+
+
+# Exceptions
+class BrowserIncorrectPassword(Exception):
+    pass
+
+class BrowserBanned(BrowserIncorrectPassword):
+    pass
+
+class BrowserUnavailable(Exception):
+    pass
+
+class BrowserRetry(Exception):
+    pass
+
+class NoHistory(object):
+    """
+    We don't want to fill memory with history
+    """
+    def __init__(self): pass
+    def add(self, request, response): pass
+    def back(self, n, _response): pass
+    def clear(self): pass
+    def close(self): pass
+
+class BasePage(object):
+    """
+    Base page
+    """
+    def __init__(self, browser, document, url=''):
+        self.browser = browser
+        self.document = document
+        self.url = url
+
+    def on_loaded(self):
+        """
+        Called when the page is loaded.
+        """
+        pass
+
+class BaseBrowser(mechanize.Browser):
+    """
+    Base browser class to navigate on a website.
+    """
+
+    # ------ Class attributes --------------------------------------
+
+    DOMAIN = None
+    PROTOCOL = 'http'
+    ENCODING = 'utf-8'
+    PAGES = {}
+    USER_AGENT = 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.4) Gecko/2008111318 Ubuntu/8.10 (intrepid) Firefox/3.0.3'
+
+    # ------ Abstract methods --------------------------------------
+
+    def home(self):
+        """
+        Go to the home page.
+        """
+        self.location('%s://%s' % (self.PROTOCOL, self.DOMAIN))
+
+    def login(self):
+        """
+        Login to the website.
+
+        This function is called when is_logged() returns False and the password
+        attribute is not None.
+        """
+        raise NotImplementedError()
+
+    def is_logged(self):
+        """
+        Return True if we are logged on website. When Browser tries to access
+        to a page, if this method returns False, it calls login().
+
+        It is never called if the password attribute is None.
+        """
+        raise NotImplementedError()
+
+    # ------ Browser methods ---------------------------------------
+
+    # I'm not a robot, so disable the check of permissions in robots.txt.
+    default_features = copy(mechanize.Browser.default_features)
+    default_features.remove('_robots')
+
+    def __init__(self, username=None, password=None, firefox_cookies=None,
+                 parser=None, history=NoHistory(), proxy=None):
+        """
+        Constructor of Browser.
+
+        @param username [str] username on website.
+        @param password [str] password on website. If it is None, Browser will
+                              not try to login.
+        @param filefox_cookies [str] Path to cookies' sqlite file.
+        @param parser [IParser]  parser to use on HTML files.
+        @param hisory [object]  History manager. Default value is an object
+                                which does not keep history.
+        """
+        mechanize.Browser.__init__(self, history=history)
+        self.addheaders = [
+                ['User-agent', self.USER_AGENT]
+            ]
+
+        # Use a proxy
+        if proxy:
+            proto = 'http'
+            if proxy.find('://') >= 0:
+                proto, domain = proxy.split('://', 1)
+            else:
+                domain = proxy
+            self.set_proxies({proto: domain})
+
+        # Share cookies with firefox
+        if firefox_cookies and HAVE_COOKIES:
+            self._cookie = FirefoxCookieJar(self.DOMAIN, firefox_cookies)
+            self._cookie.load()
+            self.set_cookiejar(self._cookie)
+        else:
+            self._cookie = None
+
+        if parser is None:
+            parser = get_parser()()
+        elif isinstance(parser, (tuple,list)):
+            parser = get_parser(parser)()
+        self.parser = parser
+        self.page = None
+        self.last_update = 0.0
+        self.username = username
+        self.password = password
+        self.lock = RLock()
+        if self.password:
+            try:
+                self.home()
+            except BrowserUnavailable:
+                pass
+
+    def __enter__(self):
+        self.lock.acquire()
+
+    def __exit__(self, t, v, tb):
+        self.lock.release()
+
+    def pageaccess(func):
+        def inner(self, *args, **kwargs):
+            if not self.page or self.password and not self.page.is_logged():
+                self.home()
+
+            return func(self, *args, **kwargs)
+        return inner
+
+    @pageaccess
+    def keepalive(self):
+        self.home()
+
+    def check_location(func):
+        def inner(self, *args, **kwargs):
+            if args and isinstance(args[0], (str,unicode)) and args[0][0] == '/' and \
+               (not self.request or self.request.host != self.DOMAIN):
+                args = ('%s://%s%s' % (self.PROTOCOL, self.DOMAIN, args[0]),) + args[1:]
+
+            return func(self, *args, **kwargs)
+        return inner
+
+    @check_location
+    @retry(BrowserUnavailable, tries=3)
+    def openurl(self, *args, **kwargs):
+        """
+        Open an URL but do not create a Page object.
+        """
+        debug('Opening URL "%s", %s' % (args, kwargs))
+        try:
+            return mechanize.Browser.open(self, *args, **kwargs)
+        except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError), e:
+            raise BrowserUnavailable('%s (url="%s")' % (e, args and args[0] or 'None'))
+        except (mechanize.BrowserStateError, BrowserRetry):
+            self.home()
+            return mechanize.Browser.open(self, *args, **kwargs)
+
+    def submit(self, *args, **kwargs):
+        """
+        Submit the selected form.
+        """
+        try:
+            self._change_location(mechanize.Browser.submit(self, *args, **kwargs))
+        except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError), e:
+            self.page = None
+            raise BrowserUnavailable(e)
+        except (mechanize.BrowserStateError, BrowserRetry), e:
+            self.home()
+            raise BrowserUnavailable(e)
+
+    def is_on_page(self, pageCls):
+        return isinstance(self.page, pageCls)
+
+    def follow_link(self, *args, **kwargs):
+        try:
+            self._change_location(mechanize.Browser.follow_link(self, *args, **kwargs))
+        except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError), e:
+            self.page = None
+            raise BrowserUnavailable('%s (url="%s")' % (e, args and args[0] or 'None'))
+        except (mechanize.BrowserStateError, BrowserRetry), e:
+            self.home()
+            raise BrowserUnavailable(e)
+
+    @check_location
+    @retry(BrowserUnavailable, tries=3)
+    def location(self, *args, **kwargs):
+        """
+        Change location of browser on an URL.
+
+        When the page is loaded, it looks up PAGES to find a regexp which
+        matches, and create the object. Then, the 'on_loaded' method of
+        this object is called.
+
+        If a password is set, and is_logged() returns False, it tries to login
+        with login() and reload the page.
+        """
+        keep_args = copy(args)
+        keep_kwargs = kwargs.copy()
+
+        try:
+            self._change_location(mechanize.Browser.open(self, *args, **kwargs))
+        except BrowserRetry:
+            if not self.page or not args or self.page.url != args[0]:
+                self.location(keep_args, keep_kwargs)
+        except (mechanize.response_seek_wrapper, urllib2.HTTPError, urllib2.URLError), e:
+            self.page = None
+            raise BrowserUnavailable('%s (url="%s")' % (e, args and args[0] or 'None'))
+        except mechanize.BrowserStateError:
+            self.home()
+            self.location(*keep_args, **keep_kwargs)
+
+    def _change_location(self, result):
+        """
+        This function is called when we have moved to a page, to load a Page
+        object.
+        """
+
+        # Find page from url
+        pageCls = None
+        for key, value in self.PAGES.items():
+            regexp = re.compile('^%s$' % key)
+            m = regexp.match(result.geturl())
+            if m:
+                pageCls = value
+                break
+
+        # Not found
+        if not pageCls:
+            self.page = None
+            r = result.read()
+            if isinstance(r, unicode):
+                r = r.encode('iso-8859-15', 'replace')
+            print r
+            warning('Ho my fucking god, there isn\'t any page named %s' % result.geturl())
+            return
+
+        debug('[%s] Went on %s' % (self.username, result.geturl()))
+        self.last_update = time.time()
+
+        document = self.parser.parse(result, self.ENCODING)
+        self.page = pageCls(self, document, result.geturl())
+        self.page.on_loaded()
+
+        if self.password is not None and not self.is_logged():
+            debug('!! Relogin !!')
+            self.login()
+            return
+
+        if self._cookie:
+            self._cookie.save()
+
+    def buildurl(self, base, *args, **kwargs):
+        """
+        Build an URL and escape arguments.
+        You can give a serie of tuples in *args (and the order is keept), or
+        a dict in **kwargs (but the order is lost).
+
+        Example:
+        >>> buildurl('/blah.php', ('a', '&'), ('c', '=')
+        '/blah.php?a=%26&b=%3D'
+        >>> buildurl('/blah.php', a='&', 'c'='=')
+        '/blah.php?b=%3D&a=%26'
+
+        """
+
+        if not args:
+            args = kwargs
+        if not args:
+            return base
+        else:
+            return '%s?%s' % (base, urllib.urlencode(args))
+
+    def str(self, s):
+        if isinstance(s, unicode):
+            s = s.encode('iso-8859-15', 'replace')
+        return s
+
+    def set_field(self, args, label, field=None, value=None, is_list=False):
+        """
+        Set a value to a form field.
+
+        @param args [dict]  arguments where to look for value.
+        @param label [str]  label in args.
+        @param field [str]  field name. If None, use label instead.
+        @param value [str]  value to give on field.
+        @param is_list [bool]  the field is a list.
+        """
+        try:
+            if not field:
+                field = label
+            if args.get(label, None) is not None:
+                if not value:
+                    if is_list:
+                        if isinstance(is_list, (list, tuple)):
+                            try:
+                                value = [self.str(is_list.index(args[label]))]
+                            except ValueError, e:
+                                if args[label]:
+                                    print '[%s] %s: %s' % (label, args[label], e)
+                                return
+                        else:
+                            value = [self.str(args[label])]
+                    else:
+                        value = self.str(args[label])
+                self[field] = value
+        except ClientForm.ControlNotFoundError:
+            return

weboob/tools/browser/decorators.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2010  Christophe Benz
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+
+__all__ = ['check_domain', 'id2url']
+
+
+def check_domain(domain):
+    def wrapper(func):
+        def inner(self, *args, **kwargs):
+            if self.DOMAIN not in args[0]:
+                return None
+            return func(self, *args, **kwargs)
+        return inner
+    return wrapper
+
+
+def id2url(id2url):
+    def wrapper(func):
+        def inner(self, *args, **kwargs):
+            arg = unicode(args[0])
+            if arg.startswith('http://'):
+                if self.DOMAIN in arg:
+                    url = arg
+                else:
+                    return None
+            else:
+                if '@' in arg:
+                    _id, provider = arg.split('@')
+                    if provider in self.DOMAIN:
+                        url = id2url(_id)
+                    else:
+                        return None
+                else:
+                    url = id2url(arg)
+            new_args = [url]
+            new_args.extend(args[1:])
+            return func(self, *new_args, **kwargs)
+        return inner
+    return wrapper