support repositories to manage backends (closes #747)

modules/pastebin/__init__.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2011 Laurent Bachelier
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+from .backend import PastebinBackend
+
+__all__ = ['PastebinBackend']

modules/pastebin/backend.py

@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2011 Laurent Bachelier
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+
+from __future__ import with_statement
+
+from weboob.tools.capabilities.paste import BasePasteBackend
+from weboob.tools.backend import BaseBackend, BackendConfig
+from weboob.capabilities.base import NotLoaded
+from weboob.tools.value import Value, ValueBackendPassword
+
+from .browser import PastebinBrowser
+from .paste import PastebinPaste
+
+
+__all__ = ['PastebinBackend']
+
+
+class PastebinBackend(BaseBackend, BasePasteBackend):
+    NAME = 'pastebin'
+    MAINTAINER = 'Laurent Bachelier'
+    EMAIL = 'laurent@bachelier.name'
+    VERSION = '0.a'
+    DESCRIPTION = 'Pastebin paste tool'
+    LICENSE = 'AGPLv3+'
+    BROWSER = PastebinBrowser
+    CONFIG = BackendConfig(
+        Value('username', label='Optional username', default=''),
+        ValueBackendPassword('password', label='Optional password', default=''),
+        ValueBackendPassword('api_key',  label='Optional API key',  default='', noprompt=True),
+    )
+
+    EXPIRATIONS = {
+        600: '10M',
+        3600: '1H',
+        3600*24: '1D',
+        3600*24*30: '1M',
+        False: 'N',
+    }
+
+    def create_default_browser(self):
+        username = self.config['username'].get()
+        if username:
+            password = self.config['password'].get()
+        else:
+            password = None
+        return self.create_browser(self.config['api_key'].get() if self.config['api_key'].get() else None,
+                                   username, password, get_home=False)
+
+    def new_paste(self, *args, **kwargs):
+        return PastebinPaste(*args, **kwargs)
+
+    def can_post(self, contents, title=None, public=None, max_age=None):
+        if max_age is not None:
+            if self.get_closest_expiration(max_age) is None:
+                return 0
+        if not title or len(title) <= 60:
+            return 2
+        return 1
+
+    def get_paste(self, _id):
+        with self.browser:
+            return self.browser.get_paste(_id)
+
+    def fill_paste(self, paste, fields):
+        # if we only want the contents
+        if fields == ['contents']:
+            if paste.contents is NotLoaded:
+                with self.browser:
+                    contents = self.browser.get_contents(paste.id)
+                paste.contents = contents
+        # get all fields
+        elif fields is None or len(fields):
+            with self.browser:
+                self.browser.fill_paste(paste)
+        return paste
+
+    def post_paste(self, paste, max_age=None, use_api=True):
+        if max_age is not None:
+            expiration = self.get_closest_expiration(max_age)
+        else:
+            expiration = None
+        with self.browser:
+            if use_api and self.config.get('api_key').get():
+                self.browser.api_post_paste(paste, expiration=self.EXPIRATIONS.get(expiration))
+            else:
+                self.browser.post_paste(paste, expiration=self.EXPIRATIONS.get(expiration))
+
+    OBJECTS = {PastebinPaste: fill_paste}

modules/pastebin/browser.py

@@ -0,0 +1,140 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2011 Laurent Bachelier
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+
+from weboob.tools.browser import BaseBrowser, BrowserHTTPNotFound, BrowserIncorrectPassword
+from weboob.tools.browser.decorators import id2url, check_url
+
+from weboob.capabilities.paste import PasteNotFound
+
+from .pages import PastePage, PostPage, UserPage
+from .paste import PastebinPaste
+
+import urllib
+import re
+
+__all__ = ['PastebinBrowser']
+
+class BadAPIRequest(Exception):
+    pass
+
+
+class PastebinBrowser(BaseBrowser):
+    DOMAIN = 'pastebin.com'
+    ENCODING = 'UTF-8'
+    PASTE_URL = 'http://%s/(?P<id>\w+)' % DOMAIN
+    API_URL = 'http://%s/api/api_post.php' % DOMAIN
+    PAGES = {PASTE_URL: PastePage,
+            'http://%s/u/(?P<username>.+)' % DOMAIN: UserPage,
+            'http://%s/' % DOMAIN: PostPage}
+
+    def __init__(self, api_key, *args, **kwargs):
+        self.api_key = api_key
+        self.user_key = None
+
+        BaseBrowser.__init__(self, *args, **kwargs)
+
+    def fill_paste(self, paste):
+        """
+        Get as much as information possible from the paste page
+        """
+        try:
+            self.location(paste.page_url, no_login=True)
+            return self.page.fill_paste(paste)
+        except BrowserHTTPNotFound:
+            raise PasteNotFound()
+
+    @id2url(PastebinPaste.id2url)
+    @check_url(PASTE_URL)
+    def get_paste(self, url):
+        _id = re.match('^%s$' % self.PASTE_URL, url).groupdict()['id']
+        return PastebinPaste(_id)
+
+    def get_contents(self, _id):
+        """
+        Get the contents from the raw URL
+        This is the fastest and safest method if you only want the content.
+        Returns unicode.
+        """
+        try:
+            return self.readurl('http://%s/raw.php?i=%s' % (self.DOMAIN, _id), if_fail='raise').decode(self.ENCODING)
+        except BrowserHTTPNotFound:
+            raise PasteNotFound()
+
+    def post_paste(self, paste, expiration=None):
+        self.home()
+        if not self.is_on_page(PostPage):
+            self.home()
+        self.page.post(paste, expiration=expiration)
+        paste.id = self.page.get_id()
+
+    def api_post_paste(self, paste, expiration=None):
+        data = {'api_dev_key': self.api_key,
+                'api_option': 'paste',
+                'api_paste_code': paste.contents.encode(self.ENCODING),
+        }
+        if self.password:
+            data['api_user_key'] = self.api_login()
+        if paste.public is True:
+            data['api_paste_private'] = '0'
+        elif paste.public is False:
+            data['api_paste_private'] = '1'
+        if paste.title:
+            data['api_paste_name'] = paste.title.encode(self.ENCODING)
+        if expiration:
+            data['api_paste_expire_date'] = expiration
+        res = self.readurl(self.API_URL, urllib.urlencode(data)).decode(self.ENCODING)
+        self._validate_api_response(res)
+        paste.id = re.match('^%s$' % self.PASTE_URL, res).groupdict()['id']
+
+    def api_login(self):
+        # "The api_user_key does not expire."
+        # TODO store it on disk
+        if self.user_key:
+            return self.user_key
+
+        data = {'api_dev_key': self.api_key,
+                'api_user_name': self.username,
+                'api_user_password': self.password
+        }
+        res = self.readurl('http://%s/api/api_login.php' % self.DOMAIN,
+                urllib.urlencode(data)).decode(self.ENCODING)
+        try:
+            self._validate_api_response(res)
+        except BadAPIRequest, e:
+            if str(e) == 'invalid login':
+                raise BrowserIncorrectPassword
+            else:
+                raise e
+        self.user_key = res
+        return res
+
+    def _validate_api_response(self, res):
+        matches = re.match('Bad API request, (?P<error>.+)', res)
+        if matches:
+            raise BadAPIRequest(matches.groupdict().get('error'))
+
+    def is_logged(self):
+        return self.page and self.page.is_logged()
+
+    def login(self):
+        self.location('http://%s/login' % self.DOMAIN, no_login=True)
+        self.page.login(self.username, self.password)
+        if not self.is_logged():
+            raise BrowserIncorrectPassword()

modules/pastebin/pages.py

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2011 Laurent Bachelier
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+
+from weboob.tools.browser import BasePage, BrokenPageError
+
+__all__ = ['PastePage', 'PostPage']
+
+class BasePastebinPage(BasePage):
+    def is_logged(self):
+        header = self.parser.select(self.document.getroot(),
+                'id("header_bottom")/ul[@class="top_menu"]', 1, 'xpath')
+        for link in header.xpath('//ul/li/a'):
+            if link.text == 'logout':
+                return True
+            if link.text == 'login':
+                return False
+
+# XXX hack, since all pages are detected as a PostPage we make PastePage
+# inherit LoginPage
+class LoginPage(BasePastebinPage):
+    def login(self, username, password):
+        self.browser.select_form(nr=1)
+        self.browser['user_name'] = username.encode(self.browser.ENCODING)
+        self.browser['user_password'] = password.encode(self.browser.ENCODING)
+        self.browser.submit()
+
+
+class PastePage(LoginPage):
+    def fill_paste(self, paste):
+        header = self.parser.select(self.document.getroot(),
+                'id("content_left")//div[@class="paste_box_info"]', 1, 'xpath')
+        paste.title = self.parser.select(header,
+                '//div[@class="paste_box_line1"]//h1', 1, 'xpath').text
+        paste.contents = self.parser.select(self.document.getroot(),
+                '//textarea[@id="paste_code"]', 1, 'xpath').text
+        visibility_text = self.parser.select(header,
+                '//div[@class="paste_box_line1"]//img', 1, 'xpath').attrib['alt']
+        if visibility_text.startswith('Public'):
+            paste.public = True
+        elif visibility_text.startswith('Private'):
+            paste.public = False
+        else:
+            raise BrokenPageError('Unable to get the paste visibility')
+        return paste
+
+    def get_id(self):
+        """
+        Find out the ID from the URL
+        """
+        return self.group_dict['id']
+
+
+class PostPage(BasePastebinPage):
+    def post(self, paste, expiration=None):
+        self.browser.select_form(name='myform')
+        self.browser['paste_code'] = paste.contents.encode(self.browser.ENCODING)
+        self.browser['paste_name'] = paste.title.encode(self.browser.ENCODING)
+        if paste.public is True:
+            self.browser['paste_private'] = ['0']
+        elif paste.public is False:
+            self.browser['paste_private'] = ['1']
+        if expiration:
+            self.browser['paste_expire_date'] = [expiration]
+        self.browser.submit()
+
+class UserPage(BasePastebinPage):
+    pass

modules/pastebin/paste.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2011 Laurent Bachelier
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+
+from weboob.capabilities.paste import BasePaste
+
+
+__all__ = ['PastebinPaste']
+
+
+class PastebinPaste(BasePaste):
+    @classmethod
+    def id2url(cls, _id):
+        return 'http://pastebin.com/%s' % _id

modules/pastebin/test.py

@@ -0,0 +1,92 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2011 Laurent Bachelier
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+from weboob.tools.test import BackendTest
+from weboob.capabilities.base import NotLoaded
+from weboob.capabilities.paste import PasteNotFound
+
+class PastebinTest(BackendTest):
+    BACKEND = 'pastebin'
+
+    def test_get_paste(self):
+        for _id in ('7HmXwzyt', 'http://pastebin.com/7HmXwzyt'):
+            # html method
+            p = self.backend.get_paste(_id)
+            self.backend.fillobj(p, ['title'])
+            assert p.title == 'plop'
+            assert p.page_url == 'http://pastebin.com/7HmXwzyt'
+            assert p.contents == 'prout'
+            assert p.public is True
+
+            # raw method
+            p = self.backend.get_paste(_id)
+            self.backend.fillobj(p, ['contents'])
+            assert p.title is NotLoaded
+            assert p.page_url == 'http://pastebin.com/7HmXwzyt'
+            assert p.contents == 'prout'
+            assert p.public is NotLoaded
+
+    def test_post(self):
+        p = self.backend.new_paste(None, title='ouiboube', contents='Weboob Test', public=True)
+        self.backend.post_paste(p, max_age=600)
+        assert p.id
+        self.backend.fill_paste(p, ['title'])
+        assert p.title == 'ouiboube'
+        assert p.id in p.page_url
+        assert p.public is True
+
+    def test_specialchars(self):
+        # post a paste and get the contents through the HTML response
+        p1 = self.backend.new_paste(None, title='ouiboube', contents=u'Weboob <test>¿¡', public=False)
+        self.backend.post_paste(p1, max_age=600)
+        assert p1.id
+        assert p1.public is False
+
+        # this should use the raw method to get the contents
+        p2 = self.backend.get_paste(p1.id)
+        self.backend.fillobj(p2, ['contents'])
+        assert p2.contents == p1.contents
+        assert p2.public is NotLoaded
+
+    def test_notfound(self):
+        for _id in ('weboooooooooooooooooooooooooob', 'http://pastebin.com/weboooooooooooooooooooooooooob'):
+            # html method
+            p = self.backend.get_paste(_id)
+            self.assertRaises(PasteNotFound, self.backend.fillobj, p, ['title'])
+
+            # raw method
+            p = self.backend.get_paste(_id)
+            self.assertRaises(PasteNotFound, self.backend.fillobj, p, ['contents'])
+
+    def test_checkurl(self):
+        # call with an URL we can't handle with this backend
+        assert self.backend.get_paste('http://pastealacon.com/1') is None
+
+    def test_can_post(self):
+        assert self.backend.can_post('hello', public=None) > 0
+        assert self.backend.can_post('hello', public=True) > 0
+        assert self.backend.can_post('hello', public=False) > 0
+        assert self.backend.can_post('hello', public=True, max_age=600) > 0
+        assert self.backend.can_post('hello', public=True, max_age=3600*24) > 0
+        assert self.backend.can_post('hello', public=True, max_age=3600*24*3) > 0
+        assert self.backend.can_post('hello', public=True, max_age=False) > 0
+        assert self.backend.can_post('hello', public=None, max_age=False) > 0
+        assert self.backend.can_post('hello', public=True, max_age=3600*24*40) > 0
+        assert self.backend.can_post(u'héhé', public=True) > 0
+        assert self.backend.can_post(u'hello ♥', public=True) > 0