support repositories to manage backends (closes #747)

modules/kickass/__init__.py

@@ -0,0 +1,3 @@
+from .backend import KickassBackend
+
+__all__ = ['KickassBackend']

modules/kickass/backend.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2010-2011 Julien Veyssier
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+from weboob.capabilities.torrent import ICapTorrent
+from weboob.tools.backend import BaseBackend
+
+from .browser import KickassBrowser
+
+
+__all__ = ['KickassBackend']
+
+
+class KickassBackend(BaseBackend, ICapTorrent):
+    NAME = 'kickass'
+    MAINTAINER = 'Julien Veyssier'
+    EMAIL = 'julien.veyssier@aiur.fr'
+    VERSION = '0.a'
+    DESCRIPTION = 'kickasstorrent.com bittorrent tracker'
+    LICENSE = 'AGPLv3+'
+    BROWSER = KickassBrowser
+
+    def create_default_browser(self):
+        return self.create_browser()
+
+    def get_torrent(self, id):
+        return self.browser.get_torrent(id)
+
+    def get_torrent_file(self, id):
+        torrent = self.browser.get_torrent(id)
+        if not torrent:
+            return None
+
+        return self.browser.openurl(torrent.url.encode('utf-8')).read()
+
+    def iter_torrents(self, pattern):
+        return self.browser.iter_torrents(pattern.replace(' ','+'))

modules/kickass/browser.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2010-2011 Julien Veyssier
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+
+from weboob.tools.browser import BaseBrowser
+
+from .pages.torrents import TorrentsPage, TorrentPage
+
+
+__all__ = ['KickassBrowser']
+
+
+class KickassBrowser(BaseBrowser):
+    DOMAIN = 'kickasstorrents.com'
+    PROTOCOL = 'http'
+    ENCODING = 'utf-8'
+    USER_AGENT = BaseBrowser.USER_AGENTS['wget']
+    PAGES = {
+        'http://fr.(kickasstorrents.com|kat.ph)/new/.*field=seeders&sorder=desc': TorrentsPage,
+        'http://fr.(kickasstorrents.com|kat.ph)/.*.html': TorrentPage,
+        }
+
+    def home(self):
+        return self.location('http://kickasstorrents.com')
+
+    def iter_torrents(self, pattern):
+        self.location('http://fr.kickasstorrents.com/new/?q=%s&field=seeders&sorder=desc' % pattern.encode('utf-8'))
+        assert self.is_on_page(TorrentsPage)
+        return self.page.iter_torrents()
+
+    def get_torrent(self, id):
+        self.location('http://fr.kickasstorrents.com/%s.html' % id)
+        assert self.is_on_page(TorrentPage)
+        return self.page.get_torrent(id)

modules/kickass/pages/torrents.py

@@ -0,0 +1,123 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2010-2011 Julien Veyssier, Laurent Bachelier
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+
+try:
+    from urlparse import parse_qs
+except ImportError:
+    from cgi import parse_qs
+from urlparse import urlsplit
+
+from weboob.capabilities.torrent import Torrent
+from weboob.tools.browser import BasePage
+from weboob.tools.misc import get_bytes_size
+
+
+__all__ = ['TorrentsPage']
+
+
+class TorrentsPage(BasePage):
+    def iter_torrents(self):
+        for tr in self.document.getiterator('tr'):
+            if tr.attrib.get('class', '') == 'odd' or tr.attrib.get('class', '') == ' even':
+                if not 'id' in tr.attrib:
+                    continue
+                title = tr.getchildren()[0].getchildren()[1].getchildren()[1].text
+                if not title:
+                    title = ''
+                for red in tr.getchildren()[0].getchildren()[1].getchildren()[1].getchildren():
+                    title += red.text_content()
+                idt = tr.getchildren()[0].getchildren()[1].getchildren()[1].attrib.get('href', '').replace('/', '') \
+                    .replace('.html', '')
+
+                # look for url
+                for a in tr.getchildren()[0].getiterator('a'):
+                    if '.torrent' in a.attrib.get('href', ''):
+                        url = a.attrib['href']
+
+                size = tr.getchildren()[1].text
+                u = tr.getchildren()[1].getchildren()[0].text
+                size = size = size.replace(',', '.')
+                size = float(size)
+                seed = tr.getchildren()[4].text
+                leech = tr.getchildren()[5].text
+
+                yield Torrent(idt,
+                              title,
+                              url=url,
+                              filename=parse_qs(urlsplit(url).query).get('title', [None])[0],
+                              size=get_bytes_size(size, u),
+                              seeders=int(seed),
+                              leechers=int(leech))
+
+
+class TorrentPage(BasePage):
+    def get_torrent(self, id):
+        seed = 0
+        leech = 0
+        description = 'No description'
+        url = 'No Url found'
+        for div in self.document.getiterator('div'):
+            if div.attrib.get('id', '') == 'desc':
+                try:
+                    description = div.text_content()
+                except UnicodeDecodeError:
+                    description = 'Description with invalid UTF-8.'
+            elif div.attrib.get('class', '') == 'seedBlock':
+                if div.getchildren()[1].text is not None:
+                    seed = int(div.getchildren()[1].text)
+                else:
+                    seed = 0
+            elif div.attrib.get('class', '') == 'leechBlock':
+                if div.getchildren()[1].text is not None:
+                    leech = int(div.getchildren()[1].text)
+                else:
+                    leech = 0
+
+        for h in self.document.getiterator('h1'):
+            if h.attrib.get('class', '') == 'torrentName':
+                title = h.getchildren()[0].getchildren()[0].text
+
+        for a in self.document.getiterator('a'):
+            if ('Download' in a.attrib.get('title', '')) and ('torrent file' in a.attrib.get('title', '')):
+                url = a.attrib.get('href', '')
+
+        size = 0
+        for span in self.document.getiterator('span'):
+            # sometimes there are others span, this is not so sure but the size of the children list
+            # is enough to know if this is the right span
+            if (span.attrib.get('class', '') == 'folder' or span.attrib.get('class', '') == 'folderopen') and len(span.getchildren())>2:
+                size = span.getchildren()[1].tail
+                u = span.getchildren()[2].text
+                size = float(size.split(': ')[1].replace(',', '.'))
+
+        files = []
+        for td in self.document.getiterator('td'):
+            if td.attrib.get('class', '') == 'torFileName':
+                files.append(td.text)
+
+        torrent = Torrent(id, title)
+        torrent.url = url
+        torrent.filename = parse_qs(urlsplit(url).query).get('title', [None])[0]
+        torrent.size = get_bytes_size(size, u)
+        torrent.seeders = int(seed)
+        torrent.leechers = int(leech)
+        torrent.description = description
+        torrent.files = files
+        return torrent

modules/kickass/test.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2010-2011 Julien Veyssier, Laurent Bachelier
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+from weboob.tools.test import BackendTest
+from weboob.capabilities.base import NotLoaded
+
+import urllib
+from random import choice
+
+class KickassTest(BackendTest):
+    BACKEND = 'kickass'
+
+    def test_torrent(self):
+        torrents = list(self.backend.iter_torrents('debian'))
+        for torrent in torrents:
+            path, qs = urllib.splitquery(torrent.url)
+            assert path.endswith('.torrent')
+            if qs:
+                assert torrent.filename
+            assert torrent.id
+            assert torrent.name
+            assert torrent.description is NotLoaded
+            full_torrent = self.backend.get_torrent(torrent.id)
+            # do not assert torrent.name is full_torrent.name
+            # (or even that one contains another), it isn't always true!
+            assert full_torrent.name
+            assert full_torrent.url
+            assert full_torrent.description is not NotLoaded
+
+        # get the file of a random torrent
+        # from the list (getting them all would be too long)
+        if len(torrents):
+            torrent = choice(torrents)
+            self.backend.get_torrent_file(torrent.id)