btmon speeded up, all ICapTorrent backends clarified and fillobj cleaned

modules/btmon/backend.py

@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with weboob. If not, see <http://www.gnu.org/licenses/>.
 
-from weboob.capabilities.torrent import ICapTorrent
+from weboob.capabilities.torrent import ICapTorrent, Torrent
 from weboob.tools.backend import BaseBackend
 
 from .browser import BtmonBrowser
@@ -50,3 +50,16 @@ class BtmonBackend(BaseBackend, ICapTorrent):
 
     def iter_torrents(self, pattern):
         return self.browser.iter_torrents(quote_plus(pattern.encode('utf-8')))
+
+    def fill_torrent(self, torrent, fields):
+        if 'description' in fields:
+            tor = self.get_torrent(torrent.id)
+            torrent.description = tor.description
+            torrent.magnet = tor.magnet
+            torrent.files = tor.files
+            torrent.url = tor.url
+        return torrent
+
+    OBJECTS = {
+        Torrent:fill_torrent
+    }

modules/btmon/pages.py

@@ -21,7 +21,7 @@
 import string
 
 from weboob.capabilities.torrent import Torrent
-from weboob.capabilities.base import NotAvailable
+from weboob.capabilities.base import NotAvailable, NotLoaded
 from weboob.tools.browser import BasePage
 from weboob.tools.misc import get_bytes_size
 
@@ -32,22 +32,50 @@ __all__ = ['TorrentsPage','TorrentPage']
 class TorrentsPage(BasePage):
     def iter_torrents(self):
         for div in self.parser.select(self.document.getroot(),'div.list_tor'):
+            name = NotAvailable
+            size = NotAvailable
+            seeders = NotAvailable
+            leechers = NotAvailable
+            right_div = self.parser.select(div,'div.list_tor_right',1)
+            try:
+                seeders = int(self.parser.select(right_div,'b.green',1).text)
+            except ValueError:
+                seeders = 0
+            try:
+                leechers = int(self.parser.select(right_div,'b.red',1).text)
+            except ValueError:
+                leechers = 0
+            sizep = self.parser.select(right_div,'p')[0]
+            sizespan = self.parser.select(sizep,'span')[0]
+            nsize = float(sizespan.text_content().split(':')[1].split()[0])
+            usize = sizespan.text_content().split()[-1].upper()
+            size = get_bytes_size(nsize,usize)
             a = self.parser.select(div,'a.list_tor_title',1)
             href = a.attrib.get('href','')
-            self.browser.location('http://%s%s'%(self.browser.DOMAIN,href))
+            name = unicode(a.text_content())
-            assert self.browser.is_on_page(TorrentPage)
+            id = unicode(href.strip('/').split('.html')[0])
-            yield self.browser.page.get_torrent()
+            torrent = Torrent(id,name)
+            torrent.url = NotLoaded
+            torrent.filename = id
+            torrent.magnet = NotLoaded
+            torrent.size = size
+            torrent.seeders = seeders
+            torrent.leechers = leechers
+            torrent.description = NotLoaded
+            torrent.files = NotLoaded
+            yield torrent
+            
 
 
 class TorrentPage(BasePage):
     def get_torrent(self):
         seed = 0
         leech = 0
-        description = NotAvailable.__unicode__()
+        description = NotAvailable
         url = NotAvailable
         magnet = NotAvailable
         title = NotAvailable
-        id = self.browser.geturl().split('.html')[0].split('/')[-1]
+        id = unicode(self.browser.geturl().split('.html')[0].split('/')[-1])
 
         div = self.parser.select(self.document.getroot(),'div#middle_content',1)
         title = u'%s'%self.parser.select(self.document.getroot(),'div#middle_content > h1',1).text
@@ -56,8 +84,8 @@ class TorrentPage(BasePage):
             seed = slblock_values[0].text
             leech = slblock_values[1].text
         href_t = self.parser.select(div,'a.down',1).attrib.get('href','')
-        url = 'http://%s%s'%(self.browser.DOMAIN,href_t)
+        url = u'http://%s%s'%(self.browser.DOMAIN,href_t)
-        magnet = self.parser.select(div,'a.magnet',1).attrib.get('href','')
+        magnet = unicode(self.parser.select(div,'a.magnet',1).attrib.get('href',''))
         
         divtabs = self.parser.select(div,'div#tabs',1)
         files_div = self.parser.select(divtabs,'div.body > div.doubleblock > div.leftblock')
@@ -74,7 +102,7 @@ class TorrentPage(BasePage):
         u = size_text.split(',')[1].strip().translate(None,string.digits).strip('.').strip().upper()
         div_desc = self.parser.select(divtabs,'div#descriptionContent')
         if len(div_desc) > 0:
-            description = div_desc[0].text_content()
+            description = unicode(div_desc[0].text_content())
 
         torrent = Torrent(id, title)
         torrent.url = url

modules/btmon/test.py

@@ -29,11 +29,10 @@ class BtmonTest(BackendTest):
     def test_torrent(self):
         torrents = list(self.backend.iter_torrents('spiderman'))
         for torrent in torrents:
-            path, qs = urllib.splitquery(torrent.url)
-            assert path.endswith('.torrent')
             assert torrent.id
             assert torrent.name
             assert torrent.description is NotLoaded
+            assert torrent.files is NotLoaded
 
         # get the file of a random torrent
         # from the list (getting them all would be too long)

modules/isohunt/backend.py

@@ -52,8 +52,13 @@ class IsohuntBackend(BaseBackend, ICapTorrent):
         return self.browser.iter_torrents(pattern.replace(' ','+'))
 
     def fill_torrent(self, torrent, fields):
-        if 'description' in fields or fields == None or 'files' in fields:
+        if 'description' in fields or 'files' in fields:
-            return self.get_torrent(torrent.id)
+            tor = self.get_torrent(torrent.id)
+            torrent.description = tor.description
+            torrent.magnet = tor.magnet
+            torrent.files = tor.files
+            torrent.url = tor.url
+        return torrent
 
     OBJECTS = {
             Torrent:fill_torrent

modules/isohunt/test.py

@@ -19,10 +19,16 @@
 
 from weboob.tools.test import BackendTest
 
+from random import choice
+
 class IsohuntTest(BackendTest):
     BACKEND = 'isohunt'
 
     def test_torrent(self):
         l = list(self.backend.iter_torrents('debian'))
-        if len(l) > 0:
+        if len(l):
-            self.backend.get_torrent_file(l[0].id)
+            torrent = choice(l)
+            full_torrent = self.backend.get_torrent(torrent.id)
+            assert torrent.name
+            assert full_torrent.name
+            assert self.backend.get_torrent_file(torrent.id)

modules/kickass/backend.py

@@ -63,8 +63,13 @@ class KickassBackend(BaseBackend, ICapTorrent):
         return self.browser.iter_torrents(quote_plus(pattern.encode('utf-8')))
 
     def fill_torrent(self, torrent, fields):
-        if 'description' in fields or fields == None:
+        if 'description' in fields or 'files' in fields:
-            return self.get_torrent(torrent.id)
+            tor = self.get_torrent(torrent.id)
+            torrent.description = tor.description
+            torrent.magnet = tor.magnet
+            torrent.files = tor.files
+            torrent.url = tor.url
+        return torrent
 
     OBJECTS = {
         Torrent:fill_torrent

modules/kickass/pages.py

@@ -44,7 +44,9 @@ class TorrentsPage(BasePage):
                     continue
                 title = tr.getchildren()[0].getchildren()[1].getchildren()[1].text
                 if not title:
-                    title = ''
+                    title = u''
+                else:
+                    title = unicode(title)
                 for red in tr.getchildren()[0].getchildren()[1].getchildren()[1].getchildren():
                     title += red.text_content()
                 idt = tr.getchildren()[0].getchildren()[1].getchildren()[1].attrib.get('href', '').replace('/', '') \
@@ -54,11 +56,11 @@ class TorrentsPage(BasePage):
                 for a in self.parser.select(tr,'div.iaconbox a'):
                     href = a.attrib.get('href', '')
                     if href.startswith('magnet'):
-                        magnet = href
+                        magnet = unicode(href)
                     elif href.startswith('http'):
-                        url = href
+                        url = unicode(href)
                     elif href.startswith('//'):
-                        url = 'http:%s'%href
+                        url = u'http:%s'%href
 
                 size = tr.getchildren()[1].text
                 u = tr.getchildren()[1].getchildren()[0].text
@@ -72,7 +74,7 @@ class TorrentsPage(BasePage):
                 torrent.magnet = magnet
                 torrent.description = NotLoaded
                 torrent.files = NotLoaded
-                torrent.filename = parse_qs(urlsplit(url).query).get('title', [None])[0]
+                torrent.filename = unicode(parse_qs(urlsplit(url).query).get('title', [None])[0])
                 torrent.size = get_bytes_size(size, u)
                 torrent.seeders = int(seed)
                 torrent.leechers = int(leech)
@@ -90,7 +92,7 @@ class TorrentPage(BasePage):
         for div in self.document.getiterator('div'):
             if div.attrib.get('id', '') == 'desc':
                 try:
-                    description = div.text_content().strip()
+                    description = unicode(div.text_content().strip())
                 except UnicodeDecodeError:
                     description = 'Description with invalid UTF-8.'
             elif div.attrib.get('class', '') == 'seedBlock':
@@ -106,17 +108,17 @@ class TorrentPage(BasePage):
 
         title = self.parser.select(self.document.getroot(),
                 'h1.torrentName span', 1)
-        title = title.text
+        title = unicode(title.text)
 
         for a in self.parser.select(self.document.getroot(),
                 'div.downloadButtonGroup a'):
             href = a.attrib.get('href', '')
             if href.startswith('magnet'):
-                magnet = href
+                magnet = unicode(href)
             elif href.startswith('//'):
-                url = 'http:%s'%href
+                url = u'http:%s'%href
             elif href.startswith('http'):
-                url = href
+                url = unicode(href)
 
         size = 0
         u = ''

modules/piratebay/backend.py

@@ -54,8 +54,13 @@ class PiratebayBackend(BaseBackend, ICapTorrent):
         return self.browser.iter_torrents(pattern.replace(' ', '+'))
 
     def fill_torrent(self, torrent, fields):
-        if 'description' in fields or fields == None:
+        if 'description' in fields or 'files' in fields:
-            return self.get_torrent(torrent.id)
+            tor = self.get_torrent(torrent.id)
+            torrent.description = tor.description
+            torrent.magnet = tor.magnet
+            torrent.files = tor.files
+            torrent.url = tor.url
+        return torrent
 
     OBJECTS = {
         Torrent:fill_torrent

modules/piratebay/pages/torrents.py

@@ -20,7 +20,7 @@
 
 from weboob.tools.browser import BasePage,BrokenPageError
 from weboob.capabilities.torrent import Torrent
-from weboob.capabilities.base import NotAvailable
+from weboob.capabilities.base import NotAvailable, NotLoaded
 
 
 __all__ = ['TorrentsPage']
@@ -68,15 +68,16 @@ class TorrentsPage(BasePage):
                 torrent.size = self.unit(float(size), u)
                 torrent.seeders = int(seed)
                 torrent.leechers = int(leech)
-                torrent.description = NotAvailable
+                torrent.description = NotLoaded
-                torrent.files = NotAvailable
+                torrent.files = NotLoaded
+                torrent.magnet = NotLoaded
                 yield torrent
 
 
 class TorrentPage(BasePage):
     def get_torrent(self, id):
-        url = None
+        url = NotAvailable
-        magnet = None
+        magnet = NotAvailable
         for div in self.document.getiterator('div'):
             if div.attrib.get('id', '') == 'title':
                 title = unicode(div.text.strip())
@@ -107,7 +108,7 @@ class TorrentPage(BasePage):
                         leech = ch.text
                     prev_child_txt = ch.text
             elif div.attrib.get('class', '') == 'nfo':
-                description = unicode(div.getchildren()[0].text.strip())
+                description = unicode(div.getchildren()[0].text_content().strip())
         torrent = Torrent(id, title)
         torrent.url = url or NotAvailable
         torrent.magnet = magnet
@@ -115,6 +116,6 @@ class TorrentPage(BasePage):
         torrent.seeders = int(seed)
         torrent.leechers = int(leech)
         torrent.description = description
-        torrent.files = ['NYI']
+        torrent.files = NotAvailable
 
         return torrent

weboob/applications/qcineoob/subtitle.py

@@ -47,10 +47,8 @@ class Subtitle(QFrame):
         else:
             self.ui.langLabel.parent().hide()
         if not empty(subtitle.description):
-            print 'plop'
             self.ui.descriptionPlain.setPlainText(u'%s'%subtitle.description)
         else:
-            print 'ploppppp'
             self.ui.descriptionPlain.parent().hide()
         if not empty(subtitle.url):
             self.ui.urlEdit.setText(u'%s'%subtitle.url)