From 690a100e2ba7fc5fa189e14c2b972f7f41302a8e Mon Sep 17 00:00:00 2001 From: Julien Veyssier Date: Thu, 28 Feb 2013 03:21:30 +0100 Subject: [PATCH] [torrent] new backend Btmon --- modules/btmon/__init__.py | 3 ++ modules/btmon/backend.py | 52 +++++++++++++++++++++ modules/btmon/browser.py | 47 +++++++++++++++++++ modules/btmon/favicon.png | Bin 0 -> 1775 bytes modules/btmon/pages.py | 94 ++++++++++++++++++++++++++++++++++++++ modules/btmon/test.py | 42 +++++++++++++++++ 6 files changed, 238 insertions(+) create mode 100644 modules/btmon/__init__.py create mode 100644 modules/btmon/backend.py create mode 100644 modules/btmon/browser.py create mode 100644 modules/btmon/favicon.png create mode 100644 modules/btmon/pages.py create mode 100644 modules/btmon/test.py diff --git a/modules/btmon/__init__.py b/modules/btmon/__init__.py new file mode 100644 index 00000000..b88fb31a --- /dev/null +++ b/modules/btmon/__init__.py @@ -0,0 +1,3 @@ +from .backend import BtmonBackend + +__all__ = ['BtmonBackend'] diff --git a/modules/btmon/backend.py b/modules/btmon/backend.py new file mode 100644 index 00000000..4641fce2 --- /dev/null +++ b/modules/btmon/backend.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2010-2011 Julien Veyssier +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from weboob.capabilities.torrent import ICapTorrent +from weboob.tools.backend import BaseBackend + +from .browser import BtmonBrowser + +from urllib import quote_plus + +__all__ = ['KickassBackend'] + + +class BtmonBackend(BaseBackend, ICapTorrent): + NAME = 'btmon' + MAINTAINER = u'Julien Veyssier' + EMAIL = 'julien.veyssier@aiur.fr' + VERSION = '0.f' + DESCRIPTION = 'Btmon BitTorrent tracker' + LICENSE = 'AGPLv3+' + BROWSER = BtmonBrowser + + def create_default_browser(self): + return self.create_browser() + + def get_torrent(self, id): + return self.browser.get_torrent(id) + + def get_torrent_file(self, id): + torrent = self.browser.get_torrent(id) + if not torrent: + return None + return self.browser.openurl(torrent.url.encode('utf-8')).read() + + def iter_torrents(self, pattern): + return self.browser.iter_torrents(quote_plus(pattern.encode('utf-8'))) diff --git a/modules/btmon/browser.py b/modules/btmon/browser.py new file mode 100644 index 00000000..52c78c01 --- /dev/null +++ b/modules/btmon/browser.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2010-2011 Julien Veyssier +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.tools.browser import BaseBrowser + +from .pages import TorrentsPage, TorrentPage + + +__all__ = ['BtmonBrowser'] + + +class BtmonBrowser(BaseBrowser): + DOMAIN = 'www.btmon.com' + PROTOCOL = 'http' + ENCODING = 'utf-8' + USER_AGENT = BaseBrowser.USER_AGENTS['wget'] + PAGES = { + 'http://www.btmon.com/torrent/\?f=.*': TorrentsPage, + 'http://www.btmon.com/.*torrent.html': TorrentPage, + } + + def iter_torrents(self, pattern): + self.location('http://www.btmon.com/torrent/?f=%s' % pattern.encode('utf-8')) + assert self.is_on_page(TorrentsPage) + return self.page.iter_torrents() + + def get_torrent(self, id): + self.location('http://www.btmon.com/%s.html' % id) + assert self.is_on_page(TorrentPage) + return self.page.get_torrent() diff --git a/modules/btmon/favicon.png b/modules/btmon/favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..f7976cbe3204b31c30c94481a121d1fa4c855abb GIT binary patch literal 1775 zcmVPx#24YJ`L;(K){{a7>y{D4^000SaNLh0L01FcU01FcV0GgZ_00007bV*G`2i*c3 z0uchdD2PM=00wVKL_t(|+U=TcY!pQp$A7dGmPQIFR928y(8`;<9(oE3Y!wtxqA?hw zi3S1vfW{B%H$zA?YK#(Jq6r!`M2%773*skj80bhj%7LH=qC^3KZ7e8IfdgsvLuZre zxOcZ(?r4MiPcApJ`|Ql@f1c-=XP#Mz5hF&77%^hR_@AOjF&fPzlCyyMKn*Yqs01oh ze}4gcfla`>Ryy@f!1H$jA9kYaU#)a%c$5cv7KzyBfJs^I^#jJM--DS%@*QBYl}@#3 zGBK}R${3rbUqb8n{z)kYUxopac}!2~Le5 zI}1o>63M|A1Ob^ua+I=j9k!KD^-ys^r3#OCYWHh^XL7XJ4;ZIiY5}%&R}(PRf47xR zHL1X+4QK||WD?1Jz;;b;P9BZYC#k_or&_zK377<|%ObZYU~h2Vm%m~V;DjW=+$8W>UowJ{qlb>U4J{z_kBr;8>n#nt}PgqwEUxFQgBM5&*RVtAL4CI+f~5 z2UtiAcvGot4KNfK09a}-&;lF+z6Uk{pIYhE?|FPmPt~4IS_3g+#E20iMnMtjWXScL zv1&Q<@w%|((g3+i;3sXPb6D7NP=MS-;C_`G7y|UBJw;XrydZ43C(Afy7@_u4!j{Vd zmD|bETEno}qC@@;dTfhk5Az-`fIk#qMe}++C z5Hgrrf3A_B+0p0#))Ta!_7FsX>p25dRQEt04J-#fa6P9w#EXq|J!e#BqREno>R)4A z9_V_`#hOeH;MIT^+okT80v`jDwR=A(6DSA97#W|7fX9q+l+k|rWa9Q>Yrq{^`Gi*1 zNx4r8VUGrAz{9|eMsj4Duw|XFt@TkWYz1fZoBE zTDeMf*q8QGVSvw+L-X;S>p2y`a_w3hut3=IsL@Z_s1M6CtNs@GI@fasMcqu5gjD~* zMsnhFwb9WBxC|Jq2J{8(2cZB>!j>Me1n94|@xHKSP=fQ6Q7_Ho0N>EwWlj|JkD>wd z47*+j98u*VuIJP!Zw~M>pr7t;RsV~OHeboomzg6xrcFdC9aHaZO6&Z|s4a>H0Qz#Y zOW3kWCqSjS%7DYPr@8fkY*h|a_tHNLTc)%0Wv15jgVPDLe`(@lyRhXUV585GBB~#7 zjh=i_{v~=o!1bJ7bo^jJsQ=^Y|7M`CDlZ4l((!YGETb-j_D3T6Lc37Dsq z|ItFwB5N#&C?W_%FUmTMd$hCSa~s-XUyxIDh~l zY`G@j1vaVwp+=i~vzXfy;7Xk+Vapw=JO;Q#Hw|ISBLKpdr!`G+j6mzCv408g16zxQA-;}m^&eWfd1J$yMafH`rA2{KXX<$DD^k@CsNq5t+N_1*Qkp;@-vM# z{j9Hfo@WJ+Jr20nAZcZUWo>hoG3CrAMYD}a+)eWKb0o>Hrd{$b^}YLCzw zHLMNLf(E0VNCeD@X#cg@)_P&fUx0f8;(QbD0n>#o|Hu>n*M}q^pULK21#|*h$$vIi zn=9B6wp{Ic&Um%I_fueJR0?b@uu9mnF=S>ZECEG1PKyyEMvNFSV#K&0@GmOI*6K9; RMP2{^002ovPDHLkV1m^SHgNy| literal 0 HcmV?d00001 diff --git a/modules/btmon/pages.py b/modules/btmon/pages.py new file mode 100644 index 00000000..9c83b5d2 --- /dev/null +++ b/modules/btmon/pages.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2010-2012 Julien Veyssier +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +try: + from urlparse import parse_qs +except ImportError: + from cgi import parse_qs # NOQA + +from urlparse import urlsplit +import string + +from weboob.capabilities.torrent import Torrent +from weboob.capabilities.base import NotAvailable +from weboob.tools.browser import BasePage +from weboob.tools.misc import get_bytes_size + + +__all__ = ['TorrentsPage','TorrentPage'] + + +class TorrentsPage(BasePage): + def iter_torrents(self): + for div in self.parser.select(self.document.getroot(),'div.list_tor'): + a = self.parser.select(div,'a.list_tor_title',1) + href = a.attrib.get('href','') + self.browser.location('http://%s%s'%(self.browser.DOMAIN,href)) + assert self.browser.is_on_page(TorrentPage) + yield self.browser.page.get_torrent() + + +class TorrentPage(BasePage): + def get_torrent(self): + seed = 0 + leech = 0 + description = NotAvailable.__unicode__() + url = NotAvailable + magnet = NotAvailable + title = NotAvailable + id = self.browser.geturl().split('.html')[0].split('/')[-1] + + div = self.parser.select(self.document.getroot(),'div#middle_content',1) + title = u'%s'%self.parser.select(self.document.getroot(),'div#middle_content > h1',1).text + slblock_values = self.parser.select(div,'div.sl_block b') + if len(slblock_values) >= 2: + seed = slblock_values[0].text + leech = slblock_values[1].text + href_t = self.parser.select(div,'a.down',1).attrib.get('href','') + url = 'http://%s%s'%(self.browser.DOMAIN,href_t) + magnet = self.parser.select(div,'a.magnet',1).attrib.get('href','') + + divtabs = self.parser.select(div,'div#tabs',1) + files_div = self.parser.select(divtabs,'div.body > div.doubleblock > div.leftblock') + files = [] + if len(files_div) > 0: + size_text = self.parser.select(files_div,'h5',1).text + for b in self.parser.select(files_div,'b'): + div = b.getparent() + files.append(div.text_content()) + else: + size_text = self.parser.select(divtabs,'h5',1).text_content() + size_text = size_text.split('(')[1].split(')')[0].strip() + size = float(size_text.split(',')[1].strip(string.letters)) + u = size_text.split(',')[1].strip().translate(None,string.digits).strip('.').strip().upper() + div_desc = self.parser.select(divtabs,'div#descriptionContent') + if len(div_desc) > 0: + description = div_desc[0].text_content() + + torrent = Torrent(id, title) + torrent.url = url + torrent.filename = id + torrent.magnet = magnet + torrent.size = get_bytes_size(size, u) + torrent.seeders = int(seed) + torrent.leechers = int(leech) + torrent.description = description + torrent.files = files + return torrent diff --git a/modules/btmon/test.py b/modules/btmon/test.py new file mode 100644 index 00000000..3d463149 --- /dev/null +++ b/modules/btmon/test.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2010-2011 Julien Veyssier, Laurent Bachelier +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from weboob.tools.test import BackendTest +from weboob.capabilities.base import NotLoaded + +import urllib +from random import choice + +class BtmonTest(BackendTest): + BACKEND = 'btmon' + + def test_torrent(self): + torrents = list(self.backend.iter_torrents('spiderman')) + for torrent in torrents: + path, qs = urllib.splitquery(torrent.url) + assert path.endswith('.torrent') + assert torrent.id + assert torrent.name + assert torrent.description is NotLoaded + + # get the file of a random torrent + # from the list (getting them all would be too long) + if len(torrents): + torrent = choice(torrents) + self.backend.get_torrent_file(torrent.id)