simplify and factorize code, remove dead code, follow conventions, use new-style classes
This commit is contained in:
Christophe Benz
Romain Bignon
parent
55a1574df5
commit
f1b3264a67
28 changed files with 202 additions and 251 deletions
|
|
@ -36,7 +36,8 @@ class ArteBackend(BaseBackend, ICapVideo):
|
|||
VERSION = '0.5'
|
||||
DESCRIPTION = 'Arte french TV'
|
||||
LICENSE = 'GPLv3'
|
||||
CONFIG = ValuesDict(Value('lang', label='Lang of videos', choices={'fr': 'French', 'de': 'Deutsch', 'en': 'English'}, default='fr'),
|
||||
CONFIG = ValuesDict(Value('lang', label='Lang of videos',
|
||||
choices={'fr': 'French', 'de': 'Deutsch', 'en': 'English'}, default='fr'),
|
||||
Value('quality', label='Quality of videos', choices=['hd', 'sd'], default='hd'))
|
||||
BROWSER = ArteBrowser
|
||||
|
||||
|
|
|
|||
|
|
@ -25,7 +25,8 @@ class TileError(Exception):
|
|||
Exception.__init__(self, msg)
|
||||
self.tile = tile
|
||||
|
||||
class Captcha:
|
||||
|
||||
class Captcha(object):
|
||||
def __init__(self, file):
|
||||
self.inim = Image.open(file)
|
||||
self.nx,self.ny = self.inim.size
|
||||
|
|
@ -77,7 +78,8 @@ class Captcha:
|
|||
y += 25
|
||||
ty += 1
|
||||
|
||||
class Tile:
|
||||
|
||||
class Tile(object):
|
||||
hash = {'b2d25ae11efaaaec6dd6a4c00f0dfc29': 1,
|
||||
'600873fa288e75ca6cca092ae95bf129': 2,
|
||||
'da24ac28930feee169adcbc9bad4acaf': 3,
|
||||
|
|
@ -117,4 +119,3 @@ class Tile:
|
|||
for pxl in pxls:
|
||||
sys.stdout.write('%02d' % pxl)
|
||||
sys.stdout.write('\n')
|
||||
|
||||
|
|
|
|||
|
|
@ -24,7 +24,6 @@ __all__ = ['AccountComing']
|
|||
|
||||
|
||||
class AccountComing(BasePage):
|
||||
|
||||
def on_loaded(self):
|
||||
self.operations = []
|
||||
|
||||
|
|
|
|||
|
|
@ -24,7 +24,6 @@ __all__ = ['AccountHistory']
|
|||
|
||||
|
||||
class AccountHistory(BasePage):
|
||||
|
||||
def on_loaded(self):
|
||||
self.operations = []
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,10 @@ from weboob.tools.browser import BasePage
|
|||
|
||||
from ..errors import PasswordExpired
|
||||
|
||||
|
||||
__all__ = ['AccountsList']
|
||||
|
||||
|
||||
class AccountsList(BasePage):
|
||||
LINKID_REGEXP = re.compile(".*ch4=(\w+).*")
|
||||
|
||||
|
|
|
|||
|
|
@ -55,9 +55,11 @@ class LoginPage(BasePage):
|
|||
self.browser['ch5'] = img.get_codes(password)
|
||||
self.browser.submit()
|
||||
|
||||
|
||||
class ConfirmPage(BasePage):
|
||||
pass
|
||||
|
||||
|
||||
class ChangePasswordPage(BasePage):
|
||||
def change_password(self, current, new):
|
||||
img = Captcha(self.browser.openurl('/NSImgGrille'))
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
|
||||
|
||||
import re
|
||||
|
||||
from weboob.tools.browser import BasePage
|
||||
|
|
@ -56,6 +57,7 @@ class TransferPage(BasePage):
|
|||
self.browser['T5'] = reason
|
||||
self.browser.submit()
|
||||
|
||||
|
||||
class TransferConfirmPage(BasePage):
|
||||
def on_loaded(self):
|
||||
for td in self.document.getroot().cssselect('td#size2'):
|
||||
|
|
@ -67,6 +69,7 @@ class TransferConfirmPage(BasePage):
|
|||
self.browser.location('/NS_VIRDA?stp=%s' % m.group(1))
|
||||
return
|
||||
|
||||
|
||||
class TransferCompletePage(BasePage):
|
||||
def get_id(self):
|
||||
return self.group_dict['id']
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@
|
|||
from __future__ import with_statement
|
||||
|
||||
from weboob.capabilities.messages import CantSendMessage, ICapMessages, ICapMessagesPost
|
||||
#from weboob.capabilities.account import ICapAccount, StatusField
|
||||
from weboob.tools.backend import BaseBackend
|
||||
from weboob.tools.value import ValuesDict, Value
|
||||
|
||||
|
|
|
|||
|
|
@ -23,6 +23,9 @@ from weboob.tools.value import ValuesDict, Value
|
|||
from .browser import BPBrowser
|
||||
|
||||
|
||||
__all__ = ['BPBackend']
|
||||
|
||||
|
||||
class BPBackend(BaseBackend, ICapBank):
|
||||
NAME = 'bp'
|
||||
MAINTAINER = 'Nicolas Duhamel'
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from weboob.tools.value import ValuesDict, Value
|
|||
from .browser import CanalplusBrowser
|
||||
from .pages import CanalplusVideo
|
||||
|
||||
|
||||
__all__ = ['CanalplusBackend']
|
||||
|
||||
|
||||
|
|
@ -57,7 +58,6 @@ class CanalplusBackend(BaseBackend, ICapVideo):
|
|||
if 'thumbnail' in fields:
|
||||
with self.browser:
|
||||
video.thumbnail.data = self.browser.readurl(video.thumbnail.url)
|
||||
|
||||
return video
|
||||
|
||||
OBJECTS = {CanalplusVideo: fill_video}
|
||||
|
|
|
|||
|
|
@ -15,16 +15,21 @@
|
|||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
|
||||
|
||||
import urllib
|
||||
|
||||
import lxml.etree
|
||||
|
||||
from weboob.tools.browser import BaseBrowser
|
||||
from weboob.tools.browser.decorators import id2url
|
||||
|
||||
from .pages import InitPage, CanalplusVideo, VideoPage
|
||||
|
||||
import lxml.etree
|
||||
|
||||
class XMLParser:
|
||||
__all__ = ['CanalplusBrowser']
|
||||
|
||||
|
||||
class XMLParser(object):
|
||||
def parse(self, data, encoding=None):
|
||||
if encoding is None:
|
||||
parser = None
|
||||
|
|
@ -33,22 +38,21 @@ class XMLParser:
|
|||
return lxml.etree.XML(data.get_data(), parser)
|
||||
|
||||
|
||||
__all__ = ['CanalplusBrowser']
|
||||
|
||||
|
||||
class CanalplusBrowser(BaseBrowser):
|
||||
DOMAIN = u'service.canal-plus.com'
|
||||
ENCODING = 'utf-8'
|
||||
PAGES = {r"http://service.canal-plus.com/video/rest/initPlayer/cplus/": InitPage,
|
||||
r"http://service.canal-plus.com/video/rest/search/cplus/.*": VideoPage,
|
||||
r"http://service.canal-plus.com/video/rest/getVideosLiees/cplus/(?P<id>.+)": VideoPage,
|
||||
}
|
||||
PAGES = {
|
||||
r'http://service.canal-plus.com/video/rest/initPlayer/cplus/': InitPage,
|
||||
r'http://service.canal-plus.com/video/rest/search/cplus/.*': VideoPage,
|
||||
r'http://service.canal-plus.com/video/rest/getVideosLiees/cplus/(?P<id>.+)': VideoPage,
|
||||
}
|
||||
|
||||
#We need lxml.etree.XMLParser for read CDATA
|
||||
PARSER = XMLParser()
|
||||
FORMATS = { 'sd': 'BAS_DEBIT',
|
||||
'hd': 'HD'
|
||||
}
|
||||
FORMATS = {
|
||||
'sd': 'BAS_DEBIT',
|
||||
'hd': 'HD',
|
||||
}
|
||||
|
||||
def __init__(self, quality, *args, **kwargs):
|
||||
BaseBrowser.__init__(self, parser= self.PARSER, *args, **kwargs)
|
||||
|
|
@ -58,15 +62,13 @@ class CanalplusBrowser(BaseBrowser):
|
|||
self.quality = 'HD'
|
||||
|
||||
def home(self):
|
||||
self.location("http://service.canal-plus.com/video/rest/initPlayer/cplus/")
|
||||
self.location('http://service.canal-plus.com/video/rest/initPlayer/cplus/')
|
||||
|
||||
def iter_search_results(self, pattern):
|
||||
self.location("http://service.canal-plus.com/video/rest/search/cplus/" + urllib.quote_plus(pattern))
|
||||
self.location('http://service.canal-plus.com/video/rest/search/cplus/' + urllib.quote_plus(pattern))
|
||||
return self.page.iter_results()
|
||||
|
||||
@id2url(CanalplusVideo.id2url)
|
||||
def get_video(self, url, video=None):
|
||||
self.location(url)
|
||||
return self.page.get_video(video, self.quality)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -15,26 +15,28 @@
|
|||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
|
||||
|
||||
from weboob.tools.browser import BasePage
|
||||
|
||||
|
||||
__all__ = ['InitPage']
|
||||
|
||||
|
||||
class InitPage(BasePage):
|
||||
def on_loaded(self):
|
||||
channels = []
|
||||
### Parse liste des channels
|
||||
for elem in self.document[2].getchildren():
|
||||
channel = {}
|
||||
for e in elem.getchildren():
|
||||
subchannels = []
|
||||
if e.tag == "NOM":
|
||||
channel['nom'] = e.text
|
||||
elif e.tag == "SELECTIONS":
|
||||
|
||||
for select in e:
|
||||
subchannel = {}
|
||||
subchannel['id'] = select[0].text
|
||||
subchannel['nom'] = select[1].text
|
||||
subchannels.append(subchannel)
|
||||
channel['subchannels'] = subchannels
|
||||
channels.append(channel)
|
||||
def on_loaded(self):
|
||||
channels = []
|
||||
### Parse liste des channels
|
||||
for elem in self.document[2].getchildren():
|
||||
channel = {}
|
||||
for e in elem.getchildren():
|
||||
subchannels = []
|
||||
if e.tag == "NOM":
|
||||
channel['nom'] = e.text
|
||||
elif e.tag == "SELECTIONS":
|
||||
for select in e:
|
||||
subchannel = {}
|
||||
subchannel['id'] = select[0].text
|
||||
subchannel['nom'] = select[1].text
|
||||
subchannels.append(subchannel)
|
||||
channel['subchannels'] = subchannels
|
||||
channels.append(channel)
|
||||
|
|
|
|||
|
|
@ -17,9 +17,14 @@
|
|||
|
||||
|
||||
from datetime import datetime, date, time
|
||||
|
||||
from weboob.tools.browser import BaseBrowser
|
||||
from weboob.tools.parsers.lxmlparser import SelectElementException
|
||||
from weboob.tools.misc import to_unicode
|
||||
from weboob.tools.parsers.lxmlparser import SelectElementException
|
||||
|
||||
|
||||
__all__ = ['CanalTP']
|
||||
|
||||
|
||||
class CanalTP(BaseBrowser):
|
||||
DOMAIN = 'widget.canaltp.fr'
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ class FourChanBackend(BaseBackend, ICapMessages):
|
|||
EMAIL = 'romain@weboob.org'
|
||||
VERSION = '0.5'
|
||||
LICENSE = 'GPLv3'
|
||||
DESCRIPTION = "4chan website"
|
||||
DESCRIPTION = '4chan website'
|
||||
CONFIG = ValuesDict(Value('boards', label='Boards to fetch'))
|
||||
STORAGE = {'boards': {}}
|
||||
BROWSER = FourChan
|
||||
|
|
|
|||
|
|
@ -15,26 +15,27 @@
|
|||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
|
||||
|
||||
from weboob.tools.browser import BaseBrowser
|
||||
|
||||
from .pages.board import BoardPage
|
||||
|
||||
|
||||
class FourChan(BaseBrowser):
|
||||
DOMAIN = 'boards.4chan.org'
|
||||
PAGES = {'http://boards.4chan.org/\w+/': BoardPage,
|
||||
'http://boards.4chan.org/\w+/res/\d+': BoardPage,
|
||||
}
|
||||
PAGES = {
|
||||
'http://boards.4chan.org/\w+/': BoardPage,
|
||||
'http://boards.4chan.org/\w+/res/\d+': BoardPage,
|
||||
}
|
||||
|
||||
def is_logged(self):
|
||||
return True
|
||||
|
||||
def get_threads(self, board):
|
||||
self.location('http://boards.4chan.org/%s/' % board)
|
||||
|
||||
return self.page.articles
|
||||
|
||||
def get_thread(self, board, id):
|
||||
self.location('http://boards.4chan.org/%s/res/%d' % (board, long(id)))
|
||||
|
||||
assert len(self.page.articles) == 1
|
||||
return self.page.articles[0]
|
||||
|
|
|
|||
|
|
@ -15,12 +15,17 @@
|
|||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
import re
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from weboob.tools.browser import BasePage
|
||||
|
||||
|
||||
__all__ = ['BoardPage']
|
||||
|
||||
|
||||
class Message(object):
|
||||
def __init__(self, browser, board, id, filename=u'', url=u''):
|
||||
self.id = id
|
||||
|
|
@ -42,6 +47,7 @@ class Message(object):
|
|||
def __repr__(self):
|
||||
return '<Message id=%s filename=%s url=%s comments=%d>' % (self.id, self.filename, self.url, len(self.comments))
|
||||
|
||||
|
||||
class BoardPage(BasePage):
|
||||
URL_REGEXP = re.compile('http://boards.4chan.org/(\w+)/')
|
||||
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ class IndexPage(BasePage):
|
|||
def is_logged(self):
|
||||
return 'id' in self.document.find('body').attrib
|
||||
|
||||
|
||||
class LoginPage(BasePage):
|
||||
def login(self, login, password):
|
||||
self.browser.select_form(nr=0)
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
|
||||
|
||||
from weboob.capabilities.torrent import ICapTorrent
|
||||
from weboob.tools.backend import BaseBackend
|
||||
|
||||
|
|
@ -31,10 +32,6 @@ class IsohuntBackend(BaseBackend, ICapTorrent):
|
|||
VERSION = '0.5'
|
||||
DESCRIPTION = 'isohunt.com bittorrent tracker'
|
||||
LICENSE = 'GPLv3'
|
||||
#CONFIG = ValuesDict(Value('domain', label='Domain (example "ssl.what.cd")'),
|
||||
# Value('protocol', label='Protocol to use', choices=('http', 'https')),
|
||||
# Value('username', label='Username'),
|
||||
# Value('password', label='Password', masked=True))
|
||||
BROWSER = IsohuntBrowser
|
||||
|
||||
def create_default_browser(self):
|
||||
|
|
@ -47,7 +44,6 @@ class IsohuntBackend(BaseBackend, ICapTorrent):
|
|||
torrent = self.browser.get_torrent(id)
|
||||
if not torrent:
|
||||
return None
|
||||
|
||||
return self.browser.openurl(torrent.url.encode('utf-8')).read()
|
||||
|
||||
def iter_torrents(self, pattern):
|
||||
|
|
|
|||
|
|
@ -30,24 +30,19 @@ class IsohuntBrowser(BaseBrowser):
|
|||
ENCODING = 'utf-8'
|
||||
USER_AGENT = BaseBrowser.USER_AGENTS['wget']
|
||||
PAGES = {
|
||||
'https://isohunt.com/torrents/.*iht=-1&ihp=1&ihs1=2&iho1=d' : TorrentsPage,
|
||||
'https://isohunt.com/torrent_details.*tab=summary' : TorrentPage
|
||||
}
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
BaseBrowser.__init__(self, *args, **kwargs)
|
||||
'https://isohunt.com/torrents/.*iht=-1&ihp=1&ihs1=2&iho1=d' : TorrentsPage,
|
||||
'https://isohunt.com/torrent_details.*tab=summary' : TorrentPage,
|
||||
}
|
||||
|
||||
def home(self):
|
||||
return self.location('https://isohunt.com')
|
||||
|
||||
def iter_torrents(self, pattern):
|
||||
self.location('https://isohunt.com/torrents/%s?iht=-1&ihp=1&ihs1=2&iho1=d' % pattern)
|
||||
|
||||
assert self.is_on_page(TorrentsPage)
|
||||
return self.page.iter_torrents()
|
||||
|
||||
def get_torrent(self, id):
|
||||
self.location('https://isohunt.com/torrent_details/%s/?tab=summary' % id)
|
||||
|
||||
assert self.is_on_page(TorrentPage)
|
||||
return self.page.get_torrent(id)
|
||||
|
|
|
|||
|
|
@ -16,29 +16,18 @@
|
|||
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
|
||||
|
||||
|
||||
from weboob.tools.browser import BasePage
|
||||
from weboob.capabilities.torrent import Torrent
|
||||
from weboob.tools.browser import BasePage
|
||||
from weboob.tools.misc import get_bytes_size
|
||||
|
||||
|
||||
__all__ = ['TorrentsPage']
|
||||
|
||||
|
||||
class TorrentsPage(BasePage):
|
||||
def unit(self, n, u):
|
||||
m = {'KB': 1024,
|
||||
'MB': 1024*1024,
|
||||
'GB': 1024*1024*1024,
|
||||
'TB': 1024*1024*1024*1024,
|
||||
}
|
||||
return float(n*m[u])
|
||||
|
||||
def iter_torrents(self):
|
||||
|
||||
|
||||
|
||||
for tr in self.document.getiterator('tr'):
|
||||
if tr.attrib.get('class','') == 'hlRow':
|
||||
if tr.attrib.get('class', '') == 'hlRow':
|
||||
# TODO à corriger
|
||||
atitle = tr.getchildren()[2].getchildren()[1]
|
||||
title = atitle.text
|
||||
|
|
@ -57,41 +46,32 @@ class TorrentsPage(BasePage):
|
|||
seed = tr.getchildren()[4].text
|
||||
leech = tr.getchildren()[5].text
|
||||
url = 'https://isohunt.com/download/%s/mon_joli_torrent.torrent' % idt
|
||||
yield Torrent(idt,
|
||||
title,
|
||||
url=url,
|
||||
size=get_bytes_size(size, u),
|
||||
seeders=int(seed),
|
||||
leechers=int(leech))
|
||||
|
||||
torrent = Torrent(idt,
|
||||
title,
|
||||
url=url,
|
||||
size=self.unit(size,u),
|
||||
seeders=int(seed),
|
||||
leechers=int(leech))
|
||||
yield torrent
|
||||
|
||||
class TorrentPage(BasePage):
|
||||
def unit(self, n, u):
|
||||
m = {'KB': 1024,
|
||||
'MB': 1024*1024,
|
||||
'GB': 1024*1024*1024,
|
||||
'TB': 1024*1024*1024*1024,
|
||||
}
|
||||
return float(n*m[u])
|
||||
|
||||
def get_torrent(self, id):
|
||||
title = ''
|
||||
url = 'https://isohunt.com/download/%s/%s.torrent' % (id , id)
|
||||
url = 'https://isohunt.com/download/%s/%s.torrent' % (id, id)
|
||||
for a in self.document.getiterator('a'):
|
||||
if 'Search more torrents of' in a.attrib.get('title',''):
|
||||
if 'Search more torrents of' in a.attrib.get('title', ''):
|
||||
title = a.tail
|
||||
for span in self.document.getiterator('span'):
|
||||
if span.attrib.get('style','') == 'color:green;' and ('ShowTip' in span.attrib.get('onmouseover','')):
|
||||
if span.attrib.get('style', '') == 'color:green;' and ('ShowTip' in span.attrib.get('onmouseover', '')):
|
||||
seed = span.tail.split(' ')[1]
|
||||
tip_id = span.attrib.get('onmouseover','').split("'")[1]
|
||||
tip_id = span.attrib.get('onmouseover', '').split("'")[1]
|
||||
for div in self.document.getiterator('div'):
|
||||
# find the corresponding super tip which appears on super mouse hover!
|
||||
if div.attrib.get('class','') == 'dirs ydsf' and tip_id in div.attrib.get('id',''):
|
||||
if div.attrib.get('class', '') == 'dirs ydsf' and tip_id in div.attrib.get('id', ''):
|
||||
leech = div.getchildren()[0].getchildren()[1].tail.split(' ')[2]
|
||||
# the <b> with the size in it doesn't have a distinction
|
||||
# have to get it by higher
|
||||
elif div.attrib.get('id','') == 'torrent_details':
|
||||
elif div.attrib.get('id', '') == 'torrent_details':
|
||||
size = div.getchildren()[6].getchildren()[0].getchildren()[0].text
|
||||
u = size[-2:]
|
||||
size = float(size[:-3])
|
||||
|
|
@ -104,7 +84,7 @@ class TorrentPage(BasePage):
|
|||
files = []
|
||||
count_p_found = 0
|
||||
for p in self.document.getiterator('p'):
|
||||
if p.attrib.get('style','') == "line-height:1.2em;margin-top:1.8em":
|
||||
if p.attrib.get('style', '') == 'line-height:1.2em;margin-top:1.8em':
|
||||
count_p_found += 1
|
||||
if count_p_found == 1:
|
||||
description = p.getchildren()[1].tail
|
||||
|
|
@ -115,24 +95,20 @@ class TorrentPage(BasePage):
|
|||
files.append(p.getchildren()[0].tail.strip())
|
||||
|
||||
for td in self.document.getiterator('td'):
|
||||
#print td.attrib.get('class')
|
||||
if td.attrib.get('class','') == 'fileRows':
|
||||
if td.attrib.get('class', '') == 'fileRows':
|
||||
filename = td.text
|
||||
#print "len"+str(len(td.getchildren()))
|
||||
for slash in td.getchildren():
|
||||
filename += '/'
|
||||
filename += slash.tail
|
||||
files.append(filename)
|
||||
|
||||
|
||||
#--------------------------TODO
|
||||
|
||||
torrent = Torrent(id, title)
|
||||
torrent.url = url
|
||||
torrent.size = self.unit(size,u)
|
||||
torrent.size = get_bytes_size(size, u)
|
||||
torrent.seeders = int(seed)
|
||||
torrent.leechers = int(leech)
|
||||
torrent.description = description
|
||||
torrent.files = files
|
||||
|
||||
return torrent
|
||||
|
|
|
|||
|
|
@ -31,10 +31,6 @@ class KickassBackend(BaseBackend, ICapTorrent):
|
|||
VERSION = '0.5'
|
||||
DESCRIPTION = 'kickasstorrent.com bittorrent tracker'
|
||||
LICENSE = 'GPLv3'
|
||||
#CONFIG = ValuesDict(Value('domain', label='Domain (example "ssl.what.cd")'),
|
||||
# Value('protocol', label='Protocol to use', choices=('http', 'https')),
|
||||
# Value('username', label='Username'),
|
||||
# Value('password', label='Password', masked=True))
|
||||
BROWSER = KickassBrowser
|
||||
|
||||
def create_default_browser(self):
|
||||
|
|
|
|||
|
|
@ -30,43 +30,19 @@ class KickassBrowser(BaseBrowser):
|
|||
ENCODING = 'utf-8'
|
||||
USER_AGENT = BaseBrowser.USER_AGENTS['wget']
|
||||
PAGES = {
|
||||
'http://fr.kickasstorrents.com/new/.*field=seeders&sorder=desc' : TorrentsPage,
|
||||
'http://fr.kickasstorrents.com/.*.html' : TorrentPage
|
||||
}
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
#self.DOMAIN = domain
|
||||
#self.PROTOCOL = protocol
|
||||
#self.PAGES = {}
|
||||
#for key, value in PiratebayBrowser.PAGES.iteritems():
|
||||
# self.PAGES[key % domain] = value
|
||||
|
||||
BaseBrowser.__init__(self, *args, **kwargs)
|
||||
|
||||
#def login(self):
|
||||
# if not self.is_on_page(LoginPage):
|
||||
# self.home()
|
||||
# self.page.login(self.username, self.password)
|
||||
|
||||
#def is_logged(self):
|
||||
# if not self.page or self.is_on_page(LoginPage):
|
||||
# return False
|
||||
# if self.is_on_page(IndexPage):
|
||||
# return self.page.is_logged()
|
||||
# return True
|
||||
'http://fr.kickasstorrents.com/new/.*field=seeders&sorder=desc': TorrentsPage,
|
||||
'http://fr.kickasstorrents.com/.*.html': TorrentPage,
|
||||
}
|
||||
|
||||
def home(self):
|
||||
return self.location('http://kickasstorrents.com')
|
||||
|
||||
def iter_torrents(self, pattern):
|
||||
#self.location(self.buildurl('/torrents.php', searchstr=pattern))
|
||||
self.location('http://fr.kickasstorrents.com/new/?q=%s&field=seeders&sorder=desc' % pattern)
|
||||
|
||||
assert self.is_on_page(TorrentsPage)
|
||||
return self.page.iter_torrents()
|
||||
|
||||
def get_torrent(self, id):
|
||||
self.location('http://fr.kickasstorrents.com/%s.html' % id)
|
||||
|
||||
assert self.is_on_page(TorrentPage)
|
||||
return self.page.get_torrent(id)
|
||||
|
|
|
|||
|
|
@ -16,28 +16,18 @@
|
|||
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
|
||||
|
||||
|
||||
from weboob.tools.browser import BasePage
|
||||
from weboob.capabilities.torrent import Torrent
|
||||
from weboob.tools.browser import BasePage
|
||||
from weboob.tools.misc import get_bytes_size
|
||||
|
||||
|
||||
__all__ = ['TorrentsPage']
|
||||
|
||||
|
||||
class TorrentsPage(BasePage):
|
||||
def unit(self, n, u):
|
||||
m = {'bytes': 1,
|
||||
'KB': 1024,
|
||||
'MB': 1024*1024,
|
||||
'GB': 1024*1024*1024,
|
||||
'TB': 1024*1024*1024*1024,
|
||||
}
|
||||
return float(n*m[u])
|
||||
|
||||
def iter_torrents(self):
|
||||
|
||||
for tr in self.document.getiterator('tr'):
|
||||
if tr.attrib.get('class','') == 'odd' or tr.attrib.get('class','') == ' even':
|
||||
if tr.attrib.get('class', '') == 'odd' or tr.attrib.get('class', '') == ' even':
|
||||
title = tr.getchildren()[0].getchildren()[1].getchildren()[1].text
|
||||
if not title:
|
||||
title = ''
|
||||
|
|
@ -46,78 +36,67 @@ class TorrentsPage(BasePage):
|
|||
title += red.text
|
||||
if red.tail:
|
||||
title += red.tail
|
||||
idt = tr.getchildren()[0].getchildren()[1].getchildren()[1].attrib.get('href','').replace('/','').replace('.html','')
|
||||
url = tr.getchildren()[0].getchildren()[0].getchildren()[0].getchildren()[0].attrib.get('href','')
|
||||
idt = tr.getchildren()[0].getchildren()[1].getchildren()[1].attrib.get('href', '').replace('/', '') \
|
||||
.replace('.html', '')
|
||||
url = tr.getchildren()[0].getchildren()[0].getchildren()[0].getchildren()[0].attrib.get('href', '')
|
||||
size = tr.getchildren()[1].text
|
||||
u = tr.getchildren()[1].getchildren()[0].text
|
||||
size = size = size.replace(',','.')
|
||||
size = size = size.replace(',', '.')
|
||||
size = float(size)
|
||||
seed = tr.getchildren()[4].text
|
||||
leech = tr.getchildren()[5].text
|
||||
|
||||
torrent = Torrent(idt,
|
||||
title,
|
||||
url=url,
|
||||
size=self.unit(size,u),
|
||||
seeders=int(seed),
|
||||
leechers=int(leech))
|
||||
yield torrent
|
||||
yield Torrent(idt,
|
||||
title,
|
||||
url=url,
|
||||
size=get_bytes_size(size, u),
|
||||
seeders=int(seed),
|
||||
leechers=int(leech))
|
||||
|
||||
|
||||
class TorrentPage(BasePage):
|
||||
def unit(self, n, u):
|
||||
m = {'bytes': 1,
|
||||
'KB': 1024,
|
||||
'MB': 1024*1024,
|
||||
'GB': 1024*1024*1024,
|
||||
'TB': 1024*1024*1024*1024,
|
||||
}
|
||||
return float(n*m[u])
|
||||
|
||||
def get_torrent(self, id):
|
||||
|
||||
seed = 0
|
||||
leech = 0
|
||||
description = "No description"
|
||||
description = 'No description'
|
||||
url = 'No Url found'
|
||||
for div in self.document.getiterator('div'):
|
||||
if div.attrib.get('id','') == 'desc':
|
||||
if div.attrib.get('id', '') == 'desc':
|
||||
description = div.text.strip()
|
||||
for ch in div.getchildren():
|
||||
if ch.tail != None:
|
||||
description += ' '+ch.tail.strip()
|
||||
if div.attrib.get('class','') == 'seedBlock':
|
||||
if div.attrib.get('class', '') == 'seedBlock':
|
||||
seed = int(div.getchildren()[1].text)
|
||||
if div.attrib.get('class','') == 'leechBlock':
|
||||
if div.attrib.get('class', '') == 'leechBlock':
|
||||
leech = int(div.getchildren()[1].text)
|
||||
|
||||
for h in self.document.getiterator('h1'):
|
||||
if h.attrib.get('class','') == 'torrentName':
|
||||
if h.attrib.get('class', '') == 'torrentName':
|
||||
title = h.getchildren()[0].getchildren()[0].text
|
||||
|
||||
for a in self.document.getiterator('a'):
|
||||
if ('Download' in a.attrib.get('title','')) and ('torrent file' in a.attrib.get('title','')):
|
||||
url = a.attrib.get('href','')
|
||||
if ('Download' in a.attrib.get('title', '')) and ('torrent file' in a.attrib.get('title', '')):
|
||||
url = a.attrib.get('href', '')
|
||||
|
||||
size = 0
|
||||
for span in self.document.getiterator('span'):
|
||||
if span.attrib.get('class','') == "folder" or span.attrib.get('class','') == "folderopen":
|
||||
if span.attrib.get('class', '') == 'folder' or span.attrib.get('class', '') == 'folderopen':
|
||||
size = span.getchildren()[1].tail
|
||||
u = span.getchildren()[2].text
|
||||
size = float(size.split(': ')[1].replace(',','.'))
|
||||
size = float(size.split(': ')[1].replace(',', '.'))
|
||||
|
||||
files = []
|
||||
for td in self.document.getiterator('td'):
|
||||
if td.attrib.get('class','') == 'torFileName':
|
||||
if td.attrib.get('class', '') == 'torFileName':
|
||||
files.append(td.text)
|
||||
|
||||
|
||||
torrent = Torrent(id, title)
|
||||
torrent = Torrent(id, title)
|
||||
torrent.url = url
|
||||
torrent.size = self.unit(size,u)
|
||||
torrent.size = get_bytes_size(size, u)
|
||||
torrent.seeders = int(seed)
|
||||
torrent.leechers = int(leech)
|
||||
torrent.description = description
|
||||
torrent.files = files
|
||||
|
||||
return torrent
|
||||
|
|
|
|||
|
|
@ -30,10 +30,11 @@ class LCLBrowser(BaseBrowser):
|
|||
DOMAIN = 'particuliers.secure.lcl.fr'
|
||||
ENCODING = 'utf-8'
|
||||
USER_AGENT = BaseBrowser.USER_AGENTS['wget']
|
||||
PAGES = {'https://particuliers.secure.lcl.fr/index.html': LoginPage,
|
||||
'https://particuliers.secure.lcl.fr/everest/UWBI/UWBIAccueil\?DEST=IDENTIFICATION': LoginErrorPage,
|
||||
'https://particuliers.secure.lcl.fr/outil/UWSP/Synthese/accesSynthese': AccountsPage
|
||||
}
|
||||
PAGES = {
|
||||
'https://particuliers.secure.lcl.fr/index.html': LoginPage,
|
||||
'https://particuliers.secure.lcl.fr/everest/UWBI/UWBIAccueil\?DEST=IDENTIFICATION': LoginErrorPage,
|
||||
'https://particuliers.secure.lcl.fr/outil/UWSP/Synthese/accesSynthese': AccountsPage,
|
||||
}
|
||||
|
||||
def __init__(self, agency, *args, **kwargs):
|
||||
self.agency = agency
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
|
||||
|
||||
from weboob.capabilities.weather import ICapWeather
|
||||
from weboob.tools.backend import BaseBackend
|
||||
|
||||
|
|
@ -31,10 +32,6 @@ class MeteofranceBackend(BaseBackend, ICapWeather):
|
|||
VERSION = '0.5'
|
||||
DESCRIPTION = 'Get forecasts from the MeteoFrance website'
|
||||
LICENSE = 'GPLv3'
|
||||
#CONFIG = ValuesDict(Value('domain', label='Domain (example "ssl.what.cd")'),
|
||||
# Value('protocol', label='Protocol to use', choices=('http', 'https')),
|
||||
# Value('username', label='Username'),
|
||||
# Value('password', label='Password', masked=True))
|
||||
BROWSER = MeteofranceBrowser
|
||||
|
||||
def create_default_browser(self):
|
||||
|
|
|
|||
|
|
@ -31,14 +31,16 @@ class MeteofranceBrowser(BaseBrowser):
|
|||
PROTOCOL = 'http'
|
||||
ENCODING = 'utf-8'
|
||||
USER_AGENT = BaseBrowser.USER_AGENTS['wget']
|
||||
WEATHER_URL = "{0}://{1}/france/meteo?PREVISIONS_PORTLET.path=previsionsville/{{cityid}}".format(PROTOCOL, DOMAIN)
|
||||
CITY_SEARCH_URL="{0}://{1}/france/accueil/resultat?RECHERCHE_RESULTAT_PORTLET.path=rechercheresultat&query={{city_pattern}}&type=PREV_FRANCE&satellite=france".format(PROTOCOL, DOMAIN)
|
||||
WEATHER_URL = '{0}://{1}/france/meteo?PREVISIONS_PORTLET.path=previsionsville/{{cityid}}'.format(PROTOCOL, DOMAIN)
|
||||
CITY_SEARCH_URL = '{0}://{1}/france/accueil/resultat?RECHERCHE_RESULTAT_PORTLET.path=rechercheresultat&' \
|
||||
'query={{city_pattern}}&type=PREV_FRANCE&satellite=france'.format(PROTOCOL, DOMAIN)
|
||||
PAGES = {
|
||||
WEATHER_URL.format(cityid=".*") : WeatherPage,
|
||||
CITY_SEARCH_URL.format(city_pattern=".*") : CityPage,
|
||||
"http://france.meteofrance.com/france/accueil/resultat.*" : CityPage,
|
||||
"http://france.meteofrance.com/france/meteo.*" : WeatherPage
|
||||
}
|
||||
WEATHER_URL.format(cityid=".*"): WeatherPage,
|
||||
CITY_SEARCH_URL.format(city_pattern=".*"): CityPage,
|
||||
'http://france.meteofrance.com/france/accueil/resultat.*': CityPage,
|
||||
'http://france.meteofrance.com/france/meteo.*': WeatherPage,
|
||||
}
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
BaseBrowser.__init__(self, *args, **kwargs)
|
||||
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ from weboob.capabilities.weather import Forecast, Current, City
|
|||
|
||||
import datetime
|
||||
|
||||
|
||||
__all__ = ['WeatherPage', 'CityPage']
|
||||
|
||||
|
||||
|
|
@ -60,16 +61,15 @@ class WeatherPage(BasePage):
|
|||
mdate = datetime.datetime.now()
|
||||
return Current(mdate, temp, mtxt, "C")
|
||||
|
||||
|
||||
def get_city(self):
|
||||
"""Return the city from the forecastpage
|
||||
"""
|
||||
Return the city from the forecastpage.
|
||||
"""
|
||||
for div in self.document.getiterator('div'):
|
||||
if div.attrib.has_key("class") and div.attrib.get("class") == "choix":
|
||||
for strong in div.getiterator("strong"):
|
||||
city_name=strong.text +" "+ strong.tail.replace("(","").replace(")","")
|
||||
city_id=self.url.split("/")[-1]
|
||||
|
||||
return City(city_id, city_name)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -16,14 +16,68 @@
|
|||
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
|
||||
|
||||
from logging import warning
|
||||
from dateutil import tz
|
||||
from logging import warning
|
||||
import sys
|
||||
import traceback
|
||||
import types
|
||||
|
||||
|
||||
__all__ = ['to_unicode', 'local2utc', 'html2text', 'get_backtrace', 'iter_fields']
|
||||
__all__ = ['get_backtrace', 'get_bytes_size', 'html2text', 'iter_fields', 'local2utc', 'to_unicode', 'utc2local']
|
||||
|
||||
|
||||
def get_backtrace(empty="Empty backtrace."):
|
||||
"""
|
||||
Try to get backtrace as string.
|
||||
Returns "Error while trying to get backtrace" on failure.
|
||||
"""
|
||||
try:
|
||||
info = sys.exc_info()
|
||||
trace = traceback.format_exception(*info)
|
||||
sys.exc_clear()
|
||||
if trace[0] != "None\n":
|
||||
return "".join(trace)
|
||||
except:
|
||||
# No i18n here (imagine if i18n function calls error...)
|
||||
return "Error while trying to get backtrace"
|
||||
return empty
|
||||
|
||||
|
||||
def get_bytes_size(size, unit_name):
|
||||
unit_data = {
|
||||
'bytes': 1,
|
||||
'KB': 1024,
|
||||
'MB': 1024 * 1024,
|
||||
'GB': 1024 * 1024 * 1024,
|
||||
'TB': 1024 * 1024 * 1024 * 1024,
|
||||
}
|
||||
return float(size * unit_data[unit_name])
|
||||
|
||||
|
||||
try:
|
||||
import html2text as h2t
|
||||
h2t.UNICODE_SNOB = 1
|
||||
h2t.SKIP_INTERNAL_LINKS = True
|
||||
html2text = h2t.html2text
|
||||
except ImportError:
|
||||
warning('python-html2text is not present. HTML pages will not be converted into text.')
|
||||
def html2text(s):
|
||||
return s
|
||||
|
||||
|
||||
def iter_fields(obj):
|
||||
for attribute_name in dir(obj):
|
||||
if attribute_name.startswith('_'):
|
||||
continue
|
||||
attribute = getattr(obj, attribute_name)
|
||||
if not isinstance(attribute, types.MethodType):
|
||||
yield attribute_name, attribute
|
||||
|
||||
|
||||
def local2utc(d):
|
||||
d = d.replace(tzinfo=tz.tzlocal())
|
||||
d = d.astimezone(tz.tzutc())
|
||||
return d
|
||||
|
||||
|
||||
def to_unicode(text):
|
||||
|
|
@ -47,57 +101,8 @@ def to_unicode(text):
|
|||
except UnicodeError:
|
||||
return unicode(text, 'windows-1252')
|
||||
|
||||
def local2utc(d):
|
||||
d = d.replace(tzinfo=tz.tzlocal())
|
||||
d = d.astimezone(tz.tzutc())
|
||||
return d
|
||||
|
||||
def utc2local(d):
|
||||
d = d.replace(tzinfo=tz.tzutc())
|
||||
d = d.astimezone(tz.tzlocal())
|
||||
return d
|
||||
|
||||
try:
|
||||
import html2text as h2t
|
||||
h2t.UNICODE_SNOB = 1
|
||||
h2t.SKIP_INTERNAL_LINKS = True
|
||||
html2text = h2t.html2text
|
||||
except ImportError:
|
||||
warning('python-html2text is not present. HTML pages will not be converted into text.')
|
||||
def html2text(s):
|
||||
return s
|
||||
|
||||
def get_backtrace(empty="Empty backtrace."):
|
||||
"""
|
||||
Try to get backtrace as string.
|
||||
Returns "Error while trying to get backtrace" on failure.
|
||||
"""
|
||||
try:
|
||||
info = sys.exc_info()
|
||||
trace = traceback.format_exception(*info)
|
||||
sys.exc_clear()
|
||||
if trace[0] != "None\n":
|
||||
return "".join(trace)
|
||||
except:
|
||||
# No i18n here (imagine if i18n function calls error...)
|
||||
return "Error while trying to get backtrace"
|
||||
return empty
|
||||
|
||||
def iter_fields(obj):
|
||||
for attribute_name in dir(obj):
|
||||
if attribute_name.startswith('_'):
|
||||
continue
|
||||
attribute = getattr(obj, attribute_name)
|
||||
if not isinstance(attribute, types.MethodType):
|
||||
yield attribute_name, attribute
|
||||
|
||||
def iternb(it, nb=0):
|
||||
"""
|
||||
Iter 'nb' times on the generator
|
||||
"""
|
||||
i = 0
|
||||
for v in it:
|
||||
if i >= nb:
|
||||
raise StopIteration()
|
||||
yield v
|
||||
i += 1
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue