diff --git a/weboob/backends/arte/backend.py b/weboob/backends/arte/backend.py index 754e815a..4c998747 100644 --- a/weboob/backends/arte/backend.py +++ b/weboob/backends/arte/backend.py @@ -36,7 +36,8 @@ class ArteBackend(BaseBackend, ICapVideo): VERSION = '0.5' DESCRIPTION = 'Arte french TV' LICENSE = 'GPLv3' - CONFIG = ValuesDict(Value('lang', label='Lang of videos', choices={'fr': 'French', 'de': 'Deutsch', 'en': 'English'}, default='fr'), + CONFIG = ValuesDict(Value('lang', label='Lang of videos', + choices={'fr': 'French', 'de': 'Deutsch', 'en': 'English'}, default='fr'), Value('quality', label='Quality of videos', choices=['hd', 'sd'], default='hd')) BROWSER = ArteBrowser diff --git a/weboob/backends/bnporc/captcha.py b/weboob/backends/bnporc/captcha.py index ae11b8df..029664ac 100644 --- a/weboob/backends/bnporc/captcha.py +++ b/weboob/backends/bnporc/captcha.py @@ -25,7 +25,8 @@ class TileError(Exception): Exception.__init__(self, msg) self.tile = tile -class Captcha: + +class Captcha(object): def __init__(self, file): self.inim = Image.open(file) self.nx,self.ny = self.inim.size @@ -77,7 +78,8 @@ class Captcha: y += 25 ty += 1 -class Tile: + +class Tile(object): hash = {'b2d25ae11efaaaec6dd6a4c00f0dfc29': 1, '600873fa288e75ca6cca092ae95bf129': 2, 'da24ac28930feee169adcbc9bad4acaf': 3, @@ -117,4 +119,3 @@ class Tile: for pxl in pxls: sys.stdout.write('%02d' % pxl) sys.stdout.write('\n') - diff --git a/weboob/backends/bnporc/pages/account_coming.py b/weboob/backends/bnporc/pages/account_coming.py index 7b072670..522208cf 100644 --- a/weboob/backends/bnporc/pages/account_coming.py +++ b/weboob/backends/bnporc/pages/account_coming.py @@ -24,7 +24,6 @@ __all__ = ['AccountComing'] class AccountComing(BasePage): - def on_loaded(self): self.operations = [] diff --git a/weboob/backends/bnporc/pages/account_history.py b/weboob/backends/bnporc/pages/account_history.py index a5df0c98..db582b35 100644 --- a/weboob/backends/bnporc/pages/account_history.py +++ b/weboob/backends/bnporc/pages/account_history.py @@ -24,7 +24,6 @@ __all__ = ['AccountHistory'] class AccountHistory(BasePage): - def on_loaded(self): self.operations = [] diff --git a/weboob/backends/bnporc/pages/accounts_list.py b/weboob/backends/bnporc/pages/accounts_list.py index b457938c..a167133c 100644 --- a/weboob/backends/bnporc/pages/accounts_list.py +++ b/weboob/backends/bnporc/pages/accounts_list.py @@ -24,6 +24,10 @@ from weboob.tools.browser import BasePage from ..errors import PasswordExpired + +__all__ = ['AccountsList'] + + class AccountsList(BasePage): LINKID_REGEXP = re.compile(".*ch4=(\w+).*") diff --git a/weboob/backends/bnporc/pages/login.py b/weboob/backends/bnporc/pages/login.py index cc79c250..92b518e0 100644 --- a/weboob/backends/bnporc/pages/login.py +++ b/weboob/backends/bnporc/pages/login.py @@ -55,9 +55,11 @@ class LoginPage(BasePage): self.browser['ch5'] = img.get_codes(password) self.browser.submit() + class ConfirmPage(BasePage): pass + class ChangePasswordPage(BasePage): def change_password(self, current, new): img = Captcha(self.browser.openurl('/NSImgGrille')) diff --git a/weboob/backends/bnporc/pages/transfer.py b/weboob/backends/bnporc/pages/transfer.py index 782d9d1f..bdef646e 100644 --- a/weboob/backends/bnporc/pages/transfer.py +++ b/weboob/backends/bnporc/pages/transfer.py @@ -15,6 +15,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + import re from weboob.tools.browser import BasePage @@ -56,6 +57,7 @@ class TransferPage(BasePage): self.browser['T5'] = reason self.browser.submit() + class TransferConfirmPage(BasePage): def on_loaded(self): for td in self.document.getroot().cssselect('td#size2'): @@ -67,6 +69,7 @@ class TransferConfirmPage(BasePage): self.browser.location('/NS_VIRDA?stp=%s' % m.group(1)) return + class TransferCompletePage(BasePage): def get_id(self): return self.group_dict['id'] diff --git a/weboob/backends/bouygues/backend.py b/weboob/backends/bouygues/backend.py index 43090f44..18358e33 100644 --- a/weboob/backends/bouygues/backend.py +++ b/weboob/backends/bouygues/backend.py @@ -19,7 +19,6 @@ from __future__ import with_statement from weboob.capabilities.messages import CantSendMessage, ICapMessages, ICapMessagesPost -#from weboob.capabilities.account import ICapAccount, StatusField from weboob.tools.backend import BaseBackend from weboob.tools.value import ValuesDict, Value diff --git a/weboob/backends/bp/backend.py b/weboob/backends/bp/backend.py index 994098d7..4d1aea43 100644 --- a/weboob/backends/bp/backend.py +++ b/weboob/backends/bp/backend.py @@ -23,6 +23,9 @@ from weboob.tools.value import ValuesDict, Value from .browser import BPBrowser +__all__ = ['BPBackend'] + + class BPBackend(BaseBackend, ICapBank): NAME = 'bp' MAINTAINER = 'Nicolas Duhamel' diff --git a/weboob/backends/canalplus/backend.py b/weboob/backends/canalplus/backend.py index 933daa44..e205fe43 100644 --- a/weboob/backends/canalplus/backend.py +++ b/weboob/backends/canalplus/backend.py @@ -25,6 +25,7 @@ from weboob.tools.value import ValuesDict, Value from .browser import CanalplusBrowser from .pages import CanalplusVideo + __all__ = ['CanalplusBackend'] @@ -57,7 +58,6 @@ class CanalplusBackend(BaseBackend, ICapVideo): if 'thumbnail' in fields: with self.browser: video.thumbnail.data = self.browser.readurl(video.thumbnail.url) - return video OBJECTS = {CanalplusVideo: fill_video} diff --git a/weboob/backends/canalplus/browser.py b/weboob/backends/canalplus/browser.py index e4a74c1c..f7e3827c 100644 --- a/weboob/backends/canalplus/browser.py +++ b/weboob/backends/canalplus/browser.py @@ -15,16 +15,21 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + import urllib +import lxml.etree + from weboob.tools.browser import BaseBrowser from weboob.tools.browser.decorators import id2url from .pages import InitPage, CanalplusVideo, VideoPage -import lxml.etree -class XMLParser: +__all__ = ['CanalplusBrowser'] + + +class XMLParser(object): def parse(self, data, encoding=None): if encoding is None: parser = None @@ -33,22 +38,21 @@ class XMLParser: return lxml.etree.XML(data.get_data(), parser) -__all__ = ['CanalplusBrowser'] - - class CanalplusBrowser(BaseBrowser): DOMAIN = u'service.canal-plus.com' ENCODING = 'utf-8' - PAGES = {r"http://service.canal-plus.com/video/rest/initPlayer/cplus/": InitPage, - r"http://service.canal-plus.com/video/rest/search/cplus/.*": VideoPage, - r"http://service.canal-plus.com/video/rest/getVideosLiees/cplus/(?P.+)": VideoPage, - } + PAGES = { + r'http://service.canal-plus.com/video/rest/initPlayer/cplus/': InitPage, + r'http://service.canal-plus.com/video/rest/search/cplus/.*': VideoPage, + r'http://service.canal-plus.com/video/rest/getVideosLiees/cplus/(?P.+)': VideoPage, + } #We need lxml.etree.XMLParser for read CDATA PARSER = XMLParser() - FORMATS = { 'sd': 'BAS_DEBIT', - 'hd': 'HD' - } + FORMATS = { + 'sd': 'BAS_DEBIT', + 'hd': 'HD', + } def __init__(self, quality, *args, **kwargs): BaseBrowser.__init__(self, parser= self.PARSER, *args, **kwargs) @@ -58,15 +62,13 @@ class CanalplusBrowser(BaseBrowser): self.quality = 'HD' def home(self): - self.location("http://service.canal-plus.com/video/rest/initPlayer/cplus/") + self.location('http://service.canal-plus.com/video/rest/initPlayer/cplus/') def iter_search_results(self, pattern): - self.location("http://service.canal-plus.com/video/rest/search/cplus/" + urllib.quote_plus(pattern)) + self.location('http://service.canal-plus.com/video/rest/search/cplus/' + urllib.quote_plus(pattern)) return self.page.iter_results() @id2url(CanalplusVideo.id2url) def get_video(self, url, video=None): self.location(url) return self.page.get_video(video, self.quality) - - diff --git a/weboob/backends/canalplus/pages/initpage.py b/weboob/backends/canalplus/pages/initpage.py index 847fb251..549948c3 100644 --- a/weboob/backends/canalplus/pages/initpage.py +++ b/weboob/backends/canalplus/pages/initpage.py @@ -15,26 +15,28 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + from weboob.tools.browser import BasePage + __all__ = ['InitPage'] + class InitPage(BasePage): - def on_loaded(self): - channels = [] - ### Parse liste des channels - for elem in self.document[2].getchildren(): - channel = {} - for e in elem.getchildren(): - subchannels = [] - if e.tag == "NOM": - channel['nom'] = e.text - elif e.tag == "SELECTIONS": - - for select in e: - subchannel = {} - subchannel['id'] = select[0].text - subchannel['nom'] = select[1].text - subchannels.append(subchannel) - channel['subchannels'] = subchannels - channels.append(channel) + def on_loaded(self): + channels = [] + ### Parse liste des channels + for elem in self.document[2].getchildren(): + channel = {} + for e in elem.getchildren(): + subchannels = [] + if e.tag == "NOM": + channel['nom'] = e.text + elif e.tag == "SELECTIONS": + for select in e: + subchannel = {} + subchannel['id'] = select[0].text + subchannel['nom'] = select[1].text + subchannels.append(subchannel) + channel['subchannels'] = subchannels + channels.append(channel) diff --git a/weboob/backends/canaltp/browser.py b/weboob/backends/canaltp/browser.py index c1820ee4..4bffc95e 100644 --- a/weboob/backends/canaltp/browser.py +++ b/weboob/backends/canaltp/browser.py @@ -17,9 +17,14 @@ from datetime import datetime, date, time + from weboob.tools.browser import BaseBrowser -from weboob.tools.parsers.lxmlparser import SelectElementException from weboob.tools.misc import to_unicode +from weboob.tools.parsers.lxmlparser import SelectElementException + + +__all__ = ['CanalTP'] + class CanalTP(BaseBrowser): DOMAIN = 'widget.canaltp.fr' diff --git a/weboob/backends/fourchan/backend.py b/weboob/backends/fourchan/backend.py index 7b618e93..2f1df67a 100644 --- a/weboob/backends/fourchan/backend.py +++ b/weboob/backends/fourchan/backend.py @@ -34,7 +34,7 @@ class FourChanBackend(BaseBackend, ICapMessages): EMAIL = 'romain@weboob.org' VERSION = '0.5' LICENSE = 'GPLv3' - DESCRIPTION = "4chan website" + DESCRIPTION = '4chan website' CONFIG = ValuesDict(Value('boards', label='Boards to fetch')) STORAGE = {'boards': {}} BROWSER = FourChan diff --git a/weboob/backends/fourchan/browser.py b/weboob/backends/fourchan/browser.py index 5e6c86b3..6d4100d3 100644 --- a/weboob/backends/fourchan/browser.py +++ b/weboob/backends/fourchan/browser.py @@ -15,26 +15,27 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + from weboob.tools.browser import BaseBrowser from .pages.board import BoardPage + class FourChan(BaseBrowser): DOMAIN = 'boards.4chan.org' - PAGES = {'http://boards.4chan.org/\w+/': BoardPage, - 'http://boards.4chan.org/\w+/res/\d+': BoardPage, - } + PAGES = { + 'http://boards.4chan.org/\w+/': BoardPage, + 'http://boards.4chan.org/\w+/res/\d+': BoardPage, + } def is_logged(self): return True def get_threads(self, board): self.location('http://boards.4chan.org/%s/' % board) - return self.page.articles def get_thread(self, board, id): self.location('http://boards.4chan.org/%s/res/%d' % (board, long(id))) - assert len(self.page.articles) == 1 return self.page.articles[0] diff --git a/weboob/backends/fourchan/pages/board.py b/weboob/backends/fourchan/pages/board.py index 24efb923..3fbb0088 100644 --- a/weboob/backends/fourchan/pages/board.py +++ b/weboob/backends/fourchan/pages/board.py @@ -15,12 +15,17 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -from datetime import datetime import re +from datetime import datetime + from weboob.tools.browser import BasePage + +__all__ = ['BoardPage'] + + class Message(object): def __init__(self, browser, board, id, filename=u'', url=u''): self.id = id @@ -42,6 +47,7 @@ class Message(object): def __repr__(self): return '' % (self.id, self.filename, self.url, len(self.comments)) + class BoardPage(BasePage): URL_REGEXP = re.compile('http://boards.4chan.org/(\w+)/') diff --git a/weboob/backends/gazelle/pages/index.py b/weboob/backends/gazelle/pages/index.py index 1d727a93..c6da22b8 100644 --- a/weboob/backends/gazelle/pages/index.py +++ b/weboob/backends/gazelle/pages/index.py @@ -26,6 +26,7 @@ class IndexPage(BasePage): def is_logged(self): return 'id' in self.document.find('body').attrib + class LoginPage(BasePage): def login(self, login, password): self.browser.select_form(nr=0) diff --git a/weboob/backends/isohunt/backend.py b/weboob/backends/isohunt/backend.py index e512d8a9..1348c330 100644 --- a/weboob/backends/isohunt/backend.py +++ b/weboob/backends/isohunt/backend.py @@ -15,6 +15,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + from weboob.capabilities.torrent import ICapTorrent from weboob.tools.backend import BaseBackend @@ -31,10 +32,6 @@ class IsohuntBackend(BaseBackend, ICapTorrent): VERSION = '0.5' DESCRIPTION = 'isohunt.com bittorrent tracker' LICENSE = 'GPLv3' - #CONFIG = ValuesDict(Value('domain', label='Domain (example "ssl.what.cd")'), - # Value('protocol', label='Protocol to use', choices=('http', 'https')), - # Value('username', label='Username'), - # Value('password', label='Password', masked=True)) BROWSER = IsohuntBrowser def create_default_browser(self): @@ -47,7 +44,6 @@ class IsohuntBackend(BaseBackend, ICapTorrent): torrent = self.browser.get_torrent(id) if not torrent: return None - return self.browser.openurl(torrent.url.encode('utf-8')).read() def iter_torrents(self, pattern): diff --git a/weboob/backends/isohunt/browser.py b/weboob/backends/isohunt/browser.py index 47d79325..5e29e10d 100644 --- a/weboob/backends/isohunt/browser.py +++ b/weboob/backends/isohunt/browser.py @@ -30,24 +30,19 @@ class IsohuntBrowser(BaseBrowser): ENCODING = 'utf-8' USER_AGENT = BaseBrowser.USER_AGENTS['wget'] PAGES = { - 'https://isohunt.com/torrents/.*iht=-1&ihp=1&ihs1=2&iho1=d' : TorrentsPage, - 'https://isohunt.com/torrent_details.*tab=summary' : TorrentPage - } - - def __init__(self, *args, **kwargs): - BaseBrowser.__init__(self, *args, **kwargs) + 'https://isohunt.com/torrents/.*iht=-1&ihp=1&ihs1=2&iho1=d' : TorrentsPage, + 'https://isohunt.com/torrent_details.*tab=summary' : TorrentPage, + } def home(self): return self.location('https://isohunt.com') def iter_torrents(self, pattern): self.location('https://isohunt.com/torrents/%s?iht=-1&ihp=1&ihs1=2&iho1=d' % pattern) - assert self.is_on_page(TorrentsPage) return self.page.iter_torrents() def get_torrent(self, id): self.location('https://isohunt.com/torrent_details/%s/?tab=summary' % id) - assert self.is_on_page(TorrentPage) return self.page.get_torrent(id) diff --git a/weboob/backends/isohunt/pages/torrents.py b/weboob/backends/isohunt/pages/torrents.py index f70d503a..382d8100 100644 --- a/weboob/backends/isohunt/pages/torrents.py +++ b/weboob/backends/isohunt/pages/torrents.py @@ -16,29 +16,18 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -from weboob.tools.browser import BasePage from weboob.capabilities.torrent import Torrent +from weboob.tools.browser import BasePage +from weboob.tools.misc import get_bytes_size __all__ = ['TorrentsPage'] class TorrentsPage(BasePage): - def unit(self, n, u): - m = {'KB': 1024, - 'MB': 1024*1024, - 'GB': 1024*1024*1024, - 'TB': 1024*1024*1024*1024, - } - return float(n*m[u]) - def iter_torrents(self): - - - for tr in self.document.getiterator('tr'): - if tr.attrib.get('class','') == 'hlRow': + if tr.attrib.get('class', '') == 'hlRow': # TODO à corriger atitle = tr.getchildren()[2].getchildren()[1] title = atitle.text @@ -57,41 +46,32 @@ class TorrentsPage(BasePage): seed = tr.getchildren()[4].text leech = tr.getchildren()[5].text url = 'https://isohunt.com/download/%s/mon_joli_torrent.torrent' % idt + yield Torrent(idt, + title, + url=url, + size=get_bytes_size(size, u), + seeders=int(seed), + leechers=int(leech)) - torrent = Torrent(idt, - title, - url=url, - size=self.unit(size,u), - seeders=int(seed), - leechers=int(leech)) - yield torrent class TorrentPage(BasePage): - def unit(self, n, u): - m = {'KB': 1024, - 'MB': 1024*1024, - 'GB': 1024*1024*1024, - 'TB': 1024*1024*1024*1024, - } - return float(n*m[u]) - def get_torrent(self, id): title = '' - url = 'https://isohunt.com/download/%s/%s.torrent' % (id , id) + url = 'https://isohunt.com/download/%s/%s.torrent' % (id, id) for a in self.document.getiterator('a'): - if 'Search more torrents of' in a.attrib.get('title',''): + if 'Search more torrents of' in a.attrib.get('title', ''): title = a.tail for span in self.document.getiterator('span'): - if span.attrib.get('style','') == 'color:green;' and ('ShowTip' in span.attrib.get('onmouseover','')): + if span.attrib.get('style', '') == 'color:green;' and ('ShowTip' in span.attrib.get('onmouseover', '')): seed = span.tail.split(' ')[1] - tip_id = span.attrib.get('onmouseover','').split("'")[1] + tip_id = span.attrib.get('onmouseover', '').split("'")[1] for div in self.document.getiterator('div'): # find the corresponding super tip which appears on super mouse hover! - if div.attrib.get('class','') == 'dirs ydsf' and tip_id in div.attrib.get('id',''): + if div.attrib.get('class', '') == 'dirs ydsf' and tip_id in div.attrib.get('id', ''): leech = div.getchildren()[0].getchildren()[1].tail.split(' ')[2] # the with the size in it doesn't have a distinction # have to get it by higher - elif div.attrib.get('id','') == 'torrent_details': + elif div.attrib.get('id', '') == 'torrent_details': size = div.getchildren()[6].getchildren()[0].getchildren()[0].text u = size[-2:] size = float(size[:-3]) @@ -104,7 +84,7 @@ class TorrentPage(BasePage): files = [] count_p_found = 0 for p in self.document.getiterator('p'): - if p.attrib.get('style','') == "line-height:1.2em;margin-top:1.8em": + if p.attrib.get('style', '') == 'line-height:1.2em;margin-top:1.8em': count_p_found += 1 if count_p_found == 1: description = p.getchildren()[1].tail @@ -115,24 +95,20 @@ class TorrentPage(BasePage): files.append(p.getchildren()[0].tail.strip()) for td in self.document.getiterator('td'): - #print td.attrib.get('class') - if td.attrib.get('class','') == 'fileRows': + if td.attrib.get('class', '') == 'fileRows': filename = td.text - #print "len"+str(len(td.getchildren())) for slash in td.getchildren(): filename += '/' filename += slash.tail files.append(filename) - #--------------------------TODO torrent = Torrent(id, title) torrent.url = url - torrent.size = self.unit(size,u) + torrent.size = get_bytes_size(size, u) torrent.seeders = int(seed) torrent.leechers = int(leech) torrent.description = description torrent.files = files - return torrent diff --git a/weboob/backends/kickass/backend.py b/weboob/backends/kickass/backend.py index 4e045a8b..748809cf 100644 --- a/weboob/backends/kickass/backend.py +++ b/weboob/backends/kickass/backend.py @@ -31,10 +31,6 @@ class KickassBackend(BaseBackend, ICapTorrent): VERSION = '0.5' DESCRIPTION = 'kickasstorrent.com bittorrent tracker' LICENSE = 'GPLv3' - #CONFIG = ValuesDict(Value('domain', label='Domain (example "ssl.what.cd")'), - # Value('protocol', label='Protocol to use', choices=('http', 'https')), - # Value('username', label='Username'), - # Value('password', label='Password', masked=True)) BROWSER = KickassBrowser def create_default_browser(self): diff --git a/weboob/backends/kickass/browser.py b/weboob/backends/kickass/browser.py index 14135ab4..c1f82264 100644 --- a/weboob/backends/kickass/browser.py +++ b/weboob/backends/kickass/browser.py @@ -30,43 +30,19 @@ class KickassBrowser(BaseBrowser): ENCODING = 'utf-8' USER_AGENT = BaseBrowser.USER_AGENTS['wget'] PAGES = { - 'http://fr.kickasstorrents.com/new/.*field=seeders&sorder=desc' : TorrentsPage, - 'http://fr.kickasstorrents.com/.*.html' : TorrentPage - } - - def __init__(self, *args, **kwargs): - #self.DOMAIN = domain - #self.PROTOCOL = protocol - #self.PAGES = {} - #for key, value in PiratebayBrowser.PAGES.iteritems(): - # self.PAGES[key % domain] = value - - BaseBrowser.__init__(self, *args, **kwargs) - - #def login(self): - # if not self.is_on_page(LoginPage): - # self.home() - # self.page.login(self.username, self.password) - - #def is_logged(self): - # if not self.page or self.is_on_page(LoginPage): - # return False - # if self.is_on_page(IndexPage): - # return self.page.is_logged() - # return True + 'http://fr.kickasstorrents.com/new/.*field=seeders&sorder=desc': TorrentsPage, + 'http://fr.kickasstorrents.com/.*.html': TorrentPage, + } def home(self): return self.location('http://kickasstorrents.com') def iter_torrents(self, pattern): - #self.location(self.buildurl('/torrents.php', searchstr=pattern)) self.location('http://fr.kickasstorrents.com/new/?q=%s&field=seeders&sorder=desc' % pattern) - assert self.is_on_page(TorrentsPage) return self.page.iter_torrents() def get_torrent(self, id): self.location('http://fr.kickasstorrents.com/%s.html' % id) - assert self.is_on_page(TorrentPage) return self.page.get_torrent(id) diff --git a/weboob/backends/kickass/pages/torrents.py b/weboob/backends/kickass/pages/torrents.py index da29769e..4911178b 100644 --- a/weboob/backends/kickass/pages/torrents.py +++ b/weboob/backends/kickass/pages/torrents.py @@ -16,28 +16,18 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -from weboob.tools.browser import BasePage from weboob.capabilities.torrent import Torrent +from weboob.tools.browser import BasePage +from weboob.tools.misc import get_bytes_size __all__ = ['TorrentsPage'] class TorrentsPage(BasePage): - def unit(self, n, u): - m = {'bytes': 1, - 'KB': 1024, - 'MB': 1024*1024, - 'GB': 1024*1024*1024, - 'TB': 1024*1024*1024*1024, - } - return float(n*m[u]) - def iter_torrents(self): - for tr in self.document.getiterator('tr'): - if tr.attrib.get('class','') == 'odd' or tr.attrib.get('class','') == ' even': + if tr.attrib.get('class', '') == 'odd' or tr.attrib.get('class', '') == ' even': title = tr.getchildren()[0].getchildren()[1].getchildren()[1].text if not title: title = '' @@ -46,78 +36,67 @@ class TorrentsPage(BasePage): title += red.text if red.tail: title += red.tail - idt = tr.getchildren()[0].getchildren()[1].getchildren()[1].attrib.get('href','').replace('/','').replace('.html','') - url = tr.getchildren()[0].getchildren()[0].getchildren()[0].getchildren()[0].attrib.get('href','') + idt = tr.getchildren()[0].getchildren()[1].getchildren()[1].attrib.get('href', '').replace('/', '') \ + .replace('.html', '') + url = tr.getchildren()[0].getchildren()[0].getchildren()[0].getchildren()[0].attrib.get('href', '') size = tr.getchildren()[1].text u = tr.getchildren()[1].getchildren()[0].text - size = size = size.replace(',','.') + size = size = size.replace(',', '.') size = float(size) seed = tr.getchildren()[4].text leech = tr.getchildren()[5].text - torrent = Torrent(idt, - title, - url=url, - size=self.unit(size,u), - seeders=int(seed), - leechers=int(leech)) - yield torrent + yield Torrent(idt, + title, + url=url, + size=get_bytes_size(size, u), + seeders=int(seed), + leechers=int(leech)) + class TorrentPage(BasePage): - def unit(self, n, u): - m = {'bytes': 1, - 'KB': 1024, - 'MB': 1024*1024, - 'GB': 1024*1024*1024, - 'TB': 1024*1024*1024*1024, - } - return float(n*m[u]) - def get_torrent(self, id): - seed = 0 leech = 0 - description = "No description" + description = 'No description' url = 'No Url found' for div in self.document.getiterator('div'): - if div.attrib.get('id','') == 'desc': + if div.attrib.get('id', '') == 'desc': description = div.text.strip() for ch in div.getchildren(): if ch.tail != None: description += ' '+ch.tail.strip() - if div.attrib.get('class','') == 'seedBlock': + if div.attrib.get('class', '') == 'seedBlock': seed = int(div.getchildren()[1].text) - if div.attrib.get('class','') == 'leechBlock': + if div.attrib.get('class', '') == 'leechBlock': leech = int(div.getchildren()[1].text) for h in self.document.getiterator('h1'): - if h.attrib.get('class','') == 'torrentName': + if h.attrib.get('class', '') == 'torrentName': title = h.getchildren()[0].getchildren()[0].text for a in self.document.getiterator('a'): - if ('Download' in a.attrib.get('title','')) and ('torrent file' in a.attrib.get('title','')): - url = a.attrib.get('href','') + if ('Download' in a.attrib.get('title', '')) and ('torrent file' in a.attrib.get('title', '')): + url = a.attrib.get('href', '') size = 0 for span in self.document.getiterator('span'): - if span.attrib.get('class','') == "folder" or span.attrib.get('class','') == "folderopen": + if span.attrib.get('class', '') == 'folder' or span.attrib.get('class', '') == 'folderopen': size = span.getchildren()[1].tail u = span.getchildren()[2].text - size = float(size.split(': ')[1].replace(',','.')) + size = float(size.split(': ')[1].replace(',', '.')) files = [] for td in self.document.getiterator('td'): - if td.attrib.get('class','') == 'torFileName': + if td.attrib.get('class', '') == 'torFileName': files.append(td.text) - torrent = Torrent(id, title) torrent = Torrent(id, title) torrent.url = url - torrent.size = self.unit(size,u) + torrent.size = get_bytes_size(size, u) torrent.seeders = int(seed) torrent.leechers = int(leech) torrent.description = description torrent.files = files - return torrent diff --git a/weboob/backends/lcl/browser.py b/weboob/backends/lcl/browser.py index 4c17adb3..124e0092 100644 --- a/weboob/backends/lcl/browser.py +++ b/weboob/backends/lcl/browser.py @@ -30,10 +30,11 @@ class LCLBrowser(BaseBrowser): DOMAIN = 'particuliers.secure.lcl.fr' ENCODING = 'utf-8' USER_AGENT = BaseBrowser.USER_AGENTS['wget'] - PAGES = {'https://particuliers.secure.lcl.fr/index.html': LoginPage, - 'https://particuliers.secure.lcl.fr/everest/UWBI/UWBIAccueil\?DEST=IDENTIFICATION': LoginErrorPage, - 'https://particuliers.secure.lcl.fr/outil/UWSP/Synthese/accesSynthese': AccountsPage - } + PAGES = { + 'https://particuliers.secure.lcl.fr/index.html': LoginPage, + 'https://particuliers.secure.lcl.fr/everest/UWBI/UWBIAccueil\?DEST=IDENTIFICATION': LoginErrorPage, + 'https://particuliers.secure.lcl.fr/outil/UWSP/Synthese/accesSynthese': AccountsPage, + } def __init__(self, agency, *args, **kwargs): self.agency = agency diff --git a/weboob/backends/meteofrance/backend.py b/weboob/backends/meteofrance/backend.py index 106154c7..fea1c4a0 100644 --- a/weboob/backends/meteofrance/backend.py +++ b/weboob/backends/meteofrance/backend.py @@ -15,6 +15,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + from weboob.capabilities.weather import ICapWeather from weboob.tools.backend import BaseBackend @@ -31,10 +32,6 @@ class MeteofranceBackend(BaseBackend, ICapWeather): VERSION = '0.5' DESCRIPTION = 'Get forecasts from the MeteoFrance website' LICENSE = 'GPLv3' - #CONFIG = ValuesDict(Value('domain', label='Domain (example "ssl.what.cd")'), - # Value('protocol', label='Protocol to use', choices=('http', 'https')), - # Value('username', label='Username'), - # Value('password', label='Password', masked=True)) BROWSER = MeteofranceBrowser def create_default_browser(self): diff --git a/weboob/backends/meteofrance/browser.py b/weboob/backends/meteofrance/browser.py index fe230f2b..00e82920 100644 --- a/weboob/backends/meteofrance/browser.py +++ b/weboob/backends/meteofrance/browser.py @@ -31,14 +31,16 @@ class MeteofranceBrowser(BaseBrowser): PROTOCOL = 'http' ENCODING = 'utf-8' USER_AGENT = BaseBrowser.USER_AGENTS['wget'] - WEATHER_URL = "{0}://{1}/france/meteo?PREVISIONS_PORTLET.path=previsionsville/{{cityid}}".format(PROTOCOL, DOMAIN) - CITY_SEARCH_URL="{0}://{1}/france/accueil/resultat?RECHERCHE_RESULTAT_PORTLET.path=rechercheresultat&query={{city_pattern}}&type=PREV_FRANCE&satellite=france".format(PROTOCOL, DOMAIN) + WEATHER_URL = '{0}://{1}/france/meteo?PREVISIONS_PORTLET.path=previsionsville/{{cityid}}'.format(PROTOCOL, DOMAIN) + CITY_SEARCH_URL = '{0}://{1}/france/accueil/resultat?RECHERCHE_RESULTAT_PORTLET.path=rechercheresultat&' \ + 'query={{city_pattern}}&type=PREV_FRANCE&satellite=france'.format(PROTOCOL, DOMAIN) PAGES = { - WEATHER_URL.format(cityid=".*") : WeatherPage, - CITY_SEARCH_URL.format(city_pattern=".*") : CityPage, - "http://france.meteofrance.com/france/accueil/resultat.*" : CityPage, - "http://france.meteofrance.com/france/meteo.*" : WeatherPage - } + WEATHER_URL.format(cityid=".*"): WeatherPage, + CITY_SEARCH_URL.format(city_pattern=".*"): CityPage, + 'http://france.meteofrance.com/france/accueil/resultat.*': CityPage, + 'http://france.meteofrance.com/france/meteo.*': WeatherPage, + } + def __init__(self, *args, **kwargs): BaseBrowser.__init__(self, *args, **kwargs) diff --git a/weboob/backends/meteofrance/pages/meteo.py b/weboob/backends/meteofrance/pages/meteo.py index 6b6d0af0..930abd47 100644 --- a/weboob/backends/meteofrance/pages/meteo.py +++ b/weboob/backends/meteofrance/pages/meteo.py @@ -22,6 +22,7 @@ from weboob.capabilities.weather import Forecast, Current, City import datetime + __all__ = ['WeatherPage', 'CityPage'] @@ -60,16 +61,15 @@ class WeatherPage(BasePage): mdate = datetime.datetime.now() return Current(mdate, temp, mtxt, "C") - def get_city(self): - """Return the city from the forecastpage + """ + Return the city from the forecastpage. """ for div in self.document.getiterator('div'): if div.attrib.has_key("class") and div.attrib.get("class") == "choix": for strong in div.getiterator("strong"): city_name=strong.text +" "+ strong.tail.replace("(","").replace(")","") city_id=self.url.split("/")[-1] - return City(city_id, city_name) diff --git a/weboob/tools/misc.py b/weboob/tools/misc.py index 4f7b6fa4..b9b93048 100644 --- a/weboob/tools/misc.py +++ b/weboob/tools/misc.py @@ -16,14 +16,68 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -from logging import warning from dateutil import tz +from logging import warning import sys import traceback import types -__all__ = ['to_unicode', 'local2utc', 'html2text', 'get_backtrace', 'iter_fields'] +__all__ = ['get_backtrace', 'get_bytes_size', 'html2text', 'iter_fields', 'local2utc', 'to_unicode', 'utc2local'] + + +def get_backtrace(empty="Empty backtrace."): + """ + Try to get backtrace as string. + Returns "Error while trying to get backtrace" on failure. + """ + try: + info = sys.exc_info() + trace = traceback.format_exception(*info) + sys.exc_clear() + if trace[0] != "None\n": + return "".join(trace) + except: + # No i18n here (imagine if i18n function calls error...) + return "Error while trying to get backtrace" + return empty + + +def get_bytes_size(size, unit_name): + unit_data = { + 'bytes': 1, + 'KB': 1024, + 'MB': 1024 * 1024, + 'GB': 1024 * 1024 * 1024, + 'TB': 1024 * 1024 * 1024 * 1024, + } + return float(size * unit_data[unit_name]) + + +try: + import html2text as h2t + h2t.UNICODE_SNOB = 1 + h2t.SKIP_INTERNAL_LINKS = True + html2text = h2t.html2text +except ImportError: + warning('python-html2text is not present. HTML pages will not be converted into text.') + def html2text(s): + return s + + +def iter_fields(obj): + for attribute_name in dir(obj): + if attribute_name.startswith('_'): + continue + attribute = getattr(obj, attribute_name) + if not isinstance(attribute, types.MethodType): + yield attribute_name, attribute + + +def local2utc(d): + d = d.replace(tzinfo=tz.tzlocal()) + d = d.astimezone(tz.tzutc()) + return d def to_unicode(text): @@ -47,57 +101,8 @@ def to_unicode(text): except UnicodeError: return unicode(text, 'windows-1252') -def local2utc(d): - d = d.replace(tzinfo=tz.tzlocal()) - d = d.astimezone(tz.tzutc()) - return d def utc2local(d): d = d.replace(tzinfo=tz.tzutc()) d = d.astimezone(tz.tzlocal()) return d - -try: - import html2text as h2t - h2t.UNICODE_SNOB = 1 - h2t.SKIP_INTERNAL_LINKS = True - html2text = h2t.html2text -except ImportError: - warning('python-html2text is not present. HTML pages will not be converted into text.') - def html2text(s): - return s - -def get_backtrace(empty="Empty backtrace."): - """ - Try to get backtrace as string. - Returns "Error while trying to get backtrace" on failure. - """ - try: - info = sys.exc_info() - trace = traceback.format_exception(*info) - sys.exc_clear() - if trace[0] != "None\n": - return "".join(trace) - except: - # No i18n here (imagine if i18n function calls error...) - return "Error while trying to get backtrace" - return empty - -def iter_fields(obj): - for attribute_name in dir(obj): - if attribute_name.startswith('_'): - continue - attribute = getattr(obj, attribute_name) - if not isinstance(attribute, types.MethodType): - yield attribute_name, attribute - -def iternb(it, nb=0): - """ - Iter 'nb' times on the generator - """ - i = 0 - for v in it: - if i >= nb: - raise StopIteration() - yield v - i += 1