nojhan/weboob-devel
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

support repositories to manage backends (closes #747)

Browse source
This commit is contained in:
Romain Bignon 2012-01-03 12:10:21 +01:00
commit 14a7a1d362
410 changed files with 1079 additions and 297 deletions
Download patch file Download diff file Expand all files Collapse all files

1
modules/__init__.py Normal file
View file

@ -0,0 +1 @@

22
modules/arte/__init__.py Normal file
View file

@ -0,0 +1,22 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from .backend import ArteBackend
__all__ = ['ArteBackend']

68
modules/arte/backend.py Normal file
View file

@ -0,0 +1,68 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from __future__ import with_statement
from weboob.capabilities.video import ICapVideo
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.tools.value import Value
from .browser import ArteBrowser
from .video import ArteVideo
__all__ = ['ArteBackend']
class ArteBackend(BaseBackend, ICapVideo):
NAME = 'arte'
MAINTAINER = 'Romain Bignon'
EMAIL = 'romain@weboob.org'
VERSION = '0.a'
DESCRIPTION = 'Arte french TV'
LICENSE = 'AGPLv3+'
CONFIG = BackendConfig(Value('lang', label='Lang of videos',
choices={'fr': 'French', 'de': 'Deutsch', 'en': 'English'}, default='fr'),
Value('quality', label='Quality of videos', choices=['hd', 'sd'], default='hd'))
BROWSER = ArteBrowser
def create_default_browser(self):
return self.create_browser(lang=self.config['lang'].get(), quality=self.config['quality'].get())
def get_video(self, _id):
with self.browser:
return self.browser.get_video(_id)
def iter_search_results(self, pattern=None, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False, max_results=None):
with self.browser:
return self.browser.iter_search_results(pattern)
def fill_video(self, video, fields):
if fields != ['thumbnail']:
# if we don't want only the thumbnail, we probably want also every fields
with self.browser:
video = self.browser.get_video(ArteVideo.id2url(video.id), video)
if 'thumbnail' in fields and video.thumbnail:
with self.browser:
video.thumbnail.data = self.browser.readurl(video.thumbnail.url)
return video
OBJECTS = {ArteVideo: fill_video}

60
modules/arte/browser.py Normal file
View file

@ -0,0 +1,60 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.tools.browser import BaseBrowser
from weboob.tools.browser.decorators import id2url
from .pages import IndexPage, VideoPage
from .video import ArteVideo
__all__ = ['ArteBrowser']
class ArteBrowser(BaseBrowser):
DOMAIN = u'videos.arte.tv'
ENCODING = None
PAGES = {r'http://videos.arte.tv/\w+/videos/arte7.*': IndexPage,
r'http://videos.arte.tv/\w+/do_search/videos/.*': IndexPage,
r'http://videos.arte.tv/\w+/videos/(?P<id>.+)\.html': VideoPage
}
SEARCH_LANG = {'fr': 'recherche', 'de':'suche', 'en': 'search'}
def __init__(self, lang, quality, *args, **kwargs):
BaseBrowser.__init__(self, *args, **kwargs)
self.lang = lang
self.quality = quality
@id2url(ArteVideo.id2url)
def get_video(self, url, video=None):
self.location(url)
return self.page.get_video(video, self.lang, self.quality)
def home(self):
self.location('http://videos.arte.tv/fr/videos/arte7')
def iter_search_results(self, pattern):
if not pattern:
self.home()
else:
self.location(self.buildurl('/%s/do_search/videos/%s' % (self.lang, self.SEARCH_LANG[self.lang]), q=pattern.encode('utf-8')))
assert self.is_on_page(IndexPage)
return self.page.iter_videos()

BIN
modules/arte/favicon.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 819 B

114
modules/arte/pages.py Normal file
View file

@ -0,0 +1,114 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
import datetime
import re
import urllib
from weboob.tools.browser import BasePage, BrokenPageError
from .video import ArteVideo
__all__ = ['IndexPage', 'VideoPage']
class IndexPage(BasePage):
def iter_videos(self):
videos = self.document.getroot().cssselect("div[class=video]")
for div in videos:
title = div.find('h2').find('a').text
m = re.match(r'/fr/videos/(.*)\.html', div.find('h2').find('a').attrib['href'])
_id = ''
if m:
_id = m.group(1)
rating = rating_max = 0
rates = self.parser.select(div, 'div[class=rateContainer]', 1)
for r in rates.findall('div'):
if 'star-rating-on' in r.attrib['class']:
rating += 1
rating_max += 1
video = ArteVideo(_id)
video.title = title
video.rating = rating
video.rating_max = rating_max
thumb = self.parser.select(div, 'img[class=thumbnail]', 1)
video.thumbnail_url = 'http://videos.arte.tv' + thumb.attrib['src']
try:
parts = self.parser.select(div, 'div.duration_thumbnail', 1).text.split(':')
if len(parts) == 2:
hours = 0
minutes, seconds = parts
elif len(parts) == 3:
hours, minutes, seconds = parts
else:
raise BrokenPageError('Unable to parse duration %r' % parts)
except BrokenPageError:
pass
else:
video.duration = datetime.timedelta(hours=int(hours), minutes=int(minutes), seconds=int(seconds))
yield video
class VideoPage(BasePage):
def get_video(self, video=None, lang='fr', quality='hd'):
if not video:
video = ArteVideo(self.group_dict['id'])
video.title = self.get_title()
video.url = self.get_url(lang, quality)
return video
def get_title(self):
return self.document.getroot().cssselect('h2')[0].text
def get_url(self, lang, quality):
obj = self.parser.select(self.document.getroot(), 'object', 1)
movie_url = self.parser.select(obj, 'param[name=movie]', 1)
xml_url = urllib.unquote(movie_url.attrib['value'].split('videorefFileUrl=')[-1])
doc = self.browser.get_document(self.browser.openurl(xml_url))
videos_list = self.parser.select(doc.getroot(), 'video')
videos = {}
for v in videos_list:
videos[v.attrib['lang']] = v.attrib['ref']
if lang in videos:
xml_url = videos[lang]
else:
xml_url = videos.popitem()[1]
doc = self.browser.get_document(self.browser.openurl(xml_url))
obj = self.parser.select(doc.getroot(), 'urls', 1)
videos_list = self.parser.select(obj, 'url')
urls = {}
for v in videos_list:
urls[v.attrib['quality']] = v.text
if quality in urls:
video_url = urls[quality]
else:
video_url = urls.popitem()[1]
return video_url

31
modules/arte/test.py Normal file
View file

@ -0,0 +1,31 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.tools.test import BackendTest
class ArteTest(BackendTest):
BACKEND = 'arte'
def test_arte(self):
l = list(self.backend.iter_search_results('arte'))
if len(l) > 0:
v = l[0]
self.backend.fillobj(v, ('url',))
self.assertTrue(v.url and v.url.startswith('rtmp://'), 'URL for video "%s" not found: %s' % (v.id, v.url))

30
modules/arte/video.py Normal file
View file

@ -0,0 +1,30 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.capabilities.video import BaseVideo
__all__ = ['ArteVideo']
class ArteVideo(BaseVideo):
@classmethod
def id2url(cls, _id):
return 'http://videos.arte.tv/fr/videos/%s.html' % _id

932
modules/aum/API.txt Normal file
View file

@ -0,0 +1,932 @@
Adopte un Mec API
------------------
Constants:
APIKEY = fb0123456789abcd
URL = http://api.adopteunmec.com/api.php
ME Commands
===========
me.login
---------
Parameters:
- login
- pass
Errors:
- 1.1.1 : invalid login
Return value:
{u'errors': [],
u'result': {u'baskets': u'384',
u'events': {u'lastBasket': {u'alert': u'1',
u'birthday': u'1989-02-04',
u'city': u'Paris',
u'country': u'fr',
u'cover': u'1',
u'id': u'14471939',
u'isBan': False,
u'isOnline': False,
u'last_cnx': u'2011-09-20 13:56:36',
u'list5': u'0',
u'login': u'kloo',
u'mod_level': u'0',
u'path': u'9/3/9/1/7/4/4/',
u'pseudo': u'Kloolloo',
u'region': u'11',
u'sex': 1,
u'shard': 9,
u'style': u'0',
u'table': u'adopteun.girls',
u'url': u'/api.php?member/view/14471939/Kloolloo',
u'zip': u'75001'},
u'lastChat': {u'alert': u'1',
u'birthday': u'1987-10-10',
u'city': u'Paris 18e Arrondissement',
u'country': u'fr',
u'cover': u'0',
u'id': u'13280274',
u'isBan': False,
u'isOnline': False,
u'last_cnx': u'2010-11-20 03:37:00',
u'list5': u'0',
u'login': u'#c1a63bda81cc03ccdf080ca6e003919e',
u'mod_level': u'0',
u'path': u'4/7/2/0/8/2/3/',
u'pseudo': u'Katie Lee',
u'region': u'11',
u'sex': 1,
u'shard': 4,
u'style': u'0',
u'table': u'adopteun.girls_coma',
u'url': u'/api.php?member/view/13280274/KatieLee',
u'zip': u'75018'},
u'lastFlash': {u'alert': u'1',
u'birthday': u'1983-12-20',
u'city': u'Longjumeau',
u'country': u'fr',
u'cover': u'1',
u'id': u'13883475',
u'isBan': False,
u'isOnline': True,
u'last_cnx': u'2011-09-20 18:52:13',
u'list5': u'0',
u'login': u'@',
u'mod_level': u'0',
u'path': u'5/7/4/3/8/8/3/',
u'pseudo': u'Pas Tjs Sage',
u'region': u'11',
u'sex': 1,
u'shard': 5,
u'style': u'2',
u'table': u'adopteun.girls',
u'url': u'/api.php?member/view/13883475/PasTjsSage',
u'zip': u'91160'},
u'lastMail': {u'alert': u'1',
u'birthday': u'1989-04-28',
u'city': u'Paris 8e Arrondissement',
u'country': u'fr',
u'cover': u'5',
u'id': u'13268738',
u'isBan': False,
u'isOnline': True,
u'last_cnx': u'2011-09-20 19:24:14',
u'list5': u'0',
u'login': u'@13268738',
u'mod_level': u'0',
u'path': u'8/3/7/8/6/2/3/',
u'pseudo': u'Th\xe9na',
u'region': u'11',
u'sex': 1,
u'shard': 8,
u'style': u'0',
u'table': u'adopteun.girls',
u'url': u'/api.php?member/view/13268738/Thna',
u'zip': u'75008'},
u'lastVisit': {u'alert': u'1',
u'birthday': u'1988-02-27',
u'city': u'Paris',
u'country': u'fr',
u'cover': u'1',
u'id': u'14477637',
u'isBan': False,
u'isOnline': True,
u'last_cnx': u'2011-09-20 18:31:00',
u'list5': u'0',
u'login': u'@',
u'mod_level': u'0',
u'path': u'7/3/6/7/7/4/4/',
u'pseudo': u'Lolly',
u'region': u'11',
u'sex': 1,
u'shard': 7,
u'style': u'4',
u'table': u'adopteun.girls',
u'url': u'/api.php?member/view/14477637/Lolly',
u'zip': u'75005'}},
u'flashs': 10,
u'mails': u'731',
u'me': {u'about1': u"Je n'ai pas de temps &agrave; perdre, je n'ai ni MSN ni Facebook, je consid&egrave;re qu'on appr&eacute;cie davantage la discussion face &agrave; face autour d'un verre que dans les yeux de son &eacute;cran.\r<br>\r<br>Bon et ne venez que si vous avez quelque chose &agrave; me dire, je n'envoie jamais de charmes.",
u'about2': u'',
u'admin': u'0',
u'alert': u'0',
u'alert_add': u'6',
u'birthday': u'1986-08-13',
u'books': u"Orwell (1984, La ferme des animaux)<br>Barjavel (La nuit des temps, Le voyageur imprudent, Ravage, \x85)<br>Boris Vian (J'irai cracher sur vos tombes, L'&eacute;cume des jours\x85)<br>Bukowski, Desproges, San Antonio<br>Sartre, Le Canard",
u'cat': u'1',
u'checks1': u'0',
u'checks2': u'16686',
u'checks3': u'0',
u'checks4': u'0',
u'checks5': u'0',
u'checks6': u'2',
u'checks7': u'0',
u'cinema': u'Le Grand D&eacute;tournement \x97 La Classe Am&eacute;ricaine<br>V pour Vendetta, Pulp Fiction, The Truman Show<br>Eternal Sunshine of the Spotless Mind, Match Point<br>Idiocracy, The Big Lebowski, La cit&eacute; de la peur<br>Sin City, Orange Mecanique, Buffet Froid, L',
u'city': u'Paris',
u'country': u'fr',
u'cover': u'5',
u'drink': u'2',
u'email': u'tesiruna@parano.me',
u'eyes': u'3',
u'f': u'',
u'first_cnx': u'2010-05-16 09:13:53',
u'first_ip': u'81.57.125.104',
u'food': u'1',
u'godfather': u'0',
u'hair_color': u'5',
u'hair_size': u'3',
u'hobbies': u'',
u'id': u'22450639',
u'img_count': u'5',
u'isBan': False,
u'isOnline': True,
u'job': u'Dieu',
u'last_chat': u'-0001-11-29 23:09:21',
u'last_cnx': u'2011-09-20 19:24:25',
u'last_ip': u'88.161.27.232',
u'lat': u'48.861961',
u'latR': u'0.852802098431',
u'list1': u'2',
u'list2': u'2',
u'list3': u'2',
u'list4': u'3',
u'list5': u'0',
u'list6': u'0',
u'lng': u'2.33594',
u'lngR': u'0.040769844129',
u'login': u'@22450639',
u'mod_level': u'1',
u'music': u"Pink Floyd, Scorpions, Emperor<br>Metallica, Iron Maiden, Accept<br>Slash's Snakepit, Queen, Deep Purple<br>Led Zeppelin, Rolling Stones<br>Brassens, Souchon, Brel, Vian",
u'origins': u'1',
u'pass': u'8f3fa83cec9a243ae53c1337d2b5e1cf',
u'path': u'9/3/6/0/5/4/2/',
u'phone': u'-',
u'pictures': [{u'file': u'5',
u'height': u'427',
u'id': u'7315391',
u'md5': u'859fcd2e425617c33c16d6a1bc510ad3',
u'member': u'22450639',
u'rank': u'1',
u'valid': u'1578737',
u'width': u'500'}],
u'pseudo': u'Nazification',
u'region': u'11',
u'sex': 0,
u'shape': u'1',
u'shard': 9,
u'size': u'175',
u'smoke': u'2',
u'style': u'0',
u'subregion': u'76',
u'table': u'adopteun.boys',
u'texts1': u'',
u'texts2': u'',
u'texts3': u'',
u'texts4': u'',
u'texts5': u'',
u'texts6': u'',
u'title': u'',
u'tvs': u'Je ne poss&egrave;de pas la TV<br><br><br><br>',
u'url': u'/api.php?member/view/22450639/Nazification',
u'validated': True,
u'visites': u'0',
u'w': u'',
u'warn': u'0',
u'weight': u'55',
u'zip': u'75000'},
u'news': {u'newBaskets': 3, u'newMails': 1, u'newVisits': 113},
u'popu': u'71540',
u'subMobile': False,
u'subWebsite': False,
u'token': u'ea7bac8837f6e5bb1e2a3267d6a08b32e388d593',
u'visites': u'958'}}
me.[default]
------------
Return value:
{u'errors': [],
u'result': {u'events': {u'lastBasket': {u'alert': u'1',
u'birthday': u'1989-02-04',
u'city': u'Paris',
u'country': u'fr',
u'cover': u'1',
u'id': u'14471939',
u'isBan': False,
u'isOnline': False,
u'last_cnx': u'2011-09-20 13:56:36',
u'list5': u'0',
u'login': u'kloo',
u'mod_level': u'0',
u'path': u'9/3/9/1/7/4/4/',
u'pseudo': u'Kloolloo',
u'region': u'11',
u'sex': 1,
u'shard': 9,
u'style': u'0',
u'table': u'adopteun.girls',
u'url': u'/api.php?member/view/14471939/Kloolloo',
u'zip': u'75001'},
u'lastChat': {u'alert': u'1',
u'birthday': u'1987-10-10',
u'city': u'Paris 18e Arrondissement',
u'country': u'fr',
u'cover': u'0',
u'id': u'13280274',
u'isBan': False,
u'isOnline': False,
u'last_cnx': u'2010-11-20 03:37:00',
u'list5': u'0',
u'login': u'#c1a63bda81cc03ccdf080ca6e003919e',
u'mod_level': u'0',
u'path': u'4/7/2/0/8/2/3/',
u'pseudo': u'Katie Lee',
u'region': u'11',
u'sex': 1,
u'shard': 4,
u'style': u'0',
u'table': u'adopteun.girls_coma',
u'url': u'/api.php?member/view/13280274/KatieLee',
u'zip': u'75018'},
u'lastFlash': {u'alert': u'1',
u'birthday': u'1983-12-20',
u'city': u'Longjumeau',
u'country': u'fr',
u'cover': u'1',
u'id': u'13883475',
u'isBan': False,
u'isOnline': True,
u'last_cnx': u'2011-09-20 18:52:13',
u'list5': u'0',
u'login': u'@',
u'mod_level': u'0',
u'path': u'5/7/4/3/8/8/3/',
u'pseudo': u'Pas Tjs Sage',
u'region': u'11',
u'sex': 1,
u'shard': 5,
u'style': u'2',
u'table': u'adopteun.girls',
u'url': u'/api.php?member/view/13883475/PasTjsSage',
u'zip': u'91160'},
u'lastMail': {u'alert': u'1',
u'birthday': u'1989-04-28',
u'city': u'Paris 8e Arrondissement',
u'country': u'fr',
u'cover': u'5',
u'id': u'13268738',
u'isBan': False,
u'isOnline': True,
u'last_cnx': u'2011-09-20 19:47:28',
u'list5': u'0',
u'login': u'@13268738',
u'mod_level': u'0',
u'path': u'8/3/7/8/6/2/3/',
u'pseudo': u'Th\xe9na',
u'region': u'11',
u'sex': 1,
u'shard': 8,
u'style': u'0',
u'table': u'adopteun.girls',
u'url': u'/api.php?member/view/13268738/Thna',
u'zip': u'75008'},
u'lastVisit': {u'alert': u'1',
u'birthday': u'1988-02-27',
u'city': u'Paris',
u'country': u'fr',
u'cover': u'1',
u'id': u'14477637',
u'isBan': False,
u'isOnline': False,
u'last_cnx': u'2011-09-20 19:09:00',
u'list5': u'0',
u'login': u'@',
u'mod_level': u'0',
u'path': u'7/3/6/7/7/4/4/',
u'pseudo': u'Lolly',
u'region': u'11',
u'sex': 1,
u'shard': 7,
u'style': u'4',
u'table': u'adopteun.girls',
u'url': u'/api.php?member/view/14477637/Lolly',
u'zip': u'75005'}},
u'news': {u'newBaskets': 0, u'newMails': 1, u'newVisits': 113},
u'token': u'9a97a03774c9f440e676c78f48794a7221a67285'}}
me.basket
----------
Return value:
{u'errors': [],
u'popu': 71840,
u'result': {u'basket': [{u'alert': u'1',
u'birthday': u'1989-02-04',
u'city': u'Paris',
u'country': u'fr',
u'cover': u'1',
u'date': u'2011-09-19 01:52:29',
u'id': u'14471939',
u'isBan': False,
u'isOnline': False,
u'last_cnx': u'2011-09-20 13:56:36',
u'list5': u'0',
u'login': u'kloo',
u'mod_level': u'0',
u'path': u'9/3/9/1/7/4/4/',
u'pseudo': u'Kloolloo',
u'region': u'11',
u'sex': 1,
u'shard': 9,
u'style': u'0',
u'table': u'adopteun.girls',
u'url': u'/api.php?member/view/14471939/Kloolloo',
u'zip': u'75001'}],
u'inBasket': 57,
u'token': u'ea7bac8837f6e5bb1e2a3267d6a08b32e388d593'}}
me.flashs
---------
Return value:
{u'errors': [],
u'result': {u'all': [{u'date': u'2011-10-19 10:50:43',
u'fid': u'41311269',
u'id': u'12656592',
u'member': {u'alert': u'1',
u'birthday': u'1986-08-08',
u'city': u'Armes',
u'country': u'fr',
u'cover': u'3',
u'id': u'12656592',
u'isBan': False,
u'isOnline': False,
u'last_cnx': u'2011-10-19 17:28:00',
u'list5': u'0',
u'login': u'@12656592',
u'mod_level': u'0',
u'path': u'2/9/5/6/5/6/2/',
u'pseudo': u'Yayanne',
u'region': u'5',
u'sex': 1,
u'shard': 2,
u'style': u'0',
u'table': u'adopteun.girls',
u'url': u'/api.php?member/view/12656592/Yayanne',
u'zip': u'58500'},
u'seen': u'2011-10-19 10:54:09'}],
u'count': 1467,
u'news': [],
u'olds': [],
u'popu': u'110350',
u'token': u'3af90dd563431fe6b4c65930d18337c997fac34e'}}
me.visits
---------
Return value:
{u'errors': [],
u'result': {u'count': u'607',
u'news': [],
u'offset': 0,
u'olds': [{u'alert': u'0',
u'birthday': u'1985-01-29',
u'city': u'Albi',
u'country': u'fr',
u'cover': u'11',
u'date': u'2011-10-19 17:40:43',
u'id': u'13461054',
u'isBan': False,
u'isOnline': True,
u'last_cnx': u'2011-10-19 17:47:25',
u'list5': u'0',
u'login': u'@13461054',
u'mod_level': u'0',
u'path': u'4/5/0/1/6/4/3/',
u'pseudo': u'Bruume',
u'region': u'15',
u'seen': u'2011-10-19 17:42:55',
u'sex': 1,
u'shard': 4,
u'style': u'0',
u'table': u'adopteun.girls',
u'url': u'/api.php?member/view/13461054/Bruume',
u'vid': u'199226074',
u'zip': u'81000'}],
u'popu': u'110350',
u'token': u'd7380871794008c94971acec82b7b679dd800307'}}
MESSAGE Commands
================
message.[default]
-----------------
Arguments:
- P=<NB_ENTRIES>,<START>
Return value:
{u'errors': [],
u'result': {u'count': 1,
u'threads': [{u'cat': u'0',
u'date': u'2011-09-20 19:22:11',
u'id': u'11132125',
u'id_from': u'13268738',
u'id_to': u'22450639',
u'member': {u'alert': u'1',
u'birthday': u'1989-04-28',
u'city': u'Paris 8e Arrondissement',
u'country': u'fr',
u'cover': u'5',
u'id': u'13268738',
u'isBan': False,
u'isOnline': True,
u'last_cnx': u'2011-09-20 19:54:15',
u'list5': u'0',
u'login': u'@13268738',
u'mod_level': u'0',
u'path': u'8/3/7/8/6/2/3/',
u'pseudo': u'Th\xe9na',
u'region': u'11',
u'sex': 1,
u'shard': 8,
u'style': u'0',
u'table': u'adopteun.girls',
u'url': u'/api.php?member/view/13268738/Thna',
u'zip': u'75008'},
u'message': u'0',
u'status': u'0',
u'title': u"J'aime bien les \xe9l\xe9phants. Toi ?"}],
u'token': u'0f70c31bc6d05d45bee64e6f2eab9b537640f2f8'}}
message.thread
--------------
Parameters:
- memberId
- count
Return value:
{u'result': {u'popu': u'15910',
u'thread': {u'isNew': False,
u'member': {u'alert': u'3',
u'birthday': u'1987-04-22',
u'city': u'Maisons-Alfort',
u'country': u'fr',
u'cover': u'6',
u'id': u'14022243',
u'isBan': False,
u'isOnline': False,
u'last_cnx': u'2011-09-20 18:18:00',
u'list5': u'0',
u'login': u'@',
u'mod_level': u'0',
u'path': u'3/4/2/2/2/0/4/',
u'pseudo': u'Sophkipeut',
u'region': u'11',
u'sex': 1,
u'shard': 3,
u'style': u'0',
u'table': u'adopteun.girls',
u'url': u'/api.php?member/view/14022243/Sophkipeut',
u'zip': u'94700'},
u'messages': [{u'date': u'2011-09-19 17:52:03',
u'id': u'46583458',
u'id_from': u'14022243',
u'id_to': u'23185402',
u'message': u"Lol, moi j'ai fait attention, mais bon &ccedil;a n'emp&ecirc;che pas son bidou, ceci dit c'est planqu&eacute; par ses loooooooooooong poils ^^",
u'src': u'',
u'title': u"Lol, moi j'ai fait attention, mais bon \xe7a n'emp\xea..."},
{u'date': u'2011-09-19 17:49:39',
u'id': u'47467748',
u'id_from': u'23185402',
u'id_to': u'14022243',
u'message': u"Ah oui, justement le v&eacute;t&eacute;rinaire m'avait dit apr&egrave;s la castration de Futex qu'il fallait faire attention &agrave; son poids, du coup &ccedil;a m'a tellement vex&eacute; que j'ai fais attention au point qu'il est sans doute m&ecirc;me trop maigre.",
u'src': u'',
u'title': u"Ah oui, justement le v\xe9t\xe9rinaire m'avait dit apr\xe8..."}],
u'remoteStatus': u'2',
u'status': u'1',
u'warning': 0},
u'token': u'dbeccf96256d4f11991626707881fdba28f54d73'}}
message.new
-----------
Parameters:
- memberId
- message
Return value:
{u'errors': [],
u'result': {u'thread': {u'isNew': False,
u'member': {u'alert': u'1',
u'birthday': u'1986-12-08',
u'city': u'Rosny-sous-Bois',
u'country': u'fr',
u'cover': u'6',
u'id': u'11099536',
u'isBan': False,
u'isOnline': False,
u'last_cnx': u'2011-09-20 16:26:32',
u'list5': u'0',
u'login': u'@11099536',
u'mod_level': u'0',
u'path': u'6/3/5/9/9/0/1/',
u'pseudo': u'Debo',
u'region': u'11',
u'sex': 1,
u'shard': 6,
u'style': u'0',
u'table': u'adopteun.girls',
u'url': u'/api.php?member/view/11099536/Debo',
u'zip': u'93110'},
u'messages': [{u'date': u'2011-09-20 20:09:07',
u'id': u'46588573',
u'id_from': u'23185402',
u'id_to': u'11099536',
u'message': u'Coucou',
u'src': u'iphone',
u'title': u'Coucou'},
{u'date': u'2011-09-20 16:27:34',
u'id': u'46638469',
u'id_from': u'11099536',
u'id_to': u'23185402',
u'message': u"Coucou pour tout t'avouer je ne m'y etais pas connect&eacute; depuis septembre ! un peu moins de boulot alors j'y traine !!\r\n\r\nEt toi &ccedil;a va? tu devais pas partir au canada? tu es deja revenu.?",
u'src': u'',
u'title': u"Coucou pour tout t'avouer je ne m'y etais pas co..."}],
u'remoteStatus': u'0',
u'status': u'2',
u'warning': 0},
u'token': u'2491f8ccb6741ceee2a461dc523939796904a0fa'}}
message.delete
--------------
Parameters:
- id_user
Return Value:
Unknown
MEMBER Commands
===============
member.view
-----------
Parameters:
- id
Return value:
{u'errors': [],
u'result': {u'member': {u'about1': u"comment te dire.. j'ai autant envie te laisser boire dans ma bouteille que mettre ma langue dans ta bouche !",
u'about2': u"LES BISOUS C'EST BIEN LES BIJOUX C'EST MIEUX!\r<br>\r<br>Et l'humour encore plus, mouhaha\r<br>\r<br>PS: Si vous &ecirc;tes le sosie de Pharell Williams, adoptez moi ;)",
u'admin': u'0',
u'alert': u'1',
u'alert_add': u'0',
u'birthday': u'1991-05-31',
u'books': u"L'Analphab&egrave;te - Rendell<br>Etat limite - Assouline<br>Marie-Antoinette - Zweig<br><br>",
u'cat': u'2',
u'checks1': u'2',
u'checks2': u'1588',
u'checks3': u'0',
u'checks4': u'0',
u'checks5': u'64',
u'checks6': u'192',
u'checks7': u'0',
u'cinema': u"My Blueberry Night<br>Shinning - L'exorciste - Ester - Gothika<br>How High ! - Requiem for a dream<br>Remember Me - trainspotting<br>He gots game - Buffet froid",
u'city': u'Paris',
u'country': u'fr',
u'cover': u'1',
u'drink': u'2',
u'eyes': u'5',
u'f': u'',
u'first_cnx': u'2011-09-17 00:18:59',
u'first_ip': u'82.120.134.233',
u'food': u'3',
u'godfather': u'0',
u'hair_color': u'4',
u'hair_size': u'3',
u'hobbies': u'Le th&eacute;&acirc;tre d&eacute;finitivement ! et le sport !',
u'id': u'14465370',
u'img_count': u'6',
u'isBan': False,
u'isOnline': False,
u'job': u'etudiante',
u'last_chat': u'0000-00-00 00:00:00',
u'last_cnx': u'2011-09-24 00:09:29',
u'last_ip': u'92.151.177.164',
u'lat': u'48.8814',
u'latR': u'0.853141372984',
u'list1': u'0',
u'list2': u'42',
u'list3': u'0',
u'list4': u'0',
u'list5': u'0',
u'list6': u'0',
u'lng': u'2.3365',
u'lngR': u'0.0407796179728',
u'login': u'@',
u'mailable': True,
u'mod_level': u'0',
u'music': u'Daft Punk - Bloody Beetrots - Kid Cudi - Jamiroquai<br>Red Hot - Ray Charles - BEP - Gorillaz - Birdy nam nam<br>Citizen Cope - Angus &amp; Julia Stone - Portishead<br>50cent - Sia - Ben Harper - Busta Rhymes<br>Musiques de gansta ! ET Debussy',
u'origins': u'1',
u'path': u'0/7/3/5/6/4/4/',
u'phone': u'-',
u'popu': {u'bonus': u'6',
u'contacts': u'1',
u'flashs': u'246',
u'id': u'14465370',
u'invits': u'0',
u'mails': u'39',
u'popu': u'11345',
u'visites': u'645'},
u'pseudo': u'Ruslana',
u'region': u'11',
u'sex': 1,
u'shape': u'1',
u'shard': 0,
u'size': u'170',
u'smoke': u'2',
u'style': u'4',
u'subregion': u'76',
u'table': u'adopteun.girls',
u'texts1': u'',
u'texts2': u'',
u'texts3': u'',
u'texts4': u'',
u'texts5': u'',
u'texts6': u'MON SOURIRE HAHAHA',
u'title': u'',
u'tvs': u'Dexter<br>OC<br>True blood<br>Envoy&eacute; sp&eacute;cial ;) - Arte !<br>',
u'url': u'/api.php?member/view/14465370/Ruslana',
u'visites': u'0',
u'w': u'',
u'warn': u'0',
u'weight': u'50',
u'zip': u'75008'},
u'token': u'e0247704012e01bc32756b357b010e5206ac9c76'}}
member.pictures
---------------
Parameters:
- id
Return value:
{u'errors': [],
u'result': {u'pictures': [{u'id': u'12363004',
u'rating': 4.4473684210500002,
u'url': u'http://s0.adopteunmec.com/0/6/0/1/6/image7.jpg'}],
u'token': u'e131f1b194f2a19337882398b10b79457a638252'}}
member.addBasket
----------------
Parameters:
- id
Errors:
- 5.1.1 : member does not exist
- 5.1.5 : already sent charm to this one
- 5.1.6 : no enough charms available
Return value:
{u'errors': u'0',
u'flashs': 4,
u'result': {u'token': u'55039d0557393bb7c5e4381792143d003f0e60c0'}}
SEARH Commands
==============
search.[default]
----------------
Return value:
{u'errors': [],
u'result': {u'qsearch': {u'ageMax': u'25',
u'ageMin': u'18',
u'dist': u'0',
u'new': u'0',
u'query': u'{"sex":1,"ageMin":"18","ageMax":"25","region":"fr","new":"0","dist":"0"}',
u'region': u'fr',
u'sex': 1},
u'search': {u'ageMax': u'27',
u'ageMin': u'20',
u'checks1': u'0',
u'checks2': u'0',
u'country': u'fr',
u'dist': u'50',
u'drink': u'0',
u'eyes': u'0',
u'food': u'0',
u'hair_color': u'0',
u'hair_size': u'0',
u'origins': u'0',
u'pseudo': u'',
u'query': u'{"ageMin":"20","ageMax":"27","country":"fr","region":"11","subregion":"0","dist":"50","pseudo":"","sex":"1","sizeMin":"0","sizeMax":"0","weightMin":"0","weightMax":"75","shape":"0","hair_size":"0","hair_color":"0","eyes":"0","origins":"0","style":"0","checks1":"0","checks2":"0","smoke":"0","drink":"0","food":"0","search":"true"}',
u'region': u'11',
u'search': u'true',
u'sex': u'1',
u'shape': u'0',
u'sizeMax': u'0',
u'sizeMin': u'0',
u'smoke': u'0',
u'style': u'0',
u'subregion': u'0',
u'weightMax': u'75',
u'weightMin': u'0'},
u'token': u'3196a3365d927f2ee8738ec8dfc4a5abd75e3ee3'}}
search.quick
------------
Parameters:
- sex (int[0,1])
- ageMin (int)
- ageMax (int)
- region (str)
- new (int)
- dist (int)
Return Value:
{u'errors': [],
u'result': {u'regions': {u'be': None,
u'be_24': u' - wallonie',
u'be_25': u' - bruxelles capitale',
u'be_26': u' - flandre',
u'ca': None,
u'ca_34': u' - canada',
u'ca_35': u' - quebec',
u'ch': None,
u'ch_27': u' - r\xe9gion l\xe9manique',
u'ch_28': u' - espace Mittelland',
u'ch_29': u' - suisse du nord-ouest',
u'ch_30': u' - zurich',
u'ch_31': u' - suisse orientale',
u'ch_32': u' - suisse centrale',
u'ch_33': u' - tessin',
u'fr': None,
u'fr_1': u' - alsace',
u'fr_10': u' - haute-normandie',
u'fr_11': u' - ile-de-france',
u'fr_12': u' - languedoc-roussillon',
u'fr_13': u' - limousin',
u'fr_14': u' - lorraine',
u'fr_15': u' - midi-pyr\xe9n\xe9es',
u'fr_16': u' - nord-pas-de-calais',
u'fr_17': u' - paca',
u'fr_18': u' - pays de la loire',
u'fr_19': u' - picardie',
u'fr_2': u' - aquitaine',
u'fr_20': u' - poitou-charentes',
u'fr_21': u' - rh\xf4ne-alpes',
u'fr_22': u' - corse',
u'fr_23': u' - dom+tom',
u'fr_3': u' - auvergne',
u'fr_4': u' - basse-normandie',
u'fr_5': u' - bourgogne',
u'fr_6': u' - bretagne',
u'fr_7': u' - centre',
u'fr_8': u' - champagne-ardenne',
u'fr_9': u' - franche-comt\xe9'},
u'search': [{u'alert': u'2',
u'birthday': u'1985-11-21',
u'city': u'Boulogne-Billancourt',
u'country': u'fr',
u'cover': u'12',
u'id': u'14252744',
u'isBan': False,
u'isOnline': True,
u'last_cnx': u'2011-09-24 15:19:48',
u'list5': u'0',
u'login': u'@',
u'mod_level': u'0',
u'path': u'4/4/7/2/5/2/4/',
u'pseudo': u'Birdy',
u'region': u'11',
u'sex': 1,
u'shard': 4,
u'style': u'5',
u'table': u'adopteun.girls',
u'url': u'/api.php?member/view/14252744/Birdy',
u'zip': u'92100'}],
u'token': u'ef78ac6d812ab15f2f8efd578f4da4ef2e23aa71'}}
search.advanced
---------------
Parameters:
- ageMin (int)
- ageMax (int)
- country (str)
- region (int)
- subregion (int)
- dist (int)
- pseudo (str)
- sex (int[0,1])
- sizeMin (int)
- sizeMax (int)
- weightMin (int)
- weightMax (int)
- shape (int)
- hair_size (int)
- hair_color (int)
- eyes (int)
- origins (int)
- style (int)
- checks1 (int)
- checks2 (int)
- smoke (int)
- drink (int)
- food (int)
- search (bool)
Return Value:
{u'errors': [],
u'regions': {u'be': None,
u'be_24': u' - wallonie',
u'be_25': u' - bruxelles capitale',
u'be_26': u' - flandre',
u'ca': None,
u'ca_34': u' - canada',
u'ca_35': u' - quebec',
u'ch': None,
u'ch_27': u' - r\xe9gion l\xe9manique',
u'ch_28': u' - espace Mittelland',
u'ch_29': u' - suisse du nord-ouest',
u'ch_30': u' - zurich',
u'ch_31': u' - suisse orientale',
u'ch_32': u' - suisse centrale',
u'ch_33': u' - tessin',
u'fr': None,
u'fr_1': u' - alsace',
u'fr_10': u' - haute-normandie',
u'fr_11': u' - ile-de-france',
u'fr_12': u' - languedoc-roussillon',
u'fr_13': u' - limousin',
u'fr_14': u' - lorraine',
u'fr_15': u' - midi-pyr\xe9n\xe9es',
u'fr_16': u' - nord-pas-de-calais',
u'fr_17': u' - paca',
u'fr_18': u' - pays de la loire',
u'fr_19': u' - picardie',
u'fr_2': u' - aquitaine',
u'fr_20': u' - poitou-charentes',
u'fr_21': u' - rh\xf4ne-alpes',
u'fr_22': u' - corse',
u'fr_23': u' - dom+tom',
u'fr_3': u' - auvergne',
u'fr_4': u' - basse-normandie',
u'fr_5': u' - bourgogne',
u'fr_6': u' - bretagne',
u'fr_7': u' - centre',
u'fr_8': u' - champagne-ardenne',
u'fr_9': u' - franche-comt\xe9'},
u'result': {u'search': [{u'alert': u'1',
u'birthday': u'1988-04-07',
u'city': u'Dammartin',
u'country': u'fr',
u'cover': u'25',
u'id': u'13579115',
u'isBan': False,
u'isOnline': True,
u'last_cnx': u'2011-09-24 15:17:22',
u'list5': u'0',
u'login': u'@',
u'mod_level': u'0',
u'path': u'5/1/1/9/7/5/3/',
u'pseudo': u"S\xe9 's\xe9",
u'region': u'11',
u'sex': 1,
u'shard': 5,
u'style': u'1',
u'table': u'adopteun.girls',
u'url': u'/api.php?member/view/13579115/Ss',
u'zip': u'77230'}],
u'token': u'f92aede46118dcdba6d484146b4627777fbe7188'}}

24
modules/aum/__init__.py Normal file
View file

@ -0,0 +1,24 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from .browser import AuMBrowser
from .backend import AuMBackend
__all__ = ['AuMBrowser', 'AuMBackend']

135
modules/aum/antispam.py Normal file
View file

@ -0,0 +1,135 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
import re
__all__ = ['AntiSpam']
class AntiSpam(object):
def check_thread(self, thread):
resume = thread['title']
# Check if there is an email address in the offer.
if re.match('^[\w\d\.\-_]+@[\w\d\.]+ vous offre la pos', resume):
return False
if thread['member']['pseudo'] == 'Ekaterina':
return False
return True
def check_profile(self, profile):
# The name of profile is in form #123456789
if profile['pseudo'] == '':
return False
if profile['about1'].startswith('salut! je te donne mon msn'):
return False
if profile['about2'].startswith('cam to cam'):
return False
if profile['about2'].startswith('je suis une femme tres tres belle et je recherche un homme qui aime le sexe'):
return False
if profile['about2'].endswith('mmmmmmmmmmmmmmmm'):
return False
return True
# ipaddr is not available anymore.
for ipaddr in (profile['last_ip'], profile['first_ip']):
if ipaddr.startswith('41.202.'):
return False
if ipaddr.startswith('41.250.'):
return False
if ipaddr.startswith('41.251.'):
return False
if ipaddr.startswith('41.141.'):
return False
if ipaddr.startswith('194.177.'):
return False
if ipaddr.startswith('41.85.'):
return False
if ipaddr.startswith('41.86.'):
return False
if ipaddr.startswith('196.47.'):
return False
if re.match('105\.13\d.*', ipaddr):
return False
if ipaddr in ('62.157.186.18', '198.36.222.8', '212.234.67.61', '203.193.158.210', '41.189.34.180', '41.66.12.36', '196.47.137.21', '213.136.125.122', '41.191.87.188'):
return False
return True
def check_contact(self, contact):
if contact.id == 1:
return True
if not self.check_profile(contact.aum_profile):
return False
return True
# ipaddr is not available anymore.
first_ip = contact.profile['info']['IPaddr'].value.split(' ')[0]
last_ip = contact.profile['info']['IPaddr'].value.rstrip(')')
for ipaddr in (first_ip, last_ip):
if ipaddr.endswith('.afnet.net'):
return False
if ipaddr.endswith('.iam.net.ma'):
return False
if ipaddr.endswith('.amsterdam.ananoos.net'):
return False
if ipaddr.endswith('.tedata.net'):
return False
if ipaddr.endswith('kupo.fr'):
return False
if ipaddr.endswith('.static.virginmedia.com'):
return False
if ipaddr.endswith('frozenway.com'):
return False
if ipaddr.endswith('.rev.bgtn.net'):
return False
if ipaddr.endswith('real-vpn.com'):
return False
if ipaddr.endswith('.nl.ipodah.net'):
return False
if ipaddr.endswith('.wanamaroc.com'):
return False
if ipaddr.endswith('.ukservers.com'):
return False
if ipaddr.endswith('.startdedicated.com'):
return False
if ipaddr.endswith('.clients.your-server.de'):
return False
if ipaddr.endswith('.cba.embratel.net.br'):
return False
if ipaddr.endswith('.idstelcom.com'):
return False
if ipaddr.endswith('proxy.chg-support.com'):
return False
if ipaddr.endswith('.sprintsvc.net'):
return False
if ipaddr.endswith('.relakks.com'):
return False
return True
def check_mail(self, mail):
# Spambot with a long first-message.
if mail['message'].find('Je veux que vous m\'ayez ecrit directement sur le mon e-mail') >= 0:
return False
if mail['message'].find('ilusa12010@live.fr') >= 0:
return False
return True

555
modules/aum/backend.py Normal file
View file

@ -0,0 +1,555 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from __future__ import with_statement
import email
import time
import re
import datetime
from html2text import unescape
from dateutil import tz
from dateutil.parser import parse as _parse_dt
from weboob.capabilities.base import NotLoaded
from weboob.capabilities.chat import ICapChat
from weboob.capabilities.messages import ICapMessages, ICapMessagesPost, Message, Thread
from weboob.capabilities.dating import ICapDating, OptimizationNotFound, Event
from weboob.capabilities.contact import ICapContact, ContactPhoto, Query, QueryError
from weboob.capabilities.account import ICapAccount, StatusField
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.tools.browser import BrowserUnavailable
from weboob.tools.value import Value, ValuesDict, ValueBool, ValueBackendPassword
from weboob.tools.log import getLogger
from weboob.tools.misc import local2utc
from .contact import Contact
from .captcha import CaptchaError
from .antispam import AntiSpam
from .browser import AuMBrowser
from .optim.profiles_walker import ProfilesWalker
from .optim.visibility import Visibility
from .optim.queries_queue import QueriesQueue
__all__ = ['AuMBackend']
def parse_dt(s):
d = _parse_dt(s)
return local2utc(d)
class AuMBackend(BaseBackend, ICapMessages, ICapMessagesPost, ICapDating, ICapChat, ICapContact, ICapAccount):
NAME = 'aum'
MAINTAINER = 'Romain Bignon'
EMAIL = 'romain@weboob.org'
VERSION = '0.a'
LICENSE = 'AGPLv3+'
DESCRIPTION = u"“Adopte un mec” french dating website"
CONFIG = BackendConfig(Value('username', label='Username'),
ValueBackendPassword('password', label='Password'),
ValueBool('antispam', label='Enable anti-spam', default=False),
ValueBool('baskets', label='Get baskets with new messages', default=True))
STORAGE = {'profiles_walker': {'viewed': []},
'queries_queue': {'queue': []},
'sluts': {},
'notes': {},
}
BROWSER = AuMBrowser
MAGIC_ID_BASKET = 1
def __init__(self, *args, **kwargs):
BaseBackend.__init__(self, *args, **kwargs)
if self.config['antispam'].get():
self.antispam = AntiSpam()
else:
self.antispam = None
def create_default_browser(self):
return self.create_browser(self.config['username'].get(), self.config['password'].get())
def report_spam(self, id):
with self.browser:
self.browser.delete_thread(id)
# Do not report fakes to website, to let them to other guys :)
#self.browser.report_fake(id)
# ---- ICapDating methods ---------------------
def init_optimizations(self):
self.add_optimization('PROFILE_WALKER', ProfilesWalker(self.weboob.scheduler, self.storage, self.browser))
self.add_optimization('VISIBILITY', Visibility(self.weboob.scheduler, self.browser))
self.add_optimization('QUERIES_QUEUE', QueriesQueue(self.weboob.scheduler, self.storage, self.browser))
def iter_events(self):
all_events = {}
with self.browser:
all_events['baskets'] = (self.browser.get_baskets, 'You were put into %s\'s basket')
all_events['flashs'] = (self.browser.get_flashs, 'You sent a charm to %s')
all_events['visits'] = (self.browser.get_visits, 'Visited by %s')
for type, (events, message) in all_events.iteritems():
for event in events():
try:
e = Event(event['%sid' % type[0]])
except KeyError:
e = Event(event['id'])
e.date = parse_dt(event['date'])
e.type = type
if 'member' in event:
e.contact = self._get_partial_contact(event['member'])
else:
e.contact = self._get_partial_contact(event)
if not e.contact:
continue
e.message = message % e.contact.name
yield e
# ---- ICapMessages methods ---------------------
def fill_thread(self, thread, fields):
return self.get_thread(thread)
def iter_threads(self):
with self.browser:
threads = self.browser.get_threads_list()
for thread in threads:
if thread['member'].get('isBan', thread['member'].get('dead', False)):
with self.browser:
self.browser.delete_thread(thread['member']['id'])
continue
if self.antispam and not self.antispam.check_thread(thread):
self.logger.info('Skipped a spam-thread from %s' % thread['pseudo'])
self.report_spam(thread['member']['id'])
continue
t = Thread(int(thread['member']['id']))
t.flags = Thread.IS_DISCUSSION
t.title = 'Discussion with %s' % thread['member']['pseudo']
yield t
def get_thread(self, id, contacts=None, get_profiles=False):
"""
Get a thread and its messages.
The 'contacts' parameters is only used for internal calls.
"""
thread = None
if isinstance(id, Thread):
thread = id
id = thread.id
if not thread:
thread = Thread(int(id))
thread.flags = Thread.IS_DISCUSSION
full = False
else:
full = True
with self.browser:
mails = self.browser.get_thread_mails(id, 100)
my_name = self.browser.get_my_name()
child = None
msg = None
slut = self._get_slut(id)
if contacts is None:
contacts = {}
if not thread.title:
thread.title = u'Discussion with %s' % mails['member']['pseudo']
self.storage.set('sluts', thread.id, 'status', mails['status'])
self.storage.save()
for mail in mails['messages']:
flags = 0
if self.antispam and not self.antispam.check_mail(mail):
self.logger.info('Skipped a spam-mail from %s' % mails['member']['pseudo'])
self.report_spam(thread.id)
break
if parse_dt(mail['date']) > slut['lastmsg']:
flags |= Message.IS_UNREAD
if get_profiles:
if not mail['id_from'] in contacts:
with self.browser:
contacts[mail['id_from']] = self.get_contact(mail['id_from'])
if self.antispam and not self.antispam.check_contact(contacts[mail['id_from']]):
self.logger.info('Skipped a spam-mail-profile from %s' % mails['member']['pseudo'])
self.report_spam(thread.id)
break
if int(mail['id_from']) == self.browser.my_id:
if int(mails['remoteStatus']) == 0 and msg is None:
flags |= Message.IS_NOT_ACCUSED
else:
flags |= Message.IS_ACCUSED
signature = u''
if mail.get('src', None):
signature += u'Sent from my %s\n\n' % mail['src']
if mail['id_from'] in contacts:
signature += contacts[mail['id_from']].get_text()
msg = Message(thread=thread,
id=int(time.strftime('%Y%m%d%H%M%S', parse_dt(mail['date']).timetuple())),
title=thread.title,
sender=my_name if int(mail['id_from']) == self.browser.my_id else mails['member']['pseudo'],
receivers=[my_name if int(mail['id_from']) != self.browser.my_id else mails['member']['pseudo']],
date=parse_dt(mail['date']),
content=unescape(mail['message']).strip(),
signature=signature,
children=[],
flags=flags)
if child:
msg.children.append(child)
child.parent = msg
child = msg
if full and msg:
# If we have get all the messages, replace NotLoaded with None as
# parent.
msg.parent = None
if not full and not msg:
# Perhaps there are hidden messages
msg = NotLoaded
thread.root = msg
return thread
def iter_unread_messages(self, thread=None):
try:
contacts = {}
with self.browser:
threads = self.browser.get_threads_list()
for thread in threads:
if thread['member'].get('isBan', thread['member'].get('dead', False)):
with self.browser:
self.browser.delete_thread(int(thread['member']['id']))
continue
if self.antispam and not self.antispam.check_thread(thread):
self.logger.info('Skipped a spam-unread-thread from %s' % thread['member']['pseudo'])
self.report_spam(thread['member']['id'])
continue
slut = self._get_slut(thread['member']['id'])
if parse_dt(thread['date']) > slut['lastmsg'] or int(thread['status']) != int(slut['status']):
t = self.get_thread(thread['member']['id'], contacts, get_profiles=True)
for m in t.iter_all_messages():
if m.flags & m.IS_UNREAD:
yield m
if not self.config['baskets'].get():
return
# Send mail when someone added me in her basket.
# XXX possibly race condition if a slut adds me in her basket
# between the aum.nb_new_baskets() and aum.get_baskets().
with self.browser:
slut = self._get_slut(-self.MAGIC_ID_BASKET)
new_baskets = self.browser.nb_new_baskets()
if new_baskets > 0:
baskets = self.browser.get_baskets()
my_name = self.browser.get_my_name()
for basket in baskets:
if basket['isBan'] or parse_dt(basket['date']) <= slut['lastmsg']:
continue
contact = self.get_contact(basket['id'])
if self.antispam and not self.antispam.check_contact(contact):
self.logger.info('Skipped a spam-basket from %s' % contact.name)
self.report_spam(basket['id'])
continue
thread = Thread(int(basket['id']))
thread.title = 'Basket of %s' % contact.name
thread.root = Message(thread=thread,
id=self.MAGIC_ID_BASKET,
title=thread.title,
sender=contact.name,
receivers=[my_name],
date=parse_dt(basket['date']),
content='You are taken in her basket!',
signature=contact.get_text(),
children=[],
flags=Message.IS_UNREAD)
yield thread.root
except BrowserUnavailable, e:
self.logger.debug('No messages, browser is unavailable: %s' % e)
pass # don't care about waiting
def set_message_read(self, message):
if message.id == self.MAGIC_ID_BASKET:
# Save the last baskets checks.
slut = self._get_slut(-self.MAGIC_ID_BASKET)
if slut['lastmsg'] < message.date:
slut['lastmsg'] = message.date
self.storage.set('sluts', -self.MAGIC_ID_BASKET, slut)
self.storage.save()
return
slut = self._get_slut(message.thread.id)
if slut['lastmsg'] < message.date:
slut['lastmsg'] = message.date
self.storage.set('sluts', message.thread.id, slut)
self.storage.save()
def _get_slut(self, id):
id = int(id)
sluts = self.storage.get('sluts')
if not sluts or not id in sluts:
slut = {'lastmsg': datetime.datetime(1970,1,1),
'status': 0}
else:
slut = self.storage.get('sluts', id)
slut['lastmsg'] = slut.get('lastmsg', datetime.datetime(1970,1,1)).replace(tzinfo=tz.tzutc())
slut['status'] = int(slut.get('status', 0))
return slut
# ---- ICapMessagesPost methods ---------------------
def post_message(self, message):
with self.browser:
self.browser.post_mail(message.thread.id, message.content)
# ---- ICapContact methods ---------------------
def fill_contact(self, contact, fields):
if 'profile' in fields:
contact = self.get_contact(contact)
if contact and 'photos' in fields:
for name, photo in contact.photos.iteritems():
with self.browser:
if photo.url and not photo.data:
data = self.browser.openurl(photo.url).read()
contact.set_photo(name, data=data)
if photo.thumbnail_url and not photo.thumbnail_data:
data = self.browser.openurl(photo.thumbnail_url).read()
contact.set_photo(name, thumbnail_data=data)
def fill_photo(self, photo, fields):
with self.browser:
if 'data' in fields and photo.url and not photo.data:
photo.data = self.browser.readurl(photo.url)
if 'thumbnail_data' in fields and photo.thumbnail_url and not photo.thumbnail_data:
photo.thumbnail_data = self.browser.readurl(photo.thumbnail_url)
return photo
def get_contact(self, contact):
with self.browser:
if isinstance(contact, Contact):
_id = contact.id
elif isinstance(contact, (int,long,basestring)):
_id = contact
else:
raise TypeError("The parameter 'contact' isn't a contact nor a int/long/str/unicode: %s" % contact)
profile = self.browser.get_profile(_id)
if not profile:
return None
_id = profile['id']
if isinstance(contact, Contact):
contact.id = _id
contact.name = profile['pseudo']
else:
contact = Contact(_id, profile['pseudo'], Contact.STATUS_ONLINE)
contact.url = self.browser.id2url(_id)
contact.parse_profile(profile, self.browser.get_consts())
return contact
def _get_partial_contact(self, contact):
if contact.get('isBan', contact.get('dead', False)):
with self.browser:
self.browser.delete_thread(int(contact['id']))
return None
s = 0
if contact.get('isOnline', False):
s = Contact.STATUS_ONLINE
else:
s = Contact.STATUS_OFFLINE
c = Contact(contact['id'], contact['pseudo'], s)
c.url = self.browser.id2url(contact['id'])
if 'birthday' in contact:
birthday = _parse_dt(contact['birthday'])
age = int((datetime.datetime.now() - birthday).days / 365.25)
c.status_msg = u'%s old, %s' % (age, contact['city'])
if contact['cover'].isdigit() and int(contact['cover']) > 0:
url = 'http://s%s.adopteunmec.com/%s%%(type)s%s.jpg' % (contact['shard'], contact['path'], contact['cover'])
else:
url = 'http://s.adopteunmec.com/www/img/thumb0.gif'
c.set_photo('image%s' % contact['cover'],
url=url % {'type': 'image'},
thumbnail_url=url % {'type': 'thumb0_'})
return c
def iter_contacts(self, status=Contact.STATUS_ALL, ids=None):
with self.browser:
threads = self.browser.get_threads_list(count=100)
for thread in threads:
c = self._get_partial_contact(thread['member'])
if c and (c.status & status) and (not ids or c.id in ids):
yield c
def send_query(self, id):
if isinstance(id, Contact):
id = id.id
queries_queue = None
try:
queries_queue = self.get_optimization('QUERIES_QUEUE')
except OptimizationNotFound:
pass
if queries_queue and queries_queue.is_running():
if queries_queue.enqueue_query(id):
return Query(id, 'A charm has been sent')
else:
return Query(id, 'Unable to send charm: it has been enqueued')
else:
with self.browser:
if not self.browser.send_charm(id):
raise QueryError('No enough charms available')
return Query(id, 'A charm has been sent')
def get_notes(self, id):
if isinstance(id, Contact):
id = id.id
return self.storage.get('notes', id)
def save_notes(self, id, notes):
if isinstance(id, Contact):
id = id.id
self.storage.set('notes', id, notes)
self.storage.save()
# ---- ICapChat methods ---------------------
def iter_chat_messages(self, _id=None):
with self.browser:
return self.browser.iter_chat_messages(_id)
def send_chat_message(self, _id, message):
with self.browser:
return self.browser.send_chat_message(_id, message)
#def start_chat_polling(self):
#self._profile_walker = ProfilesWalker(self.weboob.scheduler, self.storage, self.browser)
# ---- ICapAccount methods ---------------------
ACCOUNT_REGISTER_PROPERTIES = ValuesDict(
Value('username', label='Email address', regexp='^[^ ]+@[^ ]+\.[^ ]+$'),
Value('password', label='Password', regexp='^[^ ]+$', masked=True),
Value('sex', label='Sex', choices={'m': 'Male', 'f': 'Female'}),
Value('birthday', label='Birthday (dd/mm/yyyy)', regexp='^\d+/\d+/\d+$'),
Value('zipcode', label='Zipcode'),
Value('country', label='Country', choices={'fr': 'France', 'be': 'Belgique', 'ch': 'Suisse', 'ca': 'Canada'}, default='fr'),
Value('godfather',label='Godfather', regexp='^\d*$', default=''),
)
@classmethod
def register_account(klass, account):
"""
Register an account on website
This is a static method, it would be called even if the backend is
instancied.
@param account an Account object which describe the account to create
"""
browser = None
bday, bmonth, byear = account.properties['birthday'].get().split('/', 2)
while not browser:
try:
browser = klass.BROWSER(account.properties['username'].get())
browser.register(password= account.properties['password'].get(),
sex= (0 if account.properties['sex'].get() == 'm' else 1),
birthday_d= int(bday),
birthday_m= int(bmonth),
birthday_y= int(byear),
zipcode= account.properties['zipcode'].get(),
country= account.properties['country'].get(),
godfather= account.properties['godfather'].get())
except CaptchaError:
getLogger('aum').info('Unable to resolve captcha. Retrying...')
browser = None
REGISTER_REGEXP = re.compile('.*http://www.adopteunmec.com/register4.php\?([^\' ]*)\'')
def confirm_account(self, mail):
msg = email.message_from_string(mail)
content = u''
for part in msg.walk():
s = part.get_payload(decode=True)
content += unicode(s, 'iso-8859-15')
url = None
for s in content.split():
m = self.REGISTER_REGEXP.match(s)
if m:
url = '/register4.php?' + m.group(1)
break
if url:
browser = self.create_browser('')
browser.openurl(url)
return True
return False
def get_account(self):
"""
Get the current account.
"""
raise NotImplementedError()
def update_account(self, account):
"""
Update the current account.
"""
raise NotImplementedError()
def get_account_status(self):
with self.browser:
return (
StatusField('myname', 'My name', self.browser.get_my_name()),
StatusField('score', 'Score', self.browser.score()),
StatusField('avcharms', 'Available charms', self.browser.nb_available_charms()),
StatusField('godchilds', 'Number of godchilds', self.browser.nb_godchilds()),
)
OBJECTS = {Thread: fill_thread,
Contact: fill_contact,
ContactPhoto: fill_photo
}

419
modules/aum/browser.py Normal file
View file

@ -0,0 +1,419 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2008-2011 Romain Bignon, Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
import math
import re
import datetime
import random
import urllib
from htmlentitydefs import codepoint2name
try:
import json
except ImportError:
import simplejson as json
from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword, BrowserUnavailable, BrowserHTTPNotFound
from weboob.capabilities.chat import ChatException, ChatMessage
from weboob.capabilities.messages import CantSendMessage
__all__ = ['AuMBrowser']
class AuMException(Exception):
ERRORS = {"0.0.0": "Bad signature",
"0.0.1": "Malformed request",
"0.0.2": "Not logged",
"1.1.1": "No member has this login",
"1.1.2": "Password don't match",
"1.1.3": "User has been banned",
"1.12.1": "Invalid country",
"1.12.1": "Invalid region",
"4.0.1": "Member not found",
"4.1.1": "Thread doesn't exist",
"4.1.2": "Cannot write to this member",
"5.1.1": "Member tergeted doesn't exist",
"5.1.2": "Sex member targeted is not the opposite of the member logged",
"5.1.3": "Not possible to send a charm",
"5.1.4": "Not possible to send a charm because the 5 charms has been already used",
"5.1.5": "Not possible because the guy has already send a charm to this girl",
"5.1.6": "No more money",
"5.1.7": "Not possible to add to basket",
"5.2.1": "Member doesn't exist",
"5.3.1": "Member doesn't exist",
}
def __init__(self, code):
Exception.__init__(self, self.ERRORS.get(code, code))
self.code = code
class AuMBrowser(BaseBrowser):
DOMAIN = 'api.adopteunmec.com'
APIKEY = 'fb0123456789abcd'
consts = None
search_query = None
my_sex = 0
my_id = 0
my_name = u''
my_coords = (0,0)
def id2url(self, id):
return 'http://www.adopteunmec.com/index.php/profile/%s' % id
def url2id(func):
def inner(self, id, *args, **kwargs):
m = re.match('^http://.*adopteunmec.com.*/(\d+)$', str(id))
if m:
id = int(m.group(1))
else:
m = re.match('^http://.*adopteunmec.com/index.php/profile/(\d+).*', str(id))
if m:
id = int(m.group(1))
return func(self, id, *args, **kwargs)
return inner
def api_request(self, command, action, parameter='', data=None, nologin=False):
if data is None:
# Always do POST requests.
data = ''
elif isinstance(data, (list,tuple,dict)):
data = urllib.urlencode(data)
elif isinstance(data, unicode):
data = data.encode('utf-8')
url = self.buildurl(self.absurl('/api.php'), S=self.APIKEY,
C=command,
A=action,
P=parameter,
O='json')
buf = self.openurl(url, data)
try:
r = json.load(buf)
except ValueError:
buf.seek(0)
raise ValueError(buf.read())
if 'errors' in r and r['errors'] != '0' and len(r['errors']) > 0:
code = r['errors'][0]
if code in (u'0.0.2', u'1.1.1', u'1.1.2'):
if not nologin:
self.login()
return self.api_request(command, action, parameter, data, nologin=True)
else:
raise BrowserIncorrectPassword(AuMException.ERRORS[code])
else:
raise AuMException(code)
return r
def login(self):
r = self.api_request('me', 'login', data={'login': self.username,
'pass': self.password,
}, nologin=True)
self.my_sex = r['result']['me']['sex']
self.my_id = int(r['result']['me']['id'])
self.my_name = r['result']['me']['pseudo']
self.my_coords = (float(r['result']['me']['lat']), float(r['result']['me']['lng']))
return r
#def register(self, password, sex, birthday_d, birthday_m, birthday_y, zipcode, country, godfather=None):
# if not self.is_on_page(RegisterPage):
# self.location('http://www.adopteunmec.com/register2.php')
# self.page.register(password, sex, birthday_d, birthday_m, birthday_y, zipcode, country)
# if godfather:
# if not self.is_on_page(AccountPage):
# self.location('http://www.adopteunmec.com/account.php')
# self.page.set_godfather(godfather)
#@pageaccess
#def add_photo(self, name, f):
# if not self.is_on_page(EditPhotoPage):
# self.location('/edit.php?type=1')
# return self.page.add_photo(name, f)
#@pageaccess
#def set_nickname(self, nickname):
# if not self.is_on_page(EditAnnouncePage):
# self.location('/edit.php?type=2')
# return self.page.set_nickname(nickname)
#@pageaccess
#def set_announce(self, title=None, description=None, lookingfor=None):
# if not self.is_on_page(EditAnnouncePage):
# self.location('/edit.php?type=2')
# return self.page.set_announce(title, description, lookingfor)
#@pageaccess
#def set_description(self, **args):
# if not self.is_on_page(EditDescriptionPage):
# self.location('/edit.php?type=3')
# return self.page.set_description(**args)
def check_login(func):
def inner(self, *args, **kwargs):
if self.my_id == 0:
self.login()
return func(self, *args, **kwargs)
return inner
def get_consts(self):
if self.consts is not None:
return self.consts
self.consts = []
for i in xrange(2):
r = self.api_request('me', 'all_values', data={'sex': i})
self.consts.append(r['result']['values'])
return self.consts
@check_login
def score(self):
r = self.api_request('member', 'view', data={'id': self.my_id})
return int(r['result']['member']['popu']['popu'])
@check_login
def get_my_name(self):
return self.my_name
@check_login
def get_my_id(self):
return self.my_id
@check_login
def nb_new_mails(self):
r = self.api_request('me', '[default]')
return r['result']['news']['newMails']
@check_login
def nb_new_baskets(self):
r = self.api_request('me', '[default]')
return r['result']['news']['newBaskets']
@check_login
def nb_new_visites(self):
r = self.api_request('me', '[default]')
return r['result']['news']['newVisits']
@check_login
def nb_available_charms(self):
r = self.login()
return r['result']['flashs']
@check_login
def nb_godchilds(self):
r = self.api_request('member', 'view', data={'id': self.my_id})
return int(r['result']['member']['popu']['invits'])
@check_login
def get_baskets(self):
r = self.api_request('me', 'basket')
return r['result']['basket']
@check_login
def get_flashs(self):
r = self.api_request('me', 'flashs')
return r['result']['all']
@check_login
def get_visits(self):
r = self.api_request('me', 'visits')
return r['result']['news'] + r['result']['olds']
@check_login
def get_threads_list(self, count=30):
r = self.api_request('message', '[default]', '%d,0' % count)
return r['result']['threads']
@check_login
@url2id
def get_thread_mails(self, id, count=30):
r = self.api_request('message', 'thread', data={'memberId': id, 'count': count})
return r['result']['thread']
@check_login
@url2id
def post_mail(self, id, content):
new_content = u''
for c in content:
try:
new_content += '&%s;' % codepoint2name[ord(c)]
except KeyError:
new_content += c
content = new_content.replace('\n', '\r\n').encode('Windows-1252', 'replace')
try:
self.api_request('message', 'new', data={'memberId': id, 'message': content})
except AuMException, e:
raise CantSendMessage(unicode(e))
@check_login
@url2id
def delete_thread(self, id):
r = self.api_request('message', 'delete', data={'id_user': id})
self.logger.debug('Thread deleted: %r' % r)
@check_login
@url2id
def send_charm(self, id):
try:
self.api_request('member', 'addBasket', data={'id': id})
except AuMException:
return False
else:
return True
@check_login
@url2id
def add_basket(self, id):
try:
self.api_request('member', 'addBasket', data={'id': id})
except AuMException:
return False
else:
return True
@url2id
def deblock(self, id):
self.readurl('http://www.adopteunmec.com/fajax_postMessage.php?action=deblock&to=%s' % id)
return True
@url2id
def report_fake(self, id):
return self.readurl('http://www.adopteunmec.com/fake.php', 'id=%s' % id)
@url2id
def rate(self, id, what, rating):
result = self.openurl('http://www.adopteunmec.com/fajax_vote.php', 'member=%s&what=%s&rating=%s' % (id, what, rating)).read()
return float(result)
def search_profiles(self, **kwargs):
if self.search_query is None:
r = self.api_request('searchs', '[default]')
self.search_query = r['result']['search']['query']
params = {}
for key, value in json.loads(self.search_query).iteritems():
if isinstance(value, dict):
for k, v in value.iteritems():
params['%s%s' % (key, k.capitalize())] = v
else:
params[key] = value or ''
r = self.api_request('searchs', 'advanced', '30,0', params)
ids = [s['id'] for s in r['result']['search']]
return set(ids)
@url2id
def get_profile(self, id, with_pics=True):
r = self.api_request('member', 'view', data={'id': id})
if not 'result' in r:
print r
profile = r['result']['member']
# Calculate distance in km.
profile['dist'] = 0.0
if 'lat' in profile and 'lng' in profile:
coords = (float(profile['lat']), float(profile['lng']))
R = 6371
lat1 = math.radians(self.my_coords[0])
lat2 = math.radians(coords[0])
lon1 = math.radians(self.my_coords[1])
lon2 = math.radians(coords[1])
dLat = lat2 - lat1
dLong = lon2 - lon1
a= pow(math.sin(dLat/2), 2) + math.cos(lat1) * math.cos(lat2) * pow(math.sin(dLong/2), 2)
c= 2 * math.atan2(math.sqrt(a), math.sqrt(1-a))
profile['dist'] = R * c
if with_pics:
r = self.api_request('member', 'pictures', data={'id': id})
profile['pictures'] = []
for pic in r['result']['pictures']:
d = {'hidden': False}
d.update(pic)
profile['pictures'].append(d)
base_url = 'http://s%s.adopteunmec.com/%s' % (profile['shard'], profile['path'])
if len(profile['pictures']) > 0:
pic_regex = re.compile('(?P<base_url>http://.+\.adopteunmec\.com/.+/)image(?P<id>.+)\.jpg')
pic_max_id = max((int((lambda m: m and m.groupdict()['id'] or 0)(pic_regex.match(pic['url'])))) for pic in profile['pictures'])
for id in xrange(1, pic_max_id + 1):
url = u'%simage%s.jpg' % (base_url, id)
if not url in [pic['url'] for pic in profile['pictures']]:
profile['pictures'].append({'url': url, u'hidden': True, 'id': u'0', 'rating': 0.0})
else:
url = '%simage1.jpg' % base_url
try:
self.openurl(url)
except BrowserHTTPNotFound:
pass
else:
profile['pictures'].append({'url': url, u'hidden': True, 'id': u'0', 'rating': 0.0})
return profile
def _get_chat_infos(self):
try:
data = json.load(self.openurl('http://www.adopteunmec.com/1.1_cht_get.php?anticache=%f' % random.random()))
except ValueError:
raise BrowserUnavailable()
if data['error']:
raise ChatException(u'Error while getting chat infos. json:\n%s' % data)
return data
def iter_contacts(self):
def iter_dedupe(contacts):
yielded_ids = set()
for contact in contacts:
if contact['id'] not in yielded_ids:
yield contact
yielded_ids.add(contact['id'])
data = self._get_chat_infos()
return iter_dedupe(data['contacts'])
def iter_chat_messages(self, _id=None):
data = self._get_chat_infos()
if data['messages'] is not None:
for message in data['messages']:
yield ChatMessage(id_from=message['id_from'], id_to=message['id_to'], message=message['message'], date=message['date'])
def send_chat_message(self, _id, message):
url = 'http://www.adopteunmec.com/1.1_cht_send.php?anticache=%f' % random.random()
data = dict(id=_id, message=message)
headers = {
'Content-type': 'application/x-www-form-urlencoded',
'Accept': 'text/plain',
'Referer': 'http://www.adopteunmec.com/chat.php',
'Origin': 'http://www.adopteunmec.com',
}
request = self.request_class(url, urllib.urlencode(data), headers)
response = self.openurl(request).read()
try:
datetime.datetime.strptime(response, '%Y-%m-%d %H:%M:%S')
return True
except ValueError:
return False

204
modules/aum/captcha.py Normal file
View file

@ -0,0 +1,204 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
import hashlib
import sys
import Image
class CaptchaError(Exception): pass
class Tile:
hash = {
'bc8d52d96058478a6def26226145d53b': 'A',
'c62ecdfddb72b2feaed96cd9fe7c2802': 'A',
'8b61cda8a3240d8fa5f2610424271300': 'AD',
'f5dc63d37c7ea3375d86180f0ae62d05': 'AE',
'fd562be230da7f767f4454148632201d': 'AF',
'1860de576d8b0d1d87edc9dcb0b2a64c': 'AG',
'53afa108d36186e6bd23631711ec3d8c': 'AJ',
'6f2f9a1082a9230272c45117320f159d': 'AL',
'e14249a774d24bacc6e2bcadd7f3df65': 'AM',
'389330dbf3d85dea2dc40c6f9cf77d52': 'AN',
'17526a3c2261b55f9cd237c4aa195099': 'AQ',
'7e4820a9cc6c83a9fa60ff73ceb52157': 'AW',
'90690d1209753a2bcfeafa890082a585': 'B',
'2cf22e9ceace03a5f8ed3999e92d877e': 'C',
'a1d0bf1a29600a82a6aa2b8b21651b0f': 'D',
'9bb6909d647a0be3b2e7352d37374228': 'E',
'38120c8346f16cd07a9194283787ee5e': 'F',
'd41ff948fbc50a628c858b8e3e9e931c': 'G',
'4cc9322d3361eb3f9fea7fc83579e40f': 'H',
'837cd0f04e2d47ca6975745bdd0da640': 'I',
'da0204fa51b38414051376cc1c27ba72': 'J',
'199b1a9f9e1df1c2eddadcc4582957d7': 'JW',
'5e8d3d5bd5f683d84b089f2cecc1e196': 'JX',
'bc1fcf3546057d40d2db5454caacb3a5': 'JZ',
'c2f5866ba3bf799ece8b202492d199bf': 'K',
'7abe4091e11921afe6dac16509999010': 'KT',
'281ef08e623184e5621a73b9ccec7c9a': 'KX',
'b28e3fc06411de2ac7f53569bc3b42db': 'L',
'd58a6c26649926f1145fb4b7b42d0554': 'LT',
'4add630c6d124899fef814211975e344': 'M',
'9740cefe1629d6bc149a72d5f2a4586d': 'N',
'396f816f7e78e5c98de6404f8c4bd2ee': 'O',
'31ae7c9536b6c6a96e30a77b70e4b2fd': 'P',
'98ad9b1c32c05e6efc06637a166e4c42': 'PA',
'a05cce33683025fb2c6708ee06f6028e': 'Q',
'2852f51e8939bf9664fe064f7dacf310': 'R',
'3798513fe87e786faa67552a140fd86f': 'S',
'350b13811e34eeb63e3d7fb4b5eade5b': 'T',
'a01b186cbc767e17d948ed04eff114a1': 'U',
'8405f4d80ce80c4e6e9680fcfac4fe40': 'V',
'17ed80e9cb9a585098ae6a55d8d1f5c0': 'W',
'ae54ca77be5561330781a08dfbaff7a7': 'W',
'bbded6a2ba5f521bba276bb843bf4c98': 'WXT',
'ea662dd25fc528b84b832ce71ae3de61': 'WZ',
'4eb23916138e7c01714431dbecfe8b96': 'X',
'c02093d35d852339ff34f2b26873bf5a': 'XW',
'65744e0c6ce0c56d04873dfd732533a7': 'Y',
'315fb7dba7032004bd362cf0bb076733': 'YA',
'ce12a68a4f15657bc5297a6cf698bc0a': 'YAQ',
'275478ea2280351f7433a0606f962175': 'Z',
}
def __init__(self):
self.map = []
def append(self, pxls):
self.map.append(pxls)
def display(self):
print '-' * (len(self.map) * 2 + 2)
for y in xrange(len(self.map[0])):
sys.stdout.write('|')
for x in xrange(len(self.map)):
sys.stdout.write('%s' % ('XX' if self.map[x][y] else ' '))
print '|'
print '-' * (len(self.map) * 2 + 2)
def checksum(self):
s = ''
for pxls in self.map:
for pxl in pxls:
s += '%d' % (1 if pxl else 0)
return hashlib.md5(s).hexdigest()
@property
def letter(self):
checksum = self.checksum()
try:
return self.hash[checksum]
except KeyError:
print 'Unable te resolve:'
self.display()
print 'hash: %s' % checksum
raise CaptchaError()
class Captcha:
def __init__(self, f):
self.img = Image.open(f)
self.w, self.h = self.img.size
self.map = self.img.load()
self.tiles = []
tile = None
for x in xrange(self.w):
blank = True
pxls = []
for y in xrange(self.h):
pxls.append(self[x,y])
if self[x,y] != 0:
blank = False
if tile:
if blank:
tile = None
else:
tile.append(pxls)
elif not blank:
tile = Tile()
tile.append(pxls)
self.tiles.append(tile)
def __getitem__(self, (x, y)):
return self.map[x % self.w, y % self.h]
def __iter__(self):
for tile in self.tiles:
yield tile
@property
def text(self):
s = ''
for tile in self.tiles:
s += tile.letter
return s
class Decoder:
def __init__(self):
self.hash = {}
def process(self):
from aum.browser import AuMBrowser
browser = AuMBrowser('')
browser.openurl('/register2.php')
c = Captcha(browser.openurl('/captcha.php'))
for tile in c:
checksum = tile.checksum()
if checksum in self.hash:
print 'Skipping %s' % self.hash[checksum]
continue
tile.display()
print 'Checksum: %s' % checksum
ntry = 2
while ntry:
sys.stdout.write('Enter the letter: ')
l = sys.stdin.readline().strip()
ntry -= 1
if len(l) != 1:
print 'Error: please enter only one letter'
elif l in self.hash.itervalues():
print 'Warning! This letter has already been catched!'
else:
ntry = 0
self.hash[checksum] = l
def main(self):
try:
while 1:
self.process()
except KeyboardInterrupt:
print ''
print 'hash = {'
l = sorted(self.hash.iteritems(), key=lambda (k,v): (v,k))
for hash, value in l:
print ' \'%s\': %s' % (hash, value)
print '}'
if __name__ == '__main__':
d = Decoder()
d.main()

281
modules/aum/contact.py Normal file
View file

@ -0,0 +1,281 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2008-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
import socket
from datetime import datetime
from dateutil.parser import parse as parse_dt
from weboob.tools.ordereddict import OrderedDict
from weboob.capabilities.contact import Contact as _Contact, ProfileNode
from weboob.tools.misc import html2text
class FieldBase:
def __init__(self, key, key2=None):
self.key = key
self.key2 = key2
def get_value(self, value, consts):
raise NotImplementedError
class FieldStr(FieldBase):
def get_value(self, profile, consts):
return html2text(unicode(profile[self.key])).strip()
class FieldBool(FieldBase):
def get_value(self, profile, consts):
return bool(int(profile[self.key]))
class FieldDist(FieldBase):
def get_value(self, profile, consts):
return '%.2f km' % float(profile[self.key])
class FieldIP(FieldBase):
def get_hostname(self, s):
try:
return socket.gethostbyaddr(s)[0]
except (socket.gaierror, socket.herror):
return s
def get_value(self, profile, consts):
s = self.get_hostname(profile[self.key])
if profile[self.key] != profile[self.key2]:
s += ' (first %s)' % self.get_hostname(profile[self.key2])
return s
class FieldProfileURL(FieldBase):
def get_value(self, profile, consts):
id = int(profile[self.key])
if id > 0:
return 'http://www.adopteunmec.com/index.php/profile/%d' % id
else:
return ''
class FieldPopu(FieldBase):
def get_value(self, profile, consts):
return unicode(profile['popu'][self.key])
class FieldPopuRatio(FieldBase):
def get_value(self, profile, consts):
v1 = float(profile['popu'][self.key])
v2 = float(profile['popu'][self.key2])
if v2 == 0.0:
return 'NaN'
else:
return '%.2f' % (v1 / v2)
class FieldOld(FieldBase):
def get_value(self, profile, consts):
birthday = parse_dt(profile[self.key])
return int((datetime.now() - birthday).days / 365.25)
class FieldSplit(FieldBase):
def get_value(self, profile, consts):
return [html2text(s).strip() for s in profile[self.key].split(self.key2) if len(s.strip()) > 0]
class FieldBMI(FieldBase):
def __init__(self, key, key2, fat=False):
FieldBase.__init__(self, key, key2)
self.fat = fat
def get_value(self, profile, consts):
height = int(profile[self.key])
weight = int(profile[self.key2])
if height == 0 or weight == 0:
return ''
bmi = (weight/float(pow(height/100.0, 2)))
if not self.fat:
return bmi
elif bmi < 15.5:
return 'severely underweight'
elif bmi < 18.4:
return 'underweight'
elif bmi < 24.9:
return 'normal'
elif bmi < 30:
return 'overweight'
else:
return 'obese'
class FieldFlags(FieldBase):
def get_value(self, profile, consts):
i = int(profile[self.key])
labels = []
for d in consts[self.key]:
if i & (1 << int(d['value'])):
labels.append(html2text(d['label']).strip())
return labels
class FieldList(FieldBase):
def get_value(self, profile, consts):
i = int(profile[self.key])
for d in consts[self.key]:
if i == int(d['value']):
return html2text(d['label']).strip()
return ''
class Contact(_Contact):
TABLE = OrderedDict((
('_info', OrderedDict((
('title', FieldStr('title')),
# ipaddr is not available anymore.
#('IPaddr', FieldIP('last_ip', 'first_ip')),
('admin', FieldBool('admin')),
('ban', FieldBool('isBan')),
('first', FieldStr('first_cnx')),
('godfather', FieldProfileURL('godfather')),
))),
('_stats', OrderedDict((
('mails', FieldPopu('mails')),
('baskets', FieldPopu('contacts')),
('charms', FieldPopu('flashs')),
('visites', FieldPopu('visites')),
('invits', FieldPopu('invits')),
('bonus', FieldPopu('bonus')),
('score', FieldPopu('popu')),
('ratio', FieldPopuRatio('mails', 'flashs')),
))),
('details', OrderedDict((
('old', FieldOld('birthday')),
('birthday', FieldStr('birthday')),
('zipcode', FieldStr('zip')),
('location', FieldStr('city')),
('distance', FieldDist('dist')),
('country', FieldStr('country')),
('phone', FieldStr('phone')),
('eyes', FieldList('eyes')),
('hair_color', FieldList('hair_color')),
('hair_size', FieldList('hair_size')),
('height', FieldList('size')),
('weight', FieldList('weight')),
('BMI', FieldBMI('size', 'weight')),
('fat', FieldBMI('size', 'weight', fat=True)),
('shape', FieldList('shape')),
('origins', FieldList('origins')),
('signs', FieldFlags('checks1')),
('job', FieldStr('job')),
('style', FieldList('style')),
('food', FieldList('food')),
('drink', FieldList('drink')),
('smoke', FieldList('smoke')),
))),
('tastes', OrderedDict((
('hobbies', FieldStr('hobbies')),
('music', FieldSplit('music', '<br>')),
('cinema', FieldSplit('cinema', '<br>')),
('books', FieldSplit('books', '<br>')),
('tv', FieldSplit('tvs', '<br>')),
))),
('sex', OrderedDict((
('underwear', FieldFlags('checks7')),
('practices', FieldFlags('checks5')),
('favorite', FieldFlags('checks3')),
('toys', FieldFlags('checks6')),
))),
('personality', OrderedDict((
('snap', FieldStr('texts1')),
('exciting', FieldStr('texts2')),
('hate', FieldStr('texts3')),
('vices', FieldStr('texts4')),
('assets', FieldStr('texts6')),
('fantasies', FieldStr('texts5')),
('is', FieldFlags('checks2')),
)))
))
def parse_profile(self, profile, consts):
cat = int(profile.get('cat', 3))
if cat == 1:
self.status = Contact.STATUS_ONLINE
self.status_msg = u'online'
self.status_msg = u'since %s' % profile['last_cnx']
elif cat == 2:
self.status = Contact.STATUS_AWAY
self.status_msg = u'away'
self.status_msg = u'connection at %s' % profile['last_cnx']
elif cat == 3:
self.status = Contact.STATUS_OFFLINE
self.status_msg = u'last connection %s' % profile['last_cnx']
self.summary = html2text(profile.get('about1', '')).strip().replace('\n\n', '\n')
if len(profile.get('about2', '')) > 0:
self.summary += u'\n\nLooking for:\n%s' % html2text(profile['about2']).strip().replace('\n\n', '\n')
for photo in profile['pictures']:
self.set_photo(photo['url'].split('/')[-1],
url=photo['url'],
thumbnail_url=photo['url'].replace('image', 'thumb1_'),
hidden=photo['hidden'])
self.profile = OrderedDict()
if 'sex' in profile:
for section, d in self.TABLE.iteritems():
flags = ProfileNode.SECTION
if section.startswith('_'):
flags |= ProfileNode.HEAD
section = section.lstrip('_')
s = ProfileNode(section, section.capitalize(), OrderedDict(), flags=flags)
for key, builder in d.iteritems():
value = builder.get_value(profile, consts[int(profile['sex'])])
s.value[key] = ProfileNode(key, key.capitalize().replace('_', ' '), value)
self.profile[section] = s
self.aum_profile = profile
def get_text(self):
def print_node(node, level=1):
result = u''
if node.flags & node.SECTION:
result += u'\t' * level + node.label + '\n'
for sub in node.value.itervalues():
result += print_node(sub, level+1)
else:
if isinstance(node.value, (tuple,list)):
value = ', '.join(unicode(v) for v in node.value)
elif isinstance(node.value, float):
value = '%.2f' % node.value
else:
value = node.value
result += u'\t' * level + u'%-20s %s\n' % (node.label + ':', value)
return result
result = u'Nickname: %s\n' % self.name
if self.status & Contact.STATUS_ONLINE:
s = 'online'
elif self.status & Contact.STATUS_OFFLINE:
s = 'offline'
elif self.status & Contact.STATUS_AWAY:
s = 'away'
else:
s = 'unknown'
result += u'Status: %s (%s)\n' % (s, self.status_msg)
result += u'URL: %s\n' % self.url
result += u'Photos:\n'
for name, photo in self.photos.iteritems():
result += u'\t%s%s\n' % (photo, ' (hidden)' if photo.hidden else '')
result += u'\nProfile:\n'
for head in self.profile.itervalues():
result += print_node(head)
result += u'Description:\n'
for s in self.summary.split('\n'):
result += u'\t%s\n' % s
return result

BIN
modules/aum/favicon.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 2.1 KiB

0
modules/aum/optim/__init__.py Normal file
View file

185
modules/aum/optim/priority_connection.py Normal file
View file

@ -0,0 +1,185 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from __future__ import with_statement
import random
from weboob.tools.browser import BrowserUnavailable, BrowserIncorrectPassword
from weboob.capabilities.dating import Optimization
from weboob.capabilities.account import AccountRegisterError
from weboob.tools.log import getLogger
from weboob.tools.value import Value, ValuesDict, ValueInt
from aum.captcha import CaptchaError
from aum.exceptions import AdopteWait, AdopteBanned
from aum.browser import AuMBrowser
__all__ = ['PriorityConnection']
class PriorityConnection(Optimization):
CONFIG = ValuesDict(ValueInt('minimal', label='Minimal of godchilds', default=5),
Value('domain', label='Domain to use for fake accounts emails', default='aum.example.com'),
ValueInt('interval', label='Interval of checks (seconds)', default=3600)
)
def __init__(self, sched, storage, browser):
self.sched = sched
self.storage = storage
self.browser = browser
self.logger = getLogger('priorityconn', browser.logger)
self.config = storage.get('priority_connection', 'config', default=None)
if self.config == {}:
self.config = None
self.check_cron = None
self.activity_cron = None
def start(self):
if self.config is None:
return False
self.check_cron = self.sched.repeat(int(self.config['interval']), self.check_godchilds)
self.activity_cron = self.sched.repeat(600, self.activity_fakes)
return True
def stop(self):
self.sched.cancel(self.check_cron)
self.check_cron = None
self.sched.cancel(self.activity_cron)
self.activity_cron = None
return True
def is_running(self):
return self.check_cron is not None
def set_config(self, params):
self.config = params
self.storage.set('priority_connection', 'config', self.config)
self.storage.save()
def get_config(self):
return self.config
def generate_name(self):
login = u''
for x in xrange(8):
if x % 2:
login += random.choice(u'aeiou')
else:
login += random.choice(u'bcdfghjklmnprstv')
fakes = self.storage.get('priority_connection', 'fakes')
while ('%s@%s' % (login, self.config['domain'])) in fakes.iterkeys():
login += '_'
return login
def generate_password(self):
return '%08x' % random.randint(1, int('ffffffff', 16))
def check_godchilds(self):
with self.browser:
try:
my_id = self.browser.get_my_id()
nb_godchilds = self.browser.nb_godchilds()
except AdopteWait:
nb_godchilds = 0
except BrowserUnavailable:
# We'll check later
return
missing_godchilds = int(self.config['minimal']) - nb_godchilds
self.logger.info('Missing godchilds: %s' % missing_godchilds)
if missing_godchilds <= 0:
return
for i in xrange(missing_godchilds):
registered = False
while not registered:
name = self.generate_name()
password = self.generate_password()
browser = AuMBrowser('%s@%s' % (name, self.config['domain']), proxy=self.browser.proxy)
try:
browser.register(password= password,
sex= 1, #slut
birthday_d= random.randint(1,28),
birthday_m= random.randint(1,12),
birthday_y= random.randint(1975, 1990),
zipcode= 75001,
country= 'fr',
godfather= my_id)
except AccountRegisterError, e:
self.logger.warning('Unable to register account: %s' % e)
except CaptchaError:
self.logger.warning('Unable to solve captcha... Retrying')
else:
registered = True
# set nickname
browser.set_nickname(name.strip('_').capitalize())
# rate my own profile with good score
for i in xrange(4):
browser.rate(my_id, i, 5.0)
# save fake in storage
fake = {'username': browser.username,
'password': password}
self.storage.set('priority_connection', 'fakes', name, fake)
self.storage.save()
self.logger.info('Fake account "%s" created (godfather=%s)' % (name, my_id))
def activity_fakes(self):
try:
fakes = self.storage.get('priority_connection', 'fakes', default={})
if len(fakes) == 0:
return
while 1:
name = random.choice(fakes.keys())
fake = fakes[name]
try:
browser = AuMBrowser(fake['username'], fake['password'], proxy=self.browser.proxy)
except (AdopteBanned,BrowserIncorrectPassword), e:
self.logger.warning('Fake %s can\'t login: %s' % (name, e))
continue
profiles = browser.search_profiles(country="fr",
dist='10',
save=True)
if not profiles:
continue
id = profiles.pop()
profile = browser.get_profile(id)
# bad rate
for i in xrange(4):
browser.rate(profile.get_id(), i, 0.6)
# deblock
browser.deblock(profile.get_id())
return
except BrowserUnavailable:
# don't care
pass

104
modules/aum/optim/profiles_walker.py Normal file
View file

@ -0,0 +1,104 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon, Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from __future__ import with_statement
from random import randint
from weboob.tools.browser import BrowserUnavailable
from weboob.capabilities.dating import Optimization
from weboob.tools.log import getLogger
__all__ = ['ProfilesWalker']
class ProfilesWalker(Optimization):
def __init__(self, sched, storage, browser):
self.sched = sched
self.storage = storage
self.browser = browser
self.logger = getLogger('walker', browser.logger)
self.walk_cron = None
self.view_cron = None
self.visited_profiles = set(storage.get('profiles_walker', 'viewed'))
self.logger.info(u'Loaded %d already visited profiles from storage.' % len(self.visited_profiles))
self.profiles_queue = set()
def save(self):
self.storage.set('profiles_walker', 'viewed', list(self.visited_profiles))
self.storage.save()
def start(self):
self.walk_cron = self.sched.repeat(60, self.enqueue_profiles)
self.view_cron = self.sched.schedule(randint(10,40), self.view_profile)
return True
def stop(self):
self.sched.cancel(self.walk_cron)
self.sched.cancel(self.view_cron)
self.walk_cron = None
self.view_cron = None
return True
def is_running(self):
return self.walk_cron is not None
def enqueue_profiles(self):
try:
with self.browser:
profiles_to_visit = self.browser.search_profiles().difference(self.visited_profiles)
self.logger.info(u'Enqueuing profiles to visit: %s' % profiles_to_visit)
self.profiles_queue = set(profiles_to_visit)
self.save()
except BrowserUnavailable:
return
def view_profile(self):
try:
try:
id = self.profiles_queue.pop()
except KeyError:
return # empty queue
try:
with self.browser:
profile = self.browser.get_profile(id)
self.logger.info(u'Visited profile %s (%s)' % (profile['pseudo'], id))
# Get score from the aum_score module
#d = self.nucentral_core.callService(context.Context.fromComponent(self), 'aum_score', 'score', profile)
# d.addCallback(self.score_cb, profile.getID())
# deferredlist.append(d)
# do not forget that we visited this profile, to avoid re-visiting it.
self.visited_profiles.add(id)
self.save()
except BrowserUnavailable:
# We consider this profil hasn't been [correctly] analysed
self.profiles_queue.add(id)
return
except Exception, e:
print e
finally:
if self.view_cron is not None:
self.view_cron = self.sched.schedule(randint(10,40), self.view_profile)

107
modules/aum/optim/queries_queue.py Normal file
View file

@ -0,0 +1,107 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from __future__ import with_statement
from weboob.tools.browser import BrowserUnavailable
from weboob.capabilities.dating import Optimization
from weboob.capabilities.contact import QueryError
from weboob.tools.log import getLogger
__all__ = ['QueriesQueue']
class QueriesQueue(Optimization):
def __init__(self, sched, storage, browser):
self.sched = sched
self.storage = storage
self.browser = browser
self.logger = getLogger('queriesqueue', browser.logger)
self.queue = storage.get('queries_queue', 'queue', default=[])
self.check_cron = None
def save(self):
self.storage.set('queries_queue', 'queue', self.queue)
self.storage.save()
def start(self):
self.check_cron = self.sched.repeat(3600, self.flush_queue)
return True
def stop(self):
self.sched.cancel(self.check_cron)
self.check_cron = None
return True
def is_running(self):
return self.check_cron is not None
def enqueue_query(self, id, priority=999):
id_queue = [_id[1] for _id in self.queue]
if int(id) in id_queue:
raise QueryError('This id is already queued')
self.queue.append((int(priority), int(id)))
self.save()
# Try to flush queue to send it now.
self.flush_queue()
# Check if the enqueued query has been sent
for p, i in self.queue:
if i == int(id):
return False
return True
def flush_queue(self):
self.queue.sort()
priority = 0
id = None
try:
try:
while len(self.queue) > 0:
priority, id = self.queue.pop()
if not id:
continue
with self.browser:
if self.browser.send_charm(id):
self.logger.info('Charm sent to %s' % id)
else:
self.queue.append((priority, id))
self.logger.info("Charm can't be send to %s" % id)
break
# As the charm has been correctly sent (no exception raised),
# we don't store anymore ID, because if nbAvailableCharms()
# fails, we don't want to re-queue this ID.
id = None
priority = 0
except BrowserUnavailable:
# We consider this profil hasn't been [correctly] analysed
if not id is None:
self.queue.append((priority, id))
finally:
self.save()

55
modules/aum/optim/visibility.py Normal file
View file

@ -0,0 +1,55 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.tools.browser import BrowserUnavailable
from weboob.capabilities.dating import Optimization
from ..browser import AuMBrowser
__all__ = ['Visibility']
class Visibility(Optimization):
def __init__(self, sched, browser):
self.sched = sched
self.browser = browser
self.cron = None
def start(self):
self.cron = self.sched.repeat(60*5, self.reconnect)
return True
def stop(self):
self.sched.cancel(self.cron)
self.cron = None
return True
def is_running(self):
return self.cron is not None
def reconnect(self):
try:
AuMBrowser(self.browser.username,
self.browser.password,
proxy=self.browser.proxy)
except BrowserUnavailable, e:
print str(e)
pass

49
modules/aum/test.py Normal file
View file

@ -0,0 +1,49 @@
# -*- CODing: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.tools.test import BackendTest
from weboob.tools.browser import BrowserUnavailable
__all__ = ['AuMTest']
class AuMTest(BackendTest):
BACKEND = 'aum'
def test_new_messages(self):
try:
for message in self.backend.iter_unread_messages():
pass
except BrowserUnavailable:
# enough frequent to do not care about.
pass
def test_contacts(self):
try:
contacts = list(self.backend.iter_contacts())
if len(contacts) == 0:
# so bad, we can't test that...
return
self.backend.fillobj(contacts[0], ['photos', 'profile'])
except BrowserUnavailable:
# enough frequent to do not care about.
pass

22
modules/batoto/__init__.py Normal file
View file

@ -0,0 +1,22 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Noé Rubinstein
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from .backend import BatotoBackend
__all__ = ['BatotoBackend']

34
modules/batoto/backend.py Normal file
View file

@ -0,0 +1,34 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Noé Rubinstein
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.tools.capabilities.gallery.genericcomicreader import GenericComicReaderBackend, DisplayPage
__all__ = ['BatotoBackend']
class BatotoBackend(GenericComicReaderBackend):
NAME = 'batoto'
DESCRIPTION = 'Batoto manga reading site'
DOMAIN = 'www.batoto.com'
BROWSER_PARAMS = dict(
img_src_xpath="//img[@id='comic_page']/@src",
page_list_xpath="(//select[@id='page_select'])[1]/option/@value")
ID_REGEXP = r'[^/]+/[^/]+'
URL_REGEXP = r'.+batoto.(?:com|net)/read/_/(%s).+' % ID_REGEXP
ID_TO_URL = 'http://www.batoto.net/read/_/%s'
PAGES = { URL_REGEXP: DisplayPage }

26
modules/batoto/test.py Normal file
View file

@ -0,0 +1,26 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Noé Rubinstein
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.tools.capabilities.gallery.genericcomicreader import GenericComicReaderTest
class BatotoTest(GenericComicReaderTest):
BACKEND = 'batoto'
def test_download(self):
return self._test_download('26287/yurumates_ch4_by_primitive-scans')

23
modules/bnporc/__init__.py Normal file
View file

@ -0,0 +1,23 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from .backend import BNPorcBackend
__all__ = ['BNPorcBackend']

106
modules/bnporc/backend.py Normal file
View file

@ -0,0 +1,106 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
# python2.5 compatibility
from __future__ import with_statement
from weboob.capabilities.bank import ICapBank, AccountNotFound, Account, Recipient
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.tools.value import ValueBackendPassword
from .browser import BNPorc
__all__ = ['BNPorcBackend']
class BNPorcBackend(BaseBackend, ICapBank):
NAME = 'bnporc'
MAINTAINER = 'Romain Bignon'
EMAIL = 'romain@weboob.org'
VERSION = '0.a'
LICENSE = 'AGPLv3+'
DESCRIPTION = 'BNP Paribas french bank\' website'
CONFIG = BackendConfig(ValueBackendPassword('login', label='Account ID', masked=False),
ValueBackendPassword('password', label='Password', regexp='^(\d{6}|)$'),
ValueBackendPassword('rotating_password',
label='Password to set when the allowed uses are exhausted (6 digits)',
regexp='^(\d{6}|)$'))
BROWSER = BNPorc
def create_default_browser(self):
if self.config['rotating_password'].get().isdigit() and len(self.config['rotating_password'].get()) == 6:
rotating_password = self.config['rotating_password'].get()
else:
rotating_password = None
return self.create_browser(self.config['login'].get(),
self.config['password'].get(),
password_changed_cb=self._password_changed_cb,
rotating_password=rotating_password)
def _password_changed_cb(self, old, new):
self.config['password'].set(new)
self.config['rotating_password'].set(old)
self.config.save()
def iter_accounts(self):
for account in self.browser.get_accounts_list():
yield account
def get_account(self, _id):
if not _id.isdigit():
raise AccountNotFound()
with self.browser:
account = self.browser.get_account(_id)
if account:
return account
else:
raise AccountNotFound()
def iter_history(self, account):
with self.browser:
for history in self.browser.get_history(account):
yield history
def iter_operations(self, account):
with self.browser:
for coming in self.browser.get_coming_operations(account):
yield coming
def iter_transfer_recipients(self, ignored):
for account in self.browser.get_transfer_accounts().itervalues():
recipient = Recipient()
recipient.id = account.id
recipient.label = account.label
yield recipient
def transfer(self, account, to, amount, reason=None):
if isinstance(account, Account):
account = account.id
try:
assert account.isdigit()
assert to.isdigit()
amount = float(amount)
except (AssertionError, ValueError):
raise AccountNotFound()
with self.browser:
return self.browser.transfer(account, to, amount, reason)

158
modules/bnporc/browser.py Normal file
View file

@ -0,0 +1,158 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2009-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from datetime import datetime
from logging import warning
from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword
from weboob.capabilities.bank import TransferError, Transfer
from bnporc import pages
from .errors import PasswordExpired
__all__ = ['BNPorc']
class BNPorc(BaseBrowser):
DOMAIN = 'www.secure.bnpparibas.net'
PROTOCOL = 'https'
ENCODING = None # refer to the HTML encoding
PAGES = {'.*identifiant=DOSSIER_Releves_D_Operation.*': pages.AccountsList,
'.*SAF_ROP.*': pages.AccountHistory,
'.*Action=SAF_CHM.*': pages.ChangePasswordPage,
'.*NS_AVEDT.*': pages.AccountComing,
'.*NS_AVEDP.*': pages.AccountPrelevement,
'.*NS_VIRDF.*': pages.TransferPage,
'.*NS_VIRDC.*': pages.TransferConfirmPage,
'.*/NS_VIRDA\?stp=(?P<id>\d+).*': pages.TransferCompletePage,
'.*Action=DSP_VGLOBALE.*': pages.LoginPage,
'.*type=homeconnex.*': pages.LoginPage,
'.*layout=HomeConnexion.*': pages.ConfirmPage,
'.*SAF_CHM_VALID.*': pages.ConfirmPage,
}
def __init__(self, *args, **kwargs):
self.rotating_password = kwargs.pop('rotating_password', None)
self.password_changed_cb = kwargs.pop('password_changed_cb', None)
BaseBrowser.__init__(self, *args, **kwargs)
def home(self):
self.location('https://www.secure.bnpparibas.net/banque/portail/particulier/HomeConnexion?type=homeconnex')
def is_logged(self):
return not self.is_on_page(pages.LoginPage)
def login(self):
assert isinstance(self.username, basestring)
assert isinstance(self.password, basestring)
assert self.password.isdigit()
if not self.is_on_page(pages.LoginPage):
self.location('https://www.secure.bnpparibas.net/banque/portail/particulier/HomeConnexion?type=homeconnex')
self.page.login(self.username, self.password)
self.location('/NSFR?Action=DSP_VGLOBALE', no_login=True)
if self.is_on_page(pages.LoginPage):
raise BrowserIncorrectPassword()
def change_password(self, new_password):
assert new_password.isdigit() and len(new_password) == 6
self.location('https://www.secure.bnpparibas.net/SAF_CHM?Action=SAF_CHM')
assert self.is_on_page(pages.ChangePasswordPage)
self.page.change_password(self.password, new_password)
if not self.is_on_page(pages.ConfirmPage):
self.logger.error('Oops, unable to change password')
return
self.password, self.rotating_password = (new_password, self.password)
if self.password_changed_cb:
self.password_changed_cb(self.rotating_password, self.password)
def check_expired_password(func):
def inner(self, *args, **kwargs):
try:
return func(self, *args, **kwargs)
except PasswordExpired:
if self.rotating_password is not None:
warning('[%s] Your password has expired. Switching...' % self.username)
self.change_password(self.rotating_password)
return func(self, *args, **kwargs)
else:
raise
return inner
@check_expired_password
def get_accounts_list(self):
if not self.is_on_page(pages.AccountsList):
self.location('/NSFR?Action=DSP_VGLOBALE')
return self.page.get_list()
def get_account(self, id):
assert isinstance(id, basestring)
if not self.is_on_page(pages.AccountsList):
self.location('/NSFR?Action=DSP_VGLOBALE')
l = self.page.get_list()
for a in l:
if a.id == id:
return a
return None
def get_history(self, account):
if not self.is_on_page(pages.AccountHistory) or self.page.account.id != account.id:
self.location('/SAF_ROP?ch4=%s' % account.link_id)
return self.page.get_operations()
def get_coming_operations(self, account):
if not self.is_on_page(pages.AccountComing) or self.page.account.id != account.id:
self.location('/NS_AVEDT?ch4=%s' % account.link_id)
return self.page.get_operations()
def get_transfer_accounts(self):
if not self.is_on_page(pages.TransferPage):
self.location('/NS_VIRDF')
assert self.is_on_page(pages.TransferPage)
return self.page.get_accounts()
def transfer(self, from_id, to_id, amount, reason=None):
if not self.is_on_page(pages.TransferPage):
self.location('/NS_VIRDF')
accounts = self.page.get_accounts()
self.page.transfer(from_id, to_id, amount, reason)
if not self.is_on_page(pages.TransferCompletePage):
raise TransferError('An error occured during transfer')
transfer = Transfer(self.page.get_id())
transfer.amount = amount
transfer.origin = accounts[from_id].label
transfer.recipient = accounts[to_id].label
transfer.date = datetime.now()
return transfer

25
modules/bnporc/errors.py Normal file
View file

@ -0,0 +1,25 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2009-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
__all__ = ['PasswordExpired']
class PasswordExpired(Exception):
pass

BIN
modules/bnporc/favicon.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 5.1 KiB

31
modules/bnporc/pages/__init__.py Normal file
View file

@ -0,0 +1,31 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2009-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from .accounts_list import AccountsList
from .account_coming import AccountComing
from .account_history import AccountHistory
from .transfer import TransferPage, TransferConfirmPage, TransferCompletePage
from .login import LoginPage, ConfirmPage, ChangePasswordPage
class AccountPrelevement(AccountsList): pass
__all__ = ['AccountsList', 'AccountComing', 'AccountHistory', 'LoginPage',
'ConfirmPage', 'AccountPrelevement', 'ChangePasswordPage',
'TransferPage', 'TransferConfirmPage', 'TransferCompletePage']

71
modules/bnporc/pages/account_coming.py Normal file
View file

@ -0,0 +1,71 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2009-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
import re
from datetime import date
from weboob.tools.browser import BasePage
from weboob.capabilities.bank import Operation
__all__ = ['AccountComing']
class AccountComing(BasePage):
LABEL_PATTERNS = [('^FACTURECARTEDU(?P<dd>\d{2})(?P<mm>\d{2})(?P<yy>\d{2})(?P<text>.*)',
u'CB %(yy)s-%(mm)s-%(dd)s: %(text)s'),
('^PRELEVEMENT(?P<text>.*)', 'Order: %(text)s'),
('^ECHEANCEPRET(?P<text>.*)', u'Loan payment n°%(text)s'),
]
def on_loaded(self):
self.operations = []
for tr in self.document.getiterator('tr'):
if tr.attrib.get('class', '') == 'hdoc1' or tr.attrib.get('class', '') == 'hdotc1':
tds = tr.findall('td')
if len(tds) != 3:
continue
d = tds[0].getchildren()[0].attrib.get('name', '')
d = date(int(d[0:4]), int(d[4:6]), int(d[6:8]))
label = u''
label += tds[1].text or u''
label = label.replace(u'\xa0', u'')
for child in tds[1].getchildren():
if child.text: label += child.text
if child.tail: label += child.tail
if tds[1].tail: label += tds[1].tail
label = label.strip()
for pattern, text in self.LABEL_PATTERNS:
m = re.match(pattern, label)
if m:
label = text % m.groupdict()
amount = tds[2].text.replace('.', '').replace(',', '.')
operation = Operation(len(self.operations))
operation.date = d
operation.label = label
operation.amount = float(amount)
self.operations.append(operation)
def get_operations(self):
return self.operations

80
modules/bnporc/pages/account_history.py Normal file
View file

@ -0,0 +1,80 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2009-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
import re
from datetime import date
from weboob.tools.browser import BasePage
from weboob.capabilities.bank import Operation
from weboob.capabilities.base import NotAvailable
__all__ = ['AccountHistory']
class AccountHistory(BasePage):
LABEL_PATTERNS = [(u'^CHEQUEN°(?P<no>.*)', u'CHEQUE', u'N°%(no)s')]
def on_loaded(self):
self.operations = []
for tr in self.document.getiterator('tr'):
if tr.attrib.get('class', '') == 'hdoc1' or tr.attrib.get('class', '') == 'hdotc1':
tds = tr.findall('td')
if len(tds) != 4:
continue
d = date(*reversed([int(x) for x in tds[0].text.split('/')]))
label = u''
label += tds[1].text
label = label.replace(u'\xa0', u'')
for child in tds[1].getchildren():
if child.text: label += child.text
if child.tail: label += child.tail
if tds[1].tail: label += tds[1].tail
label = label.strip()
category = NotAvailable
for pattern, _cat, _lab in self.LABEL_PATTERNS:
m = re.match(pattern, label)
if m:
category = _cat % m.groupdict()
label = _lab % m.groupdict()
break
else:
if ' ' in label:
category, useless, label = [part.strip() for part in label.partition(' ')]
amount = tds[2].text.replace('.', '').replace(',', '.')
# if we don't have exactly one '.', this is not a floatm try the next
operation = Operation(len(self.operations))
if amount.count('.') != 1:
amount = tds[3].text.replace('.', '').replace(',', '.')
operation.amount = float(amount)
else:
operation.amount = - float(amount)
operation.date = d
operation.label = label
operation.category = category
self.operations.append(operation)
def get_operations(self):
return self.operations

78
modules/bnporc/pages/accounts_list.py Normal file
View file

@ -0,0 +1,78 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2009-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
import re
from weboob.capabilities.bank import Account
from weboob.capabilities.base import NotAvailable
from weboob.tools.browser import BasePage
from ..errors import PasswordExpired
__all__ = ['AccountsList']
class AccountsList(BasePage):
LINKID_REGEXP = re.compile(".*ch4=(\w+).*")
def on_loaded(self):
pass
def get_list(self):
l = []
for tr in self.document.getiterator('tr'):
if tr.attrib.get('class', '') == 'comptes':
account = Account()
for td in tr.getiterator('td'):
if td.attrib.get('headers', '').startswith('Numero_'):
id = td.text
account.id = ''.join(id.split(' ')).strip()
elif td.attrib.get('headers', '').startswith('Libelle_'):
a = td.findall('a')
label = unicode(a[0].text)
account.label = label.strip()
m = self.LINKID_REGEXP.match(a[0].attrib.get('href', ''))
if m:
account.link_id = m.group(1)
elif td.attrib.get('headers', '').startswith('Solde'):
a = td.findall('a')
balance = a[0].text
balance = balance.replace('.','').replace(',','.')
account.balance = float(balance)
elif td.attrib.get('headers', '').startswith('Avenir'):
a = td.findall('a')
# Some accounts don't have a "coming"
if len(a):
coming = a[0].text
coming = coming.replace('.','').replace(',','.')
account.coming = float(coming)
else:
account.coming = NotAvailable
l.append(account)
if len(l) == 0:
# oops, no accounts? check if we have not exhausted the allowed use
# of this password
for div in self.document.getroot().cssselect('div.Style_texte_gras'):
if div.text.strip() == 'Vous avez atteint la date de fin de vie de votre code secret.':
raise PasswordExpired(div.text.strip())
return l

114
modules/bnporc/pages/login.py Normal file
View file

@ -0,0 +1,114 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2009-2011 Romain Bignon, Pierre Mazière
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.tools.mech import ClientForm
import urllib
from logging import error
from weboob.tools.browser import BasePage, BrowserUnavailable
from weboob.tools.captcha.virtkeyboard import MappedVirtKeyboard,VirtKeyboardError
import tempfile
__all__ = ['LoginPage', 'ConfirmPage', 'ChangePasswordPage']
class BNPVirtKeyboard(MappedVirtKeyboard):
symbols={'0':'9cc4789a2cb223e8f2d5e676e90264b5',
'1':'e10b58fc085f9683052d5a63c96fc912',
'2':'04ec647e7b3414bcc069f0c54eb55a4c',
'3':'fde84fd9bac725db8463554448f1e469',
'4':'2359eea8671bf112b58264bec0294f71',
'5':'82b55b63480114f04fad8c5c4fa5673a',
'6':'e074864faeaeabb3be3d118192cd8879',
'7':'af5740e4ca71fadc6f4ae1412d864a1c',
'8':'cab759c574038ad89a0e35cc76ab7214',
'9':'828cf0faf86ac78e7f43208907620527'
}
url="/NSImgGrille"
color=27
def __init__(self,basepage):
img=basepage.document.find("//img[@usemap='#MapGril']")
MappedVirtKeyboard.__init__(self,basepage.browser.openurl(self.url),basepage.document,img,self.color)
if basepage.browser.responses_dirname is None:
basepage.browser.responses_dirname = \
tempfile.mkdtemp(prefix='weboob_session_')
self.check_symbols(self.symbols,basepage.browser.responses_dirname)
def get_symbol_code(self,md5sum):
code=MappedVirtKeyboard.get_symbol_code(self,md5sum)
return code[-4:-2]
def get_string_code(self,string):
code=''
for c in string:
code+=self.get_symbol_code(self.symbols[c])
return code
class LoginPage(BasePage):
def on_loaded(self):
for td in self.document.getroot().cssselect('td.LibelleErreur'):
if td.text is None:
continue
msg = td.text.strip()
if 'indisponible' in msg:
raise BrowserUnavailable(msg)
def login(self, login, password):
try:
vk=BNPVirtKeyboard(self)
except VirtKeyboardError,err:
error("Error: %s"%err)
return False
self.browser.select_form('logincanalnet')
# HACK because of fucking malformed HTML, the field isn't recognized by mechanize.
self.browser.controls.append(ClientForm.TextControl('text', 'ch1', {'value': ''}))
self.browser.set_all_readonly(False)
self.browser['ch1'] = login
self.browser['ch5'] = vk.get_string_code(password)
self.browser.submit()
class ConfirmPage(BasePage):
pass
class ChangePasswordPage(BasePage):
def change_password(self, current, new):
try:
vk=BNPVirtKeyboard(self)
except VirtKeyboardError,err:
error("Error: %s"%err)
return False
code_current=vk.get_string_code(current)
code_new=vk.get_string_code(new)
data = {'ch1': code_current,
'ch2': code_new,
'ch3': code_new
}
self.browser.location('/SAF_CHM_VALID', urllib.urlencode(data))

96
modules/bnporc/pages/transfer.py Normal file
View file

@ -0,0 +1,96 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
import re
from weboob.tools.browser import BasePage
from weboob.tools.ordereddict import OrderedDict
from weboob.capabilities.bank import TransferError
__all__ = ['TransferPage', 'TransferConfirmPage', 'TransferCompletePage']
class Account(object):
def __init__(self, id, label, send_checkbox, receive_checkbox):
self.id = id
self.label = label
self.send_checkbox = send_checkbox
self.receive_checkbox = receive_checkbox
class TransferPage(BasePage):
def get_accounts(self):
accounts = OrderedDict()
for table in self.document.getiterator('table'):
if table.attrib.get('cellspacing') == '2':
for tr in table.cssselect('tr.hdoc1, tr.hdotc1'):
tds = tr.findall('td')
id = tds[1].text.replace(u'\xa0', u'')
label = tds[0].text
if label is None and tds[0].find('nobr') is not None:
label = tds[0].find('nobr').text
send_checkbox = tds[4].find('input').attrib['value'] if tds[4].find('input') is not None else None
receive_checkbox = tds[5].find('input').attrib['value'] if tds[5].find('input') is not None else None
account = Account(id, label, send_checkbox, receive_checkbox)
accounts[id] = account
return accounts
def transfer(self, from_id, to_id, amount, reason):
accounts = self.get_accounts()
try:
sender = accounts[from_id]
except KeyError:
raise TransferError('Account %s not found' % from_id)
try:
recipient = accounts[to_id]
except KeyError:
raise TransferError('Recipient %s not found' % to_id)
if sender.send_checkbox is None:
raise TransferError('Unable to make a transfer from %s' % sender.label)
if recipient.receive_checkbox is None:
raise TransferError('Unable to make a transfer to %s' % recipient.label)
self.browser.select_form(nr=0)
self.browser['C1'] = [sender.send_checkbox]
self.browser['C2'] = [recipient.receive_checkbox]
self.browser['T6'] = str(amount).replace('.', ',')
if reason:
self.browser['T5'] = reason.encode('utf-8')
self.browser.submit()
class TransferConfirmPage(BasePage):
def on_loaded(self):
for td in self.document.getroot().cssselect('td#size2'):
raise TransferError(td.text.strip())
for a in self.document.getiterator('a'):
m = re.match('/NSFR\?Action=VIRDA&stp=(\d+)', a.attrib['href'])
if m:
self.browser.location('/NS_VIRDA?stp=%s' % m.group(1))
return
class TransferCompletePage(BasePage):
def get_id(self):
return self.group_dict['id']

31
modules/bnporc/test.py Normal file
View file

@ -0,0 +1,31 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.tools.test import BackendTest
class BNPorcTest(BackendTest):
BACKEND = 'bnporc'
def test_bnporc(self):
l = list(self.backend.iter_accounts())
if len(l) > 0:
a = l[0]
list(self.backend.iter_operations(a))
list(self.backend.iter_history(a))

24
modules/boursorama/__init__.py Normal file
View file

@ -0,0 +1,24 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Gabriel Kerneis
# Copyright(C) 2010-2011 Jocelyn Jaubert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from .backend import BoursoramaBackend
__all__ = ['BoursoramaBackend']

73
modules/boursorama/backend.py Normal file
View file

@ -0,0 +1,73 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Gabriel Kerneis
# Copyright(C) 2010-2011 Jocelyn Jaubert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
# python2.5 compatibility
from __future__ import with_statement
from weboob.capabilities.bank import ICapBank, AccountNotFound
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.tools.value import ValueBackendPassword
from .browser import Boursorama
__all__ = ['BoursoramaBackend']
class BoursoramaBackend(BaseBackend, ICapBank):
NAME = 'boursorama'
MAINTAINER = 'Gabriel Kerneis'
EMAIL = 'gabriel@kerneis.info'
VERSION = '0.a'
LICENSE = 'AGPLv3+'
DESCRIPTION = u'Boursorama french bank\'s website'
CONFIG = BackendConfig(ValueBackendPassword('login', label='Account ID', masked=False),
ValueBackendPassword('password', label='Password'))
BROWSER = Boursorama
def create_default_browser(self):
return self.create_browser(self.config['login'].get(),
self.config['password'].get())
def iter_accounts(self):
for account in self.browser.get_accounts_list():
yield account
def get_account(self, _id):
if not _id.isdigit():
raise AccountNotFound()
with self.browser:
account = self.browser.get_account(_id)
if account:
return account
else:
raise AccountNotFound()
def iter_history(self, account):
with self.browser:
for history in self.browser.get_history(account):
yield history
def iter_operations(self, account):
with self.browser:
for coming in self.browser.get_coming_operations(account):
yield coming

98
modules/boursorama/browser.py Normal file
View file

@ -0,0 +1,98 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Gabriel Kerneis
# Copyright(C) 2010-2011 Jocelyn Jaubert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword
from boursorama import pages
from datetime import date
from dateutil.relativedelta import relativedelta
__all__ = ['Boursorama']
class Boursorama(BaseBrowser):
DOMAIN = 'www.boursorama.com'
PROTOCOL = 'https'
ENCODING = None # refer to the HTML encoding
PAGES = {
'.*connexion.phtml.*': pages.LoginPage,
'.*/comptes/synthese.phtml': pages.AccountsList,
'.*/comptes/banque/detail/mouvements.phtml.*': pages.AccountHistory,
}
def __init__(self, *args, **kwargs):
BaseBrowser.__init__(self, *args, **kwargs)
def home(self):
self.location('https://' + self.DOMAIN + '/connexion.phtml')
def is_logged(self):
return not self.is_on_page(pages.LoginPage)
def login(self):
assert isinstance(self.username, basestring)
assert isinstance(self.password, basestring)
assert self.password.isdigit()
if not self.is_on_page(pages.LoginPage):
self.location('https://' + self.DOMAIN + '/connexion.phtml')
self.page.login(self.username, self.password)
if self.is_on_page(pages.LoginPage):
raise BrowserIncorrectPassword()
def get_accounts_list(self):
if not self.is_on_page(pages.AccountsList):
self.location('/comptes/synthese.phtml')
return self.page.get_list()
def get_account(self, id):
assert isinstance(id, basestring)
if not self.is_on_page(pages.AccountsList):
self.location('/comptes/synthese.phtml')
l = self.page.get_list()
for a in l:
if a.id == id:
return a
return None
def get_history(self, account):
self.location(account.link_id)
operations = self.page.get_operations()
# load last month as well
target = date.today() - relativedelta( months = 1 )
self.location(account.link_id + ("&month=%d&year=%d" % (target.month, target.year)))
operations += self.page.get_operations()
# and the month before, just in case you're greedy
target = date.today() - relativedelta( months = 2 )
self.location(account.link_id + ("&month=%d&year=%d" % (target.month, target.year)))
operations += self.page.get_operations()
return operations
def transfer(self, from_id, to_id, amount, reason=None):
raise NotImplementedError()

31
modules/boursorama/pages/__init__.py Normal file
View file

@ -0,0 +1,31 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Gabriel Kerneis
# Copyright(C) 2010-2011 Jocelyn Jaubert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from .account_history import AccountHistory
from .accounts_list import AccountsList
from .login import LoginPage
class AccountPrelevement(AccountsList): pass
__all__ = ['LoginPage',
'AccountsList',
'AccountHistory',
]

68
modules/boursorama/pages/account_history.py Normal file
View file

@ -0,0 +1,68 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Gabriel Kerneis
# Copyright(C) 2009-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from datetime import date
from weboob.tools.browser import BasePage
from weboob.capabilities.bank import Operation
__all__ = ['AccountHistory']
class AccountHistory(BasePage):
def on_loaded(self):
self.operations = []
for form in self.document.getiterator('form'):
if form.attrib.get('name', '') == 'marques':
for tr in form.getiterator('tr'):
tds = tr.findall('td')
if len(tds) != 6:
continue
# tds[0]: operation
# tds[1]: valeur
d = date(*reversed([int(x) for x in tds[1].text.split('/')]))
labeldiv = tds[2].find('div')
label = u''
label += labeldiv.text
label = label.strip(u' \n\t')
category = labeldiv.attrib.get('title', '')
useless, sep, category = [part.strip() for part in category.partition(':')]
amount = tds[3].text
if amount == None:
amount = tds[4].text
amount = amount.strip(u' \n\t\x80').replace(' ', '').replace(',', '.')
# if we don't have exactly one '.', this is not a floatm try the next
operation = Operation(len(self.operations))
operation.amount = float(amount)
operation.date = d
operation.label = label
operation.category = category
self.operations.append(operation)
def get_operations(self):
return self.operations

67
modules/boursorama/pages/accounts_list.py Normal file
View file

@ -0,0 +1,67 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Gabriel Kerneis
# Copyright(C) 2010-2011 Jocelyn Jaubert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.capabilities.bank import Account
from weboob.tools.browser import BasePage
class AccountsList(BasePage):
def on_loaded(self):
pass
def get_list(self):
l = []
for div in self.document.getiterator('div'):
if div.attrib.get('id', '') == 'synthese-list':
for tr in div.getiterator('tr'):
account = Account()
for td in tr.getiterator('td'):
if td.attrib.get('class', '') == 'account-cb':
break
elif td.attrib.get('class', '') == 'account-name':
a = td.find('a')
account.label = a.text
account.link_id = a.get('href', '')
elif td.attrib.get('class', '') == 'account-number':
id = td.text
id = id.strip(u' \n\t')
account.id = id
elif td.attrib.get('class', '') == 'account-total':
span = td.find('span')
if span == None:
balance = td.text
else:
balance = span.text
balance = balance.strip(u' \n\t€+').replace(',','.').replace(' ','')
if balance != "":
account.balance = float(balance)
else:
account.balance = 0.0
else:
# because of some weird useless <tr>
if account.id != 0:
l.append(account)
return l

48
modules/boursorama/pages/login.py Normal file
View file

@ -0,0 +1,48 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Jocelyn Jaubert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.tools.browser import BasePage
__all__ = ['LoginPage']
class LoginPage(BasePage):
def on_loaded(self):
pass
# for td in self.document.getroot().cssselect('td.LibelleErreur'):
# if td.text is None:
# continue
# msg = td.text.strip()
# if 'indisponible' in msg:
# raise BrowserUnavailable(msg)
def login(self, login, password):
DOMAIN = self.browser.DOMAIN
url_login = 'https://' + DOMAIN + '/connexion.phtml'
self.browser.openurl(url_login)
self.browser.select_form('identification')
self.browser.set_all_readonly(False)
self.browser['login'] = login
self.browser['password'] = password
self.browser.submit()

32
modules/boursorama/test.py Normal file
View file

@ -0,0 +1,32 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Gabriel Kerneis
# Copyright(C) 2010-2011 Jocelyn Jaubert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.tools.test import BackendTest
class BoursoramaTest(BackendTest):
BACKEND = 'boursorama'
def test_boursorama(self):
l = list(self.backend.iter_accounts())
if len(l) > 0:
a = l[0]
list(self.backend.iter_operations(a))
list(self.backend.iter_history(a))

3
modules/bouygues/__init__.py Normal file
View file

@ -0,0 +1,3 @@
from .backend import BouyguesBackend
__all__ = ['BouyguesBackend']

52
modules/bouygues/backend.py Normal file
View file

@ -0,0 +1,52 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from __future__ import with_statement
from weboob.capabilities.messages import CantSendMessage, ICapMessages, ICapMessagesPost
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.tools.value import ValueBackendPassword, Value
from .browser import BouyguesBrowser
__all__ = ['BouyguesBackend']
class BouyguesBackend(BaseBackend, ICapMessages, ICapMessagesPost):
NAME = 'bouygues'
MAINTAINER = 'Christophe Benz'
EMAIL = 'christophe.benz@gmail.com'
VERSION = '0.a'
DESCRIPTION = 'Bouygues french mobile phone provider'
LICENSE = 'AGPLv3+'
CONFIG = BackendConfig(Value('login', label='Login'),
ValueBackendPassword('password', label='Password'))
BROWSER = BouyguesBrowser
ACCOUNT_REGISTER_PROPERTIES = None
def create_default_browser(self):
return self.create_browser(self.config['login'].get(), self.config['password'].get())
def post_message(self, message):
if not message.content.strip():
raise CantSendMessage(u'Message content is empty.')
with self.browser:
self.browser.post_message(message)

62
modules/bouygues/browser.py Normal file
View file

@ -0,0 +1,62 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from .pages.compose import ComposeFrame, ComposePage, ConfirmPage, SentPage
from .pages.login import LoginPage, LoginSASPage
from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword
__all__ = ['BouyguesBrowser']
class BouyguesBrowser(BaseBrowser):
DOMAIN = 'www.bouyguestelecom.fr'
PAGES = {
'http://www.espaceclient.bouyguestelecom.fr/ECF/jsf/client/envoiSMS/viewEnvoiSMS.jsf': ComposePage,
'http://www.mobile.service.bbox.bouyguestelecom.fr/services/SMSIHD/sendSMS.phtml': ComposeFrame,
'http://www.mobile.service.bbox.bouyguestelecom.fr/services/SMSIHD/confirmSendSMS.phtml': ConfirmPage,
'https://www.espaceclient.bouyguestelecom.fr/ECF/jsf/submitLogin.jsf': LoginPage,
'https://www.espaceclient.bouyguestelecom.fr/ECF/SasUnifie': LoginSASPage,
'http://www.mobile.service.bbox.bouyguestelecom.fr/services/SMSIHD/resultSendSMS.phtml': SentPage,
}
def home(self):
self.location('http://www.espaceclient.bouyguestelecom.fr/ECF/jsf/client/envoiSMS/viewEnvoiSMS.jsf')
def is_logged(self):
return 'code' not in [form.name for form in self.forms()]
def login(self):
self.location('https://www.espaceclient.bouyguestelecom.fr/ECF/jsf/submitLogin.jsf', no_login=True)
self.page.login(self.username, self.password)
assert self.is_on_page(LoginSASPage)
self.page.login()
if not self.is_logged():
raise BrowserIncorrectPassword()
def post_message(self, message):
if not self.is_on_page(ComposeFrame):
self.home()
self.location('http://www.mobile.service.bbox.bouyguestelecom.fr/services/SMSIHD/sendSMS.phtml')
self.page.post_message(message)
assert self.is_on_page(ConfirmPage)
self.page.confirm()
assert self.is_on_page(SentPage)

BIN
modules/bouygues/favicon.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 5.1 KiB

0
modules/bouygues/pages/__init__.py Normal file
View file

53
modules/bouygues/pages/compose.py Normal file
View file

@ -0,0 +1,53 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
import re
from weboob.capabilities.messages import CantSendMessage
from weboob.tools.browser import BasePage
__all__ = ['ComposeFrame', 'ComposePage', 'ConfirmPage', 'SentPage']
class ComposeFrame(BasePage):
phone_regex = re.compile('^(\+33|0033|0)(6|7)(\d{8})$')
def post_message(self, message):
receiver = message.thread.id
if self.phone_regex.match(receiver) is None:
raise CantSendMessage(u'Invalid receiver: %s' % receiver)
self.browser.select_form(nr=0)
self.browser['fieldMsisdn'] = receiver
self.browser['fieldMessage'] = message.content.encode('utf-8')
self.browser.submit()
class ComposePage(BasePage):
pass
class ConfirmPage(BasePage):
def confirm(self):
self.browser.location('http://www.mobile.service.bbox.bouyguestelecom.fr/services/SMSIHD/resultSendSMS.phtml')
class SentPage(BasePage):
pass

38
modules/bouygues/pages/login.py Normal file
View file

@ -0,0 +1,38 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.tools.browser import BasePage
__all__ = ['LoginPage', 'LoginSASPage']
class LoginPage(BasePage):
def login(self, login, password):
self.browser.select_form(name='code')
self.browser['j_username'] = login
self.browser['j_password'] = password
self.browser.submit()
class LoginSASPage(BasePage):
def login(self):
self.browser.select_form(name='redirect')
self.browser.submit()

27
modules/bouygues/test.py Normal file
View file

@ -0,0 +1,27 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.tools.test import BackendTest
class BouyguesTest(BackendTest):
BACKEND = 'bouygues'
def test_bouygues(self):
pass

22
modules/bp/__init__.py Normal file
View file

@ -0,0 +1,22 @@
# -*- coding: utf-8 -*-
#
# Copyright(C) 2010-2011 Nicolas Duhamel <nicolas@NicolasDesktop>
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from .backend import BPBackend
__all__ = ['BPBackend']

66
modules/bp/backend.py Normal file
View file

@ -0,0 +1,66 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.capabilities.bank import ICapBank, AccountNotFound
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.tools.value import ValueBackendPassword
from .browser import BPBrowser
__all__ = ['BPBackend']
class BPBackend(BaseBackend, ICapBank):
NAME = 'bp'
MAINTAINER = 'Nicolas Duhamel'
EMAIL = 'nicolas@jombi.fr'
VERSION = '0.a'
LICENSE = 'AGPLv3+'
DESCRIPTION = u'La banque postale, French bank'
CONFIG = BackendConfig(ValueBackendPassword('login', label='Account ID', masked=False),
ValueBackendPassword('password', label='Password'))
BROWSER = BPBrowser
def create_default_browser(self):
return self.create_browser(self.config['login'].get(), self.config['password'].get())
def iter_accounts(self):
for account in self.browser.get_accounts_list():
yield account
def get_account(self, _id):
account = self.browser.get_account(_id)
if account:
return account
else:
raise AccountNotFound()
def iter_history(self, account):
for history in self.browser.get_history(account):
yield history
def transfer(self, id_from, id_to, amount, reason=None):
from_account = self.get_account(id_from)
to_account = self.get_account(id_to)
#TODO: retourner le numero du virement
#TODO: support the 'reason' parameter
return self.browser.make_transfer(from_account, to_account, amount)

111
modules/bp/browser.py Normal file
View file

@ -0,0 +1,111 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from datetime import datetime
from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword, BrowserBanned
from .pages import LoginPage, Initident, CheckPassword, repositionnerCheminCourant, BadLoginPage, AccountDesactivate, \
AccountList, AccountHistory, \
TransferChooseAccounts, CompleteTransfer, TransferConfirm, TransferSummary
from weboob.capabilities.bank import Transfer
__all__ = ['BPBrowser']
class BPBrowser(BaseBrowser):
DOMAIN = 'voscomptesenligne.labanquepostale.fr'
PROTOCOL = 'https'
ENCODING = None # refer to the HTML encoding
PAGES = {r'.*wsost/OstBrokerWeb/loginform.*' : LoginPage,
r'.*authentification/repositionnerCheminCourant-identif.ea' : repositionnerCheminCourant,
r'.*authentification/initialiser-identif.ea' : Initident,
r'.*authentification/verifierMotDePasse-identif.ea' : CheckPassword,
r'.*synthese_assurancesEtComptes/afficheSynthese-synthese\.ea' : AccountList,
r'.*synthese_assurancesEtComptes/rechercheContratAssurance-synthese.ea' : AccountList,
r'.*CCP/releves_ccp/releveCPP-releve_ccp\.ea' : AccountHistory,
r'.*CNE/releveCNE/releveCNE-releve_cne\.ea' : AccountHistory,
r'.*/virementSafran_aiguillage/init-saisieComptes\.ea' : TransferChooseAccounts,
r'.*/virementSafran_aiguillage/formAiguillage-saisieComptes\.ea' : CompleteTransfer,
r'.*/virementSafran_national/validerVirementNational-virementNational.ea' : TransferConfirm,
r'.*/virementSafran_national/confirmerVirementNational-virementNational.ea' : TransferSummary,
r'.*ost/messages\.CVS\.html\?param=0x132120c8.*' : BadLoginPage,
r'.*ost/messages\.CVS\.html\?param=0x132120cb.*' : AccountDesactivate,
}
def __init__(self, *args, **kwargs):
kwargs['parser'] = ('lxml',)
BaseBrowser.__init__(self, *args, **kwargs)
def home(self):
self.location('https://voscomptesenligne.labanquepostale.fr/wsost/OstBrokerWeb/loginform?TAM_OP=login&'
'ERROR_CODE=0x00000000&URL=%2Fvoscomptes%2FcanalXHTML%2Fidentif.ea%3Forigin%3Dparticuliers')
def is_logged(self):
return not self.is_on_page(LoginPage)
def login(self):
if not self.is_on_page(LoginPage):
self.location('https://voscomptesenligne.labanquepostale.fr/wsost/OstBrokerWeb/loginform?TAM_OP=login&'
'ERROR_CODE=0x00000000&URL=%2Fvoscomptes%2FcanalXHTML%2Fidentif.ea%3Forigin%3Dparticuliers',
no_login=True)
self.page.login(self.username, self.password)
if self.is_on_page(BadLoginPage):
raise BrowserIncorrectPassword()
if self.is_on_page(AccountDesactivate):
raise BrowserBanned()
def get_accounts_list(self):
self.location("https://voscomptesenligne.labanquepostale.fr/voscomptes/canalXHTML/comptesCommun/synthese_assurancesEtComptes/rechercheContratAssurance-synthese.ea")
return self.page.get_accounts_list()
def get_account(self, id):
if not self.is_on_page(AccountList):
self.location("https://voscomptesenligne.labanquepostale.fr/voscomptes/canalXHTML/comptesCommun/synthese_assurancesEtComptes/rechercheContratAssurance-synthese.ea")
return self.page.get_account(id)
def get_history(self, Account):
self.location(Account.link_id)
return self.page.get_history()
def make_transfer(self, from_account, to_account, amount):
self.location('https://voscomptesenligne.labanquepostale.fr/voscomptes/canalXHTML/virement/virementSafran_aiguillage/init-saisieComptes.ea')
self.page.set_accouts(from_account, to_account)
#TODO: Check
self.page.complete_transfer(amount)
self.page.confirm()
id_transfer = self.page.get_transfer_id()
transfer = Transfer(id_transfer)
transfer.amount = amount
transfer.origin = from_account.label
transfer.recipient = to_account.label
transfer.date = datetime.now()
return transfer

BIN
modules/bp/favicon.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 1.7 KiB

28
modules/bp/pages/__init__.py Normal file
View file

@ -0,0 +1,28 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from .login import LoginPage, Initident, CheckPassword,repositionnerCheminCourant, BadLoginPage, AccountDesactivate
from .accountlist import AccountList
from .accounthistory import AccountHistory
from .transfer import TransferChooseAccounts, CompleteTransfer, TransferConfirm, TransferSummary
__all__ = ['LoginPage','Initident', 'CheckPassword', 'repositionnerCheminCourant', "AccountList", 'AccountHistory', 'BadLoginPage',
'AccountDesactivate', 'TransferChooseAccounts', 'CompleteTransfer', 'TransferConfirm', 'TransferSummary']

60
modules/bp/pages/accounthistory.py Normal file
View file

@ -0,0 +1,60 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
import re
from weboob.capabilities.bank import Operation
from weboob.tools.browser import BasePage
__all__ = ['AccountHistory']
class AccountHistory(BasePage):
def get_history(self):
mvt_table = self.document.xpath("//table[@id='mouvements']", smart_strings=False)[0]
mvt_ligne = mvt_table.xpath("./tbody/tr")
operations = []
for mvt in mvt_ligne:
operation = Operation(len(operations))
operation.date = mvt.xpath("./td/span")[0].text
tmp = mvt.xpath("./td/span")[1]
operation.label = unicode(self.parser.tocleanstring(tmp))
r = re.compile(r'\d+')
tmp = mvt.xpath("./td/span/strong")
if not tmp:
tmp = mvt.xpath("./td/span")
amount = None
for t in tmp:
if r.search(t.text):
amount = t.text
amount = ''.join( amount.replace('.', '').replace(',', '.').split() )
if amount[0] == "-":
operation.amount = -float(amount[1:])
else:
operation.amount = float(amount)
operations.append(operation)
return operations

69
modules/bp/pages/accountlist.py Normal file
View file

@ -0,0 +1,69 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.capabilities.bank import Account, AccountNotFound
from weboob.tools.browser import BasePage
__all__ = ['AccountList']
class AccountList(BasePage):
def on_loaded(self):
self.account_list = []
self.parse_table('comptes')
self.parse_table('comptesEpargne')
self.parse_table('comptesTitres')
self.parse_table('comptesVie')
self.parse_table('comptesRetraireEuros')
def get_accounts_list(self):
return self.account_list
def parse_table(self, what):
tables = self.document.xpath("//table[@id='%s']" % what, smart_strings=False)
if len(tables) < 1:
return
lines = tables[0].xpath(".//tbody/tr")
for line in lines:
account = Account()
tmp = line.xpath("./td//a")[0]
account.label = tmp.text
account.link_id = tmp.get("href")
tmp = line.xpath("./td//strong")
if len(tmp) != 2:
tmp_id = line.xpath("./td//span")[1].text
tmp_balance = tmp[0].text
else:
tmp_id = tmp[0].text
tmp_balance = tmp[1].text
account.id = tmp_id
account.balance = float(''.join(tmp_balance.replace('.','').replace(',','.').split()))
self.account_list.append(account)
def get_account(self, id):
for account in self.account_list:
if account.id == id:
return account
raise AccountNotFound('Unable to find account: %s' % id)

74
modules/bp/pages/login.py Normal file
View file

@ -0,0 +1,74 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
import hashlib
from weboob.tools.browser import BasePage
__all__ = ['LoginPage', 'BadLoginPage', 'AccountDesactivate', 'Initident', 'CheckPassword', 'repositionnerCheminCourant']
def md5(f):
md5 = hashlib.md5()
md5.update(f.read())
return md5.hexdigest()
class LoginPage(BasePage):
def on_loaded(self):
pass
def login(self, login, pwd):
LOCAL_HASH = ['a02574d7bf67677d2a86b7bfc5e864fe', 'eb85e1cc45dd6bdb3cab65c002d7ac8a',
'596e6fbd54d5b111fe5df8a4948e80a4', '9cdc989a4310554e7f5484d0d27a86ce',
'0183943de6c0e331f3b9fc49c704ac6d', '291b9987225193ab1347301b241e2187',
'163279f1a46082408613d12394e4042a', 'b0a9c740c4cada01eb691b4acda4daea',
'3c4307ee92a1f3b571a3c542eafcb330', 'dbccecfa2206bfdb4ca891476404cc68']
process = lambda i: md5(self.browser.openurl(('https://voscomptesenligne.labanquepostale.fr/wsost/OstBrokerWeb/loginform?imgid=%d&0.25122230781963073' % i)))
keypad = [process(i) for i in range(10)]
correspondance = [keypad.index(i) for i in LOCAL_HASH]
newpassword = ''.join(str(correspondance[int(c)]) for c in pwd)
self.browser.select_form(name='formAccesCompte')
self.browser.set_all_readonly(False)
self.browser['password'] = newpassword
self.browser['username'] = login
self.browser.submit()
class repositionnerCheminCourant(BasePage):
def on_loaded(self):
self.browser.open("https://voscomptesenligne.labanquepostale.fr/voscomptes/canalXHTML/securite/authentification/initialiser-identif.ea")
class Initident(BasePage):
def on_loaded(self):
self.browser.open("https://voscomptesenligne.labanquepostale.fr/voscomptes/canalXHTML/securite/authentification/verifierMotDePasse-identif.ea")
class CheckPassword(BasePage):
def on_loaded(self):
self.browser.open("https://voscomptesenligne.labanquepostale.fr/voscomptes/canalXHTML/comptesCommun/synthese_assurancesEtComptes/init-synthese.ea")
class BadLoginPage(BasePage):
pass
class AccountDesactivate(BasePage):
pass

74
modules/bp/pages/transfer.py Normal file
View file

@ -0,0 +1,74 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
import re
from weboob.capabilities.bank import TransferError
from weboob.tools.browser import BasePage
from weboob.tools.misc import to_unicode
__all__ = ['TransferChooseAccounts', 'CompleteTransfer', 'TransferConfirm', 'TransferSummary']
class TransferChooseAccounts(BasePage):
def set_accouts(self, from_account, to_account):
self.browser.select_form(name="AiguillageForm")
self.browser["idxCompteEmetteur"] = [from_account.id]
self.browser["idxCompteReceveur"] = [to_account.id]
self.browser.submit()
class CompleteTransfer(BasePage):
def complete_transfer(self, amount):
self.browser.select_form(name="virement_unitaire_saisie_saisie_virement_sepa")
self.browser["montant"] = str(amount)
self.browser.submit()
class TransferConfirm(BasePage):
def confirm(self):
self.browser.location('https://voscomptesenligne.labanquepostale.fr/voscomptes/canalXHTML/virement/virementSafran_national/confirmerVirementNational-virementNational.ea')
class TransferSummary(BasePage):
def get_transfer_id(self):
p = self.document.xpath("//div[@id='main']/div/p")[0]
#HACK for deal with bad encoding ...
try:
text = p.text
except UnicodeDecodeError, error:
text = error.object.strip()
match = re.search("Votre virement N.+ ([0-9]+) ", text)
if match:
id_transfer = match.groups()[0]
return id_transfer
if text.startswith(u"Votre virement n'a pas pu"):
if p.find('br') is not None:
errmsg = to_unicode(p.find('br').tail).strip()
raise TransferError('Unable to process transfer: %s' % errmsg)
else:
self.browser.logger.warning('Unable to find the error reason')
self.browser.logger.error('Unable to parse the text result: %r' % text)
raise TransferError('Unable to process transfer: %r' % text)

22
modules/canalplus/__init__.py Normal file
View file

@ -0,0 +1,22 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from .backend import CanalplusBackend
__all__ = ['CanalplusBackend']

71
modules/canalplus/backend.py Normal file
View file

@ -0,0 +1,71 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from __future__ import with_statement
from weboob.capabilities.video import ICapVideo
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.tools.value import Value
from .browser import CanalplusBrowser
from .pages import CanalplusVideo
from weboob.capabilities.collection import ICapCollection
__all__ = ['CanalplusBackend']
class CanalplusBackend(BaseBackend, ICapVideo, ICapCollection):
NAME = 'canalplus'
MAINTAINER = 'Nicolas Duhamel'
EMAIL = 'nicolas@jombi.fr'
VERSION = '0.a'
DESCRIPTION = 'Canal plus french TV'
LICENSE = 'AGPLv3+'
CONFIG = BackendConfig(Value('quality', label='Quality of videos', choices=['hd', 'sd'], default='hd'))
BROWSER = CanalplusBrowser
def create_default_browser(self):
return self.create_browser(quality=self.config['quality'].get())
def iter_search_results(self, pattern=None, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False, max_results=None):
with self.browser:
return self.browser.iter_search_results(pattern)
def get_video(self, _id):
with self.browser:
return self.browser.get_video(_id)
def fill_video(self, video, fields):
if fields != ['thumbnail']:
# if we don't want only the thumbnail, we probably want also every fields
with self.browser:
video = self.browser.get_video(CanalplusVideo.id2url(video.id), video)
if 'thumbnail' in fields and video.thumbnail:
with self.browser:
video.thumbnail.data = self.browser.readurl(video.thumbnail.url)
return video
OBJECTS = {CanalplusVideo: fill_video}
def iter_resources(self, splited_path):
with self.browser:
return self.browser.iter_resources(splited_path)

93
modules/canalplus/browser.py Normal file
View file

@ -0,0 +1,93 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
import urllib
import lxml.etree
from weboob.tools.browser import BaseBrowser
from weboob.tools.browser.decorators import id2url
from .pages import InitPage, CanalplusVideo, VideoPage
from weboob.capabilities.collection import Collection, CollectionNotFound
__all__ = ['CanalplusBrowser']
class XMLParser(object):
def parse(self, data, encoding=None):
if encoding is None:
parser = None
else:
parser = lxml.etree.XMLParser(encoding=encoding, strip_cdata=False)
return lxml.etree.XML(data.get_data(), parser)
class CanalplusBrowser(BaseBrowser):
DOMAIN = u'service.canal-plus.com'
ENCODING = 'utf-8'
PAGES = {
r'http://service.canal-plus.com/video/rest/initPlayer/cplus/': InitPage,
r'http://service.canal-plus.com/video/rest/search/cplus/.*': VideoPage,
r'http://service.canal-plus.com/video/rest/getVideosLiees/cplus/(?P<id>.+)': VideoPage,
r'http://service.canal-plus.com/video/rest/getMEAs/cplus/.*': VideoPage,
}
#We need lxml.etree.XMLParser for read CDATA
PARSER = XMLParser()
FORMATS = {
'sd': 'BAS_DEBIT',
'hd': 'HD',
}
def __init__(self, quality, *args, **kwargs):
BaseBrowser.__init__(self, parser= self.PARSER, *args, **kwargs)
if quality in self.FORMATS:
self.quality = self.FORMATS[quality]
else:
self.quality = 'HD'
def home(self):
self.location('http://service.canal-plus.com/video/rest/initPlayer/cplus/')
def iter_search_results(self, pattern):
self.location('http://service.canal-plus.com/video/rest/search/cplus/' + urllib.quote_plus(pattern.encode('utf-8')))
return self.page.iter_results()
@id2url(CanalplusVideo.id2url)
def get_video(self, url, video=None):
self.location(url)
return self.page.get_video(video, self.quality)
def iter_resources(self, splited_path):
self.home()
collections = self.page.collections
def walk_res(path, collections):
if len(path) == 0 or not isinstance(collections, (list, Collection)):
return collections
i = path[0]
if i not in [collection.title for collection in collections]:
raise CollectionNotFound()
return walk_res(path[1:], [collection.children for collection in collections if collection.title == i][0])
return walk_res(splited_path, collections)

BIN
modules/canalplus/favicon.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 652 B

24
modules/canalplus/pages/__init__.py Normal file
View file

@ -0,0 +1,24 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from .initpage import InitPage
from .video import CanalplusVideo
from .videopage import VideoPage
__all__ = ['InitPage', 'VideoPage', 'CanalplusVideo']

49
modules/canalplus/pages/initpage.py Normal file
View file

@ -0,0 +1,49 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.tools.browser import BasePage
from weboob.capabilities.collection import Collection
__all__ = ['InitPage']
class InitPage(BasePage):
def on_loaded(self):
self.collections = []
def do(id):
self.browser.location("http://service.canal-plus.com/video/rest/getMEAs/cplus/" + id)
return self.browser.page.iter_channel()
### Parse liste des channels
for elem in self.document[2].getchildren():
coll = Collection()
for e in elem.getchildren():
if e.tag == "NOM":
coll.title = e.text.strip().encode('utf-8')
elif e.tag == "SELECTIONS":
for select in e:
sub = Collection(title=select[1].text.strip().encode('utf-8'))
sub.id = select[0].text
sub.children = do
coll.appendchild(sub)
self.collections.append(coll)

31
modules/canalplus/pages/video.py Normal file
View file

@ -0,0 +1,31 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.capabilities.video import BaseVideo
__all__ = ['CanalplusVideo']
class CanalplusVideo(BaseVideo):
swf_player = False
@classmethod
def id2url(cls, _id):
return 'http://service.canal-plus.com/video/rest/getVideosLiees/cplus/%s' % _id

96
modules/canalplus/pages/videopage.py Normal file
View file

@ -0,0 +1,96 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Nicolas Duhamel
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from datetime import datetime
from weboob.capabilities.base import NotAvailable
from weboob.tools.capabilities.thumbnail import Thumbnail
from weboob.tools.browser import BasePage
from .video import CanalplusVideo
__all__ = ['VideoPage']
class VideoPage(BasePage):
def parse_video(self, el, video=None, quality=None):
_id = el.find('ID').text
if _id == '-1':
# means the video is not found
return None
if not video:
video = CanalplusVideo(_id)
infos = el.find('INFOS')
video.title = u''
for part in infos.find('TITRAGE'):
if len(part.text.strip()) == 0:
continue
if len(video.title) > 0:
video.title += u' — '
video.title += part.text.strip()
video.description = infos.find('DESCRIPTION').text
media = el.find('MEDIA')
url = media.find('IMAGES').find('PETIT').text
if url:
video.thumbnail = Thumbnail(url)
else:
video.thumbnail = NotAvailable
lastest_format = None
for format in media.find('VIDEOS'):
if format.text is None:
continue
if format.tag == quality:
video.url = format.text
break
lastest_format = format
if not video.url and lastest_format is not None:
video.url = lastest_format.text
day, month, year = map(int, infos.find('PUBLICATION').find('DATE').text.split('/'))
hour, minute, second = map(int, infos.find('PUBLICATION').find('HEURE').text.split(':'))
video.date = datetime(year, month, day, hour, minute, second)
return video
def iter_results(self):
for vid in self.document.getchildren():
yield self.parse_video(vid)
def iter_channel(self):
for vid in self.document.getchildren():
yield self.parse_video_channel(vid)
def parse_video_channel(self,el):
_id = el[0].text
video = CanalplusVideo(_id)
video.title = el[2][3][0].text
video.date = datetime.now()
return video
def get_video(self, video, quality):
_id = self.group_dict['id']
for vid in self.document.getchildren():
if not _id in vid.find('ID').text:
continue
return self.parse_video(vid, video, quality)

31
modules/canalplus/test.py Normal file
View file

@ -0,0 +1,31 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.tools.test import BackendTest
class CanalPlusTest(BackendTest):
BACKEND = 'canalplus'
def test_canalplus(self):
l = list(self.backend.iter_search_results('guignol'))
self.assertTrue(len(l) > 0)
v = l[0]
self.backend.fillobj(v, ('url',))
self.assertTrue(v.url and v.url.startswith('rtmp://'), 'URL for video "%s" not found: %s' % (v.id, v.url))

23
modules/canaltp/__init__.py Normal file
View file

@ -0,0 +1,23 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from .backend import CanalTPBackend
__all__ = ['CanalTPBackend']

49
modules/canaltp/backend.py Normal file
View file

@ -0,0 +1,49 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.capabilities.travel import ICapTravel, Station, Departure
from weboob.tools.backend import BaseBackend
from .browser import CanalTP
__all__ = ['CanalTPBackend']
class CanalTPBackend(BaseBackend, ICapTravel):
NAME = 'canaltp'
MAINTAINER = 'Romain Bignon'
EMAIL = 'romain@weboob.org'
VERSION = '0.a'
LICENSE = 'AGPLv3+'
DESCRIPTION = "French trains"
BROWSER = CanalTP
def iter_station_search(self, pattern):
for _id, name in self.browser.iter_station_search(pattern):
yield Station(_id, name)
def iter_station_departures(self, station_id, arrival_id=None):
for i, d in enumerate(self.browser.iter_station_departures(station_id, arrival_id)):
departure = Departure(i, d['type'], d['time'])
departure.departure_station = d['departure']
departure.arrival_station = d['arrival']
departure.late = d['late']
departure.information = d['late_reason']
yield departure

76
modules/canaltp/browser.py Normal file
View file

@ -0,0 +1,76 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from datetime import datetime, date, time
from weboob.tools.browser import BaseBrowser
from weboob.tools.misc import to_unicode
from weboob.tools.browser import BrokenPageError
__all__ = ['CanalTP']
class CanalTP(BaseBrowser):
DOMAIN = 'widget.canaltp.fr'
def __init__(self, **kwargs):
BaseBrowser.__init__(self, '', **kwargs)
def iter_station_search(self, pattern):
url = u'http://widget.canaltp.fr/Prochains_departs_15122009/dev/gare.php?txtrech=%s' % unicode(pattern)
result = self.openurl(url.encode('utf-8')).read()
for station in result.split('&'):
try:
_id, name = station.split('=')
except ValueError:
continue
else:
yield _id, to_unicode(name)
def iter_station_departures(self, station_id, arrival_id=None):
url = u'http://widget.canaltp.fr/Prochains_departs_15122009/dev/index.php?gare=%s' % unicode(station_id)
result = self.openurl(url.encode('utf-8')).read()
result = result
departure = ''
for line in result.split('&'):
if not '=' in line:
raise BrokenPageError('Unable to parse result: %s' % line)
key, value = line.split('=', 1)
if key == 'nomgare':
departure = value
elif key.startswith('ligne'):
_type, unknown, _time, arrival, served, late, late_reason = value.split(';', 6)
yield {'type': to_unicode(_type),
'time': datetime.combine(date.today(), time(*[int(x) for x in _time.split(':')])),
'departure': to_unicode(departure),
'arrival': to_unicode(arrival).strip(),
'late': late and time(0, int(late.split()[0])) or time(),
'late_reason': to_unicode(late_reason).replace('\n', '').strip()}
def home(self):
pass
def login(self):
pass
def is_logged(self):
""" Do not need to be logged """
return True

BIN
modules/canaltp/favicon.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 2.4 KiB

30
modules/canaltp/test.py Normal file
View file

@ -0,0 +1,30 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.tools.test import BackendTest
class CanalTPTest(BackendTest):
BACKEND = 'canaltp'
def test_canaltp(self):
stations = list(self.backend.iter_station_search('defense'))
self.assertTrue(len(stations) > 0)
list(self.backend.iter_station_departures(stations[0].id))

23
modules/cmb/__init__.py Normal file
View file

@ -0,0 +1,23 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Johann Broudin
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from .backend import CmbBackend
__all__ = ['CmbBackend']

231
modules/cmb/backend.py Normal file
View file

@ -0,0 +1,231 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2012 Johann Broudin
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.capabilities.bank import ICapBank, AccountNotFound
from weboob.capabilities.bank import Account, Recipient, Operation
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.tools.value import ValueBackendPassword
from weboob.capabilities.base import NotAvailable
from weboob.tools.browser import BrowserIncorrectPassword, BrokenPageError
from re import match
from httplib import HTTPSConnection
from urllib import urlencode
from lxml import etree
from datetime import date
from StringIO import StringIO
__all__ = ['CmbBackend']
class WrongLoginOrPassword(Exception):
def __init__(self):
return
class CmbBackend(BaseBackend, ICapBank):
NAME = 'cmb'
MAINTAINER = 'Johann Broudin'
EMAIL = 'Johann.Broudin@6-8.fr'
VERSION = '0.a'
LICENSE = 'AGPLv3+'
DESCRIPTION = 'Credit Mutuel de Bretagne'
CONFIG = BackendConfig(
ValueBackendPassword('login', label='Account ID', masked=False),
ValueBackendPassword('password', label='Password', masked=True))
cookie = None
headers = {
'User-Agent':
'Mozilla/5.0 (iPad; U; CPU OS 3_2_1 like Mac OSX; en-us) ' +
'AppleWebKit/531.21.10 (KHTML, like Gecko) Mobile/7B405'
}
def login(self):
params = urlencode({
'codeEspace': 'NO',
'codeEFS': '01',
'codeSi': '001',
'noPersonne': self.config['login'].get(),
'motDePasse': self.config['password'].get()
})
conn = HTTPSConnection("www.cmb.fr")
headers = {'Content-Type': 'application/x-www-form-urlencoded'}
conn.request("POST",
"/domiweb/servlet/Identification",
params,
headers)
response = conn.getresponse()
if response.status == 302:
self.cookie = response.getheader('Set-Cookie').split(';')[0]
self.cookie += ';'
return True
else:
raise BrowserIncorrectPassword()
return false
def iter_accounts(self):
if not self.cookie:
self.login()
def do_http():
conn = HTTPSConnection("www.cmb.fr")
headers = self.headers
headers['Cookie'] = self.cookie
conn.request("GET",
'/domiweb/prive/particulier/releve/0-releve.act',
{},
headers)
response = conn.getresponse()
data = response.read()
conn.close()
return data
data = do_http()
parser = etree.HTMLParser()
tree = etree.parse(StringIO(data), parser)
table = tree.xpath('/html/body/table')
if len(table) == 0:
title = tree.xpath('/html/head/title')[0].text
if title == u"Utilisateur non identifié":
self.login()
data = do_http()
parser = etree.HTMLParser()
tree = etree.parse(StringIO(data), parser)
table = tree.xpath('/html/body/table')
if len(table) == 0:
raise BrokenPageError
else:
raise BrokenPageError
for tr in table[1].getiterator('tr'):
if tr.get('class') != 'LnTit' and tr.get('class') != 'LnTot':
account = Account()
td = tr.xpath('td')
a = td[0].xpath('a')
account.label = unicode(a[0].text).strip()
href = a[0].get('href')
m = match(r"javascript:releve\((.*),'(.*)','(.*)'\)",
href)
account.id = unicode(m.group(1) + m.group(2) + m.group(3))
account.cmbvaleur = m.group(1)
account.cmbvaleur2 = m.group(2)
account.cmbtype = m.group(3)
balance = td[1].text
balance = balance.replace(',','.').replace(u"\xa0",'')
account.balance = float(balance)
span = td[3].xpath('a/span')
if len(span):
coming = span[0].text.replace(' ','').replace(',','.')
coming = coming.replace(u"\xa0",'')
account.coming = float(coming)
else:
account.coming = NotAvailable
yield account
def get_account(self, _id):
for account in self.iter_accounts():
if account.id == _id:
return account
raise AccountNotFound()
def iter_history(self, account):
if not self.cookie:
self.login()
page = "/domiweb/prive/particulier/releve/"
if account.cmbtype == 'D':
page += "10-releve.act"
else:
page += "2-releve.act"
page +="?noPageReleve=1&indiceCompte="
page += account.cmbvaleur
page += "&typeCompte="
page += account.cmbvaleur2
page += "&deviseOrigineEcran=EUR"
def do_http():
conn = HTTPSConnection("www.cmb.fr")
headers = self.headers
headers['Cookie'] = self.cookie
conn.request("GET", page, {}, headers)
response = conn.getresponse()
data = response.read()
conn.close
return data
data = do_http()
parser = etree.HTMLParser()
tree = etree.parse(StringIO(data), parser)
tables = tree.xpath('/html/body/table')
if len(tables) == 0:
title = tree.xpath('/html/head/title')[0].text
if title == u"Utilisateur non identifié":
self.login()
data = do_http()
parser = etree.HTMLParser()
tree = etree.parse(StringIO(data), parser)
tables = tree.xpath('/html/body/table')
if len(tables) == 0:
raise BrokenPageError
else:
raise BrokenPageError
i = 0
for table in tables:
if table.get('id') != "tableMouvements":
continue
for tr in table.getiterator('tr'):
if (tr.get('class') != 'LnTit' and
tr.get('class') != 'LnTot'):
operation = Operation(i)
td = tr.xpath('td')
div = td[1].xpath('div')
d = div[0].text.split('/')
operation.date = date(*reversed([int(x) for x in d]))
div = td[2].xpath('div')
label = div[0].xpath('a')[0].text.replace('\n','')
operation.label = unicode(' '.join(label.split()))
amount = td[3].text
if amount.count(',') != 1:
amount = td[4].text
amount = amount.replace(',','.').replace(u'\xa0','')
print repr(amount)
operation.amount = float(amount)
else:
amount = amount.replace(',','.').replace(u'\xa0','')
operation.amount = - float(amount)
i += 1
yield operation

23
modules/cragr/__init__.py Normal file
View file

@ -0,0 +1,23 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from .backend import CragrBackend
__all__ = ['CragrBackend']

105
modules/cragr/backend.py Normal file
View file

@ -0,0 +1,105 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon, Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.capabilities.bank import ICapBank, AccountNotFound
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.tools.ordereddict import OrderedDict
from weboob.tools.value import ValueBackendPassword, Value
from .browser import Cragr
__all__ = ['CragrBackend']
class CragrBackend(BaseBackend, ICapBank):
NAME = 'cragr'
MAINTAINER = 'Xavier Guerrin'
EMAIL = 'xavier@tuxfamily.org'
VERSION = '0.a'
DESCRIPTION = 'Credit Agricole french bank\'s website'
LICENSE = 'AGPLv3+'
website_choices = OrderedDict([(k, u'%s (%s)' % (v, k)) for k, v in sorted({
'm.ca-alpesprovence.fr': u'Alpes Provence',
'm.ca-anjou-maine.fr': u'Anjou Maine',
'm.ca-atlantique-vendee.fr': u'Atlantique Vendée',
'm.ca-aquitaine.fr': u'Aquitaine',
'm.ca-briepicardie.fr': u'Brie Picardie',
'm.ca-centrest.fr': u'Centre Est',
'm.ca-centrefrance.fr': u'Centre France',
'm.ca-centreloire.fr': u'Centre Loire',
'm.ca-centreouest.fr': u'Centre Ouest',
'm.ca-cb.fr': u'Champagne Bourgogne',
'm.ca-charente-perigord.fr': u'Charente Périgord',
'm.ca-cmds.fr': u'Charente-Maritime Deux-Sèvres',
'm.ca-corse.fr': u'Corse',
'm.ca-cotesdarmor.fr': u'Côtes d\'Armor',
'm.ca-des-savoie.fr': u'Des Savoie',
'm.ca-finistere.fr': u'Finistere',
'm.ca-paris.fr': u'Ile-de-France',
'm.ca-illeetvilaine.fr': u'Ille-et-Vilaine',
'm.ca-languedoc.fr': u'Languedoc',
'm.ca-loirehauteloire.fr': u'Loire Haute Loire',
'm.ca-lorraine.fr': u'Lorraine',
'm.ca-martinique.fr': u'Martinique Guyane',
'm.ca-morbihan.fr': u'Morbihan',
'm.ca-norddefrance.fr': u'Nord de France',
'm.ca-nord-est.fr': u'Nord Est',
'm.ca-nmp.fr': u'Nord Midi-Pyrénées',
'm.ca-normandie.fr': u'Normandie',
'm.ca-normandie-seine.fr': u'Normandie Seine',
'm.ca-pca.fr': u'Provence Côte d\'Azur',
'm.lefil.com': u'Pyrénées Gascogne',
'm.ca-reunion.fr': u'Réunion',
'm.ca-sudrhonealpes.fr': u'Sud Rhône Alpes',
'm.ca-sudmed.fr': u'Sud Méditerranée',
'm.ca-toulouse31.fr': u'Toulouse 31', # m.ca-toulousain.fr redirects here
'm.ca-tourainepoitou.fr': u'Tourraine Poitou',
}.iteritems())])
CONFIG = BackendConfig(Value('website', label='Website to use', choices=website_choices),
ValueBackendPassword('login', label='Account ID', masked=False),
ValueBackendPassword('password', label='Password'))
BROWSER = Cragr
def create_default_browser(self):
return self.create_browser(self.config['website'].get(),
self.config['login'].get(),
self.config['password'].get())
def iter_accounts(self):
for account in self.browser.get_accounts_list():
yield account
def get_account(self, _id):
if not _id.isdigit():
raise AccountNotFound()
account = self.browser.get_account(_id)
if account:
return account
else:
raise AccountNotFound()
def iter_history(self, account):
for history in self.browser.get_history(account):
yield history
def transfer(self, account, to, amount, reason=None):
return self.browser.do_transfer(account, to, amount, reason)

259
modules/cragr/browser.py Normal file
View file

@ -0,0 +1,259 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2009-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword
from weboob.capabilities.bank import Transfer, TransferError
from cragr import pages
import mechanize
from datetime import datetime
import re
class Cragr(BaseBrowser):
PROTOCOL = 'https'
ENCODING = 'utf-8'
USER_AGENT = BaseBrowser.USER_AGENTS['wget']
# a session id that is sometimes added, and should be ignored when matching pages
SESSION_REGEXP = '(?:|%s[A-Z0-9]+)' % re.escape(r';jsessionid=')
is_logging = False
def __init__(self, website, *args, **kwargs):
self.DOMAIN = website
self.PAGES = {'https://[^/]+/': pages.LoginPage,
'https://[^/]+/.*\.c.*': pages.AccountsList,
'https://[^/]+/login/process%s' % self.SESSION_REGEXP: pages.AccountsList,
'https://[^/]+/accounting/listAccounts': pages.AccountsList,
'https://[^/]+/accounting/listOperations': pages.AccountsList,
'https://[^/]+/accounting/showAccountDetail.+': pages.AccountsList,
'https://[^/]+/accounting/showMoreAccountOperations.*': pages.AccountsList,
}
BaseBrowser.__init__(self, *args, **kwargs)
def viewing_html(self):
"""
As the fucking HTTP server returns a document in unknown mimetype
'application/vnd.wap.xhtml+xml' it is not recognized by mechanize.
So this is a fucking hack.
"""
return True
def is_logged(self):
logged = self.page and self.page.is_logged() or self.is_logging
self.logger.debug('logged: %s' % (logged and 'yes' or 'no'))
return logged
def login(self):
"""
Attempt to log in.
Note: this method does nothing if we are already logged in.
"""
assert isinstance(self.username, basestring)
assert isinstance(self.password, basestring)
# Do we really need to login?
if self.is_logged():
self.logger.debug('already logged in')
return
self.is_logging = True
# Are we on the good page?
if not self.is_on_page(pages.LoginPage):
self.logger.debug('going to login page')
BaseBrowser.home(self)
self.logger.debug('attempting to log in')
self.page.login(self.username, self.password)
self.is_logging = False
if not self.is_logged():
raise BrowserIncorrectPassword()
def get_accounts_list(self):
self.logger.debug('accounts list required')
self.home()
return self.page.get_list()
def home(self):
"""
Ensure we are both logged and on the accounts list.
"""
self.logger.debug('accounts list page required')
if self.is_on_page(pages.AccountsList) and self.page.is_accounts_list():
self.logger.debug('already on accounts list')
return
# simply go to http(s)://the.doma.in/
BaseBrowser.home(self)
if self.is_on_page(pages.LoginPage):
if not self.is_logged():
# So, we are not logged on the login page -- what about logging ourselves?
self.login()
# we assume we are logged in
# for some regions, we may stay on the login page once we're
# logged in, without being redirected...
if self.is_on_page(pages.LoginPage):
# ... so we have to move by ourselves
self.move_to_accounts_list()
def move_to_accounts_list(self):
"""
For regions where you can stay on http(s)://the.doma.in/ while you are
logged in, move to the accounts list
"""
self.location('%s://%s/accounting/listAccounts' % (self.PROTOCOL, self.DOMAIN))
def get_account(self, id):
assert isinstance(id, basestring)
l = self.get_accounts_list()
for a in l:
if a.id == ('%s' % id):
return a
return None
def get_history(self, account):
history_url = account.link_id
operations_count = 0
# 1st, go on the account page
self.logger.debug('going on: %s' % history_url)
self.location('https://%s%s' % (self.DOMAIN, history_url))
# Some regions have a "Show more" (well, actually "Voir les 25
# suivants") link we have to use to get all the operations.
# However, it does not show only the 25 next results, it *adds* them
# to the current view. Therefore, we have to parse each new page using
# an offset, in order to ignore all already-fetched operations.
# This especially occurs on CA Centre.
use_expand_url = bool(self.page.expand_history_page_url())
while (history_url):
# we skip "operations_count" operations on each page if we are in the case described above
operations_offset = operations_count if use_expand_url else 0
for page_operation in self.page.get_history(operations_count, operations_offset):
operations_count += 1
yield page_operation
history_url = self.page.expand_history_page_url() if use_expand_url else self.page.next_page_url()
self.logger.debug('going on: %s' % history_url)
self.location('https://%s%s' % (self.DOMAIN, history_url))
def dict_find_value(self, dictionary, value):
"""
Returns the first key pointing on the given value, or None if none
is found.
"""
for k, v in dictionary.iteritems():
if v == value:
return k
return None
def do_transfer(self, account, to, amount, reason=None):
"""
Transfer the given amount of money from an account to another,
tagging the transfer with the given reason.
"""
# access the transfer page
transfer_page_unreachable_message = u'Could not reach the transfer page.'
self.home()
if not self.page.is_accounts_list():
raise TransferError(transfer_page_unreachable_message)
operations_url = self.page.operations_page_url()
self.location('https://%s%s' % (self.DOMAIN, operations_url))
transfer_url = self.page.transfer_page_url()
abs_transfer_url = 'https://%s%s' % (self.DOMAIN, transfer_url)
self.location(abs_transfer_url)
if not self.page.is_transfer_page():
raise TransferError(transfer_page_unreachable_message)
source_accounts = self.page.get_transfer_source_accounts()
target_accounts = self.page.get_transfer_target_accounts()
# check that the given source account can be used
if not account in source_accounts.values():
raise TransferError('You cannot use account %s as a source account.' % account)
# check that the given source account can be used
if not to in target_accounts.values():
raise TransferError('You cannot use account %s as a target account.' % to)
# separate euros from cents
amount_euros = int(amount)
amount_cents = int((amount * 100) - (amount_euros * 100))
# let's circumvent https://github.com/jjlee/mechanize/issues/closed#issue/17
# using http://wwwsearch.sourceforge.net/mechanize/faq.html#usage
adjusted_response = self.response().get_data().replace('<br/>', '<br />')
response = mechanize.make_response(adjusted_response, [('Content-Type', 'text/html')], abs_transfer_url, 200, 'OK')
self.set_response(response)
# fill the form
self.select_form(nr=0)
self['numCompteEmetteur'] = ['%s' % self.dict_find_value(source_accounts, account)]
self['numCompteBeneficiaire'] = ['%s' % self.dict_find_value(target_accounts, to)]
self['montantPartieEntiere'] = '%s' % amount_euros
self['montantPartieDecimale'] = '%02d' % amount_cents
if reason != None:
self['libelle'] = reason
self.submit()
# look for known errors
content = unicode(self.response().get_data(), 'utf-8')
insufficient_amount_message = u'Montant insuffisant.'
maximum_allowed_balance_message = u'Solde maximum autorisé dépassé.'
if content.find(insufficient_amount_message) != -1:
raise TransferError('The amount you tried to transfer is too low.')
if content.find(maximum_allowed_balance_message) != -1:
raise TransferError('The maximum allowed balance for the target account has been / would be reached.')
# look for the known "all right" message
ready_for_transfer_message = u'Vous allez effectuer un virement'
if not content.find(ready_for_transfer_message):
raise TransferError('The expected message "%s" was not found.' % ready_for_transfer_message)
# submit the last form
self.select_form(nr=0)
submit_date = datetime.now()
self.submit()
# look for the known "everything went well" message
content = unicode(self.response().get_data(), 'utf-8')
transfer_ok_message = u'Vous venez d\'effectuer un virement du compte'
if not content.find(transfer_ok_message):
raise TransferError('The expected message "%s" was not found.' % transfer_ok_message)
# We now have to return a Transfer object
# the final page does not provide any transfer id, so we'll use the submit date
transfer = Transfer(submit_date.strftime('%Y%m%d%H%M%S'))
transfer.amount = amount
transfer.origin = account
transfer.recipient = to
transfer.date = submit_date
return transfer
#def get_coming_operations(self, account):
# if not self.is_on_page(pages.AccountComing) or self.page.account.id != account.id:
# self.location('/NS_AVEEC?ch4=%s' % account.link_id)
# return self.page.get_operations()

BIN
modules/cragr/favicon.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 2.2 KiB

24
modules/cragr/pages/__init__.py Normal file
View file

@ -0,0 +1,24 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from .accounts_list import AccountsList
from .login import LoginPage
__all__ = ['AccountsList', 'LoginPage']

283
modules/cragr/pages/accounts_list.py Normal file
View file

@ -0,0 +1,283 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
import re
from weboob.capabilities.bank import Account
from .base import CragrBasePage
from weboob.capabilities.bank import Operation
def clean_amount(amount):
"""
Removes weird characters and converts to a float
>>> clean_amount(u'1 000,00 $')
1000.0
"""
data = amount.replace(',', '.').replace(' ', '').replace(u'\xa0', '')
matches = re.findall('^(-?[0-9]+\.[0-9]{2}).*$', data)
return float(matches[0]) if (matches) else 0.0
class AccountsList(CragrBasePage):
def get_list(self):
"""
Returns the list of available bank accounts
"""
l = []
for div in self.document.getiterator('div'):
if div.attrib.get('class', '') == 'dv' and div.getchildren()[0].tag in ('a', 'br'):
account = Account()
if div.getchildren()[0].tag == 'a':
# This is at least present on CA Nord-Est
account.label = ' '.join(div.find('a').text.split()[:-1])
account.link_id = div.find('a').get('href', '')
account.id = div.find('a').text.split()[-1]
s = div.find('div').find('b').find('span').text
else:
# This is at least present on CA Toulouse
account.label = div.find('a').text.strip()
account.link_id = div.find('a').get('href', '')
account.id = div.findall('br')[1].tail.strip()
s = div.find('div').find('b').text
account.balance = clean_amount(s)
l.append(account)
return l
def is_accounts_list(self):
"""
Returns True if the current page appears to be the page dedicated to
list the accounts.
"""
# we check for the presence of a "mes comptes titres" link_id
link = self.document.xpath('/html/body//a[contains(text(), "comptes titres")]')
return bool(link)
def is_account_page(self):
"""
Returns True if the current page appears to be a page dedicated to list
the history of a specific account.
"""
# tested on CA Lorraine, Paris, Toulouse
title_spans = self.document.xpath('/html/body//div[@class="dv"]/span')
for title_span in title_spans:
title_text = title_span.text_content().strip().replace("\n", '')
if (re.match('.*Compte.*n.*[0-9]+.*au.*', title_text)):
return True
return False
def is_transfer_page(self):
"""
Returns True if the current page appears to be the page dedicated to
order transfers between accounts.
"""
source_account_select_field = self.document.xpath('/html/body//form//select[@name="numCompteEmetteur"]')
target_account_select_field = self.document.xpath('/html/body//form//select[@name="numCompteBeneficiaire"]')
return bool(source_account_select_field) and bool(target_account_select_field)
def get_transfer_accounts(self, select_name):
"""
Returns the accounts proposed for a transfer in a select field.
This method assumes the current page is the one dedicated to transfers.
select_name is the name of the select field to analyze
"""
if not self.is_transfer_page():
return False
source_accounts = {}
source_account_options = self.document.xpath('/html/body//form//select[@name="%s"]/option' % select_name)
for option in source_account_options:
source_account_value = option.get('value', -1)
if (source_account_value != -1):
matches = re.findall('^[A-Z0-9]+.*([0-9]{11}).*$', self.extract_text(option))
if matches:
source_accounts[source_account_value] = matches[0]
return source_accounts
def get_transfer_source_accounts(self):
return self.get_transfer_accounts('numCompteEmetteur')
def get_transfer_target_accounts(self):
return self.get_transfer_accounts('numCompteBeneficiaire')
def expand_history_page_url(self):
"""
When on a page dedicated to list the history of a specific account (see
is_account_page), returns the link to expand the history with 25 more results,
or False if the link is not present.
"""
# tested on CA centre france
a = self.document.xpath('/html/body//div[@class="navlink"]//a[contains(text(), "Voir les 25 suivants")]')
if not a:
return False
else:
return a[0].get('href', '')
def next_page_url(self):
"""
When on a page dedicated to list the history of a specific account (see
is_account_page), returns the link to the next page, or False if the
link is not present.
"""
# tested on CA Lorraine, Paris, Toulouse
a = self.document.xpath('/html/body//div[@class="navlink"]//a[contains(text(), "Suite")]')
if not a:
return False
else:
return a[0].get('href', '')
def operations_page_url(self):
"""
Returns the link to the "Opérations" page. This function assumes the
current page is the accounts list (see is_accounts_list)
"""
link = self.document.xpath(u'/html/body//a[contains(text(), "Opérations")]')
return link[0].get('href')
def transfer_page_url(self):
"""
Returns the link to the "Virements" page. This function assumes the
current page is the operations list (see operations_page_url)
"""
link = self.document.xpath('/html/body//a[@accesskey=1]/@href')
return link[0]
def is_right_aligned_div(self, div_elmt):
"""
Returns True if the given div element is right-aligned
"""
return(re.match('.*text-align: ?right.*', div_elmt.get('style', '')))
def extract_text(self, xml_elmt):
"""
Given an XML element, returns its inner text in a reasonably readable way
"""
data = u''
for text in xml_elmt.itertext():
data = data + u'%s ' % text
data = re.sub(' +', ' ', data.replace("\n", ' ').strip())
return data
def get_history(self, start_index = 0, start_offset = 0):
"""
Returns the history of a specific account. Note that this function
expects the current page to be the one dedicated to this history.
start_index is the id used for the first created operation.
start_offset allows ignoring the `n' first Operations on the page.
"""
# tested on CA Lorraine, Paris, Toulouse
# avoir parsing the page as an account-dedicated page if it is not the case
if not self.is_account_page():
return
index = start_index
operation = False
skipped = 0
body_elmt_list = self.document.xpath('/html/body/*')
# type of separator used in the page
separators = 'hr'
# How many <hr> elements do we have under the <body>?
sep_expected = len(self.document.xpath('/html/body/hr'))
if (not sep_expected):
# no <hr>? Then how many class-less <div> used as separators instead?
sep_expected = len(self.document.xpath('/html/body/div[not(@class) and not(@style)]'))
separators = 'div'
# the interesting divs are after the <hr> elements
interesting_divs = []
right_div_count = 0
left_div_count = 0
sep_found = 0
for body_elmt in body_elmt_list:
if (separators == 'hr' and body_elmt.tag == 'hr'):
sep_found += 1
elif (separators == 'div' and body_elmt.tag == 'div' and body_elmt.get('class', 'nope') == 'nope'):
sep_found += 1
elif (sep_found >= sep_expected and body_elmt.tag == 'div'):
# we just want <div> with dv class and a style attribute
if (body_elmt.get('class', '') != 'dv'):
continue
if (body_elmt.get('style', 'nope') == 'nope'):
continue
interesting_divs.append(body_elmt)
if (self.is_right_aligned_div(body_elmt)):
right_div_count += 1
else:
left_div_count += 1
# new layout that is somewhat easier to parse (found at Toulouse)
table_layout = len(self.document.xpath("id('operationsHeader')")) > 0
# So, how are data laid out?
alternate_layout = (left_div_count == 2 * right_div_count)
# we'll have: one left-aligned div for the date, one right-aligned
# div for the amount, and one left-aligned div for the label. Each time.
if table_layout:
lines = self.document.xpath('id("operationsContent")//table[@class="tb"]/tr')
for line in lines:
if skipped < start_offset:
skipped += 1
continue
operation = Operation(index)
index += 1
operation.date = self.extract_text(line[0])
operation.label = self.extract_text(line[1])
operation.amount = clean_amount(self.extract_text(line[2]))
yield operation
elif (not alternate_layout):
for body_elmt in interesting_divs:
if skipped < start_offset:
if self.is_right_aligned_div(body_elmt):
skipped += 1
continue
if (self.is_right_aligned_div(body_elmt)):
# this is the second line of an operation entry, displaying the amount
operation.amount = clean_amount(self.extract_text(body_elmt))
yield operation
else:
# this is the first line of an operation entry, displaying the date and label
data = self.extract_text(body_elmt)
matches = re.findall('^([012][0-9]|3[01])/(0[1-9]|1[012]).(.+)$', data)
operation = Operation(index)
index += 1
if (matches):
operation.date = u'%s/%s' % (matches[0][0], matches[0][1])
operation.label = u'%s' % matches[0][2]
else:
operation.date = u'01/01'
operation.label = u'Unknown'
else:
for i in range(0, len(interesting_divs)/3):
if skipped < start_offset:
skipped += 1
continue
operation = Operation(index)
index += 1
# amount
operation.amount = clean_amount(self.extract_text(interesting_divs[(i*3)+1]))
# date
data = self.extract_text(interesting_divs[i*3])
matches = re.findall('^([012][0-9]|3[01])/(0[1-9]|1[012])', data)
operation.date = u'%s/%s' % (matches[0][0], matches[0][1]) if (matches) else u'01/01'
#label
data = self.extract_text(interesting_divs[(i*3)+2])
data = re.sub(' +', ' ', data)
operation.label = u'%s' % data
yield operation

41
modules/cragr/pages/base.py Normal file
View file

@ -0,0 +1,41 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.tools.browser import BasePage
from weboob.tools.browser import BrowserUnavailable
class CragrBasePage(BasePage):
def on_loaded(self):
# Check for an error
for div in self.document.getiterator('div'):
if div.attrib.get('class', '') == 'dv' and div.getchildren()[0].tag in ('img') and div.getchildren()[0].attrib.get('alt', '') == 'Attention':
# Try to find a detailed error message
if div.getchildren()[1].tag == 'span':
raise BrowserUnavailable(div.find('span').find('b').text)
elif div.getchildren()[1].tag == 'b':
# I haven't encountered this variation in the wild,
# but I wouldn't be surprised if it existed
# given the similar differences between regions.
raise BrowserUnavailable(div.find('b').find('span').text)
raise BrowserUnavailable()
def is_logged(self):
return not self.document.xpath('/html/body//form//input[@name = "code"]') and \
not self.document.xpath('/html/body//form//input[@name = "userPassword"]')

51
modules/cragr/pages/login.py Normal file
View file

@ -0,0 +1,51 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.tools.mech import ClientForm
ControlNotFoundError = ClientForm.ControlNotFoundError
from .base import CragrBasePage
__all__ = ['LoginPage']
class LoginPage(CragrBasePage):
def login(self, login, password):
self.browser.select_form(nr=0)
try:
self.browser['numero'] = login
self.browser['code'] = password
except ControlNotFoundError:
try:
self.browser['userLogin'] = login
self.browser['userPassword'] = password
except ControlNotFoundError:
self.browser.controls.append(ClientForm.TextControl('text', 'numero', {'value': ''}))
self.browser.controls.append(ClientForm.TextControl('text', 'code', {'value': ''}))
self.browser.controls.append(ClientForm.TextControl('text', 'userLogin', {'value': ''}))
self.browser.controls.append(ClientForm.TextControl('text', 'userPassword', {'value': ''}))
self.browser.set_all_readonly(False)
self.browser['numero'] = login
self.browser['code'] = password
self.browser['userLogin'] = login
self.browser['userPassword'] = password
self.browser.submit()

30
modules/cragr/test.py Normal file
View file

@ -0,0 +1,30 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.tools.test import BackendTest
class CrAgrTest(BackendTest):
BACKEND = 'cragr'
def test_cragr(self):
l = list(self.backend.iter_accounts())
if len(l) > 0:
a = l[0]
list(self.backend.iter_history(a))

23
modules/creditmutuel/__init__.py Normal file
View file

@ -0,0 +1,23 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Veyssier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from .backend import CreditMutuelBackend
__all__ = ['CreditMutuelBackend']

64
modules/creditmutuel/backend.py Normal file
View file

@ -0,0 +1,64 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Veyssier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.capabilities.bank import ICapBank, AccountNotFound
from weboob.tools.backend import BaseBackend, BackendConfig
from weboob.tools.value import ValueBackendPassword
from .browser import CreditMutuelBrowser
__all__ = ['CreditMutuelBackend']
class CreditMutuelBackend(BaseBackend, ICapBank):
NAME = 'creditmutuel'
MAINTAINER = 'Julien Veyssier'
EMAIL = 'julien.veyssier@aiur.fr'
VERSION = '0.a'
DESCRIPTION = u'Crédit Mutuel french bank'
LICENSE = 'AGPLv3+'
CONFIG = BackendConfig(ValueBackendPassword('login', label='Account ID', regexp='^\d{1,13}\w$', masked=False),
ValueBackendPassword('password', label='Password of account'))
BROWSER = CreditMutuelBrowser
def create_default_browser(self):
return self.create_browser(self.config['login'].get(), self.config['password'].get())
def iter_accounts(self):
for account in self.browser.get_accounts_list():
yield account
def get_account(self, _id):
if not _id.isdigit():
raise AccountNotFound()
account = self.browser.get_account(_id)
if account:
return account
else:
raise AccountNotFound()
def iter_operations(self, account):
""" TODO Not supported yet """
return iter([])
def iter_history(self, account):
for history in self.browser.get_history(account):
yield history

117
modules/creditmutuel/browser.py Normal file
View file

@ -0,0 +1,117 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Veyssier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword
from .pages import LoginPage, LoginErrorPage, AccountsPage, OperationsPage, InfoPage
__all__ = ['CreditMutuelBrowser']
# Browser
class CreditMutuelBrowser(BaseBrowser):
PROTOCOL = 'https'
DOMAIN = 'www.creditmutuel.fr'
ENCODING = 'iso-8859-1'
USER_AGENT = BaseBrowser.USER_AGENTS['wget']
PAGES = {'https://www.creditmutuel.fr/groupe/fr/index.html': LoginPage,
'https://www.creditmutuel.fr/groupe/fr/identification/default.cgi': LoginErrorPage,
'https://www.creditmutuel.fr/.*/fr/banque/situation_financiere.cgi': AccountsPage,
'https://www.creditmutuel.fr/.*/fr/banque/mouvements.cgi.*' : OperationsPage,
'https://www.creditmutuel.fr/.*/fr/banque/BAD.*' : InfoPage
}
def __init__(self, *args, **kwargs):
BaseBrowser.__init__(self, *args, **kwargs)
#self.SUB_BANKS = ['cmdv','cmcee','cmse', 'cmidf', 'cmsmb', 'cmma', 'cmmabn', 'cmc', 'cmlaco', 'cmnormandie', 'cmm']
#self.currentSubBank = None
def is_logged(self):
return self.page and not self.is_on_page(LoginPage)
def home(self):
return self.location('https://www.creditmutuel.fr/groupe/fr/index.html')
def login(self):
assert isinstance(self.username, basestring)
assert isinstance(self.password, basestring)
if not self.is_on_page(LoginPage):
self.location('https://www.creditmutuel.fr/', no_login=True)
self.page.login( self.username, self.password)
if not self.is_logged() or self.is_on_page(LoginErrorPage):
raise BrowserIncorrectPassword()
self.SUB_BANKS = ['cmdv','cmcee','cmse', 'cmidf', 'cmsmb', 'cmma', 'cmmabn', 'cmc', 'cmlaco', 'cmnormandie', 'cmm']
self.getCurrentSubBank()
def get_accounts_list(self):
if not self.is_on_page(AccountsPage):
self.location('https://www.creditmutuel.fr/%s/fr/banque/situation_financiere.cgi'%self.currentSubBank)
return self.page.get_list()
def get_account(self, id):
assert isinstance(id, basestring)
l = self.get_accounts_list()
for a in l:
if a.id == id:
return a
return None
def getCurrentSubBank(self):
# the account list and history urls depend on the sub bank of the user
current_url = self.geturl()
current_url_parts = current_url.split('/')
for subbank in self.SUB_BANKS:
if subbank in current_url_parts:
self.currentSubBank = subbank
def get_history(self, account):
page_url = account.link_id
#operations_count = 0
l_ret = []
while (page_url):
self.location('https://%s/%s/fr/banque/%s' % (self.DOMAIN, self.currentSubBank, page_url))
#for page_operation in self.page.get_history(operations_count):
# operations_count += 1
# yield page_operation
## FONCTIONNE
#for op in self.page.get_history():
# yield op
## FONTIONNE
#return self.page.get_history()
for op in self.page.get_history():
l_ret.append(op)
page_url = self.page.next_page_url()
return l_ret
#def get_coming_operations(self, account):
# if not self.is_on_page(AccountComing) or self.page.account.id != account.id:
# self.location('/NS_AVEEC?ch4=%s' % account.link_id)
# return self.page.get_operations()

BIN
modules/creditmutuel/favicon.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 1.7 KiB

95
modules/creditmutuel/pages.py Normal file
View file

@ -0,0 +1,95 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Veyssier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.tools.browser import BasePage
from weboob.capabilities.bank import Account
from weboob.capabilities.bank import Operation
class LoginPage(BasePage):
def login(self, login, passwd):
self.browser.select_form(nr=0)
self.browser['_cm_user'] = login
self.browser['_cm_pwd'] = passwd
self.browser.submit()
class LoginErrorPage(BasePage):
pass
class InfoPage(BasePage):
pass
class AccountsPage(BasePage):
def get_list(self):
l = []
for tr in self.document.getiterator('tr'):
first_td = tr.getchildren()[0]
if first_td.attrib.get('class', '') == 'i g' or first_td.attrib.get('class', '') == 'p g':
account = Account()
account.label = u"%s"%first_td.find('a').text
account.link_id = first_td.find('a').get('href', '')
account.id = first_td.find('a').text.split(' ')[0]+first_td.find('a').text.split(' ')[1]
s = tr.getchildren()[2].text
if s.strip() == "":
s = tr.getchildren()[1].text
balance = u''
for c in s:
if c.isdigit() or c == '-':
balance += c
if c == ',':
balance += '.'
account.balance = float(balance)
l.append(account)
#raise NotImplementedError()
return l
def next_page_url(self):
""" TODO pouvoir passer à la page des comptes suivante """
return 0
class OperationsPage(BasePage):
def get_history(self):
index = 0
for tr in self.document.getiterator('tr'):
first_td = tr.getchildren()[0]
if first_td.attrib.get('class', '') == 'i g' or first_td.attrib.get('class', '') == 'p g':
operation = Operation(index)
index += 1
operation.date = first_td.text
operation.label = u"%s"%tr.getchildren()[2].text.replace('\n',' ')
if len(tr.getchildren()[3].text) > 2:
s = tr.getchildren()[3].text
elif len(tr.getchildren()[4].text) > 2:
s = tr.getchildren()[4].text
else:
s = "0"
balance = u''
for c in s:
if c.isdigit() or c == "-":
balance += c
if c == ',':
balance += '.'
operation.amount = float(balance)
yield operation
def next_page_url(self):
""" TODO pouvoir passer à la page des opérations suivantes """
return 0

27
modules/creditmutuel/test.py Normal file
View file

@ -0,0 +1,27 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Julien Veyssier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.tools.test import BackendTest
class CreditMutuelTest(BackendTest):
BACKEND = 'crmut'
def test_crmut(self):
list(self.backend.iter_accounts())

3
modules/dailymotion/__init__.py Normal file
View file

@ -0,0 +1,3 @@
from .backend import DailymotionBackend
__all__ = ['DailymotionBackend']

62
modules/dailymotion/backend.py Normal file
View file

@ -0,0 +1,62 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from __future__ import with_statement
from weboob.capabilities.video import ICapVideo
from weboob.tools.backend import BaseBackend
from .browser import DailymotionBrowser
from .video import DailymotionVideo
__all__ = ['DailymotionBackend']
class DailymotionBackend(BaseBackend, ICapVideo):
NAME = 'dailymotion'
MAINTAINER = 'Romain Bignon'
EMAIL = 'romain@weboob.org'
VERSION = '0.a'
DESCRIPTION = 'Dailymotion videos website'
LICENSE = 'AGPLv3+'
BROWSER = DailymotionBrowser
def get_video(self, _id):
with self.browser:
return self.browser.get_video(_id)
SORTBY = ['relevance', 'rated', 'visited', None]
def iter_search_results(self, pattern=None, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False, max_results=None):
with self.browser:
return self.browser.iter_search_results(pattern, self.SORTBY[sortby])
def fill_video(self, video, fields):
if fields != ['thumbnail']:
# if we don't want only the thumbnail, we probably want also every fields
with self.browser:
video = self.browser.get_video(DailymotionVideo.id2url(video.id), video)
if 'thumbnail' in fields and video.thumbnail:
with self.browser:
video.thumbnail.data = self.browser.readurl(video.thumbnail.url)
return video
OBJECTS = {DailymotionVideo: fill_video}

56
modules/dailymotion/browser.py Normal file
View file

@ -0,0 +1,56 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from urllib import quote_plus
from weboob.tools.browser import BaseBrowser
from weboob.tools.browser.decorators import id2url
from .pages import IndexPage, VideoPage
from .video import DailymotionVideo
__all__ = ['DailymotionBrowser']
class DailymotionBrowser(BaseBrowser):
DOMAIN = 'dailymotion.com'
ENCODING = None
PAGES = {r'http://[w\.]*dailymotion\.com/?': IndexPage,
r'http://[w\.]*dailymotion\.com/(\w+/)?search/.*': IndexPage,
r'http://[w\.]*dailymotion\.com/video/(?P<id>.+)': VideoPage,
}
@id2url(DailymotionVideo.id2url)
def get_video(self, url, video=None):
self.location(url)
return self.page.get_video(video)
def iter_search_results(self, pattern, sortby):
if not pattern:
self.home()
else:
if sortby is None:
url = '/search/%s/1' % quote_plus(pattern.encode('utf-8'))
else:
url = '/%s/search/%s/1' % (sortby, quote_plus(pattern.encode('utf-8')))
self.location(url)
assert self.is_on_page(IndexPage)
return self.page.iter_videos()

BIN
modules/dailymotion/favicon.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 2.6 KiB

112
modules/dailymotion/pages.py Normal file
View file

@ -0,0 +1,112 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
import datetime
import urllib
import re
from weboob.tools.capabilities.thumbnail import Thumbnail
from weboob.capabilities.base import NotAvailable
from weboob.tools.misc import html2text
from weboob.tools.browser import BasePage, BrokenPageError
from .video import DailymotionVideo
__all__ = ['IndexPage', 'VideoPage']
class IndexPage(BasePage):
def iter_videos(self):
for div in self.parser.select(self.document.getroot(), 'div.dmpi_video_item'):
_id = 0
for cls in div.attrib['class'].split():
if cls.startswith('id_'):
_id = int(cls[3:])
break
if _id == 0:
self.browser.logger.warning('Unable to find the ID of a video')
continue
video = DailymotionVideo(int(_id))
video.title = self.parser.select(div, 'h3 a', 1).text
video.author = self.parser.select(div, 'div.dmpi_user_login', 1).find('a').text
video.description = html2text(self.parser.tostring(self.parser.select(div, 'div.dmpi_video_description', 1))).strip()
try:
parts = self.parser.select(div, 'div.duration', 1).text.split(':')
except BrokenPageError:
# it's probably a live, np.
video.duration = NotAvailable
else:
if len(parts) == 1:
seconds = parts[0]
hours = minutes = 0
elif len(parts) == 2:
minutes, seconds = parts
hours = 0
elif len(parts) == 3:
hours, minutes, seconds = parts
else:
raise BrokenPageError('Unable to parse duration %r' % self.parser.select(div, 'div.duration', 1).text)
video.duration = datetime.timedelta(hours=int(hours), minutes=int(minutes), seconds=int(seconds))
url = self.parser.select(div, 'img.dmco_image', 1).attrib['src']
video.thumbnail = Thumbnail(url)
rating_div = self.parser.select(div, 'div.small_stars', 1)
video.rating_max = self.get_rate(rating_div)
video.rating = self.get_rate(rating_div.find('div'))
# XXX missing date
video.date = NotAvailable
yield video
def get_rate(self, div):
m = re.match('width: *(\d+)px', div.attrib['style'])
if m:
return int(m.group(1))
else:
self.browser.logger.warning('Unable to parse rating: %s' % div.attrib['style'])
return 0
class VideoPage(BasePage):
def get_video(self, video=None):
if video is None:
video = DailymotionVideo(self.group_dict['id'])
div = self.parser.select(self.document.getroot(), 'div#content', 1)
video.title = self.parser.select(div, 'span.title', 1).text
video.author = self.parser.select(div, 'a.name', 1).text
try:
video.description = self.parser.select(div, 'div#video_description', 1).text
except BrokenPageError:
video.description = u''
for script in self.parser.select(self.document.getroot(), 'div.dmco_html'):
if 'id' in script.attrib and script.attrib['id'].startswith('container_player_'):
text = script.find('script').text
mobj = re.search(r'(?i)addVariable\(\"video\"\s*,\s*\"([^\"]*)\"\)', text)
if mobj is None:
mobj = re.search('"sdURL":.*?"(.*?)"', urllib.unquote(text))
mediaURL = mobj.group(1).replace("\\", "")
else:
mediaURL = urllib.unquote(mobj.group(1))
video.url = mediaURL
return video

32
modules/dailymotion/test.py Normal file
View file

@ -0,0 +1,32 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.tools.test import BackendTest
class DailymotionTest(BackendTest):
BACKEND = 'dailymotion'
def test_dailymotion(self):
l = list(self.backend.iter_search_results('chirac'))
self.assertTrue(len(l) > 0)
v = l[0]
self.backend.fillobj(v, ('url',))
self.assertTrue(v.url and v.url.startswith('http://'), 'URL for video "%s" not found: %s' % (v.id, v.url))
self.backend.browser.openurl(v.url)

34
modules/dailymotion/video.py Normal file
View file

@ -0,0 +1,34 @@
# -*- coding: utf-8 -*-
# Copyright(C) 2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from weboob.capabilities.video import BaseVideo
__all__ = ['DailymotionVideo']
class DailymotionVideo(BaseVideo):
def __init__(self, *args, **kwargs):
BaseVideo.__init__(self, *args, **kwargs)
self.ext = 'flv'
@classmethod
def id2url(cls, _id):
return 'http://www.dailymotion.com/video/%s' % _id

Some files were not shown because too many files have changed in this diff Show more