support repositories to manage backends (closes #747)
This commit is contained in:
Romain Bignon
parent
ef16a5b726
commit
14a7a1d362
410 changed files with 1079 additions and 297 deletions
23
modules/mediawiki/__init__.py
Normal file
23
modules/mediawiki/__init__.py
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright(C) 2011 Clément Schreiner
|
||||
#
|
||||
# This file is part of weboob.
|
||||
#
|
||||
# weboob is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# weboob is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
from .backend import MediawikiBackend
|
||||
|
||||
__all__ = ['MediawikiBackend']
|
||||
73
modules/mediawiki/backend.py
Normal file
73
modules/mediawiki/backend.py
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright(C) 2011 Clément Schreiner
|
||||
#
|
||||
# This file is part of weboob.
|
||||
#
|
||||
# weboob is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# weboob is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
from __future__ import with_statement
|
||||
|
||||
from weboob.tools.backend import BaseBackend, BackendConfig
|
||||
from weboob.capabilities.content import ICapContent, Content
|
||||
from weboob.tools.value import ValueBackendPassword, Value
|
||||
|
||||
|
||||
from .browser import MediawikiBrowser
|
||||
|
||||
|
||||
__all__ = ['MediawikiBackend']
|
||||
|
||||
class MediawikiBackend(BaseBackend, ICapContent):
|
||||
NAME = 'mediawiki'
|
||||
MAINTAINER = u'Clément Schreiner'
|
||||
EMAIL = 'clemux@clemux.info'
|
||||
VERSION = '0.a'
|
||||
LICENSE = 'AGPLv3+'
|
||||
DESCRIPTION = 'Mediawiki wiki software application'
|
||||
CONFIG = BackendConfig(Value('url', label='URL of the Mediawiki website', default='http://en.wikipedia.org/', regexp='https?://.*'),
|
||||
Value('apiurl', label='URL of the Mediawiki website\'s API', default='http://en.wikipedia.org/w/api.php', regexp='https?://.*'),
|
||||
Value('username', label='Login', default=''),
|
||||
ValueBackendPassword('password', label='Password', default=''))
|
||||
|
||||
BROWSER = MediawikiBrowser
|
||||
def create_default_browser(self):
|
||||
username = self.config['username'].get()
|
||||
if len(username) > 0:
|
||||
password = self.config['password'].get()
|
||||
else:
|
||||
password = None
|
||||
return self.create_browser(self.config['url'].get(),
|
||||
self.config['apiurl'].get(),
|
||||
username, password)
|
||||
|
||||
def get_content(self, _id):
|
||||
_id = _id.replace(' ', '_').encode('utf-8')
|
||||
content = Content(_id)
|
||||
page = _id
|
||||
with self.browser:
|
||||
data = self.browser.get_wiki_source(page)
|
||||
|
||||
content.content = data
|
||||
return content
|
||||
|
||||
def iter_revisions(self, _id, max_results=10):
|
||||
for rev in self.browser.iter_wiki_revisions(_id, max_results):
|
||||
yield rev
|
||||
|
||||
def push_content(self, content, message=None, minor=False):
|
||||
self.browser.set_wiki_source(content, message, minor)
|
||||
|
||||
def get_content_preview(self, content):
|
||||
return self.browser.get_wiki_preview(content)
|
||||
190
modules/mediawiki/browser.py
Normal file
190
modules/mediawiki/browser.py
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright(C) 2011 Clément Schreiner
|
||||
#
|
||||
# This file is part of weboob.
|
||||
#
|
||||
# weboob is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# weboob is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
from urlparse import urlsplit
|
||||
import urllib
|
||||
import datetime
|
||||
|
||||
|
||||
from weboob.tools.browser import BaseBrowser, BrowserIncorrectPassword
|
||||
from weboob.capabilities.content import Revision
|
||||
|
||||
try:
|
||||
import simplejson
|
||||
except ImportError:
|
||||
# Python 2.6+ has a module similar to simplejson
|
||||
import json as simplejson
|
||||
|
||||
__all__ = ['MediawikiBrowser']
|
||||
|
||||
|
||||
class APIError(Exception):
|
||||
pass
|
||||
|
||||
# Browser
|
||||
class MediawikiBrowser(BaseBrowser):
|
||||
ENCODING = 'utf-8'
|
||||
|
||||
def __init__(self, url, apiurl, *args, **kwargs):
|
||||
url_parsed = urlsplit(url)
|
||||
self.PROTOCOL = url_parsed.scheme
|
||||
self.DOMAIN = url_parsed.netloc
|
||||
self.BASEPATH = url_parsed.path
|
||||
if self.BASEPATH.endswith('/'):
|
||||
self.BASEPATH = self.BASEPATH[:-1]
|
||||
|
||||
self.apiurl = apiurl
|
||||
BaseBrowser.__init__(self, *args, **kwargs)
|
||||
|
||||
def get_wiki_source(self, page):
|
||||
assert isinstance(self.apiurl, basestring)
|
||||
|
||||
data = {'action': 'query',
|
||||
'prop': 'revisions|info',
|
||||
'titles': page,
|
||||
'rvprop': 'content|timestamp',
|
||||
'rvlimit': '1',
|
||||
'intoken': 'edit',
|
||||
}
|
||||
|
||||
|
||||
|
||||
result = self.API_get(data)
|
||||
pageid = result['query']['pages'].keys()[0]
|
||||
if pageid == "-1": # Page does not exist
|
||||
return ""
|
||||
return result['query']['pages'][str(pageid)]['revisions'][0]['*']
|
||||
|
||||
def get_token(self, page, _type):
|
||||
''' _type can be edit, delete, protect, move, block, unblock, email or import'''
|
||||
if len(self.username) > 0 and not self.is_logged():
|
||||
self.login()
|
||||
|
||||
data = {'action': 'query',
|
||||
'prop': 'info',
|
||||
'titles': page,
|
||||
'intoken': _type,
|
||||
}
|
||||
result = self.API_get(data)
|
||||
pageid = result['query']['pages'].keys()[0]
|
||||
return result['query']['pages'][str(pageid)][_type+'token']
|
||||
|
||||
|
||||
def set_wiki_source(self, content, message=None, minor=False):
|
||||
if len(self.username) > 0 and not self.is_logged():
|
||||
self.login()
|
||||
|
||||
page = content.id
|
||||
token = self.get_token(page, 'edit')
|
||||
|
||||
data = {'action': 'edit',
|
||||
'title': page,
|
||||
'token': token,
|
||||
'text': content.content.encode('utf-8'),
|
||||
'summary': message,
|
||||
}
|
||||
if minor:
|
||||
data['minor'] = 'true'
|
||||
|
||||
self.API_post(data)
|
||||
|
||||
def get_wiki_preview(self, content, message=None):
|
||||
data = {'action': 'parse',
|
||||
'title': content.id,
|
||||
'text': content.content.encode('utf-8'),
|
||||
'summary': message,
|
||||
}
|
||||
result = self.API_post(data)
|
||||
return result['parse']['text']['*']
|
||||
|
||||
def is_logged(self):
|
||||
data = {'action': 'query',
|
||||
'meta': 'userinfo',
|
||||
}
|
||||
result = self.API_get(data)
|
||||
return result['query']['userinfo']['id'] != 0
|
||||
|
||||
def login(self):
|
||||
assert isinstance(self.username, basestring)
|
||||
assert isinstance(self.password, basestring)
|
||||
assert isinstance(self.apiurl, basestring)
|
||||
|
||||
data = {'action': 'login',
|
||||
'lgname': self.username,
|
||||
'lgpassword': self.password,
|
||||
}
|
||||
result = self.API_post(data)
|
||||
if result['login']['result'] == 'WrongPass':
|
||||
raise BrowserIncorrectPassword()
|
||||
|
||||
if result['login']['result'] == 'NeedToken':
|
||||
data['lgtoken'] = result['login']['token']
|
||||
self.API_post(data)
|
||||
|
||||
def iter_wiki_revisions(self, page, nb_entries):
|
||||
'''Yield 'Revision' objects for the last <nb_entries> revisions of the specified page.'''
|
||||
if len(self.username) > 0 and not self.is_logged():
|
||||
self.login()
|
||||
data = {'action': 'query',
|
||||
'titles': page,
|
||||
'prop': 'revisions',
|
||||
'rvprop': 'ids|timestamp|comment|user|flags',
|
||||
'rvlimit': str(nb_entries),
|
||||
}
|
||||
|
||||
result = self.API_get(data)
|
||||
pageid = str(result['query']['pages'].keys()[0])
|
||||
|
||||
if pageid != "-1":
|
||||
for rev in result['query']['pages'][pageid]['revisions']:
|
||||
rev_content = Revision(str(rev['revid']))
|
||||
rev_content.comment = rev['comment']
|
||||
rev_content.revision = str(rev['revid'])
|
||||
rev_content.author = rev['user']
|
||||
rev_content.timestamp = datetime.datetime.strptime(rev['timestamp'], '%Y-%m-%dT%H:%M:%SZ')
|
||||
if rev.has_key('minor'):
|
||||
rev_content.minor = True
|
||||
else:
|
||||
rev_content.minor = False
|
||||
yield rev_content
|
||||
|
||||
def home(self):
|
||||
'''We don't need to change location, we're using the JSON API here.'''
|
||||
pass
|
||||
|
||||
def check_result(self, result):
|
||||
if 'error' in result:
|
||||
raise APIError('%s' % result['error']['info'])
|
||||
|
||||
def API_get(self, data):
|
||||
'''Submit a GET request to the website
|
||||
The JSON data is parsed and returned as a dictionary'''
|
||||
data['format'] = 'json'
|
||||
result = simplejson.loads(self.readurl(self.buildurl(self.apiurl, **data)), 'utf-8')
|
||||
self.check_result(result)
|
||||
return result
|
||||
|
||||
def API_post(self, data):
|
||||
'''Submit a POST request to the website
|
||||
The JSON data is parsed and returned as a dictionary'''
|
||||
|
||||
data['format'] = 'json'
|
||||
result = simplejson.loads(self.readurl(self.apiurl, urllib.urlencode(data)), 'utf-8')
|
||||
self.check_result(result)
|
||||
return result
|
||||
BIN
modules/mediawiki/favicon.png
Normal file
BIN
modules/mediawiki/favicon.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 1.1 KiB |
43
modules/mediawiki/test.py
Normal file
43
modules/mediawiki/test.py
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright(C) 2011 Clément Schreiner
|
||||
#
|
||||
# This file is part of weboob.
|
||||
#
|
||||
# weboob is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# weboob is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
from weboob.tools.test import BackendTest
|
||||
from datetime import datetime
|
||||
|
||||
class MediawikiTest(BackendTest):
|
||||
BACKEND = 'mediawiki'
|
||||
|
||||
def test_get_content(self):
|
||||
self.backend.get_content(u"Utilisateur:Clemux/Test")
|
||||
|
||||
def test_iter_revisions(self):
|
||||
for rev in self.backend.iter_revisions(u"Utilisateur:Clemux/Test"):
|
||||
pass
|
||||
|
||||
def test_push_content(self):
|
||||
content = self.backend.get_content(u"Utilisateur:Clemux/Test")
|
||||
content.content = "test "+str(datetime.now())
|
||||
self.backend.push_content(content, message="test weboob", minor=True)
|
||||
new_content = self.backend.get_content(u"Utilisateur:Clemux/Test")
|
||||
assert content.content == new_content.content
|
||||
|
||||
def test_content_preview(self):
|
||||
content = self.backend.get_content(u"Utilisateur:Clemux/Test")
|
||||
self.backend.get_content_preview(content)
|
||||
Loading…
Add table
Add a link
Reference in a new issue