a3b75fc2bb
It was moved to the right directory, but Repositories tries to find it in the datadir, not in the workdir.
513 lines
17 KiB
Python
513 lines
17 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
# Copyright(C) 2010-2012 Romain Bignon
|
|
#
|
|
# This file is part of weboob.
|
|
#
|
|
# weboob is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU Affero General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# weboob is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Affero General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Affero General Public License
|
|
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
from __future__ import with_statement
|
|
|
|
import tarfile
|
|
import posixpath
|
|
import shutil
|
|
import re
|
|
import sys
|
|
import os
|
|
from datetime import datetime
|
|
from contextlib import closing
|
|
|
|
from .modules import Module
|
|
from weboob.tools.log import getLogger
|
|
from weboob.tools.misc import to_unicode
|
|
from weboob.tools.browser import StandardBrowser, BrowserUnavailable
|
|
from ConfigParser import RawConfigParser, DEFAULTSECT
|
|
|
|
class ModuleInfo(object):
|
|
def __init__(self, name):
|
|
self.name = name
|
|
|
|
# path to the local directory containing this module.
|
|
self.path = None
|
|
self.url = None
|
|
|
|
self.version = 0
|
|
self.capabilities = ()
|
|
self.description = u''
|
|
self.maintainer = u''
|
|
self.license = u''
|
|
self.icon = u''
|
|
self.urls = u''
|
|
|
|
def load(self, items):
|
|
self.version = int(items['version'])
|
|
self.capabilities = items['capabilities'].split()
|
|
self.description = to_unicode(items['description'])
|
|
self.maintainer = to_unicode(items['maintainer'])
|
|
self.license = to_unicode(items['license'])
|
|
self.icon = items['icon']
|
|
self.urls = items['urls']
|
|
|
|
def has_caps(self, caps):
|
|
if not isinstance(caps, (list,tuple)):
|
|
caps = [caps]
|
|
for c in caps:
|
|
if type(c) == type:
|
|
c = c.__name__
|
|
if c in self.capabilities:
|
|
return True
|
|
return False
|
|
|
|
def is_installed(self):
|
|
return self.path is not None
|
|
|
|
def is_local(self):
|
|
return self.url is None
|
|
|
|
def dump(self):
|
|
return (('version', self.version),
|
|
('capabilities', ' '.join(self.capabilities)),
|
|
('description', self.description),
|
|
('maintainer', self.maintainer),
|
|
('license', self.license),
|
|
('icon', self.icon),
|
|
('urls', self.urls),
|
|
)
|
|
|
|
class RepositoryUnavailable(Exception):
|
|
pass
|
|
|
|
class Repository(object):
|
|
INDEX = 'modules.list'
|
|
|
|
def __init__(self, url):
|
|
self.url = url
|
|
self.name = u''
|
|
self.update = 0
|
|
self.maintainer = u''
|
|
self.local = None
|
|
|
|
self.modules = {}
|
|
|
|
if self.url.startswith('file://'):
|
|
self.local = True
|
|
elif re.match('https?://.*', self.url):
|
|
self.local = False
|
|
else:
|
|
# This is probably a file in ~/.weboob/repositories/, we
|
|
# don't know if this is a local or a remote repository.
|
|
with open(self.url, 'r') as fp:
|
|
self.parse_index(fp)
|
|
|
|
def localurl2path(self):
|
|
"""
|
|
Get a local path of a file:// URL.
|
|
"""
|
|
assert self.local == True
|
|
|
|
if self.url.startswith('file://'):
|
|
return self.url[len('file://'):]
|
|
return self.url
|
|
|
|
def retrieve_index(self, dest_path):
|
|
"""
|
|
Retrieve the index file of this repository. It can use network
|
|
if this is a remote repository.
|
|
|
|
@param dest_path [str] path to save the downloaded index file.
|
|
"""
|
|
if self.local:
|
|
# Repository is local, open the file.
|
|
filename = os.path.join(self.localurl2path(), self.INDEX)
|
|
try:
|
|
fp = open(filename, 'r')
|
|
except IOError, e:
|
|
# This local repository doesn't contain a built modules.list index.
|
|
self.name = self.url.replace(os.path.sep, '_')
|
|
self.build_index(self.localurl2path(), filename)
|
|
fp = open(filename, 'r')
|
|
else:
|
|
# This is a remote repository, download file
|
|
browser = StandardBrowser()
|
|
try:
|
|
fp = browser.openurl(posixpath.join(self.url, self.INDEX))
|
|
except BrowserUnavailable, e:
|
|
raise RepositoryUnavailable(unicode(e))
|
|
|
|
self.parse_index(fp)
|
|
|
|
if self.local:
|
|
# Always rebuild index of a local repository.
|
|
self.build_index(self.localurl2path(), filename)
|
|
|
|
# Save the repository index in ~/.weboob/repositories/
|
|
self.save(dest_path, private=True)
|
|
|
|
def parse_index(self, fp):
|
|
"""
|
|
Parse index of a repository
|
|
|
|
@param fp [buffer] file descriptor to read
|
|
"""
|
|
config = RawConfigParser()
|
|
config.readfp(fp)
|
|
|
|
# Read default parameters
|
|
items = dict(config.items(DEFAULTSECT))
|
|
try:
|
|
self.name = items['name']
|
|
self.update = int(items['update'])
|
|
self.maintainer = items['maintainer']
|
|
except KeyError, e:
|
|
raise RepositoryUnavailable('Missing global parameters in repository: %s' % e)
|
|
except ValueError, e:
|
|
raise RepositoryUnavailable('Incorrect value in repository parameters: %s' % e)
|
|
|
|
if len(self.name) == 0:
|
|
raise RepositoryUnavailable('Name is empty')
|
|
|
|
if 'url' in items:
|
|
self.url = items['url']
|
|
self.local = self.url.startswith('file://')
|
|
elif self.local is None:
|
|
raise RepositoryUnavailable('Missing "url" key in settings')
|
|
|
|
# Load modules
|
|
self.modules.clear()
|
|
for section in config.sections():
|
|
module = ModuleInfo(section)
|
|
module.load(dict(config.items(section)))
|
|
if not self.local:
|
|
module.url = posixpath.join(self.url, '%s.tar.gz' % module.name)
|
|
self.modules[section] = module
|
|
|
|
def build_index(self, path, filename):
|
|
"""
|
|
Rebuild index of modules of repository.
|
|
|
|
@param path [str] path of the repository
|
|
@param filename [str] file to save index
|
|
"""
|
|
print 'Rebuild index'
|
|
self.modules.clear()
|
|
|
|
sys.path.append(path)
|
|
for name in sorted(os.listdir(path)):
|
|
module_path = os.path.join(path, name)
|
|
if not os.path.isdir(module_path) or '.' in name:
|
|
continue
|
|
|
|
try:
|
|
module = Module(__import__(name, fromlist=[str(name)]))
|
|
except Exception, e:
|
|
print 'ERROR: %s' % e
|
|
else:
|
|
m = ModuleInfo(module.name)
|
|
m.version = int(datetime.fromtimestamp(os.path.getmtime(module_path)).strftime('%Y%m%d%H%M'))
|
|
m.capabilities = [c.__name__ for c in module.iter_caps()]
|
|
m.description = module.description
|
|
m.maintainer = module.maintainer
|
|
m.license = module.license
|
|
m.icon = module.icon or ''
|
|
self.modules[module.name] = m
|
|
sys.path.remove(path)
|
|
|
|
self.update = int(datetime.now().strftime('%Y%m%d%H%M'))
|
|
self.save(filename)
|
|
|
|
def save(self, filename, private=False):
|
|
"""
|
|
Save repository into a file (modules.list for example).
|
|
|
|
@param filename [str] path to file to save repository.
|
|
@param private [bool] if enabled, save URL of repository.
|
|
"""
|
|
config = RawConfigParser()
|
|
config.set(DEFAULTSECT, 'name', self.name)
|
|
config.set(DEFAULTSECT, 'update', self.update)
|
|
config.set(DEFAULTSECT, 'maintainer', self.maintainer)
|
|
if private:
|
|
config.set(DEFAULTSECT, 'url', self.url)
|
|
|
|
for module in self.modules.itervalues():
|
|
config.add_section(module.name)
|
|
for key, value in module.dump():
|
|
config.set(module.name, key, to_unicode(value).encode('utf-8'))
|
|
|
|
with open(filename, 'wb') as f:
|
|
config.write(f)
|
|
|
|
class Versions(object):
|
|
VERSIONS_LIST = 'versions.list'
|
|
|
|
def __init__(self, path):
|
|
self.path = path
|
|
self.versions = {}
|
|
|
|
try:
|
|
with open(os.path.join(self.path, self.VERSIONS_LIST), 'r') as fp:
|
|
config = RawConfigParser()
|
|
config.readfp(fp)
|
|
|
|
# Read default parameters
|
|
for key, value in config.items(DEFAULTSECT):
|
|
self.versions[key] = int(value)
|
|
except IOError:
|
|
pass
|
|
|
|
def get(self, name):
|
|
return self.versions.get(name, None)
|
|
|
|
def set(self, name, version):
|
|
self.versions[name] = int(version)
|
|
self.save()
|
|
|
|
def save(self):
|
|
config = RawConfigParser()
|
|
for name, version in self.versions.iteritems():
|
|
config.set(DEFAULTSECT, name, version)
|
|
with open(os.path.join(self.path, self.VERSIONS_LIST), 'wb') as fp:
|
|
config.write(fp)
|
|
|
|
class IProgress:
|
|
def progress(self, percent, message):
|
|
print '=== [%3.0f%%] %s' % (percent*100, message)
|
|
|
|
class ModuleInstallError(Exception):
|
|
pass
|
|
|
|
DEFAULT_SOURCES_LIST = \
|
|
"""# List of Weboob repositories
|
|
#
|
|
# The entries below override the entries above (with
|
|
# backends of the same name).
|
|
|
|
http://updates.weboob.org/%(version)s/main/
|
|
|
|
# To enable Not Safe For Work backends, uncomment the
|
|
# following line:
|
|
#http://updates.weboob.org/%(version)s/nsfw/
|
|
|
|
# DEVELOPMENT
|
|
# If you want to hack on Weboob modules, you may add a
|
|
# reference to sources, for example:
|
|
#file:///home/rom1/src/weboob/modules/
|
|
"""
|
|
|
|
class Repositories(object):
|
|
SOURCES_LIST = 'sources.list'
|
|
MODULES_DIR = 'modules'
|
|
REPOSITORIES_DIR = 'repositories'
|
|
ICONS_DIR = 'icons'
|
|
|
|
SHARE_DIRS = [MODULES_DIR, REPOSITORIES_DIR, ICONS_DIR]
|
|
|
|
def __init__(self, workdir, datadir, version):
|
|
self.logger = getLogger('repositories')
|
|
self.version = version
|
|
self.workdir = workdir
|
|
self.datadir = datadir
|
|
self.sources_list = os.path.join(self.workdir, self.SOURCES_LIST)
|
|
self.modules_dir = os.path.join(self.datadir, self.MODULES_DIR)
|
|
self.repos_dir = os.path.join(self.datadir, self.REPOSITORIES_DIR)
|
|
self.icons_dir = os.path.join(self.datadir, self.ICONS_DIR)
|
|
|
|
self.create_dir(self.datadir)
|
|
self.create_dir(self.repos_dir)
|
|
self.create_dir(self.modules_dir)
|
|
self.create_dir(self.icons_dir)
|
|
|
|
self.versions = Versions(self.modules_dir)
|
|
|
|
self.repositories = []
|
|
|
|
if not os.path.exists(self.sources_list):
|
|
with open(self.sources_list, 'w') as f:
|
|
f.write(DEFAULT_SOURCES_LIST)
|
|
self.update()
|
|
else:
|
|
self.load()
|
|
|
|
def create_dir(self, name):
|
|
if not os.path.exists(name):
|
|
os.makedirs(name)
|
|
elif not os.path.isdir(name):
|
|
self.logger.error(u'"%s" is not a directory' % name)
|
|
|
|
def _extend_module_info(self, repos, info):
|
|
if repos.local:
|
|
info.path = repos.localurl2path()
|
|
elif self.versions.get(info.name) is not None:
|
|
info.path = self.modules_dir
|
|
return info
|
|
|
|
def get_all_modules_info(self, caps=None):
|
|
"""
|
|
Get all ModuleInfo instances available.
|
|
|
|
@param caps [list(str)] Filter on capabilities.
|
|
@return [dict(ModuleInfo)]
|
|
"""
|
|
modules = {}
|
|
for repos in reversed(self.repositories):
|
|
for name, info in repos.modules.iteritems():
|
|
if not name in modules and (not caps or info.has_caps(caps)):
|
|
modules[name] = self._extend_module_info(repos, info)
|
|
return modules
|
|
|
|
def get_module_info(self, name):
|
|
"""
|
|
Get ModuleInfo object of a module.
|
|
|
|
It tries all repositories from last to first, and set
|
|
the 'path' attribute of ModuleInfo if it is installed.
|
|
"""
|
|
for repos in reversed(self.repositories):
|
|
if name in repos.modules:
|
|
m = repos.modules[name]
|
|
self._extend_module_info(repos, m)
|
|
return m
|
|
return None
|
|
|
|
def load(self):
|
|
"""
|
|
Load repositories from ~/.weboob/repositories/.
|
|
"""
|
|
self.repositories = []
|
|
for name in os.listdir(self.repos_dir):
|
|
repository = Repository(os.path.join(self.repos_dir, name))
|
|
self.repositories.append(repository)
|
|
|
|
def retrieve_icon(self, module):
|
|
"""
|
|
Retrieve the icon of a module and save it in ~/.weboob/icons/.
|
|
"""
|
|
if not isinstance(module, ModuleInfo):
|
|
module = self.get_module_info(module)
|
|
|
|
dest_path = os.path.join(self.icons_dir, '%s.png' % module.name)
|
|
|
|
if module.is_local():
|
|
icon_path = os.path.join(module.path, module.name, 'favicon.png')
|
|
if module.path and os.path.exists(icon_path):
|
|
shutil.copy(icon_path, dest_path)
|
|
return
|
|
|
|
if module.icon:
|
|
icon_url = module.icon
|
|
else:
|
|
icon_url = module.url.replace('.tar.gz', '.png')
|
|
|
|
browser = StandardBrowser()
|
|
try:
|
|
icon = browser.openurl(icon_url)
|
|
except BrowserUnavailable:
|
|
pass # no icon, no problem
|
|
else:
|
|
with open(dest_path, 'wb') as fp:
|
|
fp.write(icon.read())
|
|
|
|
def update(self, progress=IProgress()):
|
|
"""
|
|
Update list of repositories by downloading them
|
|
and put them in ~/.weboob/repositories/.
|
|
|
|
@param progress [IProgress] observer object.
|
|
"""
|
|
self.repositories = []
|
|
for name in os.listdir(self.repos_dir):
|
|
os.remove(os.path.join(self.repos_dir, name))
|
|
|
|
with open(self.sources_list, 'r') as f:
|
|
for line in f.xreadlines():
|
|
line = line.strip() % {'version': self.version}
|
|
m = re.match('(file|https?)://.*', line)
|
|
if m:
|
|
print 'Getting %s' % line
|
|
repository = Repository(line)
|
|
dest_path = os.path.join(self.repos_dir, '%02d-%s' % (len(self.repositories),
|
|
repository.url.replace(os.path.sep, '_')))
|
|
try:
|
|
repository.retrieve_index(dest_path)
|
|
except RepositoryUnavailable, e:
|
|
print 'Error: %s' % e
|
|
else:
|
|
self.repositories.append(repository)
|
|
|
|
to_update = []
|
|
for name, info in self.get_all_modules_info().iteritems():
|
|
if not info.is_local() and info.is_installed():
|
|
to_update.append(info)
|
|
|
|
class InstallProgress(IProgress):
|
|
def __init__(self, n):
|
|
self.n = n
|
|
|
|
def progress(self, percent, message):
|
|
progress.progress(float(self.n)/len(to_update) + 1.0/len(to_update)*percent, message)
|
|
|
|
for n, info in enumerate(to_update):
|
|
inst_progress = InstallProgress(n)
|
|
try:
|
|
self.install(info, inst_progress)
|
|
except ModuleInstallError, e:
|
|
inst_progress.progress(1.0, unicode(e))
|
|
|
|
def install(self, module, progress=IProgress()):
|
|
"""
|
|
Install a module.
|
|
|
|
@paran module [str,ModuleInfo] module to install.
|
|
@param progress [IProgress] observer object.
|
|
"""
|
|
if isinstance(module, ModuleInfo):
|
|
info = module
|
|
elif isinstance(module, basestring):
|
|
progress.progress(0.0, 'Looking for module %s' % module)
|
|
info = self.get_module_info(module)
|
|
if not info:
|
|
raise ModuleInstallError('Module "%s" does not exist' % module)
|
|
else:
|
|
raise ValueError('"module" parameter might be a ModuleInfo object or a string, not %r' % module)
|
|
|
|
if info.is_local():
|
|
raise ModuleInstallError('%s is available on local.' % info.name)
|
|
|
|
installed = self.versions.get(info.name)
|
|
if installed is None:
|
|
progress.progress(0.3, 'Module is not installed yet')
|
|
elif info.version > installed:
|
|
progress.progress(0.3, 'A new version of this module is available')
|
|
else:
|
|
raise ModuleInstallError('The last version of %s is already installed' % info.name)
|
|
|
|
browser = StandardBrowser()
|
|
progress.progress(0.2, 'Downloading module...')
|
|
try:
|
|
fp = browser.openurl(info.url)
|
|
except BrowserUnavailable, e:
|
|
raise ModuleInstallError('Unable to fetch module: %s' % e)
|
|
|
|
progress.progress(0.7, 'Setting up module...')
|
|
|
|
# Extract module from tarball.
|
|
with closing(tarfile.open('', 'r:gz', fp)) as tar:
|
|
tar.extractall(self.modules_dir)
|
|
|
|
self.versions.set(info.name, info.version)
|
|
|
|
progress.progress(0.9, 'Downloading icon...')
|
|
self.retrieve_icon(info)
|
|
|
|
progress.progress(1.0, 'Module %s has been installed!' % info.name)
|