nojhan/weboob-devel
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Centralize encoding guesses, default to UTF-8

Browse source 
This might not be enough for print() and could need a locale.setlocale()
even though it is generally discouraged.

closes #1352
This commit is contained in:
Laurent Bachelier 2014-09-03 00:49:51 +02:00
commit 19a95dc0d6
4 changed files with 96 additions and 90 deletions
Download patch file Download diff file Expand all files Collapse all files

67
weboob/tools/application/console.py
View file

@ -76,11 +76,22 @@ class ConsoleApplication(BaseApplication):
stdin = sys.stdin
stdout = sys.stdout
stderr = sys.stderr
def __init__(self, option_parser=None):
BaseApplication.__init__(self, option_parser)
self.weboob.callbacks['login'] = self.login_cb
self.enabled_backends = set()
self.encoding = self.guess_encoding()
def guess_encoding(self, stdio=None):
if stdio is None:
stdio = self.stdout
encoding = stdio.encoding or locale.getpreferredencoding()
# ASCII or ANSII is most likely a user mistake
if not encoding or encoding.lower() == 'ascii' or encoding.lower().startswith('ansi'):
encoding = 'UTF-8'
return encoding
def login_cb(self, backend_name, value):
return self.ask('[%s] %s' % (backend_name,
@ -109,7 +120,7 @@ class ConsoleApplication(BaseApplication):
ret = super(ConsoleApplication, self).load_backends(*args, **kwargs)
for err in errors:
print('Error(%s): %s' % (err.backend_name, err), file=sys.stderr)
print('Error(%s): %s' % (err.backend_name, err), file=self.stderr)
if self.ask('Do you want to reconfigure this backend?', default=True):
self.edit_backend(err.backend_name)
self.load_backends(names=[err.backend_name])
@ -124,7 +135,7 @@ class ConsoleApplication(BaseApplication):
def check_loaded_backends(self, default_config=None):
while len(self.enabled_backends) == 0:
print('Warning: there is currently no configured backend for %s' % self.APPNAME)
if not os.isatty(sys.stdout.fileno()) or not self.ask('Do you want to configure backends?', default=True):
if not os.isatty(self.stdout.fileno()) or not self.ask('Do you want to configure backends?', default=True):
return False
self.prompt_create_backends(default_config)
@ -158,7 +169,7 @@ class ConsoleApplication(BaseApplication):
if str(r).isdigit():
i = int(r) - 1
if i < 0 or i >= len(modules):
print('Error: %s is not a valid choice' % r, file=sys.stderr)
print('Error: %s is not a valid choice' % r, file=self.stderr)
continue
name = modules[i]
try:
@ -245,11 +256,11 @@ class ConsoleApplication(BaseApplication):
backend = None
if not backend:
print('Backend "%s" does not exist.' % name, file=sys.stderr)
print('Backend "%s" does not exist.' % name, file=self.stderr)
return 1
if not backend.has_caps(CapAccount) or backend.klass.ACCOUNT_REGISTER_PROPERTIES is None:
print('You can\'t register a new account with %s' % name, file=sys.stderr)
print('You can\'t register a new account with %s' % name, file=self.stderr)
return 1
account = Account()
@ -294,7 +305,7 @@ class ConsoleApplication(BaseApplication):
try:
self.weboob.repositories.install(name)
except ModuleInstallError as e:
print('Unable to install module "%s": %s' % (name, e), file=sys.stderr)
print('Unable to install module "%s": %s' % (name, e), file=self.stderr)
return False
print('')
@ -326,7 +337,7 @@ class ConsoleApplication(BaseApplication):
params = items
config = module.config.load(self.weboob, bname, name, params, nofail=True)
except ModuleLoadError as e:
print('Unable to load module "%s": %s' % (name, e), file=sys.stderr)
print('Unable to load module "%s": %s' % (name, e), file=self.stderr)
return 1
# ask for params non-specified on command-line arguments
@ -345,7 +356,7 @@ class ConsoleApplication(BaseApplication):
print('-------------------------%s' % ('-' * len(module.name)))
while not edit and self.weboob.backends_config.backend_exists(name):
print('Backend instance "%s" already exists in "%s"' % (name, self.weboob.backends_config.confpath), file=sys.stderr)
print('Backend instance "%s" already exists in "%s"' % (name, self.weboob.backends_config.confpath), file=self.stderr)
if not self.ask('Add new backend for module "%s"?' % module.name, default=False):
return 1
@ -361,7 +372,7 @@ class ConsoleApplication(BaseApplication):
print('Backend "%s" successfully %s.' % (name, 'edited' if edit else 'added'))
return name
except BackendAlreadyExists:
print('Backend "%s" already exists.' % name, file=sys.stderr)
print('Backend "%s" already exists.' % name, file=self.stderr)
return 1
def ask(self, question, default=None, masked=None, regexp=None, choices=None, tiny=None):
@ -406,7 +417,7 @@ class ConsoleApplication(BaseApplication):
question = u'[%s] %s' % (v.id, question)
if isinstance(v, ValueBackendPassword):
print(question.encode(sys.stdout.encoding or locale.getpreferredencoding()) + ':')
print(question.encode(self.encoding) + ':')
question = v.label
choices = OrderedDict()
choices['c'] = 'Run an external tool during backend load'
@ -476,9 +487,9 @@ class ConsoleApplication(BaseApplication):
if sys.platform == 'win32':
line = getpass.getpass(str(question))
else:
line = getpass.getpass(question.encode(sys.stdout.encoding or locale.getpreferredencoding()))
line = getpass.getpass(self.encoding)
else:
self.stdout.write(question.encode(sys.stdout.encoding or locale.getpreferredencoding()))
self.stdout.write(question.encode(self.encoding))
self.stdout.flush()
line = self.stdin.readline()
if len(line) == 0:
@ -497,7 +508,7 @@ class ConsoleApplication(BaseApplication):
try:
v.set(line)
except ValueError as e:
print(u'Error: %s' % e, file=sys.stderr)
print(u'Error: %s' % e, file=self.stderr)
else:
break
@ -512,7 +523,7 @@ class ConsoleApplication(BaseApplication):
filename = f.name
if content is not None:
if isinstance(content, unicode):
content = content.encode(sys.stdin.encoding or locale.getpreferredencoding())
content = content.encode(self.encoding)
f.write(content)
f.flush()
try:
@ -527,7 +538,7 @@ class ConsoleApplication(BaseApplication):
print('Reading content from stdin... Type ctrl-D ' \
'from an empty line to stop.')
text = sys.stdin.read()
return text.decode(sys.stdin.encoding or locale.getpreferredencoding())
return text.decode(self.encoding)
def bcall_error_handler(self, backend, error, backtrace):
"""
@ -539,7 +550,7 @@ class ConsoleApplication(BaseApplication):
msg = unicode(error)
if not msg:
msg = 'invalid login/password.'
print('Error(%s): %s' % (backend.name, msg), file=sys.stderr)
print('Error(%s): %s' % (backend.name, msg), file=self.stderr)
if self.ask('Do you want to reconfigure this backend?', default=True):
self.unload_backends(names=[backend.name])
self.edit_backend(backend.name)
@ -548,21 +559,21 @@ class ConsoleApplication(BaseApplication):
msg = unicode(error)
if not msg:
msg = 'website is unavailable.'
print(u'Error(%s): %s' % (backend.name, msg), file=sys.stderr)
print(u'Error(%s): %s' % (backend.name, msg), file=self.stderr)
elif isinstance(error, BrowserForbidden):
print(u'Error(%s): %s' % (backend.name, msg or 'Forbidden'), file=sys.stderr)
print(u'Error(%s): %s' % (backend.name, msg or 'Forbidden'), file=self.stderr)
elif isinstance(error, NotImplementedError):
print(u'Error(%s): this feature is not supported yet by this backend.' % backend.name, file=sys.stderr)
print(u' %s To help the maintainer of this backend implement this feature,' % (' ' * len(backend.name)), file=sys.stderr)
print(u' %s please contact: %s <%s@issues.weboob.org>' % (' ' * len(backend.name), backend.MAINTAINER, backend.NAME), file=sys.stderr)
print(u'Error(%s): this feature is not supported yet by this backend.' % backend.name, file=self.stderr)
print(u' %s To help the maintainer of this backend implement this feature,' % (' ' * len(backend.name)), file=self.stderr)
print(u' %s please contact: %s <%s@issues.weboob.org>' % (' ' * len(backend.name), backend.MAINTAINER, backend.NAME), file=self.stderr)
elif isinstance(error, UserError):
print(u'Error(%s): %s' % (backend.name, to_unicode(error)), file=sys.stderr)
print(u'Error(%s): %s' % (backend.name, to_unicode(error)), file=self.stderr)
elif isinstance(error, MoreResultsAvailable):
print(u'Hint: There are more results for backend %s' % (backend.name), file=sys.stderr)
print(u'Hint: There are more results for backend %s' % (backend.name), file=self.stderr)
elif isinstance(error, BrowserSSLError):
print(u'FATAL(%s): ' % backend.name + self.BOLD + '/!\ SERVER CERTIFICATE IS INVALID /!\\' + self.NC, file=sys.stderr)
print(u'FATAL(%s): ' % backend.name + self.BOLD + '/!\ SERVER CERTIFICATE IS INVALID /!\\' + self.NC, file=self.stderr)
else:
print(u'Bug(%s): %s' % (backend.name, to_unicode(error)), file=sys.stderr)
print(u'Bug(%s): %s' % (backend.name, to_unicode(error)), file=self.stderr)
minfo = self.weboob.repositories.get_module_info(backend.NAME)
if minfo and not minfo.is_local():
@ -577,7 +588,7 @@ class ConsoleApplication(BaseApplication):
return
if logging.root.level == logging.DEBUG:
print(backtrace, file=sys.stderr)
print(backtrace, file=self.stderr)
else:
return True
@ -596,6 +607,6 @@ class ConsoleApplication(BaseApplication):
ask_debug_mode = True
if ask_debug_mode:
print(debugmsg, file=sys.stderr)
print(debugmsg, file=self.stderr)
elif len(more_results) > 0:
print('Hint: There are more results available for %s (use option -n or count command)' % (', '.join(more_results)), file=sys.stderr)
print('Hint: There are more results available for %s (use option -n or count command)' % (', '.join(more_results)), file=self.stderr)

68
weboob/tools/application/repl.py
View file

@ -22,11 +22,9 @@ from __future__ import print_function
import atexit
from cmd import Cmd
import logging
import locale
import re
from optparse import OptionGroup, OptionParser, IndentedHelpFormatter
import os
import sys
from weboob.capabilities.base import FieldNotFound, BaseObject, UserError
from weboob.core import CallErrors
@ -113,9 +111,11 @@ class ReplApplication(Cmd, ConsoleApplication):
def __init__(self):
Cmd.__init__(self)
ConsoleApplication.__init__(self, ReplOptionParser(self.SYNOPSIS, version=self._get_optparse_version()))
self.intro = '\n'.join(('Welcome to %s%s%s v%s' % (self.BOLD, self.APPNAME, self.NC, self.VERSION),
'',
self.COPYRIGHT.encode(sys.stdout.encoding or locale.getpreferredencoding()),
self.COPYRIGHT.encode(self.encoding),
'This program is free software: you can redistribute it and/or modify',
'it under the terms of the GNU Affero General Public License as published by',
'the Free Software Foundation, either version 3 of the License, or',
@ -130,8 +130,6 @@ class ReplApplication(Cmd, ConsoleApplication):
self.formatter = None
self.commands_formatters = self.COMMANDS_FORMATTERS.copy()
ConsoleApplication.__init__(self, ReplOptionParser(self.SYNOPSIS, version=self._get_optparse_version()))
commands_help = self.get_commands_doc()
self._parser.commands = commands_help
self._parser.formatter = ReplOptionFormatter()
@ -170,7 +168,7 @@ class ReplApplication(Cmd, ConsoleApplication):
# of the same line instead of new line.
#self.prompt = self.BOLD + '%s> ' % self.APPNAME + self.NC
if len(self.working_path.get()):
wp_enc = unicode(self.working_path).encode(sys.stdout.encoding or locale.getpreferredencoding())
wp_enc = unicode(self.working_path).encode(self.encoding)
self.prompt = '%s:%s> ' % (self.APPNAME, wp_enc)
else:
self.prompt = '%s> ' % (self.APPNAME)
@ -210,13 +208,13 @@ class ReplApplication(Cmd, ConsoleApplication):
i = self.ask('Select a backend to proceed with "%s"' % id)
if not i.isdigit():
if not i in dict(e.backends):
print('Error: %s is not a valid backend' % i, file=sys.stderr)
print('Error: %s is not a valid backend' % i, file=self.stderr)
continue
backend_name = i
else:
i = int(i)
if i < 0 or i > len(e.backends):
print('Error: %s is not a valid choice' % i, file=sys.stderr)
print('Error: %s is not a valid choice' % i, file=self.stderr)
continue
backend_name = e.backends[i-1][0]
@ -350,7 +348,7 @@ class ReplApplication(Cmd, ConsoleApplication):
missing_fields = set(self.formatter.MANDATORY_FIELDS) - set(fields)
# If a mandatory field is not selected, do not use the customized formatter
if missing_fields:
print('Warning: you do not select enough mandatory fields for the formatter. Fallback to another. Hint: use option -f', file=sys.stderr)
print('Warning: you do not select enough mandatory fields for the formatter. Fallback to another. Hint: use option -f', file=self.stderr)
self.formatter = self.formatters_loader.build_formatter(ReplApplication.DEFAULT_FORMATTER)
if self.formatter.DISPLAYED_FIELDS is not None:
@ -360,7 +358,7 @@ class ReplApplication(Cmd, ConsoleApplication):
missing_fields = set(fields) - set(self.formatter.DISPLAYED_FIELDS + self.formatter.MANDATORY_FIELDS)
# If a selected field is not displayed, do not use the customized formatter
if missing_fields:
print('Warning: some selected fields will not be displayed by the formatter. Fallback to another. Hint: use option -f', file=sys.stderr)
print('Warning: some selected fields will not be displayed by the formatter. Fallback to another. Hint: use option -f', file=self.stderr)
self.formatter = self.formatters_loader.build_formatter(ReplApplication.DEFAULT_FORMATTER)
return self.weboob.do(self._do_complete, self.options.count, fields, function, *args, **kwargs)
@ -426,9 +424,9 @@ class ReplApplication(Cmd, ConsoleApplication):
except CallErrors as e:
self.bcall_errors_handler(e)
except BackendNotGiven as e:
print('Error: %s' % str(e), file=sys.stderr)
print('Error: %s' % str(e), file=self.stderr)
except NotEnoughArguments as e:
print('Error: not enough arguments. %s' % str(e), file=sys.stderr)
print('Error: not enough arguments. %s' % str(e), file=self.stderr)
except (KeyboardInterrupt, EOFError):
# ^C during a command process doesn't exit application.
print('\nAborted.')
@ -443,12 +441,12 @@ class ReplApplication(Cmd, ConsoleApplication):
pass
def default(self, line):
print('Unknown command: "%s"' % line, file=sys.stderr)
print('Unknown command: "%s"' % line, file=self.stderr)
cmd, arg, ignore = Cmd.parseline(self, line)
if cmd is not None:
names = set(name[3:] for name in self.get_names() if name.startswith('do_' + cmd))
if len(names) > 0:
print('Do you mean: %s?' % ', '.join(names), file=sys.stderr)
print('Do you mean: %s?' % ', '.join(names), file=self.stderr)
return 2
def completenames(self, text, *ignored):
@ -602,7 +600,7 @@ class ReplApplication(Cmd, ConsoleApplication):
lines[0] = '%s%s%s' % (self.BOLD, lines[0], self.NC)
self.stdout.write('%s\n' % '\n'.join(lines))
else:
print('Unknown command: "%s"' % arg, file=sys.stderr)
print('Unknown command: "%s"' % arg, file=self.stderr)
else:
cmds = self._parser.formatter.format_commands(self._parser.commands)
self.stdout.write('%s\n' % cmds)
@ -665,20 +663,20 @@ class ReplApplication(Cmd, ConsoleApplication):
if action in ('add', 'register'):
minfo = self.weboob.repositories.get_module_info(backend_name)
if minfo is None:
print('Module "%s" does not exist.' % backend_name, file=sys.stderr)
print('Module "%s" does not exist.' % backend_name, file=self.stderr)
return 1
else:
if not minfo.has_caps(self.CAPS):
print('Module "%s" is not supported by this application => skipping.' % backend_name, file=sys.stderr)
print('Module "%s" is not supported by this application => skipping.' % backend_name, file=self.stderr)
return 1
else:
if backend_name not in [backend.name for backend in self.weboob.iter_backends()]:
print('Backend "%s" does not exist => skipping.' % backend_name, file=sys.stderr)
print('Backend "%s" does not exist => skipping.' % backend_name, file=self.stderr)
return 1
if action in ('enable', 'disable', 'only', 'add', 'register', 'edit', 'remove'):
if not given_backend_names:
print('Please give at least a backend name.', file=sys.stderr)
print('Please give at least a backend name.', file=self.stderr)
return 2
given_backends = set(backend for backend in self.weboob.iter_backends() if backend.name in given_backend_names)
@ -691,7 +689,7 @@ class ReplApplication(Cmd, ConsoleApplication):
try:
self.enabled_backends.remove(backend)
except KeyError:
print('%s is not enabled' % backend.name, file=sys.stderr)
print('%s is not enabled' % backend.name, file=self.stderr)
elif action == 'only':
self.enabled_backends = set()
for backend in given_backends:
@ -743,11 +741,11 @@ class ReplApplication(Cmd, ConsoleApplication):
print('[%s] %s%-15s%s %s' % (loaded, self.BOLD, name, self.NC, info.description))
else:
print('Unknown action: "%s"' % action, file=sys.stderr)
print('Unknown action: "%s"' % action, file=self.stderr)
return 1
if len(self.enabled_backends) == 0:
print('Warning: no more backends are loaded. %s is probably unusable.' % self.APPNAME.capitalize(), file=sys.stderr)
print('Warning: no more backends are loaded. %s is probably unusable.' % self.APPNAME.capitalize(), file=self.stderr)
def complete_logging(self, text, line, begidx, endidx):
levels = ('debug', 'info', 'warning', 'error', 'quiet', 'default')
@ -788,8 +786,8 @@ class ReplApplication(Cmd, ConsoleApplication):
try:
level = levels[args[0]]
except KeyError:
print('Level "%s" does not exist.' % args[0], file=sys.stderr)
print('Availables: %s' % ' '.join(levels.iterkeys()), file=sys.stderr)
print('Level "%s" does not exist.' % args[0], file=self.stderr)
print('Availables: %s' % ' '.join(levels.iterkeys()), file=self.stderr)
return 2
else:
logging.root.setLevel(level)
@ -813,7 +811,7 @@ class ReplApplication(Cmd, ConsoleApplication):
try:
self.condition = ResultsCondition(line)
except ResultsConditionError as e:
print('%s' % e, file=sys.stderr)
print('%s' % e, file=self.stderr)
return 2
else:
if self.condition is None:
@ -840,7 +838,7 @@ class ReplApplication(Cmd, ConsoleApplication):
try:
count = int(line)
except ValueError:
print('Could not interpret "%s" as a number.' % line, file=sys.stderr)
print('Could not interpret "%s" as a number.' % line, file=self.stderr)
return 2
else:
if count > 0:
@ -902,7 +900,7 @@ class ReplApplication(Cmd, ConsoleApplication):
print('off' if self.options.no_keys else 'on')
else:
if args[2] not in ('on', 'off'):
print('Invalid value "%s". Please use "on" or "off" values.' % args[2], file=sys.stderr)
print('Invalid value "%s". Please use "on" or "off" values.' % args[2], file=self.stderr)
return 2
else:
if args[1] == 'header':
@ -910,7 +908,7 @@ class ReplApplication(Cmd, ConsoleApplication):
elif args[1] == 'keys':
self.options.no_keys = True if args[2] == 'off' else False
else:
print('Don\'t know which option to set. Available options: header, keys.', file=sys.stderr)
print('Don\'t know which option to set. Available options: header, keys.', file=self.stderr)
return 2
else:
if args[0] in self.formatters_loader.get_available_formatters():
@ -921,7 +919,7 @@ class ReplApplication(Cmd, ConsoleApplication):
self.DEFAULT_FORMATTER = self.set_formatter(args[0])
else:
print('Formatter "%s" is not available.\n' \
'Available formatters: %s.' % (args[0], ', '.join(self.formatters_loader.get_available_formatters())), file=sys.stderr)
'Available formatters: %s.' % (args[0], ', '.join(self.formatters_loader.get_available_formatters())), file=self.stderr)
return 1
else:
print('Default formatter: %s' % self.DEFAULT_FORMATTER)
@ -1077,7 +1075,7 @@ class ReplApplication(Cmd, ConsoleApplication):
if len(collections) == 1:
self.working_path.split_path = collections[0].split_path
else:
print(u"Path: %s not found" % unicode(self.working_path), file=sys.stderr)
print(u"Path: %s not found" % unicode(self.working_path), file=self.stderr)
self.working_path.restore()
return 1
@ -1150,7 +1148,7 @@ class ReplApplication(Cmd, ConsoleApplication):
collections = self.all_collections()
for collection in collections:
directories.add(collection.basename.encode(sys.stdout.encoding or locale.getpreferredencoding()))
directories.add(collection.basename.encode(self.encoding))
return [s[offs:] for s in directories if s.startswith(mline)]
@ -1170,10 +1168,10 @@ class ReplApplication(Cmd, ConsoleApplication):
try:
self.formatter = self.formatters_loader.build_formatter(name)
except FormatterLoadError as e:
print('%s' % e, file=sys.stderr)
print('%s' % e, file=self.stderr)
if self.DEFAULT_FORMATTER == name:
self.DEFAULT_FORMATTER = ReplApplication.DEFAULT_FORMATTER
print('Falling back to "%s".' % (self.DEFAULT_FORMATTER), file=sys.stderr)
print('Falling back to "%s".' % (self.DEFAULT_FORMATTER), file=self.stderr)
self.formatter = self.formatters_loader.build_formatter(self.DEFAULT_FORMATTER)
name = self.DEFAULT_FORMATTER
if self.options.no_header:
@ -1211,9 +1209,9 @@ class ReplApplication(Cmd, ConsoleApplication):
try:
self.formatter.format(obj=result, selected_fields=fields, alias=alias)
except FieldNotFound as e:
print(e, file=sys.stderr)
print(e, file=self.stderr)
except MandatoryFieldsNotFound as e:
print('%s Hint: select missing fields or use another formatter (ex: multiline).' % e, file=sys.stderr)
print('%s Hint: select missing fields or use another formatter (ex: multiline).' % e, file=self.stderr)
def flush(self):
self.formatter.flush()