pax_global_header 0000666 0000000 0000000 00000000064 13414137563 0014521 g ustar 00root root 0000000 0000000 52 comment=130005535d1d257a835857d4e42b2541ce490b40
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/ 0000775 0000000 0000000 00000000000 13414137563 0021417 5 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/ 0000775 0000000 0000000 00000000000 13414137563 0022674 5 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/ 0000775 0000000 0000000 00000000000 13414137563 0024034 5 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/__init__.py 0000664 0000000 0000000 00000000000 13414137563 0026133 0 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/ 0000775 0000000 0000000 00000000000 13414137563 0026337 5 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/__init__.py 0000664 0000000 0000000 00000000000 13414137563 0030436 0 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/base.py 0000664 0000000 0000000 00000044413 13414137563 0027631 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2012 Romain Bignon, Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import print_function
import codecs
import logging
import optparse
from optparse import OptionGroup, OptionParser
from datetime import datetime
import os
import sys
import warnings
from weboob.capabilities.base import ConversionWarning, BaseObject
from weboob.core import Weboob, CallErrors
from weboob.core.backendscfg import BackendsConfig
from weboob.tools.config.iconfig import ConfigError
from weboob.exceptions import FormFieldConversionWarning
from weboob.tools.log import createColoredFormatter, getLogger, DEBUG_FILTERS, settings as log_settings
from weboob.tools.misc import to_unicode, guess_encoding
from weboob.tools.compat import unicode
from .results import ResultsConditionError
__all__ = ['Application']
class MoreResultsAvailable(Exception):
pass
class ApplicationStorage(object):
def __init__(self, name, storage):
self.name = name
self.storage = storage
def set(self, *args):
if self.storage:
return self.storage.set('applications', self.name, *args)
def delete(self, *args):
if self.storage:
return self.storage.delete('applications', self.name, *args)
def get(self, *args, **kwargs):
if self.storage:
return self.storage.get('applications', self.name, *args, **kwargs)
else:
return kwargs.get('default', None)
def load(self, default):
if self.storage:
return self.storage.load('applications', self.name, default)
def save(self):
if self.storage:
return self.storage.save('applications', self.name)
class Application(object):
"""
Base application.
This class can be herited to have some common code within weboob
applications.
"""
# ------ Class attributes --------------------------------------
# Application name
APPNAME = ''
# Configuration and work directory (if None, use the Weboob instance one)
CONFDIR = None
# Default configuration dict (can only contain key/values)
CONFIG = {}
# Default storage tree
STORAGE = {}
# Synopsis
SYNOPSIS = 'Usage: %prog [-h] [-dqv] [-b backends] ...\n'
SYNOPSIS += ' %prog [--help] [--version]'
# Description
DESCRIPTION = None
# Version
VERSION = None
# Copyright
COPYRIGHT = None
# Verbosity of DEBUG
DEBUG_FILTER = 2
stdin = sys.stdin
stdout = sys.stdout
stderr = sys.stderr
# ------ Abstract methods --------------------------------------
def create_weboob(self):
return Weboob()
def _get_completions(self):
"""
Overload this method in subclasses if you want to enrich shell completion.
@return a set object
"""
return set()
def _handle_options(self):
"""
Overload this method in application type subclass
if you want to handle options defined in subclass constructor.
"""
pass
def add_application_options(self, group):
"""
Overload this method if your application needs extra options.
These options will be displayed in an option group.
"""
pass
def handle_application_options(self):
"""
Overload this method in your application if you want to handle options defined in add_application_options.
"""
pass
# ------ Application methods -------------------------------
def __init__(self, option_parser=None):
super(Application, self).__init__()
self.encoding = self.guess_encoding()
self.logger = getLogger(self.APPNAME)
self.weboob = self.create_weboob()
if self.CONFDIR is None:
self.CONFDIR = self.weboob.workdir
self.config = None
self.options = None
self.condition = None
self.storage = None
if option_parser is None:
self._parser = OptionParser(self.SYNOPSIS, version=self._get_optparse_version())
else:
self._parser = option_parser
if self.DESCRIPTION:
self._parser.description = self.DESCRIPTION
app_options = OptionGroup(self._parser, '%s Options' % self.APPNAME.capitalize())
self.add_application_options(app_options)
if len(app_options.option_list) > 0:
self._parser.add_option_group(app_options)
self._parser.add_option('-b', '--backends', help='what backend(s) to enable (comma separated)')
self._parser.add_option('-e', '--exclude-backends', help='what backend(s) to exclude (comma separated)')
self._parser.add_option('-I', '--insecure', action='store_true', help='do not validate SSL')
self._parser.add_option('--nss', action='store_true', help='Use NSS instead of OpenSSL')
logging_options = OptionGroup(self._parser, 'Logging Options')
logging_options.add_option('-d', '--debug', action='count', help='display debug messages. Set up it twice to more verbosity', default=0)
logging_options.add_option('-q', '--quiet', action='store_true', help='display only error messages')
logging_options.add_option('-v', '--verbose', action='store_true', help='display info messages')
logging_options.add_option('--logging-file', action='store', type='string', dest='logging_file', help='file to save logs')
logging_options.add_option('-a', '--save-responses', action='store_true', help='save every response')
logging_options.add_option('--export-session', action='store_true', help='log browser session cookies after login')
self._parser.add_option_group(logging_options)
self._parser.add_option('--shell-completion', action='store_true', help=optparse.SUPPRESS_HELP)
self._is_default_count = True
def guess_encoding(self, stdio=None):
return guess_encoding(stdio or self.stdout)
def deinit(self):
self.weboob.want_stop()
self.weboob.deinit()
def create_storage(self, path=None, klass=None, localonly=False):
"""
Create a storage object.
:param path: An optional specific path
:type path: :class:`str`
:param klass: What class to instance
:type klass: :class:`weboob.tools.storage.IStorage`
:param localonly: If True, do not set it on the :class:`Weboob` object.
:type localonly: :class:`bool`
:rtype: :class:`weboob.tools.storage.IStorage`
"""
if klass is None:
from weboob.tools.storage import StandardStorage
klass = StandardStorage
if path is None:
path = os.path.join(self.CONFDIR, self.APPNAME + '.storage')
elif os.path.sep not in path:
path = os.path.join(self.CONFDIR, path)
storage = klass(path)
self.storage = ApplicationStorage(self.APPNAME, storage)
self.storage.load(self.STORAGE)
if not localonly:
self.weboob.storage = storage
return storage
def load_config(self, path=None, klass=None):
"""
Load a configuration file and get his object.
:param path: An optional specific path
:type path: :class:`str`
:param klass: What class to instance
:type klass: :class:`weboob.tools.config.iconfig.IConfig`
:rtype: :class:`weboob.tools.config.iconfig.IConfig`
"""
if klass is None:
from weboob.tools.config.iniconfig import INIConfig
klass = INIConfig
if path is None:
path = os.path.join(self.CONFDIR, self.APPNAME)
elif os.path.sep not in path:
path = os.path.join(self.CONFDIR, path)
self.config = klass(path)
self.config.load(self.CONFIG)
if self.config.get('use_nss', default=False):
self.setup_nss()
if self.config.get('export_session', default=False):
log_settings['export_session'] = True
def main(self, argv):
"""
Main method
Called by run
"""
raise NotImplementedError()
def load_backends(self, caps=None, names=None, exclude=None, *args, **kwargs):
if names is None and self.options.backends:
names = self.options.backends.split(',')
if exclude is None and self.options.exclude_backends:
exclude = self.options.exclude_backends.split(',')
loaded = self.weboob.load_backends(caps, names, exclude=exclude, *args, **kwargs)
if not loaded:
logging.info(u'No backend loaded')
return loaded
def _get_optparse_version(self):
version = None
if self.VERSION:
if self.COPYRIGHT:
copyright = self.COPYRIGHT.replace('YEAR', '%d' % datetime.today().year)
version = '%s v%s %s' % (self.APPNAME, self.VERSION, copyright)
else:
version = '%s v%s' % (self.APPNAME, self.VERSION)
return version
def _do_complete_obj(self, backend, fields, obj):
if not obj:
return obj
if not isinstance(obj, BaseObject):
return obj
obj.backend = backend.name
if fields is None or len(fields) > 0:
obj = backend.fillobj(obj, fields) or obj
return obj
def _do_complete_iter(self, backend, count, fields, res):
modif = 0
for i, sub in enumerate(res):
sub = self._do_complete_obj(backend, fields, sub)
if self.condition and self.condition.limit and \
self.condition.limit == i:
return
if self.condition and not self.condition.is_valid(sub):
modif += 1
else:
if count and i - modif == count:
if self._is_default_count:
raise MoreResultsAvailable()
else:
return
yield sub
def _do_complete(self, backend, count, selected_fields, function, *args, **kwargs):
assert count is None or count > 0
if callable(function):
res = function(backend, *args, **kwargs)
else:
res = getattr(backend, function)(*args, **kwargs)
if hasattr(res, '__iter__') and not isinstance(res, (bytes, unicode)):
return self._do_complete_iter(backend, count, selected_fields, res)
else:
return self._do_complete_obj(backend, selected_fields, res)
def bcall_error_handler(self, backend, error, backtrace):
"""
Handler for an exception inside the CallErrors exception.
This method can be overridden to support more exceptions types.
"""
# Ignore this error.
if isinstance(error, MoreResultsAvailable):
return False
print(u'Error(%s): %s' % (backend.name, error), file=self.stderr)
if logging.root.level <= logging.DEBUG:
print(backtrace, file=self.stderr)
else:
return True
def bcall_errors_handler(self, errors, debugmsg='Use --debug option to print backtraces', ignore=()):
"""
Handler for the CallErrors exception.
It calls `bcall_error_handler` for each error.
:param errors: Object containing errors from backends
:type errors: :class:`weboob.core.bcall.CallErrors`
:param debugmsg: Default message asking to enable the debug mode
:type debugmsg: :class:`basestring`
:param ignore: Exceptions to ignore
:type ignore: tuple[:class:`Exception`]
"""
err = 0
ask_debug_mode = False
for backend, error, backtrace in errors.errors:
if isinstance(error, ignore):
continue
elif self.bcall_error_handler(backend, error, backtrace):
ask_debug_mode = True
if not isinstance(error, MoreResultsAvailable):
err = 1
if ask_debug_mode:
print(debugmsg, file=self.stderr)
return err
def _shell_completion_items(self):
items = set()
for ol in [self._parser.option_list] + [og.option_list for og in self._parser.option_groups]:
for option in ol:
if option.help is not optparse.SUPPRESS_HELP:
items.update(str(option).split('/'))
items.update(self._get_completions())
return items
def parse_args(self, args):
self.options, args = self._parser.parse_args(args)
if self.options.shell_completion:
items = self._shell_completion_items()
print(' '.join(items))
sys.exit(0)
if self.options.debug >= self.DEBUG_FILTER:
level = DEBUG_FILTERS
elif self.options.debug or self.options.save_responses:
level = logging.DEBUG
elif self.options.verbose:
level = logging.INFO
elif self.options.quiet:
level = logging.ERROR
else:
level = logging.WARNING
if self.options.insecure:
log_settings['ssl_insecure'] = True
if self.options.nss:
self.setup_nss()
# this only matters to developers
if not self.options.debug and not self.options.save_responses:
warnings.simplefilter('ignore', category=ConversionWarning)
warnings.simplefilter('ignore', category=FormFieldConversionWarning)
handlers = []
if self.options.save_responses:
import tempfile
responses_dirname = tempfile.mkdtemp(prefix='weboob_session_')
print('Debug data will be saved in this directory: %s' % responses_dirname, file=self.stderr)
log_settings['responses_dirname'] = responses_dirname
handlers.append(self.create_logging_file_handler(os.path.join(responses_dirname, 'debug.log')))
if self.options.export_session:
log_settings['export_session'] = True
# file logger
if self.options.logging_file:
handlers.append(self.create_logging_file_handler(self.options.logging_file))
else:
handlers.append(self.create_default_logger())
self.setup_logging(level, handlers)
self._handle_options()
self.handle_application_options()
return args
@classmethod
def create_default_logger(cls):
# stderr logger
format = '%(asctime)s:%(levelname)s:%(name)s:' + cls.VERSION +\
':%(filename)s:%(lineno)d:%(funcName)s %(message)s'
handler = logging.StreamHandler(cls.stderr)
handler.setFormatter(createColoredFormatter(cls.stderr, format))
return handler
@classmethod
def setup_logging(cls, level, handlers):
logging.root.handlers = []
logging.root.setLevel(level)
for handler in handlers:
logging.root.addHandler(handler)
def setup_nss(self):
from weboob.browser.nss import (
init_nss, inject_in_urllib3, create_cert_db, certificate_db_filename,
)
path = self.CONFDIR
if not os.path.exists(os.path.join(path, certificate_db_filename())):
create_cert_db(path)
init_nss(path)
inject_in_urllib3()
def create_logging_file_handler(self, filename):
try:
stream = open(os.path.expanduser(filename), 'w')
except IOError as e:
self.logger.error('Unable to create the logging file: %s' % e)
sys.exit(1)
else:
format = '%(asctime)s:%(levelname)s:%(name)s:' + self.VERSION +\
':%(filename)s:%(lineno)d:%(funcName)s %(message)s'
handler = logging.StreamHandler(stream)
handler.setFormatter(logging.Formatter(format))
return handler
@classmethod
def run(cls, args=None):
"""
This static method can be called to run the application.
It creates the application object, handles options, setups logging, calls
the main() method, and catches common exceptions.
You can't do anything after this call, as it *always* finishes with
a call to sys.exit().
For example:
>>> from weboob.application.myapplication import MyApplication
>>> MyApplication.run()
"""
cls.setup_logging(logging.INFO, [cls.create_default_logger()])
if sys.version_info.major == 2:
encoding = sys.stdout.encoding
if encoding is None:
encoding = guess_encoding(sys.stdout)
cls.stdout = sys.stdout = codecs.getwriter(encoding)(sys.stdout)
# can't do the same with stdin, codecs.getreader buffers too much to be usable in a REPL
if args is None:
args = [(cls.stdin.encoding and isinstance(arg, bytes) and arg.decode(cls.stdin.encoding) or to_unicode(arg)) for arg in sys.argv]
try:
app = cls()
except BackendsConfig.WrongPermissions as e:
print(e, file=cls.stderr)
sys.exit(1)
try:
try:
args = app.parse_args(args)
sys.exit(app.main(args))
except KeyboardInterrupt:
print('Program killed by SIGINT', file=cls.stderr)
sys.exit(0)
except EOFError:
sys.exit(0)
except ConfigError as e:
print('Configuration error: %s' % e, file=cls.stderr)
sys.exit(1)
except CallErrors as e:
try:
ret = app.bcall_errors_handler(e)
except KeyboardInterrupt:
pass
else:
sys.exit(ret)
sys.exit(1)
except ResultsConditionError as e:
print('%s' % e, file=cls.stderr)
sys.exit(1)
finally:
app.deinit()
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/captcha.py 0000664 0000000 0000000 00000002663 13414137563 0030323 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
from __future__ import print_function
from threading import Lock, Event
from weboob.capabilities.captcha import CapCaptchaSolver
from weboob.core.ouiboube import Weboob
__all__ = ['CaptchaMixin']
class CaptchaMixin(object):
def __init__(self, *args, **kwargs):
super(CaptchaMixin, self).__init__(*args, **kwargs)
self.captcha_weboob = Weboob()
self.captcha_weboob.load_backends(caps=[CapCaptchaSolver])
def solve_captcha(self, job, backend):
def call_solver(solver_backend, job):
with lock:
if solved.is_set():
solver_backend.logger.info('already solved, ignoring')
return
ret = solver_backend.solve_catpcha_blocking(job)
if ret:
solver_backend.logger.info('backend solved job')
backend.config['captcha_response'].set(ret.solution)
solved.set()
def all_solvers_finished():
if not solved.is_set():
print('Error(%s): CAPTCHA could not be solved.' % backend.name, file=self.stderr)
else:
print('Info(%s): CAPTCHA was successfully solved. Please retry operation.' % backend.name, file=self.stderr)
lock = Lock()
solved = Event()
bres = self.captcha_weboob.do(call_solver, job)
bres.callback_thread(None, None, all_solvers_finished)
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/console.py 0000664 0000000 0000000 00000064743 13414137563 0030371 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2012 Christophe Benz, Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import print_function
from collections import OrderedDict
from copy import copy
import getpass
import logging
import subprocess
from subprocess import check_output
import sys
import os
from weboob.capabilities import UserError
from weboob.capabilities.account import CapAccount, Account, AccountRegisterError
from weboob.core.backendscfg import BackendAlreadyExists
from weboob.core.repositories import IProgress
from weboob.exceptions import BrowserUnavailable, BrowserIncorrectPassword, BrowserForbidden, \
BrowserSSLError, BrowserQuestion, BrowserHTTPSDowngrade, \
ModuleInstallError, ModuleLoadError, NoAccountsException, \
ActionNeeded, CaptchaQuestion
from weboob.tools.value import Value, ValueBool, ValueFloat, ValueInt, ValueBackendPassword
from weboob.tools.misc import to_unicode
from weboob.tools.compat import unicode, long
from .base import Application, MoreResultsAvailable
__all__ = ['ConsoleApplication', 'BackendNotGiven']
class BackendNotGiven(Exception):
def __init__(self, id, backends):
self.id = id
self.backends = sorted(backends)
super(BackendNotGiven, self).__init__('Please specify a backend to use for this argument (%s@backend_name). '
'Availables: %s.' % (id, ', '.join(name for name, backend in backends)))
class BackendNotFound(Exception):
pass
class ConsoleProgress(IProgress):
def __init__(self, app):
self.app = app
def progress(self, percent, message):
try:
quiet = self.app.options.quiet
except AttributeError:
quiet = False
if not quiet:
self.app.stdout.write('=== [%3.0f%%] %s\n' % (percent*100, message))
def error(self, message):
self.app.stderr.write('ERROR: %s\n' % message)
def prompt(self, message):
return self.app.ask(message, default=True)
class ConsoleApplication(Application):
"""
Base application class for CLI applications.
"""
CAPS = None
# shell escape strings
if sys.platform == 'win32' \
or not sys.stdout.isatty() \
or os.getenv('ANSI_COLORS_DISABLED') is not None:
#workaround to disable bold
BOLD = ''
NC = '' # no color
else:
BOLD = '[1m'
NC = '[0m' # no color
def __init__(self, option_parser=None):
super(ConsoleApplication, self).__init__(option_parser)
self.weboob.requests.register('login', self.login_cb)
self.enabled_backends = set()
self._parser.add_option('--auto-update', action='store_true',
help='Automatically check for updates when a bug in a module is encountered')
def login_cb(self, backend_name, value):
return self.ask('[%s] %s' % (backend_name,
value.label),
masked=True,
default='',
regexp=value.regexp)
def unload_backends(self, *args, **kwargs):
unloaded = self.weboob.unload_backends(*args, **kwargs)
for backend in unloaded.values():
try:
self.enabled_backends.remove(backend)
except KeyError:
pass
return unloaded
def is_module_loadable(self, info):
return self.CAPS is None or info.has_caps(self.CAPS)
def load_backends(self, *args, **kwargs):
if 'errors' in kwargs:
errors = kwargs['errors']
else:
kwargs['errors'] = errors = []
ret = super(ConsoleApplication, self).load_backends(*args, **kwargs)
for err in errors:
print(u'Error(%s): %s' % (err.backend_name, err), file=self.stderr)
if self.ask('Do you want to reconfigure this backend?', default=True):
self.edit_backend(err.backend_name)
self.load_backends(names=[err.backend_name])
for name, backend in ret.items():
self.enabled_backends.add(backend)
self.check_loaded_backends()
return ret
def check_loaded_backends(self, default_config=None):
while len(self.enabled_backends) == 0:
print('Warning: there is currently no configured backend for %s' % self.APPNAME)
if not self.stdout.isatty() or not self.ask('Do you want to configure backends?', default=True):
return False
self.prompt_create_backends(default_config)
return True
def prompt_create_backends(self, default_config=None):
r = ''
while r != 'q':
modules = []
print('\nAvailable modules:')
for name, info in sorted(self.weboob.repositories.get_all_modules_info().items()):
if not self.is_module_loadable(info):
continue
modules.append(name)
loaded = ' '
for bi in self.weboob.iter_backends():
if bi.NAME == name:
if loaded == ' ':
loaded = 'X'
elif loaded == 'X':
loaded = 2
else:
loaded += 1
print('%s%d)%s [%s] %s%-15s%s %s' % (self.BOLD, len(modules), self.NC, loaded,
self.BOLD, name, self.NC,
info.description))
print('%sa) --all--%s install all backends' % (self.BOLD, self.NC))
print('%sq)%s --stop--\n' % (self.BOLD, self.NC))
r = self.ask('Select a backend to create (q to stop)', regexp='^(\d+|q|a)$')
if str(r).isdigit():
i = int(r) - 1
if i < 0 or i >= len(modules):
print('Error: %s is not a valid choice' % r, file=self.stderr)
continue
name = modules[i]
try:
inst = self.add_backend(name, name, default_config)
if inst:
self.load_backends(names=[inst])
except (KeyboardInterrupt, EOFError):
print('\nAborted.')
elif r == 'a':
try:
for name in modules:
if name in [b.NAME for b in self.weboob.iter_backends()]:
continue
inst = self.add_backend(name, name, default_config)
if inst:
self.load_backends(names=[inst])
except (KeyboardInterrupt, EOFError):
print('\nAborted.')
else:
break
print('Right right!')
def _handle_options(self):
self.load_default_backends()
def load_default_backends(self):
"""
By default loads all backends.
Applications can overload this method to restrict backends loaded.
"""
if len(self.STORAGE) > 0:
self.load_backends(self.CAPS, storage=self.create_storage())
else:
self.load_backends(self.CAPS)
@classmethod
def run(klass, args=None):
try:
super(ConsoleApplication, klass).run(args)
except BackendNotFound as e:
print('Error: Backend "%s" not found.' % e)
sys.exit(1)
def do(self, function, *args, **kwargs):
if 'backends' not in kwargs:
kwargs['backends'] = self.enabled_backends
return self.weboob.do(function, *args, **kwargs)
def parse_id(self, _id, unique_backend=False):
try:
_id, backend_name = _id.rsplit('@', 1)
except ValueError:
backend_name = None
backends = [(b.name, b) for b in self.enabled_backends]
if unique_backend and not backend_name:
if len(backends) == 1:
backend_name = backends[0][0]
else:
raise BackendNotGiven(_id, backends)
if backend_name is not None and backend_name not in dict(backends):
# Is the backend a short version of a real one?
found = False
for key in dict(backends):
if backend_name in key:
# two choices, ambiguous command
if found:
raise BackendNotFound(backend_name)
else:
found = True
_back = key
if found:
return _id, _back
raise BackendNotFound(backend_name)
return _id, backend_name
# user interaction related methods
def register_backend(self, name, ask_add=True):
try:
backend = self.weboob.modules_loader.get_or_load_module(name)
except ModuleLoadError:
backend = None
if not backend:
print('Backend "%s" does not exist.' % name, file=self.stderr)
return 1
if not backend.has_caps(CapAccount) or backend.klass.ACCOUNT_REGISTER_PROPERTIES is None:
print('You can\'t register a new account with %s' % name, file=self.stderr)
return 1
account = Account()
account.properties = {}
if backend.website:
website = 'on website %s' % backend.website
else:
website = 'with backend %s' % backend.name
while True:
asked_config = False
for key, prop in backend.klass.ACCOUNT_REGISTER_PROPERTIES.items():
if not asked_config:
asked_config = True
print('Configuration of new account %s' % website)
print('-----------------------------%s' % ('-' * len(website)))
p = copy(prop)
p.set(self.ask(prop, default=account.properties[key].get() if (key in account.properties) else prop.default))
account.properties[key] = p
if asked_config:
print('-----------------------------%s' % ('-' * len(website)))
try:
backend.klass.register_account(account)
except AccountRegisterError as e:
print(u'%s' % e)
if self.ask('Do you want to try again?', default=True):
continue
else:
return None
else:
break
backend_config = {}
for key, value in account.properties.items():
if key in backend.config:
backend_config[key] = value.get()
if ask_add and self.ask('Do you want to add the new register account?', default=True):
return self.add_backend(name, name, backend_config, ask_register=False)
return backend_config
def install_module(self, name):
try:
self.weboob.repositories.install(name, ConsoleProgress(self))
except ModuleInstallError as e:
print('Unable to install module "%s": %s' % (name, e), file=self.stderr)
return False
print('')
return True
def edit_backend(self, name, params=None):
return self.add_backend(name, name, params, True)
def add_backend(self, module_name, backend_name, params=None, edit=False, ask_register=True):
if params is None:
params = {}
module = None
config = None
try:
if not edit:
minfo = self.weboob.repositories.get_module_info(module_name)
if minfo is None:
raise ModuleLoadError(module_name, 'Module does not exist')
if not minfo.is_installed():
print('Module "%s" is available but not installed.' % minfo.name)
self.install_module(minfo)
module = self.weboob.modules_loader.get_or_load_module(module_name)
config = module.config
else:
module_name, items = self.weboob.backends_config.get_backend(backend_name)
module = self.weboob.modules_loader.get_or_load_module(module_name)
items.update(params)
params = items
config = module.config.load(self.weboob, module_name, backend_name, params, nofail=True)
except ModuleLoadError as e:
print('Unable to load module "%s": %s' % (module_name, e), file=self.stderr)
return 1
# ask for params non-specified on command-line arguments
asked_config = False
for key, value in config.items():
if not asked_config:
asked_config = True
print('')
print('Configuration of backend %s' % module.name)
print('-------------------------%s' % ('-' * len(module.name)))
if key not in params or edit:
params[key] = self.ask(value, default=params[key] if (key in params) else value.default)
else:
print(u'[%s] %s: %s' % (key, value.description, '(masked)' if value.masked else to_unicode(params[key])))
if asked_config:
print('-------------------------%s' % ('-' * len(module.name)))
i = 2
while not edit and self.weboob.backends_config.backend_exists(backend_name):
if not self.ask('Backend "%s" already exists. Add a new one for module %s?' % (backend_name, module.name), default=False):
return 1
backend_name = backend_name.rstrip('0123456789')
while self.weboob.backends_config.backend_exists('%s%s' % (backend_name, i)):
i += 1
backend_name = self.ask('Please give new instance name', default='%s%s' % (backend_name, i), regexp=r'^[\w\-_]+$')
try:
config = config.load(self.weboob, module.name, backend_name, params, nofail=True)
for key, value in params.items():
if key not in config:
continue
config[key].set(value)
config.save(edit=edit)
print('Backend "%s" successfully %s.' % (backend_name, 'edited' if edit else 'added'))
return backend_name
except BackendAlreadyExists:
print('Backend "%s" already exists.' % backend_name, file=self.stderr)
return 1
def ask(self, question, default=None, masked=None, regexp=None, choices=None, tiny=None):
"""
Ask a question to user.
@param question text displayed (str)
@param default optional default value (str)
@param masked if True, do not show typed text (bool)
@param regexp text must match this regexp (str)
@param choices choices to do (list)
@param tiny ask for the (small) value of the choice (bool)
@return entered text by user (str)
"""
if isinstance(question, Value):
v = copy(question)
if default is not None:
v.default = to_unicode(default) if isinstance(default, str) else default
if masked is not None:
v.masked = masked
if regexp is not None:
v.regexp = regexp
if choices is not None:
v.choices = choices
if tiny is not None:
v.tiny = tiny
else:
if isinstance(default, bool):
klass = ValueBool
elif isinstance(default, float):
klass = ValueFloat
elif isinstance(default, (int,long)):
klass = ValueInt
else:
klass = Value
v = klass(label=question, default=default, masked=masked, regexp=regexp, choices=choices, tiny=tiny)
question = v.label
if v.description and v.description != v.label:
question = u'%s: %s' % (question, v.description)
if v.id:
question = u'[%s] %s' % (v.id, question)
if isinstance(v, ValueBackendPassword):
print(question + ':')
question = v.label
choices = OrderedDict()
choices['c'] = 'Run an external tool during backend load'
if not v.noprompt:
choices['p'] = 'Prompt value when needed (do not store it)'
choices['s'] = 'Store value in config'
if v.is_command(v.default):
d = 'c'
elif v.default == '' and not v.noprompt:
d = 'p'
else:
d = 's'
r = self.ask('*** How do you want to store it?', choices=choices, tiny=True, default=d)
if r == 'p':
return ''
if r == 'c':
print('Enter the shell command that will print the required value on the standard output')
if v.is_command(v.default):
print(': %s' % v.default[1:-1])
else:
d = None
while True:
cmd = self.ask('')
try:
check_output(cmd, shell=True)
except subprocess.CalledProcessError as e:
print('%s' % e)
else:
return '`%s`' % cmd
aliases = {}
if isinstance(v, ValueBool):
question = u'%s (%s/%s)' % (question, 'Y' if v.default else 'y', 'n' if v.default else 'N')
elif v.choices:
if v.tiny is None:
v.tiny = True
for key in v.choices:
if len(key) > 5 or ' ' in key:
v.tiny = False
break
if v.tiny:
question = u'%s (%s)' % (question, '/'.join((s.upper() if s == v.default else s)
for s in v.choices))
for s in v.choices:
if s == v.default:
aliases[s.upper()] = s
for key, value in v.choices.items():
print(' %s%s%s: %s' % (self.BOLD, key, self.NC, value))
else:
for n, (key, value) in enumerate(v.choices.items()):
print(' %s%2d)%s %s' % (self.BOLD, n + 1, self.NC, value))
aliases[str(n + 1)] = key
question = u'%s (choose in list)' % question
if v.masked:
question = u'%s (hidden input)' % question
if not isinstance(v, ValueBool) and not v.tiny and v.default not in (None, ''):
question = u'%s [%s]' % (question, '*******' if v.masked else v.default)
question += ': '
while True:
if v.masked:
if sys.version_info.major < 3 and isinstance(question, unicode):
question = question.encode(self.encoding)
line = getpass.getpass(question)
if sys.platform != 'win32':
if isinstance(line, bytes): # only for python2
line = line.decode(self.encoding)
else:
self.stdout.write(question)
self.stdout.flush()
line = self._readline()
if len(line) == 0:
raise EOFError()
else:
line = line.rstrip('\r\n')
if not line and v.default is not None:
line = v.default
if line in aliases:
line = aliases[line]
try:
v.set(line)
except ValueError as e:
print(u'Error: %s' % e, file=self.stderr)
else:
break
v.noprompt = True
return v.get()
def print(self, txt):
print(txt)
def _readall(self):
if sys.version_info.major == 2:
return self.stdin.read().decode(self.encoding)
else:
return self.stdin.read()
def _readline(self):
if sys.version_info.major == 2:
return self.stdin.readline().decode(self.encoding)
else:
return self.stdin.readline()
def acquire_input(self, content=None, editor_params=None):
editor = os.getenv('EDITOR', 'vi')
if self.stdin.isatty() and editor:
from tempfile import NamedTemporaryFile
with NamedTemporaryFile() as f:
filename = f.name
if content is not None:
if isinstance(content, unicode):
content = content
f.write(content)
f.flush()
try:
params = editor_params[os.path.basename(editor)]
except (KeyError,TypeError):
params = ''
os.system("%s %s %s" % (editor, params, filename))
f.seek(0)
text = f.read()
else:
if self.stdin.isatty():
print('Reading content from stdin... Type ctrl-D '
'from an empty line to stop.')
text = self._readall()
return to_unicode(text)
def bcall_error_handler(self, backend, error, backtrace):
"""
Handler for an exception inside the CallErrors exception.
This method can be overridden to support more exceptions types.
"""
if isinstance(error, BrowserQuestion):
for field in error.fields:
v = self.ask(field)
if v:
backend.config[field.id].set(v)
elif isinstance(error, CaptchaQuestion):
print(u'Warning(%s): Captcha has been found on login page' % backend.name, file=self.stderr)
elif isinstance(error, BrowserIncorrectPassword):
msg = unicode(error)
if not msg:
msg = 'invalid login/password.'
print('Error(%s): %s' % (backend.name, msg), file=self.stderr)
if self.ask('Do you want to reconfigure this backend?', default=True):
self.unload_backends(names=[backend.name])
self.edit_backend(backend.name)
self.load_backends(names=[backend.name])
elif isinstance(error, BrowserSSLError):
print(u'FATAL(%s): ' % backend.name + self.BOLD + '/!\ SERVER CERTIFICATE IS INVALID /!\\' + self.NC, file=self.stderr)
elif isinstance(error, BrowserHTTPSDowngrade):
print(u'FATAL(%s): ' % backend.name + 'Downgrade from HTTPS to HTTP')
elif isinstance(error, BrowserForbidden):
msg = unicode(error)
print(u'Error(%s): %s' % (backend.name, msg or 'Forbidden'), file=self.stderr)
elif isinstance(error, BrowserUnavailable):
msg = unicode(error)
print(u'Error(%s): %s' % (backend.name, msg or 'Website is unavailable.'), file=self.stderr)
elif isinstance(error, ActionNeeded):
msg = unicode(error)
print(u'Error(%s): Action needed on website: %s' % (backend.name, msg), file=self.stderr)
elif isinstance(error, NotImplementedError):
print(u'Error(%s): this feature is not supported yet by this backend.' % backend.name, file=self.stderr)
print(u' %s To help the maintainer of this backend implement this feature,' % (' ' * len(backend.name)), file=self.stderr)
print(u' %s please contact us on the project mailing list' % (' ' * len(backend.name)), file=self.stderr)
elif isinstance(error, UserError):
print(u'Error(%s): %s' % (backend.name, to_unicode(error)), file=self.stderr)
elif isinstance(error, MoreResultsAvailable):
print(u'Hint: There are more results for backend %s' % (backend.name), file=self.stderr)
elif isinstance(error, NoAccountsException):
print(u'Error(%s): %s' % (backend.name, to_unicode(error) or 'No account on this backend'), file=self.stderr)
else:
print(u'Bug(%s): %s' % (backend.name, to_unicode(error)), file=self.stderr)
minfo = self.weboob.repositories.get_module_info(backend.NAME)
if minfo and not minfo.is_local():
if self.options.auto_update:
self.weboob.repositories.update_repositories(ConsoleProgress(self))
# minfo of the new available module
minfo = self.weboob.repositories.get_module_info(backend.NAME)
if minfo and minfo.version > self.weboob.repositories.versions.get(minfo.name) and \
self.ask('A new version of %s is available. Do you want to install it?' % minfo.name, default=True) and \
self.install_module(minfo):
print('New version of module %s has been installed. Retry to call the command.' % minfo.name)
return
else:
print('(If --auto-update is passed on the command-line, new versions of the module will be checked automatically)')
if logging.root.level <= logging.DEBUG:
print(backtrace, file=self.stderr)
else:
return True
def bcall_errors_handler(self, errors, debugmsg='Use --debug option to print backtraces', ignore=()):
"""
Handler for the CallErrors exception.
"""
ask_debug_mode = False
more_results = set()
err = 0
for backend, error, backtrace in errors.errors:
if isinstance(error, MoreResultsAvailable):
more_results.add(backend.name)
elif isinstance(error, ignore):
continue
else:
err = 1
if self.bcall_error_handler(backend, error, backtrace):
ask_debug_mode = True
if ask_debug_mode:
print(debugmsg, file=self.stderr)
elif len(more_results) > 0:
print('Hint: There are more results available for %s (use option -n or count command)' % (', '.join(more_results)), file=self.stderr)
return err
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/formatters/ 0000775 0000000 0000000 00000000000 13414137563 0030525 5 ustar 00root root 0000000 0000000 __init__.py 0000664 0000000 0000000 00000000000 13414137563 0032545 0 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/formatters csv.py 0000664 0000000 0000000 00000003554 13414137563 0031622 0 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/formatters # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import absolute_import
from codecs import open
import csv
import sys
from weboob.tools.compat import basestring
from weboob.tools.misc import to_unicode
from .iformatter import IFormatter
__all__ = ['CSVFormatter']
class CSVFormatter(IFormatter):
def __init__(self, field_separator=";"):
super(CSVFormatter, self).__init__()
self.started = False
self.field_separator = field_separator
def flush(self):
self.started = False
def format_dict(self, item):
if not isinstance(self.outfile, basestring):
return self.write_dict(item, self.outfile)
with open(self.outfile, "a+", encoding='utf-8') as fp:
return self.write_dict(item, fp)
def write_dict(self, item, fp):
writer = csv.writer(fp, delimiter=self.field_separator)
if not self.started:
writer.writerow([to_unicode(v) for v in item.keys()])
self.started = True
if sys.version_info.major >= 3:
writer.writerow([str(v) for v in item.values()])
else:
writer.writerow([to_unicode(v).encode('utf-8') for v in item.values()])
iformatter.py 0000664 0000000 0000000 00000023237 13414137563 0033203 0 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/formatters # -*- coding: utf-8 -*-
# Copyright(C) 2010-2013 Christophe Benz, Julien Hebert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import print_function
from codecs import open
from collections import OrderedDict
import os
import sys
import subprocess
try:
from termcolor import colored
except ImportError:
def colored(s, color=None, on_color=None, attrs=None):
if os.getenv('ANSI_COLORS_DISABLED') is None \
and attrs is not None and 'bold' in attrs:
return '%s%s%s' % (IFormatter.BOLD, s, IFormatter.NC)
else:
return s
try:
import tty
import termios
except ImportError:
PROMPT = '--Press return to continue--'
def readch():
return sys.stdin.readline()
else:
PROMPT = '--Press a key to continue--'
def readch():
fd = sys.stdin.fileno()
old_settings = termios.tcgetattr(fd)
tty.setcbreak(fd)
try:
c = sys.stdin.read(1)
# XXX do not read magic number
if c == '\x03':
raise KeyboardInterrupt()
return c
finally:
termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
from weboob.capabilities.base import BaseObject
from weboob.tools.application.console import ConsoleApplication
from weboob.tools.compat import basestring
from weboob.tools.misc import guess_encoding
__all__ = ['IFormatter', 'MandatoryFieldsNotFound']
class MandatoryFieldsNotFound(Exception):
def __init__(self, missing_fields):
super(MandatoryFieldsNotFound, self).__init__(u'Mandatory fields not found: %s.' % ', '.join(missing_fields))
class IFormatter(object):
# Tuple of fields mandatory to not crash
MANDATORY_FIELDS = None
# Tuple of displayed field. Set to None if all available fields are
# displayed
DISPLAYED_FIELDS = None
BOLD = ConsoleApplication.BOLD
NC = ConsoleApplication.NC
def colored(self, string, color, attrs=None, on_color=None):
if self.outfile != sys.stdout or not self.outfile.isatty():
return string
if isinstance(attrs, basestring):
attrs = [attrs]
return colored(string, color, on_color=on_color, attrs=attrs)
def __init__(self, display_keys=True, display_header=True, outfile=None):
self.display_keys = display_keys
self.display_header = display_header
self.interactive = False
self.print_lines = 0
self.termrows = 0
self.termcols = None
if outfile is None:
outfile = sys.stdout
self.outfile = outfile
# XXX if stdin is not a tty, it seems that the command fails.
if sys.stdout.isatty() and sys.stdin.isatty():
if sys.platform == 'win32':
from ctypes import windll, create_string_buffer
h = windll.kernel32.GetStdHandle(-12)
csbi = create_string_buffer(22)
res = windll.kernel32.GetConsoleScreenBufferInfo(h, csbi)
if res:
import struct
(bufx, bufy, curx, cury, wattr,
left, top, right, bottom, maxx, maxy) = struct.unpack("hhhhHhhhhhh", csbi.raw)
self.termrows = right - left + 1
self.termcols = bottom - top + 1
else:
self.termrows = 80 # can't determine actual size - return default values
self.termcols = 80
else:
self.termrows = int(
subprocess.Popen('stty size', shell=True, stdout=subprocess.PIPE).communicate()[0].split()[0]
)
self.termcols = int(
subprocess.Popen('stty size', shell=True, stdout=subprocess.PIPE).communicate()[0].split()[1]
)
def output(self, formatted):
if self.outfile != sys.stdout:
encoding = guess_encoding(sys.stdout)
with open(self.outfile, "a+", encoding=encoding, errors='replace') as outfile:
outfile.write(formatted + os.linesep)
else:
for line in formatted.split('\n'):
if self.termrows and (self.print_lines + 1) >= self.termrows:
self.outfile.write(PROMPT)
self.outfile.flush()
readch()
self.outfile.write('\b \b' * len(PROMPT))
self.print_lines = 0
plen = len(line.replace(self.BOLD, '').replace(self.NC, ''))
print(line)
if self.termcols:
self.print_lines += int(plen/self.termcols) + 1
else:
self.print_lines += 1
def start_format(self, **kwargs):
pass
def flush(self):
pass
def format(self, obj, selected_fields=None, alias=None):
"""
Format an object to be human-readable.
An object has fields which can be selected.
:param obj: object to format
:type obj: BaseObject or dict
:param selected_fields: fields to display. If None, all fields are selected
:type selected_fields: tuple
:param alias: an alias to use instead of the object's ID
:type alias: unicode
"""
if isinstance(obj, BaseObject):
if selected_fields: # can be an empty list (nothing to do), or None (return all fields)
obj = obj.copy()
for name, value in list(obj.iter_fields()):
if name not in selected_fields:
delattr(obj, name)
if self.MANDATORY_FIELDS:
missing_fields = set(self.MANDATORY_FIELDS) - set([name for name, value in obj.iter_fields()])
if missing_fields:
raise MandatoryFieldsNotFound(missing_fields)
formatted = self.format_obj(obj, alias)
else:
try:
obj = OrderedDict(obj)
except ValueError:
raise TypeError('Please give a BaseObject or a dict')
if selected_fields:
obj = obj.copy()
for name, value in obj.items():
if name not in selected_fields:
obj.pop(name)
if self.MANDATORY_FIELDS:
missing_fields = set(self.MANDATORY_FIELDS) - set(obj)
if missing_fields:
raise MandatoryFieldsNotFound(missing_fields)
formatted = self.format_dict(obj)
if formatted:
self.output(formatted)
return formatted
def format_obj(self, obj, alias=None):
"""
Format an object to be human-readable.
Called by format().
This method has to be overridden in child classes.
:param obj: object to format
:type obj: BaseObject
:rtype: str
"""
return self.format_dict(obj.to_dict())
def format_dict(self, obj):
"""
Format a dict to be human-readable.
:param obj: dict to format
:type obj: dict
:rtype: str
"""
raise NotImplementedError()
def format_collection(self, collection, only):
"""
Format a collection to be human-readable.
:param collection: collection to format
:type collection: BaseCollection
:rtype: str
"""
if only is False or collection.basename in only:
if collection.basename and collection.title:
self.output(u'%s~ (%s) %s (%s)%s' %
(self.BOLD, collection.basename, collection.title, collection.backend, self.NC))
else:
self.output(u'%s~ (%s) (%s)%s' %
(self.BOLD, collection.basename, collection.backend, self.NC))
class PrettyFormatter(IFormatter):
def format_obj(self, obj, alias):
title = self.get_title(obj)
desc = self.get_description(obj)
if alias is not None:
result = u'%s %s %s (%s)' % (self.colored('%2s' % alias, 'red', 'bold'),
self.colored(u'—', 'cyan', 'bold'),
self.colored(title, 'yellow', 'bold'),
self.colored(obj.backend, 'blue', 'bold'))
else:
result = u'%s %s %s' % (self.colored(obj.fullid, 'red', 'bold'),
self.colored(u'—', 'cyan', 'bold'),
self.colored(title, 'yellow', 'bold'))
if desc is not None:
result += u'%s\t%s' % (os.linesep, self.colored(desc, 'white'))
return result
def get_title(self, obj):
raise NotImplementedError()
def get_description(self, obj):
return None
def formatter_test_output(Formatter, obj):
"""
Formats an object and returns output as a string.
For test purposes only.
"""
from tempfile import mkstemp
from os import remove
_, name = mkstemp()
fmt = Formatter()
fmt.outfile = name
fmt.format(obj)
fmt.flush()
with open(name) as f:
res = f.read()
remove(name)
return res
json.py 0000664 0000000 0000000 00000003425 13414137563 0031775 0 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/formatters # -*- coding: utf-8 -*-
# Copyright(C) 2013-2014 Julien Hebert, Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.json import json, WeboobEncoder
from .iformatter import IFormatter
__all__ = ['JsonFormatter', 'JsonLineFormatter']
class JsonFormatter(IFormatter):
"""
Formats the whole list as a single JSON list object.
"""
def __init__(self):
super(JsonFormatter, self).__init__()
self.queue = []
def flush(self):
self.output(json.dumps(self.queue, cls=WeboobEncoder))
def format_dict(self, item):
self.queue.append(item)
def format_collection(self, collection, only):
self.queue.append(collection.to_dict())
class JsonLineFormatter(IFormatter):
"""
Formats the list as received, with a JSON object per line.
The advantage is that it can be streamed.
"""
def format_dict(self, item):
self.output(json.dumps(item, cls=WeboobEncoder))
def test():
from .iformatter import formatter_test_output as fmt
assert fmt(JsonFormatter, {'foo': 'bar'}) == '[{"foo": "bar"}]\n'
assert fmt(JsonLineFormatter, {'foo': 'bar'}) == '{"foo": "bar"}\n'
load.py 0000664 0000000 0000000 00000005267 13414137563 0031751 0 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/formatters # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz, Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
__all__ = ['FormattersLoader', 'FormatterLoadError']
class FormatterLoadError(Exception):
pass
class FormattersLoader(object):
BUILTINS = ['htmltable', 'multiline', 'simple', 'table', 'csv', 'webkit', 'json', 'json_line']
def __init__(self):
self.formatters = {}
def register_formatter(self, name, klass):
self.formatters[name] = klass
def get_available_formatters(self):
l = set(self.formatters)
l = l.union(self.BUILTINS)
l = sorted(l)
return l
def build_formatter(self, name):
if name not in self.formatters:
try:
self.formatters[name] = self.load_builtin_formatter(name)
except ImportError as e:
FormattersLoader.BUILTINS.remove(name)
raise FormatterLoadError('Unable to load formatter "%s": %s' % (name, e))
return self.formatters[name]()
def load_builtin_formatter(self, name):
if name not in self.BUILTINS:
raise FormatterLoadError('Formatter "%s" does not exist' % name)
if name == 'htmltable':
from .table import HTMLTableFormatter
return HTMLTableFormatter
elif name == 'table':
from .table import TableFormatter
return TableFormatter
elif name == 'simple':
from .simple import SimpleFormatter
return SimpleFormatter
elif name == 'multiline':
from .multiline import MultilineFormatter
return MultilineFormatter
elif name == 'webkit':
from .webkit import WebkitGtkFormatter
return WebkitGtkFormatter
elif name == 'csv':
from .csv import CSVFormatter
return CSVFormatter
elif name == 'json':
from .json import JsonFormatter
return JsonFormatter
elif name == 'json_line':
from .json import JsonLineFormatter
return JsonLineFormatter
multiline.py 0000664 0000000 0000000 00000002723 13414137563 0033026 0 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/formatters # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.base import NotLoaded, NotAvailable
from .iformatter import IFormatter
__all__ = ['MultilineFormatter']
class MultilineFormatter(IFormatter):
def __init__(self, key_value_separator=u': ', after_item=u'\n'):
super(MultilineFormatter, self).__init__()
self.key_value_separator = key_value_separator
self.after_item = after_item
def flush(self):
pass
def format_dict(self, item):
result = u'\n'.join(u'%s%s' % (
(u'%s%s' % (k, self.key_value_separator) if self.display_keys else ''), v)
for k, v in item.items() if (v is not NotLoaded and v is not NotAvailable))
if len(item) > 1:
result += self.after_item
return result
simple.py 0000664 0000000 0000000 00000002375 13414137563 0032320 0 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/formatters # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .iformatter import IFormatter
__all__ = ['SimpleFormatter']
class SimpleFormatter(IFormatter):
def __init__(self, field_separator=u'\t', key_value_separator=u'='):
super(SimpleFormatter, self).__init__()
self.field_separator = field_separator
self.key_value_separator = key_value_separator
def format_dict(self, item):
return self.field_separator.join(u'%s%s' % (
(u'%s%s' % (k, self.key_value_separator) if self.display_keys else ''), v)
for k, v in item.items())
table.py 0000664 0000000 0000000 00000007257 13414137563 0032122 0 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/formatters # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from prettytable import PrettyTable
from weboob.capabilities.base import empty
from weboob.tools.compat import range
from .iformatter import IFormatter
__all__ = ['TableFormatter', 'HTMLTableFormatter']
class TableFormatter(IFormatter):
HTML = False
def __init__(self):
super(TableFormatter, self).__init__()
self.queue = []
self.keys = None
self.header = None
def flush(self):
s = self.get_formatted_table()
if s is not None:
self.output(s)
def get_formatted_table(self):
if len(self.queue) == 0:
return
if self.interactive:
# Insert indices at the beginning of each line
self.keys.insert(0, '#')
for i in range(len(self.queue)):
self.queue[i].insert(0, i+1)
queue = [() for i in range(len(self.queue))]
column_headers = []
# Do not display columns when all values are NotLoaded or NotAvailable
maxrow = 0
for i in range(len(self.keys)):
available = False
for line in self.queue:
if len(line)> i and not empty(line[i]):
maxrow += 1
available = True
break
if available:
column_headers.append(self.keys[i].capitalize().replace('_', ' '))
for j in range(len(self.queue)):
if len(self.queue[j]) > i:
queue[j] += (self.queue[j][i],)
s = ''
if self.display_header and self.header:
if self.HTML:
s += '
%s
' % self.header
else:
s += self.header
s += "\n"
table = PrettyTable(list(column_headers))
for column_header in column_headers:
# API changed in python-prettytable. The try/except is a bad hack to support both versions
# Note: two versions are not exactly the same...
# (first one: header in center. Second one: left align for header too)
try:
table.set_field_align(column_header, 'l')
except:
table.align[column_header] = 'l'
for line in queue:
for _ in range(maxrow - len(line)):
line += ('',)
table.add_row(line)
if self.HTML:
s += table.get_html_string()
else:
s += table.get_string()
self.queue = []
return s
def format_dict(self, item):
if self.keys is None:
self.keys = list(item.keys())
self.queue.append(list(item.values()))
def set_header(self, string):
self.header = string
class HTMLTableFormatter(TableFormatter):
HTML = True
def test():
from .iformatter import formatter_test_output as fmt
assert fmt(TableFormatter, {'foo': 'bar'}) == \
'+-----+\n' \
'| Foo |\n' \
'+-----+\n' \
'| bar |\n' \
'+-----+\n'
webkit/ 0000775 0000000 0000000 00000000000 13414137563 0031733 5 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/formatters __init__.py 0000664 0000000 0000000 00000001456 13414137563 0034052 0 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/formatters/webkit # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from .webkitgtk import WebkitGtkFormatter
__all__ = ['WebkitGtkFormatter']
webkitgtk.py 0000664 0000000 0000000 00000005144 13414137563 0034304 0 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/formatters/webkit # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import os
import gtk
import webkit
from weboob.tools.application.javascript import get_javascript
from ..table import HTMLTableFormatter
__all__ = ['WebkitGtkFormatter']
class WebBrowser(gtk.Window):
def __init__(self):
super(WebBrowser, self).__init__()
self.connect('destroy', gtk.main_quit)
self.set_default_size(800, 600)
self.web_view = webkit.WebView()
sw = gtk.ScrolledWindow()
sw.add(self.web_view)
self.add(sw)
self.show_all()
class WebkitGtkFormatter(HTMLTableFormatter):
def flush(self):
table_string = self.get_formatted_table()
js_filepaths = []
js_filepaths.append(get_javascript('jquery'))
js_filepaths.append(get_javascript('tablesorter'))
scripts = ['' % js_filepath for js_filepath in js_filepaths]
html_string_params = dict(table=table_string)
if scripts:
html_string_params['scripts'] = ''.join(scripts)
html_string = """
%(scripts)s
%(table)s
""" % html_string_params
web_browser = WebBrowser()
web_browser.web_view.load_html_string(html_string, 'file://%s' % os.path.abspath(os.getcwd()))
gtk.main()
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/javascript.py 0000664 0000000 0000000 00000003444 13414137563 0031064 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import os
__all__ = ['get_javascript']
def get_javascript(name, load_order=('local', 'web'), minified=True):
if name == 'jquery':
for src in load_order:
if src == 'local':
# try Debian paths
if minified:
filepath = '/usr/share/javascript/jquery/jquery.min.js'
else:
filepath = '/usr/share/javascript/jquery/jquery.js'
if os.path.exists(filepath):
return filepath
elif src == 'web':
# return Google-hosted URLs
if minified:
return 'http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js'
else:
return 'http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.js'
elif name == 'tablesorter':
if 'web' in load_order:
if minified:
return 'http://tablesorter.com/jquery.tablesorter.min.js'
else:
return 'http://tablesorter.com/jquery.tablesorter.js'
return None
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/media_player.py 0000664 0000000 0000000 00000015363 13414137563 0031354 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz, Romain Bignon, John Obbele
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import print_function
from contextlib import closing
import os
from subprocess import PIPE, Popen
import requests
from weboob.tools.log import getLogger
__all__ = ['InvalidMediaPlayer', 'MediaPlayer', 'MediaPlayerNotFound']
PLAYERS = (
('mpv', '-'),
('mplayer2', '-'),
('mplayer', '-'),
('vlc', '-'),
('parole', 'fd://0'),
('totem', 'fd://0'),
('xine', 'stdin:/'),
)
class MediaPlayerNotFound(Exception):
def __init__(self):
super(MediaPlayerNotFound, self).__init__(u'No media player found on this system. Please install one of them: %s.' %
', '.join(player[0] for player in PLAYERS))
class InvalidMediaPlayer(Exception):
def __init__(self, player_name):
super(InvalidMediaPlayer, self).__init__(u'Invalid media player: %s. Valid media players: %s.' % (
player_name, ', '.join(player[0] for player in PLAYERS)))
class MediaPlayer(object):
"""
Black magic invoking a media player to this world.
Presently, due to strong disturbances in the holidays of the ether
world, the media player used is chosen from a static list of
programs. See PLAYERS for more information.
"""
def __init__(self, logger=None):
self.logger = getLogger('mediaplayer', logger)
def guess_player_name(self):
for player_name in [player[0] for player in PLAYERS]:
if self._find_in_path(os.environ['PATH'], player_name):
return player_name
return None
def play(self, media, player_name=None, player_args=None):
"""
Play a media object, using programs from the PLAYERS list.
This function dispatch calls to either _play_default or
_play_rtmp for special rtmp streams using SWF verification.
"""
player_names = [player[0] for player in PLAYERS]
if not player_name:
self.logger.debug(u'No media player given. Using the first available from: %s.' %
', '.join(player_names))
player_name = self.guess_player_name()
if player_name is None:
raise MediaPlayerNotFound()
if media.url.startswith('rtmp'):
self._play_rtmp(media, player_name, args=player_args)
else:
self._play_default(media, player_name, args=player_args)
def _play_default(self, media, player_name, args=None):
"""
Play media.url with the media player.
"""
# if flag play_proxy...
if hasattr(media, '_play_proxy') and media._play_proxy is True:
# use requests to handle redirect and cookies
self._play_proxy(media, player_name, args)
return None
args = player_name.split(' ')
player_name = args[0]
args.append(media.url)
print('Invoking "%s".' % (' '.join(args)))
os.spawnlp(os.P_WAIT, player_name, *args)
def _play_proxy(self, media, player_name, args):
"""
Load data with python requests and pipe data to a media player.
We need this function for url that use redirection and cookies.
This function is used if the non-standard,
non-API compliant '_play_proxy' attribute of the 'media' object is defined and is True.
"""
if args is None:
for (binary, stdin_args) in PLAYERS:
if binary == player_name:
args = stdin_args
assert args is not None
print(':: Play_proxy streaming from %s' % media.url)
print(':: to %s %s' % (player_name, args))
print(player_name + ' ' + args)
proc = Popen(player_name + ' ' + args, stdin=PIPE, shell=True)
# Handle cookies (and redirection 302...)
session = requests.sessions.Session()
with closing(proc.stdin):
with closing(session.get(media.url, stream=True)) as response:
for buffer in response.iter_content(8192):
try:
proc.stdin.write(buffer)
except:
print("play_proxy broken pipe. Can't write anymore.")
break
def _play_rtmp(self, media, player_name, args):
"""
Download data with rtmpdump and pipe them to a media player.
You need a working version of rtmpdump installed and the SWF
object url in order to comply with SWF verification requests
from the server. The last one is retrieved from the non-standard
non-API compliant 'swf_player' attribute of the 'media' object.
"""
if not self._find_in_path(os.environ['PATH'], 'rtmpdump'):
self.logger.warning('"rtmpdump" binary not found')
return self._play_default(media, player_name)
media_url = media.url
try:
player_url = media.swf_player
if media.swf_player:
rtmp = 'rtmpdump -r %s --swfVfy %s' % (media_url, player_url)
else:
rtmp = 'rtmpdump -r %s' % media_url
except AttributeError:
self.logger.warning('Your media object does not have a "swf_player" attribute. SWF verification will be '
'disabled and may prevent correct media playback.')
return self._play_default(media, player_name)
rtmp += ' --quiet'
if args is None:
for (binary, stdin_args) in PLAYERS:
if binary == player_name:
args = stdin_args
assert args is not None
player_name = player_name.split(' ')
args = args.split(' ')
print(':: Streaming from %s' % media_url)
print(':: to %s %s' % (player_name, args))
print(':: %s' % rtmp)
p1 = Popen(rtmp.split(), stdout=PIPE)
Popen(player_name + args, stdin=p1.stdout, stderr=PIPE)
@classmethod
def _find_in_path(cls, path, filename):
for i in path.split(':'):
if os.path.exists('/'.join([i, filename])):
return True
return False
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/qt5/ 0000775 0000000 0000000 00000000000 13414137563 0027050 5 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/qt5/Makefile 0000664 0000000 0000000 00000000264 13414137563 0030512 0 ustar 00root root 0000000 0000000 UI_FILES = $(wildcard *.ui)
UI_PY_FILES = $(UI_FILES:%.ui=%_ui.py)
PYUIC = pyuic5
all: $(UI_PY_FILES)
%_ui.py: %.ui
$(PYUIC) -o $@ $^
clean:
rm -f *.pyc
rm -f $(UI_PY_FILES)
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/qt5/__init__.py 0000664 0000000 0000000 00000000301 13414137563 0031153 0 ustar 00root root 0000000 0000000 from .qt import QtApplication, QtMainWindow, QtDo, HTMLDelegate
from .backendcfg import BackendCfg
__all__ = ['QtApplication', 'QtMainWindow', 'QtDo', 'HTMLDelegate',
'BackendCfg']
backendcfg.py 0000664 0000000 0000000 00000050404 13414137563 0031415 0 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/qt5 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2012 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from PyQt5.QtWidgets import QDialog, QTreeWidgetItem, QLabel, QFormLayout, \
QMessageBox, QHeaderView, \
QListWidgetItem, QVBoxLayout, \
QDialogButtonBox, QProgressDialog
from PyQt5.QtGui import QTextDocument, QPixmap, QImage, QIcon
from PyQt5.QtCore import Qt, QVariant, QUrl, QThread
from PyQt5.QtCore import pyqtSignal as Signal, pyqtSlot as Slot
from collections import OrderedDict
import re
import os
from logging import warning
from weboob.core.repositories import IProgress
from weboob.core.backendscfg import BackendAlreadyExists
from weboob.capabilities.account import CapAccount, Account, AccountRegisterError
from weboob.exceptions import ModuleInstallError, ModuleLoadError
from .backendcfg_ui import Ui_BackendCfg
from .reposdlg_ui import Ui_RepositoriesDlg
from weboob.tools.misc import to_unicode
from weboob.tools.compat import unicode
from .qt import QtValue
class RepositoriesDialog(QDialog):
def __init__(self, filename, parent=None):
super(RepositoriesDialog, self).__init__(parent)
self.filename = filename
self.ui = Ui_RepositoriesDlg()
self.ui.setupUi(self)
self.ui.buttonBox.accepted.connect(self.save)
with open(self.filename, 'r') as fp:
self.ui.reposEdit.setPlainText(fp.read())
@Slot()
def save(self):
with open(self.filename, 'w') as fp:
fp.write(self.ui.reposEdit.toPlainText())
self.accept()
class IconFetcher(QThread):
retrieved = Signal()
def __init__(self, weboob, item, minfo):
super(IconFetcher, self).__init__()
self.weboob = weboob
self.items = [item]
self.minfo = minfo
def run(self):
self.weboob.repositories.retrieve_icon(self.minfo)
self.retrieved.emit()
class ProgressDialog(IProgress, QProgressDialog):
def __init__(self, *args, **kwargs):
super(ProgressDialog, self).__init__(*args, **kwargs)
def progress(self, percent, message):
self.setValue(int(percent * 100))
self.setLabelText(message)
def error(self, message):
QMessageBox.critical(self, self.tr('Error'), '%s' % message, QMessageBox.Ok)
def prompt(self, message):
reply = QMessageBox.question(self, '', unicode(message), QMessageBox.Yes|QMessageBox.No)
return reply == QMessageBox.Yes
class BackendCfg(QDialog):
def __init__(self, weboob, caps=None, parent=None):
super(BackendCfg, self).__init__(parent)
self.ui = Ui_BackendCfg()
self.ui.setupUi(self)
self.ui.backendsList.sortByColumn(0, Qt.AscendingOrder)
self.to_unload = set()
self.to_load = set()
self.weboob = weboob
self.caps = caps
self.config_widgets = {}
# This attribute is set when itemChanged it called, because when
# a backend is enabled/disabled, we don't want to display its config
# frame, and the itemClicked event is always emit just after a
# itemChanged event.
# is_enabling is a counter to prevent race conditions.
self.is_enabling = 0
self.ui.backendsList.header().setSectionResizeMode(QHeaderView.ResizeToContents)
self.ui.configFrame.hide()
self.icon_cache = {}
self.icon_threads = {}
self.loadModules()
self.loadBackendsList()
self.ui.updateButton.clicked.connect(self.updateModules)
self.ui.repositoriesButton.clicked.connect(self.editRepositories)
self.ui.backendsList.itemClicked.connect(self.backendClicked)
self.ui.backendsList.itemChanged.connect(self.backendEnabled)
self.ui.modulesList.itemSelectionChanged.connect(self.moduleSelectionChanged)
self.ui.proxyBox.toggled.connect(self.proxyEditEnabled)
self.ui.addButton.clicked.connect(self.addEvent)
self.ui.removeButton.clicked.connect(self.removeEvent)
self.ui.registerButton.clicked.connect(self.registerEvent)
self.ui.configButtonBox.accepted.connect(self.acceptBackend)
self.ui.configButtonBox.rejected.connect(self.rejectBackend)
def get_icon_cache(self, path):
if path not in self.icon_cache:
img = QImage(path)
self.icon_cache[path] = QIcon(QPixmap.fromImage(img))
return self.icon_cache[path]
def set_icon(self, item, minfo):
icon_path = self.weboob.repositories.get_module_icon_path(minfo)
icon = self.icon_cache.get(icon_path, None)
if icon is None and not os.path.exists(icon_path):
if minfo.name in self.icon_threads:
self.icon_threads[minfo.name].items.append(item)
else:
thread = IconFetcher(self.weboob, item, minfo)
thread.retrieved.connect(self._set_icon_slot)
self.icon_threads[minfo.name] = thread
thread.start()
return
self._set_icon([item], minfo)
@Slot()
def _set_icon_slot(self):
thread = self.sender()
self._set_icon(thread.items, thread.minfo)
def _set_icon(self, items, minfo):
icon_path = self.weboob.repositories.get_module_icon_path(minfo)
icon = self.get_icon_cache(icon_path)
if icon is None:
return
for item in items:
try:
item.setIcon(icon)
except TypeError:
item.setIcon(0, icon)
self.icon_threads.pop(minfo.name, None)
@Slot()
def updateModules(self):
self.ui.configFrame.hide()
pd = ProgressDialog('Update of modules', "Cancel", 0, 100, self)
pd.setWindowModality(Qt.WindowModal)
try:
self.weboob.repositories.update(pd)
except ModuleInstallError as err:
QMessageBox.critical(self, self.tr('Update error'),
self.tr('Unable to update modules: %s' % (err)),
QMessageBox.Ok)
pd.setValue(100)
self.loadModules()
QMessageBox.information(self, self.tr('Update of modules'),
self.tr('Modules updated!'), QMessageBox.Ok)
@Slot()
def editRepositories(self):
if RepositoriesDialog(self.weboob.repositories.sources_list).exec_():
self.updateModules()
def loadModules(self):
self.ui.modulesList.clear()
for name, module in sorted(self.weboob.repositories.get_all_modules_info(self.caps).items()):
item = QListWidgetItem(name.capitalize())
self.set_icon(item, module)
self.ui.modulesList.addItem(item)
def askInstallModule(self, minfo):
reply = QMessageBox.question(self, self.tr('Install a module'),
self.tr("Module %s is not installed. Do you want to install it?") % minfo.name,
QMessageBox.Yes|QMessageBox.No)
if reply != QMessageBox.Yes:
return False
return self.installModule(minfo)
def installModule(self, minfo):
pd = ProgressDialog('Installation of %s' % minfo.name, "Cancel", 0, 100, self)
pd.setWindowModality(Qt.WindowModal)
try:
self.weboob.repositories.install(minfo, pd)
except ModuleInstallError as err:
QMessageBox.critical(self, self.tr('Install error'),
self.tr('Unable to install module %s: %s' % (minfo.name, err)),
QMessageBox.Ok)
pd.setValue(100)
return True
def loadBackendsList(self):
self.ui.backendsList.clear()
for backend_name, module_name, params in self.weboob.backends_config.iter_backends():
info = self.weboob.repositories.get_module_info(module_name)
if not info or (self.caps and not info.has_caps(self.caps)):
continue
item = QTreeWidgetItem(None, [backend_name, module_name])
item.setCheckState(0, Qt.Checked if params.get('_enabled', '1').lower() in ('1', 'y', 'true', 'on', 'yes')
else Qt.Unchecked)
self.set_icon(item, info)
self.ui.backendsList.addTopLevelItem(item)
@Slot(QTreeWidgetItem, int)
def backendEnabled(self, item, col):
self.is_enabling += 1
backend_name = item.text(0)
module_name = item.text(1)
if item.checkState(0) == Qt.Checked:
self.to_load.add(backend_name)
enabled = 'true'
else:
self.to_unload.add(backend_name)
try:
self.to_load.remove(backend_name)
except KeyError:
pass
enabled = 'false'
self.weboob.backends_config.edit_backend(backend_name, module_name, {'_enabled': enabled})
@Slot(QTreeWidgetItem, int)
def backendClicked(self, item, col):
if self.is_enabling:
self.is_enabling -= 1
return
backend_name = item.text(0)
self.editBackend(backend_name)
@Slot()
def addEvent(self):
self.editBackend()
@Slot()
def removeEvent(self):
item = self.ui.backendsList.currentItem()
if not item:
return
backend_name = item.text(0)
reply = QMessageBox.question(self, self.tr('Remove a backend'),
self.tr("Are you sure you want to remove the backend '%s'?") % backend_name,
QMessageBox.Yes|QMessageBox.No)
if reply != QMessageBox.Yes:
return
self.weboob.backends_config.remove_backend(backend_name)
self.to_unload.add(backend_name)
try:
self.to_load.remove(backend_name)
except KeyError:
pass
self.ui.configFrame.hide()
self.loadBackendsList()
def editBackend(self, backend_name=None):
self.ui.registerButton.hide()
self.ui.configFrame.show()
if backend_name is not None:
module_name, params = self.weboob.backends_config.get_backend(backend_name)
items = self.ui.modulesList.findItems(module_name, Qt.MatchFixedString)
if not items:
warning('Backend not found')
else:
self.ui.modulesList.setCurrentItem(items[0])
self.ui.modulesList.setEnabled(False)
self.ui.nameEdit.setText(backend_name)
self.ui.nameEdit.setEnabled(False)
if '_proxy' in params:
self.ui.proxyBox.setChecked(True)
self.ui.proxyEdit.setText(params.pop('_proxy'))
else:
self.ui.proxyBox.setChecked(False)
self.ui.proxyEdit.clear()
params.pop('_enabled', None)
info = self.weboob.repositories.get_module_info(module_name)
if info and (info.is_installed() or self.installModule(info)):
module = self.weboob.modules_loader.get_or_load_module(module_name)
for key, value in module.config.load(self.weboob, module_name, backend_name, params, nofail=True).items():
try:
l, widget = self.config_widgets[key]
except KeyError:
warning('Key "%s" is not found' % key)
else:
# Do not prompt user for value (for example a password if it is empty).
value.noprompt = True
widget.set_value(value)
return
self.ui.nameEdit.clear()
self.ui.nameEdit.setEnabled(True)
self.ui.proxyBox.setChecked(False)
self.ui.proxyEdit.clear()
self.ui.modulesList.setEnabled(True)
self.ui.modulesList.setCurrentRow(-1)
@Slot()
def moduleSelectionChanged(self):
for key, (label, value) in self.config_widgets.items():
label.hide()
value.hide()
self.ui.configLayout.removeWidget(label)
self.ui.configLayout.removeWidget(value)
label.deleteLater()
value.deleteLater()
self.config_widgets = {}
self.ui.moduleInfo.clear()
selection = self.ui.modulesList.selectedItems()
if not selection:
return
minfo = self.weboob.repositories.get_module_info(selection[0].text().lower())
if not minfo:
warning('Module not found')
return
if not minfo.is_installed() and not self.installModule(minfo):
self.editBackend(None)
return
module = self.weboob.modules_loader.get_or_load_module(minfo.name)
icon_path = os.path.join(self.weboob.repositories.icons_dir, '%s.png' % minfo.name)
img = QImage(icon_path)
self.ui.moduleInfo.document().addResource(QTextDocument.ImageResource, QUrl('mydata://logo.png'),
QVariant(img))
if module.name not in [n for n, ign, ign2 in self.weboob.backends_config.iter_backends()]:
self.ui.nameEdit.setText(module.name)
else:
self.ui.nameEdit.setText('')
self.ui.moduleInfo.setText(to_unicode(self.tr(
u'%s Module %s
'
'Version: %s
'
'Maintainer: %s
'
'License: %s
'
'%s'
'Description: %s
'
'Capabilities: %s
'))
% ('',
module.name.capitalize(),
module.version,
to_unicode(module.maintainer).replace(u'&', u'&').replace(u'<', u'<').replace(u'>', u'>'),
module.license,
(self.tr('Website: %s
') % module.website) if module.website else '',
module.description,
', '.join(sorted(cap.__name__.replace('Cap', '') for cap in module.iter_caps()))))
if module.has_caps(CapAccount) and self.ui.nameEdit.isEnabled() and \
module.klass.ACCOUNT_REGISTER_PROPERTIES is not None:
self.ui.registerButton.show()
else:
self.ui.registerButton.hide()
for key, field in module.config.items():
label = QLabel(u'%s:' % field.label)
qvalue = QtValue(field)
self.ui.configLayout.addRow(label, qvalue)
self.config_widgets[key] = (label, qvalue)
@Slot(bool)
def proxyEditEnabled(self, state):
self.ui.proxyEdit.setEnabled(state)
@Slot()
def acceptBackend(self):
backend_name = self.ui.nameEdit.text()
selection = self.ui.modulesList.selectedItems()
if not selection:
QMessageBox.critical(self, self.tr('Unable to add a backend'),
self.tr('Please select a module'))
return
try:
module = self.weboob.modules_loader.get_or_load_module(selection[0].text().lower())
except ModuleLoadError:
module = None
if not module:
QMessageBox.critical(self, self.tr('Unable to add a backend'),
self.tr('The selected module does not exist.'))
return
params = {}
if not backend_name:
QMessageBox.critical(self, self.tr('Missing field'), self.tr('Please specify a backend name'))
return
if self.ui.nameEdit.isEnabled():
if not re.match(r'^[\w\-_]+$', backend_name):
QMessageBox.critical(self, self.tr('Invalid value'),
self.tr('The backend name can only contain letters and digits'))
return
if self.weboob.backends_config.backend_exists(backend_name):
QMessageBox.critical(self, self.tr('Unable to create backend'),
self.tr('Unable to create backend "%s": it already exists') % backend_name)
return
if self.ui.proxyBox.isChecked():
params['_proxy'] = self.ui.proxyEdit.text()
if not params['_proxy']:
QMessageBox.critical(self, self.tr('Missing field'), self.tr('Please specify a proxy URL'))
return
config = module.config.load(self.weboob, module.name, backend_name, {}, nofail=True)
for key, field in config.items():
label, qtvalue = self.config_widgets[key]
try:
value = qtvalue.get_value()
except ValueError as e:
QMessageBox.critical(self, self.tr('Invalid value'),
self.tr('Invalid value for field "%s":
%s') % (field.label, e))
return
field.set(value.get())
try:
config.save(edit=not self.ui.nameEdit.isEnabled(), params=params)
except BackendAlreadyExists:
QMessageBox.critical(self, self.tr('Unable to create backend'),
self.tr('Unable to create backend "%s": it already exists') % backend_name)
return
self.to_load.add(backend_name)
self.ui.configFrame.hide()
self.loadBackendsList()
@Slot()
def rejectBackend(self):
self.ui.configFrame.hide()
@Slot()
def registerEvent(self):
selection = self.ui.modulesList.selectedItems()
if not selection:
return
try:
module = self.weboob.modules_loader.get_or_load_module(selection[0].text().lower())
except ModuleLoadError:
module = None
if not module:
return
dialog = QDialog(self)
vbox = QVBoxLayout(dialog)
if module.website:
website = 'on the website %s' % module.website
else:
website = 'with the module %s' % module.name
vbox.addWidget(QLabel('To create an account %s, please provide this information:' % website))
formlayout = QFormLayout()
props_widgets = OrderedDict()
for key, prop in module.klass.ACCOUNT_REGISTER_PROPERTIES.items():
widget = QtValue(prop)
formlayout.addRow(QLabel(u'%s:' % prop.label), widget)
props_widgets[prop.id] = widget
vbox.addLayout(formlayout)
buttonBox = QDialogButtonBox(dialog)
buttonBox.setStandardButtons(QDialogButtonBox.Ok|QDialogButtonBox.Cancel)
buttonBox.accepted.connect(dialog.accept)
buttonBox.rejected.connect(dialog.reject)
vbox.addWidget(buttonBox)
end = False
while not end:
end = True
if dialog.exec_():
account = Account()
account.properties = {}
for key, widget in props_widgets.items():
try:
v = widget.get_value()
except ValueError as e:
QMessageBox.critical(self, self.tr('Invalid value'),
self.tr('Invalid value for field "%s":
%s') % (key, e))
end = False
break
else:
account.properties[key] = v
if end:
try:
module.klass.register_account(account)
except AccountRegisterError as e:
QMessageBox.critical(self, self.tr('Error during register'),
self.tr('Unable to register account %s:
%s') % (website, e))
end = False
else:
for key, value in account.properties.items():
if key in self.config_widgets:
self.config_widgets[key][1].set_value(value)
def run(self):
self.exec_()
ret = (len(self.to_load) > 0 or len(self.to_unload) > 0)
self.weboob.unload_backends(self.to_unload)
self.weboob.load_backends(names=self.to_load)
return ret
backendcfg.ui 0000664 0000000 0000000 00000022764 13414137563 0031412 0 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/qt5
BackendCfg
0
0
622
516
Backends configuration
6
4
-
QFrame::StyledPanel
QFrame::Raised
4
-
Update modules
-
Repositories
-
Qt::Horizontal
40
20
-
Qt::Vertical
-
QAbstractItemView::NoEditTriggers
24
24
false
false
false
true
true
false
false
false
true
Name
Module
-
QFrame::StyledPanel
QFrame::Raised
-
Add
-
Remove
-
Qt::Vertical
20
40
QFrame::StyledPanel
QFrame::Raised
-
-
Available modules:
-
0
0
24
24
1
true
-
1
0
QFrame::NoFrame
QFrame::Plain
-
0
0
true
-
QFormLayout::ExpandingFieldsGrow
-
-
Proxy:
-
false
-
Register an account...
-
Name:
-
Qt::Horizontal
QDialogButtonBox::Cancel|QDialogButtonBox::Ok
-
QDialogButtonBox::Close
backendsList
addButton
removeButton
modulesList
moduleInfo
nameEdit
proxyBox
proxyEdit
registerButton
configButtonBox
buttonBox
buttonBox
clicked(QAbstractButton*)
BackendCfg
accept()
312
591
312
306
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/qt5/models.py 0000664 0000000 0000000 00000040076 13414137563 0030714 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2016 weboob project
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import print_function
from collections import deque
from weakref import WeakKeyDictionary
from PyQt5.QtCore import Qt, QObject, QAbstractItemModel, QModelIndex, \
QSortFilterProxyModel, QVariant, pyqtSignal as Signal,\
pyqtSlot as Slot
from PyQt5.QtGui import QIcon, QImage, QPixmap, QPixmapCache, \
QStandardItemModel, QStandardItem
from PyQt5.QtWidgets import QApplication
from weboob.capabilities.base import NotAvailable, NotLoaded
from weboob.capabilities.collection import BaseCollection
from weboob.capabilities.file import BaseFile
from weboob.capabilities.gallery import BaseGallery, BaseImage as GBaseImage
from weboob.capabilities.gauge import Gauge, GaugeSensor
from weboob.tools.compat import basestring
# TODO expand other cap objects when needed
from .qt import QtDo
from .thumbnails import try_get_thumbnail, store_thumbnail
__all__ = ['BackendListModel', 'ResultModel', 'FilterTypeModel']
class BackendListModel(QStandardItemModel):
"""Model for displaying a backends list with icons"""
RoleBackendName = Qt.UserRole
RoleCapability = Qt.UserRole + 1
def __init__(self, weboob, *args, **kwargs):
super(BackendListModel, self).__init__(*args, **kwargs)
self.weboob = weboob
def addBackends(self, cap=None, entry_all=True, entry_title=False):
"""
Populate the model by adding backends.
Appends backends to the model, without clearing previous entries.
For each entry in the model, the cap name is stored under role
RoleBackendName and the capability object under role
RoleCapability.
:param cap: capabilities to add (None to add all loaded caps)
:param entry_all: if True, add a "All backends" entry
:param entry_title: if True, add a disabled entry with the cap name
"""
if entry_title:
if cap:
capname = cap.__name__
else:
capname = '(All capabilities)'
item = QStandardItem(capname)
item.setEnabled(False)
self.appendRow(item)
first = True
for backend in self.weboob.iter_backends(caps=cap):
if first and entry_all:
item = QStandardItem('(All backends)')
item.setData('', self.RoleBackendName)
item.setData(cap, self.RoleCapability)
self.appendRow(item)
first = False
item = QStandardItem(backend.name)
item.setData(backend.name, self.RoleBackendName)
item.setData(cap, self.RoleCapability)
minfo = self.weboob.repositories.get_module_info(backend.NAME)
icon_path = self.weboob.repositories.get_module_icon_path(minfo)
if icon_path:
pixmap = QPixmapCache.find(icon_path)
if not pixmap:
pixmap = QPixmap(QImage(icon_path))
item.setIcon(QIcon(pixmap))
self.appendRow(item)
class DoWrapper(QtDo):
"""Wrapper for QtDo to use in DoQueue."""
def __init__(self, *args, **kwargs):
super(DoWrapper, self).__init__(*args, **kwargs)
self.do_args = None
def do(self, *args, **kwargs):
self.do_args = (args, kwargs)
def start(self):
super(DoWrapper, self).do(*self.do_args[0], **self.do_args[1])
self.do_args = None
class DoQueue(QObject):
"""Queue to limit the number of parallel Do processes."""
def __init__(self):
super(DoQueue, self).__init__()
self.max_tasks = 10
self.running = set()
self.queue = deque()
def add(self, doer):
doer.finished.connect(self._finished)
if len(self.running) < self.max_tasks:
self.running.add(doer)
doer.start()
else:
self.queue.append(doer)
@Slot()
def _finished(self):
try:
self.running.remove(self.sender())
except KeyError:
return
try:
doer = self.queue.popleft()
except IndexError:
return
self.running.add(doer)
doer.start()
def stop(self):
doers = list(self.running) + list(self.queue)
self.running, self.queue = set(), deque()
for do in doers:
do.stop()
class Item(object):
def __init__(self, obj, parent):
self.obj = obj
self.parent = parent
self.children = None
class ResultModel(QAbstractItemModel):
"""Model for displaying objects and collections"""
RoleObject = Qt.UserRole
RoleCapability = Qt.UserRole + 1
RoleBackendName = Qt.UserRole + 2
jobAdded = Signal()
jobFinished = Signal()
def __init__(self, weboob, *args, **kwargs):
super(ResultModel, self).__init__(*args, **kwargs)
self.weboob = weboob
self.resource_classes = []
self.root = Item(None, None)
self.columns = []
self.limit = None
self.jobs = DoQueue()
self.jobExpanders = WeakKeyDictionary()
self.jobFillers = WeakKeyDictionary()
def __del__(self):
try:
self.jobs.stop()
except:
pass
# configuration/general operation
def setLimit(self, limit):
self.limit = limit
def clear(self):
"""Empty the model completely"""
self.jobs.stop()
#n = len(self.children.get(None, []))
self.beginResetModel()
#if n:
# self.beginRemoveRows(QModelIndex(), 0, max(0, n - 1))
self.root = Item(None, None)
self.endResetModel()
#if n:
# self.endRemoveRows()
@Slot(object)
def _gotRootDone(self, obj):
self._addToRoot(obj)
def addRootDo(self, *args, **kwargs):
"""Make a weboob.do and add returned items to root of model"""
process = DoWrapper(self.weboob, None)
process.gotResponse.connect(self._gotRootDone)
process.finished.connect(self.jobFinished)
process.do(*args, **kwargs)
self.jobAdded.emit()
self.jobs.add(process)
def addRootDoLimit(self, cls, *args, **kwargs):
app = QApplication.instance()
if cls is None:
fields = None
else:
fields = self._expandableFields(cls)
return self.addRootDo(app._do_complete, self.limit, fields, *args, **kwargs)
def addRootItems(self, objs):
for obj in objs:
self._addToRoot(obj)
def setColumnFields(self, columns):
self.columns = tuple(((c,) if isinstance(c, basestring) else c) for c in columns)
def setResourceClasses(self, classes):
"""Set accepted object classes for CapCollection.iter_resources"""
self.resource_classes = classes
def removeItem(self, qidx):
item = qidx.internalPointer()
assert item
# TODO recursive?
parent_qidx = qidx.parent()
parent_item = item.parent
assert parent_item is parent_qidx.internalPointer()
n = parent_item.children.index(item)
self.beginRemoveRows(parent_qidx, n, n)
del parent_item.children[n]
item.parent = None
self.endRemoveRows()
# internal operation
def _addToRoot(self, obj):
self._addItem(obj, self.root, QModelIndex())
def _addItem(self, obj, parent, parent_qidx):
item = Item(obj, parent)
if parent.children is None:
parent.children = []
children = parent.children
n = len(children)
self.beginInsertRows(parent_qidx, n, n)
children.append(item)
self.endInsertRows()
@Slot(object)
def _expanderGotResponse(self, obj):
parent_obj, parent_item = self.jobExpanders[self.sender()]
row = parent_item.parent.children.index(parent_item)
parent_qidx = self.createIndex(row, 0, parent_item)
self._addItem(obj, parent_item, parent_qidx)
def _prepareExpanderJob(self, obj, qidx):
item = qidx.internalPointer()
process = DoWrapper(self.weboob, None)
process.finished.connect(self.jobFinished)
process.gotResponse.connect(self._expanderGotResponse)
self.jobExpanders[process] = (obj, item)
return process
def _expandableFields(self, cls):
fields = set()
for col in self.columns:
for f in col:
if f == 'id' or f in cls._fields:
fields.add(f)
if 'thumbnail' in cls._fields:
fields.add('thumbnail')
return list(fields)
def expandGauge(self, gauge, qidx):
app = QApplication.instance()
fields = self._expandableFields(GaugeSensor)
process = self._prepareExpanderJob(gauge, qidx)
process.do(app._do_complete, self.limit, fields, 'iter_sensors', gauge.id, backends=[gauge.backend])
self.jobAdded.emit()
self.jobs.add(process)
def expandGallery(self, gall, qidx):
app = QApplication.instance()
fields = self._expandableFields(GBaseImage)
process = self._prepareExpanderJob(gall, qidx)
process.do(app._do_complete, self.limit, fields, 'iter_gallery_images', gall, backends=[gall.backend])
self.jobAdded.emit()
self.jobs.add(process)
def expandCollection(self, coll, qidx):
app = QApplication.instance()
if len(self.resource_classes) == 1:
fields = self._expandableFields(self.resource_classes[0])
else:
fields = None
# at this point, we don't know the class of each object
# FIXME reimplement _do_complete obj to filter dynamically
process = self._prepareExpanderJob(coll, qidx)
process.do(app._do_complete, self.limit, fields, 'iter_resources', self.resource_classes, coll.split_path, backends=[coll.backend])
self.jobAdded.emit()
self.jobs.add(process)
def expandObj(self, obj, qidx):
if isinstance(obj, BaseCollection):
self.expandCollection(obj, qidx)
elif isinstance(obj, BaseGallery):
self.expandGallery(obj, qidx)
elif isinstance(obj, Gauge):
self.expandGauge(obj, qidx)
def _getBackend(self, qidx):
while qidx.isValid():
item = qidx.internalPointer()
if item.obj.backend:
return item.obj.backend
qidx = qidx.parent()
def fillObj(self, obj, fields, qidx):
assert qidx.isValid()
item = qidx.internalPointer()
process = DoWrapper(self.weboob, None)
self.jobFillers[process] = item
process.gotResponse.connect(self._fillerGotResponse)
process.finished.connect(self.jobFinished)
process.do('fillobj', obj, fields, backends=qidx.data(self.RoleBackendName))
self.jobAdded.emit()
self.jobs.add(process)
@Slot(object)
def _fillerGotResponse(self, new_obj):
item = self.jobFillers[self.sender()]
if new_obj is not None:
item.obj = new_obj
row = item.parent.children.index(item)
qidx = self.createIndex(row, 0, item) # FIXME col 0 ?
self.dataChanged.emit(qidx, qidx)
# Qt model methods
def index(self, row, col, parent_qidx):
parent = parent_qidx.internalPointer() or self.root
children = parent.children or ()
if row >= len(children) or col >= len(self.columns):
return QModelIndex()
return self.createIndex(row, col, children[row])
def parent(self, qidx):
item = qidx.internalPointer() or self.root
parent = item.parent
if parent is None or parent.parent is None:
return QModelIndex()
gparent = parent.parent
row = gparent.children.index(parent)
return self.createIndex(row, 0, parent)
def flags(self, qidx):
obj = qidx.internalPointer()
if obj is None:
return Qt.NoItemFlags
else:
return Qt.ItemIsSelectable | Qt.ItemIsEnabled
def rowCount(self, qidx):
if qidx.column() != 0 and qidx.isValid():
return 0
item = qidx.internalPointer() or self.root
return len(item.children or ())
def columnCount(self, qidx):
if qidx.column() != 0 and qidx.isValid():
return 0
return len(self.columns)
def data(self, qidx, role):
item = qidx.internalPointer()
if item is None or item.obj is None:
return QVariant()
obj = item.obj
if role == self.RoleBackendName:
return QVariant(self._getBackend(qidx))
elif role == self.RoleObject:
return QVariant(obj)
elif role == Qt.DecorationRole:
return self._dataIcon(obj, qidx)
elif role == Qt.DisplayRole:
return self._dataText(obj, qidx)
return QVariant()
def headerData(self, section, orientation, role):
if role != Qt.DisplayRole:
return QVariant()
elif section >= len(self.columns):
return QVariant()
return '/'.join(self.columns[section])
def hasChildren(self, qidx):
item = qidx.internalPointer() or self.root
obj = item.obj
if isinstance(obj, BaseFile):
return False
# assume there are children, so a view may ask fetching
children = item.children or [True]
return bool(len(children))
def canFetchMore(self, qidx):
item = qidx.internalPointer()
if item is None:
return False
return item.children is None
def fetchMore(self, qidx):
if not self.canFetchMore(qidx):
return
item = qidx.internalPointer()
if item.children is None:
item.children = []
self.expandObj(item.obj, qidx)
# overridable
def _dataText(self, obj, qidx):
fields = self.columns[qidx.column()]
for field in fields:
if hasattr(obj, field):
data = getattr(obj, field)
if data:
return QVariant(data)
return QVariant()
def _dataIcon(self, obj, qidx):
if qidx.column() != 0:
return QVariant()
var = try_get_thumbnail(obj)
if var:
return var
try:
thumbnail = obj.thumbnail
except AttributeError:
return QVariant()
if thumbnail is NotLoaded:
self.fillObj(obj, ['thumbnail'], qidx)
elif thumbnail is NotAvailable:
return QVariant()
elif thumbnail.data is NotLoaded:
self.fillObj(thumbnail, ['data'], qidx)
elif thumbnail.data is NotAvailable:
return QVariant()
else:
img = QImage.fromData(thumbnail.data)
store_thumbnail(obj)
return QVariant(QIcon(QPixmap(img)))
return QVariant()
class FilterTypeModel(QSortFilterProxyModel):
def __init__(self, *args):
super(FilterTypeModel, self).__init__(*args)
self.types = []
def setAcceptedTypes(self, types):
self.types = types
def filterAcceptsRow(self, row, parent_qidx):
default = super(FilterTypeModel, self).filterAcceptsRow(row, parent_qidx)
if not default:
return False
mdl = self.sourceModel()
qidx = mdl.index(row, 0, parent_qidx)
obj = mdl.data(qidx, ResultModel.RoleObject).value()
actual = type(obj)
for accepted in self.types:
if issubclass(actual, accepted):
return True
return False
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/qt5/qt.py 0000664 0000000 0000000 00000033760 13414137563 0030057 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import print_function
import sys
import logging
import re
import gc
from threading import Event
from traceback import print_exc
from copy import copy
from PyQt5.QtCore import QTimer, QObject, QSize, QVariant, QMutex, Qt
from PyQt5.QtCore import pyqtSignal as Signal, pyqtSlot as Slot
from PyQt5.QtWidgets import QApplication, QCheckBox, QComboBox, QInputDialog, \
QLineEdit, QMainWindow, QMessageBox, QSpinBox, \
QStyle, QStyledItemDelegate, QStyleOptionViewItem
from PyQt5.QtGui import QTextDocument, QAbstractTextDocumentLayout, QPalette
from weboob.core.ouiboube import Weboob, VersionsMismatchError
from weboob.core.scheduler import IScheduler
from weboob.tools.compat import range, unicode
from weboob.tools.config.iconfig import ConfigError
from weboob.exceptions import BrowserUnavailable, BrowserIncorrectPassword, BrowserForbidden, ModuleInstallError
from weboob.tools.value import ValueInt, ValueBool, ValueBackendPassword
from weboob.tools.misc import to_unicode
from weboob.capabilities import UserError
from ..base import Application, MoreResultsAvailable
__all__ = ['QtApplication', 'QtMainWindow', 'QtDo', 'HTMLDelegate']
class QtScheduler(QObject, IScheduler):
def __init__(self, app):
super(QtScheduler, self).__init__(parent=app)
self.params = {}
def schedule(self, interval, function, *args):
timer = QTimer()
timer.setInterval(interval * 1000)
timer.setSingleShot(True)
self.params[timer] = (None, function, args)
timer.timeout.connect(self.timeout)
timer.start()
def repeat(self, interval, function, *args):
timer = QTimer()
timer.setSingleShot(False)
self.params[timer] = (interval, function, args)
timer.start(0)
timer.timeout.connect(self.timeout, Qt.QueuedConnection)
@Slot()
def timeout(self):
timer = self.sender()
interval, function, args = self.params[timer]
function(*args)
if interval is None:
self.timers.pop(timer)
else:
timer.setInterval(interval * 1000)
def want_stop(self):
QApplication.instance().quit()
def run(self):
return QApplication.instance().exec_()
class QCallbacksManager(QObject):
class Request(object):
def __init__(self):
self.event = Event()
self.answer = None
def __call__(self):
raise NotImplementedError()
class LoginRequest(Request):
def __init__(self, backend_name, value):
super(QCallbacksManager.LoginRequest, self).__init__()
self.backend_name = backend_name
self.value = value
def __call__(self):
password, ok = QInputDialog.getText(None,
'%s request' % self.value.label,
'Please enter %s for %s' % (self.value.label,
self.backend_name),
QLineEdit.Password)
return password
new_request = Signal()
def __init__(self, weboob, parent=None):
super(QCallbacksManager, self).__init__(parent)
self.weboob = weboob
self.weboob.requests.register('login', self.callback(self.LoginRequest))
self.mutex = QMutex()
self.requests = []
self.new_request.connect(self.do_request)
def callback(self, klass):
def cb(*args, **kwargs):
return self.add_request(klass(*args, **kwargs))
return cb
@Slot()
def do_request(self):
self.mutex.lock()
request = self.requests.pop()
request.answer = request()
request.event.set()
self.mutex.unlock()
def add_request(self, request):
self.mutex.lock()
self.requests.append(request)
self.mutex.unlock()
self.new_request.emit()
request.event.wait()
return request.answer
class QtApplication(QApplication, Application):
def __init__(self):
super(QtApplication, self).__init__(sys.argv)
self.setApplicationName(self.APPNAME)
self.cbmanager = QCallbacksManager(self.weboob, self)
def create_weboob(self):
return Weboob(scheduler=QtScheduler(self))
def load_backends(self, *args, **kwargs):
while True:
last_exc = None
try:
return Application.load_backends(self, *args, **kwargs)
except VersionsMismatchError as e:
msg = 'Versions of modules mismatch with version of weboob.'
last_exc = e
except ConfigError as e:
msg = unicode(e)
last_exc = e
res = QMessageBox.question(None, 'Configuration error', u'%s\n\nDo you want to update repositories?' % msg, QMessageBox.Yes|QMessageBox.No)
if res == QMessageBox.No:
raise last_exc
# Do not import it globally, it causes circular imports
from .backendcfg import ProgressDialog
pd = ProgressDialog('Update of repositories', "Cancel", 0, 100)
pd.setWindowModality(Qt.WindowModal)
try:
self.weboob.update(pd)
except ModuleInstallError as err:
QMessageBox.critical(None, self.tr('Update error'),
self.tr('Unable to update repositories: %s' % err),
QMessageBox.Ok)
pd.setValue(100)
QMessageBox.information(None, self.tr('Update of repositories'),
self.tr('Repositories updated!'), QMessageBox.Ok)
def deinit(self):
super(QtApplication, self).deinit()
gc.collect()
class QtMainWindow(QMainWindow):
pass
class QtDo(QObject):
gotResponse = Signal(object)
gotError = Signal(object, object, object)
finished = Signal()
def __init__(self, weboob, cb, eb=None, fb=None, retain=False):
super(QtDo, self).__init__()
if not eb:
eb = self.default_eb
self.weboob = weboob
self.process = None
self.cb = cb
self.eb = eb
self.fb = fb
self.gotResponse.connect(self.local_cb)
self.gotError.connect(self.local_eb)
self.finished.connect(self.local_fb)
if not retain:
QApplication.instance().aboutToQuit.connect(self.stop)
def __del__(self):
try:
self.stop()
except Exception:
print_exc()
def do(self, *args, **kwargs):
assert self.process is None
self.process = self.weboob.do(*args, **kwargs)
self.process.callback_thread(self.thread_cb, self.thread_eb, self.thread_fb)
@Slot()
def stop(self, wait=False):
if self.process is not None:
self.process.stop(wait)
@Slot(object, object, object)
def default_eb(self, backend, error, backtrace):
if isinstance(error, MoreResultsAvailable):
# This is not an error, ignore.
return
msg = unicode(error)
if isinstance(error, BrowserIncorrectPassword):
if not msg:
msg = 'Invalid login/password.'
elif isinstance(error, BrowserUnavailable):
if not msg:
msg = 'Website is unavailable.'
elif isinstance(error, BrowserForbidden):
if not msg:
msg = 'This action is forbidden.'
elif isinstance(error, NotImplementedError):
msg = u'This feature is not supported by this backend.\n\n' \
u'To help the maintainer of this backend implement this feature, please contact: %s <%s>' % (backend.MAINTAINER, backend.EMAIL)
elif isinstance(error, UserError):
if not msg:
msg = type(error).__name__
elif logging.root.level <= logging.DEBUG:
msg += u'
'
ul_opened = False
for line in backtrace.split('\n'):
m = re.match(' File (.*)', line)
if m:
if not ul_opened:
msg += u''
ul_opened = True
else:
msg += u''
msg += u'- %s' % m.group(1)
else:
msg += u'
%s' % to_unicode(line)
if ul_opened:
msg += u'
'
print(error, file=sys.stderr)
print(backtrace, file=sys.stderr)
QMessageBox.critical(None, self.tr('Error with backend %s') % backend.name,
msg, QMessageBox.Ok)
@Slot(object)
def local_cb(self, data):
if self.cb:
self.cb(data)
@Slot(object, object, object)
def local_eb(self, backend, error, backtrace):
if self.eb:
self.eb(backend, error, backtrace)
@Slot()
def local_fb(self):
if self.fb:
self.fb()
self.gotResponse.disconnect(self.local_cb)
self.gotError.disconnect(self.local_eb)
self.finished.disconnect(self.local_fb)
self.process = None
def thread_cb(self, data):
self.gotResponse.emit(data)
def thread_eb(self, backend, error, backtrace):
self.gotError.emit(backend, error, backtrace)
def thread_fb(self):
self.finished.emit()
class HTMLDelegate(QStyledItemDelegate):
def paint(self, painter, option, index):
option = QStyleOptionViewItem(option) # copy option
self.initStyleOption(option, index)
style = option.widget.style() if option.widget else QApplication.style()
doc = QTextDocument()
doc.setHtml(option.text)
# painting item without text
option.text = ""
style.drawControl(QStyle.CE_ItemViewItem, option, painter)
ctx = QAbstractTextDocumentLayout.PaintContext()
# Hilight text if item is selected
if option.state & QStyle.State_Selected:
ctx.palette.setColor(QPalette.Text, option.palette.color(QPalette.Active, QPalette.HighlightedText))
textRect = style.subElementRect(QStyle.SE_ItemViewItemText, option)
painter.save()
painter.translate(textRect.topLeft())
painter.setClipRect(textRect.translated(-textRect.topLeft()))
doc.documentLayout().draw(painter, ctx)
painter.restore()
def sizeHint(self, option, index):
self.initStyleOption(option, index)
doc = QTextDocument()
doc.setHtml(option.text)
doc.setTextWidth(option.rect.width())
return QSize(doc.idealWidth(), max(doc.size().height(), option.decorationSize.height()))
class _QtValueStr(QLineEdit):
def __init__(self, value):
super(_QtValueStr, self).__init__()
self._value = value
if value.default:
self.setText(unicode(value.default))
if value.masked:
self.setEchoMode(self.Password)
def set_value(self, value):
self._value = value
self.setText(self._value.get())
def get_value(self):
self._value.set(self.text())
return self._value
class _QtValueBackendPassword(_QtValueStr):
def get_value(self):
self._value._domain = None
return _QtValueStr.get_value(self)
class _QtValueBool(QCheckBox):
def __init__(self, value):
super(_QtValueBool, self).__init__()
self._value = value
if value.default:
self.setChecked(True)
def set_value(self, value):
self._value = value
self.setChecked(self._value.get())
def get_value(self):
self._value.set(self.isChecked())
return self._value
class _QtValueInt(QSpinBox):
def __init__(self, value):
super(_QtValueInt, self).__init__()
self._value = value
if value.default:
self.setValue(int(value.default))
def set_value(self, value):
self._value = value
self.setValue(self._value.get())
def get_value(self):
self._value.set(self.getValue())
return self._value
class _QtValueChoices(QComboBox):
def __init__(self, value):
super(_QtValueChoices, self).__init__()
self._value = value
for k, l in value.choices.items():
self.addItem(l, QVariant(k))
if value.default == k:
self.setCurrentIndex(self.count()-1)
def set_value(self, value):
self._value = value
for i in range(self.count()):
if self.itemData(i) == self._value.get():
self.setCurrentIndex(i)
return
def get_value(self):
self._value.set(self.itemData(self.currentIndex()))
return self._value
def QtValue(value):
if isinstance(value, ValueBool):
klass = _QtValueBool
elif isinstance(value, ValueInt):
klass = _QtValueInt
elif isinstance(value, ValueBackendPassword):
klass = _QtValueBackendPassword
elif value.choices is not None:
klass = _QtValueChoices
else:
klass = _QtValueStr
return klass(copy(value))
# if the default excepthook is used, PyQt 5.5 *aborts* the app when an unhandled exception occurs
# see http://pyqt.sourceforge.net/Docs/PyQt5/incompatibilities.html
# as this behaviour is questionable, we restore the old one
if sys.excepthook is sys.__excepthook__:
sys.excepthook = lambda *args: sys.__excepthook__(*args)
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/qt5/reposdlg.ui 0000664 0000000 0000000 00000002235 13414137563 0031230 0 ustar 00root root 0000000 0000000
RepositoriesDlg
0
0
400
300
Repositories
-
-
Qt::Horizontal
QDialogButtonBox::Cancel|QDialogButtonBox::Ok
buttonBox
rejected()
RepositoriesDlg
reject()
316
260
286
274
search_history.py 0000664 0000000 0000000 00000004172 13414137563 0032375 0 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/qt5 # -*- coding: utf-8 -*-
# Copyright(C) 2016 weboob project
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import codecs
import os
from PyQt5.QtWidgets import QCompleter
from PyQt5.QtCore import QStringListModel
__all__ = ['HistoryCompleter']
class HistoryCompleter(QCompleter):
def __init__(self, hist_path, *args, **kwargs):
super(HistoryCompleter, self).__init__(*args, **kwargs)
self.setModel(QStringListModel())
self.max_history = 50
self.search_history = []
self.hist_path = hist_path
def addString(self, s):
if not s:
return
if len(self.search_history) > self.max_history:
self.search_history.pop(0)
if s not in self.search_history:
self.search_history.append(s)
self.updateCompletion()
def updateCompletion(self):
self.model().setStringList(self.search_history)
def load(self):
""" Return search string history list loaded from history file """
self.search_history = []
if os.path.exists(self.hist_path):
with codecs.open(self.hist_path, 'r', 'utf-8') as f:
conf_hist = f.read().strip()
if conf_hist:
self.search_history = conf_hist.split('\n')
self.updateCompletion()
def save(self):
""" Save search history in history file. """
if len(self.search_history) > 0:
with codecs.open(self.hist_path, 'w', 'utf-8') as f:
f.write('\n'.join(self.search_history))
thumbnails.py 0000664 0000000 0000000 00000005324 13414137563 0031515 0 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/qt5 # -*- coding: utf-8 -*-
# Copyright(C) 2016 Vincent A
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from time import mktime
from PyQt5.QtCore import QVariant, Qt, QSize
from PyQt5.QtGui import QIcon, QPixmap, QImage
from weboob.capabilities.image import BaseImage
try:
import vignette # https://github.com/hydrargyrum/vignette
except ImportError:
vignette = None
__all__ = ('try_get_thumbnail', 'store_thumbnail')
def find_url(obj):
if isinstance(obj, BaseImage):
return obj.url
try:
obj.thumbnail
except AttributeError:
pass
else:
return obj.thumbnail.url
def try_get_thumbnail(obj):
if vignette is None:
return
url = find_url(obj)
if not url:
return
try:
ts = mktime(obj.date.timetuple())
except AttributeError:
return
path = vignette.try_get_thumbnail(url, mtime=ts)
if path:
return QVariant(QIcon(QPixmap(path)))
def ideal_thumb_size(size):
if size[0] <= 128 and size[1] <= 128:
return 'normal', None
if size[0] <= 256 and size[1] <= 256:
return 'large', None
return 'large', 256
def load_qimg(obj, *attrs):
try:
for attr in attrs:
obj = getattr(obj, attr)
except AttributeError:
return QImage()
if not obj:
return QImage()
return QImage.fromData(obj)
def store_thumbnail(obj):
if vignette is None:
return
url = find_url(obj)
if not url:
return
try:
ts = mktime(obj.date.timetuple())
except AttributeError:
return
path = vignette.try_get_thumbnail(url, mtime=ts)
if path:
# thumbnail already exists
return
qimg = load_qimg(obj, 'data')
if qimg.isNull():
qimg = load_qimg(obj, 'thumbnail', 'data')
if qimg.isNull():
return
format, size = ideal_thumb_size((qimg.width(), qimg.height()))
if size:
qimg = qimg.scaled(QSize(size, size), Qt.KeepAspectRatio, Qt.SmoothTransformation)
path = vignette.create_temp(format)
qimg.save(path)
vignette.put_thumbnail(url, format, path, mtime=ts)
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/repl.py 0000664 0000000 0000000 00000137416 13414137563 0027667 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2012 Christophe Benz, Romain Bignon, Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import print_function
import atexit
import logging
import os
import re
import shlex
import signal
import sys
from cmd import Cmd
from collections import OrderedDict
from datetime import datetime
from optparse import IndentedHelpFormatter, OptionGroup, OptionParser
from weboob.capabilities.base import BaseObject, FieldNotFound, UserError, empty
from weboob.capabilities.collection import BaseCollection, CapCollection, Collection, CollectionNotFound
from weboob.core import CallErrors
from weboob.tools.application.formatters.iformatter import MandatoryFieldsNotFound
from weboob.tools.compat import basestring, range, unicode
from weboob.tools.misc import to_unicode
from weboob.tools.path import WorkingPath
from .console import BackendNotGiven, ConsoleApplication
from .formatters.load import FormatterLoadError, FormattersLoader
from .results import ResultsCondition, ResultsConditionError
__all__ = ['NotEnoughArguments', 'TooManyArguments', 'ArgSyntaxError',
'ReplApplication']
class NotEnoughArguments(Exception):
pass
class TooManyArguments(Exception):
pass
class ArgSyntaxError(Exception):
pass
class ReplOptionParser(OptionParser):
def format_option_help(self, formatter=None):
if not formatter:
formatter = self.formatter
return '%s\n%s' % (formatter.format_commands(self.commands),
OptionParser.format_option_help(self, formatter))
class ReplOptionFormatter(IndentedHelpFormatter):
def format_commands(self, commands):
s = u''
for section, cmds in commands.items():
if len(cmds) == 0:
continue
if len(s) > 0:
s += '\n'
s += '%s Commands:\n' % section
for c in cmds:
c = c.split('\n')[0]
s += ' %s\n' % c
return s
def defaultcount(default_count=10):
def deco(f):
def inner(self, *args, **kwargs):
oldvalue = self.options.count
if self._is_default_count:
self.options.count = default_count
try:
return f(self, *args, **kwargs)
finally:
self.options.count = oldvalue
inner.__doc__ = f.__doc__
assert inner.__doc__ is not None, "A command must have a docstring"
inner.__doc__ += '\nDefault is limited to %s results.' % default_count
return inner
return deco
class MyCmd(Cmd, object):
# Hack for Python 2, because Cmd doesn't inherit object, so its __init__ was not called when using super only
pass
class ReplApplication(ConsoleApplication, MyCmd):
"""
Base application class for Repl applications.
"""
SYNOPSIS = 'Usage: %prog [-dqv] [-b backends] [-cnfs] [command [arguments..]]\n'
SYNOPSIS += ' %prog [--help] [--version]'
DISABLE_REPL = False
EXTRA_FORMATTERS = {}
DEFAULT_FORMATTER = 'multiline'
COMMANDS_FORMATTERS = {}
# Objects to allow in do_ls / do_cd
COLLECTION_OBJECTS = tuple()
weboob_commands = set(['backends', 'condition', 'count', 'formatter', 'logging', 'select', 'quit', 'ls', 'cd'])
hidden_commands = set(['EOF'])
def __init__(self):
super(ReplApplication, self).__init__(ReplOptionParser(self.SYNOPSIS, version=self._get_optparse_version()))
copyright = self.COPYRIGHT.replace('YEAR', '%d' % datetime.today().year)
self.intro = '\n'.join(('Welcome to %s%s%s v%s' % (self.BOLD, self.APPNAME, self.NC, self.VERSION),
'',
copyright,
'This program is free software: you can redistribute it and/or modify',
'it under the terms of the GNU Affero General Public License as published by',
'the Free Software Foundation, either version 3 of the License, or',
'(at your option) any later version.',
'',
'Type "help" to display available commands.',
'',
))
self.formatters_loader = FormattersLoader()
for key, klass in self.EXTRA_FORMATTERS.items():
self.formatters_loader.register_formatter(key, klass)
self.formatter = None
self.commands_formatters = self.COMMANDS_FORMATTERS.copy()
commands_help = self.get_commands_doc()
self._parser.commands = commands_help
self._parser.formatter = ReplOptionFormatter()
results_options = OptionGroup(self._parser, 'Results Options')
results_options.add_option('-c', '--condition', help='filter result items to display given a boolean expression. See CONDITION section for the syntax')
results_options.add_option('-n', '--count', type='int',
help='limit number of results (from each backends)')
results_options.add_option('-s', '--select', help='select result item keys to display (comma separated)')
self._parser.add_option_group(results_options)
formatting_options = OptionGroup(self._parser, 'Formatting Options')
available_formatters = self.formatters_loader.get_available_formatters()
formatting_options.add_option('-f', '--formatter', choices=available_formatters,
help='select output formatter (%s)' % u', '.join(available_formatters))
formatting_options.add_option('--no-header', dest='no_header', action='store_true', help='do not display header')
formatting_options.add_option('--no-keys', dest='no_keys', action='store_true', help='do not display item keys')
formatting_options.add_option('-O', '--outfile', dest='outfile', help='file to export result')
self._parser.add_option_group(formatting_options)
self._interactive = False
self.working_path = WorkingPath()
self._change_prompt()
@property
def interactive(self):
return self._interactive
def _change_prompt(self):
self.objects = []
self.collections = []
# XXX can't use bold prompt because:
# 1. it causes problems when trying to get history (lines don't start
# at the right place).
# 2. when typing a line longer than term width, cursor goes at start
# of the same line instead of new line.
#self.prompt = self.BOLD + '%s> ' % self.APPNAME + self.NC
if len(self.working_path.get()):
wp_enc = unicode(self.working_path)
self.prompt = '%s:%s> ' % (self.APPNAME, wp_enc)
else:
self.prompt = '%s> ' % (self.APPNAME)
def change_path(self, split_path):
self.working_path.location(split_path)
self._change_prompt()
def add_object(self, obj):
self.objects.append(obj)
def _complete_object(self):
return [obj.fullid for obj in self.objects]
def parse_id(self, id, unique_backend=False):
if self.interactive:
try:
obj = self.objects[int(id) - 1]
except (IndexError, ValueError):
# Try to find a shortcut in the cache
for obj in self.objects:
if id in obj.id:
id = obj.fullid
break
else:
if isinstance(obj, BaseObject):
id = obj.fullid
try:
return ConsoleApplication.parse_id(self, id, unique_backend)
except BackendNotGiven as e:
backend_name = None
while not backend_name:
print('This command works with an unique backend. Availables:')
for index, (name, backend) in enumerate(e.backends):
print('%s%d)%s %s%-15s%s %s' % (self.BOLD, index + 1, self.NC, self.BOLD, name, self.NC,
backend.DESCRIPTION))
i = self.ask('Select a backend to proceed with "%s"' % id)
if not i.isdigit():
if i not in dict(e.backends):
print('Error: %s is not a valid backend' % i, file=self.stderr)
continue
backend_name = i
else:
i = int(i)
if i < 0 or i > len(e.backends):
print('Error: %s is not a valid choice' % i, file=self.stderr)
continue
backend_name = e.backends[i-1][0]
return id, backend_name
def get_object(self, _id, method, fields=None, caps=None):
if self.interactive:
try:
obj = self.objects[int(_id) - 1]
except (IndexError, ValueError):
pass
else:
try:
backend = self.weboob.get_backend(obj.backend)
actual_method = getattr(backend, method, None)
if actual_method is None:
return None
else:
if callable(actual_method):
obj, = self.do('fillobj', obj, fields, backends=backend)
return obj
else:
return None
except UserError as e:
self.bcall_error_handler(backend, e, '')
_id, backend_name = self.parse_id(_id)
kargs = {}
if caps is not None:
kargs = {'caps': caps}
backend_names = (backend_name,) if backend_name is not None else self.enabled_backends
# if backend's service returns several objects, try to find the one
# with wanted ID. If not found, get the last not None object.
obj = None
# remove backends that do not have the required method
new_backend_names = []
for backend in backend_names:
if isinstance(backend, (str, unicode)):
actual_backend = self.weboob.get_backend(backend)
else:
actual_backend = backend
if getattr(actual_backend, method, None) is not None:
new_backend_names.append(backend)
backend_names = tuple(new_backend_names)
try:
for objiter in self.do(method, _id, backends=backend_names, fields=fields, **kargs):
if objiter:
obj = objiter
if objiter.id == _id:
return obj
except CallErrors as e:
if obj is not None:
self.bcall_errors_handler(e)
else:
raise
return obj
def get_object_list(self, method=None, *args, **kwargs):
# return cache if not empty
if len(self.objects) > 0:
return self.objects
elif method is not None:
kwargs['backends'] = self.enabled_backends
for _object in self.do(method, *args, **kwargs):
self.add_object(_object)
return self.objects
# XXX: what can we do without method?
return tuple()
def unload_backends(self, *args, **kwargs):
self.objects = []
self.collections = []
return ConsoleApplication.unload_backends(self, *args, **kwargs)
def load_backends(self, *args, **kwargs):
self.objects = []
self.collections = []
return ConsoleApplication.load_backends(self, *args, **kwargs)
def main(self, argv):
cmd_args = argv[1:]
if cmd_args:
cmd_line = u' '.join(cmd_args)
cmds = cmd_line.split(';')
for cmd in cmds:
ret = self.onecmd(cmd)
if ret:
return ret
elif self.DISABLE_REPL:
self._parser.print_help()
self._parser.exit()
else:
try:
import readline
except ImportError:
pass
else:
# Remove '-' from delims
readline.set_completer_delims(readline.get_completer_delims().replace('-', ''))
history_filepath = os.path.join(self.weboob.workdir, '%s_history' % self.APPNAME)
try:
readline.read_history_file(history_filepath)
except IOError:
pass
def savehist():
readline.write_history_file(history_filepath)
atexit.register(savehist)
self.intro += '\nLoaded backends: %s\n' % ', '.join(sorted(backend.name for backend in self.weboob.iter_backends()))
self._interactive = True
self.cmdloop()
def do(self, function, *args, **kwargs):
"""
Call Weboob.do(), passing count and selected fields given by user.
"""
backends = kwargs.pop('backends', None)
if backends is None:
kwargs['backends'] = []
for backend in self.enabled_backends:
actual_function = getattr(backend, function, None) if isinstance(function, basestring) else function
if callable(actual_function):
kwargs['backends'].append(backend)
else:
kwargs['backends'] = backends
fields = kwargs.pop('fields', self.selected_fields)
if not fields and fields != []:
fields = self.selected_fields
fields = self.parse_fields(fields)
if fields and self.formatter.MANDATORY_FIELDS is not None:
missing_fields = set(self.formatter.MANDATORY_FIELDS) - set(fields)
# If a mandatory field is not selected, do not use the customized formatter
if missing_fields:
print('Warning: you do not select enough mandatory fields for the formatter. Fallback to another. Hint: use option -f', file=self.stderr)
self.formatter = self.formatters_loader.build_formatter(ReplApplication.DEFAULT_FORMATTER)
if self.formatter.DISPLAYED_FIELDS is not None:
if fields is None:
missing_fields = True
else:
missing_fields = set(fields) - set(self.formatter.DISPLAYED_FIELDS + self.formatter.MANDATORY_FIELDS)
# If a selected field is not displayed, do not use the customized formatter
if missing_fields:
print('Warning: some selected fields will not be displayed by the formatter. Fallback to another. Hint: use option -f', file=self.stderr)
self.formatter = self.formatters_loader.build_formatter(ReplApplication.DEFAULT_FORMATTER)
return self.weboob.do(self._do_complete, self.options.count, fields, function, *args, **kwargs)
# -- command tools ------------
def parse_command_args(self, line, nb, req_n=None):
try:
if sys.version_info.major >= 3:
args = shlex.split(line)
else:
args = [arg.decode('utf-8') for arg in shlex.split(line.encode('utf-8'))]
except ValueError as e:
raise ArgSyntaxError(str(e))
if nb < len(args):
raise TooManyArguments('Command takes at most %d arguments' % nb)
if req_n is not None and (len(args) < req_n):
raise NotEnoughArguments('Command needs %d arguments' % req_n)
if len(args) < nb:
args += tuple(None for i in range(nb - len(args)))
return args
# -- cmd.Cmd methods ---------
def postcmd(self, stop, line):
"""
This REPL method is overridden to return None instead of integers
to prevent stopping cmdloop().
"""
if not isinstance(stop, bool):
stop = None
return stop
def parseline(self, line):
"""
This REPL method is overridden to search "short" alias of commands
"""
cmd, arg, ignored = Cmd.parseline(self, line)
if cmd is not None:
names = set(name for name in self.get_names() if name.startswith('do_'))
if 'do_' + cmd not in names:
long = set(name for name in names if name.startswith('do_' + cmd))
# if more than one result, ambiguous command, do nothing (error will display suggestions)
if len(long) == 1:
cmd = long.pop()[3:]
return cmd, arg, ignored
def onecmd(self, line):
"""
This REPL method is overridden to catch some particular exceptions.
"""
line = to_unicode(line)
cmd, arg, ignored = self.parseline(line)
# Set the right formatter for the command.
try:
formatter_name = self.commands_formatters[cmd]
except KeyError:
formatter_name = self.DEFAULT_FORMATTER
self.set_formatter(formatter_name)
try:
try:
return super(ReplApplication, self).onecmd(line)
except CallErrors as e:
return self.bcall_errors_handler(e)
except BackendNotGiven as e:
print('Error: %s' % str(e), file=self.stderr)
return os.EX_DATAERR
except NotEnoughArguments as e:
print('Error: not enough arguments. %s' % str(e), file=self.stderr)
return os.EX_USAGE
except TooManyArguments as e:
print('Error: too many arguments. %s' % str(e), file=self.stderr)
return os.EX_USAGE
except ArgSyntaxError as e:
print('Error: invalid arguments. %s' % str(e), file=self.stderr)
return os.EX_USAGE
except (KeyboardInterrupt, EOFError):
# ^C during a command process doesn't exit application.
print('\nAborted.')
return signal.SIGINT + 128
finally:
self.flush()
def emptyline(self):
"""
By default, an emptyline repeats the previous command.
Overriding this function disables this behaviour.
"""
pass
def default(self, line):
print('Unknown command: "%s"' % line, file=self.stderr)
cmd, arg, ignore = Cmd.parseline(self, line)
if cmd is not None:
names = set(name[3:] for name in self.get_names() if name.startswith('do_' + cmd))
if len(names) > 0:
print('Do you mean: %s?' % ', '.join(names), file=self.stderr)
return os.EX_USAGE
def completenames(self, text, *ignored):
return [name for name in Cmd.completenames(self, text, *ignored) if name not in self.hidden_commands]
def _shell_completion_items(self):
items = super(ReplApplication, self)._shell_completion_items()
items.update(
set(self.completenames('')) -
set(('debug', 'condition', 'count', 'formatter', 'logging', 'select', 'quit')))
return items
def path_completer(self, arg):
dirname = os.path.dirname(arg)
try:
children = os.listdir(dirname or '.')
except OSError:
return ()
l = []
for child in children:
path = os.path.join(dirname, child)
if os.path.isdir(path):
child += '/'
l.append(child)
return l
def complete(self, text, state):
"""
Override of the Cmd.complete() method to:
* add a space at end of proposals
* display only proposals for words which match the
text already written by user.
"""
super(ReplApplication, self).complete(text, state)
# When state = 0, Cmd.complete() set the 'completion_matches' attribute by
# calling the completion function. Then, for other states, it only tries to
# get the right item in list.
# So that's the good place to rework the choices.
if state == 0:
self.completion_matches = [choice for choice in self.completion_matches if choice.startswith(text)]
try:
match = self.completion_matches[state]
except IndexError:
return None
else:
if match[-1] != '/':
return '%s ' % match
return match
# -- errors management -------------
def bcall_error_handler(self, backend, error, backtrace):
"""
Handler for an exception inside the CallErrors exception.
This method can be overridden to support more exceptions types.
"""
return super(ReplApplication, self).bcall_error_handler(backend, error, backtrace)
def bcall_errors_handler(self, errors, ignore=()):
if self.interactive:
return super(ReplApplication, self).bcall_errors_handler(errors, 'Use "logging debug" option to print backtraces.', ignore)
else:
return super(ReplApplication, self).bcall_errors_handler(errors, ignore=ignore)
# -- options related methods -------------
def _handle_options(self):
if self.options.formatter:
self.commands_formatters = {}
self.DEFAULT_FORMATTER = self.options.formatter
self.set_formatter(self.DEFAULT_FORMATTER)
if self.options.select:
self.selected_fields = self.options.select.split(',')
else:
self.selected_fields = ['$direct']
if self.options.count is not None:
self._is_default_count = False
if self.options.count <= 0:
# infinite search
self.options.count = None
if self.options.condition:
self.condition = ResultsCondition(self.options.condition)
else:
self.condition = None
return super(ReplApplication, self)._handle_options()
def get_command_help(self, command, short=False):
try:
func = getattr(self, 'do_' + command)
except AttributeError:
return None
doc = func.__doc__
assert doc is not None, "A command must have a docstring"
lines = [line.strip() for line in doc.strip().split('\n')]
if not lines[0].startswith(command):
lines = [command, ''] + lines
if short:
return lines[0]
return '\n'.join(lines)
def get_commands_doc(self):
names = set(name for name in self.get_names() if name.startswith('do_'))
appname = self.APPNAME.capitalize()
d = OrderedDict(((appname, []), ('Weboob', [])))
for name in sorted(names):
cmd = name[3:]
if cmd in self.hidden_commands.union(self.weboob_commands).union(['help']):
continue
d[appname].append(self.get_command_help(cmd))
if not self.DISABLE_REPL:
for cmd in self.weboob_commands:
d['Weboob'].append(self.get_command_help(cmd))
return d
# -- default REPL commands ---------
def do_quit(self, arg):
"""
Quit the application.
"""
return True
def do_EOF(self, arg):
"""
Quit the command line interpreter when ^D is pressed.
"""
# print empty line for the next shell prompt to appear on the first column of the terminal
print()
return self.do_quit(arg)
def do_help(self, arg=None):
"""
help [COMMAND]
List commands, or get information about a command.
"""
if arg:
cmd_names = set(name[3:] for name in self.get_names() if name.startswith('do_'))
if arg in cmd_names:
command_help = self.get_command_help(arg)
if command_help is None:
logging.warning(u'Command "%s" is undocumented' % arg)
else:
lines = command_help.split('\n')
lines[0] = '%s%s%s' % (self.BOLD, lines[0], self.NC)
self.stdout.write('%s\n' % '\n'.join(lines))
else:
print('Unknown command: "%s"' % arg, file=self.stderr)
else:
cmds = self._parser.formatter.format_commands(self._parser.commands)
self.stdout.write('%s\n' % cmds)
self.stdout.write('Type "help " for more info about a command.\n')
return 2
def complete_backends(self, text, line, begidx, endidx):
choices = []
commands = ['enable', 'disable', 'only', 'list', 'add', 'register', 'edit', 'remove', 'list-modules']
available_backends_names = set(backend.name for backend in self.weboob.iter_backends())
enabled_backends_names = set(backend.name for backend in self.enabled_backends)
args = line.split(' ')
if len(args) == 2:
choices = commands
elif len(args) >= 3:
if args[1] == 'enable':
choices = sorted(available_backends_names - enabled_backends_names)
elif args[1] == 'only':
choices = sorted(available_backends_names)
elif args[1] == 'disable':
choices = sorted(enabled_backends_names)
elif args[1] in ('add', 'register') and len(args) == 3:
for name, module in sorted(self.weboob.repositories.get_all_modules_info(self.CAPS).items()):
choices.append(name)
elif args[1] == 'edit':
choices = sorted(available_backends_names)
elif args[1] == 'remove':
choices = sorted(available_backends_names)
return choices
def do_backends(self, line):
"""
backends [ACTION] [BACKEND_NAME]...
Select used backends.
ACTION is one of the following (default: list):
* enable enable given backends
* disable disable given backends
* only enable given backends and disable the others
* list list backends
* add add a backend
* register register a new account on a website
* edit edit a backend
* remove remove a backend
* list-modules list modules
"""
line = line.strip()
if line:
args = line.split()
else:
args = ['list']
action = args[0]
given_backend_names = args[1:]
for backend_name in given_backend_names:
if action in ('add', 'register'):
minfo = self.weboob.repositories.get_module_info(backend_name)
if minfo is None:
print('Module "%s" does not exist.' % backend_name, file=self.stderr)
return 1
else:
if not minfo.has_caps(self.CAPS):
print('Module "%s" is not supported by this application => skipping.' % backend_name, file=self.stderr)
return 1
else:
if backend_name not in [backend.name for backend in self.weboob.iter_backends()]:
print('Backend "%s" does not exist => skipping.' % backend_name, file=self.stderr)
return 1
if action in ('enable', 'disable', 'only', 'add', 'register', 'edit', 'remove'):
if not given_backend_names:
print('Please give at least a backend name.', file=self.stderr)
return 2
given_backends = set(backend for backend in self.weboob.iter_backends() if backend.name in given_backend_names)
if action == 'enable':
for backend in given_backends:
self.enabled_backends.add(backend)
elif action == 'disable':
for backend in given_backends:
try:
self.enabled_backends.remove(backend)
except KeyError:
print('%s is not enabled' % backend.name, file=self.stderr)
elif action == 'only':
self.enabled_backends = set()
for backend in given_backends:
self.enabled_backends.add(backend)
elif action == 'list':
enabled_backends_names = set(backend.name for backend in self.enabled_backends)
disabled_backends_names = set(backend.name for backend in self.weboob.iter_backends()) - enabled_backends_names
print('Enabled: %s' % ', '.join(enabled_backends_names))
if len(disabled_backends_names) > 0:
print('Disabled: %s' % ', '.join(disabled_backends_names))
elif action == 'add':
for name in given_backend_names:
instname = self.add_backend(name, name)
if instname:
self.load_backends(names=[instname])
elif action == 'register':
for name in given_backend_names:
instname = self.register_backend(name)
if isinstance(instname, basestring):
self.load_backends(names=[instname])
elif action == 'edit':
for backend in given_backends:
enabled = backend in self.enabled_backends
self.unload_backends(names=[backend.name])
self.edit_backend(backend.name)
for newb in self.load_backends(names=[backend.name]).values():
if not enabled:
self.enabled_backends.remove(newb)
elif action == 'remove':
for backend in given_backends:
self.weboob.backends_config.remove_backend(backend.name)
self.unload_backends(backend.name)
elif action == 'list-modules':
modules = []
print('Modules list:')
for name, info in sorted(self.weboob.repositories.get_all_modules_info().items()):
if not self.is_module_loadable(info):
continue
modules.append(name)
loaded = ' '
for bi in self.weboob.iter_backends():
if bi.NAME == name:
if loaded == ' ':
loaded = 'X'
elif loaded == 'X':
loaded = 2
else:
loaded += 1
print('[%s] %s%-15s%s %s' % (loaded, self.BOLD, name, self.NC, info.description))
else:
print('Unknown action: "%s"' % action, file=self.stderr)
return 1
if len(self.enabled_backends) == 0:
print('Warning: no more backends are loaded. %s is probably unusable.' % self.APPNAME.capitalize(), file=self.stderr)
def complete_logging(self, text, line, begidx, endidx):
levels = ('debug', 'info', 'warning', 'error', 'quiet', 'default')
args = line.split(' ')
if len(args) == 2:
return levels
return ()
def do_logging(self, line):
"""
logging [LEVEL]
Set logging level.
Availables: debug, info, warning, error.
* quiet is an alias for error
* default is an alias for warning
"""
args = self.parse_command_args(line, 1, 0)
levels = (('debug', logging.DEBUG),
('info', logging.INFO),
('warning', logging.WARNING),
('error', logging.ERROR),
('quiet', logging.ERROR),
('default', logging.WARNING)
)
if not args[0]:
current = None
for label, level in levels:
if logging.root.level == level:
current = label
break
print('Current level: %s' % current)
return
levels = dict(levels)
try:
level = levels[args[0]]
except KeyError:
print('Level "%s" does not exist.' % args[0], file=self.stderr)
print('Availables: %s' % ' '.join(levels), file=self.stderr)
return 2
else:
logging.root.setLevel(level)
for handler in logging.root.handlers:
handler.setLevel(level)
def do_condition(self, line):
"""
condition [EXPRESSION | off]
If an argument is given, set the condition expression used to filter the results. See CONDITION section for more details and the expression.
If the "off" value is given, conditional filtering is disabled.
If no argument is given, print the current condition expression.
"""
line = line.strip()
if line:
if line == 'off':
self.condition = None
else:
try:
self.condition = ResultsCondition(line)
except ResultsConditionError as e:
print('%s' % e, file=self.stderr)
return 2
else:
if self.condition is None:
print('No condition is set.')
else:
print(str(self.condition))
def do_count(self, line):
"""
count [NUMBER | off]
If an argument is given, set the maximum number of results fetched.
NUMBER must be at least 1.
"off" value disables counting, and allows infinite searches.
If no argument is given, print the current count value.
"""
line = line.strip()
if line:
if line == 'off':
self.options.count = None
self._is_default_count = False
else:
try:
count = int(line)
except ValueError:
print('Could not interpret "%s" as a number.' % line, file=self.stderr)
return 2
else:
if count > 0:
self.options.count = count
self._is_default_count = False
else:
self.options.count = None
self._is_default_count = False
else:
if self.options.count is None:
print('Counting disabled.')
else:
print(self.options.count)
def complete_formatter(self, text, line, *ignored):
formatters = self.formatters_loader.get_available_formatters()
commands = ['list', 'option'] + formatters
options = ['header', 'keys']
option_values = ['on', 'off']
args = line.split(' ')
if len(args) == 2:
return commands
if args[1] == 'option':
if len(args) == 3:
return options
if len(args) == 4:
return option_values
elif args[1] in formatters:
return list(set(name[3:] for name in self.get_names() if name.startswith('do_')))
def do_formatter(self, line):
"""
formatter [list | FORMATTER [COMMAND] | option OPTION_NAME [on | off]]
If a FORMATTER is given, set the formatter to use.
You can add a COMMAND to apply the formatter change only to
a given command.
If the argument is "list", print the available formatters.
If the argument is "option", set the formatter options.
Valid options are: header, keys.
If on/off value is given, set the value of the option.
If not, print the current value for the option.
If no argument is given, print the current formatter.
"""
args = line.strip().split()
if args:
if args[0] == 'list':
print(', '.join(self.formatters_loader.get_available_formatters()))
elif args[0] == 'option':
if len(args) > 1:
if len(args) == 2:
if args[1] == 'header':
print('off' if self.options.no_header else 'on')
elif args[1] == 'keys':
print('off' if self.options.no_keys else 'on')
else:
if args[2] not in ('on', 'off'):
print('Invalid value "%s". Please use "on" or "off" values.' % args[2], file=self.stderr)
return 2
else:
if args[1] == 'header':
self.options.no_header = True if args[2] == 'off' else False
elif args[1] == 'keys':
self.options.no_keys = True if args[2] == 'off' else False
else:
print('Don\'t know which option to set. Available options: header, keys.', file=self.stderr)
return 2
else:
if args[0] in self.formatters_loader.get_available_formatters():
if len(args) > 1:
self.commands_formatters[args[1]] = self.set_formatter(args[0])
else:
self.commands_formatters = {}
self.DEFAULT_FORMATTER = self.set_formatter(args[0])
else:
print('Formatter "%s" is not available.\n'
'Available formatters: %s.' % (args[0], ', '.join(self.formatters_loader.get_available_formatters())), file=self.stderr)
return 1
else:
print('Default formatter: %s' % self.DEFAULT_FORMATTER)
for key, klass in self.commands_formatters.items():
print('Command "%s": %s' % (key, klass))
def do_select(self, line):
"""
select [FIELD_NAME]... | "$direct" | "$full"
If an argument is given, set the selected fields.
$direct selects all fields loaded in one http request.
$full selects all fields using as much http requests as necessary.
If no argument is given, print the currently selected fields.
"""
line = line.strip()
if line:
split = line.split()
self.selected_fields = split
else:
print(' '.join(self.selected_fields))
# First sort in alphabetical of backend
# Second, sort with ID
def comp_key(self, obj):
return (obj.backend, obj.id)
@defaultcount(40)
def do_ls(self, line):
"""
ls [-d] [-U] [PATH]
List objects in current path.
If an argument is given, list the specified path.
Use -U option to not sort results. It allows you to use a "fast path" to
return results as soon as possible.
Use -d option to display information about a collection (and to not
display the content of it). It has the same behavior than the well
known UNIX "ls" command.
"""
# TODO: real parsing of options
path = line.strip()
only = False
sort = True
if '-U' in line.strip().partition(' '):
path = line.strip().partition(' ')[-1]
sort = False
if '-d' in line.strip().partition(' '):
path = None
only = line.strip().partition(' ')[-1]
if path:
for _path in path.split('/'):
# We have an argument, let's ch to the directory before the ls
self.working_path.cd1(_path)
objects = []
collections = []
self.objects = []
self.start_format()
for res in self._fetch_objects(objs=self.COLLECTION_OBJECTS):
if isinstance(res, Collection):
collections.append(res)
if sort is False:
self.formatter.format_collection(res, only)
else:
if sort:
objects.append(res)
else:
self._format_obj(res, only)
if sort:
objects.sort(key=self.comp_key)
collections = self._merge_collections_with_same_path(collections)
collections.sort(key=self.comp_key)
for collection in collections:
self.formatter.format_collection(collection, only)
for obj in objects:
self._format_obj(obj, only)
if path:
for _path in path.split('/'):
# Let's go back to the parent directory
self.working_path.up()
else:
# Save collections only if we listed the current path.
self.collections = collections
def _find_collection(self, collection, collections):
for col in collections:
if col.split_path == collection.split_path:
return col
return None
def _merge_collections_with_same_path(self, collections):
to_return = []
for collection in collections:
col = self._find_collection(collection, to_return)
if col:
col.backend += " %s" % collection.backend
else:
to_return.append(collection)
return to_return
def _format_obj(self, obj, only):
if only is False or not hasattr(obj, 'id') or obj.id in only:
self.cached_format(obj)
def do_cd(self, line):
"""
cd [PATH]
Follow a path.
".." is a special case and goes up one directory.
"" is a special case and goes home.
"""
if not len(line.strip()):
self.working_path.home()
elif line.strip() == '..':
self.working_path.up()
else:
self.working_path.cd1(line)
collections = []
try:
for res in self.do('get_collection', objs=self.COLLECTION_OBJECTS,
split_path=self.working_path.get(),
caps=CapCollection):
if res:
collections.append(res)
except CallErrors as errors:
self.bcall_errors_handler(errors, CollectionNotFound)
if len(collections):
# update the path from the collection if possible
if len(collections) == 1:
self.working_path.split_path = collections[0].split_path
else:
print(u"Path: %s not found" % unicode(self.working_path), file=self.stderr)
self.working_path.restore()
return 1
self._change_prompt()
def _fetch_objects(self, objs):
split_path = self.working_path.get()
try:
for res in self.do('iter_resources', objs=objs,
split_path=split_path,
caps=CapCollection):
yield res
except CallErrors as errors:
self.bcall_errors_handler(errors, CollectionNotFound)
def all_collections(self):
"""
Get all objects that are collections: regular objects and fake dumb objects.
"""
obj_collections = [obj for obj in self.objects if isinstance(obj, BaseCollection)]
return obj_collections + self.collections
def obj_to_filename(self, obj, dest=None, default=None):
"""
This method can be used to get a filename from an object, using a mask
filled by information of this object.
All patterns are braces-enclosed, and are name of available fields in
the object.
:param obj: object
:type obj: BaseObject
:param dest: dest given by user (default None)
:type dest: str
:param default: default file mask (if not given, this is '{id}-{title}.{ext}')
:type default: str
:rtype: str
"""
if default is None:
default = '{id}-{title}.{ext}'
if dest is None:
dest = '.'
if os.path.isdir(dest):
dest = os.path.join(dest, default)
def repl(m):
field = m.group(1)
if hasattr(obj, field):
value = getattr(obj, field)
if empty(value):
value = 'unknown'
return re.sub('[?:/]', '-', '%s' % value)
else:
return m.group(0)
return re.sub(r'\{(.+?)\}', repl, dest)
# for cd & ls
def complete_path(self, text, line, begidx, endidx):
directories = set()
if len(self.working_path.get()):
directories.add('..')
mline = line.partition(' ')[2]
offs = len(mline) - len(text)
# refresh only if needed
if len(self.objects) == 0 and len(self.collections) == 0:
try:
self.objects, self.collections = self._fetch_objects(objs=self.COLLECTION_OBJECTS)
except CallErrors as errors:
self.bcall_errors_handler(errors, CollectionNotFound)
collections = self.all_collections()
for collection in collections:
directories.add(collection.basename)
return [s[offs:] for s in directories if s.startswith(mline)]
def complete_ls(self, text, line, begidx, endidx):
return self.complete_path(text, line, begidx, endidx)
def complete_cd(self, text, line, begidx, endidx):
return self.complete_path(text, line, begidx, endidx)
# -- formatting related methods -------------
def set_formatter(self, name):
"""
Set the current formatter from name.
It returns the name of the formatter which has been really set.
"""
try:
self.formatter = self.formatters_loader.build_formatter(name)
except FormatterLoadError as e:
print('%s' % e, file=self.stderr)
if self.DEFAULT_FORMATTER == name:
self.DEFAULT_FORMATTER = ReplApplication.DEFAULT_FORMATTER
print('Falling back to "%s".' % (self.DEFAULT_FORMATTER), file=self.stderr)
self.formatter = self.formatters_loader.build_formatter(self.DEFAULT_FORMATTER)
name = self.DEFAULT_FORMATTER
if self.options.no_header:
self.formatter.display_header = False
if self.options.no_keys:
self.formatter.display_keys = False
if self.options.outfile:
self.formatter.outfile = self.options.outfile
if self.interactive:
self.formatter.interactive = True
return name
def set_formatter_header(self, string):
pass
def start_format(self, **kwargs):
self.formatter.start_format(**kwargs)
def cached_format(self, obj):
self.add_object(obj)
alias = None
if self.interactive:
alias = '%s' % len(self.objects)
self.format(obj, alias=alias)
def parse_fields(self, fields):
if '$direct' in fields:
return []
if '$full' in fields:
return None
return fields
def format(self, result, alias=None):
fields = self.parse_fields(self.selected_fields)
try:
self.formatter.format(obj=result, selected_fields=fields, alias=alias)
except FieldNotFound as e:
print(e, file=self.stderr)
except MandatoryFieldsNotFound as e:
print('%s Hint: select missing fields or use another formatter (ex: multiline).' % e, file=self.stderr)
def flush(self):
self.formatter.flush()
def do_debug(self, line):
"""
debug
Launch a debug Python shell
"""
from weboob.applications.weboobdebug import weboobdebug
app = weboobdebug.WeboobDebug()
locs = dict(application=self, weboob=self.weboob)
banner = ('Weboob debug shell\n\nAvailable variables:\n'
+ '\n'.join([' %s: %s' % (k, v) for k, v in locs.items()]))
funcs = [app.ipython, app.bpython, app.python]
app.launch(funcs, locs, banner)
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/application/results.py 0000664 0000000 0000000 00000013730 13414137563 0030416 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities import UserError
from weboob.tools.compat import unicode
__all__ = ['ResultsCondition', 'ResultsConditionError']
class IResultsCondition(object):
def is_valid(self, obj):
raise NotImplementedError()
class ResultsConditionError(UserError):
pass
class Condition(object):
def __init__(self, left, op, right):
self.left = left # Field of the object to test
self.op = op
self.right = right
def is_egal(left, right):
return left == right
def is_notegal(left, right):
return left != right
def is_sup(left, right):
return left < right
def is_inf(left, right):
return left > right
def is_in(left, right):
return left in right
functions = {'!=': is_notegal, '=': is_egal, '>': is_sup, '<': is_inf, '|': is_in}
class ResultsCondition(IResultsCondition):
condition_str = None
# Supported operators
# =, !=, <, > for float/int/decimal
# =, != for strings
# We build a list of list. Return true if each conditions of one list is TRUE
def __init__(self, condition_str):
self.limit = None
or_list = []
_condition_str = condition_str.split(' LIMIT ')
if len(_condition_str) == 2:
try:
self.limit = int(_condition_str[1])
except ValueError:
raise ResultsConditionError(u'Syntax error in the condition expression, please check documentation')
condition_str= _condition_str[0]
for _or in condition_str.split(' OR '):
and_list = []
for _and in _or.split(' AND '):
operator = None
for op in ['!=', '=', '>', '<', '|']:
if op in _and:
operator = op
break
if operator is None:
raise ResultsConditionError(u'Could not find a valid operator in sub-expression "%s". Protect the complete condition expression with quotes, or read the documentation in the man manual.' % _and)
try:
l, r = _and.split(operator)
except ValueError:
raise ResultsConditionError(u'Syntax error in the condition expression, please check documentation')
and_list.append(Condition(l.strip(), operator, r.strip()))
or_list.append(and_list)
self.condition = or_list
self.condition_str = condition_str
def is_valid(self, obj):
import weboob.tools.date as date_utils
import re
from datetime import date, datetime, timedelta
d = obj.to_dict()
# We evaluate all member of a list at each iteration.
for _or in self.condition:
myeval = True
for condition in _or:
if condition.left in d:
# in the case of id, test id@backend and id
if condition.left == 'id':
tocompare = condition.right
evalfullid = functions[condition.op](tocompare, d['id'])
evalid = functions[condition.op](tocompare, obj.id)
myeval = evalfullid or evalid
else:
# We have to change the type of v, always gived as string by application
typed = type(d[condition.left])
try:
if isinstance(d[condition.left], date_utils.date):
tocompare = date(*[int(x) for x in condition.right.split('-')])
elif isinstance(d[condition.left], date_utils.datetime):
splitted_datetime = condition.right.split(' ')
tocompare = datetime(*([int(x) for x in splitted_datetime[0].split('-')] +
[int(x) for x in splitted_datetime[1].split(':')]))
elif isinstance(d[condition.left], timedelta):
time_dict = re.match('^\s*((?P\d+)\s*h)?\s*((?P\d+)\s*m)?\s*((?P\d+)\s*s)?\s*$',
condition.right).groupdict()
tocompare = timedelta(seconds=int(time_dict['seconds'] or "0"),
minutes=int(time_dict['minutes'] or "0"),
hours=int(time_dict['hours'] or "0"))
else:
tocompare = typed(condition.right)
myeval = functions[condition.op](tocompare, d[condition.left])
except:
myeval = False
else:
raise ResultsConditionError(u'Field "%s" is not valid.' % condition.left)
# Do not try all AND conditions if one is false
if not myeval:
break
# Return True at the first OR valid condition
if myeval:
return True
# If we are here, all OR conditions are False
return False
def __str__(self):
return unicode(self).encode('utf-8')
def __unicode__(self):
return self.condition_str
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/backend.py 0000664 0000000 0000000 00000037404 13414137563 0026005 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import os
from copy import copy
from threading import RLock
from weboob.capabilities.base import BaseObject, Capability, FieldNotFound, NotAvailable, NotLoaded
from weboob.exceptions import ModuleInstallError
from weboob.tools.compat import basestring, getproxies
from weboob.tools.log import getLogger
from weboob.tools.misc import iter_fields
from weboob.tools.value import ValuesDict
__all__ = ['BackendStorage', 'BackendConfig', 'Module']
class BackendStorage(object):
"""
This is an abstract layer to store data in storages (:mod:`weboob.tools.storage`)
easily.
It is instancied automatically in constructor of :class:`Module`, in the
:attr:`Module.storage` attribute.
:param name: name of backend
:param storage: storage object
:type storage: :class:`weboob.tools.storage.IStorage`
"""
def __init__(self, name, storage):
self.name = name
self.storage = storage
def set(self, *args):
"""
Set value in the storage.
Example:
>>> from weboob.tools.storage import StandardStorage
>>> backend = BackendStorage('blah', StandardStorage('/tmp/cfg'))
>>> backend.storage.set('config', 'nb_of_threads', 10) # doctest: +SKIP
>>>
:param args: the path where to store value
"""
if self.storage:
return self.storage.set('backends', self.name, *args)
def delete(self, *args):
"""
Delete a value from the storage.
:param args: path to delete.
"""
if self.storage:
return self.storage.delete('backends', self.name, *args)
def get(self, *args, **kwargs):
"""
Get a value or a dict of values in storage.
Example:
>>> from weboob.tools.storage import StandardStorage
>>> backend = BackendStorage('blah', StandardStorage('/tmp/cfg'))
>>> backend.storage.get('config', 'nb_of_threads') # doctest: +SKIP
10
>>> backend.storage.get('config', 'unexistant', 'path', default='lol') # doctest: +SKIP
'lol'
>>> backend.storage.get('config') # doctest: +SKIP
{'nb_of_threads': 10, 'other_things': 'blah'}
:param args: path to get
:param default: if specified, default value when path is not found
"""
if self.storage:
return self.storage.get('backends', self.name, *args, **kwargs)
else:
return kwargs.get('default', None)
def load(self, default):
"""
Load storage.
It is made automatically when your backend is created, and use the
``STORAGE`` class attribute as default.
:param default: this is the default tree if storage is empty
:type default: :class:`dict`
"""
if self.storage:
return self.storage.load('backends', self.name, default)
def save(self):
"""
Save storage.
"""
if self.storage:
return self.storage.save('backends', self.name)
class BackendConfig(ValuesDict):
"""
Configuration of a backend.
This class is firstly instanced as a :class:`weboob.tools.value.ValuesDict`,
containing some :class:`weboob.tools.value.Value` (and derivated) objects.
Then, using the :func:`load` method will load configuration from file and
create a copy of the :class:`BackendConfig` object with the loaded values.
"""
modname = None
instname = None
weboob = None
def load(self, weboob, modname, instname, config, nofail=False):
"""
Load configuration from dict to create an instance.
:param weboob: weboob object
:type weboob: :class:`weboob.core.ouiboube.Weboob`
:param modname: name of the module
:type modname: :class:`str`
:param instname: name of this backend
:type instname: :class:`str`
:param params: parameters to load
:type params: :class:`dict`
:param nofail: if true, this call can't fail
:type nofail: :class:`bool`
:rtype: :class:`BackendConfig`
"""
cfg = BackendConfig()
cfg.modname = modname
cfg.instname = instname
cfg.weboob = weboob
for name, field in self.items():
value = config.get(name, None)
if value is None:
if not nofail and field.required:
raise Module.ConfigError('Backend(%s): Configuration error: Missing parameter "%s" (%s)'
% (cfg.instname, name, field.description))
value = field.default
field = copy(field)
try:
field.load(cfg.instname, value, cfg.weboob.requests)
except ValueError as v:
if not nofail:
raise Module.ConfigError(
'Backend(%s): Configuration error for field "%s": %s' % (cfg.instname, name, v))
cfg[name] = field
return cfg
def dump(self):
"""
Dump config in a dictionary.
:rtype: :class:`dict`
"""
settings = {}
for name, value in self.items():
settings[name] = value.dump()
return settings
def save(self, edit=True, params=None):
"""
Save backend config.
:param edit: if true, it changes config of an existing backend
:type edit: :class:`bool`
:param params: if specified, params to merge with the ones of the current object
:type params: :class:`dict`
"""
assert self.modname is not None
assert self.instname is not None
assert self.weboob is not None
dump = self.dump()
if params is not None:
dump.update(params)
self.weboob.backends_config.add_backend(self.instname, self.modname, dump, edit)
class Module(object):
"""
Base class for modules.
You may derivate it, and also all capabilities you want to implement.
:param weboob: weboob instance
:type weboob: :class:`weboob.core.ouiboube.Weboob`
:param name: name of backend
:type name: :class:`str`
:param config: configuration of backend
:type config: :class:`dict`
:param storage: storage object
:type storage: :class:`weboob.tools.storage.IStorage`
:param logger: logger
:type logger: :class:`logging.Logger`
"""
# Module name.
NAME = None
# Name of the maintainer of this module.
MAINTAINER = u''
# Email address of the maintainer.
EMAIL = ''
# Version of module (for information only).
VERSION = ''
# Description
DESCRIPTION = ''
# License of this module.
LICENSE = ''
# Configuration required for backends.
# Values must be weboob.tools.value.Value objects.
CONFIG = BackendConfig()
# Storage
STORAGE = {}
# Browser class
BROWSER = None
# URL to an optional icon.
# If you want to create your own icon, create a 'favicon.ico' ico in
# the module's directory, and keep the ICON value to None.
ICON = None
# Supported objects to fill
# The key is the class and the value the method to call to fill
# Method prototype: method(object, fields)
# When the method is called, fields are only the one which are
# NOT yet filled.
OBJECTS = {}
class ConfigError(Exception):
"""
Raised when the config can't be loaded.
"""
def __enter__(self):
self.lock.acquire()
def __exit__(self, t, v, tb):
self.lock.release()
def __repr__(self):
return "" % self.name
def __init__(self, weboob, name, config=None, storage=None, logger=None, nofail=False):
self.logger = getLogger(name, parent=logger)
self.weboob = weboob
self.name = name
self.lock = RLock()
if config is None:
config = {}
# Private fields (which start with '_')
self._private_config = dict((key, value) for key, value in config.items() if key.startswith('_'))
# Load configuration of backend.
self.config = self.CONFIG.load(weboob, self.NAME, self.name, config, nofail)
self.storage = BackendStorage(self.name, storage)
self.storage.load(self.STORAGE)
def dump_state(self):
if hasattr(self.browser, 'dump_state'):
self.storage.set('browser_state', self.browser.dump_state())
self.storage.save()
def deinit(self):
"""
This abstract method is called when the backend is unloaded.
"""
if self._browser is None:
return
try:
self.dump_state()
finally:
if hasattr(self.browser, 'deinit'):
self.browser.deinit()
_browser = None
@property
def browser(self):
"""
Attribute 'browser'. The browser is created at the first call
of this attribute, to avoid useless pages access.
Note that the :func:`create_default_browser` method is called to create it.
"""
if self._browser is None:
self._browser = self.create_default_browser()
return self._browser
def create_default_browser(self):
"""
Method to overload to build the default browser in
attribute 'browser'.
"""
return self.create_browser()
def create_browser(self, *args, **kwargs):
"""
Build a browser from the BROWSER class attribute and the
given arguments.
:param klass: optional parameter to give another browser class to instanciate
:type klass: :class:`weboob.browser.browsers.Browser`
"""
klass = kwargs.pop('klass', self.BROWSER)
if not klass:
return None
kwargs['proxy'] = self.get_proxy()
kwargs['logger'] = self.logger
if self.logger.settings['responses_dirname']:
kwargs.setdefault('responses_dirname', os.path.join(self.logger.settings['responses_dirname'],
self._private_config.get('_debug_dir', self.name)))
elif os.path.isabs(self._private_config.get('_debug_dir', '')):
kwargs.setdefault('responses_dirname', self._private_config['_debug_dir'])
if self._private_config.get('_highlight_el', ''):
kwargs.setdefault('highlight_el', bool(int(self._private_config['_highlight_el'])))
browser = klass(*args, **kwargs)
if hasattr(browser, 'load_state'):
browser.load_state(self.storage.get('browser_state', default={}))
return browser
def get_proxy(self):
# Get proxies from environment variables
proxies = getproxies()
# Override them with backend-specific config
if '_proxy' in self._private_config:
proxies['http'] = self._private_config['_proxy']
if '_proxy_ssl' in self._private_config:
proxies['https'] = self._private_config['_proxy_ssl']
# Remove empty values
for key in list(proxies.keys()):
if not proxies[key]:
del proxies[key]
return proxies
@classmethod
def iter_caps(klass):
"""
Iter capabilities implemented by this backend.
:rtype: iter[:class:`weboob.capabilities.base.Capability`]
"""
for base in klass.mro():
if issubclass(base, Capability) and base != Capability and base != klass:
yield base
def has_caps(self, *caps):
"""
Check if this backend implements at least one of these capabilities.
"""
for c in caps:
if (isinstance(c, basestring) and c in [cap.__name__ for cap in self.iter_caps()]) or \
isinstance(self, c):
return True
return False
def fillobj(self, obj, fields=None):
"""
Fill an object with the wanted fields.
:param fields: what fields to fill; if None, all fields are filled
:type fields: :class:`list`
"""
if obj is None:
return obj
def not_loaded_or_incomplete(v):
return (v is NotLoaded or isinstance(v, BaseObject) and not v.__iscomplete__())
def not_loaded(v):
return v is NotLoaded
def filter_missing_fields(obj, fields, check_cb):
missing_fields = []
if fields is None:
# Select all fields
if isinstance(obj, BaseObject):
fields = [item[0] for item in obj.iter_fields()]
else:
fields = [item[0] for item in iter_fields(obj)]
for field in fields:
if not hasattr(obj, field):
raise FieldNotFound(obj, field)
value = getattr(obj, field)
missing = False
if hasattr(value, '__iter__'):
for v in (value.values() if isinstance(value, dict) else value):
if check_cb(v):
missing = True
break
elif check_cb(value):
missing = True
if missing:
missing_fields.append(field)
return missing_fields
if isinstance(fields, basestring):
fields = (fields,)
missing_fields = filter_missing_fields(obj, fields, not_loaded_or_incomplete)
if not missing_fields:
return obj
for key, value in self.OBJECTS.items():
if isinstance(obj, key):
self.logger.debug(u'Fill %r with fields: %s' % (obj, missing_fields))
obj = value(self, obj, missing_fields) or obj
break
missing_fields = filter_missing_fields(obj, fields, not_loaded)
# Object is not supported by backend. Do not notice it to avoid flooding user.
# That's not so bad.
for field in missing_fields:
setattr(obj, field, NotAvailable)
return obj
class AbstractModuleMissingParentError(Exception):
pass
class AbstractModule(Module):
""" Abstract module allow inheritance between modules.
Sometimes, several websites are based on the same website engine. This module
allow to simplify code with a fake inheritance: weboob will install (if needed) and
load a PARENT module and build our AbstractModule on top of this class.
PARENT is a mandatory attribute of any AbstractModule
Note that you must pass a valid weboob instance as first argument of the constructor.
"""
PARENT = None
def __new__(cls, weboob, name, config=None, storage=None, logger=None, nofail=False):
if cls.PARENT is None:
raise AbstractModuleMissingParentError("PARENT is not defined for module %s" % cls.__name__)
try:
parent = weboob.load_or_install_module(cls.PARENT).klass
except ModuleInstallError as err:
raise ModuleInstallError('The module %s depends on %s module but %s\'s installation failed with: %s' % (name, cls.PARENT, cls.PARENT, err))
cls.__bases__ = tuple([parent] + list(cls.iter_caps()))
return object.__new__(cls)
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/capabilities/ 0000775 0000000 0000000 00000000000 13414137563 0026465 5 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/capabilities/__init__.py 0000664 0000000 0000000 00000000000 13414137563 0030564 0 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/capabilities/audio/ 0000775 0000000 0000000 00000000000 13414137563 0027566 5 ustar 00root root 0000000 0000000 __init__.py 0000775 0000000 0000000 00000000000 13414137563 0031611 0 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/capabilities/audio woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/capabilities/audio/audio.py 0000775 0000000 0000000 00000002521 13414137563 0031244 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2009-2015 Bezleputh
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.browser.filters.standard import Format
class AlbumIdFilter(Format):
"""
Filter that help to fill Albums id field
"""
def __init__(self, *args):
super(AlbumIdFilter, self).__init__(u'album.%s', *args)
class PlaylistIdFilter(Format):
"""
Filter that help to fill Albums id field
"""
def __init__(self, *args):
super(PlaylistIdFilter, self).__init__(u'playlist.%s', *args)
class BaseAudioIdFilter(Format):
"""
Filter that help to fill Albums id field
"""
def __init__(self, *args):
super(BaseAudioIdFilter, self).__init__(u'audio.%s', *args)
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/capabilities/bank/ 0000775 0000000 0000000 00000000000 13414137563 0027400 5 ustar 00root root 0000000 0000000 __init__.py 0000664 0000000 0000000 00000000000 13414137563 0031420 0 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/capabilities/bank woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/capabilities/bank/iban.py 0000664 0000000 0000000 00000006331 13414137563 0030666 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2016 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import re
from weboob.tools.compat import unicode
_country2length = dict(
AL=28, AD=24, AT=20, AZ=28, BE=16, BH=22, BA=20, BR=29,
BG=22, CR=21, HR=21, CY=28, CZ=24, DK=18, DO=28, EE=20,
FO=18, FI=18, FR=27, GE=22, DE=22, GI=23, GR=27, GL=18,
GT=28, HU=28, IS=26, IE=22, IL=23, IT=27, KZ=20, KW=30,
LV=21, LB=28, LI=21, LT=20, LU=20, MK=19, MT=31, MR=27,
MU=30, MC=27, MD=24, ME=22, NL=18, NO=15, PK=24, PS=29,
PL=28, PT=25, RO=24, SM=27, SA=24, RS=22, SK=24, SI=19,
ES=24, SE=24, CH=21, TN=24, TR=26, AE=23, GB=22, VG=24,
MA=28, JO=30, TL=23, XK=20, QA=29,
)
def clean(iban):
return iban.replace(' ','').replace('\t', '')
def is_iban_valid(iban):
# Ensure upper alphanumeric input.
iban = clean(iban)
if not re.match(r'^[A-Z]{2}\d{2}[\dA-Z]+$', iban):
return False
# Validate country code against expected length.
if iban[:2] in _country2length and len(iban) != _country2length[iban[:2]]:
return False
digits = iban2numeric(iban)
return digits % 97 == 1
def iban2numeric(iban):
# Shift and convert.
iban = iban[4:] + iban[:4]
# BASE 36: 0..9,A..Z -> 0..35
digits = int(''.join(str(int(ch, 36)) for ch in iban))
return digits
def find_iban_checksum(iban):
iban = iban[:2] + '00' + iban[4:]
digits = str(iban2numeric(iban))
checksum = 0
for char in digits:
checksum *= 10
checksum += int(char)
checksum %= 97
return 98-checksum
def rebuild_iban(iban):
return unicode(iban[:2] + ('%02d' % find_iban_checksum(iban)) + iban[4:])
def rib2iban(rib):
return rebuild_iban('FR00' + rib)
def find_rib_checksum(bank, counter, account):
letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
digits = '12345678912345678923456789'
account = ''.join([(char if char.isdigit() else digits[letters.find(char.upper())]) for char in account])
rest = (89*int(bank) + 15*int(counter) + 3*int(account)) % 97
return 97 - rest
def is_rib_valid(rib):
if len(rib) != 23:
return False
return find_rib_checksum(rib[:5], rib[5:10], rib[10:21]) == int(rib[21:23])
def rebuild_rib(rib):
rib = clean(rib)
assert len(rib) >= 21
key = find_rib_checksum(rib[:5], rib[5:10], rib[10:21])
return unicode(rib[:21] + ('%02d' % key))
def test():
assert rebuild_iban('FR0013048379405300290000355') == "FR7613048379405300290000355"
assert rebuild_iban('GB87BARC20658244971655') == "GB87BARC20658244971655"
assert rebuild_rib('30003021990005077567600') == "30003021990005077567667"
investments.py 0000664 0000000 0000000 00000007114 13414137563 0032255 0 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/capabilities/bank # -*- coding: utf-8 -*-
# Copyright(C) 2017 Jonathan Schmidt
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import unicode_literals
import re
from weboob.tools.compat import basestring
from weboob.capabilities.base import NotAvailable
from weboob.capabilities.bank import Investment
def is_isin_valid(isin):
"""
Méthode générale
Table de conversion des lettres en chiffres
A=10 B=11 C=12 D=13 E=14 F=15 G=16 H=17 I=18
J=19 K=20 L=21 M=22 N=23 O=24 P=25 Q=26 R=27
S=28 T=29 U=30 V=31 W=32 X=33 Y=34 Z=35
1 - Mettre de côté la clé, qui servira de référence à la fin de la vérification.
2 - Convertir toutes les lettres en nombres via la table de conversion ci-contre. Si le nombre obtenu est supérieur ou égal à 10, prendre les deux chiffres du nombre séparément (exemple : 27 devient 2 et 7).
3 - Pour chaque chiffre, multiplier sa valeur par deux si sa position est impaire en partant de la droite. Si le nombre obtenu est supérieur ou égal à 10, garder les deux chiffres du nombre séparément (exemple : 14 devient 1 et 4).
4 - Faire la somme de tous les chiffres.
5 - Soustraire cette somme de la dizaine supérieure ou égale la plus proche (exemples : si la somme vaut 22, la dizaine « supérieure ou égale » est 30, et la clé vaut donc 8 ; si la somme vaut 30, la dizaine « supérieure ou égale » est 30, et la clé vaut 0 ; si la somme vaut 31, la dizaine « supérieure ou égale » est 40, et la clé vaut 9).
6 - Comparer la valeur obtenue à la clé mise initialement de côté.
Étapes 1 et 2 :
F R 0 0 0 3 5 0 0 0 0 (+ 8 : clé)
15 27 0 0 0 3 5 0 0 0 0
Étape 3 : le traitement se fait sur des chiffres
1 5 2 7 0 0 0 3 5 0 0 0 0
I P I P I P I P I P I P I : position en partant de la droite (P = Pair, I = Impair)
2 1 2 1 2 1 2 1 2 1 2 1 2 : coefficient multiplicateur
2 5 4 7 0 0 0 3 10 0 0 0 0 : résultat
Étape 4 :
2 + 5 + 4 + 7 + 0 + 0 + 0 + 3 + (1 + 0)+ 0 + 0 + 0 + 0 = 22
Étapes 5 et 6 : 30 - 22 = 8 (valeur de la clé)
"""
if not isinstance(isin, basestring):
return False
if not re.match(r'^[A-Z]{2}[A-Z0-9]{9}\d$', isin):
return False
isin_in_digits = ''.join(str(ord(x) - ord('A') + 10) if not x.isdigit() else x for x in isin[:-1])
key = isin[-1:]
result = ''
for k, val in enumerate(isin_in_digits[::-1], start=1):
if k % 2 == 0:
result = ''.join((result, val))
else:
result = ''.join((result, str(int(val)*2)))
return str(sum(int(x) for x in result) + int(key))[-1] == '0'
def create_french_liquidity(valuation):
"""
Automatically fills a liquidity investment with label, code and code_type.
"""
liquidity = Investment()
liquidity.label = "Liquidités"
liquidity.code = "XX-liquidity"
liquidity.code_type = NotAvailable
liquidity.valuation = valuation
return liquidity
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/capabilities/bank/test.py 0000664 0000000 0000000 00000017646 13414137563 0030747 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2017 Vincent Ardisson
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from datetime import date
from weboob.capabilities.base import empty
from weboob.capabilities.bank import CapBankTransfer, CapBankWealth, CapBankPockets
from weboob.exceptions import NoAccountsException
from weboob.tools.capabilities.bank.iban import is_iban_valid
from weboob.tools.capabilities.bank.investments import is_isin_valid
from weboob.tools.date import new_date
__all__ = ('BankStandardTest',)
class BankStandardTest(object):
"""Mixin for simple tests on CapBank backends.
This checks:
* there are accounts
* accounts have an id, a label and a balance
* history is implemented (optional)
* transactions have a date, a label and an amount
* investments are implemented (optional)
* investments have a label and a valuation
* recipients are implemented (optional)
* recipients have an id and a label
"""
allow_notimplemented_history = False
allow_notimplemented_coming = False
allow_notimplemented_investments = False
allow_notimplemented_pockets = False
allow_notimplemented_recipients = False
def test_basic(self):
try:
accounts = list(self.backend.iter_accounts())
except NoAccountsException:
return
self.assertTrue(accounts)
for account in accounts:
self.check_account(account)
try:
self.check_history(account)
except NotImplementedError:
self.assertTrue(self.allow_notimplemented_history, 'iter_history should not raise NotImplementedError')
try:
self.check_coming(account)
except NotImplementedError:
self.assertTrue(self.allow_notimplemented_coming, 'iter_coming should not raise NotImplementedError')
try:
self.check_investments(account)
except NotImplementedError:
self.assertTrue(self.allow_notimplemented_investments, 'iter_investment should not raise NotImplementedError')
try:
self.check_pockets(account)
except NotImplementedError:
self.assertTrue(self.allow_notimplemented_pockets, 'iter_pocket should not raise NotImplementedError')
try:
self.check_recipients(account)
except NotImplementedError:
self.assertTrue(self.allow_notimplemented_recipients, 'iter_transfer_recipients should not raise NotImplementedError')
def check_account(self, account):
self.assertTrue(account.id, 'account %r has no id' % account)
self.assertTrue(account.label, 'account %r has no label' % account)
self.assertFalse(empty(account.balance) and empty(account.coming), 'account %r should have balance or coming' % account)
self.assertTrue(account.type, 'account %r is untyped' % account)
self.assertTrue(account.currency, 'account %r has no currency' % account)
if account.iban:
self.assertTrue(is_iban_valid(account.iban), 'account %r IBAN is invalid: %r' % (account, account.iban))
if account.type in (account.TYPE_LOAN,):
self.assertLessEqual(account.balance, 0, 'loan %r should not have positive balance' % account)
elif account.type == account.TYPE_CHECKING:
self.assertTrue(account.iban, 'account %r has no IBAN' % account)
elif account.type == account.TYPE_CARD:
if not account.parent:
self.backend.logger.warning('card account %r has no parent account', account)
else:
self.assertEqual(account.parent.type, account.TYPE_CHECKING, 'parent account of %r should have checking type' % account)
def check_history(self, account):
for tr in self.backend.iter_history(account):
self.check_transaction(account, tr, False)
def check_coming(self, account):
for tr in self.backend.iter_coming(account):
self.check_transaction(account, tr, True)
def check_transaction(self, account, tr, coming):
today = date.today()
self.assertFalse(empty(tr.date), 'transaction %r has no debit date' % tr)
if tr.amount != 0:
self.assertTrue(tr.amount, 'transaction %r has no amount' % tr)
self.assertFalse(empty(tr.raw) and empty(tr.label), 'transaction %r has no raw or label' % tr)
if coming:
self.assertGreaterEqual(new_date(tr.date), today, 'coming transaction %r should be in the future' % tr)
else:
self.assertLessEqual(new_date(tr.date), today, 'history transaction %r should be in the past' % tr)
if tr.rdate:
self.assertGreaterEqual(new_date(tr.date), new_date(tr.rdate), 'transaction %r rdate should be before date' % tr)
self.assertLess(abs(tr.date.year - tr.rdate.year), 2, 'transaction %r date (%r) and rdate (%r) are too far away' % (tr, tr.date, tr.rdate))
if tr.original_amount or tr.original_currency:
self.assertTrue(tr.original_amount and tr.original_currency, 'transaction %r has missing foreign info' % tr)
for inv in (tr.investments or []):
self.assertTrue(inv.label, 'transaction %r investment %r has no label' % (tr, inv))
self.assertTrue(inv.valuation, 'transaction %r investment %r has no valuation' % (tr, inv))
def check_investments(self, account):
if not isinstance(self.backend, CapBankWealth):
return
total = 0
for inv in self.backend.iter_investment(account):
self.check_investment(account, inv)
if not empty(inv.valuation):
total += inv.valuation
if total:
self.assertEqual(total, account.balance, 'investments total (%s) is different than account balance (%s)' % (total, account.balance))
def check_investment(self, account, inv):
self.assertTrue(inv.label, 'investment %r has no label' % inv)
self.assertTrue(inv.valuation, 'investment %r has no valuation' % inv)
if inv.code and inv.code != 'XX-liquidity':
self.assertTrue(inv.code_type, 'investment %r has code but no code type' % inv)
if inv.code_type == inv.CODE_TYPE_ISIN and inv.code and not inv.code.startswith('XX'):
self.assertTrue(is_isin_valid(inv.code), 'investment %r has invalid ISIN: %r' % (inv, inv.code))
if not empty(inv.portfolio_share):
self.assertTrue(0 < inv.portfolio_share <= 1, 'investment %r has invalid portfolio_share' % inv)
def check_pockets(self, account):
if not isinstance(self.backend, CapBankPockets):
return
for pocket in self.backend.iter_pocket(account):
self.check_pocket(account, pocket)
def check_pocket(self, account, pocket):
self.assertTrue(pocket.amount, 'pocket %r has no amount' % pocket)
self.assertTrue(pocket.label, 'pocket %r has no label' % pocket)
def check_recipients(self, account):
if not isinstance(self.backend, CapBankTransfer):
return
for rcpt in self.backend.iter_transfer_recipients(account):
self.check_recipient(account, rcpt)
def check_recipient(self, account, rcpt):
self.assertTrue(rcpt.id, 'recipient %r has no id' % rcpt)
self.assertTrue(rcpt.label, 'recipient %r has no label' % rcpt)
transactions.py 0000664 0000000 0000000 00000037032 13414137563 0032410 0 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/capabilities/bank # -*- coding: utf-8 -*-
# Copyright(C) 2009-2012 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from collections import OrderedDict
from decimal import Decimal, InvalidOperation
import datetime
import re
from weboob.capabilities.bank import Transaction, Account
from weboob.capabilities import NotAvailable, NotLoaded
from weboob.tools.misc import to_unicode
from weboob.tools.log import getLogger
from weboob.tools.date import new_datetime
from weboob.exceptions import ParseError
from weboob.browser.elements import TableElement, ItemElement
from weboob.browser.filters.standard import Filter, CleanText, CleanDecimal
from weboob.browser.filters.html import TableCell
__all__ = [
'FrenchTransaction', 'AmericanTransaction',
'sorted_transactions', 'merge_iterators', 'keep_only_card_transactions',
'omit_deferred_transactions',
]
class classproperty(object):
def __init__(self, f):
self.f = f
def __get__(self, obj, owner):
return self.f(owner)
class FrenchTransaction(Transaction):
"""
Transaction with some helpers for french bank websites.
"""
PATTERNS = []
def __init__(self, id='', *args, **kwargs):
super(FrenchTransaction, self).__init__(id, *args, **kwargs)
self._logger = getLogger('FrenchTransaction')
@classmethod
def clean_amount(klass, text):
"""
Clean a string containing an amount.
"""
text = text.replace('.','').replace(',','.')
return re.sub(u'[^\d\-\.]', '', text)
def set_amount(self, credit='', debit=''):
"""
Set an amount value from a string.
Can take two strings if there are both credit and debit
columns.
"""
credit = self.clean_amount(credit)
debit = self.clean_amount(debit)
if len(debit) > 0:
self.amount = - abs(Decimal(debit))
elif len(credit) > 0:
self.amount = Decimal(credit)
else:
self.amount = Decimal('0')
def parse_date(self, date):
if date is None:
return NotAvailable
if not isinstance(date, (datetime.date, datetime.datetime)):
if date.isdigit() and len(date) == 8:
date = datetime.date(int(date[4:8]), int(date[2:4]), int(date[0:2]))
elif '/' in date:
date = datetime.date(*reversed([int(x) for x in date.split('/')]))
if not isinstance(date, (datetime.date, datetime.datetime)):
self._logger.warning('Unable to parse date %r' % date)
date = NotAvailable
elif date.year < 100:
date = date.replace(year=2000 + date.year)
return date
def parse(self, date, raw, vdate=None):
"""
Parse date and raw strings to create datetime.date objects,
determine the type of transaction, and create a simplified label
When calling this method, you should have defined patterns (in the
PATTERN class attribute) with a list containing tuples of regexp
and the associated type, for example::
PATTERNS = [(re.compile(r'^VIR(EMENT)? (?P.*)'), FrenchTransaction.TYPE_TRANSFER),
(re.compile(r'^PRLV (?P.*)'), FrenchTransaction.TYPE_ORDER),
(re.compile(r'^(?P.*) CARTE \d+ PAIEMENT CB (?P\d{2})(?P\d{2}) ?(.*)$'),
FrenchTransaction.TYPE_CARD)
]
In regexps, you can define this patterns:
* text: part of label to store in simplified label
* category: part of label representing the category
* yy, mm, dd, HH, MM: date and time parts
"""
self.date = self.parse_date(date)
self.vdate = self.parse_date(vdate)
self.rdate = self.date
self.raw = to_unicode(raw.replace(u'\n', u' ').strip())
self.category = NotAvailable
if ' ' in self.raw:
self.category, _, self.label = [part.strip() for part in self.raw.partition(' ')]
else:
self.label = self.raw
for pattern, _type in self.PATTERNS:
m = pattern.match(self.raw)
if m:
args = m.groupdict()
def inargs(key):
"""
inner function to check if a key is in args,
and is not None.
"""
return args.get(key, None) is not None
self.type = _type
labels = [args[name].strip() for name in ('text', 'text2') if inargs(name)]
if labels:
self.label = ' '.join(labels)
if inargs('category'):
self.category = args['category'].strip()
# Set date from information in raw label.
if inargs('dd') and inargs('mm'):
dd = int(args['dd'])
mm = int(args['mm'])
if inargs('yy'):
yy = int(args['yy'])
else:
d = self.date
try:
d = d.replace(month=mm, day=dd)
except ValueError:
d = d.replace(year=d.year-1, month=mm, day=dd)
yy = d.year
if d > self.date:
yy -= 1
if yy < 100:
yy += 2000
try:
if inargs('HH') and inargs('MM'):
self.rdate = datetime.datetime(yy, mm, dd, int(args['HH']), int(args['MM']))
else:
self.rdate = datetime.date(yy, mm, dd)
except ValueError as e:
self._logger.warning('Unable to date in label %r: %s' % (self.raw, e))
return
@classproperty
def TransactionElement(k):
class _TransactionElement(ItemElement):
klass = k
obj_date = klass.Date(TableCell('date'))
obj_vdate = klass.Date(TableCell('vdate', 'date'))
obj_raw = klass.Raw(TableCell('raw'))
obj_amount = klass.Amount(TableCell('credit'), TableCell('debit', default=''))
return _TransactionElement
@classproperty
def TransactionsElement(klass):
class _TransactionsElement(TableElement):
col_date = [u'Date']
col_vdate = [u'Valeur']
col_raw = [u'Opération', u'Libellé', u'Intitulé opération']
col_credit = [u'Crédit', u'Montant']
col_debit = [u'Débit']
item = klass.TransactionElement
return _TransactionsElement
class Date(CleanText):
def __call__(self, item):
date = super(FrenchTransaction.Date, self).__call__(item)
return date
def filter(self, date):
date = super(FrenchTransaction.Date, self).filter(date)
if date is None:
return NotAvailable
if not isinstance(date, (datetime.date, datetime.datetime)):
if date.isdigit() and len(date) == 8:
date = datetime.date(int(date[4:8]), int(date[2:4]), int(date[0:2]))
elif '/' in date:
date = datetime.date(*reversed([int(x) for x in date.split('/')]))
if not isinstance(date, (datetime.date, datetime.datetime)):
date = NotAvailable
elif date.year < 100:
date = date.replace(year=2000 + date.year)
return date
@classmethod
def Raw(klass, *args, **kwargs):
patterns = klass.PATTERNS
class Filter(CleanText):
def __call__(self, item):
raw = super(Filter, self).__call__(item)
if item.obj.rdate is NotLoaded:
item.obj.rdate = item.obj.date
item.obj.category = NotAvailable
if ' ' in raw:
item.obj.category, useless, item.obj.label = [part.strip() for part in raw.partition(' ')]
else:
item.obj.label = raw
for pattern, _type in patterns:
m = pattern.match(raw)
if m:
args = m.groupdict()
def inargs(key):
"""
inner function to check if a key is in args,
and is not None.
"""
return args.get(key, None) is not None
item.obj.type = _type
labels = [args[name].strip() for name in ('text', 'text2') if inargs(name)]
if labels:
item.obj.label = ' '.join(labels)
if inargs('category'):
item.obj.category = args['category'].strip()
# Set date from information in raw label.
if inargs('dd') and inargs('mm'):
dd = int(args['dd']) if args['dd'] != '00' else 1
mm = int(args['mm'])
if inargs('yy'):
yy = int(args['yy'])
else:
d = item.obj.date
try:
d = d.replace(month=mm, day=dd)
except ValueError:
d = d.replace(year=d.year-1, month=mm, day=dd)
yy = d.year
if d > item.obj.date:
yy -= 1
if yy < 100:
yy += 2000
try:
if inargs('HH') and inargs('MM'):
item.obj.rdate = datetime.datetime(yy, mm, dd, int(args['HH']), int(args['MM']))
else:
item.obj.rdate = datetime.date(yy, mm, dd)
except ValueError as e:
raise ParseError('Unable to parse date in label %r: %s' % (raw, e))
break
return raw
def filter(self, text):
text = super(Filter, self).filter(text)
return to_unicode(text.replace(u'\n', u' ').strip())
return Filter(*args, **kwargs)
class Currency(CleanText):
def filter(self, text):
text = super(FrenchTransaction.Currency, self).filter(text)
return Account.get_currency(text)
class Amount(Filter):
def __init__(self, credit, debit=None, replace_dots=True):
self.credit_selector = credit
self.debit_selector = debit
self.replace_dots = replace_dots
def __call__(self, item):
if self.debit_selector:
try:
return - abs(CleanDecimal(self.debit_selector, replace_dots=self.replace_dots)(item))
except InvalidOperation:
pass
if self.credit_selector:
try:
return CleanDecimal(self.credit_selector, replace_dots=self.replace_dots)(item)
except InvalidOperation:
pass
return Decimal('0')
class AmericanTransaction(Transaction):
"""
Transaction with some helpers for american bank websites.
"""
@classmethod
def clean_amount(klass, text):
"""
Clean a string containing an amount.
"""
# Convert "American" UUU.CC format to "French" UUU,CC format
if re.search(r'\d\.\d\d(?: [A-Z]+)?$', text):
text = text.replace(',', ' ').replace('.', ',')
return FrenchTransaction.clean_amount(text)
@classmethod
def decimal_amount(klass, text):
"""
Convert a string containing an amount to Decimal.
"""
amnt = AmericanTransaction.clean_amount(text)
return Decimal(amnt) if amnt else Decimal('0')
def sorted_transactions(iterable):
"""Sort an iterable of transactions in reverse chronological order"""
return sorted(iterable, reverse=True, key=lambda tr: (tr.date, new_datetime(tr.rdate) if tr.rdate else datetime.datetime.min))
def merge_iterators(*iterables):
"""Merge transactions iterators keeping sort order.
Each iterator must already be sorted in reverse chronological order.
"""
def keyfunc(kv):
return (kv[1].date, kv[1].rdate)
its = OrderedDict((iter(it), None) for it in iterables)
for k in list(its):
try:
its[k] = next(k)
except StopIteration:
del its[k]
while its:
k, v = max(its.items(), key=keyfunc)
yield v
try:
its[k] = next(k)
except StopIteration:
del its[k]
def keep_only_card_transactions(it, match_func=None):
"""Filter iterator to keep transactions with card types.
This helper should typically be used when a banking site returns card and non-card
transactions mixed on the same checking account.
Types kept are `TYPE_DEFERRED_CARD` and `TYPE_CARD_SUMMARY`.
Additionally, the amount is inversed for transactions with type `TYPE_CARD_SUMMARY`.
This is because on the deferred debit card account, summaries should be positive
as the amount is debitted from checking account to credit the card account.
The `match_func` can be provided in case of multiple cards, to only return
transactions of one card.
:param match_func: optional function to filter transactions further
:type match_func: callable or None
"""
for tr in it:
if tr.type == tr.TYPE_DEFERRED_CARD:
if match_func is None or match_func(tr):
yield tr
elif tr.type == tr.TYPE_CARD_SUMMARY:
if match_func is None or match_func(tr):
tr.amount = -tr.amount
yield tr
def omit_deferred_transactions(it):
"""Filter iterator to omit transactions with type `TYPE_DEFERRED_CARD`.
This helper should typically be used when a banking site returns card and non-card
transactions mixed on the same checking account.
"""
for tr in it:
if tr.type != tr.TYPE_DEFERRED_CARD:
yield tr
def test():
clean_amount = AmericanTransaction.clean_amount
assert clean_amount('42') == '42'
assert clean_amount('42,12') == '42.12'
assert clean_amount('42.12') == '42.12'
assert clean_amount('$42.12 USD') == '42.12'
assert clean_amount('$12.442,12 USD') == '12442.12'
assert clean_amount('$12,442.12 USD') == '12442.12'
decimal_amount = AmericanTransaction.decimal_amount
assert decimal_amount('$12,442.12 USD') == Decimal('12442.12')
assert decimal_amount('') == Decimal('0')
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/capabilities/housing/ 0000775 0000000 0000000 00000000000 13414137563 0030141 5 ustar 00root root 0000000 0000000 __init__.py 0000775 0000000 0000000 00000000000 13414137563 0032164 0 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/capabilities/housing housing.py 0000775 0000000 0000000 00000002414 13414137563 0032114 0 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/capabilities/housing # -*- coding: utf-8 -*-
# Copyright(C) 2009-2015 Bezleputh
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.browser.filters.standard import _Filter, Field, debug
from weboob.capabilities.base import empty
from decimal import Decimal
class PricePerMeterFilter(_Filter):
"""
Filter that help to fill PricePerMeter field
"""
def __init__(self):
super(PricePerMeterFilter, self).__init__()
@debug()
def __call__(self, item):
cost = Field('cost')(item)
area = Field('area')(item)
if not (empty(cost) or empty(area)):
return Decimal(cost or 0) / Decimal(area or 1)
return Decimal(0)
housing_test.py 0000775 0000000 0000000 00000013535 13414137563 0033161 0 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/capabilities/housing # -*- coding: utf-8 -*-
# Copyright(C) 2018 Phyks
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import unicode_literals
import itertools
from collections import Counter
from weboob.capabilities.base import empty
from weboob.capabilities.housing import POSTS_TYPES
class HousingTest(object):
"""
Testing class to standardize the housing modules tests.
"""
# Fields to be checked for values across all items in housings list
FIELDS_ALL_HOUSINGS_LIST = [
"id", "type", "advert_type", "house_type", "url", "title", "area",
"cost", "currency", "utilities", "date", "location", "station", "text",
"phone", "rooms", "bedrooms", "DPE", "GES", "details"
]
# Fields to be checked for at least one item in housings list
FIELDS_ANY_HOUSINGS_LIST = [
"photos"
]
# Fields to be checked for values across all items when querying
# individually
FIELDS_ALL_SINGLE_HOUSING = [
"id", "url", "type", "advert_type", "house_type", "title", "area",
"cost", "currency", "utilities", "date", "location", "station", "text",
"phone", "rooms", "bedrooms", "DPE", "GES", "details"
]
# Fields to be checked for values at least once for all items when querying
# individually
FIELDS_ANY_SINGLE_HOUSING = [
"photos"
]
# Some backends cannot distinguish between rent and furnished rent for
# single housing post. Set this to True if this is the case.
DO_NOT_DISTINGUISH_FURNISHED_RENT = False
def assertNotEmpty(self, obj, field):
self.assertFalse(
empty(getattr(obj, field)),
'Field "%s" is empty and should not be.' % field
)
def check_housing_lists(self, query):
results = list(itertools.islice(
self.backend.search_housings(query),
20
))
self.assertGreater(len(results), 0)
for field in self.FIELDS_ANY_HOUSINGS_LIST:
self.assertTrue(
any(not empty(getattr(x, field)) for x in results),
'Missing a "%s" field.' % field
)
for x in results:
if 'type' in self.FIELDS_ALL_HOUSINGS_LIST:
self.assertEqual(x.type, query.type)
if 'advert_type' in self.FIELDS_ALL_HOUSINGS_LIST:
self.assertIn(x.advert_type, query.advert_types)
if 'house_type' in self.FIELDS_ALL_HOUSINGS_LIST:
self.assertIn(x.house_type, query.house_types)
for field in self.FIELDS_ALL_HOUSINGS_LIST:
self.assertNotEmpty(x, field)
if not empty(x.cost):
self.assertNotEmpty(x, 'price_per_meter')
for photo in x.photos:
self.assertRegexpMatches(photo.url, r'^http(s?)://')
return results
def check_single_housing_all(self, housing,
type, house_types, advert_type):
for field in self.FIELDS_ALL_SINGLE_HOUSING:
self.assertNotEmpty(housing, field)
if 'type' in self.FIELDS_ALL_SINGLE_HOUSING:
if (
self.DO_NOT_DISTINGUISH_FURNISHED_RENT and
type in [POSTS_TYPES.RENT, POSTS_TYPES.FURNISHED_RENT]
):
self.assertIn(housing.type,
[POSTS_TYPES.RENT, POSTS_TYPES.FURNISHED_RENT])
else:
self.assertEqual(housing.type, type)
if 'house_type' in self.FIELDS_ALL_SINGLE_HOUSING:
if not empty(house_types):
self.assertEqual(housing.house_type, house_types)
else:
self.assertNotEmpty(housing, 'house_type')
if 'advert_type' in self.FIELDS_ALL_SINGLE_HOUSING:
self.assertEqual(housing.advert_type, advert_type)
def check_single_housing_any(self, housing, counter):
for field in self.FIELDS_ANY_SINGLE_HOUSING:
if not empty(getattr(housing, field)):
counter[field] += 1
for photo in housing.photos:
self.assertRegexpMatches(photo.url, r'^http(s?)://')
return counter
def check_against_query(self, query):
# Check housing listing results
results = self.check_housing_lists(query)
# Check mandatory fields in all housings
housing = self.backend.get_housing(results[0].id)
self.backend.fillobj(housing, 'phone') # Fetch phone
self.check_single_housing_all(
housing,
results[0].type,
results[0].house_type,
results[0].advert_type
)
# Check fields that should appear in at least one housing
counter = Counter()
counter = self.check_single_housing_any(housing, counter)
for result in results[1:]:
if all(counter[field] > 0 for field in
self.FIELDS_ANY_SINGLE_HOUSING):
break
housing = self.backend.get_housing(result.id)
self.backend.fillobj(housing, 'phone') # Fetch phone
counter = self.check_single_housing_any(housing, counter)
for field in self.FIELDS_ANY_SINGLE_HOUSING:
self.assertGreater(
counter[field],
0,
'Optional field "%s" should appear at least once.' % field
)
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/capabilities/paste.py 0000664 0000000 0000000 00000007510 13414137563 0030156 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from base64 import b64decode, b64encode
import binascii
from weboob.capabilities.paste import CapPaste
class BasePasteModule(CapPaste):
EXPIRATIONS = {}
"""
List of expirations and their corresponding remote codes (any type can be used).
The expirations, i.e. the keys, are integers representing the duration
in seconds. There also can be one False key, for the "forever" expiration.
"""
def get_closest_expiration(self, max_age):
"""
Get the expiration closest (and less or equal to) max_age (int, in seconds).
max_age set to False means we want it to never expire.
@return int or False if found, else None
"""
# "forever"
if max_age is False and False in self.EXPIRATIONS:
return max_age
# get timed expirations, longest first
expirations = sorted([e for e in self.EXPIRATIONS if e is not False], reverse=True)
# find the first expiration that is below or equal to the maximum wanted age
for e in expirations:
if max_age is False or max_age >= e:
return e
def image_mime(data_base64, supported_formats=('gif', 'jpeg', 'png')):
"""
Return the MIME type of an image or None.
:param data_base64: data to detect, base64 encoded
:type data_base64: str
:param supported_formats: restrict list of formats to test
"""
try:
beginning = b64decode(data_base64[:24])
except binascii.Error:
return None
if 'gif' in supported_formats and b'GIF8' in beginning:
return 'image/gif'
elif 'jpeg' in supported_formats and b'JFIF' in beginning:
return 'image/jpeg'
elif 'png' in supported_formats and b'\x89PNG' in beginning:
return 'image/png'
elif 'xcf' in supported_formats and b'gimp xcf' in beginning:
return 'image/x-xcf'
elif 'pdf' in supported_formats and b'%PDF' in beginning:
return 'application/pdf'
elif 'tiff' in supported_formats and (b'II\x00\x2a' in beginning or
b'MM\x2a\x00' in beginning):
return 'image/tiff'
def bin_to_b64(b):
return b64encode(b).decode('ascii')
def test():
class MockPasteModule(BasePasteModule):
def __init__(self, expirations):
self.EXPIRATIONS = expirations
# all expirations are too high
assert MockPasteModule({1337: '', 42: '', False: ''}).get_closest_expiration(1) is None
# we found a suitable lower or equal expiration
assert MockPasteModule({1337: '', 42: '', False: ''}).get_closest_expiration(84) == 42
assert MockPasteModule({1337: '', 42: '', False: ''}).get_closest_expiration(False) is False
assert MockPasteModule({1337: '', 42: ''}).get_closest_expiration(False) == 1337
assert MockPasteModule({1337: '', 42: '', False: ''}).get_closest_expiration(1336) == 42
assert MockPasteModule({1337: '', 42: '', False: ''}).get_closest_expiration(1337) == 1337
assert MockPasteModule({1337: '', 42: '', False: ''}).get_closest_expiration(1338) == 1337
# this format should work, though of doubtful usage
assert MockPasteModule([1337, 42, False]).get_closest_expiration(84) == 42
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/capabilities/recipe.py 0000664 0000000 0000000 00000010627 13414137563 0030314 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2018 Julien Veyssier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import unicode_literals
import base64
import re
import lxml.etree as ET
import requests
from weboob.capabilities.base import empty
__all__ = ['recipe_to_krecipes_xml']
def recipe_to_krecipes_xml(recipe, author=None):
"""
Export recipe to KRecipes XML string
"""
sauthor = u''
if not empty(recipe.author):
sauthor += '%s@' % recipe.author
if author is None:
sauthor += 'Cookboob'
else:
sauthor += author
header = u'\n'
initial_xml = '''\
'''
doc = ET.fromstring(initial_xml)
xrecipe = doc.find('krecipes-recipe')
desc = ET.SubElement(xrecipe, 'krecipes-description')
title = ET.SubElement(desc, 'title')
title.text = recipe.title
authors = ET.SubElement(desc, 'author')
authors.text = sauthor
eyield = ET.SubElement(desc, 'yield')
if not empty(recipe.nb_person):
amount = ET.SubElement(eyield, 'amount')
if len(recipe.nb_person) == 1:
amount.text = '%s' % recipe.nb_person[0]
else:
mini = ET.SubElement(amount, 'min')
mini.text = u'%s' % recipe.nb_person[0]
maxi = ET.SubElement(amount, 'max')
maxi.text = u'%s' % recipe.nb_person[1]
etype = ET.SubElement(eyield, 'type')
etype.text = 'persons'
if not empty(recipe.preparation_time):
preptime = ET.SubElement(desc, 'preparation-time')
preptime.text = '%02d:%02d' % (recipe.preparation_time / 60, recipe.preparation_time % 60)
if recipe.picture and recipe.picture.url:
data = requests.get(recipe.picture.url).content
datab64 = base64.encodestring(data)[:-1]
pictures = ET.SubElement(desc, 'pictures')
pic = ET.SubElement(pictures, 'pic', {'format': 'JPEG', 'id': '1'})
pic.text = ET.CDATA(datab64)
if not empty(recipe.ingredients):
ings = ET.SubElement(xrecipe, 'krecipes-ingredients')
pat = re.compile('^[0-9,.]*')
for i in recipe.ingredients:
sname = u'%s' % i
samount = ''
sunit = ''
first_nums = pat.match(i).group()
if first_nums != '':
samount = first_nums
sname = i.lstrip('0123456789 ')
ing = ET.SubElement(ings, 'ingredient')
am = ET.SubElement(ing, 'amount')
am.text = samount
unit = ET.SubElement(ing, 'unit')
unit.text = sunit
name = ET.SubElement(ing, 'name')
name.text = sname
if not empty(recipe.instructions):
instructions = ET.SubElement(xrecipe, 'krecipes-instructions')
instructions.text = recipe.instructions
if not empty(recipe.comments):
ratings = ET.SubElement(xrecipe, 'krecipes-ratings')
for c in recipe.comments:
rating = ET.SubElement(ratings, 'rating')
if c.author:
rater = ET.SubElement(rating, 'rater')
rater.text = c.author
if c.text:
com = ET.SubElement(rating, 'comment')
com.text = c.text
crits = ET.SubElement(rating, 'criterion')
if c.rate:
crit = ET.SubElement(crits, 'criteria')
critname = ET.SubElement(crit, 'name')
critname.text = 'Overall'
critstars = ET.SubElement(crit, 'stars')
critstars.text = c.rate.split('/')[0]
return header + ET.tostring(doc, encoding='UTF-8', pretty_print=True).decode('utf-8')
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/capabilities/streaminfo.py 0000664 0000000 0000000 00000002353 13414137563 0031211 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2013 Pierre Mazière
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.capabilities.base import BaseObject, StringField, NotLoaded
__all__ = ['StreamInfo']
class StreamInfo(BaseObject):
"""
Stream related information.
"""
who = StringField('Who is currently on air')
what = StringField('What is currently on air')
def __iscomplete__(self):
return self.who is not NotLoaded or self.what is not NotLoaded
def __unicode__(self):
if self.who:
return u'%s - %s' % (self.who, self.what)
else:
return self.what
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/capabilities/video/ 0000775 0000000 0000000 00000000000 13414137563 0027573 5 ustar 00root root 0000000 0000000 __init__.py 0000664 0000000 0000000 00000000000 13414137563 0031613 0 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/capabilities/video woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/capabilities/video/ytdl.py 0000664 0000000 0000000 00000004427 13414137563 0031130 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2017 Vincent A
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import os
import subprocess
from weboob.capabilities.base import NotAvailable
from weboob.capabilities.image import Thumbnail
from weboob.capabilities.video import BaseVideo
from weboob.tools.date import parse_date
from weboob.tools.json import json
from weboob.tools.application.media_player import MediaPlayer
from weboob.tools.compat import unicode
from datetime import timedelta
__all__ = ('video_info',)
def video_info(url):
"""Fetch info about a video using youtube-dl
:param url: URL of the web page containing the video
:rtype: :class:`weboob.capabilities.video.Video`
"""
if not MediaPlayer._find_in_path(os.environ['PATH'], 'youtube-dl'):
raise Exception('Please install youtube-dl')
try:
j = json.loads(subprocess.check_output(['youtube-dl', '-f', 'best', '-J', url]))
except subprocess.CalledProcessError:
return
v = BaseVideo(id=url)
v.title = unicode(j.get('title')) if j.get('title') else NotAvailable
v.ext = unicode(j.get('ext')) if j.get('ext') else NotAvailable
v.description = unicode(j.get('description')) if j.get('description') else NotAvailable
v.url = unicode(j['url'])
v.duration = timedelta(seconds=j.get('duration')) if j.get('duration') else NotAvailable
v.author = unicode(j.get('uploader')) if j.get('uploader') else NotAvailable
v.rating = j.get('average_rating') or NotAvailable
if j.get('thumbnail'):
v.thumbnail = Thumbnail(unicode(j['thumbnail']))
d = j.get('upload_date', j.get('release_date'))
if d:
v.date = parse_date(d)
return v
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/captcha/ 0000775 0000000 0000000 00000000000 13414137563 0025437 5 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/captcha/__init__.py 0000664 0000000 0000000 00000000000 13414137563 0027536 0 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/captcha/virtkeyboard.py 0000664 0000000 0000000 00000043402 13414137563 0030521 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2011 Pierre Mazière
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import division
import hashlib
import tempfile
try:
from PIL import Image
except ImportError:
raise ImportError('Please install python-imaging')
from weboob.tools.compat import basestring
class VirtKeyboardError(Exception):
pass
class VirtKeyboard(object):
"""
Handle a virtual keyboard.
:attribute margin: Margin used by :meth:`get_symbol_coords` to reduce size
of each "key" of the virtual keyboard. This attribute is always
converted to a 4-tuple, and has the same semantic as the CSS
``margin`` property (top, right, bottom, right), in pixels.
:type margin: int or float or (2|3|4)-tuple
"""
margin = None
codesep = ''
"""Output separator between code strings.
See :func:`get_string_code`.
"""
def __init__(self, file=None, coords=None, color=None, convert=None):
# file: virtual keyboard image
# coords: dictionary :
# color: color of the symbols in the image
# depending on the image, it can be a single value or a tuple
# convert: if not None, convert image to this target type (for example 'RGB')
if file is not None:
assert color, 'No color provided !'
self.load_image(file, color, convert)
if type(self.margin) in (int, float):
self.margin = (self.margin,) * 4
elif self.margin is not None:
if len(self.margin) == 2:
self.margin = self.margin + self.margin
elif len(self.margin) == 3:
self.margin = self.margin + (self.margin[1],)
assert len(self.margin) == 4
if coords is not None:
self.load_symbols(coords)
def load_image(self, file, color, convert=None):
self.image = Image.open(file)
if convert is not None:
self.image = self.image.convert(convert)
self.bands = self.image.getbands()
if isinstance(color, int) and not isinstance(self.bands, str) and len(self.bands) != 1:
raise VirtKeyboardError("Color requires %i component but only 1 is provided"
% len(self.bands))
if not isinstance(color, int) and len(color) != len(self.bands):
raise VirtKeyboardError("Color requires %i components but %i are provided"
% (len(self.bands), len(color)))
self.color = color
self.width, self.height = self.image.size
self.pixar = self.image.load()
def load_symbols(self, coords):
self.coords = {}
self.md5 = {}
for i in coords:
coord = self.get_symbol_coords(coords[i])
if coord == (-1, -1, -1, -1):
continue
self.coords[i] = coord
self.md5[i] = self.checksum(self.coords[i])
def check_color(self, pixel):
return pixel == self.color
def get_symbol_coords(self, coords):
"""Return narrow coordinates around symbol."""
(x1, y1, x2, y2) = coords
if self.margin:
top, right, bottom, left = self.margin
x1, y1, x2, y2 = x1 + left, y1 + top, x2 - right, y2 - bottom
newY1 = -1
newY2 = -1
for y in range(y1, min(y2 + 1, self.height)):
empty_line = True
for x in range(x1, min(x2 + 1, self.width)):
if self.check_color(self.pixar[x, y]):
empty_line = False
if newY1 < 0:
newY1 = y
break
if newY1 >= 0 and not empty_line:
newY2 = y
newX1 = -1
newX2 = -1
for x in range(x1, min(x2 + 1, self.width)):
empty_column = True
for y in range(y1, min(y2 + 1, self.height)):
if self.check_color(self.pixar[x, y]):
empty_column = False
if newX1 < 0:
newX1 = x
break
if newX1 >= 0 and not empty_column:
newX2 = x
return (newX1, newY1, newX2, newY2)
def checksum(self, coords):
(x1, y1, x2, y2) = coords
s = b''
for y in range(y1, min(y2 + 1, self.height)):
for x in range(x1, min(x2 + 1, self.width)):
if self.check_color(self.pixar[x, y]):
s += b"."
else:
s += b" "
return hashlib.md5(s).hexdigest()
def get_symbol_code(self, md5sum_list):
if isinstance(md5sum_list, basestring):
md5sum_list = [md5sum_list]
for md5sum in md5sum_list:
for i in self.md5:
if md5sum == self.md5[i]:
return i
raise VirtKeyboardError('Symbol not found for hash "%s".' % md5sum)
def get_string_code(self, string):
return self.codesep.join(self.get_symbol_code(self.symbols[c]) for c in string)
def check_symbols(self, symbols, dirname):
# symbols: dictionary :
for s in symbols:
try:
self.get_symbol_code(symbols[s])
except VirtKeyboardError:
if dirname is None:
dirname = tempfile.mkdtemp(prefix='weboob_session_')
self.generate_MD5(dirname)
raise VirtKeyboardError("Symbol '%s' not found; all symbol hashes are available in %s"
% (s, dirname))
def generate_MD5(self, dir):
for i in self.coords:
width = self.coords[i][2] - self.coords[i][0] + 1
height = self.coords[i][3] - self.coords[i][1] + 1
img = Image.new(''.join(self.bands), (width, height))
matrix = img.load()
for y in range(height):
for x in range(width):
matrix[x, y] = self.pixar[self.coords[i][0] + x, self.coords[i][1] + y]
img.save(dir + "/" + self.md5[i] + ".png")
self.image.save(dir + "/image.png")
class MappedVirtKeyboard(VirtKeyboard):
def __init__(self, file, document, img_element, color, map_attr="onclick", convert=None):
map_id = img_element.attrib.get("usemap")[1:]
map = document.find('//map[@id="%s"]' % map_id)
if map is None:
map = document.find('//map[@name="%s"]' % map_id)
coords = {}
for area in map.getiterator("area"):
code = area.attrib.get(map_attr)
area_coords = []
for coord in area.attrib.get("coords").split(' ')[0].split(','):
area_coords.append(int(coord))
coords[code] = tuple(area_coords)
super(MappedVirtKeyboard, self).__init__(file, coords, color, convert)
class GridVirtKeyboard(VirtKeyboard):
"""
Make a virtual keyboard where "keys" are distributed on a grid.
For example: https://www.esgbl.com/part/fr/idehom.html
Parameters:
:param symbols: Sequence of symbols, ordered in the grid from left to
right and up to down
:type symbols: iterable
:param cols: Column count of the grid
:type cols: int
:param rows: Row count of the grid
:type rows: int
:param image: File-like object to be used as data source
:type image: file
:param color: Color of the meaningful pixels
:type color: 3-tuple
:param convert: Mode to which convert color of pixels, see
:meth:`Image.Image.convert` for more information
Attributes:
:attribute symbols: Association table between symbols and md5s
:type symbols: dict
"""
symbols = {}
def __init__(self, symbols, cols, rows, image, color, convert=None):
self.load_image(image, color, convert)
tileW = self.width / cols
tileH = self.height / rows
positions = ((s, i * tileW % self.width, i // cols * tileH)
for i, s in enumerate(symbols))
coords = dict((s, tuple(map(int, (x, y, x + tileW, y + tileH))))
for (s, x, y) in positions)
super(GridVirtKeyboard, self).__init__()
self.load_symbols(coords)
class SplitKeyboard(object):
"""Virtual keyboard for when the chars are in individual images, not a single grid"""
char_to_hash = None
"""dict mapping password characters to image hashes"""
codesep = ''
"""Output separator between symbols"""
def __init__(self, code_to_filedata):
"""Create a SplitKeyboard
:param code_to_filedata: dict mapping site codes to images data
:type code_to_filedata: dict[str, str]
"""
hash_to_code = {
self.checksum(data): code for code, data in code_to_filedata.items()
}
self.char_to_code = {}
for char, hashes in self.char_to_hash.items():
if isinstance(hashes, basestring):
hashes = (hashes,)
for hash in hash_to_code:
if hash in hashes:
self.char_to_code[char] = hash_to_code.pop(hash)
break
else:
path = tempfile.mkdtemp(prefix='weboob_session_')
self.dump(code_to_filedata.values(), path)
raise VirtKeyboardError("Symbol '%s' not found; all symbol hashes are available in %s" % (char, path))
def checksum(self, buffer):
return hashlib.md5(buffer).hexdigest()
def dump(self, files, path):
for dat in files:
md5 = hashlib.md5(dat).hexdigest()
with open('%s/%s.png' % (path, md5), 'wb') as fd:
fd.write(dat)
def get_string_code(self, password):
symbols = []
for c in password:
symbols.append(self.char_to_code[c])
return self.codesep.join(symbols)
@classmethod
def create_from_url(cls, browser, code_to_url):
code_to_file = {
code: browser.open(url).content for code, url in code_to_url
}
return cls(code_to_file)
class Tile(object):
"""Tile of a image grid for SimpleVirtualKeyboard"""
def __init__(self, matching_symbol, coords, image=None, md5=None):
self.matching_symbol = matching_symbol
self.coords = coords
self.image = image
self.md5 = md5
class SimpleVirtualKeyboard(object):
"""Handle a virtual keyboard where "keys" are distributed on a simple grid.
Parameters:
:param cols: Column count of the grid
:type cols: int
:param rows: Row count of the grid
:type rows: int
:param image: File-like object to be used as data source
:type image: file
:param convert: Mode to which convert color of pixels, see
:meth:`Image.Image.convert` for more information
:param matching_symbols: symbol that match all case of image grid from left to right and top
to down, European reading way.
:type matching_symbols: iterable
:param matching_symbols_coords: dict mapping matching website symbols to their image coords
(x0, y0, x1, y1) on grid image from left to right and top to
down, European reading way. It's not symbols in the image.
:type matching_symbols_coords: dict[str:4-tuple(int)]
Attributes:
:attribute codesep: Output separator between matching symbols
:type codesep: str
:param margin: Useless image pixel to cut.
See :func:`cut_margin`.
:type margin: 4-tuple(int), same as HTML margin: (top, right, bottom, left).
or 2-tuple(int), (top = bottom, right = left),
or int, top = right = bottom = left
:attribute tile_margin: Useless tile pixel to cut.
See :func:`cut_margin`.
:attribute symbols: Association table between image symbols and md5s
:type symbols: dict[str:str] or dict[str:n-tuple(str)]
:attribute convert: Mode to which convert color of pixels, see
:meth:`Image.Image.convert` for more information
:attribute alter: Allow custom main image alteration. Then overwrite :func:`alter_image`.
:type alter: boolean
"""
codesep = ''
margin = None
tile_margin = None
symbols = None
convert = None
def __init__(self, file, cols, rows, matching_symbols=None, matching_symbols_coords=None):
self.cols = cols
self.rows = rows
self.path = tempfile.mkdtemp(prefix='weboob_session_')
# Get self.image
self.load_image(file, self.margin, self.convert)
# Get self.tiles
self.get_tiles( matching_symbols=matching_symbols,
matching_symbols_coords=matching_symbols_coords)
# Tiles processing
self.cut_tiles(self.tile_margin)
self.hash_md5_tiles()
def load_image(self, file, margin=None, convert=None):
self.image = Image.open(file)
# Resize image if margin is given
if margin:
self.image = self.cut_margin(self.image, margin)
# Give possibility to alter image before get tiles, overwrite :func:`alter_image`.
self.alter_image()
if convert:
self.image = self.image.convert(convert)
self.width, self.height = self.image.size
def alter_image(self):
pass
def cut_margin(self, image, margin):
width, height = image.size
# Verify the magin value format
if type(margin) is int:
margin = (margin, margin, margin, margin)
elif len(margin) == 2:
margin = (margin[0], margin[1], margin[0], margin[1])
elif len(margin) == 4:
margin = margin
else:
assert (len(margin) == 3) & (len(margin) > 4), \
"Margin format is wrong."
assert ((margin[0] + margin[2]) < height) & ((margin[1] + margin[3]) < width), \
"Margin is too high, there is not enough pixel to cut."
image = image.crop((0 + margin[3],
0 + margin[0],
width - margin[1],
height - margin[2]
))
return image
def get_tiles(self, matching_symbols=None, matching_symbols_coords=None):
self.tiles = []
# Tiles coords are given
if matching_symbols_coords:
for matching_symbol in matching_symbols_coords:
self.tiles.append(Tile( matching_symbol=matching_symbol,
coords=matching_symbols_coords[matching_symbol]
))
return
assert (not self.width%self.cols) & (not self.height%self.rows), \
"Image width and height are not multiple of cols and rows. Please resize image with attribute `margin`."
# Tiles coords aren't given, calculate them
self.tileW = self.width // self.cols
self.tileH = self.height // self.rows
# Matching symbols aren't given, default value is range(columns*rows)
if not matching_symbols:
matching_symbols = ['%s' % i for i in range(self.cols*self.rows)]
assert len(matching_symbols) == (self.cols*self.rows), \
"Number of website matching symbols is not equal to the number of cases on the image."
# Calculate tiles coords for each matching symbol from 1-dimension to 2-dimensions
for index, matching_symbol in enumerate(matching_symbols):
coords = self.get_tile_coords_in_grid(index)
self.tiles.append(Tile(matching_symbol=matching_symbol, coords=coords))
def get_tile_coords_in_grid(self, case_index):
lenH = self.tileH * case_index
# Get the top left pixel coords of the tile
x0 = self.tileW * case_index%self.width
y0 = self.tileH * ((lenH - lenH%self.width) // self.width)
# Get the bottom right coords of the tile
x1 = x0 + self.tileW
y1 = y0 + self.tileH
coords = (x0, y0, x1, y1)
return(coords)
def cut_tiles(self, tile_margin=None):
for tile in self.tiles:
tile.image = self.image.crop(tile.coords)
# Resize tile if margin is given
if tile_margin:
for tile in self.tiles:
tile.image = self.cut_margin(tile.image, tile_margin)
def hash_md5_tiles(self):
for tile in self.tiles:
tile.md5 = hashlib.md5(tile.image.tobytes()).hexdigest()
def dump_tiles(self, path):
for tile in self.tiles:
tile.image.save('{}/{}.png'.format(path, tile.md5))
def get_string_code(self, password):
word = []
for digit in password:
for tile in self.tiles:
if tile.md5 in self.symbols[digit]:
word.append(tile.matching_symbol)
break
else:
# Dump file only if the symbol is not found
self.dump_tiles(self.path)
raise VirtKeyboardError("Symbol '%s' not found; all symbol hashes are available in %s"
% (digit, self.path))
return self.codesep.join(word)
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/compat.py 0000664 0000000 0000000 00000012373 13414137563 0025677 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2014 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import sys
import pickle
__all__ = ['unicode', 'long', 'basestring', 'range',
'with_metaclass', 'unpickle',
'quote', 'quote_plus', 'unquote', 'unquote_plus',
'urlparse', 'urlunparse', 'urlsplit', 'urlunsplit',
'urlencode', 'urljoin', 'parse_qs', 'parse_qsl',
'getproxies',
]
try:
unicode = unicode
except NameError:
unicode = str
try:
long = long
except NameError:
long = int
try:
basestring = basestring
except NameError:
basestring = str
try:
range = xrange
except NameError:
range = range
try:
from future.utils import with_metaclass
except ImportError:
from six import with_metaclass
if sys.version_info.major == 2:
class StrConv(object):
def __str__(self):
if hasattr(self, '__unicode__'):
return self.__unicode__().encode('utf-8')
else:
return repr(self)
else:
class StrConv(object):
def __str__(self):
if hasattr(self, '__unicode__'):
return self.__unicode__()
else:
return repr(self)
try:
from urllib import quote as _quote, quote_plus as _quote_plus, unquote as _unquote, unquote_plus as _unquote_plus, urlencode as _urlencode, getproxies
from urlparse import urlparse, urlunparse, urljoin, urlsplit, urlunsplit, parse_qsl as _parse_qsl, parse_qs as _parse_qs
def _reencode(s):
if isinstance(s, unicode):
s = s.encode('utf-8')
return s
def quote(p, *args, **kwargs):
return _quote(_reencode(p), *args, **kwargs)
def quote_plus(p, *args, **kwargs):
return _quote_plus(_reencode(p), *args, **kwargs)
def urlencode(d, *args, **kwargs):
if hasattr(d, 'items'):
d = list(d.items())
else:
d = list(d)
d = [(_reencode(k), _reencode(v)) for k, v in d]
return _urlencode(d, *args, **kwargs)
def unquote(s):
s = _reencode(s)
return _unquote(s).decode('utf-8')
def unquote_plus(s):
s = _reencode(s)
return _unquote_plus(s).decode('utf-8')
def parse_qs(s):
s = _reencode(s)
orig = _parse_qs(s)
return {k.decode('utf-8'): [vv.decode('utf-8') for vv in v] for k, v in orig.items()}
def parse_qsl(s):
s = _reencode(s)
return [(k.decode('utf-8'), v.decode('utf-8')) for k, v in _parse_qsl(s)]
except ImportError:
from urllib.parse import (
urlparse, urlunparse, urlsplit, urlunsplit, urljoin, urlencode,
quote, quote_plus, unquote, unquote_plus, parse_qsl, parse_qs,
)
from urllib.request import getproxies
def unpickle(pickled_data):
if sys.version_info.major < 3:
pyobject = pickle.loads(pickled_data)
else: # Assuming future Python versions will not remove encoding argument
pyobject = pickle.loads(pickled_data, encoding='UTF-8')
return pyobject
def test_base():
assert type(range(3)) != list
assert type(u'') == unicode
assert type(b'') == bytes
assert isinstance(u'', basestring)
def test_url():
assert quote( 'foo=é&bar=qux ,/%') == u'foo%3D%C3%A9%26bar%3Dqux%20%2C/%25'
assert quote(u'foo=é&bar=qux ,/%') == u'foo%3D%C3%A9%26bar%3Dqux%20%2C/%25'
assert quote_plus( 'foo=é&bar=qux ,/%') == u'foo%3D%C3%A9%26bar%3Dqux+%2C%2F%25'
assert quote_plus(u'foo=é&bar=qux ,/%') == u'foo%3D%C3%A9%26bar%3Dqux+%2C%2F%25'
assert unquote( 'foo%3D%C3%A9%26bar%3Dqux%20%2C/%25') == u'foo=é&bar=qux ,/%'
assert unquote(u'foo%3D%C3%A9%26bar%3Dqux%20%2C/%25') == u'foo=é&bar=qux ,/%'
assert unquote_plus( 'foo%3D%C3%A9%26bar%3Dqux+%2C%2F%25') == u'foo=é&bar=qux ,/%'
assert unquote_plus(u'foo%3D%C3%A9%26bar%3Dqux+%2C%2F%25') == u'foo=é&bar=qux ,/%'
assert urlencode([( 'foo', u'é'), ( 'bar', 'qux ,/%')]) == u'foo=%C3%A9&bar=qux+%2C%2F%25'
assert urlencode([(u'foo', u'é'), (u'bar', u'qux ,/%')]) == u'foo=%C3%A9&bar=qux+%2C%2F%25'
assert urlencode(dict([( 'foo', u'é'), ( 'bar', 'qux ,/%')])) == u'foo=%C3%A9&bar=qux+%2C%2F%25'
assert urlencode(dict([(u'foo', u'é'), (u'bar', u'qux ,/%')])) == u'foo=%C3%A9&bar=qux+%2C%2F%25'
assert parse_qs( 'foo=%C3%A9&bar=qux+%2C%2F%25') == dict([(u'foo', [u'é']), (u'bar', [u'qux ,/%'])])
assert parse_qs(u'foo=%C3%A9&bar=qux+%2C%2F%25') == dict([(u'foo', [u'é']), (u'bar', [u'qux ,/%'])])
assert parse_qsl( 'foo=%C3%A9&bar=qux+%2C%2F%25') == [(u'foo', u'é'), (u'bar', u'qux ,/%')]
assert parse_qsl(u'foo=%C3%A9&bar=qux+%2C%2F%25') == [(u'foo', u'é'), (u'bar', u'qux ,/%')]
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/config/ 0000775 0000000 0000000 00000000000 13414137563 0025301 5 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/config/__init__.py 0000664 0000000 0000000 00000000000 13414137563 0027400 0 ustar 00root root 0000000 0000000 woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/config/iconfig.py 0000664 0000000 0000000 00000003623 13414137563 0027275 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
class ConfigError(Exception):
pass
class IConfig(object):
"""
Interface for config storage.
Config stores keys and values. Each key is a path of components, allowing
to group multiple options.
"""
def load(self, default={}):
"""
Load config.
:param default: default values for the config
:type default: dict[:class:`str`]
"""
raise NotImplementedError()
def save(self):
"""Save config."""
raise NotImplementedError()
def set(self, *args):
"""
Set a config value.
:param args: all args except the last arg are the path of the option key.
:type args: str or object
"""
raise NotImplementedError()
def delete(self, *args):
"""
Delete an option from config.
:param args: path to the option key.
:type args: str
"""
raise NotImplementedError()
def get(self, *args, **kwargs):
"""
Get the value of an option.
:param args: path of the option key.
:param default: if specified, default value when path is not found
"""
raise NotImplementedError()
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/config/iniconfig.py 0000664 0000000 0000000 00000010437 13414137563 0027625 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
try:
from configparser import RawConfigParser, DEFAULTSECT
except ImportError:
from ConfigParser import RawConfigParser, DEFAULTSECT
from collections import OrderedDict
from decimal import Decimal
import logging
import os
import io
import sys
from weboob.tools.compat import basestring, unicode
from .iconfig import IConfig
__all__ = ['INIConfig']
class INIConfig(IConfig):
ROOTSECT = 'ROOT'
def __init__(self, path):
self.path = path
self.values = OrderedDict()
self.config = RawConfigParser()
def load(self, default={}):
self.values = OrderedDict(default)
if os.path.exists(self.path):
logging.debug(u'Loading application configuration file: %s.' % self.path)
if sys.version_info.major < 3:
self.config.readfp(io.open(self.path, "r", encoding='utf-8'))
else:
self.config.read(self.path, encoding='utf-8')
for section in self.config.sections():
args = section.split(':')
if args[0] == self.ROOTSECT:
args.pop(0)
for key, value in self.config.items(section):
self.set(*(args + [key, value]))
# retro compatibility
if len(self.config.sections()) == 0:
first = True
for key, value in self.config.items(DEFAULTSECT):
if first:
logging.warning('The configuration file "%s" uses an old-style' % self.path)
logging.warning('Please rename the %s section to %s' % (DEFAULTSECT, self.ROOTSECT))
first = False
self.set(key, value)
logging.debug(u'Application configuration file loaded: %s.' % self.path)
else:
self.save()
logging.debug(u'Application configuration file created with default values: %s. '
'Please customize it.' % self.path)
return self.values
def save(self):
def save_section(values, root_section=self.ROOTSECT):
for k, v in values.items():
if isinstance(v, (int, Decimal, float, basestring)):
if not self.config.has_section(root_section):
self.config.add_section(root_section)
self.config.set(root_section, k, unicode(v))
elif isinstance(v, dict):
new_section = ':'.join((root_section, k)) if (root_section != self.ROOTSECT or k == self.ROOTSECT) else k
if not self.config.has_section(new_section):
self.config.add_section(new_section)
save_section(v, new_section)
save_section(self.values)
with io.open(self.path, 'w', encoding='utf-8') as f:
self.config.write(f)
def get(self, *args, **kwargs):
default = None
if 'default' in kwargs:
default = kwargs['default']
v = self.values
for k in args[:-1]:
if k in v:
v = v[k]
else:
return default
try:
return v[args[-1]]
except KeyError:
return default
def set(self, *args):
v = self.values
for k in args[:-2]:
if k not in v:
v[k] = OrderedDict()
v = v[k]
v[args[-2]] = args[-1]
def delete(self, *args):
v = self.values
for k in args[:-1]:
if k not in v:
return
v = v[k]
v.pop(args[-1], None)
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/config/yamlconfig.py 0000664 0000000 0000000 00000007122 13414137563 0030005 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import logging
import os
import tempfile
import sys
import weboob.tools.date
import yaml
from .iconfig import ConfigError, IConfig
try:
from yaml import CLoader as Loader
from yaml import CDumper as Dumper
except ImportError:
from yaml import Loader
from yaml import Dumper
__all__ = ['YamlConfig']
class WeboobDumper(Dumper):
pass
WeboobDumper.add_representer(weboob.tools.date.date,
WeboobDumper.represent_date)
WeboobDumper.add_representer(weboob.tools.date.datetime,
WeboobDumper.represent_datetime)
class YamlConfig(IConfig):
def __init__(self, path):
self.path = path
self.values = {}
def load(self, default={}):
self.values = default.copy()
logging.debug(u'Loading application configuration file: %s.' % self.path)
try:
with open(self.path, 'r') as f:
self.values = yaml.load(f, Loader=Loader)
logging.debug(u'Application configuration file loaded: %s.' % self.path)
except IOError:
self.save()
logging.debug(u'Application configuration file created with default values: %s. Please customize it.' % self.path)
if self.values is None:
self.values = {}
def save(self):
# write in a temporary file to avoid corruption problems
if sys.version_info.major == 2:
f = tempfile.NamedTemporaryFile(dir=os.path.dirname(self.path), delete=False)
else:
f = tempfile.NamedTemporaryFile(mode='w', dir=os.path.dirname(self.path), delete=False, encoding='utf-8')
with f:
yaml.dump(self.values, f, Dumper=WeboobDumper, default_flow_style=False)
if os.path.isfile(self.path):
os.remove(self.path)
os.rename(f.name, self.path)
def get(self, *args, **kwargs):
v = self.values
for a in args[:-1]:
try:
v = v[a]
except KeyError:
if 'default' in kwargs:
v[a] = {}
v = v[a]
else:
raise ConfigError()
except TypeError:
raise ConfigError()
try:
v = v[args[-1]]
except KeyError:
v = kwargs.get('default')
return v
def set(self, *args):
v = self.values
for a in args[:-2]:
try:
v = v[a]
except KeyError:
v[a] = {}
v = v[a]
except TypeError:
raise ConfigError()
v[args[-2]] = args[-1]
def delete(self, *args):
v = self.values
for a in args[:-1]:
try:
v = v[a]
except KeyError:
return
except TypeError:
raise ConfigError()
v.pop(args[-1], None)
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/date.py 0000664 0000000 0000000 00000035700 13414137563 0025330 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2013 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import dateutil.parser
from datetime import date as real_date, datetime as real_datetime, timedelta
import time
import re
try:
from dateutil import tz
except ImportError:
raise ImportError('Please install python-dateutil')
from .compat import range
__all__ = ['local2utc', 'utc2local', 'LinearDateGuesser', 'date', 'datetime', 'new_date', 'new_datetime', 'closest_date']
def local2utc(dateobj):
dateobj = dateobj.replace(tzinfo=tz.tzlocal())
dateobj = dateobj.astimezone(tz.tzutc())
return dateobj
def utc2local(dateobj):
dateobj = dateobj.replace(tzinfo=tz.tzutc())
dateobj = dateobj.astimezone(tz.tzlocal())
return dateobj
class date(real_date):
def strftime(self, fmt):
return strftime(self, fmt)
@classmethod
def from_date(cls, d):
return cls(d.year, d.month, d.day)
class datetime(real_datetime):
def strftime(self, fmt):
return strftime(self, fmt)
def combine(self, date, time):
return datetime(date.year, date.month, date.day, time.hour, time.minute, time.microsecond, time.tzinfo)
def date(self):
return date(self.year, self.month, self.day)
@classmethod
def from_datetime(cls, dt):
return cls(dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, dt.microsecond, dt.tzinfo)
def new_date(d):
""" Generate a safe date from a datetime.date object """
return date(d.year, d.month, d.day)
def new_datetime(d):
"""
Generate a safe datetime from a datetime.date or datetime.datetime object
"""
kw = [d.year, d.month, d.day]
if isinstance(d, real_datetime):
kw.extend([d.hour, d.minute, d.second, d.microsecond, d.tzinfo])
return datetime(*kw)
# No support for strftime's "%s" or "%y".
# Allowed if there's an even number of "%"s because they are escaped.
_illegal_formatting = re.compile(r"((^|[^%])(%%)*%[sy])")
def _findall(text, substr):
# Also finds overlaps
sites = []
i = 0
while True:
j = text.find(substr, i)
if j == -1:
break
sites.append(j)
i = j+1
return sites
def strftime(dt, fmt):
if dt.year >= 1900:
return super(type(dt), dt).strftime(fmt)
illegal_formatting = _illegal_formatting.search(fmt)
if illegal_formatting:
raise TypeError("strftime of dates before 1900 does not handle" + illegal_formatting.group(0))
year = dt.year
# For every non-leap year century, advance by
# 6 years to get into the 28-year repeat cycle
delta = 2000 - year
off = 6*(delta // 100 + delta // 400)
year = year + off
# Move to around the year 2000
year = year + ((2000 - year)//28)*28
timetuple = dt.timetuple()
s1 = time.strftime(fmt, (year,) + timetuple[1:])
sites1 = _findall(s1, str(year))
s2 = time.strftime(fmt, (year+28,) + timetuple[1:])
sites2 = _findall(s2, str(year+28))
sites = []
for site in sites1:
if site in sites2:
sites.append(site)
s = s1
syear = "%4d" % (dt.year,)
for site in sites:
s = s[:site] + syear + s[site+4:]
return s
def cmp(a, b):
return (a > b) - (a < b)
class LinearDateGuesser(object):
"""
The aim of this class is to guess the exact date object from
a day and a month, but not a year.
It works with a start date (default is today), and all dates must be
sorted from recent to older.
"""
def __init__(self, current_date=None, date_max_bump=timedelta(31)):
self.date_max_bump = date_max_bump
if current_date is None:
current_date = date.today()
self.current_date = current_date
def try_assigning_year(self, day, month, start_year, max_year):
"""
Tries to create a date object with day, month and start_year and returns
it.
If it fails due to the year not matching the day+month combination
(i.e. due to a ValueError -- TypeError and OverflowError are not
handled), the previous or next years are tried until max_year is
reached.
In case initialization still fails with max_year, this function raises
a ValueError.
"""
while True:
try:
return date(start_year, month, day)
except ValueError as e:
if start_year == max_year:
raise e
start_year += cmp(max_year, start_year)
def set_current_date(self, current_date):
self.current_date = current_date
def guess_date(self, day, month, change_current_date=True):
""" Returns a date object built from a given day/month pair. """
today = self.current_date
# The website only provides dates using the 'DD/MM' string, so we have to
# determine the most possible year by ourselves. This implies tracking
# the current date.
# However, we may also encounter "bumps" in the dates, e.g. "12/11,
# 10/11, 10/11, 12/11, 09/11", so we have to be, well, quite tolerant,
# by accepting dates in the near future (say, 7 days) of the current
# date. (Please, kill me...)
# We first try to keep the current year
naively_parsed_date = self.try_assigning_year(day, month, today.year, today.year - 5)
if (naively_parsed_date.year != today.year):
# we most likely hit a 29/02 leading to a change of year
if change_current_date:
self.set_current_date(naively_parsed_date)
return naively_parsed_date
if (naively_parsed_date > today + self.date_max_bump):
# if the date ends up too far in the future, consider it actually
# belongs to the previous year
parsed_date = date(today.year - 1, month, day)
if change_current_date:
self.set_current_date(parsed_date)
elif (naively_parsed_date > today and naively_parsed_date <= today + self.date_max_bump):
# if the date is in the near future, consider it is a bump
parsed_date = naively_parsed_date
# do not keep it as current date though
else:
# if the date is in the past, as expected, simply keep it
parsed_date = naively_parsed_date
# and make it the new current date
if change_current_date:
self.set_current_date(parsed_date)
return parsed_date
class ChaoticDateGuesser(LinearDateGuesser):
"""
This class aim to find the guess the date when you know the
day and month and the minimum year
"""
def __init__(self, min_date, current_date=None, date_max_bump=timedelta(31)):
if min_date is None:
raise ValueError("min_date is not set")
self.min_date = min_date
super(ChaoticDateGuesser, self).__init__(current_date, date_max_bump)
def guess_date(self, day, month):
"""Returns a possible date between min_date and current_date"""
parsed_date = super(ChaoticDateGuesser, self).guess_date(day, month, False)
if parsed_date >= self.min_date:
return parsed_date
else:
raise ValueError("%s is inferior to min_date %s" % (parsed_date, self.min_date))
DATE_TRANSLATE_FR = [(re.compile(u'janvier', re.I), u'january'),
(re.compile(u'f[eé]vrier', re.I | re.U), u'february'),
(re.compile(u'mars', re.I), u'march'),
(re.compile(u'avril', re.I), u'april'),
(re.compile(u'mai', re.I), u'may'),
(re.compile(u'juin', re.I), u'june'),
(re.compile(u'juillet', re.I), u'july'),
(re.compile(u'ao[uû]t?', re.I | re.U), u'august'),
(re.compile(u'septembre', re.I), u'september'),
(re.compile(u'octobre', re.I), u'october'),
(re.compile(u'novembre', re.I), u'november'),
(re.compile(u'd[eé]cembre', re.I | re.U),u'december'),
(re.compile(u'jan\\.', re.I), u'january'),
(re.compile(u'janv\\.', re.I), u'january'),
(re.compile(u'\\bjan\\b', re.I), u'january'),
(re.compile(u'f[eé]v\\.', re.I | re.U), u'february'),
(re.compile(u'f[eé]vr\\.', re.I | re.U), u'february'),
(re.compile(u'\\bf[eé]v\\b', re.I | re.U), u'february'),
(re.compile(u'avr\\.', re.I), u'april'),
(re.compile(u'\\bavr\\b', re.I), u'april'),
(re.compile(u'juil\\.', re.I), u'july'),
(re.compile(u'juill\\.', re.I), u'july'),
(re.compile(u'\\bjuil\\b', re.I),u'july'),
(re.compile(u'sep\\.', re.I), u'september'),
(re.compile(u'sept\\.', re.I), u'september'),
(re.compile(u'\\bsep\\b', re.I), u'september'),
(re.compile(u'oct\\.', re.I), u'october'),
(re.compile(u'\\boct\\b', re.I), u'october'),
(re.compile(u'nov\.', re.I), u'november'),
(re.compile(u'\\bnov\\b', re.I), u'november'),
(re.compile(u'd[eé]c\\.', re.I | re.U), u'december'),
(re.compile(u'\\bd[eé]c\\b', re.I | re.U), u'december'),
(re.compile(u'lundi', re.I), u'monday'),
(re.compile(u'mardi', re.I), u'tuesday'),
(re.compile(u'mercredi', re.I), u'wednesday'),
(re.compile(u'jeudi', re.I), u'thursday'),
(re.compile(u'vendredi', re.I), u'friday'),
(re.compile(u'samedi', re.I), u'saturday'),
(re.compile(u'dimanche', re.I), u'sunday')]
DATE_TRANSLATE_IT = [(re.compile(u'gennaio', re.I), u'january'),
(re.compile(u'febbraio', re.I), u'february'),
(re.compile(u'marzo', re.I), u'march'),
(re.compile(u'aprile', re.I), u'april'),
(re.compile(u'maggio', re.I), u'may'),
(re.compile(u'giugno', re.I), u'june'),
(re.compile(u'luglio', re.I), u'july'),
(re.compile(u'agosto', re.I), u'august'),
(re.compile(u'ago', re.I), u'august'),
(re.compile(u'settembre', re.I), u'september'),
(re.compile(u'ottobre', re.I), u'october'),
(re.compile(u'novembre', re.I), u'november'),
(re.compile(u'dicembre', re.I), u'december'),
(re.compile(u'luned[iì]', re.I | re.U), u'monday'),
(re.compile(u'marted[iì]', re.I | re.U), u'tuesday'),
(re.compile(u'mercoled[iì]', re.I | re.U), u'wednesday'),
(re.compile(u'gioved[iì]', re.I | re.U), u'thursday'),
(re.compile(u'venerd[iì]', re.I | re.U), u'friday'),
(re.compile(u'sabato', re.I), u'saturday'),
(re.compile(u'domenica', re.I), u'sunday')]
def parse_french_date(date, **kwargs):
for fr, en in DATE_TRANSLATE_FR:
date = fr.sub(en, date)
if 'dayfirst' not in kwargs:
kwargs['dayfirst'] = True
return dateutil.parser.parse(date, **kwargs)
WEEK = {'MONDAY': 0,
'TUESDAY': 1,
'WEDNESDAY': 2,
'THURSDAY': 3,
'FRIDAY': 4,
'SATURDAY': 5,
'SUNDAY': 6,
'LUNDI': 0,
'MARDI': 1,
'MERCREDI': 2,
'JEUDI': 3,
'VENDREDI': 4,
'SAMEDI': 5,
'DIMANCHE': 6,
}
def get_date_from_day(day):
today = date.today()
today_day_number = today.weekday()
requested_day_number = WEEK[day.upper()]
if today_day_number < requested_day_number:
day_to_go = requested_day_number - today_day_number
else:
day_to_go = 7 - today_day_number + requested_day_number
requested_date = today + timedelta(day_to_go)
return date(requested_date.year, requested_date.month, requested_date.day)
def parse_date(string):
matches = re.search('\s*([012]?[0-9]|3[01])\s*/\s*(0?[1-9]|1[012])\s*/?(\d{2}|\d{4})?$', string)
if matches:
year = matches.group(3)
if not year:
year = date.today().year
elif len(year) == 2:
year = 2000 + int(year)
return date(int(year), int(matches.group(2)), int(matches.group(1)))
elif string.upper() in list(WEEK.keys()):
return get_date_from_day(string)
elif string.upper() == "TODAY":
return date.today()
def closest_date(date, date_from, date_to):
"""
Adjusts year so that the date is closest to the given range.
Transactions dates in a statement usually contain only day and month.
Statement dates range have a year though.
Merge them all together to get a full transaction date.
"""
# If the date is within given range, we're done.
if date_from <= date <= date_to:
return date
dates = [real_datetime(year, date.month, date.day)
for year in range(date_from.year, date_to.year+1)]
# Ideally, pick the date within given range.
for d in dates:
if date_from <= d <= date_to:
return d
# Otherwise, return the most recent date in the past.
return min(dates, key=lambda d: abs(d-date_from))
def test():
dt = real_datetime
range1 = [dt(2012,12,20), dt(2013,1,10)]
assert closest_date(dt(2012,12,15), *range1) == dt(2012,12,15)
assert closest_date(dt(2000,12,15), *range1) == dt(2012,12,15)
assert closest_date(dt(2020,12,15), *range1) == dt(2012,12,15)
assert closest_date(dt(2013,1,15), *range1) == dt(2013,1,15)
assert closest_date(dt(2000,1,15), *range1) == dt(2013,1,15)
assert closest_date(dt(2020,1,15), *range1) == dt(2013,1,15)
assert closest_date(dt(2013,1,1), *range1) == dt(2013,1,1)
assert closest_date(dt(2000,1,1), *range1) == dt(2013,1,1)
assert closest_date(dt(2020,1,1), *range1) == dt(2013,1,1)
range2 = [dt(2012,12,20), dt(2014,1,10)]
assert closest_date(dt(2012,12,15), *range2) == dt(2013,12,15)
assert closest_date(dt(2014,1,15), *range2) == dt(2013,1,15)
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/decorators.py 0000664 0000000 0000000 00000003577 13414137563 0026567 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import logging
import time
__all__ = ['retry']
def retry(exceptions_to_check, exc_handler=None, tries=3, delay=2, backoff=2):
"""
Retry decorator
from http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
original from http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
"""
def deco_retry(f):
def f_retry(*args, **kwargs):
mtries = kwargs.pop('_tries', tries)
mdelay = kwargs.pop('_delay', delay)
while mtries > 1:
try:
return f(*args, **kwargs)
except exceptions_to_check as exc:
if exc_handler:
exc_handler(exc, **kwargs)
try:
logging.debug(u'%s, Retrying in %d seconds...' % (exc, mdelay))
except UnicodeDecodeError:
logging.debug(u'%s, Retrying in %d seconds...' % (repr(exc), mdelay))
time.sleep(mdelay)
mtries -= 1
mdelay *= backoff
return f(*args, **kwargs)
return f_retry # true decorator
return deco_retry
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/html.py 0000664 0000000 0000000 00000002212 13414137563 0025347 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2014 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from weboob.tools.compat import unicode
__all__ = ['html2text']
from html2text import HTML2Text
def html2text(html, **options):
h = HTML2Text()
defaults = dict(
unicode_snob=True,
skip_internal_links=True,
inline_links=False,
links_each_paragraph=True,
)
defaults.update(options)
for k, v in defaults.items():
setattr(h, k, v)
return unicode(h.handle(html))
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/js.py 0000664 0000000 0000000 00000006212 13414137563 0025023 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2015 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
__all__ = ['Javascript']
from weboob.tools.log import getLogger
class Javascript(object):
HEADER = """
function btoa(str) {
var buffer;
if (str instanceof Buffer) {
buffer = str;
} else {
buffer = new Buffer(str.toString(), 'binary');
}
return buffer.toString('base64');
}
function atob(str) {
return new Buffer(str, 'base64').toString('binary');
}
document = {
createAttribute: null,
styleSheets: null,
characterSet: "UTF-8",
documentElement: {}
};
history = {};
screen = {
width: 1280,
height: 800
};
var XMLHttpRequest = function() {};
XMLHttpRequest.prototype.onreadystatechange = function(){};
XMLHttpRequest.prototype.open = function(){};
XMLHttpRequest.prototype.setRequestHeader = function(){};
XMLHttpRequest.prototype.send = function(){};
/* JS code checks that some PhantomJS globals aren't defined on the
* global window object; put an empty window object, so that all these
* tests fail.
* It then tests the user agent against some known scrappers; just put
* the default Tor user agent in there.
*/
window = {
document: document,
history: history,
screen: screen,
XMLHttpRequest: XMLHttpRequest,
innerWidth: 1280,
innerHeight: 800,
close: function(){}
};
navigator = {
userAgent: "Mozilla/5.0 (X11; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0",
appName: "Netscape"
};
"""
def __init__(self, script, logger=None, domain=""):
try:
import execjs
except ImportError:
raise ImportError('Please install PyExecJS')
self.runner = execjs.get()
self.logger = getLogger('js', logger)
window_emulator = self.HEADER
if domain:
window_emulator += "document.domain = '" + domain + "';"
window_emulator += """
if (typeof(location) === "undefined") {
var location = window.location = {
host: document.domain
};
}
"""
self.ctx = self.runner.compile(window_emulator + script)
def call(self, *args, **kwargs):
retval = self.ctx.call(*args, **kwargs)
self.logger.debug('Calling %s%s = %s', args[0], args[1:], retval)
return retval
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/json.py 0000664 0000000 0000000 00000007104 13414137563 0025361 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2012 Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
# because we don't want to import this file by "import json"
from __future__ import absolute_import
from decimal import Decimal
from datetime import datetime, date, time, timedelta
__all__ = ['json', 'mini_jsonpath']
try:
# try simplejson first because it is faster
# However, note that simplejson has very different behaviors from the
# stdlib json module. In particular, it is handling Decimal in a very
# peculiar way and is not returning a string for them.
import simplejson as json
except ImportError:
# Python 2.6+ has a module similar to simplejson
import json
from weboob.capabilities.base import BaseObject, NotAvailable, NotLoaded
from weboob.tools.compat import basestring
def mini_jsonpath(node, path):
"""
Evaluates a dot separated path against JSON data. Path can contains
star wilcards. Always returns a generator.
Relates to http://goessner.net/articles/JsonPath/ but in a really basic
and simpler form.
>>> list(mini_jsonpath({"x": 95, "y": 77, "z": 68}, 'y'))
[77]
>>> list(mini_jsonpath({"x": {"y": {"z": "nested"}}}, 'x.y.z'))
['nested']
>>> list(mini_jsonpath('{"data": [{"x": "foo", "y": 13}, {"x": "bar", "y": 42}, {"x": "baz", "y": 128}]}', 'data.*.y'))
[13, 42, 128]
"""
def iterkeys(i):
return range(len(i)) if isinstance(i, list) else i
def cut(s):
p = s.split('.', 1) if s else [None]
return p + [None] if len(p) == 1 else p
if isinstance(node, basestring):
node = json.loads(node)
queue = [(node, cut(path))]
while queue:
node, (name, rest) = queue.pop(0)
if name is None:
yield node
continue
elif name == '*':
keys = iterkeys(node)
elif type(node) not in (dict, list) or name not in node:
continue
else:
keys = [int(name) if type(node) is list else name]
for k in keys:
queue.append((node[k], cut(rest)))
class WeboobEncoder(json.JSONEncoder):
"""JSON encoder class for weboob objects (and Decimal and dates)
>>> json.dumps(object, cls=WeboobEncoder)
'{"id": "1234@backend", "url": null}'
"""
def __init__(self, *args, **kwargs):
# avoid simplejson internal Decimal handling
if 'use_decimal' in kwargs:
kwargs['use_decimal'] = False
super(WeboobEncoder, self).__init__(*args, **kwargs)
def default(self, o):
if o is NotAvailable:
return None
elif o is NotLoaded:
return None
elif isinstance(o, BaseObject):
return o.to_dict()
elif isinstance(o, Decimal):
return str(o)
elif isinstance(o, (datetime, date, time)):
return o.isoformat()
elif isinstance(o, timedelta):
return o.total_seconds()
return super(WeboobEncoder, self).default(o)
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/log.py 0000664 0000000 0000000 00000004326 13414137563 0025174 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import print_function
import sys
from collections import defaultdict
from logging import addLevelName, Formatter, getLogger as _getLogger
__all__ = ['getLogger', 'createColoredFormatter', 'settings']
RESET_SEQ = "\033[0m"
COLOR_SEQ = "%s%%s" + RESET_SEQ
COLORS = {
'DEBUG': COLOR_SEQ % "\033[0;36m",
'INFO': COLOR_SEQ % "\033[32m",
'WARNING': COLOR_SEQ % "\033[1;33m",
'ERROR': COLOR_SEQ % "\033[1;31m",
'CRITICAL': COLOR_SEQ % ("\033[1;33m\033[1;41m"),
'DEBUG_FILTERS': COLOR_SEQ % "\033[0;35m",
}
DEBUG_FILTERS = 8
addLevelName(DEBUG_FILTERS, 'DEBUG_FILTERS')
# Global settings f logger.
settings = defaultdict(lambda: None)
def getLogger(name, parent=None):
if parent:
name = parent.name + '.' + name
logger = _getLogger(name)
logger.settings = settings
return logger
class ColoredFormatter(Formatter):
"""
Class written by airmind:
http://stackoverflow.com/questions/384076/how-can-i-make-the-python-logging-output-to-be-colored
"""
def format(self, record):
levelname = record.levelname
msg = Formatter.format(self, record)
if levelname in COLORS:
msg = COLORS[levelname] % msg
return msg
def createColoredFormatter(stream, format):
if (sys.platform != 'win32') and stream.isatty():
return ColoredFormatter(format)
else:
return Formatter(format)
if __name__ == '__main__':
for levelname, cs in COLORS.items():
print(cs % levelname, end=' ')
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/lrudict.py 0000664 0000000 0000000 00000002715 13414137563 0026061 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2012-2016 weboob project
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from collections import OrderedDict
__all__ = ['LimitedLRUDict', 'LRUDict']
class LRUDict(OrderedDict):
"""dict to store items in the order the keys were last added/fetched."""
def __setitem__(self, key, value):
if key in self:
del self[key]
super(LRUDict, self).__setitem__(key, value)
def __getitem__(self, key):
value = super(LRUDict, self).__getitem__(key)
self[key] = value
return value
class LimitedLRUDict(LRUDict):
"""dict to store only the N most recent items."""
max_entries = 100
def __setitem__(self, key, value):
super(LimitedLRUDict, self).__setitem__(key, value)
if len(self) > self.max_entries:
self.popitem(last=False)
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/misc.py 0000664 0000000 0000000 00000013230 13414137563 0025340 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2014 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from time import time, sleep
import locale
import os
import sys
import traceback
import types
# keep compatibility
from .compat import unicode
__all__ = ['get_backtrace', 'get_bytes_size', 'iter_fields',
'to_unicode', 'input', 'limit', 'find_exe']
def get_backtrace(empty="Empty backtrace."):
"""
Try to get backtrace as string.
Returns "Error while trying to get backtrace" on failure.
"""
try:
info = sys.exc_info()
trace = traceback.format_exception(*info)
if trace[0] != "None\n":
return "".join(trace)
except:
return "Error while trying to get backtrace"
return empty
def get_bytes_size(size, unit_name):
r"""Converts a unit and a number into a number of bytes.
>>> get_bytes_size(2, 'KB')
2048.0
"""
unit_data = {
'bytes': 1,
'KB': 1024,
'KiB': 1024,
'MB': 1024 * 1024,
'MiB': 1024 * 1024,
'GB': 1024 * 1024 * 1024,
'GiB': 1024 * 1024 * 1024,
'TB': 1024 * 1024 * 1024 * 1024,
'TiB': 1024 * 1024 * 1024 * 1024,
}
return float(size * unit_data.get(unit_name, 1))
def iter_fields(obj):
for attribute_name in dir(obj):
if attribute_name.startswith('_'):
continue
attribute = getattr(obj, attribute_name)
if not isinstance(attribute, types.MethodType):
yield attribute_name, attribute
def to_unicode(text):
r"""
>>> to_unicode('ascii') == u'ascii'
True
>>> to_unicode(u'utf\xe9'.encode('UTF-8')) == u'utf\xe9'
True
>>> to_unicode(u'unicode') == u'unicode'
True
"""
if isinstance(text, unicode):
return text
if not isinstance(text, bytes):
if sys.version_info.major >= 3:
return unicode(text)
else:
try:
return unicode(text)
except UnicodeError:
text = bytes(text)
try:
return text.decode('utf-8')
except UnicodeError:
pass
try:
return text.decode('iso-8859-15')
except UnicodeError:
pass
return text.decode('windows-1252', 'replace')
# Get Python 3 input function in Python2, impl here because of reliance on to_unicode
if sys.version_info.major >= 3:
raw_input = input # pyflakes3 is satisfied
else:
def input(prompt=''):
return raw_input(to_unicode(prompt).encode(sys.stdout.encoding or 'UTF-8'))
def guess_encoding(stdio):
try:
encoding = stdio.encoding or locale.getpreferredencoding()
except AttributeError:
encoding = None
# ASCII or ANSI is most likely a user mistake
if not encoding or encoding.lower() == 'ascii' or encoding.lower().startswith('ansi'):
encoding = 'UTF-8'
return encoding
def limit(iterator, lim):
"""Iterate on the lim first elements of iterator."""
count = 0
iterator = iter(iterator)
while count < lim:
yield next(iterator)
count += 1
def ratelimit(group, delay):
"""
Simple rate limiting.
Waits if the last call of lastlimit with this group name was less than
delay seconds ago. The rate limiting is global, shared between any instance
of the application and any call to this function sharing the same group
name. The same group name should not be used with different delays.
This function is intended to be called just before the code that should be
rate-limited.
This function is not thread-safe. For reasonably non-critical rate
limiting (like accessing a website), it should be sufficient nevertheless.
@param group [string] rate limiting group name, alphanumeric
@param delay [int] delay in seconds between each call
"""
from tempfile import gettempdir
path = os.path.join(gettempdir(), 'weboob_ratelimit.%s' % group)
while True:
try:
offset = time() - os.stat(path).st_mtime
except OSError:
with open(path, 'w'):
pass
offset = 0
if delay < offset:
break
sleep(delay - offset)
os.utime(path, None)
def find_exe(basename):
"""
Find the path to an executable by its base name (such as 'gpg').
The executable can be overriden using an environment variable in the form
`NAME_EXECUTABLE` where NAME is the specified base name in upper case.
If the environment variable is not provided, the PATH will be searched
both without and with a ".exe" suffix for Windows compatibility.
If the executable can not be found, None is returned.
"""
env_exe = os.getenv('%s_EXECUTABLE' % basename.upper())
if env_exe and os.path.exists(env_exe) and os.access(env_exe, os.X_OK):
return env_exe
paths = os.getenv('PATH', os.defpath).split(os.pathsep)
for path in paths:
for ex in (basename, basename + '.exe'):
fpath = os.path.join(path, ex)
if os.path.exists(fpath) and os.access(fpath, os.X_OK):
return fpath
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/newsfeed.py 0000664 0000000 0000000 00000005573 13414137563 0026220 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Clément Schreiner
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import datetime
try:
import feedparser
except ImportError:
raise ImportError('Please install python-feedparser')
if '5.1' > feedparser.__version__ >= '5.0':
# feedparser 5.0.x replaces this regexp on sgmllib
# and mechanize < 0.2 fails with malformed pages.
import sgmllib
import re
sgmllib.endbracket = re.compile('[<>]')
__all__ = ['Entry', 'Newsfeed']
class Entry(object):
def __init__(self, entry, rssid_func=None):
if hasattr(entry, 'id'):
self.id = entry.id
else:
self.id = None
if "link" in entry:
self.link = entry["link"]
if not self.id:
self.id = entry["link"]
else:
self.link = None
if "title" in entry:
self.title = entry["title"]
else:
self.title = None
if "author" in entry:
self.author = entry["author"]
else:
self.author = None
if "updated_parsed" in entry:
self.datetime = datetime.datetime(*entry['updated_parsed'][:7])
elif "published_parsed" in entry:
self.datetime = datetime.datetime(*entry['published_parsed'][:7])
else:
self.datetime = None
if "summary" in entry:
self.summary = entry["summary"]
else:
self.summary = None
self.content = []
if "content" in entry:
for i in entry["content"]:
self.content.append(i.value)
elif self.summary:
self.content.append(self.summary)
if "wfw_commentrss" in entry:
self.rsscomment = entry["wfw_commentrss"]
else:
self.rsscomment = None
if rssid_func:
self.id = rssid_func(self)
class Newsfeed(object):
def __init__(self, url, rssid_func=None):
self.feed = feedparser.parse(url)
self.rssid_func = rssid_func
def iter_entries(self):
for entry in self.feed['entries']:
yield Entry(entry, self.rssid_func)
def get_entry(self, id):
for entry in self.iter_entries():
if entry.id == id:
return entry
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/path.py 0000664 0000000 0000000 00000005211 13414137563 0025341 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2012 Nicolas Duhamel, Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from copy import copy
from posixpath import sep, join
from .compat import StrConv, unicode
class WorkingPath(StrConv, object):
def __init__(self):
self.split_path = []
self.previous = copy(self.split_path)
def cd1(self, user_input):
"""
Append *one* level to the current path.
This means that separators (/) will get escaped.
"""
split_path = self.get()
split_path.append(user_input)
self.location(split_path)
def location(self, split_path):
"""
Go to a new path, and store the previous path.
"""
self.previous = self.get()
self.split_path = split_path
def restore(self):
"""
Go to the previous path
"""
self.split_path, self.previous = self.previous, self.split_path
def home(self):
"""
Go to the root
"""
self.location([])
def up(self):
"""
Go up one directory
"""
self.location(self.split_path[:-1])
def get(self):
"""
Get the current working path
"""
return copy(self.split_path)
def __unicode__(self):
return join(sep, *[s.replace(u'/', u'\/') for s in self.split_path])
def test():
wp = WorkingPath()
assert wp.get() == []
assert unicode(wp) == u'/'
wp.cd1(u'lol')
assert wp.get() == [u'lol']
assert unicode(wp) == u'/lol'
wp.cd1(u'cat')
assert wp.get() == [u'lol', u'cat']
assert unicode(wp) == u'/lol/cat'
wp.restore()
assert unicode(wp) == u'/lol'
wp.home()
assert wp.get() == []
assert unicode(wp) == u'/'
wp.up()
assert wp.get() == []
assert unicode(wp) == u'/'
wp.location(['aa / aa', 'bbbb'])
assert unicode(wp) == u'/aa \/ aa/bbbb'
wp.up()
assert unicode(wp) == u'/aa \/ aa'
wp.cd1(u'héhé/hé')
assert unicode(wp) == u'/aa \/ aa/héhé\/hé'
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/pdf.py 0000664 0000000 0000000 00000037434 13414137563 0025172 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2014 Oleg Plakhotniuk
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from io import BytesIO
from collections import namedtuple
import logging
import os
import subprocess
from tempfile import mkstemp
from .compat import range
__all__ = ['decompress_pdf', 'get_pdf_rows']
def decompress_pdf(inpdf):
"""
Takes PDF file contents as a string and returns decompressed version
of the file contents, suitable for text parsing.
External dependencies:
MuPDF (http://www.mupdf.com).
"""
inh, inname = mkstemp(suffix='.pdf')
outh, outname = mkstemp(suffix='.pdf')
os.write(inh, inpdf)
os.close(inh)
os.close(outh)
subprocess.call(['mutool', 'clean', '-d', inname, outname])
with open(outname, 'rb') as f:
outpdf = f.read()
os.remove(inname)
os.remove(outname)
return outpdf
Rect = namedtuple('Rect', ('x0', 'y0', 'x1', 'y1'))
TextRect = namedtuple('TextRect', ('x0', 'y0', 'x1', 'y1', 'text'))
def almost_eq(a, b):
return abs(a - b) < 2
def lt_to_coords(obj, ltpage):
# in a pdf, 'y' coords are bottom-to-top
# in a pdf, coordinates are very often almost equal but not strictly equal
x0 = (min(obj.x0, obj.x1))
y0 = (min(ltpage.y1 - obj.y0, ltpage.y1 - obj.y1))
x1 = (max(obj.x0, obj.x1))
y1 = (max(ltpage.y1 - obj.y0, ltpage.y1 - obj.y1))
x0 = round(x0)
y0 = round(y0)
x1 = round(x1)
y1 = round(y1)
# in a pdf, straight lines are actually rects, make them as thin as possible
if almost_eq(x1, x0):
x1 = x0
if almost_eq(y1, y0):
y1 = y0
return Rect(x0, y0, x1, y1)
def lttext_to_multilines(obj, ltpage):
# text lines within 'obj' are probably the same height
x0 = (min(obj.x0, obj.x1))
y0 = (min(ltpage.y1 - obj.y0, ltpage.y1 - obj.y1))
x1 = (max(obj.x0, obj.x1))
y1 = (max(ltpage.y1 - obj.y0, ltpage.y1 - obj.y1))
lines = obj.get_text().rstrip('\n').split('\n')
h = (y1 - y0) / len(lines)
for n, line in enumerate(lines):
yield TextRect((x0), (y0 + n * h), (x1), (y0 + n * h + h), line)
# fuzzy floats to smooth comparisons because lines are actually rects
# and seemingly-contiguous lines are actually not contiguous
class ApproxFloat(float):
def __eq__(self, other):
return almost_eq(self, other)
def __ne__(self, other):
return not self == other
def __lt__(self, other):
return self - other < 0 and self != other
def __le__(self, other):
return self - other <= 0 or self == other
def __gt__(self, other):
return not self <= other
def __ge__(self, other):
return not self < other
ANGLE_VERTICAL = 0
ANGLE_HORIZONTAL = 1
ANGLE_OTHER = 2
def angle(r):
if r.x0 == r.x1:
return ANGLE_VERTICAL
elif r.y0 == r.y1:
return ANGLE_HORIZONTAL
return ANGLE_OTHER
class ApproxVecDict(dict):
# since coords are never strictly equal, search coords around
# store vectors and points
def __getitem__(self, coords):
x, y = coords
for i in (0, -1, 1):
for j in (0, -1, 1):
try:
return super(ApproxVecDict, self).__getitem__((x+i, y+j))
except KeyError:
pass
raise KeyError()
def get(self, k, v=None):
try:
return self[k]
except KeyError:
return v
class ApproxRectDict(dict):
# like ApproxVecDict, but store rects
def __getitem__(self, coords):
x0, y0, x1, y1 = coords
for i in (0, -1, 1):
for j in (0, -1, 1):
if x0 == x1:
for j2 in (0, -1, 1):
try:
return super(ApproxRectDict, self).__getitem__((x0+i, y0+j, x0+i, y1+j2))
except KeyError:
pass
elif y0 == y1:
for i2 in (0, -1, 1):
try:
return super(ApproxRectDict, self).__getitem__((x0+i, y0+j, x1+i2, y0+j))
except KeyError:
pass
else:
return super(ApproxRectDict, self).__getitem__((x0, y0, x1, y1))
raise KeyError()
def uniq_lines(lines):
new = ApproxRectDict()
for line in lines:
line = tuple(line)
try:
new[line]
except KeyError:
new[line] = None
return [Rect(*k) for k in new.keys()]
def build_rows(lines):
points = ApproxVecDict()
# for each top-left point, build tuple with lines going down and lines going right
for line in lines:
a = angle(line)
if a not in (ANGLE_HORIZONTAL, ANGLE_VERTICAL):
continue
coord = (line.x0, line.y0)
plines = points.get(coord)
if plines is None:
plines = points[coord] = tuple([] for _ in range(2))
plines[a].append(line)
boxes = ApproxVecDict()
for plines in points.values():
if not (plines[ANGLE_HORIZONTAL] and plines[ANGLE_VERTICAL]):
continue
plines[ANGLE_HORIZONTAL].sort(key=lambda l: (l.y0, l.x1))
plines[ANGLE_VERTICAL].sort(key=lambda l: (l.x0, l.y1))
for hline in plines[ANGLE_HORIZONTAL]:
try:
vparallels = points[hline.x1, hline.y0][ANGLE_VERTICAL]
except KeyError:
continue
if not vparallels:
continue
for vline in plines[ANGLE_VERTICAL]:
try:
hparallels = points[vline.x0, vline.y1][ANGLE_HORIZONTAL]
except KeyError:
continue
if not hparallels:
continue
hparallels = [hpar for hpar in hparallels if almost_eq(hpar.x1, hline.x1)]
if not hparallels:
continue
vparallels = [vpar for vpar in vparallels if almost_eq(vpar.y1, vline.y1)]
if not vparallels:
continue
assert len(hparallels) == 1 and len(vparallels) == 1
assert almost_eq(hparallels[0].y0, vparallels[0].y1)
assert almost_eq(vparallels[0].x0, hparallels[0].x1)
box = Rect(hline.x0, hline.y0, hline.x1, vline.y1)
boxes.setdefault((vline.y0, vline.y1), []).append(box)
rows = list(boxes.values())
new_rows = []
for row in rows:
row.sort(key=lambda box: box.x0)
if row:
row = [row[0]] + [c for n, c in enumerate(row[1:], 1) if row[n-1].x0 != c.x0]
new_rows.append(row)
rows = new_rows
rows.sort(key=lambda row: row[0].y0)
return rows
def find_in_table(rows, rect):
for j, row in enumerate(rows):
if ApproxFloat(row[0].y0) > rect.y1:
break
if not (ApproxFloat(row[0].y0) <= rect.y0 and ApproxFloat(row[0].y1) >= rect.y1):
continue
for i, box in enumerate(row):
if ApproxFloat(box.x0) <= rect.x0 and ApproxFloat(box.x1) >= rect.x1:
return i, j
def arrange_texts_in_rows(rows, trects):
table = [[[] for _ in row] for row in rows]
for trect in trects:
pos = find_in_table(rows, trect)
if not pos:
continue
table[pos[1]][pos[0]].append(trect.text)
return table
LOGGER = logging.getLogger('pdf')
DEBUGFILES = logging.DEBUG - 1
def get_pdf_rows(data, miner_layout=True):
"""
Takes PDF file content as string and yield table row data for each page.
For each page in the PDF, the function yields a list of rows.
Each row is a list of cells. Each cell is a list of strings present in the cell.
Note that the rows may belong to different tables.
There are no logic tables in PDF format, so this parses PDF drawing instructions
and tries to find rectangles and arrange them in rows, then arrange text in
the rectangles.
External dependencies:
PDFMiner (http://www.unixuser.org/~euske/python/pdfminer/index.html).
"""
try:
from pdfminer.pdfparser import PDFParser, PDFSyntaxError
except ImportError:
raise ImportError('Please install python-pdfminer')
try:
from pdfminer.pdfdocument import PDFDocument
from pdfminer.pdfpage import PDFPage
newapi = True
except ImportError:
from pdfminer.pdfparser import PDFDocument
newapi = False
from pdfminer.converter import PDFPageAggregator
from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
from pdfminer.layout import LAParams, LTRect, LTTextBox, LTTextLine, LTLine, LTChar, LTCurve
parser = PDFParser(BytesIO(data))
try:
if newapi:
doc = PDFDocument(parser)
else:
doc = PDFDocument()
parser.set_document(doc)
doc.set_parser(parser)
except PDFSyntaxError:
return
rsrcmgr = PDFResourceManager()
if miner_layout:
device = PDFPageAggregator(rsrcmgr, laparams=LAParams())
else:
device = PDFPageAggregator(rsrcmgr)
interpreter = PDFPageInterpreter(rsrcmgr, device)
if newapi:
pages = PDFPage.get_pages(BytesIO(data), check_extractable=True)
else:
doc.initialize()
pages = doc.get_pages()
if LOGGER.isEnabledFor(DEBUGFILES):
import tempfile
import PIL.Image as Image
import PIL.ImageDraw as ImageDraw
import random
path = tempfile.mkdtemp(prefix='pdf')
for npage, page in enumerate(pages):
LOGGER.debug('processing page %s', npage)
interpreter.process_page(page)
page_layout = device.get_result()
texts = sum([list(lttext_to_multilines(obj, page_layout)) for obj in page_layout._objs if isinstance(obj, (LTTextBox, LTTextLine, LTChar))], [])
LOGGER.debug('found %d text objects', len(texts))
if LOGGER.isEnabledFor(DEBUGFILES):
img = Image.new('RGB', (int(page.mediabox[2]), int(page.mediabox[3])), (255, 255, 255))
draw = ImageDraw.Draw(img)
for t in texts:
color = (random.randint(127, 255), random.randint(127, 255), random.randint(127, 255))
draw.rectangle((t.x0, t.y0, t.x1, t.y1), outline=color)
draw.text((t.x0, t.y0), t.text.encode('utf-8'), color)
fpath = '%s/1text-%03d.png' % (path, npage)
img.save(fpath)
LOGGER.log(DEBUGFILES, 'saved %r', fpath)
if not miner_layout:
texts.sort(key=lambda t: (t.y0, t.x0))
# TODO filter ltcurves that are not lines?
# TODO convert rects to 4 lines?
lines = [lt_to_coords(obj, page_layout) for obj in page_layout._objs if isinstance(obj, (LTRect, LTLine, LTCurve))]
LOGGER.debug('found %d lines', len(lines))
if LOGGER.isEnabledFor(DEBUGFILES):
img = Image.new('RGB', (int(page.mediabox[2]), int(page.mediabox[3])), (255, 255, 255))
draw = ImageDraw.Draw(img)
for l in lines:
color = (random.randint(127, 255), random.randint(127, 255), random.randint(127, 255))
draw.rectangle((l.x0, l.y0, l.x1, l.y1), outline=color)
fpath = '%s/2lines-%03d.png' % (path, npage)
img.save(fpath)
LOGGER.log(DEBUGFILES, 'saved %r', fpath)
lines = list(uniq_lines(lines))
LOGGER.debug('found %d unique lines', len(lines))
rows = build_rows(lines)
LOGGER.debug('built %d rows (%d boxes)', len(rows), sum(len(row) for row in rows))
if LOGGER.isEnabledFor(DEBUGFILES):
img = Image.new('RGB', (int(page.mediabox[2]), int(page.mediabox[3])), (255, 255, 255))
draw = ImageDraw.Draw(img)
for r in rows:
for b in r:
color = (random.randint(127, 255), random.randint(127, 255), random.randint(127, 255))
draw.rectangle((b.x0 + 1, b.y0 + 1, b.x1 - 1, b.y1 - 1), outline=color)
fpath = '%s/3rows-%03d.png' % (path, npage)
img.save(fpath)
LOGGER.log(DEBUGFILES, 'saved %r', fpath)
textrows = arrange_texts_in_rows(rows, texts)
LOGGER.debug('assigned %d strings', sum(sum(len(c) for c in r) for r in textrows))
if LOGGER.isEnabledFor(DEBUGFILES):
img = Image.new('RGB', (int(page.mediabox[2]), int(page.mediabox[3])), (255, 255, 255))
draw = ImageDraw.Draw(img)
for row, trow in zip(rows, textrows):
for b, tlines in zip(row, trow):
color = (random.randint(127, 255), random.randint(127, 255), random.randint(127, 255))
draw.rectangle((b.x0 + 1, b.y0 + 1, b.x1 - 1, b.y1 - 1), outline=color)
draw.text((b.x0 + 1, b.y0 + 1), '\n'.join(tlines).encode('utf-8'), color)
fpath = '%s/4cells-%03d.png' % (path, npage)
img.save(fpath)
LOGGER.log(DEBUGFILES, 'saved %r', fpath)
yield textrows
device.close()
# Export part #
def html_to_pdf(browser, url=None, data=None, extra_options=None):
"""
Convert html to PDF.
:param browser: browser instance
:param url: link to the html ressource
:param data: HTML content
:return: the document converted in PDF
:rtype: bytes
"""
try:
import pdfkit # https://pypi.python.org/pypi/pdfkit
except ImportError:
raise ImportError('Please install python-pdfkit')
assert (url or data) and not (url and data), 'Please give only url or data parameter'
callback = pdfkit.from_url if url else pdfkit.from_string
options = {}
try:
cookies = browser.session.cookies
except AttributeError:
pass
else:
options.update({
'cookie': [(cookie, value) for cookie, value in cookies.items() if value], # cookies of browser
})
if extra_options:
options.update(extra_options)
return callback(url or data, False, options=options)
# extract all text from PDF
def extract_text(data):
try:
try:
from pdfminer.pdfdocument import PDFDocument
from pdfminer.pdfpage import PDFPage
newapi = True
except ImportError:
from pdfminer.pdfparser import PDFDocument
newapi = False
from pdfminer.pdfparser import PDFParser, PDFSyntaxError
from pdfminer.converter import TextConverter
from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
except ImportError:
raise ImportError('Please install python-pdfminer to parse PDF')
else:
parser = PDFParser(BytesIO(data))
try:
if newapi:
doc = PDFDocument(parser)
else:
doc = PDFDocument()
parser.set_document(doc)
doc.set_parser(parser)
except PDFSyntaxError:
return
rsrcmgr = PDFResourceManager()
out = BytesIO()
device = TextConverter(rsrcmgr, out)
interpreter = PDFPageInterpreter(rsrcmgr, device)
if newapi:
pages = PDFPage.create_pages(doc)
else:
doc.initialize()
pages = doc.get_pages()
for page in pages:
interpreter.process_page(page)
return out.getvalue()
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/regex_helper.py 0000664 0000000 0000000 00000033665 13414137563 0027074 0 ustar 00root root 0000000 0000000 # Copyright (c) Django Software Foundation and individual contributors.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# 3. Neither the name of Django nor the names of its contributors may be used
# to endorse or promote products derived from this software without
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""
Functions for reversing a regular expression (used in reverse URL resolving).
Used internally by Django and not intended for external use.
This is not, and is not intended to be, a complete reg-exp decompiler. It
should be good enough for a large class of URLS, however.
"""
from weboob.tools.compat import basestring
# Mapping of an escape character to a representative of that class. So, e.g.,
# "\w" is replaced by "x" in a reverse URL. A value of None means to ignore
# this sequence. Any missing key is mapped to itself.
ESCAPE_MAPPINGS = {
"A": None,
"b": None,
"B": None,
"d": u"0",
"D": u"x",
"s": u" ",
"S": u"x",
"w": u"x",
"W": u"!",
"Z": None,
}
class Choice(list):
"""
Used to represent multiple possibilities at this point in a pattern string.
We use a distinguished type, rather than a list, so that the usage in the
code is clear.
"""
class Group(list):
"""
Used to represent a capturing group in the pattern string.
"""
class NonCapture(list):
"""
Used to represent a non-capturing group in the pattern string.
"""
def normalize(pattern):
"""
Given a reg-exp pattern, normalizes it to a list of forms that suffice for
reverse matching. This does the following:
(1) For any repeating sections, keeps the minimum number of occurrences
permitted (this means zero for optional groups).
(2) If an optional group includes parameters, include one occurrence of
that group (along with the zero occurrence case from step (1)).
(3) Select the first (essentially an arbitrary) element from any character
class. Select an arbitrary character for any unordered class (e.g. '.'
or '\w') in the pattern.
(4) Ignore comments and any of the reg-exp flags that won't change
what we construct ("iLmsu"). "(?x)" is an error, however.
(5) Raise an error on all other non-capturing (?...) forms (e.g.
look-ahead and look-behind matches) and any disjunctive ('|')
constructs.
Django's URLs for forward resolving are either all positional arguments or
all keyword arguments. That is assumed here, as well. Although reverse
resolving can be done using positional args when keyword args are
specified, the two cannot be mixed in the same reverse() call.
"""
# Do a linear scan to work out the special features of this pattern. The
# idea is that we scan once here and collect all the information we need to
# make future decisions.
result = []
non_capturing_groups = []
consume_next = True
pattern_iter = next_char(iter(pattern))
num_args = 0
# A "while" loop is used here because later on we need to be able to peek
# at the next character and possibly go around without consuming another
# one at the top of the loop.
try:
ch, escaped = next(pattern_iter)
except StopIteration:
return list(zip([u''], [[]]))
try:
while True:
if escaped:
result.append(ch)
elif ch == '.':
# Replace "any character" with an arbitrary representative.
result.append(u".")
elif ch == '|':
# FIXME: One day we'll should do this, but not in 1.0.
raise NotImplementedError()
elif ch == "^":
pass
elif ch == '$':
break
elif ch == ')':
# This can only be the end of a non-capturing group, since all
# other unescaped parentheses are handled by the grouping
# section later (and the full group is handled there).
#
# We regroup everything inside the capturing group so that it
# can be quantified, if necessary.
start = non_capturing_groups.pop()
inner = NonCapture(result[start:])
result = result[:start] + [inner]
elif ch == '[':
# Replace ranges with the first character in the range.
ch, escaped = next(pattern_iter)
result.append(ch)
ch, escaped = next(pattern_iter)
while escaped or ch != ']':
ch, escaped = next(pattern_iter)
elif ch == '(':
# Some kind of group.
ch, escaped = next(pattern_iter)
if ch != '?' or escaped:
# A positional group
name = "_%d" % num_args
num_args += 1
result.append(Group(((u"%%(%s)s" % name), name)))
walk_to_end(ch, pattern_iter)
else:
ch, escaped = next(pattern_iter)
if ch in "iLmsu#":
# All of these are ignorable. Walk to the end of the
# group.
walk_to_end(ch, pattern_iter)
elif ch == ':':
# Non-capturing group
non_capturing_groups.append(len(result))
elif ch != 'P':
# Anything else, other than a named group, is something
# we cannot reverse.
raise ValueError("Non-reversible reg-exp portion: '(?%s'" % ch)
else:
ch, escaped = next(pattern_iter)
if ch not in ('<', '='):
raise ValueError("Non-reversible reg-exp portion: '(?P%s'" % ch)
# We are in a named capturing group. Extra the name and
# then skip to the end.
if ch == '<':
terminal_char = '>'
# We are in a named backreference.
else:
terminal_char = ')'
name = []
ch, escaped = next(pattern_iter)
while ch != terminal_char:
name.append(ch)
ch, escaped = next(pattern_iter)
param = ''.join(name)
# Named backreferences have already consumed the
# parenthesis.
if terminal_char != ')':
result.append(Group(((u"%%(%s)s" % param), param)))
walk_to_end(ch, pattern_iter)
else:
result.append(Group(((u"%%(%s)s" % param), None)))
elif ch in "*?+{":
# Quanitifers affect the previous item in the result list.
count, ch = get_quantifier(ch, pattern_iter)
if ch:
# We had to look ahead, but it wasn't need to compute the
# quanitifer, so use this character next time around the
# main loop.
consume_next = False
if count == 0:
if contains(result[-1], Group):
# If we are quantifying a capturing group (or
# something containing such a group) and the minimum is
# zero, we must also handle the case of one occurrence
# being present. All the quantifiers (except {0,0},
# which we conveniently ignore) that have a 0 minimum
# also allow a single occurrence.
result[-1] = Choice([None, result[-1]])
else:
result.pop()
elif count > 1:
result.extend([result[-1]] * (count - 1))
else:
# Anything else is a literal.
result.append(ch)
if consume_next:
ch, escaped = next(pattern_iter)
else:
consume_next = True
except StopIteration:
pass
except NotImplementedError:
# A case of using the disjunctive form. No results for you!
return list(zip([u''], [[]]))
return list(zip(*flatten_result(result)))
def next_char(input_iter):
"""
An iterator that yields the next character from "pattern_iter", respecting
escape sequences. An escaped character is replaced by a representative of
its class (e.g. \w -> "x"). If the escaped character is one that is
skipped, it is not returned (the next character is returned instead).
Yields the next character, along with a boolean indicating whether it is a
raw (unescaped) character or not.
"""
for ch in input_iter:
if ch != '\\':
yield ch, False
continue
ch = next(input_iter)
representative = ESCAPE_MAPPINGS.get(ch, ch)
if representative is None:
continue
yield representative, True
def walk_to_end(ch, input_iter):
"""
The iterator is currently inside a capturing group. We want to walk to the
close of this group, skipping over any nested groups and handling escaped
parentheses correctly.
"""
if ch == '(':
nesting = 1
else:
nesting = 0
for ch, escaped in input_iter:
if escaped:
continue
elif ch == '(':
nesting += 1
elif ch == ')':
if not nesting:
return
nesting -= 1
def get_quantifier(ch, input_iter):
"""
Parse a quantifier from the input, where "ch" is the first character in the
quantifier.
Returns the minimum number of occurences permitted by the quantifier and
either None or the next character from the input_iter if the next character
is not part of the quantifier.
"""
if ch in '*?+':
try:
ch2, escaped = next(input_iter)
except StopIteration:
ch2 = None
if ch2 == '?':
ch2 = None
if ch == '+':
return 1, ch2
return 0, ch2
quant = []
while ch != '}':
ch, escaped = next(input_iter)
quant.append(ch)
quant = quant[:-1]
values = ''.join(quant).split(',')
# Consume the trailing '?', if necessary.
try:
ch, escaped = next(input_iter)
except StopIteration:
ch = None
if ch == '?':
ch = None
return int(values[0]), ch
def contains(source, inst):
"""
Returns True if the "source" contains an instance of "inst". False,
otherwise.
"""
if isinstance(source, inst):
return True
if isinstance(source, NonCapture):
for elt in source:
if contains(elt, inst):
return True
return False
def flatten_result(source):
"""
Turns the given source sequence into a list of reg-exp possibilities and
their arguments. Returns a list of strings and a list of argument lists.
Each of the two lists will be of the same length.
"""
if source is None:
return [u''], [[]]
if isinstance(source, Group):
if source[1] is None:
params = []
else:
params = [source[1]]
return [source[0]], [params]
result = [u'']
result_args = [[]]
pos = last = 0
for pos, elt in enumerate(source):
if isinstance(elt, basestring):
continue
piece = u''.join(source[last:pos])
if isinstance(elt, Group):
piece += elt[0]
param = elt[1]
else:
param = None
last = pos + 1
for i in range(len(result)):
result[i] += piece
if param:
result_args[i].append(param)
if isinstance(elt, (Choice, NonCapture)):
if isinstance(elt, NonCapture):
elt = [elt]
inner_result, inner_args = [], []
for item in elt:
res, args = flatten_result(item)
inner_result.extend(res)
inner_args.extend(args)
new_result = []
new_args = []
for item, args in zip(result, result_args):
for i_item, i_args in zip(inner_result, inner_args):
new_result.append(item + i_item)
new_args.append(args[:] + i_args)
result = new_result
result_args = new_args
if pos >= last:
piece = u''.join(source[last:])
for i in range(len(result)):
result[i] += piece
return result, result_args
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/storage.py 0000664 0000000 0000000 00000004376 13414137563 0026064 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from copy import deepcopy
from .config.yamlconfig import YamlConfig
class IStorage(object):
def load(self, what, name, default={}):
"""
Load data from storage.
"""
raise NotImplementedError()
def save(self, what, name):
"""
Write changes in storage on the disk.
"""
raise NotImplementedError()
def set(self, what, name, *args):
"""
Set data in a path.
"""
raise NotImplementedError()
def delete(self, what, name, *args):
"""
Delete a value or a path.
"""
raise NotImplementedError()
def get(self, what, name, *args, **kwargs):
"""
Get a value or a path.
"""
raise NotImplementedError()
class StandardStorage(IStorage):
def __init__(self, path):
self.config = YamlConfig(path)
self.config.load()
def load(self, what, name, default={}):
d = {}
if what not in self.config.values:
self.config.values[what] = {}
else:
d = self.config.values[what].get(name, {})
self.config.values[what][name] = deepcopy(default)
self.config.values[what][name].update(d)
def save(self, what, name):
self.config.save()
def set(self, what, name, *args):
self.config.set(what, name, *args)
def delete(self, what, name, *args):
self.config.delete(what, name, *args)
def get(self, what, name, *args, **kwargs):
return self.config.get(what, name, *args, **kwargs)
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/test.py 0000664 0000000 0000000 00000007651 13414137563 0025376 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon, Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import print_function
import sys
from functools import wraps
from unittest import TestCase
from weboob.core import Weboob
# This is what nose does for Python 2.6 and lower compatibility
# We do the same so nose becomes optional
try:
from unittest.case import SkipTest
except:
from nose.plugins.skip import SkipTest
__all__ = ['BackendTest', 'SkipTest', 'skip_without_config']
class BackendTest(TestCase):
MODULE = None
def __init__(self, *args, **kwargs):
super(BackendTest, self).__init__(*args, **kwargs)
self.backends = {}
self.backend_instance = None
self.backend = None
self.weboob = Weboob()
# Skip tests when passwords are missing
self.weboob.requests.register('login', self.login_cb)
if self.weboob.load_backends(modules=[self.MODULE]):
# provide the tests with all available backends
self.backends = self.weboob.backend_instances
def login_cb(self, backend_name, value):
raise SkipTest('missing config \'%s\' is required for this test' % value.label)
def run(self, result):
"""
Call the parent run() for each backend instance.
Skip the test if we have no backends.
"""
# This is a hack to fix an issue with nosetests running
# with many tests. The default is 1000.
sys.setrecursionlimit(10000)
try:
if not len(self.backends):
self.backend = self.weboob.build_backend(self.MODULE, nofail=True)
TestCase.run(self, result)
else:
# Run for all backend
for backend_instance in self.backends.keys():
print(backend_instance)
self.backend = self.backends[backend_instance]
TestCase.run(self, result)
finally:
self.weboob.deinit()
def shortDescription(self):
"""
Generate a description with the backend instance name.
"""
# do not use TestCase.shortDescription as it returns None
return '%s [%s]' % (str(self), self.backend_instance)
def is_backend_configured(self):
"""
Check if the backend is in the user configuration file
"""
return self.weboob.backends_config.backend_exists(self.backend.config.instname)
def skip_without_config(*keys):
"""Decorator to skip a test if backend config is missing
:param keys: if any of these keys is missing in backend config, skip test. Can be empty.
"""
for key in keys:
if callable(key):
raise TypeError('skip_without_config() must be called with arguments')
def decorator(func):
@wraps(func)
def wrapper(self, *args, **kwargs):
config = self.backend.config
if not self.is_backend_configured():
raise SkipTest('a backend must be declared in configuration for this test')
for key in keys:
if not config[key].get():
raise SkipTest('config key %r is required for this test' %
key)
return func(self, *args, **kwargs)
return wrapper
return decorator
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/tokenizer.py 0000664 0000000 0000000 00000006334 13414137563 0026426 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2014 Oleg Plakhotniuk
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import re
__all__ = ['ReTokenizer']
class ReTokenizer(object):
"""
Simple regex-based tokenizer (AKA lexer or lexical analyser).
Useful for PDF statements parsing.
1. There's a lexing table consisting of type-regex tuples.
2. Lexer splits text into chunks using the separator character.
3. Text chunk is sequentially matched against regexes and first
successful match defines the type of the token.
Check out test() function below for examples.
"""
def __init__(self, text, sep, lex):
self._lex = lex
self._tok = [ReToken(lex, chunk) for chunk in text.split(sep)]
def tok(self, index):
if 0 <= index < len(self._tok):
return self._tok[index]
else:
return ReToken(self._lex, eof=True)
def simple_read(self, token_type, pos, transform=lambda v: v):
t = self.tok(pos)
is_type = getattr(t, 'is_%s' % token_type)()
return (pos+1, transform(t.value())) if is_type else (pos, None)
class ReToken(object):
def __init__(self, lex, chunk=None, eof=False):
self._lex = lex
self._eof = eof
self._value = None
self._type = None
if chunk is not None:
for type_, regex in self._lex:
m = re.match(regex, chunk, flags=re.UNICODE)
if m:
self._type = type_
if len(m.groups()) == 1:
self._value = m.groups()[0]
elif m.groups():
self._value = m.groups()
else:
self._value = m.group(0)
break
def is_eof(self):
return self._eof
def value(self):
return self._value
def __getattr__(self, name):
if name.startswith('is_'):
return lambda: self._type == name[3:]
raise AttributeError()
def test():
t = ReTokenizer('foo bar baz', ' ', [('f', r'^f'), ('b', r'^b')])
assert t.tok(0).is_f()
assert t.tok(1).is_b()
assert t.tok(2).is_b()
assert t.tok(-1).is_eof()
assert t.tok(3).is_eof()
assert not t.tok(-1).is_f()
assert not t.tok(0).is_b()
assert not t.tok(0).is_eof()
t = ReTokenizer('nogroup onegroup multigroup', ' ', [
('ng', r'^n.*$'),
('og', r'^one(g.*)$'),
('mg', r'^(m.*)(g.*)$')])
assert t.tok(-1).value() is None
assert t.tok(0).value() == 'nogroup'
assert t.tok(1).value() == 'group'
assert t.tok(2).value() == ('multi', 'group')
woob-130005535d1d257a835857d4e42b2541ce490b40-weboob-tools/weboob/tools/value.py 0000664 0000000 0000000 00000024770 13414137563 0025534 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
import re
import time
import subprocess
from collections import OrderedDict
from subprocess import check_output
from weboob.tools.compat import basestring, unicode
from .misc import to_unicode
__all__ = ['ValuesDict', 'Value', 'ValueBackendPassword', 'ValueInt', 'ValueFloat', 'ValueBool']
class ValuesDict(OrderedDict):
"""
Ordered dictionarry which can take values in constructor.
>>> ValuesDict(Value('a', label='Test'), ValueInt('b', label='Test2'))
"""
def __init__(self, *values):
super(ValuesDict, self).__init__()
for v in values:
self[v.id] = v
class Value(object):
"""
Value.
:param label: human readable description of a value
:type label: str
:param required: if ``True``, the backend can't load if the key isn't found in its configuration
:type required: bool
:param default: an optional default value, used when the key is not in config. If there is no default value and the key
is not found in configuration, the **required** parameter is implicitly set
:param masked: if ``True``, the value is masked. It is useful for applications to know if this key is a password
:type masked: bool
:param regexp: if specified, on load the specified value is checked against this regexp, and an error is raised if it doesn't match
:type regexp: str
:param choices: if this parameter is set, the value must be in the list
:param aliases: mapping of old choices values that should be accepted but not presented
:type aliases: dict
:param tiny: the value of choices can be entered by an user (as they are small)
:type choices: (list,dict)
"""
def __init__(self, *args, **kwargs):
if len(args) > 0:
self.id = args[0]
else:
self.id = ''
self.label = kwargs.get('label', kwargs.get('description', None))
self.description = kwargs.get('description', kwargs.get('label', None))
self.default = kwargs.get('default', None)
if isinstance(self.default, str):
self.default = to_unicode(self.default)
self.regexp = kwargs.get('regexp', None)
self.choices = kwargs.get('choices', None)
self.aliases = kwargs.get('aliases')
if isinstance(self.choices, (list, tuple)):
self.choices = OrderedDict(((v, v) for v in self.choices))
self.tiny = kwargs.get('tiny', None)
self.masked = kwargs.get('masked', False)
self.required = kwargs.get('required', self.default is None)
self._value = kwargs.get('value', None)
def show_value(self, v):
if self.masked:
return u''
else:
return v
def check_valid(self, v):
"""
Check if the given value is valid.
:raises: ValueError
"""
if self.default is not None and v == self.default:
return
if self.required and v is None:
raise ValueError('Value is required and thus must be set')
if v == '' and self.default != '' and (self.choices is None or v not in self.choices):
raise ValueError('Value can\'t be empty')
if self.regexp is not None and not re.match(self.regexp + '$', unicode(v) if v is not None else ''):
raise ValueError('Value "%s" does not match regexp "%s"' % (self.show_value(v), self.regexp))
if self.choices is not None and v not in self.choices:
if not self.aliases or v not in self.aliases:
raise ValueError('Value "%s" is not in list: %s' % (
self.show_value(v), ', '.join(unicode(s) for s in self.choices)))
def load(self, domain, v, requests):
"""
Load value.
:param domain: what is the domain of this value
:type domain: str
:param v: value to load
:param requests: list of weboob requests
:type requests: weboob.core.requests.Requests
"""
return self.set(v)
def set(self, v):
"""
Set a value.
"""
if isinstance(v, str):
v = to_unicode(v)
self.check_valid(v)
if self.aliases and v in self.aliases:
v = self.aliases[v]
self._value = v
def dump(self):
"""
Dump value to be stored.
"""
return self.get()
def get(self):
"""
Get the value.
"""
return self._value
def is_command(self, v):
"""
Test if a value begin with ` and end with `
(`command` is used to call external programms)
"""
return isinstance(v, basestring) and v.startswith(u'`') and v.endswith(u'`')
class ValueBackendPassword(Value):
_domain = None
_requests = None
_stored = True
def __init__(self, *args, **kwargs):
kwargs['masked'] = kwargs.pop('masked', True)
self.noprompt = kwargs.pop('noprompt', False)
super(ValueBackendPassword, self).__init__(*args, **kwargs)
self.default = kwargs.get('default', '')
def load(self, domain, password, requests):
if self.is_command(password):
cmd = password[1:-1]
try:
password = check_output(cmd, shell=True)
except subprocess.CalledProcessError as e:
raise ValueError(u'The call to the external tool failed: %s' % e)
else:
password = password.decode('utf-8')
password = password.partition('\n')[0].strip('\r\n\t')
self.check_valid(password)
self._domain = domain
self._value = to_unicode(password)
self._requests = requests
def check_valid(self, passwd):
if passwd == '':
# always allow empty passwords
return True
return super(ValueBackendPassword, self).check_valid(passwd)
def set(self, passwd):
if self.is_command(passwd):
self._value = passwd
return
self.check_valid(passwd)
if passwd is None:
# no change
return
self._value = ''
if passwd == '':
return
if self._domain is None:
self._value = to_unicode(passwd)
return
try:
raise ImportError('Keyrings are disabled (see #706)')
import keyring
keyring.set_password(self._domain, self.id, passwd)
except Exception:
self._value = to_unicode(passwd)
else:
self._value = ''
def dump(self):
if self._stored:
return self._value
else:
return ''
def get(self):
if self._value != '' or self._domain is None:
return self._value
try:
raise ImportError('Keyrings are disabled (see #706)')
import keyring
except ImportError:
passwd = None
else:
passwd = keyring.get_password(self._domain, self.id)
if passwd is not None:
# Password has been read in the keyring.
return to_unicode(passwd)
# Prompt user to enter password by hand.
if not self.noprompt and self._requests:
self._value = self._requests.request('login', self._domain, self)
if self._value is None:
self._value = ''
else:
self._value = to_unicode(self._value)
self._stored = False
return self._value
class ValueInt(Value):
def __init__(self, *args, **kwargs):
kwargs['regexp'] = '^\d+$'
super(ValueInt, self).__init__(*args, **kwargs)
self.default = kwargs.get('default', 0)
def get(self):
return int(self._value)
class ValueFloat(Value):
def __init__(self, *args, **kwargs):
kwargs['regexp'] = '^[\d\.]+$'
super(ValueFloat, self).__init__(*args, **kwargs)
self.default = kwargs.get('default', 0.0)
def check_valid(self, v):
try:
float(v)
except ValueError:
raise ValueError('Value "%s" is not a float value' % self.show_value(v))
def get(self):
return float(self._value)
class ValueBool(Value):
def __init__(self, *args, **kwargs):
kwargs['choices'] = {'y': 'True', 'n': 'False'}
super(ValueBool, self).__init__(*args, **kwargs)
self.default = kwargs.get('default', False)
def check_valid(self, v):
if not isinstance(v, bool) and \
unicode(v).lower() not in ('y', 'yes', '1', 'true', 'on',
'n', 'no', '0', 'false', 'off'):
raise ValueError('Value "%s" is not a boolean (y/n)' % self.show_value(v))
def get(self):
return (isinstance(self._value, bool) and self._value) or \
unicode(self._value).lower() in ('y', 'yes', '1', 'true', 'on')
class ValueDate(Value):
DEFAULT_FORMATS = ('%Y-%m-%d',)
def __init__(self, *args, **kwargs):
super(ValueDate, self).__init__(*args, **kwargs)
self.formats = tuple(kwargs.get('formats', ()))
self.formats_tuple = self.DEFAULT_FORMATS + self.formats
def get_format(self, v=None):
for format in self.formats_tuple:
try:
dateval = time.strptime(v or self._value, format)
# year < 1900 is handled by strptime but not strftime, check it
time.strftime(self.formats_tuple[0], dateval)
except ValueError:
continue
return format
def check_valid(self, v):
super(ValueDate, self).check_valid(v)
if not self.get_format(v):
raise ValueError('Value "%s" does not match format in %s' % (self.show_value(v), self.show_value(self.formats_tuple)))
def get(self):
if self.formats:
self._value = time.strftime(self.formats[0], time.strptime(self._value, self.get_format()))
return self._value