pax_global_header 0000666 0000000 0000000 00000000064 13264636610 0014521 g ustar 00root root 0000000 0000000 52 comment=a5989d1421c9248c5dd6e24358d7e98797f2438a
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/ 0000775 0000000 0000000 00000000000 13264636610 0020375 5 ustar 00root root 0000000 0000000 woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/ 0000775 0000000 0000000 00000000000 13264636610 0021535 5 ustar 00root root 0000000 0000000 woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/boilerplate.py 0000775 0000000 0000000 00000016131 13264636610 0024416 0 ustar 00root root 0000000 0000000 #! /usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright(C) 2013 Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import print_function
import argparse
import subprocess
import datetime
import importlib
import os
import sys
import codecs
from mako.lookup import TemplateLookup
MODULE_PATH = os.getenv(
'MODULE_PATH',
os.path.realpath(os.path.join(os.path.dirname(__file__), '../modules')))
TEMPLATE_PATH = os.getenv(
'TEMPLATE_PATH',
os.path.realpath(os.path.join(os.path.dirname(__file__), 'boilerplate_data')))
VERSION = '1.4'
TEMPLATES = TemplateLookup(directories=[TEMPLATE_PATH])
def u8(s):
if isinstance(s, bytes):
return s.decode('utf-8')
return s
def gitconfig(entry):
return u8(subprocess.check_output('git config -z --get %s' % entry, shell=True)[:-1])
def write(target, contents):
if not os.path.isdir(os.path.dirname(target)):
os.makedirs(os.path.dirname(target))
if os.path.exists(target):
print("%s already exists." % target, file=sys.stderr)
sys.exit(4)
with codecs.open(target, mode='w', encoding='utf-8') as f:
f.write(contents)
print('Created %s' % target)
class Recipe(object):
@classmethod
def configure_subparser(cls, subparsers):
subparser = subparsers.add_parser(cls.NAME)
subparser.add_argument('name', help='Module name')
subparser.set_defaults(recipe=cls)
return subparser
def __init__(self, args):
self.name = args.name.lower().replace(' ', '')
self.classname = args.name.title().replace(' ', '')
self.year = datetime.date.today().year
self.author = args.author
self.email = args.email
self.version = VERSION
self.login = False
def write(self, filename, contents):
return write(os.path.join(MODULE_PATH, self.name, filename), contents)
def template(self, name, **kwargs):
if '.' not in name:
name += '.py'
return TEMPLATES.get_template(name) \
.render(r=self,
# workaround, as it's also a mako directive
coding='# -*- coding: utf-8 -*-',
login=self.login,
**kwargs)
def generate(self):
raise NotImplementedError()
class BaseRecipe(Recipe):
NAME = 'base'
def generate(self):
self.write('__init__.py', self.template('init'))
self.write('module.py', self.template('base_module'))
self.write('browser.py', self.template('base_browser'))
self.write('pages.py', self.template('base_pages'))
self.write('test.py', self.template('base_test'))
class CapRecipe(Recipe):
NAME = 'cap'
def __init__(self, args):
super(CapRecipe, self).__init__(args)
self.capname = args.capname
self.login = args.login
@classmethod
def configure_subparser(cls, subparsers):
subparser = super(CapRecipe, cls).configure_subparser(subparsers)
subparser.add_argument('--login', action='store_true', help='The site requires login')
subparser.add_argument('capname', help='Capability name')
return subparser
def find_module_cap(self):
if '.' not in self.capname:
return self.search_cap()
PREFIX = 'weboob.capabilities.'
if not self.capname.startswith(PREFIX):
self.capname = PREFIX + self.capname
try:
self.capmodulename, self.capname = self.capname.rsplit('.', 1)
except ValueError:
self.error('Cap name must be in format module.CapSomething or CapSomething')
try:
module = importlib.import_module(self.capmodulename)
except ImportError:
self.error('Module %r not found' % self.capmodulename)
try:
cap = getattr(module, self.capname)
except AttributeError:
self.error('Module %r has no such capability %r' % (self.capmodulename, self.capname))
return cap
def search_cap(self):
import pkgutil
import weboob.capabilities
modules = pkgutil.walk_packages(weboob.capabilities.__path__, prefix='weboob.capabilities.')
for _, capmodulename, __ in modules:
module = importlib.import_module(capmodulename)
if hasattr(module, self.capname):
self.capmodulename = capmodulename
return getattr(module, self.capname)
self.error('Capability %r not found' % self.capname)
def error(self, message):
print(message, file=sys.stderr)
sys.exit(1)
def methods_code(self, klass):
import inspect
methods = []
for name, member in inspect.getmembers(klass):
if inspect.ismethod(member) and name in klass.__dict__:
lines, _ = inspect.getsourcelines(member)
methods.append(lines)
return methods
def generate(self):
cap = self.find_module_cap()
self.methods = self.methods_code(cap)
self.write('__init__.py', self.template('init'))
self.write('module.py', self.template('cap_module'))
self.write('browser.py', self.template('base_browser'))
self.write('pages.py', self.template('base_pages'))
self.write('test.py', self.template('base_test'))
class ComicRecipe(Recipe):
NAME = 'comic'
def generate(self):
self.write('__init__.py', self.template('init'))
self.write('module.py', self.template('comic_module'))
class ComicTestRecipe(Recipe):
NAME = 'comic.test'
@classmethod
def configure_subparser(cls, subparsers):
subparser = super(ComicTestRecipe, cls).configure_subparser(subparsers)
subparser.add_argument('download_id', help='Download ID')
return subparser
def __init__(self, args):
super(ComicTestRecipe, self).__init__(args)
self.download_id = args.download_id
def generate(self):
self.write('test.py', self.template('comic_test'))
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'-a', '--author',
default=gitconfig('user.name'), type=u8)
parser.add_argument(
'-e', '--email',
default=gitconfig('user.email'), type=u8)
subparsers = parser.add_subparsers()
recipes = [BaseRecipe, ComicRecipe, ComicTestRecipe, CapRecipe]
for recipe in recipes:
recipe.configure_subparser(subparsers)
args = parser.parse_args()
recipe = args.recipe(args)
recipe.generate()
if __name__ == '__main__':
main()
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/boilerplate_data/ 0000775 0000000 0000000 00000000000 13264636610 0025030 5 ustar 00root root 0000000 0000000 woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/boilerplate_data/base_browser.py 0000664 0000000 0000000 00000001172 13264636610 0030060 0 ustar 00root root 0000000 0000000 <%inherit file="layout.py"/>
from weboob.browser import ${'LoginBrowser, need_login' if r.login else 'PagesBrowser'}, URL
from .pages import Page1, Page2
class ${r.classname}Browser(${'Login' if r.login else 'Pages'}Browser):
BASEURL = 'http://www.${r.name}.com'
page1 = URL('/page1\?id=(?P.+)', Page1)
page2 = URL('/page2', Page2)
% if login:
def do_login(self):
pass
@need_login
% endif
def get_stuff(self, _id):
self.page1.go(id=_id)
assert self.page1.is_here()
self.page.do_stuff(_id)
assert self.page2.is_here()
return self.page.do_more_stuff()
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/boilerplate_data/base_module.py 0000664 0000000 0000000 00000000612 13264636610 0027660 0 ustar 00root root 0000000 0000000 <%inherit file="layout.py"/>
from weboob.tools.backend import Module
from .browser import ${r.classname}Browser
__all__ = ['${r.classname}Module']
class ${r.classname}Module(Module):
NAME = '${r.name}'
DESCRIPTION = '${r.name} website'
MAINTAINER = '${r.author}'
EMAIL = '${r.email}'
LICENSE = 'AGPLv3+'
VERSION = '${r.version}'
BROWSER = ${r.classname}Browser
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/boilerplate_data/base_pages.py 0000664 0000000 0000000 00000000373 13264636610 0027476 0 ustar 00root root 0000000 0000000 <%inherit file="layout.py"/>
from weboob.browser.pages import HTMLPage
class Page1(HTMLPage):
def do_stuff(self, _id):
raise NotImplementedError()
class Page2(HTMLPage):
def do_more_stuff(self):
raise NotImplementedError()
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/boilerplate_data/base_test.py 0000664 0000000 0000000 00000000211 13264636610 0027345 0 ustar 00root root 0000000 0000000 <%inherit file="layout.py"/>
from weboob.tools.test import BackendTest
class ${r.classname}Test(BackendTest):
MODULE = '${r.name}'
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/boilerplate_data/cap_module.py 0000664 0000000 0000000 00000001625 13264636610 0027516 0 ustar 00root root 0000000 0000000 <%inherit file="layout.py"/>
from weboob.tools.backend import Module${', BackendConfig' if r.login else ''}
% if login:
from weboob.tools.value import Value, ValueBackendPassword
% endif
from ${r.capmodulename} import ${r.capname}
from .browser import ${r.classname}Browser
__all__ = ['${r.classname}Module']
class ${r.classname}Module(Module, ${r.capname}):
NAME = '${r.name}'
DESCRIPTION = '${r.name} website'
MAINTAINER = '${r.author}'
EMAIL = '${r.email}'
LICENSE = 'AGPLv3+'
VERSION = '${r.version}'
BROWSER = ${r.classname}Browser
% if login:
CONFIG = BackendConfig(
Value('username', help='Username'),
ValueBackendPassword('password', help='Password'),
)
def create_default_browser(self):
return self.create_browser(self.config['username'].get(), self.config['password'].get())
% endif
% for meth in r.methods:
${''.join(meth)}
% endfor
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/boilerplate_data/comic_module.py 0000664 0000000 0000000 00000001377 13264636610 0030051 0 ustar 00root root 0000000 0000000 <%inherit file="layout.py"/>
from weboob.tools.capabilities.gallery.genericcomicreader import GenericComicReaderModule, DisplayPage
__all__ = ['${r.classname}Module']
class ${r.classname}Module(GenericComicReaderModule):
NAME = '${r.name}'
DESCRIPTION = u'${r.name} manga reading site'
MAINTAINER = u'${r.author}'
EMAIL = '${r.email}'
VERSION = '${r.version}'
LICENSE = 'AGPLv3+'
DOMAIN = 'www.${r.name}.com'
BROWSER_PARAMS = dict(
img_src_xpath="//img[@id='comic_page']/@src",
page_list_xpath="(//select[@id='page_select'])[1]/option/@value")
ID_REGEXP = r'[^/]+/[^/]+'
URL_REGEXP = r'.+${r.name}.com/(%s).+' % ID_REGEXP
ID_TO_URL = 'http://www.${r.name}.com/%s'
PAGES = {URL_REGEXP: DisplayPage}
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/boilerplate_data/comic_test.py 0000664 0000000 0000000 00000000442 13264636610 0027533 0 ustar 00root root 0000000 0000000 <%inherit file="layout.py"/>
from weboob.tools.capabilities.gallery.genericcomicreadertest import GenericComicReaderTest
class ${r.classname}BackendTest(GenericComicReaderTest):
MODULE = '${r.name}'
def test_download(self):
return self._test_download('${r.download_id}')
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/boilerplate_data/init.py 0000664 0000000 0000000 00000000153 13264636610 0026344 0 ustar 00root root 0000000 0000000 <%inherit file="layout.py"/>
from .module import ${r.classname}Module
__all__ = ['${r.classname}Module']
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/boilerplate_data/layout.py 0000664 0000000 0000000 00000001415 13264636610 0026720 0 ustar 00root root 0000000 0000000 ${coding}
# Copyright(C) ${r.year} ${r.author}
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import unicode_literals
${self.body()}\
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/certhash.py 0000775 0000000 0000000 00000000256 13264636610 0023716 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python
from __future__ import print_function
import sys
from weboob.deprecated.browser import StandardBrowser
print(StandardBrowser()._certhash(sys.argv[1]))
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/check_xpath.py 0000775 0000000 0000000 00000011204 13264636610 0024371 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python
# Copyright(C) 2017 Vincent A
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import print_function
import ast
import fnmatch
import os
import traceback
import lxml.etree
from weboob.browser.filters import standard
class Error(SyntaxError):
def __init__(self, file, line, message):
super(Error, self).__init__('%s:%s: %s' % (file, line, message))
self.file = file
self.line = line
def do_visits(*funcs):
def wrapper(self, node):
for func in funcs:
func(self, node)
self.generic_visit(node)
return wrapper
class Visitor(ast.NodeVisitor):
def __init__(self, file, *args, **kwargs):
super(Visitor, self).__init__(*args, **kwargs)
self.warnings = kwargs.pop('warnings', False)
self.file = file
self.filters = []
self.filters.extend(f for f in dir(standard) if isinstance(getattr(standard, f), type) and issubclass(getattr(standard, f), standard.CleanText))
self.filters.extend(['Regexp', 'XPath', 'Attr', 'Link'])
self.element_context = []
def check_xpath(self, s, lineno):
try:
lxml.etree.XPath(s)
except lxml.etree.XPathSyntaxError as exc:
raise Error(self.file, lineno, exc)
if self.warnings:
if not s.startswith('.') and len(self.element_context) >= 2:
if self.element_context[-1] == 'ItemElement' and self.element_context[-2] in ('TableElement', 'ListElement'):
print('%s:%s: probable missing "." at start of XPath' % (self.file, lineno))
def _item_xpath(self, node):
try:
target, = node.targets
except ValueError:
return
if not isinstance(target, ast.Name) or target.id != 'item_xpath':
return
try:
if self.element_context[-1] not in ('TableElement', 'ListElement'):
return
except IndexError:
return
if not isinstance(node.value, ast.Str):
return
self.check_xpath(node.value.s, node.lineno)
visit_Assign = do_visits(_item_xpath)
def _xpath_call(self, node):
if not isinstance(node.func, ast.Attribute):
return
if node.func.attr != 'xpath':
return
try:
if not isinstance(node.args[0], ast.Str):
return
except IndexError:
return
self.check_xpath(node.args[0].s, node.lineno)
def _filter_call(self, node):
if not isinstance(node.func, ast.Name):
return
if node.func.id not in self.filters:
return
try:
if not isinstance(node.args[0], ast.Str):
return
except IndexError:
return
self.check_xpath(node.args[0].s, node.lineno)
visit_Call = do_visits(_xpath_call, _filter_call)
def visit_ClassDef(self, node):
has_element = False
proceed = False
for deco in node.decorator_list:
if isinstance(deco, ast.Name) and deco.id == 'method':
proceed = True
break
if proceed:
for basenode in node.bases:
if isinstance(basenode, ast.Name) and basenode.id in ('ListElement', 'ItemElement', 'TableElement'):
self.element_context.append(basenode.id)
has_element = True
break
self.generic_visit(node)
if has_element:
self.element_context.pop()
def search_py(root):
for path, dirs, files in os.walk(root):
dirs.sort()
for f in fnmatch.filter(files, '*.py'):
yield os.path.join(path, f)
for fn in search_py(os.path.normpath(os.path.dirname(__file__) + '/../modules')):
with open(fn) as fd:
try:
node = ast.parse(fd.read(), fn)
except SyntaxError as exc:
print('In file', fn)
traceback.print_exc(exc)
try:
Visitor(fn).visit(node)
except SyntaxError as exc:
print(exc)
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/common.sh 0000664 0000000 0000000 00000001371 13264636610 0023363 0 ustar 00root root 0000000 0000000 VER=2
if [ "${1-}" = "-3" ]; then
VER=3
shift
fi
if [ -z "${PYTHON2-}" ]; then
which python2.7 >/dev/null 2>&1 && PYTHON2=$(which python2.7)
which python2 >/dev/null 2>&1 && PYTHON2=$(which python2)
fi
if [ -z "${PYTHON3-}" ]; then
which python3.4 >/dev/null 2>&1 && PYTHON3=$(which python3.4)
which python3.5 >/dev/null 2>&1 && PYTHON3=$(which python3.5)
which python3.6 >/dev/null 2>&1 && PYTHON3=$(which python3.6)
which python3 >/dev/null 2>&1 && PYTHON3=$(which python3)
fi
if [ -z "${PYTHON-}" ]; then
which python >/dev/null 2>&1 && PYTHON=$(which python)
if [ $VER -eq 2 -a -n "${PYTHON2}" ]; then
PYTHON=${PYTHON2}
elif [ $VER -eq 3 -a -n "${PYTHON3}" ]; then
PYTHON=${PYTHON3}
fi
fi
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/debpydep.py 0000775 0000000 0000000 00000002431 13264636610 0023706 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
from __future__ import print_function
import os
import subprocess
import sys
selection = set()
dependencies = set()
for root, dirs, files in os.walk(sys.argv[1]):
for f in files:
if f.endswith('.py') and f != '__init__.py':
s = "from %s import %s" % (root.strip('/').replace('/', '.'), f[:-3])
try:
exec(s)
except ImportError as e:
print(str(e), file=sys.stderr)
else:
m = eval(f[:-3])
for attrname in dir(m):
try:
attr = getattr(m, attrname)
selection.add(attr.__file__)
except AttributeError:
pass
for f in selection:
f = f.replace('.pyc', '.py')
try:
f = os.path.abspath(os.path.join(os.path.split(f)[0], os.readlink(f)))
except OSError:
pass
p = subprocess.Popen(['/usr/bin/dpkg', '-S', f], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if p.wait() == 0:
for line in p.stdout.readlines():
line = line.decode('utf-8')
dependencies.add(line.strip().split(':')[0])
else:
print('not found: %s' % f)
for d in dependencies:
print(d)
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/generate_changelog_modules.sh 0000775 0000000 0000000 00000001160 13264636610 0027423 0 ustar 00root root 0000000 0000000 #!/bin/bash
# Usage: weboob$ tools/generate_changelog_modules.sh TAG "list of hash" [show]
BEGIN=$1
EXCLUDE=$2
SHOW=$3
for a in modules/*
do
if [ -d $a ]
then
MODULE=`basename $a`
LOG=`git log --format="%H:::* %s" --date-order --reverse "$BEGIN..HEAD" -- $a`
for b in $EXCLUDE
do
LOG=$(echo "$LOG" |grep -v $b)
done
if [ -n "$LOG" ]
then
if [ -n "$SHOW" ]
then
echo "$LOG" | awk -F ":::" '{print $1}' | git show --stdin
else
echo -e "\tModules: $MODULE"
echo "$LOG" | awk -F ":::" '{print "\t"$2}' | sed "s/$MODULE: //" | sed "s/\[$MODULE\] //"
echo ""
fi
fi
fi
done
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/local_install.py 0000664 0000000 0000000 00000003255 13264636610 0024734 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function
import os
import subprocess
import sys
if '--deps' in sys.argv:
sys.argv.remove('--deps')
deps = []
else:
deps = ['--nodeps']
print("Weboob local installer")
print()
if len(sys.argv) < 2:
print("This tool will install Weboob to be usuable without requiring")
print("messing with your system, which should only be touched by a package manager.")
print()
print("Usage: %s DESTINATION [OPTIONS]" % sys.argv[0])
print()
print("By default, no dependencies are installed, as you should try")
print("to install them from your package manager as much as possible.")
print("To install all the missing dependencies, add the option --deps")
print("at the end of the command line.")
print()
print("Error: Please provide a destination, "
"for example ‘%s/bin’" % os.getenv('HOME'), file=sys.stderr)
sys.exit(1)
else:
dest = os.path.expanduser(sys.argv[1])
print("Installing weboob applications into ‘%s’." % dest)
subprocess.check_call(
[sys.executable, 'setup.py',
'install', '--user', '--install-scripts=%s' % dest] + sys.argv[2:] + deps,
cwd=os.path.join(os.path.dirname(__file__), os.pardir))
subprocess.check_call([sys.executable, os.path.join(dest, 'weboob-config'), 'update'])
print()
print("Installation done. Applications are available in ‘%s’." % dest)
print("You can remove the source files.")
print()
print("To have easy access to the Weboob applications,")
print("you should add the following line to your ~/.bashrc or ~/.zshrc file:")
print("export PATH=\"$PATH:%s\"" % dest)
print("And then restart your shells.")
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/local_install.sh 0000775 0000000 0000000 00000000210 13264636610 0024705 0 ustar 00root root 0000000 0000000 #!/bin/sh
set -e
. "$(dirname $0)/common.sh"
$PYTHON "$(dirname $0)/stale_pyc.py"
exec $PYTHON "$(dirname $0)/local_install.py" "$@"
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/local_run.py 0000664 0000000 0000000 00000003160 13264636610 0024065 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function
import subprocess
import sys
import os
if len(sys.argv) < 2:
print("Usage: %s SCRIPTNAME [args]" % sys.argv[0])
sys.exit(1)
else:
script = sys.argv[1]
args = sys.argv[2:]
project = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))
wd = os.path.join(project, 'localconfig')
if not os.path.isdir(wd):
os.makedirs(wd)
paths = os.getenv('PYTHONPATH', None)
if not paths:
paths = sys.path
else:
paths = paths.split(':')
if project not in paths:
paths.insert(0, project)
env = os.environ.copy()
env['PYTHONPATH'] = ':'.join(p for p in paths if p)
env['WEBOOB_WORKDIR'] = wd
env['WEBOOB_DATADIR'] = wd
env['WEBOOB_BACKENDS'] = os.getenv('WEBOOB_LOCAL_BACKENDS',
os.getenv('WEBOOB_BACKENDS',
os.path.join(os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config')), 'weboob', 'backends')))
modpath = os.getenv('WEBOOB_MODULES', os.path.join(project, 'modules'))
with open(os.path.join(wd, 'sources.list'), 'w') as f:
f.write("file://%s\n" % modpath)
# Hide output unless there is an error
p = subprocess.Popen(
[sys.executable, os.path.join(project, 'scripts', 'weboob-config'), 'update'],
env=env,
stdout=subprocess.PIPE)
s = p.communicate()
if p.returncode != 0:
print(s[0])
sys.exit(p.returncode)
if os.path.exists(script):
spath = script
else:
spath = os.path.join(project, 'scripts', script)
os.execvpe(
sys.executable,
['-Wall', '-s', spath] + args,
env)
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/local_run.sh 0000775 0000000 0000000 00000000204 13264636610 0024046 0 ustar 00root root 0000000 0000000 #!/bin/sh
set -e
. "$(dirname $0)/common.sh"
$PYTHON "$(dirname $0)/stale_pyc.py"
exec $PYTHON "$(dirname $0)/local_run.py" "$@"
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/make_man.py 0000775 0000000 0000000 00000022370 13264636610 0023666 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2014 Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
from __future__ import absolute_import, print_function
import imp
import inspect
import optparse
import os
import re
import sys
import tempfile
import time
from datetime import datetime
from weboob.tools.application.base import Application
BASE_PATH = os.path.join(os.path.dirname(__file__), os.pardir)
DEST_DIR = 'man'
class ManpageHelpFormatter(optparse.HelpFormatter):
def __init__(self,
app,
indent_increment=0,
max_help_position=0,
width=80,
short_first=1):
optparse.HelpFormatter.__init__(self, indent_increment, max_help_position, width, short_first)
self.app = app
def format_heading(self, heading):
return ".SH %s\n" % heading.upper()
def format_usage(self, usage):
txt = ''
for line in usage.split('\n'):
line = line.lstrip().split(' ', 1)
if len(txt) > 0:
txt += '.br\n'
txt += '.B %s\n' % line[0]
arg_re = re.compile(r'([\[\s])([\w_]+)')
args = re.sub(arg_re, r"\1\\fI\2\\fR", line[1])
txt += args
txt += '\n'
return '.SH SYNOPSIS\n%s' % txt
def format_description(self, description):
desc = u'.SH DESCRIPTION\n.LP\n\n%s\n' % description
if hasattr(self.app, 'CAPS'):
self.app.weboob.modules_loader.load_all()
caps = self.app.CAPS if isinstance(self.app.CAPS, tuple) else (self.app.CAPS,)
modules = []
for name, module in self.app.weboob.modules_loader.loaded.items():
if module.has_caps(*caps):
modules.append(u'* %s (%s)' % (name, module.description))
if len(modules) > 0:
desc += u'\n.SS Supported websites:\n'
desc += u'\n.br\n'.join(sorted(modules))
return desc
def format_commands(self, commands):
s = u''
for section, cmds in commands.items():
if len(cmds) == 0:
continue
s += '.SH %s COMMANDS\n' % section.upper()
for cmd in sorted(cmds):
s += '.TP\n'
h = cmd.split('\n')
if ' ' in h[0]:
cmdname, args = h[0].split(' ', 1)
arg_re = re.compile(r'([A-Z_]+)')
args = re.sub(arg_re, r"\\fI\1\\fR", args)
s += '\\fB%s\\fR %s' % (cmdname, args)
else:
s += '\\fB%s\\fR' % h[0]
s += '%s\n' % '\n.br\n'.join(h[1:])
return s
def format_option_strings(self, option):
opts = optparse.HelpFormatter.format_option_strings(self, option).split(", ")
return ".TP\n" + ", ".join("\\fB%s\\fR" % opt for opt in opts)
def main():
scripts_path = os.path.join(BASE_PATH, "scripts")
files = os.listdir(scripts_path)
# Create a fake "scripts" modules to import the scripts into
sys.modules["scripts"] = imp.new_module("scripts")
for fname in files:
fpath = os.path.join(scripts_path, fname)
if os.path.isfile(fpath) and os.access(fpath, os.X_OK):
with open(fpath) as f:
# Python will likely want create a compiled file, we provide a place
tmpdir = os.path.join(tempfile.gettempdir(), "weboob", "make_man")
if not os.path.isdir(tmpdir):
os.makedirs(tmpdir)
tmpfile = os.path.join(tmpdir, fname)
desc = ("", "U", imp.PY_SOURCE)
try:
script = imp.load_module("scripts.%s" % fname, f, tmpfile, desc)
except ImportError as e:
print("Unable to load the %s script (%s)"
% (fname, e), file=sys.stderr)
else:
print("Loaded %s" % fname)
# Find the applications we can handle
for klass in script.__dict__.values():
if inspect.isclass(klass) and issubclass(klass, Application) and klass.VERSION:
analyze_application(klass, fname)
finally:
# Cleanup compiled files if needed
if (os.path.isfile(tmpfile + "c")):
os.unlink(tmpfile + "c")
def format_title(title):
return re.sub(r'^(.+):$', r'.SH \1\n.TP', title.group().upper())
# XXX useful because the PyQt QApplication destructor crashes sometimes. By
# keeping every applications until program end, it prevents to stop before
# every manpages have been generated. If it crashes at exit, it's not a
# really a problem.
applications = []
def analyze_application(app, script_name):
application = app()
applications.append(application)
formatter = ManpageHelpFormatter(application)
# patch the application
application._parser.prog = "%s" % script_name
application._parser.formatter = formatter
helptext = application._parser.format_help(formatter)
cmd_re = re.compile(r'^.+ Commands:$', re.MULTILINE)
helptext = re.sub(cmd_re, format_title, helptext)
helptext = helptext.replace("-", r"\-")
coding = r'.\" -*- coding: utf-8 -*-'
comment = r'.\" This file was generated automatically by tools/make_man.sh.'
header = '.TH %s 1 "%s" "%s %s"' % (script_name.upper(), time.strftime("%d %B %Y"),
script_name, app.VERSION.replace('.', '\\&.'))
name = ".SH NAME\n%s \- %s" % (script_name, application.SHORT_DESCRIPTION)
condition = """.SH CONDITION
The \-c and \-\-condition is a flexible way to filter and get only interesting results. It supports conditions on numerical values, dates, durations and strings. Dates are given in YYYY\-MM\-DD or YYYY\-MM\-DD HH:MM format. Durations look like XhYmZs where X, Y and Z are integers. Any of them may be omitted. For instance, YmZs, XhZs or Ym are accepted.
The syntax of one expression is "\\fBfield operator value\\fR". The field to test is always the left member of the expression.
.LP
The field is a member of the objects returned by the command. For example, a bank account has "balance", "coming" or "label" fields.
.SS The following operators are supported:
.TP
=
Test if object.field is equal to the value.
.TP
!=
Test if object.field is not equal to the value.
.TP
>
Test if object.field is greater than the value. If object.field is date, return true if value is before that object.field.
.TP
<
Test if object.field is less than the value. If object.field is date, return true if value is after that object.field.
.TP
|
This operator is available only for string fields. It works like the Unix standard \\fBgrep\\fR command, and returns True if the pattern specified in the value is in object.field.
.SS Expression combination
.LP
You can make a expression combinations with the keywords \\fB" AND "\\fR, \\fB" OR "\\fR an \\fB" LIMIT "\\fR.
.LP
The \\fBLIMIT\\fR keyword can be used to limit the number of items upon which running the expression. \\fBLIMIT\\fR can only be placed at the end of the expression followed by the number of elements you want.
.SS Examples:
.nf
.B boobank ls \-\-condition 'label=Livret A'
.fi
Display only the "Livret A" account.
.PP
.nf
.B boobank ls \-\-condition 'balance>10000'
.fi
Display accounts with a lot of money.
.PP
.nf
.B boobank history account@backend \-\-condition 'label|rewe'
.fi
Get transactions containing "rewe".
.PP
.nf
.B boobank history account@backend \-\-condition 'date>2013\-12\-01 AND date<2013\-12\-09'
.fi
Get transactions betweens the 2th December and 8th December 2013.
.PP
.nf
.B boobank history account@backend \-\-condition 'date>2013\-12\-01 LIMIT 10'
.fi
Get transactions after the 2th December in the last 10 transactions
"""
footer = """.SH COPYRIGHT
%s
.LP
For full copyright information see the COPYING file in the weboob package.
.LP
.RE
.SH FILES
"~/.config/weboob/backends" """ % application.COPYRIGHT.replace('YEAR', '%d' % datetime.today().year)
if len(app.CONFIG) > 0:
footer += '\n\n "~/.config/weboob/%s"' % app.APPNAME
# Skip internal applications.
footer += "\n\n.SH SEE ALSO\nHome page: http://weboob.org/applications/%s" % application.APPNAME
mantext = u"%s\n%s\n%s\n%s\n%s\n%s\n%s" % (coding, comment, header, name, helptext, condition, footer)
with open(os.path.join(BASE_PATH, DEST_DIR, "%s.1" % script_name), 'w+') as manfile:
for line in mantext.split('\n'):
manfile.write('%s\n' % line.lstrip().encode('utf-8'))
print("wrote %s/%s.1" % (DEST_DIR, script_name))
if __name__ == '__main__':
sys.exit(main())
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/make_man.sh 0000775 0000000 0000000 00000001507 13264636610 0023647 0 ustar 00root root 0000000 0000000 #!/bin/sh
# stop on failure
set -e
. "$(dirname $0)/common.sh"
# Use C local to avoid local dates in headers
export LANG=en_US.utf8
# disable termcolor
export ANSI_COLORS_DISABLED=1
[ -z "${TMPDIR}" ] && TMPDIR="/tmp"
# do not allow undefined variables anymore
set -u
WEBOOB_TMPDIR=$(mktemp -d "${TMPDIR}/weboob_man.XXXXX")
# path to sources
WEBOOB_DIR=$(cd $(dirname $0)/.. && pwd -P)
touch "${WEBOOB_TMPDIR}/backends"
chmod 600 "${WEBOOB_TMPDIR}/backends"
echo "file://$WEBOOB_DIR/modules" > "${WEBOOB_TMPDIR}/sources.list"
export WEBOOB_WORKDIR="${WEBOOB_TMPDIR}"
export WEBOOB_DATADIR="${WEBOOB_TMPDIR}"
export PYTHONPATH="${WEBOOB_DIR}"
$PYTHON "${WEBOOB_DIR}/scripts/weboob-config" update
$PYTHON "${WEBOOB_DIR}/tools/make_man.py"
# allow failing commands past this point
STATUS=$?
rm -rf "${WEBOOB_TMPDIR}"
exit $STATUS
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/modules_testing_grid.py 0000775 0000000 0000000 00000004003 13264636610 0026321 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Script to format XUNIT output from unittests as a JSON string ready to be sent
to a [Weboob-CI](https://github.com/Phyks/weboob-ci) instance.
* `XUNIT` is the XUNIT file to handle.
* `ORIGIN` is an origin string as described in the Weboob-CI documentation
(basically just a string to identify the source of the unittests results).
"""
from __future__ import print_function
import json
import sys
import xunitparser
def main(xunit, origin):
with open(xunit, "r") as fh:
ts, tr = xunitparser.parse(fh)
# Get test results for each module
modules = {}
other_testcases = []
for tc in ts:
if tc.classname.startswith("weboob."):
other_testcases.append(repr(tc))
continue
module = tc.classname.split(".")[0]
# In the following, we consider
# bad > skipped > good
# and only make update of a module status according to this order
if tc.good:
if tc.skipped:
# Set to skipped only if previous test was good
if module not in modules or modules[module] == "good":
modules[module] = "skipped"
else:
# Set to good only if no previous result
if module not in modules:
modules[module] = "good"
else:
# Always set to bad on failed test
modules[module] = "bad"
# Agregate results by test result rather than module
results = {
"good": [],
"bad": [],
"skipped": []
}
for module in modules:
results[modules[module]].append(module)
return {
"origin": origin,
"modules": results,
"others": other_testcases
}
if __name__ == "__main__":
if len(sys.argv) < 3:
sys.exit("Usage: %s XUNIT_FILE ORIGIN" % (sys.argv[0]))
print(
json.dumps(
main(sys.argv[1], sys.argv[2]),
sort_keys=True, indent=4, separators=(',', ': ')
)
)
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/newsince.py 0000775 0000000 0000000 00000002472 13264636610 0023732 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python3
import os
import re
import subprocess
import sys
import configparser
since = sys.argv[1]
def changed_modules(changes, changetype):
for change in changes:
change = change.decode('utf-8').split()
if change[0] == changetype:
m = re.match('modules/([^/]+)/__init__\.py', change[1])
if m:
yield m.group(1)
def get_caps(module, config):
try:
return sorted(c for c in config[module]['capabilities'].split() if c != 'CapCollection')
except KeyError:
return ['**** FILL ME **** (running weboob update could help)']
os.chdir(os.path.join(os.path.dirname(__file__), os.path.pardir))
modules_info = configparser.ConfigParser()
with open('modules/modules.list') as f:
modules_info.read_file(f)
git_cmd = ['git', 'diff', '--no-renames', '--name-status', '%s..HEAD' % since, '--', 'modules/']
added_modules = sorted(changed_modules(subprocess.check_output(git_cmd).splitlines(), 'A'))
deleted_modules = sorted(changed_modules(subprocess.check_output(git_cmd).splitlines(), 'D'))
for added_module in added_modules:
print(' * New %s module (%s)' % (added_module, ', '.join(get_caps(added_module, modules_info))))
for deleted_module in deleted_modules:
print(' * Deleted %s module' % deleted_module)
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/py3-compatible.modules 0000664 0000000 0000000 00000002332 13264636610 0025757 0 ustar 00root root 0000000 0000000 ##
# this file declares which modules are compatible with python3 and should be checked with pyflakes3
##
750g/
adecco/
agendaculturel/
agendadulibre/
allrecipes/
americanexpress/
apec/
arte/
axabanque/
bandcamp/
banquepopulaire/
barclays/
bforbank/
bibliothequesparis/
billetreduc/
biplan/
blablacar/
blogspot/
bnppere/
boursorama/
/bp/
bred/
btmon/
caissedepargne/
capeasi/
colissimo/
cragr/
creditcooperatif/
creditdunord/
cuisineaz/
delubac/
dlfp/
entreparticuliers/
erehsbc/
esalia/
explorimmo/
feedly/
fortuneo/
francetelevisions/
freeteknomusic/
funmooc/
github/
googletranslate/
groupamaes/
hsbc/
hybride/
imdb/
imgur/
indeed/
infomaniak/
ing/
ipinfodb/
jcvelaux/
jirafeau/
lameteoagricole/
larousse/
lcl/
leboncoin/
limetorrents/
linebourse/
logicimmo/
lolix/
lutim/
lyricsmode/
manpower/
marmiton/
mediawiki/
meteofrance/
monster/
n26/
nectarine/
nova/
opensubtitles/
ouifm/
pap/
pariskiwi/
paroles2chansons/
parolesmania/
pastebin/
pastealacon/
pixtoilelibre/
popolemploi/
pornhub/
ratp/
razibus/
regionsjob/
relaiscolis/
s2e/
seloger/
senscritique/
societegenerale/
somafm/
spirica/
sprunge/
sueurdemetal/
supertoinette/
tumblr/
tvsubtitles/
twitter/
vimeo/
vlille/
wordreference/
xhamster/
yggtorrents/
yomoni/
youtube/
zerobin/
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/pyflakes.sh 0000775 0000000 0000000 00000005321 13264636610 0023713 0 ustar 00root root 0000000 0000000 #!/bin/bash -u
. "$(dirname $0)/common.sh"
err=0
PY3MODS=./tools/py3-compatible.modules
cd $(dirname $0)/..
MODULE_FILES=$(git ls-files|grep '^modules/.*\.py$')
MODULE_FILES3=$(printf "%s\n" $MODULE_FILES|grep -F -f $PY3MODS)
PYFILES=$(git ls-files | grep '^scripts\|\.py$'|grep -v boilerplate_data|grep -v stable_backport_data|grep -v '^modules'|grep -v '^contrib')
PYFILES3="$(printf "%s\n" $PYFILES | grep -v /deprecated/) $MODULE_FILES3"
PYFILES="$PYFILES $MODULE_FILES"
grep -n 'class [^( ]\+:$' ${PYFILES} && echo 'Error: old class style found, always inherit object' && err=3
grep -n $'\t\|\s$' ${PYFILES} && echo 'Error: tabs or trailing whitespace found, remove them' && err=4
grep -Fn '.setlocale' ${PYFILES} && echo 'Error: do not use setlocale' && err=5
grep -Fn '__future__ import with_statement' ${PYFILES} && echo 'Error: with_statement useless as we do not support Python 2.5' && err=6
grep -nE '^[[:space:]]+except [[:alnum:] ]+,[[:alnum:] ]+' ${PYFILES} && echo 'Error: use new "as" way of naming exceptions' && err=7
grep -nE "^ *print " ${PYFILES} && echo 'Error: Use the print function' && err=8
grep -Fn ".has_key" ${PYFILES} && echo 'Error: Deprecated, use operator "in"' && err=9
grep -Fn "os.isatty" ${PYFILES} && echo 'Error: Use stream.isatty() instead of os.isatty(stream.fileno())' && err=10
grep -Fn "raise StopIteration" ${PYFILES} && echo 'Error: PEP 479' && err=11
grep -nE "\.iter(keys|values|items)\(\)" ${PYFILES3} | grep -Fv "six.iter" && echo 'Error: iterkeys/itervalues/iteritems is forbidden' && err=12
grep -nE "^ *print(\(| )" ${MODULE_FILES} && echo 'Error: Use of print in modules is forbidden, use logger instead' && err=20
grep -n xrange ${MODULE_FILES3} && echo 'Error: xrange is forbidden' && err=21
grep -nE "from (urllib|urlparse) import" ${MODULE_FILES3} && echo 'Error: python2 urllib is forbidden' && err=22
grep -nE "^import (urllib|urlparse)$" ${MODULE_FILES3} && echo 'Error: python2 urllib is forbidden' && err=22
if [ ${VER} -eq 2 ]
then
if ${PYTHON2} -c "import flake8" 2>/dev/null; then
FLAKER2=flake8
OPT2="--select=E9,F"
elif ${PYTHON2} -c "import pyflakes" 2>/dev/null; then
FLAKER2=pyflakes
OPT2=
else
echo "flake8 or pyflakes for python2 not found"
err=1
fi
if [ ${err} -ne 1 ]; then
$PYTHON2 -m ${FLAKER2} ${OPT2} ${PYFILES} || err=32
fi
fi
if [ ${VER} -eq 3 ]
then
if ${PYTHON3} -c "import flake8" 2>/dev/null; then
FLAKER3=flake8
OPT3="--select=E9,F"
elif ${PYTHON3} -c "import pyflakes" 2>/dev/null; then
FLAKER3=pyflakes
OPT3=
else
echo "flake8 or pyflakes for python3 not found"
err=1
fi
if [ ${err} -ne 1 ]; then
$PYTHON3 -m ${FLAKER3} ${OPT3} ${PYFILES3} || exit 33
fi
fi
exit $err
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/pyreverse.sh 0000775 0000000 0000000 00000000433 13264636610 0024120 0 ustar 00root root 0000000 0000000 #!/bin/bash
#
# Examples:
# pyreverse.sh weboob.backends.aum
#
# pyreverse is included in pylint Debian package
function usage() {
echo "pyreverse.sh "
exit
}
[ -z "$1" ] && usage
PYTHONPATH="$(dirname $0)/../modules/$1" pyreverse -p "$1" -o pdf -a1 -s1 "."
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/run_tests.sh 0000775 0000000 0000000 00000011656 13264636610 0024133 0 ustar 00root root 0000000 0000000 #!/bin/bash
# Mai available environment variables
# * RSYNC_TARGET: target on which to rsync the xunit output.
# * XUNIT_OUT: file in which xunit output should be saved.
# * WEBOOB_BACKENDS: path to the Weboob backends file to use.
# * WEBOOB_CI_TARGET: URL of your Weboob-CI instance.
# * WEBOOB_CI_ORIGIN: origin for the Weboob-CI data.
# stop on failure
set -e
. "$(dirname $0)/common.sh"
if [ -z "${PYTHON}" ]; then
echo "Python required"
exit 1
fi
if ! $PYTHON -c "import nose" 2>/dev/null; then
echo "python-nose required"
exit 1
fi
# path to sources
WEBOOB_DIR=$(cd $(dirname $0)/.. && pwd -P)
BACKEND="${1}"
if [ -z "${WEBOOB_WORKDIR}" ]; then
# use the old workdir by default
WEBOOB_WORKDIR="${HOME}/.weboob"
# but if we can find a valid xdg workdir, switch to it
[ "${XDG_CONFIG_HOME}" != "" ] || XDG_CONFIG_HOME="${HOME}/.config"
[ -d "${XDG_CONFIG_HOME}/weboob" ] && WEBOOB_WORKDIR="${XDG_CONFIG_HOME}/weboob"
fi
[ -z "${TMPDIR}" ] && TMPDIR="/tmp"
WEBOOB_TMPDIR=$(mktemp -d "${TMPDIR}/weboob_test.XXXXX")
[ -z "${WEBOOB_BACKENDS}" ] && WEBOOB_BACKENDS="${WEBOOB_WORKDIR}/backends"
[ -z "${WEBOOB_MODULES}" ] && WEBOOB_MODULES="${WEBOOB_DIR}/modules"
[ -z "${PYTHONPATH}" ] && PYTHONPATH=""
# allow private environment setup
[ -f "${WEBOOB_WORKDIR}/pre-test.sh" ] && source "${WEBOOB_WORKDIR}/pre-test.sh"
# setup xunit reporting (buildbot slaves only)
if [ -n "${RSYNC_TARGET}" ]; then
# by default, builder name is containing directory name
[ -z "${BUILDER_NAME}" ] && BUILDER_NAME=$(basename $(readlink -e $(dirname $0)/../..))
XUNIT_OUT="${WEBOOB_TMPDIR}/xunit.xml"
else
RSYNC_TARGET=""
fi
# Avoid undefined variables
if [ ! -n "${XUNIT_OUT}" ]; then
XUNIT_OUT=""
fi
# Handle Weboob-CI variables
if [ -n "${WEBOOB_CI_TARGET}" ]; then
if [ ! -n "${WEBOOB_CI_ORIGIN}" ]; then
WEBOOB_CI_ORIGIN="Weboob unittests run"
fi
# Set up xunit reporting
XUNIT_OUT="${WEBOOB_TMPDIR}/xunit.xml"
else
WEBOOB_CI_TARGET=""
fi
# do not allow undefined variables anymore
set -u
if [ -f "${WEBOOB_BACKENDS}" ]; then
cp "${WEBOOB_BACKENDS}" "${WEBOOB_TMPDIR}/backends"
else
touch "${WEBOOB_TMPDIR}/backends"
chmod go-r "${WEBOOB_TMPDIR}/backends"
fi
# xunit nose setup
if [ -n "${XUNIT_OUT}" ]; then
XUNIT_ARGS="--with-xunit --xunit-file=${XUNIT_OUT}"
else
XUNIT_ARGS=""
fi
${PYTHON} "$(dirname $0)/stale_pyc.py"
echo "file://${WEBOOB_MODULES}" > "${WEBOOB_TMPDIR}/sources.list"
export WEBOOB_WORKDIR="${WEBOOB_TMPDIR}"
export WEBOOB_DATADIR="${WEBOOB_TMPDIR}"
export PYTHONPATH="${WEBOOB_DIR}:${PYTHONPATH}"
export NOSE_NOPATH="1"
${PYTHON} "${WEBOOB_DIR}/scripts/weboob-config" update
# allow failing commands past this point
set +e
set -o pipefail
if [ -n "${BACKEND}" ]; then
${PYTHON} -m nose -c /dev/null -sv "${WEBOOB_MODULES}/${BACKEND}/test.py" ${XUNIT_ARGS}
STATUS=$?
STATUS_CORE=0
else
echo "=== Weboob ==="
CORE_TESTS=$(mktemp)
${PYTHON} -m nose --cover-package weboob -c ${WEBOOB_DIR}/setup.cfg -sv 2>&1 | tee "${CORE_TESTS}"
STATUS_CORE=$?
echo "=== Modules ==="
MODULES_TESTS=$(mktemp)
MODULES_TO_TEST=$(find "${WEBOOB_MODULES}" -name "test.py" | sort | xargs echo)
${PYTHON} -m nose --with-coverage --cover-package modules -c /dev/null -sv ${XUNIT_ARGS} ${MODULES_TO_TEST} 2>&1 | tee ${MODULES_TESTS}
STATUS=$?
# Compute total coverage
echo "=== Total coverage ==="
CORE_STMTS=$(grep "TOTAL" ${CORE_TESTS} | awk '{ print $2; }')
CORE_MISS=$(grep "TOTAL" ${CORE_TESTS} | awk '{ print $3; }')
CORE_COVERAGE=$(grep "TOTAL" ${CORE_TESTS} | awk '{ print $4; }')
MODULES_STMTS=$(grep "TOTAL" ${MODULES_TESTS} | awk '{ print $2; }')
MODULES_MISS=$(grep "TOTAL" ${MODULES_TESTS} | awk '{ print $3; }')
MODULES_COVERAGE=$(grep "TOTAL" ${MODULES_TESTS} | awk '{ print $4; }')
echo "CORE COVERAGE: ${CORE_COVERAGE}"
echo "MODULES COVERAGE: ${MODULES_COVERAGE}"
TOTAL_STMTS=$((${CORE_STMTS} + ${MODULES_STMTS}))
TOTAL_MISS=$((${CORE_MISS} + ${MODULES_MISS}))
TOTAL_COVERAGE=$((100 * (${TOTAL_STMTS} - ${TOTAL_MISS}) / ${TOTAL_STMTS}))
echo "TOTAL: ${TOTAL_COVERAGE}%"
# removal of temp files
rm ${CORE_TESTS}
rm ${MODULES_TESTS}
fi
# Rsync xunit transfer
if [ -n "${RSYNC_TARGET}" ]; then
rsync -iz "${XUNIT_OUT}" "${RSYNC_TARGET}/${BUILDER_NAME}-$(date +%s).xml"
rm "${XUNIT_OUT}"
fi
# Weboob-CI upload
if [ -n "${WEBOOB_CI_TARGET}" ]; then
JSON_MODULE_MATRIX=$(${PYTHON} "${WEBOOB_DIR}/tools/modules_testing_grid.py" "${XUNIT_OUT}" "${WEBOOB_CI_ORIGIN}")
curl -H "Content-Type: application/json" --data "${JSON_MODULE_MATRIX}" "${WEBOOB_CI_TARGET}/api/v1/modules"
rm "${XUNIT_OUT}"
fi
# safe removal
rm -r "${WEBOOB_TMPDIR}/icons" "${WEBOOB_TMPDIR}/repositories" "${WEBOOB_TMPDIR}/modules" "${WEBOOB_TMPDIR}/keyrings"
rm "${WEBOOB_TMPDIR}/backends" "${WEBOOB_TMPDIR}/sources.list"
rmdir "${WEBOOB_TMPDIR}"
[ $STATUS_CORE -gt 0 ] && exit $STATUS_CORE
exit $STATUS
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/setup-virtualenv.sh 0000775 0000000 0000000 00000002725 13264636610 0025437 0 ustar 00root root 0000000 0000000 #!/bin/sh -e
# install weboob inside a virtualenv, optionally with an associated weboob workdir
# can be combined with git-worktree
cd "$(dirname $0)/.."
SRC=$PWD
source=
VDIR=
usage () {
cat << EOF
Usage: $0 [-s] [-d DIR]
-s point sources.list to $SRC/modules instead of updates.weboob.org
-d DIR install virtualenv in DIR instead of a new dir
EOF
}
while getopts hsd: name
do
case $name in
s) source=y;;
d) VDIR="$OPTARG";;
h) usage
exit 0;;
?) usage
exit 2;;
esac
done
shift $(($OPTIND - 1))
PYTHON=${PYTHON-python}
echo "Using weboob source $SRC"
if [ -z "$VDIR" ]
then
VDIR=$(mktemp -d /tmp/weboob.venv.XXXXXX)
fi
cd "$VDIR"
echo "Creating env in $VDIR"
virtualenv -p "$(which "$PYTHON")" --system-site-packages "$VDIR"
. ./bin/activate
echo "Installing weboob in $VDIR"
"$PYTHON" -m pip install "$SRC"
mkdir workdir
export WEBOOB_WORKDIR=$VDIR/workdir
if [ "$source" = y ]
then
echo "file://$SRC/modules" > "$WEBOOB_WORKDIR/sources.list"
fi
./bin/weboob-config update
cat > use-weboob-local.sh << EOF
VDIR="$VDIR"
. "$VDIR/bin/activate"
export WEBOOB_WORKDIR="$VDIR/workdir"
EOF
cat << EOF
Installation complete in $VDIR.
Run ". $VDIR/use-weboob-local.sh" to start using it.
Run "$PYTHON -m pip install -U $SRC" to reinstall the core.
EOF
if [ "$source" != y ]
then
echo "You can add file://$SRC/modules into $VDIR/workdir/sources.list to use local modules instead of downloading modules."
fi
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/stable_backport.py 0000775 0000000 0000000 00000016367 13264636610 0025266 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python3
from __future__ import print_function
import time
import sys
import re
from contextlib import contextmanager
from os import system, path, makedirs
from subprocess import check_output, STDOUT, CalledProcessError
from collections import defaultdict
import shutil
from termcolor import colored
STABLE_VERSION = '1.3'
DEVEL_BRANCH = 'master'
@contextmanager
def log(message, success='done'):
print('%s... ' % message, end='', flush=True)
start = time.time()
try:
yield
except KeyboardInterrupt:
print(colored('abort', 'red'))
sys.exit(1)
except Exception as e:
print(colored('fail: %s' % e, 'red'))
raise
else:
print('%s %s' % (colored(success, 'green'),
colored('(%.2fs)' % (time.time() - start), 'blue')))
def create_compat_dir(name):
if not path.exists(name):
makedirs(name)
with open(path.join(name, '__init__.py'), 'w'):
pass
MANUAL_PORTS = [
'weboob.capabilities.bank',
'weboob.browser.pages',
'weboob.browser.exceptions',
'weboob.exceptions',
'weboob.browser.filters.html',
]
MANUAL_PORT_DIR = path.join(path.dirname(__file__), 'stable_backport_data')
class Error(object):
def __init__(self, filename, linenum, message):
self.filename = filename
self.linenum = linenum
self.message = message
self.compat_dir = path.join(path.dirname(filename), 'compat')
def __repr__(self):
return '<%s filename=%r linenum=%s message=%r>' % (type(self).__name__, self.filename, self.linenum, self.message)
def reimport_module(self, module):
# not a weboob module, probably a false positive.
if not module.startswith('weboob'):
return
dirname = module.replace('.', '/')
filename = dirname + '.py'
new_module = module.replace('.', '_')
target = path.join(self.compat_dir, '%s.py' % new_module)
base_module = '.'.join(module.split('.')[:-1])
try:
r = check_output('git show %s:%s' % (DEVEL_BRANCH, filename), shell=True, stderr=STDOUT).decode('utf-8')
except CalledProcessError:
# this file does not exist, perhaps a directory.
return
if module in MANUAL_PORTS:
shutil.copyfile(path.join(MANUAL_PORT_DIR, path.basename(target)), target)
else:
# Copy module from devel to a compat/ sub-module
with open(target, 'w') as fp:
for line in r.split('\n'):
# Replace relative imports to absolute ones
m = re.match(r'^from (\.\.?)([\w_\.]+) import (.*)', line)
if m:
if m.group(1) == '..':
base_module = '.'.join(base_module.split('.')[:-1])
fp.write('from %s.%s import %s\n' % (base_module, m.group(2), m.group(3)))
continue
# Inherit all classes by previous ones, if they already existed.
m = re.match(r'^class (\w+)\(([\w,\s]+)\):(.*)', line)
if m and path.exists(filename) and system('grep "^class %s" %s >/dev/null' % (m.group(1), filename)) == 0:
symbol = m.group(1)
trailing = m.group(3)
fp.write('from %s import %s as _%s\n' % (module, symbol, symbol))
fp.write('class %s(_%s):%s\n' % (symbol, symbol, trailing))
continue
fp.write('%s\n' % line)
# Particular case, in devel some imports have been added to
# weboob/browser/__init__.py
system(r'sed -i -e "s/from weboob.browser import/from weboob.browser.browsers import/g" %s'
% self.filename)
# Replace import to this module by a relative import to the copy in
# compat/
system(r'sed -i -e "%ss/from \([A-Za-z0-9_\.]\+\) import \(.*\)/from .compat.%s import \2/g" %s'
% (self.linenum, new_module, self.filename))
def remove_block(name, start):
lines = []
with open(name, 'r') as fd:
it = iter(fd)
for n in range(start - 1):
lines.append(next(it))
line = next(it)
level = len(re.match(r'^( *)', line).group(1))
for line in it:
if not line.strip():
continue
new = len(re.match(r'^( *)', line).group(1))
if new <= level:
lines.append(line)
break
lines.extend(it)
with open(name, 'w') as fd:
fd.write(''.join(lines))
class NoNameInModuleError(Error):
def fixup(self):
m = re.match(r"No name '(\w+)' in module '([\w\.]+)'", self.message)
module = m.group(2)
self.reimport_module(module)
class ImportErrorError(Error):
def fixup(self):
m = re.match(r"Unable to import '([\w\.]+)'", self.message)
module = m.group(1)
self.reimport_module(module)
def replace_all(expr, dest):
system(r"""for file in $(git ls-files modules | grep '\.py$');
do
sed -i -e "s/""" + expr + '/' + dest + """/g" $file
done""")
def output_lines(cmd):
return check_output(cmd, shell=True, stderr=STDOUT).decode('utf-8').rstrip().split('\n')
class StableBackport(object):
errors = {'E0611': NoNameInModuleError,
'E0401': ImportErrorError,
}
def main(self):
with log('Copying last version of modules from devel'):
system('git checkout --theirs %s modules' % DEVEL_BRANCH)
with log('Replacing version number'):
replace_all(r"""^\(\s*\)\(VERSION\)\( *\)=\( *\)[\"'][0-9]\+\..\+[\"']\(,\?\)$""",
r"""\1\2\3=\4'""" + STABLE_VERSION + r"""'\5""")
with log('Removing staling data'):
system('tools/stale_pyc.py')
system('find modules -type d -empty -delete')
system('git add -u')
with log('Lookup modules errors'):
r = check_output("pylint modules -f parseable -E -d all -e no-name-in-module,import-error; exit 0", shell=True, stderr=STDOUT).decode('utf-8')
dirnames = defaultdict(list)
for line in r.split('\n'):
m = re.match(r'([\w\./]+):(\d+): \[(\w+)[^\]]+\] (.*)', line)
if not m:
continue
filename = m.group(1)
linenum = m.group(2)
error = m.group(3)
msg = m.group(4)
dirnames[path.dirname(filename)].append(self.errors[error](filename, linenum, msg))
for dirname, errors in sorted(dirnames.items()):
with log('Fixing up %s errors in %s' % (colored(str(len(errors)), 'magenta'),
colored(dirname, 'yellow'))):
compat_dirname = path.join(dirname, 'compat')
create_compat_dir(compat_dirname)
for error in errors:
error.fixup()
system('git add %s' % compat_dirname)
with log('Custom fixups'):
replace_all("from weboob.browser.exceptions import LoggedOut", "from .weboob_browser_exceptions import LoggedOut")
system('git add -u')
if __name__ == '__main__':
StableBackport().main()
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/stable_backport_data/ 0000775 0000000 0000000 00000000000 13264636610 0025665 5 ustar 00root root 0000000 0000000 weboob_browser_exceptions.py 0000664 0000000 0000000 00000000120 13264636610 0033432 0 ustar 00root root 0000000 0000000 woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/stable_backport_data
from weboob.browser.exceptions import *
class LoggedOut(Exception):
pass
weboob_browser_filters_html.py 0000664 0000000 0000000 00000000150 13264636610 0033750 0 ustar 00root root 0000000 0000000 woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/stable_backport_data
from weboob.browser.filters.html import *
from weboob.browser.filters.standard import ColumnNotFound
weboob_browser_pages.py 0000664 0000000 0000000 00000000371 13264636610 0032360 0 ustar 00root root 0000000 0000000 woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/stable_backport_data
from weboob.browser.pages import *
from .weboob_browser_exceptions import LoggedOut
class LoginPage(object):
def on_load(self):
if not self.browser.logging_in:
raise LoggedOut()
super(LoginPage, self).on_load()
weboob_capabilities_bank.py 0000664 0000000 0000000 00000000552 13264636610 0033143 0 ustar 00root root 0000000 0000000 woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/stable_backport_data
import weboob.capabilities.bank as OLD
# can't import *, __all__ is incomplete...
for attr in dir(OLD):
globals()[attr] = getattr(OLD, attr)
__all__ = OLD.__all__
class CapBankWealth(CapBank):
pass
class CapBankPockets(CapBank):
pass
Account.TYPE_MORTGAGE = 17
Account.TYPE_CONSUMER_CREDIT = 18
Account.TYPE_REVOLVING_CREDIT = 19
weboob_capabilities_housing.py 0000664 0000000 0000000 00000001147 13264636610 0033705 0 ustar 00root root 0000000 0000000 woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/stable_backport_data
from weboob.capabilities.housing import *
ENERGY_CLASS = enum(A=u'A', B=u'B', C=u'C', D=u'D', E=u'E', F=u'F', G=u'G')
POSTS_TYPES = enum(RENT=u'RENT',
SALE=u'SALE',
SHARING=u'SHARING',
FURNISHED_RENT=u'FURNISHED_RENT',
VIAGER=u'VIAGER')
ADVERT_TYPES = enum(PROFESSIONAL=u'Professional', PERSONAL=u'Personal')
HOUSE_TYPES = enum(APART=u'Apartment',
HOUSE=u'House',
PARKING=u'Parking',
LAND=u'Land',
OTHER=u'Other',
UNKNOWN=u'Unknown')
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/stable_backport_data/weboob_exceptions.py 0000664 0000000 0000000 00000002343 13264636610 0031757 0 ustar 00root root 0000000 0000000
from weboob.exceptions import *
class AuthMethodNotImplemented(Exception):
pass
class CaptchaQuestion(Exception):
"""Site requires solving a CAPTCHA (base class)"""
# could be improved to pass the name of the backendconfig key
def __init__(self, type=None, **kwargs):
super(CaptchaQuestion, self).__init__("The site requires solving a captcha")
self.type = type
for key, value in kwargs.items():
setattr(self, key, value)
class ImageCaptchaQuestion(CaptchaQuestion):
type = 'image_captcha'
image_data = None
def __init__(self, image_data):
super(ImageCaptchaQuestion, self).__init__(self.type, image_data=image_data)
class NocaptchaQuestion(CaptchaQuestion):
type = 'g_recaptcha'
website_key = None
website_url = None
def __init__(self, website_key, website_url):
super(NocaptchaQuestion, self).__init__(self.type, website_key=website_key, website_url=website_url)
class RecaptchaQuestion(CaptchaQuestion):
type = 'g_recaptcha'
website_key = None
website_url = None
def __init__(self, website_key, website_url):
super(RecaptchaQuestion, self).__init__(self.type, website_key=website_key, website_url=website_url)
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/stale_pyc.py 0000775 0000000 0000000 00000001324 13264636610 0024075 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python
from __future__ import print_function
import os
import sys
root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))
verbose = '-v' in sys.argv
excludes = ('.git', '.svn', '__pycache__')
for dirpath, dirnames, filenames in os.walk(root):
for exclude in excludes:
try:
dirnames.remove(exclude)
except ValueError:
pass
for filename in filenames:
if filename.endswith('.pyc') or filename.endswith('pyo'):
if not os.path.exists(os.path.join(dirpath, filename[:-1])):
os.unlink(os.path.join(dirpath, filename))
if verbose:
print(os.path.join(dirpath, filename))
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/weboob_bash_completion 0000664 0000000 0000000 00000002422 13264636610 0026163 0 ustar 00root root 0000000 0000000 # Weboob completion for Bash
#
# vim: filetype=sh expandtab softtabstop=4 shiftwidth=4
#
# Copyright(C) 2010-2011 Christophe Benz
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see .
#
# This script can be distributed under the same license as the
# weboob or bash packages.
_weboob()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="$(${COMP_WORDS[0]} --shell-completion)"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
hash weboob-config 2>/dev/null &&
weboob_applications=$(weboob-config applications 2>/dev/null)
hash -d weboob-config
for application in $weboob_applications
do
complete -F _weboob $application
done
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/weboob_lint.py 0000775 0000000 0000000 00000002626 13264636610 0024423 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python3
from __future__ import print_function
# Hint: use this script with file:///path/to/local/modules/ in sources.list
# if you want to correctly check all modules.
from weboob.core import Weboob
import os
import sys
import subprocess
weboob = Weboob()
weboob.modules_loader.load_all()
backends_without_tests = []
backends_without_icons = []
backends_using_deprecated = []
for name, backend in weboob.modules_loader.loaded.items():
path = backend.package.__path__[0]
if not os.path.exists(os.path.join(path, 'test.py')):
backends_without_tests.append(name)
if not os.path.exists(os.path.join(path, 'favicon.png')) and \
not os.path.exists(os.path.join(weboob.repositories.icons_dir, '%s.png' % name)) and \
not backend.icon:
backends_without_icons.append(name)
if subprocess.call(['grep', '-q', '-r', 'weboob.deprecated.browser', path]) == 0:
backends_using_deprecated.append(name)
if backends_without_tests:
backends_without_tests.sort()
print('Modules without tests: %s' % backends_without_tests)
if backends_without_icons:
backends_without_icons.sort()
print('Modules without icons: %s' % backends_without_icons)
if backends_using_deprecated:
backends_using_deprecated.sort()
print('Modules using deprecated Browser 1: %s' % backends_using_deprecated)
if backends_without_tests or backends_without_icons:
sys.exit(1)
woob-a5989d1421c9248c5dd6e24358d7e98797f2438a-tools/tools/weboob_lint.sh 0000775 0000000 0000000 00000001322 13264636610 0024375 0 ustar 00root root 0000000 0000000 #!/bin/sh
# stop on failure
set -e
. "$(dirname $0)/common.sh"
[ -z "${TMPDIR}" ] && TMPDIR="/tmp"
# do not allow undefined variables anymore
set -u
WEBOOB_TMPDIR=$(mktemp -d "${TMPDIR}/weboob_lint.XXXXX")
# path to sources
WEBOOB_DIR=$(cd $(dirname $0)/.. && pwd -P)
touch "${WEBOOB_TMPDIR}/backends"
chmod 600 "${WEBOOB_TMPDIR}/backends"
echo "file://$WEBOOB_DIR/modules" > "${WEBOOB_TMPDIR}/sources.list"
export WEBOOB_WORKDIR="${WEBOOB_TMPDIR}"
export WEBOOB_DATADIR="${WEBOOB_TMPDIR}"
export PYTHONPATH="${WEBOOB_DIR}"
$PYTHON "${WEBOOB_DIR}/scripts/weboob-config" update
$PYTHON "${WEBOOB_DIR}/tools/weboob_lint.py"
# allow failing commands past this point
STATUS=$?
rm -rf "${WEBOOB_TMPDIR}"
exit $STATUS