pax_global_header 0000666 0000000 0000000 00000000064 14014251244 0014507 g ustar 00root root 0000000 0000000 52 comment=93297fa1488f03afedd5801ce30540b427cb2d96
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/ 0000775 0000000 0000000 00000000000 14014251244 0020536 5 ustar 00root root 0000000 0000000 woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/ 0000775 0000000 0000000 00000000000 14014251244 0021676 5 ustar 00root root 0000000 0000000 woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/boilerplate/ 0000775 0000000 0000000 00000000000 14014251244 0024200 5 ustar 00root root 0000000 0000000 woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/boilerplate/__init__.py 0000664 0000000 0000000 00000000000 14014251244 0026277 0 ustar 00root root 0000000 0000000 woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/boilerplate/boilerplate.py 0000775 0000000 0000000 00000004376 14014251244 0027071 0 ustar 00root root 0000000 0000000 #! /usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright(C) 2013-2019 Laurent Bachelier, Sébastien Jean
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with weboob. If not, see .
from __future__ import print_function
import argparse
import os
import subprocess
import sys
from importlib import import_module
BOILERPLATE_PATH = os.getenv(
'BOILERPLATE_PATH',
os.path.realpath(os.path.join(os.path.dirname(__file__), 'boilerplate_data')))
sys.path.append(os.path.dirname(__file__))
sys.path.append(BOILERPLATE_PATH)
from recipe import Recipe # NOQA
def u8(s):
if isinstance(s, bytes):
return s.decode('utf-8')
return s
def gitconfig(entry):
return u8(subprocess.check_output('git config -z --get %s' % entry, shell=True)[:-1])
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'-a', '--author',
default=gitconfig('user.name'), type=u8)
parser.add_argument(
'-e', '--email',
default=gitconfig('user.email'), type=u8)
subparsers = parser.add_subparsers(dest='recipe')
subparsers.required = True
recipes_module = import_module('recipes', package='boilerplate_data')
if hasattr(recipes_module, '__all__'):
for k in recipes_module.__all__:
getattr(recipes_module, k).configure_subparser(subparsers)
else:
for k in dir(recipes_module):
print(k)
if issubclass(getattr(recipes_module, k), Recipe) and not k.startswith('_'):
getattr(recipes_module, k).configure_subparser(subparsers)
args = parser.parse_args()
recipe = args.recipe_class(args)
recipe.generate()
if __name__ == '__main__':
main()
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/boilerplate/boilerplate_data/ 0000775 0000000 0000000 00000000000 14014251244 0027473 5 ustar 00root root 0000000 0000000 base_browser.pyt 0000664 0000000 0000000 00000001173 14014251244 0032631 0 ustar 00root root 0000000 0000000 woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/boilerplate/boilerplate_data <%inherit file="layout.pyt"/>
from weboob.browser import ${'LoginBrowser, need_login' if r.login else 'PagesBrowser'}, URL
from .pages import Page1, Page2
class ${r.classname}Browser(${'Login' if r.login else 'Pages'}Browser):
BASEURL = 'http://www.${r.name}.com'
page1 = URL('/page1\?id=(?P.+)', Page1)
page2 = URL('/page2', Page2)
% if login:
def do_login(self):
pass
@need_login
% endif
def get_stuff(self, _id):
self.page1.go(id=_id)
assert self.page1.is_here()
self.page.do_stuff(_id)
assert self.page2.is_here()
return self.page.do_more_stuff()
base_module.pyt 0000664 0000000 0000000 00000000613 14014251244 0032431 0 ustar 00root root 0000000 0000000 woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/boilerplate/boilerplate_data <%inherit file="layout.pyt"/>
from weboob.tools.backend import Module
from .browser import ${r.classname}Browser
__all__ = ['${r.classname}Module']
class ${r.classname}Module(Module):
NAME = '${r.name}'
DESCRIPTION = '${r.name} website'
MAINTAINER = '${r.author}'
EMAIL = '${r.email}'
LICENSE = 'LGPLv3+'
VERSION = '${r.version}'
BROWSER = ${r.classname}Browser
base_pages.pyt 0000664 0000000 0000000 00000000374 14014251244 0032247 0 ustar 00root root 0000000 0000000 woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/boilerplate/boilerplate_data <%inherit file="layout.pyt"/>
from weboob.browser.pages import HTMLPage
class Page1(HTMLPage):
def do_stuff(self, _id):
raise NotImplementedError()
class Page2(HTMLPage):
def do_more_stuff(self):
raise NotImplementedError()
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/boilerplate/boilerplate_data/base_test.pyt0000664 0000000 0000000 00000000212 14014251244 0032175 0 ustar 00root root 0000000 0000000 <%inherit file="layout.pyt"/>
from weboob.tools.test import BackendTest
class ${r.classname}Test(BackendTest):
MODULE = '${r.name}'
cap_module.pyt 0000664 0000000 0000000 00000001626 14014251244 0032267 0 ustar 00root root 0000000 0000000 woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/boilerplate/boilerplate_data <%inherit file="layout.pyt"/>
from weboob.tools.backend import Module${', BackendConfig' if r.login else ''}
% if login:
from weboob.tools.value import Value, ValueBackendPassword
% endif
from ${r.capmodulename} import ${r.capname}
from .browser import ${r.classname}Browser
__all__ = ['${r.classname}Module']
class ${r.classname}Module(Module, ${r.capname}):
NAME = '${r.name}'
DESCRIPTION = '${r.name} website'
MAINTAINER = '${r.author}'
EMAIL = '${r.email}'
LICENSE = 'LGPLv3+'
VERSION = '${r.version}'
BROWSER = ${r.classname}Browser
% if login:
CONFIG = BackendConfig(
Value('username', help='Username'),
ValueBackendPassword('password', help='Password'),
)
def create_default_browser(self):
return self.create_browser(self.config['username'].get(), self.config['password'].get())
% endif
% for meth in r.methods:
${''.join(meth)}
% endfor
comic_module.pyt 0000664 0000000 0000000 00000001400 14014251244 0032604 0 ustar 00root root 0000000 0000000 woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/boilerplate/boilerplate_data <%inherit file="layout.pyt"/>
from weboob.tools.capabilities.gallery.genericcomicreader import GenericComicReaderModule, DisplayPage
__all__ = ['${r.classname}Module']
class ${r.classname}Module(GenericComicReaderModule):
NAME = '${r.name}'
DESCRIPTION = u'${r.name} manga reading site'
MAINTAINER = u'${r.author}'
EMAIL = '${r.email}'
VERSION = '${r.version}'
LICENSE = 'LGPLv3+'
DOMAIN = 'www.${r.name}.com'
BROWSER_PARAMS = dict(
img_src_xpath="//img[@id='comic_page']/@src",
page_list_xpath="(//select[@id='page_select'])[1]/option/@value")
ID_REGEXP = r'[^/]+/[^/]+'
URL_REGEXP = r'.+${r.name}.com/(%s).+' % ID_REGEXP
ID_TO_URL = 'http://www.${r.name}.com/%s'
PAGES = {URL_REGEXP: DisplayPage}
comic_test.pyt 0000664 0000000 0000000 00000000443 14014251244 0032304 0 ustar 00root root 0000000 0000000 woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/boilerplate/boilerplate_data <%inherit file="layout.pyt"/>
from weboob.tools.capabilities.gallery.genericcomicreadertest import GenericComicReaderTest
class ${r.classname}BackendTest(GenericComicReaderTest):
MODULE = '${r.name}'
def test_download(self):
return self._test_download('${r.download_id}')
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/boilerplate/boilerplate_data/init.pyt 0000664 0000000 0000000 00000000154 14014251244 0031174 0 ustar 00root root 0000000 0000000 <%inherit file="layout.pyt"/>
from .module import ${r.classname}Module
__all__ = ['${r.classname}Module']
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/boilerplate/boilerplate_data/layout.pyt 0000664 0000000 0000000 00000001472 14014251244 0031552 0 ustar 00root root 0000000 0000000 ${coding}
# Copyright(C) ${r.year} ${r.author}
#
# This file is part of a weboob module.
#
# This weboob module is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This weboob module is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this weboob module. If not, see .
from __future__ import unicode_literals
${self.body()}\
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/boilerplate/boilerplate_data/recipes.py 0000664 0000000 0000000 00000011100 14014251244 0031470 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2013-2019 Laurent Bachelier, Sébastien Jean
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with weboob. If not, see .
from __future__ import print_function
import importlib
import sys
from recipe import Recipe
__all__ = ['BaseRecipe', 'CapRecipe', 'ComicRecipe', 'ComicTestRecipe']
class BaseRecipe(Recipe):
NAME = 'base'
def generate(self):
self.write('__init__.py', self.template('init'))
self.write('module.py', self.template('base_module'))
self.write('browser.py', self.template('base_browser'))
self.write('pages.py', self.template('base_pages'))
self.write('test.py', self.template('base_test'))
class CapRecipe(Recipe):
NAME = 'cap'
def __init__(self, args):
super(CapRecipe, self).__init__(args)
self.capname = args.capname
self.login = args.login
@classmethod
def configure_subparser(cls, subparsers):
subparser = super(CapRecipe, cls).configure_subparser(subparsers)
subparser.add_argument('--login', action='store_true', help='The site requires login')
subparser.add_argument('capname', help='Capability name')
return subparser
def find_module_cap(self):
if '.' not in self.capname:
return self.search_cap()
PREFIX = 'weboob.capabilities.'
if not self.capname.startswith(PREFIX):
self.capname = PREFIX + self.capname
try:
self.capmodulename, self.capname = self.capname.rsplit('.', 1)
except ValueError:
self.error('Cap name must be in format module.CapSomething or CapSomething')
try:
module = importlib.import_module(self.capmodulename)
except ImportError:
self.error('Module %r not found' % self.capmodulename)
try:
cap = getattr(module, self.capname)
except AttributeError:
self.error('Module %r has no such capability %r' % (self.capmodulename, self.capname))
return cap
def search_cap(self):
import pkgutil
import weboob.capabilities
modules = pkgutil.walk_packages(weboob.capabilities.__path__, prefix='weboob.capabilities.')
for _, capmodulename, __ in modules:
module = importlib.import_module(capmodulename)
if hasattr(module, self.capname):
self.capmodulename = capmodulename
return getattr(module, self.capname)
self.error('Capability %r not found' % self.capname)
def error(self, message):
print(message, file=sys.stderr)
sys.exit(1)
def methods_code(self, klass):
import inspect
methods = []
for name, member in inspect.getmembers(klass):
if inspect.ismethod(member) and name in klass.__dict__:
lines, _ = inspect.getsourcelines(member)
methods.append(lines)
return methods
def generate(self):
cap = self.find_module_cap()
self.methods = self.methods_code(cap)
self.write('__init__.py', self.template('init'))
self.write('module.py', self.template('cap_module'))
self.write('browser.py', self.template('base_browser'))
self.write('pages.py', self.template('base_pages'))
self.write('test.py', self.template('base_test'))
class ComicRecipe(Recipe):
NAME = 'comic'
def generate(self):
self.write('__init__.py', self.template('init'))
self.write('module.py', self.template('comic_module'))
class ComicTestRecipe(Recipe):
NAME = 'comic.test'
@classmethod
def configure_subparser(cls, subparsers):
subparser = super(ComicTestRecipe, cls).configure_subparser(subparsers)
subparser.add_argument('download_id', help='Download ID')
return subparser
def __init__(self, args):
super(ComicTestRecipe, self).__init__(args)
self.download_id = args.download_id
def generate(self):
self.write('test.py', self.template('comic_test'))
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/boilerplate/recipe.py 0000664 0000000 0000000 00000005365 14014251244 0026032 0 ustar 00root root 0000000 0000000 # -*- coding: utf-8 -*-
# Copyright(C) 2013-2019 Laurent Bachelier, Sébastien Jean
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with weboob. If not, see .
from __future__ import print_function
import codecs
import datetime
import os
import sys
from mako.lookup import TemplateLookup
from weboob import __version__
WEBOOB_MODULES = os.getenv(
'WEBOOB_MODULES',
os.path.realpath(os.path.join(os.path.dirname(__file__), '../../modules')))
BOILERPLATE_PATH = os.getenv(
'BOILERPLATE_PATH',
os.path.realpath(os.path.join(os.path.dirname(__file__), 'boilerplate_data')))
TEMPLATES = TemplateLookup(directories=[BOILERPLATE_PATH], input_encoding='utf-8')
def write(target, contents):
if not os.path.isdir(os.path.dirname(target)):
os.makedirs(os.path.dirname(target))
if os.path.exists(target):
print("%s already exists." % target, file=sys.stderr)
sys.exit(4)
with codecs.open(target, mode='w', encoding='utf-8') as f:
f.write(contents)
print('Created %s' % target)
class Recipe(object):
@classmethod
def configure_subparser(cls, subparsers):
subparser = subparsers.add_parser(cls.NAME)
subparser.add_argument('name', help='Module name')
subparser.set_defaults(recipe_class=cls)
return subparser
def __init__(self, args):
self.name = args.name.lower().replace(' ', '')
self.classname = args.name.title().replace(' ', '').replace('_', '')
self.year = datetime.date.today().year
self.author = args.author
self.email = args.email
self.version = __version__
self.login = False
def write(self, filename, contents):
return write(os.path.join(WEBOOB_MODULES, self.name, filename), contents)
def template(self, name, **kwargs):
if '.' not in name:
name += '.pyt'
return TEMPLATES.get_template(name) \
.render(r=self,
# workaround, as it's also a mako directive
coding='# -*- coding: utf-8 -*-',
login=self.login,
**kwargs).strip() + u'\n'
def generate(self):
raise NotImplementedError()
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/check_xpath.py 0000775 0000000 0000000 00000011403 14014251244 0024533 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python3
# Copyright(C) 2017 Vincent A
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with weboob. If not, see .
from __future__ import print_function
import ast
import fnmatch
import os
import traceback
import lxml.etree
from weboob.browser.filters import standard
class Error(SyntaxError):
def __init__(self, file, line, message):
super(Error, self).__init__('%s:%s: %s' % (file, line, message))
self.file = file
self.line = line
def do_visits(*funcs):
def wrapper(self, node):
for func in funcs:
func(self, node)
self.generic_visit(node)
return wrapper
class Visitor(ast.NodeVisitor):
def __init__(self, file, *args, **kwargs):
self.warnings = kwargs.pop('warnings', False)
super(Visitor, self).__init__(*args, **kwargs)
self.file = file
self.filters = []
self.filters.extend(f for f in dir(standard) if isinstance(getattr(standard, f), type) and issubclass(getattr(standard, f), standard.CleanText))
self.filters.extend(['Regexp', 'XPath', 'Attr', 'Link'])
self.element_context = []
def check_xpath(self, s, lineno):
try:
lxml.etree.XPath(s)
except lxml.etree.XPathSyntaxError as exc:
raise Error(self.file, lineno, exc)
if self.warnings:
if not s.lstrip('(').startswith('.') and len(self.element_context) >= 2:
if self.element_context[-1] == 'ItemElement' and self.element_context[-2] in ('TableElement', 'ListElement'):
print('%s:%s: probable missing "." at start of XPath' % (self.file, lineno))
def _item_xpath(self, node):
try:
target, = node.targets
except ValueError:
return
if not isinstance(target, ast.Name) or target.id != 'item_xpath':
return
try:
if self.element_context[-1] not in ('TableElement', 'ListElement'):
return
except IndexError:
return
if not isinstance(node.value, ast.Str):
return
self.check_xpath(node.value.s, node.lineno)
visit_Assign = do_visits(_item_xpath)
def _xpath_call(self, node):
if not isinstance(node.func, ast.Attribute):
return
if node.func.attr != 'xpath':
return
try:
if not isinstance(node.args[0], ast.Str):
return
except IndexError:
return
self.check_xpath(node.args[0].s, node.lineno)
def _filter_call(self, node):
if not isinstance(node.func, ast.Name):
return
if node.func.id not in self.filters:
return
try:
if not isinstance(node.args[0], ast.Str):
return
except IndexError:
return
self.check_xpath(node.args[0].s, node.lineno)
visit_Call = do_visits(_xpath_call, _filter_call)
def visit_ClassDef(self, node):
has_element = False
for basenode in node.bases:
if isinstance(basenode, ast.Name) and basenode.id in ('ListElement', 'ItemElement', 'TableElement'):
self.element_context.append(basenode.id)
has_element = True
break
self.generic_visit(node)
if has_element:
self.element_context.pop()
def search_py(root):
for path, dirs, files in os.walk(root):
dirs.sort()
for f in fnmatch.filter(files, '*.py'):
yield os.path.join(path, f)
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser(description="Check XPath definitions")
parser.add_argument('-w', '--warnings', action='store_true')
args = parser.parse_args()
modpath = os.getenv('WEBOOB_MODULES', os.path.normpath(os.path.dirname(__file__) + '/../modules'))
for fn in search_py(modpath):
with open(fn) as fd:
try:
node = ast.parse(fd.read(), fn)
except SyntaxError as exc:
print('In file', fn)
traceback.print_exc(exc)
try:
Visitor(fn, warnings=args.warnings).visit(node)
except SyntaxError as exc:
print(exc)
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/common.sh 0000664 0000000 0000000 00000001657 14014251244 0023533 0 ustar 00root root 0000000 0000000 VER=3
if [ "${1-}" = "-2" ]; then
VER=2
shift
fi
python2_not_supported () {
echo "Python2 is not installed" >&2
return 1
}
if [ -z "${PYTHON2-}" ]; then
PYTHON2=python2_not_supported
which python2.7 >/dev/null 2>&1 && PYTHON2=$(which python2.7)
which python2 >/dev/null 2>&1 && PYTHON2=$(which python2)
fi
if [ -z "${PYTHON3-}" ]; then
which python3.5 >/dev/null 2>&1 && PYTHON3=$(which python3.5)
which python3.6 >/dev/null 2>&1 && PYTHON3=$(which python3.6)
which python3.7 >/dev/null 2>&1 && PYTHON3=$(which python3.7)
which python3.8 >/dev/null 2>&1 && PYTHON3=$(which python3.8)
which python3 >/dev/null 2>&1 && PYTHON3=$(which python3)
fi
if [ -z "${PYTHON-}" ]; then
which python >/dev/null 2>&1 && PYTHON=$(which python)
if [ $VER -eq 2 -a -n "${PYTHON2}" ]; then
PYTHON=${PYTHON2}
elif [ $VER -eq 3 -a -n "${PYTHON3}" ]; then
PYTHON=${PYTHON3}
fi
fi
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/debpydep.py 0000775 0000000 0000000 00000002431 14014251244 0024047 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
from __future__ import print_function
import os
import subprocess
import sys
selection = set()
dependencies = set()
for root, dirs, files in os.walk(sys.argv[1]):
for f in files:
if f.endswith('.py') and f != '__init__.py':
s = "from %s import %s" % (root.strip('/').replace('/', '.'), f[:-3])
try:
exec(s)
except ImportError as e:
print(str(e), file=sys.stderr)
else:
m = eval(f[:-3])
for attrname in dir(m):
try:
attr = getattr(m, attrname)
selection.add(attr.__file__)
except AttributeError:
pass
for f in selection:
f = f.replace('.pyc', '.py')
try:
f = os.path.abspath(os.path.join(os.path.split(f)[0], os.readlink(f)))
except OSError:
pass
p = subprocess.Popen(['/usr/bin/dpkg', '-S', f], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if p.wait() == 0:
for line in p.stdout.readlines():
line = line.decode('utf-8')
dependencies.add(line.strip().split(':')[0])
else:
print('not found: %s' % f)
for d in dependencies:
print(d)
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/har-to-old.py 0000775 0000000 0000000 00000005773 14014251244 0024235 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python3
from argparse import ArgumentParser, FileType
from base64 import b64decode
import json
import mimetypes
import os
from pathlib import Path
from urllib.parse import urlparse
def write_request(entry, fd):
entry = entry['request']
# we should put the path, but since requests does not output the Host header
# we would not know what was the host
fd.write(f"{entry['method']} {entry['url']} {entry['httpVersion']}\n".encode())
for header in entry['headers']:
fd.write(f"{header['name']}: {header['value']}\n".encode())
fd.write(b'\n')
if 'postData' in entry:
if entry['postData'].get('x-binary'):
# non-standard key emitted by weboob
body = entry['postData']['text'].encode('latin-1')
else:
body = entry['postData']['text'].encode()
fd.write(body)
def write_response(entry, fd):
entry = entry['response']
fd.write(f"{entry['httpVersion']} {entry['status']} {entry['statusText']}\n")
for header in entry['headers']:
fd.write(f"{header['name']}: {header['value']}\n")
def write_body(entry, fd):
entry = entry['response']
if entry['content'].get('encoding') == 'base64':
data = b64decode(entry['content']['text'])
else:
data = entry['content']['text'].encode('utf-8')
fd.write(data)
def guess_extension(entry):
headers = entry['response']['headers']
ctype = next((header['value'] for header in headers if header['name'].lower() == 'content-type'), '')
# due to http://bugs.python.org/issue1043134
if ctype == 'text/plain':
ext = '.txt'
else:
# try to get an extension (and avoid adding 'None')
ext = mimetypes.guess_extension(ctype, False) or ''
return ext
def main():
def extract(n, destdir):
os.makedirs(destdir, exist_ok=True)
entry = data['log']['entries'][n]
ext = guess_extension(entry)
name = Path(urlparse(entry['request']['url']).path).stem
prefix = f'{destdir}/{n + 1:03d}-{entry["response"]["status"]}{name and f"-{name}"}{ext}'
with open(f'{prefix}-request.txt', 'wb') as fd:
write_request(entry, fd)
with open(f'{prefix}-response.txt', 'w') as fd:
write_response(entry, fd)
with open(prefix, 'wb') as fd:
write_body(entry, fd)
parser = ArgumentParser()
parser.add_argument('file', type=FileType('r'), help='HAR file to extract')
parser.add_argument('destdir', nargs='?', default=None, help='Destination directory for extracted files')
args = parser.parse_args()
if args.destdir is None:
# Automatically generate destdir if not provided
if args.file.name.endswith('.har'):
args.destdir = args.file.name[:-4]
else:
args.destdir = f'{args.file.name}_content'
data = json.load(args.file)
for n in range(len(data['log']['entries'])):
print('extracting request', n)
extract(n, args.destdir)
if __name__ == '__main__':
main()
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/hooks/ 0000775 0000000 0000000 00000000000 14014251244 0023021 5 ustar 00root root 0000000 0000000 woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/hooks/check-continuations.py 0000775 0000000 0000000 00000002405 14014251244 0027347 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python3
from pathlib import Path
import runpy
import sys
import tokenize
mod = runpy.run_path(str(Path(__file__).with_name('checkerlib.py')))
Checker = mod['Checker']
class ContinuationChecker(Checker):
def __init__(self, filename):
super().__init__(filename)
self.parse_tokens()
def check_lines(self):
previous_token = None
for token in self.tokens:
if token.type in (tokenize.NEWLINE, tokenize.NL):
previous_token = None
continue
# if two adjacent tokens don't "touch" on the same line
# and no NL/NEWLINE is involved, it's a line continuation!
if previous_token and previous_token.end[0] != token.start[0]:
assert previous_token.line.endswith('\\\n')
self.add_error(
"backslashed line-continuations are forbidden",
line=token.start[0], col=token.start[1],
)
previous_token = token
return self.ok
args = mod['parser'].parse_args()
exit_code = 0
for file in mod['files_to_check'](args):
checker = ContinuationChecker(file)
checker.parse_noqa()
if not checker.check_lines():
exit_code = 1
sys.exit(exit_code)
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/hooks/check-ifexpr.py 0000775 0000000 0000000 00000001415 14014251244 0025747 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python3
import ast
from pathlib import Path
import runpy
import sys
mod = runpy.run_path(str(Path(__file__).with_name('checkerlib.py')))
Checker = mod['Checker']
class IfExprKiller(Checker, ast.NodeVisitor):
def __init__(self, filename):
super().__init__(filename)
self.parse_ast()
def check(self):
self.visit(self.tree)
return self.ok
def visit_IfExp(self, node):
self.add_error(
"if-expressions are forbidden",
line=node.lineno, col=node.col_offset,
)
args = mod['parser'].parse_args()
exit_code = 0
for file in mod['files_to_check'](args):
checker = IfExprKiller(file)
checker.parse_noqa()
if not checker.check():
exit_code = 1
sys.exit(exit_code)
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/hooks/check-line-length.py 0000775 0000000 0000000 00000005271 14014251244 0026664 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python3
from pathlib import Path
import runpy
import sys
import tokenize
mod = runpy.run_path(str(Path(__file__).with_name('checkerlib.py')))
Checker = mod['Checker']
class LineLengthChecker(Checker):
def __init__(self, filename):
super().__init__(filename)
self.parse_tokens()
def check_lines(self):
crossing = False
offending_token = None
# string and comments are the only tokens that are allowed to cross line border
# for comments, only NL or NEWLINE can be after COMMENT
# STRING OP* (NL | NEWLINE): ok
# STRING
# .start[0]: line number of start, starting at 1
# .end[0]: line number of end, included, starting at 1
# .start[1]: column number of start, starting at 0
# .end[1]: column number of end, excluded, starting at 0
# ->
# .start[1] + 1 == column number of start, starting at 1
# .end[1] == column number of end, included, starting at 1
for token in self.tokens:
if token.type == tokenize.STRING:
if token.start[1] + 1 >= args.line_length:
self.add_error(
"string starts after max line length",
line=token.start[0]
)
elif token.end[1] >= args.line_length:
crossing = True
elif token.type == tokenize.COMMENT:
if token.start[1] + 1 >= args.line_length:
self.add_error(
"comment starts after max line length",
line=token.start[0]
)
elif token.end[1] >= args.line_length:
crossing = True
elif token.type in (tokenize.NL, tokenize.NEWLINE):
if token.start[1] > args.line_length and not crossing:
if not offending_token:
offending_token = token
self.add_error(
"line too long not due to a string",
line=offending_token.start[0]
)
crossing = False
offending_token = None
elif token.type != tokenize.OP:
if not offending_token:
offending_token = token
crossing = False
return self.ok
parser = mod['parser']
parser.add_argument('-l', '--line-length', type=int, default=80)
args = parser.parse_args()
exit_code = 0
for file in mod['files_to_check'](args):
checker = LineLengthChecker(file)
checker.parse_noqa()
if not checker.check_lines():
exit_code = 1
sys.exit(exit_code)
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/hooks/check-op-precedence.py 0000775 0000000 0000000 00000014445 14014251244 0027172 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python3
import ast
from pathlib import Path
import runpy
import sys
import tokenize
from asttokens import ASTTokens
mod = runpy.run_path(str(Path(__file__).with_name('checkerlib.py')))
Checker = mod['Checker']
bit_ops = (ast.BitOr, ast.BitAnd, ast.BitXor, ast.LShift, ast.RShift)
math_ops = (ast.Add, ast.Sub, ast.Mult, ast.MatMult, ast.Div, ast.FloorDiv, ast.Mod)
dubious_ops = [
bit_ops,
]
dubious_ops_groups = [
(bit_ops, math_ops),
]
class OpPrioVerifier(Checker, ast.NodeVisitor):
def __init__(self, filename):
super().__init__(filename)
with open(self.filename) as fd:
self.astt = ASTTokens(fd.read(), parse=True)
self.tree = self.astt.tree
self.tokens = self.astt.tokens
def has_paren(self, tokens, expected_paren):
for token in tokens:
if token.type in (tokenize.NL, tokenize.COMMENT):
continue
elif token.type == tokenize.OP and token.string == expected_paren:
return True
else:
return False
else:
return False
def same_types(self, a, b):
ta = type(a)
tb = type(b)
return ta is tb
def visit_BoolOp(self, node):
first = True
for child in node.values:
if isinstance(child, ast.BoolOp) and not self.same_types(child.op, node.op):
if first:
# There are at least 2 elements in a BoolOp, so for the first, the
# tokens are:
# "(" (COMMENT? NL)* child (COMMENT? NL)* ")" (COMMENT? NL)* node.op
#
# Don't look for the open paren because since it's the first element,
# the open paren could be something else. For example:
#
# f(a and b or c)
#
# node.values are (roughly) ["a and b", "c"]. There is an open paren
# before "a and b" but it's not related to "a and b".
# Instead, look for a closing paren between "a and b" and the `node`
# bool operator "or".
if not self.has_paren(self.tokens[child.last_token.index + 1:], ')'):
op_token = self.search_boolop_token(child)
self.add_error(
"ambiguous precedence between 'or' and 'and'",
line=op_token.start[0]
)
else:
# Conversely, there must be a `node` bool operator before `child`'s
# tokens. Look for an open paren before `child`.
# node.op (COMMENT? NL)* "(" (COMMENT? NL)* child (COMMENT? NL)* ")"
if not self.has_paren(self.tokens[child.first_token.index - 1::-1], '('):
op_token = self.search_boolop_token(child)
self.add_error(
"ambiguous precedence between 'or' and 'and'",
line=op_token.start[0]
)
first = False
self.generic_visit(node)
def search_boolop_token(self, node):
# search for the first and/or operator token
assert isinstance(node, ast.BoolOp)
first_operand = node.values[0].last_token
for token in self.tokens[first_operand.index + 1:]:
if token.type in (tokenize.NL, tokenize.COMMENT):
continue
else:
assert token.type == tokenize.NAME, "expected an operator after left node"
assert token.string in ('and', 'or')
return token
raise AssertionError("expected an operator after left node")
def visit_BinOp(self, node):
if isinstance(node.left, ast.BinOp):
if not self.check_binop(node, node.left):
# same as for BoolOp, look for a closing paren after first operand
if not self.has_paren(self.tokens[node.left.last_token.index + 1:], ')'):
op_token = self.search_binop_token(node.left)
parent_token = self.search_binop_token(node)
self.add_error(
f"ambiguous precedence between {parent_token.string!r} and {op_token.string!r}",
line=op_token.start[0]
)
if isinstance(node.right, ast.BinOp):
if not self.check_binop(node, node.right):
# same as for BoolOp, look for an opening paren before second operand
if not self.has_paren(self.tokens[node.right.first_token.index - 1::-1], '('):
op_token = self.search_binop_token(node.right)
parent_token = self.search_binop_token(node)
self.add_error(
f"ambiguous precedence between {parent_token.string!r} and {op_token.string!r}",
line=op_token.start[0]
)
self.generic_visit(node)
def search_binop_token(self, node):
# search for the operator token
assert isinstance(node, ast.BinOp)
for token in self.tokens[node.left.last_token.index + 1:]:
if token.type in (tokenize.NL, tokenize.COMMENT):
continue
else:
assert token.type == tokenize.OP, "expected an operator after left node"
return token
raise AssertionError("expected an operator after left node")
def check_binop(self, parent, child):
if self.same_types(parent.op, child.op):
return True
for dubious in dubious_ops:
if isinstance(parent.op, dubious) and isinstance(child.op, dubious):
return False
for dub1, dub2 in dubious_ops_groups:
if (
(isinstance(parent.op, dub1) and isinstance(child.op, dub2))
or (isinstance(parent.op, dub2) and isinstance(child.op, dub1))
):
return False
return True
def check(self):
self.visit(self.tree)
return self.ok
args = mod['parser'].parse_args()
exit_code = 0
for file in mod['files_to_check'](args):
verifier = OpPrioVerifier(file)
if not verifier.check():
exit_code = 1
sys.exit(exit_code)
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/hooks/check-stringcut.py 0000775 0000000 0000000 00000003674 14014251244 0026505 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python3
from pathlib import Path
import runpy
import sys
import tokenize
mod = runpy.run_path(str(Path(__file__).with_name('checkerlib.py')))
Checker = mod['Checker']
# typical code that we absolutely want to prevent (look closely at the commas):
# foo = URL(
# "/one/url/",
# "/a/second/url"
# "/that/is/actually"
# "/very/long",
# "/and/a/third/url"
# )
class StringChecker(Checker):
def __init__(self, filename):
super().__init__(filename)
self.parse_tokens()
def check_strings(self):
# STRING NEWLINE STRING: ok
# STRING (COMMENT? NL)* STRING: bad
in_str = False
for token in self.tokens:
if token.type == tokenize.STRING:
self.check_continuation(token)
if in_str:
if token.type == tokenize.STRING:
self.add_error(
"implicitly concatenated strings are forbidden",
line=token.start[0], col=token.start[1],
)
elif token.type not in (tokenize.NL, tokenize.COMMENT):
in_str = False
elif token.type == tokenize.STRING:
in_str = True
return self.ok
# check_continuation avoids such code:
# foo = "bar\
# baz"
def check_continuation(self, token):
if (
token.start[0] == token.end[0]
or token.string.endswith('"""') or token.string.endswith("'''")
):
return True
assert '\\\n' in token.string
self.add_error(
"line-continuations in a string are forbidden",
line=token.start[0], col=token.start[1],
)
args = mod['parser'].parse_args()
exit_code = 0
for file in mod['files_to_check'](args):
checker = StringChecker(file)
checker.parse_noqa()
if not checker.check_strings():
exit_code = 1
sys.exit(exit_code)
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/hooks/check-trailing-commas.py 0000775 0000000 0000000 00000030130 14014251244 0027534 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python3
import ast
from collections import namedtuple
from pathlib import Path
import runpy
import sys
import tokenize
from asttokens import ASTTokens
mod = runpy.run_path(str(Path(__file__).with_name('checkerlib.py')))
Checker = mod['Checker']
# these are ok:
# {1: 2, 3: 4}
# {
# 1: 2,
# }
# {
# 1: 2, # foo
# }
# these are bad:
# {
# 1: 2
# }
# {
# 1: 2
# ,
# }
# {
# 1: 2,}
class TrailingCommaVerifier(Checker, ast.NodeVisitor):
def __init__(self, filename):
super().__init__(filename)
with open(self.filename) as fd:
self.astt = ASTTokens(fd.read(), parse=True)
self.tree = self.astt.tree
self.tokens = self.astt.tokens
def should_skip(self, node, attr):
if not getattr(node, attr):
# it's an empty container
return True
first_elem = getattr(node, attr)[0]
return (
# that's not multiline, we don't care
node.first_token.start[0] == node.last_token.start[0]
# single multiline element, acceptable for compacity if glued to container
# e.g. [[[\n1,\n2,\n3,\n]]]
or (
len(getattr(node, attr)) == 1
and node.first_token.start[0] == first_elem.first_token.start[0]
and node.last_token.end[0] == first_elem.last_token.end[0]
)
)
def check_trailing(self, node, attr):
if self.should_skip(node, attr):
return
last_elt_token = getattr(node, attr)[-1].last_token
# for list "[0,(1+2),]", last_elt_token is "2"
# after last_elt_token, we want this:
# RPAR* COMMA (COMMENT? NL)+ end-delimiter
all_tokens = self.tokens
has_comma = False
has_nl = False
for idx in range(last_elt_token.index + 1, node.last_token.index):
if all_tokens[idx].type == tokenize.OP:
if all_tokens[idx].string == ',':
assert not has_comma
has_comma = True
else:
assert all_tokens[idx].string == ')'
elif all_tokens[idx].type == tokenize.NL:
has_nl = True
break
else:
assert all_tokens[idx].type == tokenize.COMMENT
if not has_comma:
self.add_error(
'expected a comma after element',
line=last_elt_token.end[0],
)
elif not has_nl:
self.add_error(
'expected end of line between comma and literal end',
line=last_elt_token.end[0],
)
def check_first_indent(self, node, attr):
if self.should_skip(node, attr):
return
if getattr(node, attr)[0]:
first_elt_token = getattr(node, attr)[0].first_token
else:
assert isinstance(node, ast.Dict) and attr == 'keys', "None node should only be in dict keys"
# a None node in ast.Dict.keys happens in case of **mapping
# use the tokens of the value then, nevermind the "**" tokens
first_elt_token = node.values[0].first_token
if first_elt_token.start[0] == node.first_token.start[0]:
self.add_error(
'first element should start on a new line',
line=first_elt_token.start[0],
)
def visit_Tuple(self, node):
# no assert to verify tokens, because there might be no delimiters:
# foo = a, b # no parentheses around that tuple
# such a tuple cannot be multiline, so we won't check trailing commas
self.visit_simple_container(node)
def visit_simple_container(self, node):
self.check_trailing(node, 'elts')
self.check_first_indent(node, 'elts')
self.generic_visit(node)
def visit_List(self, node):
assert node.first_token.string == '['
assert node.last_token.string == ']'
self.visit_simple_container(node)
def visit_Set(self, node):
assert node.first_token.string == '{'
assert node.last_token.string == '}'
self.visit_simple_container(node)
def visit_Dict(self, node):
assert node.first_token.string == '{'
assert node.last_token.string == '}'
self.check_trailing(node, 'values')
self.check_first_indent(node, 'keys')
self.generic_visit(node)
def visit_Call(self, node):
assert node.last_token.string == ')'
self.generic_visit(node)
if not (node.args or node.keywords):
# no params at all, don't bother
return
callable_last = node.func.last_token
# for call "(foo)(bar)", node.func.last_token is "foo"
# after callable_last, there must be only this:
# (COMMENT? NL)* RPAR* (COMMENT? NL)* LPAR
for token in self.tokens[callable_last.index + 1:]:
if token.type == tokenize.OP:
if token.string == '(':
open_paren = token
break
assert token.string == ')'
else:
assert token.type in (tokenize.NL, tokenize.COMMENT)
else:
raise AssertionError('could not find opening paren')
if open_paren.start[0] == node.last_token.start[0]:
# '(' and ')' on the same line
return
if node.args:
param_first_token = node.args[0].first_token
else:
assert node.keywords
# here we go again, the keyword is just a string, no .token attribute
for token in self.tokens[open_paren.index + 1:]:
if token.type == tokenize.NAME:
param_first_token = token
break
elif token.type == tokenize.OP:
assert token.string == '**' and not node.keywords[0].arg
param_first_token = token
break
assert token.type in (tokenize.NL, tokenize.COMMENT)
else:
raise AssertionError('could not find first keyword')
if open_paren.start[0] == param_first_token.start[0]:
# allow compact multiline call if single parameter
# e.g. "foo([\n1,\n2,\n])"
if (
len(node.args) == 1 and not node.keywords
and node.last_token.end[0] == node.args[0].last_token.end[0]
):
return
elif (
len(node.keywords) == 1 and not node.args
and node.last_token.end[0] == node.keywords[0].value.last_token.end[0]
):
return
self.add_error(
'first param should start on a new line',
line=param_first_token.start[0],
)
def visit_ImportFrom(self, node):
assert node.first_token.string == 'from' # asttokens bug
proc = SetTokensOnImport(node, self.tokens[node.first_token.index:node.last_token.index + 1])
proc.process()
# TODO reimplement using tokens only, we don't really need the AST for imports
self.check_trailing(node, 'names')
def check(self):
self.visit(self.tree)
return self.ok
class Tokens:
Matcher = namedtuple('Matcher', ('type', 'string'))
# string = None means whatever string
FROM = Matcher(tokenize.NAME, 'from')
IMPORT = Matcher(tokenize.NAME, 'import')
AS = Matcher(tokenize.NAME, 'as')
ANY_NAME = Matcher(tokenize.NAME, None)
DOT = Matcher(tokenize.OP, '.')
COMMA = Matcher(tokenize.OP, ',')
OPEN = Matcher(tokenize.OP, '(')
CLOSE = Matcher(tokenize.OP, ')')
STAR = Matcher(tokenize.OP, '*')
class RuleBase:
# helper for passing in a stream of tokens and checking if we have the desired tokens
def __init__(self, tokens):
self.tokens = tokens
self.current = 0
@staticmethod
def token_match(token, matcher):
if matcher.type != token.type:
return False
if matcher.string is not None and matcher.string != token.string:
return False
return True
def peek(self):
# just peek current token without advancing cursor
return self.tokens[self.current]
def peek_prev(self):
assert self.current > 0
return self.tokens[self.current - 1]
def peek_normal(self):
# peek the first non-NL, non-COMMENT token
# (and advance reading cursor to it)
while self.current < len(self.tokens):
token = self.tokens[self.current]
if token.type not in (tokenize.NL, tokenize.COMMENT):
return token
self.current += 1
def read(self):
# get current token and advance cursor
val = self.peek()
self.current += 1
return val
def has(self, matcher):
token = self.peek_normal()
if token is None:
# end of tokens
return False
return self.token_match(token, matcher)
def probe(self, matcher):
# if token matches, advances cursor
# else, stay in place
if self.has(matcher):
self.current += 1
return True
return False
def match(self, matcher):
assert self.probe(matcher)
class SetTokensOnImport(RuleBase):
# this is due to https://github.com/gristlabs/asttokens/issues/60
# we browse the given Import node and set the correct tokens as first_token/last_token
# token sequences are listed in https://docs.python.org/3/reference/simple_stmts.html#the-import-statement
def __init__(self, node, tokens):
super().__init__(tokens)
self.node = node
self.aliases_iter = iter(node.names)
self.current_alias = None
def process(self):
if self.probe(Tokens.IMPORT):
# "import foo" and related forms
self.do_basic_import()
elif self.probe(Tokens.FROM):
# "from foo import bar" and related forms
self.do_from_import()
else:
raise AssertionError('nothing was probed')
def do_basic_import(self):
self.do_module_as()
while self.probe(Tokens.COMMA):
self.do_module_as()
def do_from_import(self):
while self.probe(Tokens.DOT):
# example: from .foo import bar
pass
if not self.probe(Tokens.IMPORT):
# here, we are not in: from . import foo
self.do_module()
self.match(Tokens.IMPORT)
if self.probe(Tokens.STAR):
# example: from foo import *
return
paren = self.probe(Tokens.OPEN)
self.do_identifier_as()
while self.probe(Tokens.COMMA):
if not self.has(Tokens.ANY_NAME):
# that was a trailing comma
break
self.do_identifier_as()
if paren:
self.match(Tokens.CLOSE)
def do_module_as(self):
# match: module ["as" identifier]
# and set tokens on nodes
self.current_alias = next(self.aliases_iter)
self.current_alias.first_token = self.peek_normal()
self.do_module()
self.current_alias.last_token = self.peek_prev()
if self.probe(Tokens.AS):
self.match(Tokens.ANY_NAME)
self.current_alias.last_token = self.peek_prev()
def do_identifier_as(self):
# match: identifier ["as" identifier]
# and set tokens on nodes
self.current_alias = next(self.aliases_iter)
self.current_alias.first_token = self.peek_normal()
self.current_alias.last_token = self.peek()
self.match(Tokens.ANY_NAME)
if self.probe(Tokens.AS):
self.match(Tokens.ANY_NAME)
self.current_alias.last_token = self.peek_prev()
def do_module(self):
# match: identifier ("." identifier)*
self.match(Tokens.ANY_NAME)
while self.probe(Tokens.DOT):
self.match(Tokens.ANY_NAME)
args = mod['parser'].parse_args()
exit_code = 0
for file in mod['files_to_check'](args):
verifier = TrailingCommaVerifier(file)
if not verifier.check():
exit_code = 1
sys.exit(exit_code)
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/hooks/checkerlib.py 0000664 0000000 0000000 00000004505 14014251244 0025472 0 ustar 00root root 0000000 0000000 # python3-only
import ast
import argparse
from pathlib import Path
import subprocess
import sys
import tokenize
def get_lines(cmd):
return subprocess.check_output(cmd, encoding='utf-8').strip('\n').split('\n')
parser = argparse.ArgumentParser()
parser.add_argument('files', nargs='*')
current_file = Path(__file__).resolve()
git_root = current_file.parent.parent.parent
def files_to_check(args, pattern=None):
if pattern is None:
pattern = '^# flake8: compatible'
if args.files:
to_check = args.files
else:
try:
to_check = get_lines([
'git', 'grep', '-l', pattern,
git_root / 'modules/**/*.py', # git will interpret wildcards by itself
git_root / 'weboob/**/*.py',
])
except subprocess.CalledProcessError as exc:
if exc.returncode != 1:
raise
# when no results found
to_check = []
return to_check
def run_on_files(cmd):
args = parser.parse_args()
to_check = files_to_check(args)
if to_check:
subprocess.check_call([*cmd, *to_check])
def parse_tokens(filename):
with open(filename) as fd:
tokens = list(tokenize.generate_tokens(fd.readline))
return tokens
class Checker:
def __init__(self, filename):
super().__init__()
self.filename = filename
self.noqa_lines = set()
self.tokens = None
self.ok = True
def parse_tokens(self):
self.tokens = parse_tokens(self.filename)
def parse_ast(self):
with open(self.filename) as fd:
self.tree = ast.parse(fd.read(), self.filename)
def parse_noqa(self):
if self.tokens is None:
self.parse_tokens()
for token in self.tokens:
if token.type == tokenize.COMMENT and token.string == "# noqa":
self.noqa_lines.add(token.start[0])
def add_error(self, message, line, col=None, code=None):
if line in self.noqa_lines:
return
col_text = ''
if col is not None:
col_text = f"{col}:"
code_text = ''
if code is not None:
code_text = f" {code}"
print(
f"{self.filename}:{line}:{col_text}{code_text} {message}",
file=sys.stderr
)
self.ok = False
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/hooks/run-flake8.py 0000775 0000000 0000000 00000003171 14014251244 0025354 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python3
from pathlib import Path
import runpy
import sys
from types import ModuleType
from flake8.main.cli import main
from flake8_import_order.styles import PEP8
import pkg_resources
mod = runpy.run_path(str(Path(__file__).with_name('checkerlib.py')))
args = mod['parser'].parse_args()
# flake8-import-order's PEP8 style merges app imports and relative imports
class AllSeparateStyle(PEP8):
@staticmethod
def same_section(previous, current):
return current.type == previous.type
# flake8-import-order is almost only configurable through entry_points
# ugly hack to avoid having to create a separate package with entry_point, setup, etc.
style_mod = ModuleType('fake_module_all_separate')
style_mod.AllSeparateStyle = AllSeparateStyle
sys.modules['fake_module_all_separate'] = style_mod
d = pkg_resources.Distribution('/lol.py')
ep = pkg_resources.EntryPoint.parse('fake_module_all_separate = fake_module_all_separate:AllSeparateStyle')
ep.dist = d
d._ep_map = {'flake8_import_order.styles': {'fake_module_all_separate': ep}}
pkg_resources.working_set.add(d, 'fake_module_all_separate')
# E501: Line too long
# Disabled because it doesn't allow exceptions, for example URLs or log
# messages shouldn't be split, less readable or searchable.
# W503: Line break occurred before a binary operator
# Disabling it follows pep8 (see W504).
# E266: Too many leading '#' for block comment
# But it's a nice visual separator sometimes.
main([
'--ignore=E501,W503,E266',
'--application-import-names=weboob',
'--import-order-style=fake_module_all_separate',
*map(str, mod['files_to_check'](args)),
])
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/local_install.py 0000664 0000000 0000000 00000003632 14014251244 0025074 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
from __future__ import print_function
import os
import subprocess
import sys
if '--local-modules' in sys.argv:
local_modules = True
sys.argv.remove('--local-modules')
else:
local_modules = False
print("Weboob local installer")
print()
if len(sys.argv) < 2:
print("This tool will install Weboob to be usuable without requiring")
print("messing with your system, which should only be touched by a package manager.")
print()
print("Usage: %s DESTINATION" % sys.argv[0])
print()
print("Error: Please provide a destination, "
"for example ‘%s/bin’" % os.getenv('HOME'), file=sys.stderr)
sys.exit(1)
else:
dest = os.path.expanduser(sys.argv[1])
print("Installing weboob applications into ‘%s’." % dest)
if local_modules:
sourceslist = os.path.join(
os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config')),
'weboob', 'sources.list')
if not os.path.isdir(os.path.dirname(sourceslist)):
os.makedirs(os.path.dirname(sourceslist))
if not os.path.exists(sourceslist):
with open(sourceslist, 'w') as f:
f.write('file://' + os.path.realpath(
os.path.join(os.path.dirname(__file__), os.pardir, 'modules')
))
subprocess.check_call(
[sys.executable, 'setup.py',
'install', '--user', '--install-scripts=%s' % dest] + sys.argv[2:],
cwd=os.path.join(os.path.dirname(__file__), os.pardir))
subprocess.call([sys.executable, os.path.join(dest, 'weboob-config'), 'update'])
print()
print("Installation done. Applications are available in ‘%s’." % dest)
print("You can remove the source files.")
print()
print("To have easy access to the Weboob applications,")
print("you should add the following line to your ~/.bashrc or ~/.zshrc file:")
print("export PATH=\"$PATH:%s\"" % dest)
print("And then restart your shells.")
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/local_install.sh 0000775 0000000 0000000 00000000232 14014251244 0025052 0 ustar 00root root 0000000 0000000 #!/bin/sh
set -e
. "$(dirname $0)/common.sh"
[ $VER -eq 2 ] && $PYTHON "$(dirname $0)/stale_pyc.py"
exec $PYTHON "$(dirname $0)/local_install.py" "$@"
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/local_run.py 0000664 0000000 0000000 00000004074 14014251244 0024233 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
from __future__ import print_function
import os
import subprocess
import sys
import tempfile
if len(sys.argv) < 2:
print("Usage: %s SCRIPTNAME [args]" % sys.argv[0])
sys.exit(1)
else:
args = sys.argv[1:]
pyargs = []
while args and args[0].startswith('-'):
pyargs.append(args.pop(0))
script = args.pop(0)
project = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))
def get_project_dir(name):
wd = os.path.join(project, name)
if not os.path.isdir(wd):
os.makedirs(wd)
return wd
wd = get_project_dir('localconfig')
venv = get_project_dir('localenv')
env = os.environ.copy()
env['WEBOOB_WORKDIR'] = wd
env['WEBOOB_DATADIR'] = wd
env['WEBOOB_BACKENDS'] = os.getenv('WEBOOB_LOCAL_BACKENDS',
os.getenv('WEBOOB_BACKENDS',
os.path.join(os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config')), 'weboob', 'backends')))
modpath = os.getenv('WEBOOB_MODULES', os.path.join(project, 'modules'))
with tempfile.NamedTemporaryFile(mode='w', dir=wd, delete=False) as f:
f.write("file://%s\n" % modpath)
os.rename(f.name, os.path.join(wd, 'sources.list'))
# Hide output unless there is an error
def run_quiet(cmd):
p = subprocess.Popen(
cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env,
)
s = p.communicate()
if p.returncode != 0:
print(s[0].decode('utf-8'))
if p.returncode > 1:
sys.exit(p.returncode)
venv_exe = os.path.join(venv, 'bin', 'python')
run_quiet([
sys.executable, '-m', 'virtualenv', '--system-site-packages',
'--python', sys.executable, venv,
])
run_quiet([
venv_exe, '-m', 'pip', 'install', '--no-deps', '--editable', project,
])
run_quiet([os.path.join(venv, 'bin', 'weboob-config'), 'update', '-d'])
if os.path.exists(script):
spath = script
else:
spath = os.path.join(venv, 'bin', script)
os.execvpe(
venv_exe,
[venv_exe, '-s'] + pyargs + [spath] + args,
env)
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/local_run.sh 0000775 0000000 0000000 00000000226 14014251244 0024213 0 ustar 00root root 0000000 0000000 #!/bin/sh
set -e
. "$(dirname $0)/common.sh"
[ $VER -eq 2 ] && $PYTHON "$(dirname $0)/stale_pyc.py"
exec $PYTHON "$(dirname $0)/local_run.py" "$@"
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/make_man.py 0000775 0000000 0000000 00000023224 14014251244 0024026 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright(C) 2010-2018 Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with weboob. If not, see .
from __future__ import absolute_import, print_function
import imp
import inspect
import optparse
import os
import re
import sys
import time
from datetime import datetime
from textwrap import dedent
from weboob.tools.application.base import Application
BASE_PATH = os.path.join(os.path.dirname(__file__), os.pardir)
DEST_DIR = 'man'
COMP_PATH = 'tools/weboob_bash_completion'
class ManpageHelpFormatter(optparse.HelpFormatter):
def __init__(self,
app,
indent_increment=0,
max_help_position=0,
width=80,
short_first=1):
optparse.HelpFormatter.__init__(self, indent_increment, max_help_position, width, short_first)
self.app = app
def format_heading(self, heading):
return ".SH %s\n" % heading.upper()
def format_usage(self, usage):
txt = ''
for line in usage.split('\n'):
line = line.lstrip().split(' ', 1)
if len(txt) > 0:
txt += '.br\n'
txt += '.B %s\n' % line[0]
arg_re = re.compile(r'([\[\s])([\w_]+)')
args = re.sub(arg_re, r"\1\\fI\2\\fR", line[1])
txt += args
txt += '\n'
return '.SH SYNOPSIS\n%s' % txt
def format_description(self, description):
desc = u'.SH DESCRIPTION\n.LP\n\n%s\n' % description
if hasattr(self.app, 'CAPS'):
self.app.weboob.modules_loader.load_all()
caps = self.app.CAPS if isinstance(self.app.CAPS, tuple) else (self.app.CAPS,)
modules = []
for name, module in self.app.weboob.modules_loader.loaded.items():
if module.has_caps(*caps):
modules.append(u'* %s (%s)' % (name, module.description))
if len(modules) > 0:
desc += u'\n.SS Supported websites:\n'
desc += u'\n.br\n'.join(sorted(modules))
return desc
def format_commands(self, commands):
s = u''
for section, cmds in commands.items():
if len(cmds) == 0:
continue
s += '.SH %s COMMANDS\n' % section.upper()
for cmd in sorted(cmds):
s += '.TP\n'
h = cmd.split('\n')
if ' ' in h[0]:
cmdname, args = h[0].split(' ', 1)
arg_re = re.compile(r'([A-Z_]+)')
args = re.sub(arg_re, r"\\fI\1\\fR", args)
s += '\\fB%s\\fR %s' % (cmdname, args)
else:
s += '\\fB%s\\fR' % h[0]
s += '%s\n' % '\n.br\n'.join(h[1:])
return s
def format_option_strings(self, option):
opts = optparse.HelpFormatter.format_option_strings(self, option).split(", ")
return ".TP\n" + ", ".join("\\fB%s\\fR" % opt for opt in opts)
def main():
scripts_path = os.path.join(BASE_PATH, 'weboob', 'applications')
files = os.listdir(scripts_path)
completions = dict()
for fname in files:
fpath = os.path.join(scripts_path, fname)
if os.path.isdir(fpath) and not fname.startswith('_'):
try:
fp, pathname, description = imp.find_module(fname, [scripts_path])
module = imp.load_module(fname, fp, pathname, description)
except OSError as e:
print("Unable to load the %s application (%s)"
% (fname, e), file=sys.stderr)
else:
print("Loaded %s" % fname)
# Find the applications we can handle
for klass in module.__dict__.values():
if inspect.isclass(klass) and issubclass(klass, Application) and klass.VERSION:
completions[klass.APPNAME] = analyze_application(klass, klass.APPNAME)
write_completions(completions)
def format_title(title):
return re.sub(r'^(.+):$', r'.SH \1\n.TP', title.group().upper())
# XXX useful because the PyQt QApplication destructor crashes sometimes. By
# keeping every applications until program end, it prevents to stop before
# every manpages have been generated. If it crashes at exit, it's not a
# really a problem.
applications = []
def analyze_application(app, script_name):
application = app()
applications.append(application)
formatter = ManpageHelpFormatter(application)
# patch the application
application._parser.prog = "%s" % script_name
application._parser.formatter = formatter
helptext = application._parser.format_help(formatter)
cmd_re = re.compile(r'^.+ Commands:$', re.MULTILINE)
helptext = re.sub(cmd_re, format_title, helptext)
helptext = helptext.replace("-", r"\-")
coding = r'.\" -*- coding: utf-8 -*-'
comment = r'.\" This file was generated automatically by tools/make_man.sh.'
header = '.TH %s 1 "%s" "%s %s"' % (script_name.upper(), time.strftime("%d %B %Y"),
script_name, app.VERSION.replace('.', '\\&.'))
name = ".SH NAME\n%s \- %s" % (script_name, application.SHORT_DESCRIPTION)
condition = """.SH CONDITION
The \-c and \-\-condition is a flexible way to filter and get only interesting results. It supports conditions on numerical values, dates, durations and strings. Dates are given in YYYY\-MM\-DD or YYYY\-MM\-DD HH:MM format. Durations look like XhYmZs where X, Y and Z are integers. Any of them may be omitted. For instance, YmZs, XhZs or Ym are accepted.
The syntax of one expression is "\\fBfield operator value\\fR". The field to test is always the left member of the expression.
.LP
The field is a member of the objects returned by the command. For example, a bank account has "balance", "coming" or "label" fields.
.SS The following operators are supported:
.TP
=
Test if object.field is equal to the value.
.TP
!=
Test if object.field is not equal to the value.
.TP
>
Test if object.field is greater than the value. If object.field is date, return true if value is before that object.field.
.TP
<
Test if object.field is less than the value. If object.field is date, return true if value is after that object.field.
.TP
|
This operator is available only for string fields. It works like the Unix standard \\fBgrep\\fR command, and returns True if the pattern specified in the value is in object.field.
.SS Expression combination
.LP
You can make a expression combinations with the keywords \\fB" AND "\\fR, \\fB" OR "\\fR an \\fB" LIMIT "\\fR.
.LP
The \\fBLIMIT\\fR keyword can be used to limit the number of items upon which running the expression. \\fBLIMIT\\fR can only be placed at the end of the expression followed by the number of elements you want.
.SS Examples:
.nf
.B boobank ls \-\-condition 'label=Livret A'
.fi
Display only the "Livret A" account.
.PP
.nf
.B boobank ls \-\-condition 'balance>10000'
.fi
Display accounts with a lot of money.
.PP
.nf
.B boobank history account@backend \-\-condition 'label|rewe'
.fi
Get transactions containing "rewe".
.PP
.nf
.B boobank history account@backend \-\-condition 'date>2013\-12\-01 AND date<2013\-12\-09'
.fi
Get transactions betweens the 2th December and 8th December 2013.
.PP
.nf
.B boobank history account@backend \-\-condition 'date>2013\-12\-01 LIMIT 10'
.fi
Get transactions after the 2th December in the last 10 transactions
"""
footer = """.SH COPYRIGHT
%s
.LP
For full copyright information see the COPYING file in the weboob package.
.LP
.RE
.SH FILES
"~/.config/weboob/backends" """ % application.COPYRIGHT.replace('YEAR', '%d' % datetime.today().year)
if len(app.CONFIG) > 0:
footer += '\n\n "~/.config/weboob/%s"' % app.APPNAME
# Skip internal applications.
footer += "\n\n.SH SEE ALSO\nHome page: http://weboob.org/applications/%s" % application.APPNAME
mantext = u"%s\n%s\n%s\n%s\n%s\n%s\n%s" % (coding, comment, header, name, helptext, condition, footer)
with open(os.path.join(BASE_PATH, DEST_DIR, "%s.1" % script_name), 'w+') as manfile:
for line in mantext.split('\n'):
manfile.write('%s\n' % line.lstrip())
print("wrote %s/%s.1" % (DEST_DIR, script_name))
return application._shell_completion_items()
def write_completions(completions):
compscript = dedent('''
# Weboob completion for Bash (automatically generated by tools/make_man.sh)
#
# vim: filetype=sh expandtab softtabstop=4 shiftwidth=4
#
# This file is part of weboob.
#
# This script can be distributed under the same license as the
# weboob or bash packages.
''')
for name, items in completions.items():
compscript += dedent('''
_weboob_{1}()
{{
local cur args
COMPREPLY=()
cur=${{COMP_WORDS[COMP_CWORD]}}
args="{2}"
COMPREPLY=( $(compgen -o default -W "${{args}}" -- "$cur" ) )
}}
complete -F _weboob_{1} {0}
''').format(name, name.replace('-', '_'), ' '.join(items))
with open(os.path.join(BASE_PATH, COMP_PATH), 'w') as f:
f.write(compscript)
if __name__ == '__main__':
sys.exit(main())
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/make_man.sh 0000775 0000000 0000000 00000002074 14014251244 0024010 0 ustar 00root root 0000000 0000000 #!/bin/sh
# stop on failure
set -e
. "$(dirname $0)/common.sh"
# Use C local to avoid local dates in headers
export LANG=en_US.utf8
# disable termcolor
export ANSI_COLORS_DISABLED=1
[ -z "${TMPDIR}" ] && TMPDIR="/tmp"
# do not allow undefined variables anymore
set -u
WEBOOB_TMPDIR=$(mktemp -d "${TMPDIR}/weboob_man.XXXXXX")
# path to sources
WEBOOB_DIR=$(cd $(dirname $0)/.. && pwd -P)
touch "${WEBOOB_TMPDIR}/backends"
chmod 600 "${WEBOOB_TMPDIR}/backends"
echo "file://$WEBOOB_DIR/modules" > "${WEBOOB_TMPDIR}/sources.list"
export WEBOOB_WORKDIR="${WEBOOB_TMPDIR}"
export WEBOOB_DATADIR="${WEBOOB_TMPDIR}"
export PYTHONPATH="${WEBOOB_DIR}"
# TODO can we require weboob to be installed before being able to run run_tests.sh?
# if we can, then weboob-config is present in PATH (virtualenv or whatever)
${PYTHON} -c "import sys; sys.argv='weboob-config update'.split(); from weboob.applications.weboobcfg import WeboobCfg; WeboobCfg.run()"
$PYTHON "${WEBOOB_DIR}/tools/make_man.py"
# allow failing commands past this point
STATUS=$?
rm -rf "${WEBOOB_TMPDIR}"
exit $STATUS
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/modules_testing_grid.py 0000775 0000000 0000000 00000004004 14014251244 0026463 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Script to format XUNIT output from unittests as a JSON string ready to be sent
to a [Weboob-CI](https://github.com/Phyks/weboob-ci) instance.
* `XUNIT` is the XUNIT file to handle.
* `ORIGIN` is an origin string as described in the Weboob-CI documentation
(basically just a string to identify the source of the unittests results).
"""
from __future__ import print_function
import json
import sys
import xunitparser
def main(xunit, origin):
with open(xunit, "r") as fh:
ts, tr = xunitparser.parse(fh)
# Get test results for each module
modules = {}
other_testcases = []
for tc in ts:
if tc.classname.startswith("weboob."):
other_testcases.append(repr(tc))
continue
module = tc.classname.split(".")[0]
# In the following, we consider
# bad > skipped > good
# and only make update of a module status according to this order
if tc.good:
if tc.skipped:
# Set to skipped only if previous test was good
if module not in modules or modules[module] == "good":
modules[module] = "skipped"
else:
# Set to good only if no previous result
if module not in modules:
modules[module] = "good"
else:
# Always set to bad on failed test
modules[module] = "bad"
# Agregate results by test result rather than module
results = {
"good": [],
"bad": [],
"skipped": []
}
for module in modules:
results[modules[module]].append(module)
return {
"origin": origin,
"modules": results,
"others": other_testcases
}
if __name__ == "__main__":
if len(sys.argv) < 3:
sys.exit("Usage: %s XUNIT_FILE ORIGIN" % (sys.argv[0]))
print(
json.dumps(
main(sys.argv[1], sys.argv[2]),
sort_keys=True, indent=4, separators=(',', ': ')
)
)
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/pyflakes-strict.sh 0000775 0000000 0000000 00000001663 14014251244 0025367 0 ustar 00root root 0000000 0000000 #!/bin/sh -e
. "$(dirname $0)/common.sh"
cd "$(dirname $0)/.."
die () {
echo "$@" >&2
exit 1
}
$PYTHON3 -c 'import flake8' || die "Please install flake8 (e.g. apt install flake8)"
$PYTHON3 -c 'import flake8_import_order' || die "Please install flake8-import-order (e.g. pip3 install flake8-import-order)"
$PYTHON3 -c 'import bugbear' || die "Please install flake8-bugbear (e.g. pip3 install flake8-bugbear)"
$PYTHON3 -c 'import asttokens' || die "Please install asttokens (e.g. apt install python3-asttokens)"
err=0
$PYTHON3 ./tools/hooks/run-flake8.py "$@" || err=1
$PYTHON3 ./tools/hooks/check-ifexpr.py "$@" || err=1
$PYTHON3 ./tools/hooks/check-stringcut.py "$@" || err=1
$PYTHON3 ./tools/hooks/check-continuations.py "$@" || err=1
$PYTHON3 ./tools/hooks/check-trailing-commas.py "$@" || err=1
$PYTHON3 ./tools/hooks/check-line-length.py -l 120 "$@" || err=1
$PYTHON3 ./tools/hooks/check-op-precedence.py "$@" || err=1
exit $err
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/pyflakes.sh 0000775 0000000 0000000 00000005235 14014251244 0024060 0 ustar 00root root 0000000 0000000 #!/bin/sh -u
. "$(dirname $0)/common.sh"
err=0
cd $(dirname $0)/..
MODULE_FILES=$(git ls-files modules|grep '\.py$')
# Takes PYFILES from env, if empty use all git tracked files
: ${PYFILES:=}
if [ -z "${PYFILES}" ]; then
PYFILES="$(git ls-files | grep '^scripts\|\.py$'|grep -v boilerplate_data|grep -v stable_backport_data|grep -v '^modules'|grep -v '^contrib')"
PYFILES="$PYFILES $MODULE_FILES"
fi
grep -n '[[:space:]]$' ${PYFILES} && echo 'Error: tabs or trailing whitespace found, remove them' && err=4
grep -Fn '.setlocale' ${PYFILES} && echo 'Error: do not use setlocale' && err=5
grep -Fn '__future__ import with_statement' ${PYFILES} && echo 'Error: with_statement useless as we do not support Python 2.5' && err=6
grep -nE '^[[:space:]]+except [[:alnum:] ]+,[[:alnum:] ]+' ${PYFILES} && echo 'Error: use new "as" way of naming exceptions' && err=7
grep -nE "^ *print " ${PYFILES} && echo 'Error: Use the print function' && err=8
grep -Fn ".has_key" ${PYFILES} && echo 'Error: Deprecated, use operator "in"' && err=9
grep -Fn "os.isatty" ${PYFILES} && echo 'Error: Use stream.isatty() instead of os.isatty(stream.fileno())' && err=10
grep -Fn "raise StopIteration" ${PYFILES} && echo 'Error: PEP 479' && err=11
grep -nE "\.iter(keys|values|items)\(\)" ${PYFILES} | grep -Fv "six.iter" && echo 'Error: iterkeys/itervalues/iteritems is forbidden' && err=12
grep -nE "^ *print(\(| )" ${MODULE_FILES} && echo 'Error: Use of print in modules is forbidden, use logger instead' && err=20
grep -n xrange ${MODULE_FILES} && echo 'Error: xrange is forbidden' && err=21
grep -nE "from (urllib|urlparse) import" ${MODULE_FILES} && echo 'Error: python2 urllib is forbidden' && err=22
grep -nE "^import (urllib|urlparse)$" ${MODULE_FILES} && echo 'Error: python2 urllib is forbidden' && err=22
grep -nE "HEADLESS[[:space:]]*=[[:space:]]*False" ${MODULE_FILES} && echo 'Error: HEADLESS must be set back to True' && err=23
if [ ${VER} -eq 2 ]
then
if ${PYTHON2} -c "import flake8" 2>/dev/null; then
FLAKER2=flake8
OPT2="--select=E9,F"
elif ${PYTHON2} -c "import pyflakes" 2>/dev/null; then
FLAKER2=pyflakes
OPT2=
else
echo "flake8 or pyflakes for python2 not found"
err=1
fi
if [ ${err} -ne 1 ]; then
$PYTHON2 -m ${FLAKER2} ${OPT2} ${PYFILES} || err=32
fi
fi
if [ ${VER} -eq 3 ]
then
if ${PYTHON3} -c "import flake8" 2>/dev/null; then
FLAKER3=flake8
OPT3="--select=E9,F"
elif ${PYTHON3} -c "import pyflakes" 2>/dev/null; then
FLAKER3=pyflakes
OPT3=
else
echo "flake8 or pyflakes for python3 not found"
err=1
fi
if [ ${err} -ne 1 ]; then
$PYTHON3 -m ${FLAKER3} ${OPT3} ${PYFILES} || exit 33
fi
fi
exit $err
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/pyreverse.sh 0000775 0000000 0000000 00000000420 14014251244 0024255 0 ustar 00root root 0000000 0000000 #!/bin/sh
#
# Examples:
# pyreverse.sh weboob.backends.aum
#
# pyreverse is included in pylint Debian package
usage() {
echo "pyreverse.sh "
exit
}
[ -z "$1" ] && usage
PYTHONPATH="$(dirname $0)/../modules/$1" pyreverse -p "$1" -o pdf -a1 -s1 "."
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/release.py 0000775 0000000 0000000 00000015410 14014251244 0023674 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python3
import argparse
import configparser
import os
import re
import sys
import datetime
from subprocess import check_call, check_output
from weboob.tools.misc import to_unicode
WORKTREE = 'release_tmp'
def make_tarball(tag, wheel):
# Create and enter a temporary worktree
if os.path.isdir(WORKTREE):
check_call(['git', 'worktree', 'remove', '--force', WORKTREE])
check_call(['git', 'worktree', 'add', WORKTREE, tag])
assert os.path.isdir(WORKTREE)
os.chdir(WORKTREE)
check_call([sys.executable, 'setup.py'] +
['sdist',
'--keep',
'--dist-dir', '../dist'])
if wheel:
check_call([sys.executable, 'setup.py'] +
['bdist_wheel',
'--keep',
'--dist-dir', '../dist'])
# Clean up the temporary worktree
os.chdir(os.pardir)
check_call(['git', 'worktree', 'remove', '--force', WORKTREE])
assert not os.path.isdir(WORKTREE)
files = ['dist/weboob-%s.tar.gz' % tag]
if wheel:
files.append('dist/weboob-%s-py2.py3-none-any.whl' % tag)
for f in files:
if not os.path.exists(f):
raise Exception('Generated file not found at %s' % f)
else:
print('Generated file: %s' % f)
print('To upload to PyPI, run: twine upload -s %s' % ' '.join(files))
def changed_modules(changes, changetype):
for change in changes:
change = change.decode('utf-8').split()
if change[0] == changetype:
m = re.match(r'modules/([^/]+)/__init__\.py', change[1])
if m:
yield m.group(1)
def get_caps(module, config):
try:
return sorted(c for c in config[module]['capabilities'].split() if c != 'CapCollection')
except KeyError:
return ['**** FILL ME **** (running weboob update could help)']
def new_modules(start, end):
os.chdir(os.path.join(os.path.dirname(__file__), os.path.pardir))
modules_info = configparser.ConfigParser()
with open('modules/modules.list') as f:
modules_info.read_file(f)
git_cmd = ['git', 'diff', '--no-renames', '--name-status', '%s..%s' % (start, end), '--', 'modules/']
added_modules = sorted(changed_modules(check_output(git_cmd).splitlines(), 'A'))
deleted_modules = sorted(changed_modules(check_output(git_cmd).splitlines(), 'D'))
for added_module in added_modules:
yield 'New %s module (%s)' % (added_module, ', '.join(get_caps(added_module, modules_info)))
for deleted_module in deleted_modules:
yield 'Deleted %s module' % deleted_module
def changelog(start, end='HEAD'):
def sortkey(d):
"""Put the commits with multiple domains at the end"""
return (len(d), d)
commits = {}
for commithash in check_output(['git', 'rev-list', '{}..{}'.format(start, end)]).splitlines():
title, domains = commitinfo(commithash)
commits.setdefault(domains, []).append(title)
for line in new_modules(start, end):
commits.setdefault(('General',), []).append(line)
cl = ''
for domains in sorted(commits.keys(), key=sortkey):
cl += '\n\n\t' + '\n\t'.join(domains)
for title in commits[domains]:
cl += '\n\t* ' + title
return cl.lstrip('\n')
def domain(path):
dirs = os.path.dirname(path).split('/')
if dirs == ['']:
return 'General: Core'
if dirs[0] == 'man' or path == 'tools/py3-compatible.modules':
return None
if dirs[0] == 'weboob':
try:
if dirs[1] in ('core', 'tools'):
return 'General: Core'
elif dirs[1] == 'capabilities':
return 'Capabilities'
elif dirs[1] == 'browser':
try:
if dirs[2] == 'filters':
return 'Browser: Filters'
except IndexError:
return 'Browser'
elif dirs[1] == 'applications':
try:
return 'Applications: {}'.format(dirs[2])
except IndexError:
return 'Applications'
elif dirs[1] == 'application':
try:
return 'Applications: {}'.format(dirs[2].title())
except IndexError:
return 'Applications'
except IndexError:
return 'General: Core'
if dirs[0] in ('contrib', 'tools'):
return 'Tools'
if dirs[0] in ('docs', 'icons'):
return 'Documentation'
if dirs[0] == 'modules':
try:
return 'Modules: {}'.format(dirs[1])
except IndexError:
return 'General: Core'
return 'Unknown'
def commitinfo(commithash):
info = check_output(['git', 'show', '--format=%s', '--name-only', commithash]).decode('utf-8').splitlines()
title = to_unicode(info[0])
domains = set([domain(p) for p in info[2:] if domain(p)])
if 'Unknown' in domains and len(domains) > 1:
domains.remove('Unknown')
if not domains or len(domains) > 5:
domains = set(['Unknown'])
if 'Unknown' not in domains:
# When the domains are known, hide the title prefixes
title = re.sub(r'^(?:[\w\./\s]+:|\[[\w\./\s]+\])\s*', '', title, flags=re.UNICODE)
return title, tuple(sorted(domains))
def previous_version():
"""
Get the highest version tag
"""
for v in check_output(['git', 'tag', '-l', '*.*', '--sort=-v:refname']).splitlines():
return v.decode()
def prepare(start, end, version):
print('Weboob %s (%s)\n' % (version, datetime.date.today().strftime('%Y-%m-%d')))
print(changelog(start, end))
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description="Prepare and export a release.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
epilog='This is mostly meant to be called from release.sh for now.',
)
subparsers = parser.add_subparsers()
prepare_parser = subparsers.add_parser(
'prepare',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
prepare_parser.add_argument('version')
prepare_parser.add_argument('--start', default=previous_version(), help='Commit of the previous release')
prepare_parser.add_argument('--end', default='HEAD', help='Last commit before the new release')
prepare_parser.set_defaults(mode='prepare')
tarball_parser = subparsers.add_parser(
'tarball',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
tarball_parser.add_argument('tag')
tarball_parser.add_argument('--no-wheel', action='store_false', dest='wheel')
tarball_parser.set_defaults(mode='tarball')
args = parser.parse_args()
if args.mode == 'prepare':
prepare(args.start, args.end, args.version)
elif args.mode == 'tarball':
make_tarball(args.tag, args.wheel)
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/release.sh 0000775 0000000 0000000 00000002413 14014251244 0023655 0 ustar 00root root 0000000 0000000 #!/usr/bin/env bash
# This script is used to release a version.
set -e
cd "$(dirname $0)/.."
function set_version {
echo -n "Replacing version in source files to $1... "
sed -i "s/^\(\s*\)\(VERSION\|version\|release\|__version__\)\( *\)=\( *\)\([\"']\?\)[0-9]\+\.[0-9a-z]\+\([\"']\?\)\(,\?\)$/\1\2\3=\4\5$1\6\7/g" $(git ls-files -x contrib | grep -v "\.svg$")
echo -e "done.\n"
}
if [ -z "$1" ]; then
echo "Syntax: $0 VERSION"
exit 1
fi
VERSION=$1
echo "Generating ChangeLog..."
export LANG=en_US.utf8
mv ChangeLog ChangeLog.old
tools/release.py prepare $VERSION > ChangeLog
echo -e "\n" >> ChangeLog
cat ChangeLog.old >> ChangeLog
rm -f ChangeLog.old
vi +2 ChangeLog
set_version $VERSION
echo "Generating manpages..."
tools/make_man.sh
echo -e "done!\n"
# in case there are new manpages not included in the git tree.
git add man/*
echo "Release commit:"
git commit -a -m "Weboob $VERSION released"
echo -ne "\n"
echo "Release tag:"
git tag $VERSION -s -m "Weboob $VERSION"
echo -ne "\n"
tools/release.py tarball $VERSION
echo -ne "\nDo you want to change the version number (y/n) "
read change_version
if [ "$change_version" = "y" ]; then
echo -n "Enter the new version number: "
read NEW_VERSION
set_version $NEW_VERSION
git commit -a -m "bump to $NEW_VERSION"
fi
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/run_tests.sh 0000775 0000000 0000000 00000013572 14014251244 0024273 0 ustar 00root root 0000000 0000000 #!/usr/bin/env bash
# Mai available environment variables
# * RSYNC_TARGET: target on which to rsync the xunit output.
# * XUNIT_OUT: file in which xunit output should be saved.
# * WEBOOB_BACKENDS: path to the Weboob backends file to use.
# * WEBOOB_CI_TARGET: URL of your Weboob-CI instance.
# * WEBOOB_CI_ORIGIN: origin for the Weboob-CI data.
# stop on failure
set -e
. "$(dirname $0)/common.sh"
if [ -z "${PYTHON}" ]; then
echo "Python required"
exit 1
fi
if ! $PYTHON -c "import nose" 2>/dev/null; then
echo "python-nose required"
exit 1
fi
TEST_CORE=1
TEST_MODULES=1
for i in "$@"
do
case $i in
--no-modules)
TEST_MODULES=0
shift
;;
--no-core)
TEST_CORE=0
shift
;;
*)
;;
esac
done
# path to sources
WEBOOB_DIR=$(cd $(dirname $0)/.. && pwd -P)
BACKEND="${1}"
if [ -z "${WEBOOB_WORKDIR}" ]; then
# use the old workdir by default
WEBOOB_WORKDIR="${HOME}/.weboob"
# but if we can find a valid xdg workdir, switch to it
[ "${XDG_CONFIG_HOME}" != "" ] || XDG_CONFIG_HOME="${HOME}/.config"
[ -d "${XDG_CONFIG_HOME}/weboob" ] && WEBOOB_WORKDIR="${XDG_CONFIG_HOME}/weboob"
fi
[ -z "${TMPDIR}" ] && TMPDIR="/tmp"
WEBOOB_TMPDIR=$(mktemp -d "${TMPDIR}/weboob_test.XXXXXX")
[ -z "${WEBOOB_BACKENDS}" ] && WEBOOB_BACKENDS="${WEBOOB_WORKDIR}/backends"
[ -z "${WEBOOB_MODULES}" ] && WEBOOB_MODULES="${WEBOOB_DIR}/modules"
[ -z "${PYTHONPATH}" ] && PYTHONPATH=""
# allow private environment setup
[ -f "${WEBOOB_WORKDIR}/pre-test.sh" ] && source "${WEBOOB_WORKDIR}/pre-test.sh"
# setup xunit reporting (buildbot slaves only)
if [ -n "${RSYNC_TARGET}" ]; then
# by default, builder name is containing directory name
[ -z "${BUILDER_NAME}" ] && BUILDER_NAME=$(basename $(readlink -e $(dirname $0)/../..))
XUNIT_OUT="${WEBOOB_TMPDIR}/xunit.xml"
else
RSYNC_TARGET=""
fi
# Avoid undefined variables
if [ ! -n "${XUNIT_OUT}" ]; then
XUNIT_OUT=""
fi
# Handle Weboob-CI variables
if [ -n "${WEBOOB_CI_TARGET}" ]; then
if [ ! -n "${WEBOOB_CI_ORIGIN}" ]; then
WEBOOB_CI_ORIGIN="Weboob unittests run"
fi
# Set up xunit reporting
XUNIT_OUT="${WEBOOB_TMPDIR}/xunit.xml"
else
WEBOOB_CI_TARGET=""
fi
# do not allow undefined variables anymore
set -u
if [ -f "${WEBOOB_BACKENDS}" ]; then
cp "${WEBOOB_BACKENDS}" "${WEBOOB_TMPDIR}/backends"
else
touch "${WEBOOB_TMPDIR}/backends"
chmod go-r "${WEBOOB_TMPDIR}/backends"
fi
# xunit nose setup
if [ -n "${XUNIT_OUT}" ]; then
XUNIT_ARGS="--with-xunit --xunit-file=${XUNIT_OUT}"
else
XUNIT_ARGS=""
fi
[ $VER -eq 2 ] && $PYTHON "$(dirname $0)/stale_pyc.py"
echo "file://${WEBOOB_MODULES}" > "${WEBOOB_TMPDIR}/sources.list"
export WEBOOB_WORKDIR="${WEBOOB_TMPDIR}"
export WEBOOB_DATADIR="${WEBOOB_TMPDIR}"
export PYTHONPATH="${WEBOOB_DIR}:${PYTHONPATH}"
export NOSE_NOPATH="1"
if [[ ($TEST_MODULES = 1) || (-n "${BACKEND}") ]]; then
# TODO can we require weboob to be installed before being able to run run_tests.sh?
# if we can, then weboob-config is present in PATH (virtualenv or whatever)
${PYTHON} -c "import sys; sys.argv='weboob-config update'.split(); from weboob.applications.weboobcfg import WeboobCfg; WeboobCfg.run()"
fi
# allow failing commands past this point
set +e
set -o pipefail
STATUS_CORE=0
STATUS=0
if [ -n "${BACKEND}" ]; then
${PYTHON} -m nose -c /dev/null --logging-level=DEBUG -sv "${WEBOOB_MODULES}/${BACKEND}/test.py" ${XUNIT_ARGS}
STATUS=$?
else
if [ $TEST_CORE = 1 ]; then
echo "=== Weboob ==="
CORE_TESTS=$(mktemp)
${PYTHON} -m nose --cover-package weboob -c ${WEBOOB_DIR}/setup.cfg --logging-level=DEBUG -sv 2>&1 | tee "${CORE_TESTS}"
STATUS_CORE=$?
CORE_STMTS=$(grep "TOTAL" ${CORE_TESTS} | awk '{ print $2; }')
CORE_MISS=$(grep "TOTAL" ${CORE_TESTS} | awk '{ print $3; }')
CORE_COVERAGE=$(grep "TOTAL" ${CORE_TESTS} | awk '{ print $4; }')
rm ${CORE_TESTS}
fi
if [ $TEST_MODULES = 1 ]; then
echo "=== Modules ==="
MODULES_TESTS=$(mktemp)
MODULES_TO_TEST=$(find "${WEBOOB_MODULES}" -name "test.py" | sort | xargs echo)
${PYTHON} -m nose --with-coverage --cover-package modules -c /dev/null --logging-level=DEBUG -sv ${XUNIT_ARGS} ${MODULES_TO_TEST} 2>&1 | tee ${MODULES_TESTS}
STATUS=$?
MODULES_STMTS=$(grep "TOTAL" ${MODULES_TESTS} | awk '{ print $2; }')
MODULES_MISS=$(grep "TOTAL" ${MODULES_TESTS} | awk '{ print $3; }')
MODULES_COVERAGE=$(grep "TOTAL" ${MODULES_TESTS} | awk '{ print $4; }')
rm ${MODULES_TESTS}
fi
# Compute total coverage
echo "=== Total coverage ==="
if [ $TEST_CORE = 1 ]; then
echo "CORE COVERAGE: ${CORE_COVERAGE}"
fi
if [ $TEST_MODULES = 1 ]; then
echo "MODULES COVERAGE: ${MODULES_COVERAGE}"
fi
if [[ ($TEST_CORE = 1) && ($TEST_MODULES = 1) ]]; then
TOTAL_STMTS=$((${CORE_STMTS} + ${MODULES_STMTS}))
TOTAL_MISS=$((${CORE_MISS} + ${MODULES_MISS}))
TOTAL_COVERAGE=$((100 * (${TOTAL_STMTS} - ${TOTAL_MISS}) / ${TOTAL_STMTS}))
echo "TOTAL: ${TOTAL_COVERAGE}%"
fi
fi
# Rsync xunit transfer
if [ -n "${RSYNC_TARGET}" ]; then
rsync -iz "${XUNIT_OUT}" "${RSYNC_TARGET}/${BUILDER_NAME}-$(date +%s).xml"
rm "${XUNIT_OUT}"
fi
# Weboob-CI upload
if [ -n "${WEBOOB_CI_TARGET}" ]; then
JSON_MODULE_MATRIX=$(${PYTHON} "${WEBOOB_DIR}/tools/modules_testing_grid.py" "${XUNIT_OUT}" "${WEBOOB_CI_ORIGIN}")
curl -H "Content-Type: application/json" --data "${JSON_MODULE_MATRIX}" "${WEBOOB_CI_TARGET}/api/v1/modules"
rm "${XUNIT_OUT}"
fi
# safe removal
if [[ ($TEST_MODULES = 1) || (-n "${BACKEND}") ]]; then
rm -r "${WEBOOB_TMPDIR}/icons" "${WEBOOB_TMPDIR}/repositories" "${WEBOOB_TMPDIR}/modules" "${WEBOOB_TMPDIR}/keyrings"
fi
rm "${WEBOOB_TMPDIR}/backends" "${WEBOOB_TMPDIR}/sources.list"
rmdir "${WEBOOB_TMPDIR}"
[ $STATUS_CORE -gt 0 ] && exit $STATUS_CORE
exit $STATUS
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/setup-virtualenv.sh 0000775 0000000 0000000 00000002725 14014251244 0025600 0 ustar 00root root 0000000 0000000 #!/bin/sh -e
# install weboob inside a virtualenv, optionally with an associated weboob workdir
# can be combined with git-worktree
cd "$(dirname $0)/.."
SRC=$PWD
source=
VDIR=
usage () {
cat << EOF
Usage: $0 [-s] [-d DIR]
-s point sources.list to $SRC/modules instead of updates.weboob.org
-d DIR install virtualenv in DIR instead of a new dir
EOF
}
while getopts hsd: name
do
case $name in
s) source=y;;
d) VDIR="$OPTARG";;
h) usage
exit 0;;
?) usage
exit 2;;
esac
done
shift $(($OPTIND - 1))
PYTHON=${PYTHON-python3}
echo "Using weboob source $SRC"
if [ -z "$VDIR" ]
then
VDIR=$(mktemp -d /tmp/weboob.venv.XXXXXX)
fi
cd "$VDIR"
echo "Creating env in $VDIR"
virtualenv -p "$(which "$PYTHON")" --system-site-packages "$VDIR"
. ./bin/activate
echo "Installing weboob in $VDIR"
"$PYTHON" -m pip install "$SRC"
mkdir workdir
export WEBOOB_WORKDIR=$VDIR/workdir
if [ "$source" = y ]
then
echo "file://$SRC/modules" > "$WEBOOB_WORKDIR/sources.list"
fi
cat > use-weboob-local.sh << EOF
VDIR="$VDIR"
. "$VDIR/bin/activate"
export WEBOOB_WORKDIR="$VDIR/workdir"
EOF
cat << EOF
Installation complete in $VDIR.
Run ". $VDIR/use-weboob-local.sh" to start using it.
Run "$PYTHON -m pip install -U $SRC" to reinstall the core.
EOF
if [ "$source" != y ]
then
echo "You can add file://$SRC/modules into $VDIR/workdir/sources.list to use local modules instead of downloading modules."
fi
./bin/weboob-config update
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/stable_backport.py 0000775 0000000 0000000 00000020034 14014251244 0025411 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python3
from __future__ import print_function
import time
import sys
import re
from contextlib import contextmanager
from os import system, path, makedirs, getenv
from subprocess import check_output, STDOUT, CalledProcessError
from collections import defaultdict
import shutil
from termcolor import colored
STABLE_VERSION = getenv('WEBOOB_BACKPORT_STABLE', '1.3')
DEVEL_BRANCH = getenv('WEBOOB_BACKPORT_DEVEL', 'master')
@contextmanager
def log(message, success='done'):
print('%s... ' % message, end='', flush=True)
start = time.time()
try:
yield
except KeyboardInterrupt:
print(colored('abort', 'red'))
sys.exit(1)
except Exception as e:
print(colored('fail: %s' % e, 'red'))
raise
else:
print('%s %s' % (colored(success, 'green'),
colored('(%.2fs)' % (time.time() - start), 'blue')))
def create_compat_dir(name):
if not path.exists(name):
makedirs(name)
with open(path.join(name, '__init__.py'), 'w'):
pass
MANUAL_PORTS = [
'weboob.tools.captcha.virtkeyboard',
]
MANUAL_PORT_DIR = path.join(path.dirname(__file__), 'stable_backport_data')
class Error(object):
def __init__(self, filename, linenum, message):
self.filename = filename
self.linenum = linenum
self.message = message
self.compat_dir = path.join(path.dirname(filename), 'compat')
def __repr__(self):
return '<%s filename=%r linenum=%s message=%r>' % (type(self).__name__, self.filename, self.linenum, self.message)
def reimport_module(self, module):
# not a weboob module, probably a false positive.
if not module.startswith('weboob'):
return
dirname = module.replace('.', '/')
filename = dirname + '.py'
new_module = module.replace('.', '_')
target = path.join(self.compat_dir, '%s.py' % new_module)
base_module = '.'.join(module.split('.')[:-1])
if module in MANUAL_PORTS:
shutil.copyfile(path.join(MANUAL_PORT_DIR, path.basename(target)), target)
else:
try:
r = check_output('git show %s:%s' % (DEVEL_BRANCH, filename), shell=True, stderr=STDOUT).decode('utf-8')
except CalledProcessError:
# this file does not exist, perhaps a directory.
return
# Copy module from devel to a compat/ sub-module
with open(target, 'w') as fp:
for line in r.split('\n'):
# Replace relative imports to absolute ones
m = re.match(r'^from (\.\.?)([\w_\.]+) import (.*)', line)
if m:
if m.group(1) == '..':
base_module = '.'.join(base_module.split('.')[:-1])
fp.write('from %s.%s import %s\n' % (base_module, m.group(2), m.group(3)))
continue
# Inherit all classes by previous ones, if they already existed.
m = re.match(r'^class (\w+)\(([\w,\s]+)\):(.*)', line)
if m and path.exists(filename) and system('grep "^class %s" %s >/dev/null' % (m.group(1), filename)) == 0:
symbol = m.group(1)
trailing = m.group(3)
fp.write('from %s import %s as _%s\n' % (module, symbol, symbol))
fp.write('class %s(_%s):%s\n' % (symbol, symbol, trailing))
continue
fp.write('%s\n' % line)
# Particular case, in devel some imports have been added to
# weboob/browser/__init__.py
system(r'sed -i -e "s/from weboob.browser import/from weboob.browser.browsers import/g" %s'
% self.filename)
# Replace import to this module by a relative import to the copy in
# compat/
system(r'sed -i -e "%ss/from \([A-Za-z0-9_\.]\+\) import \(.*\)/from .compat.%s import \2/g" %s'
% (self.linenum, new_module, self.filename))
def remove_block(name, start):
lines = []
with open(name, 'r') as fd:
it = iter(fd)
for n in range(start - 1):
lines.append(next(it))
line = next(it)
level = len(re.match(r'^( *)', line).group(1))
for line in it:
if not line.strip():
continue
new = len(re.match(r'^( *)', line).group(1))
if new <= level:
lines.append(line)
break
lines.extend(it)
with open(name, 'w') as fd:
fd.write(''.join(lines))
class NoNameInModuleError(Error):
def fixup(self):
m = re.match(r"No name '(\w+)' in module '([\w\.]+)'", self.message)
module = m.group(2)
self.reimport_module(module)
class ImportErrorError(Error):
def fixup(self):
m = re.match(r"Unable to import '([\w\.]+)'", self.message)
module = m.group(1)
self.reimport_module(module)
class ManualBackport(Error):
def fixup(self):
self.reimport_module(self.message)
def replace_all(expr, dest):
system(r"""for file in $(git ls-files modules | grep '\.py$');
do
sed -i -e "s/""" + expr + '/' + dest + """/g" $file
done""")
def output_lines(cmd):
return check_output(cmd, shell=True, stderr=STDOUT).decode('utf-8').rstrip().split('\n')
class StableBackport(object):
errors = {'E0611': NoNameInModuleError,
'E0401': ImportErrorError,
}
def main(self):
with log('Removing previous compat files'):
system('git rm -q "modules/*/compat/*.py"')
with log('Copying last version of modules from devel'):
system('git checkout --theirs %s modules' % DEVEL_BRANCH)
with log('Replacing version number'):
replace_all(r"""^\(\s*\)\(VERSION\)\( *\)=\( *\)[\"'][0-9]\+\..\+[\"']\(,\?\)$""",
r"""\1\2\3=\4'""" + STABLE_VERSION + r"""'\5""")
with log('Removing staling data'):
system('tools/stale_pyc.py')
system('find modules -type d -empty -delete')
system('git add -u')
with log('Lookup modules errors'):
r = check_output("pylint modules/* -f parseable -E -d all -e no-name-in-module,import-error; exit 0", shell=True, stderr=STDOUT).decode('utf-8')
dirnames = defaultdict(list)
for line in r.split('\n'):
m = re.match(r'([\w\./]+):(\d+): \[(\w+)[^\]]+\] (.*)', line)
if not m:
continue
filename = m.group(1)
linenum = m.group(2)
error = m.group(3)
msg = m.group(4)
dirnames[path.dirname(filename)].append(self.errors[error](filename, linenum, msg))
with log('Searching manual backports'):
for manual in MANUAL_PORTS:
r = check_output("grep -nEr '^from %s import ' modules" % manual, shell=True).strip().decode('utf-8')
for line in r.split('\n'):
m = re.match(r'([\w\./]+):(\d+):.*', line)
filename = m.group(1)
linenum = m.group(2)
target = dirnames[path.dirname(filename)]
for err in target:
if err.filename == filename and err.linenum == linenum:
# an error was already spot on this line
break
else:
target.append(ManualBackport(filename, linenum, manual))
for dirname, errors in sorted(dirnames.items()):
with log('Fixing up %s errors in %s' % (colored(str(len(errors)), 'magenta'),
colored(dirname, 'yellow'))):
compat_dirname = path.join(dirname, 'compat')
create_compat_dir(compat_dirname)
for error in errors:
error.fixup()
system('git add %s' % compat_dirname)
system('git add -u')
if __name__ == '__main__':
StableBackport().main()
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/stable_backport_data/ 0000775 0000000 0000000 00000000000 14014251244 0026026 5 ustar 00root root 0000000 0000000 weboob_tools_captcha_virtkeyboard.py 0000664 0000000 0000000 00000016721 14014251244 0035275 0 ustar 00root root 0000000 0000000 woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/stable_backport_data
import weboob.tools.captcha.virtkeyboard as OLD
# can't import *, __all__ is incomplete...
for attr in dir(OLD):
globals()[attr] = getattr(OLD, attr)
try:
__all__ = OLD.__all__
except AttributeError:
pass
class SimpleVirtualKeyboard(object):
"""Handle a virtual keyboard where "keys" are distributed on a simple grid.
Parameters:
:param cols: Column count of the grid
:type cols: int
:param rows: Row count of the grid
:type rows: int
:param image: File-like object to be used as data source
:type image: file
:param convert: Mode to which convert color of pixels, see
:meth:`Image.Image.convert` for more information
:param matching_symbols: symbol that match all case of image grid from left to right and top
to down, European reading way.
:type matching_symbols: iterable
:param matching_symbols_coords: dict mapping matching website symbols to their image coords
(x0, y0, x1, y1) on grid image from left to right and top to
down, European reading way. It's not symbols in the image.
:type matching_symbols_coords: dict[str:4-tuple(int)]
:param browser: Browser of weboob session.
Allow to dump tiles files in same directory than session folder
:type browser: obj(Browser)
Attributes:
:attribute codesep: Output separator between matching symbols
:type codesep: str
:param margin: Useless image pixel to cut.
See :func:`cut_margin`.
:type margin: 4-tuple(int), same as HTML margin: (top, right, bottom, left).
or 2-tuple(int), (top = bottom, right = left),
or int, top = right = bottom = left
:attribute tile_margin: Useless tile pixel to cut.
See :func:`cut_margin`.
:attribute symbols: Association table between image symbols and md5s
:type symbols: dict[str:str] or dict[str:n-tuple(str)]
:attribute convert: Mode to which convert color of pixels, see
:meth:`Image.Image.convert` for more information
:attribute alter: Allow custom main image alteration. Then overwrite :func:`alter_image`.
:type alter: boolean
"""
codesep = ''
margin = None
tile_margin = None
symbols = None
convert = None
def __init__(self, file, cols, rows, matching_symbols=None, matching_symbols_coords=None, browser=None):
self.cols = cols
self.rows = rows
# Needed even if init is overwrite
self.path = self.build_path(browser)
# Get self.image
self.load_image(file, self.margin, self.convert)
# Get self.tiles
self.get_tiles( matching_symbols=matching_symbols,
matching_symbols_coords=matching_symbols_coords)
# Tiles processing
self.cut_tiles(self.tile_margin)
self.hash_md5_tiles()
def build_path(self, browser=None):
if browser and browser.responses_dirname:
return browser.responses_dirname
else:
return tempfile.mkdtemp(prefix='weboob_session_')
def load_image(self, file, margin=None, convert=None):
self.image = Image.open(file)
# Resize image if margin is given
if margin:
self.image = self.cut_margin(self.image, margin)
if convert:
self.image = self.image.convert(convert)
# Give possibility to alter image before get tiles, overwrite :func:`alter_image`.
self.alter_image()
self.width, self.height = self.image.size
def alter_image(self):
pass
def cut_margin(self, image, margin):
width, height = image.size
# Verify the magin value format
if type(margin) is int:
margin = (margin, margin, margin, margin)
elif len(margin) == 2:
margin = (margin[0], margin[1], margin[0], margin[1])
elif len(margin) == 4:
margin = margin
else:
assert (len(margin) == 3) & (len(margin) > 4), \
"Margin format is wrong."
assert ((margin[0] + margin[2]) < height) & ((margin[1] + margin[3]) < width), \
"Margin is too high, there is not enough pixel to cut."
image = image.crop((0 + margin[3],
0 + margin[0],
width - margin[1],
height - margin[2]
))
return image
def get_tiles(self, matching_symbols=None, matching_symbols_coords=None):
self.tiles = []
# Tiles coords are given
if matching_symbols_coords:
for matching_symbol in matching_symbols_coords:
self.tiles.append(Tile( matching_symbol=matching_symbol,
coords=matching_symbols_coords[matching_symbol]
))
return
assert (not self.width%self.cols) & (not self.height%self.rows), \
"Image width and height are not multiple of cols and rows. Please resize image with attribute `margin`."
# Tiles coords aren't given, calculate them
self.tileW = self.width // self.cols
self.tileH = self.height // self.rows
# Matching symbols aren't given, default value is range(columns*rows)
if not matching_symbols:
matching_symbols = ['%s' % i for i in range(self.cols*self.rows)]
assert len(matching_symbols) == (self.cols*self.rows), \
"Number of website matching symbols is not equal to the number of cases on the image."
# Calculate tiles coords for each matching symbol from 1-dimension to 2-dimensions
for index, matching_symbol in enumerate(matching_symbols):
coords = self.get_tile_coords_in_grid(index)
self.tiles.append(Tile(matching_symbol=matching_symbol, coords=coords))
def get_tile_coords_in_grid(self, case_index):
# Get the top left pixel coords of the tile
x0 = (case_index % self.cols) * self.tileW
y0 = (case_index // self.cols) * self.tileH
# Get the bottom right coords of the tile
x1 = x0 + self.tileW
y1 = y0 + self.tileH
coords = (x0, y0, x1, y1)
return(coords)
def cut_tiles(self, tile_margin=None):
for tile in self.tiles:
tile.image = self.image.crop(tile.coords)
# Resize tile if margin is given
if tile_margin:
for tile in self.tiles:
tile.image = self.cut_margin(tile.image, tile_margin)
def hash_md5_tiles(self):
for tile in self.tiles:
tile.md5 = hashlib.md5(tile.image.tobytes()).hexdigest()
def dump_tiles(self, path):
for tile in self.tiles:
tile.image.save('{}/{}.png'.format(path, tile.md5))
def get_string_code(self, password):
word = []
for digit in password:
for tile in self.tiles:
if tile.md5 in self.symbols[digit]:
word.append(tile.matching_symbol)
break
else:
# Dump file only if the symbol is not found
self.dump_tiles(self.path)
raise VirtKeyboardError("Symbol '%s' not found; all symbol hashes are available in %s"
% (digit, self.path))
return self.codesep.join(word)
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/stale_pyc.py 0000775 0000000 0000000 00000001325 14014251244 0024237 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python3
from __future__ import print_function
import os
import sys
root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))
verbose = '-v' in sys.argv
excludes = ('.git', '.svn', '__pycache__')
for dirpath, dirnames, filenames in os.walk(root):
for exclude in excludes:
try:
dirnames.remove(exclude)
except ValueError:
pass
for filename in filenames:
if filename.endswith('.pyc') or filename.endswith('pyo'):
if not os.path.exists(os.path.join(dirpath, filename[:-1])):
os.unlink(os.path.join(dirpath, filename))
if verbose:
print(os.path.join(dirpath, filename))
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/weboob_bash_completion 0000664 0000000 0000000 00000046272 14014251244 0026337 0 ustar 00root root 0000000 0000000
# Weboob completion for Bash (automatically generated by tools/make_man.sh)
#
# vim: filetype=sh expandtab softtabstop=4 shiftwidth=4
#
# This file is part of weboob.
#
# This script can be distributed under the same license as the
# weboob or bash packages.
_weboob_boobmsg()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-O --insecure -t -c --backends cd --export-session --quiet --outfile --help export_all -b -I --select -E -d export_thread help -n --debug photos --condition --save-responses -s --version ls backends show --accept-empty -v -f --no-header --no-keys --formatter -h --nss -q --logging-file --exclude-backends --count list post profile status --verbose -e -a --auto-update --title"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_boobmsg boobmsg
_weboob_weboob_cli()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-O --insecure -c --backends cd --export-session --quiet --outfile --help -b -I --select -d help -n --debug --condition --save-responses -s --version ls backends -v -f --no-header --no-keys --formatter -h --nss -q --logging-file --count --verbose -e -a --auto-update --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_weboob_cli weboob-cli
_weboob_webcontentedit()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-O --insecure -c --backends cd --export-session --quiet --outfile --help -b -I --select -d help -n --debug --condition --save-responses edit -s --version ls backends -v -f --no-header --no-keys --formatter -h --nss get -q --logging-file --count log --verbose -e -a --auto-update --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_webcontentedit webcontentedit
_weboob_geolooc()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-O --insecure -c --backends cd --export-session --quiet --outfile --help -b -I --select -d help -n --debug --condition --save-responses -s --version ls backends -v -f --no-header --no-keys --formatter -h --nss -q --logging-file --count --verbose -e -a --auto-update --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_geolooc geolooc
_weboob_weboorrents()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-O --insecure -c --backends cd --export-session --quiet --outfile --help search -b -I --select -d help -n --debug info --condition --save-responses -s --version ls backends -v -f --no-header --no-keys --formatter -h --nss -q --logging-file --count --verbose -e -a getfile --auto-update --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_weboorrents weboorrents
_weboob_suboob()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-O --insecure -c --backends cd --export-session --quiet --outfile --help search -b -I --select -d help -n --debug info download --condition --save-responses -s --version ls backends -v -f --no-header --no-keys --formatter -h --nss -q --logging-file --count --verbose -e -a --auto-update --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_suboob suboob
_weboob_boomoney()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-O --insecure -c --backends budgea --export-session --quiet --outfile cd add_recipient --help -b -I --select -d --noimport advisor help -n --debug -P --condition transfer --save-responses -A -s --version ls --force --display backends --parallel -v -f --no-header --no-keys --formatter -D recipients -h --nss investment -q --logging-file --count coming -F list --account profile --verbose history pocket -e -a --auto-update -N --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_boomoney boomoney
_weboob_havedate()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-O --insecure -t -c --backends cd --export-session --quiet --outfile --help export_all -b -I --select -E -d export_thread help -n --debug photos --condition --save-responses -s --version ls backends show --accept-empty -v -f --no-header --no-keys --formatter events -h --nss -q --logging-file query --exclude-backends --count list post profile status --verbose -e -a --auto-update optim --title"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_havedate havedate
_weboob_parceloob()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-O --insecure -c --backends cd --export-session --quiet --outfile --help -b -I --select -d untrack help -n --debug info --condition --save-responses -s --version ls backends track -v -f --no-header --no-keys --formatter -h --nss -q --logging-file --count status --verbose -e -a --auto-update --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_parceloob parceloob
_weboob_handjoob()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-O --insecure -c --backends cd --export-session --quiet --outfile --help search -b -I --select -d help -n --debug info --condition --save-responses -s --version ls backends -v -f --no-header --no-keys --formatter -h --nss -q --logging-file --count --verbose -e -a --auto-update --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_handjoob handjoob
_weboob_boobathon()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-O --insecure -c --backends cd --export-session --quiet --outfile remtask members --help -b -I --select progress -d help -n --debug info --condition --save-responses close edit -s --version ls cancel done backends join -v tasks -f --no-header --no-keys --formatter start -h leave --nss addtask -q --logging-file --count --verbose -e -a --auto-update --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_boobathon boobathon
_weboob_traveloob()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-O --insecure -c --backends cd --export-session --quiet --outfile --departure-time --help -b -I --select -d departures help -n --debug --condition --save-responses -s --version ls backends -v stations -f --no-header --no-keys --formatter -h --nss --arrival-time roadmap -q --logging-file --count --verbose -e -a --auto-update --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_traveloob traveloob
_weboob_comparoob()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-O --insecure -c --backends cd --export-session --quiet --outfile --help -b -I --select -d help -n --debug info --condition --save-responses -s --version ls backends -v -f --no-header --no-keys --formatter -h --nss -q --logging-file --count prices --verbose -e -a --auto-update --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_comparoob comparoob
_weboob_booblyrics()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-O --insecure -c --backends cd --export-session --quiet --outfile --help search -b -I --select -d help -n --debug --condition --save-responses -s --version ls backends -v -f --no-header --no-keys --formatter -h --nss get -q --logging-file --count --verbose -e -a --auto-update --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_booblyrics booblyrics
_weboob_weboob_debug()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="--insecure --backends --export-session --quiet --help -b -I -d -B --debug --save-responses --bpython --version -v -h --nss -q --logging-file --verbose -e -a --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_weboob_debug weboob-debug
_weboob_boobands()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-O suggestions --insecure -c --backends cd --export-session --quiet --outfile --help search -b -I --select -d help -n --debug info --condition --save-responses -s --version ls backends -v -f --no-header --no-keys --formatter -h --nss favorites albums -q --logging-file --count --verbose -e -a --auto-update --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_boobands boobands
_weboob_monboob()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-O --insecure -c --backends cd --export-session --quiet --outfile once --help -b -I --select -d help --smtpd -n --debug --condition --save-responses -s --version ls backends run -v -f --no-header --no-keys --formatter -h --nss -S -q --logging-file --count post --verbose -e -a --auto-update --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_monboob monboob
_weboob_boobsize()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-O --insecure -c --backends cd --export-session --quiet --outfile --help search -b -I --select -d help -n --debug --condition --save-responses last_sensor_measure -s --version ls backends -v -f --no-header --no-keys --formatter -h --nss -q --logging-file --count --verbose history -e -a details --auto-update --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_boobsize boobsize
_weboob_flatboob()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-O --insecure -c --backends cd --export-session --quiet --outfile --help search load -b -I --select -d help -n --debug info --condition --save-responses -s --version ls backends -v -f --no-header --no-keys --formatter -h --nss -q --logging-file --count --verbose -e -a --auto-update --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_flatboob flatboob
_weboob_weboob_repos()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-O --insecure -c --backends cd --export-session --quiet --outfile create --help -b -I --select -d help -n --debug --condition --save-responses -s --version ls backends -v -f --no-header --no-keys --formatter -h --nss -q --logging-file build --count --verbose -e -a --auto-update --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_weboob_repos weboob-repos
_weboob_translaboob()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-O --insecure -c --backends cd --export-session --quiet --outfile --help -b -I --select -d help -n --debug --condition --save-responses -s --version ls translate backends -v -f --no-header --no-keys --formatter -h --nss -q --logging-file --count --verbose -e -a --auto-update --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_translaboob translaboob
_weboob_radioob()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-O --insecure -c --backends cd --export-session --quiet --outfile playlist --help play search -b -I --select -d help -n --debug info download --condition --save-responses -s --version ls backends -v -f --no-header --no-keys --formatter -h --nss -q --logging-file --count --verbose -e -a --auto-update --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_radioob radioob
_weboob_wetboobs()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-O --insecure -c --backends cd --export-session --quiet --outfile --help -b -I --select -d help -n --debug --condition --save-responses -s --version ls backends -v -f --no-header --no-keys --formatter -h --nss cities -q --logging-file --count forecasts --verbose -e -a --auto-update current --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_wetboobs wetboobs
_weboob_shopoob()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-O --insecure -c --backends cd --export-session --quiet --outfile --help -b -I --select -d payments help -n --debug --condition --save-responses orders -s --version ls backends -v -f --no-header --no-keys --formatter -h --nss -q --logging-file --count items --verbose -e -a --auto-update --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_shopoob shopoob
_weboob_weboob_config()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-O --insecure -c --backends cd --export-session --quiet --outfile --help -b -I --select -d help -n --debug info --condition --save-responses edit -s --version ls backends -v -f --no-header --no-keys --formatter update add enable -h --nss register -q --logging-file --count disable list --verbose modules remove -e -a confirm --auto-update --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_weboob_config weboob-config
_weboob_videoob()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-O --insecure -c --backends cd --export-session --quiet --outfile playlist --help play search -b -I --select nsfw -d help -n --debug info download --condition --save-responses -s --version ls backends -v -f --no-header --no-keys --formatter -h --nss -q --logging-file --count --verbose -e -a --auto-update --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_videoob videoob
_weboob_weboob()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="--insecure --backends --export-session --quiet --help -b -I -d --debug --save-responses --version -v -h --nss -q --logging-file --verbose -e -a --auto-update --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_weboob weboob
_weboob_boobcoming()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-O --insecure -c --backends cd --export-session --quiet --outfile --help load search -b -I --select -d help -n --debug info --condition --save-responses export -s --version ls backends attends -v -f --no-header --no-keys --formatter -h --nss -q --logging-file --count list unattends --verbose -e -a --auto-update --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_boobcoming boobcoming
_weboob_cookboob()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-O --insecure -c --backends cd --export-session --quiet --outfile --help search -b -I --select -d help -n --debug info --condition --save-responses export -s --version ls backends -v -f --no-header --no-keys --formatter -h --nss -q --logging-file --count --verbose -e -a --auto-update --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_cookboob cookboob
_weboob_boobill()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="subscriptions -O --insecure -c --backends cd --export-session --quiet --outfile --help -b -I --select -d help -n --debug download_pdf download --condition --save-responses documents bills -s --version ls backends -v -f --no-header --no-keys --formatter balance -h --nss -q --logging-file --count profile --verbose history -e -a details --auto-update --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_boobill boobill
_weboob_boobooks()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-O --insecure -c --backends cd --export-session --quiet --outfile --help search -b -I --select -d help -n --debug --condition --save-responses -s --version ls backends -v -f --no-header --no-keys --formatter -h --nss renew -q --logging-file --count --verbose rented -e -a --auto-update --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_boobooks boobooks
_weboob_pastoob()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-m -O --insecure -t -c --backends cd --export-session --quiet --max-age --outfile --help -b -I --select -E -d help -n --debug info --condition --save-responses -s --version ls get_bin backends post_bin -v -f --no-header --no-keys --formatter -h --nss get -q --logging-file --exclude-backends --count --encoding --public post --verbose -p -e -a --auto-update --title"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_pastoob pastoob
_weboob_cineoob()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="search_torrent -O --insecure -c --backends cd --export-session --quiet --outfile getfile_subtitle --help -b -I --select -d info_movie persons_in_common info_person help search_subtitle -n --debug --condition --save-responses info_torrent -s --version releases ls search_person backends search_movie_torrent -v info_subtitle filmography -f --no-header --no-keys --formatter search_movie movies_in_common -h --nss getfile_torrent -q --logging-file search_movie_subtitle --count casting --verbose biography -e -a --auto-update --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_cineoob cineoob
_weboob_boobtracker()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="--status -O --insecure -c --backends cd --export-session --quiet --outfile --help search -b -I --select --start -d help -n --debug --condition --save-responses edit attach -s --version ls backends -v comment -f --tracker --no-header --formatter --no-keys logtime -h --nss --category --author --due get -q --logging-file --priority --exclude-backends --count --assignee post --target-version --verbose remove -e -a --auto-update --title"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_boobtracker boobtracker
_weboob_galleroob()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-O --insecure -c --backends cd --export-session --quiet --outfile --help search -b -I --select -d help -n --debug info download --condition --save-responses -s --version ls backends -v -f --no-header --no-keys --formatter -h --nss -q --logging-file --count --verbose -e -a --auto-update --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_galleroob galleroob
_weboob_boobank()
{
local cur args
COMPREPLY=()
cur=${COMP_WORDS[COMP_CWORD]}
args="-O --insecure -c --backends budgea --export-session --quiet --outfile cd add_recipient --help -b -I --select -d advisor help -n --debug transfer --condition --save-responses -s --version ls backends -v -f --no-header --no-keys --formatter recipients -h --nss investment -q --logging-file --count coming list profile --verbose history pocket -e -a --auto-update --exclude-backends"
COMPREPLY=( $(compgen -o default -W "${args}" -- "$cur" ) )
}
complete -F _weboob_boobank boobank
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/weboob_lint.py 0000775 0000000 0000000 00000002116 14014251244 0024556 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python3
from __future__ import print_function
import logging
import os
import sys
from weboob.core import Weboob
# Hint: use this script with file:///path/to/local/modules/ in sources.list
# if you want to correctly check all modules.
logging.basicConfig()
weboob = Weboob()
weboob.modules_loader.load_all()
modules_without_tests = []
modules_without_icons = []
for name, module in weboob.modules_loader.loaded.items():
path = module.package.__path__[0]
if not os.path.exists(os.path.join(path, 'test.py')):
modules_without_tests.append(name)
if not os.path.exists(os.path.join(path, 'favicon.png')) and \
not os.path.exists(os.path.join(weboob.repositories.icons_dir, '%s.png' % name)) and \
not module.icon:
modules_without_icons.append(name)
if modules_without_tests:
print('\nModules without tests: %s' % ', '.join(sorted(modules_without_tests)))
if modules_without_icons:
print('\nModules without icons: %s' % ', '.join(sorted(modules_without_icons)))
if modules_without_tests or modules_without_icons:
sys.exit(1)
woob-93297fa1488f03afedd5801ce30540b427cb2d96-tools/tools/weboob_lint.sh 0000775 0000000 0000000 00000001716 14014251244 0024545 0 ustar 00root root 0000000 0000000 #!/bin/sh
# stop on failure
set -e
. "$(dirname $0)/common.sh"
[ -z "${TMPDIR}" ] && TMPDIR="/tmp"
# do not allow undefined variables anymore
set -u
WEBOOB_TMPDIR=$(mktemp -d "${TMPDIR}/weboob_lint.XXXXXX")
# path to sources
WEBOOB_DIR=$(cd $(dirname $0)/.. && pwd -P)
touch "${WEBOOB_TMPDIR}/backends"
chmod 600 "${WEBOOB_TMPDIR}/backends"
echo "file://$WEBOOB_DIR/modules" > "${WEBOOB_TMPDIR}/sources.list"
export WEBOOB_WORKDIR="${WEBOOB_TMPDIR}"
export WEBOOB_DATADIR="${WEBOOB_TMPDIR}"
export PYTHONPATH="${WEBOOB_DIR}"
set +e
# TODO can we require weboob to be installed before being able to run run_tests.sh?
# if we can, then weboob-config is present in PATH (virtualenv or whatever)
${PYTHON} -c "import sys; sys.argv='weboob-config update'.split(); from weboob.applications.weboobcfg import WeboobCfg; WeboobCfg.run()"
$PYTHON "${WEBOOB_DIR}/tools/weboob_lint.py"
# allow failing commands past this point
STATUS=$?
rm -rf "${WEBOOB_TMPDIR}"
exit $STATUS