From 434c0f15744e1febb2134ec1e8b56727b9afdd5f Mon Sep 17 00:00:00 2001 From: "Phyks (Lucas Verney)" Date: Tue, 29 Nov 2016 08:57:52 -0500 Subject: [PATCH] Gitlab-CI continuous integration This commit adds the necessary files to run the CI using Gitlab-CI. For now, it checks that Weboob builds, then runs the linting script (checking that every module as an icon and some tests + PyFlakes) and the unittests. Most of modules unittests cannot run because there is no backend configured. Some changes were needed in the pre-existing scripts: * Edit `weboob_lint` to exit with non-zero code if it finds modules without icons or tests, so that the build could fail in such a case. * Edit `run_tests.sh` to set correct exit code on failure and rework generation of XUNIT output. Also added some doc about useful environment variables. Added a way to generate an xunit output file when running modules unittests, passing a `XUNIT_OUT` env variable to `run_tests.sh` script. * Modification of `setup.cfg` and `run_tests` scripts to handle code coverage generation. The matching regex in Gitlab for the total code coverage is `TOTAL: (\d+\%\s*)$)`. I also added a script to generate a JSON module status matrix from modules unittests, ready to be sent to a [Weboob-CI](https://github.com/Phyks/weboob-ci) instance. NOTE: Required Python modules are taken from the `setup.py` script. `.ci/requirements.txt` contains the requirements to run the unittests and the CI, whereas `.ci/requirements_modules.txt` contains the specific Python modules required at runtime by Weboob modules. The latter could eventually be replaced by a proper call to `debpydep` script. --- .ci/requirements.txt | 5 +++ .ci/requirements_modules.txt | 3 ++ .gitignore | 1 + .gitlab-ci.yml | 27 ++++++++++++++ setup.cfg | 8 +++-- tools/modules_testing_grid.py | 66 +++++++++++++++++++++++++++++++++++ tools/run_tests.sh | 47 ++++++++++++++++++++++--- tools/weboob_lint.py | 4 +++ 8 files changed, 155 insertions(+), 6 deletions(-) create mode 100644 .ci/requirements.txt create mode 100644 .ci/requirements_modules.txt create mode 100644 .gitlab-ci.yml create mode 100755 tools/modules_testing_grid.py diff --git a/.ci/requirements.txt b/.ci/requirements.txt new file mode 100644 index 0000000000..b6095b85c4 --- /dev/null +++ b/.ci/requirements.txt @@ -0,0 +1,5 @@ +coverage==4.2 +flake8==3.2.1 +mock==2.0.0 +nose==1.3.7 +pyflakes==1.3.0 diff --git a/.ci/requirements_modules.txt b/.ci/requirements_modules.txt new file mode 100644 index 0000000000..18304475bc --- /dev/null +++ b/.ci/requirements_modules.txt @@ -0,0 +1,3 @@ +BeautifulSoup +html2text +simplejson diff --git a/.gitignore b/.gitignore index 5d0a145826..5abb875a83 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ modules/modules.list /localconfig *.idea/ *.DS_Store +*.coverage* diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000000..5c5d037c34 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,27 @@ +image: "python:2.7" + +before_script: + - "pip install -r .ci/requirements.txt" + - "REQUIREMENTS=$(mktemp) && python setup.py requirements > ${REQUIREMENTS} && pip install -r ${REQUIREMENTS} && rm ${REQUIREMENTS}" + - "pip install -r .ci/requirements_modules.txt" + +build: + stage: "build" + script: + - "./tools/local_install.sh ~/bin" + +lint: + stage: "test" + script: + - "./tools/pyflakes.sh" + - "./tools/weboob_lint.sh" + +unittests: + stage: "test" + script: + - "NOSE_PROCESSES=4 ./tools/run_tests.sh" + +doc: + stage: "deploy" + script: + - "cd ./docs && make html" diff --git a/setup.cfg b/setup.cfg index a5fcc4f905..86da6eb73d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,6 +2,7 @@ verbosity = 2 detailed-errors = 1 with-doctest = 1 +with-coverage = 1 where = weboob tests = weboob.tools.capabilities.bank.transactions, weboob.tools.capabilities.paste, @@ -18,9 +19,12 @@ tests = weboob.tools.capabilities.bank.transactions, weboob.browser.tests.url [isort] -known_first_party=weboob -line_length=120 +known_first_party = weboob +line_length = 120 [flake8] max-line-length = 120 exclude = dist,*.egg-info,build,.git,__pycache__ + +[easy_install] + diff --git a/tools/modules_testing_grid.py b/tools/modules_testing_grid.py new file mode 100755 index 0000000000..dec3fcfc64 --- /dev/null +++ b/tools/modules_testing_grid.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Script to format XUNIT output from unittests as a JSON string ready to be sent +to a [Weboob-CI](https://github.com/Phyks/weboob-ci) instance. + +* `XUNIT` is the XUNIT file to handle. +* `ORIGIN` is an origin string as described in the Weboob-CI documentation +(basically just a string to identify the source of the unittests results). +""" +from __future__ import print_function + +import collections +import json +import sys +import xunitparser + + +def main(xunit, origin): + with open(xunit, "r") as fh: + ts, tr = xunitparser.parse(fh) + + # Get test results for each module + modules = {} + other_testcases = [] + for tc in ts: + if not tc.classname.startswith("modules."): + other_testcases.append(repr(tc)) + continue + module = tc.classname.split(".")[1] + # In the following, we consider + # good > skipped > bad + # and only make update of a module status according to this order + if tc.good: + if tc.skipped: + # Set to skipped only if previous test was good + if module not in modules or modules[module] == "good": + modules[module] = "skipped" + else: + # Set to good only if no previous result + if module not in modules: + modules[module] = "good" + else: + # Always set to bad on failed test + modules[module] = "bad" + # Agregate results by test result rather than module + results = collections.defaultdict(list) + for module in modules: + results[modules[module]].append(module) + return { + "origin": origin, + "modules": results, + "others": other_testcases + } + + +if __name__ == "__main__": + if len(sys.argv) < 3: + sys.exit("Usage: %s XUNIT_FILE ORIGIN" % (sys.argv[0])) + + print( + json.dumps( + main(sys.argv[1], sys.argv[2]), + sort_keys=True, indent=4, separators=(',', ': ') + ) + ) diff --git a/tools/run_tests.sh b/tools/run_tests.sh index 4ef2d20cdc..14da4aeaae 100755 --- a/tools/run_tests.sh +++ b/tools/run_tests.sh @@ -1,4 +1,9 @@ -#!/bin/sh +#!/bin/bash + +# Mai available environment variables +# * RSYNC_TARGET: target on which to rsync the xunit output. +# * XUNIT_OUT: file in which xunit output should be saved. +# * WEBOOB_BACKENDS: path to the Weboob backends file to use. # stop on failure set -e @@ -30,6 +35,10 @@ else RSYNC_TARGET="" fi +if [ ! -n "${XUNIT_OUT}" ]; then + XUNIT_OUT="" +fi + # find executables if [ -z "${PYTHON}" ]; then which python >/dev/null 2>&1 && PYTHON=$(which python) @@ -56,11 +65,18 @@ fi # do not allow undefined variables anymore set -u WEBOOB_TMPDIR=$(mktemp -d "${TMPDIR}/weboob_test.XXXXX") -cp "${WEBOOB_BACKENDS}" "${WEBOOB_TMPDIR}/backends" +if [ -f "${WEBOOB_BACKENDS}" ]; then + cp "${WEBOOB_BACKENDS}" "${WEBOOB_TMPDIR}/backends" +else + touch "${WEBOOB_TMPDIR}/backends" + chmod go-r "${WEBOOB_TMPDIR}/backends" +fi # xunit nose setup if [ -n "${RSYNC_TARGET}" ]; then XUNIT_ARGS="--with-xunit --xunit-file=${WEBOOB_TMPDIR}/xunit.xml" +elif [ -n "${XUNIT_OUT}" ]; then + XUNIT_ARGS="--with-xunit --xunit-file=${XUNIT_OUT}" else XUNIT_ARGS="" fi @@ -77,17 +93,40 @@ ${PYTHON} "${WEBOOB_DIR}/scripts/weboob-config" update # allow failing commands past this point set +e +set -o pipefail if [ -n "${BACKEND}" ]; then ${PYTHON} ${NOSE} -c /dev/null -sv "${WEBOOB_MODULES}/${BACKEND}/test.py" ${XUNIT_ARGS} STATUS=$? STATUS_CORE=0 else echo "=== Weboob ===" - ${PYTHON} ${NOSE} -c ${WEBOOB_DIR}/setup.cfg -sv + CORE_TESTS=$(mktemp) + ${PYTHON} ${NOSE} --cover-package weboob -c ${WEBOOB_DIR}/setup.cfg -sv 2>&1 | tee "${CORE_TESTS}" STATUS_CORE=$? echo "=== Modules ===" - find "${WEBOOB_MODULES}" -name "test.py" | sort | xargs ${PYTHON} ${NOSE} -c /dev/null -sv ${XUNIT_ARGS} + MODULES_TESTS=$(mktemp) + MODULES_TO_TEST=$(find "${WEBOOB_MODULES}" -name "test.py" | sort | xargs echo) + ${PYTHON} ${NOSE} --with-coverage --cover-package modules -c /dev/null -sv ${XUNIT_ARGS} ${MODULES_TO_TEST} 2>&1 | tee ${MODULES_TESTS} STATUS=$? + + # Compute total coverage + echo "=== Total coverage ===" + CORE_STMTS=$(grep "TOTAL" ${CORE_TESTS} | awk '{ print $2; }') + CORE_MISS=$(grep "TOTAL" ${CORE_TESTS} | awk '{ print $3; }') + CORE_COVERAGE=$(grep "TOTAL" ${CORE_TESTS} | awk '{ print $4; }') + MODULES_STMTS=$(grep "TOTAL" ${MODULES_TESTS} | awk '{ print $2; }') + MODULES_MISS=$(grep "TOTAL" ${MODULES_TESTS} | awk '{ print $3; }') + MODULES_COVERAGE=$(grep "TOTAL" ${MODULES_TESTS} | awk '{ print $4; }') + echo "CORE COVERAGE: ${CORE_COVERAGE}" + echo "MODULES COVERAGE: ${MODULES_COVERAGE}" + TOTAL_STMTS=$((${CORE_STMTS} + ${MODULES_STMTS})) + TOTAL_MISS=$((${CORE_MISS} + ${MODULES_MISS})) + TOTAL_COVERAGE=$((100 * (${TOTAL_STMTS} - ${TOTAL_MISS}) / ${TOTAL_STMTS})) + echo "TOTAL: ${TOTAL_COVERAGE}%" + + # removal of temp files + rm ${CORE_TESTS} + rm ${MODULES_TESTS} fi # xunit transfer diff --git a/tools/weboob_lint.py b/tools/weboob_lint.py index 7b8bcbcab1..1524addf17 100755 --- a/tools/weboob_lint.py +++ b/tools/weboob_lint.py @@ -6,6 +6,7 @@ from weboob.core import Weboob import os +import sys weboob = Weboob() weboob.modules_loader.load_all() @@ -26,3 +27,6 @@ print('Modules without tests: %s' % backends_without_tests) if backends_without_icons: print('Modules without icons: %s' % backends_without_icons) + +if backends_without_tests or backends_without_icons: + sys.exit(1) -- GitLab