stable_backport.py 7.11 KB
Newer Older
1 2 3 4 5 6 7 8
#!/usr/bin/env python3

from __future__ import print_function

import time
import sys
import re
from contextlib import contextmanager
9
from os import system, path, makedirs, getenv
10 11
from subprocess import check_output, STDOUT, CalledProcessError
from collections import defaultdict
12
import shutil
13 14 15 16

from termcolor import colored


17 18
STABLE_VERSION = getenv('WEBOOB_BACKPORT_STABLE', '1.3')
DEVEL_BRANCH = getenv('WEBOOB_BACKPORT_DEVEL', 'master')
19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44


@contextmanager
def log(message, success='done'):
    print('%s... ' % message, end='', flush=True)
    start = time.time()
    try:
        yield
    except KeyboardInterrupt:
        print(colored('abort', 'red'))
        sys.exit(1)
    except Exception as e:
        print(colored('fail: %s' % e, 'red'))
        raise
    else:
        print('%s %s' % (colored(success, 'green'),
                         colored('(%.2fs)' % (time.time() - start), 'blue')))


def create_compat_dir(name):
    if not path.exists(name):
        makedirs(name)
        with open(path.join(name, '__init__.py'), 'w'):
            pass


45 46 47 48 49 50
MANUAL_PORTS = [
]

MANUAL_PORT_DIR = path.join(path.dirname(__file__), 'stable_backport_data')


51 52 53 54 55 56 57
class Error(object):
    def __init__(self, filename, linenum, message):
        self.filename = filename
        self.linenum = linenum
        self.message = message
        self.compat_dir = path.join(path.dirname(filename), 'compat')

58 59 60
    def __repr__(self):
        return '<%s filename=%r linenum=%s message=%r>' % (type(self).__name__, self.filename, self.linenum, self.message)

61 62 63 64 65 66 67
    def reimport_module(self, module):
        # not a weboob module, probably a false positive.
        if not module.startswith('weboob'):
            return

        dirname = module.replace('.', '/')
        filename = dirname + '.py'
68 69
        new_module = module.replace('.', '_')
        target = path.join(self.compat_dir, '%s.py' % new_module)
70 71 72 73 74 75 76 77
        base_module = '.'.join(module.split('.')[:-1])

        try:
            r = check_output('git show %s:%s' % (DEVEL_BRANCH, filename), shell=True, stderr=STDOUT).decode('utf-8')
        except CalledProcessError:
            # this file does not exist, perhaps a directory.
            return

78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101
        if module in MANUAL_PORTS:
            shutil.copyfile(path.join(MANUAL_PORT_DIR, path.basename(target)), target)
        else:
            # Copy module from devel to a compat/ sub-module
            with open(target, 'w') as fp:
                for line in r.split('\n'):
                    # Replace relative imports to absolute ones
                    m = re.match(r'^from (\.\.?)([\w_\.]+) import (.*)', line)
                    if m:
                        if m.group(1) == '..':
                            base_module = '.'.join(base_module.split('.')[:-1])
                        fp.write('from %s.%s import %s\n' % (base_module, m.group(2), m.group(3)))
                        continue

                    # Inherit all classes by previous ones, if they already existed.
                    m = re.match(r'^class (\w+)\(([\w,\s]+)\):(.*)', line)
                    if m and path.exists(filename) and system('grep "^class %s" %s >/dev/null' % (m.group(1), filename)) == 0:
                        symbol = m.group(1)
                        trailing = m.group(3)
                        fp.write('from %s import %s as _%s\n' % (module, symbol, symbol))
                        fp.write('class %s(_%s):%s\n' % (symbol, symbol, trailing))
                        continue

                    fp.write('%s\n' % line)
102 103 104 105 106 107 108 109

        # Particular case, in devel some imports have been added to
        # weboob/browser/__init__.py
        system(r'sed -i -e "s/from weboob.browser import/from weboob.browser.browsers import/g" %s'
               % self.filename)
        # Replace import to this module by a relative import to the copy in
        # compat/
        system(r'sed -i -e "%ss/from \([A-Za-z0-9_\.]\+\) import \(.*\)/from .compat.%s import \2/g" %s'
110
               % (self.linenum, new_module, self.filename))
111 112


113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135
def remove_block(name, start):
    lines = []
    with open(name, 'r') as fd:
        it = iter(fd)
        for n in range(start - 1):
            lines.append(next(it))
        line = next(it)

        level = len(re.match(r'^( *)', line).group(1))
        for line in it:
            if not line.strip():
                continue
            new = len(re.match(r'^( *)', line).group(1))
            if new <= level:
                lines.append(line)
                break

        lines.extend(it)

    with open(name, 'w') as fd:
        fd.write(''.join(lines))


136 137 138 139 140 141 142 143 144 145 146 147 148 149
class NoNameInModuleError(Error):
    def fixup(self):
        m = re.match(r"No name '(\w+)' in module '([\w\.]+)'", self.message)
        module = m.group(2)
        self.reimport_module(module)


class ImportErrorError(Error):
    def fixup(self):
        m = re.match(r"Unable to import '([\w\.]+)'", self.message)
        module = m.group(1)
        self.reimport_module(module)


150 151 152 153 154 155 156
def replace_all(expr, dest):
    system(r"""for file in $(git ls-files modules | grep '\.py$');
               do
                   sed -i -e "s/""" + expr + '/' + dest + """/g" $file
               done""")


157 158
def output_lines(cmd):
    return check_output(cmd, shell=True, stderr=STDOUT).decode('utf-8').rstrip().split('\n')
159 160


161 162 163 164 165 166
class StableBackport(object):
    errors = {'E0611': NoNameInModuleError,
              'E0401': ImportErrorError,
             }

    def main(self):
167 168 169
        with log('Removing previous compat files'):
            system('git rm -q "modules/*/compat/*.py"')

170 171 172 173
        with log('Copying last version of modules from devel'):
            system('git checkout --theirs %s modules' % DEVEL_BRANCH)

        with log('Replacing version number'):
174 175
            replace_all(r"""^\(\s*\)\(VERSION\)\( *\)=\( *\)[\"'][0-9]\+\..\+[\"']\(,\?\)$""",
                        r"""\1\2\3=\4'""" + STABLE_VERSION + r"""'\5""")
176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206

        with log('Removing staling data'):
            system('tools/stale_pyc.py')
            system('find modules -type d -empty -delete')
            system('git add -u')

        with log('Lookup modules errors'):
            r = check_output("pylint modules -f parseable -E -d all -e no-name-in-module,import-error; exit 0", shell=True, stderr=STDOUT).decode('utf-8')

        dirnames = defaultdict(list)
        for line in r.split('\n'):
            m = re.match(r'([\w\./]+):(\d+): \[(\w+)[^\]]+\] (.*)', line)
            if not m:
                continue

            filename = m.group(1)
            linenum = m.group(2)
            error = m.group(3)
            msg = m.group(4)

            dirnames[path.dirname(filename)].append(self.errors[error](filename, linenum, msg))

        for dirname, errors in sorted(dirnames.items()):
            with log('Fixing up %s errors in %s' % (colored(str(len(errors)), 'magenta'),
                                                    colored(dirname, 'yellow'))):
                compat_dirname = path.join(dirname, 'compat')
                create_compat_dir(compat_dirname)
                for error in errors:
                    error.fixup()
                system('git add %s' % compat_dirname)

207 208
        system('git add -u')

209 210 211

if __name__ == '__main__':
    StableBackport().main()