stable_backport.py 8.01 KB
Newer Older
1 2 3 4 5 6 7 8
#!/usr/bin/env python3

from __future__ import print_function

import time
import sys
import re
from contextlib import contextmanager
9
from os import system, path, makedirs, getenv
10 11
from subprocess import check_output, STDOUT, CalledProcessError
from collections import defaultdict
12
import shutil
13 14 15 16

from termcolor import colored


17 18
STABLE_VERSION = getenv('WEBOOB_BACKPORT_STABLE', '1.3')
DEVEL_BRANCH = getenv('WEBOOB_BACKPORT_DEVEL', 'master')
19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40


@contextmanager
def log(message, success='done'):
    print('%s... ' % message, end='', flush=True)
    start = time.time()
    try:
        yield
    except KeyboardInterrupt:
        print(colored('abort', 'red'))
        sys.exit(1)
    except Exception as e:
        print(colored('fail: %s' % e, 'red'))
        raise
    else:
        print('%s %s' % (colored(success, 'green'),
                         colored('(%.2fs)' % (time.time() - start), 'blue')))


def create_compat_dir(name):
    if not path.exists(name):
        makedirs(name)
41 42
    with open(path.join(name, '__init__.py'), 'w'):
        pass
43 44


45
MANUAL_PORTS = [
46
    'weboob.tools.captcha.virtkeyboard',
47 48 49 50 51
]

MANUAL_PORT_DIR = path.join(path.dirname(__file__), 'stable_backport_data')


52 53 54 55 56 57 58
class Error(object):
    def __init__(self, filename, linenum, message):
        self.filename = filename
        self.linenum = linenum
        self.message = message
        self.compat_dir = path.join(path.dirname(filename), 'compat')

59 60 61
    def __repr__(self):
        return '<%s filename=%r linenum=%s message=%r>' % (type(self).__name__, self.filename, self.linenum, self.message)

62 63 64 65 66 67 68
    def reimport_module(self, module):
        # not a weboob module, probably a false positive.
        if not module.startswith('weboob'):
            return

        dirname = module.replace('.', '/')
        filename = dirname + '.py'
69 70
        new_module = module.replace('.', '_')
        target = path.join(self.compat_dir, '%s.py' % new_module)
71 72 73 74 75 76 77 78
        base_module = '.'.join(module.split('.')[:-1])

        try:
            r = check_output('git show %s:%s' % (DEVEL_BRANCH, filename), shell=True, stderr=STDOUT).decode('utf-8')
        except CalledProcessError:
            # this file does not exist, perhaps a directory.
            return

79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102
        if module in MANUAL_PORTS:
            shutil.copyfile(path.join(MANUAL_PORT_DIR, path.basename(target)), target)
        else:
            # Copy module from devel to a compat/ sub-module
            with open(target, 'w') as fp:
                for line in r.split('\n'):
                    # Replace relative imports to absolute ones
                    m = re.match(r'^from (\.\.?)([\w_\.]+) import (.*)', line)
                    if m:
                        if m.group(1) == '..':
                            base_module = '.'.join(base_module.split('.')[:-1])
                        fp.write('from %s.%s import %s\n' % (base_module, m.group(2), m.group(3)))
                        continue

                    # Inherit all classes by previous ones, if they already existed.
                    m = re.match(r'^class (\w+)\(([\w,\s]+)\):(.*)', line)
                    if m and path.exists(filename) and system('grep "^class %s" %s >/dev/null' % (m.group(1), filename)) == 0:
                        symbol = m.group(1)
                        trailing = m.group(3)
                        fp.write('from %s import %s as _%s\n' % (module, symbol, symbol))
                        fp.write('class %s(_%s):%s\n' % (symbol, symbol, trailing))
                        continue

                    fp.write('%s\n' % line)
103 104 105 106 107 108 109 110

        # Particular case, in devel some imports have been added to
        # weboob/browser/__init__.py
        system(r'sed -i -e "s/from weboob.browser import/from weboob.browser.browsers import/g" %s'
               % self.filename)
        # Replace import to this module by a relative import to the copy in
        # compat/
        system(r'sed -i -e "%ss/from \([A-Za-z0-9_\.]\+\) import \(.*\)/from .compat.%s import \2/g" %s'
111
               % (self.linenum, new_module, self.filename))
112 113


114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136
def remove_block(name, start):
    lines = []
    with open(name, 'r') as fd:
        it = iter(fd)
        for n in range(start - 1):
            lines.append(next(it))
        line = next(it)

        level = len(re.match(r'^( *)', line).group(1))
        for line in it:
            if not line.strip():
                continue
            new = len(re.match(r'^( *)', line).group(1))
            if new <= level:
                lines.append(line)
                break

        lines.extend(it)

    with open(name, 'w') as fd:
        fd.write(''.join(lines))


137 138 139 140 141 142 143 144 145 146 147 148 149 150
class NoNameInModuleError(Error):
    def fixup(self):
        m = re.match(r"No name '(\w+)' in module '([\w\.]+)'", self.message)
        module = m.group(2)
        self.reimport_module(module)


class ImportErrorError(Error):
    def fixup(self):
        m = re.match(r"Unable to import '([\w\.]+)'", self.message)
        module = m.group(1)
        self.reimport_module(module)


151 152 153 154 155
class ManualBackport(Error):
    def fixup(self):
        self.reimport_module(self.message)


156 157 158 159 160 161 162
def replace_all(expr, dest):
    system(r"""for file in $(git ls-files modules | grep '\.py$');
               do
                   sed -i -e "s/""" + expr + '/' + dest + """/g" $file
               done""")


163 164
def output_lines(cmd):
    return check_output(cmd, shell=True, stderr=STDOUT).decode('utf-8').rstrip().split('\n')
165 166


167 168 169 170 171 172
class StableBackport(object):
    errors = {'E0611': NoNameInModuleError,
              'E0401': ImportErrorError,
             }

    def main(self):
173 174 175
        with log('Removing previous compat files'):
            system('git rm -q "modules/*/compat/*.py"')

176 177 178 179
        with log('Copying last version of modules from devel'):
            system('git checkout --theirs %s modules' % DEVEL_BRANCH)

        with log('Replacing version number'):
180 181
            replace_all(r"""^\(\s*\)\(VERSION\)\( *\)=\( *\)[\"'][0-9]\+\..\+[\"']\(,\?\)$""",
                        r"""\1\2\3=\4'""" + STABLE_VERSION + r"""'\5""")
182 183 184 185 186 187 188

        with log('Removing staling data'):
            system('tools/stale_pyc.py')
            system('find modules -type d -empty -delete')
            system('git add -u')

        with log('Lookup modules errors'):
189
            r = check_output("pylint modules/* -f parseable -E -d all -e no-name-in-module,import-error; exit 0", shell=True, stderr=STDOUT).decode('utf-8')
190 191 192 193 194 195 196 197 198 199 200 201 202 203

        dirnames = defaultdict(list)
        for line in r.split('\n'):
            m = re.match(r'([\w\./]+):(\d+): \[(\w+)[^\]]+\] (.*)', line)
            if not m:
                continue

            filename = m.group(1)
            linenum = m.group(2)
            error = m.group(3)
            msg = m.group(4)

            dirnames[path.dirname(filename)].append(self.errors[error](filename, linenum, msg))

204 205 206 207 208 209 210 211 212 213 214 215 216 217 218
        with log('Searching manual backports'):
            for manual in MANUAL_PORTS:
                r = check_output("grep -nEr '^from %s import ' modules" % manual, shell=True).strip().decode('utf-8')
                for line in r.split('\n'):
                    m = re.match(r'([\w\./]+):(\d+):.*', line)
                    filename = m.group(1)
                    linenum = m.group(2)
                    target = dirnames[path.dirname(filename)]
                    for err in target:
                        if err.filename == filename and err.linenum == linenum:
                            # an error was already spot on this line
                            break
                    else:
                        target.append(ManualBackport(filename, linenum, manual))

219 220 221 222 223 224 225 226 227
        for dirname, errors in sorted(dirnames.items()):
            with log('Fixing up %s errors in %s' % (colored(str(len(errors)), 'magenta'),
                                                    colored(dirname, 'yellow'))):
                compat_dirname = path.join(dirname, 'compat')
                create_compat_dir(compat_dirname)
                for error in errors:
                    error.fixup()
                system('git add %s' % compat_dirname)

228 229
        system('git add -u')

230 231 232

if __name__ == '__main__':
    StableBackport().main()