repositories.py 30.4 KB
Newer Older
1 2
# -*- coding: utf-8 -*-

3
# Copyright(C) 2010-2014 Romain Bignon, Laurent Bachelier
4 5 6 7
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
8
# it under the terms of the GNU Lesser General Public License as published by
9 10 11 12 13 14
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU Lesser General Public License for more details.
16
#
17
# You should have received a copy of the GNU Lesser General Public License
18 19
# along with weboob. If not, see <http://www.gnu.org/licenses/>.

20

21
from __future__ import print_function
22
import imp
23 24 25 26 27
import posixpath
import shutil
import re
import sys
import os
28
import subprocess
29
import hashlib
30
from compileall import compile_dir
31 32
from contextlib import closing, contextmanager
from datetime import datetime
33
from io import BytesIO, StringIO
34
from tempfile import NamedTemporaryFile
35

36
from weboob.exceptions import BrowserHTTPError, BrowserHTTPNotFound, ModuleInstallError
37
from .modules import LoadedModule
38
from weboob.tools.log import getLogger
39
from weboob.tools.misc import get_backtrace, to_unicode, find_exe
40
from weboob.tools.compat import basestring, unicode
41 42
try:
    from ConfigParser import RawConfigParser, DEFAULTSECT
43 44
except ImportError:
    from configparser import RawConfigParser, DEFAULTSECT
45

46

47
@contextmanager
48 49
def open_for_config(filename):
    if sys.version_info.major == 2:
50 51 52
        f = NamedTemporaryFile(mode='wb',
                               dir=os.path.dirname(filename),
                               delete=False)
53
    else:
54 55 56 57 58
        f = NamedTemporaryFile(mode='w', encoding='utf-8',
                               dir=os.path.dirname(filename),
                               delete=False)
    yield f
    os.rename(f.name, filename)
59 60


61
class ModuleInfo(object):
62 63 64
    """
    Information about a module available on a repository.
    """
65

66 67 68 69 70 71
    def __init__(self, name):
        self.name = name

        # path to the local directory containing this module.
        self.path = None
        self.url = None
72
        self.repo_url = None
73 74 75 76 77 78 79 80 81 82 83 84 85 86 87

        self.version = 0
        self.capabilities = ()
        self.description = u''
        self.maintainer = u''
        self.license = u''
        self.icon = u''
        self.urls = u''

    def load(self, items):
        self.version = int(items['version'])
        self.capabilities = items['capabilities'].split()
        self.description = to_unicode(items['description'])
        self.maintainer = to_unicode(items['maintainer'])
        self.license = to_unicode(items['license'])
88
        self.icon = items['icon'].strip() or None
89 90
        self.urls = items['urls']

91
    def has_caps(self, *caps):
Vincent A's avatar
Vincent A committed
92
        """Return True if module implements at least one of the caps."""
93 94
        if len(caps) == 1 and isinstance(caps[0], (list, tuple)):
            caps = caps[0]
95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113
        for c in caps:
            if type(c) == type:
                c = c.__name__
            if c in self.capabilities:
                return True
        return False

    def is_installed(self):
        return self.path is not None

    def is_local(self):
        return self.url is None

    def dump(self):
        return (('version', self.version),
                ('capabilities', ' '.join(self.capabilities)),
                ('description', self.description),
                ('maintainer', self.maintainer),
                ('license', self.license),
114
                ('icon', self.icon or ''),
115 116 117
                ('urls', self.urls),
               )

118

119
class RepositoryUnavailable(Exception):
120 121 122
    """
    Repository in not available.
    """
123

124

125
class Repository(object):
126 127 128
    """
    Represents a repository.
    """
129
    INDEX = 'modules.list'
130 131
    KEYDIR = '.keys'
    KEYRING = 'trusted.gpg'
132

133
    def __init__(self, url):
134 135 136 137 138
        self.url = url
        self.name = u''
        self.update = 0
        self.maintainer = u''
        self.local = None
139 140
        self.signed = False
        self.key_update = 0
141
        self.obsolete = False
142
        self.logger = getLogger('repository')
143
        self.errors = {}
144 145 146 147 148 149 150 151 152 153 154 155 156

        self.modules = {}

        if self.url.startswith('file://'):
            self.local = True
        elif re.match('https?://.*', self.url):
            self.local = False
        else:
            # This is probably a file in ~/.weboob/repositories/, we
            # don't know if this is a local or a remote repository.
            with open(self.url, 'r') as fp:
                self.parse_index(fp)

157 158 159
    def __repr__(self):
        return '<Repository %r>' % self.name

160 161 162 163
    def localurl2path(self):
        """
        Get a local path of a file:// URL.
        """
164
        assert self.local is True
165 166 167 168 169

        if self.url.startswith('file://'):
            return self.url[len('file://'):]
        return self.url

170
    def retrieve_index(self, browser, repo_path):
171 172 173 174
        """
        Retrieve the index file of this repository. It can use network
        if this is a remote repository.

175 176
        :param repo_path: path to save the downloaded index file (if any).
        :type repo_path: str or None
177
        """
178
        built = False
179 180 181 182 183
        if self.local:
            # Repository is local, open the file.
            filename = os.path.join(self.localurl2path(), self.INDEX)
            try:
                fp = open(filename, 'r')
184
            except IOError:
185
                # This local repository doesn't contain a built modules.list index.
186
                self.name = Repositories.url2filename(self.url)
187
                self.build_index(self.localurl2path(), filename)
188
                built = True
189 190 191 192
                fp = open(filename, 'r')
        else:
            # This is a remote repository, download file
            try:
193
                fp = StringIO(browser.open(posixpath.join(self.url, self.INDEX)).text)
194
            except BrowserHTTPError as e:
195
                raise RepositoryUnavailable(unicode(e))
196 197 198

        self.parse_index(fp)

199 200
        # this value can be changed by parse_index
        if self.local and not built:
201 202 203 204
            # Always rebuild index of a local repository.
            self.build_index(self.localurl2path(), filename)

        # Save the repository index in ~/.weboob/repositories/
205 206
        if repo_path:
            self.save(repo_path, private=True)
207

208
    def retrieve_keyring(self, browser, keyring_path, progress):
209 210 211 212 213 214 215 216 217 218 219 220 221 222
        # ignore local
        if self.local:
            return

        keyring = Keyring(keyring_path)
        # prevent previously signed repos from going unsigned
        if not self.signed and keyring.exists():
            raise RepositoryUnavailable('Previously signed repository can not go unsigned')
        if not self.signed:
            return

        if not keyring.exists() or self.key_update > keyring.version:
            # This is a remote repository, download file
            try:
223 224
                keyring_data = browser.open(posixpath.join(self.url, self.KEYRING)).content
                sig_data = browser.open(posixpath.join(self.url, self.KEYRING + '.sig')).content
225
            except BrowserHTTPError as e:
226
                raise RepositoryUnavailable(unicode(e))
227 228 229
            if keyring.exists():
                if not keyring.is_valid(keyring_data, sig_data):
                    raise InvalidSignature('the keyring itself')
230 231 232
                progress.progress(0.0, 'The keyring was updated (and validated by the previous one).')
            elif not progress.prompt('The repository %s isn\'t trusted yet.\nFingerprint of keyring is %s\nAre you sure you want to continue?' % (self.url, hashlib.sha1(keyring_data).hexdigest())):
                raise RepositoryUnavailable('Repository not trusted')
233
            keyring.save(keyring_data, self.key_update)
234
            progress.progress(0.0, str(keyring))
235 236 237 238 239

    def parse_index(self, fp):
        """
        Parse index of a repository

240 241
        :param fp: file descriptor to read
        :type fp: buffer
242 243 244 245 246 247 248 249 250 251
        """
        config = RawConfigParser()
        config.readfp(fp)

        # Read default parameters
        items = dict(config.items(DEFAULTSECT))
        try:
            self.name = items['name']
            self.update = int(items['update'])
            self.maintainer = items['maintainer']
252 253
            self.signed = bool(int(items.get('signed', '0')))
            self.key_update = int(items.get('key_update', '0'))
254
            self.obsolete = bool(int(items.get('obsolete', '0')))
255
        except KeyError as e:
256
            raise RepositoryUnavailable('Missing global parameters in repository: %s' % e)
257
        except ValueError as e:
258
            raise RepositoryUnavailable('Incorrect value in repository parameters: %s' % e)
259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275

        if len(self.name) == 0:
            raise RepositoryUnavailable('Name is empty')

        if 'url' in items:
            self.url = items['url']
            self.local = self.url.startswith('file://')
        elif self.local is None:
            raise RepositoryUnavailable('Missing "url" key in settings')

        # Load modules
        self.modules.clear()
        for section in config.sections():
            module = ModuleInfo(section)
            module.load(dict(config.items(section)))
            if not self.local:
                module.url = posixpath.join(self.url, '%s.tar.gz' % module.name)
276 277
                module.repo_url = self.url
                module.signed = self.signed
278 279 280 281 282 283
            self.modules[section] = module

    def build_index(self, path, filename):
        """
        Rebuild index of modules of repository.

284 285 286 287
        :param path: path of the repository
        :type path: str
        :param filename: file to save index
        :type filename: str
288
        """
289
        self.logger.debug('Rebuild index')
290
        self.modules.clear()
291
        self.errors.clear()
292

293 294 295 296 297 298 299
        if os.path.isdir(os.path.join(path, self.KEYDIR)):
            self.signed = True
            self.key_update = self.get_tree_mtime(os.path.join(path, self.KEYDIR), True)
        else:
            self.signed = False
            self.key_update = 0

300 301
        for name in sorted(os.listdir(path)):
            module_path = os.path.join(path, name)
302
            if not os.path.isdir(module_path) or '.' in name or name == self.KEYDIR or not os.path.exists(os.path.join(module_path, '__init__.py')):
303 304 305
                continue

            try:
306 307
                fp, pathname, description = imp.find_module(name, [path])
                try:
308
                    module = LoadedModule(imp.load_module(name, fp, pathname, description))
309 310 311
                finally:
                    if fp:
                        fp.close()
312
            except Exception as e:
313 314 315 316
                self.logger.warning('Unable to build module %s: [%s] %s' % (name, type(e).__name__, e))
                bt = get_backtrace(e)
                self.logger.debug(bt)
                self.errors[name] = bt
317 318
            else:
                m = ModuleInfo(module.name)
319
                m.version = self.get_tree_mtime(module_path)
320
                m.capabilities = list(set([c.__name__ for c in module.iter_caps()]))
321 322 323 324 325 326 327 328 329
                m.description = module.description
                m.maintainer = module.maintainer
                m.license = module.license
                m.icon = module.icon or ''
                self.modules[module.name] = m

        self.update = int(datetime.now().strftime('%Y%m%d%H%M'))
        self.save(filename)

330
    @staticmethod
331
    def get_tree_mtime(path, include_root=False):
332
        mtime = 0
333 334
        if include_root:
            mtime = int(datetime.fromtimestamp(os.path.getmtime(path)).strftime('%Y%m%d%H%M'))
335 336 337 338 339 340 341 342 343
        for root, dirs, files in os.walk(path):
            for f in files:
                if f.endswith('.pyc'):
                    continue
                m = int(datetime.fromtimestamp(os.path.getmtime(os.path.join(root, f))).strftime('%Y%m%d%H%M'))
                mtime = max(mtime, m)

        return mtime

344 345 346 347
    def save(self, filename, private=False):
        """
        Save repository into a file (modules.list for example).

348 349 350 351
        :param filename: path to file to save repository.
        :type filename: str
        :param private: if enabled, save URL of repository.
        :type private: bool
352 353 354 355 356
        """
        config = RawConfigParser()
        config.set(DEFAULTSECT, 'name', self.name)
        config.set(DEFAULTSECT, 'update', self.update)
        config.set(DEFAULTSECT, 'maintainer', self.maintainer)
357 358
        config.set(DEFAULTSECT, 'signed', int(self.signed))
        config.set(DEFAULTSECT, 'key_update', self.key_update)
359 360 361
        if private:
            config.set(DEFAULTSECT, 'url', self.url)

362
        for module in self.modules.values():
363 364
            config.add_section(module.name)
            for key, value in module.dump():
365 366 367 368 369
                if sys.version_info.major == 2:
                    # python2's configparser enforces bytes coercion with str(value)...
                    config.set(module.name, key, to_unicode(value).encode('utf-8'))
                else:
                    config.set(module.name, key, value)
370

371
        with open_for_config(filename) as f:
372 373
            config.write(f)

374

375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401
class Versions(object):
    VERSIONS_LIST = 'versions.list'

    def __init__(self, path):
        self.path = path
        self.versions = {}

        try:
            with open(os.path.join(self.path, self.VERSIONS_LIST), 'r') as fp:
                config = RawConfigParser()
                config.readfp(fp)

                # Read default parameters
                for key, value in config.items(DEFAULTSECT):
                    self.versions[key] = int(value)
        except IOError:
            pass

    def get(self, name):
        return self.versions.get(name, None)

    def set(self, name, version):
        self.versions[name] = int(version)
        self.save()

    def save(self):
        config = RawConfigParser()
402
        for name, version in self.versions.items():
403
            config.set(DEFAULTSECT, name, version)
404 405

        with open_for_config(os.path.join(self.path, self.VERSIONS_LIST)) as fp:
406 407
            config.write(fp)

408

409
class IProgress(object):
410 411 412 413 414 415
    def progress(self, percent, message):
        raise NotImplementedError()

    def error(self, message):
        raise NotImplementedError()

416 417 418
    def prompt(self, message):
        raise NotImplementedError()

419 420 421
    def __repr__(self):
        return '<%s>' % self.__class__.__name__

422

423
class PrintProgress(IProgress):
424
    def progress(self, percent, message):
425
        print('=== [%3.0f%%] %s' % (percent*100, message), file=sys.stderr)
426

427
    def error(self, message):
428
        print('ERROR: %s' % message, file=sys.stderr)
429

430
    def prompt(self, message):
431
        print('%s (Y/n): *** ASSUMING YES ***' % message, file=sys.stderr)
432 433
        return True

434

435 436
DEFAULT_SOURCES_LIST = \
"""# List of Weboob repositories
437 438 439
#
# The entries below override the entries above (with
# backends of the same name).
440

441
https://updates.weboob.org/%(version)s/main/
442

443
# DEVELOPMENT
444 445
# If you want to hack on Weboob modules, you may add a
# reference to sources, for example:
446 447 448
#file:///home/rom1/src/weboob/modules/
"""

449

450 451 452
class Repositories(object):
    SOURCES_LIST = 'sources.list'
    MODULES_DIR = 'modules'
453 454
    REPOS_DIR = 'repositories'
    KEYRINGS_DIR = 'keyrings'
455 456
    ICONS_DIR = 'icons'

457
    SHARE_DIRS = [MODULES_DIR, REPOS_DIR, KEYRINGS_DIR, ICONS_DIR]
458

459
    def __init__(self, workdir, datadir, version):
460 461
        self.logger = getLogger('repositories')
        self.version = version
462

463
        self.browser = None
464

465
        self.workdir = workdir
466
        self.datadir = datadir
467
        self.sources_list = os.path.join(self.workdir, self.SOURCES_LIST)
468
        self.modules_dir = os.path.join(self.datadir, self.MODULES_DIR, self.version)
469 470
        self.repos_dir = os.path.join(self.datadir, self.REPOS_DIR)
        self.keyrings_dir = os.path.join(self.datadir, self.KEYRINGS_DIR)
471
        self.icons_dir = os.path.join(self.datadir, self.ICONS_DIR)
472

473
        self.create_dir(self.datadir)
474
        self.create_dir(self.modules_dir)
475 476
        self.create_dir(self.repos_dir)
        self.create_dir(self.keyrings_dir)
477 478 479 480 481 482 483
        self.create_dir(self.icons_dir)

        self.versions = Versions(self.modules_dir)

        self.repositories = []

        if not os.path.exists(self.sources_list):
484
            with open_for_config(self.sources_list) as f:
485 486 487 488 489
                f.write(DEFAULT_SOURCES_LIST)
            self.update()
        else:
            self.load()

490
    def load_browser(self):
491 492
        from weboob.browser.browsers import Browser
        from weboob.browser.profiles import Weboob as WeboobProfile
493
        from weboob.tools.compat import getproxies
494

495
        class WeboobBrowser(Browser):
496 497
            PROFILE = WeboobProfile(self.version)
        if self.browser is None:
498 499
            self.browser = WeboobBrowser(
                logger=getLogger('browser', parent=self.logger),
500
                proxy=getproxies())
501

502 503
    def create_dir(self, name):
        if not os.path.exists(name):
504
            os.makedirs(name)
505
        elif not os.path.isdir(name):
506
            self.logger.error(u'"%s" is not a directory' % name)
507

508 509 510
    def _extend_module_info(self, repo, info):
        if repo.local:
            info.path = repo.localurl2path()
511 512
        elif self.versions.get(info.name) is not None:
            info.path = self.modules_dir
513

514 515 516 517 518 519
        return info

    def get_all_modules_info(self, caps=None):
        """
        Get all ModuleInfo instances available.

520 521 522
        :param caps: filter on capabilities:
        :type caps: list[str]
        :rtype: dict[:class:`ModuleInfo`]
523 524 525
        """
        modules = {}
        for repos in reversed(self.repositories):
526
            for name, info in repos.modules.items():
527
                if name not in modules and (not caps or info.has_caps(caps)):
528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546
                    modules[name] = self._extend_module_info(repos, info)
        return modules

    def get_module_info(self, name):
        """
        Get ModuleInfo object of a module.

        It tries all repositories from last to first, and set
        the 'path' attribute of ModuleInfo if it is installed.
        """
        for repos in reversed(self.repositories):
            if name in repos.modules:
                m = repos.modules[name]
                self._extend_module_info(repos, m)
                return m
        return None

    def load(self):
        """
Florent Fourcot's avatar
Florent Fourcot committed
547
        Load repositories from ~/.local/share/weboob/repositories/.
548 549
        """
        self.repositories = []
550
        for name in sorted(os.listdir(self.repos_dir)):
551 552
            path = os.path.join(self.repos_dir, name)
            try:
553
                repository = Repository(path)
554 555
                self.repositories.append(repository)
            except RepositoryUnavailable as e:
556
                print('Unable to load repository %s (%s), try to update repositories.' % (name, e), file=sys.stderr)
557

558 559 560
    def get_module_icon_path(self, module):
        return os.path.join(self.icons_dir, '%s.png' % module.name)

561 562
    def retrieve_icon(self, module):
        """
Florent Fourcot's avatar
Florent Fourcot committed
563
        Retrieve the icon of a module and save it in ~/.local/share/weboob/icons/.
564
        """
565
        self.load_browser()
566 567 568
        if not isinstance(module, ModuleInfo):
            module = self.get_module_info(module)

569
        dest_path = self.get_module_icon_path(module)
570

571 572 573 574 575 576 577 578 579
        icon_url = module.icon
        if not icon_url:
            if module.is_local():
                icon_path = os.path.join(module.path, module.name, 'favicon.png')
                if module.path and os.path.exists(icon_path):
                    shutil.copy(icon_path, dest_path)
                return
            else:
                icon_url = module.url.replace('.tar.gz', '.png')
580 581

        try:
582
            icon = self.browser.open(icon_url)
583
        except BrowserHTTPNotFound:
584
            pass  # no icon, no problem
585 586
        else:
            with open(dest_path, 'wb') as fp:
587
                fp.write(icon.content)
588

589 590 591
    def _parse_source_list(self):
        l = []
        with open(self.sources_list, 'r') as f:
592
            for line in f:
593 594 595 596 597 598
                line = line.strip() % {'version': self.version}
                m = re.match('(file|https?)://.*', line)
                if m:
                    l.append(line)
        return l

599
    def update_repositories(self, progress=PrintProgress()):
600
        self.load_browser()
601 602
        """
        Update list of repositories by downloading them
Florent Fourcot's avatar
Florent Fourcot committed
603
        and put them in ~/.local/share/weboob/repositories/.
604

605 606
        :param progress: observer object.
        :type progress: :class:`IProgress`
607 608 609 610 611
        """
        self.repositories = []
        for name in os.listdir(self.repos_dir):
            os.remove(os.path.join(self.repos_dir, name))

612
        gpg_found = Keyring.find_gpg() or Keyring.find_gpgv()
613 614
        for line in self._parse_source_list():
            progress.progress(0.0, 'Getting %s' % line)
615
            repository = Repository(line)
616 617 618 619 620
            filename = self.url2filename(repository.url)
            prio_filename = '%02d-%s' % (len(self.repositories), filename)
            repo_path = os.path.join(self.repos_dir, prio_filename)
            keyring_path = os.path.join(self.keyrings_dir, filename)
            try:
621
                repository.retrieve_index(self.browser, repo_path)
622
                if gpg_found:
623
                    repository.retrieve_keyring(self.browser, keyring_path, progress)
624
                else:
625 626
                    progress.error('Cannot find gpg or gpgv to check for repository authenticity.\n'
                                   'You should install GPG for better security.')
627
            except RepositoryUnavailable as e:
628 629 630
                progress.error('Unable to load repository: %s' % e)
            else:
                self.repositories.append(repository)
631 632 633 634
                if repository.obsolete:
                    last_update = datetime.strptime(str(repository.update), '%Y%m%d%H%M').strftime('%Y-%m-%d')
                    progress.error('This repository does not receive updates anymore (since %s).\n'
                                   'Your weboob version is probably obsolete and should be upgraded.' % last_update)
635 636 637 638 639 640 641

    def check_repositories(self):
        """
        Check if sources.list is consistent with repositories
        """
        l = []
        for line in self._parse_source_list():
642
            repository = Repository(line)
643 644 645 646 647 648 649
            filename = self.url2filename(repository.url)
            prio_filename = '%02d-%s' % (len(l), filename)
            repo_path = os.path.join(self.repos_dir, prio_filename)
            if not os.path.isfile(repo_path):
                return False
            l.append(repository)
        return True
650

651
    def update(self, progress=PrintProgress()):
652 653 654 655 656 657
        """
        Update repositories and install new packages versions.

        :param progress: observer object.
        :type progress: :class:`IProgress`
        """
658
        self.update_repositories(progress)
659

660
        to_update = []
661
        for name, info in self.get_all_modules_info().items():
662
            if not info.is_local() and info.is_installed():
663 664
                if self.versions.get(name) != info.version:
                    to_update.append(info)
665

666 667 668 669
        if len(to_update) == 0:
            progress.progress(1.0, 'All modules are up-to-date.')
            return

670
        class InstallProgress(PrintProgress):
671 672 673 674 675 676 677 678 679 680
            def __init__(self, n):
                self.n = n

            def progress(self, percent, message):
                progress.progress(float(self.n)/len(to_update) + 1.0/len(to_update)*percent, message)

        for n, info in enumerate(to_update):
            inst_progress = InstallProgress(n)
            try:
                self.install(info, inst_progress)
681
            except ModuleInstallError as e:
682 683
                inst_progress.progress(1.0, unicode(e))

684
    def install(self, module, progress=PrintProgress()):
685 686 687
        """
        Install a module.

688 689 690 691
        :param module: module to install
        :type module: :class:`str` or :class:`ModuleInfo`
        :param progress: observer object
        :type progress: :class:`IProgress`
692
        """
693 694 695
        import tarfile
        self.load_browser()

696 697 698 699 700 701 702 703 704 705
        if isinstance(module, ModuleInfo):
            info = module
        elif isinstance(module, basestring):
            progress.progress(0.0, 'Looking for module %s' % module)
            info = self.get_module_info(module)
            if not info:
                raise ModuleInstallError('Module "%s" does not exist' % module)
        else:
            raise ValueError('"module" parameter might be a ModuleInfo object or a string, not %r' % module)

Romain Bignon's avatar
Romain Bignon committed
706
        module = info
707

Romain Bignon's avatar
Romain Bignon committed
708 709 710 711 712
        if module.is_local():
            raise ModuleInstallError('%s is available on local.' % module.name)

        module_dir = os.path.join(self.modules_dir, module.name)
        installed = self.versions.get(module.name)
713
        if installed is None or not os.path.exists(module_dir):
714
            progress.progress(0.2, 'Module %s is not installed yet' % module.name)
Romain Bignon's avatar
Romain Bignon committed
715
        elif module.version > installed:
716
            progress.progress(0.2, 'A new version of %s is available' % module.name)
717
        else:
718
            raise ModuleInstallError('The latest version of %s is already installed' % module.name)
719

720
        progress.progress(0.3, 'Downloading module...')
721
        try:
722
            tardata = self.browser.open(module.url).content
723
        except BrowserHTTPError as e:
724
            raise ModuleInstallError('Unable to fetch module: %s' % e)
725

726
        # Check signature
727
        if module.signed and (Keyring.find_gpg() or Keyring.find_gpgv()):
728
            progress.progress(0.5, 'Checking module authenticity...')
729
            sig_data = self.browser.open(posixpath.join(module.url + '.sig')).content
Romain Bignon's avatar
Romain Bignon committed
730
            keyring_path = os.path.join(self.keyrings_dir, self.url2filename(module.repo_url))
731 732 733
            keyring = Keyring(keyring_path)
            if not keyring.exists():
                raise ModuleInstallError('No keyring found, please update repos.')
734
            if not keyring.is_valid(tardata, sig_data):
Romain Bignon's avatar
Romain Bignon committed
735
                raise ModuleInstallError('Invalid signature for %s.' % module.name)
736

Romain Bignon's avatar
Romain Bignon committed
737
        # Extract module from tarball.
738 739
        if os.path.isdir(module_dir):
            shutil.rmtree(module_dir)
740
        progress.progress(0.7, 'Setting up module...')
741
        with closing(tarfile.open('', 'r:gz', BytesIO(tardata))) as tar:
742
            tar.extractall(self.modules_dir)
743
        if not os.path.isdir(module_dir):
Romain Bignon's avatar
Romain Bignon committed
744
            raise ModuleInstallError('The archive for %s looks invalid.' % module.name)
745 746
        # Precompile
        compile_dir(module_dir, quiet=True)
747

Romain Bignon's avatar
Romain Bignon committed
748
        self.versions.set(module.name, module.version)
749 750

        progress.progress(0.9, 'Downloading icon...')
Romain Bignon's avatar
Romain Bignon committed
751
        self.retrieve_icon(module)
752

Romain Bignon's avatar
Romain Bignon committed
753
        progress.progress(1.0, 'Module %s has been installed!' % module.name)
754 755 756 757

    @staticmethod
    def url2filename(url):
        """
758 759
        Get a safe file name for an URL.

760 761 762 763
        All non-alphanumeric characters are replaced by _.
        """
        return ''.join([l if l.isalnum() else '_' for l in url])

764 765 766 767 768 769 770 771 772 773 774
    def __iter__(self):
        for repository in self.repositories:
            yield repository

    @property
    def errors(self):
        errors = {}
        for repository in self:
            errors.update(repository.errors)
        return errors

775 776 777 778

class InvalidSignature(Exception):
    def __init__(self, filename):
        self.filename = filename
779
        super(InvalidSignature, self).__init__('Invalid signature for %s' % filename)
780 781 782 783 784 785 786 787 788 789 790 791 792


class Keyring(object):
    EXTENSION = '.gpg'

    def __init__(self, path):
        self.path = path + self.EXTENSION
        self.vpath = path + '.version'
        self.version = 0

        if self.exists():
            with open(self.vpath, 'r') as f:
                self.version = int(f.read().strip())
793 794 795 796 797
        else:
            if os.path.exists(self.path):
                os.remove(self.path)
            if os.path.exists(self.vpath):
                os.remove(self.vpath)
798 799

    def exists(self):
800
        if not os.path.exists(self.vpath):
Florent Fourcot's avatar
Florent Fourcot committed
801
            return False
802 803 804
        if os.path.exists(self.path):
            # Check the file is not empty.
            # This is because there was a bug creating empty keyring files.
805
            with open(self.path, 'rb') as fp:
806 807 808
                if len(fp.read().strip()):
                    return True
        return False
809 810 811 812 813

    def save(self, keyring_data, version):
        with open(self.path, 'wb') as fp:
            fp.write(keyring_data)
        self.version = version
814
        with open_for_config(self.vpath) as fp:
815 816 817 818
            fp.write(str(version))

    @staticmethod
    def find_gpgv():
819 820 821 822 823
        return find_exe('gpgv2') or find_exe('gpgv')

    @staticmethod
    def find_gpg():
        return find_exe('gpg2') or find_exe('gpg')
824 825 826 827 828 829

    def is_valid(self, data, sigdata):
        """
        Check if the data is signed by an accepted key.
        data and sigdata should be strings.
        """
830
        gpg = self.find_gpg()
831
        gpgv = self.find_gpgv()
832 833

        if gpg:
834 835
            from tempfile import mkdtemp
            gpg_homedir = mkdtemp(prefix='weboob_gpg_')
836
            verify_command = [gpg, '--verify', '--no-options',
837 838
                              '--no-default-keyring', '--quiet',
                              '--homedir', gpg_homedir]
839 840 841
        elif gpgv:
            verify_command = [gpgv]

842
        from tempfile import NamedTemporaryFile
843 844 845 846 847 848 849 850
        with NamedTemporaryFile(suffix='.sig', delete=False) as sigfile:
            temp_filename = sigfile.name
            return_code = None
            out = ''
            err = ''
            try:
                sigfile.write(sigdata)
                sigfile.flush()  # very important
851
                sigfile.close()
852
                assert isinstance(data, bytes)
853
                # Yes, all of it is necessary
854
                proc = subprocess.Popen(verify_command + [
855 856 857 858 859 860 861 862 863 864 865
                        '--status-fd', '1',
                        '--keyring', os.path.realpath(self.path),
                        os.path.realpath(sigfile.name),
                        '-'],
                    stdin=subprocess.PIPE,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE)
                out, err = proc.communicate(data)
                return_code = proc.returncode
            finally:
                os.unlink(temp_filename)
866 867
                if gpg:
                    shutil.rmtree(gpg_homedir)
868

869
            if return_code or b'GOODSIG' not in out or b'VALIDSIG' not in out:
870
                print(out, err, file=sys.stderr)
871 872
                return False
        return True
873 874 875

    def __str__(self):
        if self.exists():
876
            with open(self.path, 'rb') as f:
877 878 879
                h = hashlib.sha1(f.read()).hexdigest()
            return 'Keyring version %s, checksum %s' % (self.version, h)
        return 'NO KEYRING'