browser.py 31.1 KB
Newer Older
1 2 3 4
# -*- coding: utf-8 -*-

# Copyright(C) 2012 Romain Bignon
#
5
# This file is part of a weboob module.
6
#
7
# This weboob module is free software: you can redistribute it and/or modify
8
# it under the terms of the GNU Lesser General Public License as published by
9 10 11
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
12
# This weboob module is distributed in the hope that it will be useful,
13 14
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU Lesser General Public License for more details.
16
#
17
# You should have received a copy of the GNU Lesser General Public License
18
# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
19

20
from __future__ import unicode_literals
21 22 23

import re

24
from datetime import datetime
25
from collections import OrderedDict
26
from functools import wraps
27

28
from dateutil.relativedelta import relativedelta
29
from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable
30
from weboob.browser.exceptions import HTTPNotFound, ServerError
31
from weboob.browser import LoginBrowser, URL, need_login
32
from weboob.capabilities.bank import Account, AccountOwnership
33
from weboob.capabilities.base import NotAvailable, find_object
34
from weboob.tools.capabilities.bank.investments import create_french_liquidity
35

36
from .pages import (
37
    LoggedOut,
38 39
    LoginPage, IndexPage, AccountsPage, AccountsFullPage, CardsPage, TransactionsPage,
    UnavailablePage, RedirectPage, HomePage, Login2Page, ErrorPage,
40
    IbanPage, AdvisorPage, TransactionDetailPage, TransactionsBackPage,
41
    NatixisPage, EtnaPage, NatixisInvestPage, NatixisHistoryPage, NatixisErrorPage,
42
    NatixisDetailsPage, NatixisChoicePage, NatixisRedirect,
43
    LineboursePage, AlreadyLoginPage,
44
)
45

46 47
from .document_pages import BasicTokenPage, SubscriberPage, SubscriptionsPage, DocumentsPage

48 49
from .linebourse_browser import LinebourseBrowser

50 51 52 53

__all__ = ['BanquePopulaire']


54 55
class BrokenPageError(Exception):
    pass
56 57


58
def retry(exc_check, tries=4):
59 60 61 62 63 64 65 66 67 68 69 70 71 72 73
    """Decorate a function to retry several times in case of exception.

    The decorated function is called at max 4 times. It is retried only when it
    raises an exception of the type `exc_check`.
    If the function call succeeds and returns an iterator, a wrapper to the
    iterator is returned. If iterating on the result raises an exception of type
    `exc_check`, the iterator is recreated by re-calling the function, but the
    values already yielded will not be re-yielded.
    For consistency, the function MUST always return values in the same order.
    """
    def decorator(func):
        @wraps(func)
        def wrapper(browser, *args, **kwargs):
            cb = lambda: func(browser, *args, **kwargs)

74
            for i in range(tries, 0, -1):
75 76 77 78 79 80
                try:
                    ret = cb()
                except exc_check as exc:
                    browser.logger.debug('%s raised, retrying', exc)
                    continue

81
                if not (hasattr(ret, '__next__') or hasattr(ret, 'next')):
82
                    return ret  # simple value, no need to retry on items
83
                return iter_retry(cb, value=ret, remaining=i, exc_check=exc_check, logger=browser.logger)
84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102

            raise BrowserUnavailable('Site did not reply successfully after multiple tries')

        return wrapper
    return decorator


def no_need_login(func):
    # indicate a login is in progress, so LoggedOut should not be raised
    def wrapper(browser, *args, **kwargs):
        browser.no_login += 1
        try:
            return func(browser, *args, **kwargs)
        finally:
            browser.no_login -= 1

    return wrapper


103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124
class BanquePopulaire(LoginBrowser):
    login_page = URL(r'https://[^/]+/auth/UI/Login.*', LoginPage)
    index_page = URL(r'https://[^/]+/cyber/internet/Login.do', IndexPage)
    accounts_page = URL(r'https://[^/]+/cyber/internet/StartTask.do\?taskInfoOID=mesComptes.*',
                        r'https://[^/]+/cyber/internet/StartTask.do\?taskInfoOID=maSyntheseGratuite.*',
                        r'https://[^/]+/cyber/internet/StartTask.do\?taskInfoOID=accueilSynthese.*',
                        r'https://[^/]+/cyber/internet/StartTask.do\?taskInfoOID=equipementComplet.*',
                        r'https://[^/]+/cyber/internet/ContinueTask.do\?.*dialogActionPerformed=VUE_COMPLETE.*',
                        AccountsPage)

    iban_page = URL(r'https://[^/]+/cyber/internet/StartTask.do\?taskInfoOID=cyberIBAN.*',
                    r'https://[^/]+/cyber/internet/ContinueTask.do\?.*dialogActionPerformed=DETAIL_IBAN_RIB.*',
                    IbanPage)

    accounts_full_page = URL(r'https://[^/]+/cyber/internet/ContinueTask.do\?.*dialogActionPerformed=EQUIPEMENT_COMPLET.*',
                             AccountsFullPage)

    cards_page = URL(r'https://[^/]+/cyber/internet/ContinueTask.do\?.*dialogActionPerformed=ENCOURS_COMPTE.*', CardsPage)

    transactions_page = URL(r'https://[^/]+/cyber/internet/ContinueTask.do\?.*dialogActionPerformed=SELECTION_ENCOURS_CARTE.*',
                            r'https://[^/]+/cyber/internet/ContinueTask.do\?.*dialogActionPerformed=SOLDE.*',
                            r'https://[^/]+/cyber/internet/ContinueTask.do\?.*dialogActionPerformed=CONTRAT.*',
125
                            r'https://[^/]+/cyber/internet/ContinueTask.do\?.*ConsultationDetail.*ActionPerformed=BACK.*',
126
                            r'https://[^/]+/cyber/internet/StartTask.do\?taskInfoOID=ordreBourseCTJ.*',
127 128 129
                            r'https://[^/]+/cyber/internet/Page.do\?.*',
                            r'https://[^/]+/cyber/internet/Sort.do\?.*',
                            TransactionsPage)
130

131 132
    transactions_back_page = URL(r'https://[^/]+/cyber/internet/ContinueTask.do\?.*ActionPerformed=BACK.*', TransactionsBackPage)

133 134
    transaction_detail_page = URL(r'https://[^/]+/cyber/internet/ContinueTask.do\?.*dialogActionPerformed=DETAIL_ECRITURE.*', TransactionDetailPage)

135 136
    error_page = URL(r'https://[^/]+/cyber/internet/ContinueTask.do',
                     r'https://[^/]+/_layouts/error.aspx',
137
                     r'https://[^/]+/portailinternet/_layouts/Ibp.Cyi.Administration/RedirectPageError.aspx',
138 139
                     ErrorPage)

140
    unavailable_page = URL(r'https://[^/]+/s3f-web/.*',
141 142 143
                           r'https://[^/]+/static/errors/nondispo.html',
                           r'/i-RIA/swc/1.0.0/desktop/index.html',
                           UnavailablePage)
144 145 146 147

    redirect_page = URL(r'https://[^/]+/portailinternet/_layouts/Ibp.Cyi.Layouts/RedirectSegment.aspx.*', RedirectPage)
    home_page = URL(r'https://[^/]+/portailinternet/Catalogue/Segments/.*.aspx(\?vary=(?P<vary>.*))?',
                    r'https://[^/]+/portailinternet/Pages/.*.aspx\?vary=(?P<vary>.*)',
148
                    r'https://[^/]+/portailinternet/Pages/[dD]efault.aspx',
149
                    r'https://[^/]+/portailinternet/Transactionnel/Pages/CyberIntegrationPage.aspx',
150
                    r'https://[^/]+/cyber/internet/ShowPortal.do\?token=.*',
151 152
                    HomePage)

153 154
    already_login_page = URL(r'https://[^/]+/dacswebssoissuer.*',
                             r'https://[^/]+/WebSSO_BP/_(?P<bankid>\d+)/index.html\?transactionID=(?P<transactionID>.*)', AlreadyLoginPage)
155 156 157
    login2_page = URL(r'https://[^/]+/WebSSO_BP/_(?P<bankid>\d+)/index.html\?transactionID=(?P<transactionID>.*)', Login2Page)

    # natixis
158
    natixis_redirect = URL(r'https://www.assurances.natixis.fr/espaceinternet-bp/views/common/routage.xhtml.*?windowId=[a-f0-9]+$', NatixisRedirect)
159
    natixis_choice = URL(r'https://www.assurances.natixis.fr/espaceinternet-bp/views/contrat/list.xhtml\?.*', NatixisChoicePage)
160
    natixis_page = URL(r'https://www.assurances.natixis.fr/espaceinternet-bp/views/common.*', NatixisPage)
161 162 163
    etna = URL(r'https://www.assurances.natixis.fr/etna-ihs-bp/#/contratVie/(?P<id1>\w+)/(?P<id2>\w+)/(?P<id3>\w+).*',
               r'https://www.assurances.natixis.fr/espaceinternet-bp/views/contrat/detail/vie/view.xhtml\?windowId=.*&reference=(?P<id3>\d+)&codeSociete=(?P<id1>[^&]*)&codeProduit=(?P<id2>[^&]*).*',
               EtnaPage)
164 165 166
    natixis_error_page = URL(r'https://www.assurances.natixis.fr/espaceinternet-bp/error-redirect.*',
                             r'https://www.assurances.natixis.fr/etna-ihs-bp/#/equipement;codeEtab=.*\?windowId=.*',
                             NatixisErrorPage)
167 168 169
    natixis_invest = URL(r'https://www.assurances.natixis.fr/espaceinternet-bp/rest/v2/contratVie/load/(?P<id1>\w+)/(?P<id2>\w+)/(?P<id3>\w+)', NatixisInvestPage)
    natixis_history = URL(r'https://www.assurances.natixis.fr/espaceinternet-bp/rest/v2/contratVie/load-operation/(?P<id1>\w+)/(?P<id2>\w+)/(?P<id3>\w+)', NatixisHistoryPage)
    natixis_pdf = URL(r'https://www.assurances.natixis.fr/espaceinternet-bp/rest/v2/contratVie/load-releve/(?P<id1>\w+)/(?P<id2>\w+)/(?P<id3>\w+)/(?P<year>\d+)', NatixisDetailsPage)
170

171 172
    linebourse_home = URL(r'https://www.linebourse.fr/ReroutageSJR', LineboursePage)

173 174 175
    advisor = URL(r'https://[^/]+/cyber/internet/StartTask.do\?taskInfoOID=accueil.*',
                  r'https://[^/]+/cyber/internet/StartTask.do\?taskInfoOID=contacter.*', AdvisorPage)

176 177 178 179 180
    basic_token_page = URL(r'/SRVATE/context/mde/1.1.5', BasicTokenPage)
    subscriber_page = URL(r'https://[^/]+/api-bp/wapi/2.0/abonnes/current/mes-documents-electroniques', SubscriberPage)
    subscription_page = URL(r'https://[^/]+/api-bp/wapi/2.0/abonnes/current/contrats', SubscriptionsPage)
    documents_page = URL(r'/api-bp/wapi/2.0/abonnes/current/documents/recherche-avancee', DocumentsPage)

181 182
    def __init__(self, website, *args, **kwargs):
        self.BASEURL = 'https://%s' % website
183 184 185 186 187
        # this url is required because the creditmaritime abstract uses an other url
        if 'cmgo.creditmaritime' in self.BASEURL:
            self.redirect_url = 'https://www.icgauth.creditmaritime.groupe.banquepopulaire.fr/dacsrest/api/v1u0/transaction/'
        else:
            self.redirect_url = 'https://www.icgauth.banquepopulaire.fr/dacsrest/api/v1u0/transaction/'
188
        self.token = None
189
        self.weboob = kwargs['weboob']
190
        super(BanquePopulaire, self).__init__(*args, **kwargs)
191

192 193 194
        dirname = self.responses_dirname
        if dirname:
            dirname += '/bourse'
195
        self.linebourse = LinebourseBrowser('https://www.linebourse.fr', logger=self.logger, responses_dirname=dirname, weboob=self.weboob, proxy=self.PROXIES)
196

197
        self.investments = {}
198
        self.documents_headers = None
199

200 201 202 203
    def deinit(self):
        super(BanquePopulaire, self).deinit()
        self.linebourse.deinit()

204 205
    no_login = 0

206 207 208 209 210 211 212 213 214 215 216 217 218 219
    def follow_back_button_if_any(self, params=None, actions=None):
        """
        Look for a Retour button and follow it using a POST
        :param params: Optional form params to use (default: call self.page.get_params())
        :param actions: Optional actions to use (default: call self.page.get_button_actions())
        :return: None
        """
        if not self.page:
            return

        data = self.page.get_back_button_params(params=params, actions=actions)
        if data:
            self.location('/cyber/internet/ContinueTask.do', data=data)

220
    @no_need_login
221
    def do_login(self):
222
        self.location(self.BASEURL)
223 224 225
        # avoids trying to relog in while it's already on home page
        if self.home_page.is_here():
            return
226

227 228 229 230 231 232 233 234 235 236
        try:
            self.page.login(self.username, self.password)
        except BrowserUnavailable as ex:
            # HACK: some accounts with legacy password fails (legacy means not only digits).
            # The website crashes, even on a web browser.
            # So, if we get a specific exception AND if we have a legacy password,
            # we raise WrongPass instead of BrowserUnavailable.
            if 'Cette page est indisponible' in ex.message and not self.password.isdigit():
                raise BrowserIncorrectPassword()
            raise
237

238
        if self.login_page.is_here():
239
            raise BrowserIncorrectPassword()
240 241 242 243
        if 'internetRescuePortal' in self.url:
            # 1 more request is necessary
            data = {'integrationMode':	'INTERNET_RESCUE'}
            self.location('/cyber/internet/Login.do', data=data)
244

245
    ACCOUNT_URLS = ['mesComptes', 'mesComptesPRO', 'maSyntheseGratuite', 'accueilSynthese', 'equipementComplet']
246

247
    @retry(BrokenPageError)
248
    @need_login
Romain Bignon's avatar
Romain Bignon committed
249 250
    def go_on_accounts_list(self):
        for taskInfoOID in self.ACCOUNT_URLS:
251
            data = OrderedDict([('taskInfoOID', taskInfoOID), ('token', self.token)])
252
            self.location(self.absurl('/cyber/internet/StartTask.do', base=True), params=data)
Romain Bignon's avatar
Romain Bignon committed
253
            if not self.page.is_error():
254
                if self.page.pop_up():
255
                    self.logger.debug('Popup displayed, retry')
256
                    data = OrderedDict([('taskInfoOID', taskInfoOID), ('token', self.token)])
257
                    self.location('/cyber/internet/StartTask.do', params=data)
Romain Bignon's avatar
Romain Bignon committed
258 259 260
                self.ACCOUNT_URLS = [taskInfoOID]
                break
        else:
261
            raise BrokenPageError('Unable to go on the accounts list page')
Romain Bignon's avatar
Romain Bignon committed
262

263
        if self.page.is_short_list():
264 265 266 267
            form = self.page.get_form(nr=0)
            form['dialogActionPerformed'] = 'EQUIPEMENT_COMPLET'
            form['token'] = self.page.build_token(form['token'])
            form.submit()
Romain Bignon's avatar
Romain Bignon committed
268

269
        # In case of prevAction maybe we have reached an expanded accounts list page, need to go back
270
        self.follow_back_button_if_any()
271

272
    @retry(LoggedOut)
273
    @need_login
274 275 276
    def get_accounts_list(self, get_iban=True):
        # We have to parse account list in 2 different way depending if we want the iban number or not
        # thanks to stateful website
Romain Bignon's avatar
Romain Bignon committed
277
        next_pages = []
278
        accounts = []
279 280 281
        owner_name = re.search(r' (.+)', self.get_profile().name).group(1).upper()

        self.go_on_accounts_list()
Romain Bignon's avatar
Romain Bignon committed
282 283

        for a in self.page.iter_accounts(next_pages):
284
            self.set_account_ownership(a, owner_name)
285 286
            accounts.append(a)
            if not get_iban:
287
                yield a
Romain Bignon's avatar
Romain Bignon committed
288

289 290 291
        while len(next_pages) > 0:
            next_page = next_pages.pop()

292
            if not self.accounts_full_page.is_here():
Romain Bignon's avatar
Romain Bignon committed
293
                self.go_on_accounts_list()
294 295 296 297
            # If there is an action needed to go to the "next page", do it.
            if 'prevAction' in next_page:
                params = self.page.get_params()
                params['dialogActionPerformed'] = next_page.pop('prevAction')
298
                params['token'] = self.page.build_token(self.token)
299
                self.location('/cyber/internet/ContinueTask.do', data=params)
Romain Bignon's avatar
Romain Bignon committed
300

301
            next_page['token'] = self.page.build_token(self.token)
302
            self.location('/cyber/internet/ContinueTask.do', data=next_page)
Romain Bignon's avatar
Romain Bignon committed
303

304
            for a in self.page.iter_accounts(next_pages, accounts_parsed=accounts):
305
                self.set_account_ownership(a, owner_name)
306 307
                accounts.append(a)
                if not get_iban:
308 309 310 311
                    yield a

        if get_iban:
            for a in accounts:
312
                a.iban = self.get_iban_number(a)
313 314
            for a in accounts:
                self.get_investment(a)
Romain Bignon's avatar
Romain Bignon committed
315
                yield a
316

317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335
    # TODO: see if there's other type of account with a label without name which
    # is not ATTORNEY (cf. 'COMMUN'). Didn't find one right now.
    def set_account_ownership(self, account, owner_name):
        if not account.ownership:
            label = account.label.upper()
            if account.parent:
                if not account.parent.ownership:
                    self.set_account_ownership(account.parent, owner_name)
                account.ownership = account.parent.ownership
            elif owner_name in label:
                if re.search(r'(m|mr|me|mme|mlle|mle|ml)\.? (.*)\bou (m|mr|me|mme|mlle|mle|ml)\b(.*)', label, re.IGNORECASE):
                    account.ownership = AccountOwnership.CO_OWNER
                else:
                    account.ownership = AccountOwnership.OWNER
            elif 'COMMUN' in label:
                account.ownership = AccountOwnership.CO_OWNER
            else:
                account.ownership = AccountOwnership.ATTORNEY

336
    @need_login
337
    def get_iban_number(self, account):
338 339
        url = self.absurl('/cyber/internet/StartTask.do?taskInfoOID=cyberIBAN&token=%s' % self.page.build_token(self.token), base=True)
        self.location(url)
340
        # Sometimes we can't choose an account
341
        if account.type in [Account.TYPE_LIFE_INSURANCE, Account.TYPE_MARKET] or (self.page.need_to_go() and not self.page.go_iban(account)):
342
            return NotAvailable
343
        return self.page.get_iban(account.id)
344

345
    @retry(LoggedOut)
346
    @need_login
347
    def get_account(self, id):
348
        return find_object(self.get_accounts_list(False), id=id)
349

350 351 352
    def set_gocardless_transaction_details(self, transaction):
        # Setting references for a GoCardless transaction
        data = self.page.get_params()
353
        data['validationStrategy'] = self.page.get_gocardless_strategy_param(transaction)
354 355 356 357 358 359 360 361 362 363
        data['dialogActionPerformed'] = 'DETAIL_ECRITURE'
        attribute_key, attribute_value = self.page.get_transaction_table_id(transaction._ref)
        data[attribute_key] = attribute_value
        data['token'] = self.page.build_token(data['token'])

        self.location(self.absurl('/cyber/internet/ContinueTask.do', base=True), data=data)
        ref = self.page.get_reference()
        transaction.raw = '%s %s' % (transaction.raw, ref)

        # Needed to preserve navigation.
364
        self.follow_back_button_if_any()
365

366
    @retry(LoggedOut)
367
    @need_login
Romain Bignon's avatar
Romain Bignon committed
368
    def get_history(self, account, coming=False):
369 370
        def get_history_by_receipt(account, coming, sel_tbl1=None):
            account = self.get_account(account.id)
371

372 373
            if account is None:
                raise BrowserUnavailable()
374

375 376 377 378 379
            if account._invest_params or (account.id.startswith('TIT') and account._params):
                if not coming:
                    for tr in self.get_invest_history(account):
                        yield tr
                return
380

381 382 383 384
            if coming:
                params = account._coming_params
            else:
                params = account._params
Romain Bignon's avatar
Romain Bignon committed
385

386 387 388
            if params is None:
                return
            params['token'] = self.page.build_token(params['token'])
389

390 391
            if sel_tbl1 != None:
                params['attribute($SEL_$tbl1)'] = str(sel_tbl1)
392

393
            self.location(self.absurl('/cyber/internet/ContinueTask.do', base=True), data=params)
394

395 396
            if not self.page or self.error_page.is_here() or self.page.no_operations():
                return
397

398 399 400 401 402 403
            # Sort by values dates (see comment in TransactionsPage.get_history)
            if len(self.page.doc.xpath('//a[@id="tcl4_srt"]')) > 0:
                form = self.page.get_form(id='myForm')
                form.url = self.absurl('/cyber/internet/Sort.do?property=tbl1&sortBlocId=blc2&columnName=dateValeur')
                params['token'] = self.page.build_token(params['token'])
                form.submit()
404

405
            transactions_next_page = True
406

407 408
            while transactions_next_page:
                assert self.transactions_page.is_here()
409

410 411 412 413 414 415
                transaction_list = self.page.get_history(account, coming)
                for tr in transaction_list:
                    # Add information about GoCardless
                    if 'GoCardless' in tr.label and tr._has_link:
                        self.set_gocardless_transaction_details(tr)
                    yield tr
416

417 418 419 420 421 422 423 424
                next_params = self.page.get_next_params()
                # Go to the next transaction page only if it exists:
                if next_params is None:
                    transactions_next_page = False
                else:
                    self.location('/cyber/internet/Page.do', params=next_params)

        if coming and account._coming_count:
425 426
            for i in range(account._coming_start,
                           account._coming_start + account._coming_count):
427 428 429 430 431
                for tr in get_history_by_receipt(account, coming, sel_tbl1=i):
                    yield tr
        else:
            for tr in get_history_by_receipt(account, coming):
                yield tr
432

433
    @need_login
434
    def go_investments(self, account, get_account=False):
435 436

        if not account._invest_params and not (account.id.startswith('TIT') or account.id.startswith('PRV')):
437 438
            raise NotImplementedError()

439 440
        if get_account:
            account = self.get_account(account.id)
441 442 443 444 445 446 447 448 449 450

        if account._params:
            params = {'taskInfoOID':            "ordreBourseCTJ",
                      'controlPanelTaskAction': "true",
                      'token':                  self.page.build_token(account._params['token'])
                     }
            self.location(self.absurl('/cyber/internet/StartTask.do', base=True), params=params)
        else:
            params = account._invest_params
            params['token'] = self.page.build_token(params['token'])
451 452 453 454
            try:
                self.location(self.absurl('/cyber/internet/ContinueTask.do', base=True), data=params)
            except BrowserUnavailable:
                return False
455

456
        if self.error_page.is_here():
457 458
            raise NotImplementedError()

459
        url, params = self.page.get_investment_page_params()
460
        if params:
461 462 463 464
            try:
                self.location(url, data=params)
            except BrowserUnavailable:
                return False
465 466 467 468

            if "linebourse" in self.url:
                self.linebourse.session.cookies.update(self.session.cookies)
                self.linebourse.invest.go()
469

470
            if self.natixis_error_page.is_here():
471
                self.logger.warning("natixis site doesn't work")
472
                return False
473 474 475 476 477 478

            if self.natixis_redirect.is_here():
                url = self.page.get_redirect()
                if re.match(r'https://www.assurances.natixis.fr/etna-ihs-bp/#/equipement;codeEtab=\d+\?windowId=[a-f0-9]+$', url):
                    self.logger.warning('there may be no contract associated with %s, skipping', url)
                    return False
479
        return True
480

481 482
    @need_login
    def get_investment(self, account):
483 484 485 486
        if account.type in (Account.TYPE_LOAN,):
            self.investments[account.id] = []
            return []

487 488
        # Add "Liquidities" investment if the account is a "Compte titres PEA":
        if account.type == Account.TYPE_PEA and account.id.startswith('CPT'):
489
            self.investments[account.id] = [create_french_liquidity(account.balance)]
490 491
            return self.investments[account.id]

492 493
        if account.id in self.investments.keys() and self.investments[account.id] is False:
            raise NotImplementedError()
494

495 496 497 498
        if account.id not in self.investments.keys():
            self.investments[account.id] = []
            try:
                if self.go_investments(account, get_account=True):
499
                    # Redirection URL is https://www.linebourse.fr/ReroutageSJR
500
                    if "linebourse" in self.url:
501 502 503 504
                        # Eliminating the 3 letters prefix to match IDs on Linebourse:
                        linebourse_id = account.id[3:]
                        for inv in self.linebourse.iter_investment(linebourse_id):
                            self.investments[account.id].append(inv)
505

506
                    if self.etna.is_here():
507 508 509 510 511 512
                        params = self.page.params
                    elif self.natixis_redirect.is_here():
                        # the url may contain a "#", so we cannot make a request to it, the params after "#" would be dropped
                        url = self.page.get_redirect()
                        self.logger.debug('using redirect url %s', url)
                        m = self.etna.match(url)
513 514 515 516 517
                        if not m:
                            # url can be contratPrev which is not investments
                            self.logger.debug('Unable to handle this kind of contract')
                            raise NotImplementedError()

518 519 520
                        params = m.groupdict()

                    if self.natixis_redirect.is_here() or self.etna.is_here():
521
                        try:
522
                            self.natixis_invest.go(**params)
523
                        except ServerError:
524
                            # Broken website .. nothing to do.
525 526
                            self.investments[account.id] = iter([])
                            return self.investments[account.id]
527 528
                        self.investments[account.id] = list(self.page.get_investments())
            except NotImplementedError:
529
                self.investments[account.id] = []
530
        return self.investments[account.id]
531 532 533 534 535

    @need_login
    def get_invest_history(self, account):
        if not self.go_investments(account):
            return
536 537 538 539 540
        if "linebourse" in self.url:
            for tr in self.linebourse.iter_history(re.sub('[^0-9]', '', account.id)):
                yield tr
            return

541 542
        if self.etna.is_here():
            params = self.page.params
543 544 545 546
        elif self.natixis_redirect.is_here():
            url = self.page.get_redirect()
            self.logger.debug('using redirect url %s', url)
            m = self.etna.match(url)
547 548 549 550 551
            if not m:
                # url can be contratPrev which is not investments
                self.logger.debug('Unable to handle this kind of contract')
                return

552
            params = m.groupdict()
553 554
        else:
            return
555

556 557 558
        self.natixis_history.go(**params)
        items_from_json = list(self.page.get_history())
        items_from_json.sort(reverse=True, key=lambda item: item.date)
559

560 561
        years = list(set(item.date.year for item in items_from_json))
        years.sort(reverse=True)
562

563 564 565 566 567 568 569
        for year in years:
            try:
                self.natixis_pdf.go(year=year, **params)
            except HTTPNotFound:
                self.logger.debug('no pdf for year %s, fallback on json transactions', year)
                for tr in items_from_json:
                    if tr.date.year == year:
570
                        yield tr
571 572 573 574 575 576 577
            except ServerError:
                return
            else:
                history = list(self.page.get_history())
                history.sort(reverse=True, key=lambda item: item.date)
                for tr in history:
                    yield tr
578

579 580 581 582 583
    @need_login
    def get_profile(self):
        self.location('/cyber/internet/StartTask.do?taskInfoOID=accueil&token=%s' % self.token)
        return self.page.get_profile()

584 585 586 587 588
    @retry(LoggedOut)
    @need_login
    def get_advisor(self):
        for taskInfoOID in ['accueil', 'contacter']:
            data = OrderedDict([('taskInfoOID', taskInfoOID), ('token', self.token)])
589
            self.location(self.absurl('/cyber/internet/StartTask.do', base=True), params=data)
590 591 592 593 594 595 596 597
            if taskInfoOID == "accueil":
                advisor = self.page.get_advisor()
                if not advisor:
                    break
            else:
                self.page.update_agency(advisor)
        return iter([advisor])

598 599 600 601 602 603 604 605 606
    @need_login
    def iter_subscriptions(self):
        self.location('/SRVATE/context/mde/1.1.5')
        headers = {'Authorization': 'Basic %s' % self.page.get_basic_token()}
        response = self.location('/as-bp/as/2.0/tokens', method='POST', headers=headers)
        self.documents_headers = {'Authorization': 'Bearer %s' % response.json()['access_token']}

        self.location('/api-bp/wapi/2.0/abonnes/current/mes-documents-electroniques', headers=self.documents_headers)

607 608 609 610 611 612
        if self.page.get_status_dematerialized() == 'CGDN':
            # A status different than 1 means either the demateralization isn't enabled
            # or not available for this connection
            return []

        subscriber = self.page.get_subscriber()
613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641
        params = {'type': 'dematerialisationEffective'}
        self.location('/api-bp/wapi/2.0/abonnes/current/contrats', params=params, headers=self.documents_headers)
        return self.page.get_subscriptions(subscriber=subscriber)

    @need_login
    def iter_documents(self, subscription):
        now = datetime.now()
        # website says we can't get documents more than one year range at once but it seems it's just a javascript check
        # no problem here so far
        first_date = now - relativedelta(years=5)
        start_date = first_date.strftime('%Y-%m-%dT00:00:00.000+00:00')
        end_date = now.strftime('%Y-%m-%dT%H:%M:%S.000+00:00')
        body = {
            'inTypeRecherche': {'type': 'typeRechercheDocument', 'code': 'DEMAT'},
            'inDateDebut': start_date,
            'inDateFin': end_date,
            'inListeIdentifiantsContrats': [
                {'identifiantContrat': {'identifiant': subscription.id, 'codeBanque': subscription._bank_code}}
            ],
            'inListeTypesDocuments': [
                {'typeDocument': {'code': 'EXTRAIT', 'label': 'Extrait de compte', 'type': 'referenceLogiqueDocument'}}
            ]
        }
        self.location('/api-bp/wapi/2.0/abonnes/current/documents/recherche-avancee', json=body, headers=self.documents_headers)
        return self.page.iter_documents(subid=subscription.id)

    def download_document(self, document):
        return self.open(document.url, headers=self.documents_headers).content

642 643 644 645 646

class iter_retry(object):
    # when the callback is retried, it will create a new iterator, but we may already yielded
    # some values, so we need to keep track of them and seek in the middle of the iterator

647
    def __init__(self, cb, remaining=4, value=None, exc_check=Exception, logger=None):
648 649 650
        self.cb = cb
        self.it = value
        self.items = []
651
        self.remaining = remaining
652 653 654 655 656 657 658
        self.exc_check = exc_check
        self.logger = logger

    def __iter__(self):
        return self

    def __next__(self):
659
        if self.remaining <= 0:
660 661 662 663 664 665 666
            raise BrowserUnavailable('Site did not reply successfully after multiple tries')

        if self.it is None:
            self.it = self.cb()

            # recreated iterator, consume previous items
            try:
667 668
                nb = -1
                for nb, sent in enumerate(self.items):
669
                    new = next(self.it)
670 671 672 673 674
                    if hasattr(new, 'to_dict'):
                        equal = sent.to_dict() == new.to_dict()
                    else:
                        equal = sent == new
                    if not equal:
675
                        # safety is not guaranteed
676
                        raise BrowserUnavailable('Site replied inconsistently between retries, %r vs %r', sent, new)
677
            except StopIteration:
678
                raise BrowserUnavailable('Site replied fewer elements (%d) than last iteration (%d)', nb + 1, len(self.items))
679 680 681 682
            except self.exc_check as exc:
                if self.logger:
                    self.logger.info('%s raised, retrying', exc)
                self.it = None
683
                self.remaining -= 1
684 685 686 687 688 689 690 691 692
                return next(self)

        # return one item
        try:
            obj = next(self.it)
        except self.exc_check as exc:
            if self.logger:
                self.logger.info('%s raised, retrying', exc)
            self.it = None
693
            self.remaining -= 1
694 695 696 697 698 699
            return next(self)
        else:
            self.items.append(obj)
            return obj

    next = __next__