browser.py 34 KB
Newer Older
1 2
# -*- coding: utf-8 -*-

Baptiste Delpey's avatar
Baptiste Delpey committed
3
# Copyright(C) 2016       Baptiste Delpey
4
#
5
# This file is part of a weboob module.
6
#
7
# This weboob module is free software: you can redistribute it and/or modify
8
# it under the terms of the GNU Lesser General Public License as published by
9 10 11
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
12
# This weboob module is distributed in the hope that it will be useful,
13 14
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU Lesser General Public License for more details.
16
#
17
# You should have received a copy of the GNU Lesser General Public License
18
# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
19

20 21
# flake8: compatible

22
from __future__ import unicode_literals
23

24
import requests
25

26
from datetime import date, datetime
Baptiste Delpey's avatar
Baptiste Delpey committed
27
from dateutil.relativedelta import relativedelta
28

29
from weboob.browser.retry import login_method, retry_on_logout, RetryLoginBrowser
30
from weboob.browser.browsers import need_login, TwoFactorBrowser
Baptiste Delpey's avatar
Baptiste Delpey committed
31
from weboob.browser.url import URL
32
from weboob.exceptions import BrowserIncorrectPassword, BrowserHTTPNotFound, NoAccountsException, BrowserUnavailable
33
from weboob.browser.exceptions import LoggedOut, ClientError
34 35 36
from weboob.capabilities.bank import (
    Account, AccountNotFound, TransferError, TransferInvalidAmount,
    TransferInvalidEmitter, TransferInvalidLabel, TransferInvalidRecipient,
37
    AddRecipientStep, Rate, TransferBankError, AccountOwnership, RecipientNotFound,
38
    AddRecipientTimeout, TransferDateType, Emitter,
39
)
40
from weboob.capabilities.base import empty, find_object
41
from weboob.capabilities.contact import Advisor
42
from weboob.tools.value import Value
43
from weboob.tools.compat import basestring, urlsplit
44
from weboob.tools.capabilities.bank.transactions import sorted_transactions
45
from weboob.tools.capabilities.bank.bank_transfer import sorted_transfers
46

47
from .pages import (
48
    VirtKeyboardPage, AccountsPage, AsvPage, HistoryPage, AuthenticationPage,
49
    MarketPage, LoanPage, SavingMarketPage, ErrorPage, IncidentPage, IbanPage, ProfilePage, ExpertPage,
50
    CardsNumberPage, CalendarPage, HomePage, PEPPage,
51
    TransferAccounts, TransferRecipients, TransferCharac, TransferConfirm, TransferSent,
52
    AddRecipientPage, StatusPage, CardHistoryPage, CardCalendarPage, CurrencyListPage, CurrencyConvertPage,
53 54
    AccountsErrorPage, NoAccountPage, TransferMainPage, PasswordPage, NewTransferRecipients,
    NewTransferAccounts,
55
)
56
from .transfer_pages import TransferListPage, TransferInfoPage
57

Romain Bignon's avatar
Romain Bignon committed
58

Baptiste Delpey's avatar
Baptiste Delpey committed
59
__all__ = ['BoursoramaBrowser']
60 61


62 63 64 65
class BrowserIncorrectAuthenticationCode(BrowserIncorrectPassword):
    pass


66
class BoursoramaBrowser(RetryLoginBrowser, TwoFactorBrowser):
Baptiste Delpey's avatar
Baptiste Delpey committed
67
    BASEURL = 'https://clients.boursorama.com'
68
    TIMEOUT = 60.0
69 70
    HAS_CREDENTIALS_ONLY = True
    TWOFA_DURATION = 60 * 24 * 90
Baptiste Delpey's avatar
Baptiste Delpey committed
71

72
    home = URL('/$', HomePage)
73
    keyboard = URL(r'/connexion/clavier-virtuel\?_hinclude=1', VirtKeyboardPage)
74
    status = URL(r'/aide/messages/dashboard\?showza=0&_hinclude=1', StatusPage)
75
    calendar = URL('/compte/cav/.*/calendrier', CalendarPage)
76
    card_calendar = URL('https://api.boursorama.com/services/api/files/download.phtml.*', CardCalendarPage)
77 78 79 80 81
    error = URL(
        '/connexion/compte-verrouille',
        '/infos-profil',
        ErrorPage
    )
82
    login = URL(r'/connexion/saisie-mot-de-passe/', PasswordPage)
83

84 85
    accounts = URL(r'/dashboard/comptes\?_hinclude=300000', AccountsPage)
    accounts_error = URL(r'/dashboard/comptes\?_hinclude=300000', AccountsErrorPage)
86
    pro_accounts = URL(r'/dashboard/comptes-professionnels\?_hinclude=1', AccountsPage)
87 88 89 90 91
    no_account = URL(
        r'/dashboard/comptes\?_hinclude=300000',
        r'/dashboard/comptes-professionnels\?_hinclude=1',
        NoAccountPage
    )
92

93
    history = URL(r'/compte/(cav|epargne)/(?P<webid>.*)/mouvements.*', HistoryPage)
94
    card_transactions = URL('/compte/cav/(?P<webid>.*)/carte/.*', HistoryPage)
95
    deffered_card_history = URL('https://api.boursorama.com/services/api/files/download.phtml.*', CardHistoryPage)
Baptiste Delpey's avatar
Baptiste Delpey committed
96 97
    budget_transactions = URL('/budget/compte/(?P<webid>.*)/mouvements.*', HistoryPage)
    other_transactions = URL('/compte/cav/(?P<webid>.*)/mouvements.*', HistoryPage)
98
    saving_transactions = URL('/compte/epargne/csl/(?P<webid>.*)/mouvements.*', HistoryPage)
99
    saving_pep = URL('/compte/epargne/pep', PEPPage)
Vincent Paredes's avatar
Vincent Paredes committed
100
    incident = URL('/compte/cav/(?P<webid>.*)/mes-incidents.*', IncidentPage)
101

102
    # transfer
103 104 105 106 107 108 109
    transfer_list = URL(
        r'/compte/(?P<acc_type>[^/]+)/(?P<webid>\w+)/virements/suivi/(?P<type>\w+)$',
        # next url is for pagination, token is very long
        # make sure you don't match "details" or it could break "transfer_info" URL
        r'/compte/(?P<acc_type>[^/]+)/(?P<webid>\w+)/virements/suivi/(?P<type>\w+)/[a-zA-Z0-9]{30,}$',
        TransferListPage
    )
110 111 112 113
    transfer_info = URL(
        r'/compte/(?P<acc_type>[^/]+)/(?P<webid>\w+)/virements/suivi/(?P<type>\w+)/details/[\w-]{40,}',
        TransferInfoPage
    )
114
    transfer_main_page = URL(r'/compte/(?P<acc_type>[^/]+)/(?P<webid>\w+)/virements$', TransferMainPage)
115 116 117 118 119 120 121 122 123 124
    transfer_accounts = URL(
        r'/compte/(?P<acc_type>[^/]+)/(?P<webid>\w+)/virements/nouveau$',
        r'/compte/(?P<type>[^/]+)/(?P<webid>\w+)/virements/nouveau/(?P<id>\w+)/1',
        TransferAccounts
    )
    recipients_page = URL(
        r'/compte/(?P<type>[^/]+)/(?P<webid>\w+)/virements$',
        r'/compte/(?P<type>[^/]+)/(?P<webid>\w+)/virements/nouveau/(?P<id>\w+)/2',
        TransferRecipients
    )
125 126 127 128 129 130 131 132 133
    new_transfer_accounts = URL(
        r'/compte/(?P<acc_type>[^/]+)/(?P<webid>\w+)/virements/immediat/nouveau/?$',
        r'/compte/(?P<type>[^/]+)/(?P<webid>\w+)/virements/immediat/nouveau/(?P<id>\w+)/1',
        NewTransferAccounts
    )
    new_recipients_page = URL(
        r'/compte/(?P<type>[^/]+)/(?P<webid>\w+)/virements/immediat/nouveau/(?P<id>\w+)/2',
        NewTransferRecipients
    )
134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149
    transfer_charac = URL(
        r'/compte/(?P<type>[^/]+)/(?P<webid>\w+)/virements/nouveau/(?P<id>\w+)/3',
        TransferCharac
    )
    transfer_confirm = URL(
        r'/compte/(?P<type>[^/]+)/(?P<webid>\w+)/virements/nouveau/(?P<id>\w+)/4',
        TransferConfirm
    )
    transfer_sent = URL(
        r'/compte/(?P<type>[^/]+)/(?P<webid>\w+)/virements/nouveau/(?P<id>\w+)/5',
        TransferSent
    )
    rcpt_page = URL(
        r'/compte/(?P<type>[^/]+)/(?P<webid>\w+)/virements/comptes-externes/nouveau/(?P<id>\w+)/\d',
        AddRecipientPage
    )
150

Baptiste Delpey's avatar
Baptiste Delpey committed
151
    asv = URL('/compte/assurance-vie/.*', AsvPage)
152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172
    saving_history = URL(
        '/compte/cefp/.*/(positions|mouvements)',
        '/compte/.*ord/.*/mouvements',
        '/compte/pea/.*/mouvements',
        '/compte/0%25pea/.*/mouvements',
        '/compte/pea-pme/.*/mouvements',
        SavingMarketPage
    )
    market = URL(
        r'/compte/(?!assurance|cav|epargne).*/(positions|mouvements|ordres)',
        r'/compte/ord/.*/positions',
        MarketPage
    )
    loans = URL(
        r'/credit/paiement-3x/.*/informations',
        r'/credit/immobilier/.*/informations',
        r'/credit/immobilier/.*/caracteristiques',
        r'/credit/consommation/.*/informations',
        r'/credit/lombard/.*/caracteristiques',
        LoanPage
    )
Baptiste Delpey's avatar
Baptiste Delpey committed
173
    authentication = URL('/securisation', AuthenticationPage)
Baptiste Delpey's avatar
Baptiste Delpey committed
174
    iban = URL('/compte/(?P<webid>.*)/rib', IbanPage)
175
    profile = URL('/mon-profil/', ProfilePage)
176
    profile_children = URL('/mon-profil/coordonnees/enfants', ProfilePage)
Baptiste Delpey's avatar
Baptiste Delpey committed
177

178
    expert = URL('/compte/derive/', ExpertPage)
179

180 181
    cards = URL('/compte/cav/cb', CardsNumberPage)

182
    currencylist = URL('https://www.boursorama.com/bourse/devises/parite/_detail-parite', CurrencyListPage)
183 184 185 186
    currencyconvert = URL(
        'https://www.boursorama.com/bourse/devises/convertisseur-devises/convertir',
        CurrencyConvertPage
    )
187

188
    __states__ = ('auth_token', 'recipient_form',)
189 190 191 192

    def __init__(self, config=None, *args, **kwargs):
        self.config = config
        self.auth_token = None
Vincent Paredes's avatar
Vincent Paredes committed
193
        self.accounts_list = None
194
        self.cards_list = None
195
        self.deferred_card_calendar = None
196
        self.recipient_form = None
197 198
        kwargs['username'] = self.config['login'].get()
        kwargs['password'] = self.config['password'].get()
199 200 201 202 203 204

        self.AUTHENTICATION_METHODS = {
            'pin_code': self.handle_sms,
        }

        super(BoursoramaBrowser, self).__init__(config, *args, **kwargs)
205

206 207 208 209 210 211
    def locate_browser(self, state):
        try:
            self.location(state['url'])
        except (requests.exceptions.HTTPError, requests.exceptions.TooManyRedirects, LoggedOut):
            pass

212
    def load_state(self, state):
213 214 215 216 217
        # needed to continue the session while adding recipient with otp
        # it keeps the form to continue to submit the otp
        if state.get('recipient_form'):
            state.pop('url', None)

218
        super(BoursoramaBrowser, self).load_state(state)
219

220
    def handle_authentication(self):
Baptiste Delpey's avatar
Baptiste Delpey committed
221
        if self.authentication.is_here():
222
            self.check_interactive()
223 224 225 226 227 228 229

            confirmation_link = self.page.get_confirmation_link()
            if confirmation_link:
                self.location(confirmation_link)

            self.page.sms_first_step()
            self.page.sms_second_step()
230

231 232 233 234 235 236 237
    def handle_sms(self):
        # regular 2FA way
        if self.auth_token:
            self.page.authenticate()
        # PSD2 way
        else:
            # we can't access form without sending a SMS again
238 239 240 241 242 243 244
            self.location(
                '/securisation/authentification/validation',
                data={
                    'strong_authentication_confirm[code]': self.config['pin_code'].get(),
                    'strong_authentication_confirm[type]': 'brs-otp-sms',
                }
            )
245 246 247 248 249 250

        if self.authentication.is_here():
            raise BrowserIncorrectAuthenticationCode()

    def init_login(self):
        self.login.go()
251
        self.page.enter_password(self.username, self.password)
252

253
        if self.error.is_here():
254
            raise BrowserIncorrectPassword()
255 256 257 258
        elif self.login.is_here():
            error = self.page.get_error()
            assert error, 'Should not be on login page without error message'

259 260 261 262 263 264
            wrongpass_messages = (
                'Identifiant ou mot de passe invalide',
                "Erreur d'authentification",
                "Cette valeur n'est pas valide",
                "votre identifiant ou votre mot de passe n'est pas valide",
            )
265 266 267 268 269 270

            if 'vous pouvez actuellement rencontrer des difficultés pour accéder à votre Espace Client' in error:
                raise BrowserUnavailable()
            elif any(msg in error for msg in wrongpass_messages):
                raise BrowserIncorrectPassword(error)

271
            raise AssertionError('Unhandled error message : "%s"' % error)
272

273 274
        # After login, we might be redirected to the two factor authentication page.
        self.handle_authentication()
275

276 277 278
    @login_method
    def do_login(self):
        return super(BoursoramaBrowser, self).do_login()
279

280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312
    def ownership_guesser(self):
        ownerless_accounts = [account for account in self.accounts_list if empty(account.ownership)]

        # On Boursorama website, all mandatory accounts have the real owner name in their label, and
        # children names are findable in the PSU profile.
        self.profile_children.go()
        children_names = self.page.get_children_firstnames()

        for ownerless_account in ownerless_accounts:
            for child_name in children_names:
                if child_name in ownerless_account.label:
                    ownerless_account.ownership = AccountOwnership.ATTORNEY
                    break

        # If there are two deferred card for with the same parent account, we assume that's the parent checking
        # account is a 'CO_OWNER' account
        parent_accounts = []
        for account in self.accounts_list:
            if account.type == Account.TYPE_CARD and empty(account.parent.ownership):
                if account.parent in parent_accounts:
                    account.parent.ownership = AccountOwnership.CO_OWNER
                parent_accounts.append(account.parent)

        # We set all accounts without ownership as if they belong to the credential owner
        for account in self.accounts_list:
            if empty(account.ownership) and account.type != Account.TYPE_CARD:
                account.ownership = AccountOwnership.OWNER

        # Account cards should be set with the same ownership of their parents accounts
        for account in self.accounts_list:
            if account.type == Account.TYPE_CARD:
                account.ownership = account.parent.ownership

313 314
    def go_cards_number(self, link):
        self.location(link)
315
        self.location(self.page.get_cards_number_link())
Baptiste Delpey's avatar
Baptiste Delpey committed
316

317
    @retry_on_logout()
Baptiste Delpey's avatar
Baptiste Delpey committed
318
    @need_login
319
    def get_accounts_list(self):
320
        self.status.go()
321 322

        exc = None
323
        for _ in range(3):
324 325
            if self.accounts_list is not None:
                break
326

327
            self.accounts_list = []
328
            self.loans_list = []
329 330 331 332 333 334 335 336 337 338 339
            # Check that there is at least one account for this user
            has_account = False
            self.pro_accounts.go()
            if self.pro_accounts.is_here():
                self.accounts_list.extend(self.page.iter_accounts())
                has_account = True
            else:
                # We dont want to let has_account=False if we landed on an unknown page
                # it has to be the no_accounts page
                assert self.no_account.is_here()

340 341 342 343 344 345 346 347
            try:
                self.accounts.go()
            except BrowserUnavailable as e:
                self.logger.warning('par accounts seem unavailable, retrying')
                exc = e
                self.accounts_list = None
                continue
            else:
348 349 350 351 352 353 354 355
                if self.accounts.is_here():
                    self.accounts_list.extend(self.page.iter_accounts())
                    has_account = True
                else:
                    # We dont want to let has_account=False if we landed on an unknown page
                    # it has to be the no_accounts page
                    assert self.no_account.is_here()

356
                exc = None
357

358 359 360 361
            if not has_account:
                # if we landed twice on NoAccountPage, it means there is neither pro accounts nor pp accounts
                raise NoAccountsException()

362
            for account in list(self.accounts_list):
363
                if account.type == Account.TYPE_LOAN:
364 365
                    # Loans details are present on another page so we create
                    # a Loan object and remove the corresponding Account:
366
                    self.location(account.url)
367 368 369
                    loan = self.page.get_loan()
                    loan.url = account.url
                    self.loans_list.append(loan)
370 371
                    self.accounts_list.remove(account)
            self.accounts_list.extend(self.loans_list)
372

373 374 375
            self.cards_list = [acc for acc in self.accounts_list if acc.type == Account.TYPE_CARD]
            if self.cards_list:
                self.go_cards_number(self.cards_list[0].url)
376
                if self.cards.is_here():
377
                    self.page.populate_cards_number(self.cards_list)
378
            # Cards without a number are not activated yet:
379
            for card in self.cards_list:
380 381
                if not card.number:
                    self.accounts_list.remove(card)
382

383 384 385 386 387 388
            type_with_iban = (
                Account.TYPE_CHECKING,
                Account.TYPE_SAVINGS,
                Account.TYPE_MARKET,
                Account.TYPE_PEA,
            )
Baptiste Delpey's avatar
Baptiste Delpey committed
389
            for account in self.accounts_list:
390
                if account.type in type_with_iban:
391 392
                    account.iban = self.iban.go(webid=account._webid).get_iban()

393
            for card in self.cards_list:
394 395 396 397 398
                checking, = [
                    account
                    for account in self.accounts_list
                    if account.type == Account.TYPE_CHECKING and account.url in card.url
                ]
399 400
                card.parent = checking

401 402 403
        if exc:
            raise exc

404
        self.ownership_guesser()
405
        return self.accounts_list
406 407 408 409

    def get_account(self, id):
        assert isinstance(id, basestring)

410
        for a in self.get_accounts_list():
411 412 413 414
            if a.id == id:
                return a
        return None

415 416
    def get_debit_date(self, debit_date):
        for i, j in zip(self.deferred_card_calendar, self.deferred_card_calendar[1:]):
417 418
            if i[0] < debit_date <= j[0]:
                return j[1]
419

420 421 422 423 424 425 426 427 428 429 430 431 432
    @retry_on_logout()
    @need_login
    def get_history(self, account, coming=False):
        if account.type in (Account.TYPE_LOAN, Account.TYPE_CONSUMER_CREDIT) or '/compte/derive' in account.url:
            return []
        if account.type is Account.TYPE_SAVINGS and u"PLAN D'ÉPARGNE POPULAIRE" in account.label:
            return []
        if account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_MARKET):
            return self.get_invest_transactions(account, coming)
        elif account.type == Account.TYPE_CARD:
            return self.get_card_transactions(account, coming)
        return self.get_regular_transactions(account, coming)

433 434 435 436 437 438 439 440 441 442 443 444 445
    def otp_location(self, *args, **kwargs):
        # this method is used in `otp_pagination` decorator from pages
        # without this header, we don't get a 401 but a 302 that logs us out
        kwargs.setdefault('headers', {}).update({'X-Requested-With': "XMLHttpRequest"})
        try:
            return super(BoursoramaBrowser, self).location(*args, **kwargs)
        except ClientError as e:
            # as done in boursorama's js : a 401 results in a popup
            # asking to send an otp to get more than x months of transactions
            # so... we don't want it :)
            if e.response.status_code != 401:
                raise e

446
    def get_regular_transactions(self, account, coming):
447
        if not coming:
448 449 450 451 452
            # We look for 3 years of history.
            params = {}
            params['movementSearch[toDate]'] = (date.today() + relativedelta(days=40)).strftime('%d/%m/%Y')
            params['movementSearch[fromDate]'] = (date.today() - relativedelta(years=3)).strftime('%d/%m/%Y')
            params['movementSearch[selectedAccounts][]'] = account._webid
453 454 455
            if self.otp_location('%s/mouvements' % account.url.rstrip('/'), params=params) is None:
                return

456 457
            for transaction in self.page.iter_history():
                yield transaction
458

459 460
        # Note: Checking accounts have a 'Mes prélèvements à venir' tab,
        # but these transactions have no date anymore so we ignore them.
461

462 463 464 465 466 467 468
    def get_card_transaction(self, coming, tr):
        if coming and tr.date > date.today():
            tr._is_coming = True
            return True
        elif not coming and tr.date <= date.today():
            return True

469
    def get_card_transactions(self, account, coming):
470 471
        # All card transactions can be found in the CSV (history and coming),
        # however the CSV shows a maximum of 1000 transactions from all accounts.
472
        self.location(account.url)
473 474 475 476
        if self.home.is_here():
            # for some cards, the site redirects us to '/'...
            return

477 478
        if self.deferred_card_calendar is None:
            self.location(self.page.get_calendar_link())
479

480 481 482
        params = {}
        params['movementSearch[fromDate]'] = (date.today() - relativedelta(years=3)).strftime('%d/%m/%Y')
        params['fullSearch'] = 1
483 484 485
        if self.otp_location(account.url, params=params) is None:
            return

486
        csv_link = self.page.get_csv_link()
487
        if csv_link and self.otp_location(csv_link):
488 489 490
            # Yield past transactions as 'history' and
            # transactions in the future as 'coming':
            for tr in sorted_transactions(self.page.iter_history(account_number=account.number)):
491 492 493 494 495 496 497
                if self.get_card_transaction(coming, tr):
                    yield tr
        else:
            # if the export link is hidden or we got a 401 on csv link,
            # we need to get transactions from current page or we will just get nothing
            for tr in self.open(account.url).page.iter_history(is_card=True):
                if self.get_card_transaction(coming, tr):
498
                    yield tr
499 500 501

            for tr in self.page.iter_history(is_card=True):
                if self.get_card_transaction(coming, tr):
502
                    yield tr
503 504 505 506 507

    def get_invest_transactions(self, account, coming):
        if coming:
            return
        transactions = []
508
        self.location('%s/mouvements' % account.url.rstrip('/'))
509 510 511 512 513 514 515 516
        account._history_pages = []
        for t in self.page.iter_history(account=account):
            transactions.append(t)
        for t in self.page.get_transactions_from_detail(account):
            transactions.append(t)
        for t in sorted(transactions, key=lambda tr: tr.date, reverse=True):
            yield t

517
    @retry_on_logout()
Baptiste Delpey's avatar
Baptiste Delpey committed
518
    @need_login
519
    def iter_investment(self, account):
520 521 522 523
        if (
            '/compte/derive' in account.url
            or account.type not in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_MARKET, Account.TYPE_PEA)
        ):
524
            return []
525
        self.location(account.url)
Baptiste Delpey's avatar
Baptiste Delpey committed
526
        return self.page.iter_investment()
527

528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543
    @retry_on_logout()
    @need_login
    def iter_market_orders(self, account):
        # Only Market & PEA accounts have the Market Orders tab
        if '/compte/derive' in account.url or account.type not in (Account.TYPE_MARKET, Account.TYPE_PEA):
            return []
        self.location(account.url)

        # Go to Market Orders tab ('Mes ordres')
        market_order_link = self.page.get_market_order_link()
        if not market_order_link:
            self.logger.warning('Could not find market orders link for account "%s".', account.label)
            return []
        self.location(market_order_link)
        return self.page.iter_market_orders()

544 545 546
    @need_login
    def get_profile(self):
        return self.profile.stay_or_go().get_profile()
547

548 549 550 551 552 553 554 555
    @need_login
    def get_advisor(self):
        # same for everyone
        advisor = Advisor()
        advisor.name = u"Service clientèle"
        advisor.phone = u"0146094949"
        return iter([advisor])

556
    def go_recipients_list(self, account_url, account_id):
557
        # url transfer preparation
558
        url = urlsplit(account_url)
559 560
        parts = [part for part in url.path.split('/') if part]

561
        assert len(parts) > 2, 'Account url missing some important part to iter recipient'
562
        account_type = parts[1]  # cav, ord, epargne ...
563
        account_webid = parts[-1]
564

565
        self.transfer_main_page.go(acc_type=account_type, webid=account_webid)  # may raise a BrowserHTTPNotFound
566

567 568 569 570
        # can check all account available transfer option
        if self.transfer_main_page.is_here():
            self.transfer_accounts.go(acc_type=account_type, webid=account_webid)

571
        if self.transfer_accounts.is_here():
572
            self.page.submit_account(account_id)  # may raise AccountNotFound
573 574 575
        elif self.transfer_main_page.is_here():
            self.new_transfer_accounts.go(acc_type=account_type, webid=account_webid)
            self.page.submit_account(account_id)  # may raise AccountNotFound
576 577 578 579 580 581 582 583 584 585 586

    @need_login
    def iter_transfer_recipients(self, account):
        if account.type in (Account.TYPE_LOAN, Account.TYPE_LIFE_INSURANCE):
            return []
        assert account.url

        try:
            self.go_recipients_list(account.url, account.id)
        except (BrowserHTTPNotFound, AccountNotFound):
            return []
587

588 589 590 591 592
        assert (
            self.recipients_page.is_here()
            or self.new_recipients_page.is_here()
        ), 'Should be on recipients page'

593 594 595 596
        return self.page.iter_recipients()

    def check_basic_transfer(self, transfer):
        if transfer.amount <= 0:
597
            raise TransferInvalidAmount('transfer amount must be positive')
598
        if transfer.recipient_id == transfer.account_id:
599
            raise TransferInvalidRecipient('recipient must be different from emitter')
600
        if not transfer.label:
601
            raise TransferInvalidLabel('transfer label cannot be empty')
602 603 604 605 606 607 608 609 610 611 612

    @need_login
    def init_transfer(self, transfer, **kwargs):
        self.check_basic_transfer(transfer)

        account = self.get_account(transfer.account_id)
        if not account:
            raise AccountNotFound()

        recipients = list(self.iter_transfer_recipients(account))
        if not recipients:
613
            raise TransferInvalidEmitter('The account cannot emit transfers')
614 615 616

        recipients = [rcpt for rcpt in recipients if rcpt.id == transfer.recipient_id]
        if len(recipients) == 0:
617
            raise TransferInvalidRecipient('The recipient cannot be used with the emitter account')
618 619 620 621 622 623 624 625
        assert len(recipients) == 1

        self.page.submit_recipient(recipients[0]._tempid)
        assert self.transfer_charac.is_here()

        self.page.submit_info(transfer.amount, transfer.label, transfer.exec_date)
        assert self.transfer_confirm.is_here()

626 627 628 629
        if self.page.need_refresh():
            # In some case we are not yet in the transfer_charac page, you need to refresh the page
            self.location(self.url)
            assert not self.page.need_refresh()
630 631 632 633
        ret = self.page.get_transfer()

        # at this stage, the site doesn't show the real ids/ibans, we can only guess
        if recipients[0].label != ret.recipient_label:
634 635 636 637
            self.logger.info(
                'Recipients from iter_recipient and from the transfer are diffent: "%s" and "%s"',
                recipients[0].label, ret.recipient_label
            )
638 639 640
            if not ret.recipient_label.startswith('%s - ' % recipients[0].label):
                # the label displayed here is  "<name> - <bank>"
                # but in the recipients list it is "<name>"...
641 642 643 644
                raise AssertionError(
                    'Recipient label changed during transfer (from "%s" to "%s")'
                    % (recipients[0].label, ret.recipient_label)
                )
645 646 647 648
        ret.recipient_id = recipients[0].id
        ret.recipient_iban = recipients[0].iban

        if account.label != ret.account_label:
649
            raise TransferError('Account label changed during transfer')
650 651 652 653 654 655 656 657 658 659 660 661

        ret.account_id = account.id
        ret.account_iban = account.iban

        return ret

    @need_login
    def execute_transfer(self, transfer, **kwargs):
        assert self.transfer_confirm.is_here()
        self.page.submit()

        assert self.transfer_sent.is_here()
662 663
        transfer_error = self.page.get_transfer_error()
        if transfer_error:
664
            raise TransferBankError(message=transfer_error)
665

666 667
        # the last page contains no info, return the last transfer object from init_transfer
        return transfer
668 669

    @need_login
670 671
    def init_new_recipient(self, recipient):
        self.recipient_form = None  # so it is reset when a new recipient is added
672

673
        # get url
674 675
        account = None
        for account in self.get_accounts_list():
676
            if account.url:
677 678 679
                break

        suffix = 'virements/comptes-externes/nouveau'
680 681
        if account.url.endswith('/'):
            target = account.url + suffix
682
        else:
683
            target = account.url + '/' + suffix
684 685

        self.location(target)
686
        assert self.page.is_charac(), 'Not on the page to add recipients.'
687

688
        # fill recipient form
689
        self.page.submit_recipient(recipient)
690 691 692 693 694
        recipient.origin_account_id = account.id

        # confirm sending sms
        assert self.page.is_confirm_send_sms(), 'Cannot reach the page asking to send a sms.'
        self.page.confirm_send_sms()
695 696

        if self.page.is_send_sms():
697
            # send sms
698 699
            self.page.send_otp()
            assert self.page.is_confirm_otp(), 'The sms was not sent.'
700

701
            self.recipient_form = self.page.get_confirm_otp_form()
702
            self.recipient_form['account_url'] = account.url
703
            raise AddRecipientStep(recipient, Value('otp_sms', label='Veuillez saisir le code recu par sms'))
704

705 706
        # if the add recipient is restarted after the sms has been confirmed recently, the sms step is not presented again
        return self.rcpt_after_sms()
707

708 709
    def new_recipient(self, recipient, **kwargs):
        # step 2 of new_recipient
710
        if 'otp_sms' in kwargs:
711 712
            # there is no confirmation to check the recipient
            # validating the sms code directly adds the recipient
713
            account_url = self.send_recipient_form(kwargs['otp_sms'])
714
            return self.rcpt_after_sms(recipient, account_url)
715 716 717 718
        # step 3 of new_recipient (not always used)
        elif 'otp_email' in kwargs:
            account_url = self.send_recipient_form(kwargs['otp_email'])
            return self.check_and_update_recipient(recipient, account_url)
719 720 721 722

        # step 1 of new recipient
        return self.init_new_recipient(recipient)

723 724 725 726 727 728 729 730 731 732 733 734 735
    def send_recipient_form(self, value):
        if not self.recipient_form:
            # The session expired
            raise AddRecipientTimeout()

        url = self.recipient_form.pop('url')
        account_url = self.recipient_form.pop('account_url')
        self.recipient_form['strong_authentication_confirm[code]'] = value
        self.location(url, data=self.recipient_form)

        self.recipient_form = None
        return account_url

736
    def rcpt_after_sms(self, recipient, account_url):
737 738 739 740 741 742 743 744 745 746 747 748 749 750
        if self.page.is_send_email():
            # Sometimes after validating the sms code, the user is also
            # asked to validate a code received by email (observed when
            # adding a non-french recipient).
            self.page.send_otp()
            assert self.page.is_confirm_otp(), 'The email was not sent.'

            self.recipient_form = self.page.get_confirm_otp_form()
            self.recipient_form['account_url'] = account_url
            raise AddRecipientStep(recipient, Value('otp_email', label='Veuillez saisir le code recu par email'))

        return self.check_and_update_recipient(recipient, account_url)

    def check_and_update_recipient(self, recipient, account_url):
751 752
        assert self.page.is_created(), 'The recipient was not added.'

753 754 755
        # At this point, the recipient was added to the website,
        # here we just want to return the right Recipient object.
        # We are taking it from the recipient list page
756 757
        # because there is no summary of the adding
        self.go_recipients_list(account_url, recipient.origin_account_id)
758
        return find_object(self.page.iter_recipients(), id=recipient.id, error=RecipientNotFound)
759

760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816
    @need_login
    def iter_transfers(self, account):
        if account is not None:
            if not (isinstance(account, Account) or isinstance(account, Emitter)):
                self.logger.debug('we have only the emitter id %r, fetching full object', account)
                account = find_object(self.iter_emitters(), id=account)

            return sorted_transfers(self.iter_transfers_for_emitter(account))

        transfers = []
        self.logger.debug('no account given: fetching all emitters')
        for emitter in self.iter_emitters():
            self.logger.debug('fetching transfers for emitter %r', emitter.id)
            transfers.extend(self.iter_transfers_for_emitter(emitter))
        transfers = sorted_transfers(transfers)
        return transfers

    @need_login
    def iter_transfers_for_emitter(self, emitter):
        # We fetch original transfers from 2 pages (single transfers vs periodic).
        # Each page is sorted, but since we list from the 2 pages in sequence,
        # the result is not sorted as is.
        # TODO Maybe the site is not stateful and we could do parallel navigation
        # on both lists, to merge the sorted iterators.

        self.transfer_list.go(acc_type='temp', webid=emitter._bourso_id, type='ponctuels')
        for transfer in self.page.iter_transfers():
            transfer.account_id = emitter.id
            transfer.date_type = TransferDateType.FIRST_OPEN_DAY
            if transfer._is_instant:
                transfer.date_type = TransferDateType.INSTANT
            elif transfer.exec_date > date.today():
                # The site does not indicate when transfer was created
                # we only have the date of its execution.
                # So, for a DONE transfer, we cannot know if it was deferred or not...
                transfer.date_type = TransferDateType.DEFERRED

            self.location(transfer.url)
            self.page.fill_transfer(obj=transfer)

            # build id with account id because get_transfer will receive only the account id
            assert transfer.id, 'transfer should have an id from site'
            transfer.id = '%s.%s' % (emitter.id, transfer.id)
            yield transfer

        self.transfer_list.go(acc_type='temp', webid=emitter._bourso_id, type='permanents')
        for transfer in self.page.iter_transfers():
            transfer.account_id = emitter.id
            transfer.date_type = TransferDateType.PERIODIC
            self.location(transfer.url)
            self.page.fill_transfer(obj=transfer)
            self.page.fill_periodic_transfer(obj=transfer)

            assert transfer.id, 'transfer should have an id from site'
            transfer.id = '%s.%s' % (emitter.id, transfer.id)
            yield transfer

817 818 819 820 821 822 823 824
    def iter_currencies(self):
        return self.currencylist.go().get_currency_list()

    def get_rate(self, curr_from, curr_to):
        r = Rate()
        params = {
            'from': curr_from,
            'to': curr_to,
825
            'amount': '1',
826 827 828 829 830 831 832 833 834 835 836
        }
        r.currency_from = curr_from
        r.currency_to = curr_to
        r.datetime = datetime.now()
        try:
            self.currencyconvert.go(params=params)
            r.value = self.page.get_rate()
        # if a rate is no available the site return a 401 error...
        except ClientError:
            return
        return r
837 838 839 840 841 842 843

    @need_login
    def iter_emitters(self):
        # It seems that if we give a wrong acc_type and webid to the transfer page
        # we are redirected to a page where we can choose the emitter account
        self.transfer_accounts.go(acc_type='temp', webid='temp')
        return self.page.iter_emitters()