browser.py 18.5 KB
Newer Older
smurail's avatar
smurail committed
1 2
# -*- coding: utf-8 -*-

3
# Copyright(C) 2009-2016  Romain Bignon
smurail's avatar
smurail committed
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.

20
from __future__ import unicode_literals
smurail's avatar
smurail committed
21

22 23
from datetime import datetime
from dateutil.relativedelta import relativedelta
smurail's avatar
smurail committed
24
import time
25
from requests.exceptions import ConnectionError
smurail's avatar
smurail committed
26

Baptiste Delpey's avatar
Baptiste Delpey committed
27
from weboob.browser.browsers import LoginBrowser, URL, need_login
smurail's avatar
smurail committed
28
from weboob.capabilities.base import find_object
Baptiste Delpey's avatar
Baptiste Delpey committed
29
from weboob.capabilities.bank import AccountNotFound, Account, TransferError, AddRecipientStep
30
from weboob.capabilities.profile import ProfileMissing
31
from weboob.tools.decorators import retry
32
from weboob.tools.capabilities.bank.transactions import sorted_transactions
smurail's avatar
smurail committed
33
from weboob.tools.json import json
34
from weboob.browser.exceptions import ServerError
35
from weboob.browser.elements import DataError
Baptiste Delpey's avatar
Baptiste Delpey committed
36
from weboob.exceptions import BrowserIncorrectPassword
Baptiste Delpey's avatar
Baptiste Delpey committed
37
from weboob.tools.value import Value
smurail's avatar
smurail committed
38

39 40 41
from .pages import (
    LoginPage, AccountsPage, AccountsIBANPage, HistoryPage, TransferInitPage,
    ConnectionThresholdPage, LifeInsurancesPage, LifeInsurancesHistoryPage,
42
    LifeInsurancesDetailPage, NatioVieProPage, CapitalisationPage,
43
    MarketListPage, MarketPage, MarketHistoryPage, MarketSynPage, BNPKeyboard,
44
    RecipientsPage, ValidateTransferPage, RegisterTransferPage, AdvisorPage,
45
    AddRecipPage, ActivateRecipPage, ProfilePage, ListDetailCardPage, ListErrorPage,
46
)
smurail's avatar
smurail committed
47 48


Kitof's avatar
Kitof committed
49
__all__ = ['BNPPartPro', 'HelloBank']
smurail's avatar
smurail committed
50 51 52 53 54 55 56 57 58 59 60 61 62 63


def JSON(data):
    return ('json', data)


def isJSON(obj):
    return type(obj) is tuple and obj and obj[0] == 'json'


class JsonBrowserMixin(object):
    def open(self, *args, **kwargs):
        if isJSON(kwargs.get('data')):
            kwargs['data'] = json.dumps(kwargs['data'][1])
Laurent Bachelier's avatar
Laurent Bachelier committed
64
            if 'headers' not in kwargs:
smurail's avatar
smurail committed
65 66 67 68 69
                kwargs['headers'] = {}
            kwargs['headers']['Content-Type'] = 'application/json'

        return super(JsonBrowserMixin, self).open(*args, **kwargs)

70 71

class BNPParibasBrowser(JsonBrowserMixin, LoginBrowser):
smurail's avatar
smurail committed
72 73
    TIMEOUT = 30.0

74
    login = URL(r'identification-wspl-pres/identification\?acceptRedirection=true&timestamp=(?P<timestamp>\d+)',
smurail's avatar
smurail committed
75 76 77
                'SEEA-pa01/devServer/seeaserver',
                'https://mabanqueprivee.bnpparibas.net/fr/espace-prive/comptes-et-contrats\?u=%2FSEEA-pa01%2FdevServer%2Fseeaserver',
                LoginPage)
78

79
    list_error_page = URL('https://mabanque.bnpparibas/rsc/contrib/document/properties/identification-fr-part-V1.json', ListErrorPage)
80
    con_threshold = URL('/fr/connexion/100-connexions',
81
                        '/fr/connexion/mot-de-passe-expire',
82 83
                        '/fr/espace-prive/100-connexions.*',
                        '/fr/espace-pro/100-connexions-pro.*',
84
                        '/fr/espace-pro/changer-son-mot-de-passe',
85
                        '/fr/espace-client/100-connexions',
86
                        '/fr/espace-prive/mot-de-passe-expire',
87
                        '/fr/client/100-connexion',
88
                        '/fr/systeme/page-indisponible', ConnectionThresholdPage)
smurail's avatar
smurail committed
89 90 91
    accounts = URL('udc-wspl/rest/getlstcpt', AccountsPage)
    ibans = URL('rib-wspl/rpc/comptes', AccountsIBANPage)
    history = URL('rop-wspl/rest/releveOp', HistoryPage)
92
    transfer_init = URL('virement-wspl/rest/initialisationVirement', TransferInitPage)
smurail's avatar
smurail committed
93

94 95 96 97
    lifeinsurances = URL('mefav-wspl/rest/infosContrat', LifeInsurancesPage)
    lifeinsurances_history = URL('mefav-wspl/rest/listMouvements', LifeInsurancesHistoryPage)
    lifeinsurances_detail = URL('mefav-wspl/rest/detailMouvement', LifeInsurancesDetailPage)

98 99 100
    natio_vie_pro = URL('/mefav-wspl/rest/natioViePro', NatioVieProPage)
    capitalisation_page = URL('https://www.clients.assurance-vie.fr/servlets/helios.cinrj.htmlnav.runtime.FrontServlet', CapitalisationPage)

101
    market_list = URL('pe-war/rpc/SAVaccountDetails/get', MarketListPage)
102
    market_syn = URL('pe-war/rpc/synthesis/get', MarketSynPage)
103 104 105
    market = URL('pe-war/rpc/portfolioDetails/get', MarketPage)
    market_history = URL('/pe-war/rpc/turnOverHistory/get', MarketHistoryPage)

Baptiste Delpey's avatar
Baptiste Delpey committed
106
    recipients = URL('/virement-wspl/rest/listerBeneficiaire', RecipientsPage)
Baptiste Delpey's avatar
Baptiste Delpey committed
107 108
    add_recip = URL('/virement-wspl/rest/ajouterBeneficiaire', AddRecipPage)
    activate_recip = URL('/virement-wspl/rest/activerBeneficiaire', ActivateRecipPage)
Baptiste Delpey's avatar
Baptiste Delpey committed
109 110
    validate_transfer = URL('/virement-wspl/rest/validationVirement', ValidateTransferPage)
    register_transfer = URL('/virement-wspl/rest/enregistrerVirement', RegisterTransferPage)
Baptiste Delpey's avatar
Baptiste Delpey committed
111

112 113
    advisor = URL('/conseiller-wspl/rest/monConseiller', AdvisorPage)

114
    profile = URL(r'/kyc-wspl/rest/informationsClient', ProfilePage)
115
    list_detail_card = URL(r'/udcarte-wspl/rest/listeDetailCartes', ListDetailCardPage)
116

117 118
    def __init__(self, config, *args, **kwargs):
        super(BNPParibasBrowser, self).__init__(config['login'].get(), config['password'].get(), *args, **kwargs)
119 120
        self.accounts_list = None
        self.card_to_transaction_type = {}
121
        self.rotating_password = config['rotating_password'].get()
122

123 124 125 126
    @retry(ConnectionError, tries=3)
    def open(self, *args, **kwargs):
        return super(BNPParibasBrowser, self).open(*args, **kwargs)

smurail's avatar
smurail committed
127
    def do_login(self):
Baptiste Delpey's avatar
Baptiste Delpey committed
128 129
        if not (self.username.isdigit() and self.password.isdigit()):
            raise BrowserIncorrectPassword()
smurail's avatar
smurail committed
130 131 132 133 134
        timestamp = lambda: int(time.time() * 1e3)
        self.login.go(timestamp=timestamp())
        if self.login.is_here():
            self.page.login(self.username, self.password)

135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151
    def change_pass(self, oldpass, newpass):
        res = self.open('/identification-wspl-pres/grille?accessible=false')
        url = '/identification-wspl-pres/grille/%s' % res.json()['data']['idGrille']
        keyboard = self.open(url)
        vk = BNPKeyboard(self, keyboard)
        data = {}
        data['codeAppli'] = 'PORTAIL'
        data['idGrille'] = res.json()['data']['idGrille']
        data['typeGrille'] = res.json()['data']['typeGrille']
        data['confirmNouveauPassword'] = vk.get_string_code(newpass)
        data['nouveauPassword'] = vk.get_string_code(newpass)
        data['passwordActuel'] = vk.get_string_code(oldpass)
        response = self.location('/mcs-wspl/rpc/modifiercodesecret', data=data)
        if response.json().get('messageIden').lower() == 'nouveau mot de passe invalide':
            return False
        return True

152 153
    @need_login
    def get_profile(self):
154
        self.profile.go(data=JSON({}))
155
        profile = self.page.get_profile()
156 157
        if profile:
            return profile
158
        raise ProfileMissing(self.page.get_error_message())
159

smurail's avatar
smurail committed
160 161
    @need_login
    def get_accounts_list(self):
162 163
        if self.accounts_list is None:
            self.accounts_list = []
164 165 166
            # In case of password renewal, we need to go on ibans twice.
            self.ibans.go()
            ibans = self.page.get_ibans_dict() if self.ibans.is_here() else self.ibans.go().get_ibans_dict()
167 168 169
            # This page might be unavailable.
            try:
                ibans.update(self.transfer_init.go(data=JSON({'modeBeneficiaire': '0'})).get_ibans_dict('Crediteur'))
170
            except (TransferError, AttributeError):
171 172
                pass

173 174 175 176
            accounts = list(self.accounts.go().iter_accounts(ibans))
            self.market_syn.go(data=JSON({}))  # do a post on the given URL
            market_accounts = self.page.get_list()  # get the list of 'Comptes Titres'
            checked_accounts = set()
177
            for account in accounts:
178 179 180 181 182 183 184 185 186 187 188 189
                for market_acc in market_accounts:
                    if all((
                        market_acc['securityAccountNumber'].endswith(account.number[-4:]),
                        account.type in (Account.TYPE_MARKET, Account.TYPE_PEA),
                        account.label == market_acc['securityAccountName'],
                        not account.iban,
                    )):
                        if account.id in checked_accounts:
                            # in this case, we have identified two accounts for the same CompteTitre
                            raise DataError('we have two market accounts mapped to a same "CompteTitre" dictionary')

                        checked_accounts.add(account.id)
190
                        account.balance = market_acc.get('valorisation', account.balance)
191 192 193
                        account.valuation_diff = market_acc['profitLoss']
                        break
                self.accounts_list.append(account)
194

195 196
            # Fetching capitalisation contracts from the "Assurances Vie" space (some are not in the BNP API):
            params = self.natio_vie_pro.go().get_params()
197 198 199 200 201 202 203 204 205 206 207 208 209 210
            try:
                self.capitalisation_page.go(params=params)
            except ServerError:
                self.logger.warning("An Internal Server Error occurred")
            else:
                if self.capitalisation_page.is_here() and self.page.has_contracts():
                    for account in self.page.iter_capitalisation():
                        # Life Insurance accounts may appear BOTH in the API and the "Assurances Vie" domain,
                        # It is better to keep the API version since it contains the unitvalue:
                        if account.number not in [a.number for a in self.accounts_list]:
                            self.logger.warning("We found an account that only appears on the old BNP website.")
                            self.accounts_list.append(account)
                        else:
                            self.logger.warning("This account was skipped because it already appears in the API.")
211

212
        return iter(self.accounts_list)
smurail's avatar
smurail committed
213 214 215 216 217 218 219

    @need_login
    def get_account(self, _id):
        return find_object(self.get_accounts_list(), id=_id, error=AccountNotFound)

    @need_login
    def iter_history(self, account, coming=False):
220 221 222
        # The accounts from the "Assurances Vie" space have no available history:
        if hasattr(account, '_details'):
            return []
223 224
        if account.type == Account.TYPE_PEA and account.label.endswith('Espèces'):
            return []
225 226
        if account.type == account.TYPE_LIFE_INSURANCE:
            return self.iter_lifeinsurance_history(account, coming)
227
        elif account.type in (account.TYPE_MARKET, Account.TYPE_PEA) and not coming:
228
            try:
229
                self.market_list.go(data=JSON({}))
230
            except ServerError:
231
                self.logger.warning("An Internal Server Error occurred")
232
                return iter([])
233
            for market_acc in self.page.get_list():
234
                if account.number[-4:] == market_acc['securityAccountNumber'][-4:]:
235 236 237 238 239
                    self.page = self.market_history.go(data=JSON({
                        "securityAccountNumber": market_acc['securityAccountNumber'],
                    }))
                    return self.page.iter_history()
            return iter([])
240
        else:
241 242 243 244
            if not self.card_to_transaction_type:
                self.list_detail_card.go()
                self.card_to_transaction_type = self.page.get_card_to_transaction_type()

245
            self.history.go(data=JSON({
246
                "ibanCrypte": account.id,
247 248
                "pastOrPending": 1,
                "triAV": 0,
249 250
                "startDate": (datetime.now() - relativedelta(years=2)).strftime('%d%m%Y'),
                "endDate": datetime.now().strftime('%d%m%Y')
251
            }))
252 253 254 255 256

            if coming:
                return sorted_transactions(self.page.iter_coming())
            else:
                return sorted_transactions(self.page.iter_history())
257 258 259

    @need_login
    def iter_lifeinsurance_history(self, account, coming=False):
260
        self.lifeinsurances_history.go(data=JSON({
261
            "ibanCrypte": account.id,
smurail's avatar
smurail committed
262
        }))
263 264 265

        for tr in self.page.iter_history(coming):
            page = self.lifeinsurances_detail.go(data=JSON({
266
                "ibanCrypte": account.id,
267 268 269 270 271 272 273
                "idMouvement": tr._op.get('idMouvement'),
                "ordreMouvement": tr._op.get('ordreMouvement'),
                "codeTypeMouvement": tr._op.get('codeTypeMouvement'),
            }))
            tr.investments = list(page.iter_investments())
            yield tr

smurail's avatar
smurail committed
274 275 276 277 278 279
    @need_login
    def iter_coming_operations(self, account):
        return self.iter_history(account, coming=True)

    @need_login
    def iter_investment(self, account):
280 281
        if account.type == Account.TYPE_PEA and account.label.endswith('Espèces'):
            return []
282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301

        # Life insurances and PERP may be scraped from the API or from the "Assurance Vie" space,
        # so we need to discriminate between both using account._details:
        if account.type in (account.TYPE_LIFE_INSURANCE, account.TYPE_PERP, account.TYPE_CAPITALISATION):
            if hasattr(account, '_details'):
                # Going to the "Assurances Vie" page
                natiovie_params = self.natio_vie_pro.go().get_params()
                self.capitalisation_page.go(params=natiovie_params)
                # Fetching the form to get the contract investments:
                capitalisation_params = self.page.get_params(account)
                self.capitalisation_page.go(params=capitalisation_params)
                return self.page.iter_investments()
            else:
                # No capitalisation contract has yet been found in the API:
                assert account.type != account.TYPE_CAPITALISATION
                self.lifeinsurances.go(data=JSON({
                    "ibanCrypte": account.id,
                }))
                return self.page.iter_investments()

302
        elif account.type in (account.TYPE_MARKET, account.TYPE_PEA):
303
            try:
304
                self.market_list.go(data=JSON({}))
305
            except ServerError:
306
                self.logger.warning("An Internal Server Error occurred")
307
                return iter([])
308
            for market_acc in self.page.get_list():
309
                if account.number[-4:] == market_acc['securityAccountNumber'][-4:] and not account.iban:
310 311
                    # Sometimes generate an Internal Server Error ...
                    try:
312
                        self.market.go(data=JSON({
313 314 315
                            "securityAccountNumber": market_acc['securityAccountNumber'],
                        }))
                    except ServerError:
316
                        self.logger.warning("An Internal Server Error occurred")
317
                        break
318
                    return self.page.iter_investments()
319

320
        return iter([])
smurail's avatar
smurail committed
321

Baptiste Delpey's avatar
Baptiste Delpey committed
322
    @need_login
323
    def iter_recipients(self, origin_account_id):
324 325 326 327 328 329
        try:
            if not origin_account_id in self.transfer_init.go(data=JSON({'modeBeneficiaire': '0'})).get_ibans_dict('Debiteur'):
                raise NotImplementedError()
        except TransferError:
            return

330
        for recipient in self.page.transferable_on(origin_account_ibancrypte=origin_account_id):
Baptiste Delpey's avatar
Baptiste Delpey committed
331
            yield recipient
332
        if self.page.can_transfer_to_recipients(origin_account_id):
Baptiste Delpey's avatar
Baptiste Delpey committed
333 334 335
            for recipient in self.recipients.go(data=JSON({'type': 'TOUS'})).iter_recipients():
                yield recipient

336 337 338 339 340 341 342 343 344 345 346 347 348 349 350
    @need_login
    def new_recipient(self, recipient, **params):
        if 'code' in params:
            return self.send_code(recipient, **params)
        # needed to get the phone number, enabling the possibility to send sms.
        self.recipients.go(data=JSON({'type': 'TOUS'}))
        # post recipient data sending sms with same request
        data = {}
        data['adresseBeneficiaire'] = ''
        data['iban'] = recipient.iban
        data['libelleBeneficiaire'] = recipient.label
        data['notification'] = True
        data['typeBeneficiaire'] = ''
        data['typeEnvoi'] = 'SMS'
        recipient = self.add_recip.go(data=json.dumps(data), headers={'Content-Type': 'application/json'}).get_recipient(recipient)
351
        raise AddRecipientStep(recipient, Value('code', label='Saisissez le code.'))
352

Baptiste Delpey's avatar
Baptiste Delpey committed
353
    @need_login
Sylvie Ye's avatar
Sylvie Ye committed
354
    def prepare_transfer(self, account, recipient, amount, reason, exec_date):
Baptiste Delpey's avatar
Baptiste Delpey committed
355 356 357
        data = {}
        data['devise'] = account.currency
        data['motif'] = reason
Sylvie Ye's avatar
Sylvie Ye committed
358
        data['dateExecution'] = exec_date.strftime('%d-%m-%Y')
Baptiste Delpey's avatar
Baptiste Delpey committed
359 360 361
        data['compteDebiteur'] = account.id
        data['montant'] = str(amount)
        data['typeVirement'] = 'SEPA'
362
        if recipient.category == u'Externe':
Baptiste Delpey's avatar
Baptiste Delpey committed
363 364 365 366 367 368
            data['idBeneficiaire'] = recipient.id
        else:
            data['compteCrediteur'] = recipient.id
        return data

    @need_login
Sylvie Ye's avatar
Sylvie Ye committed
369 370
    def init_transfer(self, account, recipient, amount, reason, exec_date):
        data = self.prepare_transfer(account, recipient, amount, reason, exec_date)
371 372
        return self.validate_transfer.go(data=JSON(data)).handle_response(account, recipient, amount, reason)

Baptiste Delpey's avatar
Baptiste Delpey committed
373 374 375 376 377 378 379 380 381 382 383
    @need_login
    def send_code(self, recipient, **params):
        # depending on whether recipient is a weboob or a budgea backend object.
        _id = recipient.webid if hasattr(recipient, 'webid') else recipient.id
        data = {}
        data['idBeneficiaire'] = _id
        data['typeActivation'] = 1
        data['codeActivation'] = params['code']
        return self.activate_recip.go(data=json.dumps(data), headers={'Content-Type': 'application/json'}).get_recipient(recipient)

    @need_login
384 385 386
    def execute_transfer(self, transfer):
        self.register_transfer.go(data=JSON({'referenceVirement': transfer.id}))
        return self.page.handle_response(transfer)
Baptiste Delpey's avatar
Baptiste Delpey committed
387

388 389
    @need_login
    def get_advisor(self):
390 391 392 393
        self.advisor.stay_or_go()
        if self.page.has_error():
            return None
        return self.page.get_advisor()
394

395 396 397 398 399 400 401 402 403 404 405 406 407 408 409
    @need_login
    def iter_threads(self):
        raise NotImplementedError()

    @need_login
    def get_thread(self, thread):
        raise NotImplementedError()


class BNPPartPro(BNPParibasBrowser):
    BASEURL_TEMPLATE = r'https://%s.bnpparibas/'
    BASEURL = BASEURL_TEMPLATE % 'mabanque'

    def __init__(self, config=None, *args, **kwargs):
        self.config = config
410
        super(BNPPartPro, self).__init__(self.config, *args, **kwargs)
411 412 413 414

    def switch(self, subdomain):
        self.BASEURL = self.BASEURL_TEMPLATE % subdomain

Baptiste Delpey's avatar
Baptiste Delpey committed
415

Kitof's avatar
Kitof committed
416 417
class HelloBank(BNPParibasBrowser):
    BASEURL = 'https://www.hellobank.fr/'