diff --git a/modules/cragr/api/__init__.py b/modules/cragr/api/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/modules/cragr/api/browser.py b/modules/cragr/api/browser.py new file mode 100644 index 0000000000000000000000000000000000000000..aa8dcb3c57d9213a7ee2371bea480bf08121abe1 --- /dev/null +++ b/modules/cragr/api/browser.py @@ -0,0 +1,247 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012-2019 Romain Bignon +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from __future__ import unicode_literals + +import re + +from weboob.capabilities.bank import ( + Account, +) +from weboob.capabilities.base import find_object, empty, NotAvailable +from weboob.browser import LoginBrowser, URL, need_login +from weboob.exceptions import BrowserUnavailable, BrowserIncorrectPassword +from weboob.browser.exceptions import ServerError +from weboob.tools.capabilities.bank.iban import is_iban_valid + +from .pages import ( + LoginPage, LoggedOutPage, KeypadPage, SecurityPage, ContractsPage, AccountsPage, AccountDetailsPage, + IbanPage, HistoryPage, ProfilePage, +) + + +__all__ = ['CragrAPI'] + + +class CragrAPI(LoginBrowser): + login_page = URL(r'particulier/acceder-a-mes-comptes.html$', LoginPage) + keypad = URL(r'particulier/acceder-a-mes-comptes.authenticationKeypad.json', KeypadPage) + security_check = URL(r'particulier/acceder-a-mes-comptes.html/j_security_check', SecurityPage) + logged_out = URL(r'.*', LoggedOutPage) + + contracts_page = URL(r'particulier/operations/.rechargement.contexte.html\?idBamIndex=(?P)', + r'association/operations/.rechargement.contexte.html\?idBamIndex=(?P)', + r'professionnel/operations/.rechargement.contexte.html\?idBamIndex=(?P)', ContractsPage) + + accounts_page = URL(r'particulier/operations/synthese.html', + r'association/operations/synthese.html', + r'professionnel/operations/synthese.html', AccountsPage) + + account_details = URL(r'particulier/operations/synthese/jcr:content.produits-valorisation.json/(?P)', + r'association/operations/synthese/jcr:content.produits-valorisation.json/(?P)', + r'professionnel/operations/synthese/jcr:content.produits-valorisation.json/(?P)', AccountDetailsPage) + + account_iban = URL(r'particulier/operations/operations-courantes/editer-rib/jcr:content.ibaninformation.json', + r'association/operations/operations-courantes/editer-rib/jcr:content.ibaninformation.json', + r'professionnel/operations/operations-courantes/editer-rib/jcr:content.ibaninformation.json', IbanPage) + + history_page = URL(r'particulier/operations/synthese/detail-comptes/jcr:content.n3.compte.infos.json', + r'association/operations/synthese/detail-comptes/jcr:content.n3.compte.infos.json', + r'professionnel/operations/synthese/detail-comptes/jcr:content.n3.compte.infos.json', HistoryPage) + + profile_page = URL(r'particulier/operations/synthese/jcr:content.npc.store.client.json', + r'association/operations/synthese/jcr:content.npc.store.client.json', + r'professionnel/operations/synthese/jcr:content.npc.store.client.json', ProfilePage) + + + def __init__(self, website, *args, **kwargs): + super(CragrAPI, self).__init__(*args, **kwargs) + website = website.replace('.fr', '') + self.region = re.sub('^m\.', 'www.credit-agricole.fr/', website) + self.BASEURL = 'https://%s/' % self.region + self.accounts_url = None + + def do_login(self): + self.keypad.go() + keypad_password = self.page.build_password(self.password[:6]) + keypad_id = self.page.get_keypad_id() + assert keypad_password, 'Could not obtain keypad password' + assert keypad_id, 'Could not obtain keypad id' + + self.login_page.go() + # Get the form data to POST the security check: + form = self.page.get_login_form(self.username, keypad_password, keypad_id) + try: + self.security_check.go(data=form) + except ServerError as exc: + # Wrongpass returns a 500 server error... + error = exc.response.json().get('error') + if error: + message = error.get('message', '') + if 'Votre identification est incorrecte' in message: + raise BrowserIncorrectPassword() + assert False, 'Unhandled Server Error encountered: %s' % error.get('message', '') + + # accounts_url may contain '/particulier', '/professionnel' or '/association' + self.accounts_url = self.page.get_accounts_url() + assert self.accounts_url, 'Could not get accounts url from security check' + self.location(self.accounts_url) + assert self.accounts_page.is_here(), 'We failed to login after the security check!' + # Once the security check is passed, we are logged in. + + @need_login + def get_accounts_list(self): + # Determine how many spaces are present on the connection: + self.location(self.accounts_url) + total_spaces = self.page.count_spaces() + self.logger.info('The total number of spaces on this connection is %s.' % total_spaces) + + for contract in range(total_spaces): + # This request often returns a 500 error so we retry several times. + try: + self.contracts_page.go(id_contract=contract) + except ServerError: + self.logger.warning('Server returned error 500 when trying to access space %s, we try again' % contract) + try: + self.contracts_page.go(id_contract=contract) + except ServerError: + self.logger.warning('Server returned error 500 twice when trying to access space %s, this space will be skipped' % contract) + continue + + # The main account is not located at the same place in the JSON. + main_account = self.page.get_main_account() + main_account.owner_type = self.page.get_owner_type() + main_account._contract = contract + + accounts_list = list(self.page.iter_accounts()) + for account in accounts_list: + account._contract = contract + account.owner_type = self.page.get_owner_type() + + # Some accounts have no balance in the main JSON, so we must + # get all the (id, balance) pairs in the account_details JSON: + categories = {int(account._category) for account in accounts_list if account._category != None} + account_balances = {} + loan_ids = {} + for category in categories: + self.account_details.go(category=category) + account_balances.update(self.page.get_account_balances()) + loan_ids.update(self.page.get_loan_ids()) + + # Getting IBANs for checking accounts + if main_account.type == Account.TYPE_CHECKING: + params = { + 'compteIdx': int(main_account._index), + 'grandeFamilleCode': 1, + } + self.account_iban.go(params=params) + iban = self.page.get_iban() + if is_iban_valid(iban): + main_account.iban = iban + yield main_account + + for card in main_account._cards: + card.parent = main_account + card.currency = main_account.currency + card.owner_type = main_account.owner_type + card._contract = contract + yield card + + for account in accounts_list: + if empty(account.balance): + account.balance = account_balances.get(account.id, NotAvailable) + if account.type == Account.TYPE_CHECKING: + try: + params = { + 'compteIdx': int(account._index), + 'grandeFamilleCode': 1, + } + self.account_iban.go(params=params) + iban = self.page.get_iban() + if is_iban_valid(iban): + account.iban = iban + except ServerError: + self.logger.warning('Could not fetch IBAN for checking account "%s %s"', account.label, account.id) + pass + + # TO-DO: Create Loan() object with its related attributes + # Loans have a specific ID that we need to fetch + # so the backend can match loans properly. + # If no there is no loan ID, we keep the account ID. + if account.type == Account.TYPE_LOAN: + account.id = loan_ids.get(account.id, account.id) + account.balance = -account.balance + elif account.type == Account.TYPE_REVOLVING_CREDIT: + account.id = loan_ids.get(account.id, account.id) + account.balance = 0 + yield account + + @need_login + def go_to_account_space(self, contract): + # TO-DO: Figure out a way to determine whether + # we already are on the right account space + self.contracts_page.go(id_contract=contract) + assert self.accounts_page.is_here() + + @need_login + def get_card(self, id): + return find_object(self.get_cards(), id=id) + + @need_login + def get_cards(self, accounts_list=None): + # accounts_list is only used by get_list + raise BrowserUnavailable() + + @need_login + def get_history(self, account): + raise BrowserUnavailable() + + @need_login + def iter_investment(self, account): + raise BrowserUnavailable() + + @need_login + def iter_advisor(self): + raise BrowserUnavailable() + + @need_login + def get_profile(self): + #self.profile.go() + raise BrowserUnavailable() + + @need_login + def iter_transfer_recipients(self, account): + raise BrowserUnavailable() + + @need_login + def init_transfer(self, transfer, **params): + raise BrowserUnavailable() + + @need_login + def execute_transfer(self, transfer, **params): + raise BrowserUnavailable() + + @need_login + def build_recipient(self, recipient): + raise BrowserUnavailable() + + @need_login + def new_recipient(self, recipient, **params): + raise BrowserUnavailable() diff --git a/modules/cragr/api/pages.py b/modules/cragr/api/pages.py new file mode 100644 index 0000000000000000000000000000000000000000..85c92a93c3cf6d8e5e79ab7bec59bd1dc82051c9 --- /dev/null +++ b/modules/cragr/api/pages.py @@ -0,0 +1,263 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012-2019 Romain Bignon +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +from decimal import Decimal +import re +import json + +from weboob.browser.pages import HTMLPage, JsonPage, LoggedPage +from weboob.exceptions import BrowserUnavailable +from weboob.capabilities import NotAvailable +from weboob.capabilities.bank import ( + Account, AccountOwnerType, +) + +from weboob.browser.elements import DictElement, ItemElement, method +from weboob.browser.filters.standard import ( + CleanText, CleanDecimal, Currency as CleanCurrency, Format, Field, Map, Eval +) +from weboob.browser.filters.json import Dict + + +def float_to_decimal(f): + return Decimal(str(f)) + + +class KeypadPage(JsonPage): + def build_password(self, password): + # Fake Virtual Keyboard: just get the positions of each digit. + key_positions = [i for i in Dict('keyLayout')(self.doc)] + return str(','.join([str(key_positions.index(i)) for i in password])) + + def get_keypad_id(self): + return Dict('keypadId')(self.doc) + + +class LoginPage(HTMLPage): + def get_login_form(self, username, keypad_password, keypad_id): + form = self.get_form(id="loginForm") + form['j_username'] = username + form['j_password'] = keypad_password + form['keypadId'] = keypad_id + return form + + +class LoggedOutPage(HTMLPage): + def is_here(self): + return self.doc.xpath('//b[text()="FIN DE CONNEXION"]') + + def on_load(self): + self.logger.warning('We have been logged out!') + raise BrowserUnavailable() + + +class SecurityPage(JsonPage): + def get_accounts_url(self): + return Dict('url')(self.doc) + + +class ContractsPage(LoggedPage, HTMLPage): + pass + + +ACCOUNT_TYPES = { + 'CCHQ': Account.TYPE_CHECKING, # par + 'CCOU': Account.TYPE_CHECKING, # pro + 'AUTO ENTRP': Account.TYPE_CHECKING, # pro + 'DEVISE USD': Account.TYPE_CHECKING, + 'EKO': Account.TYPE_CHECKING, + 'DAV NANTI': Account.TYPE_SAVINGS, + 'LIV A': Account.TYPE_SAVINGS, + 'LIV A ASS': Account.TYPE_SAVINGS, + 'LDD': Account.TYPE_SAVINGS, + 'PEL': Account.TYPE_SAVINGS, + 'CEL': Account.TYPE_SAVINGS, + 'CODEBIS': Account.TYPE_SAVINGS, + 'LJMO': Account.TYPE_SAVINGS, + 'CSL': Account.TYPE_SAVINGS, + 'LEP': Account.TYPE_SAVINGS, + 'LEF': Account.TYPE_SAVINGS, + 'TIWI': Account.TYPE_SAVINGS, + 'CSL LSO': Account.TYPE_SAVINGS, + 'CSL CSP': Account.TYPE_SAVINGS, + 'ESPE INTEG': Account.TYPE_SAVINGS, + 'DAV TIGERE': Account.TYPE_SAVINGS, + 'CPTEXCPRO': Account.TYPE_SAVINGS, + 'CPTEXCENT': Account.TYPE_SAVINGS, + 'PRET PERSO': Account.TYPE_LOAN, + 'P. ENTREPR': Account.TYPE_LOAN, + 'P. HABITAT': Account.TYPE_LOAN, + 'PRET 0%': Account.TYPE_LOAN, + 'INV PRO': Account.TYPE_LOAN, + 'TRES. PRO': Account.TYPE_LOAN, + 'PEA': Account.TYPE_PEA, + 'PEAP': Account.TYPE_PEA, + 'DAV PEA': Account.TYPE_PEA, + 'CPS': Account.TYPE_MARKET, + 'TITR': Account.TYPE_MARKET, + 'TITR CTD': Account.TYPE_MARKET, + 'PVERT VITA': Account.TYPE_PERP, + 'réserves de crédit': Account.TYPE_CHECKING, + 'prêts personnels': Account.TYPE_LOAN, + 'crédits immobiliers': Account.TYPE_LOAN, + 'épargne disponible': Account.TYPE_SAVINGS, + 'épargne à terme': Account.TYPE_DEPOSIT, + 'épargne boursière': Account.TYPE_MARKET, + 'assurance vie et capitalisation': Account.TYPE_LIFE_INSURANCE, + 'PRED': Account.TYPE_LIFE_INSURANCE, + 'PREDI9 S2': Account.TYPE_LIFE_INSURANCE, + 'V.AVENIR': Account.TYPE_LIFE_INSURANCE, + 'FLORIA': Account.TYPE_LIFE_INSURANCE, + 'ATOUT LIB': Account.TYPE_REVOLVING_CREDIT, +} + + +class AccountsPage(LoggedPage, JsonPage): + def build_doc(self, content): + # Store the HTML doc to count the number of spaces + self.html_doc = HTMLPage(self.browser, self.response).doc + + # Transform the HTML tag containing the accounts list into a JSON + raw = re.search("syntheseController\.init\((.*)\)'>", content).group(1) + d = json.JSONDecoder() + # De-comment this line to debug the JSON accounts: + # print json.dumps(d.raw_decode(raw)[0]) + return d.raw_decode(raw)[0] + + def count_spaces(self): + # The total number of spaces corresponds to the number + # of available space choices plus the one we are on now: + return len(self.html_doc.xpath('//div[@class="HubAccounts-content"]/a')) + 1 + + def get_owner_type(self): + OWNER_TYPES = { + 'PARTICULIER': AccountOwnerType.PRIVATE, + 'PROFESSIONNEL': AccountOwnerType.ORGANIZATION, + 'ASSOC_CA_MODERE': AccountOwnerType.ORGANIZATION, + } + return OWNER_TYPES.get(Dict('marche')(self.doc), NotAvailable) + + @method + class get_main_account(ItemElement): + klass = Account + + obj_id = CleanText(Dict('comptePrincipal/numeroCompte')) + obj_number = CleanText(Dict('comptePrincipal/numeroCompte')) + obj_label = CleanText(Dict('comptePrincipal/libelleProduit')) + obj_balance = Eval(float_to_decimal, Dict('comptePrincipal/solde')) + obj_currency = CleanCurrency(Dict('comptePrincipal/idDevise')) + obj__index = Dict('comptePrincipal/index') + obj__category = None # Main accounts have no category + obj__id_element_contrat = CleanText(Dict('comptePrincipal/idElementContrat')) + + def obj_type(self): + _type = Map(CleanText(Dict('comptePrincipal/libelleUsuelProduit')), ACCOUNT_TYPES, Account.TYPE_UNKNOWN)(self) + if _type == Account.TYPE_UNKNOWN: + self.logger.warning('We got an untyped account: please add "%s" to ACCOUNT_TYPES.' % CleanText(Dict('comptePrincipal/libelleUsuelProduit'))(self)) + return _type + + class obj__cards(DictElement): + item_xpath = 'comptePrincipal/cartesDD' + + class item(ItemElement): + klass = Account + + def obj_id(self): + return CleanText(Dict('idCarte'))(self).replace(' ', '') + + obj_label = Format('Carte %s %s', Field('id'), CleanText(Dict('titulaire'))) + obj_type = Account.TYPE_CARD + obj_coming = Eval(float_to_decimal, Dict('encoursCarteM')) + obj_balance = CleanDecimal(0) + obj__index = Dict('index') + obj__category = None + + @method + class iter_accounts(DictElement): + item_xpath = 'grandesFamilles/*/elementsContrats' + + class item(ItemElement): + IGNORED_ACCOUNTS = ("MES ASSURANCES",) + + klass = Account + + obj_id = CleanText(Dict('numeroCompteBam')) + obj_number = CleanText(Dict('numeroCompteBam')) + obj_label = CleanText(Dict('libelleProduit')) + obj_currency = CleanCurrency(Dict('idDevise')) + obj__index = Dict('index') + obj__category = Dict('grandeFamilleProduitCode', default=None) + obj__id_element_contrat = CleanText(Dict('idElementContrat')) + + def obj_type(self): + _type = Map(CleanText(Dict('libelleUsuelProduit')), ACCOUNT_TYPES, Account.TYPE_UNKNOWN)(self) + if _type == Account.TYPE_UNKNOWN: + self.logger.warning('We got an untyped account: please add "%s" to ACCOUNT_TYPES.' % CleanText(Dict('libelleUsuelProduit'))(self)) + return _type + + def obj_balance(self): + balance = Dict('solde', default=None)(self) + if balance: + return Eval(float_to_decimal, balance)(self) + # We will fetch the balance with account_details + return NotAvailable + + def condition(self): + # Ignore insurances (plus they all have identical IDs) + return CleanText(Dict('familleProduit/libelle', default=''))(self) not in self.IGNORED_ACCOUNTS + + +class AccountDetailsPage(LoggedPage, JsonPage): + def get_account_balances(self): + account_balances = {} + for el in self.doc: + value = el.get('solde', el.get('encoursActuel', el.get('valorisationContrat', el.get('montantRestantDu', el.get('capitalDisponible'))))) + assert value is not None, 'Could not find the account balance' + account_balances[Dict('numeroCompte')(el)] = float_to_decimal(value) + return account_balances + + def get_loan_ids(self): + loan_ids = {} + for el in self.doc: + if el.get('numeroCredit'): + # Loans + loan_ids[Dict('numeroCompte')(el)] = Dict('numeroCredit')(el) + elif el.get('numeroContrat'): + # Revolving credits + loan_ids[Dict('numeroCompte')(el)] = Dict('numeroContrat')(el) + return loan_ids + + +class IbanPage(LoggedPage, JsonPage): + def get_iban(self): + return Dict('ibanData/ibanCode', default=NotAvailable)(self.doc) + + +class HistoryPage(LoggedPage, JsonPage): + pass + + +class InvestmentPage(LoggedPage, JsonPage): + pass + + +class ProfilePage(LoggedPage, JsonPage): + pass \ No newline at end of file diff --git a/modules/cragr/proxy_browser.py b/modules/cragr/proxy_browser.py new file mode 100644 index 0000000000000000000000000000000000000000..d4f21cbaeaedd0ecbe80448b368e8c7218d83836 --- /dev/null +++ b/modules/cragr/proxy_browser.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012-2019 Romain Bignon +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.browser.switch import SwitchingBrowser + +from .api.browser import CragrAPI +from .web.browser import Cragr + + +class ProxyBrowser(SwitchingBrowser): + BROWSERS = { + 'main': Cragr, + 'api': CragrAPI, + } diff --git a/modules/pradoepargne/pages.py b/modules/pradoepargne/pages.py new file mode 100644 index 0000000000000000000000000000000000000000..06a4fc60ab8fe89a0ce75e3670660d38040df586 --- /dev/null +++ b/modules/pradoepargne/pages.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012-2019 Budget Insight + + +from weboob.browser.pages import AbstractPage + + +class LoginPage(AbstractPage): + PARENT = 'cmes' + PARENT_URL = 'login' + BROWSER_ATTR = 'package.browser.CmesBrowser'