From 91f1f22f1795c283f1361589cddd8f105d1f8748 Mon Sep 17 00:00:00 2001 From: Quentin Defenouillere Date: Tue, 15 Oct 2019 16:57:04 +0200 Subject: [PATCH] [ganassurances] Make this module standalone with former code from groupama Groupama has shifted to the new website and is now a child of ganpatrimoine, however ganassurances (a child of groupama) is still using the old website, to I shifted all the code from groupama here so ganassurances can still work correctly on the old website. --- modules/ganassurances/browser.py | 128 ++++++++++++- modules/ganassurances/module.py | 46 ++--- modules/ganassurances/pages.py | 304 +++++++++++++++++++++++++++++++ 3 files changed, 448 insertions(+), 30 deletions(-) create mode 100644 modules/ganassurances/pages.py diff --git a/modules/ganassurances/browser.py b/modules/ganassurances/browser.py index e18e680eea..25638009e6 100644 --- a/modules/ganassurances/browser.py +++ b/modules/ganassurances/browser.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright(C) 2012-2019 Budget Insight +# Copyright(C) 2012 Romain Bignon # # This file is part of a weboob module. # @@ -17,19 +17,129 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . - import re -from weboob.browser import AbstractBrowser +from weboob.browser import LoginBrowser, URL, need_login +from weboob.exceptions import BrowserIncorrectPassword +from weboob.capabilities.bank import Account +from weboob.capabilities.base import empty + +from .pages import LoginPage, AccountsPage, TransactionsPage, AVAccountPage, AVHistoryPage, FormPage, IbanPage, AvJPage + +__all__ = ['GanAssurancesBrowser'] -class GanAssurancesBrowser(AbstractBrowser): - PARENT = 'groupama' - PARENT_ATTR = 'package.browser.GroupamaBrowser' - def __init__(self, website, *args, **kwargs): +class GanAssurancesBrowser(LoginBrowser): + BASEURL = 'https://espaceclient.ganassurances.fr' + + login = URL(r'https://authentification.(?P.*).fr/cas/login', LoginPage) + iban = URL(r'/wps/myportal/!ut/(.*)/\?paramNumCpt=(.*)', IbanPage) + accounts = URL(r'/wps/myportal/TableauDeBord', AccountsPage) + transactions = URL(r'/wps/myportal/!ut', TransactionsPage) + av_account_form = URL(r'/wps/myportal/assurancevie/', FormPage) + av_account = URL(r'https://secure-rivage.ganassurances.fr/contratVie.rivage.syntheseContratEparUc.gsi', + r'/front/vie/epargne/contrat/(.*)', AVAccountPage) + av_history = URL(r'https://secure-rivage.(?P.*).fr/contratVie.rivage.mesOperations.gsi', AVHistoryPage) + av_secondary = URL(r'/api/ecli/vie/contrats/(?P.*)', AvJPage) + + def __init__(self, *args, **kwargs): super(GanAssurancesBrowser, self).__init__(*args, **kwargs) - self.BASEURL = 'https://%s' % website - self.website = re.findall('espaceclient.(.*?).fr', self.BASEURL)[0] + self.website = 'ganassurances' self.domain = 'ganass' + def do_login(self): + login_url = 'https://espaceclient.%s.fr/login-%s' % (self.website, self.domain) + self.login.go(website=self.website, params={'service': login_url}) + self.page.login(self.username, self.password) + + if self.login.is_here(): + error_msg = self.page.get_error() + if error_msg and "LOGIN_ERREUR_MOT_PASSE_INVALIDE" in error_msg: + raise BrowserIncorrectPassword() + assert False, 'Unhandled error at login: %s' % error_msg + + # For life asssurance accounts, to get balance we use the link from the account. + # And to get history (or other) we need to use the link again but the link works only once. + # So we get balance only for iter_account to not use the new link each time. + @need_login + def get_accounts_list(self, balance=True, need_iban=False): + accounts = [] + self.accounts.stay_or_go() + for account in self.page.get_list(): + if account.type == Account.TYPE_LIFE_INSURANCE and balance: + assert empty(account.balance) + self.location(account._link) + if self.av_account_form.is_here(): + self.page.av_account_form() + account.balance, account.currency = self.page.get_av_balance() + # New page where some AV are stored + elif "front/vie/" in account._link: + link = re.search(r'contrat\/(.+)-Groupama', account._link) + if link: + self.av_secondary.go(id_contrat=link.group(1)) + account.balance, account.currency = self.page.get_av_balance() + + self.accounts.stay_or_go() + if account.balance or not balance: + if account.type != Account.TYPE_LIFE_INSURANCE and need_iban: + self.location(account._link) + if self.transactions.is_here() and self.page.has_iban(): + self.page.go_iban() + account.iban = self.page.get_iban() + accounts.append(account) + return accounts + + def _get_history(self, account): + if "front/vie" in account._link: + return [] + accounts = self.get_accounts_list(balance=False) + for a in accounts: + if a.id == account.id: + self.location(a._link) + if a.type == Account.TYPE_LIFE_INSURANCE: + if not self.page.av_account_form(): + self.logger.warning('history form not found for %s', account) + return [] + self.av_history.go(website=self.website) + return self.page.get_av_history() + assert self.transactions.is_here() + return self.page.get_history(accid=account.id) + return [] + + # Duplicate line in case of arbitration because the site has only one line for the 2 transactions (debit and credit on the same line) + def get_history(self, account): + for tr in self._get_history(account): + yield tr + if getattr(tr, '_arbitration', False): + tr = tr.copy() + tr.amount = -tr.amount + yield tr + + def get_coming(self, account): + if account.type == Account.TYPE_LIFE_INSURANCE: + return [] + for a in self.get_accounts_list(): + if a.id == account.id: + self.location(a._link) + assert self.transactions.is_here() + link = self.page.get_coming_link() + if link is not None: + self.location(self.page.get_coming_link()) + assert self.transactions.is_here() + return self.page.get_history(accid=account.id) + return [] + + def get_investment(self, account): + if account.type != Account.TYPE_LIFE_INSURANCE: + return [] + for a in self.get_accounts_list(balance=False): + if a.id == account.id: + # There isn't any invest on AV having front/vie + # in theirs url + if "front/vie/" not in account._link: + self.location(a._link) + self.page.av_account_form() + if self.av_account.is_here(): + return self.page.get_av_investments() + return [] diff --git a/modules/ganassurances/module.py b/modules/ganassurances/module.py index eb1d2b2cf0..8b2bf530b5 100644 --- a/modules/ganassurances/module.py +++ b/modules/ganassurances/module.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright(C) 2012-2013 Romain Bignon +# Copyright(C) 2012-2019 Romain Bignon # # This file is part of a weboob module. # @@ -17,14 +17,13 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . -from __future__ import unicode_literals - -from collections import OrderedDict +from __future__ import unicode_literals -from weboob.capabilities.bank import CapBank -from weboob.tools.backend import AbstractModule, BackendConfig -from weboob.tools.value import ValueBackendPassword, Value +from weboob.capabilities.bank import CapBankWealth, AccountNotFound +from weboob.capabilities.base import find_object +from weboob.tools.backend import Module, BackendConfig +from weboob.tools.value import Value, ValueBackendPassword from .browser import GanAssurancesBrowser @@ -32,31 +31,36 @@ __all__ = ['GanAssurancesModule'] -class GanAssurancesModule(AbstractModule, CapBank): +class GanAssurancesModule(Module, CapBankWealth): NAME = 'ganassurances' MAINTAINER = 'Romain Bignon' EMAIL = 'romain@weboob.org' VERSION = '1.6' DESCRIPTION = 'Gan Assurances' LICENSE = 'LGPLv3+' - website_choices = OrderedDict([(k, '%s (%s)' % (v, k)) for k, v in sorted({ - 'espaceclient.groupama.fr': 'Groupama Banque', - 'espaceclient.ganassurances.fr': 'Gan Assurances', - 'espaceclient.ganpatrimoine.fr': 'Gan Patrimoine', - }.items(), key=lambda k_v: (k_v[1], k_v[0]))]) CONFIG = BackendConfig( - Value('website', label='Banque', choices=website_choices, default='espaceclient.ganassurances.fr'), - ValueBackendPassword('login', label='Identifiant / N° Client ou Email ou Mobile', masked=False), - ValueBackendPassword('password', label='Mon mot de passe', regexp=r'^\d+$') + Value('login', label='Numéro client'), + ValueBackendPassword('password', label="Code d'accès") ) - - PARENT = 'groupama' BROWSER = GanAssurancesBrowser def create_default_browser(self): return self.create_browser( - self.config['website'].get(), self.config['login'].get(), - self.config['password'].get(), - weboob=self.weboob + self.config['password'].get() ) + + def iter_accounts(self): + return self.browser.get_accounts_list(need_iban=True) + + def get_account(self, _id): + return find_object(self.browser.get_accounts_list(need_iban=True), id=_id, error=AccountNotFound) + + def iter_history(self, account): + return self.browser.get_history(account) + + def iter_coming(self, account): + return self.browser.get_coming(account) + + def iter_investment(self, account): + return self.browser.get_investment(account) diff --git a/modules/ganassurances/pages.py b/modules/ganassurances/pages.py new file mode 100644 index 0000000000..54f65eb75d --- /dev/null +++ b/modules/ganassurances/pages.py @@ -0,0 +1,304 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012 Romain Bignon +# +# This file is part of a weboob module. +# +# This weboob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This weboob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this weboob module. If not, see . + + +from __future__ import unicode_literals + +import re +import requests +import ast + +from decimal import Decimal + +from weboob.browser.pages import HTMLPage, pagination, LoggedPage, FormNotFound, JsonPage +from weboob.browser.elements import method, TableElement, ItemElement +from weboob.browser.filters.standard import Env, CleanDecimal, CleanText, Date, Regexp, Eval, Field +from weboob.browser.filters.html import Attr, Link, TableCell +from weboob.browser.filters.javascript import JSVar +from weboob.capabilities.bank import Account, Investment +from weboob.capabilities.base import NotAvailable +from weboob.tools.capabilities.bank.transactions import FrenchTransaction +from weboob.tools.capabilities.bank.investments import is_isin_valid +from weboob.browser.filters.json import Dict + + +class LoginPage(HTMLPage): + def login(self, login, passwd): + tab = re.search(r'clavierAChristian = (\[[\d,\s]*\])', self.content).group(1) + number_list = ast.literal_eval(tab) + key_map = {} + for i, number in enumerate(number_list): + if number < 10: + key_map[number] = chr(ord('A') + i) + pass_string = ''.join(key_map[int(n)] for n in passwd) + form = self.get_form(name='loginForm') + form['username'] = login + form['password'] = pass_string + form.submit() + + def get_error(self): + return CleanText('//div[@id="msg"]')(self.doc) + + +class AccountsPage(LoggedPage, HTMLPage): + ACCOUNT_TYPES = { + 'Solde des comptes bancaires - Groupama Banque': Account.TYPE_CHECKING, + 'Solde des comptes bancaires': Account.TYPE_CHECKING, + 'Epargne bancaire constituée - Groupama Banque': Account.TYPE_SAVINGS, + 'Epargne bancaire constituée': Account.TYPE_SAVINGS, + 'Mes crédits': Account.TYPE_LOAN, + 'Assurance Vie': Account.TYPE_LIFE_INSURANCE, + 'Certificats Mutualistes': Account.TYPE_SAVINGS, + } + + ACCOUNT_TYPES2 = { + 'plan epargne actions': Account.TYPE_PEA, + } + + def get_list(self): + account_type = Account.TYPE_UNKNOWN + accounts = [] + + for tr in self.doc.xpath('//div[@class="finance"]/form/table[@class="ecli"]/tr'): + if tr.attrib.get('class', '') == 'entete': + account_type = self.ACCOUNT_TYPES.get(tr.find('th').text.strip(), Account.TYPE_UNKNOWN) + continue + + tds = tr.findall('td') + a = tds[0].find('a') + + # Skip accounts that can't be accessed + if a is None: + continue + + balance = tds[-1].text.strip() + + account = Account() + account.label = ' '.join([txt.strip() for txt in tds[0].itertext()]) + account.label = re.sub(r'[\s\xa0\u2022]+', ' ', account.label).strip() + + # take "N° (FOO123 456)" but "N° (FOO123) MR. BAR" + account.id = re.search(r'N° (\w+( \d+)*)', account.label).group(1).replace(' ', '') + account.number = account.id + account.type = account_type + + for patt, type in self.ACCOUNT_TYPES2.items(): + if patt in account.label.lower(): + account.type = type + break + + if balance: + account.balance = Decimal(FrenchTransaction.clean_amount(balance)) + account.currency = account.get_currency(balance) + + if 'onclick' in a.attrib: + m = re.search(r"javascript:submitForm\(([\w]+),'([^']+)'\);", a.attrib['onclick']) + if not m: + self.logger.warning('Unable to find link for %r', account.label) + account._link = None + else: + account._link = m.group(2) + else: + account._link = a.attrib['href'].strip() + + if accounts and accounts[-1].label == account.label and account.type == Account.TYPE_PEA: + self.logger.warning('%s seems to be a duplicate of %s, skipping', account, accounts[-1]) + continue + accounts.append(account) + return accounts + + +class Transaction(FrenchTransaction): + PATTERNS = [ + (re.compile(r'^Facture (?P
\d{2})/(?P\d{2})-(?P.*) carte .*'), FrenchTransaction.TYPE_CARD), + (re.compile(r'^(Prlv( de)?|Ech(éance|\.)) (?P.*)'), FrenchTransaction.TYPE_ORDER), + (re.compile(r'^(Vir|VIR)( de)? (?P.*)'), FrenchTransaction.TYPE_TRANSFER), + (re.compile(r'^CHEQUE.*? (N° \w+)?$'), FrenchTransaction.TYPE_CHECK), + (re.compile(r'^Cotis(ation)? (?P.*)'), FrenchTransaction.TYPE_BANK), + (re.compile(r'(?PInt .*)'), FrenchTransaction.TYPE_BANK), + (re.compile(r'^SOUSCRIPTION|REINVESTISSEMENT'), FrenchTransaction.TYPE_DEPOSIT), + ] + + +class TransactionsPage(HTMLPage): + logged = True + + @pagination + @method + class get_history(Transaction.TransactionsElement): + head_xpath = '//table[@id="releve_operation"]//tr/th' + item_xpath = '//table[@id="releve_operation"]//tr' + + col_date = ['Date opé', 'Date', 'Date d\'opé', 'Date opération'] + col_vdate = ['Date valeur'] + col_credit = ['Crédit', 'Montant', 'Valeur'] + col_debit = ['Débit'] + + def next_page(self): + url = Attr('//a[contains(text(), "Page suivante")]', 'onclick', default=None)(self) + if url: + m = re.search(r'\'([^\']+).*([\d]+)', url) + return requests.Request("POST", m.group(1), + data={ + 'numCompte': Env('accid')(self), + 'vue': "ReleveOperations", + 'tri': "DateOperation", + 'sens': "DESC", + 'page': m.group(2), + 'nb_element': "25"} + ) + + class item(Transaction.TransactionElement): + def condition(self): + return len(self.el.xpath('./td')) > 3 + + def get_coming_link(self): + try: + a = self.doc.getroot().cssselect('div#sous_nav ul li a.bt_sans_off')[0] + except IndexError: + return None + return re.sub(r'[\s]+', '', a.attrib['href']) + + def has_iban(self): + return self.doc.xpath('//a[@class="rib"]') + + def go_iban(self): + js_event = Attr("//a[@class='rib']", 'onclick')(self.doc) + m = re.search(r'envoyer(.*);', js_event) + iban_params = ast.literal_eval(m.group(1)) + self.browser.location("{}?paramNumCpt={}".format(iban_params[1], iban_params[0])) + + +class IbanPage(LoggedPage, HTMLPage): + + def get_iban(self): + return CleanText('(//b[contains(text(), "IBAN")])[1]/../text()')(self.doc) + + +class AVAccountPage(LoggedPage, HTMLPage): + """ + Get balance + + :return: decimal balance, currency + :rtype: tuple + """ + def get_av_balance(self): + balance_xpath = '//p[contains(text(), "Épargne constituée")]/span' + balance = CleanDecimal(balance_xpath)(self.doc) + currency = Account.get_currency(CleanText(balance_xpath)(self.doc)) + return balance, currency + + @method + class get_av_investments(TableElement): + item_xpath = '//table[@id="repartition_epargne3"]/tr[position() > 1]' + head_xpath = '//table[@id="repartition_epargne3"]/tr/th[position() > 1]' + + col_quantity = 'Nombre d’unités de compte' + col_unitvalue = "Valeur de l’unité de compte" + col_valuation = 'Épargne constituée en euros' + col_portfolio_share = 'Répartition %' + + class item(ItemElement): + klass = Investment + + def condition(self): + return (CleanText('./th')(self) != 'Total épargne constituée') and ('Détail' not in Field('label')(self)) + + obj_label = CleanText('./th') + obj_quantity = CleanDecimal(TableCell('quantity'), default=NotAvailable) + obj_unitvalue = CleanDecimal(TableCell('unitvalue'), default=NotAvailable) + obj_valuation = CleanDecimal(TableCell('valuation'), default=NotAvailable) + obj_portfolio_share = Eval(lambda x: x / 100, CleanDecimal(TableCell('portfolio_share'))) + + def obj_code(self): + code = Regexp(Link('./th/a', default=''), r'isin=(\w+)|/(\w+)\.pdf', default=NotAvailable)(self) + return code if is_isin_valid(code) else NotAvailable + + def obj_code_type(self): + return Investment.CODE_TYPE_ISIN if is_isin_valid(Field('code')(self)) else NotAvailable + + +class AvJPage(LoggedPage, JsonPage): + def get_av_balance(self): + balance = CleanDecimal(Dict('montant'))(self.doc) + currency = "EUR" + return balance, currency + + +class AVHistoryPage(LoggedPage, HTMLPage): + @method + class get_av_history(TableElement): + item_xpath = '//table[@id="enteteTableSupports"]/tbody/tr' + head_xpath = '//table[@id="enteteTableSupports"]/thead/tr/th' + + col_date = 'Date' + col_label = 'Type de mouvement' + col_debit = 'Montant Désinvesti' + col_credit = ['Montant investi', 'Montant Net Perçu'] + # There is several types of life insurances, so multiple columns + col_credit2 = ['Montant Brut Versé'] + + class item(ItemElement): + klass = Transaction + + def condition(self): + return CleanText(TableCell('date'))(self) != 'en cours' + + obj_label = CleanText(TableCell('label')) + obj_type = Transaction.TYPE_BANK + obj_date = Date(CleanText(TableCell('date')), dayfirst=True) + obj__arbitration = False + + def obj_amount(self): + credit = CleanDecimal(TableCell('credit'), default=Decimal(0))(self) + # Different types of life insurances, use different columns. + if TableCell('debit', default=None)(self): + debit = CleanDecimal(TableCell('debit'), default=Decimal(0))(self) + # In case of financial arbitration, both columns are equal + if credit and debit: + assert credit == debit + self.obj._arbitration = True + return credit + else: + return credit - abs(debit) + else: + credit2 = CleanDecimal(TableCell('credit2'), default=Decimal(0))(self) + assert not (credit and credit2) + return credit + credit2 + + +class FormPage(LoggedPage, HTMLPage): + def get_av_balance(self): + balance_xpath = '//p[contains(text(), "montant de votre épargne")]' + balance = CleanDecimal(Regexp(CleanText(balance_xpath), r'est de ([\s\d,]+)', default=NotAvailable), + replace_dots=True, default=NotAvailable)(self.doc) + currency = Account.get_currency(CleanText(balance_xpath)(self.doc)) + return balance, currency + + def av_account_form(self): + try: + form = self.get_form(id="formGoToRivage") + form['gfr_numeroContrat'] = JSVar(var='numContrat').filter(CleanText('//script[contains(text(), "var numContrat")]')(self.doc)) + form['gfr_data'] = JSVar(var='pCryptage').filter(CleanText('//script[contains(text(), "var pCryptage")]')(self.doc)) + form['gfr_adrSite'] = 'https://espaceclient.%s.fr' % self.browser.website + form.url = 'https://secure-rivage.%s.fr/contratVie.rivage.syntheseContratEparUc.gsi' % self.browser.website + form.submit() + return True + except FormNotFound: + return False -- GitLab