From f0f3aceafa2f60260c2bcda77f7733b094dedf4e Mon Sep 17 00:00:00 2001 From: Quentin Defenouillere Date: Mon, 15 Jul 2019 18:19:47 +0200 Subject: [PATCH] [caissedepargne] Handled new API space for Life Insurances (invests & history) The website has a new web space for life insurance and capitalisation contracts that uses an API for invests and transactions. Closes: 12267@zendesk, 12411@zendesk --- modules/caissedepargne/browser.py | 44 +++++++------ modules/caissedepargne/pages.py | 104 +++++++++++++++++++++--------- 2 files changed, 98 insertions(+), 50 deletions(-) diff --git a/modules/caissedepargne/browser.py b/modules/caissedepargne/browser.py index 54f3401fd2..87a2b4bfcc 100644 --- a/modules/caissedepargne/browser.py +++ b/modules/caissedepargne/browser.py @@ -46,8 +46,7 @@ from weboob.tools.decorators import retry from .pages import ( - IndexPage, ErrorPage, MarketPage, LifeInsurance, GarbagePage, - MessagePage, LoginPage, + IndexPage, ErrorPage, MarketPage, LifeInsurance, LifeInsuranceHistory, LifeInsuranceInvestments, GarbagePage, MessagePage, LoginPage, TransferPage, ProTransferPage, TransferConfirmPage, TransferSummaryPage, ProTransferConfirmPage, ProTransferSummaryPage, ProAddRecipientOtpPage, ProAddRecipientPage, SmsPage, SmsPageOption, SmsRequest, AuthentPage, RecipientPage, CanceledAuth, CaissedepargneKeyboard, @@ -103,18 +102,21 @@ class CaisseEpargne(LoginBrowser, StatesMixin): natixis_redirect = URL(r'/NaAssuranceRedirect/NaAssuranceRedirect.aspx', r'https://www.espace-assurances.caisse-epargne.fr/espaceinternet-ce/views/common/routage-itce.xhtml\?windowId=automatedEntryPoint', NatixisRedirectPage) - life_insurance = URL('https://.*/Assurance/Pages/Assurance.aspx', - 'https://www.extranet2.caisse-epargne.fr.*', LifeInsurance) - natixis_life_ins_his = URL('https://www.espace-assurances.caisse-epargne.fr/espaceinternet-ce/rest/v2/contratVie/load-operation/(?P\w+)/(?P\w+)/(?P)', NatixisLIHis) - natixis_life_ins_inv = URL('https://www.espace-assurances.caisse-epargne.fr/espaceinternet-ce/rest/v2/contratVie/load/(?P\w+)/(?P\w+)/(?P)', NatixisLIInv) - message = URL('https://www.caisse-epargne.offrebourse.com/DetailMessage\?refresh=O', MessagePage) - garbage = URL('https://www.caisse-epargne.offrebourse.com/Portefeuille', - 'https://www.caisse-epargne.fr/particuliers/.*/emprunter.aspx', - 'https://.*/particuliers/emprunter.*', - 'https://.*/particuliers/epargner.*', GarbagePage) - sms = URL('https://www.icgauth.caisse-epargne.fr/dacswebssoissuer/AuthnRequestServlet', SmsPage) - sms_option = URL('https://www.icgauth.caisse-epargne.fr/dacstemplate-SOL/index.html\?transactionID=.*', SmsPageOption) - request_sms = URL('https://www.icgauth.caisse-epargne.fr/dacsrest/api/v1u0/transaction/(?P)', SmsRequest) + life_insurance_history = URL(r'https://www.extranet2.caisse-epargne.fr/cin-front/contrats/evenements', LifeInsuranceHistory) + life_insurance_investments = URL(r'https://www.extranet2.caisse-epargne.fr/cin-front/contrats/details', LifeInsuranceInvestments) + life_insurance = URL(r'https://.*/Assurance/Pages/Assurance.aspx', + r'https://www.extranet2.caisse-epargne.fr.*', LifeInsurance) + natixis_life_ins_his = URL(r'https://www.espace-assurances.caisse-epargne.fr/espaceinternet-ce/rest/v2/contratVie/load-operation/(?P\w+)/(?P\w+)/(?P)', NatixisLIHis) + natixis_life_ins_inv = URL(r'https://www.espace-assurances.caisse-epargne.fr/espaceinternet-ce/rest/v2/contratVie/load/(?P\w+)/(?P\w+)/(?P)', NatixisLIInv) + message = URL(r'https://www.caisse-epargne.offrebourse.com/DetailMessage\?refresh=O', MessagePage) + garbage = URL(r'https://www.caisse-epargne.offrebourse.com/Portefeuille', + r'https://www.caisse-epargne.fr/particuliers/.*/emprunter.aspx', + r'https://.*/particuliers/emprunter.*', + r'https://.*/particuliers/epargner.*', GarbagePage) + sms = URL(r'https://www.icgauth.caisse-epargne.fr/dacswebssoissuer/AuthnRequestServlet', SmsPage) + sms_option = URL(r'https://www.icgauth.caisse-epargne.fr/dacstemplate-SOL/index.html\?transactionID=.*', SmsPageOption) + request_sms = URL(r'https://www.icgauth.caisse-epargne.fr/dacsrest/api/v1u0/transaction/(?P)', SmsRequest) + __states__ = ('BASEURL', 'multi_type', 'typeAccount', 'is_cenet_website', 'recipient_form', 'is_send_sms') # Accounts managed in life insurance space (not in linebourse) @@ -647,7 +649,7 @@ def _get_history_invests(self, account): self.home.go() self.page.go_history(account._info) - if account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_PERP): + if account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_CAPITALISATION, Account.TYPE_PERP): if self.page.is_account_inactive(account.id): self.logger.warning('Account %s %s is inactive.' % (account.label, account.id)) return [] @@ -683,8 +685,9 @@ def _get_history_invests(self, account): # life insurance website is not always available raise BrowserUnavailable() self.page.submit() - self.location('https://www.extranet2.caisse-epargne.fr%s' % self.page.get_cons_histo()) - + self.life_insurance_history.go() + # Life insurance transactions are not sorted by date in the JSON + return sorted_transactions(self.page.iter_history()) except (IndexError, AttributeError) as e: self.logger.error(e) return [] @@ -705,7 +708,7 @@ def match_cb(tr): if not hasattr(account, '_info'): raise NotImplementedError - if account.type is Account.TYPE_LIFE_INSURANCE and 'measure_id' not in account._info: + if account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_CAPITALISATION) and 'measure_id' not in account._info: return self._get_history_invests(account) if account.type in (Account.TYPE_MARKET, Account.TYPE_PEA): self.page.go_history(account._info) @@ -810,16 +813,15 @@ def get_investment(self, account): try: self.page.go_life_insurance(account) - if not self.market.is_here() and not self.message.is_here(): # life insurance website is not always available raise BrowserUnavailable() - self.page.submit() - self.location('https://www.extranet2.caisse-epargne.fr%s' % self.page.get_cons_repart()) + self.life_insurance_investments.go() except (IndexError, AttributeError) as e: self.logger.error(e) return + if self.garbage.is_here(): self.page.come_back() return diff --git a/modules/caissedepargne/pages.py b/modules/caissedepargne/pages.py index fa4be334c2..3eb093ce8a 100644 --- a/modules/caissedepargne/pages.py +++ b/modules/caissedepargne/pages.py @@ -161,7 +161,6 @@ class Transaction(FrenchTransaction): (re.compile(r'^RACHAT PARTIEL', re.IGNORECASE), FrenchTransaction.TYPE_BANK), ] - class IndexPage(LoggedPage, HTMLPage): ACCOUNT_TYPES = {u'Epargne liquide': Account.TYPE_SAVINGS, u'Compte Courant': Account.TYPE_CHECKING, @@ -1098,42 +1097,89 @@ def come_back(self): class LifeInsurance(MarketPage): - def get_cons_repart(self): - return self.doc.xpath('//tr[@id="sousMenuConsultation3"]/td/div/a')[0].attrib['href'] + pass - def get_cons_histo(self): - return self.doc.xpath('//tr[@id="sousMenuConsultation4"]/td/div/a')[0].attrib['href'] - def iter_history(self): - for tr in self.doc.xpath(u'//table[@class="boursedetail"]/tbody/tr[td]'): - t = Transaction() +class LifeInsuranceHistory(LoggedPage, JsonPage): + @method + class iter_history(DictElement): - t.label = CleanText('.')(tr.xpath('./td[2]')[0]) - t.date = Date(dayfirst=True).filter(CleanText('.')(tr.xpath('./td[1]')[0])) - t.amount = self.parse_decimal(tr.xpath('./td[3]')[0]) + def find_elements(self): + return self.el or [] # JSON contains 'null' if no transaction - yield t + class item(ItemElement): + klass = Transaction - def iter_investment(self): - for tr in self.doc.xpath(u'//table[@class="boursedetail"]/tr[@class and not(@class="total")]'): + obj_raw = Transaction.Raw(Dict('type/libelleLong')) + obj_amount = Eval(float_to_decimal, Dict('montantBrut/valeur')) - inv = Investment() - libelle = CleanText('.')(tr.xpath('./td[1]')[0]).split(' ') - inv.label, inv.code = self.split_label_code(libelle) - inv.code_type = Investment.CODE_TYPE_ISIN if is_isin_valid(inv.code) else NotAvailable - inv.quantity = self.parse_decimal(tr.xpath('./td[2]')[0]) - inv.unitvalue = self.parse_decimal(tr.xpath('./td[3]')[0]) - date = CleanText('.')(tr.xpath('./td[4]')[0]) - inv.vdate = Date(dayfirst=True).filter(date) if date and date != '-' else NotAvailable - inv.valuation = self.parse_decimal(tr.xpath('./td[5]')[0]) - inv.diff_percent = self.parse_decimal(tr.xpath('./td[6]')[0], percentage=True) + def obj_date(self): + date = Dict('dateTraitement')(self) + if date: + return datetime.fromtimestamp(date/1000) + return NotAvailable - yield inv + obj_rdate = obj_date - def split_label_code(self, libelle): - if is_isin_valid(libelle[-1]): - return ' '.join(libelle[:-1]), libelle[-1] - return ' '.join(libelle), NotAvailable + def obj_vdate(self): + vdate = Dict('dateEffet')(self) + if vdate: + return datetime.fromtimestamp(vdate/1000) + return NotAvailable + + +class LifeInsuranceInvestments(LoggedPage, JsonPage): + @method + class iter_investment(DictElement): + + def find_elements(self): + return self.el['repartition']['supports'] or [] # JSON contains 'null' if no investment + + class item(ItemElement): + klass = Investment + + # For whatever reason some labels start with a '.' (for example '.INVESTMENT') + obj_label = CleanText(Dict('libelleSupport'), replace=[('.', '')]) + obj_valuation = Eval(float_to_decimal, Dict('montantBrutInvesti/valeur')) + obj_portfolio_share = Eval(lambda x: float_to_decimal(x)/100, Dict('pourcentageInvesti')) + + # Note: the following attributes are not available for euro funds + def obj_vdate(self): + vdate = Dict('cotation/date')(self) + if vdate: + return datetime.fromtimestamp(vdate/1000) + return NotAvailable + + def obj_quantity(self): + if Dict('nombreParts')(self): + return Eval(float_to_decimal, Dict('nombreParts'))(self) + return NotAvailable + + def obj_diff(self): + if Dict('tauxPlusValue')(self): + return Eval(float_to_decimal, Dict('tauxPlusValue'))(self) + return NotAvailable + + def obj_diff_percent(self): + if Dict('tauxPlusValue')(self): + return Eval(lambda x: float_to_decimal(x)/100, Dict('tauxPlusValue'))(self) + return NotAvailable + + def obj_unitvalue(self): + if Dict('cotation/montant')(self): + return Eval(float_to_decimal, Dict('cotation/montant/valeur'))(self) + return NotAvailable + + def obj_code(self): + code = Dict('codeISIN')(self) + if is_isin_valid(code): + return code + return NotAvailable + + def obj_code_type(self): + if Field('code')(self) == NotAvailable: + return NotAvailable + return Investment.CODE_TYPE_ISIN class NatixisLIHis(LoggedPage, JsonPage): -- GitLab