From 7ca85b03554c85aa9b138f9fa80db45820d4211d Mon Sep 17 00:00:00 2001 From: Sylvie Ye Date: Tue, 30 Apr 2019 10:26:40 +0200 Subject: [PATCH] [lcl] retrieve all life insurance accounts on external website Go on lcl external life insurance to get accounts for account with "routage" option For life insurance accounts with "popup" option, stay on main website --- modules/lcl/browser.py | 49 ++++++++++++++++++++++------ modules/lcl/pages.py | 73 ++++++++++++++++++++++++++++++++++-------- 2 files changed, 100 insertions(+), 22 deletions(-) diff --git a/modules/lcl/browser.py b/modules/lcl/browser.py index bd61755104..48743a717f 100644 --- a/modules/lcl/browser.py +++ b/modules/lcl/browser.py @@ -34,7 +34,7 @@ from .pages import LoginPage, AccountsPage, AccountHistoryPage, \ CBListPage, CBHistoryPage, ContractsPage, ContractsChoicePage, BoursePage, \ - AVPage, AVDetailPage, DiscPage, NoPermissionPage, RibPage, \ + AVListPage, AVPage, AVDetailPage, DiscPage, NoPermissionPage, RibPage, \ HomePage, LoansPage, TransferPage, AddRecipientPage, \ RecipientPage, RecipConfirmPage, SmsPage, RecipRecapPage, \ LoansProPage, Form2Page, DocumentsPage, ClientPage, SendTokenPage, \ @@ -84,7 +84,7 @@ class LCLBrowser(LoginBrowser, StatesMixin): r'https://assurance-vie-et-prevoyance.secure.lcl.fr/filiale/ServletReroutageCookie', '/outil/UAUT/RetourPartenaire/retourCar', DiscPage) - form2 = URL(r'/outil/UWVI/Routage/', Form2Page) + form2 = URL(r'/outil/UWVI/Routage', Form2Page) send_token = URL('/outil/UWVI/AssuranceVie/envoyerJeton', SendTokenPage) calie = URL('https://www.my-calie.fr/FO.HoldersWebSite/Disclaimer/Disclaimer.aspx.*', 'https://www.my-calie.fr/FO.HoldersWebSite/Contract/ContractDetails.aspx.*', @@ -94,9 +94,10 @@ class LCLBrowser(LoginBrowser, StatesMixin): '/outil/UWVI/AssuranceVie/accesDetail.*', AVPage) + av_list = URL('https://assurance-vie-et-prevoyance.secure.lcl.fr/rest/assurance/synthesePartenaire', AVListPage) avdetail = URL('https://assurance-vie-et-prevoyance.secure.lcl.fr/consultation/epargne', AVDetailPage) av_history = URL('https://assurance-vie-et-prevoyance.secure.lcl.fr/rest/assurance/historique', AVHistoryPage) - av_investments = URL('https://assurance-vie-et-prevoyance.secure.lcl.fr/rest/detailEpargne/contrat', AVInvestmentsPage) + av_investments = URL('https://assurance-vie-et-prevoyance.secure.lcl.fr/rest/detailEpargne/contrat/(?P\w+)', AVInvestmentsPage) loans = URL('/outil/UWCR/SynthesePar/', LoansPage) loans_pro = URL('/outil/UWCR/SynthesePro/', LoansProPage) @@ -179,6 +180,24 @@ def connexion_bourse(self): def deconnexion_bourse(self): self.disc.stay_or_go() + @need_login + def go_life_insurance_website(self): + self.assurancevie.stay_or_go() + life_insurance_routage_url = self.page.get_routage_url() + if life_insurance_routage_url: + self.location(life_insurance_routage_url) + self.av_list.go() + + @need_login + def update_life_insurance_account(self, life_insurance): + self.av_investments.go(life_insurance_id=life_insurance.id) + return self.page.update_life_insurance_account(life_insurance) + + @need_login + def go_back_from_life_insurance_website(self): + self.avdetail.stay_or_go() + self.page.come_back() + def select_contract(self, id_contract): if self.current_contract and id_contract != self.current_contract: # when we go on bourse page, we can't change contract anymore... we have to logout. @@ -215,20 +234,27 @@ def set_deposit_account_id(self, account): @need_login def get_accounts(self): - self.assurancevie.stay_or_go() # This is required in case the browser is left in the middle of add_recipient and the session expires. if self.login.is_here(): return self.get_accounts_list() - if self.accounts_list is None: - self.accounts_list = [] + # retrieve life insurance accounts + self.assurancevie.stay_or_go() if self.no_perm.is_here(): self.logger.warning('Life insurances are unavailable.') else: - for a in self.page.get_list(): + for a in self.page.get_popup_life_insurance(): self.update_accounts(a) - - self.accounts.stay_or_go() + # retrieve life insurance on special lcl life insurance website + if self.page.is_website_life_insurance(): + self.go_life_insurance_website() + for life_insurance in self.page.iter_life_insurance(): + life_insurance = self.update_life_insurance_account(life_insurance) + self.update_accounts(life_insurance) + self.go_back_from_life_insurance_website() + + # retrieve accounts on main page + self.accounts.go() for a in self.page.get_list(): if not self.check_accounts(a): continue @@ -246,6 +272,7 @@ def get_accounts(self): a.iban = iban if iban is not None else NotAvailable self.update_accounts(a) + # retrieve loans accounts self.loans.stay_or_go() if self.no_perm.is_here(): self.logger.warning('Loans are unavailable.') @@ -253,6 +280,7 @@ def get_accounts(self): for a in self.page.get_list(): self.update_accounts(a) + # retrieve pro loans accounts self.loans_pro.stay_or_go() if self.no_perm.is_here(): self.logger.warning('Loans are unavailable.') @@ -267,6 +295,7 @@ def get_accounts(self): # Disconnecting from bourse portal before returning account list # to be sure that we are on the banque portal + # retrieve deposit accounts self.deposit.stay_or_go() if self.no_perm.is_here(): self.logger.warning('Deposits are unavailable.') @@ -280,6 +309,8 @@ def get_accounts(self): @need_login def get_accounts_list(self): if self.accounts_list is None: + self.accounts_list = [] + if self.contracts and self.current_contract: for id_contract in self.contracts: self.select_contract(id_contract) diff --git a/modules/lcl/pages.py b/modules/lcl/pages.py index d0bfe586db..eb50aabff3 100644 --- a/modules/lcl/pages.py +++ b/modules/lcl/pages.py @@ -41,7 +41,8 @@ from weboob.browser.pages import LoggedPage, HTMLPage, JsonPage, FormNotFound, pagination from weboob.browser.filters.html import Attr, Link, TableCell, AttributeNotFound from weboob.browser.filters.standard import ( - CleanText, Field, Regexp, Format, Date, CleanDecimal, Map, AsyncLoad, Async, Env, Slugify, BrowserURL, Eval, + CleanText, Field, Regexp, Format, Date, CleanDecimal, Map, AsyncLoad, Async, Env, + Slugify, BrowserURL, Eval, Lower, ) from weboob.browser.filters.json import Dict from weboob.exceptions import BrowserUnavailable, BrowserIncorrectPassword @@ -687,8 +688,18 @@ def get_error_msg(self): class AVPage(LoggedPage, HTMLPage): + def get_routage_url(self): + for account in self.doc.xpath('//table[@class]/tbody/tr'): + if account.xpath('.//td[has-class("nomContrat")]//a[has-class("routageCAR")]'): + return Link('.//td[has-class("nomContrat")]//a[has-class("routageCAR")]')(account) + + def is_website_life_insurance(self): + # no need specific account to go on life insurance external website + # because we just need to go on life insurance external website + return bool(self.get_routage_url()) + @method - class get_list(ListElement): + class get_popup_life_insurance(ListElement): item_xpath = '//table[@class]/tbody/tr' class item(ItemElement): @@ -698,7 +709,8 @@ def condition(self): if self.obj_balance(self) == 0 and not self.el.xpath('.//td[has-class("nomContrat")]//a'): self.logger.warning("ignoring an AV account because there's no link for it") return False - return True + # there is life insurance detail page link but check if it's a popup + return self.el.xpath('.//td[has-class("nomContrat")]//a[has-class("clickPopupDetail")]') obj__owner = CleanText('.//td[2]') obj_label = Format(u'%s %s', CleanText('.//td/text()[following-sibling::br]'), obj__owner) @@ -710,19 +722,17 @@ def condition(self): obj__coming_links = [] obj__transfer_id = None obj_number = Field('id') + obj__external_website = False def obj_id(self): + _id = CleanText('.//td/@id')(self) + # in old code, we use _id, it seems that is not used anymore + # but check if it's the case for all users + assert not _id, '_id is still used to retrieve life insurance' + try: - _id = CleanText('.//td/@id')(self) - if not _id: - self.page.browser.assurancevie.go() - ac_details_page = self.page.browser.open(Link('.//td[has-class("nomContrat")]//a')(self)).page - else: - if '-' in _id: - split = _id.split('-') - ac_details_page = self.page.browser.open('/outil/UWVI/AssuranceVie/accesDetail?ID_CONTRAT=%s&PRODUCTEUR=%s' % (split[0], split[1])).page - else: - ac_details_page = self.page.browser.open('/outil/UWVI/AssuranceVie/accesDetail?ID_CONTRAT=%s' % (_id)).page + self.page.browser.assurancevie.go() + ac_details_page = self.page.browser.open(Link('.//td[has-class("nomContrat")]//a')(self)).page return CleanText('(//tr[3])/td[2]')(ac_details_page.doc) except ServerError: self.logger.debug("link didn't work, trying with the form instead") @@ -737,6 +747,7 @@ def obj_id(self): return account_id def obj__form(self): + # maybe deprecated form_id = Attr('.//td[has-class("nomContrat")]//a', 'id', default=None)(self) if form_id: if '-' in form_id: @@ -851,6 +862,34 @@ def come_back(self): return self.browser.location('https://assurance-vie-et-prevoyance.secure.lcl.fr/filiale/entreeBam', params=params) +class AVListPage(LoggedPage, JsonPage): + @method + class iter_life_insurance(DictElement): + item_xpath = 'syntheseContrats' + + class item(ItemElement): + def condition(self): + return ( + Lower(Dict('lcstacntgen'))(self) == 'actif' + and Lower(Dict('lcgampdt'))(self) == 'epargne' + ) + + klass = Account + + obj_id = obj_number = Dict('idcntcar') + obj_balance = CleanDecimal(Dict('mtvalcnt')) + obj_label = Dict('lnpdt') + obj_type = Account.TYPE_LIFE_INSURANCE + obj_currency = 'EUR' + + obj__external_website = True + obj__form = None + obj__link_id = None + obj__market_link = None + obj__coming_links = [] + obj__transfer_id = None + + class AVHistoryPage(LoggedPage, JsonPage): @method class iter_history(DictElement): @@ -885,6 +924,14 @@ def obj_rdate(self): class AVInvestmentsPage(LoggedPage, JsonPage): + def update_life_insurance_account(self, life_insurance): + life_insurance._owner = Format('%s %s', + Dict('situationAdministrativeEpargne/lppeoscp'), + Dict('situationAdministrativeEpargne/lnpeoscp'))(self.doc) + life_insurance.label = '%s %s' % (Dict('situationAdministrativeEpargne/lcofc')(self.doc), life_insurance._owner) + life_insurance.valuation_diff = CleanDecimal(Dict('situationFinanciereEpargne/mtpmvcnt'), default=NotAvailable)(self.doc) + return life_insurance + @method class iter_investment(DictElement): item_xpath = 'listeSupports/support' -- GitLab