From 9f3ff71e73a46c6b7f8b6047cde74fa7c48691d7 Mon Sep 17 00:00:00 2001 From: Sylvie Ye Date: Thu, 19 Sep 2019 11:52:02 +0200 Subject: [PATCH] [sgpe] Handle new market pages for pro website Market pages are Json pages now, all SGPE connections crashed because of the 'assert'. This patch handles the new JSON Market Pages and their investments. Closes: 46992@sibi --- modules/societegenerale/sgpe/browser.py | 68 +++++++------------ modules/societegenerale/sgpe/json_pages.py | 56 ++++++++++++++- modules/societegenerale/sgpe/pages.py | 79 ++-------------------- 3 files changed, 85 insertions(+), 118 deletions(-) diff --git a/modules/societegenerale/sgpe/browser.py b/modules/societegenerale/sgpe/browser.py index 554bbb87f4..c24c524415 100644 --- a/modules/societegenerale/sgpe/browser.py +++ b/modules/societegenerale/sgpe/browser.py @@ -26,7 +26,7 @@ from weboob.browser.url import URL from weboob.browser.exceptions import ClientError from weboob.exceptions import BrowserIncorrectPassword, ActionNeeded, NoAccountsException -from weboob.capabilities.base import find_object, NotAvailable +from weboob.capabilities.base import find_object from weboob.capabilities.bank import ( AccountNotFound, RecipientNotFound, AddRecipientStep, AddRecipientBankError, Recipient, TransferBankError, AccountOwnerType, @@ -36,10 +36,11 @@ from .pages import ( LoginPage, CardsPage, CardHistoryPage, IncorrectLoginPage, ProfileProPage, ProfileEntPage, ChangePassPage, SubscriptionPage, InscriptionPage, - ErrorPage, UselessPage, MainPage, MarketAccountPage, MarketInvestmentPage, + ErrorPage, UselessPage, MainPage, ) from .json_pages import ( AccountsJsonPage, BalancesJsonPage, HistoryJsonPage, BankStatementPage, + MarketAccountPage, MarketInvestmentPage, ) from .transfer_pages import ( EasyTransferPage, RecipientsJsonPage, TransferPage, SignTransferPage, TransferDatesPage, @@ -251,31 +252,34 @@ class SGProfessionalBrowser(SGEnterpriseBrowser, StatesMixin): CERTHASH = '9f5232c9b2283814976608bfd5bba9d8030247f44c8493d8d205e574ea75148e' STATE_DURATION = 5 - incorrect_login = URL('/authent.html', IncorrectLoginPage) - profile = URL('/gao/modifier-donnees-perso-saisie.html', ProfileProPage) + incorrect_login = URL(r'/authent.html', IncorrectLoginPage) + profile = URL(r'/gao/modifier-donnees-perso-saisie.html', ProfileProPage) - transfer_dates = URL('/ord-web/ord//get-dates-execution.json', TransferDatesPage) - easy_transfer = URL('/ord-web/ord//ord-virement-simplifie-emetteur.html', EasyTransferPage) - internal_recipients = URL('/ord-web/ord//ord-virement-simplifie-beneficiaire.html', EasyTransferPage) - external_recipients = URL('/ord-web/ord//ord-liste-compte-beneficiaire-externes.json', RecipientsJsonPage) + transfer_dates = URL(r'/ord-web/ord//get-dates-execution.json', TransferDatesPage) + easy_transfer = URL(r'/ord-web/ord//ord-virement-simplifie-emetteur.html', EasyTransferPage) + internal_recipients = URL(r'/ord-web/ord//ord-virement-simplifie-beneficiaire.html', EasyTransferPage) + external_recipients = URL(r'/ord-web/ord//ord-liste-compte-beneficiaire-externes.json', RecipientsJsonPage) - init_transfer_page = URL('/ord-web/ord//ord-enregistrer-ordre-simplifie.json', TransferPage) - sign_transfer_page = URL('/ord-web/ord//ord-verifier-habilitation-signature-ordre.json', SignTransferPage) - confirm_transfer = URL('/ord-web/ord//ord-valider-signature-ordre.json', TransferPage) + init_transfer_page = URL(r'/ord-web/ord//ord-enregistrer-ordre-simplifie.json', TransferPage) + sign_transfer_page = URL(r'/ord-web/ord//ord-verifier-habilitation-signature-ordre.json', SignTransferPage) + confirm_transfer = URL(r'/ord-web/ord//ord-valider-signature-ordre.json', TransferPage) - recipients = URL('/ord-web/ord//ord-gestion-tiers-liste.json', RecipientsJsonPage) - add_recipient = URL('/ord-web/ord//ord-fragment-form-tiers.html\?cl_action=ajout&cl_idTiers=', + recipients = URL(r'/ord-web/ord//ord-gestion-tiers-liste.json', RecipientsJsonPage) + add_recipient = URL(r'/ord-web/ord//ord-fragment-form-tiers.html\?cl_action=ajout&cl_idTiers=', AddRecipientPage) - add_recipient_step = URL('/ord-web/ord//ord-tiers-calcul-bic.json', - '/ord-web/ord//ord-preparer-signature-destinataire.json', + add_recipient_step = URL(r'/ord-web/ord//ord-tiers-calcul-bic.json', + r'/ord-web/ord//ord-preparer-signature-destinataire.json', AddRecipientStepPage) - confirm_new_recipient = URL('/ord-web/ord//ord-creer-destinataire.json', ConfirmRecipientPage) + confirm_new_recipient = URL(r'/ord-web/ord//ord-creer-destinataire.json', ConfirmRecipientPage) - bank_statement_menu = URL('/icd/syd-front/data/syd-rce-accederDepuisMenu.json', BankStatementPage) - bank_statement_search = URL('/icd/syd-front/data/syd-rce-lancerRecherche.json', BankStatementPage) + bank_statement_menu = URL(r'/icd/syd-front/data/syd-rce-accederDepuisMenu.json', BankStatementPage) + bank_statement_search = URL(r'/icd/syd-front/data/syd-rce-lancerRecherche.json', BankStatementPage) - useless_page = URL('/icd-web/syd-front/index-comptes.html', UselessPage) - error_page = URL('https://static.societegenerale.fr/pro/erreur.html', ErrorPage) + useless_page = URL(r'/icd-web/syd-front/index-comptes.html', UselessPage) + error_page = URL(r'https://static.societegenerale.fr/pro/erreur.html', ErrorPage) + + markets_page = URL(r'/icd/npe/data/comptes-titres/findComptesTitresClasseurs-authsec.json', MarketAccountPage) + investments_page = URL(r'/icd/npe/data/comptes-titres/findLignesCompteTitre-authsec.json', MarketInvestmentPage) date_max = None date_min = None @@ -295,21 +299,7 @@ def load_state(self, state): @need_login def iter_market_accounts(self): - self.main_page.go() - # retrieve market accounts if exist - market_accounts_link = self.page.get_market_accounts_link() - if market_accounts_link is NotAvailable: - return [] - assert market_accounts_link, 'Market accounts link xpath may have changed' - - # need to be on market accounts page to get the accounts iframe - self.location(market_accounts_link) - market_accounts_list_link = self.page.get_table_iframe_link() - if market_accounts_list_link is NotAvailable: - return [] - assert market_accounts_link, 'Market accounts iframe link xpath may have changed' - - self.location(market_accounts_list_link) + self.markets_page.go() return self.page.iter_market_accounts() @need_login @@ -317,13 +307,7 @@ def iter_investment(self, account): if account.type not in (account.TYPE_MARKET, ): return [] - assert account._url_data, 'This account has no url to retrieve investments' - # need to be on market accounts investment page to get the invetment iframe - self.location('/Pgn/NavigationServlet?%s' % account._url_data) - - invests_list_link = self.page.get_table_iframe_link() - assert invests_list_link, 'It seems that this market account has no investment' - self.location(invests_list_link) + self.investments_page.go(data={'cl2000_numeroPrestation': account._prestation_number}) return self.page.iter_investment() def copy_recipient_obj(self, recipient): diff --git a/modules/societegenerale/sgpe/json_pages.py b/modules/societegenerale/sgpe/json_pages.py index 94b5837035..fc03c6c243 100644 --- a/modules/societegenerale/sgpe/json_pages.py +++ b/modules/societegenerale/sgpe/json_pages.py @@ -24,12 +24,12 @@ from weboob.browser.elements import ItemElement, method, DictElement from weboob.browser.filters.standard import ( CleanDecimal, CleanText, Date, Format, BrowserURL, Env, - Field, Regexp, + Field, Regexp, Currency as CurrencyFilter, ) from weboob.browser.filters.json import Dict from weboob.capabilities.base import Currency, empty from weboob.capabilities import NotAvailable -from weboob.capabilities.bank import Account +from weboob.capabilities.bank import Account, Investment from weboob.capabilities.bill import Document, Subscription, DocumentTypes from weboob.exceptions import ( BrowserUnavailable, NoAccountsException, BrowserIncorrectPassword, BrowserPasswordExpired, @@ -37,6 +37,7 @@ ) from weboob.tools.capabilities.bank.iban import is_iban_valid from weboob.tools.capabilities.bank.transactions import FrenchTransaction +from weboob.tools.capabilities.bank.investments import is_isin_valid from weboob.tools.compat import quote_plus from .pages import Transaction @@ -255,3 +256,54 @@ def iter_documents(self): d.url = '/icd/syd-front/data/syd-rce-telechargerReleve.html?b64e4000_sceau=%s' % quote_plus(document['sceau']) yield d + + +class MarketAccountPage(LoggedPage, JsonPage): + @method + class iter_market_accounts(DictElement): + item_xpath = 'donnees/comptesTitresByClasseur' + + def condition(self): + # Some 'comptesTitresByClasseur' do not have a 'list' key + # and therefore have no account list, we skip them + return Dict('list', default=None)(self) + + class iter_accounts(DictElement): + item_xpath = 'list' + + class item(ItemElement): + klass = Account + + obj__prestation_number = Dict('numeroPrestation') + + obj_id = Format('%s_TITRE', CleanText(Field('_prestation_number'), replace=[(' ', '')])) + obj_number = CleanText(Field('_prestation_number'), replace=[(' ', '')]) + obj_label = Dict('intitule') + obj_balance = CleanDecimal.French(Dict('evaluation')) + obj_currency = CurrencyFilter(Dict('evaluation')) + obj_type = Account.TYPE_MARKET + + +class MarketInvestmentPage(LoggedPage, JsonPage): + @method + class iter_investment(DictElement): + item_xpath = 'donnees' + + class item(ItemElement): + klass = Investment + + obj_label = Dict('libelle') + obj_valuation = CleanDecimal.French(Dict('valorisation')) + obj_quantity = CleanDecimal.French(Dict('quantite')) + obj_unitvalue = CleanDecimal.French(Dict('cours')) + + def obj_code(self): + code = Dict('codeISIN')(self) + if is_isin_valid(code): + return code + return NotAvailable + + def obj_code_type(self): + if empty(Field('code')(self)): + return NotAvailable + return Investment.CODE_TYPE_ISIN diff --git a/modules/societegenerale/sgpe/pages.py b/modules/societegenerale/sgpe/pages.py index 3ea356d41b..45e272341e 100644 --- a/modules/societegenerale/sgpe/pages.py +++ b/modules/societegenerale/sgpe/pages.py @@ -24,17 +24,15 @@ from io import BytesIO from weboob.browser.pages import HTMLPage, LoggedPage -from weboob.browser.elements import ListElement, ItemElement, method, TableElement +from weboob.browser.elements import ListElement, ItemElement, method from weboob.browser.filters.standard import ( CleanText, CleanDecimal, Date, - Env, Regexp, Field, Format, TableCell, + Env, Regexp, Field, Format, ) from weboob.browser.filters.html import Attr, Link -from weboob.tools.capabilities.bank.investments import is_isin_valid from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.capabilities.profile import Profile, Person from weboob.capabilities.bill import Document, Subscription, DocumentTypes -from weboob.capabilities.bank import Account, Investment from weboob.exceptions import ActionNeeded, BrowserIncorrectPassword, BrowserUnavailable from weboob.tools.json import json @@ -279,79 +277,12 @@ class UselessPage(LoggedPage, SGPEPage): class MainPage(LoggedPage, SGPEPage): def get_market_accounts_link(self): - market_accounts_link = Link('//li/a[@title="Comptes titres"]', default=None)(self.doc) + # this is for "ent" website, don't know if it works like "pro" website + market_accounts_link = Link('//li/a[@title="Comptes-titres"]', default=None)(self.doc) if market_accounts_link: return market_accounts_link - elif self.doc.xpath('//span[contains(text(), "Comptes titres") and contains(@title, "pas habilité à utiliser ce service")]'): + elif self.doc.xpath('//span[contains(text(), "Comptes-titres") and contains(@title, "pas habilité à utiliser ce service")]'): return NotAvailable # return None when we don't know if there are market accounts or not # it will be handled in `browser.py` - - -class MarketAccountPage(LoggedPage, SGPEPage): - def get_table_iframe_link(self): - if self.doc.xpath('//div[contains(text(), "Aucun compte-titres")]'): - return NotAvailable - return Attr('//iframe[@id="frameTableau"]', 'src')(self.doc) - - @method - class iter_market_accounts(TableElement): - item_xpath = '//table[@id="tab-corps"]//tr' - head_xpath = '//table[@id="tab-entete"]//td' - - col_id = 'COMPTE' - col_label = 'INTITULE' - col_balance = 'EVALUATION' - - class item(ItemElement): - def condition(self): - # table with empty row filled by empty `td` - return Field('number')(self) - - klass = Account - - obj_id = Format('%s_TITRE', CleanText(TableCell('id'), replace=[(' ', '')])) - obj_number = CleanText(TableCell('id'), replace=[(' ', '')]) - obj_label = CleanText(TableCell('label')) - obj_balance = CleanDecimal.French(CleanText(TableCell('balance'))) - obj_type = Account.TYPE_MARKET - - # all `a` balises have same `href` - obj__url_data = Regexp(Link('(.//a)[1]'), r"lienParent\('(.*)'\)", default=NotAvailable) - - -class MarketInvestmentPage(LoggedPage, SGPEPage): - def get_table_iframe_link(self): - return Attr('//iframe[@id="frameTableau"]', 'src')(self.doc) - - @method - class iter_investment(TableElement): - item_xpath = '//table[@id="tab-corps"]//tr' - head_xpath = '//table[@id="tab-entete"]//td' - - col_code = 'CODE' - col_label = 'VALEUR' - col_valuation = 'MONTANT' - col_quantity = 'QUANTITE' - col_unitvalue = 'COURS' - - class item(ItemElement): - def condition(self): - # table with empty row filled by empty `td` - return Field('valuation')(self) - - klass = Investment - - obj_code_type = Investment.CODE_TYPE_ISIN - obj_label = CleanText(TableCell('label')) - obj_valuation = CleanDecimal.French(CleanText(TableCell('valuation'))) - obj_quantity = CleanDecimal.French(CleanText(TableCell('quantity'))) - obj_unitvalue = CleanDecimal.French(CleanText(TableCell('unitvalue'))) - - def obj_code(self): - code = CleanText(TableCell('code'))(self) - # there is no example of invests without valid ISIN code - # wait for it to retrieve them corretly - assert is_isin_valid(code), 'This code is not a valid ISIN, please check what invest is it.' - return code -- GitLab