From 8938e36aaea17a93826944ff099aac61f1368f05 Mon Sep 17 00:00:00 2001 From: Romain Bignon Date: Sun, 2 Dec 2018 12:43:45 +0100 Subject: [PATCH] backport master modules fixes --- modules/ameli/browser.py | 6 +- modules/ameli/module.py | 9 +- modules/ameli/pages.py | 11 + modules/axabanque/browser.py | 7 +- modules/axabanque/module.py | 4 +- modules/axabanque/pages/login.py | 4 +- modules/banquepopulaire/pages.py | 8 +- modules/bnporc/enterprise/browser.py | 4 +- modules/bnporc/module.py | 11 +- modules/bnporc/pp/browser.py | 9 +- modules/bnporc/pp/pages.py | 7 +- modules/bolden/browser.py | 25 ++- modules/bolden/compat/__init__.py | 0 .../bolden/compat/weboob_capabilities_bank.py | 45 +++++ ...oob_tools_capabilities_bank_investments.py | 86 ++++++++ modules/bolden/module.py | 7 +- modules/bolden/pages.py | 54 ++--- modules/boursorama/browser.py | 68 ++++++- modules/boursorama/pages.py | 118 ++++++----- modules/bp/pages/transfer.py | 7 +- modules/bred/bred/browser.py | 2 +- modules/bred/bred/pages.py | 29 +-- modules/caissedepargne/browser.py | 58 ++++-- modules/caissedepargne/pages.py | 21 +- modules/cmso/par/pages.py | 3 +- modules/cmso/pro/browser.py | 38 ++-- modules/cragr/web/pages.py | 28 ++- modules/creditdunord/browser.py | 15 +- ...oob_tools_capabilities_bank_investments.py | 86 ++++++++ modules/creditdunord/pages.py | 129 ++++++------ modules/creditmutuel/browser.py | 61 ++++-- modules/creditmutuel/module.py | 2 +- modules/creditmutuel/pages.py | 190 +++++++++++------ modules/fortuneo/browser.py | 10 +- modules/fortuneo/pages/accounts_list.py | 3 +- modules/fortuneo/pages/transfer.py | 6 + modules/groupama/browser.py | 14 +- modules/groupama/pages.py | 10 +- modules/hsbc/browser.py | 191 ++++++++++++------ modules/hsbc/module.py | 4 +- modules/hsbc/pages/account_pages.py | 83 +++++--- modules/hsbc/pages/investments.py | 13 +- modules/ing/browser.py | 8 - modules/ing/pages/accounts_list.py | 5 - modules/lcl/pages.py | 6 +- modules/materielnet/browser.py | 31 ++- modules/materielnet/pages.py | 2 +- modules/paypal/browser.py | 5 +- modules/paypal/pages.py | 67 +----- modules/societegenerale/sgpe/browser.py | 44 +++- modules/societegenerale/sgpe/json_pages.py | 86 +++++--- modules/societegenerale/sgpe/pages.py | 6 + .../societegenerale/sgpe/transfer_pages.py | 12 +- modules/yomoni/browser.py | 4 +- 54 files changed, 1210 insertions(+), 552 deletions(-) create mode 100644 modules/bolden/compat/__init__.py create mode 100644 modules/bolden/compat/weboob_capabilities_bank.py create mode 100644 modules/bolden/compat/weboob_tools_capabilities_bank_investments.py create mode 100644 modules/creditdunord/compat/weboob_tools_capabilities_bank_investments.py diff --git a/modules/ameli/browser.py b/modules/ameli/browser.py index 295037b49d..715c6b888c 100644 --- a/modules/ameli/browser.py +++ b/modules/ameli/browser.py @@ -21,8 +21,11 @@ from weboob.browser import LoginBrowser, URL, need_login from weboob.exceptions import BrowserIncorrectPassword, ActionNeeded +from .pages import ( + LoginPage, HomePage, CguPage, AccountPage, LastPaymentsPage, PaymentsPage, PaymentDetailsPage, Raw, UnavailablePage, +) from weboob.tools.compat import basestring -from .pages import LoginPage, HomePage, CguPage, AccountPage, LastPaymentsPage, PaymentsPage, PaymentDetailsPage, Raw + __all__ = ['AmeliBrowser'] @@ -38,6 +41,7 @@ class AmeliBrowser(LoginBrowser): paymentdetailsp = URL(r'/PortailAS/paiements.do\?actionEvt=chargerDetailPaiements.*', PaymentDetailsPage) lastpaymentsp = URL(r'/PortailAS/paiements.do\?actionEvt=afficherPaiements.*', LastPaymentsPage) pdf_page = URL(r'PortailAS/PDFServletReleveMensuel.dopdf\?PDF.moisRecherche=.*', Raw) + unavailablep = URL(r'/vu/INDISPO_COMPTE_ASSURES.html', UnavailablePage) def do_login(self): self.logger.debug('call Browser.do_login') diff --git a/modules/ameli/module.py b/modules/ameli/module.py index f94b28bcf9..46bbe096c4 100644 --- a/modules/ameli/module.py +++ b/modules/ameli/module.py @@ -35,13 +35,8 @@ class AmeliModule(Module, CapDocument): VERSION = '1.3' LICENSE = 'AGPLv3+' BROWSER = AmeliBrowser - CONFIG = BackendConfig(ValueBackendPassword('login', - label='Numero de SS', - masked=False), - ValueBackendPassword('password', - label='Password', - masked=True) - ) + CONFIG = BackendConfig(ValueBackendPassword('login', label='Numero de SS', regexp=r'^\d{13}$', masked=False), + ValueBackendPassword('password', label='Password', masked=True)) def create_default_browser(self): return self.create_browser(self.config['login'].get(), diff --git a/modules/ameli/pages.py b/modules/ameli/pages.py index ccba3c4550..b9eb114228 100644 --- a/modules/ameli/pages.py +++ b/modules/ameli/pages.py @@ -27,9 +27,11 @@ from weboob.browser.pages import HTMLPage, RawPage, LoggedPage from weboob.capabilities.bill import Subscription, Detail, Bill from weboob.browser.filters.standard import CleanText, Regexp +from weboob.exceptions import BrowserUnavailable # Ugly array to avoid the use of french locale + FRENCH_MONTHS = ['janvier', 'février', 'mars', 'avril', 'mai', 'juin', 'juillet', 'août', 'septembre', 'octobre', 'novembre', 'décembre'] @@ -52,6 +54,10 @@ def is_error(self): if errors: return errors + errors = CleanText('//div[@class="zone-alerte"]/span')(self.doc) + if errors: + return errors + return False @@ -225,3 +231,8 @@ def iter_payment_details(self, sub): class Raw(LoggedPage, RawPage): pass + + +class UnavailablePage(HTMLPage): + def on_load(self): + raise BrowserUnavailable(CleanText('//span[@class="texte-indispo"]')(self.doc)) diff --git a/modules/axabanque/browser.py b/modules/axabanque/browser.py index 995960f41d..0fd4c7640d 100644 --- a/modules/axabanque/browser.py +++ b/modules/axabanque/browser.py @@ -60,7 +60,9 @@ class AXABrowser(LoginBrowser): denied = URL('https://connect.axa.fr/Account/AccessDenied', DeniedPage) account_space_login = URL('https://connect.axa.fr/api/accountspace', AccountSpaceLogin) - errors = URL('https://espaceclient.axa.fr/content/ecc-public/accueil-axa-connect/_jcr_content/par/text.html', ErrorPage) + errors = URL('https://espaceclient.axa.fr/content/ecc-public/accueil-axa-connect/_jcr_content/par/text.html', + 'https://espaceclient.axa.fr/content/ecc-public/errors/500.html', + ErrorPage) def do_login(self): # due to the website change, login changed too, this is for don't try to login with the wrong login @@ -109,7 +111,8 @@ class AXABanque(AXABrowser, StatesMixin): 'webapp/axabanque/jsp/.*/detail.*.faces', TransactionsPage) unavailable = URL('login_errors/indisponibilite.*', '.*page-indisponible.html.*', - '.*erreur/erreurBanque.faces', UnavailablePage) + '.*erreur/erreurBanque.faces', + 'http://www.axabanque.fr/message/maintenance.htm', UnavailablePage) # Wealth wealth_accounts = URL('https://espaceclient.axa.fr/$', 'https://espaceclient.axa.fr/accueil.html', diff --git a/modules/axabanque/module.py b/modules/axabanque/module.py index c7fe6418b3..c86e5b50c7 100644 --- a/modules/axabanque/module.py +++ b/modules/axabanque/module.py @@ -19,7 +19,7 @@ from .compat.weboob_capabilities_bank import CapBankWealth, CapBankTransferAddRecipient, AccountNotFound, RecipientNotFound -from weboob.capabilities.base import find_object, NotAvailable +from weboob.capabilities.base import find_object, NotAvailable, empty from weboob.capabilities.bank import Account, TransferInvalidLabel from weboob.capabilities.profile import CapProfile from weboob.capabilities.bill import CapDocument, Subscription, Document, DocumentNotFound, SubscriptionNotFound @@ -112,7 +112,7 @@ def transfer_check_account_id(self, old, new): def transfer_check_account_iban(self, old, new): # Skip origin account iban check and force origin account iban - if new is NotAvailable: + if empty(new) or empty(old): self.logger.warning( 'Origin account iban check (%s) is not possible because iban is currently not available', old, diff --git a/modules/axabanque/pages/login.py b/modules/axabanque/pages/login.py index a7269ce869..20b738443b 100644 --- a/modules/axabanque/pages/login.py +++ b/modules/axabanque/pages/login.py @@ -121,8 +121,10 @@ def on_load(self): error_msg = ( CleanText('//p[contains(text(), "temporairement indisponible")]')(self.doc), CleanText('//p[contains(text(), "maintenance est en cours")]')(self.doc), + # parsing for false 500 error page + CleanText('//div[contains(@class, "error-page")]//span[contains(@class, "subtitle") and contains(text(), "Chargement de page impossible")]')(self.doc) ) for error in error_msg: if error: - raise BrowserUnavailable(error_msg) + raise BrowserUnavailable(error) diff --git a/modules/banquepopulaire/pages.py b/modules/banquepopulaire/pages.py index 741b327cf2..774fe27466 100644 --- a/modules/banquepopulaire/pages.py +++ b/modules/banquepopulaire/pages.py @@ -412,13 +412,17 @@ def get_token(self): url = self.browser.absurl('/portailinternet/Transactionnel/Pages/CyberIntegrationPage.aspx') headers = {'Referer': self.url} + + # Sometime, the page is a 302 and redirect to a page where there are no information that we need, + # so we try with 2 others url to further fetch token when empty page r = self.browser.open(url, data='taskId=aUniversMesComptes', params={'vary': vary}, headers=headers) if not int(r.headers.get('Content-Length', 0)): - url = self.browser.absurl('/portailinternet/Transactionnel/Pages/CyberIntegrationPage.aspx') - headers = {'Referer': self.url} r = self.browser.open(url, data='taskId=aUniversMesComptes', headers=headers) + if not int(r.headers.get('Content-Length', 0)): + r = self.browser.open(url, data={'taskId': 'equipementDom'}, params={'vary': vary}, headers=headers) + doc = r.page.doc date = None for script in doc.xpath('//script'): diff --git a/modules/bnporc/enterprise/browser.py b/modules/bnporc/enterprise/browser.py index e7b752d27b..b644d67022 100644 --- a/modules/bnporc/enterprise/browser.py +++ b/modules/bnporc/enterprise/browser.py @@ -65,8 +65,8 @@ class BNPEnterprise(LoginBrowser): renew_pass = URL('/sommaire/PseRedirectPasswordConnect', ActionNeededPage) - def __init__(self, *args, **kwargs): - super(BNPEnterprise, self).__init__(*args, **kwargs) + def __init__(self, config, *args, **kwargs): + super(BNPEnterprise, self).__init__(config['login'].get(), config['password'].get(), *args, **kwargs) def do_login(self): self.login.go() diff --git a/modules/bnporc/module.py b/modules/bnporc/module.py index fcbaef27e4..f2662e5211 100644 --- a/modules/bnporc/module.py +++ b/modules/bnporc/module.py @@ -32,7 +32,7 @@ from weboob.capabilities.profile import CapProfile from weboob.capabilities.base import find_object from weboob.tools.backend import Module, BackendConfig -from weboob.tools.value import ValueBackendPassword, Value +from weboob.tools.value import ValueBackendPassword, Value, ValueBool from .enterprise.browser import BNPEnterprise from .company.browser import BNPCompany @@ -52,9 +52,7 @@ class BNPorcModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapMessag CONFIG = BackendConfig( ValueBackendPassword('login', label=u'Numéro client', masked=False), ValueBackendPassword('password', label=u'Code secret', regexp='^(\d{6})$'), - #ValueBackendPassword('rotating_password', default='', - # label='Password to set when the allowed uses are exhausted (6 digits)', - # regexp='^(\d{6}|)$'), + ValueBool('rotating_password', label=u'Automatically renew password every 100 connections', default=False), Value('website', label='Type de compte', default='pp', choices={'pp': 'Particuliers/Professionnels', 'hbank': 'HelloBank', @@ -73,10 +71,7 @@ def __init__(self, *args, **kwargs): def create_default_browser(self): b = {'ent': BNPEnterprise, 'ent2': BNPCompany, 'pp': BNPPartPro, 'hbank': HelloBank} self.BROWSER = b[self.config['website'].get()] - if self.BROWSER is BNPPartPro: - return self.create_browser(self.config) - return self.create_browser(self.config['login'].get(), - self.config['password'].get()) + return self.create_browser(self.config) def iter_accounts(self): for account in self.browser.get_accounts_list(): diff --git a/modules/bnporc/pp/browser.py b/modules/bnporc/pp/browser.py index 3242611064..a9b0cd13ac 100644 --- a/modules/bnporc/pp/browser.py +++ b/modules/bnporc/pp/browser.py @@ -114,10 +114,11 @@ class BNPParibasBrowser(JsonBrowserMixin, LoginBrowser): profile = URL(r'/kyc-wspl/rest/informationsClient', ProfilePage) list_detail_card = URL(r'/udcarte-wspl/rest/listeDetailCartes', ListDetailCardPage) - def __init__(self, *args, **kwargs): - super(BNPParibasBrowser, self).__init__(*args, **kwargs) + def __init__(self, config, *args, **kwargs): + super(BNPParibasBrowser, self).__init__(config['login'].get(), config['password'].get(), *args, **kwargs) self.accounts_list = None self.card_to_transaction_type = {} + self.rotating_password = config['rotating_password'] @retry(ConnectionError, tries=3) def open(self, *args, **kwargs): @@ -406,9 +407,7 @@ class BNPPartPro(BNPParibasBrowser): def __init__(self, config=None, *args, **kwargs): self.config = config - kwargs['username'] = self.config['login'].get() - kwargs['password'] = self.config['password'].get() - super(BNPPartPro, self).__init__(*args, **kwargs) + super(BNPPartPro, self).__init__(self.config, *args, **kwargs) def switch(self, subdomain): self.BASEURL = self.BASEURL_TEMPLATE % subdomain diff --git a/modules/bnporc/pp/pages.py b/modules/bnporc/pp/pages.py index e92d52f5fe..5ce42ff02e 100644 --- a/modules/bnporc/pp/pages.py +++ b/modules/bnporc/pp/pages.py @@ -102,6 +102,9 @@ def on_load(self): msg = CleanText('//div[@class="confirmation"]//span[span]')(self.doc) self.logger.warning('Password expired.') + if not self.browser.rotating_password: + raise BrowserPasswordExpired(msg) + if not self.looks_legit(self.browser.password): # we may not be able to restore the password, so reject it self.logger.warning('Unable to restore it, it is not legit.') @@ -421,7 +424,7 @@ def handle_response(self, account, recipient, amount, reason): if 'idBeneficiaire' in transfer_data and transfer_data['idBeneficiaire'] is not None: assert transfer_data['idBeneficiaire'] == recipient.id - elif 'ibanCompteCrediteur' in transfer_data and transfer_data['ibanCompteCrediteur'] is not None: + elif transfer_data.get('ibanCompteCrediteur'): assert transfer_data['ibanCompteCrediteur'] == recipient.iban transfer = Transfer() @@ -439,7 +442,7 @@ def handle_response(self, account, recipient, amount, reason): else: transfer.recipient_id = recipient.id transfer.exec_date = parse_french_date(transfer_data['dateExecution']).date() - transfer.fees = Decimal(transfer_data['montantFrais']) + transfer.fees = Decimal(transfer_data.get('montantFrais', '0')) transfer.label = transfer_data['motifVirement'] transfer.account_label = account.label diff --git a/modules/bolden/browser.py b/modules/bolden/browser.py index b76a339afa..f94007144b 100644 --- a/modules/bolden/browser.py +++ b/modules/bolden/browser.py @@ -21,8 +21,9 @@ from datetime import timedelta, datetime -from weboob.browser import LoginBrowser, need_login, URL +from weboob.browser.browsers import LoginBrowser, need_login, URL from weboob.capabilities.bill import Document +from .compat.weboob_tools_capabilities_bank_investments import create_french_liquidity from .pages import ( LoginPage, HomeLendPage, PortfolioPage, OperationsPage, MAIN_ID, ProfilePage, @@ -51,13 +52,16 @@ def iter_accounts(self): self.portfolio.go() return self.page.iter_accounts() + def iter_investments(self): + self.portfolio.go() + yield create_french_liquidity(self.page.get_liquidity()) + for inv in self.page.iter_investments(): + yield inv + @need_login def iter_history(self, account): if account.id != MAIN_ID: - return [] - return self._iter_all_history() - - def _iter_all_history(self): + return end = datetime.now() while True: start = end - timedelta(days=365) @@ -86,14 +90,13 @@ def get_profile(self): @need_login def iter_documents(self): - for acc in self.iter_accounts(): - if acc.id == MAIN_ID: + for inv in self.iter_investments(): + if inv.label == "Liquidités": continue - doc = Document() - doc.id = acc.id - doc.url = acc._docurl - doc.label = 'Contrat %s' % acc.label + doc.id = inv.id + doc.url = inv._docurl + doc.label = 'Contrat %s' % inv.label doc.type = 'other' doc.format = 'pdf' yield doc diff --git a/modules/bolden/compat/__init__.py b/modules/bolden/compat/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/modules/bolden/compat/weboob_capabilities_bank.py b/modules/bolden/compat/weboob_capabilities_bank.py new file mode 100644 index 0000000000..141097d781 --- /dev/null +++ b/modules/bolden/compat/weboob_capabilities_bank.py @@ -0,0 +1,45 @@ + +import weboob.capabilities.bank as OLD + +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) + + +__all__ = OLD.__all__ + + +class CapBankWealth(CapBank): + pass + + +class CapBankPockets(CapBank): + pass + + +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + +class CapBankTransfer(OLD.CapBankTransfer): + def transfer_check_label(self, old, new): + from unidecode import unidecode + + return unidecode(old) == unidecode(new) + + +class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipient): + pass + + +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + +Account.TYPE_MORTGAGE = 17 +Account.TYPE_CONSUMER_CREDIT = 18 +Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/bolden/compat/weboob_tools_capabilities_bank_investments.py b/modules/bolden/compat/weboob_tools_capabilities_bank_investments.py new file mode 100644 index 0000000000..2b68ff9e00 --- /dev/null +++ b/modules/bolden/compat/weboob_tools_capabilities_bank_investments.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Jonathan Schmidt +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.tools.compat import basestring +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.bank import Investment + +def is_isin_valid(isin): + """ + Méthode générale + Table de conversion des lettres en chiffres + A=10 B=11 C=12 D=13 E=14 F=15 G=16 H=17 I=18 + J=19 K=20 L=21 M=22 N=23 O=24 P=25 Q=26 R=27 + S=28 T=29 U=30 V=31 W=32 X=33 Y=34 Z=35 + + 1 - Mettre de côté la clé, qui servira de référence à la fin de la vérification. + 2 - Convertir toutes les lettres en nombres via la table de conversion ci-contre. Si le nombre obtenu est supérieur ou égal à 10, prendre les deux chiffres du nombre séparément (exemple : 27 devient 2 et 7). + 3 - Pour chaque chiffre, multiplier sa valeur par deux si sa position est impaire en partant de la droite. Si le nombre obtenu est supérieur ou égal à 10, garder les deux chiffres du nombre séparément (exemple : 14 devient 1 et 4). + 4 - Faire la somme de tous les chiffres. + 5 - Soustraire cette somme de la dizaine supérieure ou égale la plus proche (exemples : si la somme vaut 22, la dizaine « supérieure ou égale » est 30, et la clé vaut donc 8 ; si la somme vaut 30, la dizaine « supérieure ou égale » est 30, et la clé vaut 0 ; si la somme vaut 31, la dizaine « supérieure ou égale » est 40, et la clé vaut 9). + 6 - Comparer la valeur obtenue à la clé mise initialement de côté. + + Étapes 1 et 2 : + F R 0 0 0 3 5 0 0 0 0 (+ 8 : clé) + 15 27 0 0 0 3 5 0 0 0 0 + + Étape 3 : le traitement se fait sur des chiffres + 1 5 2 7 0 0 0 3 5 0 0 0 0 + I P I P I P I P I P I P I : position en partant de la droite (P = Pair, I = Impair) + 2 1 2 1 2 1 2 1 2 1 2 1 2 : coefficient multiplicateur + + 2 5 4 7 0 0 0 3 10 0 0 0 0 : résultat + + Étape 4 : + 2 + 5 + 4 + 7 + 0 + 0 + 0 + 3 + (1 + 0)+ 0 + 0 + 0 + 0 = 22 + + Étapes 5 et 6 : 30 - 22 = 8 (valeur de la clé) + """ + + if not isinstance(isin, basestring): + return False + if not re.match(r'^[A-Z]{2}[A-Z0-9]{9}\d$', isin): + return False + + isin_in_digits = ''.join(str(ord(x) - ord('A') + 10) if not x.isdigit() else x for x in isin[:-1]) + key = isin[-1:] + result = '' + for k, val in enumerate(isin_in_digits[::-1], start=1): + if k % 2 == 0: + result = ''.join((result, val)) + else: + result = ''.join((result, str(int(val)*2))) + return str(sum(int(x) for x in result) + int(key))[-1] == '0' + + +def create_french_liquidity(valuation): + """ + Automatically fills a liquidity investment with label, code and code_type. + """ + liquidity = Investment() + liquidity.label = "Liquidités" + liquidity.code = "XX-liquidity" + liquidity.code_type = NotAvailable + liquidity.valuation = valuation + return liquidity + diff --git a/modules/bolden/module.py b/modules/bolden/module.py index 7e51b2081f..d244fdced4 100644 --- a/modules/bolden/module.py +++ b/modules/bolden/module.py @@ -22,7 +22,7 @@ from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword -from weboob.capabilities.bank import CapBank, Account +from .compat.weboob_capabilities_bank import CapBankWealth, Account from weboob.capabilities.base import find_object from weboob.capabilities.bill import ( CapDocument, Subscription, SubscriptionNotFound, DocumentNotFound, Document, @@ -35,7 +35,7 @@ __all__ = ['BoldenModule'] -class BoldenModule(Module, CapBank, CapDocument, CapProfile): +class BoldenModule(Module, CapBankWealth, CapDocument, CapProfile): NAME = 'bolden' DESCRIPTION = 'Bolden' MAINTAINER = 'Vincent A' @@ -59,6 +59,9 @@ def iter_accounts(self): def iter_history(self, account): return self.browser.iter_history(account) + def iter_investment(self, account): + return self.browser.iter_investments() + def get_profile(self): return self.browser.get_profile() diff --git a/modules/bolden/pages.py b/modules/bolden/pages.py index 9f9ee35f20..7588f13e96 100644 --- a/modules/bolden/pages.py +++ b/modules/bolden/pages.py @@ -19,15 +19,14 @@ from __future__ import unicode_literals -from decimal import Decimal - from weboob.browser.elements import ListElement, ItemElement, method, TableElement from weboob.browser.filters.html import TableCell, Link, Attr from weboob.browser.filters.standard import ( CleanText, CleanDecimal, Slugify, Date, Field, Format, ) from weboob.browser.pages import HTMLPage, LoggedPage -from weboob.capabilities.bank import Account, Transaction +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.bank import Account, Transaction, Investment from weboob.capabilities.profile import Profile from weboob.exceptions import BrowserIncorrectPassword from weboob.tools.compat import urljoin @@ -55,37 +54,46 @@ class HomeLendPage(LoggedPage, HTMLPage): class PortfolioPage(LoggedPage, HTMLPage): @method class iter_accounts(ListElement): - class get_main(ItemElement): + class item(ItemElement): klass = Account obj_id = MAIN_ID obj_label = 'Compte Bolden' - obj_type = Account.TYPE_CHECKING + obj_type = Account.TYPE_MARKET obj_currency = 'EUR' - obj_balance = CleanDecimal('//div[p[has-class("investor-state") and contains(text(),"Fonds disponibles :")]]/p[has-class("investor-status")]', replace_dots=True) - #obj_coming = CleanDecimal('//div[p[has-class("investor-state") and contains(text(),"Capital restant dû :")]]/p[has-class("investor-status")]', replace_dots=True) + obj_balance = CleanDecimal('//div[p[has-class("investor-state") and contains(text(),"Total compte Bolden :")]]/p[has-class("investor-status")]', replace_dots=True) + obj_valuation_diff = CleanDecimal('//p[has-class("rent-amount strong dashboard-text")]', replace_dots=True) + + @method + class iter_investments(TableElement): + head_xpath = '//div[@class="tab-wallet"]/table/thead//td' - class iter_lends(TableElement): - head_xpath = '//div[@class="tab-wallet"]/table/thead//td' + col_label = 'Emprunteur' + col_valuation = 'Capital restant dû' + col_doc = 'Contrat' + col_diff = 'Intérêts perçus' - col_label = 'Emprunteur' - col_coming = 'Capital restant dû' - col_doc = 'Contrat' + item_xpath = '//div[@class="tab-wallet"]/table/tbody/tr' - item_xpath = '//div[@class="tab-wallet"]/table/tbody/tr' + class item(ItemElement): + klass = Investment - class item(ItemElement): - klass = Account + obj_label = CleanText(TableCell('label')) + obj_id = Slugify(Field('label')) + obj_valuation = CleanDecimal(TableCell('valuation'), replace_dots=True) + obj_diff = CleanDecimal(TableCell('diff'), replace_dots=True) + obj_code = NotAvailable + obj_code_type = NotAvailable + + def condition(self): + # Investments without valuation are expired. + return CleanDecimal(TableCell('valuation'))(self) - obj_label = CleanText(TableCell('label')) - obj_id = Slugify(Field('label')) - obj_type = Account.TYPE_SAVINGS - obj_currency = 'EUR' - obj_coming = CleanDecimal(TableCell('coming'), replace_dots=True) - obj_balance = Decimal('0') + def obj__docurl(self): + return urljoin(self.page.url, Link('.//a')(TableCell('doc')(self)[0])) - def obj__docurl(self): - return urljoin(self.page.url, Link('.//a')(TableCell('doc')(self)[0])) + def get_liquidity(self): + return CleanDecimal('//div[p[contains(text(), "Fonds disponibles")]]/p[@class="investor-status strong"]', replace_dots=True)(self.doc) class OperationsPage(LoggedPage, HTMLPage): diff --git a/modules/boursorama/browser.py b/modules/boursorama/browser.py index 1d3c19e859..7d2dc3707e 100644 --- a/modules/boursorama/browser.py +++ b/modules/boursorama/browser.py @@ -26,7 +26,7 @@ from .compat.weboob_browser_retry import login_method, retry_on_logout, RetryLoginBrowser from weboob.browser.browsers import need_login, StatesMixin from weboob.browser.url import URL -from weboob.exceptions import BrowserIncorrectPassword, BrowserHTTPNotFound +from weboob.exceptions import BrowserIncorrectPassword, BrowserHTTPNotFound, NoAccountsException, BrowserUnavailable from .compat.weboob_browser_exceptions import LoggedOut, ClientError from .compat.weboob_capabilities_bank import ( Account, AccountNotFound, TransferError, TransferInvalidAmount, @@ -45,6 +45,7 @@ CardsNumberPage, CalendarPage, HomePage, PEPPage, TransferAccounts, TransferRecipients, TransferCharac, TransferConfirm, TransferSent, AddRecipientPage, StatusPage, CardHistoryPage, CardCalendarPage, CurrencyListPage, CurrencyConvertPage, + AccountsErrorPage, NoAccountPage, ) @@ -68,8 +69,13 @@ class BoursoramaBrowser(RetryLoginBrowser, StatesMixin): error = URL('/connexion/compte-verrouille', '/infos-profil', ErrorPage) login = URL('/connexion/', LoginPage) + accounts = URL('/dashboard/comptes\?_hinclude=300000', AccountsPage) + accounts_error = URL('/dashboard/comptes\?_hinclude=300000', AccountsErrorPage) pro_accounts = URL(r'/dashboard/comptes-professionnels\?_hinclude=1', AccountsPage) + no_account = URL('/dashboard/comptes\?_hinclude=300000', + '/dashboard/comptes-professionnels\?_hinclude=1', NoAccountPage) + acc_tit = URL('/comptes/titulaire/(?P.*)\?_hinclude=1', AccbisPage) acc_rep = URL('/comptes/representative/(?P.*)\?_hinclude=1', AccbisPage) acc_pro = URL('/comptes/professionnel/(?P.*)\?_hinclude=1', AccbisPage) @@ -189,12 +195,46 @@ def go_cards_number(self, link): @need_login def get_accounts_list(self): self.status.go() + + exc = None for x in range(3): if self.accounts_list is not None: break + self.accounts_list = [] - self.accounts_list.extend(self.pro_accounts.go().iter_accounts()) - self.accounts_list.extend(self.accounts.go().iter_accounts()) + self.loans_list = [] + # Check that there is at least one account for this user + has_account = False + self.pro_accounts.go() + if self.pro_accounts.is_here(): + self.accounts_list.extend(self.page.iter_accounts()) + has_account = True + else: + # We dont want to let has_account=False if we landed on an unknown page + # it has to be the no_accounts page + assert self.no_account.is_here() + + try: + self.accounts.go() + except BrowserUnavailable as e: + self.logger.warning('par accounts seem unavailable, retrying') + exc = e + self.accounts_list = None + continue + else: + if self.accounts.is_here(): + self.accounts_list.extend(self.page.iter_accounts()) + has_account = True + else: + # We dont want to let has_account=False if we landed on an unknown page + # it has to be the no_accounts page + assert self.no_account.is_here() + + exc = None + + if not has_account: + # if we landed twice on NoAccountPage, it means there is neither pro accounts nor pp accounts + raise NoAccountsException() # discard all unvalid card accounts (if opposed or not yet activated) valid_card_url = [] @@ -210,11 +250,19 @@ def get_accounts_list(self): # there is 1 page for all accounts (one for tit and one for pro) break - for account in list(self.accounts_list): if account.type == Account.TYPE_CARD and account.url not in valid_card_url: self.accounts_list.remove(account) + elif account.type == Account.TYPE_LOAN: + # Loans details are present on another page so we create + # a Loan object and remove the corresponding Account: + self.location(account.url) + loan = self.page.get_loan() + loan.url = account.url + self.loans_list.append(loan) + self.accounts_list.remove(account) + self.accounts_list.extend(self.loans_list) cards = [acc for acc in self.accounts_list if acc.type == Account.TYPE_CARD] if cards: self.go_cards_number(cards[0].url) @@ -222,13 +270,16 @@ def get_accounts_list(self): self.page.populate_cards_number(cards) for account in self.accounts_list: - if account.type not in (Account.TYPE_CARD, Account.TYPE_LOAN, Account.TYPE_LIFE_INSURANCE): + if account.type not in (Account.TYPE_CARD, Account.TYPE_LOAN, Account.TYPE_CONSUMER_CREDIT, Account.TYPE_MORTGAGE, Account.TYPE_REVOLVING_CREDIT, Account.TYPE_LIFE_INSURANCE): account.iban = self.iban.go(webid=account._webid).get_iban() for card in cards: checking, = [account for account in self.accounts_list if account.type == Account.TYPE_CHECKING and account.url in card.url] card.parent = checking + if exc: + raise exc + return self.accounts_list def get_account(self, id): @@ -342,6 +393,9 @@ def get_advisor(self): @need_login def iter_transfer_recipients(self, account): + if account.type in (Account.TYPE_LOAN, Account.TYPE_LIFE_INSURANCE): + return [] + assert account.url url = urlsplit(account.url) @@ -399,6 +453,10 @@ def init_transfer(self, transfer, **kwargs): self.page.submit_info(transfer.amount, transfer.label, transfer.exec_date) assert self.transfer_confirm.is_here() + if self.page.need_refresh(): + # In some case we are not yet in the transfer_charac page, you need to refresh the page + self.location(self.url) + assert not self.page.need_refresh() ret = self.page.get_transfer() # at this stage, the site doesn't show the real ids/ibans, we can only guess diff --git a/modules/boursorama/pages.py b/modules/boursorama/pages.py index f97fabb95a..8c08e0451e 100644 --- a/modules/boursorama/pages.py +++ b/modules/boursorama/pages.py @@ -31,13 +31,13 @@ from weboob.browser.filters.standard import ( CleanText, CleanDecimal, Field, Format, Regexp, Date, AsyncLoad, Async, Eval, RegexpError, Env, - Currency as CleanCurrency, + Currency as CleanCurrency, Map, ) from weboob.browser.filters.json import Dict from weboob.browser.filters.html import Attr, Link, TableCell, AbsoluteLink from .compat.weboob_capabilities_bank import ( Account, Investment, Recipient, Transfer, AccountNotFound, - AddRecipientBankError, TransferInvalidAmount, + AddRecipientBankError, TransferInvalidAmount, Loan, ) from .compat.weboob_tools_capabilities_bank_investments import create_french_liquidity from weboob.capabilities.base import NotAvailable, empty, Currency @@ -207,6 +207,7 @@ def on_load(self): class AccountsPage(LoggedPage, HTMLPage): def is_here(self): + # This id appears when there are no accounts (pro and pp) return not self.doc.xpath('//div[contains(@id, "alert-random")]') ACCOUNT_TYPES = {u'comptes courants': Account.TYPE_CHECKING, @@ -240,7 +241,8 @@ class item(ItemElement): load_details = Field('url') & AsyncLoad def condition(self): - return not self.is_external() and not 'automobile' in Field('url')(self) + # Ignore externally aggregated accounts and insurances: + return not self.is_external() and not any(x in Field('url')(self) for x in ('automobile', 'assurance/protection', 'assurance/comptes')) obj_label = CleanText('.//a[has-class("account--name")] | .//div[has-class("account--name")]') @@ -325,12 +327,51 @@ def validate(self, obj): return not Async('details', CleanText(u'//h4[contains(text(), "Établissement bancaire")]'))(self) and not \ Async('details', CleanText(u'//h4/div[contains(text(), "Établissement bancaire")]'))(self) - def iter_card_ids(self): - for tr in self.doc.xpath('//table[@class="table table--accounts"]/tr[has-class("table__line--account") and count(descendant::td) > 1 and @data-line-account-href]'): - url = Attr('.//a[@class="account--name"] | .//a[2]', 'href', default='')(tr) - m = re.search(r'/([a-z0-9]+)/carte/([a-z0-9]+)', url) - if m: - yield m.groups() + +class LoanPage(LoggedPage, HTMLPage): + + LOAN_TYPES = { + "PRÊT PERSONNEL": Account.TYPE_CONSUMER_CREDIT, + } + + @method + class get_loan(ItemElement): + + klass = Loan + + obj_id = CleanText('//h3[contains(@class, "account-number")]/strong') + obj_label = CleanText('//h2[contains(@class, "page-title__account")]//div[@class="account-edit-label"]/span') + obj_total_amount = CleanDecimal('//p[contains(text(), "Montant emprunt")]/span', replace_dots=True) + obj_currency = CleanCurrency('//p[contains(text(), "Montant emprunt")]/span') + obj_duration = CleanDecimal('//p[contains(text(), "Nombre prévisionnel d\'échéances restantes")]/span') + obj_rate = CleanDecimal('//p[contains(text(), "Taux nominal en vigueur du prêt")]/span') + obj_nb_payments_left = CleanDecimal('//p[contains(text(), "Nombre prévisionnel d\'échéances restantes")]/span') + obj_next_payment_amount = CleanDecimal('//p[contains(text(), "Montant de la prochaine échéance")]/span', replace_dots=True) + obj_nb_payments_total = CleanDecimal('//p[contains(text(), "Nombre d\'écheances totales") or contains(text(), "Nombre total d\'échéances")]/span') + obj_subscription_date = Date(CleanText('//p[contains(text(), "Date de départ du prêt")]/span'), parse_func=parse_french_date) + obj_maturity_date = Date(CleanText('//p[contains(text(), "Date prévisionnelle d\'échéance finale")]/span'), parse_func=parse_french_date) + + def obj_balance(self): + balance = CleanDecimal('//p[contains(text(), "Capital restant dû")]/span', replace_dots=True)(self) + if balance > 0: + balance *= -1 + return balance + + def obj_type(self): + _type = CleanText('//h2[contains(@class, "page-title__account")]//div[@class="account-edit-label"]/span') + return Map(_type, self.page.LOAN_TYPES, default=Account.TYPE_LOAN)(self) + + def obj_next_payment_date(self): + tmp = CleanText('//p[contains(text(), "Date de la prochaine échéance")]/span')(self) + if tmp == "-": + return NotAvailable + return Date(CleanText('//p[contains(text(), "Date de la prochaine échéance")]/span'), parse_func=parse_french_date)(self) + + +class NoAccountPage(LoggedPage, HTMLPage): + def is_here(self): + err = CleanText('//div[contains(@id, "alert-random")]/text()', children=False)(self.doc) + return "compte inconnu" in err.lower() class CardCalendarPage(LoggedPage, RawPage): @@ -577,7 +618,7 @@ def obj_unitvalue(self): return CleanDecimal(replace_dots=True, default=NotAvailable).filter((TableCell('unitvalue')(self)[0]).xpath('./span[not(@class)]')) def iter_investment(self): - valuation = CleanDecimal('//li[h4[contains(text(), "Solde Espèces")]]/h3', replace_dots=True, default=None)(self.doc) + valuation = CleanDecimal('//li/span[contains(text(), "Solde Espèces")]/following-sibling::span', replace_dots=True, default=None)(self.doc) if valuation: yield create_french_liquidity(valuation) @@ -762,10 +803,6 @@ def populate(self, accounts): accounts.extend(cards) -class LoanPage(LoggedPage, HTMLPage): - pass - - class ErrorPage(HTMLPage): def on_load(self): error = (Attr('//input[@required][@id="profile_lei_type_identifier"]', 'data-message', default=None)(self.doc) or @@ -810,35 +847,13 @@ class get_profile(ItemElement): class CardsNumberPage(LoggedPage, HTMLPage): def populate_cards_number(self, cards): - """ - Checking account ID used to match credit cards. - Label useful when several cards on the same checking account. - The card_details list contains tuples including the card number, - the hash of the card's parent account, and the name of the card. - """ - card_details = [ - (CleanText('.//span')(o), CleanText('./@data-account-key')(o), CleanText('.//p')(o)) - for o in self.doc.xpath('//div[contains(@class, "zoom-carousel__item text-center credit-card-carousel__item")]') - ] - - # Remove cards whose number ends with **** (means it is not activated yet) - card_details = [ - (number, account_hash, name) for (number, account_hash, name) in card_details - if not number.endswith('****') - ] - for card in cards: - match = [(number, account_hash, name) for (number, account_hash, name) in card_details if account_hash in card.url] - - if len(match) > 1: - # Several cards matched the same bank account, so we now try - # to match them using the name of the card holder: - card_username = card.label.split('-')[1].lstrip() - match = [(number, account_hash, name) for (number, account_hash, name) in match if name.startswith(card_username)] - - assert len(match) <= 1, "only one card should be matched, or zero if the card is not yet activated" - if len(match) == 1 : - card.number = match[0][0] + # The second hash of the card's url is used to get + # the card's hash on the HTML page: + card_url_hash = re.search('carte\/(.*)', card.url).group(1) + card_hash = CleanText('//nav[ul[li[a[contains(@href, "%s")]]]]/@data-card-key' % card_url_hash)(self.doc) + # With the card hash we can get the card number: + card.number = CleanText('//div[@data-card-key="%s"]/div/span' % card_hash)(self.doc) class HomePage(LoggedPage, HTMLPage): @@ -938,8 +953,9 @@ def submit_info(self, amount, label, exec_date): else: assert self.get_option(form.el.xpath('//select[@id="Characteristics_schedulingType"]')[0], 'Différé') == '2' form['Characteristics[schedulingType]'] = '2' - form['Characteristics[scheduledDate][day]'] = exec_date.strftime('%d') - form['Characteristics[scheduledDate][month]'] = exec_date.strftime('%m') + # If we let the 0 in the front of the month or the day like 02, the website will not interpret the good date + form['Characteristics[scheduledDate][day]'] = exec_date.strftime('%d').lstrip("0") + form['Characteristics[scheduledDate][month]'] = exec_date.strftime('%m').lstrip("0") form['Characteristics[scheduledDate][year]'] = exec_date.strftime('%Y') form['Characteristics[notice]'] = 'none' @@ -952,6 +968,9 @@ def on_load(self): if errors: raise TransferInvalidAmount(message=errors) + def need_refresh(self): + return not self.doc.xpath('//form[@name="Confirm"]//button[contains(text(), "Je valide")]') + @method class get_transfer(ItemElement): klass = Transfer @@ -1073,4 +1092,13 @@ def get_currency_list(self): class CurrencyConvertPage(JsonPage): def get_rate(self): if not 'error' in self.doc: - return round(self.doc['rate'], 4) + return Decimal(str(self.doc['rate'])) + + +class AccountsErrorPage(LoggedPage, HTMLPage): + def is_here(self): + # some braindead error seems to affect many accounts until we retry + return '[E10008]' in CleanText('//div')(self.doc) + + def on_load(self): + raise BrowserUnavailable() diff --git a/modules/bp/pages/transfer.py b/modules/bp/pages/transfer.py index 0312424300..2cbe7167a2 100644 --- a/modules/bp/pages/transfer.py +++ b/modules/bp/pages/transfer.py @@ -32,6 +32,7 @@ from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.tools.capabilities.bank.iban import is_iban_valid from weboob.tools.value import Value +from weboob.exceptions import BrowserUnavailable from .base import MyHTMLPage @@ -193,6 +194,10 @@ def handle_response(self, transfer): class CreateRecipient(LoggedPage, MyHTMLPage): + def on_load(self): + if self.doc.xpath(u'//h1[contains(text(), "Service Désactivé")]'): + raise BrowserUnavailable(CleanText('//p[img[@title="attention"]]/text()')(self.doc)) + def choose_country(self, recipient, is_bp_account): # if this is present, we can't add recipient currently more_security_needed = self.doc.xpath(u'//iframe[@title="Gestion de compte par Internet"]') @@ -223,7 +228,7 @@ def populate(self, recipient): class ValidateRecipient(LoggedPage, MyHTMLPage): def is_bp_account(self): msg = CleanText('//span[has-class("app_erreur")]')(self.doc) - return msg == u'Le n° de compte que vous avez saisi appartient à La Banque Postale, veuillez vérifier votre saisie.' + return u'Le n° de compte que vous avez saisi appartient à La Banque Postale, veuillez vérifier votre saisie.' in msg def get_confirm_link(self): return Link('//a[@title="confirmer la creation"]')(self.doc) diff --git a/modules/bred/bred/browser.py b/modules/bred/bred/browser.py index 47818ab48e..5edbeda9f7 100644 --- a/modules/bred/bred/browser.py +++ b/modules/bred/bred/browser.py @@ -49,7 +49,7 @@ class BredBrowser(LoginBrowser): '/pages-gestion-des-erreurs/erreur-technique', '/pages-gestion-des-erreurs/message-tiers-oppose', ErrorPage) universe = URL('/transactionnel/services/applications/menu/getMenuUnivers', UniversePage) - token = URL('/transactionnel/services/rest/User/nonce\?random=(?P.*)', TokenPage) + token = URL(r'/transactionnel/services/rest/User/nonce\?random=(?P.*)', TokenPage) move_universe = URL('/transactionnel/services/applications/listes/(?P.*)/default', MoveUniversePage) switch = URL('/transactionnel/services/rest/User/switch', SwitchPage) loans = URL('/transactionnel/services/applications/prets/liste', LoansPage) diff --git a/modules/bred/bred/pages.py b/modules/bred/bred/pages.py index eb5bb2d775..64bcd833aa 100644 --- a/modules/bred/bred/pages.py +++ b/modules/bred/bred/pages.py @@ -35,20 +35,20 @@ class Transaction(FrenchTransaction): - PATTERNS = [(re.compile('^.*Virement (?P.*)'), FrenchTransaction.TYPE_TRANSFER), - (re.compile(u'PRELEV SEPA (?P.*)'), FrenchTransaction.TYPE_ORDER), - (re.compile(u'.*Prélèvement.*'), FrenchTransaction.TYPE_ORDER), - (re.compile(u'^(REGL|Rgt)(?P.*)'), FrenchTransaction.TYPE_ORDER), - (re.compile('^(?P.*) Carte \d+\s+ LE (?P
\d{2})/(?P\d{2})/(?P\d{2})'), + PATTERNS = [(re.compile(r'^.*Virement (?P.*)'), FrenchTransaction.TYPE_TRANSFER), + (re.compile(r'PRELEV SEPA (?P.*)'), FrenchTransaction.TYPE_ORDER), + (re.compile(r'.*Prélèvement.*'), FrenchTransaction.TYPE_ORDER), + (re.compile(r'^(REGL|Rgt)(?P.*)'), FrenchTransaction.TYPE_ORDER), + (re.compile(r'^(?P.*) Carte \d+\s+ LE (?P
\d{2})/(?P\d{2})/(?P\d{2})'), FrenchTransaction.TYPE_CARD), - (re.compile(u'^Débit mensuel.*'), FrenchTransaction.TYPE_CARD_SUMMARY), - (re.compile(u"^Retrait d'espèces à un DAB (?P.*) CARTE [X\d]+ LE (?P
\d{2})/(?P\d{2})/(?P\d{2})"), + (re.compile(r'^Débit mensuel.*'), FrenchTransaction.TYPE_CARD_SUMMARY), + (re.compile(r"^Retrait d'espèces à un DAB (?P.*) CARTE [X\d]+ LE (?P
\d{2})/(?P\d{2})/(?P\d{2})"), FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile(u'^Paiement de chèque (?P.*)'), FrenchTransaction.TYPE_CHECK), - (re.compile(u'^(Cotisation|Intérêts) (?P.*)'), FrenchTransaction.TYPE_BANK), - (re.compile(u'^(Remise Chèque|Remise de chèque)\s*(?P.*)'), FrenchTransaction.TYPE_DEPOSIT), - (re.compile('^Versement (?P.*)'), FrenchTransaction.TYPE_DEPOSIT), - (re.compile('^(?P.*)LE (?P
\d{2})/(?P\d{2})/(?P\d{2})\s*(?P.*)'), + (re.compile(r'^Paiement de chèque (?P.*)'), FrenchTransaction.TYPE_CHECK), + (re.compile(r'^(Cotisation|Intérêts) (?P.*)'), FrenchTransaction.TYPE_BANK), + (re.compile(r'^(Remise Chèque|Remise de chèque)\s*(?P.*)'), FrenchTransaction.TYPE_DEPOSIT), + (re.compile(r'^Versement (?P.*)'), FrenchTransaction.TYPE_DEPOSIT), + (re.compile(r'^(?P.*)LE (?P
\d{2})/(?P\d{2})/(?P\d{2})\s*(?P.*)'), FrenchTransaction.TYPE_UNKNOWN), ] @@ -105,6 +105,7 @@ def iter_loans(self, current_univers): class AccountsPage(MyJsonPage): ACCOUNT_TYPES = { '000': Account.TYPE_CHECKING, # Compte à vue + '001': Account.TYPE_SAVINGS, # Livret Ile de France '011': Account.TYPE_CARD, # Carte bancaire '020': Account.TYPE_SAVINGS, # Compte sur livret '021': Account.TYPE_SAVINGS, @@ -221,8 +222,8 @@ def iter_history(self, account, operation_list, seen, today, coming): seen.add(t.id) d = date.fromtimestamp(op.get('dateDebit', op.get('dateOperation'))/1000) - op['details'] = [re.sub('\s+', ' ', i).replace('\x00', '') for i in op['details'] if i] # sometimes they put "null" elements... - label = re.sub('\s+', ' ', op['libelle']).replace('\x00', '') + op['details'] = [re.sub(r'\s+', ' ', i).replace('\x00', '') for i in op['details'] if i] # sometimes they put "null" elements... + label = re.sub(r'\s+', ' ', op['libelle']).replace('\x00', '') raw = ' '.join([label] + op['details']) t.rdate = date.fromtimestamp(op.get('dateOperation', op.get('dateDebit'))/1000) vdate = date.fromtimestamp(op.get('dateValeur', op.get('dateDebit', op.get('dateOperation')))/1000) diff --git a/modules/caissedepargne/browser.py b/modules/caissedepargne/browser.py index 86dda7134a..fc2e589396 100644 --- a/modules/caissedepargne/browser.py +++ b/modules/caissedepargne/browser.py @@ -33,7 +33,7 @@ from weboob.capabilities.base import NotAvailable from weboob.capabilities.profile import Profile from weboob.browser.exceptions import BrowserHTTPNotFound, ClientError -from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable +from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable, BrowserHTTPError from weboob.tools.capabilities.bank.transactions import sorted_transactions, FrenchTransaction from .compat.weboob_tools_capabilities_bank_investments import create_french_liquidity from weboob.tools.compat import urljoin @@ -222,7 +222,20 @@ def do_login(self): # Retrieve the list of types: can contain a single type or more # - when there is a single type: all the information are available # - when there are several types: an additional request is needed - data = self.login.go(login=self.username).get_response() + try: + connection = self.login.go(login=self.username) + # The website crash sometime when the module is not on caissedepargne (on linebourse, for exemple). + # The module think is not connected anymore, so we go to the home logged page. If there are no error + # that mean we are already logged and now, on the good website + + except ValueError: + self.home.go() + if self.home.is_here(): + return + # If that not the case, that's an other error that we have to correct + raise + + data = connection.get_response() if data is None: raise BrowserIncorrectPassword() @@ -242,7 +255,7 @@ def do_login(self): assert data is not None - if data.get('authMode', '') == 'redirect': # the connection type EU could also be used as a criteria + if data.get('authMode', '') == 'redirect': # the connection type EU could also be used as a criteria raise SiteSwitch('cenet') typeAccount = data['account'][0] @@ -250,12 +263,12 @@ def do_login(self): if self.multi_type: assert typeAccount == self.typeAccount - idTokenClavier = data['keyboard']['Id'] + id_token_clavier = data['keyboard']['Id'] vk = CaissedepargneKeyboard(data['keyboard']['ImageClavier'], data['keyboard']['Num']['string']) newCodeConf = vk.get_string_code(self.password) playload = { - 'idTokenClavier': idTokenClavier, + 'idTokenClavier': id_token_clavier, 'newCodeConf': newCodeConf, 'auth_mode': 'ajax', 'nuusager': self.nuser.encode('utf-8'), @@ -294,7 +307,7 @@ def do_login(self): def loans_conso(self): days = ('Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun') - month = ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul' , 'Aug', 'Sep', 'Oct', 'Nov', 'Dec') + month = ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec') now = datetime.datetime.today() d = '%s %s %s %s:%s:%s GMT 0100 (CET)' % (days[now.weekday()], month[now.month - 1], now.year, now.hour, format(now.minute, "02"), now.second) if self.home.is_here(): @@ -302,7 +315,7 @@ def loans_conso(self): if msg: self.logger.warning('%s' % msg) return None - self.cons_loan.go(datepourie = d) + self.cons_loan.go(datepourie=d) return self.page.get_conso() # On home page there is a list of "measure" links, each one leading to one person accounts list. @@ -375,7 +388,7 @@ def get_accounts_list(self): self.page.submit() # For Caisse d'Epargne's connections - if self.url.startswith('https://www.caisse-epargne.offrebourse.com') : + if self.url.startswith('https://www.caisse-epargne.offrebourse.com'): if self.page.is_error(): continue @@ -402,7 +415,6 @@ def get_accounts_list(self): for account in self.accounts: yield account - @need_login def get_loans_list(self): if self.loans is None: @@ -504,12 +516,17 @@ def _get_history_invests(self, account): self.home.go() self.page.go_history(account._info) - if account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_PERP): if "MILLEVIE" in account.label: self.page.go_life_insurance(account) label = account.label.split()[-1] - self.natixis_life_ins_his.go(id1=label[:3], id2=label[3:5], id3=account.id) + try: + self.natixis_life_ins_his.go(id1=label[:3], id2=label[3:5], id3=account.id) + except BrowserHTTPError as e: + if e.response.status_code == 500: + error = json.loads(e.response.text) + raise BrowserUnavailable(error["error"]) + raise return sorted_transactions(self.page.get_history()) if account.label.startswith('NUANCES ') or account.label in self.insurance_accounts: @@ -590,7 +607,7 @@ def get_investment(self, account): return self.page.submit() - # For Credit Cooperatif's connections + # For Credit Cooperatif's connections if self.url.startswith('https://www.offrebourse.com'): self.update_linebourse_token() for investment in self.linebourse.iter_investments(): @@ -745,7 +762,7 @@ def new_recipient(self, recipient, **params): if 'otp_sms' in params: transactionid = re.search(r'transactionID=(.*)', self.page.url).group(1) - self.request_sms.go(param = transactionid) + self.request_sms.go(param=transactionid) validation = {} validation['validate'] = {} key = self.page.validate_key() @@ -755,11 +772,11 @@ def new_recipient(self, recipient, **params): inner_param['type'] = 'SMS' inner_param['otp_sms'] = params['otp_sms'] validation['validate'][key].append(inner_param) - headers = {'Content-Type': 'application/json', 'Accept':'application/json, text/plain, */*'} - self.location(self.url +'/step' , data=json.dumps(validation), headers=headers) + headers = {'Content-Type': 'application/json', 'Accept': 'application/json, text/plain, */*'} + self.location(self.url + '/step', data=json.dumps(validation), headers=headers) saml = self.page.get_saml() action = self.page.get_action() - self.location(action, data={'SAMLResponse':saml}) + self.location(action, data={'SAMLResponse': saml}) if self.authent.is_here(): self.page.go_on() return self.facto_post_recip(recipient) @@ -773,8 +790,13 @@ def new_recipient(self, recipient, **params): if self.sms_option.is_here(): self.is_send_sms = True - raise AddRecipientStep(self.get_recipient_obj(recipient), Value('otp_sms', - label=u'Veuillez renseigner le mot de passe unique qui vous a été envoyé par SMS dans le champ réponse.')) + raise AddRecipientStep( + self.get_recipient_obj(recipient), + Value( + 'otp_sms', + label='Veuillez renseigner le mot de passe unique qui vous a été envoyé par SMS dans le champ réponse.' + ) + ) # pro add recipient. elif self.page.need_auth(): diff --git a/modules/caissedepargne/pages.py b/modules/caissedepargne/pages.py index 5159dd4fcf..683c4bf537 100644 --- a/modules/caissedepargne/pages.py +++ b/modules/caissedepargne/pages.py @@ -43,7 +43,7 @@ from weboob.tools.capabilities.bank.iban import is_rib_valid, rib2iban, is_iban_valid from weboob.tools.captcha.virtkeyboard import GridVirtKeyboard from weboob.tools.compat import unicode -from weboob.exceptions import NoAccountsException, BrowserUnavailable +from weboob.exceptions import NoAccountsException, BrowserUnavailable, ActionNeeded from weboob.browser.filters.json import Dict def MyDecimal(*args, **kwargs): @@ -67,6 +67,11 @@ def float_to_decimal(f): class LoginPage(JsonPage): + def on_load(self): + error_msg = self.doc.get('error') + if error_msg and 'Le service est momentanément indisponible' in error_msg: + raise BrowserUnavailable(error_msg) + def get_response(self): return self.doc @@ -202,6 +207,14 @@ def build_doc(self, content): return super(IndexPage, self).build_doc(content) def on_load(self): + + # For now, we have to handle this because after this warning message, + # the user is disconnected (even if all others account are reachable) + if 'NA_OIC_QCF' in self.browser.url: + message = CleanText(self.doc.xpath('//span[contains(@id, "MM_NA_OIC_QCF")]/p'))(self) + if message and "investissement financier (QCF) n’est plus valide à ce jour ou que vous avez refusé d’y répondre" in message: + raise ActionNeeded(message) + # This page is sometimes an useless step to the market website. bourse_link = Link(u'//div[@id="MM_COMPTE_TITRE_pnlbourseoic"]//a[contains(text(), "Accédez à la consultation")]', default=None)(self.doc) @@ -1028,6 +1041,12 @@ def go_add_recipient(self): class TransferConfirmPage(TransferErrorPage, IndexPage): + def build_doc(self, content): + # The page have some tags in the label content (spaces added each 40 characters if the character is not a space). + # Consequently the label can't be matched with the original one. We delete these tags. + content = content.replace(b'', b'') + return super(TransferErrorPage, self).build_doc(content) + def is_here(self): return bool(CleanText(u'//h2[contains(text(), "Confirmer mon virement")]')(self.doc)) diff --git a/modules/cmso/par/pages.py b/modules/cmso/par/pages.py index a7ee588179..df63313186 100644 --- a/modules/cmso/par/pages.py +++ b/modules/cmso/par/pages.py @@ -77,6 +77,7 @@ class AccountsPage(LoggedPage, JsonPage): ('librissime', Account.TYPE_SAVINGS), ('epargne logement', Account.TYPE_SAVINGS), ('plan bleu', Account.TYPE_SAVINGS), + ('capital plus', Account.TYPE_SAVINGS), ]) def get_keys(self): @@ -272,7 +273,7 @@ def obj_next_payment_date(self): return NotAvailable def obj_balance(self): - return -abs(CleanDecimal().filter(Dict('montantRestant', default=None)(self) or Dict('montantUtilise')(self))) + return -abs(CleanDecimal().filter(self.el.get('montantRestant', self.el.get('montantUtilise')))) # only for revolving loans obj_available_amount = CleanDecimal(Dict('montantDisponible', default=None), default=NotAvailable) diff --git a/modules/cmso/pro/browser.py b/modules/cmso/pro/browser.py index 8e1aaddbab..9d50d66039 100644 --- a/modules/cmso/pro/browser.py +++ b/modules/cmso/pro/browser.py @@ -26,7 +26,7 @@ from weboob.tools.capabilities.bank.transactions import sorted_transactions from weboob.capabilities.base import find_object from weboob.capabilities.bank import Account -from weboob.exceptions import BrowserHTTPError, BrowserIncorrectPassword, ActionNeeded +from weboob.exceptions import BrowserHTTPError, BrowserIncorrectPassword, ActionNeeded, BrowserUnavailable from weboob.browser import LoginBrowser, URL, need_login from weboob.browser.exceptions import ServerError from weboob.tools.date import LinearDateGuesser @@ -109,17 +109,30 @@ def go_with_ssodomi(self, path): if path.startswith('/domiweb'): path = path[len('/domiweb'):] - url = self.ssoDomiweb.go(website=self.website, - headers={'Authentication': 'Bearer %s' % self.token, - 'Authorization': 'Bearer %s' % self.csrf, - 'X-Csrf-Token': self.csrf, - 'Accept': 'application/json', - 'X-REFERER-TOKEN': 'RWDPRO', - 'X-ARKEA-EFS': self.arkea, - 'ADRIM': 'isAjax:true', - }, - json={'rwdStyle': 'true', - 'service': path}).get_sso_url() + headers = { + 'Authentication': 'Bearer %s' % self.token, + 'Authorization': 'Bearer %s' % self.csrf, + 'X-Csrf-Token': self.csrf, + 'Accept': 'application/json', + 'X-REFERER-TOKEN': 'RWDPRO', + 'X-ARKEA-EFS': self.arkea, + 'ADRIM': 'isAjax:true', + } + + json = { + 'rwdStyle': 'true', + 'service': path, + } + + try: + url = self.ssoDomiweb.go(website=self.website, + headers=headers, + json=json).get_sso_url() + except BrowserHTTPError as e: + if e.response.status_code == 500: + raise BrowserUnavailable() + raise + page = self.location(url).page # each time we get a new csrf we store it because it can be used in further navigation self.last_csrf = self.url.split('csrf=')[1] @@ -213,7 +226,6 @@ def iter_history(self, account): if len(date_range_list): date_range_list = [self._build_next_date_range(date_range_list[0])] + date_range_list - for date_range in date_range_list: date_guesser = LinearDateGuesser(datetime.datetime.strptime(date_range[10:], "%d/%m/%Y")) try: diff --git a/modules/cragr/web/pages.py b/modules/cragr/web/pages.py index 0890dac326..fc60058607 100644 --- a/modules/cragr/web/pages.py +++ b/modules/cragr/web/pages.py @@ -202,6 +202,7 @@ class AccountsPage(MyLoggedPage, BasePage): TYPES = {u'CCHQ': Account.TYPE_CHECKING, # par u'CCOU': Account.TYPE_CHECKING, # pro u'AUTO ENTRP': Account.TYPE_CHECKING, # pro + u'DEVISE USD': Account.TYPE_CHECKING, u'EKO' : Account.TYPE_CHECKING, u'DAV NANTI': Account.TYPE_SAVINGS, u'LIV A': Account.TYPE_SAVINGS, @@ -246,7 +247,7 @@ class AccountsPage(MyLoggedPage, BasePage): @method class iter_accounts(TableElement): head_xpath = '//table[@class="ca-table"]//tr/th' - item_xpath = '//table[@class="ca-table"]//tr[contains(@class, "colcelligne")]' + item_xpath = '//table[@class="ca-table"]//tr[contains(@class, "colcelligne") or contains(@class, "autre-devise")]' col_id = 'N° de compte' col_label = 'Type de compte' @@ -390,8 +391,9 @@ def on_load(self): self.get_current() # sometimes the perimeter use " & " and sometimes " et " + # and can also be "&" instead of "et" (example: "metms" et "m&ms") if not (self.browser.current_perimeter in self.browser.perimeters or - self.browser.current_perimeter.replace(' et ', ' & ') in self.browser.perimeters): + self.browser.current_perimeter.replace('et', '&') in self.browser.perimeters): assert len(self.browser.perimeters) == 1 self.browser.perimeters.append(self.browser.current_perimeter) @@ -1209,15 +1211,19 @@ def parse_recipients(self, items, assume_internal=False): yield rcpt break elif opt.attrib['value'].startswith('E'): - rcpt = Recipient() - rcpt._index = opt.attrib['value'] - rcpt._raw_label = ' '.join(lines) - rcpt.category = 'Externe' - rcpt.label = lines[0] - rcpt.iban = lines[1].upper() - rcpt.id = rcpt.iban - rcpt.enabled_at = datetime.now().replace(microsecond=0) - yield rcpt + if len(lines) > 1: + # In some cases we observed beneficiaries without label, we skip them + rcpt = Recipient() + rcpt._index = opt.attrib['value'] + rcpt._raw_label = ' '.join(lines) + rcpt.category = 'Externe' + rcpt.label = lines[0] + rcpt.iban = lines[1].upper() + rcpt.id = rcpt.iban + rcpt.enabled_at = datetime.now().replace(microsecond=0) + yield rcpt + else: + self.logger.warning('The recipient associated with the iban %s has got no label' % lines[0]) def submit_accounts(self, account_id, recipient_id, amount, currency): emitters = [rcpt for rcpt in self.iter_emitters() if rcpt.id == account_id and not rcpt.iban] diff --git a/modules/creditdunord/browser.py b/modules/creditdunord/browser.py index c893504a84..ee319ee3bc 100644 --- a/modules/creditdunord/browser.py +++ b/modules/creditdunord/browser.py @@ -19,10 +19,11 @@ from __future__ import unicode_literals -from weboob.browser import LoginBrowser, URL, need_login +from weboob.browser.browsers import LoginBrowser, URL, need_login from weboob.exceptions import BrowserIncorrectPassword, BrowserPasswordExpired -from weboob.capabilities.bank import Account, Investment +from weboob.capabilities.bank import Account from weboob.capabilities.base import find_object +from .compat.weboob_tools_capabilities_bank_investments import create_french_liquidity from .pages import ( LoginPage, ProfilePage, AccountTypePage, AccountsPage, ProAccountsPage, TransactionsPage, IbanPage, RedirectPage, EntryPage, AVPage, ProIbanPage, @@ -152,20 +153,16 @@ def get_history(self, account, coming=False): @need_login def get_investment(self, account): if 'LIQUIDIT' in account.label: - inv = Investment() - inv.code = 'XX-Liquidity' - inv.label = 'Liquidité' - inv.valuation = account.balance - return [inv] + return [create_french_liquidity(account.balance)] if not account._inv: return [] if account.type in (Account.TYPE_MARKET, Account.TYPE_PEA): self.location(account._link, data=account._args) - if self.page.can_iter_investments(): + if self.page.can_iter_investments() and self.page.not_restrained(): return self.page.get_market_investment() - elif (account.type == Account.TYPE_LIFE_INSURANCE): + elif account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_CAPITALISATION): self.location(account._link, data=account._args) self.location(account._link.replace("_attente", "_detail_contrat_rep"), data=account._args) if self.page.can_iter_investments(): diff --git a/modules/creditdunord/compat/weboob_tools_capabilities_bank_investments.py b/modules/creditdunord/compat/weboob_tools_capabilities_bank_investments.py new file mode 100644 index 0000000000..2b68ff9e00 --- /dev/null +++ b/modules/creditdunord/compat/weboob_tools_capabilities_bank_investments.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Jonathan Schmidt +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.tools.compat import basestring +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.bank import Investment + +def is_isin_valid(isin): + """ + Méthode générale + Table de conversion des lettres en chiffres + A=10 B=11 C=12 D=13 E=14 F=15 G=16 H=17 I=18 + J=19 K=20 L=21 M=22 N=23 O=24 P=25 Q=26 R=27 + S=28 T=29 U=30 V=31 W=32 X=33 Y=34 Z=35 + + 1 - Mettre de côté la clé, qui servira de référence à la fin de la vérification. + 2 - Convertir toutes les lettres en nombres via la table de conversion ci-contre. Si le nombre obtenu est supérieur ou égal à 10, prendre les deux chiffres du nombre séparément (exemple : 27 devient 2 et 7). + 3 - Pour chaque chiffre, multiplier sa valeur par deux si sa position est impaire en partant de la droite. Si le nombre obtenu est supérieur ou égal à 10, garder les deux chiffres du nombre séparément (exemple : 14 devient 1 et 4). + 4 - Faire la somme de tous les chiffres. + 5 - Soustraire cette somme de la dizaine supérieure ou égale la plus proche (exemples : si la somme vaut 22, la dizaine « supérieure ou égale » est 30, et la clé vaut donc 8 ; si la somme vaut 30, la dizaine « supérieure ou égale » est 30, et la clé vaut 0 ; si la somme vaut 31, la dizaine « supérieure ou égale » est 40, et la clé vaut 9). + 6 - Comparer la valeur obtenue à la clé mise initialement de côté. + + Étapes 1 et 2 : + F R 0 0 0 3 5 0 0 0 0 (+ 8 : clé) + 15 27 0 0 0 3 5 0 0 0 0 + + Étape 3 : le traitement se fait sur des chiffres + 1 5 2 7 0 0 0 3 5 0 0 0 0 + I P I P I P I P I P I P I : position en partant de la droite (P = Pair, I = Impair) + 2 1 2 1 2 1 2 1 2 1 2 1 2 : coefficient multiplicateur + + 2 5 4 7 0 0 0 3 10 0 0 0 0 : résultat + + Étape 4 : + 2 + 5 + 4 + 7 + 0 + 0 + 0 + 3 + (1 + 0)+ 0 + 0 + 0 + 0 = 22 + + Étapes 5 et 6 : 30 - 22 = 8 (valeur de la clé) + """ + + if not isinstance(isin, basestring): + return False + if not re.match(r'^[A-Z]{2}[A-Z0-9]{9}\d$', isin): + return False + + isin_in_digits = ''.join(str(ord(x) - ord('A') + 10) if not x.isdigit() else x for x in isin[:-1]) + key = isin[-1:] + result = '' + for k, val in enumerate(isin_in_digits[::-1], start=1): + if k % 2 == 0: + result = ''.join((result, val)) + else: + result = ''.join((result, str(int(val)*2))) + return str(sum(int(x) for x in result) + int(key))[-1] == '0' + + +def create_french_liquidity(valuation): + """ + Automatically fills a liquidity investment with label, code and code_type. + """ + liquidity = Investment() + liquidity.label = "Liquidités" + liquidity.code = "XX-liquidity" + liquidity.code_type = NotAvailable + liquidity.valuation = valuation + return liquidity + diff --git a/modules/creditdunord/pages.py b/modules/creditdunord/pages.py index e3993a2563..399cdcb036 100755 --- a/modules/creditdunord/pages.py +++ b/modules/creditdunord/pages.py @@ -29,7 +29,7 @@ from weboob.browser.pages import HTMLPage, LoggedPage, JsonPage from weboob.browser.elements import method, ItemElement, TableElement -from weboob.browser.filters.standard import CleanText, Date, CleanDecimal, Regexp, Format, Field +from weboob.browser.filters.standard import CleanText, Date, CleanDecimal, Regexp, Format, Field, Eval from weboob.browser.filters.json import Dict from weboob.browser.filters.html import Attr, TableCell from weboob.exceptions import ActionNeeded, BrowserIncorrectPassword, BrowserUnavailable, BrowserPasswordExpired @@ -38,6 +38,7 @@ from weboob.capabilities.base import Currency, find_object from weboob.capabilities import NotAvailable from weboob.tools.capabilities.bank.transactions import FrenchTransaction +from weboob.tools.capabilities.bank.investments import is_isin_valid from weboob.tools.captcha.virtkeyboard import GridVirtKeyboard from weboob.tools.compat import quote, unicode from weboob.tools.json import json @@ -164,11 +165,13 @@ def get_account_type(self): class LabelsPage(LoggedPage, JsonPage): - def get_labels(self): - if not Dict('donnees')(self.doc) and Dict('commun/statut', default='')(self.doc) == 'nok': - # Dict('commun/statut') is only `GDPR` so we don't pass specific message. + def on_load(self): + if Dict('commun/statut', default='')(self.doc) == 'nok': + reason = Dict('commun/raison')(self.doc) + assert reason == 'GDPR', 'Labels page is not available with message %s' % reason raise ActionNeeded() + def get_labels(self): synthesis_labels = ["Synthèse"] loan_labels = ["Crédits en cours", "Crédits perso et immo", "Crédits"] for element in Dict('donnees/0/submenu')(self.doc): @@ -187,36 +190,36 @@ def get_profile(self): class CDNBasePage(HTMLPage): - def get_from_js(self, pattern, end_pattern, is_list=False): - """ - find a pattern in any javascript text - """ - for script in self.doc.xpath('//script'): - txt = script.text - if txt is None: - continue + def get_from_js(self, pattern, end_pattern, is_list=False): + """ + find a pattern in any javascript text + """ + for script in self.doc.xpath('//script'): + txt = script.text + if txt is None: + continue - start = txt.find(pattern) - if start < 0: - continue + start = txt.find(pattern) + if start < 0: + continue - values = [] - while start >= 0: - start += len(pattern) - end = txt.find(end_pattern, start) - values.append(txt[start:end]) + values = [] + while start >= 0: + start += len(pattern) + end = txt.find(end_pattern, start) + values.append(txt[start:end]) - if not is_list: - break + if not is_list: + break - start = txt.find(pattern, end) - return ','.join(values) + start = txt.find(pattern, end) + return ','.join(values) - def get_execution(self): - return self.get_from_js("name: 'execution', value: '", "'") + def get_execution(self): + return self.get_from_js("name: 'execution', value: '", "'") - def iban_go(self): - return '%s%s' % ('/vos-comptes/IPT/cdnProxyResource', self.get_from_js('C_PROXY.StaticResourceClientTranslation( "', '"')) + def iban_go(self): + return '%s%s' % ('/vos-comptes/IPT/cdnProxyResource', self.get_from_js('C_PROXY.StaticResourceClientTranslation( "', '"')) class AccountsPage(LoggedPage, CDNBasePage): @@ -230,6 +233,7 @@ class AccountsPage(LoggedPage, CDNBasePage): u'CARTE': Account.TYPE_CARD, u'COMPTE COURANT': Account.TYPE_CHECKING, u'CPT COURANT': Account.TYPE_CHECKING, + u'CONSEILLE RESIDENT': Account.TYPE_CHECKING, u'PEA': Account.TYPE_PEA, u'P.E.A': Account.TYPE_PEA, u'COMPTE ÉPARGNE': Account.TYPE_SAVINGS, @@ -240,12 +244,16 @@ class AccountsPage(LoggedPage, CDNBasePage): u"PLAN D'EPARGNE": Account.TYPE_SAVINGS, u'PLAN ÉPARGNE': Account.TYPE_SAVINGS, u'ASS.VIE': Account.TYPE_LIFE_INSURANCE, + u'BONS CAPI': Account.TYPE_CAPITALISATION, u'ÉTOILE AVANCE': Account.TYPE_LOAN, u'ETOILE AVANCE': Account.TYPE_LOAN, u'PRÊT': Account.TYPE_LOAN, u'CREDIT': Account.TYPE_LOAN, u'FACILINVEST': Account.TYPE_LOAN, u'TITRES': Account.TYPE_MARKET, + u'COMPTE TIT': Account.TYPE_MARKET, + u'PRDTS BLOQ. TIT': Account.TYPE_MARKET, + u'PRODUIT BLOQUE TIT': Account.TYPE_MARKET, u'COMPTE A TERME': Account.TYPE_DEPOSIT, } @@ -491,7 +499,7 @@ def get_list(self): else: self.logger.warning('The card account %s has no parent account' % a.id) - a._inv = False + a._inv = True if a.type == Account.TYPE_CHECKING: previous_checking_account = a @@ -637,36 +645,43 @@ def get_history(self, acc_type): def can_iter_investments(self): return 'Vous ne pouvez pas utiliser les fonctions de bourse.' not in CleanText('//div[@id="contenusavoir"]')(self.doc) - def get_market_investment(self): - if CleanText('//div[contains(text(), "restreint aux fonctions de bourse")]')(self.doc): - return + def not_restrained(self): + return not CleanText('//div[contains(text(), "restreint aux fonctions de bourse")]')(self.doc) - COL_LABEL = 0 - COL_QUANTITY = 1 - COL_UNITPRICE = 2 - COL_UNITVALUE = 3 - COL_VALUATION = 4 - COL_PERF = 5 + @method + class get_market_investment(TableElement): + # Fetch the tables with at least 5 head columns (browser adds a missing a ) + item_xpath = '//div[not(@id="PortefeuilleCV")]/table[@class="datas"][tr[@class="entete"][count(td)>4]]//tr[position()>1]' + head_xpath = '//div[not(@id="PortefeuilleCV")]/table[@class="datas"][tr[@class="entete"][count(td)>4]]//tr[@class="entete"]/td' - for table in self.doc.xpath('//div[not(@id="PortefeuilleCV")]/table[@class="datas"]'): - for tr in table.xpath('.//tr[not(@class="entete")]'): - cols = tr.findall('td') - if len(cols) < 7: - continue - delta = 0 - if len(cols) == 9: - delta = 1 - - inv = Investment() - inv.code = CleanText('.')(cols[COL_LABEL + delta].xpath('.//span')[1]).split(' ')[0].split(u'\xa0')[0] - inv.label = CleanText('.')(cols[COL_LABEL + delta].xpath('.//span')[0]) - inv.quantity = MyDecimal('.')(cols[COL_QUANTITY + delta]) - inv.unitprice = MyDecimal('.')(cols[COL_UNITPRICE + delta]) - inv.unitvalue = MyDecimal('.')(cols[COL_UNITVALUE + delta]) - inv.valuation = MyDecimal('.')(cols[COL_VALUATION + delta]) - inv.diff = MyDecimal('.')(cols[COL_PERF + delta]) - - yield inv + col_label = 'Valeur' + col_quantity = 'Quantité' + col_unitvalue = 'Cours' + col_valuation = 'Estimation' + col_portfolio_share = '%' + + class item(ItemElement): + klass = Investment + + obj_label = CleanText(TableCell('label')) + obj_valuation = MyDecimal(TableCell('valuation')) + obj_quantity = MyDecimal(TableCell('quantity')) + obj_unitvalue = MyDecimal(TableCell('unitvalue')) + obj_portfolio_share = Eval(lambda x: x / 100, MyDecimal(TableCell('portfolio_share'))) + + def obj_code(self): + for code in Field('label')(self).split(): + if is_isin_valid(code): + return code + return NotAvailable + + def obj_code_type(self): + if is_isin_valid(Field('code')(self)): + return Investment.CODE_TYPE_ISIN + return NotAvailable + + def condition(self): + return "Sous-total" not in Field('label')(self) @method class get_deposit_investment(TableElement): diff --git a/modules/creditmutuel/browser.py b/modules/creditmutuel/browser.py index 27633c2257..df5a7d00f4 100644 --- a/modules/creditmutuel/browser.py +++ b/modules/creditmutuel/browser.py @@ -48,7 +48,7 @@ LIAccountsPage, CardsActivityPage, CardsListPage, CardsOpePage, NewAccountsPage, InternalTransferPage, ExternalTransferPage, RevolvingLoanDetails, RevolvingLoansList, - ErrorPage, SubscriptionPage, CardsHistAvailable, CardPage2 + ErrorPage, SubscriptionPage, NewCardsListPage, CardPage2 ) @@ -126,8 +126,8 @@ class CreditMutuelBrowser(LoginBrowser, StatesMixin): cards_ope = URL(r'/(?P.*)fr/banque/pro/ENC_liste_oper', CardsOpePage) cards_ope2 = URL('/(?P.*)fr/banque/CRP8_SCIM_DEPCAR.aspx', CardPage2) - cards_hist_available = URL('/(?P.*)fr/banque/SCIM_default.aspx\?_tabi=C&_stack=SCIM_ListeActivityStep%3a%3a&_pid=ListeCartes&_fid=ChangeList&Data_ServiceListDatas_CurrentType=MyCards', CardsHistAvailable) - cards_hist_available2 = URL('/(?P.*)fr/banque/SCIM_default.aspx', CardsHistAvailable) + cards_hist_available = URL('/(?P.*)fr/banque/SCIM_default.aspx\?_tabi=C&_stack=SCIM_ListeActivityStep%3a%3a&_pid=ListeCartes&_fid=ChangeList&Data_ServiceListDatas_CurrentType=MyCards', NewCardsListPage) + cards_hist_available2 = URL('/(?P.*)fr/banque/SCIM_default.aspx', NewCardsListPage) internal_transfer = URL(r'/(?P.*)fr/banque/virements/vplw_vi.html', InternalTransferPage) external_transfer = URL(r'/(?P.*)fr/banque/virements/vplw_vee.html', ExternalTransferPage) @@ -184,17 +184,26 @@ def get_accounts_list(self): self.accounts_list = [] self.revolving_accounts = [] self.unavailablecards = [] - self.cards_histo_available = {} + self.cards_histo_available = [] + self.cards_list =[] # For some cards the validity information is only availaible on these 2 links self.cards_hist_available.go(subbank=self.currentSubBank) if self.cards_hist_available.is_here(): self.unavailablecards.extend(self.page.get_unavailable_cards()) - self.cards_histo_available.update(self.page.get_cards_list()) + for acc in self.page.iter_accounts(): + self.accounts_list.append(acc) + self.cards_list.append(acc) + self.cards_histo_available.append(acc.id) + self.cards_hist_available2.go(subbank=self.currentSubBank) if self.cards_hist_available2.is_here(): self.unavailablecards.extend(self.page.get_unavailable_cards()) - self.cards_histo_available.update(self.page.get_cards_list()) + for acc in self.page.iter_accounts(): + if acc not in self.cards_list: + self.accounts_list.append(acc) + self.cards_list.append(acc) + self.cards_histo_available.append(acc.id) for acc in self.revolving_loan_list.stay_or_go(subbank=self.currentSubBank).iter_accounts(): self.accounts_list.append(acc) @@ -206,7 +215,16 @@ def get_accounts_list(self): [self.page] if self.is_new_website else [] for company in companies: page = self.open(company).page if isinstance(company, basestring) else company - self.accounts_list.extend(page.iter_cards()) + for card in page.iter_cards(): + card2 = find_object(self.cards_list, id=card.id[:16]) + if card2: + # In order to keep the id of the card from the old space, we exchange the following values + card._link_id = card2._link_id + card._parent_id = card2._parent_id + card.coming = card2.coming + self.accounts_list.remove(card2) + self.accounts_list.append(card) + self.cards_list.append(card) # Populate accounts from old website if not self.is_new_website: @@ -224,10 +242,9 @@ def get_accounts_list(self): self.li.go(subbank=self.currentSubBank) self.accounts_list.extend(self.page.iter_li_accounts()) - for acc in self.accounts_list: - if acc.id[:16] in self.cards_histo_available: - # ex of ID card : 000000xxxxxx0000 - acc.parent = find_object(self.accounts_list, id=self.cards_histo_available[acc.id[:16]][2]) + for acc in self.cards_list: + if hasattr(acc, '_parent_id'): + acc.parent = find_object(self.accounts_list, id=acc._parent_id) excluded_label = ['etalis', 'valorisation totale'] self.accounts_list = [acc for acc in self.accounts_list if not any(w in acc.label.lower() for w in excluded_label)] @@ -338,16 +355,20 @@ def get_history(self, account): if len(account.id) >= 16 and account.id[:16] in self.cards_histo_available: # Check if '000000xxxxxx0000' card have an annual history - account._link_id = self.cards_histo_available[account.id[:16]][0] self.location(account._link_id) # The history of the card is available for 1 year with 1 month per page # Here we catch all the url needed to be the more compatible with the catch of merged subtransactions urlstogo = self.page.get_links() self.location(account._link_id) monthly_tr = [] + half_history = 'firstHalf' for url in urlstogo: self.location(url) - if 'GoMonthPrecedent' not in url: + if 'GoMonthPrecedent' in url: + # To reach the 6 last month of history you need to change this url parameter + # Moreover we are on a transition page where we see the 6 next month (no scrapping here) + half_history = 'secondHalf' + else: history = self.page.get_history() self.tr_date = self.page.get_date() if self.page.has_more_operations(): @@ -366,16 +387,16 @@ def get_history(self, account): m = re.search(r'fid=GoMonth&mois=(\d+)', self.url) if m: m = m.group(1) - self.location('CRP8_SCIM_DEPCAR.aspx?_tabi=C&a__itaret=as=SCIM_ListeActivityStep\%3a\%3a\%2fSCIM_ListeRouter%3a%3a&a__mncret=SCIM_LST&a__ecpid=EID2011&_stack=_remote::moiSelectionner={},moiAfficher=firstHalf,typeDepense=T&_pid=SCIM_DEPCAR_Details'.format(m), data=data) + self.location('CRP8_SCIM_DEPCAR.aspx?_tabi=C&a__itaret=as=SCIM_ListeActivityStep\%3a\%3a\%2fSCIM_ListeRouter%3a%3a&a__mncret=SCIM_LST&a__ecpid=EID2011&_stack=_remote::moiSelectionner={},moiAfficher={},typeDepense=T&_pid=SCIM_DEPCAR_Details'.format(m, half_history), data=data) else: self.location(self.url, data=data) if not self.page.has_more_operations_xml(): - history = self.page.iter_history_xml() - # We are now with an XML page with all the transactions of the months + history = self.page.iter_history_xml(date=self.tr_date) + # We are now with an XML page with all the transactions of the month break else: - history = self.page.get_history() + history = self.page.get_history(date=self.tr_date) merged_amount = 0 monthly_tr = [] @@ -480,7 +501,7 @@ def iter_recipients(self, origin_account): yield recipient @need_login - def init_transfer(self, account, to, amount, reason=None): + def init_transfer(self, account, to, amount, exec_date, reason=None): if to.category != 'Interne': self.external_transfer.go(subbank=self.currentSubBank) else: @@ -493,8 +514,8 @@ def init_transfer(self, account, to, amount, reason=None): break self.page.IS_PRO_PAGE = True self.page.RECIPIENT_STRING = 'data_input_indiceBen' - self.page.prepare_transfer(account, to, amount, reason) - return self.page.handle_response(account, to, amount, reason) + self.page.prepare_transfer(account, to, amount, reason, exec_date) + return self.page.handle_response(account, to, amount, reason, exec_date) @need_login def execute_transfer(self, transfer, **params): diff --git a/modules/creditmutuel/module.py b/modules/creditmutuel/module.py index 2b3aba7f8e..8845686f69 100644 --- a/modules/creditmutuel/module.py +++ b/modules/creditmutuel/module.py @@ -122,7 +122,7 @@ def init_transfer(self, transfer, **params): # drop characters that can crash website transfer.label = transfer.label.encode('cp1252', errors="ignore").decode('cp1252') - return self.browser.init_transfer(account, recipient, amount, transfer.label) + return self.browser.init_transfer(account, recipient, amount, transfer.exec_date, transfer.label) def execute_transfer(self, transfer, **params): return self.browser.execute_transfer(transfer) diff --git a/modules/creditmutuel/pages.py b/modules/creditmutuel/pages.py index 0a5a9f7bad..d1df2f2507 100644 --- a/modules/creditmutuel/pages.py +++ b/modules/creditmutuel/pages.py @@ -192,7 +192,7 @@ def filter(self, text): class Type(Filter): def filter(self, label): for pattern, actype in item_account_generic.TYPES.items(): - if label.startswith(pattern): + if pattern in label: return actype return Account.TYPE_UNKNOWN @@ -302,7 +302,7 @@ def parse(self, el): card_xpath = multiple_cards_xpath + ' | ' + single_card_xpath for elem in page.doc.xpath(card_xpath): card_id = Regexp(CleanText('.', symbols=' '), r'([\dx]{16})')(elem) - if card_id in self.page.browser.unavailablecards: + if card_id in self.page.browser.unavailablecards or card_id in [d.id for d in self.page.browser.cards_list]: raise SkipItem() if any(card_id in a.id for a in page.browser.accounts_list): @@ -593,14 +593,17 @@ def parse(self, el): self.handle_attr(attr, getattr(self, 'obj_%s' % attr)) setattr(card, attr, getattr(self.obj, attr)) - if _id in self.page.browser.cards_histo_available: - card.coming = self.page.browser.cards_histo_available[_id][1] - card._card_number = _id card.id = _id + card.number card.label = card.label.replace(' ', ' %s ' % _id) - + card2 = find_object(self.page.browser.cards_list, id=card.id[:16]) + if card2: + card._link_id = card2._link_id + card._parent_id = card2._parent_id + card.coming = card2.coming + self.page.browser.accounts_list.remove(card2) self.page.browser.accounts_list.append(card) + self.page.browser.cards_list.append(card) # Skip multi and expired cards if len(options) or len(page.doc.xpath('//span[@id="ERREUR"]')): @@ -611,7 +614,7 @@ def parse(self, el): xpath = '//table[has-class("liste")]/tbody/tr' active_card = CleanText('%s[td[text()="Active"]][1]/td[2]' % xpath, replace=[(' ', '')], default=None)(page.doc) - if not active_card and len(page.doc.xpath(xpath)) != 1: + if not active_card or len(page.doc.xpath(xpath)) != 1: raise SkipItem() self.env['id'] = active_card or CleanText('%s[1]/td[2]' % xpath, replace=[(' ', '')])(page.doc) @@ -893,6 +896,7 @@ class item(Transaction.TransactionElement): obj_raw = Transaction.Raw(Format("%s %s", CleanText(TableCell('commerce')), CleanText(TableCell('ville')))) obj_rdate = Field('vdate') + obj_date = Env('date') def obj_type(self): if not 'RELEVE' in CleanText('//td[contains(., "Aucun mouvement")]')(self): @@ -910,19 +914,8 @@ def obj_original_currency(self): if Field('original_amount')(self) and m: return m.group(2) - def obj_date(self): - debit_date = CleanText('//a[@id="C:L4"]')(self) - if "fin" in debit_date: - return self.page.browser.tr_date - m = re.search(r'(\d{2}/\d{2}/\d{4})', debit_date) - if m: - return Date().filter(re.search(r'(\d{2}/\d{2}/\d{4})', debit_date).group(1)) - def obj__is_coming(self): - debit_date = CleanText('//a[@id="C:L4"]')(self) - if "fin" in debit_date: - return True - if Date().filter(re.search(r'(\d{2}/\d{2}/\d{4})', debit_date).group(1)) > datetime.date(datetime.today()): + if Field('date')(self) > datetime.date(datetime.today()): return True return False @@ -975,6 +968,7 @@ class list_history(Transaction.TransactionsElement): class item(Transaction.TransactionElement): obj_raw = Transaction.Raw(Format("%s %s", CleanText(TableCell('commerce')), CleanText(TableCell('ville')))) obj_rdate = Field('vdate') + obj_date = Env('date') def obj_type(self): if not 'RELEVE' in CleanText('//td[contains(., "Aucun mouvement")]')(self): @@ -996,9 +990,6 @@ def obj__regroup(self): if "Regroupement" in CleanText('./td')(self): return Link('./td/span/a')(self) - def obj_date(self): - return self.page.browser.tr_date - def obj__is_coming(self): if Field('date')(self) > datetime.date(datetime.today()): return True @@ -1006,15 +997,14 @@ def obj__is_coming(self): def get_date(self): debit_date = CleanText(self.doc.xpath('//a[@id="C:L4"]'))(self) - if "fin" in debit_date: - m = re.search(r'fid=GoMonth&mois=(\d+)', self.browser.url) - y = re.search(r'annee=(\d+)', self.browser.url) - if m and y: - return date(int(y.group(1)), int(m.group(1)), 1) + relativedelta(day=31) - m = re.search(r'(\d{2}/\d{2}/\d{4})', debit_date) if m: return Date().filter(re.search(r'(\d{2}/\d{2}/\d{4})', debit_date).group(1)) + m = re.search(r'fid=GoMonth&mois=(\d+)', self.browser.url) + y = re.search(r'annee=(\d+)', self.browser.url) + if m and y: + return date(int(y.group(1)), int(m.group(1)), 1) + relativedelta(day=31) + assert False, 'No transaction date is found' def get_links(self): links = [] @@ -1340,11 +1330,12 @@ def get_transfer_form(self): # internal and external transfer form are differents return self.get_form(id='P:F', submit='//input[@type="submit" and contains(@value, "Valider")]') - def prepare_transfer(self, account, to, amount, reason): + def prepare_transfer(self, account, to, amount, reason, exec_date): form = self.get_transfer_form() form['data_input_indiceCompteADebiter'] = self.get_from_account_index(account.id) form[self.RECIPIENT_STRING] = self.get_to_account_index(to.id) form['[t:dbt%3adouble;]data_input_montant_value_0_'] = str(amount).replace('.', ',') + form['[t:dbt%3adate;]data_input_date'] = exec_date.strftime("%d/%m/%Y") form['[t:dbt%3astring;x(27)]data_input_libelleCompteDebite'] = reason form['[t:dbt%3astring;x(31)]data_input_motifCompteCredite'] = reason form['[t:dbt%3astring;x(31)]data_input_motifCompteCredite1'] = reason @@ -1362,7 +1353,8 @@ def check_errors(self): "Pour effectuer cette opération, vous devez passer par l’intermédiaire d’un compte courant", 'Montant maximum autorisé au crédit pour ce compte', 'Débit interdit sur ce compte', - 'Virement interdit sur compte clos',] + 'Virement interdit sur compte clos', + "L'intitulé du virement ne peut contenir le ou les caractères suivants",] for message in messages: if message in content: @@ -1384,7 +1376,6 @@ def check_data_consistency(self, account_id, recipient_id, amount, reason): exec_date = Date(Regexp(CleanText('//table[@summary]/tbody/tr[th[contains(text(), "Date")]]/td'), r'(\d{2}/\d{2}/\d{4})'), dayfirst=True)(self.doc) - assert exec_date == datetime.today().date() r_amount = CleanDecimal('//table[@summary]/tbody/tr[th[contains(text(), "Montant")]]/td', replace_dots=True)(self.doc) assert r_amount == Decimal(amount) @@ -1393,7 +1384,7 @@ def check_data_consistency(self, account_id, recipient_id, amount, reason): assert reason.upper()[:22].strip() in CleanText('//table[@summary]/tbody/tr[th[contains(text(), "Intitulé pour le compte à débiter")]]/td')(self.doc) return exec_date, r_amount, currency - def handle_response(self, account, recipient, amount, reason): + def handle_response(self, account, recipient, amount, reason, exec_date): self.check_errors() self.check_success() @@ -1701,21 +1692,54 @@ class iter_accounts(ListElement): class item_account(ItemElement): klass = Loan - obj_id = Regexp(Attr('.//a','href'), r'(\d{16})\d{2}$') + def condition(self): + return len(self.el.xpath('./td')) >= 5 + obj_label = CleanText('.//td[2]') obj_total_amount = MyDecimal('.//td[3]') obj_currency = FrenchTransaction.Currency(CleanText('.//td[3]')) obj_type = Account.TYPE_REVOLVING_CREDIT obj__is_inv = False obj__link_id = None + obj_number = Field('id') - load_details = Link('.//a') & AsyncLoad - obj_balance = Async('details') & MyDecimal( - Format('-%s',CleanText('//main[@id="ei_tpl_content"]/div/div[2]/table//tr[2]/td[1]'))) - obj_available_amount = Async('details') & MyDecimal('//main[@id="ei_tpl_content"]/div/div[2]/table//tr[3]/td[1]') + def obj_id(self): + if self.el.xpath('.//a') and not 'notes' in Attr('.//a','href')(self): + return Regexp(Attr('.//a','href'), r'(\d{16})\d{2}$')(self) + return Regexp(Field('label'), r'(\d+ \d+)')(self).replace(' ', '') + + def load_details(self): + self.async_load = False + if self.el.xpath('.//a') and not 'notes' in Attr('.//a','href')(self): + self.async_load = True + return self.browser.async_open(Attr('.//a','href')(self)) + return NotAvailable + + def obj_balance(self): + if self.async_load: + async_page = Async('details').loaded_page(self) + return MyDecimal( + Format('-%s',CleanText('//main[@id="ei_tpl_content"]/div/div[2]/table//tr[2]/td[1]')))(async_page) + return Field('used_amount')(self) + + def obj_available_amount(self): + if self.async_load: + async_page = Async('details').loaded_page(self) + return MyDecimal('//main[@id="ei_tpl_content"]/div/div[2]/table//tr[3]/td[1]')(async_page) + return NotAvailable + + def obj_used_amount(self): + if not self.async_load: + return MyDecimal(Regexp(CleanText('.//td[5]'), r'([\s\d-]+,\d+)'))(self) + + def obj_next_payment_date(self): + if not self.async_load: + return Date(Regexp(CleanText('.//td[4]'), r'(\d{2}/\d{2}/\d{2})'))(self) + + def obj_next_payment_amount(self): + if not self.async_load: + return MyDecimal(Regexp(CleanText('.//td[4]'), r'([\s\d-]+,\d+)'))(self) - def condition(self): - return CleanText('.//a', default=None)(self) class ErrorPage(HTMLPage): def on_load(self): @@ -1810,7 +1834,74 @@ def is_last_page(self): return False -class CardsHistAvailable(LoggedPage, HTMLPage): +class NewCardsListPage(LoggedPage, HTMLPage): + @method + class iter_accounts(ListElement): + item_xpath = '//li[@class="item"]' + + class item(ItemElement): + klass = Account + + def condition(self): + # Numerous cards are not differed card, we keep the card only if there is a coming + return CleanText('.//div[1]/p')(self) == 'Active' and 'Dépenses' in CleanText('.//tr[1]/td/a[contains(@id,"C:more-card")]')(self) + + obj_balance = 0 + obj_type = Account.TYPE_CARD + obj__new_space = True + obj__is_inv = False + load_details = Field('_link_id') & AsyncLoad + + def obj_currency(self): + curr = CleanText('.//tbody/tr[1]/td/span')(self) + return re.search(r' ([a-zA-Z]+)', curr).group(1) + + def obj_id(self): + m = re.search(r'\d{4} \d{2}XX XXXX \d{4}', CleanText('.//span')(self)) + assert m, 'Id card is not present' + return m.group(0).replace(' ', '').replace('X', 'x') + + def obj_label(self): + label = CleanText('.//span/span')(self) + return re.search(r'(.*) - ', label).group(1) + + def obj_coming(self): + coming = 0 + coming_xpath = self.el.xpath('.//tbody/tr/td/span') + if len(coming_xpath) >= 1: + for i in (1, 2): + href = Link('.//tr[%s]/td/a[contains(@id,"C:more-card")]' %(i))(self) + m = re.search(r'selectedMonthly=(.*)', href).group(1) + if date(int(m[-4:]), int(m[:-4]), 1) + relativedelta(day=31) > date.today(): + coming += CleanDecimal(coming_xpath[i-1], replace_dots=True)(self) + else: + # Sometimes only one month is available + href = Link('//tr/td/a[contains(@id,"C:more-card")]')(self) + m = re.search(r'selectedMonthly=(.*)', href).group(1) + if date(int(m[-4:]), int(m[:-4]), 1) + relativedelta(day=31) > date.today(): + coming += CleanDecimal(coming_xpath[0], replace_dots=True)(self) + return coming + + def obj__link_id(self): + return Link('.//a[contains(@id,"C:more-card")]')(self) + + def obj__parent_id(self): + return re.search(r'\d+', CleanText('./div/div/div/p', replace=[(' ', '')])(self)).group(0)[-16:] + + def parse(self, el): + # We have to reach the good page with the information of the type of card + async_page = Async('details').loaded_page(self) + card_type_page = Link('//div/ul/li/a[contains(text(), "Fonctions")]')(async_page.doc) + doc = self.page.browser.open(card_type_page).page.doc + card_type_line = doc.xpath('//tbody/tr[th[contains(text(), "Débit des paiements")]]') + if card_type_line: + if CleanText('./td')(card_type_line[0]) != 'Différé': + raise SkipItem() + elif doc.xpath('//div/p[contains(text(), "Vous n\'avez pas l\'autorisation")]'): + self.logger.warning("The user can't reach this page") + else: + assert False, 'xpath for card type information could have changed' + def get_unavailable_cards(self): cards = [] for card in self.doc.xpath('//li[@class="item"]'): @@ -1820,22 +1911,3 @@ def get_unavailable_cards(self): cards.append(m.group(0).replace(' ', '').replace('X', 'x')) return cards - def get_cards_list(self): - cards = {} - for card in self.doc.xpath('//li[@class="item"]'): - if not card.xpath('.//tbody/tr/td/span'): - # No coming/balance values and history link are available - continue - m = re.search(r'\d{4} \d{2}XX XXXX \d{4}', CleanText(card.xpath('.//span'))(self)) - assert m, 'Id card is not present' - id_card = m.group(0).replace(' ', '').replace('X', 'x') - - link = Link(card.xpath('.//a[contains(@id,"C:more-card")]'))(self) - coming_xpath = card.xpath('.//tbody/tr/td/span') - coming = CleanDecimal(coming_xpath[0], replace_dots=True)(self) - if len(coming_xpath) > 1: - coming += CleanDecimal(coming_xpath[1], replace_dots=True)(self) - parent_id = re.search(r'\d+', CleanText(card.xpath('./div/div/div/p'), replace=[(' ', '')])(self)).group(0)[-16:] or None - cards[id_card] = [link, coming, parent_id] - - return cards diff --git a/modules/fortuneo/browser.py b/modules/fortuneo/browser.py index 07410acd7a..95b8d62da1 100644 --- a/modules/fortuneo/browser.py +++ b/modules/fortuneo/browser.py @@ -207,12 +207,16 @@ def copy_recipient(self, recipient): def new_recipient(self, recipient, **params): if 'code' in params: - self.need_reload_state = None # to drop and use self.add_recipient_form instead in send_code() recipient_form = json.loads(self.add_recipient_form) self.send_code(recipient_form ,params['code']) - assert self.page.rcpt_after_sms() - return self.copy_recipient(recipient) + if self.page.rcpt_after_sms(): + self.need_reload_state = None + return self.copy_recipient(recipient) + elif self.page.is_code_expired(): + self.need_reload_state = True + raise AddRecipientStep(recipient, Value('code', label='Le code sécurité est expiré. Veuillez saisir le nouveau code reçu qui sera valable 5 minutes.')) + assert False, self.page.get_error() return self.new_recipient_before_otp(recipient, **params) @need_login diff --git a/modules/fortuneo/pages/accounts_list.py b/modules/fortuneo/pages/accounts_list.py index 0514264462..084194e8bb 100644 --- a/modules/fortuneo/pages/accounts_list.py +++ b/modules/fortuneo/pages/accounts_list.py @@ -346,7 +346,8 @@ def has_action_needed(self): //span[contains(text(), "Nouveau mot de passe")] | \ //span[contains(text(), "Renouvellement de votre mot de passe")] |\ //span[contains(text(), "Mieux vous connaître")] |\ - //span[contains(text(), "Souscrivez au Livret + en quelques clics")]' + //span[contains(text(), "Souscrivez au Livret + en quelques clics")] |\ + //p[@class="warning" and contains(text(), "Cette opération sensible doit être validée par un code sécurité envoyé par SMS ou serveur vocal")]' ) if warning: raise ActionNeeded(warning[0].text) diff --git a/modules/fortuneo/pages/transfer.py b/modules/fortuneo/pages/transfer.py index 9cbd23587d..e80c8f21e1 100644 --- a/modules/fortuneo/pages/transfer.py +++ b/modules/fortuneo/pages/transfer.py @@ -116,10 +116,16 @@ def get_send_code_form_input(self): form = self.get_form() return form + def is_code_expired(self): + return self.doc.xpath('//label[contains(text(), "Le code sécurité est expiré. Veuillez saisir le nouveau code reçu")]') + def rcpt_after_sms(self): return self.doc.xpath('//div[@class="confirmationAjoutCompteExterne"]\ /h2[contains(text(), "ajout de compte externe a bien été prise en compte")]') + def get_error(self): + return CleanText().filter(self.doc.xpath('//form[@id="CompteExterneActionForm"]//p[@class="container error"]//label[@class="error]')) + class RegisterTransferPage(LoggedPage, HTMLPage): @method diff --git a/modules/groupama/browser.py b/modules/groupama/browser.py index d227a66f23..b06b74fa58 100644 --- a/modules/groupama/browser.py +++ b/modules/groupama/browser.py @@ -17,13 +17,14 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +import re from weboob.browser import LoginBrowser, URL, need_login from weboob.exceptions import BrowserIncorrectPassword from weboob.capabilities.bank import Account from weboob.capabilities.base import empty -from .pages import LoginPage, AccountsPage, TransactionsPage, AVAccountPage, AVHistoryPage, FormPage, IbanPage +from .pages import LoginPage, AccountsPage, TransactionsPage, AVAccountPage, AVHistoryPage, FormPage, IbanPage, AvJPage __all__ = ['GroupamaBrowser'] @@ -39,8 +40,10 @@ class GroupamaBrowser(LoginBrowser): accounts = URL('/wps/myportal/TableauDeBord', AccountsPage) transactions = URL('/wps/myportal/!ut', TransactionsPage) av_account_form = URL('/wps/myportal/assurancevie/', FormPage) - av_account = URL('https://secure-rivage.(ganassurances|ganpatrimoine|groupama).fr/contratVie.rivage.syntheseContratEparUc.gsi', AVAccountPage) + av_account = URL('https://secure-rivage.(ganassurances|ganpatrimoine|groupama).fr/contratVie.rivage.syntheseContratEparUc.gsi', + '/front/vie/epargne/contrat/(.*)', AVAccountPage) av_history = URL('https://secure-rivage.(?P.*).fr/contratVie.rivage.mesOperations.gsi', AVHistoryPage) + av_secondary = URL('/api/ecli/vie/contrats/(?P.*)', AvJPage) def __init__(self, *args, **kwargs): super(GroupamaBrowser, self).__init__(*args, **kwargs) @@ -68,6 +71,13 @@ def get_accounts_list(self, balance=True, need_iban=False): if self.av_account_form.is_here(): self.page.av_account_form() account.balance, account.currency = self.page.get_av_balance() + # New page where some AV are stored + elif "front/vie/" in account._link: + link = re.search('contrat\/(.+)-Groupama', account._link) + if link: + self.av_secondary.go(id_contrat=link.group(1)) + account.balance, account.currency = self.page.get_av_balance() + self.accounts.stay_or_go() if account.balance or not balance: if account.type != Account.TYPE_LIFE_INSURANCE and need_iban: diff --git a/modules/groupama/pages.py b/modules/groupama/pages.py index fbf99ad762..b04b426bb9 100644 --- a/modules/groupama/pages.py +++ b/modules/groupama/pages.py @@ -25,7 +25,7 @@ from decimal import Decimal -from weboob.browser.pages import HTMLPage, pagination, LoggedPage, FormNotFound +from weboob.browser.pages import HTMLPage, pagination, LoggedPage, FormNotFound, JsonPage from weboob.browser.elements import method, TableElement, ItemElement from weboob.browser.filters.standard import Env, CleanDecimal, CleanText, Date, Regexp, Eval from weboob.browser.filters.html import Attr, Link, TableCell @@ -33,6 +33,7 @@ from weboob.capabilities.bank import Account, Investment from weboob.capabilities.base import NotAvailable from weboob.tools.capabilities.bank.transactions import FrenchTransaction +from weboob.browser.filters.json import Dict class LoginPage(HTMLPage): @@ -217,6 +218,13 @@ def condition(self): obj_code_type = Investment.CODE_TYPE_ISIN +class AvJPage(LoggedPage, JsonPage): + def get_av_balance(self): + balance = CleanDecimal(Dict('montant'))(self.doc) + currency = "EUR" + return balance, currency + + class AVHistoryPage(LoggedPage, HTMLPage): @method class get_av_history(TableElement): diff --git a/modules/hsbc/browser.py b/modules/hsbc/browser.py index 4f3255192e..b3e8d1a3ff 100644 --- a/modules/hsbc/browser.py +++ b/modules/hsbc/browser.py @@ -23,6 +23,7 @@ import ssl from datetime import timedelta, date from lxml.etree import XMLSyntaxError +from collections import OrderedDict from weboob.tools.date import LinearDateGuesser from weboob.capabilities.bank import Account, AccountNotFound @@ -34,8 +35,8 @@ from weboob.capabilities.base import find_object from .pages.account_pages import ( - AccountsPage, CBOperationPage, CPTOperationPage, LoginPage, AppGonePage, RibPage, - UnavailablePage, OtherPage, FrameContainer, ProfilePage, + AccountsPage, OwnersListPage, CBOperationPage, CPTOperationPage, LoginPage, + AppGonePage, RibPage, UnavailablePage, OtherPage, FrameContainer, ProfilePage, ) from .pages.life_insurances import ( LifeInsurancesPage, LifeInsurancePortal, LifeInsuranceMain, LifeInsuranceUseless, @@ -78,6 +79,7 @@ class HSBC(LoginBrowser): AppGonePage) rib = URL(r'/cgi-bin/emcgi', RibPage) accounts = URL(r'/cgi-bin/emcgi', AccountsPage) + owners_list = URL(r'/cgi-bin/emcgi', OwnersListPage) life_insurance_useless = URL(r'/cgi-bin/emcgi', LifeInsuranceUseless) profile = URL(r'/cgi-bin/emcgi', ProfilePage) unavailable = URL(r'/cgi-bin/emcgi', UnavailablePage) @@ -115,15 +117,16 @@ class HSBC(LoginBrowser): RetrieveUselessPage ) - # catch-all other_page = URL(r'/cgi-bin/emcgi', OtherPage) def __init__(self, username, password, secret, *args, **kwargs): super(HSBC, self).__init__(username, password, *args, **kwargs) - self.accounts_list = dict() + self.accounts_list = OrderedDict() + self.unique_accounts_list = dict() self.secret = secret self.PEA_LISTING = {} + self.owners = [] def load_state(self, state): return @@ -159,8 +162,9 @@ def do_login(self): if new_base_url in self.url: self.BASEURL = new_base_url - self.js_url = self.page.get_js_url() - home_url = self.page.get_frame() + if self.frame_page.is_here(): + home_url = self.page.get_frame() + self.js_url = self.page.get_js_url() if not home_url or not self.page.logged: raise BrowserIncorrectPassword() @@ -177,51 +181,99 @@ def go_post(self, url, data=None): url = url[:url.find('?')] self.location(url, data=q) - @need_login - def get_accounts_list(self): - if not self.accounts_list: - self.update_accounts_list() - - # go on cards page if there are cards accounts - for a in self.accounts_list.values(): - if a.type == Account.TYPE_CARD: - self.location(a.url) - break - - # get all couples (card, parent) on cards page - all_card_and_parent = [] - if self.cbPage.is_here(): - all_card_and_parent = self.page.get_all_parent_id() - self.go_post(self.js_url, data={'debr': 'COMPTES_PAN'}) - - # update cards parent and currency - for a in self.accounts_list.values(): - if a.type == Account.TYPE_CARD: - for card in all_card_and_parent: - if a.id in card[0].replace(' ', ''): - a.parent = find_object(self.accounts_list.values(), id=card[1]) - if a.parent and not a.currency: - a.currency = a.parent.currency - yield a + def go_to_owner_accounts(self, owner): + """ + The owners URLs change all the time so we must refresh them. + If we try to go to a person's accounts page while we are already + on this page, the website returns an empty page with the message + "Pas de TIERS", so we must always go to the owners list before + going to the owner's account page. + """ + if not self.owners_list.is_here(): + self.go_post(self.js_url, data={'debr': 'OPTIONS_TIE'}) + + if not self.owners_list.is_here(): + # Sometimes when we fetch info from a PEA account, the first POST + # fails and we are blocked on some owner's AccountsPage. + self.logger.warning('The owners list redirection failed, we must try again.') + self.go_post(self.js_url, data={'debr': 'OPTIONS_TIE'}) + + # Refresh owners URLs in case they changed: + self.owners = self.page.get_owners_urls() + self.go_post(self.owners[owner]) @need_login - def update_accounts_list(self, iban=True): - if self.accounts.is_here(): - self.go_post(self.js_url) + def iter_account_owners(self): + """ + Some connections have a "Compte de Tiers" section with several + people each having their own accounts. We must fetch the account + for each person and store the owner of each account. + """ + if self.unique_accounts_list: + for account in self.unique_accounts_list.values(): + yield account else: - data = {'debr': 'COMPTES_PAN'} - self.go_post(self.js_url, data=data) + self.go_post(self.js_url, data={'debr': 'OPTIONS_TIE'}) + if self.owners_list.is_here(): + self.owners = self.page.get_owners_urls() + + # self.accounts_list will be a dictionary of owners each + # containing a dictionary of the owner's accounts. + for owner in range(len(self.owners)): + self.accounts_list[owner] = {} + self.update_accounts_list(owner, True) + + # We must set an "_owner" attribute to each account. + for a in self.accounts_list[owner].values(): + a._owner = owner + + # go on cards page if there are cards accounts + for a in self.accounts_list[owner].values(): + if a.type == Account.TYPE_CARD: + self.location(a.url) + break + + # get all couples (card, parent) on cards page + all_card_and_parent = [] + if self.cbPage.is_here(): + all_card_and_parent = self.page.get_all_parent_id() + self.go_post(self.js_url, data={'debr': 'COMPTES_PAN'}) + + # update cards parent and currency + for a in self.accounts_list[owner].values(): + if a.type == Account.TYPE_CARD: + for card in all_card_and_parent: + if a.id in card[0].replace(' ', ''): + a.parent = find_object(self.accounts_list[owner].values(), id=card[1]) + if a.parent and not a.currency: + a.currency = a.parent.currency + + # We must get back to the owners list before moving to the next owner: + self.go_post(self.js_url, data={'debr': 'OPTIONS_TIE'}) + + # Fill a dictionary will all accounts without duplicating common accounts: + for owner in self.accounts_list.values(): + for account in owner.values(): + if account.id not in self.unique_accounts_list.keys(): + self.unique_accounts_list[account.id] = account + + for account in self.unique_accounts_list.values(): + yield account + @need_login + def update_accounts_list(self, owner, iban=True): + # Go to the owner's account page in case we are not there already: + self.go_to_owner_accounts(owner) for a in self.page.iter_spaces_account(): try: - self.accounts_list[a.id].url = a.url + self.accounts_list[owner][a.id].url = a.url except KeyError: - self.accounts_list[a.id] = a + self.accounts_list[owner][a.id] = a if iban: self.location(self.js_url, params={'debr': 'COMPTES_RIB'}) if self.rib.is_here(): - self.page.get_rib(self.accounts_list) + self.page.get_rib(self.accounts_list[owner]) @need_login def _quit_li_space(self): @@ -248,10 +300,9 @@ def _quit_li_space(self): @need_login def _go_to_life_insurance(self, account): self._quit_li_space() - self.go_post(account.url) - if self.frame_page.is_here() or self.life_insurance_useless.is_here() or self.life_not_found.is_here(): + if self.accounts.is_here() or self.frame_page.is_here() or self.life_insurance_useless.is_here() or self.life_not_found.is_here(): self.logger.warning('cannot go to life insurance %r', account) return False @@ -264,9 +315,8 @@ def _go_to_life_insurance(self, account): @need_login def get_history(self, account, coming=False, retry_li=True): self._quit_li_space() - - self.update_accounts_list(False) - account = self.accounts_list[account.id] + self.update_accounts_list(account._owner, False) + account = self.accounts_list[account._owner][account.id] if account.url is None: return [] @@ -307,18 +357,18 @@ def get_history(self, account, coming=False, retry_li=True): return history try: - self.go_post(self.accounts_list[account.id].url) + self.go_post(self.accounts_list[account._owner][account.id].url) # sometime go to hsbc life insurance space do logout except HTTPNotFound: self.app_gone = True self.do_logout() self.do_login() - # If we relogin on hsbc, all link have change + # If we relogin on hsbc, all links have changed if self.app_gone: self.app_gone = False - self.update_accounts_list() - self.location(self.accounts_list[account.id].url) + self.update_accounts_list(account._owner, False) + self.location(self.accounts_list[account._owner][account.id].url) if self.page is None: return [] @@ -326,6 +376,7 @@ def get_history(self, account, coming=False, retry_li=True): # for 'fusion' space if hasattr(account, '_is_form') and account._is_form: # go on accounts page to get account form + self.go_to_owner_accounts(account._owner) self.go_post(self.js_url, data={'debr': 'COMPTES_PAN'}) self.page.go_history_page(account) @@ -356,6 +407,8 @@ def _get_history(self): yield tr def get_investments(self, account, retry_li=True): + if not account.url: + raise NotImplementedError() if account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_CAPITALISATION): return self.get_life_investments(account, retry_li=retry_li) elif account.type == Account.TYPE_PEA: @@ -369,6 +422,8 @@ def get_investments(self, account, retry_li=True): raise NotImplementedError() def get_scpi_investments(self, account): + if not account.url: + raise NotImplementedError() # Clean account url m = re.search(r"'(.*)'", account.url) if m: @@ -377,6 +432,7 @@ def get_scpi_investments(self, account): account_url = account.url # Need to be on accounts page to go on scpi page + self.go_to_owner_accounts(account._owner) self.accounts.go() # Go on scpi page self.location(account_url) @@ -387,6 +443,7 @@ def get_scpi_investments(self, account): return self.page.iter_scpi_investment() def get_pea_investments(self, account): + self.go_to_owner_accounts(account._owner) assert account.type in (Account.TYPE_PEA, Account.TYPE_MARKET) # When invest balance is 0, there is not link to go on market page @@ -394,7 +451,10 @@ def get_pea_investments(self, account): return [] if not self.PEA_LISTING: - self._go_to_wealth_accounts() + # _go_to_wealth_accounts returns True if everything went well. + if not self._go_to_wealth_accounts(account): + self.logger.warning('Unable to connect to wealth accounts.') + return [] # Get account number without "EUR" account_id = re.search(r'\d{4,}', account.id).group(0) @@ -421,12 +481,9 @@ def get_pea_investments(self, account): return pea_invests def get_life_investments(self, account, retry_li=True): - self._quit_li_space() - - self.update_accounts_list(False) - account = self.accounts_list[account.id] - + self.update_accounts_list(account._owner, False) + account = self.accounts_list[account._owner][account.id] try: if not self._go_to_life_insurance(account): self._quit_li_space() @@ -452,16 +509,28 @@ def get_life_investments(self, account, retry_li=True): return investments - def _go_to_wealth_accounts(self): + def _go_to_wealth_accounts(self, account): if not hasattr(self.page, 'get_middle_frame_url'): # if we can catch the URL, we go directly, else we need to browse # the website - self.update_accounts_list() + self.update_accounts_list(account._owner, False) self.location(self.page.get_middle_frame_url()) + if self.page.get_patrimoine_url(): self.location(self.page.get_patrimoine_url()) self.page.go_next() + + if self.login.is_here(): + self.logger.warning('Connection to the Logon page failed, we must try again.') + self.do_login() + self.update_accounts_list(account._owner, False) + self.investment_form_page.go() + # If reloggin did not help accessing the wealth space, + # there is nothing more we can do to get there. + if not self.investment_form_page.is_here(): + return False + self.page.go_to_logon() helper = ProductViewHelper(self) # we need to go there to initialize the session @@ -469,9 +538,17 @@ def _go_to_wealth_accounts(self): self.PEA_LISTING['liquidities'] = list(helper.retrieve_liquidity()) self.PEA_LISTING['investments'] = list(helper.retrieve_invests()) self.connection.go() + return True @need_login def get_profile(self): + if not self.owners: + self.go_post(self.js_url, data={'debr': 'OPTIONS_TIE'}) + if self.owners_list.is_here(): + self.owners = self.page.get_owners_urls() + + # The main owner of the connection is always the first of the list: + self.go_to_owner_accounts(0) data = {'debr': 'PARAM'} self.go_post(self.js_url, data=data) return self.page.get_profile() diff --git a/modules/hsbc/module.py b/modules/hsbc/module.py index 1124ddcf0c..35fec58150 100644 --- a/modules/hsbc/module.py +++ b/modules/hsbc/module.py @@ -47,11 +47,11 @@ def create_default_browser(self): self.config['secret'].get()) def iter_accounts(self): - for account in self.browser.get_accounts_list(): + for account in self.browser.iter_account_owners(): yield account def get_account(self, _id): - return find_object(self.browser.get_accounts_list(), id=_id, error=AccountNotFound) + return find_object(self.browser.iter_account_owners(), id=_id, error=AccountNotFound) def iter_history(self, account): for tr in self.browser.get_history(account): diff --git a/modules/hsbc/pages/account_pages.py b/modules/hsbc/pages/account_pages.py index f129080835..089ba1ce94 100644 --- a/modules/hsbc/pages/account_pages.py +++ b/modules/hsbc/pages/account_pages.py @@ -82,40 +82,42 @@ def on_load(self): class AccountsType(Filter): PATTERNS = [ - ('c.aff', Account.TYPE_CHECKING), - ('pea', Account.TYPE_PEA), - ('invest', Account.TYPE_MARKET), - ('ptf', Account.TYPE_MARKET), - ('ldd', Account.TYPE_SAVINGS), - ('cel', Account.TYPE_SAVINGS), - ('pel', Account.TYPE_SAVINGS), - ('livret', Account.TYPE_SAVINGS), - ('livjeu', Account.TYPE_SAVINGS), - ('compte', Account.TYPE_CHECKING), - ('cpte', Account.TYPE_CHECKING), - ('scpi', Account.TYPE_MARKET), - ('account', Account.TYPE_CHECKING), - ('pret', Account.TYPE_LOAN), - ('vie', Account.TYPE_LIFE_INSURANCE), - ('strategie patr.', Account.TYPE_LIFE_INSURANCE), - ('essentiel', Account.TYPE_LIFE_INSURANCE), - ('elysee', Account.TYPE_LIFE_INSURANCE), - ('abondance', Account.TYPE_LIFE_INSURANCE), - ('ely. retraite', Account.TYPE_LIFE_INSURANCE), - ('lae option assurance', Account.TYPE_LIFE_INSURANCE), - ('carte ', Account.TYPE_CARD), - ('business ', Account.TYPE_CARD), - ('plan assur. innovat.', Account.TYPE_LIFE_INSURANCE), - ('hsbc evol pat transf', Account.TYPE_LIFE_INSURANCE), - ('hsbc evol pat capi', Account.TYPE_CAPITALISATION), - ('bourse libre', Account.TYPE_MARKET), - ('plurival', Account.TYPE_LIFE_INSURANCE), + (r'c\.aff', Account.TYPE_CHECKING), + (r'\bssmouv\b', Account.TYPE_CHECKING), + (r'\bpea\b', Account.TYPE_PEA), + (r'invest', Account.TYPE_MARKET), + (r'\bptf\b', Account.TYPE_MARKET), + (r'\bldd\b', Account.TYPE_SAVINGS), + (r'\bcel\b', Account.TYPE_SAVINGS), + (r'\bpel\b', Account.TYPE_SAVINGS), + (r'livret', Account.TYPE_SAVINGS), + (r'livjeu', Account.TYPE_SAVINGS), + (r'csljun', Account.TYPE_SAVINGS), + (r'compte', Account.TYPE_CHECKING), + (r'cpte', Account.TYPE_CHECKING), + (r'scpi', Account.TYPE_MARKET), + (r'account', Account.TYPE_CHECKING), + (r'\bpret\b', Account.TYPE_LOAN), + (r'\bvie\b', Account.TYPE_LIFE_INSURANCE), + (r'strategie patr.', Account.TYPE_LIFE_INSURANCE), + (r'essentiel', Account.TYPE_LIFE_INSURANCE), + (r'elysee', Account.TYPE_LIFE_INSURANCE), + (r'abondance', Account.TYPE_LIFE_INSURANCE), + (r'ely\. retraite', Account.TYPE_LIFE_INSURANCE), + (r'lae option assurance', Account.TYPE_LIFE_INSURANCE), + (r'carte ', Account.TYPE_CARD), + (r'business ', Account.TYPE_CARD), + (r'plan assur\. innovat\.', Account.TYPE_LIFE_INSURANCE), + (r'hsbc evol pat transf', Account.TYPE_LIFE_INSURANCE), + (r'hsbc evol pat capi', Account.TYPE_CAPITALISATION), + (r'bourse libre', Account.TYPE_MARKET), + (r'plurival', Account.TYPE_LIFE_INSURANCE), ] def filter(self, label): label = label.lower() for pattern, type in self.PATTERNS: - if pattern in label: + if re.search(pattern, label): return type return Account.TYPE_UNKNOWN @@ -158,11 +160,20 @@ class item(ItemElement): def condition(self): return len(self.el.xpath('./td')) > 2 - obj_label = Label(CleanText('./td[1]/a')) + # Some accounts have no in the first + def obj_label(self): + if self.el.xpath('./td[1]/a'): + return Label(CleanText('./td[1]/a'))(self) or 'Compte sans libellé' + return Label(CleanText('./td[1]'))(self) or 'Compte sans libellé' + obj_coming = Env('coming') obj_currency = FrenchTransaction.Currency('./td[2]') - obj_url = CleanText(AbsoluteLink('./td[1]/a'), replace=[('\n', '')]) + def obj_url(self): + # Accounts without an in the have no link + if self.el.xpath('./td[1]/a'): + return CleanText(AbsoluteLink('./td[1]/a'), default=None, replace=[('\n', '')])(self) + return None obj_type = AccountsType(Field('label')) obj_coming = NotAvailable @@ -222,6 +233,13 @@ def obj_id(self): return account_id +class OwnersListPage(AccountsPage): + is_here = '//h1[text()="Comptes de tiers"]' + + def get_owners_urls(self): + return self.doc.xpath('//div[@class="GoBack"]/a/@href') + + class RibPage(GenericLandingPage): def is_here(self): return bool(self.doc.xpath('//h1[contains(text(), "RIB/IBAN")]')) @@ -240,7 +258,8 @@ def get_rib(self, accounts): form = self.get_form(name="FORM_RIB") form['index_rib'] = str(nb+1) form.submit() - self.browser.page.link_rib(accounts) + if self.browser.rib.is_here(): + self.browser.page.link_rib(accounts) class Pagination(object): diff --git a/modules/hsbc/pages/investments.py b/modules/hsbc/pages/investments.py index 67a0e81a7a..b1e382d17d 100644 --- a/modules/hsbc/pages/investments.py +++ b/modules/hsbc/pages/investments.py @@ -9,6 +9,7 @@ from weboob.capabilities import NotAvailable from weboob.capabilities.bank import Account, Investment +from weboob.tools.capabilities.bank.investments import is_isin_valid from weboob.browser.elements import ItemElement, TableElement, DictElement, method from weboob.browser.pages import HTMLPage, JsonPage, LoggedPage @@ -359,9 +360,13 @@ class item(ItemElement): klass = Investment obj_label = CleanText(Dict('productName')) - obj_code = CleanText(Dict('productIdInformation/0/productAlternativeNumber')) - obj_code_type = Investment.CODE_TYPE_ISIN obj_quantity = CleanDecimal(Dict('holdingDetailInformation/0/productHoldingQuantityCount')) + obj_code = CleanText(Dict('productIdInformation/0/productAlternativeNumber'), replace=[('-FR', '')]) + + def obj_code_type(self): + if is_isin_valid(Field('code')(self)): + return Investment.CODE_TYPE_ISIN + return NotAvailable def obj_vdate(self): vdate = Dict('holdingDetailInformation/0/productPriceUpdateDate')(self) @@ -467,7 +472,9 @@ class item(ItemElement): def condition(self): return Dict('productTypeCode')(self) == 'INVCASH' - obj_label = CleanText(Dict('productShortName')) + obj_label = "Liquidités" + obj_code = "XX-liquidity" + obj_code_type = NotAvailable obj_valuation = CleanDecimal( Dict( 'holdingDetailInformation/0/holdingDetailMultipleCurrencyInformation/1' diff --git a/modules/ing/browser.py b/modules/ing/browser.py index 0808565444..89a9c0a07b 100644 --- a/modules/ing/browser.py +++ b/modules/ing/browser.py @@ -219,11 +219,6 @@ def get_accounts_on_space(self, space, get_iban=True): accounts_list.append(loan) yield loan - def get_coming_balance(self, account): - if account.type == Account.TYPE_CHECKING: - self.go_account_page(account) - return self.page.get_coming_balance() - return NotAvailable @need_login @start_with_main_site @@ -235,20 +230,17 @@ def get_accounts_list(self, space=None, get_iban=True): if space: for acc in self.get_accounts_on_space(space, get_iban=get_iban): - acc.coming = self.get_coming_balance(acc) yield acc elif self.multispace: for space in self.multispace: for acc in self.get_accounts_on_space(space, get_iban=get_iban): - acc.coming = self.get_coming_balance(acc) yield acc else: for acc in self.page.get_list(): acc._space = None if get_iban: self.get_iban(acc) - acc.coming = self.get_coming_balance(acc) yield acc for loan in self.iter_detailed_loans(): diff --git a/modules/ing/pages/accounts_list.py b/modules/ing/pages/accounts_list.py index 665160fc0c..687de82fc5 100644 --- a/modules/ing/pages/accounts_list.py +++ b/modules/ing/pages/accounts_list.py @@ -323,11 +323,6 @@ def load_space_page(self): self.fillup_form(form, r"\),\{(.*)\},'", on_click) form.submit() - def get_coming_balance(self): - return CleanDecimal('//div[@class="previsionnel"]/div[@class="solde_value"]//label', - replace_dots=True, - default=NotAvailable)(self.doc) - class IbanPage(LoggedPage, HTMLPage): def get_iban(self): diff --git a/modules/lcl/pages.py b/modules/lcl/pages.py index 637a8f2bcf..4a97874cc3 100644 --- a/modules/lcl/pages.py +++ b/modules/lcl/pages.py @@ -581,7 +581,11 @@ def obj_label(self): return "%s Bourse" % CleanText((TableCell('label')(self)[0]).xpath('./div[b]'))(self) def obj_type(self): - return self.page.TYPES.get(' '.join(Field('label')(self).split()[:-1]).lower(), Account.TYPE_MARKET) + _label = ' '.join(Field('label')(self).split()[:-1]).lower() + for key in self.page.TYPES: + if key in _label: + return self.page.TYPES.get(key) + return Account.TYPE_MARKET def get_logout_link(self): return Link('//a[@class="link-underline" and contains(text(), "espace client")]')(self.doc) diff --git a/modules/materielnet/browser.py b/modules/materielnet/browser.py index c81e142c5a..0b0704d5f1 100644 --- a/modules/materielnet/browser.py +++ b/modules/materielnet/browser.py @@ -27,12 +27,24 @@ class MaterielnetBrowser(LoginBrowser): BASEURL = 'https://secure.materiel.net' - login = URL(r'https://www.materiel.net/form/login', - r'/Login/PartialPublicLogin', LoginPage) + login = URL(r'/Login/Login', LoginPage) captcha = URL('/pm/client/captcha.html', CaptchaPage) - profil = URL(r'/Account/InformationsSection', ProfilPage) - documents = URL(r'/Orders/PartialCompletedOrdersHeader', DocumentsPage) - document_details = URL(r'/Orders/PartialCompletedOrderContent', DocumentsDetailsPage) + profil = URL(r'/Account/InformationsSection', + r'/pro/Account/InformationsSection', ProfilPage) + documents = URL(r'/Orders/PartialCompletedOrdersHeader', + r'/pro/Orders/PartialCompletedOrdersHeader', DocumentsPage) + document_details = URL(r'/Orders/PartialCompletedOrderContent', + r'/pro/Orders/PartialCompletedOrderContent', DocumentsDetailsPage) + + def __init__(self, *args, **kwargs): + super(MaterielnetBrowser, self).__init__(*args, **kwargs) + self.is_pro = None + + def par_or_pro_location(self, url, *args, **kwargs): + if self.is_pro: + url = '/pro' + url + + return super(MaterielnetBrowser, self).location(url, *args, **kwargs) def do_login(self): self.login.go() @@ -47,13 +59,16 @@ def do_login(self): if error: raise BrowserIncorrectPassword(error) + self.is_pro = 'pro' in self.url + @need_login def get_subscription_list(self): - return self.profil.stay_or_go().get_list() + return self.par_or_pro_location('/Account/InformationsSection').page.get_list() @need_login def iter_documents(self, subscription): - json_response = self.location('/Orders/CompletedOrdersPeriodSelection').json() + json_response = self.par_or_pro_location('/Orders/CompletedOrdersPeriodSelection').json() for data in json_response: - return self.documents.go(data=data).get_documents() + for doc in self.par_or_pro_location('/Orders/PartialCompletedOrdersHeader', data=data).page.get_documents(): + yield doc diff --git a/modules/materielnet/pages.py b/modules/materielnet/pages.py index ffe08a87e7..ace5a75527 100644 --- a/modules/materielnet/pages.py +++ b/modules/materielnet/pages.py @@ -44,7 +44,7 @@ def login(self, login, password): if not re.match(regex, login): raise BrowserIncorrectPassword(Attr('//input[@id="Email"]', 'data-val-regex')(self.doc)) - form = self.get_form(id='loginForm') + form = self.get_form(xpath='//form[contains(@action, "/Login/Login")]') form['Email'] = login form['Password'] = password form.submit() diff --git a/modules/paypal/browser.py b/modules/paypal/browser.py index 79f27bea90..4a6a00d042 100644 --- a/modules/paypal/browser.py +++ b/modules/paypal/browser.py @@ -79,7 +79,6 @@ def __init__(self, *args, **kwargs): super(Paypal, self).__init__(*args, **kwargs) def do_login(self): - raise BrowserUnavailable() assert isinstance(self.username, basestring) assert isinstance(self.password, basestring) @@ -88,6 +87,7 @@ def do_login(self): response = self.open(self.page.get_script_url()) token, csrf, key, value, sessionID, cookie = self.page.get_token_and_csrf(response.text) + self.session.cookies.update({'xppcts': cookie}) data = {} data['ads_token_js'] = token @@ -114,7 +114,8 @@ def do_login(self): self.detect_account_type() def detect_account_type(self): - self.page.detect_account_type() + if self.page: + self.page.detect_account_type() @need_login def get_accounts(self): diff --git a/modules/paypal/pages.py b/modules/paypal/pages.py index 3bd10a1d5e..458066b7b6 100644 --- a/modules/paypal/pages.py +++ b/modules/paypal/pages.py @@ -96,6 +96,9 @@ def exec_decoder(mtc): csrf = re.search(r"%s'([^']+)'" % re.escape("'&_csrf='+encodeURIComponent("), cleaner_code).group(1) key, value = re.findall(r"'(\w+)','(\w+)'", cleaner_code)[-1] + # Remove setCookie function content + cleaner_code = re.sub(r"'setCookie'.*(?=,'removeCookie')", "'setCookie':function(){}", cleaner_code) + # Detect the name of the function that computes the token, detect the # variable that stores the result and store it as a global. get_token_func_name = re.search(r"ads_token_js='\+encodeURIComponent\((\w+)\)", cleaner_code).group(1) @@ -107,64 +110,6 @@ def exec_decoder(mtc): cleaner_code = cleaner_code.replace(loop_func_name + "();", "") cleaner_code = cleaner_code.replace("data;", "return;") - # Simulate a browser environment - simulate_browser_code = """ - if (!document.createAttribute) { - document.createAttribute = null; - } - - if (!document.domain) { - document.domain = "paypal.com"; - } - - if (!document.styleSheets) { - document.styleSheets = null; - } - - if (!document.characterSet) { - document.characterSet = "UTF-8"; - } - - if (!document.documentElement) { - document.documentElement = {}; - } - - if (!window.innerWidth || !window.innerHeight) { - window.innerWidth = 1280; - window.innerHeight = 800; - } - - if (typeof(screen) === "undefined") { - var screen = window.screen = { - width: 1280, - height: 800 - }; - } - - if (typeof(history) === "undefined") { - var history = window.history = {}; - } - - if (typeof(location) === "undefined") { - var location = window.location = { - host: "paypal.com" - }; - } - - var XMLHttpRequest = function() {}; - XMLHttpRequest.prototype.onreadystatechange = function(){}; - XMLHttpRequest.prototype.open = function(){}; - XMLHttpRequest.prototype.setRequestHeader = function(){}; - XMLHttpRequest.prototype.send = function(){}; - window.XMLHttpRequest = XMLHttpRequest; - - if (!navigator.appName) { - navigator.appName = "Netscape"; - } - - navigator.userAgent = "Mozilla/5.0 (X11; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0"; - """ - # Add a function that returns the token cleaner_code += """ function GET_ADS_JS_TOKEN() @@ -173,7 +118,11 @@ def exec_decoder(mtc): } """ - token = str(Javascript(simulate_browser_code + cleaner_code).call("GET_ADS_JS_TOKEN")) + try: + token = str(Javascript(cleaner_code, None, "paypal.com").call("GET_ADS_JS_TOKEN")) + except: + raise BrowserUnavailable() + return token, csrf, key, value, sessionID, cookie def login(self, login, password, ): diff --git a/modules/societegenerale/sgpe/browser.py b/modules/societegenerale/sgpe/browser.py index 81e26a10c6..fbc43cfd01 100644 --- a/modules/societegenerale/sgpe/browser.py +++ b/modules/societegenerale/sgpe/browser.py @@ -25,7 +25,7 @@ from weboob.browser.browsers import LoginBrowser, need_login, StatesMixin from weboob.browser.url import URL from weboob.browser.exceptions import ClientError -from weboob.exceptions import BrowserIncorrectPassword +from weboob.exceptions import BrowserIncorrectPassword, ActionNeeded, NoAccountsException from weboob.capabilities.base import find_object from .compat.weboob_capabilities_bank import ( AccountNotFound, RecipientNotFound, AddRecipientStep, AddRecipientBankError, @@ -35,10 +35,12 @@ from .pages import ( LoginPage, CardsPage, CardHistoryPage, IncorrectLoginPage, - ProfileProPage, ProfileEntPage, ChangePassPage, SubscriptionPage, + ProfileProPage, ProfileEntPage, ChangePassPage, SubscriptionPage, InscriptionPage, ErrorPage, ) -from .json_pages import AccountsJsonPage, BalancesJsonPage, HistoryJsonPage, BankStatementPage +from .json_pages import ( + AccountsJsonPage, BalancesJsonPage, HistoryJsonPage, BankStatementPage, +) from .transfer_pages import ( EasyTransferPage, RecipientsJsonPage, TransferPage, SignTransferPage, TransferDatesPage, AddRecipientPage, AddRecipientStepPage, ConfirmRecipientPage, @@ -59,6 +61,7 @@ class SGPEBrowser(LoginBrowser): '/gae/afficherInscriptionUtilisateur.html', '/gae/afficherChangementCodeSecretExpire.html', ChangePassPage) + inscription_page = URL('/icd-web/gax/gax-inscription.html', InscriptionPage) def check_logged_status(self): if not self.page or self.login.is_here(): @@ -82,6 +85,9 @@ def do_login(self): except ClientError: raise BrowserIncorrectPassword() + if self.inscription_page.is_here(): + raise ActionNeeded(self.page.get_error()) + # force page change if not self.accounts.is_here(): self.go_accounts() @@ -123,23 +129,47 @@ class SGEnterpriseBrowser(SGPEBrowser): CERTHASH = '2231d5ddb97d2950d5e6fc4d986c23be4cd231c31ad530942343a8fdcc44bb99' accounts = URL('/icd/syd-front/data/syd-comptes-accederDepuisMenu.json', AccountsJsonPage) + intraday_accounts = URL('/icd/syd-front/data/syd-intraday-accederDepuisMenu.json', AccountsJsonPage) + balances = URL('/icd/syd-front/data/syd-comptes-chargerSoldes.json', BalancesJsonPage) + intraday_balances = URL('/icd/syd-front/data/syd-intraday-chargerSoldes.json', BalancesJsonPage) + history = URL('/icd/syd-front/data/syd-comptes-chargerReleve.json', '/icd/syd-front/data/syd-intraday-chargerDetail.json', HistoryJsonPage) history_next = URL('/icd/syd-front/data/syd-comptes-chargerProchainLotEcriture.json', HistoryJsonPage) + profile = URL('/gae/afficherModificationMesDonnees.html', ProfileEntPage) subscription = URL(r'/Pgn/NavigationServlet\?MenuID=BANRELRIE&PageID=ReleveRIE&NumeroPage=1&Origine=Menu', SubscriptionPage) subscription_form = URL(r'Pgn/NavigationServlet', SubscriptionPage) def go_accounts(self): - self.accounts.go() + try: + # get standard accounts + self.accounts.go() + except NoAccountsException: + # get intraday accounts + self.intraday_accounts.go() @need_login def get_accounts_list(self): - accounts = [] - accounts.extend(self.accounts.stay_or_go().iter_accounts()) - for acc in self.balances.go().populate_balances(accounts): + # 'Comptes' are standard accounts on sge website + # 'Opérations du jour' are intraday accounts on sge website + # Standard and Intraday accounts are same accounts with different detail + # User could have standard accounts with no intraday accounts or the contrary + # They also could have both, in that case, retrieve only standard accounts + try: + # get standard accounts + self.accounts.go() + accounts = list(self.page.iter_class_accounts()) + self.balances.go() + except NoAccountsException: + # get intraday accounts + self.intraday_accounts.go() + accounts = list(self.page.iter_class_accounts()) + self.intraday_balances.go() + + for acc in self.page.populate_balances(accounts): yield acc @need_login diff --git a/modules/societegenerale/sgpe/json_pages.py b/modules/societegenerale/sgpe/json_pages.py index 992e67110a..b06b284c90 100644 --- a/modules/societegenerale/sgpe/json_pages.py +++ b/modules/societegenerale/sgpe/json_pages.py @@ -22,13 +22,18 @@ from weboob.browser.pages import LoggedPage, JsonPage, pagination from weboob.browser.elements import ItemElement, method, DictElement -from weboob.browser.filters.standard import CleanDecimal, CleanText, Date, Format, BrowserURL, Env +from weboob.browser.filters.standard import ( + CleanDecimal, CleanText, Date, Format, BrowserURL, Env, + Field, +) from weboob.browser.filters.json import Dict from weboob.capabilities.base import Currency from weboob.capabilities import NotAvailable from weboob.capabilities.bank import Account from weboob.capabilities.bill import Document, Subscription -from weboob.exceptions import BrowserUnavailable, BrowserIncorrectPassword +from weboob.exceptions import ( + BrowserUnavailable, NoAccountsException, BrowserIncorrectPassword, BrowserPasswordExpired, +) from weboob.tools.capabilities.bank.iban import is_iban_valid from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.tools.compat import quote_plus @@ -52,27 +57,55 @@ class AccountsJsonPage(LoggedPage, JsonPage): u'Prêt': Account.TYPE_LOAN, } - def iter_accounts(self): - for classeur in self.doc.get('donnees', {}).get('classeurs', {}): - title = classeur['title'] - for compte in classeur.get('comptes', []): - a = Account() - a.label = CleanText().filter(compte['libelle']) - a._id = compte['id'] - a.type = self.obj_type(a.label) - a.number = compte['iban'].replace(' ', '') - # for some account that don't have Iban the account number is store under this variable in the Json - if not is_iban_valid(a.number): - a.iban = NotAvailable - else: - a.iban = a.number - # id based on iban to match ids in database. - a.id = a.number[4:-2] if len(a.number) == 27 else a.number - a._agency = compte['agenceGestionnaire'] - a._title = title - yield a - - def obj_type(self, label): + def on_load(self): + if self.doc['commun']['statut'].lower() == 'nok': + reason = self.doc['commun']['raison'] + if reason == 'SYD-COMPTES-UNAUTHORIZED-ACCESS': + raise NoAccountsException("Vous n'avez pas l'autorisation de consulter : {}".format(reason)) + elif reason == 'niv_auth_insuff': + raise BrowserIncorrectPassword('Vos identifiants sont incorrects') + elif reason == 'chgt_mdp_oblig': + raise BrowserPasswordExpired('Veuillez renouveler votre mot de passe') + raise BrowserUnavailable(reason) + + @method + class iter_class_accounts(DictElement): + item_xpath = 'donnees/classeurs' + + class iter_accounts(DictElement): + @property + def item_xpath(self): + if 'intradayComptes' in self.el: + return 'intradayComptes' + return 'comptes' + + class item(ItemElement): + klass = Account + + obj__id = Dict('id') + obj_number = CleanText(Dict('iban'), replace=[(' ', '')]) + obj_iban = Field('number') + obj_label = CleanText(Dict('libelle')) + obj__agency = Dict('agenceGestionnaire') + + def obj_id(self): + number = Field('number')(self) + if len(number) == 27: + # id based on iban to match ids in database. + return number[4:-2] + return number + + def obj_iban(self): + # for some account that don't have Iban the account number is store under this variable in the Json + number = Field('number')(self) + if not is_iban_valid(number): + return NotAvailable + return number + + def obj_type(self): + return self.page.acc_type(Field('label')(self)) + + def acc_type(self, label): for wording, acc_type in self.TYPES.items(): if wording.lower() in label.lower(): return acc_type @@ -91,14 +124,15 @@ def on_load(self): if self.doc['commun']['statut'] == 'NOK': reason = self.doc['commun']['raison'] if reason == 'SYD-COMPTES-UNAUTHORIZED-ACCESS': - raise BrowserIncorrectPassword("Vous n'avez pas l'autorisation de consulter : {}".format(reason)) + raise NoAccountsException("Vous n'avez pas l'autorisation de consulter : {}".format(reason)) raise BrowserUnavailable(reason) def populate_balances(self, accounts): for account in accounts: acc_dict = self.doc['donnees']['compteSoldesMap'][account._id] - account.balance = CleanDecimal(replace_dots=True).filter(acc_dict['soldeComptable']) - account.currency = Currency.get_currency(acc_dict['deviseSoldeComptable']) + account.balance = CleanDecimal(replace_dots=True).filter(acc_dict.get('soldeComptable', acc_dict.get('soldeInstantane'))) + account.currency = Currency.get_currency(acc_dict.get('deviseSoldeComptable', acc_dict.get('deviseSoldeInstantane'))) + account.coming = CleanDecimal(replace_dots=True, default=NotAvailable).filter(acc_dict.get('montantOperationJour')) yield account diff --git a/modules/societegenerale/sgpe/pages.py b/modules/societegenerale/sgpe/pages.py index 82f8b6d897..5fc8d32765 100644 --- a/modules/societegenerale/sgpe/pages.py +++ b/modules/societegenerale/sgpe/pages.py @@ -259,3 +259,9 @@ def on_load(self): if self.doc.xpath('//div[@class="ngo_mu_message" and contains(text(), "momentanément indisponible")]'): # Warning: it could occurs because of wrongpass, user have to change password raise BrowserUnavailable(CleanText('//div[@class="ngo_mu_message"]')(self.doc)) + + +class InscriptionPage(SGPEPage): + def get_error(self): + message = CleanText('//head/title')(self.doc) + return message diff --git a/modules/societegenerale/sgpe/transfer_pages.py b/modules/societegenerale/sgpe/transfer_pages.py index 8794a7c6c4..799c627ba4 100644 --- a/modules/societegenerale/sgpe/transfer_pages.py +++ b/modules/societegenerale/sgpe/transfer_pages.py @@ -29,7 +29,7 @@ from weboob.browser.filters.html import Attr from weboob.browser.filters.json import Dict from weboob.browser.filters.standard import Date, Eval -from weboob.capabilities.bank import Recipient, Transfer, Account +from weboob.capabilities.bank import Recipient, Transfer from .pages import LoginPage @@ -102,14 +102,8 @@ def update_origin_account(self, origin_account): origin_account._underproduct_code = json_data['codeSousProduit'] break else: - assumptions = ( - not origin_account.balance, - not origin_account.iban, - origin_account.currency != 'EUR', - origin_account.type == Account.TYPE_PEA, - ) - if not any(assumptions): - assert False, 'Account %s not found on transfer page' % (origin_account.label) + # some accounts are not able to do transfer + self.logger.warning('Account %s not found on transfer page', origin_account.label) def iter_internal_recipients(self): if self.doc.xpath('//ul[@id="idCmptToInterne"]'): diff --git a/modules/yomoni/browser.py b/modules/yomoni/browser.py index c9e86972d9..b405e840e4 100644 --- a/modules/yomoni/browser.py +++ b/modules/yomoni/browser.py @@ -143,7 +143,9 @@ def iter_investment(self, account, invs=None): i.unitvalue = CleanDecimal().filter(inv['valeurCotation']) i.valuation = CleanDecimal().filter(inv['montantEuro']) i.vdate = Date().filter(inv['datePosition']) - i.diff = CleanDecimal().filter(inv['performanceEuro']) + # performanceEuro is null sometimes in the JSON we retrieve. + if inv['performanceEuro']: + i.diff = CleanDecimal().filter(inv['performanceEuro']) self.investments[account.id].append(i) return self.investments[account.id] -- GitLab