From 99da3e9f43cd9e1e464f72fdf88ed248230fbc74 Mon Sep 17 00:00:00 2001 From: Roger Philibert Date: Sat, 5 Jun 2021 20:16:37 +0200 Subject: [PATCH] backport devel modules fixes --- modules/amazon/browser.py | 12 +- modules/amazon/pages.py | 38 +- modules/ameli/browser.py | 5 +- modules/ameli/pages.py | 7 +- modules/americanexpress/browser.py | 4 +- modules/americanexpress/pages.py | 4 + modules/banquepopulaire/pages.py | 1 + modules/bforbank/browser.py | 3 +- modules/bforbank/pages.py | 2 +- modules/bnporc/pp/browser.py | 7 +- modules/bnporc/pp/pages.py | 2 + modules/boursorama/pages.py | 4 + modules/bred/bred/browser.py | 52 +-- modules/bred/bred/pages.py | 34 +- modules/bred/bred/transfer_pages.py | 7 + modules/bred/module.py | 5 + modules/caissedepargne/browser.py | 57 +-- modules/caissedepargne/pages.py | 8 +- modules/caissedepargne/transfer_pages.py | 4 + modules/carrefourbanque/browser.py | 48 ++- modules/carrefourbanque/pages.py | 27 +- modules/cmes/browser.py | 4 +- modules/creditmutuel/browser.py | 18 +- modules/creditmutuel/pages.py | 34 +- modules/fortuneo/browser.py | 33 +- modules/fortuneo/pages/accounts_list.py | 12 +- modules/fortuneo/pages/transfer.py | 26 +- modules/hsbc/browser.py | 38 +- modules/hsbc/pages/account_pages.py | 21 +- modules/ing/api/accounts_page.py | 9 +- modules/ing/api_browser.py | 28 +- modules/lcl/browser.py | 29 +- modules/lcl/pages.py | 42 ++- modules/oney/browser.py | 419 +++++++++++++++++++---- modules/oney/module.py | 8 +- modules/oney/pages.py | 128 +++++-- modules/orange/browser.py | 23 +- modules/orange/pages/login.py | 29 +- modules/primonialreim/__init__.py | 26 ++ modules/primonialreim/browser.py | 53 +++ modules/primonialreim/favicon.png | Bin 0 -> 1943 bytes modules/primonialreim/module.py | 85 +++++ modules/primonialreim/pages.py | 90 +++++ modules/primonialreim/test.py | 48 +++ modules/wiseed/pages.py | 7 +- 45 files changed, 1249 insertions(+), 292 deletions(-) create mode 100644 modules/primonialreim/__init__.py create mode 100644 modules/primonialreim/browser.py create mode 100644 modules/primonialreim/favicon.png create mode 100644 modules/primonialreim/module.py create mode 100644 modules/primonialreim/pages.py create mode 100644 modules/primonialreim/test.py diff --git a/modules/amazon/browser.py b/modules/amazon/browser.py index 6695b3aba3..8fa91e8478 100644 --- a/modules/amazon/browser.py +++ b/modules/amazon/browser.py @@ -33,7 +33,7 @@ from .pages import ( LoginPage, SubscriptionsPage, DocumentsPage, DownloadDocumentPage, HomePage, SecurityPage, LanguagePage, HistoryPage, PasswordExpired, ApprovalPage, PollingPage, - ResetPasswordPage, + ResetPasswordPage, AccountSwitcherLoadingPage, AccountSwitcherPage, SwitchedAccountPage, ) @@ -63,10 +63,15 @@ class AmazonBrowser(LoginBrowser, StatesMixin): "Impossible de trouver un compte correspondant à cette adresse e-mail", "L'adresse e-mail est déjà utilisée", "Numéro de téléphone incorrect", + "Votre adresse e-mail ou mot de passe étaient incorrects", ] WRONG_CAPTCHA_RESPONSE = "Saisissez les caractères tels qu'ils apparaissent sur l'image." login = URL(r'/ap/signin(.*)', LoginPage) + account_switcher_loading = URL(r'/ap/signin(.*)', AccountSwitcherLoadingPage) + account_switcher = URL(r'/ap/cvf/request.embed\?arb=(?P.*)', AccountSwitcherPage) + switched_account = URL(r'/ap/switchaccount', SwitchedAccountPage) + home = URL(r'/$', r'/\?language=.+$', HomePage) subscriptions = URL(r'/gp/profile', SubscriptionsPage) history = URL( @@ -281,6 +286,11 @@ def do_login(self): else: self.history.go() + if self.account_switcher_loading.is_here(): + self.account_switcher.go(token=self.page.get_arb_token()) + self.page.validate_account() + self.location(self.page.get_redirect_url()) + if not self.login.is_here(): return diff --git a/modules/amazon/pages.py b/modules/amazon/pages.py index 287782d25c..6f97f76d78 100644 --- a/modules/amazon/pages.py +++ b/modules/amazon/pages.py @@ -20,9 +20,10 @@ from __future__ import unicode_literals from woob.browser.exceptions import ServerError -from woob.browser.pages import HTMLPage, LoggedPage, FormNotFound, PartialHTMLPage, pagination +from woob.browser.pages import HTMLPage, LoggedPage, FormNotFound, PartialHTMLPage, pagination, JsonPage from woob.browser.elements import ItemElement, ListElement, method from woob.browser.filters.html import Link, Attr +from woob.browser.filters.json import Dict from woob.browser.filters.standard import ( CleanText, CleanDecimal, Env, Regexp, Format, RawText, Field, Currency, Date, Async, AsyncLoad, @@ -135,7 +136,31 @@ class LanguagePage(HTMLPage): pass +class AccountSwitcherLoadingPage(HTMLPage): + def is_here(self): + return bool(self.doc.xpath('//div[@id="ap-account-switcher-container"]')) + + def get_arb_token(self): + # Get the token from attribute data-arbToken (data-arbtoken using the Attr filter) + return Attr('//div[@id="ap-account-switcher-container"]', 'data-arbtoken')(self.doc) + + +class AccountSwitcherPage(PartialHTMLPage): + def validate_account(self): + form = self.get_form(xpath='//form[@action="/ap/switchaccount"]') + form['switch_account_request'] = Attr('//a[@data-name="switch_account_request"]', 'data-value')(self.doc) + form.submit() + + +class SwitchedAccountPage(JsonPage): + def get_redirect_url(self): + return Dict('redirectUrl')(self.doc) + + class LoginPage(PartialHTMLPage): + def is_here(self): + return not bool(self.doc.xpath('//div[@id="ap-account-switcher-container"]')) + ENCODING = 'utf-8' def login(self, login, password, captcha=None): @@ -271,20 +296,21 @@ def obj_url(self): )(self) if not url: download_elements = async_page.doc.xpath('//a[contains(@href, "download")]') + order_summary_link = Link( + '//a[contains(text(), "Récapitulatif de commande")]|//a[contains(text(), "Order Summary")]', + default=NotAvailable + ) if download_elements and len(download_elements) > 1: # Sometimes there are multiple invoices for one order and to avoid missing the other invoices # we are taking the order summary instead - url = Link( - '//a[contains(text(), "Récapitulatif de commande")]', - default=NotAvailable - )(async_page.doc) + url = order_summary_link(async_page.doc) else: url = Coalesce( Link( '//a[contains(@href, "download")]|//a[contains(@href, "generated_invoices")]', default=NotAvailable, ), - Link('//a[contains(text(), "Récapitulatif de commande")]', default=NotAvailable), + order_summary_link, default=NotAvailable )(async_page.doc) doc_id = Field('id')(self) diff --git a/modules/ameli/browser.py b/modules/ameli/browser.py index 3fd5a8f9cb..dd0ea530b4 100644 --- a/modules/ameli/browser.py +++ b/modules/ameli/browser.py @@ -28,7 +28,7 @@ from dateutil.relativedelta import relativedelta from woob.browser import LoginBrowser, URL, need_login -from woob.exceptions import ActionNeeded, BrowserIncorrectPassword, BrowserUnavailable +from woob.exceptions import ActionNeeded, BrowserIncorrectPassword from woob.tools.capabilities.bill.documents import merge_iterators from .pages import ( @@ -86,9 +86,6 @@ def do_login(self): raise BrowserIncorrectPassword(err_msg) raise AssertionError('Unhandled error at login %s' % err_msg) - if self.error_page.is_here(): - raise BrowserUnavailable(self.page.get_error_message()) - if self.cgu_page.is_here(): raise ActionNeeded(self.page.get_cgu_message()) diff --git a/modules/ameli/pages.py b/modules/ameli/pages.py index be4e9962ce..e54cef1ef6 100644 --- a/modules/ameli/pages.py +++ b/modules/ameli/pages.py @@ -30,6 +30,7 @@ from woob.browser.filters.json import Dict from woob.browser.pages import LoggedPage, HTMLPage, PartialHTMLPage, RawPage, JsonPage from woob.capabilities.bill import Subscription, Bill, Document, DocumentTypes +from woob.exceptions import BrowserUnavailable from woob.tools.compat import html_unescape from woob.tools.date import parse_french_date from woob.tools.json import json @@ -71,8 +72,10 @@ def get_cgu_message(self): class ErrorPage(HTMLPage): - def get_error_message(self): - return html_unescape(CleanText('//div[@class="mobile"]/p')(self.doc)) + def on_load(self): + # message is: "Oups... votre compte ameli est momentanément indisponible. Il sera de retour en pleine forme très bientôt." + # nothing we can do, but retry later + raise BrowserUnavailable(html_unescape(CleanText('//div[@class="mobile"]/p')(self.doc))) class SubscriptionPage(LoggedPage, HTMLPage): diff --git a/modules/americanexpress/browser.py b/modules/americanexpress/browser.py index 66dd409078..2d3d2e63d9 100644 --- a/modules/americanexpress/browser.py +++ b/modules/americanexpress/browser.py @@ -42,7 +42,7 @@ JsonBalances2, CurrencyPage, LoginPage, NoCardPage, NotFoundPage, HomeLoginPage, ReadAuthChallengePage, UpdateAuthTokenPage, - SLoginPage, + SHomePage, SLoginPage, ) from .fingerprint import FingerprintPage @@ -487,7 +487,7 @@ def iter_coming(self, account): class AmericanExpressSeleniumFingerprintBrowser(SeleniumBrowser): BASEURL = 'https://global.americanexpress.com' - home_login = URL(r'/login\?inav=fr_utility_logout') + home_login = URL(r'/login\?inav=fr_utility_logout', SHomePage) login = URL(r'https://www.americanexpress.com/en-us/account/login', SLoginPage) HEADLESS = True # Always change to True for prod diff --git a/modules/americanexpress/pages.py b/modules/americanexpress/pages.py index cc5f0a78fc..00180a3811 100644 --- a/modules/americanexpress/pages.py +++ b/modules/americanexpress/pages.py @@ -237,5 +237,9 @@ def obj_original_amount(self): obj__ref = Dict('identifier') +class SHomePage(SeleniumPage): + pass + + class SLoginPage(SeleniumPage): pass diff --git a/modules/banquepopulaire/pages.py b/modules/banquepopulaire/pages.py index 35e3e6b45e..d72448bc57 100644 --- a/modules/banquepopulaire/pages.py +++ b/modules/banquepopulaire/pages.py @@ -758,6 +758,7 @@ class GenericAccountsPage(LoggedPage, MyHTMLPage): (re.compile(r'.*Livret.*'), Account.TYPE_SAVINGS), (re.compile(r'.*Titres Pea.*'), Account.TYPE_PEA), (re.compile(r".*Plan D'epargne En Actions.*"), Account.TYPE_PEA), + (re.compile(r".*Plan Epargne En Actions.*"), Account.TYPE_PEA), (re.compile(r".*Compte Especes Pea.*"), Account.TYPE_PEA), (re.compile(r'.*Plan Epargne Retraite.*'), Account.TYPE_PERP), (re.compile(r'.*Titres.*'), Account.TYPE_MARKET), diff --git a/modules/bforbank/browser.py b/modules/bforbank/browser.py index c6c6dcdb52..5200cb4e75 100644 --- a/modules/bforbank/browser.py +++ b/modules/bforbank/browser.py @@ -71,6 +71,7 @@ class BforbankBrowser(TwoFactorBrowser): card_history = URL('espace-client/consultation/encoursCarte/.*', CardHistoryPage) card_page = URL(r'/espace-client/carte/(?P\d+)$', CardPage) + lifeinsurance = URL(r'/espace-client/assuranceVie/(?P\d+)') lifeinsurance_list = URL(r'/client/accounts/lifeInsurance/lifeInsuranceSummary.action', LifeInsuranceList) lifeinsurance_iframe = URL( r'https://(?:www|client).bforbank.com/client/accounts/lifeInsurance/consultationDetailSpirica.action', @@ -383,7 +384,7 @@ def get_coming(self, account): raise NotImplementedError() def goto_lifeinsurance(self, account): - self.location('https://client.bforbank.com/espace-client/assuranceVie') + self.lifeinsurance.go(account_id=account.id) self.lifeinsurance_list.go() @retry(AccountNotFound, tries=5) diff --git a/modules/bforbank/pages.py b/modules/bforbank/pages.py index f7d20ba2d9..2118e88034 100644 --- a/modules/bforbank/pages.py +++ b/modules/bforbank/pages.py @@ -124,7 +124,7 @@ def populate_rib(self, accounts): if 'selected' in option.attrib: self.get_iban(accounts) else: - page = self.browser.rib.go(id=re.sub(r'[^\d]', '', Attr('.', 'value')(option))) + page = self.browser.rib.go(id=Regexp(Attr('.', 'value'), r'/(.+)')(option)) page.get_iban(accounts) def get_iban(self, accounts): diff --git a/modules/bnporc/pp/browser.py b/modules/bnporc/pp/browser.py index 36de314bf4..9174b50212 100644 --- a/modules/bnporc/pp/browser.py +++ b/modules/bnporc/pp/browser.py @@ -511,7 +511,12 @@ def iter_investment(self, account): # Life insurances and PERP may be scraped from the API or from the "Assurance Vie" space, # so we need to discriminate between both using account._details: - if account.type in (account.TYPE_LIFE_INSURANCE, account.TYPE_PERP, account.TYPE_CAPITALISATION): + if account.type in ( + account.TYPE_LIFE_INSURANCE, + account.TYPE_PERP, + account.TYPE_CAPITALISATION, + account.TYPE_PER, + ): if hasattr(account, '_details'): # Going to the "Assurances Vie" page natiovie_params = self.natio_vie_pro.go().get_params() diff --git a/modules/bnporc/pp/pages.py b/modules/bnporc/pp/pages.py index 961a0ea145..ae2e0615aa 100644 --- a/modules/bnporc/pp/pages.py +++ b/modules/bnporc/pp/pages.py @@ -250,6 +250,8 @@ def validate(self, obj): 'PEA PME Espèces': Account.TYPE_PEA, 'PEA Titres': Account.TYPE_PEA, 'PEL': Account.TYPE_SAVINGS, + 'BNP Paribas Multiplacements PER': Account.TYPE_PER, + 'BNPP Multiplacements Privilège PER': Account.TYPE_PER, 'BNPP MP PERP': Account.TYPE_PERP, 'Plan Epargne Retraite Particulier': Account.TYPE_PERP, 'Crédit immobilier': Account.TYPE_MORTGAGE, diff --git a/modules/boursorama/pages.py b/modules/boursorama/pages.py index 9da43db34a..e6b59c497f 100644 --- a/modules/boursorama/pages.py +++ b/modules/boursorama/pages.py @@ -1372,6 +1372,10 @@ def obj_label(self): if not empty(bank_name): label = label.replace('- %s' % bank_name, '').strip() + # There is an exceptional case where the recipient has an empty label. + # In such a case, at least use the name of the bank + if label == '': + label = bank_name return label def obj_category(self): diff --git a/modules/bred/bred/browser.py b/modules/bred/bred/browser.py index dbd838db0e..6ca2b583d4 100644 --- a/modules/bred/bred/browser.py +++ b/modules/bred/bred/browser.py @@ -50,6 +50,7 @@ TokenPage, MoveUniversePage, SwitchPage, LoansPage, AccountsPage, IbanPage, LifeInsurancesPage, SearchPage, ProfilePage, ErrorPage, ErrorCodePage, LinebourseLoginPage, + UnavailablePage, ) from .transfer_pages import ( RecipientListPage, EmittersListPage, ListAuthentPage, @@ -84,6 +85,7 @@ class BredBrowser(TwoFactorBrowser): search = URL(r'/transactionnel/services/applications/operations/getSearch/', SearchPage) profile = URL(r'/transactionnel/services/rest/User/user', ProfilePage) error_code = URL(r'/.*\?errorCode=.*', ErrorCodePage) + unavailable_page = URL(r'/ERREUR/', UnavailablePage) accounts_twofa = URL(r'/transactionnel/v2/services/rest/Account/accounts', AccountsTwoFAPage) list_authent = URL(r'/transactionnel/services/applications/authenticationstrong/listeAuthent/(?P\w+)', ListAuthentPage) @@ -219,6 +221,13 @@ def get_connection_twofa_method(self): # The order and tests are taken from the bred website code. # Keywords in scripts.js: showSMS showEasyOTP showOTP methods = self.context['liste'] + + # Overriding default order of tests with 'preferred_sca' configuration item + preferred_auth_methods = tuple(self.config.get('preferred_sca', '').get().split()) + for auth_method in preferred_auth_methods: + if methods.get(auth_method): + return auth_method + if methods.get('sms'): return 'sms' elif methods.get('notification') and methods.get('otp'): @@ -299,7 +308,7 @@ def enrol_device(self): self.update_headers() data = { 'uuid': self.device_id, # Called an uuid but it's just a 50 digits long string. - 'deviceName': 'Accès BudgetInsight pour agrégation', # clear message for user + 'deviceName': self.config.get('device_name', 'Accès BudgetInsight pour agrégation').get(), # clear message for user 'biometricEnabled': False, 'securedBiometricEnabled': False, 'notificationEnabled': False, @@ -336,11 +345,11 @@ def move_to_universe(self, univers): @need_login def get_accounts_list(self): accounts = [] - for universe_key in self.get_universes(): + for universe_key in sorted(self.get_universes()): self.move_to_universe(universe_key) universe_accounts = [] universe_accounts.extend(self.get_list()) - universe_accounts.extend(self.get_life_insurance_list(accounts)) + universe_accounts.extend(self.get_life_insurance_list()) universe_accounts.extend(self.get_loans_list()) linebourse_accounts = self.get_linebourse_accounts(universe_key) for account in universe_accounts: @@ -354,8 +363,17 @@ def get_accounts_list(self): accounts.extend(universe_accounts) # Life insurances are sometimes in multiple universes, we have to remove duplicates - unique_accounts = {account.id: account for account in accounts} - return sorted(unique_accounts.values(), key=operator.attrgetter('_univers')) + unique_accounts = {account.id: account for account in accounts}.values() + + # Fill parents with resulting accounts when relevant: + for account in unique_accounts: + if account.type not in [Account.TYPE_CARD, Account.TYPE_LIFE_INSURANCE]: + continue + account.parent = find_object( + unique_accounts, _number=account._parent_number, type=Account.TYPE_CHECKING + ) + + return sorted(unique_accounts, key=operator.attrgetter('_univers')) @need_login def get_linebourse_accounts(self, universe_key): @@ -387,17 +405,12 @@ def get_loans_list(self): @need_login def get_list(self): self.accounts.go() - for acc in self.page.iter_accounts(accnum=self.accnum, current_univers=self.current_univers): - yield acc + return self.page.iter_accounts(accnum=self.accnum, current_univers=self.current_univers) @need_login - def get_life_insurance_list(self, accounts): - + def get_life_insurance_list(self): self.life_insurances.go() - - for ins in self.page.iter_lifeinsurances(univers=self.current_univers): - ins.parent = find_object(accounts, _number=ins._parent_number, type=Account.TYPE_CHECKING) - yield ins + return self.page.iter_lifeinsurances(univers=self.current_univers) @need_login def _make_api_call(self, account, start_date, end_date, offset, max_length=50): @@ -422,8 +435,7 @@ def get_history(self, account, coming=False): if account.type in (Account.TYPE_LOAN, Account.TYPE_LIFE_INSURANCE) or not account._consultable: raise NotImplementedError() - if account._univers != self.current_univers: - self.move_to_universe(account._univers) + self.move_to_universe(account._univers) today = date.today() seen = set() @@ -473,8 +485,7 @@ def iter_investments(self, account): elif account.type in (Account.TYPE_PEA, Account.TYPE_MARKET): if 'Portefeuille Titres' in account.label: if account._is_in_linebourse: - if account._univers != self.current_univers: - self.move_to_universe(account._univers) + self.move_to_universe(account._univers) self.linebourse.location( self.linebourse_urls[account._univers], data={'SJRToken': self.linebourse_tokens[account._univers]} @@ -498,8 +509,7 @@ def iter_market_orders(self, account): if 'Portefeuille Titres' in account.label: if account._is_in_linebourse: - if account._univers != self.current_univers: - self.move_to_universe(account._univers) + self.move_to_universe(account._univers) self.linebourse.location( self.linebourse_urls[account._univers], data={'SJRToken': self.linebourse_tokens[account._univers]} @@ -523,8 +533,8 @@ def fill_account(self, account, fields): @need_login def iter_transfer_recipients(self, account): + self.move_to_universe(account._univers) self.update_headers() - try: self.emitters_list.go(json={ 'typeVirement': 'C', @@ -676,6 +686,8 @@ def init_new_recipient(self, recipient, **params): @need_login def init_transfer(self, transfer, account, recipient, **params): + self.move_to_universe(account._univers) + account_id = account.id.split('.')[0] poste = account.id.split('.')[1] diff --git a/modules/bred/bred/pages.py b/modules/bred/bred/pages.py index ced35af9b6..57684ae8ef 100644 --- a/modules/bred/bred/pages.py +++ b/modules/bred/bred/pages.py @@ -25,14 +25,13 @@ from woob.tools.date import parse_french_date from woob.exceptions import BrowserIncorrectPassword, BrowserUnavailable, ActionNeeded -from woob.capabilities.base import find_object from woob.browser.pages import JsonPage, LoggedPage, HTMLPage from woob.capabilities import NotAvailable from woob.capabilities.bank import Account from woob.capabilities.wealth import Investment from woob.tools.capabilities.bank.investments import is_isin_valid from woob.capabilities.profile import Person -from woob.browser.filters.standard import CleanText, CleanDecimal, Env, Eval +from woob.browser.filters.standard import CleanText, CleanDecimal, Env, Eval, Field from woob.browser.filters.html import Link from woob.browser.filters.json import Dict from woob.browser.elements import DictElement, ItemElement, method @@ -140,6 +139,7 @@ def iter_loans(self, current_univers): a.balance = -Decimal(str(content['montantCapitalDu']['valeur'])) a.currency = content['montantCapitalDu']['monnaie']['code'].strip() a._univers = current_univers + a._number = Field('id') yield a @@ -147,7 +147,11 @@ class AccountsPage(LoggedPage, MyJsonPage): ACCOUNT_TYPES = { '000': Account.TYPE_CHECKING, # Compte à vue '001': Account.TYPE_SAVINGS, # Livret Ile de France + '002': Account.TYPE_SAVINGS, # Livret Seine-et-Marne & Aisne + '003': Account.TYPE_SAVINGS, # Livret Normandie '004': Account.TYPE_SAVINGS, # Livret Guadeloupe + '005': Account.TYPE_SAVINGS, # Livret Martinique/Guyane + '006': Account.TYPE_SAVINGS, # Livret Réunion/Mayotte '011': Account.TYPE_CARD, # Carte bancaire '013': Account.TYPE_LOAN, # LCR (Lettre de Change Relevé) '020': Account.TYPE_SAVINGS, # Compte sur livret @@ -171,8 +175,6 @@ class AccountsPage(LoggedPage, MyJsonPage): def iter_accounts(self, accnum, current_univers): seen = set() - accounts_list = [] - for content in self.get_content(): if accnum != '00000000000' and content['numero'] != accnum: continue @@ -183,6 +185,7 @@ def iter_accounts(self, accnum, current_univers): a._codeSousPoste = poste['codeSousPoste'] if 'codeSousPoste' in poste else None a._consultable = poste['consultable'] a._univers = current_univers + a._parent_number = None a.id = '%s.%s' % (a._number, a._nature) if content['comptePEA']: @@ -192,8 +195,8 @@ def iter_accounts(self, accnum, current_univers): if a.type == Account.TYPE_UNKNOWN: self.logger.warning("unknown type %s" % poste['codeNature']) - if a.type == Account.TYPE_CARD: - a.parent = find_object(accounts_list, _number=a._number, type=Account.TYPE_CHECKING) + if a.type != Account.TYPE_CHECKING: + a._parent_number = a._number if 'numeroDossier' in poste and poste['numeroDossier']: a._file_number = poste['numeroDossier'] @@ -205,7 +208,8 @@ def iter_accounts(self, accnum, current_univers): a.currency = poste['montantTitres']['monnaie']['code'].strip() if not a.balance and not a.currency and 'dateTitres' not in poste: continue - accounts_list.append(a) + yield a + continue if 'libelle' not in poste: continue @@ -227,9 +231,7 @@ def iter_accounts(self, accnum, current_univers): continue seen.add(a.id) - accounts_list.append(a) - - return accounts_list + yield a class IbanPage(LoggedPage, MyJsonPage): @@ -264,6 +266,7 @@ class item(ItemElement): obj_type = Account.TYPE_LIFE_INSURANCE obj_currency = 'EUR' obj__univers = Env('univers') + obj__number = Field('id') def obj_id(self): return Eval(str, Dict('numero'))(self) @@ -399,3 +402,14 @@ def on_load(self): raise BrowserUnavailable(msg) assert False, 'Error %s is not handled yet.' % code + + +class UnavailablePage(HTMLPage): + def is_here(self): + return CleanText('//h1[contains(text(), "Site en maintenance")]', default=None)(self.doc) + + def on_load(self): + msg = CleanText('//div[contains(text(), "intervention technique est en cours")]', default=None)(self.doc) + if msg: + raise BrowserUnavailable(msg) + raise AssertionError('Ended up to this error page, message not handled yet.') diff --git a/modules/bred/bred/transfer_pages.py b/modules/bred/bred/transfer_pages.py index 9b853298a9..32f6df8fea 100644 --- a/modules/bred/bred/transfer_pages.py +++ b/modules/bred/bred/transfer_pages.py @@ -49,6 +49,10 @@ def can_account_emit_transfer(self, account_id): # Nous vous précisons que votre pouvoir ne vous permet pas # d'effectuer des virements de ce type au débit du compte sélectionné. return False + elif code == '90600': + # "Votre demande de virement ne peut être prise en compte actuellement + # The user is probably not allowed to do transfers + return False elif code != '0': raise AssertionError('Unhandled code %s in transfer emitter selection' % code) @@ -68,6 +72,9 @@ class RecipientListPage(LoggedPage, JsonPage): @method class iter_external_recipients(DictElement): item_xpath = 'content/listeComptesCExternes' + # The id is the iban, and exceptionally there could be the same + # recipient multiple times when the bic of the recipient changed + ignore_duplicate = True class item(ItemElement): klass = Recipient diff --git a/modules/bred/module.py b/modules/bred/module.py index a66d2bc07e..a994d1831c 100644 --- a/modules/bred/module.py +++ b/modules/bred/module.py @@ -48,6 +48,8 @@ class BredModule(Module, CapBankWealth, CapProfile, CapBankTransferAddRecipient) ValueBackendPassword('login', label='Identifiant', masked=False, regexp=r'.{1,32}'), ValueBackendPassword('password', label='Mot de passe'), Value('accnum', label='Numéro du compte bancaire (optionnel)', default='', masked=False), + Value('preferred_sca', label='Mécanisme(s) d\'authentification forte préferrés (optionnel, un ou plusieurs (séparés par des espaces) parmi: elcard usb sms otp mail password svi notification whatsApp)', default='', masked=False), + Value('device_name', label='Nom du device qui sera autorisé pour 90j suite à l\'authentication forte', default='', masked=False), ValueTransient('request_information'), ValueTransient('resume'), ValueTransient('otp_sms'), @@ -95,6 +97,9 @@ def fill_account(self, account, fields): def iter_transfer_recipients(self, account): if not isinstance(account, Account): account = find_object(self.iter_accounts(), id=account) + elif not hasattr(account, '_univers'): + # We need a Bred filled Account to know the "univers" associated with the account + account = find_object(self.iter_accounts(), id=account.id) return self.browser.iter_transfer_recipients(account) diff --git a/modules/caissedepargne/browser.py b/modules/caissedepargne/browser.py index 0e10b0a8a2..14fa2cdea0 100644 --- a/modules/caissedepargne/browser.py +++ b/modules/caissedepargne/browser.py @@ -1277,6 +1277,12 @@ def add_owner_accounts(self): if self.home.is_here(): for account in self.page.get_list(owner_name): if account.id not in [acc.id for acc in self.accounts]: + if account.type == Account.TYPE_LIFE_INSURANCE: + # For life insurance accounts, we check if the contract is still open + if not self.go_life_insurance_investments(account): + return + if self.page.is_contract_closed(): + continue self.accounts.append(account) wealth_not_accessible = False @@ -1500,13 +1506,11 @@ def _get_history_invests(self, account): self.life_insurance_history.go() # Life insurance transactions are not sorted by date in the JSON return sorted_transactions(self.page.iter_history()) - except (IndexError, AttributeError) as e: - self.logger.error(e) - return [] except ServerError as e: if e.response.status_code == 500: raise BrowserUnavailable() raise + return self.page.iter_history() @need_login @@ -1540,7 +1544,10 @@ def match_cb(tr): self.linebourse.session.cookies.update(self.session.cookies) self.update_linebourse_token() - return self.linebourse.iter_history(account.id) + history = self.linebourse.iter_history(account.id) + # We need to go back to the synthesis, else we can not go home later + self.home_tache.go(tache='CPTSYNT0') + return history hist = self._get_history(account._info, False) return omit_deferred_transactions(hist) @@ -1622,8 +1629,8 @@ def get_investment(self, account): else: self.home.go() - self.page.go_history(account._info) if account.type in (Account.TYPE_MARKET, Account.TYPE_PEA): + self.page.go_history(account._info) # Some users may not have access to this. if not self.market.is_here(): return @@ -1653,22 +1660,7 @@ def get_investment(self, account): yield tr return - try: - # Some life insurances are not on the accounts summary - self.home_tache.go(tache='EPASYNT0') - self.page.go_life_insurance(account) - if self.home.is_here(): - # no detail is available for this account - return - - elif not self.market.is_here() and not self.message.is_here(): - # life insurance website is not always available - raise BrowserUnavailable() - - self.page.submit() - self.life_insurance_investments.go() - except (IndexError, AttributeError) as e: - self.logger.error(e) + if not self.go_life_insurance_investments(account): return if self.garbage.is_here(): @@ -1679,6 +1671,21 @@ def get_investment(self, account): if self.market.is_here(): self.page.come_back() + @need_login + def go_life_insurance_investments(self, account): + # Returns whether it managed to go to the page + self.home_tache.go(tache='EPASYNT0') + self.page.go_life_insurance(account) + if self.home.is_here(): + # no detail is available for this account + return False + elif not self.market.is_here() and not self.message.is_here(): + # life insurance website is not always available + raise BrowserUnavailable() + self.page.submit() + self.life_insurance_investments.go() + return True + @need_login def iter_market_orders(self, account): if account.type not in (Account.TYPE_MARKET, Account.TYPE_PEA): @@ -1694,8 +1701,12 @@ def iter_market_orders(self, account): return self.linebourse.session.cookies.update(self.session.cookies) self.update_linebourse_token() - for order in self.linebourse.iter_market_orders(account.id): - yield order + try: + for order in self.linebourse.iter_market_orders(account.id): + yield order + finally: + # We need to go back to the synthesis, else we can not go home later + self.home_tache.go(tache='CPTSYNT0') @need_login def get_advisor(self): diff --git a/modules/caissedepargne/pages.py b/modules/caissedepargne/pages.py index 50b91d683f..c12acbf14a 100644 --- a/modules/caissedepargne/pages.py +++ b/modules/caissedepargne/pages.py @@ -603,6 +603,7 @@ def _get_account_info(self, a, accounts): r"PostBack(Options)?\([\"'][^\"']+[\"'],\s*['\"]([HISTORIQUE_\w|SYNTHESE_ASSURANCE_CNP|BOURSE|COMPTE_TITRE][\d\w&]+)?['\"]", a.attrib.get('href', '') ) + if m is None: return None else: @@ -876,9 +877,9 @@ def get_loan_list(self): account.currency = account.get_currency(CleanText('./a')(tds[4])) accounts[account.id] = account - website = 'old' + website = 'new' if accounts: - website = 'new' + website = 'old' self.logger.debug('we are on the %s website', website) if len(accounts) == 0: @@ -1858,6 +1859,9 @@ def obj_unitvalue(self): obj_code = IsinCode(CleanText(Dict('codeIsin', default='')), default=NotAvailable) obj_code_type = IsinType(CleanText(Dict('codeIsin', default=''))) + def is_contract_closed(self): + return Dict('etatContrat/code')(self.doc) == "01" + class NatixisLIHis(LoggedPage, JsonPage): @method diff --git a/modules/caissedepargne/transfer_pages.py b/modules/caissedepargne/transfer_pages.py index 542153513a..cf30e1d00e 100644 --- a/modules/caissedepargne/transfer_pages.py +++ b/modules/caissedepargne/transfer_pages.py @@ -275,6 +275,10 @@ def parse(self, el): # If it's an internal account, we should always find only one account with _id in it's id. # Type card account contains their parent account id, and should not be listed in recipient account. match = [acc for acc in accounts if _id in acc.id and acc.type != Account.TYPE_CARD] + # Not all internal accounts are returned by get_accounts_list + if not match: + self.logger.warning('skipping internal recipient without a matching account: %r', _id) + raise SkipItem() assert len(match) == 1 match = match[0] self.env['id'] = match.id diff --git a/modules/carrefourbanque/browser.py b/modules/carrefourbanque/browser.py index 1b55860acf..b2621cf6bd 100644 --- a/modules/carrefourbanque/browser.py +++ b/modules/carrefourbanque/browser.py @@ -27,7 +27,7 @@ from woob.browser import LoginBrowser, URL, need_login, StatesMixin from woob.exceptions import ( BrowserIncorrectPassword, RecaptchaV2Question, BrowserUnavailable, - AuthMethodNotImplemented, + ActionNeeded, AuthMethodNotImplemented, ) from woob.capabilities.bank import Account from woob.tools.compat import basestring @@ -128,9 +128,7 @@ def do_login(self): self.page.enter_password(self.password) - if not self.home.is_here(): - if self.page.has_2fa(): - raise AuthMethodNotImplemented("L'authentification forte Clé Secure n'est pas prise en charge.") + if self.login.is_here(): error = self.page.get_error_message() # Sometimes some connections aren't able to login because of a # maintenance randomly occuring. @@ -140,11 +138,45 @@ def do_login(self): elif 'saisies ne correspondent pas à l\'identifiant' in error: raise BrowserIncorrectPassword(error) raise AssertionError('Unexpected error at login: "%s"' % error) - raise AssertionError('Unexpected error at login') - if self.login.is_here(): - # Check if the website asks for strong authentication with OTP - self.page.check_action_needed() + dsp2_auth_code = self.page.get_dsp2_auth_code() + # The dsp2 authentication code gives informations on which strong authentication + # method is used by the user (Clé Secure: in-app validation or otp by SMS) + # on blocked access to the account and unavailability of the service. + if dsp2_auth_code: + if dsp2_auth_code == 'authent_cc': + raise ActionNeeded( + "Authentifiez-vous depuis l'appli Carrefour Banque avec Clé Secure." + ) + elif dsp2_auth_code == 'enrolement_cc': + # On the website 'enrolement_cc' code corresponds to a pop-in in which the Clé Secure + # authentication method is advertised. The user is presented with instructions on how + # to install Clé Secure and he is also given the option to log in using otp by SMS. + # + # Unfortunately, every time the user logs in the pop-in will show up, so we have no way + # of knowing whether we need to perform otp by SMS or if it's just the advertisement for Clé Secure. + raise AuthMethodNotImplemented( + "L'authentification forte par SMS n'est pas prise en charge." + ) + elif dsp2_auth_code == 'cle_secure_locked': + raise ActionNeeded( + "A la suite de 3 tentatives d'authentification erronées, votre Clé Secure a été bloquée." + + ' Par mesure de sécurité, créez un nouveau code Clé Secure depuis votre appli Carrefour Banque.' + ) + elif dsp2_auth_code == 'service_indisponible': + raise BrowserUnavailable( + 'Le service est momentanément indisponible. Excusez-nous pour la gêne occasionnée.' + + ' Veuillez ré-essayer ultérieurement.' + ) + elif 'acces_bloque' in dsp2_auth_code: + raise ActionNeeded( + "L'accès à votre Espace Client a été bloqué pour des raisons de sécurité." + + ' Pour le débloquer, contactez le service client de Carrefour Banque.' + ) + else: + raise AssertionError('Unhandled dsp2 authentication code at login %s' % dsp2_auth_code) + + raise AssertionError('Unexpected error at login') @need_login def get_account_list(self): diff --git a/modules/carrefourbanque/pages.py b/modules/carrefourbanque/pages.py index c8594c1f5b..31352bfaff 100644 --- a/modules/carrefourbanque/pages.py +++ b/modules/carrefourbanque/pages.py @@ -28,7 +28,6 @@ from PIL import Image -from woob.tools.json import json from woob.browser.pages import HTMLPage, LoggedPage, pagination, JsonPage from woob.browser.elements import ListElement, TableElement, ItemElement, method, DictElement from woob.browser.filters.standard import ( @@ -40,7 +39,6 @@ from woob.capabilities.wealth import Investment from woob.capabilities.base import NotAvailable, empty from woob.tools.capabilities.bank.transactions import FrenchTransaction -from woob.exceptions import ActionNeeded class CarrefourBanqueKeyboard(object): @@ -150,28 +148,15 @@ def enter_password(self, password): form.submit() - def check_action_needed(self): - # The JavaScript variable 'tc_vars' is supposed to contain the 'user_login' value - # and 'user_login'='logged'. If there is no user_login and 'user_login'='unlogged', - # the customer has to validate an OTP by SMS. - raw_text = Regexp( - CleanText('//script[contains(text(), "tc_vars")]'), - r'var tc_vars = (\{[^]]+\})' - )(self.doc) - json_text = json.loads(raw_text) - if not json_text['user_ID'] and json_text['user_login'] == 'unlogged': - # The real message contains the user's phone number, so we send a generic message. - raise ActionNeeded( - "Veuillez vous connecter sur le site de Carrefour Banque pour " - + "recevoir un code par SMS afin d'accéder à votre Espace Client." - ) - raise AssertionError('Unhandled error: password submission failed and we are still on Login Page.') - def get_error_message(self): return CleanText('//div[@class="messages error"]', default=None)(self.doc) - def has_2fa(self): - return bool(self.doc.xpath('//div[@id="region_content_dsp2"]')) + def get_dsp2_auth_code(self): + return Regexp( + CleanText('//script[contains(text(), "popin_dsp2")]', replace=[('-', '_')]), + r'"popin_dsp2":"(.+)"', + default='' + )(self.doc) class MaintenancePage(HTMLPage): diff --git a/modules/cmes/browser.py b/modules/cmes/browser.py index 983c7235e1..3f9033c27d 100644 --- a/modules/cmes/browser.py +++ b/modules/cmes/browser.py @@ -51,8 +51,8 @@ class CmesBrowser(LoginBrowser): investments = URL(r'(?P.*)(?P.*)fr/epargnants/supports/fiche-du-support.html', InvestmentPage) investment_details = URL(r'(?P.*)(?P.*)fr/epargnants/supports/epargne-sur-le-support.html', InvestmentDetailsPage) asset_management = URL( - r'https://www.cmcic-am.fr/fr/conseillers-gestion-patrimoine/nos-fonds/VALE_FicheSynthese.aspx', - r'https://www.cmcic-am.fr/fr/conseillers-gestion-patrimoine/nos-fonds/VALE_Fiche.aspx', + r'https://www.creditmutuel-am.eu/fr/conseillers-gestion-patrimoine/nos-fonds/VALE_FicheSynthese.aspx', + r'https://www.creditmutuel-am.eu/fr/conseillers-gestion-patrimoine/nos-fonds/VALE_Fiche.aspx', AssetManagementPage ) operations_list = URL(r'(?P.*)(?P.*)fr/epargnants/operations/index.html', OperationsListPage) diff --git a/modules/creditmutuel/browser.py b/modules/creditmutuel/browser.py index 8cbe68c5a4..64403d3adc 100644 --- a/modules/creditmutuel/browser.py +++ b/modules/creditmutuel/browser.py @@ -60,7 +60,8 @@ ConditionsPage, MobileConfirmationPage, UselessPage, DecoupledStatePage, CancelDecoupled, OtpValidationPage, OtpBlockedErrorPage, TwoFAUnabledPage, LoansOperationsPage, OutagePage, PorInvestmentsPage, PorHistoryPage, PorHistoryDetailsPage, - PorMarketOrdersPage, PorMarketOrderDetailsPage, SafeTransPage, + PorMarketOrdersPage, PorMarketOrderDetailsPage, SafeTransPage, PhoneNumberConfirmationPage, + AuthorityManagementPage, ) @@ -207,6 +208,10 @@ class CreditMutuelBrowser(TwoFactorBrowser): r'/(?P.*)fr/banque/coordonnees_personnelles.aspx', r'/(?P.*)fr/banque/paci_engine/paci_wsd_pdta.aspx', r'/(?P.*)fr/banque/reglementation-dsp2.html', ConditionsPage) + phone_number_confirmation_page = URL( + r'/(?P.*)fr/client/paci_engine/information-client.html', PhoneNumberConfirmationPage + ) + authority_management = URL(r'/(?P.*)fr/banque/migr_gestion_pouvoirs.html', AuthorityManagementPage) currentSubBank = None is_new_website = None @@ -330,6 +335,7 @@ def poll_decoupled(self, transactionId): def handle_polling(self): if 'polling_id' not in self.polling_data: + self.logger.info("Restarting login since we do not have the polling data") return self.init_login() try: @@ -443,6 +449,14 @@ def init_login(self): # website proposes to redo 2FA when approaching end of its validity self.page.skip_redo_twofa() + if self.phone_number_confirmation_page.is_here(): + # If we reached this point, there is no SCA since the user has to confirm its phone number + self.page.skip_confirmation() + self.logger.debug("Skipping phone confirmation") + + if self.authority_management.is_here(): + self.page.skip_authority_management() + if not self.page.logged: # 302 redirect to catch to know if polling if self.login.is_here(): @@ -932,6 +946,8 @@ def get_investment(self, account): if not account._link_inv: return [] self.location(account._link_inv) + if self.page.is_liquidity(): + return [create_french_liquidity(account.balance)] return self.page.iter_investment() if account.type is Account.TYPE_PEA: liquidities = create_french_liquidity(account.balance) diff --git a/modules/creditmutuel/pages.py b/modules/creditmutuel/pages.py index c95062d820..a4ae006a08 100644 --- a/modules/creditmutuel/pages.py +++ b/modules/creditmutuel/pages.py @@ -78,6 +78,22 @@ class UselessPage(LoggedPage, HTMLPage): pass +class AuthorityManagementPage(HTMLPage): + """This page is about authority management + + 'Votre contrat de Banque à distance évolue et vous permet désormais de gérer électroniquement les pouvoirs bancaires de vos mandataires, + quel que soit le canal qu'ils utilisent (internet, agence, ...).' + + There's a later button, so skipping it. + + """ + def skip_authority_management(self): + form = self.get_form(xpath='//form[@id="C:P:F"]') + name = '_FID_DoYes' # Name attribute of the input element that represent the part "Yes, i want to report my consent" + data = {name: form[name]} + form.submit(data=data) + + class RedirectPage(LoggedPage, HTMLPage): def on_load(self): super(RedirectPage, self).on_load() @@ -490,7 +506,7 @@ def parse(self, el): id_xpath = './preceding-sibling::tr[1]/td[1]/a/node()[contains(@class, "doux")]' else: # classical account - id_xpath = './td[1]/a/node()[contains(@class, "doux")]' + id_xpath = './td[1]/a//span[contains(@class, "doux")]' _id = CleanText(id_xpath, replace=[(' ', '')])(el) if not _id: @@ -1433,6 +1449,11 @@ def get_links(self): class LIAccountsPage(LoggedPage, HTMLPage): + def is_liquidity(self): + # if the table from iter_investment is not on the page + # it means the life insurance is 100% liquidity + return not CleanText('//table[has-class("liste") and not (@summary="Avances")]/tbody/tr[count(td)>=7]')(self.doc) + def has_accounts(self): # The form only exists if the connection has a life insurance return self.doc.xpath('//input[@name="_FID_GoBusinessSpaceLife"]') @@ -2829,3 +2850,14 @@ class ConditionsPage(LoggedPage, HTMLPage): class OutagePage(HTMLPage): pass + + +class PhoneNumberConfirmationPage(LoggedPage, HTMLPage): + def is_here(self): + return CleanText( + '//h1[@class="titlecontent"]/text()="Confirmez votre numéro de téléphone portable"' + )(self.doc) + + def skip_confirmation(self): + link = Link('//a[@class="ei_btn ei_btn_typ_quit"]')(self.doc) + self.browser.location(link) diff --git a/modules/fortuneo/browser.py b/modules/fortuneo/browser.py index 985771c882..25bf49bc57 100644 --- a/modules/fortuneo/browser.py +++ b/modules/fortuneo/browser.py @@ -190,17 +190,18 @@ def first_login_step(self): self.check_and_handle_action_needed() def handle_sms(self): + if not self.sms_form: + # An action needed can happen during the handle_sms, + # but self.sms_form will have been re-initiated to None while the user resolve it, + # and the OTP will already been submitted and accepted by the server + # So, to avoid running handle_sms a second time, we check if self.sms_form is present; + # when not, we fall back to init_login, where the SCA won't be triggered. + self.init_login() + self.sms_form['otp'] = self.code self.sms_form['typeOperationSensible'] = 'AUTHENTIFICATION_FORTE_CONNEXION' self.location('/fr/prive/valider-otp-connexion.jsp', data=self.sms_form) - # We need to clear the otp value manually as `check_and_handle_action_needed()` - # might raise an ActionNeeded. If the user takes more time than the STATE_DURATION - # to do what he needs to do to handle the ActionNeeded, the storage will be cleared. - # So we'll call `handle_sms()` again since the otp config_key has not been cleared, - # which will lead to error since `sms_form` will be None. - self.config['code'] = self.config['code'].default - self.sms_form = None self.page.check_otp_error_message() @@ -267,9 +268,18 @@ def iter_accounts(self): self.page.fill_account(obj=account) else: self.location(account._history_link) + + # Sometimes the website displays a message about preventing scams. + if self.page.send_info_form(): + self.location(account._history_link) + if self.process_skippable_message(): self.location(account._history_link) + action_needed_message = self.page.get_action_needed_message() + if action_needed_message: + raise ActionNeeded(action_needed_message) + if self.loan_contract.is_here(): loan = Loan.from_dict(account.to_dict()) loan._ca = account._ca @@ -289,6 +299,15 @@ def iter_accounts(self): time.sleep(1) # TPP can match checking accounts with this id self.page.fill_tpp_account_id(obj=account) + if not account._tpp_id: + self.register_transfer.go(ca=account._ca) + self.page.fill_tpp_account_id(obj=account) + + if not account._tpp_id: + self.logger.warning( + 'Could not find the tpp_id of account %s', + account.id + ) yield account diff --git a/modules/fortuneo/pages/accounts_list.py b/modules/fortuneo/pages/accounts_list.py index 19c4cecd3e..4519e47ed4 100644 --- a/modules/fortuneo/pages/accounts_list.py +++ b/modules/fortuneo/pages/accounts_list.py @@ -97,6 +97,7 @@ def get_action_needed_message(self): + '| //span[contains(text(), "Nouveau mot de passe")]' + '| //span[contains(text(), "Renouvellement de votre mot de passe")]' + '| //span[contains(text(), "Mieux vous connaître")]' + + '| //span[contains(text(), "mettre à jour vos informations personnelles")]/ancestor::div[1]' + '| //span[contains(text(), "Souscrivez au Livret + en quelques clics")]' + '| //p[@class="warning" and contains(text(), "Cette opération sensible doit être validée par un code sécurité envoyé par SMS")]' )(self.doc) @@ -115,6 +116,15 @@ def get_global_error_message(self): def get_local_error_message(self): return CleanText('//div[@id="error"]/p[@class="erreur_texte1"]')(self.doc) + def send_info_form(self): + try: + form = self.get_form(name='validation_messages_bloquants') + except FormNotFound: + return False + else: + form.submit() + return True + MARKET_ORDER_DIRECTIONS = { 'Achat': MarketOrderDirection.BUY, @@ -733,7 +743,7 @@ def obj__tpp_id(self): return Attr( '//input[@name="numeroCompte" and contains(@value, "%s")]/preceding-sibling::input[1]' % self.obj.id, 'value', - default=self.obj.id, + default=NotAvailable )(self) def is_loading(self): diff --git a/modules/fortuneo/pages/transfer.py b/modules/fortuneo/pages/transfer.py index 68debcc463..04537a6774 100644 --- a/modules/fortuneo/pages/transfer.py +++ b/modules/fortuneo/pages/transfer.py @@ -25,7 +25,7 @@ from datetime import date, timedelta from itertools import chain -from woob.browser.pages import HTMLPage, PartialHTMLPage, LoggedPage, FormNotFound +from woob.browser.pages import HTMLPage, PartialHTMLPage, LoggedPage from woob.browser.elements import method, ListElement, ItemElement, SkipItem, TableElement from woob.browser.filters.html import Attr, Link, TableCell from woob.browser.filters.standard import ( @@ -38,6 +38,7 @@ ) from woob.capabilities.base import NotAvailable from woob.tools.compat import parse_qs, urlparse +from woob.tools.json import json from .accounts_list import ActionNeededPage @@ -140,15 +141,6 @@ def get_send_code_form(self): form.url = urls[0] return form - def send_info_form(self): - try: - form = self.get_form(name='validation_messages_bloquants') - except FormNotFound: - return False - else: - form.submit() - return True - class RecipientSMSPage(LoggedPage, PartialHTMLPage): def on_load(self): @@ -180,6 +172,20 @@ def get_error(self): class RegisterTransferPage(LoggedPage, HTMLPage): + @method + class fill_tpp_account_id(ItemElement): + def obj__tpp_id(self): + accounts_list = Regexp( + CleanText('//script[contains(text(), "listeComptesADebiter")]'), + r'listeComptesADebiter = (.*}]); var listeComptesACrediter', + default='[]' + )(self) + accounts_list = json.loads(accounts_list) + for account in accounts_list: + if account['numero'] == self.obj.id: + return account.get('numeroContratTopaze', NotAvailable) + return NotAvailable + @method class iter_internal_recipients(ListElement): item_xpath = '//select[@name="compteACrediter"]/option[not(@selected)]' diff --git a/modules/hsbc/browser.py b/modules/hsbc/browser.py index 168bdb984c..9aaf061de6 100644 --- a/modules/hsbc/browser.py +++ b/modules/hsbc/browser.py @@ -32,7 +32,10 @@ from woob.tools.capabilities.bank.transactions import sorted_transactions, keep_only_card_transactions from woob.tools.compat import parse_qsl, urlparse from woob.tools.value import Value -from woob.exceptions import ActionNeeded, BrowserIncorrectPassword, BrowserUnavailable, BrowserQuestion +from woob.exceptions import ( + BrowserIncorrectPassword, BrowserPasswordExpired, BrowserUnavailable, + BrowserUserBanned, BrowserQuestion, +) from woob.browser import URL, need_login, TwoFactorBrowser from woob.browser.exceptions import HTTPNotFound from woob.capabilities.base import find_object @@ -167,9 +170,27 @@ def load_state(self, state): def handle_otp(self): otp = self.config['otp'].get() + + # In some scenarios relogin will be triggered (see AppGonePage). + # We need to set config['otp'] to None, otherwise we will try to validate + # the otp once again even though we might not be on the right page anymore. + self.config['otp'].set(self.config['otp'].default) + self.page.login_with_secure_key(self.secret, otp) self.end_login() + def check_login_error(self): + error_msg = self.page.get_error() + + if error_msg: + if 'Please click Reset Credentials' in error_msg or 'Please reset your HSBC Secure Key' in error_msg: + raise BrowserPasswordExpired(error_msg) + + elif 'Please retry in 30 minutes' in error_msg: + raise BrowserUserBanned(error_msg) + + raise AssertionError('Unhandled error at login: %s' % error_msg) + def init_login(self): self.session.cookies.clear() @@ -192,12 +213,7 @@ def init_login(self): if no_secure_key_link: self.location(no_secure_key_link) else: - error = self.page.get_error() - if error and 'Please click Reset Credentials' in error: - raise ActionNeeded(error) - elif error: - raise AssertionError('Unhandled error at login: %s' % error) - + self.check_login_error() self.check_interactive() raise BrowserQuestion( Value( @@ -211,6 +227,9 @@ def init_login(self): def end_login(self): for _ in range(3): if self.login.is_here(): + if not self.page.logged: + # we should be logged in at this point + self.check_login_error() self.page.useless_form() # This wonderful website has 2 baseurl with only one difference: the 's' at the end of 'client' @@ -421,6 +440,11 @@ def _quit_li_space(self): @need_login def _go_to_life_insurance(self, account): self._quit_li_space() + + # We need to be on the account's owner space if we want to access the life insurances website. + self.go_post(self.js_url, data={'debr': 'SORTIE_ACCES_TIERS'}) + self.go_to_owner_accounts(account._owner) + self.go_post(account.url) if ( diff --git a/modules/hsbc/pages/account_pages.py b/modules/hsbc/pages/account_pages.py index 1b482a21d5..4f6fdd8ff1 100644 --- a/modules/hsbc/pages/account_pages.py +++ b/modules/hsbc/pages/account_pages.py @@ -559,17 +559,18 @@ def logged(self): return False def on_load(self): - for message in self.doc.xpath('//div[has-class("csPanelErrors")]'): + for message in self.doc.xpath('//div[@class="mainBloc"]/*[@class="error"]'): # Sometimes

, sometimes

+ error_msg = CleanText('.')(message) - if any( - msg in error_msg - for msg in [ - 'Please enter valid credentials for memorable answer and password.', - 'Please enter a valid Username.', - 'mot de passe invalide', - 'Log on error', # wrong otp - ] - ): + + error_at_login_regex = re.compile( + 'Please enter valid credentials for memorable answer and password.' + + '|Please enter a valid Username.' + + '|mot de passe invalide' + + '|Log on error' # wrong otp + ) + + if error_at_login_regex.search(error_msg): raise BrowserIncorrectPassword(error_msg) else: raise BrowserUnavailable(error_msg) diff --git a/modules/ing/api/accounts_page.py b/modules/ing/api/accounts_page.py index d1cf9d42a5..cf48e819c4 100644 --- a/modules/ing/api/accounts_page.py +++ b/modules/ing/api/accounts_page.py @@ -225,7 +225,12 @@ def on_load(self): self.get_form(name='module').submit() -class BourseLandingPage(LoggedPage, HTMLPage): +class BourseLandingPage(HTMLPage): # when going to bourse space, we land on this page, which is logged # that's all what this class is for: know we're logged - pass + @property + def logged(self): + return ( + 'Réessayez après vous être de nouveau authentifié' + not in CleanText('//div[@class="error-pages-message"]')(self.doc) + ) diff --git a/modules/ing/api_browser.py b/modules/ing/api_browser.py index c7917f4fad..54fef1e16b 100644 --- a/modules/ing/api_browser.py +++ b/modules/ing/api_browser.py @@ -369,6 +369,15 @@ def fill_account_iban(self, account): return account.iban = self.page.get_iban() + @need_login + def go_to_bourse_landing_page(self, account): + self.api_to_bourse.go( + account_uid=account._uid, + headers={'Authorization': 'Bearer %s' % self.get_invest_token()} + ) + bourse_url = self.response.json()['url'] + self.location(bourse_url, data='') + @need_login def go_bourse(self, account): if 'bourse.ing.fr' in self.url: @@ -378,13 +387,18 @@ def go_bourse(self, account): assert account.type in (Account.TYPE_PEA, Account.TYPE_MARKET) self.logger.debug('going to bourse site') - self.api_to_bourse.go( - account_uid=account._uid, - headers={'Authorization': 'Bearer %s' % self.get_invest_token()} - ) - bourse_url = self.response.json()['url'] - - self.location(bourse_url, data='') + try: + self.go_to_bourse_landing_page(account) + except ClientError as e: + # Sometimes a 403 can appear with a message asking to reconnect and retry while trying to access + # the bourse's landing page + if ( + e.response.status_code == 403 and self.bourse_landing.match(e.request.url) + and not BourseLandingPage(self, e.response).logged + ): + self.go_to_bourse_landing_page(account) + else: + raise self.bourse.session.cookies.update(self.session.cookies) self.bourse.location(self.url) diff --git a/modules/lcl/browser.py b/modules/lcl/browser.py index adadaaf6f7..52fc920030 100644 --- a/modules/lcl/browser.py +++ b/modules/lcl/browser.py @@ -53,7 +53,7 @@ Form2Page, DocumentsPage, ClientPage, SendTokenPage, CaliePage, ProfilePage, DepositPage, AVHistoryPage, AVInvestmentsPage, CardsPage, AVListPage, CalieContractsPage, RedirectPage, MarketOrdersPage, AVNotAuthorized, AVReroute, TwoFAPage, AuthentStatusPage, FinalizeTwoFAPage, - PasswordExpiredPage, + PasswordExpiredPage, ContractRedirectionPage, MaintenancePage, ) @@ -80,8 +80,8 @@ class LCLBrowser(TwoFactorBrowser): r'/outil/UAUT/Contract/getContract.*', r'/outil/UAUT/Contract/selectContracts.*', r'/outil/UAUT/Accueil/preRoutageLogin', - r'/outil/UAUT/Contract/redirection', ContractsPage) + contract_redirection_page = URL(r'/outil/UAUT/Contract/redirection', ContractRedirectionPage) contracts_choice = URL(r'.*outil/UAUT/Contract/routing', ContractsChoicePage) home = URL(r'/outil/UWHO/Accueil/', HomePage) accounts = URL(r'/outil/UWSP/Synthese', AccountsPage) @@ -342,6 +342,21 @@ def handle_sms(self): self.location('/outil/UWAF/Otp/validationCodeOtp?codeOtp=%s' % self.code) self.page.check_otp_error(otp_sent=True) + def go_to_accounts(self): + try: + self.accounts.go() + except ServerError as e: + # Sometimes this page can return a 502 with a message "Pour raison de maintenance informatique, + # votre espace « gestion de comptes » est momentanément indisponible. Nous vous invitons à vous + # reconnecter ultérieurement. Nous vous prions de bien vouloir nous excuser pour la gêne occasionnée." + if e.response.status_code == 502: + maintenance_page = MaintenancePage(self, e.response) + error_message = maintenance_page.get_message() + if maintenance_page.get_error_code() == 'BPI-50': + raise BrowserUnavailable(error_message) + raise AssertionError('An unexpected error occurred: %s' % error_message) + raise + @need_login def connexion_bourse(self): self.location('/outil/UWBO/AccesBourse/temporisationCar?codeTicker=TICKERBOURSECLI') @@ -357,7 +372,9 @@ def connexion_bourse(self): def deconnexion_bourse(self): self.disc.stay_or_go() - self.accounts.go() + if self.contract_redirection_page.is_here() and self.page.should_submit_redirect_form(): + self.page.submit_redirect_form() + self.go_to_accounts() if self.login.is_here(): # When we logout we can be disconnected from the main site self.do_login() @@ -385,6 +402,8 @@ def update_life_insurance_account(self, life_insurance): def go_back_from_life_insurance_website(self): self.avdetail.stay_or_go() self.page.come_back() + if self.contract_redirection_page.is_here() and self.page.should_submit_redirect_form(): + self.page.submit_redirect_form() def select_contract(self, id_contract): if self.current_contract and id_contract != self.current_contract: @@ -486,7 +505,7 @@ def get_accounts(self): self.go_back_from_life_insurance_website() # retrieve accounts on main page - self.accounts.go() + self.go_to_accounts() for a in self.page.get_accounts_list(name=owner_name): if not self.check_accounts(a): continue @@ -560,7 +579,7 @@ def get_accounts_list(self): else: self.get_accounts() - self.accounts.go() + self.go_to_accounts() deferred_cards = self.page.get_deferred_cards() diff --git a/modules/lcl/pages.py b/modules/lcl/pages.py index 768c50bd35..dc9584362c 100644 --- a/modules/lcl/pages.py +++ b/modules/lcl/pages.py @@ -200,6 +200,18 @@ def is_here(self): return self.response.status_code == 302 +class MaintenancePage(HTMLPage): + def get_error_code(self): + return Regexp( + CleanText('//div[contains(text(), "CODE ERREUR : ")]'), + r'CODE ERREUR : (.*)', + default=None, + )(self.doc) + + def get_message(self): + return CleanText('//div[@id="indispo_texte"]')(self.doc) + + class ContractsPage(LoginPage, PartialHTMLPage): def on_load(self): # after login we are redirect in ContractsPage even if there is an error at login @@ -228,6 +240,15 @@ def select_contract(self, id_contract=None): form.submit() +class ContractRedirectionPage(ContractsPage): + def should_submit_redirect_form(self): + return bool(self.doc.xpath('//body[contains(@onload, "envoyerJeton()")]/form')) + + def submit_redirect_form(self): + form = self.get_form(id='form') + form.submit() + + class PasswordExpiredPage(LoggedPage, HTMLPage): def get_message(self): return CleanText('//form[@id="changementCodeForm"]//span[contains(., "nouveau code d’accès")]')(self.doc) @@ -619,11 +640,22 @@ def open_transaction_page(self, tr): Attr('.', 'href')(tr._el), method='POST', ) - - return self.browser.open( - '/outil/UWLM/ListeMouvementsParticulier/accesDetailsMouvement?element=%s' % row, - method='POST', - ) + try: + return self.browser.open( + '/outil/UWLM/ListeMouvementsParticulier/accesDetailsMouvement?element=%s' % row, + method='POST', + ) + except ServerError as e: + # Sometimes this page can return a 502 with a message "Pour raison de maintenance informatique, + # votre espace « gestion de comptes » est momentanément indisponible. Nous vous invitons à vous + # reconnecter ultérieurement. Nous vous prions de bien vouloir nous excuser pour la gêne occasionnée." + if e.response.status_code == 502: + maintenance_page = MaintenancePage(self.browser, e.response) + error_message = maintenance_page.get_message() + if maintenance_page.get_error_code() == 'BPI-50': + raise BrowserUnavailable(error_message) + raise AssertionError('An unexpected error occurred: %s' % error_message) + raise def fix_transaction_stuff(self, obj, tr_page): if obj.category == 'RELEVE CB': diff --git a/modules/oney/browser.py b/modules/oney/browser.py index e176e0b8b2..4c0bcb4871 100644 --- a/modules/oney/browser.py +++ b/modules/oney/browser.py @@ -17,43 +17,53 @@ # You should have received a copy of the GNU Lesser General Public License # along with this woob module. If not, see . +# flake8: compatible + from __future__ import unicode_literals +import re from datetime import datetime +import requests + +from woob.tools.compat import urlparse from woob.capabilities.bank import Account -from woob.exceptions import BrowserIncorrectPassword, BrowserPasswordExpired -from woob.browser import LoginBrowser, URL, need_login +from woob.exceptions import ( + BrowserIncorrectPassword, BrowserPasswordExpired, + AuthMethodNotImplemented, BrowserUnavailable, + BrowserQuestion, +) +from woob.browser import TwoFactorBrowser, URL, need_login +from woob.tools.compat import quote +from woob.tools.value import Value from .pages import ( LoginPage, ClientPage, OperationsPage, ChoicePage, - ContextInitPage, SendUsernamePage, SendPasswordPage, CheckTokenPage, ClientSpacePage, + ContextInitPage, SendUsernamePage, SendCompleteStepPage, ClientSpacePage, OtherDashboardPage, OAuthPage, AccountsPage, JWTTokenPage, OtherOperationsPage, + SendRiskEvaluationPage, SendInitStepPage, ) __all__ = ['OneyBrowser'] -class OneyBrowser(LoginBrowser): +class OneyBrowser(TwoFactorBrowser): BASEURL = 'https://www.oney.fr' LOGINURL = 'https://login.oney.fr' OTHERURL = 'https://middle.mobile.oney.io' home_login = URL( r'/site/s/login/login.html', - LoginPage - ) - login = URL( - r'https://login.oney.fr/login', - r'https://login.oney.fr/context', + LOGINURL + r'/context', # Target of the redirection when going on the first URL LoginPage ) - send_username = URL(LOGINURL + r'/middle/authenticationflowinit', SendUsernamePage) - send_password = URL(LOGINURL + r'/middle/completeauthflowstep', SendPasswordPage) + # Login api context_init = URL(LOGINURL + r'/middle/context', ContextInitPage) - - check_token = URL(LOGINURL + r'/middle/check_token', CheckTokenPage) + send_risk_evaluation = URL(LOGINURL + r'/middle/riskevaluation', SendRiskEvaluationPage) + send_username = URL(LOGINURL + r'/middle/initauthenticationflow', SendUsernamePage) + send_init_step = URL(LOGINURL + r'/middle/initstep', SendInitStepPage) + send_complete_step = URL(LOGINURL + r'/middle/completestrongauthenticationflow', SendCompleteStepPage) # Space selection choice = URL(r'/site/s/multimarque/choixsite.html', ChoicePage) @@ -77,67 +87,297 @@ class OneyBrowser(LoginBrowser): card_name = None is_mail = False pristine_params_headers = { - 'Environment': "PRD", - 'Origin': "Web", + 'Environment': 'PRD', + 'Origin': 'Web', 'IsLoggedIn': False, } params_headers = pristine_params_headers.copy() - def do_login(self): + HAS_CREDENTIALS_ONLY = True + + def __init__(self, *args, **kwargs): + super(OneyBrowser, self).__init__(*args, **kwargs) + + self.login_steps = None + self.login_flow_id = None + self.login_success_url = None + self.login_customer_session_id = None + self.login_additional_inputs = None + self.login_client_id = None + self.__states__ += ( + 'login_steps', + 'login_flow_id', + 'login_success_url', + 'login_customer_session_id', + 'login_additional_inputs', + 'login_client_id', + ) + + self.AUTHENTICATION_METHODS = { + 'PHONE_OTP': self.handle_phone_otp, + } + self.known_step_type = ['EMAIL_PASSWORD', 'IAD_ACCESS_CODE'] # password + self.known_step_type.extend(self.AUTHENTICATION_METHODS) # 2fa + + def locate_browser(self, state): + url = state['url'] + if self.BASEURL in url: + try: + self.location(url, params=self.other_space_params_headers()) + except (requests.exceptions.HTTPError, requests.exceptions.TooManyRedirects): + pass + else: + super(OneyBrowser, self).locate_browser(state) + + def load_state(self, state): + super(OneyBrowser, self).load_state(state) + + if self.login_client_id: + self.session.headers.update({'Client-id': self.login_client_id}) + + def dump_state(self): + state = super(OneyBrowser, self).dump_state() + if self.send_init_step.is_here(): + # We do not want to try to reload this page. + state.pop('url', None) + return state + + def clear_init_cookies(self): + # Keep the device-id to prevent an SCA + for cookie in self.session.cookies: + if cookie.name == 'did_proxy': + did_proxy = cookie + break + else: + did_proxy = None + self.session.cookies.clear() + if did_proxy: + self.session.cookies.set_cookie(did_proxy) + + @property + def device_proxy_id(self): + duration = 1800000 # Milliseconds (from proxyid script below) + proxy_id = self.session.cookies.get('did_proxy', domain='login.oney.fr') + + if not proxy_id: + text = self.open('https://argus.arcot.com/scripts/proxyid.js').text + match = re.search(r'"(.+)"', text) + if match: + proxy_id = match.group(1) + else: + raise AssertionError('Could not retrieve a new device proxy id') + + # Cannot use datetime.timestamp since it is not in python2 + expires = int((datetime.now() - datetime(1970, 1, 1)).total_seconds() * 1000 + duration) + self.session.cookies.set(name='did_proxy', value=proxy_id, domain='login.oney.fr', path='/', expires=expires) + + return proxy_id + + def send_fingerprint(self): + ddna_arcot = ( + '{"VERSION":"2.1","MFP":{"Browser":{"UserAgent":"%s","Vendor":"","VendorSubID":"","BuildID":"20181001000000","CookieEnabled":true},"IEPlugins":{},"NetscapePlugins":{},"Screen":{"FullHeight":1080,"AvlHeight":1053,"FullWidth":1920,"AvlWidth":1920,"ColorDepth":24,"PixelDepth":24},"System":{"Platform":"Linux x86_64","OSCPU":"Linux x86_64","systemLanguage":"en-US","Timezone":0}},"ExternalIP":""}' + % self.session.headers['User-Agent'] + ) + params = { + 'did_proxy': self.device_proxy_id, + 'ddna_arcot': quote(ddna_arcot), + 'ddna_arcot_time': '{"browser":0,"clientcaps":1,"plugin":0,"screen":4,"system":0,"boundingbox":2,"timetaken":7}', + } + + self.open('https://argus.arcot.com/img/zero.png', params=params) + + def init_login(self): self.reset_session_for_new_auth() - self.home_login.go(method="POST") + self.home_login.go(method='POST') context_token = self.page.get_context_token() - assert context_token is not None, "Should not have context_token=None" + assert context_token is not None, 'Should not have context_token=None' self.context_init.go(params={'contextToken': context_token}) - success_url = self.page.get_success_url() - customer_session_id = self.page.get_customer_session_id() + self.assert_no_error() + self.login_customer_session_id = self.page.get_customer_session_id() + self.login_client_id = self.page.get_client_id() + oauth_token = self.page.get_oauth_token() + self.login_success_url = self.page.get_success_url() + self.login_additional_inputs = self.page.get_additionnal_inputs() + + self.send_fingerprint() - self.session.headers.update({'Client-id': self.page.get_client_id()}) + self.session.headers.update({'Client-id': self.login_client_id}) # There is a VK on the website but it does not encode the password - self.login.go() if '@' in self.username: auth_type = 'EML' - step_type = 'EMAIL_PASSWORD' self.is_mail = True else: auth_type = 'IAD' - step_type = 'IAD_ACCESS_CODE' + + digital_identity_selector = { + 'value': self.username, + 'type': 'authentication_factor', + 'subtype': auth_type, + } + + self.send_risk_evaluation.go(json={ + 'digital_identity_selector': digital_identity_selector, + 'oauth': oauth_token, + 'device_proxy_id': self.device_proxy_id, + 'x_ca_sessionid': self.login_customer_session_id, + 'service_id': 'LOGIN', + 'client_id': self.login_client_id, + }) + self.assert_no_error() + + self.login_flow_id = self.page.get_flow_id() + niveau_authent = self.page.get_niveau_authent() + + if niveau_authent == 'O': + # Never seen in the wild but apparently if you + # receive this code, you are already logged and + # only need to use the oauth_token + # Found by reverse engineering the website code + assert oauth_token is not None + self.finish_auth_with_token(oauth_token) + return + elif niveau_authent == 'D': + # Message depuis translation.json: + # Pour des raisons de sécurité, l’opération n’a pas pu aboutir. + # Veuillez réessayez ultérieurement. + raise BrowserUnavailable() + elif niveau_authent in ['LIGHT', 'STRONG']: + pass + else: + raise AssertionError('Niveau d\'authentification inconnu. %s' % niveau_authent) self.send_username.go(json={ - 'authentication_type': 'LIGHT', - 'authentication_factor': { - 'public_value': self.username, - 'type': auth_type, - } + 'digital_identity_selector': digital_identity_selector, + 'oauth': oauth_token, + 'flowid': self.login_flow_id, + 'service_id': None, + 'client_id': None, + 'authentication_type': None, + 'x_ca_sessionid': None, }) + self.assert_no_error() + + self.login_steps = self.page.get_steps() + self.execute_login_steps() + + def execute_login_steps(self, token=None): + # The website gives us a authentification plan during the login process. + # We store this plan in the self.login_steps list. + # In this method, we execute the plan step by step until we get + # a login token or there is no more step to follow. + + # Each step has three attribute at each point in time. + # - Type: What authentification challenge to do? Password, sms, etc + # You can find the complete list of supported type in self.known_step_type + # - Action: What is the next action to do for that authentification challenge? + # Possible values: INIT, COMPLETE, DONE + # Init: Send the challenge to the user. (Ex: send the sms) + # Complete: Send the response to Oney. (Ex: send the password or the sms code) + # Done: When the step is finished. + # Some step type (ex: password) do not have a INIT action. + # - Status: Is the step optional? + # Values: TODO, OPTIONAL, DONE + # Todo: Required step. + # Optional: we directly skip optional step. + # Done: When the step is finished. + + # An authentification without SCA has 1 step to send the password. + # An authentification with SCA has at least 2. Most of them seem to have + # 3 steps with one optional that we skip. + + while token is None and self.login_steps: + step = self.login_steps[0] + step_type = step['type'] + step_action = step['action'].lower() + step_status = step['status'].lower() + + if step_status == 'optional' or step_status == 'done': + self.login_steps.pop(0) + continue + + if step_type not in self.known_step_type: + raise AuthMethodNotImplemented(step_type) + + if step_action == 'init': + if step_type in self.AUTHENTICATION_METHODS: + # Init on known 2fa + self.check_interactive() + self.send_init_step.go(json={ + 'flow_id': self.login_flow_id, + 'step_type': step_type, + # additionnal_inputs from the context request + 'additionnal_inputs': self.login_additional_inputs, + }) + self.assert_no_error() + new_step_value = self.page.get_step_of(step_type) + assert new_step_value['action'].lower() != 'init', 'The action is expected to change.' + self.login_steps[0] = new_step_value + + if step_type == 'PHONE_OTP': + extra_data = self.page.get_extra_data() + # From translation.json key: Enter_OTP_Code/Label_Code_Sent + label = 'Un nouveau code de sécurité vous a été envoyé par SMS au %s.' % extra_data['masked_phone'] + raise BrowserQuestion(Value('PHONE_OTP', label=label)) + else: + raise AssertionError('Missing handling of step type: %s' % step) + else: + raise AuthMethodNotImplemented(step) + + elif step_action == 'complete': + if step_type not in self.AUTHENTICATION_METHODS: + # Only for EMAIL_TYPE and IAD_ACCESS_CODE + token = self.complete_step(self.password) + else: + # Other type of step should be handled in handle_* + raise AssertionError('Unexpected "complete" action for step %s' % step) + else: + raise AssertionError('Unkown step action: %s' % step_action) - flow_id = self.page.get_flow_id() + if token: + self.finish_auth_with_token(token) + else: + raise BrowserIncorrectPassword() - self.send_password.go(json={ - 'flow_id': flow_id, + def complete_step(self, value): + step = self.login_steps.pop(0) + step_type = step['type'] + self.send_complete_step.go(json={ + 'flow_id': self.login_flow_id, 'step_type': step_type, - 'value': self.password, + 'value': value, }) + self.check_auth_error() + token = self.page.get_token() + new_status = self.page.get_step_of(step_type)['status'].lower() - error = self.page.get_error() - if error: - if error == 'Authenticator : Le facteur d’authentification est rattaché': - raise BrowserPasswordExpired() - raise BrowserIncorrectPassword(error) + assert new_status == 'done', 'Status should be done after a complete step' - token = self.page.get_token() + return token - self.check_token.go(params={'token': token}) - self.location(success_url, params={ - 'token': token, - 'customer_session_id': customer_session_id, - }) + def handle_phone_otp(self): + token = self.complete_step(self.PHONE_OTP) + self.execute_login_steps(token) + + def finish_auth_with_token(self, token): + self.location( + self.login_success_url, + params={ + 'token': token, + 'customer_session_id': self.login_customer_session_id, + }, + ) + + self.login_steps = None + self.login_flow_id = None + self.login_success_url = None + self.login_customer_session_id = None + self.login_additional_inputs = None + self.session.headers.pop('Client-id', None) if self.choice.is_here(): - self.other_space_url = self.page.get_redirect_other_space() self.has_other = self.has_oney = True elif self.dashboard.is_here(): self.has_other = True @@ -145,34 +385,41 @@ def do_login(self): elif self.client.is_here(): self.has_oney = True else: + parsed_url = urlparse(self.url) + netloc = parsed_url.netloc + path = parsed_url.path + self.logger.info('ONEY SUCCESS REDIRECT URL: %s%s', netloc, path) raise BrowserIncorrectPassword() def setup_headers_other_space(self): assert self.dashboard.is_here() - isaac_token = self.page.get_token() + try: + isaac_token = self.page.get_token() - self.session.headers.update({ - 'Origin': "https://espaceclient.oney.fr", - }) - self.jwt_token.go(params={ - 'localTime': datetime.now().isoformat()[:-3]+ 'Z' - }) - self.update_authorization(self.page.get_token()) + self.session.headers.update({ + 'Origin': 'https://espaceclient.oney.fr', + }) + self.jwt_token.go(params={ + 'localTime': datetime.now().isoformat()[:-3] + 'Z', + }) + self.update_authorization(self.page.get_token()) - self.oauth.go(json={ - 'header': self.params_headers, - 'isaacToken': isaac_token, - }) + self.oauth.go(json={ + 'header': self.params_headers, + 'isaacToken': isaac_token, + }) - self.params_headers.update(self.page.get_headers_from_json()) + self.params_headers.update(self.page.get_headers_from_json()) + except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout): + raise BrowserUnavailable() def update_authorization(self, token): self.session.headers.update({ - 'Authorization': 'Bearer %s' % token + 'Authorization': 'Bearer %s' % token, }) def reset_session_for_new_auth(self): - self.session.cookies.clear() + self.clear_init_cookies() self.session.headers.pop('Authorization', None) self.session.headers.pop('Origin', None) self.params_headers = self.pristine_params_headers.copy() @@ -185,7 +432,7 @@ def other_space_params_headers(self): def get_referrer(self, oldurl, newurl): if newurl.startswith(self.OTHERURL): - return "https://espaceclient.oney.fr/" + return 'https://espaceclient.oney.fr/' else: return super(OneyBrowser, self).get_referrer(oldurl, newurl) @@ -210,14 +457,7 @@ def try_go_site(self, target_site): self.do_login() assert self.choice.is_here() - # if no redirect was found in the choice_page we try the previous method. Due to a lack of example - # it might be deprecated - if self.other_space_url: - self.location(self.other_space_url) - self.client_space.go() - else: - self.choice_portal.go(data={'selectedSite': 'ONEY_HISTO'}) - + self.choice_portal.go(data={'selectedSite': 'ONEY_HISTO'}) elif target_site == 'other': if not self.has_other: return False @@ -232,9 +472,38 @@ def try_go_site(self, target_site): raise AssertionError('Unkown target_site: %s' % target_site) current_site = self.get_site() - assert current_site == target_site, 'Should be on site %s, landed on %s site instead' % (target_site, current_site) + assert current_site == target_site, ( + 'Should be on site %s, landed on %s site instead' + % (target_site, current_site) + ) return True + def assert_no_error(self): + error = self.page.get_error() + assert not error, error + + def check_auth_error(self): + error = self.page.get_error() + if error: + if error == 'Authenticator : [FunctionalError] Le facteur d’authentification est rattaché': + # Seen in the following case: the user change its login from a number to its email adress + raise BrowserIncorrectPassword() + elif error == 'Authenticator : Invalid CA response code : 504 Gateway Timeout': + raise BrowserUnavailable() + elif error == 'Authenticator : [FunctionalError] LOGIN_FAILED': + raise BrowserIncorrectPassword() + elif error == 'Authenticator : [TechnicalError] L’identité n’existe pas': + raise BrowserIncorrectPassword() + elif error == "Authenticator : [TechnicalError] Le facteur d'authentification n'existe pas": + # Website message: 'Les informations fournies ne nous permettent pas de vous identifier' + raise BrowserIncorrectPassword() + elif error == '[TechnicalError] Read timed out': + raise BrowserUnavailable() + elif error == "Authenticator : [FunctionalError] L'état de l'identifiant ne permet pas d'initialiser un flux d'authentification (BLOCKED)": + # Website message: 'Pour le débloquer, vous pouvez demander un nouveau mot de passe' + raise BrowserPasswordExpired() + raise AssertionError(error) + @need_login def iter_accounts(self): accounts = [] @@ -263,7 +532,10 @@ def iter_history(self, account): return self.page.iter_transactions(seen=set()) elif account._site == 'other' and account.type == Account.TYPE_CHECKING: - self.other_operations.go(params=self.other_space_params_headers()) + try: + self.other_operations.go(params=self.other_space_params_headers()) + except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout): + raise BrowserUnavailable() return self.page.iter_history(guid=account._guid, is_coming=False) else: return [] @@ -280,7 +552,10 @@ def iter_coming(self, account): return self.page.iter_transactions(seen=set()) elif account._site == 'other' and account.type == Account.TYPE_CHECKING: - self.other_operations.go(params=self.other_space_params_headers()) + try: + self.other_operations.go(params=self.other_space_params_headers()) + except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout): + raise BrowserUnavailable() return self.page.iter_history(guid=account._guid, is_coming=True) else: return [] diff --git a/modules/oney/module.py b/modules/oney/module.py index aa8624db22..8a3d3adcbb 100644 --- a/modules/oney/module.py +++ b/modules/oney/module.py @@ -17,12 +17,13 @@ # You should have received a copy of the GNU Lesser General Public License # along with this woob module. If not, see . +# flake8: compatible from __future__ import unicode_literals from woob.capabilities.bank import CapBank from woob.tools.backend import Module, BackendConfig -from woob.tools.value import ValueBackendPassword +from woob.tools.value import ValueBackendPassword, ValueTransient from .browser import OneyBrowser @@ -40,17 +41,20 @@ class OneyModule(Module, CapBank): CONFIG = BackendConfig( ValueBackendPassword('login', label='Identifiant', masked=False, regexp=r'([0-9]{9}|.+@.+\..+)'), ValueBackendPassword('password', label='Mot de passe'), + ValueTransient('request_information'), + ValueTransient('PHONE_OTP', regexp=r'^\d{6}$'), ) BROWSER = OneyBrowser def create_default_browser(self): return self.create_browser( + self.config, self.config['login'].get(), self.config['password'].get(), ) def iter_accounts(self): - return self.browser.iter_accounts() + return self.browser.iter_accounts() def iter_history(self, account): # To prevent issues in calcul of actual balance and coming one, all diff --git a/modules/oney/pages.py b/modules/oney/pages.py index 8956559ffb..d96fea25ef 100644 --- a/modules/oney/pages.py +++ b/modules/oney/pages.py @@ -17,15 +17,16 @@ # You should have received a copy of the GNU Lesser General Public License # along with this woob module. If not, see . +# flake8: compatible + from __future__ import unicode_literals import re -import requests - from datetime import date - from decimal import Decimal +import requests + from woob.capabilities.base import NotAvailable from woob.capabilities.bank import Account from woob.tools.capabilities.bank.transactions import FrenchTransaction, sorted_transactions @@ -37,17 +38,34 @@ ) from woob.browser.filters.html import Attr from woob.browser.filters.json import Dict +from woob.exceptions import BrowserUnavailable from woob.tools.compat import urlparse, parse_qsl class Transaction(FrenchTransaction): - PATTERNS = [(re.compile(r'^(?PRetrait .*?) - traité le \d+/\d+$'), FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile(r'^(?P(Prélèvement|Cotisation|C R C A M) .*?) - traité le \d+/\d+$'), FrenchTransaction.TYPE_ORDER), # C R C A M is a bank it is hardcoded here because some client want it typed and it would be a mess to scrap it - (re.compile(r"^(?P(Frais sur achat à l'étranger|Facturation).*?) - traité le \d+/\d+$"), FrenchTransaction.TYPE_BANK), - (re.compile(r'^Intérêts mensuels'), FrenchTransaction.TYPE_BANK), - (re.compile(r'^(?P(Avoir comptant|ANNULATION|Annulation) .*?) - traité le \d+/\d+$'), FrenchTransaction.TYPE_PAYBACK), - (re.compile(r'^(?P(RETRAIT )?DAB .*?) - traité le \d+/\d+$'), FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile(r'^(?P.*?)(, taux de change de(.*)?)? - traité le( (\d+|/\d+)*$|$)'), FrenchTransaction.TYPE_CARD)] # some labels are really badly formed so the regex needs to be this nasty to catch all edge cases + PATTERNS = [ + (re.compile(r'^(?PRetrait .*?) - traité le \d+/\d+$'), FrenchTransaction.TYPE_WITHDRAWAL), + # C R C A M is a bank it is hardcoded here because some client want it typed and it would be a mess to scrap it + ( + re.compile(r'^(?P(Prélèvement|Cotisation|C R C A M) .*?) - traité le \d+/\d+$'), + FrenchTransaction.TYPE_ORDER, + ), + ( + re.compile(r"^(?P(Frais sur achat à l'étranger|Facturation).*?) - traité le \d+/\d+$"), + FrenchTransaction.TYPE_BANK, + ), + (re.compile(r'^Intérêts mensuels'), FrenchTransaction.TYPE_BANK), + ( + re.compile(r'^(?P(Avoir comptant|ANNULATION|Annulation) .*?) - traité le \d+/\d+$'), + FrenchTransaction.TYPE_PAYBACK, + ), + (re.compile(r'^(?P(RETRAIT )?DAB .*?) - traité le \d+/\d+$'), FrenchTransaction.TYPE_WITHDRAWAL), + # some labels are really badly formed so the regex needs to be this nasty to catch all edge cases + ( + re.compile(r'^(?P.*?)(, taux de change de(.*)?)? - traité le( (\d+|/\d+)*$|$)'), + FrenchTransaction.TYPE_CARD, + ), + ] class ContextInitPage(JsonPage): @@ -60,24 +78,63 @@ def get_success_url(self): def get_customer_session_id(self): return self.doc['context']['customer_session_id'] + def get_oauth_token(self): + # Could be always null + return Dict('context/oauth_token')(self.doc) + + def get_additionnal_inputs(self): + return Dict('context/additionnal_inputs')(self.doc) + + def get_error(self): + return Dict('context/errors/0/label', default=None)(self.doc) + + +class StepsMixin: + def get_steps(self): + return Dict(self.steps_path)(self.doc) + + def get_step_of(self, step_type): + for step in self.get_steps(): + if step['type'] == step_type: + return step + + +class SendRiskEvaluationPage(JsonPage): + def get_niveau_authent(self): + return Dict('evaluatedRisk/niveau_authent')(self.doc) -class SendUsernamePage(JsonPage): def get_flow_id(self): - return self.doc['authenticationFlowInit']['flow_id'] + return Dict('evaluatedRisk/flowid')(self.doc) + + def get_error(self): + return Dict('evaluatedRisk/errors/0/label', default=None)(self.doc) -class SendPasswordPage(JsonPage): - def get_token(self): - return self.doc['completeAuthFlowStep']['token'] +class SendUsernamePage(StepsMixin, JsonPage): + steps_path = 'initAuthenticationFlow/steps' def get_error(self): - errors = self.doc['completeAuthFlowStep']['errors'] - if errors: - return errors[0]['label'] + return Dict('initAuthenticationFlow/errors/0/label', default=None)(self.doc) -class CheckTokenPage(JsonPage): - pass +class SendInitStepPage(StepsMixin, JsonPage): + steps_path = 'initStep/steps' + + def get_extra_data(self): + return Dict('initStep/extra_data/0')(self.doc) + + def get_error(self): + return Dict('initStep/errors/0/label', default=None)(self.doc) + + +class SendCompleteStepPage(StepsMixin, JsonPage): + steps_path = "completeAuthFlowStep/flow/steps" + + def get_token(self): + return Dict('completeAuthFlowStep/token', default=None)(self.doc) + + def get_error(self): + return Dict('completeAuthFlowStep/errors/0/label', default=None)(self.doc) class LoginPage(HTMLPage): @@ -95,8 +152,11 @@ def get_redirect_other_space(self): def get_pages(self): for page_attrib in self.doc.xpath('//a[@data-site]/@data-site'): - yield self.browser.open('/site/s/login/loginidentifiant.html', - data={'selectedSite': page_attrib}).page + yield self.browser.open( + '/site/s/login/loginidentifiant.html', + data={'selectedSite': page_attrib}, + ).page + class OneySpacePage(LoggedPage): def get_site(self): @@ -123,13 +183,24 @@ class item(ItemElement): def parse(self, el): self.env['label'] = CleanText('./h3/a')(self) or 'Carte Oney' - self.env['_num'] = Attr('%s%s%s' % ('//option[contains(text(), "', Field('label')(self).replace('Ma ', ''), '")]'), 'value', default='')(self) + self.env['_num'] = Attr( + '%s%s%s' % ( + '//option[contains(text(), "', + Field('label')(self).replace('Ma ', ''), + '")]', + ), 'value', default='')(self) self.env['id'] = Format('%s%s' % (self.page.browser.username, Field('_num')(self)))(self) # On the multiple accounts page, decimals are separated with dots, and separated with commas on single account page. - amount_due = CleanDecimal('./p[@class = "somme-due"]/span[@class = "synthese-montant"]', default=None)(self) + amount_due = CleanDecimal( + './p[@class = "somme-due"]/span[@class = "synthese-montant"]', + default=None + )(self) if amount_due is None: - amount_due = CleanDecimal('./div[@id = "total-sommes-dues"]/p[contains(text(), "sommes dues")]/span[@class = "montant"]', replace_dots=True)(self) + amount_due = CleanDecimal( + './div[@id = "total-sommes-dues"]/p[contains(text(), "sommes dues")]/span[@class = "montant"]', + replace_dots=True + )(self) self.env['balance'] = - amount_due @@ -204,7 +275,11 @@ def get_site(self): class OtherSpaceJsonPage(OtherSpacePage, JsonPage): def on_load(self): is_success = Dict('header/isSuccess', default=None)(self.doc) - assert is_success, "a page returned that the request was not a success" + if not is_success: + if 'InternalServerError' in Dict('header/responseCode', default='')(self.doc): + # Seen when loading the dashboard. Not account listed when it happens. + raise BrowserUnavailable() + raise AssertionError('the page %s returned that the request was not a success' % self.url) new_jwt_token = Dict('header/jwtToken/token', default=None)(self.doc) if new_jwt_token: @@ -235,6 +310,7 @@ def get_token(self): 'PP': Account.TYPE_LOAN, } + class AccountsPage(OtherSpaceJsonPage): @method class iter_accounts(DictElement): diff --git a/modules/orange/browser.py b/modules/orange/browser.py index fc5223083d..c8f277aaf9 100644 --- a/modules/orange/browser.py +++ b/modules/orange/browser.py @@ -218,8 +218,10 @@ def get_subscription_list(self): api_subscription_id_list = [] # for logging only if not subscriber: self.profile_par.go() - subscriber = self.page.get_subscriber() + if self.profile_par.is_here(): + subscriber = self.page.get_subscriber() + subscriptions = {} try: params = { 'page': 1, @@ -227,9 +229,11 @@ def get_subscription_list(self): } self.contracts.go(params=params) for sub in self.page.iter_subscriptions(): + # subscriber may be empty on some connection + # store subscription for now and get subscriber later if it's the case sub.subscriber = subscriber + subscriptions[sub.id] = sub subscription_id_list.append(sub.id) - yield sub nb_sub = self.page.doc['totalContracts'] except ServerError: pass @@ -243,9 +247,16 @@ def get_subscription_list(self): for sub in self.contracts_api.go(headers=headers).iter_subscriptions(): # subscription returned here may be duplicated with the one returned by contracts page api_subscription_id_list.append(sub.id) - if sub.id not in subscription_id_list: + if sub.id not in subscriptions.keys(): + subscriptions[sub.id] = sub nb_sub += 1 - yield sub + elif subscriptions[sub.id].subscriber is NotAvailable: + # because sometimes subscriber is only available on contracts page + # sometimes just in contracts_api + # sometimes in both page, (but it's not always the same) + # sometimes it's just not available + subscriptions[sub.id].subscriber = sub.subscriber + except (ServerError, ClientError) as e: # The orange website will return odd status codes when there are no subscriptions to return # I've seen the 404, 500 and 503 response codes @@ -253,11 +264,15 @@ def get_subscription_list(self): if e.response.status_code not in (404, 500, 503): raise + for sub in subscriptions.values(): + yield sub + # for logging purpose only for subid in subscription_id_list: if subid not in api_subscription_id_list: # there is a subscription which is returned by contracts page and not by contracts_api # we can't get rid of contracts page + # PS: still True for some connections on 25/05/21 self.logger.warning( 'there is a subscription which is returned by contracts page and not by contracts_api' ) diff --git a/modules/orange/pages/login.py b/modules/orange/pages/login.py index b706503d34..8747854473 100644 --- a/modules/orange/pages/login.py +++ b/modules/orange/pages/login.py @@ -19,10 +19,6 @@ from __future__ import unicode_literals -import lxml.html as html - -from io import StringIO - from woob.browser.pages import HTMLPage, LoggedPage, JsonPage, RawPage from woob.browser.filters.standard import CleanText, Format @@ -34,28 +30,9 @@ class LoginPage(RawPage): class PasswordPage(JsonPage): ENCODING = 'utf-8' def get_change_password_message(self): - if self.doc.get('stage') != 'changePassword': - # when stage is not present everything is okay, and if it's not changePassword we prefer do nothing here - return - - if 'mandatory' not in self.doc['options']: - # maybe there are some cases where it's optional - return - - encoding = self.encoding - if encoding == 'latin-1': - encoding = 'latin1' - if encoding: - encoding = encoding.replace('ISO8859_', 'ISO8859-') - - parser = html.HTMLParser(encoding=encoding) - html_doc = html.parse(StringIO(self.doc['view']), parser) - - # message should be: - # Votre mot de passe actuel n’est pas suffisamment sécurisé et doit être renforcé. - # Veuillez le modifier pour accéder à vos services Orange. - return CleanText('//p[@id="cnMsg"]')(html_doc) - + if self.doc.get('step') == 'mandatory': + # The password expired message on the website is fetched from a javascript file. + return 'Votre mot de passe actuel n’est pas suffisamment sécurisé et doit être renforcé.' class ManageCGI(HTMLPage): pass diff --git a/modules/primonialreim/__init__.py b/modules/primonialreim/__init__.py new file mode 100644 index 0000000000..bcf13aebcd --- /dev/null +++ b/modules/primonialreim/__init__.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Vincent A +# +# This file is part of a woob module. +# +# This woob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This woob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this woob module. If not, see . + +from __future__ import unicode_literals + + +from .module import PrimonialreimModule + + +__all__ = ['PrimonialreimModule'] diff --git a/modules/primonialreim/browser.py b/modules/primonialreim/browser.py new file mode 100644 index 0000000000..17c2703963 --- /dev/null +++ b/modules/primonialreim/browser.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Vincent A +# +# This file is part of a woob module. +# +# This woob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This woob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this woob module. If not, see . + +# flake8: compatible + +from __future__ import unicode_literals + +from woob.browser import LoginBrowser, need_login, URL + +from .pages import ( + LoginPage, AfterLoginPage, AccountsPage, TaxDocsPage, +) + + +class PrimonialreimBrowser(LoginBrowser): + BASEURL = 'https://www.primonialreim.com' + + login = URL("/login", LoginPage) + accounts = URL("/group/extranet-associes/mon-patrimoine", AccountsPage) + tax_documents = URL("/group/extranet-associes/ma-fiscalit%C3%A9", TaxDocsPage) + home = URL("/group/extranet-associes", AfterLoginPage) + + def do_login(self): + self.login.go() + self.page.do_login(self.username, self.password) + # twice because site submits username first then password + self.page.do_login(self.username, self.password) + + @need_login + def iter_accounts(self): + self.accounts.go() + return self.page.iter_accounts() + + @need_login + def iter_documents(self): + self.tax_documents.go() + return self.page.iter_documents() diff --git a/modules/primonialreim/favicon.png b/modules/primonialreim/favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..8883e5bc0a98d262862ef85e299d0f209982c8d5 GIT binary patch literal 1943 zcmZ`)dotYnfoQN=rRfyiyt+(QJCo{Ih#@@ArJ~_j^9)d++(&`#qm~v%S5@ zD7Zcx000!l&ED$NqzoIE=q!_=fZvag|-BVmFe9q?(c8!kMG<9Ow>c+rrLI!PpLSK z!SySeMj@*gPKvkRR+!e#dP28cKAf-nZqGV&eWQ*a|IPyaxS$qOu{%$nMf^Y(ifuY# zZ5*xE`Wo^ngS#`cUtWNT4g1{x>VA@9;784sWjqkW=dA}K_^$%wvxks#Pm1WmksRYR zS1T+;>qvxU4H#h?@Ru_lwG8~qdn1GPk$Ko69WNNnVkbvuou=bp`x1uDNpWU) z&nj==-QYr596@P4^la|)=tR%rE?$+ez`nB!d-%s%>6N@|a@Ob1tGE4c;0Gh(`W`<` zJ)vp~^ulSf3!waFm2HAT6`~R2c0L{eY)rp7D1nc^ssdpwil-~=Bg6n@Z>A9PVpNbG z%k>48AZbl!LhLo#uG#x^U4dSP0w&l#`zAGqMt)&Ebl5E9;HzU7CM5opSqJH691ITyN{NbjtPS%wx6=$+@)wD+V*zv!@u^B6)2 zX#CS{(7xm_09p>-_r2-EdoN?i-g;{=$CiCXs^a5nho|pAFmO!&i_Er=M_4vUwdj&A zyDelG%TBwDT3dK4ILzm9B*61SSO%0feH&z1^-sujf*ARqEPZCN0={iKhJR#DyFA?+ z9Fw0-BK(G#c2UlkD&VS^LhdEg*~*EAom&a)cQVW$;Sff}{( z%G0C!L(?lQk?mAHc84a93Yfy2p)1t;$fI^Ze#1PuS^d)50U&))MNR-9n;b}N56z1a zr43_Zq;XlFCXVpnMr?BJ9XzrMK%RuqFds7zL>;y_a0;XjG5)XQMSewvrmN43_P#JX z>D3bkXxQ5*`ya^U#wSl89aBnLh1QB>ax!IA?(Eg|EE2b z_i>TOzH?6AVoyv=RQ5<$R^WC>BpQ7oDe1&QJD+dn46ooHa9jlvS$>jcqj~Zh3O^m73z0J9)mISRO3A8f&anM(oz=FrWlc#nc3Q$ zn~O?TehnJ)KYP}q7_hLgAiiN93-l1|JbHA*%)(-6A}lP7^`@q#rgga`daC-~J-6>c zs=ZScTUx9TFE1|-1g-Jh(8x&5PusGOh=GBDD1?8@n}=_v#>cZnB8PQ|sJ;Ew>Qu-2 z=F-`moE$Z&k-k1h6455#uH*A38Rl!k`go-~oz9O+Vlq);u^3?Y_EIL>=LajgqguFu zYG3UgVpU50JrXoC+y8>%>6wd>+_-+-F(4p-#hhwSNgtf=Rux}aHZ(K@k$z~(dc6uR zEi3c!^-bq+z?R^hP438|OLX+=a`V`dYeYmuXIq9$7WbtttqXU0vXoq|!*$1z@pQ&|tX}DKIe5O~W}=kyT%R zFjURW-5s8&e800TKYZImrXI76-M{-^OH27hMY;`xOPe25XF5esqqzo$hGK7Vxu|PG ztr1&Qe`$`a^qv{5xxXC-PNw{El=fcm@F6NYJ6opu1Ah%}gO@_;1cJtf24m`;@aX6< z-5PXeH*a3g2UJj4xV|_dfc1i}1!+)q%x9g{wKEvB#KgoQ-1)y-5kMT;u`>QI5KDqO u4u>m&AUV|x#V5nC|Ea$}6=`C21JFI%U-*ah{wUS^3{YIXT!g1+oPPs6>19v= literal 0 HcmV?d00001 diff --git a/modules/primonialreim/module.py b/modules/primonialreim/module.py new file mode 100644 index 0000000000..e80e3e5aad --- /dev/null +++ b/modules/primonialreim/module.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Vincent A +# +# This file is part of a woob module. +# +# This woob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This woob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this woob module. If not, see . + +# flake8: compatible + +from __future__ import unicode_literals + +from woob.tools.backend import Module, BackendConfig +from woob.tools.value import Value, ValueBackendPassword +from woob.capabilities.base import find_object +from woob.capabilities.bank import CapBank, Account +from woob.capabilities.bill import ( + CapDocument, Subscription, SubscriptionNotFound, DocumentNotFound, +) + +from .browser import PrimonialreimBrowser + + +__all__ = ['PrimonialreimModule'] + + +class PrimonialreimModule(Module, CapBank, CapDocument): + NAME = 'primonialreim' + DESCRIPTION = 'Primonial REIM' + MAINTAINER = 'Vincent A' + EMAIL = 'dev@indigo.re' + LICENSE = 'LGPLv3+' + VERSION = '3.0' + + BROWSER = PrimonialreimBrowser + + CONFIG = BackendConfig( + Value('username', label='Identifiant'), + ValueBackendPassword('password', label='Mot de passe'), + ) + + def create_default_browser(self): + return self.create_browser(self.config['username'].get(), self.config['password'].get()) + + # CapBank + def iter_accounts(self): + return self.browser.iter_accounts() + + # CapDocument + def iter_subscription(self): + return [Subscription.from_dict(dict(id="primonial", label="Primonial"))] + + def get_subscription(self, id): + return find_object(self.iter_subscription(), id=id, error=SubscriptionNotFound) + + def iter_documents(self, subscription): + return self.browser.iter_documents() + + def get_document(self, id): + return find_object(self.iter_documents(None), id=id, error=DocumentNotFound) + + def download_document(self, document): + if isinstance(document, str): + document = find_object(self.iter_documents(None), id=document, error=DocumentNotFound) + return self.browser.open(document.url).content + + # CapCollection + def iter_resources(self, objs, split_path): + if Account in objs: + self._restrict_level(split_path) + return self.iter_accounts() + if Subscription in objs: + self._restrict_level(split_path) + return self.iter_subscription() diff --git a/modules/primonialreim/pages.py b/modules/primonialreim/pages.py new file mode 100644 index 0000000000..20769b7a2d --- /dev/null +++ b/modules/primonialreim/pages.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Vincent A +# +# This file is part of a woob module. +# +# This woob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This woob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this woob module. If not, see . + +# flake8: compatible + +from __future__ import unicode_literals + +import datetime +from decimal import Decimal +import re +import json + +from woob.capabilities.bank.base import Account +from woob.capabilities.bill import Document, DocumentTypes +from woob.browser.pages import HTMLPage, LoggedPage +from woob.browser.filters.standard import ( + CleanText, Format, Regexp, +) +from woob.browser.filters.html import AbsoluteLink +from woob.browser.elements import ListElement, ItemElement, method + + +class LoginPage(HTMLPage): + def do_login(self, username, password): + form = self.get_form(xpath="//form[contains(@action, 'login')]") + + url = form.el.attrib["action"] + token = re.search(r"INSTANCE_([a-zA-Z0-9]+)_", url)[1] + + form[f"_com_preim_portlet_login_PreimLoginPortlet_INSTANCE_{token}_username"] = username + form[f"_com_preim_portlet_login_PreimLoginPortlet_INSTANCE_{token}_password"] = password + form.submit() + + +class AfterLoginPage(LoggedPage, HTMLPage): + pass + + +class AccountsPage(LoggedPage, HTMLPage): + def iter_accounts(self): + jdata = json.loads(self.doc.xpath("//div/@js-new-graph[contains(., 'bar')]")[0]) + jdata = {item["legendText"]: item["dataPoints"] for item in jdata["data"]} + for jpoint in jdata["Valeur totale d achat"]: + yield Account.from_dict(dict( + id=jpoint["label"].lower().replace(" ", ""), + label=jpoint["label"], + balance=Decimal(str(jpoint["y"])), + type=Account.TYPE_REAL_ESTATE, + )) + + +class TaxDocsPage(LoggedPage, HTMLPage): + @method + class iter_documents(ListElement): + item_xpath = "//a[contains(@href, '.pdf')]" + + class item(ItemElement): + klass = Document + + obj_type = DocumentTypes.NOTICE + obj_format = "pdf" + + obj_url = AbsoluteLink(".") + obj_id = Regexp(obj_url, r"/([^/]+)\.pdf") + + obj__year = Regexp(obj_url, r"(\d+)\.pdf") + obj_label = Format( + "%s %s", + CleanText("."), + obj__year + ) + + def obj_date(self): + return datetime.date(int(self.obj._year) + 1, 1, 1) diff --git a/modules/primonialreim/test.py b/modules/primonialreim/test.py new file mode 100644 index 0000000000..e582510339 --- /dev/null +++ b/modules/primonialreim/test.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Vincent A +# +# This file is part of a woob module. +# +# This woob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This woob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this woob module. If not, see . + +# flake8: compatible + +from __future__ import unicode_literals + +from woob.tools.test import BackendTest + + +class PrimonialreimTest(BackendTest): + MODULE = 'primonialreim' + + def test_accounts(self): + accounts = list(self.backend.iter_accounts()) + assert accounts + for account in accounts: + assert account.id + assert account.label + assert account.balance + assert account.type + + def test_documents(self): + sub, = self.backend.iter_subscription() + docs = list(self.backend.iter_documents()) + assert docs + for doc in docs: + assert doc.id + assert doc.label + assert doc.date + assert doc.type + assert self.backend.download_document(docs[0]) diff --git a/modules/wiseed/pages.py b/modules/wiseed/pages.py index 7ab359f1da..5e13608b4a 100644 --- a/modules/wiseed/pages.py +++ b/modules/wiseed/pages.py @@ -25,7 +25,7 @@ CleanText, CleanDecimal, Regexp, Coalesce, ) from woob.browser.elements import method, ItemElement, TableElement -from woob.exceptions import BrowserIncorrectPassword +from woob.exceptions import BrowserIncorrectPassword, BrowserUserBanned from woob.capabilities.base import NotAvailable from woob.capabilities.wealth import Investment from woob.tools.capabilities.bank.investments import create_french_liquidity @@ -42,7 +42,10 @@ def raise_error(self): msg = CleanText('//div[has-class("alert-danger")]')(self.doc) if 'Email ou mot de passe invalide' in msg: raise BrowserIncorrectPassword(msg) - assert False, 'unhandled message %r' % msg + elif 'la connexion à votre compte est bloquée' in msg: + raise BrowserUserBanned(msg) + + raise AssertionError('unhandled message : %s' % msg) class LandPage(LoggedPage, HTMLPage): -- GitLab