From 5bcddf378c93ece203aa573bf98c77221dd6d5d3 Mon Sep 17 00:00:00 2001 From: Vincent A Date: Mon, 17 Jun 2019 21:59:35 +0200 Subject: [PATCH] backport master module fixes --- .../afer/compat/weboob_capabilities_bank.py | 1 + modules/amazon/compat/weboob_exceptions.py | 12 + .../compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + .../amundi/compat/weboob_capabilities_bank.py | 1 + modules/amundi/pages.py | 5 +- modules/anticaptcha/browser.py | 20 +- modules/anticaptcha/compat/__init__.py | 0 .../compat/weboob_capabilities_captcha.py | 191 ++++++++++++++++ modules/anticaptcha/module.py | 6 +- .../apivie/compat/weboob_capabilities_bank.py | 1 + modules/axabanque/browser.py | 43 ++-- .../compat/weboob_capabilities_bank.py | 1 + modules/axabanque/pages/bank.py | 1 + .../pages/compat/weboob_capabilities_bank.py | 1 + modules/axabanque/pages/wealth.py | 85 +++----- .../compat/weboob_capabilities_bank.py | 1 + modules/banquepopulaire/browser.py | 32 ++- .../compat/weboob_capabilities_bank.py | 1 + modules/banquepopulaire/pages.py | 34 ++- .../compat/weboob_capabilities_bank.py | 1 + .../becm/compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + .../binck/compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + .../bnporc/compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + modules/bnporc/pp/browser.py | 5 +- .../pp/compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + .../bolden/compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + modules/boursorama/pages.py | 1 + modules/bouygues/__init__.py | 8 + modules/bouygues/browser.py | 148 ++++--------- modules/bouygues/module.py | 51 ++--- modules/bouygues/pages.py | 205 +++++------------- modules/bp/browser.py | 8 +- modules/bp/compat/weboob_capabilities_bank.py | 1 + modules/bp/pages/accountlist.py | 20 +- .../pages/compat/weboob_capabilities_bank.py | 1 + .../bred/compat/weboob_capabilities_bank.py | 1 + modules/bred/bred/pages.py | 2 + .../bred/compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + .../caels/compat/weboob_capabilities_bank.py | 1 + modules/caissedepargne/browser.py | 97 +++++++-- .../cenet/compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + modules/caissedepargne/pages.py | 60 ++++- .../compat/weboob_capabilities_bank.py | 1 + modules/carrefourbanque/browser.py | 21 +- .../compat/weboob_capabilities_bank.py | 1 + .../cic/compat/weboob_capabilities_bank.py | 1 + .../cices/compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + .../cmb/compat/weboob_capabilities_bank.py | 1 + modules/cmes/browser.py | 7 +- .../cmes/compat/weboob_capabilities_bank.py | 1 + modules/cmes/pages.py | 33 ++- .../cmmc/compat/weboob_capabilities_bank.py | 1 + .../cmso/compat/weboob_capabilities_bank.py | 1 + .../par/compat/weboob_capabilities_bank.py | 1 + modules/cmso/par/pages.py | 40 ++-- modules/cmso/par/transfer_pages.py | 4 +- .../pro/compat/weboob_capabilities_bank.py | 1 + modules/cragr/api/browser.py | 74 ++++--- .../api/compat/weboob_capabilities_bank.py | 1 + modules/cragr/api/pages.py | 13 +- modules/cragr/api/transfer_pages.py | 6 +- .../cragr/compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + modules/cragr/regions/pages.py | 24 +- .../compat/weboob_browser_filters_standard.py | 49 ----- .../web/compat/weboob_capabilities_bank.py | 47 ---- .../compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + modules/creditdunord/pages.py | 2 + .../compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + modules/creditmutuel/pages.py | 114 +++++----- .../compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + .../esalia/compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + .../pages/compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + .../gmf/compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + modules/groupama/pages.py | 92 ++++---- .../compat/weboob_capabilities_bank.py | 1 + modules/hsbc/browser.py | 150 +++++++------ .../hsbc/compat/weboob_capabilities_bank.py | 1 + modules/hsbc/pages/account_pages.py | 103 ++++++--- .../pages/compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + .../api/compat/weboob_capabilities_bank.py | 1 + .../ing/compat/weboob_capabilities_bank.py | 1 + .../web/compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + .../lcl/compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + modules/lcl/pages.py | 13 +- .../api/compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + modules/logicimmo/pages.py | 40 ++-- modules/logicimmo/test.py | 4 +- .../compat/weboob_capabilities_bank.py | 1 + modules/myedenred/pages.py | 1 + .../n26/compat/weboob_capabilities_bank.py | 1 + .../nalo/compat/weboob_capabilities_bank.py | 1 + .../nef/compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + modules/netfinca/pages.py | 2 + .../oney/compat/weboob_capabilities_bank.py | 1 + modules/orange/browser.py | 20 +- modules/orange/module.py | 4 + modules/orange/pages/bills.py | 32 ++- .../paypal/compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + .../s2e/compat/weboob_capabilities_bank.py | 1 + modules/s2e/pages.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + .../pages/compat/weboob_capabilities_bank.py | 1 + .../sgpe/compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + .../compat/weboob_capabilities_bank.py | 1 + .../yomoni/compat/weboob_capabilities_bank.py | 1 + 136 files changed, 1198 insertions(+), 821 deletions(-) create mode 100644 modules/anticaptcha/compat/__init__.py create mode 100644 modules/anticaptcha/compat/weboob_capabilities_captcha.py delete mode 100644 modules/cragr/web/compat/weboob_browser_filters_standard.py delete mode 100644 modules/cragr/web/compat/weboob_capabilities_bank.py diff --git a/modules/afer/compat/weboob_capabilities_bank.py b/modules/afer/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/afer/compat/weboob_capabilities_bank.py +++ b/modules/afer/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/amazon/compat/weboob_exceptions.py b/modules/amazon/compat/weboob_exceptions.py index 281d3401c8..941b96a08a 100644 --- a/modules/amazon/compat/weboob_exceptions.py +++ b/modules/amazon/compat/weboob_exceptions.py @@ -124,6 +124,18 @@ def __init__(self, website_key, website_url): super(RecaptchaQuestion, self).__init__(self.type, website_key=website_key, website_url=website_url) +class RecaptchaV3Question(CaptchaQuestion): + type = 'g_recaptcha' + + website_key = None + website_url = None + action = None + + def __init__(self, website_key, website_url, action=None): + super(RecaptchaV3Question, self).__init__(self.type, website_key=website_key, website_url=website_url) + self.action = action + + from weboob.exceptions import FuncaptchaQuestion as _FuncaptchaQuestion class FuncaptchaQuestion(_FuncaptchaQuestion): type = 'funcaptcha' diff --git a/modules/amazonstorecard/compat/weboob_capabilities_bank.py b/modules/amazonstorecard/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/amazonstorecard/compat/weboob_capabilities_bank.py +++ b/modules/amazonstorecard/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/americanexpress/compat/weboob_capabilities_bank.py b/modules/americanexpress/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/americanexpress/compat/weboob_capabilities_bank.py +++ b/modules/americanexpress/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/amundi/compat/weboob_capabilities_bank.py b/modules/amundi/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/amundi/compat/weboob_capabilities_bank.py +++ b/modules/amundi/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/amundi/pages.py b/modules/amundi/pages.py index 3a7accbf86..6a0954d78e 100644 --- a/modules/amundi/pages.py +++ b/modules/amundi/pages.py @@ -116,7 +116,10 @@ def get_amount(self, instructions, account): if ('nomDispositif' in ins and 'montantNet' in ins and 'codeDispositif' in ins and '%s%s' % (ins['nomDispositif'], ins['codeDispositif']) == '%s%s' % (account.label, account.id)): - amount += ins['montantNet'] + if ins['type'] == 'RACH_TIT': + amount -= ins['montantNet'] + else: + amount += ins['montantNet'] return CleanDecimal().filter(amount) diff --git a/modules/anticaptcha/browser.py b/modules/anticaptcha/browser.py index ce8c8a238b..94a90fc0a2 100644 --- a/modules/anticaptcha/browser.py +++ b/modules/anticaptcha/browser.py @@ -23,8 +23,8 @@ from weboob.browser.browsers import APIBrowser from weboob.exceptions import BrowserIncorrectPassword, BrowserBanned -from weboob.capabilities.captcha import ( - ImageCaptchaJob, RecaptchaJob, NocaptchaJob, FuncaptchaJob, CaptchaError, +from .compat.weboob_capabilities_captcha import ( + ImageCaptchaJob, RecaptchaJob, RecaptchaV3Job, NocaptchaJob, FuncaptchaJob, CaptchaError, InsufficientFunds, UnsolvableCaptcha, InvalidCaptcha, ) @@ -74,6 +74,20 @@ def post_gcaptcha(self, url, key, prefix): r = self.request('/createTask', data=data) return str(r['taskId']) + def post_gcaptchav3(self, url, key, action): + data = { + "clientKey": self.apikey, + "task":{ + "type":"RecaptchaV3TaskProxyless", + "websiteURL": url, + "websiteKey": key, + "minScore": 0.3, + "pageAction": action + } + } + r = self.request('/createTask', data=data) + return str(r['taskId']) + def post_funcaptcha(self, url, key, sub_domain): data = { "clientKey": self.apikey, @@ -128,7 +142,7 @@ def poll(self, job): elif isinstance(job, RecaptchaJob): job.solution = sol['recaptchaResponse'] job.solution_challenge = sol['recaptchaChallenge'] - elif isinstance(job, NocaptchaJob): + elif isinstance(job, NocaptchaJob) or isinstance(job, RecaptchaV3Job): job.solution = sol['gRecaptchaResponse'] elif isinstance(job, FuncaptchaJob): job.solution = sol['token'] diff --git a/modules/anticaptcha/compat/__init__.py b/modules/anticaptcha/compat/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/modules/anticaptcha/compat/weboob_capabilities_captcha.py b/modules/anticaptcha/compat/weboob_capabilities_captcha.py new file mode 100644 index 0000000000..3b8cc0d9e8 --- /dev/null +++ b/modules/anticaptcha/compat/weboob_capabilities_captcha.py @@ -0,0 +1,191 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with weboob. If not, see . + +from time import sleep + +from weboob.capabilities.base import Capability, BaseObject, StringField, UserError, BytesField +from weboob.exceptions import ( + RecaptchaQuestion, RecaptchaV3Question, NocaptchaQuestion, FuncaptchaQuestion, ImageCaptchaQuestion +) + + +__all__ = [ + 'CapCaptchaSolver', + 'SolverJob', 'RecaptchaJob', 'NocaptchaJob', 'ImageCaptchaJob', + 'CaptchaError', 'UnsolvableCaptcha', 'InvalidCaptcha', 'InsufficientFunds', + 'exception_to_job', +] + + +from weboob.capabilities.captcha import SolverJob as _SolverJob +class SolverJob(_SolverJob): + solution = StringField('CAPTCHA solution') + + +from weboob.capabilities.captcha import RecaptchaJob as _RecaptchaJob +class RecaptchaJob(_RecaptchaJob): + site_url = StringField('Site URL for ReCaptcha service') + site_key = StringField('Site key for ReCaptcha service') + + solution_challenge = StringField('Challenge ID of the solution (output value)') + + +class RecaptchaV3Job(SolverJob): + site_url = StringField('Site URL for ReCaptcha service') + site_key = StringField('Site key for ReCaptcha service') + action = StringField('Website owner defines what user is doing on the page through this parameter.') + + +from weboob.capabilities.captcha import NocaptchaJob as _NocaptchaJob +class NocaptchaJob(_NocaptchaJob): + site_url = StringField('Site URL for NoCaptcha service') + site_key = StringField('Site key for NoCaptcha service') + + +from weboob.capabilities.captcha import FuncaptchaJob as _FuncaptchaJob +class FuncaptchaJob(_FuncaptchaJob): + site_url = StringField('Site URL for FunCaptcha service') + site_key = StringField('Site key for FunCaptcha service') + sub_domain = StringField('Required for some complex cases, but Funcaptcha integrations run without it') + + +from weboob.capabilities.captcha import ImageCaptchaJob as _ImageCaptchaJob +class ImageCaptchaJob(_ImageCaptchaJob): + image = BytesField('data of the image to solve') + + +from weboob.capabilities.captcha import CaptchaError as _CaptchaError +class CaptchaError(_CaptchaError): + """Generic solving error""" + + +from weboob.capabilities.captcha import InvalidCaptcha as _InvalidCaptcha +class InvalidCaptcha(_InvalidCaptcha): + """CAPTCHA cannot be used (e.g. invalid image format)""" + + +from weboob.capabilities.captcha import UnsolvableCaptcha as _UnsolvableCaptcha +class UnsolvableCaptcha(_UnsolvableCaptcha): + """CAPTCHA is too hard or impossible""" + + +from weboob.capabilities.captcha import InsufficientFunds as _InsufficientFunds +class InsufficientFunds(_InsufficientFunds): + """Not enough funds to pay solution""" + + +def exception_to_job(exc): + if isinstance(exc, RecaptchaQuestion): + job = RecaptchaJob() + job.site_url = exc.website_url + job.site_key = exc.website_key + elif isinstance(exc, RecaptchaV3Question): + job = RecaptchaV3Job() + job.site_url = exc.website_url + job.site_key = exc.website_key + job.action = exc.action + elif isinstance(exc, NocaptchaQuestion): + job = NocaptchaJob() + job.site_url = exc.website_url + job.site_key = exc.website_key + elif isinstance(exc, FuncaptchaQuestion): + job = FuncaptchaJob() + job.site_url = exc.website_url + job.site_key = exc.website_key + job.sub_domain = exc.sub_domain + elif isinstance(exc, ImageCaptchaQuestion): + job = ImageCaptchaJob() + job.image = exc.image_data + else: + raise NotImplementedError() + + return job + + +from weboob.capabilities.captcha import CapCaptchaSolver as _CapCaptchaSolver +class CapCaptchaSolver(_CapCaptchaSolver): + """ + Provide CAPTCHA solving + """ + + RETRIES = 30 + WAIT_TIME = 2 + + def create_job(self, job): + """Start a CAPTCHA solving job + + The `job.id` shall be filled. The CAPTCHA is not solved yet when the method returns. + + :param job: job to start + :type job: :class:`SolverJob` + :raises: :class:`NotImplementedError` if CAPTCHA type is not supported + :raises: :class:`CaptchaError` in case of other error + """ + raise NotImplementedError() + + def poll_job(self, job): + """Check if a job was solved + + If `job` is solved, return True and fill `job.solution`. + Return False if solution is still pending. + In case of solving problem, an exception may be raised. + + It should not wait for the solution but return the current state. + + :param job: job to check and to fill when solved + :type job: :class:`SolverJob` + :returns: True if the job was solved + :rtype: bool + :raises: :class:`CaptchaError` + """ + raise NotImplementedError() + + def solve_catpcha_blocking(self, job): + """Start a CAPTCHA solving job and wait for its solution + + :param job: job to start and solve + :type job: :class:`SolverJob` + :raises: :class:`CaptchaError` + """ + + self.create_job(job) + for i in range(self.RETRIES): + sleep(self.WAIT_TIME) + if self.poll_job(job): + return job + + def report_wrong_solution(self, job): + """Report a solved job as a wrong solution + + Sometimes, jobs are solved, but the solution is rejected by the CAPTCHA + site because the solution is wrong. + This method reports the solution as wrong to the CAPTCHA solver. + + :param job: job to flag + :type job: :class:`SolverJob` + """ + raise NotImplementedError() + + def get_balance(self): + """Get the prepaid balance left + + :rtype: float + """ + raise NotImplementedError() + diff --git a/modules/anticaptcha/module.py b/modules/anticaptcha/module.py index 76e0ab0fa4..acd41d413d 100644 --- a/modules/anticaptcha/module.py +++ b/modules/anticaptcha/module.py @@ -21,7 +21,9 @@ from weboob.tools.backend import Module, BackendConfig -from weboob.capabilities.captcha import CapCaptchaSolver, ImageCaptchaJob, RecaptchaJob, NocaptchaJob, FuncaptchaJob +from .compat.weboob_capabilities_captcha import ( + CapCaptchaSolver, ImageCaptchaJob, RecaptchaJob, RecaptchaV3Job, NocaptchaJob, FuncaptchaJob +) from weboob.tools.value import ValueBackendPassword from .browser import AnticaptchaBrowser @@ -53,6 +55,8 @@ def create_job(self, job): job.id = self.browser.post_image(job.image) elif isinstance(job, RecaptchaJob): job.id = self.browser.post_recaptcha(job.site_url, job.site_key) + elif isinstance(job, RecaptchaV3Job): + job.id = self.browser.post_gcaptchav3(job.site_url, job.site_key, job.action) elif isinstance(job, NocaptchaJob): job.id = self.browser.post_nocaptcha(job.site_url, job.site_key) elif isinstance(job, FuncaptchaJob): diff --git a/modules/apivie/compat/weboob_capabilities_bank.py b/modules/apivie/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/apivie/compat/weboob_capabilities_bank.py +++ b/modules/apivie/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/axabanque/browser.py b/modules/axabanque/browser.py index a0397ad907..f5ff639c22 100644 --- a/modules/axabanque/browser.py +++ b/modules/axabanque/browser.py @@ -42,7 +42,9 @@ AccountsPage as BankAccountsPage, CBTransactionsPage, TransactionsPage, UnavailablePage, IbanPage, LifeInsuranceIframe, BoursePage, BankProfilePage, ) -from .pages.wealth import AccountsPage as WealthAccountsPage, InvestmentPage, HistoryPage, ProfilePage +from .pages.wealth import ( + AccountsPage as WealthAccountsPage, InvestmentPage, HistoryPage, ProfilePage, AccountDetailsPage, +) from .pages.transfer import ( RecipientsPage, AddRecipientPage, ValidateTransferPage, RegisterTransferPage, ConfirmTransferPage, RecipientConfirmationPage, @@ -217,6 +219,7 @@ def iter_accounts(self): self.transactions.go() self.cache['accs'] = accounts + self.bank_accounts.go() return self.cache['accs'] @need_login @@ -505,13 +508,14 @@ def get_profile(self): class AXAAssurance(AXABrowser): BASEURL = 'https://espaceclient.axa.fr' - accounts = URL('/accueil.html', WealthAccountsPage) - investment = URL('/content/ecc-popin-cards/savings/[^/]+/repartition', InvestmentPage) - history = URL('.*accueil/savings/(\w+)/contract', - 'https://espaceclient.axa.fr/#', HistoryPage) - documents = URL('https://espaceclient.axa.fr/content/espace-client/accueil/mes-documents/attestations-d-assurances.content-inner.din_CERTIFICATE.html', DocumentsPage) - download = URL('/content/ecc-popin-cards/technical/detailed/document.downloadPdf.html', - '/content/ecc-popin-cards/technical/detailed/document/_jcr_content/', + accounts = URL(r'/accueil.html', WealthAccountsPage) + account_details = URL('.*accueil/savings/(\w+)/contract', + r'https://espaceclient.axa.fr/#', AccountDetailsPage) + investment = URL(r'/content/ecc-popin-cards/savings/[^/]+/repartition', InvestmentPage) + history = URL(r'/content/ecc-popin-cards/savings/savings/postsales.mawGetPostSalesOperations.json', HistoryPage) + documents = URL(r'https://espaceclient.axa.fr/content/espace-client/accueil/mes-documents/attestations-d-assurances.content-inner.din_CERTIFICATE.html', DocumentsPage) + download = URL(r'/content/ecc-popin-cards/technical/detailed/document.downloadPdf.html', + r'/content/ecc-popin-cards/technical/detailed/document/_jcr_content/', DownloadPage) profile = URL(r'/content/ecc-popin-cards/transverse/userprofile.content-inner.html\?_=\d+', ProfilePage) @@ -560,18 +564,21 @@ def iter_investment(self, account): @need_login def iter_history(self, account): - self.go_wealth_pages(account) - pagination_url = self.page.get_pagination_url() - try: - self.location(pagination_url, params={'skip': 0}) - except ClientError as e: - assert e.response.status_code == 406 - self.logger.info('not doing pagination for account %r, site seems broken', account) - for tr in self.page.iter_history(no_pagination=True): - yield tr + ''' There is now an API for the accounts history, however transactions are not + sorted by date in the JSON. The website fetches 5 years of history maximum. + For some accounts, the access to the transactions JSON is not available yet. ''' + params = { + 'startDate': (date.today() - relativedelta(years=2)).year, + 'endDate': date.today().year, + 'pid': account.id, + } + self.history.go(params=params) + error_code = self.page.get_error_code() + if error_code: + self.logger.warning('Error when trying to access the history JSON, history will be skipped for this account.') return - for tr in self.page.iter_history(): + for tr in sorted_transactions(self.page.iter_history()): yield tr def iter_coming(self, account): diff --git a/modules/axabanque/compat/weboob_capabilities_bank.py b/modules/axabanque/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/axabanque/compat/weboob_capabilities_bank.py +++ b/modules/axabanque/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/axabanque/pages/bank.py b/modules/axabanque/pages/bank.py index 57c6e6ca6f..41ae49c5b8 100644 --- a/modules/axabanque/pages/bank.py +++ b/modules/axabanque/pages/bank.py @@ -82,6 +82,7 @@ class AccountsPage(LoggedPage, MyHTMLPage): ('livret', Account.TYPE_SAVINGS), ('ldd', Account.TYPE_SAVINGS), ('pel', Account.TYPE_SAVINGS), + ('cel', Account.TYPE_SAVINGS), ('pea', Account.TYPE_PEA), ('titres', Account.TYPE_MARKET), ('valorisation', Account.TYPE_MARKET), diff --git a/modules/axabanque/pages/compat/weboob_capabilities_bank.py b/modules/axabanque/pages/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/axabanque/pages/compat/weboob_capabilities_bank.py +++ b/modules/axabanque/pages/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/axabanque/pages/wealth.py b/modules/axabanque/pages/wealth.py index 612a61b71f..29c1b84c50 100644 --- a/modules/axabanque/pages/wealth.py +++ b/modules/axabanque/pages/wealth.py @@ -17,28 +17,29 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . +from __future__ import unicode_literals import re -from weboob.browser.pages import HTMLPage, LoggedPage, pagination -from weboob.browser.elements import ListElement, ItemElement, method, TableElement +from decimal import Decimal +from weboob.browser.pages import HTMLPage, LoggedPage, JsonPage +from weboob.browser.elements import ListElement, DictElement, ItemElement, method, TableElement from .compat.weboob_browser_filters_standard import ( - Async, AsyncLoad, CleanDecimal, CleanText, Currency, Date, Eval, Field, Lower, MapIn, QueryValue, Regexp, + CleanDecimal, CleanText, Currency, Date, Eval, Field, Lower, MapIn, QueryValue, Regexp, ) from weboob.browser.filters.html import Attr, Link, TableCell +from weboob.browser.filters.json import Dict from .compat.weboob_capabilities_bank import Account, Investment from weboob.capabilities.profile import Person from weboob.capabilities.base import NotAvailable, NotLoaded from weboob.tools.capabilities.bank.transactions import FrenchTransaction -def MyDecimal(*args, **kwargs): - kwargs.update(replace_dots=True, default=NotAvailable) - return CleanDecimal(*args, **kwargs) +def float_to_decimal(f): + return Decimal(str(f)) class AccountsPage(LoggedPage, HTMLPage): - @method class iter_accounts(ListElement): item_xpath = '//div[contains(@data-route, "/savings/")]' @@ -55,10 +56,10 @@ class item(ItemElement): condition = lambda self: Field('balance')(self) is not NotAvailable - obj_id = Regexp(CleanText('.//span[has-class("small-title")]'), '(\d+)') + obj_id = Regexp(CleanText('.//span[has-class("small-title")]'), r'([\d/]+)') obj_label = CleanText('.//h3[has-class("card-title")]') - obj_balance = MyDecimal('.//p[has-class("amount-card")]') - obj_valuation_diff = MyDecimal('.//p[@class="performance"]') + obj_balance = CleanDecimal.French('.//p[has-class("amount-card")]') + obj_valuation_diff = CleanDecimal.French('.//p[@class="performance"]', default=NotAvailable) def obj_url(self): url = Attr('.', 'data-route')(self) @@ -154,66 +155,40 @@ def is_detail(self): return bool(self.doc.xpath(u'//th[contains(text(), "Valeur de la part")]')) -class Transaction(FrenchTransaction): - PATTERNS = [(re.compile(u'^(?Psouscription.*)'), FrenchTransaction.TYPE_DEPOSIT), - (re.compile(u'^(?P.*)'), FrenchTransaction.TYPE_BANK), - ] - - -class HistoryPage(LoggedPage, HTMLPage): - def build_doc(self, content): - # we got empty pages at end of pagination - if not content.strip(): - content = b"" - return super(HistoryPage, self).build_doc(content) - +class AccountDetailsPage(LoggedPage, HTMLPage): def get_account_url(self, url): - return Attr(u'//a[@href="%s"]' % url, 'data-target')(self.doc) + return Attr('//a[@href="%s"]' % url, 'data-target')(self.doc) def get_investment_url(self): return Attr('//div[has-class("card-distribution")]', 'data-url', default=None)(self.doc) - def get_pagination_url(self): - return Attr('//div[contains(@class, "default")][@data-module-card-list--current-page]', 'data-module-card-list--url')(self.doc) - - @method - class get_investments(ListElement): - item_xpath = '//div[@class="white-bg"][.//strong[contains(text(), "support")]]/following-sibling::div' - class item(ItemElement): - klass = Investment +class Transaction(FrenchTransaction): + PATTERNS = [ + (re.compile('^(?Psouscription.*)'), FrenchTransaction.TYPE_DEPOSIT), + (re.compile('^(?P.*)'), FrenchTransaction.TYPE_BANK), + ] - obj_label = CleanText('.//div[has-class("t-data__label")]') - obj_valuation = MyDecimal('.//div[has-class("t-data__amount") and has-class("desktop")]') - obj_portfolio_share = Eval(lambda x: x / 100, CleanDecimal('.//div[has-class("t-data__amount_label")]')) - @pagination +class HistoryPage(LoggedPage, JsonPage): @method - class iter_history(ListElement): - item_xpath = '//div[contains(@data-url, "savingsdetailledcard")]' - - def next_page(self): - if not CleanText(self.item_xpath, default=None)(self): - return - elif self.env.get('no_pagination'): - return - - return re.sub(r'(?<=\bskip=)(\d+)', lambda m: str(int(m.group(1)) + 10), self.page.url) + class iter_history(DictElement): class item(ItemElement): klass = Transaction - load_details = Attr('.', 'data-url') & AsyncLoad + obj_raw = Transaction.Raw(Dict('label')) + obj_date = Date(Dict('date')) + obj_amount = Eval(float_to_decimal, Dict('gross_amount/value')) - obj_raw = Transaction.Raw('.//div[has-class("desktop")]//em') - obj_date = Date(CleanText('.//div[has-class("t-data__date") and has-class("desktop")]'), dayfirst=True) - obj_amount = MyDecimal('.//div[has-class("t-data__amount") and has-class("desktop")]') + def validate(self, obj): + return CleanText(Dict('status'))(self) == 'DONE' - def obj_investments(self): - investments = list(Async('details').loaded_page(self).get_investments()) - for inv in investments: - inv.vdate = Field('date')(self) - return investments + def get_error_code(self): + # The server returns a list if it worked and a dict in case of error + if isinstance(self.doc, dict) and 'return' in self.doc: + return self.doc['return']['error']['code'] + return None class ProfilePage(LoggedPage, HTMLPage): diff --git a/modules/banqueaccord/compat/weboob_capabilities_bank.py b/modules/banqueaccord/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/banqueaccord/compat/weboob_capabilities_bank.py +++ b/modules/banqueaccord/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/banquepopulaire/browser.py b/modules/banquepopulaire/browser.py index a6342afead..dd115a6343 100644 --- a/modules/banquepopulaire/browser.py +++ b/modules/banquepopulaire/browser.py @@ -187,18 +187,39 @@ def __init__(self, website, *args, **kwargs): self.investments = {} + # HACK, the website may crash with legacy passwords (legacy means not only digits) + # If the website crashes and if we have a legacy password, we raise WrongPass instead of BrowserUnavailable + self.is_password_only_digits = None + def deinit(self): super(BanquePopulaire, self).deinit() self.linebourse.deinit() no_login = 0 + def follow_back_button_if_any(self, params=None, actions=None): + """ + Look for a Retour button and follow it using a POST + :param params: Optional form params to use (default: call self.page.get_params()) + :param actions: Optional actions to use (default: call self.page.get_button_actions()) + :return: None + """ + if not self.page: + return + + data = self.page.get_back_button_params(params=params, actions=actions) + if data: + self.location('/cyber/internet/ContinueTask.do', data=data) + @no_need_login def do_login(self): self.location(self.BASEURL) # avoids trying to relog in while it's already on home page if self.home_page.is_here(): return + + self.is_password_only_digits = self.password.isdigit() + self.page.login(self.username, self.password) if self.login_page.is_here(): raise BrowserIncorrectPassword() @@ -231,6 +252,9 @@ def go_on_accounts_list(self): form['token'] = self.page.build_token(form['token']) form.submit() + # In case of prevAction maybe we have reached an expanded accounts list page, need to go back + self.follow_back_button_if_any() + @retry(LoggedOut) @need_login def get_accounts_list(self, get_iban=True): @@ -301,13 +325,7 @@ def set_gocardless_transaction_details(self, transaction): transaction.raw = '%s %s' % (transaction.raw, ref) # Needed to preserve navigation. - btn = self.page.doc.xpath('.//button[span[text()="Retour"]]') - if len(btn): - _data = self.page.get_params() - actions = self.page.get_button_actions() - _data.update(actions[btn[0].attrib['id']]) - _data['token'] = self.page.build_token(_data['token']) - self.location('/cyber/internet/ContinueTask.do', data=_data) + self.follow_back_button_if_any() @retry(LoggedOut) @need_login diff --git a/modules/banquepopulaire/compat/weboob_capabilities_bank.py b/modules/banquepopulaire/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/banquepopulaire/compat/weboob_capabilities_bank.py +++ b/modules/banquepopulaire/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/banquepopulaire/pages.py b/modules/banquepopulaire/pages.py index e39ec9f49b..4d915dad8d 100644 --- a/modules/banquepopulaire/pages.py +++ b/modules/banquepopulaire/pages.py @@ -164,6 +164,19 @@ def get_button_actions(self): } return actions + def get_back_button_params(self, params=None, actions=None): + btn = self.doc.xpath('.//button[span[text()="Retour"]]') + if not btn: + return + + params = params or self.get_params() + actions = actions or self.get_button_actions() + key = btn[0].attrib['id'] + assert actions.get(key), "Key %s not found in actions %s" % (key, actions) # Currently it never happens + params.update(actions[key]) + params['token'] = self.build_token(params['token']) + return params + class MyHTMLPage(BasePage, HTMLPage): def build_doc(self, data, *args, **kwargs): @@ -258,6 +271,10 @@ def on_load(self): class ErrorPage(LoggedPage, MyHTMLPage): def on_load(self): + # HACK: some accounts with legacy password fails, people needs to update it + if not self.browser.is_password_only_digits: + raise BrowserIncorrectPassword() + if CleanText('//script[contains(text(), "momentanément indisponible")]')(self.doc): raise BrowserUnavailable(u"Le service est momentanément indisponible") elif CleanText('//h1[contains(text(), "Cette page est indisponible")]')(self.doc): @@ -630,6 +647,7 @@ def iter_accounts(self, next_pages): if len(tds) >= 5 and len(tds[self.COL_COMING].xpath('.//a')) > 0: _params = account._params.copy() _params['dialogActionPerformed'] = 'ENCOURS_COMPTE' + _params['attribute($SEL_$%s)' % tr.attrib['id'].split('_')[0]] = tr.attrib['id'].split('_', 1)[1] # If there is an action needed before going to the cards page, save it. m = re.search('dialogActionPerformed=([\w_]+)', self.url) @@ -645,11 +663,7 @@ def iter_accounts(self, next_pages): yield account # Needed to preserve navigation. - btn = self.doc.xpath('.//button[span[text()="Retour"]]') - if len(btn) > 0: - _params = params.copy() - _params.update(actions[btn[0].attrib['id']]) - self.browser.open('/cyber/internet/ContinueTask.do', data=_params) + self.browser.follow_back_button_if_any(params=params.copy(), actions=actions) class AccountsFullPage(AccountsPage): @@ -746,12 +760,7 @@ def iter_accounts(self, next_pages): yield account # Needed to preserve navigation. - btn = self.doc.xpath('.//button[span[text()="Retour"]]') - if len(btn) > 0: - actions = self.get_button_actions() - _params = params.copy() - _params.update(actions[btn[0].attrib['id']]) - self.browser.open('/cyber/internet/ContinueTask.do', data=_params) + self.browser.follow_back_button_if_any(params=params.copy()) class Transaction(FrenchTransaction): @@ -842,6 +851,7 @@ def get_account_history(self): debit = cleaner(tds[self.COL_DEBIT]) credit = cleaner(tds[self.COL_CREDIT]) + t.bdate = Date(dayfirst=True).filter(cleaner(tds[self.COL_COMPTA_DATE])) t.parse(date, re.sub(r'[ ]+', ' ', raw), vdate) t.set_amount(credit, debit) t._amount_type = 'debit' if t.amount == debit else 'credit' @@ -900,7 +910,7 @@ def get_card_history(self, account, coming): t.parse(debit_date, re.sub(r'[ ]+', ' ', label)) t.set_amount(amount) - t.rdate = t.parse_date(date) + t.rdate = t.bdate = t.parse_date(date) t.original_currency = currency if not t.type: t.type = Transaction.TYPE_DEFERRED_CARD diff --git a/modules/barclays/compat/weboob_capabilities_bank.py b/modules/barclays/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/barclays/compat/weboob_capabilities_bank.py +++ b/modules/barclays/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/becm/compat/weboob_capabilities_bank.py b/modules/becm/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/becm/compat/weboob_capabilities_bank.py +++ b/modules/becm/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/bforbank/compat/weboob_capabilities_bank.py b/modules/bforbank/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/bforbank/compat/weboob_capabilities_bank.py +++ b/modules/bforbank/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/binck/compat/weboob_capabilities_bank.py b/modules/binck/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/binck/compat/weboob_capabilities_bank.py +++ b/modules/binck/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/bnpcards/compat/weboob_capabilities_bank.py b/modules/bnpcards/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/bnpcards/compat/weboob_capabilities_bank.py +++ b/modules/bnpcards/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/bnpcards/corporate/compat/weboob_capabilities_bank.py b/modules/bnpcards/corporate/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/bnpcards/corporate/compat/weboob_capabilities_bank.py +++ b/modules/bnpcards/corporate/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/bnporc/company/compat/weboob_capabilities_bank.py b/modules/bnporc/company/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/bnporc/company/compat/weboob_capabilities_bank.py +++ b/modules/bnporc/company/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/bnporc/compat/weboob_capabilities_bank.py b/modules/bnporc/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/bnporc/compat/weboob_capabilities_bank.py +++ b/modules/bnporc/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/bnporc/enterprise/compat/weboob_capabilities_bank.py b/modules/bnporc/enterprise/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/bnporc/enterprise/compat/weboob_capabilities_bank.py +++ b/modules/bnporc/enterprise/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/bnporc/pp/browser.py b/modules/bnporc/pp/browser.py index 6262876c76..5c5ec161ca 100644 --- a/modules/bnporc/pp/browser.py +++ b/modules/bnporc/pp/browser.py @@ -38,6 +38,7 @@ from weboob.browser.elements import DataError from weboob.exceptions import BrowserIncorrectPassword from weboob.tools.value import Value, ValueBool +from weboob.tools.capabilities.bank.investments import create_french_liquidity from .pages import ( LoginPage, AccountsPage, AccountsIBANPage, HistoryPage, TransferInitPage, @@ -286,8 +287,8 @@ def iter_coming_operations(self, account): @need_login def iter_investment(self, account): - if account.type == Account.TYPE_PEA and account.label.endswith('Espèces'): - return [] + if account.type == Account.TYPE_PEA and 'espèces' in account.label.lower(): + return [create_french_liquidity(account.balance)] # Life insurances and PERP may be scraped from the API or from the "Assurance Vie" space, # so we need to discriminate between both using account._details: diff --git a/modules/bnporc/pp/compat/weboob_capabilities_bank.py b/modules/bnporc/pp/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/bnporc/pp/compat/weboob_capabilities_bank.py +++ b/modules/bnporc/pp/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/bnppere/compat/weboob_capabilities_bank.py b/modules/bnppere/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/bnppere/compat/weboob_capabilities_bank.py +++ b/modules/bnppere/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/bolden/compat/weboob_capabilities_bank.py b/modules/bolden/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/bolden/compat/weboob_capabilities_bank.py +++ b/modules/bolden/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/boursorama/compat/weboob_capabilities_bank.py b/modules/boursorama/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/boursorama/compat/weboob_capabilities_bank.py +++ b/modules/boursorama/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/boursorama/pages.py b/modules/boursorama/pages.py index 5e7eabc604..66868e9253 100644 --- a/modules/boursorama/pages.py +++ b/modules/boursorama/pages.py @@ -219,6 +219,7 @@ def is_here(self): 'comptes courants': Account.TYPE_CHECKING, 'cav': Account.TYPE_CHECKING, 'livret': Account.TYPE_SAVINGS, + 'livret-a': Account.TYPE_SAVINGS, 'pel': Account.TYPE_SAVINGS, 'cel': Account.TYPE_SAVINGS, 'ldd': Account.TYPE_SAVINGS, diff --git a/modules/bouygues/__init__.py b/modules/bouygues/__init__.py index 0e6b4868b7..f6b92a3988 100644 --- a/modules/bouygues/__init__.py +++ b/modules/bouygues/__init__.py @@ -1,3 +1,11 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Budget Insight + +from __future__ import unicode_literals + + from .module import BouyguesModule + __all__ = ['BouyguesModule'] diff --git a/modules/bouygues/browser.py b/modules/bouygues/browser.py index 513b2680b2..f469a38639 100644 --- a/modules/bouygues/browser.py +++ b/modules/bouygues/browser.py @@ -1,158 +1,102 @@ # -*- coding: utf-8 -*- -# Copyright(C) 2010-2015 Bezleputh +# Copyright(C) 2019 Budget Insight # # This file is part of a weboob module. # # This weboob module is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by +# it under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This weboob module is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. +# GNU Lesser General Public License for more details. # -# You should have received a copy of the GNU Affero General Public License +# You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . +from __future__ import unicode_literals + +from time import time from jose import jwt from weboob.browser import LoginBrowser, URL, need_login -from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable -from weboob.browser.exceptions import ClientError, HTTPNotFound -from weboob.tools.compat import urlparse, parse_qs +from weboob.browser.exceptions import HTTPNotFound +from weboob.tools.compat import urlparse, parse_qsl + from .pages import ( - DocumentsPage, HomePage, LoginPage, SubscriberPage, SubscriptionPage, SubscriptionDetailPage, - SendSMSPage, SendSMSErrorPage, UselessPage, DocumentFilePage, ProfilePage, + LoginPage, AppConfigPage, SubscriberPage, SubscriptionPage, SubscriptionDetail, DocumentPage, DocumentDownloadPage, + DocumentFilePage, ) -from weboob.capabilities.messages import CantSendMessage -__all__ = ['BouyguesBrowser'] +class MyURL(URL): + def go(self, *args, **kwargs): + kwargs['id_personne'] = self.browser.id_personne + kwargs['headers'] = self.browser.headers + return super(MyURL, self).go(*args, **kwargs) class BouyguesBrowser(LoginBrowser): BASEURL = 'https://api.bouyguestelecom.fr' - TIMEOUT = 20 - - login = URL(r'https://www.mon-compte.bouyguestelecom.fr/cas/login', LoginPage) - home = URL(r'https://www.bouyguestelecom.fr/mon-compte', HomePage) - subscriber = URL(r'/personnes/(?P\d+)$', SubscriberPage) - subscriptions = URL(r'/personnes/(?P\d+)/comptes-facturation', SubscriptionPage) - subscriptions_details = URL(r'/comptes-facturation/(?P\d+)/contrats-payes', SubscriptionDetailPage) - document_file = URL(r'/comptes-facturation/(?P\d+)/factures/.*/documents', DocumentFilePage) - documents = URL(r'/comptes-facturation/(?P\d+)/factures', DocumentsPage) - - sms_page = URL(r'https://www.secure.bbox.bouyguestelecom.fr/services/SMSIHD/sendSMS.phtml', - r'https://www.secure.bbox.bouyguestelecom.fr/services/SMSIHD/confirmSendSMS.phtml', - SendSMSPage) - confirm = URL(r'https://www.secure.bbox.bouyguestelecom.fr/services/SMSIHD/resultSendSMS.phtml', UselessPage) - sms_error_page = URL(r'https://www.secure.bbox.bouyguestelecom.fr/services/SMSIHD/SMS_erreur.phtml', - SendSMSErrorPage) - profile = URL(r'/personnes/(?P\d+)/coordonnees', ProfilePage) + login_page = URL(r'https://www.mon-compte.bouyguestelecom.fr/cas/login', LoginPage) + app_config = URL(r'https://www.bouyguestelecom.fr/mon-compte/data/app-config.json', AppConfigPage) + subscriber_page = MyURL(r'/personnes/(?P\d+)$', SubscriberPage) + subscriptions_page = MyURL(r'/personnes/(?P\d+)/comptes-facturation', SubscriptionPage) + subscription_detail_page = URL(r'/comptes-facturation/(?P\d+)/contrats-payes', SubscriptionDetail) + document_file_page = URL(r'/comptes-facturation/(?P\d+)/factures/.*/documents/.*', DocumentFilePage) + documents_page = URL(r'/comptes-facturation/(?P\d+)/factures(\?|$)', DocumentPage) + document_download_page = URL(r'/comptes-facturation/(?P\d+)/factures/.*(\?|$)', DocumentDownloadPage) def __init__(self, username, password, lastname, *args, **kwargs): super(BouyguesBrowser, self).__init__(username, password, *args, **kwargs) self.lastname = lastname + self.id_personne = None self.headers = None - self.id_user = None def do_login(self): - self.login.go() - - if self.home.is_here(): - return - + self.login_page.go() self.page.login(self.username, self.password, self.lastname) - if self.login.is_here(): - error = self.page.get_error() - if error and 'mot de passe' in error: - raise BrowserIncorrectPassword(error) - raise AssertionError("Unhandled error at login: {}".format(error)) + # q is timestamp millisecond + self.app_config.go(params={'q': int(time()*1000)}) + client_id = self.page.get_client_id() - # after login we need to get some tokens to use bouygues api - data = { + params = { + 'client_id': client_id, 'response_type': 'id_token token', - 'client_id': 'a360.bouyguestelecom.fr', 'redirect_uri': 'https://www.bouyguestelecom.fr/mon-compte/' } - self.location('https://oauth2.bouyguestelecom.fr/authorize', params=data) - - parsed_url = urlparse(self.response.url) - fragment = parse_qs(parsed_url.fragment) - - if not fragment: - query = parse_qs(parsed_url.query) - if 'server_error' in query.get('error', []): - raise BrowserUnavailable(query['error_description'][0]) - - claims = jwt.get_unverified_claims(fragment['id_token'][0]) - self.headers = {'Authorization': 'Bearer %s' % fragment['access_token'][0]} - self.id_user = claims['id_personne'] + self.location('https://oauth2.bouyguestelecom.fr/authorize', params=params) + fragments = dict(parse_qsl(urlparse(self.url).fragment)) - @need_login - def post_message(self, message): - self.sms_page.go() - - if self.sms_error_page.is_here(): - raise CantSendMessage(self.page.get_error_message()) - - receivers = ";".join(message.receivers) if message.receivers else self.username - self.page.send_sms(message, receivers) - - if self.sms_error_page.is_here(): - raise CantSendMessage(self.page.get_error_message()) - - self.confirm.open() + self.id_personne = jwt.get_unverified_claims(fragments['id_token'])['id_personne'] + authorization = 'Bearer ' + fragments['access_token'] + self.headers = {'Authorization': authorization} @need_login def iter_subscriptions(self): - self.subscriber.go(idUser=self.id_user, headers=self.headers) - subscriber = self.page.get_subscriber() - phone_list = self.page.get_phone_list() - - self.subscriptions.go(idUser=self.id_user, headers=self.headers) - for sub in self.page.iter_subscriptions(subscriber=subscriber): - try: - self.subscriptions_details.go(idSub=sub.id, headers=self.headers) - sub.label = self.page.get_label() - sub._is_holder = self.page.is_holder() - except ClientError: - # if another person pay for your subscription you may not have access to this page with your credentials - sub.label = phone_list - if not sub.label: - if not sub._is_holder: - sub.label = subscriber - else: - # If the subscriber is the holder but the subscription does not have a phone number anyway - # It means that the subscription has not been activated yet - continue + subscriber = self.subscriber_page.go().get_subscriber() + self.subscriptions_page.go() + for sub in self.page.iter_subscriptions(): + sub.subscriber = subscriber + sub.label = self.subscription_detail_page.go(id_account=sub.id, headers=self.headers).get_label() yield sub @need_login def iter_documents(self, subscription): try: self.location(subscription.url, headers=self.headers) - return self.page.iter_documents(subid=subscription.id) except HTTPNotFound as error: - if error.response.json()['error'] in ('facture_introuvable', 'compte_jamais_facture'): + json_response = error.response.json() + if json_response['error'] in ('facture_introuvable', 'compte_jamais_facture'): return [] raise + return self.page.iter_documents(subid=subscription.id) @need_login def download_document(self, document): - self.location(document.url, headers=self.headers) - return self.open(self.page.get_one_shot_download_url()).content - - @need_login - def get_profile(self): - self.subscriber.go(idUser=self.id_user, headers=self.headers) - subscriber = self.page.get_subscriber() - - self.profile.go(idUser=self.id_user, headers=self.headers) - - return self.page.get_profile(subscriber=subscriber) + return self.location(document.url, headers=self.headers).content diff --git a/modules/bouygues/module.py b/modules/bouygues/module.py index 579c76b7d6..16cc4b31b3 100644 --- a/modules/bouygues/module.py +++ b/modules/bouygues/module.py @@ -1,30 +1,29 @@ # -*- coding: utf-8 -*- -# Copyright(C) 2010-2015 Bezleputh +# Copyright(C) 2019 Budget Insight # # This file is part of a weboob module. # # This weboob module is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by +# it under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This weboob module is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. +# GNU Lesser General Public License for more details. # -# You should have received a copy of the GNU Affero General Public License +# You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . from __future__ import unicode_literals -from weboob.capabilities.bill import CapDocument, Subscription, Document, SubscriptionNotFound, DocumentNotFound -from weboob.capabilities.messages import CantSendMessage, CapMessages, CapMessagesPost -from weboob.capabilities.base import find_object -from weboob.capabilities.profile import CapProfile + from weboob.tools.backend import Module, BackendConfig -from weboob.tools.value import ValueBackendPassword, Value +from weboob.capabilities.base import find_object +from weboob.capabilities.bill import CapDocument, Document, SubscriptionNotFound, Subscription, DocumentNotFound +from weboob.tools.value import Value, ValueBackendPassword from .browser import BouyguesBrowser @@ -32,25 +31,20 @@ __all__ = ['BouyguesModule'] -class BouyguesModule(Module, CapMessages, CapMessagesPost, CapDocument, CapProfile): +class BouyguesModule(Module, CapDocument): NAME = 'bouygues' - MAINTAINER = 'Bezleputh' - EMAIL = 'carton_ben@yahoo.fr' + DESCRIPTION = 'Bouygues Télécom' + MAINTAINER = 'Florian Duguet' + EMAIL = 'florian.duguet@budget-insight.com' + LICENSE = 'LGPLv3+' VERSION = '1.5' - DESCRIPTION = u'Bouygues Télécom French mobile phone provider' - LICENSE = 'AGPLv3+' - CONFIG = BackendConfig(Value('login', label='E-mail / N° de Téléphone'), + CONFIG = BackendConfig(Value('login', label='Numéro de mobile, de clé/tablette ou e-mail en @bbox.fr'), ValueBackendPassword('password', label='Mot de passe'), - ValueBackendPassword('lastname', label='Nom de famille', default=u'')) + ValueBackendPassword('lastname', label='Nom de famille', default='')) BROWSER = BouyguesBrowser def create_default_browser(self): - return self.create_browser(username=self.config['login'].get(), password=self.config['password'].get(), lastname=self.config['lastname'].get()) - - def post_message(self, message): - if not message.content.strip(): - raise CantSendMessage('Message content is empty.') - self.browser.post_message(message) + return self.create_browser(self.config['login'].get(), self.config['password'].get(), self.config['lastname'].get()) def iter_subscription(self): return self.browser.iter_subscriptions() @@ -58,20 +52,17 @@ def iter_subscription(self): def get_subscription(self, _id): return find_object(self.iter_subscription(), id=_id, error=SubscriptionNotFound) - def get_document(self, _id): - subid = _id.rsplit('_', 1)[0] - subscription = self.get_subscription(subid) - return find_object(self.iter_documents(subscription), id=_id, error=DocumentNotFound) - def iter_documents(self, subscription): if not isinstance(subscription, Subscription): subscription = self.get_subscription(subscription) return self.browser.iter_documents(subscription) + def get_document(self, _id): + subid = _id.rsplit('_', 1)[0] + subscription = self.get_subscription(subid) + return find_object(self.iter_documents(subscription), id=_id, error=DocumentNotFound) + def download_document(self, document): if not isinstance(document, Document): document = self.get_document(document) return self.browser.download_document(document) - - def get_profile(self): - return self.browser.get_profile() diff --git a/modules/bouygues/pages.py b/modules/bouygues/pages.py index d2a6d76c06..6642c9f372 100644 --- a/modules/bouygues/pages.py +++ b/modules/bouygues/pages.py @@ -1,90 +1,68 @@ # -*- coding: utf-8 -*- -# Copyright(C) 2010-2015 Bezleputh + +# Copyright(C) 2019 Budget Insight # # This file is part of a weboob module. # # This weboob module is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by +# it under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This weboob module is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. +# GNU Lesser General Public License for more details. # -# You should have received a copy of the GNU Affero General Public License +# You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . from __future__ import unicode_literals import re -from datetime import datetime, timedelta - -from weboob.capabilities.messages import CantSendMessage -from weboob.exceptions import BrowserIncorrectPassword, ParseError +from datetime import timedelta -from weboob.capabilities.base import NotLoaded -from weboob.capabilities.bill import Bill, Subscription -from weboob.capabilities.profile import Profile -from weboob.browser.pages import HTMLPage, JsonPage, LoggedPage, PDFPage -from weboob.browser.filters.json import Dict -from .compat.weboob_browser_filters_standard import CleanDecimal, CleanText, Env, Format, Regexp from weboob.browser.elements import DictElement, ItemElement, method +from weboob.browser.filters.json import Dict +from weboob.browser.pages import HTMLPage, JsonPage, LoggedPage, RawPage +from weboob.capabilities import NotAvailable +from weboob.capabilities.bill import Subscription, Bill +from .compat.weboob_browser_filters_standard import Date, CleanDecimal, Env, Format +from weboob.exceptions import BrowserIncorrectPassword class LoginPage(HTMLPage): - def login(self, login, password, lastname): - form = self.get_form(id='log_data') - - form['username'] = login + def login(self, username, password, lastname): + form = self.get_form() + form['username'] = username form['password'] = password if 'lastname' in form: if not lastname: - raise BrowserIncorrectPassword('Le nom de famille est obligatoire.') + raise BrowserIncorrectPassword('Veuillez renseigner votre nom de famille.') form['lastname'] = lastname form.submit() - def get_error(self): - return CleanText('//div[@id="alert_msg"]//p')(self.doc) - -class HomePage(LoggedPage, HTMLPage): - pass +class AppConfigPage(JsonPage): + def get_client_id(self): + return self.doc['config']['oauth']['clientId'] class SubscriberPage(LoggedPage, JsonPage): def get_subscriber(self): - if self.doc['type'] == 'INDIVIDU': - sub_dict = self.doc - else: - sub_dict = self.doc['representantLegal'] - return "%s %s %s" % (sub_dict['civilite'], sub_dict['prenom'], sub_dict['nom']) - - def get_phone_list(self): - num_tel_list = [] - for phone in self.doc.get('comptesAcces', []): - num_tel_list.append(' '.join(phone[i:i + 2] for i in range(0, len(phone), 2))) - - return ' - '.join(num_tel_list) - - -class SubscriptionPage(LoggedPage, JsonPage): - @method - class iter_subscriptions(DictElement): - item_xpath = 'items' + assert self.doc['type'] in ('INDIVIDU', 'ENTREPRISE'), "%s is unknown" % self.doc['type'] - class item(ItemElement): - klass = Subscription + if self.doc['type'] == 'INDIVIDU': + subscriber_dict = self.doc + elif self.doc['type'] == 'ENTREPRISE': + subscriber_dict = self.doc['representantLegal'] - obj_id = Dict('id') - obj_url = Dict('_links/factures/href') - obj_subscriber = Env('subscriber') + return '%s %s %s' % (subscriber_dict['civilite'], subscriber_dict['prenom'], subscriber_dict['nom']) -class SubscriptionDetailPage(LoggedPage, JsonPage): +class SubscriptionDetail(LoggedPage, JsonPage): def get_label(self): label_list = [] for s in self.doc['items']: @@ -96,46 +74,33 @@ def get_label(self): return ' - '.join(label_list) - def is_holder(self): - return any(CleanText(Dict('utilisateur/libelleProfilDroits'), default=None)(s) == 'Accès titulaire' for s in self.doc['items'] if 'utilisateur' in s) - - -class SendSMSPage(HTMLPage): - def send_sms(self, message, receivers): - sms_number = CleanDecimal(Regexp(CleanText('//span[@class="txt12-o"][1]/strong'), r'(\d*) SMS.*'))(self.doc) - if sms_number == 0: - msg = CleanText('//span[@class="txt12-o"][1]')(self.doc) - raise CantSendMessage(msg) - - form = self.get_form('//form[@name="formSMS"]') - form["fieldMsisdn"] = receivers - form["fieldMessage"] = message.content +class SubscriptionPage(LoggedPage, JsonPage): + @method + class iter_subscriptions(DictElement): + item_xpath = 'items' - form.submit() + class item(ItemElement): + klass = Subscription + obj_id = Dict('id') + obj_url = Dict('_links/factures/href') -class SendSMSErrorPage(HTMLPage): - def get_error_message(self): - return CleanText('//span[@class="txt12-o"][1]')(self.doc) +class MyDate(Date): + """ + some date are datetime and contains date at GMT, and always at 22H or 23H + but date inside PDF file is at GMT +1H or +2H (depends of summer or winter hour) + so we add one day and skip time to get good date + """ + def filter(self, txt): + date = super(MyDate, self).filter(txt) + if date: + date += timedelta(days=1) + return date -class DocumentsPage(LoggedPage, JsonPage): - FRENCH_MONTHS = { - 1: 'Janvier', - 2: 'Février', - 3: 'Mars', - 4: 'Avril', - 5: 'Mai', - 6: 'Juin', - 7: 'Juillet', - 8: 'Août', - 9: 'Septembre', - 10: 'Octobre', - 11: 'Novembre', - 12: 'Décembre', - } +class DocumentPage(LoggedPage, JsonPage): @method class iter_documents(DictElement): item_xpath = 'items' @@ -144,76 +109,22 @@ class item(ItemElement): klass = Bill obj_id = Format('%s_%s', Env('subid'), Dict('idFacture')) - - def obj_url(self): - try: - link = Dict('_links/facturePDF/href')(self) - except ParseError: - # yes, sometimes it's just a misspelling word, but just sometimes... - link = Dict('_links/facturePDFDF/href')(self) - - return 'https://api.bouyguestelecom.fr%s' % link - - obj_date = Env('date') - obj_duedate = Env('duedate') - obj_format = 'pdf' - obj_label = Env('label') obj_price = CleanDecimal(Dict('mntTotFacture')) + obj_url = Dict('_links/facturePDF/href') + obj_date = MyDate(Dict('dateFacturation')) + obj_duedate = MyDate(Dict('dateLimitePaieFacture', default=NotAvailable), default=NotAvailable) + obj_label = Format('Facture %s', Dict('idFacture')) + obj_format = 'pdf' obj_currency = 'EUR' - def parse(self, el): - bill_date = datetime.strptime(Dict('dateFacturation')(self), "%Y-%m-%dT%H:%M:%SZ").date() - - # dateFacturation is like: 'YYYY-MM-DDTHH:00:00Z' where Z is UTC time and HH 23 in winter and 22 in summer - # which always correspond to the day after at midnight in French time zone - # so we remove hour and consider the day after as date (which is also the date inside pdf) - self.env['date'] = bill_date + timedelta(days=1) - duedate = Dict('dateLimitePaieFacture', default=NotLoaded)(self) - if duedate: - self.env['duedate'] = datetime.strptime(duedate, "%Y-%m-%dT%H:%M:%SZ").date() + timedelta(days=1) - else: - # for some connections we don't have duedate (why ?) - self.env['duedate'] = NotLoaded - - self.env['label'] = "%s %d" % (self.page.FRENCH_MONTHS[self.env['date'].month], self.env['date'].year) - - def get_one_shot_download_url(self): - return self.doc['_actions']['telecharger']['action'] - - -class ProfilePage(LoggedPage, JsonPage): - def get_profile(self, subscriber): - data = self.doc - - last_address = data['adressesPostales'][0] - for address in data['adressesPostales']: - if address['dateMiseAJour'] > last_address['dateMiseAJour']: - last_address = address - - p = Profile() - p.name = subscriber - p.address = '%s %s %s %s' % (last_address['numero'], last_address['rue'], - last_address['codePostal'], last_address['ville']) - p.country = last_address['pays'] - - for email in data['emails']: - if email['emailPrincipal']: - p.email = email['email'] - break - - if 'telephones' in data: - for phone in data['telephones']: - if phone['telephonePrincipal']: - p.phone = phone['numero'] - break - - return p - - -class UselessPage(HTMLPage): - pass +class DocumentDownloadPage(LoggedPage, JsonPage): + def on_load(self): + # this url change each time we want to download document, (the same one or another) + self.browser.location(self.doc['_actions']['telecharger']['action']) -class DocumentFilePage(PDFPage): +class DocumentFilePage(LoggedPage, RawPage): + # since url of this file is almost the same than url of DocumentDownloadPage (which is a JsonPage) + # we have to define it to avoid mismatching pass diff --git a/modules/bp/browser.py b/modules/bp/browser.py index 20ce99828f..a057b0b8be 100644 --- a/modules/bp/browser.py +++ b/modules/bp/browser.py @@ -75,7 +75,7 @@ class BPBrowser(LoginBrowser, StatesMixin): '/voscomptes/canalXHTML/pret/encours/detaillerOffrePretConsoListe-encoursPrets.ea', '/voscomptes/canalXHTML/pret/creditRenouvelable/init-consulterCreditRenouvelable.ea', '/voscomptes/canalXHTML/pret/encours/rechercherPret-encoursPrets.ea', - '/voscomptes/canalXHTML/sso/commun/init-integration.ea\?partenaire', + '/voscomptes/canalXHTML/sso/commun/init-integration.ea\?partenaire=cristalCEC', '/voscomptes/canalXHTML/sso/lbpf/souscriptionCristalFormAutoPost.jsp', AccountList) par_accounts_revolving = URL('https://espaceclientcreditconso.labanquepostale.fr/sav/accueil.do', AccountList) @@ -200,9 +200,9 @@ def deinit(self): super(BPBrowser, self).deinit() self.linebourse.deinit() - def location(self, url, **kwargs): + def open(self, *args, **kwargs): try: - return super(BPBrowser, self).location(url, **kwargs) + return super(BPBrowser, self).open(*args, **kwargs) except ServerError as err: if "/../" not in err.response.url: raise @@ -211,7 +211,7 @@ def location(self, url, **kwargs): self.logger.debug('site has "/../" in their url, fixing url manually') parts = list(urlsplit(err.response.url)) parts[2] = os.path.abspath(parts[2]) - return self.location(urlunsplit(parts)) + return self.open(urlunsplit(parts)) def do_login(self): self.location(self.login_url) diff --git a/modules/bp/compat/weboob_capabilities_bank.py b/modules/bp/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/bp/compat/weboob_capabilities_bank.py +++ b/modules/bp/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/bp/pages/accountlist.py b/modules/bp/pages/accountlist.py index d1139328b7..d9bf16a9db 100644 --- a/modules/bp/pages/accountlist.py +++ b/modules/bp/pages/accountlist.py @@ -50,7 +50,12 @@ class item_account_generic(ItemElement): klass = Account def condition(self): - return len(self.el.xpath('.//span[@class="number"]')) > 0 + # For some loans the following xpath is absent and we don't want to skip them + # Also a case of loan that is empty and has no information exists and will be ignored + return (len(self.el.xpath('.//span[@class="number"]')) > 0 or + (Field('type')(self) == Account.TYPE_LOAN and + (len(self.el.xpath('.//div//*[contains(text(),"pas la restitution de ces données.")]')) == 0 and + len(self.el.xpath('.//div[@class="amount"]/span[contains(text(), "Contrat résilié")]')) == 0))) obj_id = CleanText('.//abbr/following-sibling::text()') obj_currency = Currency('.//span[@class="number"]') @@ -70,7 +75,10 @@ def obj_label(self): def obj_balance(self): if Field('type')(self) == Account.TYPE_LOAN: - return -abs(CleanDecimal('.//span[@class="number"]', replace_dots=True)(self)) + balance = CleanDecimal('.//span[@class="number"]', replace_dots=True, default=NotAvailable)(self) + if balance: + balance = -abs(balance) + return balance return CleanDecimal('.//span[@class="number"]', replace_dots=True, default=NotAvailable)(self) def obj_coming(self): @@ -102,7 +110,7 @@ def obj_coming(self): return NotAvailable def obj_iban(self): - if not Field('url')(self): + if not Field('url')(self) or Field('type')(self) == Account.TYPE_LOAN: return NotAvailable details_page = self.page.browser.open(Field('url')(self)).page @@ -192,9 +200,9 @@ def get_revolving_attributes(self, account): loan.currency = account.currency loan.url = account.url - loan.available_amount = CleanDecimal('//tr[td[contains(text(), "Montant Maximum Autorisé") or contains(text(), "Montant autorisé")]]/td[2]')(self.doc) - loan.used_amount = loan.used_amount = CleanDecimal('//tr[td[contains(text(), "Montant Utilisé") or contains(text(), "Montant utilisé")]]/td[2]')(self.doc) - loan.available_amount = CleanDecimal(Regexp(CleanText('//tr[td[contains(text(), "Montant Disponible") or contains(text(), "Montant disponible")]]/td[2]'), r'(.*) au'))(self.doc) + loan.used_amount = CleanDecimal.US('//tr[td[contains(text(), "Montant Utilisé") or contains(text(), "Montant utilisé")]]/td[2]')(self.doc) + loan.available_amount = CleanDecimal.US(Regexp(CleanText('//tr[td[contains(text(), "Montant Disponible") or contains(text(), "Montant disponible")]]/td[2]'), r'(.*) au'))(self.doc) + loan.balance = -loan.used_amount loan._has_cards = False loan.type = Account.TYPE_REVOLVING_CREDIT return loan diff --git a/modules/bp/pages/compat/weboob_capabilities_bank.py b/modules/bp/pages/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/bp/pages/compat/weboob_capabilities_bank.py +++ b/modules/bp/pages/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/bred/bred/compat/weboob_capabilities_bank.py b/modules/bred/bred/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/bred/bred/compat/weboob_capabilities_bank.py +++ b/modules/bred/bred/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/bred/bred/pages.py b/modules/bred/bred/pages.py index d46823f3da..01c40d736f 100644 --- a/modules/bred/bred/pages.py +++ b/modules/bred/bred/pages.py @@ -117,10 +117,12 @@ class AccountsPage(MyJsonPage): '025': Account.TYPE_SAVINGS, # Livret Fidélis '027': Account.TYPE_SAVINGS, # Livret A '037': Account.TYPE_SAVINGS, + '070': Account.TYPE_SAVINGS, # Compte Epargne Logement '077': Account.TYPE_SAVINGS, # Livret Bambino '078': Account.TYPE_SAVINGS, # Livret jeunes '080': Account.TYPE_SAVINGS, # Plan épargne logement '081': Account.TYPE_SAVINGS, + '086': Account.TYPE_SAVINGS, # Compte épargne Moisson '097': Account.TYPE_CHECKING, # Solde en devises '730': Account.TYPE_DEPOSIT, # Compte à terme Optiplus '999': Account.TYPE_MARKET, # no label, we use 'Portefeuille Titres' if needed diff --git a/modules/bred/compat/weboob_capabilities_bank.py b/modules/bred/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/bred/compat/weboob_capabilities_bank.py +++ b/modules/bred/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/bred/dispobank/compat/weboob_capabilities_bank.py b/modules/bred/dispobank/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/bred/dispobank/compat/weboob_capabilities_bank.py +++ b/modules/bred/dispobank/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/btpbanque/compat/weboob_capabilities_bank.py b/modules/btpbanque/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/btpbanque/compat/weboob_capabilities_bank.py +++ b/modules/btpbanque/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/caels/compat/weboob_capabilities_bank.py b/modules/caels/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/caels/compat/weboob_capabilities_bank.py +++ b/modules/caels/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/caissedepargne/browser.py b/modules/caissedepargne/browser.py index 3a0b5a137f..affa2be22b 100644 --- a/modules/caissedepargne/browser.py +++ b/modules/caissedepargne/browser.py @@ -143,6 +143,7 @@ def __init__(self, nuser, *args, **kwargs): self.BASEURL = 'https://%s' % self.BASEURL self.is_cenet_website = False + self.new_website = True self.multi_type = False self.accounts = None self.loans = None @@ -433,32 +434,75 @@ def get_accounts_list(self): assert self.linebourse.portfolio.is_here() # We must declare "page" because this URL also matches MarketPage account.valuation_diff = page.get_valuation_diff() + + # We need to go back to the synthesis, else we can not go home later + self.home_tache.go(tache='CPTSYNT0') else: assert False, "new domain that hasn't been seen so far ?" + """ + Card cases are really tricky on the new website. + There are 2 kinds of page where we can find cards information + - CardsPage: List some of the PSU cards + - CardsComingPage: On the coming transaction page (for a specific checking account), + we can find all cards related to this checking account. Information to reach this + CC is in the home page + + We have to go through this both kind of page for those reasons: + - If there is no coming yet, the card will not be found in the home page and we will not + be able to reach the CardsComingPage. But we can find it on CardsPage + - Some cards are only on the CardsComingPage and not the CardsPage + - In CardsPage, there are cards (with "Business" in the label) without checking account on the + website (neither history nor coming), so we skip them. + - Some card on the CardsPage that have a checking account parent, but if we follow the link to + reach it with CardsComingPage, we find an other card that not in CardsPage. + """ + if self.new_website: + for account in self.accounts: + # Adding card's account that we find in CardsComingPage of each Checking account + if account._card_links: + self.home.go() + self.page.go_history(account._card_links) + for card in self.page.iter_cards(): + card.parent = account + card._coming_info = self.page.get_card_coming_info(card.number, card.parent._card_links.copy()) + self.accounts.append(card) + self.home.go() self.page.go_list() self.page.go_cards() - if self.cards.is_here() or self.cards_old.is_here(): - cards = list(self.page.iter_cards()) - for card in cards: + # We are on the new website. We already added some card, but we can find more of them on the CardsPage + if self.cards.is_here(): + for card in self.page.iter_cards(): card.parent = find_object(self.accounts, number=card._parent_id) - assert card.parent, 'card account %r parent was not found' % card + assert card.parent, 'card account parent %s was not found' % card + + # If we already added this card, we don't have to add it a second time + if find_object(self.accounts, number=card.number): + continue + + info = card.parent._card_links - # If we are in the new site, we have to get each card coming transaction link. - if self.cards.is_here(): - for card in cards: - info = card.parent._card_links + # If card.parent._card_links is not filled, it mean this checking account + # has no coming transactions. + card._coming_info = None + if info: + self.page.go_list() + self.page.go_history(info) + card._coming_info = self.page.get_card_coming_info(card.number, info.copy()) - # If info is filled, that mean there are comings transaction - card._coming_info = None - if info: - self.page.go_list() - self.page.go_history(info) - card._coming_info = self.page.get_card_coming_info(card.number, info.copy()) + if not card._coming_info: + self.logger.warning('Skip card %s (not found on checking account)', card.number) + continue + self.accounts.append(card) - self.accounts.extend(cards) + # We are on the old website. We add all card that we can find on the CardsPage + elif self.cards_old.is_here(): + for card in self.page.iter_cards(): + card.parent = find_object(self.accounts, number=card._parent_id) + assert card.parent, 'card account parent %s was not found' % card.number + self.accounts.append(card) # Some accounts have no available balance or label and cause issues # in the backend so we must exclude them from the accounts list: @@ -606,8 +650,17 @@ def _get_history_invests(self, account): if self.page.is_account_inactive(account.id): self.logger.warning('Account %s %s is inactive.' % (account.label, account.id)) return [] + + # There is (currently ?) no history for MILLEVIE PREMIUM accounts if "MILLEVIE" in account.label: - self.page.go_life_insurance(account) + try: + self.page.go_life_insurance(account) + except ServerError as ex: + if ex.response.status_code == 500 and 'MILLEVIE PREMIUM' in account.label: + self.logger.info("Can not reach history page for MILLEVIE PREMIUM account") + return [] + raise + label = account.label.split()[-1] try: self.natixis_life_ins_his.go(id1=label[:3], id2=label[3:5], id3=account.id) @@ -730,6 +783,9 @@ def get_investment(self, account): self.update_linebourse_token() for investment in self.linebourse.iter_investments(account.id): yield investment + + # We need to go back to the synthesis, else we can not go home later + self.home_tache.go(tache='CPTSYNT0') return elif account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_CAPITALISATION): @@ -737,7 +793,14 @@ def get_investment(self, account): self.logger.warning('Account %s %s is inactive.' % (account.label, account.id)) return if "MILLEVIE" in account.label: - self.page.go_life_insurance(account) + try: + self.page.go_life_insurance(account) + except ServerError as ex: + if ex.response.status_code == 500 and 'MILLEVIE PREMIUM' in account.label: + self.logger.info("Can not reach investment page for MILLEVIE PREMIUM account") + return + raise + label = account.label.split()[-1] self.natixis_life_ins_inv.go(id1=label[:3], id2=label[3:5], id3=account.id) for tr in self.page.get_investments(): diff --git a/modules/caissedepargne/cenet/compat/weboob_capabilities_bank.py b/modules/caissedepargne/cenet/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/caissedepargne/cenet/compat/weboob_capabilities_bank.py +++ b/modules/caissedepargne/cenet/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/caissedepargne/compat/weboob_capabilities_bank.py b/modules/caissedepargne/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/caissedepargne/compat/weboob_capabilities_bank.py +++ b/modules/caissedepargne/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/caissedepargne/pages.py b/modules/caissedepargne/pages.py index d211dc20fc..0f2c54cf98 100644 --- a/modules/caissedepargne/pages.py +++ b/modules/caissedepargne/pages.py @@ -30,7 +30,10 @@ from weboob.browser.pages import LoggedPage, HTMLPage, JsonPage, pagination, FormNotFound from weboob.browser.elements import ItemElement, method, ListElement, TableElement, SkipItem, DictElement -from .compat.weboob_browser_filters_standard import Date, CleanDecimal, Regexp, CleanText, Env, Upper, Field, Eval, Format, Currency +from .compat.weboob_browser_filters_standard import ( + Date, CleanDecimal, Regexp, CleanText, Env, Upper, + Field, Eval, Format, Currency, Coalesce, +) from weboob.browser.filters.html import Link, Attr, TableCell from weboob.capabilities import NotAvailable from .compat.weboob_capabilities_bank import ( @@ -291,6 +294,7 @@ def _add_account(self, accounts, link, label, account_type, balance, number=None return account = Account() + account._card_links = None account.id = info['id'] if is_rib_valid(info['id']): account.iban = rib2iban(info['id']) @@ -446,6 +450,7 @@ def get_loan_list(self): tds = tr.findall('td') account = Account() + account._card_links = None account.id = CleanText('./a')(tds[2]).split('-')[0].strip() account.label = CleanText('./a')(tds[2]).split('-')[-1].strip() account.type = Account.TYPE_LOAN @@ -557,7 +562,13 @@ def go_list(self): self.submit_form(form, eventargument, eventtarget, scriptmanager) def go_cards(self): + # Do not try to go the card summary if we have no card, it breaks the session + if not CleanText('//form[@id="main"]//a/span[text()="Mes cartes bancaires"]')(self.doc): + self.logger.info("Do not try to go the CardsPage, there is not link on the main page") + return + form = self.get_form(id='main') + eventargument = "" if "MM$m_CH$IsMsgInit" in form: @@ -864,6 +875,7 @@ class item(ItemElement): obj__parent_id = CleanText(TableCell('parent')) obj_balance = 0 obj_currency = Currency(TableCell('coming')) + obj__card_links = None def obj_coming(self): if CleanText(TableCell('coming'))(self) == '-': @@ -892,10 +904,37 @@ class CardsComingPage(IndexPage): def is_here(self): return CleanText('//h2[text()="Encours de carte à débit différé"]')(self.doc) - def get_card_coming_info(self, number, info): + @method + class iter_cards(ListElement): + item_xpath = '//table[contains(@class, "compte") and position() = 1]//tr[contains(@id, "MM_HISTORIQUE_CB") and position() < last()]' + + class item(ItemElement): + klass = Account + def obj_id(self): + # We must handle two kinds of Regexp because the 'X' are not + # located at the same level for sub-modules such as palatine + return Coalesce( + Regexp(CleanText(Field('label'), replace=[('*', 'X')]), r'(\d{6}\X{6}\d{4})', default=NotAvailable), + Regexp(CleanText(Field('label'), replace=[('*', 'X')]), r'(\d{4}\X{6}\d{6})', default=NotAvailable), + )(self) + + def obj_number(self): + return Coalesce( + Regexp(CleanText(Field('label')), r'(\d{6}\*{6}\d{4})', default=NotAvailable), + Regexp(CleanText(Field('label')), r'(\d{4}\*{6}\d{6})', default=NotAvailable), + )(self) + + obj_type = Account.TYPE_CARD + obj_label = CleanText('./td[1]') + obj_balance = Decimal(0) + obj_coming = CleanDecimal.French('./td[2]') + obj_currency = Currency('./td[2]') + obj__card_links = None + + def get_card_coming_info(self, number, info): # If the xpath match, that mean there are only one card - # We have enought information in `info` to get its coming transaction + # We have enough information in `info` to get its coming transaction if CleanText('//tr[@id="MM_HISTORIQUE_CB_rptMois0_ctl01_trItem"]')(self.doc): return info @@ -907,12 +946,16 @@ def get_card_coming_info(self, number, info): if Regexp(CleanText(xpath), r'(\d{6}\*{6}\d{4})')(self.doc) == number: return info - # For all card except the first one for the same check account, we have to get info through their href info - link = CleanText(Attr('//a[contains(text(),"%s")]' % number, 'href'))(self.doc) - infos = re.match(r'.*(DETAIL_OP_M0&[^\"]+).*', link) - info['link'] = infos.group(1) + # Some cards redirect to a checking account where we cannot found them. Since we have no details or history, + # we return None and skip them in the browser. + if CleanText('//a[contains(text(),"%s")]' % number)(self.doc): + # For all cards except the first one for the same check account, we have to get info through their href info + link = CleanText(Link('//a[contains(text(),"%s")]' % number))(self.doc) + infos = re.match(r'.*(DETAIL_OP_M0&[^\"]+).*', link) + info['link'] = infos.group(1) - return info + return info + return None class CardsOldWebsitePage(IndexPage): @@ -945,6 +988,7 @@ class item(ItemElement): obj_balance = 0 obj_coming = CleanDecimal.French(TableCell('coming')) obj_currency = Currency(TableCell('coming')) + obj__card_links = None def obj__parent_id(self): return self.page.get_account() diff --git a/modules/capeasi/compat/weboob_capabilities_bank.py b/modules/capeasi/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/capeasi/compat/weboob_capabilities_bank.py +++ b/modules/capeasi/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/carrefourbanque/browser.py b/modules/carrefourbanque/browser.py index 335fe6c30d..daf6c544ce 100644 --- a/modules/carrefourbanque/browser.py +++ b/modules/carrefourbanque/browser.py @@ -73,15 +73,18 @@ def do_login(self): self.incapsula_ressource.go(params={'SWCGHOEL': 'v2'}, data=data) self.login.go() - # this cookie contains an ugly \x01 and make next request fail with a 400 if not removed - ___utmvafIuFLPmB = self.session.cookies.pop('___utmvafIuFLPmB', None) - if ___utmvafIuFLPmB: - self.session.cookies['___utmvafIuFLPmB'] = ___utmvafIuFLPmB.replace('\x01', '') - - # this cookie contains an ugly \n and make next request fail with a 400 if not removed - ___utmvbfIuFLPmB = self.session.cookies.pop('___utmvbfIuFLPmB', None) - if ___utmvbfIuFLPmB: - self.session.cookies['___utmvbfIuFLPmB'] = ___utmvbfIuFLPmB.replace('\n', '') + # remove 2 cookies that make next request fail with a 400 if not removed + # cookie name can change depend on ip, but seems to be constant on same ip + # example: + # 1st cookie 2nd cookie + # ___utmvafIuFLPmB, ___utmvbfIuFLPmB + # ___utmvaYauFLPmB, ___utmvbYauFLPmB + # it may have other names... + for cookie in self.session.cookies: + if '___utmva' in cookie.name or '___utmvb' in cookie.name: + # ___utmva... contains an ugly \x01 + # ___utmvb... contains an ugly \n + self.session.cookies.pop(cookie.name) if self.incapsula_ressource.is_here(): if self.page.is_javascript: diff --git a/modules/carrefourbanque/compat/weboob_capabilities_bank.py b/modules/carrefourbanque/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/carrefourbanque/compat/weboob_capabilities_bank.py +++ b/modules/carrefourbanque/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/cic/compat/weboob_capabilities_bank.py b/modules/cic/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/cic/compat/weboob_capabilities_bank.py +++ b/modules/cic/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/cices/compat/weboob_capabilities_bank.py b/modules/cices/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/cices/compat/weboob_capabilities_bank.py +++ b/modules/cices/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/citibank/compat/weboob_capabilities_bank.py b/modules/citibank/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/citibank/compat/weboob_capabilities_bank.py +++ b/modules/citibank/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/cmb/compat/weboob_capabilities_bank.py b/modules/cmb/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/cmb/compat/weboob_capabilities_bank.py +++ b/modules/cmb/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/cmes/browser.py b/modules/cmes/browser.py index 996879a612..11ae936a27 100644 --- a/modules/cmes/browser.py +++ b/modules/cmes/browser.py @@ -78,9 +78,10 @@ def iter_investment(self, account): def iter_history(self, account): self.operations_list.stay_or_go(subsite=self.subsite, client_space=self.client_space) for idx in self.page.get_operations_idx(): - tr = self.operation.go(subsite=self.subsite, client_space=self.client_space, idx=idx).get_transaction() - if account.label == tr._account_label: - yield tr + self.operation.go(subsite=self.subsite, client_space=self.client_space, idx=idx) + for tr in self.page.get_transactions(): + if account.label == tr._account_label: + yield tr @need_login def iter_pocket(self, account): diff --git a/modules/cmes/compat/weboob_capabilities_bank.py b/modules/cmes/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/cmes/compat/weboob_capabilities_bank.py +++ b/modules/cmes/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/cmes/pages.py b/modules/cmes/pages.py index 5631ea1b95..0814423a22 100644 --- a/modules/cmes/pages.py +++ b/modules/cmes/pages.py @@ -77,14 +77,15 @@ def on_load(self): ACCOUNTS_TYPES = { "pargne entreprise": Account.TYPE_PEE, "pargne groupe": Account.TYPE_PEE, - "pargne retraite": Account.TYPE_PERCO + "pargne retraite": Account.TYPE_PERCO, + "courant bloqué": Account.TYPE_DEPOSIT, } class NewAccountsPage(LoggedPage, HTMLPage): @method class iter_accounts(ListElement): - item_xpath = '//th[text()= "Nom du support" or text()="Nom du profil"]/ancestor::table/ancestor::table' + item_xpath = '//th[text()= "Nom du support" or text()="Nom du profil" or text()="Nom du compte"]/ancestor::table/ancestor::table' class item(ItemElement): klass = Account @@ -166,15 +167,27 @@ def iter_pocket(self, inv): class OperationPage(LoggedPage, HTMLPage): + + # Most '_account_label' correspond 'account.label', but there are exceptions + ACCOUNTS_SPE_LABELS = { + 'CCB': 'Compte courant bloqué', + } + @method - class get_transaction(ItemElement): - klass = Transaction - - obj_amount = MyDecimal('//td[contains(text(), "Montant total")]/following-sibling::td') - obj_label = CleanText('(//p[contains(@id, "smltitle")])[2]') - obj_raw = Transaction.Raw(Field('label')) - obj_date = Date(Regexp(CleanText('(//p[contains(@id, "smltitle")])[1]'), r'(\d{1,2}/\d{1,2}/\d+)'), dayfirst=True) - obj__account_label = CleanText('//td[contains(text(), "Montant total")]/../following-sibling::tr/th[1]') + class get_transactions(ListElement): + item_xpath = '//tr[@id]' + + class item(ItemElement): + klass = Transaction + + obj_amount = MyDecimal('./th[@scope="rowgroup"][2]') + obj_label = CleanText('(//p[contains(@id, "smltitle")])[2]') + obj_raw = Transaction.Raw(Field('label')) + obj_date = Date(Regexp(CleanText('(//p[contains(@id, "smltitle")])[1]'), r'(\d{1,2}/\d{1,2}/\d+)'), dayfirst=True) + + def obj__account_label(self): + account_label = CleanText('./th[@scope="rowgroup"][1]')(self) + return self.page.ACCOUNTS_SPE_LABELS.get(account_label, account_label) class OperationsListPage(LoggedPage, HTMLPage): diff --git a/modules/cmmc/compat/weboob_capabilities_bank.py b/modules/cmmc/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/cmmc/compat/weboob_capabilities_bank.py +++ b/modules/cmmc/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/cmso/compat/weboob_capabilities_bank.py b/modules/cmso/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/cmso/compat/weboob_capabilities_bank.py +++ b/modules/cmso/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/cmso/par/compat/weboob_capabilities_bank.py b/modules/cmso/par/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/cmso/par/compat/weboob_capabilities_bank.py +++ b/modules/cmso/par/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/cmso/par/pages.py b/modules/cmso/par/pages.py index 4fee8920c2..f5fa833647 100644 --- a/modules/cmso/par/pages.py +++ b/modules/cmso/par/pages.py @@ -56,23 +56,27 @@ class LogoutPage(RawPage): class AccountsPage(LoggedPage, JsonPage): - TYPES = OrderedDict([('courant', Account.TYPE_CHECKING), - ('pee', Account.TYPE_PEE), - ('epargne en actions', Account.TYPE_PEA), - ('pea', Account.TYPE_PEA), - ('p.e.a.', Account.TYPE_PEA), - ('preference', Account.TYPE_LOAN), - ('livret', Account.TYPE_SAVINGS), - ('vie', Account.TYPE_LIFE_INSURANCE), - ('previ_option', Account.TYPE_LIFE_INSURANCE), - ('actions', Account.TYPE_MARKET), - ('titres', Account.TYPE_MARKET), - ('ldd cm', Account.TYPE_SAVINGS), - ('librissime', Account.TYPE_SAVINGS), - ('epargne logement', Account.TYPE_SAVINGS), - ('plan bleu', Account.TYPE_SAVINGS), - ('capital plus', Account.TYPE_SAVINGS), - ]) + TYPES = OrderedDict([ + ('courant', Account.TYPE_CHECKING), + ('pee', Account.TYPE_PEE), + ('epargne en actions', Account.TYPE_PEA), + ('pea', Account.TYPE_PEA), + ('p.e.a.', Account.TYPE_PEA), + ('preference', Account.TYPE_LOAN), + ('livret', Account.TYPE_SAVINGS), + ('vie', Account.TYPE_LIFE_INSURANCE), + ('previ_option', Account.TYPE_LIFE_INSURANCE), + ('avantage capitalisation', Account.TYPE_LIFE_INSURANCE), + ('actions', Account.TYPE_MARKET), + ('titres', Account.TYPE_MARKET), + ('ldd cm', Account.TYPE_SAVINGS), + ('librissime', Account.TYPE_SAVINGS), + ('epargne logement', Account.TYPE_SAVINGS), + ('plan bleu', Account.TYPE_SAVINGS), + ('capital plus', Account.TYPE_SAVINGS), + ('capital expansion', Account.TYPE_DEPOSIT), + ('carte', Account.TYPE_CARD), + ]) def get_keys(self): """Returns the keys for which the value is a list or dict""" @@ -355,7 +359,7 @@ def parse(self, el): class LifeinsurancePage(LoggedPage, HTMLPage): def get_account_id(self): - account_id = Regexp(CleanText('//h1[@class="portlet-title"]'), r'n° ([\d\s]+)', default=NotAvailable)(self.doc) + account_id = Regexp(CleanText('//h1[@class="portlet-title"]'), r'n° ([\s\w]+)', default=NotAvailable)(self.doc) if account_id: return re.sub(r'\s', '', account_id) diff --git a/modules/cmso/par/transfer_pages.py b/modules/cmso/par/transfer_pages.py index cb4e17850e..1f1cbb3826 100644 --- a/modules/cmso/par/transfer_pages.py +++ b/modules/cmso/par/transfer_pages.py @@ -23,7 +23,7 @@ from weboob.browser.pages import JsonPage, LoggedPage from weboob.browser.elements import DictElement, ItemElement, method -from .compat.weboob_browser_filters_standard import CleanText, CleanDecimal, Currency +from .compat.weboob_browser_filters_standard import CleanText, CleanDecimal, Currency, Coalesce from weboob.browser.filters.json import Dict from .compat.weboob_capabilities_bank import Recipient, Transfer, TransferBankError from weboob.capabilities.base import NotAvailable @@ -128,7 +128,7 @@ def condition(self): class item(ItemElement): klass = Recipient - obj_id = obj_iban = Dict('iban') + obj_id = obj_iban = Coalesce(Dict('iban'), Dict('index')) obj_label = Dict('nom') obj_category = 'Externe' obj_enabled_at = dt.date.today() diff --git a/modules/cmso/pro/compat/weboob_capabilities_bank.py b/modules/cmso/pro/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/cmso/pro/compat/weboob_capabilities_bank.py +++ b/modules/cmso/pro/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/cragr/api/browser.py b/modules/cragr/api/browser.py index e5162b4b88..80758d777c 100644 --- a/modules/cragr/api/browser.py +++ b/modules/cragr/api/browser.py @@ -119,6 +119,9 @@ def deinit(self): self.netfinca.deinit() def do_login(self): + if not self.username or not self.password: + raise BrowserIncorrectPassword() + # First we try to connect to the new website: if the connection # is on the old website, we will automatically redirected. website = self.website.replace('.fr', '') @@ -228,30 +231,34 @@ def iter_accounts(self): for contract in range(total_spaces): if not self.check_space_connection(contract): - self.logger.warning('Server returned error 500 twice when trying to access space %s, this space will be skipped' % contract) + self.logger.warning('Server returned error 500 twice when trying to access space %s, this space will be skipped', contract) continue - # The main account is not located at the same place in the JSON. - main_account = self.page.get_main_account() - if main_account.balance == NotAvailable: - self.check_space_connection(contract) + + # Some spaces have no main account + if self.page.has_main_account(): + # The main account is not located at the same place in the JSON. main_account = self.page.get_main_account() if main_account.balance == NotAvailable: - self.logger.warning('Could not fetch the balance for main account %s.' % main_account.id) - - # Get cards for the main account - if self.page.has_main_cards(): - for card in self.page.iter_main_cards(): - card.parent = main_account - card.currency = card.parent.currency - card.owner_type = card.parent.owner_type - card._category = card.parent._category - card._contract = contract - deferred_cards[card.id] = card - - main_account.owner_type = self.page.get_owner_type() - main_account._contract = contract - space_type = self.page.get_space_type() + self.check_space_connection(contract) + main_account = self.page.get_main_account() + if main_account.balance == NotAvailable: + self.logger.warning('Could not fetch the balance for main account %s.', main_account.id) + # Get cards for the main account + if self.page.has_main_cards(): + for card in self.page.iter_main_cards(): + card.parent = main_account + card.currency = card.parent.currency + card.owner_type = card.parent.owner_type + card._category = card.parent._category + card._contract = contract + deferred_cards[card.id] = card + + main_account.owner_type = self.page.get_owner_type() + main_account._contract = contract + else: + main_account = None + space_type = self.page.get_space_type() accounts_list = list(self.page.iter_accounts()) for account in accounts_list: account._contract = contract @@ -278,12 +285,13 @@ def iter_accounts(self): account_balances.update(self.page.get_account_balances()) loan_ids.update(self.page.get_loan_ids()) - if main_account.type == Account.TYPE_CHECKING: - main_account.iban = self.get_account_iban(main_account._index, 1, main_account.id) + if main_account: + if main_account.type == Account.TYPE_CHECKING: + main_account.iban = self.get_account_iban(main_account._index, 1, main_account.id) - if main_account.id not in all_accounts: - all_accounts[main_account.id] = main_account - yield main_account + if main_account.id not in all_accounts: + all_accounts[main_account.id] = main_account + yield main_account for account in accounts_list: if empty(account.balance): @@ -383,12 +391,21 @@ def switch_account_to_revolving(self, account): @need_login def go_to_account_space(self, contract): - self.contracts_page.go(space=self.space, id_contract=contract) + # This request often returns a 500 error on this quality website + try: + self.contracts_page.go(space=self.space, id_contract=contract) + except ServerError: + self.logger.warning('Space switch returned a 500 error, try again.') + self.contracts_page.go(space=self.space, id_contract=contract) if not self.accounts_page.is_here(): # We have been logged out. self.do_login() - self.contracts_page.go(space=self.space, id_contract=contract) - assert self.accounts_page.is_here() + try: + self.contracts_page.go(space=self.space, id_contract=contract) + except ServerError: + self.logger.warning('Space switch returned a 500 error, try again.') + self.contracts_page.go(space=self.space, id_contract=contract) + assert self.accounts_page.is_here() @need_login def iter_history(self, account, coming=False): @@ -646,6 +663,7 @@ def iter_transfer_recipients(self, account, transfer_space_info=None): else: space, operation, referer, _ = self.get_account_transfer_space_info(account) + self.go_to_account_space(account._contract) self.recipients.go(space=space, op=operation, headers={'Referer': referer}) if not self.page.is_sender_account(account.id): diff --git a/modules/cragr/api/compat/weboob_capabilities_bank.py b/modules/cragr/api/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/cragr/api/compat/weboob_capabilities_bank.py +++ b/modules/cragr/api/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/cragr/api/pages.py b/modules/cragr/api/pages.py index 93d1cf7cf9..f29ef41839 100644 --- a/modules/cragr/api/pages.py +++ b/modules/cragr/api/pages.py @@ -113,6 +113,8 @@ class ContractsPage(LoggedPage, HTMLPage): 'LDD': Account.TYPE_SAVINGS, 'PEL': Account.TYPE_SAVINGS, 'CEL': Account.TYPE_SAVINGS, + 'CEA': Account.TYPE_DEPOSIT, # Dépôt à terme + 'CEL2': Account.TYPE_SAVINGS, 'CODEBIS': Account.TYPE_SAVINGS, 'LJMO': Account.TYPE_SAVINGS, 'CSL': Account.TYPE_SAVINGS, @@ -126,6 +128,7 @@ class ContractsPage(LoggedPage, HTMLPage): 'CPTEXCPRO': Account.TYPE_SAVINGS, 'CPTEXCENT': Account.TYPE_SAVINGS, 'CPTDAV': Account.TYPE_SAVINGS, + 'ORCH': Account.TYPE_SAVINGS, # Orchestra / PEP 'DAT': Account.TYPE_SAVINGS, 'CB': Account.TYPE_SAVINGS, # Carré bleu / PEL 'PRET PERSO': Account.TYPE_LOAN, @@ -155,6 +158,8 @@ class ContractsPage(LoggedPage, HTMLPage): 'P. ACC.SOC': Account.TYPE_LOAN, 'PACA': Account.TYPE_LOAN, 'CAU. BANC.': Account.TYPE_LOAN, + 'CSAN': Account.TYPE_LOAN, + 'P SPE MOD': Account.TYPE_LOAN, 'épargne disponible': Account.TYPE_SAVINGS, 'épargne à terme': Account.TYPE_DEPOSIT, 'épargne boursière': Account.TYPE_MARKET, @@ -174,9 +179,12 @@ class ContractsPage(LoggedPage, HTMLPage): 'FLORIPRO': Account.TYPE_LIFE_INSURANCE, 'FLORIANE 2': Account.TYPE_LIFE_INSURANCE, 'ATOUT LIB': Account.TYPE_REVOLVING_CREDIT, - 'PACC': Account.TYPE_CONSUMER_CREDIT, + 'PACC': Account.TYPE_CONSUMER_CREDIT, # 'PAC' = 'Prêt à consommer' 'PACP': Account.TYPE_CONSUMER_CREDIT, + 'PACR': Account.TYPE_CONSUMER_CREDIT, + 'PACV': Account.TYPE_CONSUMER_CREDIT, 'SUPPLETIS': Account.TYPE_REVOLVING_CREDIT, + 'OPEN': Account.TYPE_REVOLVING_CREDIT, 'PAGR': Account.TYPE_MADELIN, } @@ -224,6 +232,9 @@ def get_connection_id(self): )(self.html_doc) return connection_id + def has_main_account(self): + return Dict('comptePrincipal', default=None)(self.doc) + @method class get_main_account(ItemElement): klass = Account diff --git a/modules/cragr/api/transfer_pages.py b/modules/cragr/api/transfer_pages.py index 0cfe0dbd09..2f4227e920 100644 --- a/modules/cragr/api/transfer_pages.py +++ b/modules/cragr/api/transfer_pages.py @@ -27,7 +27,7 @@ Account, Recipient, Transfer, TransferBankError, ) from .compat.weboob_browser_filters_standard import ( - CleanDecimal, Date, CleanText, Coalesce, + CleanDecimal, Date, CleanText, Coalesce, Format, ) from weboob.browser.filters.json import Dict @@ -77,7 +77,9 @@ def condition(self): klass = Recipient obj_id = Dict('accountNumber') - obj_label = Dict('accountNatureLongLabel', default='') + obj_label = CleanText(Format('%s %s', + Dict('accountHolderLongDesignation'), + Dict('accountNatureShortLabel', default=''))) obj_iban = Dict('ibanCode') obj_category = 'Interne' obj_enabled_at = date.today() diff --git a/modules/cragr/compat/weboob_capabilities_bank.py b/modules/cragr/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/cragr/compat/weboob_capabilities_bank.py +++ b/modules/cragr/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/cragr/regions/compat/weboob_capabilities_bank.py b/modules/cragr/regions/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/cragr/regions/compat/weboob_capabilities_bank.py +++ b/modules/cragr/regions/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/cragr/regions/pages.py b/modules/cragr/regions/pages.py index ee5bdfc819..46ad4386ca 100644 --- a/modules/cragr/regions/pages.py +++ b/modules/cragr/regions/pages.py @@ -202,6 +202,7 @@ def get_iban(self): 'DAV NANTI': Account.TYPE_SAVINGS, 'LIV A': Account.TYPE_SAVINGS, 'LIV A ASS': Account.TYPE_SAVINGS, + 'LIVCR': Account.TYPE_SAVINGS, 'LDD': Account.TYPE_SAVINGS, 'PEL': Account.TYPE_SAVINGS, 'CEL': Account.TYPE_SAVINGS, @@ -250,6 +251,7 @@ def get_iban(self): 'PREDI9 S2': Account.TYPE_LIFE_INSURANCE, 'V.AVENIR': Account.TYPE_LIFE_INSURANCE, 'FLORIA': Account.TYPE_LIFE_INSURANCE, + 'FLORIANE 2': Account.TYPE_LIFE_INSURANCE, 'CAP DECOUV': Account.TYPE_LIFE_INSURANCE, 'ESPACE LIB': Account.TYPE_LIFE_INSURANCE, 'ESP LIB 2': Account.TYPE_LIFE_INSURANCE, @@ -477,8 +479,8 @@ def condition(self): # Card label is formatted as 'Carte VISA Premier - Mr M Lastname' obj_label = Format( '%s - %s', - CleanText('.//caption/span[@class="tdb-cartes-carte"]'), - CleanText('.//caption/span[@class="tdb-cartes-prop"]') + CleanText('.//caption/span[has-class("tdb-cartes-carte")]'), + CleanText('.//caption/span[has-class("tdb-cartes-prop")]') ) obj_type = Account.TYPE_CARD @@ -490,10 +492,18 @@ def condition(self): def get_transactions_link(self, raw_number): # We cannot use Link() because the @href attribute contains line breaks and spaces. - # Always take the before the last to include the latest transactions - # (the last is just card information). - return CleanText('//table[@class="ca-table"][caption[span[text()="%s"]]]//tr[position()=last()-1]/th/a/@href' - % raw_number, replace=[(' ', '')])(self.doc) + if len(self.doc.xpath('//table[@class="ca-table"][caption[span[text()="%s"]]]//tr' % raw_number)) == 1: + # There is only one coming line (no card information link) + return CleanText('//table[@class="ca-table"][caption[span[text()="%s"]]]//tr[position()=last()]/th/a/@href' + % raw_number, replace=[(' ', '')])(self.doc) + elif self.doc.xpath('//table[@class="ca-table"][caption[span[text()="%s"]]]//tr//a[contains(text(), "Infos carte")]' % raw_number): + # There is a card information line, select the before the last + return CleanText('//table[@class="ca-table"][caption[span[text()="%s"]]]//tr[position()=last()-1]/th/a/@href' + % raw_number, replace=[(' ', '')])(self.doc) + else: + # There is no information line, return the last + return CleanText('//table[@class="ca-table"][caption[span[text()="%s"]]]//tr[position()=last()]/th/a/@href' + % raw_number, replace=[(' ', '')])(self.doc) class WealthPage(LoggedPage, CragrPage): @@ -564,7 +574,7 @@ def parse(self, obj): # History table with 4 columns (no loan details) self.env['next_payment_amount'] = NotAvailable self.env['total_amount'] = NotAvailable - self.env['balance'] = CleanDecimal.French('./td[4]//*[@class="montant3"]', default=NotAvailable)(self) + self.env['balance'] = CleanDecimal.French('./td[4]//*[@class="montant3" or @class="montant4"]', default=NotAvailable)(self) elif CleanText('//tr[contains(@class, "colcelligne")][count(td) = 6]')(self): # History table with 5 columns (contains next_payment_amount & total_amount) self.env['next_payment_amount'] = CleanDecimal.French('./td[3]//*[@class="montant3"]', default=NotAvailable)(self) diff --git a/modules/cragr/web/compat/weboob_browser_filters_standard.py b/modules/cragr/web/compat/weboob_browser_filters_standard.py deleted file mode 100644 index f382d162f0..0000000000 --- a/modules/cragr/web/compat/weboob_browser_filters_standard.py +++ /dev/null @@ -1,49 +0,0 @@ - -import weboob.browser.filters.standard as OLD - -# can't import *, __all__ is incomplete... -for attr in dir(OLD): - globals()[attr] = getattr(OLD, attr) - - -try: - __all__ = OLD.__all__ -except AttributeError: - pass - - -class Coalesce(MultiFilter): - """ - Returns the first value that is not falsy, - or default if all values are falsy. - """ - @debug() - def filter(self, values): - for value in values: - if value: - return value - return self.default_or_raise(FilterError('All falsy and no default.')) - - -class MapIn(Filter): - """ - Map the pattern of a selected value to another value using a dict. - """ - - def __init__(self, selector, map_dict, default=_NO_DEFAULT): - """ - :param selector: key from `map_dict` to use - """ - super(MapIn, self).__init__(selector, default=default) - self.map_dict = map_dict - - @debug() - def filter(self, txt): - """ - :raises: :class:`ItemNotFound` if key pattern does not exist in dict - """ - for key in self.map_dict: - if key in txt: - return self.map_dict[key] - - return self.default_or_raise(ItemNotFound('Unable to handle %r on %r' % (txt, self.map_dict))) diff --git a/modules/cragr/web/compat/weboob_capabilities_bank.py b/modules/cragr/web/compat/weboob_capabilities_bank.py deleted file mode 100644 index d82081dd5c..0000000000 --- a/modules/cragr/web/compat/weboob_capabilities_bank.py +++ /dev/null @@ -1,47 +0,0 @@ - -import weboob.capabilities.bank as OLD -from weboob.capabilities.base import StringField -from weboob.capabilities.date import DateField - -# can't import *, __all__ is incomplete... -for attr in dir(OLD): - globals()[attr] = getattr(OLD, attr) - - -try: - __all__ = OLD.__all__ -except AttributeError: - pass - - -# can't create a subclass because of CapBank.iter_resources reimplementations: -# modules will import our subclass, but boobank will call iter_resources with the OLD class -Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') - - -Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') - - -class RecipientInvalidOTP(AddRecipientError): - code = 'invalidOTP' - - -class AccountOwnership(object): - """ - Relationship between the credentials owner (PSU) and the account - """ - OWNER = u'owner' - """The PSU is the account owner""" - CO_OWNER = u'co-owner' - """The PSU is the account co-owner""" - ATTORNEY = u'attorney' - """The PSU is the account attorney""" - - -AccountOwnerType.ASSOCIATION = u'ASSO' - - -try: - __all__ += ['AccountOwnership', 'RecipientInvalidOTP'] -except NameError: - pass diff --git a/modules/creditcooperatif/compat/weboob_capabilities_bank.py b/modules/creditcooperatif/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/creditcooperatif/compat/weboob_capabilities_bank.py +++ b/modules/creditcooperatif/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/creditdunord/compat/weboob_capabilities_bank.py b/modules/creditdunord/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/creditdunord/compat/weboob_capabilities_bank.py +++ b/modules/creditdunord/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/creditdunord/pages.py b/modules/creditdunord/pages.py index 55cc65ef18..7924a55a77 100755 --- a/modules/creditdunord/pages.py +++ b/modules/creditdunord/pages.py @@ -628,6 +628,7 @@ def get_history(self, account): if account.type is Account.TYPE_CARD and MyStrip(line[self.COL_DEBIT_DATE]): date = vdate = Date(dayfirst=True).filter(MyStrip(line[self.COL_DEBIT_DATE])) + t.bdate = Date(dayfirst=True, default=NotAvailable).filter(MyStrip(line[self.COL_DATE])) else: date = Date(dayfirst=True, default=NotAvailable).filter(MyStrip(line[self.COL_DATE])) if not date: @@ -788,6 +789,7 @@ def get_history(self, account): if account.type is Account.TYPE_CARD: date = vdate = Date(dayfirst=True, default=None).filter(tr['dateval']) + t.bdate = Date(dayfirst=True, default=NotAvailable).filter(tr['date']) else: date = Date(dayfirst=True, default=None).filter(tr['date']) vdate = Date(dayfirst=True, default=None).filter(tr['dateval']) or date diff --git a/modules/creditdunordpee/compat/weboob_capabilities_bank.py b/modules/creditdunordpee/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/creditdunordpee/compat/weboob_capabilities_bank.py +++ b/modules/creditdunordpee/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/creditmutuel/compat/weboob_capabilities_bank.py b/modules/creditmutuel/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/creditmutuel/compat/weboob_capabilities_bank.py +++ b/modules/creditmutuel/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/creditmutuel/pages.py b/modules/creditmutuel/pages.py index 3c21d9005b..8ee19417c0 100644 --- a/modules/creditmutuel/pages.py +++ b/modules/creditmutuel/pages.py @@ -130,43 +130,45 @@ class item_account_generic(ItemElement): klass = Account TYPES = OrderedDict([ - ('Credits Promoteurs', Account.TYPE_CHECKING), # it doesn't fit loan's model - ('Compte Cheque', Account.TYPE_CHECKING), - ('Compte Courant', Account.TYPE_CHECKING), - ('Cpte Courant', Account.TYPE_CHECKING), - ('Contrat Personnel', Account.TYPE_CHECKING), - ('Cc Contrat Personnel', Account.TYPE_CHECKING), - ('C/C', Account.TYPE_CHECKING), - ('Start', Account.TYPE_CHECKING), - ('Comptes courants', Account.TYPE_CHECKING), - ('Service Accueil', Account.TYPE_CHECKING), - ('Catip', Account.TYPE_DEPOSIT), - ('Cic Immo', Account.TYPE_LOAN), - ('Credit', Account.TYPE_LOAN), - ('Crédits', Account.TYPE_LOAN), - ('Eco-Prêt', Account.TYPE_LOAN), - ('Mcne', Account.TYPE_LOAN), - ('Nouveau Prêt', Account.TYPE_LOAN), - ('Pret', Account.TYPE_LOAN), + ('Credits Promoteurs', Account.TYPE_CHECKING), # it doesn't fit loan's model + ('Compte Cheque', Account.TYPE_CHECKING), + ('Compte Courant', Account.TYPE_CHECKING), + ('Cpte Courant', Account.TYPE_CHECKING), + ('Contrat Personnel', Account.TYPE_CHECKING), + ('Cc Contrat Personnel', Account.TYPE_CHECKING), + ('C/C', Account.TYPE_CHECKING), + ('Start', Account.TYPE_CHECKING), + ('Comptes courants', Account.TYPE_CHECKING), + ('Service Accueil', Account.TYPE_CHECKING), + ('Eurocompte Serenite', Account.TYPE_CHECKING), + ('Catip', Account.TYPE_DEPOSIT), + ('Cic Immo', Account.TYPE_LOAN), + ('Credit', Account.TYPE_LOAN), + ('Crédits', Account.TYPE_LOAN), + ('Eco-Prêt', Account.TYPE_LOAN), + ('Mcne', Account.TYPE_LOAN), + ('Nouveau Prêt', Account.TYPE_LOAN), + ('Pret', Account.TYPE_LOAN), ('Regroupement De Credits', Account.TYPE_LOAN), - ('Nouveau Pret 0%', Account.TYPE_LOAN), - ('Passeport Credit', Account.TYPE_REVOLVING_CREDIT), - ('Allure Libre', Account.TYPE_REVOLVING_CREDIT), - ('Preference', Account.TYPE_REVOLVING_CREDIT), - ('Plan 4', Account.TYPE_REVOLVING_CREDIT), - ('P.E.A', Account.TYPE_PEA), - ('Pea', Account.TYPE_PEA), + ('Nouveau Pret 0%', Account.TYPE_LOAN), + ('Passeport Credit', Account.TYPE_REVOLVING_CREDIT), + ('Allure Libre', Account.TYPE_REVOLVING_CREDIT), + ('Preference', Account.TYPE_REVOLVING_CREDIT), + ('Plan 4', Account.TYPE_REVOLVING_CREDIT), + ('P.E.A', Account.TYPE_PEA), + ('Pea', Account.TYPE_PEA), ('Compte De Liquidite Pea', Account.TYPE_PEA), - ('Compte Epargne', Account.TYPE_SAVINGS), - ('Etalis', Account.TYPE_SAVINGS), - ('Ldd', Account.TYPE_SAVINGS), - ('Livret', Account.TYPE_SAVINGS), - ("Plan D'Epargne", Account.TYPE_SAVINGS), - ('Tonic Croissance', Account.TYPE_SAVINGS), - ('Capital Expansion', Account.TYPE_SAVINGS), - ('Épargne', Account.TYPE_SAVINGS), - ('Compte Garantie Titres', Account.TYPE_MARKET), - ]) + ('Compte Epargne', Account.TYPE_SAVINGS), + ('Etalis', Account.TYPE_SAVINGS), + ('Ldd', Account.TYPE_SAVINGS), + ('Livret', Account.TYPE_SAVINGS), + ("Plan D'Epargne", Account.TYPE_SAVINGS), + ('Tonic Croissance', Account.TYPE_SAVINGS), + ('Capital Expansion', Account.TYPE_SAVINGS), + ('Épargne', Account.TYPE_SAVINGS), + ('Capital Plus', Account.TYPE_SAVINGS), + ('Compte Garantie Titres', Account.TYPE_MARKET), + ]) REVOLVING_LOAN_LABELS = [ 'Passeport Credit', @@ -619,19 +621,18 @@ def parse(self, el): class Transaction(FrenchTransaction): - PATTERNS = [(re.compile(r'^VIR(EMENT)? (?P.*)'), FrenchTransaction.TYPE_TRANSFER), - (re.compile(r'^(PRLV|Plt|PRELEVEMENT) (?P.*)'), FrenchTransaction.TYPE_ORDER), - (re.compile(r'^(?P.*) CARTE \d+ PAIEMENT CB\s+(?P
\d{2})(?P\d{2}) ?(.*)$'), - FrenchTransaction.TYPE_CARD), - (re.compile(r'^PAIEMENT PSC\s+(?P
\d{2})(?P\d{2}) (?P.*) CARTE \d+ ?(.*)$'), - FrenchTransaction.TYPE_CARD), - (re.compile(r'^(?PRELEVE CARTE.*)'), FrenchTransaction.TYPE_CARD_SUMMARY), - (re.compile(r'^RETRAIT DAB (?P
\d{2})(?P\d{2}) (?P.*) CARTE [\*\d]+'), - FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile(r'^CHEQUE( (?P.*))?$'), FrenchTransaction.TYPE_CHECK), - (re.compile(r'^(F )?COTIS\.? (?P.*)'), FrenchTransaction.TYPE_BANK), - (re.compile(r'^(REMISE|REM CHQ) (?P.*)'), FrenchTransaction.TYPE_DEPOSIT), - ] + PATTERNS = [ + (re.compile(r'^(VIR(EMENT)?|VIRT.) (?P.*)'), FrenchTransaction.TYPE_TRANSFER), + (re.compile(r'^(PRLV|Plt|PRELEVEMENT) (?P.*)'), FrenchTransaction.TYPE_ORDER), + (re.compile(r'^(?P.*) CARTE \d+ PAIEMENT CB\s+(?P
\d{2})(?P\d{2}) ?(.*)$'), FrenchTransaction.TYPE_CARD), + (re.compile(r'^PAIEMENT PSC\s+(?P
\d{2})(?P\d{2}) (?P.*) CARTE \d+ ?(.*)$'), FrenchTransaction.TYPE_CARD), + (re.compile(r'^Regroupement \d+ PAIEMENTS (?P
\d{2})(?P\d{2}) (?P.*) CARTE \d+ ?(.*)$'), FrenchTransaction.TYPE_CARD), + (re.compile(r'^(?PRELEVE CARTE.*)'), FrenchTransaction.TYPE_CARD_SUMMARY), + (re.compile(r'^RETRAIT DAB (?P
\d{2})(?P\d{2}) (?P.*) CARTE [\*\d]+'), FrenchTransaction.TYPE_WITHDRAWAL), + (re.compile(r'^CHEQUE( (?P.*))?$'), FrenchTransaction.TYPE_CHECK), + (re.compile(r'^(F )?COTIS\.? (?P.*)'), FrenchTransaction.TYPE_BANK), + (re.compile(r'^(REMISE|REM CHQ) (?P.*)'), FrenchTransaction.TYPE_DEPOSIT), + ] _is_coming = False @@ -1258,7 +1259,20 @@ def condition(self): return not any(not x.isdigit() for x in Attr('.', 'id')(self)) obj_label = CleanText(TableCell('label'), default=NotAvailable) - obj_quantity = CleanDecimal(TableCell('quantity'), default=Decimal(0), replace_dots=True) + + def obj_quantity(self): + """ + In case of SRD actions, regular actions and SRD quantities are displayed in the same cell, + we must then add the values in text such as '4 444 + 10000 SRD' + """ + + quantity = CleanText(TableCell('quantity'))(self) + if '+' in quantity: + quantity_list = quantity.split('+') + return CleanDecimal.French().filter(quantity_list[0]) + CleanDecimal.French().filter(quantity_list[1]) + else: + return CleanDecimal.French().filter(quantity) + obj_unitprice = CleanDecimal(TableCell('unitprice'), default=Decimal(0), replace_dots=True) obj_valuation = CleanDecimal(TableCell('valuation'), default=Decimal(0), replace_dots=True) obj_diff = CleanDecimal(TableCell('diff'), default=Decimal(0), replace_dots=True) @@ -1846,7 +1860,7 @@ def iter_subscriptions(self): options = self.doc.xpath('//select[@id="SelTiers"]/option') if options: for opt in options: - subscriber = self.doc.xpath('//select[@id="SelTiers"]/option[contains(text(), "%s")]' % CleanText('.')(opt))[0] + subscriber = self.doc.xpath('//select[@id="SelTiers"]/option[contains(text(), $subscriber)]', subscriber=CleanText('.')(opt))[0] self.submit_form(Attr('.', 'value')(subscriber)) for sub in self.get_subscriptions(subscription_list, subscriber): yield sub diff --git a/modules/delubac/compat/weboob_capabilities_bank.py b/modules/delubac/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/delubac/compat/weboob_capabilities_bank.py +++ b/modules/delubac/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/erehsbc/compat/weboob_capabilities_bank.py b/modules/erehsbc/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/erehsbc/compat/weboob_capabilities_bank.py +++ b/modules/erehsbc/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/esalia/compat/weboob_capabilities_bank.py b/modules/esalia/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/esalia/compat/weboob_capabilities_bank.py +++ b/modules/esalia/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/fortuneo/compat/weboob_capabilities_bank.py b/modules/fortuneo/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/fortuneo/compat/weboob_capabilities_bank.py +++ b/modules/fortuneo/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/fortuneo/pages/compat/weboob_capabilities_bank.py b/modules/fortuneo/pages/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/fortuneo/pages/compat/weboob_capabilities_bank.py +++ b/modules/fortuneo/pages/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/ganassurances/compat/weboob_capabilities_bank.py b/modules/ganassurances/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/ganassurances/compat/weboob_capabilities_bank.py +++ b/modules/ganassurances/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/gmf/compat/weboob_capabilities_bank.py b/modules/gmf/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/gmf/compat/weboob_capabilities_bank.py +++ b/modules/gmf/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/groupama/compat/weboob_capabilities_bank.py b/modules/groupama/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/groupama/compat/weboob_capabilities_bank.py +++ b/modules/groupama/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/groupama/pages.py b/modules/groupama/pages.py index 0fb14f0285..fac929ac9f 100644 --- a/modules/groupama/pages.py +++ b/modules/groupama/pages.py @@ -56,12 +56,15 @@ def get_error(self): class AccountsPage(LoggedPage, HTMLPage): - ACCOUNT_TYPES = {u'Solde des comptes bancaires - Groupama Banque': Account.TYPE_CHECKING, - u'Solde des comptes bancaires': Account.TYPE_CHECKING, - u'Epargne bancaire constituée - Groupama Banque': Account.TYPE_SAVINGS, - u'Epargne bancaire constituée': Account.TYPE_SAVINGS, - u'Mes crédits': Account.TYPE_LOAN, - u'Assurance Vie': Account.TYPE_LIFE_INSURANCE} + ACCOUNT_TYPES = { + 'Solde des comptes bancaires - Groupama Banque': Account.TYPE_CHECKING, + 'Solde des comptes bancaires': Account.TYPE_CHECKING, + 'Epargne bancaire constituée - Groupama Banque': Account.TYPE_SAVINGS, + 'Epargne bancaire constituée': Account.TYPE_SAVINGS, + 'Mes crédits': Account.TYPE_LOAN, + 'Assurance Vie': Account.TYPE_LIFE_INSURANCE, + 'Certificats Mutualistes': Account.TYPE_SAVINGS, + } ACCOUNT_TYPES2 = { 'plan epargne actions': Account.TYPE_PEA, @@ -86,8 +89,8 @@ def get_list(self): balance = tds[-1].text.strip() account = Account() - account.label = u' '.join([txt.strip() for txt in tds[0].itertext()]) - account.label = re.sub(u'[ \xa0\u2022\r\n\t]+', u' ', account.label).strip() + account.label = ' '.join([txt.strip() for txt in tds[0].itertext()]) + account.label = re.sub(r'[\s\xa0\u2022]+', ' ', account.label).strip() # take "N° (FOO123 456)" but "N° (FOO123) MR. BAR" account.id = re.search(r'N° (\w+( \d+)*)', account.label).group(1).replace(' ', '') @@ -103,9 +106,9 @@ def get_list(self): account.currency = account.get_currency(balance) if 'onclick' in a.attrib: - m = re.search(r"javascript:submitForm\(([\w_]+),'([^']+)'\);", a.attrib['onclick']) + m = re.search(r"javascript:submitForm\(([\w]+),'([^']+)'\);", a.attrib['onclick']) if not m: - self.logger.warning('Unable to find link for %r' % account.label) + self.logger.warning('Unable to find link for %r', account.label) account._link = None else: account._link = m.group(2) @@ -120,17 +123,15 @@ def get_list(self): class Transaction(FrenchTransaction): - PATTERNS = [(re.compile('^Facture (?P
\d{2})/(?P\d{2})-(?P.*) carte .*'), - FrenchTransaction.TYPE_CARD), - (re.compile(u'^(Prlv( de)?|Ech(éance|\.)) (?P.*)'), - FrenchTransaction.TYPE_ORDER), - (re.compile('^(Vir|VIR)( de)? (?P.*)'), - FrenchTransaction.TYPE_TRANSFER), - (re.compile(u'^CHEQUE.*? (N° \w+)?$'), FrenchTransaction.TYPE_CHECK), - (re.compile('^Cotis(ation)? (?P.*)'), - FrenchTransaction.TYPE_BANK), - (re.compile('(?PInt .*)'), FrenchTransaction.TYPE_BANK), - ] + PATTERNS = [ + (re.compile(r'^Facture (?P
\d{2})/(?P\d{2})-(?P.*) carte .*'), FrenchTransaction.TYPE_CARD), + (re.compile(r'^(Prlv( de)?|Ech(éance|\.)) (?P.*)'), FrenchTransaction.TYPE_ORDER), + (re.compile(r'^(Vir|VIR)( de)? (?P.*)'), FrenchTransaction.TYPE_TRANSFER), + (re.compile(r'^CHEQUE.*? (N° \w+)?$'), FrenchTransaction.TYPE_CHECK), + (re.compile(r'^Cotis(ation)? (?P.*)'), FrenchTransaction.TYPE_BANK), + (re.compile(r'(?PInt .*)'), FrenchTransaction.TYPE_BANK), + (re.compile(r'^SOUSCRIPTION|REINVESTISSEMENT'), FrenchTransaction.TYPE_DEPOSIT), + ] class TransactionsPage(HTMLPage): @@ -142,18 +143,24 @@ class get_history(Transaction.TransactionsElement): head_xpath = '//table[@id="releve_operation"]//tr/th' item_xpath = '//table[@id="releve_operation"]//tr' - col_date = [u'Date opé', 'Date', u'Date d\'opé', u'Date opération'] - col_vdate = [u'Date valeur'] - col_credit = [u'Crédit', u'Montant', u'Valeur'] - col_debit = [u'Débit'] + col_date = ['Date opé', 'Date', 'Date d\'opé', 'Date opération'] + col_vdate = ['Date valeur'] + col_credit = ['Crédit', 'Montant', 'Valeur'] + col_debit = ['Débit'] def next_page(self): url = Attr('//a[contains(text(), "Page suivante")]', 'onclick', default=None)(self) if url: - m = re.search('\'([^\']+).*([\d]+)', url) - return requests.Request("POST", m.group(1), data={'numCompte': Env('accid')(self), \ - 'vue': "ReleveOperations", 'tri': "DateOperation", 'sens': \ - "DESC", 'page': m.group(2), 'nb_element': "25"}) + m = re.search(r'\'([^\']+).*([\d]+)', url) + return requests.Request("POST", m.group(1), + data={ + 'numCompte': Env('accid')(self), + 'vue': "ReleveOperations", + 'tri': "DateOperation", + 'sens': "DESC", + 'page': m.group(2), + 'nb_element': "25"} + ) class item(Transaction.TransactionElement): def condition(self): @@ -164,17 +171,16 @@ def get_coming_link(self): a = self.doc.getroot().cssselect('div#sous_nav ul li a.bt_sans_off')[0] except IndexError: return None - return re.sub('[ \t\r\n]+', '', a.attrib['href']) + return re.sub(r'[\s]+', '', a.attrib['href']) def has_iban(self): return self.doc.xpath('//a[@class="rib"]') def go_iban(self): js_event = Attr("//a[@class='rib']", 'onclick')(self.doc) - m = re.search("envoyer(.*);", js_event) + m = re.search(r'envoyer(.*);', js_event) iban_params = ast.literal_eval(m.group(1)) - link = iban_params[1] - self.browser.location(link+"?paramNumCpt={}".format(iban_params[0])) + self.browser.location("{}?paramNumCpt={}".format(iban_params[1], iban_params[0])) class IbanPage(LoggedPage, HTMLPage): @@ -191,7 +197,7 @@ class AVAccountPage(LoggedPage, HTMLPage): :rtype: tuple """ def get_av_balance(self): - balance_xpath = u'//p[contains(text(), "Épargne constituée")]/span' + balance_xpath = '//p[contains(text(), "Épargne constituée")]/span' balance = CleanDecimal(balance_xpath)(self.doc) currency = Account.get_currency(CleanText(balance_xpath)(self.doc)) return balance, currency @@ -201,10 +207,10 @@ class get_av_investments(TableElement): item_xpath = '//table[@id="repartition_epargne3"]/tr[position() > 1]' head_xpath = '//table[@id="repartition_epargne3"]/tr/th[position() > 1]' - col_quantity = u'Nombre d’unités de compte' - col_unitvalue = u"Valeur de l’unité de compte" - col_valuation = u'Épargne constituée en euros' - col_portfolio_share = u'Répartition %' + col_quantity = 'Nombre d’unités de compte' + col_unitvalue = "Valeur de l’unité de compte" + col_valuation = 'Épargne constituée en euros' + col_portfolio_share = 'Répartition %' class item(ItemElement): klass = Investment @@ -241,10 +247,10 @@ class get_av_history(TableElement): col_date = 'Date' col_label = 'Type de mouvement' - col_debit = u'Montant Désinvesti' - col_credit = ['Montant investi', u'Montant Net Perçu'] + col_debit = 'Montant Désinvesti' + col_credit = ['Montant investi', 'Montant Net Perçu'] # There is several types of life insurances, so multiple columns - col_credit2 = [u'Montant Brut Versé'] + col_credit2 = ['Montant Brut Versé'] class item(ItemElement): klass = Transaction @@ -259,7 +265,7 @@ def condition(self): def obj_amount(self): credit = CleanDecimal(TableCell('credit'), default=Decimal(0))(self) - # Different types of life insurances, use different columns. + # Different types of life insurances, use different columns. if TableCell('debit', default=None)(self): debit = CleanDecimal(TableCell('debit'), default=Decimal(0))(self) # In case of financial arbitration, both columns are equal @@ -277,7 +283,7 @@ def obj_amount(self): class FormPage(LoggedPage, HTMLPage): def get_av_balance(self): - balance_xpath = u'//p[contains(text(), "montant de votre épargne")]' + balance_xpath = '//p[contains(text(), "montant de votre épargne")]' balance = CleanDecimal(Regexp(CleanText(balance_xpath), r'est de ([\s\d,]+)', default=NotAvailable), replace_dots=True, default=NotAvailable)(self.doc) currency = Account.get_currency(CleanText(balance_xpath)(self.doc)) diff --git a/modules/groupamaes/compat/weboob_capabilities_bank.py b/modules/groupamaes/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/groupamaes/compat/weboob_capabilities_bank.py +++ b/modules/groupamaes/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/hsbc/browser.py b/modules/hsbc/browser.py index 010a25d6a2..3600c971ad 100644 --- a/modules/hsbc/browser.py +++ b/modules/hsbc/browser.py @@ -124,11 +124,12 @@ class HSBC(LoginBrowser): def __init__(self, username, password, secret, *args, **kwargs): super(HSBC, self).__init__(username, password, *args, **kwargs) - self.accounts_list = OrderedDict() - self.unique_accounts_list = dict() + self.accounts_dict = OrderedDict() + self.unique_accounts_dict = dict() self.secret = secret self.PEA_LISTING = {} self.owners = [] + self.web_space = None def load_state(self, state): return @@ -184,6 +185,10 @@ def go_to_owner_accounts(self, owner): "Pas de TIERS", so we must always go to the owners list before going to the owner's account page. """ + # In case of only one owner, do nothing and exit + if len(self.owners) == 1: + return + if not self.owners_list.is_here(): self.go_post(self.js_url, data={'debr': 'OPTIONS_TIE'}) @@ -204,71 +209,85 @@ def iter_account_owners(self): people each having their own accounts. We must fetch the account for each person and store the owner of each account. """ - if self.unique_accounts_list: - for account in self.unique_accounts_list.values(): + self.web_space = self.page.get_web_space() + if not self.unique_accounts_dict and self.web_space == 'new_space': + """ + With the new space the "Mes comptes de tiers" service is not activated by default, so this page is empty. + We must declare here the only owner in 'self.owners' + This could change in the future with more people migrating. + """ + self.owners = [0] + self.accounts_dict[self.owners[0]] = {} + self.update_accounts_dict(self.owners[0]) + for a in self.accounts_dict[self.owners[0]].values(): + a._owner = self.owners[0] + self.unique_accounts_dict = self.accounts_dict[self.owners[0]] + self.go_post(self.js_url, data={'debr': 'OPTIONS_TIE'}) + + if self.unique_accounts_dict: + for account in self.unique_accounts_dict.values(): yield account else: self.go_post(self.js_url, data={'debr': 'OPTIONS_TIE'}) - if self.owners_list.is_here(): - self.owners = self.page.get_owners_urls() - - # self.accounts_list will be a dictionary of owners each - # containing a dictionary of the owner's accounts. - for owner in range(len(self.owners)): - self.accounts_list[owner] = {} - self.update_accounts_list(owner, True) - - # We must set an "_owner" attribute to each account. - for a in self.accounts_list[owner].values(): - a._owner = owner - - # go on cards page if there are cards accounts - for a in self.accounts_list[owner].values(): - if a.type == Account.TYPE_CARD: - self.location(a.url) - break - - # get all couples (card, parent) on cards page - all_card_and_parent = [] - if self.cbPage.is_here(): - all_card_and_parent = self.page.get_all_parent_id() - self.go_post(self.js_url, data={'debr': 'COMPTES_PAN'}) - - # update cards parent and currency - for a in self.accounts_list[owner].values(): - if a.type == Account.TYPE_CARD: - for card in all_card_and_parent: - if a.id in card[0].replace(' ', ''): - a.parent = find_object(self.accounts_list[owner].values(), id=card[1]) - if a.parent and not a.currency: - a.currency = a.parent.currency - - # We must get back to the owners list before moving to the next owner: - self.go_post(self.js_url, data={'debr': 'OPTIONS_TIE'}) - - # Fill a dictionary will all accounts without duplicating common accounts: - for owner in self.accounts_list.values(): - for account in owner.values(): - if account.id not in self.unique_accounts_list.keys(): - self.unique_accounts_list[account.id] = account - - for account in self.unique_accounts_list.values(): - yield account + self.owners = self.page.get_owners_urls() + + # self.accounts_dict will be a dictionary of owners each + # containing a dictionary of the owner's accounts. + for owner in range(len(self.owners)): + self.accounts_dict[owner] = {} + self.update_accounts_dict(owner) + + # We must set an "_owner" attribute to each account. + for a in self.accounts_dict[owner].values(): + a._owner = owner + + # go on cards page if there are cards accounts + for a in self.accounts_dict[owner].values(): + if a.type == Account.TYPE_CARD: + self.location(a.url) + break + + # get all couples (card, parent) on cards page + all_card_and_parent = [] + if self.cbPage.is_here(): + all_card_and_parent = self.page.get_all_parent_id() + self.go_post(self.js_url, data={'debr': 'COMPTES_PAN'}) + + # update cards parent and currency + for a in self.accounts_dict[owner].values(): + if a.type == Account.TYPE_CARD: + for card in all_card_and_parent: + if a.id in card[0].replace(' ', ''): + a.parent = find_object(self.accounts_dict[owner].values(), id=card[1]) + if a.parent and not a.currency: + a.currency = a.parent.currency + + # We must get back to the owners list before moving to the next owner: + self.go_post(self.js_url, data={'debr': 'OPTIONS_TIE'}) + + # Fill a dictionary will all accounts without duplicating common accounts: + for owner in self.accounts_dict.values(): + for account in owner.values(): + if account.id not in self.unique_accounts_dict.keys(): + self.unique_accounts_dict[account.id] = account + for account in self.unique_accounts_dict.values(): + yield account @need_login - def update_accounts_list(self, owner, iban=True): + def update_accounts_dict(self, owner, iban=True): # Go to the owner's account page in case we are not there already: self.go_to_owner_accounts(owner) - for a in self.page.iter_spaces_account(): + + for a in self.page.iter_spaces_account(self.web_space): try: - self.accounts_list[owner][a.id].url = a.url + self.accounts_dict[owner][a.id].url = a.url except KeyError: - self.accounts_list[owner][a.id] = a + self.accounts_dict[owner][a.id] = a if iban: self.location(self.js_url, params={'debr': 'COMPTES_RIB'}) if self.rib.is_here(): - self.page.get_rib(self.accounts_list[owner]) + self.page.get_rib(self.accounts_dict[owner]) @need_login def _quit_li_space(self): @@ -310,8 +329,10 @@ def _go_to_life_insurance(self, account): @need_login def get_history(self, account, coming=False, retry_li=True): self._quit_li_space() - self.update_accounts_list(account._owner, False) - account = self.accounts_list[account._owner][account.id] + # Update accounts list only in case of several owners + if len(self.owners) > 1: + self.update_accounts_dict(account._owner, iban=False) + account = self.accounts_dict[account._owner][account.id] if account.url is None: return [] @@ -367,23 +388,22 @@ def get_history(self, account, coming=False, retry_li=True): return history try: - self.go_post(self.accounts_list[account._owner][account.id].url) + self.go_post(account.url) # sometime go to hsbc life insurance space do logout except HTTPNotFound: self.app_gone = True self.do_logout() self.do_login() - # If we relogin on hsbc, all links have changed if self.app_gone: self.app_gone = False - self.update_accounts_list(account._owner, False) - self.location(self.accounts_list[account._owner][account.id].url) + self.update_accounts_dict(account._owner, iban=False) + self.location(self.accounts_dict[account._owner][account.id].url) if self.page is None: return [] - # for 'fusion' space + # for 'fusion' and 'new' space there is a form to submit on the page to go the account's history if hasattr(account, '_is_form') and account._is_form: # go on accounts page to get account form self.go_to_owner_accounts(account._owner) @@ -492,8 +512,8 @@ def get_pea_investments(self, account): def get_life_investments(self, account, retry_li=True): self._quit_li_space() - self.update_accounts_list(account._owner, False) - account = self.accounts_list[account._owner][account.id] + self.update_accounts_dict(account._owner, False) + account = self.accounts_dict[account._owner][account.id] try: if not self._go_to_life_insurance(account): self._quit_li_space() @@ -523,7 +543,7 @@ def _go_to_wealth_accounts(self, account): if not hasattr(self.page, 'get_middle_frame_url'): # if we can catch the URL, we go directly, else we need to browse # the website - self.update_accounts_list(account._owner, False) + self.update_accounts_dict(account._owner, False) self.location(self.page.get_middle_frame_url()) @@ -534,7 +554,7 @@ def _go_to_wealth_accounts(self, account): if self.login.is_here(): self.logger.warning('Connection to the Logon page failed, we must try again.') self.do_login() - self.update_accounts_list(account._owner, False) + self.update_accounts_dict(account._owner, False) self.investment_form_page.go() # If reloggin did not help accessing the wealth space, # there is nothing more we can do to get there. diff --git a/modules/hsbc/compat/weboob_capabilities_bank.py b/modules/hsbc/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/hsbc/compat/weboob_capabilities_bank.py +++ b/modules/hsbc/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/hsbc/pages/account_pages.py b/modules/hsbc/pages/account_pages.py index d5435a3f7d..f40ace57b7 100644 --- a/modules/hsbc/pages/account_pages.py +++ b/modules/hsbc/pages/account_pages.py @@ -31,24 +31,23 @@ from .compat.weboob_browser_filters_standard import ( Filter, Env, CleanText, CleanDecimal, Field, DateGuesser, Regexp, Currency, Format, Date ) -from weboob.browser.filters.html import AbsoluteLink, TableCell +from weboob.browser.filters.html import AbsoluteLink, Attr, TableCell from weboob.browser.filters.javascript import JSVar from weboob.capabilities.profile import Person from .landing_pages import GenericLandingPage class Transaction(FrenchTransaction): - PATTERNS = [(re.compile(r'^VIR(EMENT)? (?P.*)'), FrenchTransaction.TYPE_TRANSFER), - (re.compile(r'^PRLV (?P.*)'), FrenchTransaction.TYPE_ORDER), - (re.compile(r'^CB (?P.*?)\s+(?P
\d+)/(?P[01]\d)\s+(?P.*)'), - FrenchTransaction.TYPE_CARD), - (re.compile(r'^DAB (?P
\d{2})/(?P\d{2}) ((?P\d{2})H(?P\d{2}) )?(?P.*?)( CB N°.*)?$'), - FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile(r'^CHEQUE( \d+)?$'), FrenchTransaction.TYPE_CHECK), - (re.compile(r'^COTIS\.? (?P.*)'), FrenchTransaction.TYPE_BANK), - (re.compile(r'^REMISE (?P.*)'), FrenchTransaction.TYPE_DEPOSIT), - (re.compile(r'^FACTURES CB (?P.*)'), FrenchTransaction.TYPE_CARD_SUMMARY), - ] + PATTERNS = [ + (re.compile(r'^VIR(EMENT)? (?P.*)'), FrenchTransaction.TYPE_TRANSFER), + (re.compile(r'^PRLV (?P.*)'), FrenchTransaction.TYPE_ORDER), + (re.compile(r'^CB (?P.*?)\s+(?P
\d+)/(?P[01]\d)\s+(?P.*)'), FrenchTransaction.TYPE_CARD), + (re.compile(r'^DAB (?P
\d{2})/(?P\d{2}) ((?P\d{2})H(?P\d{2}) )?(?P.*?)( CB N°.*)?$'), FrenchTransaction.TYPE_WITHDRAWAL), + (re.compile(r'^CHEQUE( \d+)?$'), FrenchTransaction.TYPE_CHECK), + (re.compile(r'^COTIS\.? (?P.*)'), FrenchTransaction.TYPE_BANK), + (re.compile(r'^REMISE (?P.*)'), FrenchTransaction.TYPE_DEPOSIT), + (re.compile(r'^FACTURES CB (?P.*)'), FrenchTransaction.TYPE_CARD_SUMMARY), + ] class FrameContainer(GenericLandingPage): @@ -57,7 +56,7 @@ class FrameContainer(GenericLandingPage): # main page, a frameset def on_load(self): txt = CleanText('//p[@class="debit"]', default='')(self.doc) - if u"Vos données d'identification (identifiant - code secret) sont incorrectes" in txt: + if "Vos données d'identification (identifiant - code secret) sont incorrectes" in txt: raise BrowserIncorrectPassword() def get_js_url(self): @@ -66,7 +65,7 @@ def get_js_url(self): def get_frame(self): try: - a = self.doc.xpath(u'//frame["@name=FrameWork"]')[0] + a = self.doc.xpath('//frame["@name=FrameWork"]')[0] except IndexError: return None else: @@ -130,26 +129,56 @@ def filter(self, text): class AccountsPage(GenericLandingPage): - is_here = '//h1[text()="Synthèse"]' + def is_here(self): + return CleanText('//h1[text()="Synthèse"]')(self.doc) or CleanText('//span[@class="hsbc-pib-title-text"][text()="Tous mes comptes au "]')(self.doc) - def iter_spaces_account(self): + def get_web_space(self): + """ Several spaces on HSBC, need to get which one we are on to adapt parsing to owners""" if self.doc.xpath('//p[text()="HSBC Fusion"]'): - space = 'fusion' + return 'fusion' + elif self.doc.xpath('//span[contains(@class, "screen-reader-text") and text()="Aller vers hsbc.fr"]'): + return 'new_space' else: - space = 'default' + return 'default' + def iter_spaces_account(self, space): accounts = { 'fusion': self.iter_fusion_accounts, 'default': self.iter_accounts, + 'new_space': self.iter_new_space_accounts, } return accounts[space]() def go_history_page(self, account): - for acc in self.doc.xpath('//div[@onclick]'): - # label contains account number, it's enough to check if it's the right account - if account.label == Label(CleanText('.//p[@class="title"]'))(acc): - form_id = CleanText('.//form/@id')(acc) - return self.get_form(id=form_id).submit() + if self.browser.web_space == 'new_space': + # Must iterate through forms and find a match between accound number and the 'value' tag to know which form to submit + for form in self.doc.xpath('//form[@id]'): + value = Attr('.//input[@name="CPT_IdPrestation"]', 'value')(form) + if account.id in value: + form_id = Attr('.', 'id')(form) + return self.get_form(id=form_id).submit() + else: + for acc in self.doc.xpath('//div[@onclick]'): + # label contains account number, it's enough to check if it's the right account + if account.label == Label(CleanText('.//p[@class="title"]'))(acc): + form_id = CleanText('.//form/@id')(acc) + return self.get_form(id=form_id).submit() + + @method + class iter_new_space_accounts(ListElement): + item_xpath = '//div[@class="hsbc-pib-bloc-row-container"]' + + class item(ItemElement): + klass = Account + + # TODO: 'obj_id' will need redefinition when we find connections using the new space and Investment account and main account that have the same id + obj_id = CleanText('./p/span[@class="hsbc-pib-text--xsmall uk-text-gray"]', replace=[('.', ''), (' ', '')]) + obj_label = Label(CleanText('./p/span[@class="hsbc-pib-text hsbc-pib-bloc-account-name"]')) + obj_type = AccountsType(Field('label')) + obj_balance = CleanDecimal('./p/span[@class="hsbc-pib-text uk-text-bold"]', replace_dots=True) + obj_currency = Currency('./p/span[@class="hsbc-pib-text uk-text-bold"]') + obj_url = CleanText('.//form/@action') + obj__is_form = bool(CleanText('.//form/@id')) @method class iter_accounts(ListElement): @@ -182,7 +211,7 @@ def obj_url(self): @property def obj_balance(self): - if self.el.xpath('./parent::*/tr/th') and self.el.xpath('./parent::*/tr/th')[0].text in [u'Credits', u'Crédits']: + if self.el.xpath('./parent::*/tr/th') and self.el.xpath('./parent::*/tr/th')[0].text in ['Credits', 'Crédits']: return CleanDecimal(replace_dots=True, sign=lambda x: -1).filter(self.el.xpath('./td[3]')) return CleanDecimal(replace_dots=True).filter(self.el.xpath('./td[3]')) @@ -236,7 +265,8 @@ def obj_id(self): class OwnersListPage(AccountsPage): - is_here = '//h1[text()="Comptes de tiers"]' + def is_here(self): + return CleanText('//h1[text()="Comptes de tiers"]')(self.doc) or CleanText('//h1[text()="Gérer les comptes de mes tiers"]')(self.doc) def get_owners_urls(self): return self.doc.xpath('//div[@class="GoBack"]/a/@href') @@ -250,9 +280,9 @@ def link_rib(self, accounts): for id, acc in accounts.items(): if acc.iban or acc.type is not Account.TYPE_CHECKING: continue - digit_id = ''.join(re.findall('\d', id)) + digit_id = ''.join(re.findall(r'\d', id)) if digit_id in CleanText('//div[@class="RIB_content"]')(self.doc): - acc.iban = re.search('(FR\d{25})', CleanText('//div[strong[contains(text(), "IBAN")]]', replace=[(' ', '')])(self.doc)).group(1) + acc.iban = re.search(r'(FR\d{25})', CleanText('//div[strong[contains(text(), "IBAN")]]', replace=[(' ', '')])(self.doc)).group(1) def get_rib(self, accounts): self.link_rib(accounts) @@ -316,11 +346,16 @@ def get_all_parent_id(self): class CPTOperationPage(GenericLandingPage): - is_here = '''//h1[text()="Historique des opérations"] and //h2[text()="Recherche d'opération"]''' + def is_here(self): + return (CleanText('//h1[text()="Historique des opérations"]')(self.doc) + and (CleanText('''//h2[text()="Recherche d'opération"]''')(self.doc) + or CleanText('//div[@class="hsbc-datatable-search"]/label[text()="Rechercher"]') + ) + ) def get_history(self): if self.doc.xpath('//form[@name="FORM_SUITE"]'): - m = re.search('suite[\s]+=[\s]+([\w]+)', CleanText().filter(self.doc.xpath('//script[contains(text(), "var suite")]'))) + m = re.search(r'suite[\s]+=[\s]+([\w]+)', CleanText().filter(self.doc.xpath('//script[contains(text(), "var suite")]'))) if m and m.group(1) == "true": form = self.get_form(name="FORM_SUITE") self.doc = self.browser.location("%s" % form.url, params=dict(form)).page.doc @@ -332,7 +367,7 @@ def get_history(self): first_history = None for m in re.finditer(r"CL\((\d+),'(.+)','(.+)','(.+)','([\d -\.,]+)',('([\d -\.,]+)',)?'\d+','\d+','[\w\s]+'\);", script.text, flags=re.MULTILINE | re.UNICODE): op = Transaction() - raw = re.sub(u'[ ]+', u' ', m.group(4).replace(u'\n', u' ').replace(r"\'", "'")) + raw = re.sub(r'\s+', ' ', m.group(4).replace('\n', ' ').replace("\'", "'")) op.parse(date=m.group(3), raw=raw) op.set_amount(m.group(5)) op._coming = (re.match(r'\d+/\d+/\d+', m.group(2)) is None) @@ -355,7 +390,7 @@ def on_load(self): class LoginPage(HTMLPage): @property def logged(self): - if self.doc.xpath(u'//p[contains(text(), "You are now being redirected to your Personal Internet Banking.")]'): + if self.doc.xpath('//p[contains(text(), "You are now being redirected to your Personal Internet Banking.")]'): return True return False @@ -378,7 +413,7 @@ def login(self, login): def get_no_secure_key(self): try: - a = self.doc.xpath(u'//a[contains(text(), "Without HSBC Secure Key")]')[0] + a = self.doc.xpath('//a[contains(text(), "Without HSBC Secure Key")]')[0] except IndexError: return None else: @@ -387,8 +422,8 @@ def get_no_secure_key(self): def login_w_secure(self, password, secret): form = self.get_form(nr=0) form['memorableAnswer'] = secret - inputs = self.doc.xpath(u'//input[starts-with(@id, "keyrcc_password_first")]') - split_pass = u'' + inputs = self.doc.xpath('//input[starts-with(@id, "keyrcc_password_first")]') + split_pass = '' if len(password) < len(inputs): raise BrowserIncorrectPassword('The password must be at least %d characters' % len(inputs)) elif len(password) > len(inputs): diff --git a/modules/hsbc/pages/compat/weboob_capabilities_bank.py b/modules/hsbc/pages/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/hsbc/pages/compat/weboob_capabilities_bank.py +++ b/modules/hsbc/pages/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/humanis/compat/weboob_capabilities_bank.py b/modules/humanis/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/humanis/compat/weboob_capabilities_bank.py +++ b/modules/humanis/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/ing/api/compat/weboob_capabilities_bank.py b/modules/ing/api/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/ing/api/compat/weboob_capabilities_bank.py +++ b/modules/ing/api/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/ing/compat/weboob_capabilities_bank.py b/modules/ing/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/ing/compat/weboob_capabilities_bank.py +++ b/modules/ing/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/ing/web/compat/weboob_capabilities_bank.py b/modules/ing/web/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/ing/web/compat/weboob_capabilities_bank.py +++ b/modules/ing/web/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/kiwibank/compat/weboob_capabilities_bank.py b/modules/kiwibank/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/kiwibank/compat/weboob_capabilities_bank.py +++ b/modules/kiwibank/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/lcl/compat/weboob_capabilities_bank.py b/modules/lcl/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/lcl/compat/weboob_capabilities_bank.py +++ b/modules/lcl/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/lcl/enterprise/compat/weboob_capabilities_bank.py b/modules/lcl/enterprise/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/lcl/enterprise/compat/weboob_capabilities_bank.py +++ b/modules/lcl/enterprise/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/lcl/pages.py b/modules/lcl/pages.py index b8e080e49b..8fc66f6678 100644 --- a/modules/lcl/pages.py +++ b/modules/lcl/pages.py @@ -42,7 +42,7 @@ from weboob.browser.filters.html import Attr, Link, TableCell, AttributeNotFound from .compat.weboob_browser_filters_standard import ( CleanText, Field, Regexp, Format, Date, CleanDecimal, Map, AsyncLoad, Async, Env, Slugify, - BrowserURL, Eval, Lower, Currency, + BrowserURL, Eval, Currency, ) from weboob.browser.filters.json import Dict from weboob.exceptions import BrowserUnavailable, BrowserIncorrectPassword @@ -963,9 +963,14 @@ class iter_life_insurance(DictElement): class item(ItemElement): def condition(self): - return ( - Lower(Dict('lcstacntgen'))(self) == 'actif' - and Lower(Dict('lcgampdt'))(self) == 'epargne' + activity = Dict('lcstacntgen')(self) + account_type = Dict('lcgampdt')(self) + # We ignore accounts without activities or when the activity is 'Closed', + # they are inactive and closed and they don't appear on the website. + return bool( + activity and account_type + and activity.lower() == 'actif' + and account_type.lower() == 'epargne' ) klass = Account diff --git a/modules/linebourse/api/compat/weboob_capabilities_bank.py b/modules/linebourse/api/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/linebourse/api/compat/weboob_capabilities_bank.py +++ b/modules/linebourse/api/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/linebourse/compat/weboob_capabilities_bank.py b/modules/linebourse/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/linebourse/compat/weboob_capabilities_bank.py +++ b/modules/linebourse/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/logicimmo/pages.py b/modules/logicimmo/pages.py index 9d147977dd..944733f5f6 100644 --- a/modules/logicimmo/pages.py +++ b/modules/logicimmo/pages.py @@ -68,10 +68,10 @@ def obj_type(self): elif 'location' in url: isFurnished = False for li in XPath('//ul[@itemprop="description"]/li')(self): - label = CleanText('./div[has-class("criteria-label")]')(li) + label = CleanText('./span[has-class("criteria-label")]')(li) if label.lower() == "meublé": isFurnished = ( - CleanText('./div[has-class("criteria-value")]')(li).lower() == 'oui' + CleanText('./span[has-class("criteria-value")]')(li).lower() == 'oui' ) if isFurnished: return POSTS_TYPES.FURNISHED_RENT @@ -186,8 +186,8 @@ def obj_details(self): ) for li in XPath('//ul[@itemprop="description"]/li')(self): - label = CleanText('./div[has-class("criteria-label")]')(li) - value = CleanText('./div[has-class("criteria-value")]')(li) + label = CleanText('./span[has-class("criteria-label")]')(li) + value = CleanText('./span[has-class("criteria-value")]')(li) details[label] = value return details @@ -252,7 +252,7 @@ class iter_housings(ListElement): class item(ItemElement): offer_details_wrapper = ( - './div/div/div[has-class("offer-details-wrapper")]' + './/div[has-class("offer-details-wrapper")]' ) klass = Housing @@ -265,7 +265,7 @@ class item(ItemElement): obj_advert_type = ADVERT_TYPES.PROFESSIONAL def obj_house_type(self): - house_type = CleanText('.//p[has-class("offer-type")]')(self).lower() + house_type = CleanText('.//div[has-class("offer-details-type")]/a')(self).split(' ')[0].lower() if house_type == "appartement": return HOUSE_TYPES.APART elif house_type == "maison": @@ -277,14 +277,11 @@ def obj_house_type(self): else: return HOUSE_TYPES.OTHER - obj_title = Attr( - offer_details_wrapper + '/div/div/p[@class="offer-type"]/a', - 'title' - ) + obj_title = CleanText('.//div[has-class("offer-details-type")]/a/@title') obj_url = Format(u'%s%s', - CleanText('./div/div/div/div/div/p/a[@class="offer-link"]/@href'), - CleanText('./div/div/div/div/div/p/a[@class="offer-link"]/\ + CleanText('.//div/a[@class="offer-link"]/@href'), + CleanText('.//div/a[@class="offer-link"]/\ @data-orpi', default="")) obj_area = CleanDecimal( @@ -304,15 +301,14 @@ def obj_house_type(self): '/span[has-class("offer-rooms")]' + '/span[has-class("offer-rooms-number")]' ), - default=NotLoaded + default=NotAvailable ) - obj_price_per_meter = PricePerMeterFilter() obj_cost = CleanDecimal( Regexp( CleanText( ( offer_details_wrapper + - '/div/div/p[@class="offer-price"]/span' + '/div/p[@class="offer-price"]/span' ), default=NotLoaded ), @@ -322,22 +318,16 @@ def obj_house_type(self): default=NotLoaded ) obj_currency = Currency( - offer_details_wrapper + '/div/div/p[has-class("offer-price")]/span' + offer_details_wrapper + '/div/p[has-class("offer-price")]/span' ) + obj_price_per_meter = PricePerMeterFilter() obj_utilities = UTILITIES.UNKNOWN - obj_date = Date( - Regexp( - CleanText( - './div/div/div[has-class("offer-picture-more")]/div/p[has-class("offer-update")]' - ), - ".*(\d{2}/\d{2}/\d{4}).*") - ) obj_text = CleanText( offer_details_wrapper + '/div/div/div/p[has-class("offer-description")]/span' ) obj_location = CleanText( - offer_details_wrapper + - '//div[has-class("offer-places-block")]' + offer_details_wrapper + '/div[@class="offer-details-location"]', + replace=[('Voir sur la carte','')] ) def obj_photos(self): diff --git a/modules/logicimmo/test.py b/modules/logicimmo/test.py index 73ac92948b..46f9ef3648 100644 --- a/modules/logicimmo/test.py +++ b/modules/logicimmo/test.py @@ -28,11 +28,10 @@ class LogicimmoTest(BackendTest, HousingTest): FIELDS_ALL_HOUSINGS_LIST = [ "id", "type", "advert_type", "house_type", "url", "title", "area", "cost", "currency", "utilities", "date", "location", "text", - "details" + "details", "rooms" ] FIELDS_ANY_HOUSINGS_LIST = [ "photos", - "rooms" ] FIELDS_ALL_SINGLE_HOUSING = [ "id", "url", "type", "advert_type", "house_type", "title", "area", @@ -47,6 +46,7 @@ class LogicimmoTest(BackendTest, HousingTest): "DPE", "GES" ] + DO_NOT_DISTINGUISH_FURNISHED_RENT = True def test_logicimmo_rent(self): query = Query() diff --git a/modules/myedenred/compat/weboob_capabilities_bank.py b/modules/myedenred/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/myedenred/compat/weboob_capabilities_bank.py +++ b/modules/myedenred/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/myedenred/pages.py b/modules/myedenred/pages.py index f259a26348..176a87db7b 100644 --- a/modules/myedenred/pages.py +++ b/modules/myedenred/pages.py @@ -58,6 +58,7 @@ class get_account(ItemElement): obj_label = obj_id obj_currency = u'EUR' obj_balance = MyDecimal('//p[@class="num"]/a') + obj_cardlimit = MyDecimal('//div[has-class("solde_actu")]') # Every subscription a product token and a type ex: card = 240 obj__product_token = Regexp(CleanText('//div[contains(@id, "product")]/@id'), r'productLine_(\d*)') diff --git a/modules/n26/compat/weboob_capabilities_bank.py b/modules/n26/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/n26/compat/weboob_capabilities_bank.py +++ b/modules/n26/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/nalo/compat/weboob_capabilities_bank.py b/modules/nalo/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/nalo/compat/weboob_capabilities_bank.py +++ b/modules/nalo/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/nef/compat/weboob_capabilities_bank.py b/modules/nef/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/nef/compat/weboob_capabilities_bank.py +++ b/modules/nef/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/netfinca/compat/weboob_capabilities_bank.py b/modules/netfinca/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/netfinca/compat/weboob_capabilities_bank.py +++ b/modules/netfinca/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/netfinca/pages.py b/modules/netfinca/pages.py index 9593e9ed39..750672d3e1 100644 --- a/modules/netfinca/pages.py +++ b/modules/netfinca/pages.py @@ -34,6 +34,8 @@ ACCOUNT_TYPES = { 'D.A.T.': Account.TYPE_DEPOSIT, 'COMPTE PEA': Account.TYPE_PEA, + 'INTEGRAL PEA': Account.TYPE_PEA, + 'COMPTE PEA-PME': Account.TYPE_PEA, 'COMPTE TITRES': Account.TYPE_MARKET, 'CTO VENDOME PRIVILEGE': Account.TYPE_MARKET, 'PARTS SOCIALES': Account.TYPE_MARKET, diff --git a/modules/oney/compat/weboob_capabilities_bank.py b/modules/oney/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/oney/compat/weboob_capabilities_bank.py +++ b/modules/oney/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/orange/browser.py b/modules/orange/browser.py index fc942b5601..56d0b72c8b 100644 --- a/modules/orange/browser.py +++ b/modules/orange/browser.py @@ -25,7 +25,7 @@ from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable, ActionNeeded from .pages import LoginPage, BillsPage from .pages.login import ManageCGI, HomePage -from .pages.bills import SubscriptionsPage, BillsApiPage, ContractsPage +from .pages.bills import SubscriptionsPage, BillsApiProPage, BillsApiParPage, ContractsPage from .pages.profile import ProfilePage from weboob.browser.exceptions import ClientError, ServerError from weboob.tools.compat import basestring @@ -47,6 +47,7 @@ class OrangeBillBrowser(LoginBrowser): subscriptions = URL(r'https://espaceclientv3.orange.fr/js/necfe.php\?zonetype=bandeau&idPage=gt-home-page', SubscriptionsPage) manage_cgi = URL('https://eui.orange.fr/manage_eui/bin/manage.cgi', ManageCGI) + # is billspage deprecated ? billspage = URL('https://m.espaceclientv3.orange.fr/\?page=factures-archives', 'https://.*.espaceclientv3.orange.fr/\?page=factures-archives', 'https://espaceclientv3.orange.fr/\?page=factures-archives', @@ -56,10 +57,13 @@ class OrangeBillBrowser(LoginBrowser): 'https://espaceclientv3.orange.fr/\?page=factures-historique&idContrat=(?P.*)', BillsPage) - bills_api = URL('https://espaceclientpro.orange.fr/api/contract/(?P\d+)/bills\?count=(?P)', - BillsApiPage) + bills_api_pro = URL('https://espaceclientpro.orange.fr/api/contract/(?P\d+)/bills\?count=(?P)', + BillsApiProPage) - doc_api = URL('https://espaceclientpro.orange.fr/api/contract/(?P\d+)/bill/(?P.*)/(?P.*)/\?(?P)') + bills_api_par = URL(r'https://sso-f.orange.fr/omoi_erb/facture/v2.0/billsAndPaymentInfos/users/current/contracts/(?P\d+)', BillsApiParPage) + doc_api_par = URL(r'https://sso-f.orange.fr/omoi_erb/facture/v1.0/pdf') + + doc_api_pro = URL('https://espaceclientpro.orange.fr/api/contract/(?P\d+)/bill/(?P.*)/(?P.*)/\?(?P)') profile = URL('/\?page=profil-infosPerso', ProfilePage) def do_login(self): @@ -71,6 +75,9 @@ def do_login(self): except ClientError as error: if error.response.status_code == 401: raise BrowserIncorrectPassword() + if error.response.status_code == 403: + # occur when user try several times with a bad password, orange block his account for a short time + raise BrowserIncorrectPassword(error.response.json()) raise def get_nb_remaining_free_sms(self): @@ -132,12 +139,13 @@ def get_subscription_list(self): def iter_documents(self, subscription): documents = [] if subscription._is_pro: - for d in self.bills_api.go(subid=subscription.id, count=72).get_bills(subid=subscription.id): + for d in self.bills_api_pro.go(subid=subscription.id, count=72).get_bills(subid=subscription.id): documents.append(d) # check pagination for this subscription assert len(documents) != 72 else: - self.billspage.go(subid=subscription.id) + headers = {'x-orange-caller-id': 'ECQ'} + self.bills_api_par.go(subid=subscription.id, headers=headers) for b in self.page.get_bills(subid=subscription.id): documents.append(b) return iter(documents) diff --git a/modules/orange/module.py b/modules/orange/module.py index 71d2df1706..60cf6e6d33 100644 --- a/modules/orange/module.py +++ b/modules/orange/module.py @@ -72,6 +72,10 @@ def download_document(self, document): document = self.get_document(document) if document.url is NotAvailable: return + + if document._is_v2: + # get 404 without this header + return self.browser.open(document.url, headers={'x-orange-caller-id': 'ECQ'}).content return self.browser.open(document.url).content def get_profile(self): diff --git a/modules/orange/pages/bills.py b/modules/orange/pages/bills.py index 08eaef6a00..d58f0a9ea4 100644 --- a/modules/orange/pages/bills.py +++ b/modules/orange/pages/bills.py @@ -28,7 +28,9 @@ from weboob.browser.pages import HTMLPage, LoggedPage, JsonPage from weboob.capabilities.bill import Subscription from weboob.browser.elements import DictElement, ListElement, ItemElement, method, TableElement -from .compat.weboob_browser_filters_standard import CleanDecimal, CleanText, Env, Field, Regexp, Date, Currency, BrowserURL, Format +from .compat.weboob_browser_filters_standard import ( + CleanDecimal, CleanText, Env, Field, Regexp, Date, Currency, BrowserURL, Format, Eval +) from weboob.browser.filters.html import Link, TableCell from weboob.browser.filters.javascript import JSValue from weboob.browser.filters.json import Dict @@ -38,7 +40,7 @@ from weboob.tools.compat import urlencode -class BillsApiPage(LoggedPage, JsonPage): +class BillsApiProPage(LoggedPage, JsonPage): @method class get_bills(DictElement): item_xpath = 'bills' @@ -64,9 +66,33 @@ def get_params(self): params = {'billid': Dict('id')(self), 'billDate': Dict('dueDate')(self)} return urlencode(params) - obj_url = BrowserURL('doc_api', subid=Env('subid'), dir=Dict('documents/0/mainDir'), fact_type=Dict('documents/0/subDir'), billparams=get_params) + obj_url = BrowserURL('doc_api_pro', subid=Env('subid'), dir=Dict('documents/0/mainDir'), fact_type=Dict('documents/0/subDir'), billparams=get_params) + obj__is_v2 = False +class BillsApiParPage(LoggedPage, JsonPage): + @method + class get_bills(DictElement): + item_xpath = 'billsHistory/billList' + + class item(ItemElement): + klass = Bill + + obj_date = Date(Dict('date'), default=NotAvailable) + obj_price = Eval(lambda x: x / 100, CleanDecimal(Dict('amount'))) + obj_format = 'pdf' + + def obj_label(self): + return 'Facture du %s' % Field('date')(self) + + def obj_id(self): + return '%s_%s' % (Env('subid')(self), Field('date')(self).strftime('%d%m%Y')) + + obj_url = Format('%s%s', BrowserURL('doc_api_par'), Dict('hrefPdf')) + obj__is_v2 = True + + +# is BillsPage deprecated ? class BillsPage(LoggedPage, HTMLPage): @method class get_bills(TableElement): diff --git a/modules/paypal/compat/weboob_capabilities_bank.py b/modules/paypal/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/paypal/compat/weboob_capabilities_bank.py +++ b/modules/paypal/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/pradoepargne/compat/weboob_capabilities_bank.py b/modules/pradoepargne/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/pradoepargne/compat/weboob_capabilities_bank.py +++ b/modules/pradoepargne/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/s2e/compat/weboob_capabilities_bank.py b/modules/s2e/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/s2e/compat/weboob_capabilities_bank.py +++ b/modules/s2e/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/s2e/pages.py b/modules/s2e/pages.py index 7744660ee3..6ba9acbbb7 100644 --- a/modules/s2e/pages.py +++ b/modules/s2e/pages.py @@ -340,6 +340,7 @@ def on_load(self): 'PEEG': Account.TYPE_PEE, 'PEG': Account.TYPE_PEE, 'PLAN': Account.TYPE_PEE, + 'PAGA': Account.TYPE_PEE, 'PERCO': Account.TYPE_PERCO, 'PERCOI': Account.TYPE_PERCO, 'SWISS': Account.TYPE_MARKET, diff --git a/modules/societegenerale/compat/weboob_capabilities_bank.py b/modules/societegenerale/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/societegenerale/compat/weboob_capabilities_bank.py +++ b/modules/societegenerale/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/societegenerale/pages/compat/weboob_capabilities_bank.py b/modules/societegenerale/pages/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/societegenerale/pages/compat/weboob_capabilities_bank.py +++ b/modules/societegenerale/pages/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/societegenerale/sgpe/compat/weboob_capabilities_bank.py b/modules/societegenerale/sgpe/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/societegenerale/sgpe/compat/weboob_capabilities_bank.py +++ b/modules/societegenerale/sgpe/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/sogecartenet/compat/weboob_capabilities_bank.py b/modules/sogecartenet/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/sogecartenet/compat/weboob_capabilities_bank.py +++ b/modules/sogecartenet/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/spirica/compat/weboob_capabilities_bank.py b/modules/spirica/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/spirica/compat/weboob_capabilities_bank.py +++ b/modules/spirica/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/suravenir/compat/weboob_capabilities_bank.py b/modules/suravenir/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/suravenir/compat/weboob_capabilities_bank.py +++ b/modules/suravenir/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/themisbanque/compat/weboob_capabilities_bank.py b/modules/themisbanque/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/themisbanque/compat/weboob_capabilities_bank.py +++ b/modules/themisbanque/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/ticketscesu/compat/weboob_capabilities_bank.py b/modules/ticketscesu/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/ticketscesu/compat/weboob_capabilities_bank.py +++ b/modules/ticketscesu/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/vicseccard/compat/weboob_capabilities_bank.py b/modules/vicseccard/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/vicseccard/compat/weboob_capabilities_bank.py +++ b/modules/vicseccard/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/wellsfargo/compat/weboob_capabilities_bank.py b/modules/wellsfargo/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/wellsfargo/compat/weboob_capabilities_bank.py +++ b/modules/wellsfargo/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') diff --git a/modules/yomoni/compat/weboob_capabilities_bank.py b/modules/yomoni/compat/weboob_capabilities_bank.py index d82081dd5c..4d5ed8d992 100644 --- a/modules/yomoni/compat/weboob_capabilities_bank.py +++ b/modules/yomoni/compat/weboob_capabilities_bank.py @@ -17,6 +17,7 @@ # can't create a subclass because of CapBank.iter_resources reimplementations: # modules will import our subclass, but boobank will call iter_resources with the OLD class Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') -- GitLab