From 687184c989e760e52186add462462cc616cf4ab9 Mon Sep 17 00:00:00 2001 From: Quentin Defenouillere Date: Tue, 18 Jun 2019 15:29:39 +0200 Subject: [PATCH] [axabanque] Repaired iter_history for AXAAssurance browser The website recently changed and the xpaths were obsolete. The transactions are now contained in a JSON. Closes: 11985@zendesk --- modules/axabanque/browser.py | 42 +++++++++------- modules/axabanque/pages/wealth.py | 83 +++++++++++-------------------- 2 files changed, 53 insertions(+), 72 deletions(-) diff --git a/modules/axabanque/browser.py b/modules/axabanque/browser.py index e9da2c8d3d..c19a0c75d9 100644 --- a/modules/axabanque/browser.py +++ b/modules/axabanque/browser.py @@ -42,7 +42,9 @@ AccountsPage as BankAccountsPage, CBTransactionsPage, TransactionsPage, UnavailablePage, IbanPage, LifeInsuranceIframe, BoursePage, BankProfilePage, ) -from .pages.wealth import AccountsPage as WealthAccountsPage, InvestmentPage, HistoryPage, ProfilePage +from .pages.wealth import ( + AccountsPage as WealthAccountsPage, InvestmentPage, HistoryPage, ProfilePage, AccountDetailsPage, +) from .pages.transfer import ( RecipientsPage, AddRecipientPage, ValidateTransferPage, RegisterTransferPage, ConfirmTransferPage, RecipientConfirmationPage, @@ -506,13 +508,14 @@ def get_profile(self): class AXAAssurance(AXABrowser): BASEURL = 'https://espaceclient.axa.fr' - accounts = URL('/accueil.html', WealthAccountsPage) - investment = URL('/content/ecc-popin-cards/savings/[^/]+/repartition', InvestmentPage) - history = URL('.*accueil/savings/(\w+)/contract', - 'https://espaceclient.axa.fr/#', HistoryPage) - documents = URL('https://espaceclient.axa.fr/content/espace-client/accueil/mes-documents/attestations-d-assurances.content-inner.din_CERTIFICATE.html', DocumentsPage) - download = URL('/content/ecc-popin-cards/technical/detailed/document.downloadPdf.html', - '/content/ecc-popin-cards/technical/detailed/document/_jcr_content/', + accounts = URL(r'/accueil.html', WealthAccountsPage) + account_details = URL('.*accueil/savings/(\w+)/contract', + r'https://espaceclient.axa.fr/#', AccountDetailsPage) + investment = URL(r'/content/ecc-popin-cards/savings/[^/]+/repartition', InvestmentPage) + history = URL(r'/content/ecc-popin-cards/savings/savings/postsales.mawGetPostSalesOperations.json', HistoryPage) + documents = URL(r'https://espaceclient.axa.fr/content/espace-client/accueil/mes-documents/attestations-d-assurances.content-inner.din_CERTIFICATE.html', DocumentsPage) + download = URL(r'/content/ecc-popin-cards/technical/detailed/document.downloadPdf.html', + r'/content/ecc-popin-cards/technical/detailed/document/_jcr_content/', DownloadPage) profile = URL(r'/content/ecc-popin-cards/transverse/userprofile.content-inner.html\?_=\d+', ProfilePage) @@ -561,18 +564,21 @@ def iter_investment(self, account): @need_login def iter_history(self, account): - self.go_wealth_pages(account) - pagination_url = self.page.get_pagination_url() - try: - self.location(pagination_url, params={'skip': 0}) - except ClientError as e: - assert e.response.status_code == 406 - self.logger.info('not doing pagination for account %r, site seems broken', account) - for tr in self.page.iter_history(no_pagination=True): - yield tr + ''' There is now an API for the accounts history, however transactions are not + sorted by date in the JSON. The website fetches 5 years of history maximum. + For some accounts, the access to the transactions JSON is not available yet. ''' + params = { + 'startDate': (date.today() - relativedelta(years=2)).year, + 'endDate': date.today().year, + 'pid': account.id, + } + self.history.go(params=params) + error_code = self.page.get_error_code() + if error_code: + self.logger.warning('Error when trying to access the history JSON, history will be skipped for this account.') return - for tr in self.page.iter_history(): + for tr in sorted_transactions(self.page.iter_history()): yield tr def iter_coming(self, account): diff --git a/modules/axabanque/pages/wealth.py b/modules/axabanque/pages/wealth.py index 0eb6039322..b607fc85de 100644 --- a/modules/axabanque/pages/wealth.py +++ b/modules/axabanque/pages/wealth.py @@ -17,28 +17,29 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . +from __future__ import unicode_literals import re -from weboob.browser.pages import HTMLPage, LoggedPage, pagination -from weboob.browser.elements import ListElement, ItemElement, method, TableElement +from decimal import Decimal +from weboob.browser.pages import HTMLPage, LoggedPage, JsonPage +from weboob.browser.elements import ListElement, DictElement, ItemElement, method, TableElement from weboob.browser.filters.standard import ( - Async, AsyncLoad, CleanDecimal, CleanText, Currency, Date, Eval, Field, Lower, MapIn, QueryValue, Regexp, + CleanDecimal, CleanText, Currency, Date, Eval, Field, Lower, MapIn, QueryValue, Regexp, ) from weboob.browser.filters.html import Attr, Link, TableCell +from weboob.browser.filters.json import Dict from weboob.capabilities.bank import Account, Investment from weboob.capabilities.profile import Person from weboob.capabilities.base import NotAvailable, NotLoaded from weboob.tools.capabilities.bank.transactions import FrenchTransaction -def MyDecimal(*args, **kwargs): - kwargs.update(replace_dots=True, default=NotAvailable) - return CleanDecimal(*args, **kwargs) +def float_to_decimal(f): + return Decimal(str(f)) class AccountsPage(LoggedPage, HTMLPage): - @method class iter_accounts(ListElement): item_xpath = '//div[contains(@data-route, "/savings/")]' @@ -57,8 +58,8 @@ class item(ItemElement): obj_id = Regexp(CleanText('.//span[has-class("small-title")]'), '(\d+)') obj_label = CleanText('.//h3[has-class("card-title")]') - obj_balance = MyDecimal('.//p[has-class("amount-card")]') - obj_valuation_diff = MyDecimal('.//p[@class="performance"]') + obj_balance = CleanDecimal.French('.//p[has-class("amount-card")]') + obj_valuation_diff = CleanDecimal.French('.//p[@class="performance"]', default=NotAvailable) def obj_url(self): url = Attr('.', 'data-route')(self) @@ -154,66 +155,40 @@ def is_detail(self): return bool(self.doc.xpath(u'//th[contains(text(), "Valeur de la part")]')) -class Transaction(FrenchTransaction): - PATTERNS = [(re.compile(u'^(?Psouscription.*)'), FrenchTransaction.TYPE_DEPOSIT), - (re.compile(u'^(?P.*)'), FrenchTransaction.TYPE_BANK), - ] - - -class HistoryPage(LoggedPage, HTMLPage): - def build_doc(self, content): - # we got empty pages at end of pagination - if not content.strip(): - content = b"" - return super(HistoryPage, self).build_doc(content) - +class AccountDetailsPage(LoggedPage, HTMLPage): def get_account_url(self, url): - return Attr(u'//a[@href="%s"]' % url, 'data-target')(self.doc) + return Attr('//a[@href="%s"]' % url, 'data-target')(self.doc) def get_investment_url(self): return Attr('//div[has-class("card-distribution")]', 'data-url', default=None)(self.doc) - def get_pagination_url(self): - return Attr('//div[contains(@class, "default")][@data-module-card-list--current-page]', 'data-module-card-list--url')(self.doc) - - @method - class get_investments(ListElement): - item_xpath = '//div[@class="white-bg"][.//strong[contains(text(), "support")]]/following-sibling::div' - class item(ItemElement): - klass = Investment +class Transaction(FrenchTransaction): + PATTERNS = [ + (re.compile('^(?Psouscription.*)'), FrenchTransaction.TYPE_DEPOSIT), + (re.compile('^(?P.*)'), FrenchTransaction.TYPE_BANK), + ] - obj_label = CleanText('.//div[has-class("t-data__label")]') - obj_valuation = MyDecimal('.//div[has-class("t-data__amount") and has-class("desktop")]') - obj_portfolio_share = Eval(lambda x: x / 100, CleanDecimal('.//div[has-class("t-data__amount_label")]')) - @pagination +class HistoryPage(LoggedPage, JsonPage): @method - class iter_history(ListElement): - item_xpath = '//div[contains(@data-url, "savingsdetailledcard")]' - - def next_page(self): - if not CleanText(self.item_xpath, default=None)(self): - return - elif self.env.get('no_pagination'): - return - - return re.sub(r'(?<=\bskip=)(\d+)', lambda m: str(int(m.group(1)) + 10), self.page.url) + class iter_history(DictElement): class item(ItemElement): klass = Transaction - load_details = Attr('.', 'data-url') & AsyncLoad + obj_raw = Transaction.Raw(Dict('label')) + obj_date = Date(Dict('date')) + obj_amount = Eval(float_to_decimal, Dict('gross_amount/value')) - obj_raw = Transaction.Raw('.//div[has-class("desktop")]//em') - obj_date = Date(CleanText('.//div[has-class("t-data__date") and has-class("desktop")]'), dayfirst=True) - obj_amount = MyDecimal('.//div[has-class("t-data__amount") and has-class("desktop")]') + def validate(self, obj): + return CleanText(Dict('status'))(self) == 'DONE' - def obj_investments(self): - investments = list(Async('details').loaded_page(self).get_investments()) - for inv in investments: - inv.vdate = Field('date')(self) - return investments + def get_error_code(self): + # The server returns a list if it worked and a dict in case of error + if isinstance(self.doc, dict) and 'return' in self.doc: + return self.doc['return']['error']['code'] + return None class ProfilePage(LoggedPage, HTMLPage): -- GitLab