From 0d5e1a62508aeb585744d9e98ab335263c9e581b Mon Sep 17 00:00:00 2001 From: Quentin Defenouillere Date: Thu, 21 Feb 2019 16:39:11 +0100 Subject: [PATCH] [cragr] Retry request when main account balance is unavailable This commit fixes several bugs in iter_accounts, linked to untyped accounts and unavailable main account balances. I factorized the try/except when trying to go the an accounts space. I also added a try.except on the "cards" requests because it often returns a 400 error that crashes the whole connection. The count_spaces xpath was corrected to fit specific professional spaces too. Closes: 35460@sibi --- modules/cragr/api/browser.py | 67 ++++++++++++++++++++++++------------ modules/cragr/api/pages.py | 32 +++++++++++++---- 2 files changed, 70 insertions(+), 29 deletions(-) diff --git a/modules/cragr/api/browser.py b/modules/cragr/api/browser.py index cee467c139..c0bbaf287f 100644 --- a/modules/cragr/api/browser.py +++ b/modules/cragr/api/browser.py @@ -27,7 +27,7 @@ from weboob.capabilities.base import empty, NotAvailable, strict_find_object from weboob.browser import LoginBrowser, URL, need_login from weboob.exceptions import BrowserUnavailable, BrowserIncorrectPassword, ActionNeeded -from weboob.browser.exceptions import ServerError, BrowserHTTPNotFound +from weboob.browser.exceptions import ServerError, ClientError, BrowserHTTPNotFound from weboob.capabilities.bank import Loan from weboob.tools.capabilities.bank.iban import is_iban_valid from weboob.tools.capabilities.bank.transactions import sorted_transactions @@ -205,6 +205,20 @@ def get_security_form(self): form = self.page.get_login_form(self.username, keypad_password, keypad_id) return form + @need_login + def check_space_connection(self, contract): + # Going to a specific space often returns a 500 error + # so we might have to retry several times. + try: + self.go_to_account_space(contract) + except ServerError: + self.logger.warning('Server returned error 500 when trying to access space %s, we try again' % contract) + try: + self.go_to_account_space(contract) + except ServerError: + return False + return True + @need_login def get_accounts_list(self): # Determine how many spaces are present on the connection: @@ -221,19 +235,17 @@ def get_accounts_list(self): deferred_cards = {} for contract in range(total_spaces): - # This request often returns a 500 error so we retry several times. - try: - self.go_to_account_space(contract) - except ServerError: - self.logger.warning('Server returned error 500 when trying to access space %s, we try again' % contract) - try: - self.go_to_account_space(contract) - except ServerError: - self.logger.warning('Server returned error 500 twice when trying to access space %s, this space will be skipped' % contract) - continue - + if not self.check_space_connection(contract): + self.logger.warning('Server returned error 500 twice when trying to access space %s, this space will be skipped' % contract) + continue # The main account is not located at the same place in the JSON. main_account = self.page.get_main_account() + if main_account.balance == NotAvailable: + self.check_space_connection(contract) + main_account = self.page.get_main_account() + if main_account.balance == NotAvailable: + self.logger.warning('Could not fetch the balance for main account %s.' % main_account.id) + main_account.owner_type = self.page.get_owner_type() main_account._contract = contract @@ -301,16 +313,27 @@ def get_accounts_list(self): yield account # Fetch all deferred credit cards for this space - self.cards.go() - for card in self.page.iter_card_parents(): - card.number = card.id - card.parent = all_accounts.get(card._parent_id, NotAvailable) - card.currency = card.parent.currency - card.owner_type = card.parent.owner_type - card._category = card.parent._category - card._contract = contract - if card.id not in deferred_cards: - deferred_cards[card.id] = card + # Once again, this request tends to crash often. + try: + self.cards.go() + except ClientError: + self.logger.warning('Request to cards failed, we try again') + try: + self.check_space_connection(contract) + self.cards.go() + except ClientError: + self.logger.warning('Request to cards failed twice, cards of this space will be skipped.') + + if self.cards.is_here(): + for card in self.page.iter_card_parents(): + card.number = card.id + card.parent = all_accounts.get(card._parent_id, NotAvailable) + card.currency = card.parent.currency + card.owner_type = card.parent.owner_type + card._category = card.parent._category + card._contract = contract + if card.id not in deferred_cards: + deferred_cards[card.id] = card # We must check if cards are unique on their parent account; # if not, we cannot retrieve their summaries in iter_history. diff --git a/modules/cragr/api/pages.py b/modules/cragr/api/pages.py index 9f67c20745..31ad84d0b7 100644 --- a/modules/cragr/api/pages.py +++ b/modules/cragr/api/pages.py @@ -27,6 +27,7 @@ from weboob.browser.pages import HTMLPage, JsonPage, LoggedPage from weboob.exceptions import ActionNeeded from weboob.capabilities import NotAvailable +from weboob.capabilities.base import empty from weboob.capabilities.bank import ( Account, AccountOwnerType, Transaction, Investment, ) @@ -159,9 +160,13 @@ def build_doc(self, content): return d.raw_decode(raw)[0] def count_spaces(self): - # The total number of spaces corresponds to the number - # of available space choices plus the one we are on now: - return len(self.html_doc.xpath('//div[@class="HubAccounts-content"]/a')) + 1 + ''' The total number of spaces corresponds to the number + of available space choices plus the one we are on now. + Some professional connections have a very specific xpath + so we must look for nodes with 'idBamIndex' as well as + "HubAccounts-link--cael" otherwise there might be space duplicates.''' + return len(self.html_doc.xpath('//a[contains(@class, "HubAccounts-link--cael") and contains(@href, "idBamIndex=")]')) + 1 + def get_owner_type(self): OWNER_TYPES = { @@ -185,7 +190,13 @@ class get_main_account(ItemElement): obj_id = CleanText(Dict('comptePrincipal/numeroCompte')) obj_number = CleanText(Dict('comptePrincipal/numeroCompte')) obj_label = CleanText(Dict('comptePrincipal/libelleProduit')) - obj_balance = Eval(float_to_decimal, Dict('comptePrincipal/solde')) + + def obj_balance(self): + balance = Dict('comptePrincipal/solde', default=NotAvailable)(self) + if not empty(balance): + return Eval(float_to_decimal, balance)(self) + return NotAvailable + obj_currency = CleanCurrency(Dict('comptePrincipal/idDevise')) obj__index = Dict('comptePrincipal/index') obj__category = Dict('comptePrincipal/grandeFamilleProduitCode', default=None) @@ -204,7 +215,7 @@ class iter_accounts(DictElement): item_xpath = 'grandesFamilles/*/elementsContrats' class item(ItemElement): - IGNORED_ACCOUNTS = ("MES ASSURANCES",) + IGNORED_ACCOUNTS = ('MES ASSURANCES', 'VOS ASSURANCES',) klass = Account @@ -253,7 +264,14 @@ def get_account_balances(self): # Insurances have no balance, we skip them if el.get('typeProduit') == 'assurance': continue - value = el.get('solde', el.get('encoursActuel', el.get('valorisationContrat', el.get('montantRestantDu', el.get('capitalDisponible', el.get('montantUtilise')))))) + value = el.get('solde', + el.get('encoursActuel', + el.get('valorisationContrat', + el.get('montantRestantDu', + el.get('capitalDisponible', + el.get('montantUtilise', + el.get('montantPlafondAutorise'))))))) + if value is None: continue account_balances[Dict('idElementContrat')(el)] = float_to_decimal(value) @@ -343,7 +361,7 @@ def obj_id(self): def condition(self): assert CleanText(Dict('codeTypeDebitPaiementCarte'))(self) in ('D', 'I') - return CleanText(Dict('codeTypeDebitPaiementCarte'))(self)=='D' + return CleanText(Dict('codeTypeDebitPaiementCarte'))(self) == 'D' obj_label = Format('Carte %s %s', Field('id'), CleanText(Dict('titulaire'))) obj_type = Account.TYPE_CARD -- GitLab