From 2eb9f5720b7b51c134e136a18d46f2b8db83fba9 Mon Sep 17 00:00:00 2001 From: Quentin Defenouillere Date: Fri, 17 May 2019 16:14:39 +0200 Subject: [PATCH] [spirica] Remove iter_transactions_investments Going to each transactions details to get its investments leads to too many requests and since we have been blocked in prod we decided to reduce the number of requests to iter_accounts, iter_investments and iter_history. --- modules/spirica/browser.py | 41 +------------------------------------ modules/spirica/pages.py | 42 -------------------------------------- 2 files changed, 1 insertion(+), 82 deletions(-) diff --git a/modules/spirica/browser.py b/modules/spirica/browser.py index 351c1d53e2..cdd06a6e36 100644 --- a/modules/spirica/browser.py +++ b/modules/spirica/browser.py @@ -21,7 +21,6 @@ from weboob.browser import LoginBrowser, URL, need_login from weboob.exceptions import BrowserIncorrectPassword -from weboob.browser.exceptions import ClientError, ServerError from .pages import LoginPage, AccountsPage, DetailsPage, MaintenancePage @@ -76,28 +75,6 @@ def check_if_logged_in(self, url): # Store new transaction_page after login: self.transaction_page = self.page - @need_login - def get_transactions_with_investments(self, max_count, url): - transactions = [] - for index, transaction in enumerate(self.page.iter_history()): - self.check_if_logged_in(url) - if index < max_count: - try: - self.transaction_page.go_investments_form(transaction._index) - except (ClientError, ServerError) as e: - self.logger.warning(e) - # Check if we are logged out - if self.login.is_here(): - self.check_if_logged_in(url) - if self.details.is_here(): - transaction.investments = [] - for inv in self.page.iter_transactions_investments(): - # Only keep investments that have at least a label and a valuation: - if inv.label and inv.valuation: - transaction.investments.append(inv) - transactions.append(transaction) - return transactions - @need_login def iter_history(self, account): self.location(account.url) @@ -106,23 +83,7 @@ def iter_history(self, account): # Determining the number of transaction pages: total_pages = int(self.page.count_transactions()) // 100 - - # Scraping transactions with their investments for the 20 first transactions. - # Sometimes go_historyall fails so we go back to the accounts page and retry. - if self.transaction_page.go_historyall(page_number=0): - for tr in self.get_transactions_with_investments(20, account.url): - yield tr - else: - self.logger.warning('The first go_historyall() failed, go back to account details and retry.') - self.location(account.url) - self.page.go_historytab() - self.transaction_page = self.page - if self.transaction_page.go_historyall(page_number=0): - for tr in self.get_transactions_with_investments(20, account.url): - yield tr - - # Scraping other transaction pages without their investments: - for page_number in range(1, total_pages + 1): + for page_number in range(total_pages + 1): self.check_if_logged_in(account.url) if not self.transaction_page.go_historyall(page_number): self.logger.warning('The first go_historyall() failed, go back to account details and retry.') diff --git a/modules/spirica/pages.py b/modules/spirica/pages.py index 3dc94ecf7d..adc60cca5c 100644 --- a/modules/spirica/pages.py +++ b/modules/spirica/pages.py @@ -244,20 +244,6 @@ def go_historyall(self, page_number): form.submit() return True - def go_investments_form(self, index): - form = self.get_form(xpath='//form[contains(@id, "ongletHistoOperations:ongletHistoriqueOperations")]') - form['javax.faces.behavior.event'] = 'rowToggle' - form['javax.faces.partial.event'] = 'rowToggle' - id_ = Attr('//div[contains(@id, "ongletHistoOperations:ongletHistoriqueOperations")][has-class("listeAvecDetail")]', 'id')(self.doc) - form['javax.faces.source'] = id_ - form['javax.faces.partial.execute'] = id_ - form['javax.faces.partial.render'] = id_ + ':detail ' + id_ - form[id_ + '_rowExpansion'] = 'true' - form[id_ + '_encodeFeature'] = 'true' - form[id_ + '_expandedRowIndex'] = index - form.submit() - - @method class iter_history(ListElement): item_xpath = '//tr[@role="row"]' @@ -287,31 +273,3 @@ def condition(self): and "Arrêté annuel" not in Field('label')(self) and "Fusion-absorption" not in Field('label')(self) ) - - @method - class iter_transactions_investments(TableInvestment): - item_xpath = '//table[thead[.//span[text()="ISIN"]]]/tbody/tr' - head_xpath = '//thead[.//span[text()="ISIN"]]//th' - - col_isin = 'ISIN' - col_valuation = 'Montant net' - col_portfolio_share = '%' - - class item(ItemElement): - klass = Investment - - # Columns do not always appear depending on transactions so we need - # to precise "default=NotAvailable" for all TableCell filters. - obj_label = CleanText(TableCell('label', default=NotAvailable), default=NotAvailable) - obj_vdate = Date(CleanText(TableCell('vdate', default="")), dayfirst=True, default=NotAvailable) - obj_unitvalue = MyDecimal(TableCell('unitvalue', default=NotAvailable), default=NotAvailable) - obj_quantity = MyDecimal(TableCell('quantity', default=NotAvailable), default=NotAvailable) - obj_valuation = MyDecimal(TableCell('valuation', default=NotAvailable), default=NotAvailable) - obj_portfolio_share = MyDecimal(TableCell('portfolio_share', default=NotAvailable), default=NotAvailable) - - def obj_code(self): - code = CleanText(TableCell('isin', default=NotAvailable), default=NotAvailable)(self) - return code if code != '-' else NotAvailable - - def obj_code_type(self): - return Investment.CODE_TYPE_ISIN if Field('code')(self) else NotAvailable -- GitLab