Commit 2eb9f572 authored by Quentin Defenouillere's avatar Quentin Defenouillere Committed by Romain Bignon

[spirica] Remove iter_transactions_investments

Going to each transactions details to get its investments leads to too
many requests and since we have been blocked in prod we decided to
reduce the number of requests to iter_accounts, iter_investments and
iter_history.
parent 4750be35
......@@ -21,7 +21,6 @@ from __future__ import unicode_literals
from weboob.browser import LoginBrowser, URL, need_login
from weboob.exceptions import BrowserIncorrectPassword
from weboob.browser.exceptions import ClientError, ServerError
from .pages import LoginPage, AccountsPage, DetailsPage, MaintenancePage
......@@ -76,28 +75,6 @@ class SpiricaBrowser(LoginBrowser):
# Store new transaction_page after login:
self.transaction_page = self.page
@need_login
def get_transactions_with_investments(self, max_count, url):
transactions = []
for index, transaction in enumerate(self.page.iter_history()):
self.check_if_logged_in(url)
if index < max_count:
try:
self.transaction_page.go_investments_form(transaction._index)
except (ClientError, ServerError) as e:
self.logger.warning(e)
# Check if we are logged out
if self.login.is_here():
self.check_if_logged_in(url)
if self.details.is_here():
transaction.investments = []
for inv in self.page.iter_transactions_investments():
# Only keep investments that have at least a label and a valuation:
if inv.label and inv.valuation:
transaction.investments.append(inv)
transactions.append(transaction)
return transactions
@need_login
def iter_history(self, account):
self.location(account.url)
......@@ -106,23 +83,7 @@ class SpiricaBrowser(LoginBrowser):
# Determining the number of transaction pages:
total_pages = int(self.page.count_transactions()) // 100
# Scraping transactions with their investments for the 20 first transactions.
# Sometimes go_historyall fails so we go back to the accounts page and retry.
if self.transaction_page.go_historyall(page_number=0):
for tr in self.get_transactions_with_investments(20, account.url):
yield tr
else:
self.logger.warning('The first go_historyall() failed, go back to account details and retry.')
self.location(account.url)
self.page.go_historytab()
self.transaction_page = self.page
if self.transaction_page.go_historyall(page_number=0):
for tr in self.get_transactions_with_investments(20, account.url):
yield tr
# Scraping other transaction pages without their investments:
for page_number in range(1, total_pages + 1):
for page_number in range(total_pages + 1):
self.check_if_logged_in(account.url)
if not self.transaction_page.go_historyall(page_number):
self.logger.warning('The first go_historyall() failed, go back to account details and retry.')
......
......@@ -244,20 +244,6 @@ class DetailsPage(LoggedPage, HTMLPage):
form.submit()
return True
def go_investments_form(self, index):
form = self.get_form(xpath='//form[contains(@id, "ongletHistoOperations:ongletHistoriqueOperations")]')
form['javax.faces.behavior.event'] = 'rowToggle'
form['javax.faces.partial.event'] = 'rowToggle'
id_ = Attr('//div[contains(@id, "ongletHistoOperations:ongletHistoriqueOperations")][has-class("listeAvecDetail")]', 'id')(self.doc)
form['javax.faces.source'] = id_
form['javax.faces.partial.execute'] = id_
form['javax.faces.partial.render'] = id_ + ':detail ' + id_
form[id_ + '_rowExpansion'] = 'true'
form[id_ + '_encodeFeature'] = 'true'
form[id_ + '_expandedRowIndex'] = index
form.submit()
@method
class iter_history(ListElement):
item_xpath = '//tr[@role="row"]'
......@@ -287,31 +273,3 @@ class DetailsPage(LoggedPage, HTMLPage):
and "Arrêté annuel" not in Field('label')(self)
and "Fusion-absorption" not in Field('label')(self)
)
@method
class iter_transactions_investments(TableInvestment):
item_xpath = '//table[thead[.//span[text()="ISIN"]]]/tbody/tr'
head_xpath = '//thead[.//span[text()="ISIN"]]//th'
col_isin = 'ISIN'
col_valuation = 'Montant net'
col_portfolio_share = '%'
class item(ItemElement):
klass = Investment
# Columns do not always appear depending on transactions so we need
# to precise "default=NotAvailable" for all TableCell filters.
obj_label = CleanText(TableCell('label', default=NotAvailable), default=NotAvailable)
obj_vdate = Date(CleanText(TableCell('vdate', default="")), dayfirst=True, default=NotAvailable)
obj_unitvalue = MyDecimal(TableCell('unitvalue', default=NotAvailable), default=NotAvailable)
obj_quantity = MyDecimal(TableCell('quantity', default=NotAvailable), default=NotAvailable)
obj_valuation = MyDecimal(TableCell('valuation', default=NotAvailable), default=NotAvailable)
obj_portfolio_share = MyDecimal(TableCell('portfolio_share', default=NotAvailable), default=NotAvailable)
def obj_code(self):
code = CleanText(TableCell('isin', default=NotAvailable), default=NotAvailable)(self)
return code if code != '-' else NotAvailable
def obj_code_type(self):
return Investment.CODE_TYPE_ISIN if Field('code')(self) else NotAvailable
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment