From 22b54f82624b60a554c89f70d5a0a41ae6086bd0 Mon Sep 17 00:00:00 2001 From: Lucas Ficheux Date: Thu, 4 Jun 2020 13:19:09 +0200 Subject: [PATCH] [sogecartenet] Improved stability of iter_history Sometimes the search for transactions opens a popup (even after limiting the search to one year) so now we close it, reduce the length of the search by one month and retry. Also force the return to the transaction search page after selecting a card because the website sometimes does not take us there automatically. --- modules/sogecartenet/ent_browser.py | 25 ++++++++++++++-- modules/sogecartenet/ent_pages.py | 45 +++++++++++++++++++++++++---- 2 files changed, 63 insertions(+), 7 deletions(-) diff --git a/modules/sogecartenet/ent_browser.py b/modules/sogecartenet/ent_browser.py index 4c91764ee7..cf2db7055d 100644 --- a/modules/sogecartenet/ent_browser.py +++ b/modules/sogecartenet/ent_browser.py @@ -35,7 +35,7 @@ from .ent_pages import ( LoginPage, AccueilPage, AccountsPage, HistoryPage, - AccountsXlsPage, HistoryXlsPage, + AccountsXlsPage, HistoryXlsPage, DeferredQuery, ) @@ -159,7 +159,28 @@ def iter_transactions(self, account, coming=False): self.page.select_account(account) self.selected_account = account.id - if self.page.download_transactions(): + self.page.go_transactions_list_tab() + + index_last_date = self.page.get_limit_date_index() + successful_download = False + is_retry = False + while index_last_date > 0: + try: + successful_download = self.page.download_transactions(index_last_date, retry=is_retry) + except DeferredQuery: + # This is raised when the site displays a popup to ask if we want to confirm a + # deferred query, deferred queries can be avoided by asking for less history + # here we simply retry with the xpath index of the date decreased by 1 essentialy + # decreasing the length of history requested by one month + self.logger.warning('Defered Query detected, retrying') + index_last_date -= 1 + is_retry = True + else: + break + else: + self.logger.warning('Could not get transactions without deferred query for account %s' % self.selected_account) + + if successful_download: file_path = self.retry_find_file_path('requete_cumul', '.xls') assert file_path, 'Could not find the downloaded file' diff --git a/modules/sogecartenet/ent_pages.py b/modules/sogecartenet/ent_pages.py index 1ccf852ba1..87e15fac76 100644 --- a/modules/sogecartenet/ent_pages.py +++ b/modules/sogecartenet/ent_pages.py @@ -40,6 +40,7 @@ from weboob.tools.decorators import retry from weboob.browser.selenium import ( SeleniumPage, VisibleXPath, AnyCondition, AllCondition, + StablePageCondition, ) @@ -148,6 +149,10 @@ def obj_label(self): obj__service_number = CleanText(Dict('Numéro de prestation')) +class DeferredQuery(Exception): + pass + + class HistoryPage(LoggedPage, SeleniumPage): is_here = VisibleXPath('//div[contains(text(), "Recherche opérations")]') @@ -157,7 +162,7 @@ def go_transactions_list_tab(self): self.browser.wait_xpath_clickable('//div[contains(@class, "v-widget")][div[div[@id="BTN_SEARCH"]]]') - def download_transactions(self): + def get_limit_date_index(self): # We suppose we already selected an account. el = self.driver.find_element_by_xpath('//input[contains(@placeholder, "De la date d\'arrêté")]') el.click() @@ -170,14 +175,34 @@ def download_transactions(self): last_date_index = 0 for date in dates_list: displayed_date = Date(CleanText('.'))(date) + delta = relativedelta(today, displayed_date) if ( - relativedelta(today, displayed_date).years >= 1 and - relativedelta(today, displayed_date).months > 0 + delta.years >= 1 and + (delta.months > 0 or delta.days > 0) ): break last_date_index += 1 + return last_date_index + + def is_deferred_query(self): + # If we ask for too much history sometimes the site tells + # us that the query will be answered at a later time and asks to + # confirm, we want to avoid this behaviour + self.browser.wait_until(AnyCondition( + StablePageCondition(), + VisibleXPath('//div[@role="dialog"]/div[@class="popupContent"]') + )) + return 'Votre requête sera traitée en différé' in CleanText('//div[@role="dialog"]/div[@class="popupContent"]')(self.doc) + + def download_transactions(self, last_date_index, retry=False): # Select chosen date + if retry: + el = self.driver.find_element_by_xpath('//input[contains(@placeholder, "De la date d\'arrêté")]') + el.click() + self.browser.wait_xpath_invisible('//p[contains(@class, "Notification-description")][contains(text(), "a bien été sélectionnée")]') + self.browser.wait_xpath_visible('//div[@id="VAADIN_COMBOBOX_OPTIONLIST"]') + el = self.driver.find_element_by_xpath( '//div[@id="VAADIN_COMBOBOX_OPTIONLIST"]//div[contains(@class, "suggestmenu")]//tr[%d]/td' % last_date_index ) @@ -187,6 +212,13 @@ def download_transactions(self): # Submit search for this date self.driver.execute_script("document.getElementById('BTN_SEARCH').click()") + if self.is_deferred_query(): + # Clicking no on the popup + el = self.driver.find_element_by_xpath('//div[@role="dialog"]/div[@class="popupContent"]//div[contains(@class, "button-friendly") and .//span[text()="Non"]]',) + el.click() + self.browser.wait_xpath_invisible('//div[@role="dialog"]/div[@class="popupContent"]') + raise DeferredQuery() + # Get data self.browser.wait_until(AnyCondition( VisibleXPath('//div/a/img'), @@ -233,13 +265,16 @@ def select_account(self, account): el = self.driver.find_element_by_xpath('//div[span[span[text()="Rechercher"]]]') self.click_retry_intercepted(el) - self.browser.wait_xpath_visible('//table[@role="grid"]/tbody') # Get the button of the right card (there might be multiple # card with the same service number) and click it - el = self.driver.find_element_by_xpath('//tbody/tr/td[1][following-sibling::td[contains(text(), "%s")]]' % account._card_number) + el = self.driver.find_element_by_xpath( + '//tbody/tr/td[1][following-sibling::td[contains(text(), "%s")]]//div[contains(@class, "btnGrid-action")]' % account._card_number + ) self.click_retry_intercepted(el) + self.browser.wait_xpath_visible('//p[contains(@class, "Notification-description")][contains(text(), "a bien été sélectionnée")]') + class HistoryXlsPage(LoggedPage, XLSPage): -- GitLab