From 26e5b5a0c61f3180341d083a002a91b39f4e4cd0 Mon Sep 17 00:00:00 2001 From: Damien Mat Date: Thu, 5 Nov 2020 11:03:47 +0100 Subject: [PATCH] [cmso] Fix: fetch missing transactions at the start of a month For now, transactions were fetch on a month by month basis. Each month would display transactions with at least one type of id among 2 possible, and those are collected. Finally, a global history page is parsed, without month param. A special condition would avoid duplicates based on the ids. But at the turn of the month there are transactions that don't have any id, and so would not be selected. Solution: First page fetched is the 'SIX_DERNIERES_SEMAINES'. All transactions from current month are selected here (they don't bear any id). Then the month by month selection is done. No final page without month param is fetched at the end since we already got all transactions. Tested also on CMB and BPE child modules. --- modules/cmso/par/browser.py | 24 ++++++++++++++++-------- modules/cmso/par/pages.py | 11 ++++++++--- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/modules/cmso/par/browser.py b/modules/cmso/par/browser.py index ca67bec0af..e4fd008db9 100644 --- a/modules/cmso/par/browser.py +++ b/modules/cmso/par/browser.py @@ -432,16 +432,25 @@ def iter_history(self, account): finally: self._return_from_market() - # Getting a year of history - # We have to finish by "SIX_DERNIERES_SEMAINES" to get in priority the transactions with ids. - # In "SIX_DERNIERES_SEMAINES" you can have duplicates transactions without ids of the previous two months. - nbs = ["DEUX", "TROIS", "QUATRE", "CINQ", "SIX", "SEPT", "HUIT", "NEUF", "DIX", "ONZE", "DOUZE", "SIX_DERNIERES_SEMAINES"] - trs = [] - self.history.go(json={"index": account._index}, page="pendingListOperations") - has_deferred_cards = self.page.has_deferred_cards() + # 1.fetch the last 6 weeks transactions but keep only the current month ones + # those don't have any id and include 'hier' and 'Plus tôt dans la semaine' + trs = [] + self.history.go( + json={ + 'index': account._index, + 'filtreOperationsComptabilisees': "SIX_DERNIERES_SEMAINES", + }, + page="detailcompte" + ) + for tr in self.page.iter_history(index=account._index, last_trs=True): + trs.append(tr) + + # 2. get the month by month transactions + # and avoid duplicates based on ids + nbs = ["DEUX", "TROIS", "QUATRE", "CINQ", "SIX", "SEPT", "HUIT", "NEUF", "DIX", "ONZE", "DOUZE"] self.history.go( json={ 'index': account._index, @@ -450,7 +459,6 @@ def iter_history(self, account): page="detailcompte" ) self.trs = set() - for tr in self.page.iter_history(index=account._index, nbs=nbs): # Check for duplicates if tr._operationid in self.trs or (tr.id and tr.id in self.trs): diff --git a/modules/cmso/par/pages.py b/modules/cmso/par/pages.py index ce62ecf8e1..b4a955fa07 100644 --- a/modules/cmso/par/pages.py +++ b/modules/cmso/par/pages.py @@ -409,9 +409,8 @@ class iter_history(DictElement): def next_page(self): if len(Env('nbs', default=[])(self)): data = {'index': Env('index')(self)} - if Env('nbs')(self)[0] != "SIX_DERNIERES_SEMAINES": - data.update({'filtreOperationsComptabilisees': "MOIS_MOINS_%s" % Env('nbs')(self)[0]}) - Env('nbs')(self).pop(0) + next_month = Env('nbs')(self).pop(0) + data.update({'filtreOperationsComptabilisees': "MOIS_MOINS_%s" % next_month}) return requests.Request('POST', data=json.dumps(data)) def parse(self, el): @@ -462,6 +461,12 @@ def parse(self, el): break self.obj._deferred_date = self.FromTimestamp().filter(deferred_date) + def validate(self, obj): + if Env('last_trs', default=None)(self): + # keep only current month transactions + return obj.date.month >= datetime.date.today().month + return True + class RedirectInsurancePage(LoggedPage, JsonPage): def get_url(self): -- GitLab