From 30cbc62b9794c558776cba6f422559c1d99e1aed Mon Sep 17 00:00:00 2001
From: Dorian ROLY <dorian.roly@budget-insight.com>
Date: Wed, 6 May 2020 12:05:50 +0200
Subject: [PATCH] [cmso] Change the way we handle duplicates

Previously we were filtering the transactions by id "ClefDomirama". But since some transactions got no ids,
we were filtering all the transactions with no ids too. Now with "OperationId" we can filter each transactions on a unique
session ("OperationID" change between two sessions).
---
 modules/cmso/par/browser.py | 13 +++++++------
 modules/cmso/par/pages.py   | 11 ++++++-----
 2 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/modules/cmso/par/browser.py b/modules/cmso/par/browser.py
index 334dac5674..868675f949 100644
--- a/modules/cmso/par/browser.py
+++ b/modules/cmso/par/browser.py
@@ -343,29 +343,30 @@ def iter_history(self, account):
             return sorted_transactions(self.page.iter_history())
 
         # Getting a year of history
-        nbs = ["UN", "DEUX", "TROIS", "QUATRE", "CINQ", "SIX", "SEPT", "HUIT", "NEUF", "DIX", "ONZE", "DOUZE"]
+        # We have to finish by "SIX_DERNIERES_SEMAINES" to get in priority the transactions with ids.
+        # In "SIX_DERNIERES_SEMAINES" you can have duplicates transactions without ids of the previous two months.
+        nbs = ["DEUX", "TROIS", "QUATRE", "CINQ", "SIX", "SEPT", "HUIT", "NEUF", "DIX", "ONZE", "DOUZE", "SIX_DERNIERES_SEMAINES"]
         trs = []
 
         self.history.go(data=json.dumps({"index": account._index}), page="pendingListOperations", headers=self.json_headers)
 
         has_deferred_cards = self.page.has_deferred_cards()
 
-        self.history.go(data=json.dumps({'index': account._index}), page="detailcompte", headers=self.json_headers)
-
+        self.history.go(data=json.dumps({'index': account._index, 'filtreOperationsComptabilisees': "MOIS_MOINS_UN"}), page="detailcompte", headers=self.json_headers)
         self.trs = set()
 
         for tr in self.page.iter_history(index=account._index, nbs=nbs):
             # Check for duplicates
-            if tr.id in self.trs:
+            if tr._operationid in self.trs:
                 continue
-            self.trs.add(tr.id)
+            self.trs.add(tr._operationid)
             if has_deferred_cards and tr.type == Transaction.TYPE_CARD:
                 tr.type = Transaction.TYPE_DEFERRED_CARD
                 tr.bdate = tr.rdate
 
             trs.append(tr)
 
-        return trs
+        return sorted_transactions(trs)
 
     @retry((ClientError, ServerError))
     @need_login
diff --git a/modules/cmso/par/pages.py b/modules/cmso/par/pages.py
index 9edf44c6cc..e6deabc79a 100644
--- a/modules/cmso/par/pages.py
+++ b/modules/cmso/par/pages.py
@@ -392,9 +392,9 @@ def get_keys(self):
     class iter_history(DictElement):
         def next_page(self):
             if len(Env('nbs', default=[])(self)):
-                data = {'index': Env('index')(self),
-                        'filtreOperationsComptabilisees': "MOIS_MOINS_%s" % Env('nbs')(self)[0]
-                       }
+                data = {'index': Env('index')(self)}
+                if Env('nbs')(self)[0] != "SIX_DERNIERES_SEMAINES":
+                    data.update({'filtreOperationsComptabilisees': "MOIS_MOINS_%s" % Env('nbs')(self)[0]})
                 Env('nbs')(self).pop(0)
                 return requests.Request('POST', data=json.dumps(data), headers={'Content-Type': 'application/json'})
 
@@ -432,9 +432,10 @@ def filter(self, timestamp):
             obj_raw = Transaction.Raw(Dict('libelleCourt'))
             obj_vdate = Date(Dict('dateValeur', NotAvailable), dayfirst=True, default=NotAvailable)
             obj_amount = CleanDecimal(Dict('montantEnEuro'), default=NotAvailable)
-            # DO NOT USE `OperationID` the ids aren't constant after 1 month. `clefDomirama` seems
-            # to be constant forever. Must be kept under watch though
             obj_id = Dict('clefDomirama', default='')
+            # 'operationId' is different between each session, we use it to avoid duplicates on a unique
+            # session
+            obj__operationid = Dict('operationId', default='')
 
             def parse(self, el):
                 key = Env('key', default=None)(self)
-- 
GitLab