From 87bf55593741342a26d55b0e1e537a772a2785d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?C=C3=A9lande=20Adrien?= <celande.adrien@budget-insight.com>
Date: Thu, 26 Jul 2018 09:42:52 +0200
Subject: [PATCH] [creditcooperatif] duplicate transactions

Looking at the website, you can see that some transactions are duplicates.
So it is normal to have some, since it is what is send.

But sometimes, some requests send have the same transaction list, even if they had different parameters/data.
So there are some more duplicates.
It happens only for the card type transactions.

They are compared to be sure they does not bring duplicates.
It is pretty easy since their labels always have the same format: 'TOT DIF %MONTH%'.
And the duplicates one have the same amount too.
The rest is checked just to be sure.

Closes: 6257@zendesk
---
 modules/caissedepargne/cenet/browser.py | 12 ++++++++++--
 modules/caissedepargne/cenet/pages.py   |  9 ++++++++-
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/modules/caissedepargne/cenet/browser.py b/modules/caissedepargne/cenet/browser.py
index 25173c7f8e..b721b2852e 100644
--- a/modules/caissedepargne/cenet/browser.py
+++ b/modules/caissedepargne/cenet/browser.py
@@ -23,6 +23,7 @@
 from weboob.browser.url import URL
 from weboob.browser.exceptions import ClientError
 from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable
+from weboob.capabilities.base import find_object
 from weboob.tools.capabilities.bank.transactions import sorted_transactions, FrenchTransaction
 
 from .pages import (
@@ -150,12 +151,20 @@ def get_history(self, account):
 
         items = []
         self.cenet_account_history.go(data=json.dumps(data), headers=headers)
+        # there might be some duplicate transactions regarding the card type ones
+        # because some requests lead to the same transaction list
+        # even with different parameters/data in the request
+        card_tr_list = []
         while True:
             data_out = self.page.doc['DonneesSortie']
             for tr in self.page.get_history():
                 items.append(tr)
 
                 if tr.type is FrenchTransaction.TYPE_CARD_SUMMARY:
+                    if find_object(card_tr_list, label=tr.label, amount=tr.amount, raw=tr.raw, date=tr.date, rdate=tr.rdate):
+                        self.logger.warning('Duplicate transaction: %s' % tr)
+                        continue
+                    card_tr_list.append(tr)
                     tr.deleted = True
                     tr_dict = [tr_dict for tr_dict in data_out if tr_dict['Libelle'] == tr.label]
                     donneesEntree = {}
@@ -178,10 +187,9 @@ def get_history(self, account):
             data['filtreEntree'] = json.dumps({
                 'Offset': offset,
             })
-
             self.cenet_account_history.go(data=json.dumps(data), headers=headers)
 
-        return items
+        return sorted_transactions(items)
 
     @need_login
     def get_coming(self, account):
diff --git a/modules/caissedepargne/cenet/pages.py b/modules/caissedepargne/cenet/pages.py
index 7deda862fc..5a62d4dd3f 100644
--- a/modules/caissedepargne/cenet/pages.py
+++ b/modules/caissedepargne/cenet/pages.py
@@ -184,11 +184,18 @@ class get_history(DictElement):
         class item(ItemElement):
             klass = Transaction
 
-            obj_raw = Format('%s %s', Dict('Libelle'), Dict('Libelle2'))
             obj_label = CleanText(Dict('Libelle'))
             obj_date = Date(Dict('DateGroupImputation'), dayfirst=True)
             obj_type = Transaction.TYPE_DEFERRED_CARD
 
+            def obj_raw(self):
+                label = Dict('Libelle')(self)
+                label2 = Dict('Libelle2')(self)
+                if label2 and label2 != 'None':
+                    return '%s %s' % (label, label2)
+                else:
+                    return label
+
             def obj_rdate(self):
                 rdate = re.search('(FACT\s)(\d{6})', Field('label')(self))
                 if rdate.group(2):
-- 
GitLab