From 2477826ee7032f73eeba0d7e91ebed489b64e6bb Mon Sep 17 00:00:00 2001 From: Vincent Ardisson Date: Fri, 6 Apr 2018 16:01:55 +0200 Subject: [PATCH] [caissedepargne] cenet: split deferred debit card accounts Deferred debit card are now exported as separate accounts, and the deferred debit transactions are on those accounts, not on the checking account anymore. --- modules/caissedepargne/cenet/browser.py | 120 ++++++++++++++---------- modules/caissedepargne/cenet/pages.py | 55 ++++++++--- 2 files changed, 112 insertions(+), 63 deletions(-) diff --git a/modules/caissedepargne/cenet/browser.py b/modules/caissedepargne/cenet/browser.py index b4ae70c8d5..055c52e4bc 100644 --- a/modules/caissedepargne/cenet/browser.py +++ b/modules/caissedepargne/cenet/browser.py @@ -20,6 +20,8 @@ from __future__ import unicode_literals +from collections import Counter +from fnmatch import fnmatch import json from weboob.browser import LoginBrowser, need_login, StatesMixin @@ -28,7 +30,9 @@ from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable from weboob.capabilities.base import find_object from weboob.capabilities.bank import Account -from weboob.tools.capabilities.bank.transactions import sorted_transactions, FrenchTransaction +from weboob.tools.capabilities.bank.transactions import ( + sorted_transactions, omit_deferred_transactions, keep_only_card_transactions, +) from .pages import ( ErrorPage, @@ -134,6 +138,11 @@ def do_login(self): @need_login def get_accounts_list(self): if self.accounts is None: + headers = { + 'Content-Type': 'application/json; charset=UTF-8', + 'Accept': 'application/json, text/javascript, */*; q=0.01' + } + data = { 'contexte': '', 'dateEntree': None, @@ -142,35 +151,56 @@ def get_accounts_list(self): } try: - self.accounts = [account for account in self.cenet_accounts.go(json=data).get_accounts()] + accounts = [account for account in self.cenet_accounts.go(data=json.dumps(data), headers=headers).get_accounts()] except ClientError: # Unauthorized due to wrongpass raise BrowserIncorrectPassword() + + try: + self.cenet_cards.go(data=json.dumps(data), headers=headers) + except BrowserUnavailable: + # for some accounts, the site can throw us an error, during weeks + self.logger.warning('ignoring cards because site is unavailable...') + else: + cards = list(self.page.iter_cards()) + redacted_ids = Counter(card.id[:4] + card.id[-6:] for card in cards) + for id in redacted_ids: + assert redacted_ids[id] == 1, 'there are several cards with the same %r' % id + + for card in cards: + card.parent = find_object(accounts, id=card._parent_id) + assert card.parent, 'no parent account found for card' + accounts.extend(cards) + self.cenet_loans.go(json=data) for account in self.page.get_accounts(): - self.accounts.append(account) - for account in self.accounts: - try: - account._cards = [] - self.cenet_cards.go(json=data) - - for card in self.page.get_cards(): - if card['Compte']['Numero'] == account.id: - account._cards.append(card) - except BrowserUnavailable: - # for some accounts, the site can throw us an error, during weeks - self.logger.warning('ignoring cards because site is unavailable...') - account._cards = [] - - return iter(self.accounts) + accounts.append(account) + + self.accounts = accounts + + return self.accounts def get_loans_list(self): return [] + def _matches_card(self, tr, full_id): + return fnmatch(full_id, tr.card) + @need_login def get_history(self, account): if account.type == Account.TYPE_LOAN: return [] + + if account.type == account.TYPE_CARD: + def match_cb(tr): + return self._matches_card(tr, account.number) + + hist = self.get_history_base(account.parent, deferred=account.number) + return keep_only_card_transactions(hist, match_cb) + else: + return omit_deferred_transactions(self.get_history_base(account)) + + def get_history_base(self, account, deferred=None): headers = { 'Content-Type': 'application/json; charset=UTF-8', 'Accept': 'application/json, text/javascript, */*; q=0.01' @@ -183,38 +213,31 @@ def get_history(self, account): 'donneesEntree': json.dumps(account._formated), } - items = [] self.cenet_account_history.go(data=json.dumps(data), headers=headers) - # there might be some duplicate transactions regarding the card type ones - # because some requests lead to the same transaction list - # even with different parameters/data in the request - card_tr_list = [] while True: - data_out = self.page.doc['DonneesSortie'] for tr in self.page.get_history(): - items.append(tr) - - if tr.type is FrenchTransaction.TYPE_CARD_SUMMARY: - if find_object(card_tr_list, label=tr.label, amount=tr.amount, raw=tr.raw, date=tr.date, rdate=tr.rdate): - self.logger.warning('Duplicated transaction: %s', tr) - items.pop() + yield tr + if tr.type == tr.TYPE_CARD_SUMMARY and deferred: + assert tr.card, 'card summary has no card number?' + if not self._matches_card(tr, deferred): continue - card_tr_list.append(tr) - tr.deleted = True - tr_dict = [tr_dict2 for tr_dict2 in data_out if tr_dict2['Libelle'] == tr.label] donneesEntree = {} donneesEntree['Compte'] = account._formated - donneesEntree['ListeOperations'] = [tr_dict[0]] + + donneesEntree['ListeOperations'] = [tr._data] deferred_data = { 'contexte': '', 'dateEntree': None, 'donneesEntree': json.dumps(donneesEntree).replace('/', '\\/'), - 'filtreEntree': json.dumps(tr_dict[0]).replace('/', '\\/') + 'filtreEntree': json.dumps(tr._data).replace('/', '\\/') } tr_detail_page = self.cenet_tr_detail.open(data=json.dumps(deferred_data), headers=headers) + + parent_tr = tr for tr in tr_detail_page.get_history(): - items.append(tr) + tr.card = parent_tr.card + yield tr offset = self.page.next_offset() if not offset: @@ -225,12 +248,11 @@ def get_history(self, account): }) self.cenet_account_history.go(data=json.dumps(data), headers=headers) - return sorted_transactions(items) - @need_login def get_coming(self, account): - if account.type == Account.TYPE_LOAN: + if account.type != account.TYPE_CARD: return [] + trs = [] headers = { @@ -238,17 +260,17 @@ def get_coming(self, account): 'Accept': 'application/json, text/javascript, */*; q=0.01' } - for card in account._cards: - if card['CumulEnCours']['Montant']['Valeur'] != 0: - data = { - 'contexte': '', - 'dateEntree': None, - 'donneesEntree': json.dumps(card), - 'filtreEntree': None - } - - for tr in self.cenet_account_coming.go(data=json.dumps(data), headers=headers).get_history(): - trs.append(tr) + data = { + 'contexte': '', + 'dateEntree': None, + 'donneesEntree': json.dumps(account._hist), + 'filtreEntree': None + } + + self.cenet_account_coming.go(data=json.dumps(data), headers=headers) + for tr in self.page.get_history(): + tr.type = tr.TYPE_DEFERRED_CARD + trs.append(tr) return sorted_transactions(trs) diff --git a/modules/caissedepargne/cenet/pages.py b/modules/caissedepargne/cenet/pages.py index 833356eb91..d397517a45 100644 --- a/modules/caissedepargne/cenet/pages.py +++ b/modules/caissedepargne/cenet/pages.py @@ -17,13 +17,17 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . +from copy import deepcopy +from decimal import Decimal import re import json from datetime import datetime from weboob.browser.pages import LoggedPage, HTMLPage, JsonPage from weboob.browser.elements import DictElement, ItemElement, method -from weboob.browser.filters.standard import Date, CleanDecimal, CleanText, Format, Field, Env, Regexp, Currency +from weboob.browser.filters.standard import ( + Date, CleanDecimal, CleanText, Format, Field, Env, Eval, Regexp, Currency, +) from weboob.browser.filters.json import Dict from weboob.capabilities import NotAvailable from weboob.capabilities.bank import Account, Loan @@ -181,24 +185,42 @@ def obj_next_payment_date(self): return NotAvailable +def float_to_debit(f): + return -Decimal(str(f)) + + class CenetCardsPage(LoggedPage, CenetJsonPage): - def get_cards(self): - cards = Dict('DonneesSortie')(self.doc) + @method + class iter_cards(DictElement): + item_xpath = 'DonneesSortie' - # Remove dates to prevent bad parsing - def reword_dates(card): - tmp_card = card + class item(ItemElement): + def condition(self): + assert self.el['Type'] in ('I', 'D') + return self.el['Type'] == 'D' - for k, v in tmp_card.items(): - if isinstance(v, dict): - v = reword_dates(v) - if k == "Date" and v is not None and "Date" in v: - card[k] = None + klass = Account - for card in cards: - reword_dates(card) + obj_id = obj_number = Dict('Numero') # full card number + obj__parent_id = Dict('Compte/Numero') + obj_label = Format('%s%s', Dict('Titulaire/DesignationPersonne'), Dict('Compte/LibelleComplet')) + obj_coming = Eval(float_to_debit, Dict('CumulEnCours/Montant/Valeur')) + obj_balance = 0 + obj_currency = 'EUR' + obj_type = Account.TYPE_CARD + + def obj__hist(self): + def reword_dates(card): + for k, v in card.items(): + if isinstance(v, dict): + v = reword_dates(v) + if k == "Date" and v is not None and "Date" in v: + card[k] = None + + el = deepcopy(self.el) + reword_dates(el) + return el - return cards class CenetAccountHistoryPage(LoggedPage, CenetJsonPage): @@ -256,6 +278,11 @@ def obj_amount(self): return -amount if Dict('Montant/CodeSens')(self) == "D" else amount + def obj__data(self): + return self.el + + obj_card = Regexp(obj_label, r'^CB (\d+\*+\d+)', default=None) + def next_offset(self): offset = Dict('OffsetSortie')(self.doc) if offset: -- GitLab