From e6b40dcd631eb9ef469a6ad1ca31293235f98b6f Mon Sep 17 00:00:00 2001 From: Maxime Pommier Date: Tue, 19 Feb 2019 13:21:17 +0100 Subject: [PATCH] [caissedepargne] - Separate card deferred from check account --- modules/caissedepargne/browser.py | 113 ++++++++++++--- modules/caissedepargne/pages.py | 223 +++++++++++++++++++++++++++--- 2 files changed, 300 insertions(+), 36 deletions(-) diff --git a/modules/caissedepargne/browser.py b/modules/caissedepargne/browser.py index 1ee0ff09e1..aaf7b33ca6 100644 --- a/modules/caissedepargne/browser.py +++ b/modules/caissedepargne/browser.py @@ -30,13 +30,16 @@ from weboob.browser.switch import SiteSwitch from weboob.browser.url import URL from weboob.capabilities.bank import Account, AddRecipientStep, Recipient, TransferBankError, Transaction, TransferStep -from weboob.capabilities.base import NotAvailable +from weboob.capabilities.base import NotAvailable, find_object from weboob.capabilities.profile import Profile from weboob.browser.exceptions import BrowserHTTPNotFound, ClientError, ServerError from weboob.exceptions import ( BrowserIncorrectPassword, BrowserUnavailable, BrowserHTTPError, BrowserPasswordExpired, ActionNeeded ) -from weboob.tools.capabilities.bank.transactions import sorted_transactions, FrenchTransaction +from weboob.tools.capabilities.bank.transactions import ( + sorted_transactions, FrenchTransaction, keep_only_card_transactions, + omit_deferred_transactions, +) from weboob.tools.capabilities.bank.investments import create_french_liquidity from weboob.tools.compat import urljoin, urlparse from weboob.tools.value import Value @@ -49,7 +52,7 @@ ProTransferSummaryPage, ProAddRecipientOtpPage, ProAddRecipientPage, SmsPage, SmsPageOption, SmsRequest, AuthentPage, RecipientPage, CanceledAuth, CaissedepargneKeyboard, TransactionsDetailsPage, LoadingPage, ConsLoanPage, MeasurePage, NatixisLIHis, NatixisLIInv, NatixisRedirectPage, - SubscriptionPage, CreditCooperatifMarketPage, UnavailablePage, + SubscriptionPage, CreditCooperatifMarketPage, UnavailablePage, CardsPage, CardsComingPage, CardsOldWebsitePage, ) from .linebourse_browser import LinebourseAPIBrowser @@ -81,6 +84,9 @@ class CaisseEpargne(LoginBrowser, StatesMixin): pro_add_recipient_otp = URL('https://.*/Portail.aspx.*', ProAddRecipientOtpPage) pro_add_recipient = URL('https://.*/Portail.aspx.*', ProAddRecipientPage) measure_page = URL('https://.*/Portail.aspx.*', MeasurePage) + cards_old = URL('https://.*/Portail.aspx.*', CardsOldWebsitePage) + cards = URL('https://.*/Portail.aspx.*', CardsPage) + cards_coming = URL('https://.*/Portail.aspx.*', CardsComingPage) authent = URL('https://.*/Portail.aspx.*', AuthentPage) subscription = URL('https://.*/Portail.aspx\?tache=(?P).*', SubscriptionPage) home = URL('https://.*/Portail.aspx.*', IndexPage) @@ -430,6 +436,26 @@ def get_accounts_list(self): else: assert False, "new domain that hasn't been seen so far ?" + self.home.go() + self.page.go_list() + self.page.go_cards() + + if self.cards.is_here() or self.cards_old.is_here(): + cards = list(self.page.iter_cards()) + for card in cards: + card.parent = find_object(self.accounts, number=card._parent_id) + assert card.parent, 'card account %r parent was not found' % card + + # If we are in the new site, we have to get each card coming transaction link. + if self.cards.is_here(): + for card in cards: + info = card.parent._card_links + self.page.go_list() + self.page.go_history(info) + card._coming_info = self.page.get_card_coming_info(card.number, info.copy()) + + self.accounts.extend(cards) + # Some accounts have no available balance or label and cause issues # in the backend so we must exclude them from the accounts list: self.accounts = [account for account in self.accounts if account.label and account.balance != NotAvailable] @@ -475,8 +501,13 @@ def get_loans_list(self): return iter(self.loans) + # For all account, we fill up the history with transaction. For checking account, there will have + # also deferred_card transaction too. + # From this logic, if we send "account_card", that mean we recover all transactions from the parent + # checking account of the account_card, then we filter later the deferred transaction. @need_login - def _get_history(self, info): + def _get_history(self, info, account_card=None): + # Only fetch deferred debit card transactions if `account_card` is not None if isinstance(info['link'], list): info['link'] = info['link'][0] if not info['link'].startswith('HISTORIQUE'): @@ -495,6 +526,11 @@ def _get_history(self, info): if 'netpro' in self.page.url and not self.page.is_history_of(info['id']): self.page.go_history_netpro(info) + # In this case, we want the coming transaction for the new website + # (old website return coming directly in `get_coming()` ) + if account_card and info['type'] == 'HISTORIQUE_CB': + self.page.go_coming(account_card._coming_info['link']) + info['link'] = [info['link']] for i in range(self.HISTORY_MAX_PAGE): @@ -503,14 +539,25 @@ def _get_history(self, info): # list of transactions on account page transactions_list = [] - list_form = [] + card_and_forms = [] for tr in self.page.get_history(): transactions_list.append(tr) if tr.type == tr.TYPE_CARD_SUMMARY: - list_form.append(self.page.get_form_to_detail(tr)) + if account_card: + if self.card_matches(tr.card, account_card.number): + card_and_forms.append((tr.card, self.page.get_form_to_detail(tr))) + else: + self.logger.debug('will skip summary detail (%r) for different card %r', tr, account_card.number) + + # For deferred card history only : + # + # Now that we find transactions that have TYPE_CARD_SUMMARY on the checking account AND the account_card number we want, + # we browse deferred card transactions that are resume by that list of TYPE_CARD_SUMMARY transaction. - # add detail card to list of transactions - for form in list_form: + # Checking account transaction: + # - 01/01 - Summary 5134XXXXXX103 - 900.00€ - TYPE_CARD_SUMMARY <-- We have to go in the form of this tr to get + # cards details transactions. + for card, form in card_and_forms: form.submit() if self.home.is_here() and self.page.is_access_error(): self.logger.warning('Access to card details is unavailable for this user') @@ -518,6 +565,8 @@ def _get_history(self, info): assert self.transaction_detail.is_here() for tr in self.page.get_detail(): tr.type = Transaction.TYPE_DEFERRED_CARD + if account_card: + tr.card = card transactions_list.append(tr) if self.new_website: self.page.go_newsite_back_to_summary() @@ -587,6 +636,15 @@ def _get_history_invests(self, account): def get_history(self, account): self.home.go() self.deleteCTX() + + if account.type == account.TYPE_CARD: + def match_cb(tr): + return self.card_matches(tr.card, account.number) + + hist = self._get_history(account.parent._info, account) + hist = keep_only_card_transactions(hist, match_cb) + return hist + if not hasattr(account, '_info'): raise NotImplementedError if account.type is Account.TYPE_LIFE_INSURANCE and 'measure_id' not in account._info: @@ -603,19 +661,36 @@ def get_history(self, account): self.linebourse.session.cookies.update(self.session.cookies) self.update_linebourse_token() return self.linebourse.iter_history(account.id) - return self._get_history(account._info) + + hist = self._get_history(account._info, False) + return omit_deferred_transactions(hist) @need_login def get_coming(self, account): + if account.type != account.TYPE_CARD: + return [] + trs = [] - if not hasattr(account, '_info'): + if not hasattr(account.parent, '_info'): raise NotImplementedError() - for info in account._card_links: - for tr in self._get_history(info.copy()): - tr.type = tr.TYPE_DEFERRED_CARD - tr.nopurge = True - trs.append(tr) + + # We are on the old website + if hasattr(account, '_coming_eventargument'): + + if not self.cards_old.is_here(): + self.home.go() + self.page.go_list() + self.page.go_cards() + self.page.go_card_coming(account._coming_eventargument) + + return sorted_transactions(self.page.iter_coming()) + + # We are on the new website. + info = account.parent._card_links + for tr in self._get_history(info.copy(), account): + tr.type = tr.TYPE_DEFERRED_CARD + trs.append(tr) return sorted_transactions(trs) @@ -701,7 +776,7 @@ def get_profile(self): @need_login def iter_recipients(self, origin_account): - if origin_account.type == Account.TYPE_LOAN: + if origin_account.type in [Account.TYPE_LOAN, Account.TYPE_CARD]: return [] if 'pro' in self.url: @@ -928,3 +1003,9 @@ def download_document(self, document): self.page.go_document_list(sub_id=sub_id) return self.page.download_document(document).content + + def card_matches(self, a, b): + # For the same card, depending where we scrape it, we have + # more or less visible number. `X` are visible number, `*` hidden one's. + # tr.card: XXXX******XXXXXX, account.number: XXXXXX******XXXX + return (a[:4], a[-4:]) == (b[:4], b[-4:]) diff --git a/modules/caissedepargne/pages.py b/modules/caissedepargne/pages.py index fcfed545e8..99f82ca3b9 100644 --- a/modules/caissedepargne/pages.py +++ b/modules/caissedepargne/pages.py @@ -284,7 +284,7 @@ def _get_account_info(self, a, accounts): def is_account_inactive(self, account_id): return self.doc.xpath('//tr[td[contains(text(), $id)]][@class="Inactive"]', id=account_id) - def _add_account(self, accounts, link, label, account_type, balance): + def _add_account(self, accounts, link, label, account_type, balance, number=None): info = self._get_account_info(link, accounts) if info is None: self.logger.warning('Unable to parse account %r: %r' % (label, link)) @@ -295,6 +295,7 @@ def _add_account(self, accounts, link, label, account_type, balance): if is_rib_valid(info['id']): account.iban = rib2iban(info['id']) account._info = info + account.number = number account.label = label account.type = self.ACCOUNT_TYPES.get(label, info['acc_type'] if 'acc_type' in info else account_type) if 'PERP' in account.label: @@ -309,16 +310,15 @@ def _add_account(self, accounts, link, label, account_type, balance): account.currency = account.get_currency(balance) if balance and balance is not NotAvailable else NotAvailable account._card_links = [] + # Set coming history link to the parent account. At this point, we don't have card account yet. if account._info['type'] == 'HISTORIQUE_CB' and account.id in accounts: a = accounts[account.id] - if not a.coming: - a.coming = Decimal('0.0') - if account.balance and account.balance is not NotAvailable: - a.coming += account.balance - a._card_links.append(account._info) + a.coming = Decimal('0.0') + a._card_links = account._info return accounts[account.id] = account + return account def get_balance(self, account): if account.type not in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_PERP, Account.TYPE_CAPITALISATION): @@ -368,20 +368,27 @@ def get_list(self): self.ACCOUNT_TYPES.get(CleanText('.')(tds[2])) or\ self.ACCOUNT_TYPES.get(CleanText('.')(tds[3]), Account.TYPE_UNKNOWN) else: - # On the same row, there are many accounts (for example a - # check accound and a card one). + # On the same row, there could have many accounts (check account and a card one). + # For the card line, the number will be the same than the checking account, so we skip it. if len(tds) > 4: for i, a in enumerate(tds[2].xpath('./a')): label = CleanText('.')(a) balance = CleanText('.')(tds[-2].xpath('./a')[i]) - self._add_account(accounts, a, label, account_type, balance) - # Only 4 tds on banque de la reunion website. + number = None + # if i > 0, that mean it's a card account. The number will be the same than it's + # checking parent account, we have to skip it. + if i == 0: + number = CleanText('.')(tds[-4].xpath('./a')[0]) + self._add_account(accounts, a, label, account_type, balance, number) + # Only 4 tds on "banque de la reunion" website. elif len(tds) == 4: for i, a in enumerate(tds[1].xpath('./a')): label = CleanText('.')(a) balance = CleanText('.')(tds[-1].xpath('./a')[i]) self._add_account(accounts, a, label, account_type, balance) + self.logger.debug('we are on the %s website', 'old' if accounts else 'new') + if len(accounts) == 0: # New website self.browser.new_website = True @@ -405,7 +412,9 @@ def get_list(self): label = CleanText('./strong')(tds[0]) balance = CleanText('.')(tds[-1]) - self._add_account(accounts, a, label, account_type, balance) + account = self._add_account(accounts, a, label, account_type, balance) + if account: + account.number = CleanText('.')(tds[1]) return accounts.values() @@ -444,6 +453,8 @@ def get_loan_list(self): account.currency = account.get_currency(CleanText('./a')(tds[4])) accounts[account.id] = account + self.logger.debug('we are on the %s website', 'old' if accounts else 'new') + if len(accounts) == 0: # New website for table in self.doc.xpath('//div[@class="panel"]'): @@ -522,23 +533,58 @@ class item(ItemElement): obj_next_payment_amount = MyDecimal(MyTableCell("next_payment_amount")) obj_next_payment_date = Date(CleanText(MyTableCell("next_payment_date", default=''), default=NotAvailable), default=NotAvailable) + def submit_form(self, form, eventargument, eventtarget, scriptmanager): + form['__EVENTARGUMENT'] = eventargument + form['__EVENTTARGET'] = eventtarget + form['m_ScriptManager'] = scriptmanager + fix_form(form) + form.submit() + def go_list(self): + form = self.get_form(id='main') + eventargument = "CPTSYNT0" - form['__EVENTARGUMENT'] = "CPTSYNT0" + if "MM$m_CH$IsMsgInit" in form: + # Old website + eventtarget = "Menu_AJAX" + scriptmanager = "m_ScriptManager|Menu_AJAX" + else: + # New website + eventtarget = "MM$m_PostBack" + scriptmanager = "MM$m_UpdatePanel|MM$m_PostBack" + + self.submit_form(form, eventargument, eventtarget, scriptmanager) + + def go_cards(self): + form = self.get_form(id='main') + eventargument = "" if "MM$m_CH$IsMsgInit" in form: # Old website - form['__EVENTTARGET'] = "Menu_AJAX" - form['m_ScriptManager'] = "m_ScriptManager|Menu_AJAX" + eventtarget = "Menu_AJAX" + eventargument = "HISENCB0" + scriptmanager = "m_ScriptManager|Menu_AJAX" else: # New website - form['__EVENTTARGET'] = "MM$m_PostBack" - form['m_ScriptManager'] = "MM$m_UpdatePanel|MM$m_PostBack" + eventtarget = "MM$SYNTHESE$btnSyntheseCarte" + scriptmanager = "MM$m_UpdatePanel|MM$SYNTHESE$btnSyntheseCarte" - fix_form(form) + self.submit_form(form, eventargument, eventtarget, scriptmanager) - form.submit() + # only for old website + def go_card_coming(self, eventargument): + form = self.get_form(id='main') + eventtarget = "MM$HISTORIQUE_CB" + scriptmanager = "m_ScriptManager|Menu_AJAX" + self.submit_form(form, eventargument, eventtarget, scriptmanager) + + # only for new website + def go_coming(self, eventargument): + form = self.get_form(id='main') + eventtarget = "MM$HISTORIQUE_CB" + scriptmanager = "MM$m_UpdatePanel|MM$HISTORIQUE_CB" + self.submit_form(form, eventargument, eventtarget, scriptmanager) # On some pages, navigate to indexPage does not lead to the list of measures, so we need this form ... def go_measure_list(self): @@ -679,7 +725,11 @@ def get_history(self): continue if 'tot dif' in t.raw.lower(): t._link = Link(tr.xpath('./td/a'))(self.doc) - t.deleted = True + + # "Cb" for new site, "CB" for old one + mtc = re.match(r'(Cb|CB) (\d{4}\*+\d{6}) ', raw) + if mtc is not None: + t.card = mtc.group(2) t.set_amount(credit, debit) yield t @@ -786,6 +836,137 @@ def is_transfer_allowed(self): return not self.doc.xpath('//ul/li[contains(text(), "Aucun compte tiers n\'est disponible")]') +class CardsPage(IndexPage): + def is_here(self): + return CleanText('//h3[normalize-space(text())="Mes cartes (cartes dont je suis le titulaire)"]')(self.doc) + + @method + class iter_cards(TableElement): + head_xpath = '//table[@class="cartes"]/tbody/tr/th' + + col_label = 'Carte' + col_number = 'N°' + col_parent = 'Compte dépot associé' + col_coming = 'Encours' + + item_xpath = '//table[@class="cartes"]/tbody/tr[not(th)]' + + class item(ItemElement): + klass = Account + + obj_type = Account.TYPE_CARD + obj_label = Format('%s %s', CleanText(TableCell('label')), Field('id')) + obj_number = CleanText(TableCell('number')) + obj_id = CleanText(TableCell('number'), replace=[('*', 'X')]) + obj__parent_id = CleanText(TableCell('parent')) + obj_balance = 0 + obj_currency = Currency(TableCell('coming')) + + def obj_coming(self): + if CleanText(TableCell('coming'))(self) == '-': + raise SkipItem('immediate debit card?') + return CleanDecimal.French(TableCell('coming'), sign=lambda x: -1)(self) + + +class CardsComingPage(IndexPage): + def is_here(self): + return CleanText('//h2[text()="Encours de carte à débit différé"]')(self.doc) + + def get_card_coming_info(self, number, info): + + # If the xpath match, that mean there are only one card + # We have enought information in `info` to get its coming transaction + if CleanText('//tr[@id="MM_HISTORIQUE_CB_rptMois0_ctl01_trItem"]')(self.doc): + return info + + # If the xpath match, that mean there are at least 2 cards + xpath = '//tr[@id="MM_HISTORIQUE_CB_rptMois0_trItem_0"]' + + # In case of multiple card, first card coming's transactions are reachable + # with information in `info`. + if Regexp(CleanText(xpath), r'(\d{6}\*{6}\d{4})')(self.doc) == number: + return info + + # For all card except the first one for the same check account, we have to get info through their href info + link = CleanText(Attr('//a[contains(text(),"%s")]' % number, 'href'))(self.doc) + infos = re.match(r'.*(DETAIL_OP_M0&[^\"]+).*', link) + info['link'] = infos.group(1) + + return info + + +class CardsOldWebsitePage(IndexPage): + def is_here(self): + return CleanText('//span[@id="MM_m_CH_lblTitle" and contains(text(), "Historique de vos encours CB")]')(self.doc) + + def get_account(self): + infos = CleanText('.//span[@id="MM_HISTORIQUE_CB"]/table[position()=1]//td')(self.doc) + result = re.search(r'.*(\d{11}).*', infos) + return result.group(1) + + def get_date(self): + title = CleanText('//span[@id="MM_HISTORIQUE_CB_m_TableTitle3_lblTitle"]')(self.doc) + title_date = re.match('.*le (.*) sur .*', title) + return Date(dayfirst=True).filter(title_date.group(1)) + + @method + class iter_cards(TableElement): + head_xpath = '//table[@id="MM_HISTORIQUE_CB_m_ExDGOpeM0"]//tr[@class="DataGridHeader"]/td' + item_xpath = '//table[@id="MM_HISTORIQUE_CB_m_ExDGOpeM0"]//tr[not(contains(@class, "DataGridHeader")) and position() < last()]' + + col_label = 'Libellé' + col_coming = 'Solde' + + class item(ItemElement): + klass = Account + + obj_type = Account.TYPE_CARD + obj_label = Format('%s %s', CleanText(TableCell('label')), CleanText(Field('number'))) + obj_balance = 0 + obj_coming = CleanDecimal.French(TableCell('coming')) + obj_currency = Currency(TableCell('coming')) + + def obj__parent_id(self): + return self.page.get_account() + + def obj_number(self): + return CleanText(TableCell('number'))(self).replace('*', 'X') + + def obj_id(self): + number = Field('number')(self).replace('X', '') + account_id = '%s-%s' % (self.obj__parent_id(), number) + return account_id + + def obj__coming_eventargument(self): + url = Attr('.//a', 'href')(self) + res = re.match(r'.*(DETAIL_OP_M0\&.*;\d{8})", .*', url) + return res.group(1) + + def parse(self, obj): + # There are no thead name for this column. + self._cols['number'] = 3 + + @method + class iter_coming(TableElement): + head_xpath = '//table[@id="MM_HISTORIQUE_CB_m_ExDGDetailOpe"]//tr[@class="DataGridHeader"]/td' + item_xpath = '//table[@id="MM_HISTORIQUE_CB_m_ExDGDetailOpe"]//tr[not(contains(@class, "DataGridHeader"))]' + + col_label = 'Libellé' + col_coming = 'Débit' + col_date = 'Date' + + class item(ItemElement): + klass = Transaction + + obj_type = Transaction.TYPE_DEFERRED_CARD + obj_label = CleanText(TableCell('label')) + obj_amount = CleanDecimal.French(TableCell('coming')) + obj_rdate = Date(CleanText(TableCell('date')), dayfirst=True) + + def obj_date(self): + return self.page.get_date() + + class ConsLoanPage(JsonPage): def get_conso(self): return self.doc @@ -992,7 +1173,9 @@ def parse(self, el): # TODO use after 'I'? _id = Regexp(CleanText('.'), r'- (\w+\d\w+)')(self) # at least one digit accounts = list(self.page.browser.get_accounts_list()) + list(self.page.browser.get_loans_list()) - match = [acc for acc in accounts if _id in acc.id] + # If it's an internal account, we should always find only one account with _id in it's id. + # Type card account contains their parent account id, and should not be listed in recipient account. + match = [acc for acc in accounts if _id in acc.id and acc.type != Account.TYPE_CARD] assert len(match) == 1 match = match[0] self.env['id'] = match.id -- GitLab