From 59a1e63422915d12d876b5310af95271c3789e66 Mon Sep 17 00:00:00 2001 From: Ludovic LANGE Date: Fri, 4 Jan 2019 01:45:25 +0100 Subject: [PATCH] [creditdunordpee] rewrite using s2e, make it work with OTP / device fingerprinting Credit du Nord PEE has (now) an enhanced security feature, using a kind of 'fingerprinting' of the browser device which is used to detect if you changed device ; and will trigger the sending of a OTP to email or SMS for authorization of a new device. If you fill-in the proper OTP you received, it will allow this new device from now on. The current creditdunordpee does not handle this feature, and I was not able to add it easily. Instead, I found that this bank is using what looks like a shared platform with other banks - that is already supported by weboob (module : s2e for the shared code, and modules: bnppere, capeasi, erehsbc, esalia for the banks using it) Thus I rewrote the creditdunordpee module to use s2e shared code and have the authentication working. In addition to fixing authentication, the use of s2e module gives us a new capability (CapBankPockets). Morever, it seems maintained. --- modules/creditdunordpee/browser.py | 44 +-------- modules/creditdunordpee/module.py | 44 ++++----- modules/creditdunordpee/pages.py | 144 ----------------------------- modules/creditdunordpee/test.py | 31 +++++++ modules/s2e/__init__.py | 4 +- modules/s2e/browser.py | 9 ++ modules/s2e/pages.py | 36 ++++++++ tools/py3-compatible.modules | 1 + 8 files changed, 100 insertions(+), 213 deletions(-) delete mode 100644 modules/creditdunordpee/pages.py create mode 100644 modules/creditdunordpee/test.py diff --git a/modules/creditdunordpee/browser.py b/modules/creditdunordpee/browser.py index e2520990fa..0e2724a20f 100644 --- a/modules/creditdunordpee/browser.py +++ b/modules/creditdunordpee/browser.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Copyright(C) 2016 Bezleputh +# Copyright(C) 2018 Ludovic LANGE # # This file is part of a weboob module. # @@ -18,44 +19,9 @@ # along with this weboob module. If not, see . -from weboob.browser import LoginBrowser, need_login, URL -from weboob.exceptions import BrowserIncorrectPassword -from weboob.tools.date import LinearDateGuesser -from weboob.capabilities.bank import Account -from .pages import LoginPage, HomePage, AvoirPage, HistoryPage +from weboob.browser import AbstractBrowser -class CreditdunordpeeBrowser(LoginBrowser): - BASEURL = 'https://salaries.pee.credit-du-nord.fr' - home = URL('/?/portal/fr/salarie-cdn/', HomePage) - login = URL('/portal/login', LoginPage) - avoir = URL(u'/portal/salarie-cdn/monepargne/mesavoirs', AvoirPage) - history = URL(u'/portal/salarie-cdn/operations/consulteroperations\?scenario=ConsulterOperationsEffectuees', - HistoryPage) - - def do_login(self): - self.home.stay_or_go() - passwd = self.page.get_coded_passwd(self.password) - d = {'initialURI': "/portal/fr/salarie-cdn/", - 'password': passwd, - 'username': self.username} - - self.login.go(data=d) - - if not (self.home.is_here() and self.page.is_logged()): - raise BrowserIncorrectPassword() - - @need_login - def iter_accounts(self): - account = Account(self.username) - return iter([self.avoir.go().get_account(obj=account)]) - - @need_login - def get_history(self): - transactions = list(self.history.go().get_history(date_guesser=LinearDateGuesser())) - transactions.sort(key=lambda tr: tr.date, reverse=True) - return transactions - - @need_login - def iter_investment(self): - return self.avoir.go().iter_investment() +class CreditdunordpeeBrowser(AbstractBrowser): + PARENT = 's2e' + PARENT_ATTR = 'package.browser.CreditdunordpeeBrowser' diff --git a/modules/creditdunordpee/module.py b/modules/creditdunordpee/module.py index b92b9a708c..24d4d5f246 100644 --- a/modules/creditdunordpee/module.py +++ b/modules/creditdunordpee/module.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Copyright(C) 2016 Bezleputh +# Copyright(C) 2018 Ludovic LANGE # # This file is part of a weboob module. # @@ -18,10 +19,9 @@ # along with this weboob module. If not, see . -from weboob.tools.backend import Module, BackendConfig -from weboob.tools.value import ValueBackendPassword - -from weboob.capabilities.bank import CapBankWealth +from weboob.tools.backend import AbstractModule, BackendConfig +from weboob.tools.value import ValueBackendPassword, Value +from weboob.capabilities.bank import CapBank from .browser import CreditdunordpeeBrowser @@ -29,33 +29,21 @@ __all__ = ['CreditdunordpeeModule'] -class CreditdunordpeeModule(Module, CapBankWealth): +class CreditdunordpeeModule(AbstractModule, CapBank): NAME = 'creditdunordpee' - DESCRIPTION = u'Site de gestion du PEE du groupe Credit du nord' - MAINTAINER = u'Bezleputh' - EMAIL = 'carton_ben@yahoo.fr' + DESCRIPTION = u'Crédit du Nord Épargne Salariale' + MAINTAINER = u'Ludovic LANGE' + EMAIL = 'llange@users.noreply.github.com' LICENSE = 'LGPLv3+' - VERSION = '1.6' + VERSION = '1.4' + CONFIG = BackendConfig( + ValueBackendPassword('login', label='Identifiant', masked=False), + ValueBackendPassword('password', label='Code secret', regexp='^(\d{6})$'), + Value('otp', label='Code unique temporaire', default=''), + ) BROWSER = CreditdunordpeeBrowser - - CONFIG = BackendConfig(ValueBackendPassword('login', label='Identifiant', regexp='\d{8}', masked=False), - ValueBackendPassword('password', label='Mot de passe')) + PARENT = 's2e' def create_default_browser(self): - return self.create_browser(self.config['login'].get(), self.config['password'].get()) - - def get_account(self, id): - return self.browser.iter_accounts() - - def iter_accounts(self): - return self.browser.iter_accounts() - - def iter_coming(self, account): - raise NotImplementedError() - - def iter_history(self, account): - return self.browser.get_history() - - def iter_investment(self, account): - return self.browser.iter_investment() + return self.create_browser(self.config, weboob=self.weboob) diff --git a/modules/creditdunordpee/pages.py b/modules/creditdunordpee/pages.py deleted file mode 100644 index 452652754d..0000000000 --- a/modules/creditdunordpee/pages.py +++ /dev/null @@ -1,144 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2016 Bezleputh -# -# This file is part of a weboob module. -# -# This weboob module is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This weboob module is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this weboob module. If not, see . - -from __future__ import unicode_literals - -from io import BytesIO -import re - -from weboob.browser.pages import HTMLPage, LoggedPage -from weboob.tools.captcha.virtkeyboard import MappedVirtKeyboard -from weboob.browser.elements import ItemElement, TableElement, method -from weboob.browser.filters.standard import CleanText, CleanDecimal, Format, Regexp, Date, Env, Currency, Eval -from weboob.browser.filters.html import CleanHTML, TableCell -from weboob.capabilities.bank import Account, Transaction, Investment -from weboob.capabilities.base import NotAvailable -from weboob.exceptions import NoAccountsException - - -class VirtKeyboard(MappedVirtKeyboard): - symbols = {'0': ('8adee734aaefb163fb008d26bb9b3a42', '922d79345bf824b1186d0aa523b37a7c'), - '1': ('b815d6ce999910d48619b5912b81ddf1', '4730473dcd86f205dff51c59c97cf8c0'), - '2': ('54255a70694787a4e1bd7dd473b50228', '2d8b1ab0b5ce0b88abbc0170d2e85b7e'), - '3': ('ba06373d2bfba937d00bf52a31d475eb', '08e7e7ab7b330f3cfcb819b95eba64c6'), - '4': ('3fa795ac70247922048c514115487b10', 'ffb3d035a3a335cfe32c59d8ee1302ad'), - '5': ('788963d15fa05832ee7640f7c2a21bc3', 'c4b12545020cf87223901b6b35b9a9e2'), - '6': ('c8bf62dfaed9feeb86934d8617182503', '473357666949855a0794f68f3fc40127'), - '7': ('f7543fdda3039bdd383531954dd4fc46', '5f3a71bd2f696b8dc835dfeb7f32f92a'), - '8': ('5c4210e2d8e39f7667d7a9e5534b18b7', 'b9a1a73430f724541108ed5dd862431b'), - '9': ('94520ac801883fbfb700f43cd4172d41', '12c18ca3d4350acd077f557ac74161e5')} - - def __init__(self, page): - self.img_id = page.doc.find("//input[@id='identifiantClavierVirtuel']").attrib['value'] - img = page.doc.find("//img[@id='clavier_virtuel']") - res = page.browser.open('/portal/rest/clavier_virtuel/%s' % self.img_id) - MappedVirtKeyboard.__init__(self, BytesIO(res.content), page.doc, img, (0, 0, 0), convert='RGB') - - self.check_symbols(self.symbols, page.browser.responses_dirname) - - def get_symbol_code(self, md5sum): - code = MappedVirtKeyboard.get_symbol_code(self, md5sum) - return ''.join(re.findall(r"'(\d+)'", code)[-2:]) - - def get_string_code(self, string): - code = '' - for c in string: - code += self.get_symbol_code(self.symbols[c]) - return code - - -class LoginPage(HTMLPage): - pass - - -class HomePage(HTMLPage): - def on_load(self): - if CleanText(u'//span[contains(text(), "vous ne disposez plus d\'épargne salariale")]')(self.doc): - raise NoAccountsException() - - def get_coded_passwd(self, password): - vk = VirtKeyboard(self) - return '%s|%s|#006#' % (vk.get_string_code(password), vk.img_id) - - def is_logged(self): - return len(self.doc.find('//a[@class="btn-deconnexion"]')) - - -class AvoirPage(LoggedPage, HTMLPage): - @method - class get_account(ItemElement): - klass = Account - - obj_label = Format('PEE %s', CleanText('//div[@id="pbGp_df83b8bd_2dd787_2d4d10_2db608_2d69c44af91e91_j_id1:j_idt1:j_idt2:j_idt15_body"]')) - - def obj_balance(self): - return CleanDecimal('.', replace_dots=True).filter(self.fetch_total()) - - def obj_currency(self): - return Currency('.').filter(self.fetch_total()) - - obj_type = Account.TYPE_PEE - - def fetch_total(self): - table, = self.el.xpath('//table[has-class("operation-bloc-content-tableau-synthese")]') - assert CleanText('(./thead//th)[3]')(table) == 'Total' - tr, = table.xpath('./tbody[1]/tr') - return CleanText('./td[3]/div')(tr) - - @method - class iter_investment(TableElement): - head_xpath = '//div[has-class("detail-epargne-par-support")]//table/thead//th' - item_xpath = '//div[has-class("detail-epargne-par-support")]//table/tbody[1]/tr' - - col_misc = 'Mes supports de placement' - col_portfolio_share = 'Répartition' - col_valuation = 'Montant brut (1)' - col_diff = '+ ou - value potentielle' - - class item(ItemElement): - klass = Investment - - obj_label = Regexp(CleanText(CleanHTML(TableCell('misc'))), r'^(.*? - \d+)') - obj_vdate = Date(Regexp(CleanHTML(TableCell('misc')), r'(\d{2}/\d{2}/\d{4})')) - obj_unitvalue = CleanDecimal(Regexp(CleanText(TableCell('misc')), r'([\d,]+) €'), replace_dots=True) - obj_portfolio_share = Eval(lambda x: x / 100, CleanDecimal(CleanHTML(TableCell('portfolio_share')), replace_dots=True)) - obj_valuation = CleanDecimal(CleanHTML(TableCell('valuation')), replace_dots=True) - obj_diff = CleanDecimal(CleanHTML(TableCell('diff')), replace_dots=True) - - -class HistoryPage(LoggedPage, HTMLPage): - @method - class get_history(TableElement): - head_xpath = u'//table[has-class("operation-bloc-content-tableau")]/thead/tr/th/div/div/div/div/div/div/a/text()' - item_xpath = u'//table[has-class("operation-bloc-content-tableau")]/tbody/tr[has-class("rf-dt-r")]' - - col_date = u'Date de création' - col_reference = u'Référence' - col_montant = u'Montant net' - col_type = u'Type d’opération' - - class item(ItemElement): - klass = Transaction - - obj_date = Date(CleanText(TableCell('date')), Env('date_guesser')) - obj_type = Transaction.TYPE_UNKNOWN - obj_id = CleanText(TableCell('reference')) - obj_label = CleanText(TableCell('type')) - obj_amount = CleanDecimal(CleanHTML(TableCell('montant')), - replace_dots=True, default=NotAvailable) diff --git a/modules/creditdunordpee/test.py b/modules/creditdunordpee/test.py new file mode 100644 index 0000000000..1fbcb5bb38 --- /dev/null +++ b/modules/creditdunordpee/test.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Ludovic LANGE +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with weboob. If not, see . + + +from weboob.tools.test import BackendTest + + +class CreditdunordpeeTest(BackendTest): + MODULE = 'creditdunordpee' + + def test_bank(self): + l = list(self.backend.iter_accounts()) + if len(l) > 0: + a = l[0] + list(self.backend.iter_history(a)) diff --git a/modules/s2e/__init__.py b/modules/s2e/__init__.py index 48a05b2c26..9a09ca9cf1 100644 --- a/modules/s2e/__init__.py +++ b/modules/s2e/__init__.py @@ -20,6 +20,6 @@ from .module import S2eModule -from .browser import CapeasiBrowser, ErehsbcBrowser, BnppereBrowser, EsaliaBrowser +from .browser import CapeasiBrowser, ErehsbcBrowser, BnppereBrowser, EsaliaBrowser, CreditdunordpeeBrowser -__all__ = ['S2eModule', 'CapeasiBrowser', 'ErehsbcBrowser', 'BnppereBrowser', 'EsaliaBrowser'] +__all__ = ['S2eModule', 'CapeasiBrowser', 'ErehsbcBrowser', 'BnppereBrowser', 'EsaliaBrowser', 'CreditdunordpeeBrowser'] diff --git a/modules/s2e/browser.py b/modules/s2e/browser.py index 15ae8e507b..46bf64f007 100644 --- a/modules/s2e/browser.py +++ b/modules/s2e/browser.py @@ -24,6 +24,7 @@ from .pages import ( LoginPage, AccountsPage, AMFHSBCPage, AMFAmundiPage, AMFSGPage, HistoryPage, ErrorPage, LyxorfcpePage, EcofiPage, EcofiDummyPage, LandingPage, SwissLifePage, LoginErrorPage, + EtoileGestionPage, EtoileGestionCharacteristicsPage, ) @@ -44,6 +45,8 @@ class S2eBrowser(LoginBrowser, StatesMixin): history = URL('/portal/salarie-(?P\w+)/operations/consulteroperations', HistoryPage) error = URL('/maintenance/.+/', ErrorPage) swisslife = URL('http://fr.swisslife-am.com/fr/produits/.*', SwissLifePage) + etoile_gestion = URL('http://www.etoile-gestion.com/index.php/etg_fr_fr/productsheet/view/.*', EtoileGestionPage) + etoile_gestion_characteristics = URL('http://www.etoile-gestion.com/etg_fr_fr/ezjscore/.*', EtoileGestionCharacteristicsPage) STATE_DURATION = 10 @@ -170,3 +173,9 @@ class BnppereBrowser(S2eBrowser): BASEURL = 'https://personeo.epargne-retraite-entreprises.bnpparibas.com' SLUG = 'bnp' LANG = 'fr' # ['fr', 'en'] + + +class CreditdunordpeeBrowser(S2eBrowser): + BASEURL = 'https://salaries.pee.credit-du-nord.fr' + SLUG = 'cdn' + LANG = 'fr' # ['fr', 'en'] diff --git a/modules/s2e/pages.py b/modules/s2e/pages.py index ad98f0c3b2..24b5804b0e 100644 --- a/modules/s2e/pages.py +++ b/modules/s2e/pages.py @@ -33,6 +33,7 @@ from weboob.tools.captcha.virtkeyboard import MappedVirtKeyboard from weboob.exceptions import NoAccountsException, BrowserUnavailable, ActionNeeded, BrowserQuestion, BrowserIncorrectPassword from weboob.tools.value import Value +from weboob.tools.compat import urljoin def MyDecimal(*args, **kwargs): @@ -593,3 +594,38 @@ def get_code(self): if code == "n/a": return NotAvailable return code + + +class EtoileGestionPage(HTMLPage): + CODE_TYPE = NotAvailable + + def get_code(self): + # Codes (AMF / ISIN) are available after a click on a tab + characteristics_url = urljoin(self.url, Attr(u'//a[contains(text(), "Caractéristiques")]', 'data-href', default=None)(self.doc)) + if characteristics_url is not None: + detail_page = self.browser.open(characteristics_url).page + + # We prefer to return an ISIN code by default + code_isin = detail_page.get_code_isin() + if code_isin is not None: + self.CODE_TYPE = Investment.CODE_TYPE_ISIN + return code_isin + + # But if it's unavailable we can fallback to an AMF code + code_amf = detail_page.get_code_amf() + if code_amf is not None: + self.CODE_TYPE = Investment.CODE_TYPE_AMF + return code_amf + + return NotAvailable + + +class EtoileGestionCharacteristicsPage(PartialHTMLPage): + + def get_code_isin(self): + code = CleanText('//td[contains(text(), "Code Isin")]/following-sibling::td', default=None)(self.doc) + return code + + def get_code_amf(self): + code = CleanText('//td[contains(text(), "Code AMF")]/following-sibling::td', default=None)(self.doc) + return code diff --git a/tools/py3-compatible.modules b/tools/py3-compatible.modules index 42fc1a7682..ff69ac4cc5 100644 --- a/tools/py3-compatible.modules +++ b/tools/py3-compatible.modules @@ -53,6 +53,7 @@ colissimo cragr creditcooperatif creditdunord +creditdunordpee creditmutuel cuisineaz deathbycaptcha -- GitLab