From 93035c3065a2d4c8e6c626f1aa50d3d8415c68a3 Mon Sep 17 00:00:00 2001 From: Jerome Berthier Date: Tue, 16 Apr 2019 09:14:27 +0200 Subject: [PATCH] [cmes] Handle new website, currently only for cmes and cices Note that there is less information than before for investment (no code, no unit quantity/price). The behavior is also modified, unstead of one account grouping everything, there are now several accounts (PEE, PERCO, ...). As a result, accounts id are modified. It is now 'client_number' + 'account.label', instead of 'client_name' + 'client_number'. --- modules/cmes/browser.py | 7 +- modules/cmes/module.py | 4 +- modules/cmes/new_website/__init__.py | 0 modules/cmes/new_website/browser.py | 56 +++++++++++++ modules/cmes/new_website/pages.py | 119 +++++++++++++++++++++++++++ modules/cmes/pages.py | 10 ++- modules/cmes/proxy_browser.py | 33 ++++++++ 7 files changed, 224 insertions(+), 5 deletions(-) create mode 100644 modules/cmes/new_website/__init__.py create mode 100644 modules/cmes/new_website/browser.py create mode 100644 modules/cmes/new_website/pages.py create mode 100644 modules/cmes/proxy_browser.py diff --git a/modules/cmes/browser.py b/modules/cmes/browser.py index 5cbb319b8e..1de3a3a562 100644 --- a/modules/cmes/browser.py +++ b/modules/cmes/browser.py @@ -22,7 +22,7 @@ from weboob.browser import LoginBrowser, URL, need_login from .pages import ( - LoginPage, AccountsPage, FCPEInvestmentPage, + LoginPage, NewWebsitePage, AccountsPage, FCPEInvestmentPage, CCBInvestmentPage, HistoryPage, CustomPage, ) @@ -32,6 +32,7 @@ class CmesBrowser(LoginBrowser): login = URL('/espace-client/fr/identification/authentification.html', LoginPage) accounts = URL('(?P.*)fr/espace/devbavoirs.aspx\?mode=net&menu=cpte$', AccountsPage) + new_website = URL('(?P.*)espace-client/fr/epargnants/tableau-de-bord/index.html', NewWebsitePage) fcpe_investment = URL(r'/fr/.*GoPositionsParFond.*', r'/fr/espace/devbavoirs.aspx\?.*SituationParFonds.*GoOpenDetailFond.*', r'(?P.*)fr/espace/devbavoirs.aspx\?_tabi=(C|I1)&a_mode=net&a_menu=cpte&_pid=Situation(Globale|ParPlan)&_fid=GoPositionsParFond', @@ -49,6 +50,10 @@ def __init__(self, username, password, website, subsite="", *args, **kwargs): self.password = password self.subsite = subsite + @property + def logged(self): + return 'IdSes' in self.session.cookies + def do_login(self): self.login.go() self.page.login(self.username, self.password) diff --git a/modules/cmes/module.py b/modules/cmes/module.py index 1514e65b0c..e6d54a7d72 100644 --- a/modules/cmes/module.py +++ b/modules/cmes/module.py @@ -23,7 +23,7 @@ from weboob.capabilities.bank import CapBankPockets, AccountNotFound from weboob.capabilities.base import find_object -from .browser import CmesBrowser +from .proxy_browser import ProxyBrowser __all__ = ['CmesModule'] @@ -40,7 +40,7 @@ class CmesModule(Module, CapBankPockets): ValueBackendPassword('login', label='Identifiant', masked=False), ValueBackendPassword('password', label='Mot de passe')) - BROWSER = CmesBrowser + BROWSER = ProxyBrowser def create_default_browser(self): return self.create_browser( diff --git a/modules/cmes/new_website/__init__.py b/modules/cmes/new_website/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/modules/cmes/new_website/browser.py b/modules/cmes/new_website/browser.py new file mode 100644 index 0000000000..1cb00e80a8 --- /dev/null +++ b/modules/cmes/new_website/browser.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Budget Insight +# +# This file is part of a weboob module. +# +# This weboob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This weboob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this weboob module. If not, see . + + +from weboob.browser import URL, need_login +from ..browser import CmesBrowser +from .pages import ( + NewAccountsPage, OperationsListPage, OperationPage +) + + +class CmesBrowserNew(CmesBrowser): + + accounts = URL(r'(?P.*)espace-client/fr/epargnants/mon-epargne/situation-financiere-detaillee/index.html', + r'(?P.*)espace-client/fr/epargnants/tableau-de-bord/index.html', + NewAccountsPage) + + operations_list = URL(r'(?P.*)espace-client/fr/epargnants/operations/index.html', + OperationsListPage) + + operation = URL(r'(?P.*)espace-client/fr/epargnants/operations/consulter-une-operation/index.html\?param_=(?P\d+)', + OperationPage) + + @need_login + def iter_investment(self, account): + return self.accounts.stay_or_go(subsite=self.subsite).iter_investment(account=account) + + @need_login + def iter_history(self, account): + self.operations_list.stay_or_go(subsite=self.subsite) + for idx in self.page.get_operations_idx(): + tr = self.operation.go(subsite=self.subsite, idx=idx).get_transaction() + if account.label == tr._account_label: + yield tr + + @need_login + def iter_pocket(self, account): + for inv in self.iter_investment(account=account): + for pocket in self.page.iter_pocket(inv=inv): + yield pocket diff --git a/modules/cmes/new_website/pages.py b/modules/cmes/new_website/pages.py new file mode 100644 index 0000000000..bae7d1d3af --- /dev/null +++ b/modules/cmes/new_website/pages.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Budget Insight +# +# This file is part of a weboob module. +# +# This weboob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This weboob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this weboob module. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.browser.pages import HTMLPage, LoggedPage +from weboob.browser.elements import ListElement, ItemElement, method +from weboob.browser.filters.standard import ( + CleanText, Date, Regexp, Field, Currency, Upper, MapIn, Eval +) +from weboob.capabilities.bank import Account, Investment, Pocket + +from ..pages import MyDecimal, Transaction + + +ACCOUNTS_TYPES = { + "pargne entreprise": Account.TYPE_PEE, + "pargne retraite": Account.TYPE_PERCO +} + + +class NewAccountsPage(LoggedPage, HTMLPage): + @method + class iter_accounts(ListElement): + item_xpath = '//th[text()= "Nom du support" or text()="Nom du profil"]/ancestor::table/ancestor::table' + + class item(ItemElement): + klass = Account + balance_xpath = './/span[contains(text(), "Montant total")]/following-sibling::span' + + obj_label = CleanText('./tbody/tr/th//div') + obj_balance = MyDecimal(balance_xpath) + obj_currency = Currency(balance_xpath) + obj_type = MapIn(Field('label'), ACCOUNTS_TYPES, default=Account.TYPE_UNKNOWN) + + def obj_id(self): + # Use customer number + label to build account id + number = Regexp(CleanText('//div[@id="ei_tpl_fullSite"]//div[contains(@class, "ei_tpl_profil_content")]/p'), + r'(\d+)$', '\\1')(self) + return Field('label')(self) + number + + def iter_invest_rows(self, account): + for row in self.doc.xpath('//th/div[contains(., "%s")]/ancestor::table//table/tbody/tr' % account.label): + idx = re.search(r'_(\d+)\.', row.xpath('.//a[contains(@href, "GoFund")]/@id')[0]).group(1) + yield idx, row + + def iter_investment(self, account): + for idx, row in self.iter_invest_rows(account=account): + inv = Investment() + inv._account = account + inv._idx = idx + inv.label = CleanText('.//a[contains(@href, "GoFund")]/text()')(row) + inv.valuation = MyDecimal('.//td[2]')(row) + inv.diff_ratio = Eval(lambda x: x / 100, MyDecimal('.//td[3]'))(row) + + # Get data from a popup not reachable from the current row + inv.diff = MyDecimal('//div[@id="I0:F1_%s.R20:D"]//span' % idx)(self.doc) + + if account.balance != 0: + inv.portfolio_share = inv.valuation / account.balance + yield inv + + def iter_pocket(self, inv): + for idx, _ in self.iter_invest_rows(account=inv._account): + if idx != inv._idx: + continue + + for row in self.doc.xpath('//div[@id="I0:F1_%s.R16:D"]//tr[position()>1]' % idx): + pocket = Pocket() + pocket.label = inv.label + pocket.investment = inv + pocket.amount = MyDecimal('./td[2]')(row) + + if 'DISPONIBLE' in Upper(CleanText('./td[1]'))(row): + pocket.condition = Pocket.CONDITION_AVAILABLE + else: + pocket.condition = Pocket.CONDITION_DATE + pocket.availability_date = Date(Regexp(Upper(CleanText('./td[1]')), 'AU[\s]+(.*)'), dayfirst=True)(row) + + yield pocket + + +class OperationPage(LoggedPage, HTMLPage): + @method + class get_transaction(ItemElement): + klass = Transaction + + obj_amount = MyDecimal('//td[contains(text(), "Montant total")]/following-sibling::td') + obj_label = CleanText('(//p[contains(@id, "smltitle")])[2]') + obj_raw = Transaction.Raw(Field('label')) + obj_date = Date(Regexp(CleanText('(//p[contains(@id, "smltitle")])[1]'), r'(\d{1,2}/\d{1,2}/\d+)'), dayfirst=True) + obj__account_label = CleanText('//td[contains(text(), "Montant total")]/../following-sibling::tr/th[1]') + + +class OperationsListPage(LoggedPage, HTMLPage): + def __init__(self, *a, **kw): + self._cache = [] + super(OperationsListPage, self).__init__(*a, **kw) + + def get_operations_idx(self): + return [i.split(':')[-1] for i in self.doc.xpath('.//input[contains(@name, "_FID_GoOperationDetails")]/@name')] diff --git a/modules/cmes/pages.py b/modules/cmes/pages.py index 8b49294943..72ec5bbf0c 100644 --- a/modules/cmes/pages.py +++ b/modules/cmes/pages.py @@ -28,6 +28,7 @@ CleanDecimal, Env, Async, AsyncLoad, Currency, ) from weboob.browser.filters.html import Link, TableCell, Attr +from weboob.browser.switch import SiteSwitch from weboob.capabilities.bank import Account, Investment, Pocket from weboob.capabilities.base import NotAvailable from weboob.tools.capabilities.bank.transactions import FrenchTransaction @@ -47,6 +48,11 @@ def login(self, login, password): form.submit() +class NewWebsitePage(HTMLPage, LoggedPage): + def on_load(self): + raise SiteSwitch('cmes_new') + + class AccountsPage(LoggedPage, HTMLPage): def on_load(self): if self.doc.xpath('//label[contains(@for, "AcceptCGU")]'): @@ -181,9 +187,9 @@ def parse(self, obj): class Transaction(FrenchTransaction): - PATTERNS = [(re.compile(u'^(?PVersement.*)'), FrenchTransaction.TYPE_DEPOSIT), + PATTERNS = [(re.compile(u'^(?P.*Versement.*)'), FrenchTransaction.TYPE_DEPOSIT), (re.compile(u'^(?P(Arbitrage|Prélèvements.*))'), FrenchTransaction.TYPE_ORDER), - (re.compile(u'^(?PRetrait.*)'), FrenchTransaction.TYPE_WITHDRAWAL), + (re.compile(u'^(?P(Retrait|Paiement.*))'), FrenchTransaction.TYPE_WITHDRAWAL), (re.compile(u'^(?P.*)'), FrenchTransaction.TYPE_BANK), ] diff --git a/modules/cmes/proxy_browser.py b/modules/cmes/proxy_browser.py new file mode 100644 index 0000000000..9e34251e43 --- /dev/null +++ b/modules/cmes/proxy_browser.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Budget Insight +# +# This file is part of a weboob module. +# +# This weboob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This weboob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this weboob module. If not, see . + + +from weboob.browser.switch import SwitchingBrowser + +from .browser import CmesBrowser +from .new_website.browser import CmesBrowserNew + + +class ProxyBrowser(SwitchingBrowser): + BROWSERS = { + 'main': CmesBrowser, + 'cmes_new': CmesBrowserNew, + } + + KEEP_SESSION = True -- GitLab