diff --git a/modules/wellsfargo/__init__.py b/modules/airparif/__init__.py similarity index 84% rename from modules/wellsfargo/__init__.py rename to modules/airparif/__init__.py index 6f54e99b209daf1e51059111bc20d12b10bd8496..5d0e746a963b9a802861795cd214ce23182371dc 100644 --- a/modules/wellsfargo/__init__.py +++ b/modules/airparif/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright(C) 2014 Oleg Plakhotniuk +# Copyright(C) 2019 Vincent A # # This file is part of a weboob module. # @@ -17,7 +17,10 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . +from __future__ import unicode_literals -from .module import WellsFargoModule -__all__ = ['WellsFargoModule'] +from .module import AirparifModule + + +__all__ = ['AirparifModule'] diff --git a/modules/wellsfargo/test.py b/modules/airparif/browser.py similarity index 55% rename from modules/wellsfargo/test.py rename to modules/airparif/browser.py index aadc5b32d9f7299eb3c8bf27ec06f3dda1308018..51370a6f97b3a37b4e7729330c9fb20c259f603b 100644 --- a/modules/wellsfargo/test.py +++ b/modules/airparif/browser.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright(C) 2014 Oleg Plakhotniuk +# Copyright(C) 2019 Vincent A # # This file is part of a weboob module. # @@ -17,18 +17,23 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . -from weboob.tools.test import BackendTest -from itertools import chain +from __future__ import unicode_literals -class WellsFargoTest(BackendTest): - MODULE = 'wellsfargo' +from weboob.browser import PagesBrowser, URL - def test_history(self): - """ - Test that there's at least one transaction in the whole history. - """ - b = self.backend - ts = chain(*[b.iter_history(a) for a in b.iter_accounts()]) - t = next(ts, None) - self.assertNotEqual(t, None) +from .pages import AllPage + + +class AirparifBrowser(PagesBrowser): + BASEURL = 'https://airparif.asso.fr' + + all_page = URL(r'/stations/indicepolluant/', AllPage) + + def iter_gauges(self): + self.all_page.go(method='POST', headers={ + # don't remove the following headers, site returns 404 else... + 'X-Requested-With': 'XMLHttpRequest', + 'Referer': 'https://airparif.asso.fr/stations/index/', + }) + return self.page.iter_gauges() diff --git a/modules/airparif/favicon.png b/modules/airparif/favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..9f4946015fc5b9da9c47196ba32b20e59f3718dd Binary files /dev/null and b/modules/airparif/favicon.png differ diff --git a/modules/airparif/favicon.xcf b/modules/airparif/favicon.xcf new file mode 100644 index 0000000000000000000000000000000000000000..4bc6bad44ab11a9b82ff32aec5020c24e5eb0085 Binary files /dev/null and b/modules/airparif/favicon.xcf differ diff --git a/modules/airparif/module.py b/modules/airparif/module.py new file mode 100644 index 0000000000000000000000000000000000000000..a216e3f5270e7d8fdb14e8d9a63e35ab8e86e2a3 --- /dev/null +++ b/modules/airparif/module.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Vincent A +# +# This file is part of a weboob module. +# +# This weboob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This weboob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this weboob module. If not, see . + +from __future__ import unicode_literals + +from weboob.tools.backend import Module +from weboob.capabilities.base import find_object +from weboob.capabilities.gauge import ( + CapGauge, SensorNotFound, Gauge, GaugeSensor, +) + +from .browser import AirparifBrowser + + +__all__ = ['AirparifModule'] + + +class AirparifModule(Module, CapGauge): + NAME = 'airparif' + DESCRIPTION = 'airparif website' + MAINTAINER = 'Vincent A' + EMAIL = 'dev@indigo.re' + LICENSE = 'LGPLv3+' + VERSION = '1.6' + + BROWSER = AirparifBrowser + + def iter_gauges(self, pattern=None): + if pattern: + pattern = pattern.lower() + + for gauge in self.browser.iter_gauges(): + if not pattern or pattern in gauge._searching: + yield gauge + + def _get_gauge_by_id(self, id): + return find_object(self.browser.iter_gauges(), id=id) + + def iter_sensors(self, gauge, pattern=None): + if pattern: + pattern = pattern.lower() + + if not isinstance(gauge, Gauge): + gauge = self._get_gauge_by_id(gauge) + if gauge is None: + raise SensorNotFound() + + if pattern is None: + for sensor in gauge.sensors: + yield sensor + else: + for sensor in gauge.sensors: + if pattern in sensor.name.lower(): + yield sensor + + def _get_sensor_by_id(self, id): + gid = id.partition('.')[0] + return find_object(self.iter_sensors(gid), id=id) + + def get_last_measure(self, sensor): + if not isinstance(sensor, GaugeSensor): + sensor = self._get_sensor_by_id(sensor) + if sensor is None: + raise SensorNotFound() + return sensor.lastvalue diff --git a/modules/airparif/pages.py b/modules/airparif/pages.py new file mode 100644 index 0000000000000000000000000000000000000000..3097756fe420274823675bde53a3972ea5f56955 --- /dev/null +++ b/modules/airparif/pages.py @@ -0,0 +1,111 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Vincent A +# +# This file is part of a weboob module. +# +# This weboob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This weboob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this weboob module. If not, see . + +from __future__ import unicode_literals + +from weboob.browser.pages import JsonPage +from weboob.browser.elements import ItemElement, DictElement, method +from weboob.browser.filters.standard import ( + Env, Format, Regexp, DateTime, CleanDecimal, Lower, Map, +) +from weboob.browser.filters.json import Dict +from weboob.capabilities.address import GeoCoordinates, PostalAddress +from weboob.capabilities.gauge import Gauge, GaugeSensor, GaugeMeasure + + +SENSOR_NAMES = { + 'PM25': 'PM 2.5', + 'PM10': 'PM 10', + 'O3': 'O₃', + 'NO3': 'NO₃', + 'NO2': 'NO₂', +} + + +class AllPage(JsonPage): + @method + class iter_gauges(DictElement): + def find_elements(self): + return self.el.values() + + class item(ItemElement): + klass = Gauge + + def condition(self): + # sometimes the "date" field (which contains the hour) is empty + # and no measure is present in it, so we discard it + return bool(self.el['date']) + + def parse(self, el): + for k in el: + self.env[k] = el[k] + + self.env['city'] = Regexp(Dict('commune'), r'^(\D+)')(self) + + obj_id = Dict('nom_court_sit') + obj_name = Dict('isit_long') + obj_city = Env('city') + obj_object = 'Pollution' + + obj__searching = Lower(Format( + '%s %s %s %s', + Dict('isit_long'), + Dict('commune'), + Dict('ninsee'), + Dict('adresse'), + )) + + class obj_sensors(DictElement): + def find_elements(self): + return [dict(zip(('key', 'value'), tup)) for tup in self.el['indices'].items()] + + class item(ItemElement): + klass = GaugeSensor + + obj_name = Map(Dict('key'), SENSOR_NAMES) + obj_gaugeid = Env('nom_court_sit') + obj_id = Format('%s.%s', obj_gaugeid, Dict('key')) + obj_unit = 'µg/m³' + + class obj_lastvalue(ItemElement): + klass = GaugeMeasure + + obj_date = DateTime( + Format( + '%s %s', + Env('min_donnees'), + Env('date'), # "date" contains the time... + ) + ) + obj_level = CleanDecimal(Dict('value')) + + class obj_geo(ItemElement): + klass = GeoCoordinates + + obj_latitude = CleanDecimal(Env('latitude')) + obj_longitude = CleanDecimal(Env('longitude')) + + class obj_location(ItemElement): + klass = PostalAddress + + obj_street = Env('adresse') + obj_postal_code = Env('ninsee') + obj_city = Env('city') + obj_region = 'Ile-de-France' + obj_country = 'France' diff --git a/modules/airparif/test.py b/modules/airparif/test.py new file mode 100644 index 0000000000000000000000000000000000000000..6ee2ed9cc099cc6000e3e0350dd1dc7ca3935b1c --- /dev/null +++ b/modules/airparif/test.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Vincent A +# +# This file is part of a weboob module. +# +# This weboob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This weboob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this weboob module. If not, see . + +from __future__ import unicode_literals + + +from weboob.tools.test import BackendTest + + +class AirparifTest(BackendTest): + MODULE = 'airparif' + + def test_gauges(self): + all_gauges = list(self.backend.iter_gauges()) + paris_gauges = list(self.backend.iter_gauges(pattern='paris')) + self.assertTrue(all_gauges) + self.assertTrue(paris_gauges) + + self._check_gauge(all_gauges[0]) + + def _check_gauge(self, g): + self.assertTrue(g.id) + self.assertTrue(g.name) + self.assertTrue(g.city) + self.assertTrue(g.object) + self.assertTrue(g.sensors) + + self._check_sensor(g.sensors[0], g) + + def _check_sensor(self, s, g): + self.assertTrue(s.id) + self.assertTrue(s.name) + self.assertTrue(s.unit) + self.assertTrue(s.gaugeid == g.id) + + self.assertTrue(s.lastvalue.date) + self.assertTrue(s.lastvalue.level) + + self.assertTrue(s.geo.latitude) + self.assertTrue(s.geo.longitude) + + self.assertTrue(s.location.street) + self.assertTrue(s.location.city) + self.assertTrue(s.location.region) + self.assertTrue(s.location.country) + self.assertTrue(s.location.postal_code) diff --git a/modules/allocine/browser.py b/modules/allocine/browser.py index e785277d241bdf3749683f6f864801677994e03e..5e0545fcaf68701b03394350b1385152cf298430 100644 --- a/modules/allocine/browser.py +++ b/modules/allocine/browser.py @@ -566,7 +566,7 @@ class AllocineBrowser(APIBrowser): ] if query.summary: - movie = self.iter_movies(query.summary).next() + movie = next(self.iter_movies(query.summary)) params.append(('movie', movie.id)) result = self.__do_request('showtimelist', params) diff --git a/modules/amazonstorecard/pages.py b/modules/amazonstorecard/pages.py index 5f5acc2baceb0c86c9a69ef682f6e1c3aa9b9718..26a8908e317cdf92202c3bb333ce1573bcd9d02c 100644 --- a/modules/amazonstorecard/pages.py +++ b/modules/amazonstorecard/pages.py @@ -31,13 +31,6 @@ import re import json -try: - cmp = cmp -except NameError: - def cmp(x, y): - return (x > y) - (x < y) - - class SomePage(HTMLPage): @property def logged(self): @@ -86,7 +79,7 @@ class ActivityPage(SomePage): records = json.loads(self.doc.xpath( '//div[@id="completedActivityRecords"]//input[1]/@value')[0]) recent = [x for x in records if x['PDF_LOC'] is None] - for rec in sorted(recent, ActivityPage.cmp_records, reverse=True): + for rec in sorted(recent, key=lambda rec: ActivityPage.parse_date(rec['TRANS_DATE']), reverse=True): desc = u' '.join(rec['TRANS_DESC'].split()) trans = Transaction((rec['REF_NUM'] or u'').strip()) trans.date = ActivityPage.parse_date(rec['TRANS_DATE']) @@ -97,11 +90,6 @@ class ActivityPage(SomePage): trans.amount = -AmTr.decimal_amount(rec['TRANS_AMOUNT']) yield trans - @staticmethod - def cmp_records(rec1, rec2): - return cmp(ActivityPage.parse_date(rec1['TRANS_DATE']), - ActivityPage.parse_date(rec2['TRANS_DATE'])) - @staticmethod def parse_date(recdate): return datetime.strptime(recdate, u'%B %d, %Y') @@ -141,10 +129,13 @@ class StatementPage(RawPage): self._tok = ReTokenizer(self._pdf, '\n', self.LEX) def iter_transactions(self): - return sorted(self.read_transactions(), - cmp=lambda t1, t2: cmp(t2.date, t1.date) or - cmp(t1.label, t2.label) or - cmp(t1.amount, t2.amount)) + trs = self.read_transactions() + # since the sorts are not in the same direction, we can't do in one pass + # python sorting is stable, so sorting in 2 passes can achieve a multisort + # the official docs give this way + trs = sorted(trs, key=lambda tr: (tr.label, tr.amount)) + trs = sorted(trs, key=lambda tr: tr.date, reverse=True) + return trs def read_transactions(self): # Statement typically cover one month. diff --git a/modules/americanexpress/browser.py b/modules/americanexpress/browser.py index 74bd7b64d73064f10b529ca4860cc69534e35e20..28ea130c8af9e58dc42fb0395633a22bf3370d34 100644 --- a/modules/americanexpress/browser.py +++ b/modules/americanexpress/browser.py @@ -21,7 +21,7 @@ from __future__ import unicode_literals import datetime -from weboob.exceptions import BrowserIncorrectPassword +from weboob.exceptions import BrowserIncorrectPassword, ActionNeeded from weboob.browser.browsers import LoginBrowser, need_login from weboob.browser.exceptions import HTTPNotFound, ServerError from weboob.browser.url import URL @@ -29,8 +29,8 @@ from dateutil.parser import parse as parse_date from .pages import ( AccountsPage, JsonBalances, JsonPeriods, JsonHistory, - JsonBalances2, CurrencyPage, LoginPage, WrongLoginPage, AccountSuspendedPage, - NoCardPage, NotFoundPage, + JsonBalances2, CurrencyPage, LoginPage, NoCardPage, + NotFoundPage, ) @@ -40,11 +40,9 @@ __all__ = ['AmericanExpressBrowser'] class AmericanExpressBrowser(LoginBrowser): BASEURL = 'https://global.americanexpress.com' - login = URL('/myca/logon/.*', LoginPage) - wrong_login = URL('/myca/fuidfyp/emea/.*', WrongLoginPage) - account_suspended = URL('/myca/onlinepayments/', AccountSuspendedPage) + login = URL(r'/myca/logon/emea/action/login', LoginPage) - accounts = URL(r'/accounts', AccountsPage) + accounts = URL(r'/api/servicing/v1/member', AccountsPage) js_balances = URL(r'/account-data/v1/financials/balances', JsonBalances) js_balances2 = URL(r'/api/servicing/v1/financials/transaction_summary\?type=split_by_cardmember&statement_end_date=(?P[\d-]+)', JsonBalances2) js_pending = URL(r'/account-data/v1/financials/transactions\?limit=1000&offset=(?P\d+)&status=pending', @@ -69,51 +67,49 @@ class AmericanExpressBrowser(LoginBrowser): self.cache = {} def do_login(self): - if not self.login.is_here(): - self.location('/myca/logon/emea/action?request_type=LogonHandler&DestPage=https%3A%2F%2Fglobal.americanexpress.com%2Fmyca%2Fintl%2Facctsumm%2Femea%2FaccountSummary.do%3Frequest_type%3D%26Face%3Dfr_FR%26intlink%3Dtopnavvotrecompteneligne-HPmyca&Face=fr_FR&Info=CUExpired') - - self.page.login(self.username, self.password) - if self.wrong_login.is_here() or self.login.is_here() or self.account_suspended.is_here(): + self.login.go(data={ + 'request_type': 'login', + 'UserID': self.username, + 'Password': self.password, + 'Logon': 'Logon', + }) + + if self.page.get_status_code() != 0: + if self.page.get_error_code() == 'LGON004': + # This error happens when the website needs that the user + # enter his card information and reset his password. + # There is no message returned when this error happens. + raise ActionNeeded() raise BrowserIncorrectPassword() @need_login - def get_accounts(self): - self.accounts.go() - accounts = list(self.page.iter_accounts()) + def iter_accounts(self): + loc = self.session.cookies.get_dict(domain=".americanexpress.com")['axplocale'].lower() + self.currency_page.go(locale=loc) + currency = self.page.get_currency() - for account in accounts: + self.accounts.go() + account_list = list(self.page.iter_accounts(currency=currency)) + for account in account_list: try: # for the main account - self.js_balances.go(headers={'account_tokens': account._balances_token}) + self.js_balances.go(headers={'account_tokens': account.id}) except HTTPNotFound: # for secondary accounts - self.js_periods.go(headers={'account_token': account._balances_token}) + self.js_periods.go(headers={'account_token': account._history_token}) periods = self.page.get_periods() - self.js_balances2.go(date=periods[1], headers={'account_tokens': account._balances_token}) - self.page.set_balances(accounts) - - # get currency - loc = self.session.cookies.get_dict(domain=".americanexpress.com")['axplocale'].lower() - self.currency_page.go(locale=loc) - currency = self.page.get_currency() - - for acc in accounts: - acc.currency = currency - yield acc - - @need_login - def get_accounts_list(self): - for account in self.get_accounts(): + self.js_balances2.go(date=periods[1], headers={'account_tokens': account.id}) + self.page.fill_balances(obj=account) yield account @need_login def iter_history(self, account): - self.js_periods.go(headers={'account_token': account._token}) + self.js_periods.go(headers={'account_token': account._history_token}) periods = self.page.get_periods() today = datetime.date.today() # TODO handle pagination for p in periods: - self.js_posted.go(offset=0, end=p, headers={'account_token': account._token}) + self.js_posted.go(offset=0, end=p, headers={'account_token': account._history_token}) for tr in self.page.iter_history(periods=periods): # As the website is very handy, passing account_token is not enough: # it will return every transactions of each account, so we @@ -128,14 +124,14 @@ class AmericanExpressBrowser(LoginBrowser): # ('Enregistrées' tab on the website) # "pending" have no vdate and debit date is in future - self.js_periods.go(headers={'account_token': account._token}) + self.js_periods.go(headers={'account_token': account._history_token}) periods = self.page.get_periods() date = parse_date(periods[0]).date() today = datetime.date.today() # when the latest period ends today we can't know the coming debit date if date != today: try: - self.js_pending.go(offset=0, headers={'account_token': account._token}) + self.js_pending.go(offset=0, headers={'account_token': account._history_token}) except ServerError as exc: # At certain times of the month a connection might not have pendings; # in that case, `js_pending.go` would throw a 502 error Bad Gateway @@ -150,7 +146,7 @@ class AmericanExpressBrowser(LoginBrowser): # "posted" have a vdate but debit date can be future or past for p in periods: - self.js_posted.go(offset=0, end=p, headers={'account_token': account._token}) + self.js_posted.go(offset=0, end=p, headers={'account_token': account._history_token}) for tr in self.page.iter_history(periods=periods): if tr.date > today or not tr.date: if tr._owner == account._idforJSON: diff --git a/modules/americanexpress/module.py b/modules/americanexpress/module.py index 1c80742cfb917df6f3893b4906f298eb9db2fd39..c62231a6f43a2572ca8896ddf8ccaf63bf044879 100644 --- a/modules/americanexpress/module.py +++ b/modules/americanexpress/module.py @@ -44,7 +44,7 @@ class AmericanExpressModule(Module, CapBank): self.config['password'].get()) def iter_accounts(self): - return self.browser.get_accounts_list() + return self.browser.iter_accounts() def iter_history(self, account): return self.browser.iter_history(account) diff --git a/modules/americanexpress/pages.py b/modules/americanexpress/pages.py index 6b512a228601a1941333f29940f72b96c674b964..25cf438a371e417176c8abbbc7057a1217dd71ab 100644 --- a/modules/americanexpress/pages.py +++ b/modules/americanexpress/pages.py @@ -19,18 +19,14 @@ from __future__ import unicode_literals -from ast import literal_eval from decimal import Decimal -import re from weboob.browser.pages import LoggedPage, JsonPage, HTMLPage from weboob.browser.elements import ItemElement, DictElement, method -from weboob.browser.filters.standard import Date, Eval, Env, CleanText, Field, CleanDecimal +from weboob.browser.filters.standard import Date, Eval, Env, CleanText, Field, CleanDecimal, Format, Currency from weboob.browser.filters.json import Dict from weboob.capabilities.bank import Account, Transaction from weboob.capabilities.base import NotAvailable -from weboob.tools.json import json -from weboob.tools.compat import basestring from weboob.exceptions import ActionNeeded, BrowserUnavailable from dateutil.parser import parse as parse_date @@ -48,14 +44,6 @@ def parse_decimal(s): return CleanDecimal(replace_dots=comma).filter(s) -class WrongLoginPage(HTMLPage): - pass - - -class AccountSuspendedPage(HTMLPage): - pass - - class NoCardPage(HTMLPage): def on_load(self): raise ActionNeeded() @@ -68,77 +56,70 @@ class NotFoundPage(HTMLPage): raise BrowserUnavailable(alert_header, alert_content) -class LoginPage(HTMLPage): - def login(self, username, password): - form = self.get_form(name='ssoform') - form['UserID'] = username - form['USERID'] = username - form['Password'] = password - form['PWD'] = password - form.submit() - - -class AccountsPage(LoggedPage, HTMLPage): - def iter_accounts(self): - for line in self.doc.xpath('//script[@id="initial-state"]')[0].text.split('\n'): - m = re.search('window.__INITIAL_STATE__ = (.*);', line) - if m: - data = json.loads(literal_eval(m.group(1))) - break - else: - assert False, "data was not found" - - assert data[15] == 'core' - assert len(data[16]) == 3 - - # search for products to get products list - for index, el in enumerate(data[16][2]): - if 'products' in el: - accounts_data = data[16][2][index + 1] - - assert len(accounts_data) == 2 - assert accounts_data[1][4] == 'productsList' - - accounts_data = accounts_data[1][5] - token = [] - - for account_data in accounts_data: - if isinstance(account_data, basestring): - balances_token = account_data - - elif isinstance(account_data, list) and not account_data[4][2][0] == "Canceled": - acc = Account() - if len(account_data) > 15: - token.append(account_data[-11]) - acc._idforJSON = account_data[10][-1] - else: - acc._idforJSON = account_data[-5][-1] - acc._idforJSON = re.sub(r'\s+', ' ', acc._idforJSON) - acc.number = '-%s' % account_data[2][2] - acc.label = '%s %s' % (account_data[6][4], account_data[10][-1]) - acc._balances_token = acc.id = balances_token - acc._token = token[-1] - acc.type = Account.TYPE_CARD - yield acc +class LoginPage(JsonPage): + def get_status_code(self): + # - 0 = OK + # - 1 = Incorrect login/password + return CleanDecimal(Dict('statusCode'))(self.doc) + + def get_error_code(self): + # - LGON004 = ActionNeeded + return CleanText(Dict('errorCode'))(self.doc) + + +class AccountsPage(LoggedPage, JsonPage): + @method + class iter_accounts(DictElement): + def find_elements(self): + for obj in self.page.doc.get('accounts', []): + obj['_history_token'] = obj['account_token'] + yield obj + for secondary_acc in obj.get('supplementary_accounts', []): + # Secondary accounts use the id of the parrent account + # when searching history/coming. History/coming are filtered + # on the owner name (_idforJSON). + secondary_acc['_history_token'] = obj['account_token'] + yield secondary_acc + + class item(ItemElement): + klass = Account + + def condition(self): + return any(status == 'Active' for status in Dict('status/account_status')(self)) + + obj_id = Dict('account_token') + obj__history_token = Dict('_history_token') + obj__account_type = Dict('account/relationship') + obj_number = Format('-%s', Dict('account/display_account_number')) + obj_type = Account.TYPE_CARD + obj_currency = Currency(Env('currency')) + obj__idforJSON = Dict('profile/embossed_name') + + def obj_label(self): + if Dict('account/relationship')(self) == 'SUPP': + return Format( + '%s %s', + Dict('platform/amex_region'), + Dict('profile/embossed_name'), + )(self) + return CleanText(Dict('product/description'))(self) class JsonBalances(LoggedPage, JsonPage): - def set_balances(self, accounts): - by_token = {a._balances_token: a for a in accounts} - for d in self.doc: - # coming is what should be refunded at a futur deadline - by_token[d['account_token']].coming = -float_to_decimal(d['total_debits_balance_amount']) - # balance is what is currently due - by_token[d['account_token']].balance = -float_to_decimal(d['remaining_statement_balance_amount']) + @method + class fill_balances(ItemElement): + # coming is what should be refunded at a future deadline + obj_coming = CleanDecimal.US(Dict('0/total_debits_balance_amount'), sign=lambda x: -1) + # balance is what is currently due + obj_balance = CleanDecimal.US(Dict('0/remaining_statement_balance_amount'), sign=lambda x: -1) class JsonBalances2(LoggedPage, JsonPage): - def set_balances(self, accounts): - by_token = {a._balances_token: a for a in accounts} - for d in self.doc: - by_token[d['account_token']].balance = -float_to_decimal(d['total']['payments_credits_total_amount']) - by_token[d['account_token']].coming = -float_to_decimal(d['total']['debits_total_amount']) - # warning: payments_credits_total_amount is not the coming value here + @method + class fill_balances(ItemElement): + obj_coming = CleanDecimal.US(Dict('0/total/debits_total_amount'), sign=lambda x: -1) + obj_balance = CleanDecimal.US(Dict('0/total/payments_credits_total_amount'), sign=lambda x: -1) + # warning: payments_credits_total_amount is not the coming value here class CurrencyPage(LoggedPage, JsonPage): diff --git a/modules/amundi/browser.py b/modules/amundi/browser.py index 7eb43d1b89cb4a617c000898196ac269948058bd..2d463682dd78beb2cfaf760e21d9398fb80ee84c 100644 --- a/modules/amundi/browser.py +++ b/modules/amundi/browser.py @@ -26,9 +26,9 @@ from weboob.capabilities.base import empty, NotAvailable from .pages import ( LoginPage, AccountsPage, AccountHistoryPage, AmundiInvestmentsPage, AllianzInvestmentPage, - EEInvestmentPage, EEInvestmentDetailPage, EEProductInvestmentPage, EresInvestmentPage, - CprInvestmentPage, BNPInvestmentPage, BNPInvestmentApiPage, AxaInvestmentPage, - EpsensInvestmentPage, EcofiInvestmentPage, + EEInvestmentPage, EEInvestmentPerformancePage, EEInvestmentDetailPage, EEProductInvestmentPage, + EresInvestmentPage, CprInvestmentPage, BNPInvestmentPage, BNPInvestmentApiPage, AxaInvestmentPage, + EpsensInvestmentPage, EcofiInvestmentPage, SGGestionInvestmentPage, SGGestionPerformancePage, ) @@ -44,7 +44,8 @@ class AmundiBrowser(LoginBrowser): amundi_investments = URL(r'https://www.amundi.fr/fr_part/product/view', AmundiInvestmentsPage) # EEAmundi browser investments ee_investments = URL(r'https://www.amundi-ee.com/part/home_fp&partner=PACTEO_SYS', EEInvestmentPage) - ee_investment_details = URL(r'https://www.amundi-ee.com/psAmundiEEPart/ezjscore/call', EEInvestmentDetailPage) + ee_performance_details = URL(r'https://www.amundi-ee.com/psAmundiEEPart/ezjscore/call(.*)_tab_2', EEInvestmentPerformancePage) + ee_investment_details = URL(r'https://www.amundi-ee.com/psAmundiEEPart/ezjscore/call(.*)_tab_5', EEInvestmentDetailPage) # EEAmundi product investments ee_product_investments = URL(r'https://www.amundi-ee.com/product', EEProductInvestmentPage) # Allianz GI investments @@ -62,6 +63,9 @@ class AmundiBrowser(LoginBrowser): epsens_investments = URL(r'https://www.epsens.com/information-financiere', EpsensInvestmentPage) # Ecofi investments ecofi_investments = URL(r'http://www.ecofi.fr/fr/fonds/dynamis-solidaire', EcofiInvestmentPage) + # Société Générale gestion investments + sg_gestion_investments = URL(r'https://www.societegeneralegestion.fr/psSGGestionEntr/productsheet/view/idvm', SGGestionInvestmentPage) + sg_gestion_performance = URL(r'https://www.societegeneralegestion.fr/psSGGestionEntr/ezjscore/call', SGGestionPerformancePage) def do_login(self): data = { @@ -101,7 +105,7 @@ class AmundiBrowser(LoginBrowser): handled_urls = ( 'www.amundi.fr/fr_part', # AmundiInvestmentsPage - 'www.amundi-ee.com/part/home_fp', # EEInvestmentPage + 'www.amundi-ee.com/part/home_fp', # EEInvestmentDetailPage & EEInvestmentPerformancePage 'www.amundi-ee.com/product', # EEProductInvestmentPage 'fr.allianzgi.com/fr-fr', # AllianzInvestmentPage 'www.eres-group.com/eres', # EresInvestmentPage @@ -110,6 +114,7 @@ class AmundiBrowser(LoginBrowser): 'axa-im.fr/fr/fund-page', # AxaInvestmentPage 'www.epsens.com/information-financiere', # EpsensInvestmentPage 'www.ecofi.fr/fr/fonds/dynamis-solidaire', # EcofiInvestmentPage + 'www.societegeneralegestion.fr', # SGGestionInvestmentPage ) for inv in self.page.iter_investments(account_id=account.id): @@ -156,10 +161,29 @@ class AmundiBrowser(LoginBrowser): elif self.ee_investments.is_here(): inv.recommended_period = self.page.get_recommended_period() details_url = self.page.get_details_url() + performance_url = self.page.get_performance_url() if details_url: self.location(details_url) if self.ee_investment_details.is_here(): inv.asset_category = self.page.get_asset_category() + if performance_url: + self.location(performance_url) + if self.ee_performance_details.is_here(): + # The investments JSON only contains 1 & 5 years performances + # If we can access EEInvestmentPerformancePage, we can fetch all three + # values (1, 3 and 5 years), in addition the values are more accurate here. + complete_performance_history = self.page.get_performance_history() + if complete_performance_history: + inv.performance_history = complete_performance_history + + elif self.sg_gestion_investments.is_here(): + # Fetch asset category & recommended period + self.page.fill_investment(obj=inv) + # Fetch all performances on the details page + performance_url = self.page.get_performance_url() + if performance_url: + self.location(performance_url) + inv.performance_history = self.page.get_performance_history() elif self.bnp_investments.is_here(): # We fetch the fund ID and get the attributes directly from the BNP-ERE API diff --git a/modules/amundi/module.py b/modules/amundi/module.py index 78b14bb41c71cf74c020161dbc03ba731f1ad9fa..f4dbb22cb97786d82e149510d12e6a3badccefb2 100644 --- a/modules/amundi/module.py +++ b/modules/amundi/module.py @@ -34,11 +34,19 @@ class AmundiModule(Module, CapBankWealth): EMAIL = 'james.galt.bi@gmail.com' LICENSE = 'LGPLv3+' VERSION = '1.6' - CONFIG = BackendConfig(ValueBackendPassword('login', label='Identifiant', regexp=r'\d+', masked=False), - ValueBackendPassword('password', label=u"Mot de passe", regexp=r'\d+'), - Value('website', label='Type de compte', default='ee', - choices={'ee': 'Amundi Epargne Entreprise', - 'tc': 'Amundi Tenue de Compte'})) + CONFIG = BackendConfig( + ValueBackendPassword('login', label='Identifiant', regexp=r'\d+', masked=False), + ValueBackendPassword('password', label='Mot de passe'), + Value( + 'website', + label='Type de compte', + default='ee', + choices={ + 'ee': 'Amundi Epargne Entreprise', + 'tc': 'Amundi Tenue de Compte' + } + ) + ) def create_default_browser(self): b = {'ee': EEAmundi, 'tc': TCAmundi} diff --git a/modules/amundi/pages.py b/modules/amundi/pages.py index fc6e72cfd2a6cb9a4b61799ece112df2923e7b81..a3499395240d4dbf714539d4b5be1a00620c91dd 100644 --- a/modules/amundi/pages.py +++ b/modules/amundi/pages.py @@ -48,6 +48,7 @@ ACCOUNT_TYPES = { 'PERCO': Account.TYPE_PERCO, 'PERCOI': Account.TYPE_PERCO, 'RSP': Account.TYPE_RSP, + 'ART 83': Account.TYPE_ARTICLE_83, } class AccountsPage(LoggedPage, JsonPage): @@ -84,7 +85,7 @@ class AccountsPage(LoggedPage, JsonPage): except UnicodeError: try: return Dict('libelleDispositif')(self).encode('latin1').decode('utf8') - except UnicodeDecodeError: + except UnicodeError: return Dict('libelleDispositif')(self) @method @@ -185,6 +186,21 @@ class EEInvestmentPage(LoggedPage, HTMLPage): def get_details_url(self): return Attr('//a[contains(text(), "Caractéristiques")]', 'data-href', default=None)(self.doc) + def get_performance_url(self): + return Attr('//a[contains(text(), "Performances")]', 'data-href', default=None)(self.doc) + + +class EEInvestmentPerformancePage(LoggedPage, HTMLPage): + def get_performance_history(self): + perfs = {} + if CleanDecimal.French('//tr[td[text()="Fonds"]]//td[position()=last()-2]', default=None)(self.doc): + perfs[1] = Eval(lambda x: x / 100, CleanDecimal.French('//tr[td[text()="Fonds"]]//td[position()=last()-2]'))(self.doc) + if CleanDecimal.French('//tr[td[text()="Fonds"]]//td[position()=last()-1]', default=None)(self.doc): + perfs[3] = Eval(lambda x: x / 100, CleanDecimal.French('//tr[td[text()="Fonds"]]//td[position()=last()-1]'))(self.doc) + if CleanDecimal.French('//tr[td[text()="Fonds"]]//td[position()=last()]', default=None)(self.doc): + perfs[5] = Eval(lambda x: x / 100, CleanDecimal.French('//tr[td[text()="Fonds"]]//td[position()=last()]'))(self.doc) + return perfs + class EEInvestmentDetailPage(LoggedPage, HTMLPage): def get_asset_category(self): @@ -272,3 +288,17 @@ class EcofiInvestmentPage(LoggedPage, HTMLPage): r'\/Horizon (.*)\.png' ) obj_asset_category = CleanText('//div[contains(text(), "Classification")]/following-sibling::div[1]', default=NotAvailable) + + +class SGGestionInvestmentPage(LoggedPage, HTMLPage): + @method + class fill_investment(ItemElement): + obj_asset_category = CleanText('//label[contains(text(), "Classe d\'actifs")]/following-sibling::span', default=NotAvailable) + obj_recommended_period = CleanText('//label[contains(text(), "Durée minimum")]/following-sibling::span', default=NotAvailable) + + def get_performance_url(self): + return Attr('(//li[@role="presentation"])[1]//a', 'data-href', default=None)(self.doc) + + +class SGGestionPerformancePage(EEInvestmentPerformancePage): + pass diff --git a/modules/apivie/browser.py b/modules/apivie/browser.py index 650d284f095cbe9420f3bd140dd59b868751adf3..6f599735a9b430b80b5b04fe55a79a2eb5858525 100644 --- a/modules/apivie/browser.py +++ b/modules/apivie/browser.py @@ -20,10 +20,9 @@ from weboob.browser import LoginBrowser, URL, need_login from weboob.capabilities.base import find_object -from weboob.exceptions import BrowserIncorrectPassword - -from .pages import LoginPage, AccountsPage, InvestmentsPage, OperationsPage +from weboob.exceptions import BrowserIncorrectPassword, ActionNeeded +from .pages import LoginPage, AccountsPage, InvestmentsPage, OperationsPage, InfoPage __all__ = ['ApivieBrowser'] @@ -33,6 +32,7 @@ class ApivieBrowser(LoginBrowser): r'/accueil$', r'/perte.*', LoginPage) + info = URL(r'/coordonnees.*', InfoPage) accounts = URL(r'/accueil-connect', AccountsPage) investments = URL(r'/synthese-contrat.*', InvestmentsPage) operations = URL(r'/historique-contrat.*', OperationsPage) @@ -50,6 +50,12 @@ class ApivieBrowser(LoginBrowser): self.page.login(self.username, self.password) + # If the user's contact info is too old the website asks to verify them. We're logged but we can't go further. + if self.info.is_here(): + error_message = self.page.get_error_message() + assert error_message, 'Error message location has changed on info page' + raise ActionNeeded(error_message) + if self.login.is_here() or self.page is None: raise BrowserIncorrectPassword() diff --git a/modules/apivie/pages.py b/modules/apivie/pages.py index a09ee0104721ff6aa2fc79929954156a413a5f81..521aa734b9f513af1f53f11843b67bb2dbc04738 100644 --- a/modules/apivie/pages.py +++ b/modules/apivie/pages.py @@ -36,6 +36,11 @@ class LoginPage(HTMLPage): return form.submit() +class InfoPage(LoggedPage, HTMLPage): + def get_error_message(self): + return CleanText('//span[@class="ui-messages-fatal-detail"]')(self.doc) + + class AccountsPage(LoggedPage, HTMLPage): TYPES = {u'APIVIE': Account.TYPE_LIFE_INSURANCE, u'LINXEA ZEN CLIENT': Account.TYPE_LIFE_INSURANCE, diff --git a/modules/axabanque/browser.py b/modules/axabanque/browser.py index 2fae880266195505adb8728f85b9374ba91134b5..2eacaf6c28bc671d9ec034f032db154f7361b875 100644 --- a/modules/axabanque/browser.py +++ b/modules/axabanque/browser.py @@ -56,17 +56,22 @@ from .pages.document import DocumentsPage, DownloadPage class AXABrowser(LoginBrowser): # Login - keyboard = URL('https://connect.axa.fr/keyboard/password', KeyboardPage) - login = URL('https://connect.axa.fr/api/identity/auth', LoginPage) - password = URL('https://connect.axa.fr/#/changebankpassword', ChangepasswordPage) - predisconnected = URL('https://www.axa.fr/axa-predisconnect.html', - 'https://www.axa.fr/axa-postmaw-predisconnect.html', PredisconnectedPage) - - denied = URL('https://connect.axa.fr/Account/AccessDenied', DeniedPage) - account_space_login = URL('https://connect.axa.fr/api/accountspace', AccountSpaceLogin) - errors = URL('https://espaceclient.axa.fr/content/ecc-public/accueil-axa-connect/_jcr_content/par/text.html', - 'https://espaceclient.axa.fr/content/ecc-public/errors/500.html', - ErrorPage) + keyboard = URL(r'https://connect.axa.fr/keyboard/password', KeyboardPage) + login = URL(r'https://connect.axa.fr/api/identity/auth', LoginPage) + password = URL(r'https://connect.axa.fr/#/changebankpassword', ChangepasswordPage) + predisconnected = URL( + r'https://www.axa.fr/axa-predisconnect.html', + r'https://www.axa.fr/axa-postmaw-predisconnect.html', + PredisconnectedPage + ) + + denied = URL(r'https://connect.axa.fr/Account/AccessDenied', DeniedPage) + account_space_login = URL(r'https://connect.axa.fr/api/accountspace', AccountSpaceLogin) + errors = URL( + r'https://espaceclient.axa.fr/content/ecc-public/accueil-axa-connect/_jcr_content/par/text.html', + r'https://espaceclient.axa.fr/content/ecc-public/errors/500.html', + ErrorPage + ) def do_login(self): # due to the website change, login changed too, this is for don't try to login with the wrong login @@ -98,61 +103,74 @@ class AXABrowser(LoginBrowser): class AXABanque(AXABrowser, StatesMixin): - BASEURL = 'https://www.axabanque.fr/' + BASEURL = 'https://www.axabanque.fr' STATE_DURATION = 5 # Bank - bank_accounts = URL(r'transactionnel/client/liste-comptes.html', - r'transactionnel/client/liste-(?P.*).html', - r'webapp/axabanque/jsp/visionpatrimoniale/liste_panorama_.*\.faces', - r'/webapp/axabanque/page\?code=(?P\d+)', - r'webapp/axabanque/client/sso/connexion\?token=(?P.*)', BankAccountsPage) + bank_accounts = URL( + r'/transactionnel/client/liste-comptes.html', + r'/transactionnel/client/liste-(?P.*).html', + r'/webapp/axabanque/jsp/visionpatrimoniale/liste_panorama_.*\.faces', + r'/webapp/axabanque/page\?code=(?P\d+)', + r'/webapp/axabanque/client/sso/connexion\?token=(?P.*)', + BankAccountsPage + ) iban_pdf = URL(r'http://www.axabanque.fr/webapp/axabanque/formulaire_AXA_Banque/.*\.pdf.*', IbanPage) - cbttransactions = URL(r'webapp/axabanque/jsp/detailCarteBleu.*.faces', CBTransactionsPage) - transactions = URL(r'webapp/axabanque/jsp/panorama.faces', - r'webapp/axabanque/jsp/visionpatrimoniale/panorama_.*\.faces', - r'webapp/axabanque/jsp/detail.*.faces', - r'webapp/axabanque/jsp/.*/detail.*.faces', TransactionsPage) - unavailable = URL(r'login_errors/indisponibilite.*', - r'.*page-indisponible.html.*', - r'.*erreur/erreurBanque.faces', - r'http://www.axabanque.fr/message/maintenance.htm', UnavailablePage) + cbttransactions = URL(r'/webapp/axabanque/jsp/detailCarteBleu.*.faces', CBTransactionsPage) + transactions = URL( + r'/webapp/axabanque/jsp/panorama.faces', + r'/webapp/axabanque/jsp/visionpatrimoniale/panorama_.*\.faces', + r'/webapp/axabanque/jsp/detail.*.faces', + r'/webapp/axabanque/jsp/.*/detail.*.faces', + TransactionsPage + ) + unavailable = URL( + r'/login_errors/indisponibilite.*', + r'.*page-indisponible.html.*', + r'.*erreur/erreurBanque.faces', + r'http://www.axabanque.fr/message/maintenance.htm', + UnavailablePage + ) # Wealth wealth_accounts = URL( - 'https://espaceclient.axa.fr/$', - 'https://espaceclient.axa.fr/accueil.html', - 'https://connexion.adis-assurances.com', + r'https://espaceclient.axa.fr/$', + r'https://espaceclient.axa.fr/accueil.html', + r'https://connexion.adis-assurances.com', WealthAccountsPage ) - investment = URL('https://espaceclient.axa.fr/.*content/ecc-popin-cards/savings/(\w+)/repartition', InvestmentPage) + investment = URL(r'https://espaceclient.axa.fr/.*content/ecc-popin-cards/savings/(\w+)/repartition', InvestmentPage) history = URL(r'https://espaceclient.axa.fr/accueil/savings/savings/contract/_jcr_content.eccGetSavingsOperations.json', HistoryPage) history_investments = URL(r'https://espaceclient.axa.fr/accueil/savings/savings/contract/_jcr_content.eccGetSavingOperationDetail.json', HistoryInvestmentsPage) details = URL( - 'https://espaceclient.axa.fr/.*accueil/savings/(\w+)/contract', - 'https://espaceclient.axa.fr/#', + r'https://espaceclient.axa.fr/.*accueil/savings/(\w+)/contract', + r'https://espaceclient.axa.fr/#', AccountDetailsPage ) lifeinsurance_iframe = URL( - 'https://assurance-vie.axabanque.fr/Consultation/SituationContrat.aspx', - 'https://assurance-vie.axabanque.fr/Consultation/HistoriqueOperations.aspx', + r'https://assurance-vie.axabanque.fr/Consultation/SituationContrat.aspx', + r'https://assurance-vie.axabanque.fr/Consultation/HistoriqueOperations.aspx', LifeInsuranceIframe ) # netfinca bourse - bourse = URL(r'/transactionnel/client/homepage_bourseCAT.html', - r'https://bourse.axabanque.fr/netfinca-titres/servlet/com.netfinca.*', - BoursePage) + bourse = URL( + r'/transactionnel/client/homepage_bourseCAT.html', + r'https://bourse.axabanque.fr/netfinca-titres/servlet/com.netfinca.*', + BoursePage + ) bourse_history = URL(r'https://bourse.axabanque.fr/netfinca-titres/servlet/com.netfinca.frontcr.account.AccountHistory', BoursePage) # Transfer - recipients = URL('/transactionnel/client/enregistrer-nouveau-beneficiaire.html', RecipientsPage) - add_recipient = URL('/webapp/axabanque/jsp/beneficiaireSepa/saisieBeneficiaireSepaOTP.faces', AddRecipientPage) - recipient_confirmation_page = URL('/webapp/axabanque/jsp/beneficiaireSepa/saisieBeneficiaireSepaOTP.faces', RecipientConfirmationPage) - validate_transfer = URL('/webapp/axabanque/jsp/virementSepa/saisieVirementSepa.faces', ValidateTransferPage) - register_transfer = URL('/transactionnel/client/virement.html', - 'webapp/axabanque/jsp/virementSepa/saisieVirementSepa.faces', - RegisterTransferPage) + recipients = URL(r'/transactionnel/client/enregistrer-nouveau-beneficiaire.html', RecipientsPage) + add_recipient = URL(r'/webapp/axabanque/jsp/beneficiaireSepa/saisieBeneficiaireSepaOTP.faces', AddRecipientPage) + recipient_confirmation_page = URL(r'/webapp/axabanque/jsp/beneficiaireSepa/saisieBeneficiaireSepaOTP.faces', RecipientConfirmationPage) + validate_transfer = URL(r'/webapp/axabanque/jsp/virementSepa/saisieVirementSepa.faces', ValidateTransferPage) + register_transfer = URL( + r'/transactionnel/client/virement.html', + r'/webapp/axabanque/jsp/virementSepa/saisieVirementSepa.faces', + RegisterTransferPage + ) confirm_transfer = URL('/webapp/axabanque/jsp/virementSepa/confirmationVirementSepa.faces', ConfirmTransferPage) profile_page = URL('/transactionnel/client/coordonnees.html', BankProfilePage) @@ -282,11 +300,12 @@ class AXABanque(AXABrowser, StatesMixin): def get_netfinca_account(self, account): # Important: this part is controlled by modules/lcl/pages.py + owner_name = self.get_profile().name.upper().split(' ', 1)[1] self.go_account_pages(account, None) self.page.open_market() self.page.open_market_next() self.page.open_iframe() - for bourse_account in self.page.get_list(): + for bourse_account in self.page.get_list(name=owner_name): self.logger.debug('iterating account %r', bourse_account) bourse_id = bourse_account.id.replace('bourse', '') if account.id.startswith(bourse_id): @@ -388,7 +407,7 @@ class AXABanque(AXABrowser, StatesMixin): self.go_account_pages(account, 'history') if self.page.more_history(): - for tr in self.page.get_history(): + for tr in sorted_transactions(self.page.get_history()): yield tr # Get deferred card history elif account._acctype == 'bank' and account.type == Account.TYPE_CARD: @@ -580,10 +599,28 @@ class AXAAssurance(AXABrowser): AccountDetailsPage ) investment = URL(r'/content/ecc-popin-cards/savings/[^/]+/repartition', InvestmentPage) - documents = URL(r'/content/espace-client/accueil/mes-documents/attestations-d-assurances.content-inner.din_CERTIFICATE.html', DocumentsPage) - download = URL(r'/content/ecc-popin-cards/technical/detailed/document.downloadPdf.html', - r'/content/ecc-popin-cards/technical/detailed/document/_jcr_content/', - DownloadPage) + + documents_life_insurance = URL( + r'/content/espace-client/accueil/mes-documents/situations-de-contrats-assurance-vie.content-inner.din_SAVINGS_STATEMENT.html', + DocumentsPage + ) + documents_certificates = URL( + r'/content/espace-client/accueil/mes-documents/attestations-d-assurances.content-inner.din_CERTIFICATE.html', + DocumentsPage + ) + documents_tax_area = URL( + r'https://espaceclient.axa.fr/content/espace-client/accueil/mes-documents/espace-fiscal.content-inner.din_TAX.html', + DocumentsPage + ) + documents_membership_fee = URL( + r'/content/espace-client/accueil/mes-documents/avis-d-echeance.content-inner.din_PREMIUM_STATEMENT.html', + DocumentsPage + ) + + download = URL( + r'/content/ecc-popin-cards/technical/detailed/download-document.downloadPdf.html', + DownloadPage + ) profile = URL(r'/content/ecc-popin-cards/transverse/userprofile.content-inner.html\?_=\d+', ProfilePage) def __init__(self, *args, **kwargs): @@ -621,7 +658,7 @@ class AXAAssurance(AXABrowser): else: self.cache['invs'][account.id] = [] for inv in portfolio_page.iter_investment(currency=account.currency): - i = [i for i in self.cache['invs'][account.id] if (i.valuation == inv.valuation and i.label == inv.label)] + i = [i2 for i2 in self.cache['invs'][account.id] if (i2.valuation == inv.valuation and i2.label == inv.label)] assert len(i) in (0, 1) if i: i[0].portfolio_share = inv.portfolio_share @@ -685,12 +722,20 @@ class AXAAssurance(AXABrowser): @need_login def iter_documents(self, subscription): - return self.documents.go().get_documents(subid=subscription.id) + document_urls = [ + self.documents_life_insurance, + self.documents_certificates, + self.documents_tax_area, + self.documents_membership_fee, + ] + for url in document_urls: + url.go() + for doc in self.page.get_documents(subid=subscription.id): + yield doc @need_login - def download_document(self, url): - self.location(url) - self.page.create_document() + def download_document(self, download_id): + self.download.go(data={'documentId': download_id}) return self.page.content @need_login diff --git a/modules/axabanque/module.py b/modules/axabanque/module.py index 620d37c8105a81d0a1b87c622ee2f2716bfe40be..29a61c4ece89d1c995bf653b439f8e0824a661fd 100644 --- a/modules/axabanque/module.py +++ b/modules/axabanque/module.py @@ -17,9 +17,10 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . +from __future__ import unicode_literals from weboob.capabilities.bank import CapBankWealth, CapBankTransferAddRecipient, AccountNotFound, RecipientNotFound -from weboob.capabilities.base import find_object, NotAvailable, empty +from weboob.capabilities.base import find_object, empty from weboob.capabilities.bank import Account, TransferInvalidLabel from weboob.capabilities.profile import CapProfile from weboob.capabilities.bill import CapDocument, Subscription, Document, DocumentNotFound, SubscriptionNotFound @@ -34,13 +35,15 @@ __all__ = ['AXABanqueModule'] class AXABanqueModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapDocument, CapProfile): NAME = 'axabanque' - MAINTAINER = u'Romain Bignon' + MAINTAINER = 'Romain Bignon' EMAIL = 'romain@weboob.org' VERSION = '1.6' - DESCRIPTION = u'AXA Banque' + DESCRIPTION = 'AXA Banque' LICENSE = 'LGPLv3+' - CONFIG = BackendConfig(ValueBackendPassword('login', label='Identifiant', masked=False), - ValueBackendPassword('password', label='Code', regexp='\d+')) + CONFIG = BackendConfig( + ValueBackendPassword('login', label='Identifiant', masked=False), + ValueBackendPassword('password', label='Code', regexp='\d+'), + ) BROWSER = AXABanque def create_default_browser(self): @@ -140,9 +143,7 @@ class AXABanqueModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapDoc def download_document(self, document): if not isinstance(document, Document): document = self.get_document(document) - if document.url is NotAvailable: - return - return self.browser.download_document(document.url) + return self.browser.download_document(document._download_id) def iter_resources(self, objs, split_path): if Account in objs: diff --git a/modules/axabanque/pages/bank.py b/modules/axabanque/pages/bank.py index 3d24a3a37dba4d252e3c5f46f4dd52c9c912d88e..2585a59ae6545baf80c62cb1f64b04880e55cd11 100644 --- a/modules/axabanque/pages/bank.py +++ b/modules/axabanque/pages/bank.py @@ -42,6 +42,7 @@ def MyDecimal(*args, **kwargs): kwargs.update(replace_dots=True, default=NotAvailable) return CleanDecimal(*args, **kwargs) + class UnavailablePage(HTMLPage): def on_load(self): raise BrowserUnavailable() @@ -71,23 +72,23 @@ class MyHTMLPage(HTMLPage): sub = re.search('oamSubmitForm.+?,\'([^:]+).([^\']+)', s) args['%s:_idcl' % sub.group(1)] = "%s:%s" % (sub.group(1), sub.group(2)) args['%s_SUBMIT' % sub.group(1)] = 1 - args['_form_name'] = sub.group(1) # for weboob only + args['_form_name'] = sub.group(1) # for weboob only return args class AccountsPage(LoggedPage, MyHTMLPage): ACCOUNT_TYPES = OrderedDict(( - ('visa', Account.TYPE_CARD), - ('pea', Account.TYPE_PEA), - ('valorisation', Account.TYPE_MARKET), - ('courant-titre', Account.TYPE_CHECKING), - ('courant', Account.TYPE_CHECKING), - ('livret', Account.TYPE_SAVINGS), - ('ldd', Account.TYPE_SAVINGS), - ('pel', Account.TYPE_SAVINGS), - ('cel', Account.TYPE_SAVINGS), - ('titres', Account.TYPE_MARKET), + ('visa', Account.TYPE_CARD), + ('pea', Account.TYPE_PEA), + ('valorisation', Account.TYPE_MARKET), + ('courant-titre', Account.TYPE_CHECKING), + ('courant', Account.TYPE_CHECKING), + ('livret', Account.TYPE_SAVINGS), + ('ldd', Account.TYPE_SAVINGS), + ('pel', Account.TYPE_SAVINGS), + ('cel', Account.TYPE_SAVINGS), + ('titres', Account.TYPE_MARKET), )) def get_tabs(self): @@ -169,11 +170,11 @@ class AccountsPage(LoggedPage, MyHTMLPage): self.logger.debug('Args: %r' % args) if 'paramNumCompte' not in args: - #The displaying of life insurances is very different from the other + # The displaying of life insurances is very different from the other if args.get('idPanorama:_idcl').split(":")[1] == 'tableaux-direct-solution-vie': account_details = self.browser.open("#", data=args) scripts = account_details.page.doc.xpath('//script[@type="text/javascript"]/text()') - script = filter(lambda x: "src" in x, scripts)[0] + script = list(filter(lambda x: "src" in x, scripts))[0] iframe_url = re.search("src:(.*),", script).group()[6:-2] account_details_iframe = self.browser.open(iframe_url, data=args) account.id = CleanText('//span[contains(@id,"NumeroContrat")]/text()')(account_details_iframe.page.doc) @@ -233,7 +234,7 @@ class AccountsPage(LoggedPage, MyHTMLPage): account.balance = Decimal(0) except InvalidOperation: - #The account doesn't have a amount + # The account doesn't have a amount pass account._url = self.doc.xpath('//form[contains(@action, "panorama")]/@action')[0] @@ -259,6 +260,7 @@ class AccountsPage(LoggedPage, MyHTMLPage): def get_profile_name(self): return Regexp(CleanText('//div[@id="bloc_identite"]/h5'), r'Bonjour (.*)')(self.doc) + class IbanPage(PDFPage): def get_iban(self): iban = '' @@ -269,7 +271,7 @@ class IbanPage(PDFPage): # findall will find something like # ['FRXX', '1234', ... , '9012', 'FRXX', '1234', ... , '9012'] iban += part - iban = iban[:len(iban)//2] + iban = iban[:len(iban) // 2] # we suppose that all iban are French iban iban_last_part = re.findall(r'([A-Z0-9]{3})\1\1Titulaire', extract_text(self.data), flags=re.MULTILINE) @@ -283,26 +285,21 @@ class IbanPage(PDFPage): class BankTransaction(FrenchTransaction): - PATTERNS = [(re.compile('^RET(RAIT) DAB (?P
\d{2})/(?P\d{2}) (?P.*)'), - FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile('^(CARTE|CB ETRANGER|CB) (?P
\d{2})/(?P\d{2}) (?P.*)'), - FrenchTransaction.TYPE_CARD), - (re.compile('^(?PVIR(EMEN)?T? (SEPA)?(RECU|FAVEUR)?)( /FRM)?(?P.*)'), - FrenchTransaction.TYPE_TRANSFER), - (re.compile('^PRLV (?P.*)( \d+)?$'), FrenchTransaction.TYPE_ORDER), - (re.compile('^(CHQ|CHEQUE) .*$'), FrenchTransaction.TYPE_CHECK), - (re.compile('^(AGIOS /|FRAIS) (?P.*)'), FrenchTransaction.TYPE_BANK), - (re.compile('^(CONVENTION \d+ |F )?COTIS(ATION)? (?P.*)'), - FrenchTransaction.TYPE_BANK), - (re.compile('^(F|R)-(?P.*)'), FrenchTransaction.TYPE_BANK), - (re.compile('^REMISE (?P.*)'), FrenchTransaction.TYPE_DEPOSIT), - (re.compile('^(?P.*)( \d+)? QUITTANCE .*'), - FrenchTransaction.TYPE_ORDER), - (re.compile('^.* LE (?P
\d{2})/(?P\d{2})/(?P\d{2})$'), - FrenchTransaction.TYPE_UNKNOWN), - (re.compile('^ACHATS (CARTE|CB)'), FrenchTransaction.TYPE_CARD_SUMMARY), - (re.compile('^ANNUL (?P.*)'), FrenchTransaction.TYPE_PAYBACK) - ] + PATTERNS = [ + (re.compile(r'^RET(RAIT) DAB (?P
\d{2})/(?P\d{2}) (?P.*)'), FrenchTransaction.TYPE_WITHDRAWAL), + (re.compile(r'^(CARTE|CB ETRANGER|CB) (?P
\d{2})/(?P\d{2}) (?P.*)'), FrenchTransaction.TYPE_CARD), + (re.compile(r'^(?PVIR(EMEN)?T? (SEPA)?(RECU|FAVEUR)?)( /FRM)?(?P.*)'), FrenchTransaction.TYPE_TRANSFER), + (re.compile(r'^PRLV (?P.*)( \d+)?$'), FrenchTransaction.TYPE_ORDER), + (re.compile(r'^(CHQ|CHEQUE) .*$'), FrenchTransaction.TYPE_CHECK), + (re.compile(r'^(AGIOS /|FRAIS) (?P.*)'), FrenchTransaction.TYPE_BANK), + (re.compile(r'^(CONVENTION \d+ |F )?COTIS(ATION)? (?P.*)'), FrenchTransaction.TYPE_BANK), + (re.compile(r'^(F|R)-(?P.*)'), FrenchTransaction.TYPE_BANK), + (re.compile(r'^REMISE (?P.*)'), FrenchTransaction.TYPE_DEPOSIT), + (re.compile(r'^(?P.*)( \d+)? QUITTANCE .*'), FrenchTransaction.TYPE_ORDER), + (re.compile(r'^.* LE (?P
\d{2})/(?P\d{2})/(?P\d{2})$'), FrenchTransaction.TYPE_UNKNOWN), + (re.compile(r'^ACHATS (CARTE|CB)'), FrenchTransaction.TYPE_CARD_SUMMARY), + (re.compile(r'^ANNUL (?P.*)'), FrenchTransaction.TYPE_PAYBACK) + ] class TransactionsPage(LoggedPage, MyHTMLPage): @@ -384,24 +381,26 @@ class TransactionsPage(LoggedPage, MyHTMLPage): if link is None: # this is a check account - args = {'categorieMouvementSelectionnePagination': 'afficherTout', - 'nbLigneParPageSelectionneHautPagination': -1, - 'nbLigneParPageSelectionneBasPagination': -1, - 'nbLigneParPageSelectionneComponent': -1, - 'idDetail:btnRechercherParNbLigneParPage': '', - 'idDetail_SUBMIT': 1, - 'javax.faces.ViewState': self.get_view_state(), - } + args = { + 'categorieMouvementSelectionnePagination': 'afficherTout', + 'nbLigneParPageSelectionneHautPagination': -1, + 'nbLigneParPageSelectionneBasPagination': -1, + 'nbLigneParPageSelectionneComponent': -1, + 'idDetail:btnRechercherParNbLigneParPage': '', + 'idDetail_SUBMIT': 1, + 'javax.faces.ViewState': self.get_view_state(), + } else: # something like a PEA or so value = link.attrib['id'] id = value.split(':')[0] - args = {'%s:_idcl' % id: value, - '%s:_link_hidden_' % id: '', - '%s_SUBMIT' % id: 1, - 'javax.faces.ViewState': self.get_view_state(), - 'paramNumCompte': '', - } + args = { + '%s:_idcl' % id: value, + '%s:_link_hidden_' % id: '', + '%s_SUBMIT' % id: 1, + 'javax.faces.ViewState': self.get_view_state(), + 'paramNumCompte': '', + } self.browser.location(form.attrib['action'], data=args) return True @@ -427,10 +426,10 @@ class TransactionsPage(LoggedPage, MyHTMLPage): return True def get_history(self): - #DAT account can't have transaction + # DAT account can't have transaction if self.doc.xpath('//table[@id="table-dat"]'): return - #These accounts have investments, no transactions + # These accounts have investments, no transactions if self.doc.xpath('//table[@id="InfosPortefeuille"]'): return tables = self.doc.xpath('//table[@id="table-detail-operation"]') @@ -534,7 +533,7 @@ class LifeInsuranceIframe(LoggedPage, HTMLPage): def obj_diff_ratio(self): diff_percent = MyDecimal(TableCell('diff')(self)[0])(self) - return diff_percent/100 if diff_percent != NotAvailable else diff_percent + return diff_percent / 100 if diff_percent != NotAvailable else diff_percent @method class iter_history(TableElement): diff --git a/modules/axabanque/pages/document.py b/modules/axabanque/pages/document.py index 3f2228c8cbad22c70c954b4406b632b18121ac9a..b338151e6ba42c7235a92d3f184ee340f59a8b9a 100644 --- a/modules/axabanque/pages/document.py +++ b/modules/axabanque/pages/document.py @@ -17,35 +17,45 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . +from __future__ import unicode_literals + from weboob.browser.pages import HTMLPage, LoggedPage -from weboob.browser.filters.standard import CleanText, Env, Regexp, Format -from weboob.browser.elements import ListElement, ItemElement, method, SkipItem +from weboob.browser.filters.standard import CleanText, Env, Regexp, Format, Date +from weboob.browser.elements import ListElement, ItemElement, method from weboob.capabilities.bill import Document -from weboob.tools.compat import urljoin +from weboob.tools.date import parse_french_date class DocumentsPage(LoggedPage, HTMLPage): @method class get_documents(ListElement): - item_xpath = '//article' + item_xpath = '//div[has-class("mawa-cards-item dashboard-item")]' class item(ItemElement): klass = Document - obj_id = Format('%s_%s', Env('subid'), Regexp(CleanText('./@data-route'), '#/details/(.*)')) - obj_format = u"pdf" - obj_label = CleanText('.//h2') - obj_type = u"document" - - def obj_url(self): - url = urljoin(self.page.browser.BASEURL, CleanText('./@data-url')(self)) - self.page.browser.location(url) - if self.page.doc.xpath('//form[contains(., "Afficher")]'): - return url - raise SkipItem() + obj_id = Format( + '%s_%s', + Env('subid'), + Regexp(CleanText('./@data-module-open-link--link'), '#/details/(.*)'), + ) + obj_format = 'pdf' + # eg when formatted (not complete list): + # - Situation de contrat suite à réajustement automatique Assurance Vie N° XXXXXXXXXX + # - Lettre d'information client Assurance Vie N° XXXXXXXXXX + # - Attestation de rachat partiel Assurance Vie N° XXXXXXXXXXXXXX + obj_label = Format( + '%s %s %s', + CleanText('.//h3[@class="card-title"]'), + CleanText('.//div[@class="sticker-content"]//strong'), + CleanText('.//p[@class="contract-info"]'), + ) + obj_date = Date(CleanText('.//p[@class="card-date"]'), parse_func=parse_french_date) + obj_type = 'document' + obj__download_id = Regexp(CleanText('./@data-url'), r'.did_(.*?)\.') class DownloadPage(LoggedPage, HTMLPage): def create_document(self): - form = self.get_form(xpath='//form[contains(., "Afficher")]') + form = self.get_form(xpath='//form[has-class("form-download-pdf")]') form.submit() diff --git a/modules/axabanque/pages/login.py b/modules/axabanque/pages/login.py index 70890144e68813c97748cb55254d7b515f6b09a6..9a8964a26f7b1b8c4f8c3f5d39fe905fbd87716c 100644 --- a/modules/axabanque/pages/login.py +++ b/modules/axabanque/pages/login.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . +from __future__ import unicode_literals from io import BytesIO @@ -31,35 +32,37 @@ class MyVirtKeyboard(VirtKeyboard): margin = 5, 5, 5, 5 color = (255, 255, 255) - symbols = {'0': '6959163af44cc50b3863e7e306d6e571', - '1': '98b32dff471e903b6fa8e3a0f1544b17', - '2': '32722d5b6572f9d46350aca7fb66263a', - '3': '835a9c8bf66e28f3ffa2b12994bc3f9a', - '4': 'e7457342c434da4fb0fd974f7dc37002', - '5': 'c8b74429a805e12a08c5ed87fd9730ce', - '6': '70a84c766bc323343c0c291146f652db', - '7': 'e4e7fb4f8cc90c8ad472906b5eceeb99', - '8': 'ffb78dbea5a171990e14d707d4772ba2', - '9': '063dcb4179beaeff60fb73c80cbd429d' - } - - coords = {'0': (0, 0, 40, 40), - '1': (40, 0, 80, 40), - '2': (80, 0, 120, 40), - '3': (120, 0, 160, 40), - '4': (0, 40, 40, 80), - '5': (40, 40, 80, 80), - '6': (80, 40, 120, 80), - '7': (120, 40, 160, 80), - '8': (0, 80, 40, 120), - '9': (40, 80, 80, 120), - '10': (80, 80, 120, 120), - '11': (120, 80, 160, 120), - '12': (0, 120, 40, 160), - '13': (40, 120, 80, 160), - '14': (80, 120, 120, 160), - '15': (120, 120, 160, 160) - } + symbols = { + '0': '6959163af44cc50b3863e7e306d6e571', + '1': '98b32dff471e903b6fa8e3a0f1544b17', + '2': '32722d5b6572f9d46350aca7fb66263a', + '3': '835a9c8bf66e28f3ffa2b12994bc3f9a', + '4': 'e7457342c434da4fb0fd974f7dc37002', + '5': 'c8b74429a805e12a08c5ed87fd9730ce', + '6': '70a84c766bc323343c0c291146f652db', + '7': 'e4e7fb4f8cc90c8ad472906b5eceeb99', + '8': 'ffb78dbea5a171990e14d707d4772ba2', + '9': '063dcb4179beaeff60fb73c80cbd429d', + } + + coords = { + '0': (0, 0, 40, 40), + '1': (40, 0, 80, 40), + '2': (80, 0, 120, 40), + '3': (120, 0, 160, 40), + '4': (0, 40, 40, 80), + '5': (40, 40, 80, 80), + '6': (80, 40, 120, 80), + '7': (120, 40, 160, 80), + '8': (0, 80, 40, 120), + '9': (40, 80, 80, 120), + '10': (80, 80, 120, 120), + '11': (120, 80, 160, 120), + '12': (0, 120, 40, 160), + '13': (40, 120, 80, 160), + '14': (80, 120, 120, 160), + '15': (120, 120, 160, 160), + } def __init__(self, page): VirtKeyboard.__init__(self, BytesIO(page.content), self.coords, self.color, convert='RGB') diff --git a/modules/axabanque/pages/transfer.py b/modules/axabanque/pages/transfer.py index 6afef8024f68721b50692de048355f26b4c4f867..d62972aaae32f2898aefa4c9d3dd4055b4b060e7 100644 --- a/modules/axabanque/pages/transfer.py +++ b/modules/axabanque/pages/transfer.py @@ -50,17 +50,18 @@ class TransferVirtualKeyboard(SimpleVirtualKeyboard): margin = 1 tile_margin = 10 - symbols = {'0': '715df9c139fc7b46829526229c415a67', - '1': '12d398f7f389711c5f8298ee68a8af28', - '2': 'f43ca3a5dd649d30bf02060ab65c4eff', - '3': 'b6dd7864cfd941badb0784be37f7eeb3', - '4': ('7138d0a663eef56c699d85dc6c3ac639', '0faced58777f371097a7a70bb9570dd7', ), - '5': 'b71bd38e71ce0b611642a01b6900218f', - '6': 'f71f7249413c189165da7b588c2f0493', - '7': '81fc65230d7df341e80d02e414f183d4', - '8': '8106671a6b24aee3475d6f12a650f59b', - '9': 'e8c4567eb46dba5e2a92619076441a8a' - } + symbols = { + '0': '715df9c139fc7b46829526229c415a67', + '1': '12d398f7f389711c5f8298ee68a8af28', + '2': 'f43ca3a5dd649d30bf02060ab65c4eff', + '3': 'b6dd7864cfd941badb0784be37f7eeb3', + '4': ('7138d0a663eef56c699d85dc6c3ac639', '0faced58777f371097a7a70bb9570dd7', ), + '5': 'b71bd38e71ce0b611642a01b6900218f', + '6': 'f71f7249413c189165da7b588c2f0493', + '7': '81fc65230d7df341e80d02e414f183d4', + '8': '8106671a6b24aee3475d6f12a650f59b', + '9': 'e8c4567eb46dba5e2a92619076441a8a', + } # Clean image def alter_image(self): @@ -174,9 +175,9 @@ class AddRecipientPage(LoggedPage, HTMLPage): # fill iban part _iban_rcpt_part = 4 - for i in range(3,10): + for i in range(3, 10): form_key = 'ibanContenuZone{}Hidden'.format(i) - form[form_key] = rcpt_iban[_iban_rcpt_part: _iban_rcpt_part+4] + form[form_key] = rcpt_iban[_iban_rcpt_part: _iban_rcpt_part + 4] if form[form_key]: form['ibanContenuZone{}'.format(i)] = form[form_key] _iban_rcpt_part += 4 @@ -339,7 +340,7 @@ class ValidateTransferPage(LoggedPage, HTMLPage): date = Regexp(pattern=r'(\d+/\d+/\d+)').filter(self.get_element_by_name('Date du virement')) transfer.exec_date = Date(dayfirst=True).filter(date) - account_label_id = self.get_element_by_name(u'Compte à débiter') + account_label_id = self.get_element_by_name('Compte à débiter') transfer.account_id = (Regexp(pattern=r'(\d+)').filter(account_label_id)) transfer.account_label = Regexp(pattern=r'([\w \.]+)').filter(account_label_id) # account iban is not in the summary page @@ -347,7 +348,7 @@ class ValidateTransferPage(LoggedPage, HTMLPage): transfer.recipient_id = recipient.id transfer.recipient_iban = self.get_element_by_name('IBAN').replace(' ', '') - transfer.recipient_label = self.get_element_by_name(u'Nom du bénéficiaire') + transfer.recipient_label = self.get_element_by_name('Nom du bénéficiaire') transfer.label = CleanText('//table[@id="table-confLibelle"]//p')(self.doc) return transfer @@ -357,7 +358,7 @@ class ValidateTransferPage(LoggedPage, HTMLPage): f = BytesIO(self.browser.open(img_src).content) vk = TransferVirtualKeyboard(file=f, cols=8, rows=3, - matching_symbols=string.ascii_lowercase[:8*3], browser=self.browser) + matching_symbols=string.ascii_lowercase[:8 * 3], browser=self.browser) return vk.get_string_code(password) diff --git a/modules/banquepopulaire/browser.py b/modules/banquepopulaire/browser.py index 774963a462ea27833a872844eaa9105263540e7e..d807d2596b20ceef6890b98e191b9e64917de89e 100644 --- a/modules/banquepopulaire/browser.py +++ b/modules/banquepopulaire/browser.py @@ -40,7 +40,7 @@ from .pages import ( IbanPage, AdvisorPage, TransactionDetailPage, TransactionsBackPage, NatixisPage, EtnaPage, NatixisInvestPage, NatixisHistoryPage, NatixisErrorPage, NatixisDetailsPage, NatixisChoicePage, NatixisRedirect, - LineboursePage, AlreadyLoginPage, + LineboursePage, AlreadyLoginPage, InvestmentPage, ) from .document_pages import BasicTokenPage, SubscriberPage, SubscriptionsPage, DocumentsPage @@ -128,6 +128,8 @@ class BanquePopulaire(LoginBrowser): r'https://[^/]+/cyber/internet/Sort.do\?.*', TransactionsPage) + investment_page = URL(r'https://[^/]+/cyber/ibp/ate/skin/internet/pages/webAppReroutingAutoSubmit.jsp', InvestmentPage) + transactions_back_page = URL(r'https://[^/]+/cyber/internet/ContinueTask.do\?.*ActionPerformed=BACK.*', TransactionsBackPage) transaction_detail_page = URL(r'https://[^/]+/cyber/internet/ContinueTask.do\?.*dialogActionPerformed=DETAIL_ECRITURE.*', TransactionDetailPage) @@ -168,7 +170,7 @@ class BanquePopulaire(LoginBrowser): natixis_history = URL(r'https://www.assurances.natixis.fr/espaceinternet-bp/rest/v2/contratVie/load-operation/(?P\w+)/(?P\w+)/(?P\w+)', NatixisHistoryPage) natixis_pdf = URL(r'https://www.assurances.natixis.fr/espaceinternet-bp/rest/v2/contratVie/load-releve/(?P\w+)/(?P\w+)/(?P\w+)/(?P\d+)', NatixisDetailsPage) - linebourse_home = URL(r'https://www.linebourse.fr/ReroutageSJR', LineboursePage) + linebourse_home = URL(r'https://www.linebourse.fr', LineboursePage) advisor = URL(r'https://[^/]+/cyber/internet/StartTask.do\?taskInfoOID=accueil.*', r'https://[^/]+/cyber/internet/StartTask.do\?taskInfoOID=contacter.*', AdvisorPage) @@ -194,7 +196,6 @@ class BanquePopulaire(LoginBrowser): dirname += '/bourse' self.linebourse = LinebourseBrowser('https://www.linebourse.fr', logger=self.logger, responses_dirname=dirname, weboob=self.weboob, proxy=self.PROXIES) - self.investments = {} self.documents_headers = None def deinit(self): @@ -248,7 +249,7 @@ class BanquePopulaire(LoginBrowser): raise BrowserIncorrectPassword() if 'internetRescuePortal' in self.url: # 1 more request is necessary - data = {'integrationMode': 'INTERNET_RESCUE'} + data = {'integrationMode': 'INTERNET_RESCUE'} self.location('/cyber/internet/Login.do', data=data) ACCOUNT_URLS = ['mesComptes', 'mesComptesPRO', 'maSyntheseGratuite', 'accueilSynthese', 'equipementComplet'] @@ -257,19 +258,26 @@ class BanquePopulaire(LoginBrowser): @need_login def go_on_accounts_list(self): for taskInfoOID in self.ACCOUNT_URLS: + # 4 possible URLs but we stop as soon as one of them works data = OrderedDict([('taskInfoOID', taskInfoOID), ('token', self.token)]) + + # Go from AdvisorPage to AccountsPage self.location(self.absurl('/cyber/internet/StartTask.do', base=True), params=data) + if not self.page.is_error(): if self.page.pop_up(): self.logger.debug('Popup displayed, retry') data = OrderedDict([('taskInfoOID', taskInfoOID), ('token', self.token)]) self.location('/cyber/internet/StartTask.do', params=data) + + # Set the valid ACCOUNT_URL and break the loop self.ACCOUNT_URLS = [taskInfoOID] break else: raise BrokenPageError('Unable to go on the accounts list page') if self.page.is_short_list(): + # Go from AccountsPage to AccountsFullPage to get the full accounts list form = self.page.get_form(nr=0) form['dialogActionPerformed'] = 'EQUIPEMENT_COMPLET' form['token'] = self.page.build_token(form['token']) @@ -280,16 +288,23 @@ class BanquePopulaire(LoginBrowser): @retry(LoggedOut) @need_login - def get_accounts_list(self, get_iban=True): - # We have to parse account list in 2 different way depending if we want the iban number or not - # thanks to stateful website + def iter_accounts(self, get_iban=True): + # We have to parse account list in 2 different way depending if + # we want the iban number or not thanks to stateful website next_pages = [] accounts = [] profile = self.get_profile() + if profile.name: - owner_name = re.search(r' (.+)', profile.name).group(1).upper() + name = profile.name + else: + name = profile.company_name + + # Handle names/company names without spaces + if ' ' in name: + owner_name = re.search(r' (.+)', name).group(1).upper() else: - owner_name = re.search(r' (.+)', profile.company_name).group(1).upper() + owner_name = name.upper() self.go_on_accounts_list() @@ -311,6 +326,7 @@ class BanquePopulaire(LoginBrowser): params['token'] = self.page.build_token(self.token) self.location('/cyber/internet/ContinueTask.do', data=params) + # Go to next_page with params and token next_page['token'] = self.page.build_token(self.token) self.location('/cyber/internet/ContinueTask.do', data=next_page) @@ -323,8 +339,6 @@ class BanquePopulaire(LoginBrowser): if get_iban: for a in accounts: a.iban = self.get_iban_number(a) - for a in accounts: - self.get_investment(a) yield a # TODO: see if there's other type of account with a label without name which @@ -358,7 +372,7 @@ class BanquePopulaire(LoginBrowser): @retry(LoggedOut) @need_login def get_account(self, id): - return find_object(self.get_accounts_list(False), id=id) + return find_object(self.iter_accounts(get_iban=False), id=id) def set_gocardless_transaction_details(self, transaction): # Setting references for a GoCardless transaction @@ -378,7 +392,7 @@ class BanquePopulaire(LoginBrowser): @retry(LoggedOut) @need_login - def get_history(self, account, coming=False): + def iter_history(self, account, coming=False): def get_history_by_receipt(account, coming, sel_tbl1=None): account = self.get_account(account.id) @@ -400,7 +414,7 @@ class BanquePopulaire(LoginBrowser): return params['token'] = self.page.build_token(params['token']) - if sel_tbl1 != None: + if sel_tbl1 is not None: params['attribute($SEL_$tbl1)'] = str(sel_tbl1) self.location(self.absurl('/cyber/internet/ContinueTask.do', base=True), data=params) @@ -408,12 +422,17 @@ class BanquePopulaire(LoginBrowser): if not self.page or self.error_page.is_here() or self.page.no_operations(): return - # Sort by values dates (see comment in TransactionsPage.get_history) + # Sort by operation date if len(self.page.doc.xpath('//a[@id="tcl4_srt"]')) > 0: - form = self.page.get_form(id='myForm') - form.url = self.absurl('/cyber/internet/Sort.do?property=tbl1&sortBlocId=blc2&columnName=dateValeur') - params['token'] = self.page.build_token(params['token']) - form.submit() + # The first request sort might transaction by oldest. If this is the case, + # we need to do the request a second time for the transactions to be sorted by newest. + for _ in range(2): + form = self.page.get_form(id='myForm') + form.url = self.absurl('/cyber/internet/Sort.do?property=tbl1&sortBlocId=blc2&columnName=dateOperation') + params['token'] = self.page.build_token(params['token']) + form.submit() + if self.page.is_sorted_by_most_recent(): + break transactions_next_page = True @@ -421,6 +440,7 @@ class BanquePopulaire(LoginBrowser): assert self.transactions_page.is_here() transaction_list = self.page.get_history(account, coming) + for tr in transaction_list: # Add information about GoCardless if 'GoCardless' in tr.label and tr._has_link: @@ -445,7 +465,6 @@ class BanquePopulaire(LoginBrowser): @need_login def go_investments(self, account, get_account=False): - if not account._invest_params and not (account.id.startswith('TIT') or account.id.startswith('PRV')): raise NotImplementedError() @@ -453,10 +472,11 @@ class BanquePopulaire(LoginBrowser): account = self.get_account(account.id) if account._params: - params = {'taskInfoOID': "ordreBourseCTJ", - 'controlPanelTaskAction': "true", - 'token': self.page.build_token(account._params['token']) - } + params = { + 'taskInfoOID': 'ordreBourseCTJ', + 'controlPanelTaskAction': 'true', + 'token': self.page.build_token(account._params['token']), + } self.location(self.absurl('/cyber/internet/StartTask.do', base=True), params=params) else: params = account._invest_params @@ -469,78 +489,74 @@ class BanquePopulaire(LoginBrowser): if self.error_page.is_here(): raise NotImplementedError() - url, params = self.page.get_investment_page_params() - if params: - try: - self.location(url, data=params) - except BrowserUnavailable: - return False - - if "linebourse" in self.url: - self.linebourse.session.cookies.update(self.session.cookies) - self.linebourse.invest.go() + if self.page.go_investment(): + url, params = self.page.get_investment_page_params() + if params: + try: + self.location(url, data=params) + except BrowserUnavailable: + return False - if self.natixis_error_page.is_here(): - self.logger.warning("natixis site doesn't work") - return False + if 'linebourse' in self.url: + self.linebourse.session.cookies.update(self.session.cookies) + self.linebourse.invest.go() - if self.natixis_redirect.is_here(): - url = self.page.get_redirect() - if re.match(r'https://www.assurances.natixis.fr/etna-ihs-bp/#/equipement;codeEtab=\d+\?windowId=[a-f0-9]+$', url): - self.logger.warning('there may be no contract associated with %s, skipping', url) + if self.natixis_error_page.is_here(): + self.logger.warning('Natixis site does not work.') return False + + if self.natixis_redirect.is_here(): + url = self.page.get_redirect() + if re.match(r'https://www.assurances.natixis.fr/etna-ihs-bp/#/equipement;codeEtab=\d+\?windowId=[a-f0-9]+$', url): + self.logger.warning('There may be no contract associated with %s, skipping', url) + return False return True @need_login - def get_investment(self, account): - if account.type in (Account.TYPE_LOAN,): - self.investments[account.id] = [] - return [] + def iter_investments(self, account): + if account.type not in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_PEA, Account.TYPE_MARKET, Account.TYPE_PERP): + return # Add "Liquidities" investment if the account is a "Compte titres PEA": if account.type == Account.TYPE_PEA and account.id.startswith('CPT'): - self.investments[account.id] = [create_french_liquidity(account.balance)] - return self.investments[account.id] + yield create_french_liquidity(account.balance) + return - if account.id in self.investments.keys() and self.investments[account.id] is False: - raise NotImplementedError() + if self.go_investments(account, get_account=True): + # Redirection URL is https://www.linebourse.fr/ReroutageSJR + if 'linebourse' in self.url: + self.logger.warning('Going to Linebourse space to fetch investments.') + # Eliminating the 3 letters prefix to match IDs on Linebourse: + linebourse_id = account.id[3:] + for inv in self.linebourse.iter_investment(linebourse_id): + yield inv + return - if account.id not in self.investments.keys(): - self.investments[account.id] = [] - try: - if self.go_investments(account, get_account=True): - # Redirection URL is https://www.linebourse.fr/ReroutageSJR - if "linebourse" in self.url: - # Eliminating the 3 letters prefix to match IDs on Linebourse: - linebourse_id = account.id[3:] - for inv in self.linebourse.iter_investment(linebourse_id): - self.investments[account.id].append(inv) - - if self.etna.is_here(): - params = self.page.params - elif self.natixis_redirect.is_here(): - # the url may contain a "#", so we cannot make a request to it, the params after "#" would be dropped - url = self.page.get_redirect() - self.logger.debug('using redirect url %s', url) - m = self.etna.match(url) - if not m: - # url can be contratPrev which is not investments - self.logger.debug('Unable to handle this kind of contract') - raise NotImplementedError() - - params = m.groupdict() - - if self.natixis_redirect.is_here() or self.etna.is_here(): - try: - self.natixis_invest.go(**params) - except ServerError: - # Broken website .. nothing to do. - self.investments[account.id] = iter([]) - return self.investments[account.id] - self.investments[account.id] = list(self.page.get_investments()) - except NotImplementedError: - self.investments[account.id] = [] - return self.investments[account.id] + if self.etna.is_here(): + self.logger.warning('Going to Etna space to fetch investments.') + params = self.page.params + + elif self.natixis_redirect.is_here(): + self.logger.warning('Going to Natixis space to fetch investments.') + # the url may contain a "#", so we cannot make a request to it, the params after "#" would be dropped + url = self.page.get_redirect() + self.logger.debug('using redirect url %s', url) + m = self.etna.match(url) + if not m: + # URL can be contratPrev which is not investments + self.logger.warning('Unable to handle this kind of contract.') + return + + params = m.groupdict() + + if self.natixis_redirect.is_here() or self.etna.is_here(): + try: + self.natixis_invest.go(**params) + except ServerError: + # Broken website... nothing to do. + return + for inv in self.page.iter_investments(): + yield inv @need_login def get_invest_history(self, account): @@ -591,7 +607,7 @@ class BanquePopulaire(LoginBrowser): @need_login def get_profile(self): - self.location('/cyber/internet/StartTask.do?taskInfoOID=accueil&token=%s' % self.token) + self.location(self.absurl('/cyber/internet/StartTask.do?taskInfoOID=accueil&token=%s' % self.token, base=True)) return self.page.get_profile() @retry(LoggedOut) diff --git a/modules/banquepopulaire/module.py b/modules/banquepopulaire/module.py index b569d4521f8eb4fd4bb0618fa782391a2371e014..6523bfc728d89e8696b3c004259456eba8560afc 100644 --- a/modules/banquepopulaire/module.py +++ b/modules/banquepopulaire/module.py @@ -69,9 +69,13 @@ class BanquePopulaireModule(Module, CapBankWealth, CapContact, CapProfile, CapDo 'www.ibps.sud.banquepopulaire.fr': 'Sud', 'www.ibps.valdefrance.banquepopulaire.fr': 'Val de France', }.items(), key=lambda k_v: (k_v[1], k_v[0]))]) - CONFIG = BackendConfig(Value('website', label='Région', choices=website_choices), - ValueBackendPassword('login', label='Identifiant', masked=False), - ValueBackendPassword('password', label='Mot de passe')) + + CONFIG = BackendConfig( + Value('website', label='Région', choices=website_choices), + ValueBackendPassword('login', label='Identifiant', masked=False), + ValueBackendPassword('password', label='Mot de passe') + ) + BROWSER = BanquePopulaire accepted_document_types = (DocumentTypes.STATEMENT,) @@ -89,10 +93,13 @@ class BanquePopulaireModule(Module, CapBankWealth, CapContact, CapProfile, CapDo ('ouest.banquepopulaire', 'bpgo.banquepopulaire'), ] website = reduce(lambda a, kv: a.replace(*kv), repls, self.config['website'].get()) - return self.create_browser(website, - self.config['login'].get(), - self.config['password'].get(), - weboob=self.weboob) + + return self.create_browser( + website, + self.config['login'].get(), + self.config['password'].get(), + weboob=self.weboob + ) def iter_accounts(self): return self.browser.get_accounts_list() @@ -105,13 +112,13 @@ class BanquePopulaireModule(Module, CapBankWealth, CapContact, CapProfile, CapDo raise AccountNotFound() def iter_history(self, account): - return self.browser.get_history(account) + return self.browser.iter_history(account) def iter_coming(self, account): - return self.browser.get_history(account, coming=True) + return self.browser.iter_history(account, coming=True) def iter_investment(self, account): - return self.browser.get_investment(account) + return self.browser.iter_investments(account) def iter_contacts(self): return self.browser.get_advisor() diff --git a/modules/banquepopulaire/pages.py b/modules/banquepopulaire/pages.py index a17d468ac2b178c3df4dda848b33339ee5c7a906..da36b059171169d734ab3ee68dcb9989cf52b756 100644 --- a/modules/banquepopulaire/pages.py +++ b/modules/banquepopulaire/pages.py @@ -84,7 +84,7 @@ class WikipediaARC4(object): self.x = 0 def crypt(self, input): - output = [None]*len(input) + output = [None] * len(input) for i in range(len(input)): self.x = (self.x + 1) & 0xFF self.y = (self.state[self.x] + self.y) & 0xFF @@ -113,11 +113,12 @@ class BasePage(object): def is_error(self): for script in self.doc.xpath('//script'): - if script.text is not None and \ - (u"Le service est momentanément indisponible" in script.text or - u"Le service est temporairement indisponible" in script.text or - u"Votre abonnement ne vous permet pas d'accéder à ces services" in script.text or - u'Merci de bien vouloir nous en excuser' in script.text): + if script.text is not None and ( + "Le service est momentanément indisponible" in script.text + or "Le service est temporairement indisponible" in script.text + or "Votre abonnement ne vous permet pas d'accéder à ces services" in script.text + or 'Merci de bien vouloir nous en excuser' in script.text + ): return True return False @@ -159,9 +160,10 @@ class BasePage(object): continue for id, action, strategy in re.findall(r'''attEvt\(window,"(?P[^"]+)","click","sab\('(?P[^']+)','(?P[^']+)'\);"''', script.text, re.MULTILINE): - actions[id] = {'dialogActionPerformed': action, - 'validationStrategy': strategy, - } + actions[id] = { + 'dialogActionPerformed': action, + 'validationStrategy': strategy, + } return actions def get_back_button_params(self, params=None, actions=None): @@ -272,7 +274,7 @@ class RedirectPage(LoggedPage, MyHTMLPage): class ErrorPage(LoggedPage, MyHTMLPage): def on_load(self): if CleanText('//script[contains(text(), "momentanément indisponible")]')(self.doc): - raise BrowserUnavailable(u"Le service est momentanément indisponible") + raise BrowserUnavailable("Le service est momentanément indisponible") elif CleanText('//h1[contains(text(), "Cette page est indisponible")]')(self.doc): raise BrowserUnavailable('Cette page est indisponible') return super(ErrorPage, self).on_load() @@ -381,12 +383,12 @@ class Login2Page(LoginPage): def login(self, login, password): payload = { 'validate': { - self.form_id[0]: [ { + self.form_id[0]: [{ 'id': self.form_id[1], 'login': login.upper(), 'password': password, 'type': 'PASSWORD_LOOKUP', - } ] + }] } } url = self.request_url + '/step' @@ -436,8 +438,8 @@ class Login2Page(LoginPage): if 'phase' in doc and doc['phase']['state'] == "ENROLLMENT": raise ActionNeeded() - if (('phase' in doc and doc['phase']['previousResult'] == 'FAILED_AUTHENTICATION') or - doc['response']['status'] != 'AUTHENTICATION_SUCCESS'): + if (('phase' in doc and doc['phase']['previousResult'] == 'FAILED_AUTHENTICATION') + or doc['response']['status'] != 'AUTHENTICATION_SUCCESS'): raise BrowserIncorrectPassword() data = {'SAMLResponse': doc['response']['saml2_post']['samlResponse']} @@ -523,22 +525,23 @@ class HomePage(LoggedPage, MyHTMLPage): class AccountsPage(LoggedPage, MyHTMLPage): - ACCOUNT_TYPES = {u'Mes comptes d\'épargne': Account.TYPE_SAVINGS, - u'Mon épargne': Account.TYPE_SAVINGS, - u'Placements': Account.TYPE_SAVINGS, - u'Liste complète de mon épargne': Account.TYPE_SAVINGS, - u'Mes comptes': Account.TYPE_CHECKING, - u'Comptes en euros': Account.TYPE_CHECKING, - u'Mes comptes en devises': Account.TYPE_CHECKING, - u'Liste complète de mes comptes': Account.TYPE_CHECKING, - u'Mes emprunts': Account.TYPE_LOAN, - u'Liste complète de mes emprunts': Account.TYPE_LOAN, - u'Financements': Account.TYPE_LOAN, - u'Liste complète de mes engagements': Account.TYPE_LOAN, - u'Mes services': None, # ignore this kind of accounts (no bank ones) - u'Équipements': None, # ignore this kind of accounts (no bank ones) - u'Synthèse': None, # ignore this title - } + ACCOUNT_TYPES = { + 'Mes comptes d\'épargne': Account.TYPE_SAVINGS, + 'Mon épargne': Account.TYPE_SAVINGS, + 'Placements': Account.TYPE_SAVINGS, + 'Liste complète de mon épargne': Account.TYPE_SAVINGS, + 'Mes comptes': Account.TYPE_CHECKING, + 'Comptes en euros': Account.TYPE_CHECKING, + 'Mes comptes en devises': Account.TYPE_CHECKING, + 'Liste complète de mes comptes': Account.TYPE_CHECKING, + 'Mes emprunts': Account.TYPE_LOAN, + 'Liste complète de mes emprunts': Account.TYPE_LOAN, + 'Financements': Account.TYPE_LOAN, + 'Liste complète de mes engagements': Account.TYPE_LOAN, + 'Mes services': None, # ignore this kind of accounts (no bank ones) + 'Équipements': None, # ignore this kind of accounts (no bank ones) + 'Synthèse': None, # ignore this title + } PATTERN = [ (re.compile(r'.*Titres Pea.*'), Account.TYPE_PEA), @@ -548,7 +551,7 @@ class AccountsPage(LoggedPage, MyHTMLPage): (re.compile(r'.*Titres.*'), Account.TYPE_MARKET), (re.compile(r'.*Selection Vie.*'), Account.TYPE_LIFE_INSURANCE), (re.compile(r'^Fructi Pulse.*'), Account.TYPE_MARKET), - (re.compile(r'^(Quintessa|Solevia).*'), Account.TYPE_MARKET), + (re.compile(r'^(Quintessa|Solevia).*'), Account.TYPE_LIFE_INSURANCE), (re.compile(r'^Plan Epargne Enfant Mul.*'), Account.TYPE_MARKET), (re.compile(r'^Alc Premium'), Account.TYPE_MARKET), (re.compile(r'^Plan Epargne Enfant Msu.*'), Account.TYPE_LIFE_INSURANCE), @@ -576,7 +579,7 @@ class AccountsPage(LoggedPage, MyHTMLPage): actions = self.get_button_actions() for div in self.doc.xpath('//div[has-class("btit")]'): - if div.text in (None, u'Synthèse'): + if div.text in (None, 'Synthèse'): continue account_type = self.ACCOUNT_TYPES.get(div.text.strip(), Account.TYPE_UNKNOWN) @@ -612,8 +615,9 @@ class AccountsPage(LoggedPage, MyHTMLPage): account = Account() account.id = args['identifiant'].replace(' ', '') - account.label = u' '.join([u''.join([txt.strip() for txt in tds[1].itertext()]), - u''.join([txt.strip() for txt in tds[2].itertext()])]).strip() + account.number = account.id + account.label = ' '.join([''.join([txt.strip() for txt in tds[1].itertext()]), + ''.join([txt.strip() for txt in tds[2].itertext()])]).strip() for pattern, _type in self.PATTERN: match = pattern.match(account.label) @@ -623,7 +627,7 @@ class AccountsPage(LoggedPage, MyHTMLPage): else: account.type = account_type - balance_text = u''.join([txt.strip() for txt in tds[3].itertext()]) + balance_text = ''.join([txt.strip() for txt in tds[3].itertext()]) balance = FrenchTransaction.clean_amount(balance_text) account.balance = Decimal(balance or '0.0') account.currency = currency or Account.get_currency(balance_text) @@ -637,7 +641,7 @@ class AccountsPage(LoggedPage, MyHTMLPage): account._coming_params = None account._coming_count = None account._invest_params = None - if balance != u'' and len(tds[3].xpath('.//a')) > 0: + if balance != '' and len(tds[3].xpath('.//a')) > 0: account._params = params.copy() account._params['dialogActionPerformed'] = 'SOLDE' account._params['attribute($SEL_$%s)' % tr.attrib['id'].split('_')[0]] = tr.attrib['id'].split('_', 1)[1] @@ -706,18 +710,19 @@ class CardsPage(LoggedPage, MyHTMLPage): yield account account = Account() account.id = id.replace(' ', '') + account.number = account.id account.type = Account.TYPE_CARD account.balance = account.coming = Decimal('0') account._next_debit = datetime.date.today() - account._prev_debit = datetime.date(2000,1,1) - account.label = u' '.join([CleanText(None).filter(cols[self.COL_TYPE]), + account._prev_debit = datetime.date(2000, 1, 1) + account.label = ' '.join([CleanText(None).filter(cols[self.COL_TYPE]), CleanText(None).filter(cols[self.COL_LABEL])]) account.currency = currency if accounts_parsed is not None: for account_parsed in accounts_parsed: - if (account_parsed.type == Account.TYPE_CHECKING and - account_parsed.id.replace('CPT', '') == Regexp(CleanText('//div[@class="btit"]'), r'(\d+)$')(self.doc)): + if (account_parsed.type == Account.TYPE_CHECKING + and account_parsed.id.replace('CPT', '') == Regexp(CleanText('//div[@class="btit"]'), r'(\d+)$')(self.doc)): account.parent = account_parsed account._params = None @@ -749,7 +754,7 @@ class CardsPage(LoggedPage, MyHTMLPage): date = datetime.date(*[int(c) for c in m.groups()][::-1]) if date.year < 100: - date = date.replace(year=date.year+2000) + date = date.replace(year=date.year + 2000) amount = Decimal(FrenchTransaction.clean_amount(CleanText(None).filter(cols[self.COL_AMOUNT]))) @@ -769,37 +774,51 @@ class CardsPage(LoggedPage, MyHTMLPage): class Transaction(FrenchTransaction): - PATTERNS = [(re.compile('^RET DAB (?P.*?) RETRAIT (DU|LE) (?P
\d{2})(?P\d{2})(?P\d+).*'), - FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile('^RET DAB (?P.*?) CARTE ?:.*'), - FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile('^(?P.*) RETRAIT DU (?P
\d{2})(?P\d{2})(?P\d{2}) .*'), - FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile('^(RETRAIT CARTE )?RET(RAIT)? DAB (?P.*)'), - FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile('((\w+) )?(?P
\d{2})(?P\d{2})(?P\d{2}) CB[:\*][^ ]+ (?P.*)'), - FrenchTransaction.TYPE_CARD), - (re.compile('^VIR(EMENT)? (?P.*)'), FrenchTransaction.TYPE_TRANSFER), - (re.compile('^(PRLV|PRELEVEMENT) (?P.*)'), - FrenchTransaction.TYPE_ORDER), - (re.compile('^(?PCHEQUE .*)'), FrenchTransaction.TYPE_CHECK), - (re.compile('^(AGIOS /|FRAIS) (?P.*)', re.IGNORECASE), - FrenchTransaction.TYPE_BANK), - (re.compile('^(CONVENTION \d+ )?COTIS(ATION)? (?P.*)', re.IGNORECASE), - FrenchTransaction.TYPE_BANK), - (re.compile('^REMISE (?P.*)'), FrenchTransaction.TYPE_DEPOSIT), - (re.compile('^(?PECHEANCE PRET .*)'), FrenchTransaction.TYPE_LOAN_PAYMENT), - (re.compile('^(?P.*)( \d+)? QUITTANCE .*'), - FrenchTransaction.TYPE_ORDER), - (re.compile('^.* LE (?P
\d{2})/(?P\d{2})/(?P\d{2})$'), - FrenchTransaction.TYPE_UNKNOWN), - (re.compile(r'^RELEVE CARTE'), FrenchTransaction.TYPE_CARD_SUMMARY), - (re.compile(r'^RET GAB .*'), FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile(r'^RETRAIT CARTE AGENCE \d+$'), FrenchTransaction.TYPE_WITHDRAWAL), - ] + PATTERNS = [ + (re.compile('^RET DAB (?P.*?) RETRAIT (DU|LE) (?P
\d{2})(?P\d{2})(?P\d+).*'), FrenchTransaction.TYPE_WITHDRAWAL), + (re.compile('^RET DAB (?P.*?) CARTE ?:.*'), FrenchTransaction.TYPE_WITHDRAWAL), + (re.compile('^(?P.*) RETRAIT DU (?P
\d{2})(?P\d{2})(?P\d{2}) .*'), FrenchTransaction.TYPE_WITHDRAWAL), + (re.compile('^(RETRAIT CARTE )?RET(RAIT)? DAB (?P.*)'), FrenchTransaction.TYPE_WITHDRAWAL), + (re.compile('((\w+) )?(?P
\d{2})(?P\d{2})(?P\d{2}) CB[:\*][^ ]+ (?P.*)'), FrenchTransaction.TYPE_CARD), + (re.compile('^VIR(EMENT)? (?P.*)'), FrenchTransaction.TYPE_TRANSFER), + (re.compile('^(PRLV|PRELEVEMENT) (?P.*)'), FrenchTransaction.TYPE_ORDER), + (re.compile('^(?PCHEQUE .*)'), FrenchTransaction.TYPE_CHECK), + (re.compile('^(AGIOS /|FRAIS) (?P.*)', re.IGNORECASE), FrenchTransaction.TYPE_BANK), + (re.compile('^(CONVENTION \d+ )?COTIS(ATION)? (?P.*)', re.IGNORECASE), FrenchTransaction.TYPE_BANK), + (re.compile('^REMISE (?P.*)'), FrenchTransaction.TYPE_DEPOSIT), + (re.compile('^(?PECHEANCE PRET .*)'), FrenchTransaction.TYPE_LOAN_PAYMENT), + (re.compile('^(?P.*)( \d+)? QUITTANCE .*'), FrenchTransaction.TYPE_ORDER), + (re.compile('^.* LE (?P
\d{2})/(?P\d{2})/(?P\d{2})$'), FrenchTransaction.TYPE_UNKNOWN), + (re.compile(r'^RELEVE CARTE'), FrenchTransaction.TYPE_CARD_SUMMARY), + (re.compile(r'^RET GAB .*'), FrenchTransaction.TYPE_WITHDRAWAL), + (re.compile(r'^RETRAIT CARTE AGENCE \d+$'), FrenchTransaction.TYPE_WITHDRAWAL), + ] + + +class InvestmentPage(LoggedPage, HTMLPage): + def get_investment_page_params(self): + script = self.doc.xpath('//body')[0].attrib['onload'] + url = None + m = re.search(r"','([^']+?)',\[", script, re.MULTILINE) + if m: + url = m.group(1) + + params = {} + for key, value in re.findall(r"key:'(?PSJRToken)'\,value:'(?P.*?)'}", script, re.MULTILINE): + params[key] = value + + if url and params: + return url, params + return None class TransactionsPage(LoggedPage, MyHTMLPage): + def is_sorted_by_most_recent(self): + # If the transactions are not sorted correctly, the class of this + # 'a' tag changes ('tcth' if sorted the other way, 'tctm' if not sorted + # by operation date) + return CleanText('//a[@class="tctb" and contains(text(), "Date opé")]')(self.doc) + def get_next_params(self): nxt = self.doc.xpath('//li[contains(@id, "_nxt")]') if len(nxt) == 0 or nxt[0].attrib.get('class', '') == 'nxt-dis': @@ -825,7 +844,7 @@ class TransactionsPage(LoggedPage, MyHTMLPage): COL_COMPTA_DATE = 0 COL_LABEL = 1 - COL_REF = 2 # optional + COL_REF = 2 # optional COL_OP_DATE = -4 COL_VALUE_DATE = -3 COL_DEBIT = -2 @@ -847,10 +866,8 @@ class TransactionsPage(LoggedPage, MyHTMLPage): # (only used for GoCardLess transactions so far) t._has_link = bool(tds[self.COL_DEBIT].findall('a') or tds[self.COL_CREDIT].findall('a')) - # XXX We currently take the *value* date, but it will probably - # necessary to use the *operation* one. - # Default sort on website is by compta date, so in browser.py we - # change the sort on value date. + # Default sort on website is by compta date, in browser.py we + # change the sort on operation date. cleaner = CleanText(None).filter date = cleaner(tds[self.COL_OP_DATE]) vdate = cleaner(tds[self.COL_VALUE_DATE]) @@ -897,10 +914,10 @@ class TransactionsPage(LoggedPage, MyHTMLPage): t.amount = -account._prev_balance yield t - currency = Account.get_currency(self.doc\ - .xpath('//table[@id="TabFact"]/thead//th')[self.COL_CARD_AMOUNT]\ - .text\ - .replace('(', ' ')\ + currency = Account.get_currency(self.doc + .xpath('//table[@id="TabFact"]/thead//th')[self.COL_CARD_AMOUNT] + .text + .replace('(', ' ') .replace(')', ' ')) for i, tr in enumerate(self.doc.xpath('//table[@id="TabFact"]/tbody/tr')): tds = tr.findall('td') @@ -927,22 +944,11 @@ class TransactionsPage(LoggedPage, MyHTMLPage): def no_operations(self): if len(self.doc.xpath('//table[@id="tbl1" or @id="TabFact"]//td[@colspan]')) > 0: return True - if len(self.doc.xpath(u'//div[contains(text(), "Accès à LineBourse")]')) > 0: + if len(self.doc.xpath('//div[contains(text(), "Accès à LineBourse")]')) > 0: return True return False - def get_investment_page_params(self): - script = self.doc.xpath('//body')[0].attrib['onload'] - url = None - m = re.search(r"','([^']+?)',\[", script, re.MULTILINE) - if m: - url = m.group(1) - params = {} - for key, value in re.findall(r"key:'(?PSJRToken)'\,value:'(?P.*?)'}", script, re.MULTILINE): - params[key] = value - return url, params if url and params else None - def get_transaction_table_id(self, ref): tr = self.doc.xpath('//table[@id="tbl1"]/tbody/tr[.//span[contains(text(), "%s")]]' % ref)[0] @@ -957,12 +963,12 @@ class TransactionsPage(LoggedPage, MyHTMLPage): # index in which the link lies # # To get more details about how things are done, see the following javascript functions: - #- attachTableRowEvents (atre) - #- attachActiveSelectionEventsOnRow - #- astr - #- updateSelection (uds) - #- selectActionButton (sab) - #- a script element embedded in the html page (search for "tcl5", "tcl6") + # - attachTableRowEvents (atre) + # - attachActiveSelectionEventsOnRow + # - astr + # - updateSelection (uds) + # - selectActionButton (sab) + # - a script element embedded in the html page (search for "tcl5", "tcl6") assert transaction._has_link @@ -971,6 +977,14 @@ class TransactionsPage(LoggedPage, MyHTMLPage): elif transaction._amount_type == 'credit': return 'NV' + def go_investment(self): + script = self.doc.xpath('//body')[0].attrib['onload'] + if re.search(r'startWebAppTask\(', script) is None: + return False + params = {'oid': re.search(r"'urlReturn',\w+?,'(\w+)'\)", script).group(1)} + self.browser.location(self.browser.absurl('/cyber/ibp/ate/skin/internet/pages/webAppReroutingAutoSubmit.jsp'), params=params) + return True + class NatixisChoicePage(LoggedPage, HTMLPage): def on_load(self): @@ -1006,7 +1020,7 @@ class TransactionsBackPage(TransactionsPage): class NatixisRedirect(LoggedPage, XMLPage): def get_redirect(self): url = self.doc.xpath('/partial-response/redirect/@url')[0] - return url.replace('http://', 'https://') # why do they use http on a bank site??? + return url.replace('http://', 'https://') # why do they use http on a bank site??? class NatixisErrorPage(LoggedPage, HTMLPage): @@ -1029,7 +1043,7 @@ class IbanPage(LoggedPage, MyHTMLPage): form['token'] = self.build_token(form['token']) form['dialogActionPerformed'] = "DETAIL_IBAN_RIB" tr_id = Attr(None, 'id').filter(tr.xpath('.')).split('_') - form[u'attribute($SEL_$%s)' % tr_id[0]] = tr_id[1] + form['attribute($SEL_$%s)' % tr_id[0]] = tr_id[1] form.submit() return True return False @@ -1057,7 +1071,7 @@ def float_to_decimal(f): class NatixisInvestPage(LoggedPage, JsonPage): @method - class get_investments(DictElement): + class iter_investments(DictElement): item_xpath = 'detailContratVie/valorisation/supports' class item(ItemElement): @@ -1159,7 +1173,7 @@ class NatixisDetailsPage(LoggedPage, RawPage): tr.amount = -abs(tr.amount) else: assert False, 'unhandled line %s' % label - assert not any(len(cell) for cell in row[self.COL_LABEL+1:]), 'there should be only the label' + assert not any(len(cell) for cell in row[self.COL_LABEL + 1:]), 'there should be only the label' else: if not tr: continue @@ -1188,24 +1202,26 @@ class AdvisorPage(LoggedPage, MyHTMLPage): class get_advisor(ItemElement): klass = Advisor - condition = lambda self: Field('name')(self) + def condition(self): + return Field('name')(self) - obj_name = CleanText(u'//div[label[contains(text(), "Votre conseiller")]]/span') - obj_agency = CleanText(u'//div[label[contains(text(), "Votre agence")]]/span') + obj_name = CleanText('//div[label[contains(text(), "Votre conseiller")]]/span') + obj_agency = CleanText('//div[label[contains(text(), "Votre agence")]]/span') obj_email = obj_mobile = NotAvailable @method class update_agency(ItemElement): - obj_phone = CleanText(u'//div[label[contains(text(), "Téléphone")]]/span', replace=[('.', '')]) - obj_fax = CleanText(u'//div[label[contains(text(), "Fax")]]/span', replace=[('.', '')]) - obj_address = CleanText(u'//div[div[contains(text(), "Votre agence")]]/following-sibling::div[1]//div[not(label)]/span') + obj_phone = CleanText('//div[label[contains(text(), "Téléphone")]]/span', replace=[('.', '')]) + obj_fax = CleanText('//div[label[contains(text(), "Fax")]]/span', replace=[('.', '')]) + obj_address = CleanText('//div[div[contains(text(), "Votre agence")]]/following-sibling::div[1]//div[not(label)]/span') def get_profile(self): profile = Person() - # the name is only available in a welcome message. Sometimes, the message will look like that : - # "Bienvenue M - " and sometimes just "Bienvenue M " - # Or even "Bienvenue " + # the name is only available in a welcome message. The messages can look like : + # - Bienvenue M - + # - Bienvenue M + # - Bienvenue # We need to detect wether the company name is there, and where it begins. # relying on the dash only is dangerous as people may have dashes in their name and so may companies. # but we can detect company name from a dash between space diff --git a/modules/bforbank/browser.py b/modules/bforbank/browser.py index 536b17681de6265d8ef92582c3ea3441f3d6cf5b..28aa6c1413757ef15c3927ece68edc0a657e065a 100644 --- a/modules/bforbank/browser.py +++ b/modules/bforbank/browser.py @@ -67,7 +67,7 @@ class BforbankBrowser(LoginBrowser): BoursePage) bourse_titre = URL(r'https://bourse.bforbank.com/netfinca-titres/servlet/com.netfinca.frontcr.navigation.Titre', BoursePage) # to get logout link - bourse_disco = URL(r'https://bourse.bforbank.com/netfinca-titres/servlet/com.netfinca.frontcr.login.ContextTransferDisconnect', BourseDisconnectPage) + bourse_disco = URL(r'https://bourse.bforbank.com/netfinca-titres/servlet/com.netfinca.frontcr.login.Logout', BourseDisconnectPage) profile = URL(r'/espace-client/profil/informations', ProfilePage) def __init__(self, birthdate, username, password, *args, **kwargs): @@ -96,8 +96,9 @@ class BforbankBrowser(LoginBrowser): @need_login def iter_accounts(self): if self.accounts is None: - self.home.stay_or_go() - accounts = list(self.page.iter_accounts()) + owner_name = self.get_profile().name.upper().split(' ', 1)[1] + self.home.go() + accounts = list(self.page.iter_accounts(name=owner_name)) if self.page.RIB_AVAILABLE: self.rib.go().populate_rib(accounts) @@ -144,7 +145,9 @@ class BforbankBrowser(LoginBrowser): if not bourse_account: return iter([]) - self.location(bourse_account._link_id) + self.location(bourse_account._link_id, params={ + 'nump': bourse_account._market_id, + }) assert self.bourse.is_here() history = list(self.page.iter_history()) self.leave_espace_bourse() @@ -245,6 +248,7 @@ class BforbankBrowser(LoginBrowser): return True def get_bourse_account(self, account): + owner_name = self.get_profile().name.upper().split(' ', 1)[1] self.bourse_login.go(id=account.id) # "login" to bourse page self.bourse.go() @@ -253,7 +257,7 @@ class BforbankBrowser(LoginBrowser): if self.page.password_required(): return self.logger.debug('searching account matching %r', account) - for bourse_account in self.page.get_list(): + for bourse_account in self.page.get_list(name=owner_name): self.logger.debug('iterating account %r', bourse_account) if bourse_account.id.startswith(account.id[3:]): return bourse_account @@ -295,7 +299,7 @@ class BforbankBrowser(LoginBrowser): if self.bourse.is_here(): self.location(self.bourse_titre.build()) self.location(self.page.get_logout_link()) - self.location(self.page.get_relocation()) + self.login.go() @need_login def get_profile(self): diff --git a/modules/bforbank/pages.py b/modules/bforbank/pages.py index e4dbc4bbd797a123165039c61551f9ffed4dff16..d8f9179f16810850ee09f5815bf37cb14409a268 100644 --- a/modules/bforbank/pages.py +++ b/modules/bforbank/pages.py @@ -29,7 +29,7 @@ from PIL import Image from weboob.exceptions import ActionNeeded from weboob.browser.pages import LoggedPage, HTMLPage, pagination, AbstractPage from weboob.browser.elements import method, ListElement, ItemElement, TableElement -from weboob.capabilities.bank import Account +from weboob.capabilities.bank import Account, AccountOwnership from weboob.capabilities.profile import Person from weboob.browser.filters.html import Link, Attr, TableCell from weboob.browser.filters.standard import ( @@ -170,6 +170,16 @@ class AccountsPage(LoggedPage, HTMLPage): def condition(self): return not len(self.el.xpath('./td[@class="chart"]')) + def obj_ownership(self): + owner = CleanText('./td//div[contains(@class, "-synthese-text") and not(starts-with(., "N°"))]', default=None)(self) + + if owner: + if re.search(r'(m|mr|me|mme|mlle|mle|ml)\.? (.*)\bou (m|mr|me|mme|mlle|mle|ml)\b(.*)', owner, re.IGNORECASE): + return AccountOwnership.CO_OWNER + elif all(n in owner.upper() for n in self.env['name'].split()): + return AccountOwnership.OWNER + return AccountOwnership.ATTORNEY + class Transaction(FrenchTransaction): PATTERNS = [(re.compile('^(?PVIREMENT)'), FrenchTransaction.TYPE_TRANSFER), @@ -385,13 +395,12 @@ class BoursePage(AbstractPage): PARENT = 'lcl' PARENT_URL = 'bourse' + def get_logout_link(self): + return Link('//a[@title="Retour à l\'accueil"]')(self.doc) + class BourseDisconnectPage(LoggedPage, HTMLPage): - def get_relocation(self): - link = re.search(r"window\.location= \'(.+)\';", self.content) - if link: - m = link.group(1) - return m + pass class ProfilePage(LoggedPage, HTMLPage): diff --git a/modules/binck/pages.py b/modules/binck/pages.py index 3a9b3a54fade8d8953b128b6181adb3b5cbb6b7a..8350da9185da2f325e36a28c0d99dd8f3b3bf7cd 100644 --- a/modules/binck/pages.py +++ b/modules/binck/pages.py @@ -23,7 +23,7 @@ import re from weboob.browser.pages import HTMLPage, JsonPage, LoggedPage from weboob.browser.elements import ItemElement, ListElement, DictElement, TableElement, method -from weboob.browser.filters.standard import CleanText, Date, DateTime, Format, CleanDecimal, Eval, Env, Field +from weboob.browser.filters.standard import CleanText, Date, Format, CleanDecimal, Eval, Env, Field from weboob.browser.filters.html import Attr, Link, TableCell from weboob.browser.filters.json import Dict from weboob.exceptions import BrowserPasswordExpired, ActionNeeded @@ -224,20 +224,6 @@ class InvestmentPage(LoggedPage, JsonPage): obj_original_diff = Env('o_diff', default=NotAvailable) obj__security_id = Dict('SecurityId') - def obj_vdate(self): - raw_date = CleanText(Dict('Time'))(self) - if raw_date == '---': - return NotAvailable - elif re.match(r'\d{2}\/\d{2}\/\d{4}', raw_date): - # during stocks closing hours only date (dd/mm/yyyy) is given - return Date(CleanText(Dict('Time')), dayfirst=True)(self) - elif re.match(r'\d{2}:\d{2}:\d{2}', raw_date): - # during stocks opening hours only time (hh:mm:ss) is given, - # can even be realtime, depending on user settings, - # can be given in foreign markets time, - # e.g. 10:00 is displayed at 9:00 for an action in NASDAQ Helsinki market - return DateTime(CleanText(Dict('Time')), dayfirst=True, strict=False)(self) - def obj_code(self): if is_isin_valid(Dict('IsinCode')(self)): return Dict('IsinCode')(self) diff --git a/modules/bnporc/enterprise/pages.py b/modules/bnporc/enterprise/pages.py index 0d27d8fd6ad9b459397f9e045beccb017bc1ac67..aa04b302c40b9edd1edfa7adcfd090416de83835 100644 --- a/modules/bnporc/enterprise/pages.py +++ b/modules/bnporc/enterprise/pages.py @@ -24,6 +24,7 @@ import re from datetime import datetime from io import BytesIO +import dateutil.parser from weboob.browser.pages import LoggedPage, HTMLPage, JsonPage from weboob.browser.filters.json import Dict from weboob.browser.filters.html import TableCell, Attr @@ -35,7 +36,6 @@ from weboob.browser.filters.standard import ( from weboob.capabilities.bank import Transaction, Account, Investment from weboob.capabilities.profile import Person from weboob.tools.captcha.virtkeyboard import MappedVirtKeyboard, VirtKeyboardError -from weboob.tools.date import parse_french_date from weboob.capabilities import NotAvailable from weboob.exceptions import ActionNeeded, BrowserForbidden @@ -178,7 +178,7 @@ class BnpHistoryItem(ItemElement): mtc = re.search(r'\bDU (\d{2})\.?(\d{2})\.?(\d{2})\b', raw) if mtc: date = '%s/%s/%s' % (mtc.group(1), mtc.group(2), mtc.group(3)) - return parse_french_date(date) + return dateutil.parser.parse(date, dayfirst=True) # The date can be truncated, so it is not retrieved if 'dateCreation' in self.el: @@ -270,10 +270,18 @@ class AccountHistoryPage(LoggedPage, JsonPage): def obj_rdate(self): raw = self.obj_raw() mtc = re.search(r'\bDU (\d{6}|\d{8})\b', raw) + if mtc: - date = mtc.group(1) - date = '%s/%s/%s' % (date[0:2], date[2:4], date[4:]) - return parse_french_date(date) + numbers = mtc.group(1) + # we need to create this string because dateutil crashes + # with dates in the ddmmyyyy format + # dd/mm/yy and dd/mm/yyyy + date = '%s/%s/%s' % (numbers[0:2], numbers[2:4], numbers[4:]) + try: + return dateutil.parser.parse(date, dayfirst=True) + except ValueError: + # parsing failed assuming yyyymmdd format + return dateutil.parser.parse(numbers) return fromtimestamp(Dict('dateCreation')(self)) diff --git a/modules/bnporc/pp/pages.py b/modules/bnporc/pp/pages.py index c7e8847660d7e0a91ca16168bbc9bac564f00fad..0faca2aa739d00d7f97a2be6c8003c327da30bc9 100644 --- a/modules/bnporc/pp/pages.py +++ b/modules/bnporc/pp/pages.py @@ -38,14 +38,14 @@ from weboob.browser.pages import JsonPage, LoggedPage, HTMLPage from weboob.capabilities import NotAvailable from weboob.capabilities.bank import ( Account, Investment, Recipient, Transfer, TransferBankError, - AddRecipientBankError, AddRecipientTimeout, + AddRecipientBankError, AddRecipientTimeout, AccountOwnership, ) from weboob.capabilities.base import empty from weboob.capabilities.contact import Advisor from weboob.capabilities.profile import Person, ProfileMissing from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable, BrowserPasswordExpired, ActionNeeded from weboob.tools.capabilities.bank.iban import rib2iban, rebuild_rib, is_iban_valid -from weboob.tools.capabilities.bank.transactions import FrenchTransaction +from weboob.tools.capabilities.bank.transactions import FrenchTransaction, parse_with_patterns from weboob.tools.captcha.virtkeyboard import GridVirtKeyboard from weboob.tools.date import parse_french_date from weboob.tools.capabilities.bank.investments import is_isin_valid @@ -310,6 +310,9 @@ class ProfilePage(LoggedPage, JsonPage): class AccountsPage(BNPPage): + def get_user_ikpi(self): + return self.doc['data']['infoUdc']['titulaireConnecte']['ikpi'] + @method class iter_accounts(DictElement): item_xpath = 'data/infoUdc/familleCompte' @@ -375,6 +378,17 @@ class AccountsPage(BNPPage): return iban return None + def obj_ownership(self): + indic = Dict('titulaire/indicTitulaireCollectif', default=None)(self) + # The boolean is in the form of a string ('true' or 'false') + if indic == 'true': + return AccountOwnership.CO_OWNER + elif indic == 'false': + if self.page.get_user_ikpi() == Dict('titulaire/ikpi')(self): + return AccountOwnership.OWNER + return AccountOwnership.ATTORNEY + return NotAvailable + # softcap not used TODO don't pass this key when backend is ready # deferred cb can disappear the day after the appear, so 0 as day_for_softcap obj__bisoftcap = {'deferred_cb': {'softcap_day': 1000, 'day_for_softcap': 1}} @@ -636,9 +650,12 @@ class HistoryPage(BNPPage): 'amount': op.get('montant'), 'card': op.get('numeroPorteurCarte'), }) - tr.parse(date=parse_french_date(op.get('dateOperation')), - vdate=parse_french_date(op.get('valueDate')), - raw=CleanText().filter(op.get('libelle'))) + + tr.date = parse_french_date(op.get('dateOperation')) + tr.vdate = parse_french_date(op.get('valueDate')) + tr.rdate = NotAvailable + tr.raw = CleanText().filter(op.get('libelle')) + parse_with_patterns(tr.raw, tr, Transaction.PATTERNS) if tr.type == Transaction.TYPE_CARD: tr.type = self.browser.card_to_transaction_type.get(op.get('keyCarte'), diff --git a/modules/bnppere/pages.py b/modules/bnppere/pages.py index a83d37924870fd59ce0fd9bb203e6917ab9b6c35..2ad1fe25cea2b074406864e0ea9a970876769545 100644 --- a/modules/bnppere/pages.py +++ b/modules/bnppere/pages.py @@ -125,9 +125,9 @@ class HistoryPage(LoggedPage, HTMLPage): # This wonderful website randomly displays separators as '.' or ',' # For example, numbers can look like "€12,345.67" or "12 345,67 €" try: - return CleanDecimal.French('./div[contains(@class, "accordion_header")]/div[6]')(self) + return CleanDecimal.French('./div[contains(@class, "accordion_header")]/div[position()=last()]')(self) except NumberFormatError: - return CleanDecimal.US('./div[contains(@class, "accordion_header")]/div[6]')(self) + return CleanDecimal.US('./div[contains(@class, "accordion_header")]/div[position()=last()]')(self) class InvestmentPage(LoggedPage, HTMLPage): diff --git a/modules/boursorama/browser.py b/modules/boursorama/browser.py index b3311aa251ce7969b98779726c4df175e9e475c5..05356bb41ec1ec31cc42c1e6bf5ecdc9e5b3f084 100644 --- a/modules/boursorama/browser.py +++ b/modules/boursorama/browser.py @@ -21,7 +21,6 @@ import requests from datetime import date, datetime from dateutil.relativedelta import relativedelta -from dateutil import parser from weboob.browser.retry import login_method, retry_on_logout, RetryLoginBrowser from weboob.browser.browsers import need_login, StatesMixin @@ -31,11 +30,11 @@ from weboob.browser.exceptions import LoggedOut, ClientError from weboob.capabilities.bank import ( Account, AccountNotFound, TransferError, TransferInvalidAmount, TransferInvalidEmitter, TransferInvalidLabel, TransferInvalidRecipient, - AddRecipientStep, Recipient, Rate, TransferBankError, AccountOwnership, + AddRecipientStep, Rate, TransferBankError, AccountOwnership, RecipientNotFound, + AddRecipientTimeout, ) -from weboob.capabilities.base import empty +from weboob.capabilities.base import empty, find_object from weboob.capabilities.contact import Advisor -from weboob.tools.captcha.virtkeyboard import VirtKeyboardError from weboob.tools.value import Value from weboob.tools.compat import basestring, urlsplit from weboob.tools.capabilities.bank.transactions import sorted_transactions @@ -126,7 +125,7 @@ class BoursoramaBrowser(RetryLoginBrowser, StatesMixin): currencylist = URL('https://www.boursorama.com/bourse/devises/parite/_detail-parite', CurrencyListPage) currencyconvert = URL('https://www.boursorama.com/bourse/devises/convertisseur-devises/convertir', CurrencyConvertPage) - __states__ = ('auth_token',) + __states__ = ('auth_token', 'recipient_form',) def __init__(self, config=None, *args, **kwargs): self.config = config @@ -134,6 +133,7 @@ class BoursoramaBrowser(RetryLoginBrowser, StatesMixin): self.accounts_list = None self.cards_list = None self.deferred_card_calendar = None + self.recipient_form = None kwargs['username'] = self.config['login'].get() kwargs['password'] = self.config['password'].get() super(BoursoramaBrowser, self).__init__(*args, **kwargs) @@ -145,8 +145,14 @@ class BoursoramaBrowser(RetryLoginBrowser, StatesMixin): pass def load_state(self, state): - if ('expire' in state and parser.parse(state['expire']) > datetime.now()) or state.get('auth_token'): - return super(BoursoramaBrowser, self).load_state(state) + # needed to continue the session while adding recipient with otp + # it keeps the form to continue to submit the otp + if state.get('recipient_form'): + state.pop('url', None) + super(BoursoramaBrowser, self).load_state(state) + + elif state.get('auth_token'): + super(BoursoramaBrowser, self).load_state(state) def handle_authentication(self): if self.authentication.is_here(): @@ -169,16 +175,8 @@ class BoursoramaBrowser(RetryLoginBrowser, StatesMixin): if self.auth_token and self.config['pin_code'].get(): self.page.authenticate() else: - for _ in range(3): - self.login.go() - try: - self.page.login(self.username, self.password) - except VirtKeyboardError: - self.logger.error('Failed to process VirtualKeyboard') - else: - break - else: - raise VirtKeyboardError() + self.login.go() + self.page.login(self.username, self.password) if self.login.is_here() or self.error.is_here(): raise BrowserIncorrectPassword() @@ -410,34 +408,35 @@ class BoursoramaBrowser(RetryLoginBrowser, StatesMixin): advisor.phone = u"0146094949" return iter([advisor]) - @need_login - def iter_transfer_recipients(self, account): - if account.type in (Account.TYPE_LOAN, Account.TYPE_LIFE_INSURANCE): - return [] - assert account.url - + def go_recipients_list(self, account_url, account_id): # url transfer preparation - url = urlsplit(account.url) + url = urlsplit(account_url) parts = [part for part in url.path.split('/') if part] assert len(parts) > 2, 'Account url missing some important part to iter recipient' account_type = parts[1] # cav, ord, epargne ... account_webid = parts[-1] - try: - self.transfer_main_page.go(acc_type=account_type, webid=account_webid) - except BrowserHTTPNotFound: - return [] + self.transfer_main_page.go(acc_type=account_type, webid=account_webid) # may raise a BrowserHTTPNotFound # can check all account available transfer option if self.transfer_main_page.is_here(): self.transfer_accounts.go(acc_type=account_type, webid=account_webid) if self.transfer_accounts.is_here(): - try: - self.page.submit_account(account.id) - except AccountNotFound: - return [] + self.page.submit_account(account_id) # may raise AccountNotFound + + + @need_login + def iter_transfer_recipients(self, account): + if account.type in (Account.TYPE_LOAN, Account.TYPE_LIFE_INSURANCE): + return [] + assert account.url + + try: + self.go_recipients_list(account.url, account.id) + except (BrowserHTTPNotFound, AccountNotFound): + return [] assert self.recipients_page.is_here() return self.page.iter_recipients() @@ -481,10 +480,11 @@ class BoursoramaBrowser(RetryLoginBrowser, StatesMixin): # at this stage, the site doesn't show the real ids/ibans, we can only guess if recipients[0].label != ret.recipient_label: - if not recipients[0].label.startswith('%s - ' % ret.recipient_label): - # the label displayed here is just "" - # but in the recipients list it is " - "... - raise TransferError('Recipient label changed during transfer') + self.logger.info('Recipients from iter_recipient and from the transfer are diffent: "%s" and "%s"' % (recipients[0].label, ret.recipient_label)) + if not ret.recipient_label.startswith('%s - ' % recipients[0].label): + # the label displayed here is " - " + # but in the recipients list it is ""... + assert False, 'Recipient label changed during transfer (from "%s" to "%s")' % (recipients[0].label, ret.recipient_label) ret.recipient_id = recipients[0].id ret.recipient_iban = recipients[0].iban @@ -509,26 +509,11 @@ class BoursoramaBrowser(RetryLoginBrowser, StatesMixin): # the last page contains no info, return the last transfer object from init_transfer return transfer - def build_recipient(self, recipient): - r = Recipient() - r.iban = recipient.iban - r.id = recipient.iban - r.label = recipient.label - r.category = recipient.category - r.enabled_at = date.today() - r.currency = u'EUR' - r.bank_name = recipient.bank_name - return r - @need_login - def new_recipient(self, recipient, **kwargs): - if 'code' in kwargs: - assert self.rcpt_page.is_here() - assert self.page.is_confirm_sms() - - self.page.confirm_sms(kwargs['code']) - return self.rcpt_after_sms() + def init_new_recipient(self, recipient): + self.recipient_form = None # so it is reset when a new recipient is added + # get url account = None for account in self.get_accounts_list(): if account.url: @@ -541,26 +526,57 @@ class BoursoramaBrowser(RetryLoginBrowser, StatesMixin): target = account.url + '/' + suffix self.location(target) - assert self.page.is_charac() + assert self.page.is_charac(), 'Not on the page to add recipients.' + # fill recipient form self.page.submit_recipient(recipient) + recipient.origin_account_id = account.id + + # confirm sending sms + assert self.page.is_confirm_send_sms(), 'Cannot reach the page asking to send a sms.' + self.page.confirm_send_sms() if self.page.is_send_sms(): + # send sms self.page.send_sms() - assert self.page.is_confirm_sms() - raise AddRecipientStep(self.build_recipient(recipient), Value('code', label='Veuillez saisir le code')) - # if the add recipient is restarted after the sms has been confirmed recently, the sms step is not presented again + assert self.page.is_confirm_sms(), 'The sms was not send.' - return self.rcpt_after_sms() + self.recipient_form = self.page.get_confirm_sms_form() + self.recipient_form['account_url'] = account.url + raise AddRecipientStep(recipient, Value('code', label='Veuillez saisir le code')) - def rcpt_after_sms(self): - assert self.page.is_confirm() - - ret = self.page.get_recipient() - self.page.confirm() + # if the add recipient is restarted after the sms has been confirmed recently, the sms step is not presented again + return self.rcpt_after_sms() - assert self.page.is_created() - return ret + def new_recipient(self, recipient, **kwargs): + # step 2 of new_recipient + if 'code' in kwargs: + # there is no confirmation to check the recipient + # validating the sms code directly adds the recipient + if not self.recipient_form: # the session expired + raise AddRecipientTimeout() + + url = self.recipient_form.pop('url') + account_url = self.recipient_form.pop('account_url') + self.recipient_form['strong_authentication_confirm[code]'] = kwargs['code'] + self.location(url, data=self.recipient_form) + + self.recipient_form = None + return self.rcpt_after_sms(recipient, account_url) + + # step 1 of new recipient + return self.init_new_recipient(recipient) + + def rcpt_after_sms(self, recipient, account_url): + assert self.page.is_created(), 'The recipient was not added.' + + # at this point, the recipient was added to the webiste + # we just want here to return the right Recipient object + # we are taking it from the recipient list page + # because there is no summary of the adding + self.go_recipients_list(account_url, recipient.origin_account_id) + rec = find_object(self.page.iter_recipients(), id=recipient.id, error=RecipientNotFound) + return rec def iter_currencies(self): return self.currencylist.go().get_currency_list() diff --git a/modules/boursorama/pages.py b/modules/boursorama/pages.py index 56ef045fdb8d161af6cb49f1dbedb6ee1b8f5044..b5e0b19438575f1d6e983a2da4199b07119162bc 100644 --- a/modules/boursorama/pages.py +++ b/modules/boursorama/pages.py @@ -25,6 +25,7 @@ from decimal import Decimal import re from io import BytesIO from datetime import date +from PIL import Image from weboob.browser.pages import HTMLPage, LoggedPage, pagination, NextPage, FormNotFound, PartialHTMLPage, LoginPage, CsvPage, RawPage, JsonPage from weboob.browser.elements import ListElement, ItemElement, method, TableElement, SkipItem, DictElement @@ -47,8 +48,7 @@ from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.tools.capabilities.bank.iban import is_iban_valid from weboob.tools.value import Value from weboob.tools.date import parse_french_date -from weboob.tools.captcha.virtkeyboard import VirtKeyboard, VirtKeyboardError -from weboob.tools.compat import urljoin, urlencode, urlparse +from weboob.tools.compat import urljoin, urlencode, urlparse, range from weboob.exceptions import BrowserQuestion, BrowserIncorrectPassword, BrowserHTTPNotFound, BrowserUnavailable, ActionNeeded @@ -142,45 +142,72 @@ class VirtKeyboardPage(HTMLPage): pass -class BoursoramaVirtKeyboard(VirtKeyboard): - symbols = {'0': (17, 7, 24, 17), - '1': (18, 6, 21, 18), - '2': (9, 7, 32, 34), - '3': (10, 7, 31, 34), - '4': (11, 6, 29, 34), - '5': (14, 6, 28, 34), - '6': (7, 7, 34, 34), - '7': (5, 6, 36, 34), - '8': (8, 7, 32, 34), - '9': (4, 7, 38, 34)} +class BoursoramaVirtKeyboard(object): + symbols = { + '0': '0000110000001111110001110011100110000110111000011011100001101100000110111000011011100001100110000110011100111000111111000000110000', + '1': '0000110000000111000000111100000111110000011011000000001100000000110000000011000000001100000000110000000011000000001100000000110000', + '2': '0001111000011111110001100011100000000110000000011000000001100000001110000001110000001110000001110000001110000001111111110111111111', + '3': '0001111000011111111001100011100000000110000000011000000011100001111000000111110000000001100000000110111000111001111111100001110000', + '4': '0000011100000011110000001111000001101100000100110000110011000110001100011000110011111111101111111110000000111000000011000000001100', + '5': '1111111100111111110011100000001100000000110000000011111111001111111110010000011000000001100000000110111000111011111111000001110000', + '6': '0000111000001111111001110001000110000000011000000011101111001111111110111000011011100001100110000110011100111000111111000000111000', + '7': '0111111111011111111100000001100000000110000000111000000011000000011100000001100000000110000000110000000011000000011100000001100000', + '8': '0001111000011111111011100011101110000110011000011001111111000011111000011100111011100001101110000110111000111001111111100001111000', + '9': '0001110000011111110001100011101110000110110000011011100001100111111110001111011000000001100000000110001000110001111111000001110000', + } + + def __init__(self, browser, page): + self.browser = browser + self.fingerprints = {} + col = 0 - color = (255,255,255) + keys = page.doc.xpath('//ul[@class="password-input"]//button/@data-matrix-key') - def __init__(self, page): - self.md5 = {} for button in page.doc.xpath('//ul[@class="password-input"]//button'): - c = button.attrib['data-matrix-key'] txt = button.attrib['style'].replace('background-image:url(data:image/png;base64,', '').rstrip(');') - img = BytesIO(b64decode(txt.encode('ascii'))) - self.load_image(img, self.color, convert='RGB') - self.load_symbols((0, 0, 42, 42), c) - - def load_symbols(self, coords, c): - coord = self.get_symbol_coords(coords) - if coord == (-1, -1, -1, -1): - return - self.md5[coord] = c - - def get_code(self, password): - code = '' - for i, d in enumerate(password): - if i > 0: - code += '|' - try: - code += self.md5[self.symbols[d]] - except KeyError: - raise VirtKeyboardError() - return code + + img = Image.open(BytesIO(b64decode(txt.encode('ascii')))) + width, height = img.size + + img = img.crop((16, 6, width - 16, height - 23)) + width, height = img.size + + matrix = img.load() + s = "" + for y in range(height): + for x in range(width): + (r, g, b, a) = matrix[x, y] + # If the pixel is white and opaque enough + if a > 200 and r + g + b > 740: + s += "1" + else: + s += "0" + self.fingerprints[keys[col]] = s + col += 1 + + def get_symbol_code(self, char): + fingerprint = self.symbols[char] + for code, string in self.fingerprints.items(): + if fingerprint == string: + return code + # Image contains some noise, and the match is not always perfect + # (this is why we can't use md5 hashs) + # But if we can't find the perfect one, we can take the best one + best = 0 + result = None + for code, string in self.fingerprints.items(): + match = 0 + for j, bit in enumerate(string): + if bit == fingerprint[j]: + match += 1 + if match > best: + best = match + result = code + self.browser.logger.info(self.fingerprints[result] + "(" + result + ") match " + char) + return result + + def get_string_code(self, string): + return '|'.join(self.get_symbol_code(c) for c in string) class LoginPage(LoginPage, HTMLPage): @@ -199,8 +226,8 @@ class LoginPage(LoginPage, HTMLPage): password = ''.join([c if c.isdigit() else [k for k, v in self.TO_DIGIT.items() if c in v][0] for c in password.lower()]) form = self.get_form() keyboard_page = self.browser.keyboard.open() - vk = BoursoramaVirtKeyboard(keyboard_page) - code = vk.get_code(password) + vk = BoursoramaVirtKeyboard(self.browser, keyboard_page) + code = vk.get_string_code(password) form['form[login]'] = login form['form[fakePassword]'] = len(password) * '•' form['form[password]'] = code @@ -852,6 +879,8 @@ class ProfilePage(LoggedPage, HTMLPage): klass = Person obj_name = Format('%s %s %s', MySelect('genderTitle'), MyInput('firstName'), MyInput('lastName')) + obj_firstname = MyInput('firstName') + obj_lastname = MyInput('lastName') obj_nationality = CleanText(u'//span[contains(text(), "Nationalité")]/span') obj_spouse_name = MyInput('spouseFirstName') obj_children = CleanDecimal(MyInput('dependentChildren'), default=NotAvailable) @@ -993,10 +1022,7 @@ class TransferCharac(LoggedPage, HTMLPage): else: assert self.get_option(form.el.xpath('//select[@id="Characteristics_schedulingType"]')[0], 'Différé') == '2' form['Characteristics[schedulingType]'] = '2' - # If we let the 0 in the front of the month or the day like 02, the website will not interpret the good date - form['Characteristics[scheduledDate][day]'] = exec_date.strftime('%d').lstrip("0") - form['Characteristics[scheduledDate][month]'] = exec_date.strftime('%m').lstrip("0") - form['Characteristics[scheduledDate][year]'] = exec_date.strftime('%Y') + form['Characteristics[scheduledDate]'] = exec_date.strftime('%d/%m/%Y') form['Characteristics[notice]'] = 'none' form.submit() @@ -1009,25 +1035,25 @@ class TransferConfirm(LoggedPage, HTMLPage): raise TransferInvalidAmount(message=errors) def need_refresh(self): - return not self.doc.xpath('//form[@name="Confirm"]//button[contains(text(), "Je valide")]') + return not self.doc.xpath('//form[@name="Confirm"]//button[contains(text(), "Valider")]') @method class get_transfer(ItemElement): klass = Transfer - obj_label = CleanText('//div[@id="transfer-label"]/span[@class="transfer__account-value"]') - obj_amount = CleanDecimal('//div[@id="transfer-amount"]/span[@class="transfer__account-value"]', replace_dots=True) - obj_currency = CleanCurrency('//div[@id="transfer-amount"]/span[@class="transfer__account-value"]') + obj_label = CleanText('//span[@id="transfer-label"]/span[@class="transfer__account-value"]') + obj_amount = CleanDecimal.French('//span[@id="transfer-amount"]/span[@class="transfer__account-value"]') + obj_currency = CleanCurrency('//span[@id="transfer-amount"]/span[@class="transfer__account-value"]') obj_account_label = CleanText('//span[@id="transfer-origin-account"]') obj_recipient_label = CleanText('//span[@id="transfer-destination-account"]') def obj_exec_date(self): - type_ = CleanText('//div[@id="transfer-type"]/span[@class="transfer__account-value"]')(self) + type_ = CleanText('//span[@id="transfer-type"]/span[@class="transfer__account-value"]')(self) if type_ == 'Ponctuel': return datetime.date.today() elif type_ == 'Différé': - return Date(CleanText('//div[@id="transfer-date"]/span[@class="transfer__account-value"]'), dayfirst=True)(self) + return Date(CleanText('//span[@id="transfer-date"]/span[@class="transfer__account-value"]'), dayfirst=True)(self) def submit(self): form = self.get_form(name='Confirm') @@ -1066,42 +1092,31 @@ class AddRecipientPage(LoggedPage, HTMLPage): form['externalAccountsPrepareType[beneficiaryFirstname]'] = recipient.label form['externalAccountsPrepareType[bank]'] = recipient.bank_name or 'Autre' form['externalAccountsPrepareType[iban]'] = recipient.iban + form['submit'] = '' form.submit() def is_send_sms(self): - return self._is_form(name='otp_prepare') + return self._is_form(name='strong_authentication_prepare') def send_sms(self): - form = self.get_form(name='otp_prepare') - form['otp_prepare[receiveCode]'] = '' + form = self.get_form(name='strong_authentication_prepare') form.submit() def is_confirm_sms(self): - return self._is_form(name='otp_confirm') + return self._is_form(name='strong_authentication_confirm') - def confirm_sms(self, code): - form = self.get_form(name='otp_confirm') - form['otp_confirm[otpCode]'] = code - form.submit() + def get_confirm_sms_form(self): + form = self.get_form(name='strong_authentication_confirm') + recipient_form = {k: v for k, v in form.items()} + recipient_form['url'] = form.url + return recipient_form - def is_confirm(self): + def is_confirm_send_sms(self): return self._is_form(name='externalAccountsConfirmType') - def confirm(self): - self.get_form(name='externalAccountsConfirmType').submit() - - def get_recipient(self): - div = self.doc.xpath('//div[@class="confirmation__text"]')[0] - - ret = Recipient() - ret.label = CleanText('//p[b[contains(text(),"Libellé du compte :")]]/text()')(div) - ret.iban = ret.id = CleanText('//p[b[contains(text(),"Iban :")]]/text()')(div) - ret.bank_name = CleanText(u'//p[b[contains(text(),"Établissement bancaire :")]]/text()')(div) - ret.currency = u'EUR' - ret.category = u'Externe' - ret.enabled_at = datetime.date.today() - assert ret.label - return ret + def confirm_send_sms(self): + form = self.get_form(name='externalAccountsConfirmType') + form.submit() def is_created(self): return CleanText('//p[contains(text(), "Le bénéficiaire a bien été ajouté.")]')(self.doc) != "" diff --git a/modules/bp/browser.py b/modules/bp/browser.py index d2970cd6134b6365997e8ae0cd229671f4a78c09..cb3c47a190690ccd73431f6d79e48a9cfdc2e4cb 100644 --- a/modules/bp/browser.py +++ b/modules/bp/browser.py @@ -26,12 +26,13 @@ from weboob.browser.exceptions import ServerError from weboob.capabilities.base import NotAvailable from weboob.exceptions import BrowserIncorrectPassword, BrowserBanned, NoAccountsException, BrowserUnavailable from weboob.tools.compat import urlsplit, urlunsplit, parse_qsl +from weboob.tools.decorators import retry from .pages import ( LoginPage, Initident, CheckPassword, repositionnerCheminCourant, BadLoginPage, AccountDesactivate, AccountList, AccountHistory, CardsList, UnavailablePage, AccountRIB, Advisor, TransferChooseAccounts, CompleteTransfer, TransferConfirm, TransferSummary, CreateRecipient, ValidateRecipient, - ValidateCountry, ConfirmPage, RcptSummary, SubscriptionPage, DownloadPage, ProSubscriptionPage, + ValidateCountry, ConfirmPage, RcptSummary, SubscriptionPage, DownloadPage, ProSubscriptionPage, RevolvingAttributesPage, ) from .pages.accounthistory import ( LifeInsuranceInvest, LifeInsuranceHistory, LifeInsuranceHistoryInv, RetirementHistory, @@ -76,9 +77,11 @@ class BPBrowser(LoginBrowser, StatesMixin): '/voscomptes/canalXHTML/pret/creditRenouvelable/init-consulterCreditRenouvelable.ea', '/voscomptes/canalXHTML/pret/encours/rechercherPret-encoursPrets.ea', '/voscomptes/canalXHTML/sso/commun/init-integration.ea\?partenaire=cristalCEC', - '/voscomptes/canalXHTML/sso/lbpf/souscriptionCristalFormAutoPost.jsp', AccountList) - par_accounts_revolving = URL('https://espaceclientcreditconso.labanquepostale.fr/sav/accueil.do', AccountList) + + revolving_start = URL(r'/voscomptes/canalXHTML/sso/lbpf/souscriptionCristalFormAutoPost.jsp', AccountList) + par_accounts_revolving = URL(r'https://espaceclientcreditconso.labanquepostale.fr/sav/loginlbpcrypt.do', RevolvingAttributesPage) + accounts_rib = URL(r'.*voscomptes/canalXHTML/comptesCommun/imprimerRIB/init-imprimer_rib.ea.*', '/voscomptes/canalXHTML/comptesCommun/imprimerRIB/init-selection_rib.ea', AccountRIB) @@ -141,7 +144,11 @@ class BPBrowser(LoginBrowser, StatesMixin): r'/voscomptes/canalXHTML/virement/virementSafran_sepa/valider-virementSepa.ea', r'/voscomptes/canalXHTML/virement/virementSafran_sepa/confirmerInformations-virementSepa.ea', r'/voscomptes/canalXHTML/virement/virementSafran_national/valider-creerVirementNational.ea', - r'/voscomptes/canalXHTML/virement/virementSafran_national/validerVirementNational-virementNational.ea', TransferConfirm) + r'/voscomptes/canalXHTML/virement/virementSafran_national/validerVirementNational-virementNational.ea', + # the following url is already used in transfer_summary + # but we need it to detect the case where the website displaies the list of devices + # when a transfer is made with an otp or decoupled + r'/voscomptes/canalXHTML/virement/virementSafran_sepa/confirmer-creerVirementSepa.ea', TransferConfirm) transfer_summary = URL(r'/voscomptes/canalXHTML/virement/virementSafran_national/confirmerVirementNational-virementNational.ea', r'/voscomptes/canalXHTML/virement/virementSafran_pea/confirmerInformations-virementPea.ea', r'/voscomptes/canalXHTML/virement/virementSafran_sepa/confirmer-creerVirementSepa.ea', @@ -271,8 +278,7 @@ class BPBrowser(LoginBrowser, StatesMixin): accounts.append(student_loan) else: # The main revolving page is not accessible, we can reach it by this new way - self.location(self.absurl('/voscomptes/canalXHTML/sso/lbpf/souscriptionCristalFormAutoPost.jsp')) - self.page.go_revolving() + self.go_revolving() revolving_loan = self.page.get_revolving_attributes(account) accounts.append(revolving_loan) page.go() @@ -308,6 +314,11 @@ class BPBrowser(LoginBrowser, StatesMixin): return self.accounts + @retry(BrowserUnavailable, delay=5) + def go_revolving(self): + self.revolving_start.go() + self.page.go_revolving() + def iter_cards(self, account): self.deferred_card_history.go(accountId=account.id, monthIndex=0, cardIndex=0) if self.cards_list.is_here(): @@ -482,6 +493,7 @@ class BPBrowser(LoginBrowser, StatesMixin): self.page.confirm() # Should only happen if double auth. if self.transfer_confirm.is_here(): + self.page.choose_device() self.page.double_auth(transfer) return self.page.handle_response(transfer) diff --git a/modules/bp/pages/__init__.py b/modules/bp/pages/__init__.py index 939cdd8fd78f554991b52217d2293147ec093905..b5b566aa33c3af1065d3eb9389b0df0de6b88e7d 100644 --- a/modules/bp/pages/__init__.py +++ b/modules/bp/pages/__init__.py @@ -19,7 +19,7 @@ from .login import LoginPage, Initident, CheckPassword,repositionnerCheminCourant, BadLoginPage, AccountDesactivate, UnavailablePage -from .accountlist import AccountList, AccountRIB, Advisor +from .accountlist import AccountList, AccountRIB, Advisor, RevolvingAttributesPage from .accounthistory import AccountHistory, CardsList from .transfer import TransferChooseAccounts, CompleteTransfer, TransferConfirm, TransferSummary, CreateRecipient, ValidateRecipient,\ ValidateCountry, ConfirmPage, RcptSummary @@ -29,4 +29,4 @@ from .subscription import SubscriptionPage, DownloadPage, ProSubscriptionPage __all__ = ['LoginPage', 'Initident', 'CheckPassword', 'repositionnerCheminCourant', "AccountList", 'AccountHistory', 'BadLoginPage', 'AccountDesactivate', 'TransferChooseAccounts', 'CompleteTransfer', 'TransferConfirm', 'TransferSummary', 'UnavailablePage', 'CardsList', 'AccountRIB', 'Advisor', 'CreateRecipient', 'ValidateRecipient', 'ValidateCountry', 'ConfirmPage', 'RcptSummary', - 'SubscriptionPage', 'DownloadPage', 'ProSubscriptionPage'] + 'SubscriptionPage', 'DownloadPage', 'ProSubscriptionPage', 'RevolvingAttributesPage'] diff --git a/modules/bp/pages/accounthistory.py b/modules/bp/pages/accounthistory.py index a92056fdb9043cd2a3e5271a9f56d88044c42133..88dc76eab3908048bb16760696e5e7049eac6809 100644 --- a/modules/bp/pages/accounthistory.py +++ b/modules/bp/pages/accounthistory.py @@ -76,7 +76,7 @@ class AccountHistory(LoggedPage, MyHTMLPage): def get_next_link(self): for a in self.doc.xpath('//a[@class="btn_crt"]'): - txt = u''.join([txt.strip() for txt in a.itertext()]) + txt = u''.join([txt2.strip() for txt2 in a.itertext()]) if u'mois précédent' in txt: return a.attrib['href'] diff --git a/modules/bp/pages/accountlist.py b/modules/bp/pages/accountlist.py index 2239d762759559db093843f3251279c0304bcb3a..d142eca3bc8017353b23aa9549a24f2d7d9a9588 100644 --- a/modules/bp/pages/accountlist.py +++ b/modules/bp/pages/accountlist.py @@ -189,7 +189,8 @@ class AccountList(LoggedPage, MyHTMLPage): def on_load(self): MyHTMLPage.on_load(self) - if self.doc.xpath('//h2[text()="ERREUR"]'): # website sometime crash + # website sometimes crash + if CleanText('//h2[text()="ERREUR"]')(self.doc): self.browser.location('https://voscomptesenligne.labanquepostale.fr/voscomptes/canalXHTML/securite/authentification/initialiser-identif.ea') raise BrowserUnavailable() @@ -217,22 +218,6 @@ class AccountList(LoggedPage, MyHTMLPage): def condition(self): return item_account_generic.condition(self) - - def get_revolving_attributes(self, account): - loan = Loan() - loan.id = account.id - loan.label = '%s - %s' %(account.label, account.id) - loan.currency = account.currency - loan.url = account.url - - loan.used_amount = CleanDecimal.US('//tr[td[contains(text(), "Montant Utilisé") or contains(text(), "Montant utilisé")]]/td[2]')(self.doc) - loan.available_amount = CleanDecimal.US(Regexp(CleanText('//tr[td[contains(text(), "Montant Disponible") or contains(text(), "Montant disponible")]]/td[2]'), r'(.*) au'))(self.doc) - loan.balance = -loan.used_amount - loan._has_cards = False - loan.type = Account.TYPE_REVOLVING_CREDIT - return loan - - @method class iter_revolving_loans(ListElement): item_xpath = '//div[@class="bloc Tmargin"]//dl' @@ -463,3 +448,23 @@ class ProfilePage(LoggedPage, HTMLPage): profile.job = CleanText('//div[@id="persoIdentiteDetail"]//dd[4]')(self.doc) return profile + + +class RevolvingAttributesPage(LoggedPage, HTMLPage): + def on_load(self): + if CleanText('//h1[contains(text(), "Erreur")]')(self.doc): + raise BrowserUnavailable() + + def get_revolving_attributes(self, account): + loan = Loan() + loan.id = account.id + loan.label = '%s - %s' % (account.label, account.id) + loan.currency = account.currency + loan.url = account.url + + loan.used_amount = CleanDecimal.US('//tr[td[contains(text(), "Montant Utilisé") or contains(text(), "Montant utilisé")]]/td[2]')(self.doc) + loan.available_amount = CleanDecimal.US(Regexp(CleanText('//tr[td[contains(text(), "Montant Disponible") or contains(text(), "Montant disponible")]]/td[2]'), r'(.*) au'))(self.doc) + loan.balance = -loan.used_amount + loan._has_cards = False + loan.type = Account.TYPE_REVOLVING_CREDIT + return loan diff --git a/modules/bp/pages/transfer.py b/modules/bp/pages/transfer.py index d4e0b69edcc38a25e475c31d816522d207c7cd94..043e4f76393a5737aa4213ef6acc96786832991b 100644 --- a/modules/bp/pages/transfer.py +++ b/modules/bp/pages/transfer.py @@ -33,7 +33,7 @@ from weboob.browser.elements import ListElement, ItemElement, method, SkipItem from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.tools.capabilities.bank.iban import is_iban_valid from weboob.tools.value import Value -from weboob.exceptions import BrowserUnavailable +from weboob.exceptions import BrowserUnavailable, AuthMethodNotImplemented from .base import MyHTMLPage @@ -140,9 +140,21 @@ class TransferConfirm(LoggedPage, CheckTransferError): def is_here(self): return ( not CleanText('//p[contains(text(), "Vous pouvez le consulter dans le menu")]')(self.doc) - or self.doc.xpath('//input[@title="Confirmer la demande de virement"]') + or self.doc.xpath('//input[@title="Confirmer la demande de virement"]') # appears when there is no need for otp/polling + or self.doc.xpath("//span[contains(text(), 'cliquant sur le bouton \"CONFIRMER\"')]") # appears on the page when there is a 'Confirmer' button or not ) + def choose_device(self): + # When there is no "Confirmer" button, + # it means that the device pop up appeared (it is called by js) + if ( + not self.doc.xpath('//input[@value="Confirmer"]') + or self.doc.xpath('//input[@name="codeOTPSaisi"]') + ): + # transfer validation form with sms cannot be tested yet + raise AuthMethodNotImplemented() + assert False, 'Should not be on confirmation page after posting the form.' + def double_auth(self, transfer): code_needed = CleanText('//label[@for="code_securite"]')(self.doc) if code_needed: @@ -185,6 +197,8 @@ class TransferConfirm(LoggedPage, CheckTransferError): class TransferSummary(LoggedPage, CheckTransferError): + is_here = '//h3[contains(text(), "Récapitulatif")]' + def handle_response(self, transfer): summary_filter = CleanText( '//div[contains(@class, "bloc-recapitulatif")]//p' diff --git a/modules/caissedepargne/browser.py b/modules/caissedepargne/browser.py index 8b5ed8985cd25c504a3d07e7c0c42b87cd9b8141..282e9d94fbfb13d771cefc31377e7e8cae82072a 100644 --- a/modules/caissedepargne/browser.py +++ b/modules/caissedepargne/browser.py @@ -73,60 +73,77 @@ class CaisseEpargne(LoginBrowser, StatesMixin): LINEBOURSE_BROWSER = LinebourseAPIBrowser - login = URL('/authentification/manage\?step=identification&identifiant=(?P.*)', - 'https://.*/login.aspx', LoginPage) - account_login = URL('/authentification/manage\?step=account&identifiant=(?P.*)&account=(?P.*)', LoginPage) - loading = URL('https://.*/CreditConso/ReroutageCreditConso.aspx', LoadingPage) - cons_loan = URL('https://www.credit-conso-cr.caisse-epargne.fr/websavcr-web/rest/contrat/getContrat\?datePourIe=(?P)', ConsLoanPage) - transaction_detail = URL('https://.*/Portail.aspx.*', TransactionsDetailsPage) - recipient = URL('https://.*/Portail.aspx.*', RecipientPage) - transfer = URL('https://.*/Portail.aspx.*', TransferPage) - transfer_summary = URL('https://.*/Portail.aspx.*', TransferSummaryPage) - transfer_confirm = URL('https://.*/Portail.aspx.*', TransferConfirmPage) - pro_transfer = URL('https://.*/Portail.aspx.*', ProTransferPage) - pro_transfer_confirm = URL('https://.*/Portail.aspx.*', ProTransferConfirmPage) - pro_transfer_summary = URL('https://.*/Portail.aspx.*', ProTransferSummaryPage) - pro_add_recipient_otp = URL('https://.*/Portail.aspx.*', ProAddRecipientOtpPage) - pro_add_recipient = URL('https://.*/Portail.aspx.*', ProAddRecipientPage) - measure_page = URL('https://.*/Portail.aspx.*', MeasurePage) - cards_old = URL('https://.*/Portail.aspx.*', CardsOldWebsitePage) - cards = URL('https://.*/Portail.aspx.*', CardsPage) - cards_coming = URL('https://.*/Portail.aspx.*', CardsComingPage) + login = URL( + r'/authentification/manage\?step=identification&identifiant=(?P.*)', + r'https://.*/login.aspx', + LoginPage + ) + account_login = URL(r'/authentification/manage\?step=account&identifiant=(?P.*)&account=(?P.*)', LoginPage) + loading = URL(r'https://.*/CreditConso/ReroutageCreditConso.aspx', LoadingPage) + cons_loan = URL(r'https://www.credit-conso-cr.caisse-epargne.fr/websavcr-web/rest/contrat/getContrat\?datePourIe=(?P)', ConsLoanPage) + transaction_detail = URL(r'https://.*/Portail.aspx.*', TransactionsDetailsPage) + recipient = URL(r'https://.*/Portail.aspx.*', RecipientPage) + transfer = URL(r'https://.*/Portail.aspx.*', TransferPage) + transfer_summary = URL(r'https://.*/Portail.aspx.*', TransferSummaryPage) + transfer_confirm = URL(r'https://.*/Portail.aspx.*', TransferConfirmPage) + pro_transfer = URL(r'https://.*/Portail.aspx.*', ProTransferPage) + pro_transfer_confirm = URL(r'https://.*/Portail.aspx.*', ProTransferConfirmPage) + pro_transfer_summary = URL(r'https://.*/Portail.aspx.*', ProTransferSummaryPage) + pro_add_recipient_otp = URL(r'https://.*/Portail.aspx.*', ProAddRecipientOtpPage) + pro_add_recipient = URL(r'https://.*/Portail.aspx.*', ProAddRecipientPage) + measure_page = URL(r'https://.*/Portail.aspx.*', MeasurePage) + cards_old = URL(r'https://.*/Portail.aspx.*', CardsOldWebsitePage) + cards = URL(r'https://.*/Portail.aspx.*', CardsPage) + cards_coming = URL(r'https://.*/Portail.aspx.*', CardsComingPage) old_checkings_levies = URL(r'https://.*/Portail.aspx.*', OldLeviesPage) new_checkings_levies = URL(r'https://.*/Portail.aspx.*', NewLeviesPage) - authent = URL('https://.*/Portail.aspx.*', AuthentPage) - subscription = URL('https://.*/Portail.aspx\?tache=(?P).*', SubscriptionPage) + authent = URL(r'https://.*/Portail.aspx.*', AuthentPage) + subscription = URL(r'https://.*/Portail.aspx\?tache=(?P).*', SubscriptionPage) transaction_popup = URL(r'https://.*/Portail.aspx.*', TransactionPopupPage) - home = URL('https://.*/Portail.aspx.*', IndexPage) - home_tache = URL('https://.*/Portail.aspx\?tache=(?P).*', IndexPage) - error = URL('https://.*/login.aspx', - 'https://.*/Pages/logout.aspx.*', - 'https://.*/particuliers/Page_erreur_technique.aspx.*', ErrorPage) - market = URL('https://.*/Pages/Bourse.*', - 'https://www.caisse-epargne.offrebourse.com/ReroutageSJR', - r'https://www.caisse-epargne.offrebourse.com/fr/6CE.*', MarketPage) - unavailable_page = URL('https://www.caisse-epargne.fr/.*/au-quotidien', UnavailablePage) - - creditcooperatif_market = URL('https://www.offrebourse.com/.*', CreditCooperatifMarketPage) # just to catch the landing page of the Credit Cooperatif's Linebourse - natixis_redirect = URL(r'/NaAssuranceRedirect/NaAssuranceRedirect.aspx', - r'https://www.espace-assurances.caisse-epargne.fr/espaceinternet-ce/views/common/routage-itce.xhtml\?windowId=automatedEntryPoint', - NatixisRedirectPage) + home = URL(r'https://.*/Portail.aspx.*', IndexPage) + home_tache = URL(r'https://.*/Portail.aspx\?tache=(?P).*', IndexPage) + error = URL( + r'https://.*/login.aspx', + r'https://.*/Pages/logout.aspx.*', + r'https://.*/particuliers/Page_erreur_technique.aspx.*', + ErrorPage + ) + market = URL( + r'https://.*/Pages/Bourse.*', + r'https://www.caisse-epargne.offrebourse.com/ReroutageSJR', + r'https://www.caisse-epargne.offrebourse.com/fr/6CE.*', + MarketPage + ) + unavailable_page = URL(r'https://www.caisse-epargne.fr/.*/au-quotidien', UnavailablePage) + + creditcooperatif_market = URL(r'https://www.offrebourse.com/.*', CreditCooperatifMarketPage) # just to catch the landing page of the Credit Cooperatif's Linebourse + natixis_redirect = URL( + r'/NaAssuranceRedirect/NaAssuranceRedirect.aspx', + r'https://www.espace-assurances.caisse-epargne.fr/espaceinternet-ce/views/common/routage-itce.xhtml\?windowId=automatedEntryPoint', + NatixisRedirectPage + ) life_insurance_history = URL(r'https://www.extranet2.caisse-epargne.fr/cin-front/contrats/evenements', LifeInsuranceHistory) life_insurance_investments = URL(r'https://www.extranet2.caisse-epargne.fr/cin-front/contrats/details', LifeInsuranceInvestments) - life_insurance = URL(r'https://.*/Assurance/Pages/Assurance.aspx', - r'https://www.extranet2.caisse-epargne.fr.*', LifeInsurance) + life_insurance = URL( + r'https://.*/Assurance/Pages/Assurance.aspx', + r'https://www.extranet2.caisse-epargne.fr.*', + LifeInsurance + ) natixis_life_ins_his = URL(r'https://www.espace-assurances.caisse-epargne.fr/espaceinternet-ce/rest/v2/contratVie/load-operation/(?P\w+)/(?P\w+)/(?P)', NatixisLIHis) natixis_life_ins_inv = URL(r'https://www.espace-assurances.caisse-epargne.fr/espaceinternet-ce/rest/v2/contratVie/load/(?P\w+)/(?P\w+)/(?P)', NatixisLIInv) message = URL(r'https://www.caisse-epargne.offrebourse.com/DetailMessage\?refresh=O', MessagePage) - garbage = URL(r'https://www.caisse-epargne.offrebourse.com/Portefeuille', - r'https://www.caisse-epargne.fr/particuliers/.*/emprunter.aspx', - r'https://.*/particuliers/emprunter.*', - r'https://.*/particuliers/epargner.*', GarbagePage) + garbage = URL( + r'https://www.caisse-epargne.offrebourse.com/Portefeuille', + r'https://www.caisse-epargne.fr/particuliers/.*/emprunter.aspx', + r'https://.*/particuliers/emprunter.*', + r'https://.*/particuliers/epargner.*', + GarbagePage + ) sms = URL(r'https://www.icgauth.caisse-epargne.fr/dacswebssoissuer/AuthnRequestServlet', SmsPage) sms_option = URL(r'https://www.icgauth.caisse-epargne.fr/dacstemplate-SOL/index.html\?transactionID=.*', SmsPageOption) request_sms = URL( r'https://(?Pwww.icgauth.[^/]+)/dacsrest/api/v1u0/transaction/(?P)', - SmsRequest, + SmsRequest ) __states__ = ( @@ -421,13 +438,14 @@ class CaisseEpargne(LoginBrowser, StatesMixin): if self.accounts is None: self.accounts = self.get_measure_accounts_list() if self.accounts is None: + owner_name = self.get_profile().name.upper().split(' ', 1)[1] if self.home.is_here(): self.page.check_no_accounts() self.page.go_list() else: self.home.go() - self.accounts = list(self.page.get_list()) + self.accounts = list(self.page.get_list(owner_name)) for account in self.accounts: self.deleteCTX() if account.type in (Account.TYPE_MARKET, Account.TYPE_PEA): @@ -485,6 +503,7 @@ class CaisseEpargne(LoginBrowser, StatesMixin): for card in self.page.iter_cards(): card.parent = account card._coming_info = self.page.get_card_coming_info(card.number, card.parent._card_links.copy()) + card.ownership = account.ownership self.accounts.append(card) self.home.go() @@ -506,6 +525,7 @@ class CaisseEpargne(LoginBrowser, StatesMixin): # If card.parent._card_links is not filled, it mean this checking account # has no coming transactions. card._coming_info = None + card.ownership = card.parent.ownership if info: self.page.go_list() self.page.go_history(info) @@ -715,6 +735,10 @@ class CaisseEpargne(LoginBrowser, StatesMixin): except (IndexError, AttributeError) as e: self.logger.error(e) return [] + except ServerError as e: + if e.response.status_code == 500: + raise BrowserUnavailable() + raise return self.page.iter_history() @need_login @@ -882,7 +906,7 @@ class CaisseEpargne(LoginBrowser, StatesMixin): profile = Profile() if len([k for k in self.session.cookies.keys() if k == 'CTX']) > 1: del self.session.cookies['CTX'] - elif 'username=' in self.session.cookies.get('CTX', ''): + if 'username=' in self.session.cookies.get('CTX', ''): profile.name = to_unicode(re.search('username=([^&]+)', self.session.cookies['CTX']).group(1)) elif 'nomusager=' in self.session.cookies.get('headerdei'): profile.name = to_unicode(re.search('nomusager=(?:[^&]+/ )?([^&]+)', self.session.cookies['headerdei']).group(1)) diff --git a/modules/caissedepargne/cenet/browser.py b/modules/caissedepargne/cenet/browser.py index c4f83d8b9b51f1614f5b2f9a531715be7176d1fa..52d75fb59d15f619bf654f648cab4f9392c03871 100644 --- a/modules/caissedepargne/cenet/browser.py +++ b/modules/caissedepargne/cenet/browser.py @@ -19,6 +19,7 @@ from __future__ import unicode_literals + import json from weboob.browser import LoginBrowser, need_login, StatesMixin @@ -47,26 +48,35 @@ class CenetBrowser(LoginBrowser, StatesMixin): STATE_DURATION = 5 - login = URL(r'https://(?P[^/]+)/authentification/manage\?step=identification&identifiant=(?P.*)', - r'https://.*/authentification/manage\?step=identification&identifiant=.*', - r'https://.*/login.aspx', LoginPage) - account_login = URL('https://(?P[^/]+)/authentification/manage\?step=account&identifiant=(?P.*)&account=(?P.*)', LoginPage) - cenet_vk = URL('https://www.cenet.caisse-epargne.fr/Web/Api/ApiAuthentification.asmx/ChargerClavierVirtuel') - cenet_home = URL('/Default.aspx$', CenetHomePage) - cenet_accounts = URL('/Web/Api/ApiComptes.asmx/ChargerSyntheseComptes', CenetAccountsPage) - cenet_loans = URL('/Web/Api/ApiFinancements.asmx/ChargerListeFinancementsMLT', CenetLoanPage) - cenet_account_history = URL('/Web/Api/ApiComptes.asmx/ChargerHistoriqueCompte', CenetAccountHistoryPage) - cenet_account_coming = URL('/Web/Api/ApiCartesBanquaires.asmx/ChargerEnCoursCarte', CenetAccountHistoryPage) - cenet_tr_detail = URL('/Web/Api/ApiComptes.asmx/ChargerDetailOperation', CenetCardSummaryPage) - cenet_cards = URL('/Web/Api/ApiCartesBanquaires.asmx/ChargerCartes', CenetCardsPage) - error = URL(r'https://.*/login.aspx', - r'https://.*/Pages/logout.aspx.*', - r'https://.*/particuliers/Page_erreur_technique.aspx.*', ErrorPage) - cenet_login = URL(r'https://.*/$', - r'https://.*/default.aspx', CenetLoginPage) - - subscription = URL('/Web/Api/ApiReleves.asmx/ChargerListeEtablissements', SubscriptionPage) - documents = URL('/Web/Api/ApiReleves.asmx/ChargerListeReleves', SubscriptionPage) + login = URL( + r'https://(?P[^/]+)/authentification/manage\?step=identification&identifiant=(?P.*)', + r'https://.*/authentification/manage\?step=identification&identifiant=.*', + r'https://.*/login.aspx', + LoginPage, + ) + account_login = URL(r'https://(?P[^/]+)/authentification/manage\?step=account&identifiant=(?P.*)&account=(?P.*)', LoginPage) + cenet_vk = URL(r'https://www.cenet.caisse-epargne.fr/Web/Api/ApiAuthentification.asmx/ChargerClavierVirtuel') + cenet_home = URL(r'/Default.aspx$', CenetHomePage) + cenet_accounts = URL(r'/Web/Api/ApiComptes.asmx/ChargerSyntheseComptes', CenetAccountsPage) + cenet_loans = URL(r'/Web/Api/ApiFinancements.asmx/ChargerListeFinancementsMLT', CenetLoanPage) + cenet_account_history = URL(r'/Web/Api/ApiComptes.asmx/ChargerHistoriqueCompte', CenetAccountHistoryPage) + cenet_account_coming = URL(r'/Web/Api/ApiCartesBanquaires.asmx/ChargerEnCoursCarte', CenetAccountHistoryPage) + cenet_tr_detail = URL(r'/Web/Api/ApiComptes.asmx/ChargerDetailOperation', CenetCardSummaryPage) + cenet_cards = URL(r'/Web/Api/ApiCartesBanquaires.asmx/ChargerCartes', CenetCardsPage) + error = URL( + r'https://.*/login.aspx', + r'https://.*/Pages/logout.aspx.*', + r'https://.*/particuliers/Page_erreur_technique.aspx.*', + ErrorPage, + ) + cenet_login = URL( + r'https://.*/$', + r'https://.*/default.aspx', + CenetLoginPage, + ) + + subscription = URL(r'/Web/Api/ApiReleves.asmx/ChargerListeEtablissements', SubscriptionPage) + documents = URL(r'/Web/Api/ApiReleves.asmx/ChargerListeReleves', SubscriptionPage) download = URL(r'/Default.aspx\?dashboard=ComptesReleves&lien=SuiviReleves', DownloadDocumentPage) __states__ = ('BASEURL',) @@ -192,7 +202,7 @@ class CenetBrowser(LoginBrowser, StatesMixin): card_tr_list.append(tr) tr.deleted = True - tr_dict = [tr_dict for tr_dict in data_out if tr_dict['Libelle'] == tr.label] + tr_dict = [tr_dict2 for tr_dict2 in data_out if tr_dict2['Libelle'] == tr.label] donneesEntree = {} donneesEntree['Compte'] = account._formated donneesEntree['ListeOperations'] = [tr_dict[0]] @@ -280,15 +290,15 @@ class CenetBrowser(LoginBrowser, StatesMixin): def iter_documents(self, subscription): sub_id = subscription.id input_filter = { - 'Page':0, - 'NombreParPage':0, - 'Tris':[], - 'Criteres':[ - {'Champ': 'Etablissement','TypeCritere': 'Equals','Value': sub_id}, - {'Champ': 'DateDebut','TypeCritere': 'Equals','Value': None}, - {'Champ': 'DateFin','TypeCritere': 'Equals','Value': None}, - {'Champ': 'MaxRelevesAffichesParNumero','TypeCritere': 'Equals','Value': '100'} - ] + 'Page': 0, + 'NombreParPage': 0, + 'Tris': [], + 'Criteres': [ + {'Champ': 'Etablissement', 'TypeCritere': 'Equals', 'Value': sub_id}, + {'Champ': 'DateDebut', 'TypeCritere': 'Equals', 'Value': None}, + {'Champ': 'DateFin', 'TypeCritere': 'Equals', 'Value': None}, + {'Champ': 'MaxRelevesAffichesParNumero', 'TypeCritere': 'Equals', 'Value': '100'}, + ], } json_data = { 'contexte': '', diff --git a/modules/caissedepargne/cenet/pages.py b/modules/caissedepargne/cenet/pages.py index 028a004fa708ec7bec61dc5e16b638681f02ae2b..833356eb916fb85c1f8f16ed2d41f86b91934c2b 100644 --- a/modules/caissedepargne/cenet/pages.py +++ b/modules/caissedepargne/cenet/pages.py @@ -35,36 +35,23 @@ from weboob.exceptions import BrowserUnavailable class Transaction(FrenchTransaction): - PATTERNS = [(re.compile('^CB (?P.*?) FACT (?P
\d{2})(?P\d{2})(?P\d{2})', re.IGNORECASE), - FrenchTransaction.TYPE_CARD), - (re.compile('^RET(RAIT)? DAB (?P
\d+)-(?P\d+)-.*', re.IGNORECASE), - FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile('^RET(RAIT)? DAB (?P.*?) (?P
\d{2})(?P\d{2})(?P\d{2}) (?P\d{2})H(?P\d{2})', re.IGNORECASE), - FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile('^VIR(EMENT)?(\.PERIODIQUE)? (?P.*)', re.IGNORECASE), - FrenchTransaction.TYPE_TRANSFER), - (re.compile('^PRLV (?P.*)', re.IGNORECASE), - FrenchTransaction.TYPE_ORDER), - (re.compile('^CHEQUE.*', re.IGNORECASE), FrenchTransaction.TYPE_CHECK), - (re.compile('^(CONVENTION \d+ )?COTIS(ATION)? (?P.*)', re.IGNORECASE), - FrenchTransaction.TYPE_BANK), - (re.compile(r'^\* (?P.*)', re.IGNORECASE), - FrenchTransaction.TYPE_BANK), - (re.compile('^REMISE (?P.*)', re.IGNORECASE), - FrenchTransaction.TYPE_DEPOSIT), - (re.compile('^(?P.*)( \d+)? QUITTANCE .*', re.IGNORECASE), - FrenchTransaction.TYPE_ORDER), - (re.compile('^CB [\d\*]+ TOT DIF .*', re.IGNORECASE), - FrenchTransaction.TYPE_CARD_SUMMARY), - (re.compile('^CB [\d\*]+ (?P.*)', re.IGNORECASE), - FrenchTransaction.TYPE_CARD), - (re.compile('^CB (?P.*?) (?P
\d{2})(?P\d{2})(?P\d{2})', re.IGNORECASE), - FrenchTransaction.TYPE_CARD), - (re.compile('\*CB (?P.*?) (?P
\d{2})(?P\d{2})(?P\d{2})', re.IGNORECASE), - FrenchTransaction.TYPE_CARD), - (re.compile('^FAC CB (?P.*?) (?P
\d{2})/(?P\d{2})', re.IGNORECASE), - FrenchTransaction.TYPE_CARD), - ] + PATTERNS = [ + (re.compile(r'^CB (?P.*?) FACT (?P
\d{2})(?P\d{2})(?P\d{2})', re.IGNORECASE), FrenchTransaction.TYPE_CARD), + (re.compile(r'^RET(RAIT)? DAB (?P
\d+)-(?P\d+)-.*', re.IGNORECASE), FrenchTransaction.TYPE_WITHDRAWAL), + (re.compile(r'^RET(RAIT)? DAB (?P.*?) (?P
\d{2})(?P\d{2})(?P\d{2}) (?P\d{2})H(?P\d{2})', re.IGNORECASE), FrenchTransaction.TYPE_WITHDRAWAL), + (re.compile(r'^VIR(EMENT)?(\.PERIODIQUE)? (?P.*)', re.IGNORECASE), FrenchTransaction.TYPE_TRANSFER), + (re.compile(r'^PRLV (?P.*)', re.IGNORECASE), FrenchTransaction.TYPE_ORDER), + (re.compile(r'^CHEQUE.*', re.IGNORECASE), FrenchTransaction.TYPE_CHECK), + (re.compile(r'^(CONVENTION \d+ )?COTIS(ATION)? (?P.*)', re.IGNORECASE), FrenchTransaction.TYPE_BANK), + (re.compile(r'^\* (?P.*)', re.IGNORECASE), FrenchTransaction.TYPE_BANK), + (re.compile(r'^REMISE (?P.*)', re.IGNORECASE), FrenchTransaction.TYPE_DEPOSIT), + (re.compile(r'^(?P.*)( \d+)? QUITTANCE .*', re.IGNORECASE), FrenchTransaction.TYPE_ORDER), + (re.compile(r'^CB [\d\*]+ TOT DIF .*', re.IGNORECASE), FrenchTransaction.TYPE_CARD_SUMMARY), + (re.compile(r'^CB [\d\*]+ (?P.*)', re.IGNORECASE), FrenchTransaction.TYPE_CARD), + (re.compile(r'^CB (?P.*?) (?P
\d{2})(?P\d{2})(?P\d{2})', re.IGNORECASE), FrenchTransaction.TYPE_CARD), + (re.compile(r'\*CB (?P.*?) (?P
\d{2})(?P\d{2})(?P\d{2})', re.IGNORECASE), FrenchTransaction.TYPE_CARD), + (re.compile(r'^FAC CB (?P.*?) (?P
\d{2})/(?P\d{2})', re.IGNORECASE), FrenchTransaction.TYPE_CARD), + ] class LoginPage(JsonPage): @@ -120,7 +107,7 @@ class CenetJsonPage(JsonPage): class CenetAccountsPage(LoggedPage, CenetJsonPage): - ACCOUNT_TYPES = {u'CCP': Account.TYPE_CHECKING} + ACCOUNT_TYPES = {'CCP': Account.TYPE_CHECKING} @method class get_accounts(DictElement): @@ -139,7 +126,6 @@ class CenetAccountsPage(LoggedPage, CenetJsonPage): return -absolut_amount return absolut_amount - def obj_currency(self): return CleanText(Dict('Devise'))(self).upper() @@ -214,6 +200,7 @@ class CenetCardsPage(LoggedPage, CenetJsonPage): return cards + class CenetAccountHistoryPage(LoggedPage, CenetJsonPage): TR_TYPES_LABEL = { 'VIR': Transaction.TYPE_TRANSFER, @@ -224,10 +211,10 @@ class CenetAccountHistoryPage(LoggedPage, CenetJsonPage): TR_TYPES_API = { 'VIR': Transaction.TYPE_TRANSFER, - 'PE': Transaction.TYPE_ORDER, # PRLV - 'CE': Transaction.TYPE_CHECK, # CHEQUE - 'DE': Transaction.TYPE_CASH_DEPOSIT, # APPRO - 'PI': Transaction.TYPE_CASH_DEPOSIT, # REMISE CHEQUE + 'PE': Transaction.TYPE_ORDER, # PRLV + 'CE': Transaction.TYPE_CHECK, # CHEQUE + 'DE': Transaction.TYPE_CASH_DEPOSIT, # APPRO + 'PI': Transaction.TYPE_CASH_DEPOSIT, # REMISE CHEQUE } @method diff --git a/modules/caissedepargne/module.py b/modules/caissedepargne/module.py index f3ec10e52b28f24a890c6ae6523e9edd387aaa3f..299478f0ad2fed9a5a6e33e6ec2d5f270a1f4ce6 100644 --- a/modules/caissedepargne/module.py +++ b/modules/caissedepargne/module.py @@ -17,9 +17,11 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . -from collections import OrderedDict +from __future__ import unicode_literals + import re from decimal import Decimal +from collections import OrderedDict from weboob.capabilities.bank import CapBankWealth, CapBankTransferAddRecipient, AccountNotFound, Account, RecipientNotFound from weboob.capabilities.bill import ( @@ -40,20 +42,22 @@ __all__ = ['CaisseEpargneModule'] class CaisseEpargneModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapDocument, CapContact, CapProfile): NAME = 'caissedepargne' - MAINTAINER = u'Romain Bignon' + MAINTAINER = 'Romain Bignon' EMAIL = 'romain@weboob.org' VERSION = '1.6' - DESCRIPTION = u'Caisse d\'Épargne' + DESCRIPTION = 'Caisse d\'Épargne' LICENSE = 'LGPLv3+' BROWSER = ProxyBrowser website_choices = OrderedDict([(k, u'%s (%s)' % (v, k)) for k, v in sorted({ 'www.caisse-epargne.fr': u'Caisse d\'Épargne', 'www.banquebcp.fr': u'Banque BCP', }.items(), key=lambda k_v: (k_v[1], k_v[0]))]) - CONFIG = BackendConfig(Value('website', label='Banque', choices=website_choices, default='www.caisse-epargne.fr'), - ValueBackendPassword('login', label='Identifiant client', masked=False), - ValueBackendPassword('password', label='Code personnel', regexp='\d+'), - Value('nuser', label='User ID (optional)', default='', regexp='[A-Z\d]{0,8}')) + CONFIG = BackendConfig( + Value('website', label='Banque', choices=website_choices, default='www.caisse-epargne.fr'), + ValueBackendPassword('login', label='Identifiant client', masked=False), + ValueBackendPassword('password', label='Code personnel', regexp='\d+'), + Value('nuser', label='User ID (optional)', default='', regexp='[A-Z\d]{0,8}'), + ) accepted_document_types = (DocumentTypes.OTHER,) @@ -117,7 +121,7 @@ class CaisseEpargneModule(Module, CapBankWealth, CapBankTransferAddRecipient, Ca return self.browser.execute_transfer(transfer) def new_recipient(self, recipient, **params): - #recipient.label = ' '.join(w for w in re.sub('[^0-9a-zA-Z:\/\-\?\(\)\.,\'\+ ]+', '', recipient.label).split()) + # recipient.label = ' '.join(w for w in re.sub('[^0-9a-zA-Z:\/\-\?\(\)\.,\'\+ ]+', '', recipient.label).split()) return self.browser.new_recipient(recipient, **params) def iter_resources(self, objs, split_path): diff --git a/modules/caissedepargne/pages.py b/modules/caissedepargne/pages.py index 6a9ef5f710e012ec8e68485363695fbd7ec09391..375b8d40bab5d560359ecc5afab7e7451405ffeb 100644 --- a/modules/caissedepargne/pages.py +++ b/modules/caissedepargne/pages.py @@ -20,6 +20,7 @@ from __future__ import division from __future__ import unicode_literals + from base64 import b64decode from collections import OrderedDict import re @@ -38,7 +39,7 @@ from weboob.browser.filters.html import Link, Attr, TableCell from weboob.capabilities import NotAvailable from weboob.capabilities.bank import ( Account, Investment, Recipient, TransferBankError, Transfer, - AddRecipientBankError, Loan, + AddRecipientBankError, Loan, AccountOwnership, ) from weboob.capabilities.bill import DocumentTypes, Subscription, Document from weboob.tools.capabilities.bank.investments import is_isin_valid @@ -49,15 +50,18 @@ from weboob.tools.compat import unicode from weboob.exceptions import NoAccountsException, BrowserUnavailable, ActionNeeded from weboob.browser.filters.json import Dict + def MyDecimal(*args, **kwargs): kwargs.update(replace_dots=True) return CleanDecimal(*args, **kwargs) + class MyTableCell(TableCell): def __init__(self, *names, **kwargs): super(MyTableCell, self).__init__(*names, **kwargs) self.td = './tr[%s]/td' + def fix_form(form): keys = ['MM$HISTORIQUE_COMPTE$btnCumul', 'Cartridge$imgbtnMessagerie', 'MM$m_CH$ButtonImageFondMessagerie', 'MM$m_CH$ButtonImageMessagerie'] @@ -105,7 +109,7 @@ class CaissedepargneKeyboard(GridVirtKeyboard): class GarbagePage(LoggedPage, HTMLPage): def on_load(self): - go_back_link = Link('//a[@class="btn"]', default=NotAvailable)(self.doc) + go_back_link = Link('//a[@class="btn" or @class="cta_stroke back"]', default=NotAvailable)(self.doc) if go_back_link is not NotAvailable: assert len(go_back_link) != 1 @@ -161,6 +165,7 @@ class Transaction(FrenchTransaction): (re.compile(r'^RACHAT PARTIEL', re.IGNORECASE), FrenchTransaction.TYPE_BANK), ] + class IndexPage(LoggedPage, HTMLPage): ACCOUNT_TYPES = { 'Epargne liquide': Account.TYPE_SAVINGS, @@ -218,23 +223,25 @@ class IndexPage(LoggedPage, HTMLPage): raise BrowserUnavailable(mess) # This page is sometimes an useless step to the market website. - bourse_link = Link(u'//div[@id="MM_COMPTE_TITRE_pnlbourseoic"]//a[contains(text(), "Accédez à la consultation")]', default=None)(self.doc) + bourse_link = Link('//div[@id="MM_COMPTE_TITRE_pnlbourseoic"]//a[contains(text(), "Accédez à la consultation")]', default=None)(self.doc) if bourse_link: self.browser.location(bourse_link) def need_auth(self): - return bool(CleanText(u'//span[contains(text(), "Authentification non rejouable")]')(self.doc)) + return bool(CleanText('//span[contains(text(), "Authentification non rejouable")]')(self.doc)) def check_no_loans(self): - return not bool(CleanText(u'//table[@class="menu"]//div[contains(., "Crédits")]')(self.doc)) and \ - not bool(CleanText(u'//table[@class="header-navigation_main"]//a[contains(., "Crédits")]')(self.doc)) + return ( + not bool(CleanText('//table[@class="menu"]//div[contains(., "Crédits")]')(self.doc)) + and not bool(CleanText('//table[@class="header-navigation_main"]//a[contains(., "Crédits")]')(self.doc)) + ) def check_measure_accounts(self): - return not CleanText(u'//div[@class="MessageErreur"]/ul/li[contains(text(), "Aucun compte disponible")]')(self.doc) + return not CleanText('//div[@class="MessageErreur"]/ul/li[contains(text(), "Aucun compte disponible")]')(self.doc) def check_no_accounts(self): - no_account_message = CleanText(u'//span[@id="MM_LblMessagePopinError"]/p[contains(text(), "Aucun compte disponible")]')(self.doc) + no_account_message = CleanText('//span[@id="MM_LblMessagePopinError"]/p[contains(text(), "Aucun compte disponible")]')(self.doc) if no_account_message: raise NoAccountsException(no_account_message) @@ -243,7 +250,7 @@ class IndexPage(LoggedPage, HTMLPage): # The site might be broken: id in js: 4097800039137N418S00197, id in title: 1379418S001 (N instead of 9) # So we seek for a 1 letter difference and replace if found .... (so sad) for i in range(len(info['id']) - len(acc_id) + 1): - sub_part = info['id'][i:i+len(acc_id)] + sub_part = info['id'][i:i + len(acc_id)] z = zip(sub_part, acc_id) if len([tuple_letter for tuple_letter in z if len(set(tuple_letter)) > 1]) == 1: info['link'] = info['link'].replace(sub_part, acc_id) @@ -280,7 +287,7 @@ class IndexPage(LoggedPage, HTMLPage): def is_account_inactive(self, account_id): return self.doc.xpath('//tr[td[contains(text(), $id)]][@class="Inactive"]', id=account_id) - def _add_account(self, accounts, link, label, account_type, balance, number=None): + def _add_account(self, accounts, link, label, account_type, balance, number=None, ownership=NotAvailable): info = self._get_account_info(link, accounts) if info is None: self.logger.warning('Unable to parse account %r: %r' % (label, link)) @@ -294,11 +301,14 @@ class IndexPage(LoggedPage, HTMLPage): account._info = info account.number = number account.label = label + account.ownership = ownership account.type = self.ACCOUNT_TYPES.get(label, info['acc_type'] if 'acc_type' in info else account_type) if 'PERP' in account.label: account.type = Account.TYPE_PERP if 'NUANCES CAPITALISATI' in account.label: account.type = Account.TYPE_CAPITALISATION + if account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_PERP): + account.ownership = AccountOwnership.OWNER balance = balance or self.get_balance(account) @@ -324,13 +334,13 @@ class IndexPage(LoggedPage, HTMLPage): balance = page.doc.xpath('.//tr[td[contains(@id,"NumContrat")]]/td[@class="somme"]/a[contains(@href, $id)]', id=account.id) if len(balance) > 0: balance = CleanText('.')(balance[0]) - balance = balance if balance != u'' else NotAvailable + balance = balance if balance != '' else NotAvailable else: # Specific xpath for some Life Insurances: balance = page.doc.xpath('//tr[td[contains(text(), $id)]]/td/div[contains(@id, "Solde")]', id=account.id) if len(balance) > 0: balance = CleanText('.')(balance[0]) - balance = balance if balance != u'' else NotAvailable + balance = balance if balance != '' else NotAvailable else: # sometimes the accounts are attached but no info is available balance = NotAvailable @@ -351,7 +361,7 @@ class IndexPage(LoggedPage, HTMLPage): accounts_id.append(re.search("(\d{6,})", Attr('.', 'href')(a)).group(1)) return accounts_id - def get_list(self): + def get_list(self, owner_name): accounts = OrderedDict() # Old website @@ -361,9 +371,11 @@ class IndexPage(LoggedPage, HTMLPage): for tr in table.xpath('./tr'): tds = tr.findall('td') if tr.attrib.get('class', '') == 'DataGridHeader': - account_type = self.ACCOUNT_TYPES.get(tds[1].text.strip()) or\ - self.ACCOUNT_TYPES.get(CleanText('.')(tds[2])) or\ - self.ACCOUNT_TYPES.get(CleanText('.')(tds[3]), Account.TYPE_UNKNOWN) + account_type = ( + self.ACCOUNT_TYPES.get(tds[1].text.strip()) + or self.ACCOUNT_TYPES.get(CleanText('.')(tds[2])) + or self.ACCOUNT_TYPES.get(CleanText('.')(tds[3]), Account.TYPE_UNKNOWN) + ) else: # On the same row, there could have many accounts (check account and a card one). # For the card line, the number will be the same than the checking account, so we skip it. @@ -384,7 +396,7 @@ class IndexPage(LoggedPage, HTMLPage): balance = CleanText('.')(tds[-1].xpath('./a')[i]) self._add_account(accounts, a, label, account_type, balance) - self.logger.debug('we are on the %s website', 'old' if accounts else 'new') + self.logger.warning('we are on the %s website', 'old' if accounts else 'new') if len(accounts) == 0: # New website @@ -408,12 +420,24 @@ class IndexPage(LoggedPage, HTMLPage): # (perhaps only on creditcooperatif) label = CleanText('./strong')(tds[0]) balance = CleanText('.')(tds[-1]) + ownership = self.get_ownership(tds, owner_name) - account = self._add_account(accounts, a, label, account_type, balance) + account = self._add_account(accounts, a, label, account_type, balance, ownership=ownership) if account: account.number = CleanText('.')(tds[1]) - return accounts.values() + return list(accounts.values()) + + def get_ownership(self, tds, owner_name): + if len(tds) > 2: + account_owner = CleanText('.', default=None)(tds[2]).upper() + if account_owner and any(title in account_owner for title in ('M', 'MR', 'MLLE', 'MLE', 'MME')): + if re.search(r'(m|mr|me|mme|mlle|mle|ml)\.? ?(.*)\bou (m|mr|me|mme|mlle|mle|ml)\b(.*)', account_owner, re.IGNORECASE): + return AccountOwnership.CO_OWNER + elif all(n in account_owner for n in owner_name.split()): + return AccountOwnership.OWNER + return AccountOwnership.ATTORNEY + return NotAvailable def is_access_error(self): error_message = u"Vous n'êtes pas autorisé à accéder à cette fonction" @@ -463,12 +487,12 @@ class IndexPage(LoggedPage, HTMLPage): account_type = self.ACCOUNT_TYPES.get(CleanText('.')(title), Account.TYPE_UNKNOWN) for tr in table.xpath('./table/tbody/tr[contains(@id,"MM_SYNTHESE_CREDITS") and contains(@id,"IdTrGlobal")]'): tds = tr.findall('td') - if len(tds) == 0 : + if len(tds) == 0: continue for i in tds[0].xpath('.//a/strong'): label = i.text.strip() break - if len(tds) == 3 and Decimal(FrenchTransaction.clean_amount(CleanText('.')(tds[-2]))) and any(cls in Attr('.', 'id')(tr) for cls in ['dgImmo', 'dgConso']) == False: + if len(tds) == 3 and Decimal(FrenchTransaction.clean_amount(CleanText('.')(tds[-2]))) and any(cls in Attr('.', 'id')(tr) for cls in ['dgImmo', 'dgConso']) is False: # in case of Consumer credit or revolving credit, we substract avalaible amount with max amout # to get what was spend balance = Decimal(FrenchTransaction.clean_amount(CleanText('.')(tds[-2]))) - Decimal(FrenchTransaction.clean_amount(CleanText('.')(tds[-1]))) @@ -481,6 +505,9 @@ class IndexPage(LoggedPage, HTMLPage): account.balance = -abs(balance) account.currency = account.get_currency(CleanText('.')(tds[-1])) account._card_links = [] + # The website doesn't show any information relative to the loan + # owner, we can then assume they all belong to the credentials owner. + account.ownership = AccountOwnership.OWNER if "renouvelables" in CleanText('.')(title): if 'JSESSIONID' in self.browser.session.cookies: @@ -493,7 +520,7 @@ class IndexPage(LoggedPage, HTMLPage): account.available_amount = float_to_decimal(d['situationCredit']['disponible']) account.next_payment_amount = float_to_decimal(d['situationCredit']['mensualiteEnCours']) accounts[account.id] = account - return accounts.values() + return list(accounts.values()) @method class get_real_estate_loans(ListElement): @@ -505,12 +532,12 @@ class IndexPage(LoggedPage, HTMLPage): item_xpath = './table[@class="static"][1]/tbody' head_xpath = './table[@class="static"][1]/tbody/tr/th' - col_total_amount = u'Capital Emprunté' - col_rate = u'Taux d’intérêt nominal' - col_balance = u'Capital Restant Dû' - col_last_payment_date = u'Dernière échéance' - col_next_payment_amount = u'Montant prochaine échéance' - col_next_payment_date = u'Prochaine échéance' + col_total_amount = 'Capital Emprunté' + col_rate = 'Taux d’intérêt nominal' + col_balance = 'Capital Restant Dû' + col_last_payment_date = 'Dernière échéance' + col_next_payment_amount = 'Montant prochaine échéance' + col_next_payment_date = 'Prochaine échéance' def parse(self, el): self.env['id'] = CleanText("./h2")(el).split()[-1] @@ -530,6 +557,9 @@ class IndexPage(LoggedPage, HTMLPage): obj_next_payment_amount = MyDecimal(MyTableCell("next_payment_amount")) obj_next_payment_date = Date(CleanText(MyTableCell("next_payment_date", default=''), default=NotAvailable), default=NotAvailable) obj_rate = MyDecimal(MyTableCell("rate", default=NotAvailable), default=NotAvailable) + # The website doesn't show any information relative to the loan + # owner, we can then assume they all belong to the credentials owner. + obj_ownership = AccountOwnership.OWNER def submit_form(self, form, eventargument, eventtarget, scriptmanager): form['__EVENTARGUMENT'] = eventargument @@ -726,7 +756,7 @@ class IndexPage(LoggedPage, HTMLPage): i = min(len(tds) - 4, 1) if tr.attrib.get('class', '') == 'DataGridHeader': - if tds[2].text == u'Titulaire': + if tds[2].text == 'Titulaire': ignore = True else: ignore = False @@ -742,14 +772,14 @@ class IndexPage(LoggedPage, HTMLPage): t = Transaction() - date = u''.join([txt.strip() for txt in tds[i+0].itertext()]) - raw = u' '.join([txt.strip() for txt in tds[i+1].itertext()]) - debit = u''.join([txt.strip() for txt in tds[-2].itertext()]) - credit = u''.join([txt.strip() for txt in tds[-1].itertext()]) + date = ''.join([txt.strip() for txt in tds[i + 0].itertext()]) + raw = ' '.join([txt.strip() for txt in tds[i + 1].itertext()]) + debit = ''.join([txt.strip() for txt in tds[-2].itertext()]) + credit = ''.join([txt.strip() for txt in tds[-1].itertext()]) t.parse(date, re.sub(r'[ ]+', ' ', raw)) - card_debit_date = self.doc.xpath(u'//span[@id="MM_HISTORIQUE_CB_m_TableTitle3_lblTitle"] | //label[contains(text(), "débiter le")]') + card_debit_date = self.doc.xpath('//span[@id="MM_HISTORIQUE_CB_m_TableTitle3_lblTitle"] | //label[contains(text(), "débiter le")]') if card_debit_date: t.rdate = t.bdate = Date(dayfirst=True).filter(date) m = re.search(r'\b(\d{2}/\d{2}/\d{4})\b', card_debit_date[0].text) @@ -819,7 +849,7 @@ class IndexPage(LoggedPage, HTMLPage): form.submit() def transfer_link(self): - return self.doc.xpath(u'//a[span[contains(text(), "Effectuer un virement")]] | //a[contains(text(), "Réaliser un virement")]') + return self.doc.xpath('//a[span[contains(text(), "Effectuer un virement")]] | //a[contains(text(), "Réaliser un virement")]') def go_transfer_via_history(self, account): self.go_history(account._info) @@ -844,7 +874,7 @@ class IndexPage(LoggedPage, HTMLPage): form.submit() def transfer_unavailable(self): - return CleanText(u'//li[contains(text(), "Pour accéder à cette fonctionnalité, vous devez disposer d’un moyen d’authentification renforcée")]')(self.doc) + return CleanText('//li[contains(text(), "Pour accéder à cette fonctionnalité, vous devez disposer d’un moyen d’authentification renforcée")]')(self.doc) def loan_unavailable_msg(self): msg = CleanText('//span[@id="MM_LblMessagePopinError"] | //p[@id="MM_ERREUR_PAGE_BLANCHE_pAlert"]')(self.doc) @@ -875,8 +905,8 @@ class IndexPage(LoggedPage, HTMLPage): def levies_page_enabled(self): """ Levies page does not exist in the nav bar for every connections """ return ( - CleanText('//a/span[contains(text(), "Suivre mes prélèvements reçus")]')(self.doc) or # new website - CleanText('//a[contains(text(), "Suivre les prélèvements reçus")]')(self.doc) # old website + CleanText('//a/span[contains(text(), "Suivre mes prélèvements reçus")]')(self.doc) # new website + or CleanText('//a[contains(text(), "Suivre les prélèvements reçus")]')(self.doc) # old website ) @@ -1021,8 +1051,8 @@ class CardsComingPage(IndexPage): # We must handle two kinds of Regexp because the 'X' are not # located at the same level for sub-modules such as palatine return Coalesce( - Regexp(CleanText(Field('label'), replace=[('*', 'X')]), r'(\d{6}\X{6}\d{4})', default=NotAvailable), - Regexp(CleanText(Field('label'), replace=[('*', 'X')]), r'(\d{4}\X{6}\d{6})', default=NotAvailable), + Regexp(CleanText(Field('label'), replace=[('*', 'X')]), r'(\d{6}X{6}\d{4})', default=NotAvailable), + Regexp(CleanText(Field('label'), replace=[('*', 'X')]), r'(\d{4}X{6}\d{6})', default=NotAvailable), )(self) def obj_number(self): @@ -1181,7 +1211,7 @@ class MarketPage(LoggedPage, HTMLPage): form.submit() def iter_investment(self): - for tbody in self.doc.xpath(u'//table[@summary="Contenu du portefeuille valorisé"]/tbody'): + for tbody in self.doc.xpath('//table[@summary="Contenu du portefeuille valorisé"]/tbody'): inv = Investment() inv.label = CleanText('.')(tbody.xpath('./tr[1]/td[1]/a/span')[0]) inv.code = CleanText('.')(tbody.xpath('./tr[1]/td[1]/a')[0]).split(' - ')[1] @@ -1195,7 +1225,7 @@ class MarketPage(LoggedPage, HTMLPage): yield inv def get_valuation_diff(self, account): - val = CleanText(self.doc.xpath(u'//td[contains(text(), "values latentes")]/following-sibling::*[1]')) + val = CleanText(self.doc.xpath('//td[contains(text(), "values latentes")]/following-sibling::*[1]')) account.valuation_diff = CleanDecimal(Regexp(val, '([^\(\)]+)'), replace_dots=True)(self) def is_on_right_portfolio(self, account): @@ -1205,7 +1235,7 @@ class MarketPage(LoggedPage, HTMLPage): return self.doc.xpath('//option[contains(text(), $id)]/@value', id=account._info['id'])[0] def come_back(self): - link = Link(u'//div/a[contains(text(), "Accueil accès client")]', default=NotAvailable)(self.doc) + link = Link('//div/a[contains(text(), "Accueil accès client")]', default=NotAvailable)(self.doc) if link: self.browser.location(link) @@ -1215,9 +1245,14 @@ class LifeInsurance(MarketPage): class LifeInsuranceHistory(LoggedPage, JsonPage): + def build_doc(self, text): + # If history is empty, there is no text + if not text: + return {} + return super(LifeInsuranceHistory, self).build_doc(text) + @method class iter_history(DictElement): - def find_elements(self): return self.el or [] # JSON contains 'null' if no transaction @@ -1234,7 +1269,7 @@ class LifeInsuranceHistory(LoggedPage, JsonPage): def obj_date(self): date = Dict('dateTraitement')(self) if date: - return datetime.fromtimestamp(date/1000) + return datetime.fromtimestamp(date / 1000) return NotAvailable obj_rdate = obj_date @@ -1242,7 +1277,7 @@ class LifeInsuranceHistory(LoggedPage, JsonPage): def obj_vdate(self): vdate = Dict('dateEffet')(self) if vdate: - return datetime.fromtimestamp(vdate/1000) + return datetime.fromtimestamp(vdate / 1000) return NotAvailable @@ -1270,7 +1305,7 @@ class LifeInsuranceInvestments(LoggedPage, JsonPage): def obj_vdate(self): vdate = Dict('cotation/date')(self) if vdate: - return datetime.fromtimestamp(vdate/1000) + return datetime.fromtimestamp(vdate / 1000) return NotAvailable def obj_quantity(self): @@ -1356,7 +1391,7 @@ class MyRecipient(ItemElement): klass = Recipient # Assume all recipients currency is euros. - obj_currency = u'EUR' + obj_currency = 'EUR' def obj_enabled_at(self): return datetime.now().replace(microsecond=0) @@ -1364,9 +1399,10 @@ class MyRecipient(ItemElement): class TransferErrorPage(object): def on_load(self): - errors_xpaths = ['//div[h2[text()="Information"]]/p[contains(text(), "Il ne pourra pas être crédité avant")]', - '//span[@id="MM_LblMessagePopinError"]/p | //div[h2[contains(text(), "Erreur de saisie")]]/p[1] | //span[@class="error"]/strong', - '//div[@id="MM_m_CH_ValidationSummary" and @class="MessageErreur"]', + errors_xpaths = [ + '//div[h2[text()="Information"]]/p[contains(text(), "Il ne pourra pas être crédité avant")]', + '//span[@id="MM_LblMessagePopinError"]/p | //div[h2[contains(text(), "Erreur de saisie")]]/p[1] | //span[@class="error"]/strong', + '//div[@id="MM_m_CH_ValidationSummary" and @class="MessageErreur"]', ] for error_xpath in errors_xpaths: @@ -1399,10 +1435,10 @@ class MyRecipients(ListElement): # Autres comptes if value == 'AC': raise SkipItem() - self.env['category'] = u'Interne' if value[0] == 'I' else u'Externe' - if self.env['category'] == u'Interne': + self.env['category'] = 'Interne' if value[0] == 'I' else 'Externe' + if self.env['category'] == 'Interne': # TODO use after 'I'? - _id = Regexp(CleanText('.'), r'- (\w+\d\w+)')(self) # at least one digit + _id = Regexp(CleanText('.'), r'- (\w+\d\w+)')(self) # at least one digit accounts = list(self.page.browser.get_accounts_list()) + list(self.page.browser.get_loans_list()) # If it's an internal account, we should always find only one account with _id in it's id. # Type card account contains their parent account id, and should not be listed in recipient account. @@ -1443,7 +1479,7 @@ class TransferPage(TransferErrorPage, IndexPage): RECIPIENT_XPATH = '//select[@id="MM_VIREMENT_SAISIE_VIREMENT_ddlCompteCrediter"]/option' def is_here(self): - return bool(CleanText(u'//h2[contains(text(), "Effectuer un virement")]')(self.doc)) + return bool(CleanText('//h2[contains(text(), "Effectuer un virement")]')(self.doc)) def can_transfer(self, account): for o in self.doc.xpath('//select[@id="MM_VIREMENT_SAISIE_VIREMENT_ddlCompteDebiter"]/option'): @@ -1457,10 +1493,10 @@ class TransferPage(TransferErrorPage, IndexPage): return origin_value[0] def get_recipient_value(self, recipient): - if recipient.category == u'Externe': + if recipient.category == 'Externe': recipient_value = [Attr('.', 'value')(o) for o in self.doc.xpath(self.RECIPIENT_XPATH) if Regexp(CleanText('.'), '.* - ([A-Za-z0-9]*) -', default=NotAvailable)(o) == recipient.iban] - elif recipient.category == u'Interne': + elif recipient.category == 'Interne': recipient_value = [Attr('.', 'value')(o) for o in self.doc.xpath(self.RECIPIENT_XPATH) if Regexp(CleanText('.'), '- (\d+)', default=NotAvailable)(o) and Regexp(CleanText('.'), '- (\d+)', default=NotAvailable)(o) in recipient.id] assert len(recipient_value) == 1, 'error during recipient matching' @@ -1510,7 +1546,7 @@ class TransferPage(TransferErrorPage, IndexPage): def go_add_recipient(self): form = self.get_form(id='main') - link = self.doc.xpath(u'//a[span[contains(text(), "Ajouter un compte bénéficiaire")]]')[0] + link = self.doc.xpath('//a[span[contains(text(), "Ajouter un compte bénéficiaire")]]')[0] m = re.search("PostBackOptions?\([\"']([^\"']+)[\"'],\s*['\"]([^\"']+)?['\"]", link.attrib.get('href', '')) form['__EVENTTARGET'] = m.group(1) form['__EVENTARGUMENT'] = m.group(2) @@ -1531,7 +1567,7 @@ class TransferConfirmPage(TransferErrorPage, IndexPage): return super(TransferErrorPage, self).build_doc(content) def is_here(self): - return bool(CleanText(u'//h2[contains(text(), "Confirmer mon virement")]')(self.doc)) + return bool(CleanText('//h2[contains(text(), "Confirmer mon virement")]')(self.doc)) def confirm(self): form = self.get_form(id='main') @@ -1543,9 +1579,9 @@ class TransferConfirmPage(TransferErrorPage, IndexPage): # transfer informations transfer.label = ( - CleanText(u'.//tr[td[contains(text(), "Motif de l\'opération")]]/td[not(@class)]')(self.doc) or - CleanText(u'.//tr[td[contains(text(), "Libellé")]]/td[not(@class)]')(self.doc) or - CleanText(u'.//tr[th[contains(text(), "Libellé")]]/td[not(@class)]')(self.doc) + CleanText('.//tr[td[contains(text(), "Motif de l\'opération")]]/td[not(@class)]')(self.doc) + or CleanText('.//tr[td[contains(text(), "Libellé")]]/td[not(@class)]')(self.doc) + or CleanText('.//tr[th[contains(text(), "Libellé")]]/td[not(@class)]')(self.doc) ) transfer.exec_date = Date(CleanText('.//tr[th[contains(text(), "En date du")]]/td[not(@class)]'), dayfirst=True)(self.doc) transfer.amount = CleanDecimal('.//tr[td[contains(text(), "Montant")]]/td[not(@class)] | \ @@ -1558,8 +1594,8 @@ class TransferConfirmPage(TransferErrorPage, IndexPage): transfer.recipient_label = recipient.label transfer.recipient_id = recipient.id - if recipient.category == u'Externe': - for word in Upper(CleanText(u'.//tr[th[contains(text(), "Compte à créditer")]]/td[not(@class)]'))(self.doc).split(): + if recipient.category == 'Externe': + for word in Upper(CleanText('.//tr[th[contains(text(), "Compte à créditer")]]/td[not(@class)]'))(self.doc).split(): if is_iban_valid(word): transfer.recipient_iban = word break @@ -1580,7 +1616,7 @@ class TransferConfirmPage(TransferErrorPage, IndexPage): class ProTransferConfirmPage(TransferConfirmPage): def is_here(self): - return bool(CleanText(u'//span[@id="MM_m_CH_lblTitle" and contains(text(), "Confirmez votre virement")]')(self.doc)) + return bool(CleanText('//span[@id="MM_m_CH_lblTitle" and contains(text(), "Confirmez votre virement")]')(self.doc)) def continue_transfer(self, origin_label, recipient, label): # Pro internal transfer initiation doesn't need a second step. @@ -1593,7 +1629,7 @@ class ProTransferConfirmPage(TransferConfirmPage): t.amount = CleanDecimal('//span[@id="MM_VIREMENT_CONF_VIREMENT_MontantVir"] | \ //span[@id="MM_VIREMENT_CONF_VIREMENT_lblMontantSelect"]', replace_dots=True)(self.doc) t.account_iban = account.iban - if recipient.category == u'Externe': + if recipient.category == 'Externe': for word in Upper(CleanText('//span[@id="MM_VIREMENT_CONF_VIREMENT_lblCptCrediterResult"]'))(self.doc).split(): if is_iban_valid(word): t.recipient_iban = word @@ -1618,10 +1654,10 @@ class ProTransferConfirmPage(TransferConfirmPage): class TransferSummaryPage(TransferErrorPage, IndexPage): def is_here(self): - return bool(CleanText(u'//h2[contains(text(), "Accusé de réception")]')(self.doc)) + return bool(CleanText('//h2[contains(text(), "Accusé de réception")]')(self.doc)) def populate_reference(self, transfer): - transfer.id = Regexp(CleanText(u'//p[contains(text(), "a bien été enregistré")]'), '(\d+)')(self.doc) + transfer.id = Regexp(CleanText('//p[contains(text(), "a bien été enregistré")]'), '(\d+)')(self.doc) return transfer @@ -1638,7 +1674,7 @@ class ProTransferPage(TransferPage): RECIPIENT_XPATH = '//select[@id="MM_VIREMENT_SAISIE_VIREMENT_ddlCompteCrediterPro"]/option' def is_here(self): - return CleanText(u'//span[contains(text(), "Créer une liste de virements")] | //span[contains(text(), "Réalisez un virement")]')(self.doc) + return CleanText('//span[contains(text(), "Créer une liste de virements")] | //span[contains(text(), "Réalisez un virement")]')(self.doc) @method class iter_recipients(MyRecipients): @@ -1684,7 +1720,7 @@ class SmsRequest(LoggedPage, JsonPage): return self.doc['validationUnits'][0] def get_saml(self, otp_exception): - if not 'response' in self.doc: + if 'response' not in self.doc: error = self.doc['phase']['previousResult'] if error == 'FAILED_AUTHENTICATION': @@ -1704,7 +1740,7 @@ class SmsPage(LoggedPage, HTMLPage): raise AddRecipientBankError(message='Wrongcode, ' + error) def get_prompt_text(self): - return CleanText(u'//td[@class="auth_info_prompt"]')(self.doc) + return CleanText('//td[@class="auth_info_prompt"]')(self.doc) def post_form(self): form = self.get_form(name='downloadAuthForm') @@ -1724,7 +1760,7 @@ class SmsPage(LoggedPage, HTMLPage): class AuthentPage(LoggedPage, HTMLPage): def is_here(self): - return bool(CleanText(u'//h2[contains(text(), "Authentification réussie")]')(self.doc)) + return bool(CleanText('//h2[contains(text(), "Authentification réussie")]')(self.doc)) def go_on(self): form = self.get_form(id='main') @@ -1742,7 +1778,7 @@ class RecipientPage(LoggedPage, HTMLPage): raise AddRecipientBankError(message=error) def is_here(self): - return bool(CleanText(u'//h2[contains(text(), "Ajouter un compte bénéficiaire")] |\ + return bool(CleanText('//h2[contains(text(), "Ajouter un compte bénéficiaire")] |\ //h2[contains(text(), "Confirmer l\'ajout d\'un compte bénéficiaire")]')(self.doc)) def post_recipient(self, recipient): @@ -1750,7 +1786,7 @@ class RecipientPage(LoggedPage, HTMLPage): form['__EVENTTARGET'] = '%s$m_WizardBar$m_lnkNext$m_lnkButton' % self.EVENTTARGET form['%s$m_RibIban$txtTitulaireCompte' % self.FORM_FIELD_ADD] = recipient.label for i in range(len(recipient.iban) // 4 + 1): - form['%s$m_RibIban$txtIban%s' % (self.FORM_FIELD_ADD, str(i + 1))] = recipient.iban[4*i:4*i+4] + form['%s$m_RibIban$txtIban%s' % (self.FORM_FIELD_ADD, str(i + 1))] = recipient.iban[4 * i:4 * i + 4] form.submit() def confirm_recipient(self): @@ -1775,7 +1811,7 @@ class ProAddRecipientOtpPage(IndexPage): self.browser.recipient_form['url'] = form.url def get_prompt_text(self): - return CleanText(u'////span[@id="MM_ANR_WS_AUTHENT_ANR_WS_AUTHENT_SAISIE_lblProcedure1"]')(self.doc) + return CleanText('////span[@id="MM_ANR_WS_AUTHENT_ANR_WS_AUTHENT_SAISIE_lblProcedure1"]')(self.doc) class ProAddRecipientPage(RecipientPage): @@ -1790,7 +1826,7 @@ class ProAddRecipientPage(RecipientPage): class TransactionsDetailsPage(LoggedPage, HTMLPage): def is_here(self): - return bool(CleanText(u'//h2[contains(text(), "Débits différés imputés")] | //span[@id="MM_m_CH_lblTitle" and contains(text(), "Débit différé imputé")]')(self.doc)) + return bool(CleanText('//h2[contains(text(), "Débits différés imputés")] | //span[@id="MM_m_CH_lblTitle" and contains(text(), "Débit différé imputé")]')(self.doc)) @pagination @method @@ -1798,10 +1834,10 @@ class TransactionsDetailsPage(LoggedPage, HTMLPage): item_xpath = '//table[@id="MM_ECRITURE_GLOBALE_m_ExDGEcriture"]/tr[not(@class)] | //table[has-class("small special")]//tbody/tr[@class="rowClick"]' head_xpath = '//table[@id="MM_ECRITURE_GLOBALE_m_ExDGEcriture"]/tr[@class="DataGridHeader"]/td | //table[has-class("small special")]//thead/tr/th' - col_date = u'Date' - col_label = [u'Opération', u'Libellé'] - col_debit = u'Débit' - col_credit = u'Crédit' + col_date = 'Date' + col_label = ['Opération', 'Libellé'] + col_debit = 'Débit' + col_credit = 'Crédit' def next_page(self): # only for new website, don't have any accounts with enough deferred card transactions on old webiste @@ -1827,7 +1863,7 @@ class TransactionsDetailsPage(LoggedPage, HTMLPage): def go_form_to_summary(self): # return to first page - to_history = Link(self.doc.xpath(u'//a[contains(text(), "Retour à l\'historique")]'))(self.doc) + to_history = Link(self.doc.xpath('//a[contains(text(), "Retour à l\'historique")]'))(self.doc) n = re.match('.*\([\'\"](MM\$.*?)[\'\"],.*\)$', to_history) form = self.get_form(id='main') form['__EVENTTARGET'] = n.group(1) @@ -1865,6 +1901,7 @@ class SubscriptionPage(LoggedPage, HTMLPage): class iter_documents(ListElement): # sometimes there is several documents with same label at same date and with same content ignore_duplicate = True + @property def item_xpath(self): if Env('has_subscription')(self): @@ -1876,7 +1913,7 @@ class SubscriptionPage(LoggedPage, HTMLPage): obj_format = 'pdf' obj_url = Regexp(Link('.//td[@class="telecharger"]//a'), r'WebForm_PostBackOptions\("(\S*)"') - obj_id = Format('%s_%s_%s', Env('sub_id'), CleanText('./td[2]', symbols='/', replace=[(' ', '_')]), Regexp(CleanText('./td[3]'), r'([\wé]*)')) + obj_id = Format('%s_%s_%s', Env('sub_id'), CleanText('./td[2]', symbols='/', replace=[(' ', '_')]), Regexp(CleanText('./td[3]'), r'([\wé]*)')) obj_label = Format('%s %s', CleanText('./td[3]'), CleanText('./td[2]')) obj_date = Date(CleanText('./td[2]'), dayfirst=True) diff --git a/modules/cmes/browser.py b/modules/cmes/browser.py index f75fba03b7a50992186279ec96ccea8cc419bdfe..fa15335701840177d60e72caeebc06f1812890cb 100644 --- a/modules/cmes/browser.py +++ b/modules/cmes/browser.py @@ -85,8 +85,12 @@ class CmesBrowser(LoginBrowser): self.accounts.stay_or_go(subsite=self.subsite, client_space=self.client_space) for inv in self.page.iter_investments(account=account): if inv._url: - # Go to the investment details to get performances + # Go to the investment details to get employee savings attributes self.location(inv._url) + + # Fetch SRRI, asset category & recommended period + self.page.fill_investment(obj=inv) + performances = {} # Get 1-year performance diff --git a/modules/cmes/pages.py b/modules/cmes/pages.py index c22962d7c0b93a820d4120e4e2872bd2e0431d4f..c13c3b0b02a0e298801a3226f1828800782eb30b 100644 --- a/modules/cmes/pages.py +++ b/modules/cmes/pages.py @@ -20,11 +20,12 @@ from __future__ import unicode_literals import re + from weboob.browser.pages import HTMLPage, LoggedPage from weboob.browser.elements import ListElement, ItemElement, method from weboob.browser.filters.standard import ( CleanText, CleanDecimal, Date, Regexp, Field, Currency, - Upper, MapIn, Eval, + Upper, MapIn, Eval, Title, ) from weboob.browser.filters.html import Link from weboob.capabilities.bank import Account, Investment, Pocket, NotAvailable @@ -33,11 +34,12 @@ from weboob.exceptions import ActionNeeded class Transaction(FrenchTransaction): - PATTERNS = [(re.compile(u'^(?P.*[Vv]ersement.*)'), FrenchTransaction.TYPE_DEPOSIT), - (re.compile(u'^(?P([Aa]rbitrage|[Pp]rélèvements.*))'), FrenchTransaction.TYPE_ORDER), - (re.compile(u'^(?P([Rr]etrait|[Pp]aiement.*))'), FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile(u'^(?P.*)'), FrenchTransaction.TYPE_BANK), - ] + PATTERNS = [ + (re.compile(r'^(?P.*[Vv]ersement.*)'), FrenchTransaction.TYPE_DEPOSIT), + (re.compile(r'^(?P([Aa]rbitrage|[Pp]rélèvements.*))'), FrenchTransaction.TYPE_ORDER), + (re.compile(r'^(?P([Rr]etrait|[Pp]aiement.*))'), FrenchTransaction.TYPE_WITHDRAWAL), + (re.compile(r'^(?P.*)'), FrenchTransaction.TYPE_BANK), + ] def MyDecimal(*args, **kwargs): @@ -47,7 +49,7 @@ def MyDecimal(*args, **kwargs): class LoginPage(HTMLPage): def login(self, login, password): - form = self.get_form(name="bloc_ident") + form = self.get_form(name='bloc_ident') form['_cm_user'] = login form['_cm_pwd'] = password form.submit() @@ -100,10 +102,14 @@ class AccountsPage(LoggedPage, HTMLPage): def obj_id(self): # Use customer number + label to build account id - number = Regexp(CleanText('//div[@id="ei_tpl_fullSite"]//div[contains(@class, "ei_tpl_profil_content")]/p'), - r'(\d+)$', '\\1')(self) + number = Regexp( + CleanText('//div[@id="ei_tpl_fullSite"]//div[contains(@class, "ei_tpl_profil_content")]/p'), + r'(\d+)$', '\\1' + )(self) return Field('label')(self) + number + obj_number = obj_id + def iter_invest_rows(self, account): """ Process each invest row, extract elements needed to get @@ -161,7 +167,7 @@ class AccountsPage(LoggedPage, HTMLPage): pocket.condition = Pocket.CONDITION_AVAILABLE else: pocket.condition = Pocket.CONDITION_DATE - pocket.availability_date = Date(Regexp(Upper(CleanText('./td[1]')), 'AU[\s]+(.*)'), dayfirst=True)(row) + pocket.availability_date = Date(Regexp(Upper(CleanText('./td[1]')), r'AU[\s]+(.*)'), dayfirst=True)(row) yield pocket @@ -183,6 +189,24 @@ class AccountsPage(LoggedPage, HTMLPage): class InvestmentPage(LoggedPage, HTMLPage): + @method + class fill_investment(ItemElement): + # Sometimes there is a 'LIBELLES EN EURO' string joined with the category so we remove it + obj_asset_category = Title(CleanText('//tr[th[text()="Classification AMF"]]/td', replace=[('LIBELLES EN EURO', '')])) + + def obj_srri(self): + # Extract the value from '1/7' or '6/7' for instance + srri = Regexp(CleanText('//tr[th[text()="Niveau de risque"]]/td'), r'(\d+)/7', default=None)(self) + if srri: + return int(srri) + return NotAvailable + + def obj_recommended_period(self): + period = CleanText('//tr[th[text()="Durée de placement recommandée"]]/td')(self) + if period != 'NC': + return period + return NotAvailable + def get_form_url(self): form = self.get_form(id='C:P:F') return form.url diff --git a/modules/cmso/par/browser.py b/modules/cmso/par/browser.py index 94bcae26c602906a69731daa6478d07a9746c1b6..a8e64f0ec4a7add357225bcbdb8e4856f3b12222 100644 --- a/modules/cmso/par/browser.py +++ b/modules/cmso/par/browser.py @@ -260,9 +260,13 @@ class CmsoParBrowser(LoginBrowser, StatesMixin): self.history.go(data=json.dumps({'index': account._index}), page="detailcompte", headers=self.json_headers) - self.trs = {'lastdate': None, 'list': []} + self.trs = set() for tr in self.page.iter_history(index=account._index, nbs=nbs): + # Check for duplicates + if tr.id in self.trs: + continue + self.trs.add(tr.id) if has_deferred_cards and tr.type == Transaction.TYPE_CARD: tr.type = Transaction.TYPE_DEFERRED_CARD tr.bdate = tr.rdate @@ -283,8 +287,8 @@ class CmsoParBrowser(LoginBrowser, StatesMixin): self.history.go(data=json.dumps({"index": account._index}), page="pendingListOperations", headers=self.json_headers) + # There is no ids for comings, so no check for duplicates for key in self.page.get_keys(): - self.trs = {'lastdate': None, 'list': []} for c in self.page.iter_history(key=key): if hasattr(c, '_deferred_date'): c.bdate = c.rdate diff --git a/modules/cmso/par/pages.py b/modules/cmso/par/pages.py index e0c7ff0ef63258e0d6cfc1d94c22c5afe41da54a..39608ab8bcf2116c410684baf015c48e38576c98 100644 --- a/modules/cmso/par/pages.py +++ b/modules/cmso/par/pages.py @@ -360,6 +360,9 @@ class HistoryPage(LoggedPage, JsonPage): obj_raw = Transaction.Raw(Dict('libelleCourt')) obj_vdate = Date(Dict('dateValeur', NotAvailable), dayfirst=True, default=NotAvailable) obj_amount = CleanDecimal(Dict('montantEnEuro'), default=NotAvailable) + # DO NOT USE `OperationID` the ids aren't constant after 1 month. `clefDomirama` seems + # to be constant forever. Must be kept under watch though + obj_id = Dict('clefDomirama', default='') def parse(self, el): key = Env('key', default=None)(self) @@ -370,15 +373,6 @@ class HistoryPage(LoggedPage, JsonPage): break setattr(self.obj, '_deferred_date', self.FromTimestamp().filter(deferred_date)) - # Skip duplicate transactions - amount = Dict('montantEnEuro', default=None)(self) - tr = Dict('libelleCourt')(self) + Dict('dateOperation', '')(self) + str(amount) - if amount is None or (tr in self.page.browser.trs['list'] and self.page.browser.trs['lastdate'] <= Field('date')(self)): - raise SkipItem() - - self.page.browser.trs['lastdate'] = Field('date')(self) - self.page.browser.trs['list'].append(tr) - class LifeinsurancePage(LoggedPage, HTMLPage): def get_account_id(self): diff --git a/modules/cragr/api/browser.py b/modules/cragr/api/browser.py index b2c330b19d9206d08f4435b1de06998196fcb1eb..22af0db50c933819fbbaea13d4267d2f5b901744 100644 --- a/modules/cragr/api/browser.py +++ b/modules/cragr/api/browser.py @@ -31,6 +31,7 @@ from weboob.browser.exceptions import ServerError, ClientError, BrowserHTTPNotFo from weboob.exceptions import BrowserUnavailable, BrowserIncorrectPassword, ActionNeeded from weboob.tools.capabilities.bank.iban import is_iban_valid from weboob.tools.capabilities.bank.transactions import sorted_transactions +from weboob.tools.decorators import retry from .pages import ( LoginPage, LoggedOutPage, KeypadPage, SecurityPage, ContractsPage, FirstConnectionPage, AccountsPage, AccountDetailsPage, @@ -99,7 +100,6 @@ class CragrAPI(LoginBrowser): TransferPage) old_website = URL(r'https://.*particuliers.html$', OldWebsitePage) - def __init__(self, website, *args, **kwargs): super(CragrAPI, self).__init__(*args, **kwargs) self.website = website @@ -118,6 +118,36 @@ class CragrAPI(LoginBrowser): super(CragrAPI, self).deinit() self.netfinca.deinit() + @retry(BrowserUnavailable) + def do_security_check(self): + try: + form = self.get_security_form() + self.security_check.go(data=form) + except ServerError as exc: + # Wrongpass returns a 500 server error... + exc_json = exc.response.json() + error = exc_json.get('error') + error_type = exc_json.get('erreurType') + if error: + message = error.get('message', '') + wrongpass_messages = ("Votre identification est incorrecte", "Vous n'avez plus droit") + if any(value in message for value in wrongpass_messages): + raise BrowserIncorrectPassword() + if 'obtenir un nouveau code' in message: + raise ActionNeeded(message) + + code = error.get('code', '') + technical_error_messages = ('Un incident technique', 'identifiant et votre code personnel') + # Sometimes there is no error message, so we try to use the code as well + technical_error_codes = ('technical_error',) + if any(value in message for value in technical_error_messages) or \ + any(value in code for value in technical_error_codes): + raise BrowserUnavailable(message) + elif error_type and 'UNAUTHORIZED_ERREUR_TYPE' in error_type: + # Usually appears when doing retries after a BrowserUnavailable + raise BrowserUnavailable() + raise + def do_login(self): if not self.username or not self.password: raise BrowserIncorrectPassword() @@ -133,31 +163,7 @@ class CragrAPI(LoginBrowser): self.logger.warning('This is a regional connection, switching to old website with URL %s', self.BASEURL) raise SiteSwitch('region') - form = self.get_security_form() - try: - self.security_check.go(data=form) - except ServerError as exc: - # Wrongpass returns a 500 server error... - error = exc.response.json().get('error') - if error: - message = error.get('message', '') - wrongpass_messages = ("Votre identification est incorrecte", "Vous n'avez plus droit") - if any(value in message for value in wrongpass_messages): - raise BrowserIncorrectPassword() - if 'obtenir un nouveau code' in message: - raise ActionNeeded(message) - technical_errors = ('Un incident technique', 'identifiant et votre code personnel') - if any(value in message for value in technical_errors): - # If it is a technical error, we try login again - form = self.get_security_form() - try: - self.security_check.go(data=form) - except ServerError as exc: - error = exc.response.json().get('error') - if error: - message = error.get('message', '') - if 'Un incident technique' in message: - raise BrowserUnavailable(message) + self.do_security_check() # accounts_url may contain '/particulier', '/professionnel', '/entreprise', '/agriculteur' or '/association' self.accounts_url = self.page.get_accounts_url() @@ -402,9 +408,10 @@ class CragrAPI(LoginBrowser): self.do_login() try: self.contracts_page.go(space=self.space, id_contract=contract) - except ServerError: - self.logger.warning('Space switch returned a 500 error, try again.') - self.contracts_page.go(space=self.space, id_contract=contract) + except ServerError as e: + if e.response.status_code == 500: + raise BrowserUnavailable() + raise assert self.accounts_page.is_here() @need_login diff --git a/modules/cragr/api/pages.py b/modules/cragr/api/pages.py index a47d5b9a7c1491e359b5f7453e3c44f49aa125f2..57d0d410cbddf26b918efbb787cf5c670dc43465 100644 --- a/modules/cragr/api/pages.py +++ b/modules/cragr/api/pages.py @@ -29,13 +29,15 @@ from weboob.exceptions import ActionNeeded from weboob.capabilities import NotAvailable from weboob.capabilities.base import empty from weboob.capabilities.bank import ( - Account, AccountOwnerType, Transaction, Investment, + Account, AccountOwnerType, Investment, ) +from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.capabilities.profile import Person, Company from weboob.capabilities.contact import Advisor from weboob.browser.elements import DictElement, ItemElement, method from weboob.browser.filters.standard import ( - CleanText, CleanDecimal, Currency as CleanCurrency, Format, Field, Map, Eval, Env, Regexp, Date, Coalesce, + CleanText, CleanDecimal, Currency as CleanCurrency, Format, Field, Map, Eval, Env, + Regexp, Date, Coalesce, DateTime, ) from weboob.browser.filters.html import Attr from weboob.browser.filters.json import Dict @@ -43,10 +45,26 @@ from weboob.tools.capabilities.bank.investments import is_isin_valid from weboob.exceptions import BrowserPasswordExpired + def float_to_decimal(f): return Decimal(str(f)) +class Transaction(FrenchTransaction): + # this is only used to to find the rdate + PATTERNS = [ + (re.compile(r'^(?PPAIEMENT PAR CARTE) (?P.*) (?P
\d{2})/(?P\d{2})$'), None), + (re.compile(r'^(?PPRELEVEMENT) (?P.*) (?P
\d{2})/(?P\d{2})/(?P\d{4}) .*'), None), + (re.compile(r'^(?PPRELEVEMENT) (?P.*) (?P
\d{2})\s(?P\d{2})\s(?P\d{4}) .*'), None), + (re.compile(r'^(?PVIREMENT EN VOTRE FAVEUR) (?P.*) (?P
\d{2})\.(?P\d{2})\.(?P\d{4})$'), None), + (re.compile(r'^(?PREMBOURSEMENT DE PRET) (?P.*) (?P
\d{2})/(?P\d{2})/(?P\d{2,4})$'), None), + (re.compile(r'^(?PRETRAIT AU DISTRIBUTEUR) (?P.*) (?P
\d{2})/(?P\d{2}) .*'), None), + (re.compile(r'^(?PPRELEVEMENT URSSAF) (?P.*) (du)? (?P
\d{2})/(?P\d{2})/(?P\d{2,4})$'), None), + (re.compile(r"^(?PVERSEMENT D'ESPECES) (?P.*) (?P
\d{2})/(?P\d{2})/(?P\d{4}) .*"), None), + (re.compile(r'^(?PPRELEVEMENT) (?P.*) (?P
\d{2})-(?P\d{2})$'), None), + ] + + class KeypadPage(JsonPage): def build_password(self, password): # Fake Virtual Keyboard: just get the positions of each digit. @@ -425,20 +443,29 @@ class HistoryPage(LoggedPage, JsonPage): klass = Transaction + obj_date = DateTime(Dict('dateValeur')) + # There is a key in the json called dateOperation but most of the time it is the + # same as the dateValeur, meanwhile a different rdate is available in many labels. + # So we force all rdate to NotAvailable and only fill it when we find something to + # extract from the label + obj_rdate = NotAvailable # Transactions in foreign currencies have no 'libelleTypeOperation' # and 'libelleComplementaire' keys, hence the default values. # The CleanText() gets rid of additional spaces. - obj_raw = CleanText(Format('%s %s %s', CleanText(Dict('libelleTypeOperation', default='')), CleanText(Dict('libelleOperation')), CleanText(Dict('libelleComplementaire', default='')))) + obj_raw = Transaction.Raw( + CleanText( + Format( + '%s %s %s', + CleanText(Dict('libelleTypeOperation', default='')), + CleanText(Dict('libelleOperation')), + CleanText(Dict('libelleComplementaire', default='')) + ) + ) + ) obj_label = CleanText(Format('%s %s', CleanText(Dict('libelleTypeOperation', default='')), CleanText(Dict('libelleOperation')))) obj_amount = Eval(float_to_decimal, Dict('montant')) obj_type = Map(CleanText(Dict('libelleTypeOperation', default='')), TRANSACTION_TYPES, Transaction.TYPE_UNKNOWN) - def obj_date(self): - return dateutil.parser.parse(Dict('dateValeur')(self)) - - def obj_rdate(self): - return dateutil.parser.parse(Dict('dateOperation')(self)) - class CardsPage(LoggedPage, JsonPage): @method diff --git a/modules/cragr/regions/browser.py b/modules/cragr/regions/browser.py index aec2054c49a45d1731758b60e1f74face58cd3b9..ce4e4ce0242415db05c78fbd2208b033d50c565b 100644 --- a/modules/cragr/regions/browser.py +++ b/modules/cragr/regions/browser.py @@ -324,17 +324,8 @@ class CragrRegion(LoginBrowser): return valid_accounts @need_login - def iter_perimeter_accounts(self, iban, all_accounts): - ''' - In order to use this method, we must pass the 3 accounts URLs: Regular, Wealth and Loans. - Accounts may appear on several URLs: we must check for duplicates before adding to cragr_accounts. - Once we fetched all cragr accounts, we go to the Netfinca space to get Netfinca accounts. - If there are account duplicates, we preferably yield the Netfinca version because it is more - complete ; in addition, Netfinca may contain accounts that do not appear on the cragr website. - ''' - cragr_accounts = [] - - # Regular accounts (Checking and Savings) + def iter_perimeter_regular_accounts(self, iban): + unique_ids = set() self.accounts.stay_or_go() self.page.set_cragr_code() for account in self.page.iter_accounts(): @@ -343,8 +334,23 @@ class CragrRegion(LoginBrowser): # Refresh account form in case it expired refreshed_account = find_object(self.page.iter_accounts(), id=account.id) account.iban = self.get_account_iban(refreshed_account._form) - if account.id not in [a.id for a in cragr_accounts]: - cragr_accounts.append(account) + + if account.id not in unique_ids: + # Do not yield accounts with duplicate IDs + unique_ids.add(account.id) + yield account + + @need_login + def iter_perimeter_accounts(self, iban, all_accounts): + ''' + In order to use this method, we must pass the 3 accounts URLs: Regular, Wealth and Loans. + Accounts may appear on several URLs: we must check for duplicates before adding to cragr_accounts. + Once we fetched all cragr accounts, we go to the Netfinca space to get Netfinca accounts. + If there are account duplicates, we preferably yield the Netfinca version because it is more + complete ; in addition, Netfinca may contain accounts that do not appear on the cragr website. + ''' + # Regular accounts (Checking & Savings) + cragr_accounts = list(self.iter_perimeter_regular_accounts(iban)) # Wealth accounts (PEA, Market, Life Insurances, PERP...) self.wealth.go() @@ -606,8 +612,9 @@ class CragrRegion(LoginBrowser): self.accounts.stay_or_go() self.go_to_perimeter(account._perimeter) - # No need to fetch IBANs and Netfinca accounts just to fetch an account form - refreshed_account = find_object(self.iter_perimeter_accounts(iban=False, all_accounts=False), AccountNotFound, id=account.id) + # Only fetch the perimeter's regular accounts (Checking & Savings) + # No need to go to Wealth, Loans or Netfinca for transactions + refreshed_account = find_object(self.iter_perimeter_regular_accounts(iban=False), AccountNotFound, id=account.id) refreshed_account._form.submit() if self.failed_history.is_here(): self.logger.warning('Form submission failed to reach the account history, we try again.') diff --git a/modules/cragr/regions/pages.py b/modules/cragr/regions/pages.py index 8f49ed8119b28f0a154241dff1ab68d15ccf82f7..5782e17b6ec887e2d2119217a1a1d066cbd10fe2 100644 --- a/modules/cragr/regions/pages.py +++ b/modules/cragr/regions/pages.py @@ -263,6 +263,7 @@ ACCOUNT_TYPES = { 'FLORIANE 2': Account.TYPE_LIFE_INSURANCE, 'CAP DECOUV': Account.TYPE_LIFE_INSURANCE, 'ESPACE LIB': Account.TYPE_LIFE_INSURANCE, + 'ESPACELIB3': Account.TYPE_LIFE_INSURANCE, 'ESP LIB 2': Account.TYPE_LIFE_INSURANCE, 'AST SELEC': Account.TYPE_LIFE_INSURANCE, 'PRGE': Account.TYPE_LIFE_INSURANCE, @@ -276,11 +277,6 @@ ACCOUNT_TYPES = { class AccountsPage(LoggedPage, CragrPage): - def on_load(self): - # Verify that all accounts page have the text 'Synthèse comptes' - if not CleanText('//h1[contains(text(), "Synthèse comptes")]')(self.doc): - self.logger.warning('We found an AccountsPage without the "Synthèse comptes" text.') - def no_other_perimeter(self): return not CleanText('//a[@title="Espace Autres Comptes"]')(self.doc) diff --git a/modules/creditdunord/browser.py b/modules/creditdunord/browser.py index e88e63efe3073de8d7551b688583e663dd1eee67..120a3936c2794f81fce530782d60e8bac5e259bd 100644 --- a/modules/creditdunord/browser.py +++ b/modules/creditdunord/browser.py @@ -69,7 +69,8 @@ class CreditDuNordBrowser(LoginBrowser): not self.page.doc.xpath(u'//b[contains(text(), "vous devez modifier votre code confidentiel")]') def do_login(self): - self.login.go().login(self.username, self.password) + self.login.go() + self.page.login(self.username, self.password) if self.accounts.is_here(): expired_error = self.page.get_password_expired() if expired_error: @@ -134,10 +135,15 @@ class CreditDuNordBrowser(LoginBrowser): self.multitype_iban.go() link = self.page.iban_go() - for a in [a for a in accounts if a._acc_nb]: - if a.type != Account.TYPE_CARD: - self.location(link + a._acc_nb) - a.iban = self.page.get_iban() + if link: + # For some accounts, the IBAN is displayed somewhere else behind + # an OTP validation (icd/zco/public-index.html#zco/transac/impression_rib), + # the link is None if this is the case. + # TODO when we will be able to test this OTP + for a in accounts: + if a._acc_nb and a.type != Account.TYPE_CARD: + self.location(link + a._acc_nb) + a.iban = self.page.get_iban() return accounts diff --git a/modules/creditdunord/pages.py b/modules/creditdunord/pages.py index a8ae602025eb603aa2e960bfc6a8b9864972174d..ecc8dc6d907f36475445ddefccd3de7a7055ca4c 100755 --- a/modules/creditdunord/pages.py +++ b/modules/creditdunord/pages.py @@ -79,6 +79,7 @@ class CDNVirtKeyboard(GridVirtKeyboard): def __init__(self, browser, crypto, grid): f = BytesIO(browser.open('/sec/vk/gen_ui?modeClavier=0&cryptogramme=%s' % crypto).content) + super(CDNVirtKeyboard, self).__init__(range(16), self.ncol, self.nrow, f, self.color) self.check_symbols(self.symbols, browser.responses_dirname) self.codes = grid @@ -95,7 +96,7 @@ class CDNVirtKeyboard(GridVirtKeyboard): for nbchar, c in enumerate(string): index = self.get_symbol_code(self.symbols[c]) res.append(self.codes[(nbchar * ndata) + index]) - return ','.join(res) + return ','.join(map(str, res)) class HTMLErrorPage(HTMLPage): @@ -228,7 +229,10 @@ class CDNBasePage(HTMLPage): return self.get_from_js("name: 'execution', value: '", "'") def iban_go(self): - return '%s%s' % ('/vos-comptes/IPT/cdnProxyResource', self.get_from_js('C_PROXY.StaticResourceClientTranslation( "', '"')) + value_from_js = self.get_from_js('C_PROXY.StaticResourceClientTranslation( "', '"') + if not value_from_js: + return None + return '/vos-comptes/IPT/cdnProxyResource%s' % value_from_js class ProIbanPage(CDNBasePage): diff --git a/modules/creditmutuel/browser.py b/modules/creditmutuel/browser.py index d4c95495674b1c05ea7b6f97b61e43de48be8a1b..69ef6673b51cf22e1ad9de4bc74d8d805d383872 100644 --- a/modules/creditmutuel/browser.py +++ b/modules/creditmutuel/browser.py @@ -300,7 +300,7 @@ class CreditMutuelBrowser(LoginBrowser, StatesMixin): elif acc.type in (Account.TYPE_MORTGAGE, Account.TYPE_LOAN) and acc._parent_id: acc.parent = accounts_by_id.get(acc._parent_id, NotAvailable) - self.accounts_list = accounts_by_id.values() + self.accounts_list = list(accounts_by_id.values()) if has_no_account and not self.accounts_list: raise NoAccountsException(has_no_account) diff --git a/modules/creditmutuel/pages.py b/modules/creditmutuel/pages.py index 4d9138c65d51ca0bb7792b3c2ff3b8df4e5fb279..fcbbd1341ab0049cb20a85836005c1b9e1c97ab9 100644 --- a/modules/creditmutuel/pages.py +++ b/modules/creditmutuel/pages.py @@ -20,7 +20,6 @@ from __future__ import unicode_literals import re -import hashlib import time from decimal import Decimal, InvalidOperation @@ -90,6 +89,7 @@ class LoginPage(HTMLPage): # format login/password like login/password sent by firefox or chromium browser form['_cm_user'] = login.encode('cp1252', errors='xmlcharrefreplace').decode('cp1252') form['_cm_pwd'] = passwd.encode('cp1252', errors='xmlcharrefreplace').decode('cp1252') + form['_charset_'] = 'cp1252' form.submit() @property @@ -193,6 +193,10 @@ class UserSpacePage(LoggedPage, HTMLPage): def on_load(self): if self.doc.xpath('//form[@id="GoValider"]'): raise ActionNeeded("Le site du contrat Banque à Distance a besoin d'informations supplémentaires") + personal_infos = CleanText('//form[@class="_devb_act ___Form"]//div[contains(@class, "bloctxt")]/p[1]')(self.doc) + if 'Afin de compléter vos informations personnelles, renseignez le formulaire ci-dessous' in personal_infos: + raise ActionNeeded("Le site nécessite la saisie des informations personnelles de l'utilisateur.") + super(UserSpacePage, self).on_load() @@ -241,7 +245,7 @@ class item_account_generic(ItemElement): (re.compile(r'Ldd'), Account.TYPE_SAVINGS), (re.compile(r'Livret'), Account.TYPE_SAVINGS), (re.compile(r"Plan D'Epargne"), Account.TYPE_SAVINGS), - (re.compile(r'Tonic Croissance'), Account.TYPE_SAVINGS), + (re.compile(r'Tonic Crois'), Account.TYPE_SAVINGS), # eg: 'Tonic Croissance', 'Tonic Crois Pro' (re.compile(r'Tonic Societaire'), Account.TYPE_SAVINGS), (re.compile(r'Capital Expansion'), Account.TYPE_SAVINGS), (re.compile(r'Épargne'), Account.TYPE_SAVINGS), @@ -277,7 +281,9 @@ class item_account_generic(ItemElement): def loan_condition(self, check_no_details=False): _type = Field('type')(self) label = Field('label')(self) - details_link = Link('.//a', default=None)(self) + # The 'lien_inter_sites' link leads to a 404 and is not a link to loans details. + # The link name on the website is : Vos encours mobilisation de créances + details_link = Link('.//a[not(contains(@href, "lien_inter_sites"))]', default=None)(self) # mobile accounts are leading to a 404 error when parsing history # furthermore this is not exactly a loan account @@ -957,7 +963,7 @@ class CardPage(OperationsPage, LoggedPage): return not CleanText('//td[contains(., "Aucun mouvement")]', default=False)(self) def parse(self, el): - label = CleanText('//*[contains(text(), "Achats")]')(el) + label = CleanText('//span[contains(text(), "Achats")]/following-sibling::span[2]')(el) if not label: return try: @@ -988,16 +994,24 @@ class CardPage(OperationsPage, LoggedPage): def parse(self, el): try: - self.env['raw'] = "%s %s" % (CleanText().filter(TableCell('commerce')(self)[0].text), - CleanText().filter(TableCell('ville')(self)[0].text)) - except (ColumnNotFound, AttributeError): - self.env['raw'] = "%s" % (CleanText().filter(TableCell('commerce')(self)[0].text)) - - self.env['type'] = (Transaction.TYPE_DEFERRED_CARD - if CleanText('//a[contains(text(), "Prélevé fin")]', default=None) - else Transaction.TYPE_CARD) - self.env['differed_date'] = parse_french_date(Regexp(CleanText('//*[contains(text(), "Achats")]'), - r'au[\s]+(.*)')(self)).date() + self.env['raw'] = Format( + '%s %s', + CleanText(TableCell('commerce'), children=False), + CleanText(TableCell('ville')), + )(self) + except ColumnNotFound: + self.env['raw'] = CleanText(TableCell('commerce'), chilren=False)(self) + + if CleanText('//span[contains(text(), "Prélevé fin")]', default=None)(self): + self.env['type'] = Transaction.TYPE_DEFERRED_CARD + else: + self.env['type'] = Transaction.TYPE_CARD + + self.env['differed_date'] = Date( + CleanText('//span[contains(text(), "Achats")]/following-sibling::span[2]'), + parse_func=parse_french_date, + )(self) + amount = TableCell('credit')(self)[0] if self.page.browser.is_new_website: if not len(amount.xpath('./div')): @@ -1567,18 +1581,22 @@ class InternalTransferPage(LoggedPage, HTMLPage): def check_errors(self): # look for known errors content = self.text - messages = ['Le montant du virement doit être positif, veuillez le modifier', - 'Montant maximum autorisé au débit pour ce compte', - 'Dépassement du montant journalier autorisé', - 'Le solde de votre compte est insuffisant', - 'Nom prénom du bénéficiaire différent du titulaire. Utilisez un compte courant', - "Pour effectuer cette opération, vous devez passer par l’intermédiaire d’un compte courant", - 'Montant maximum autorisé au crédit pour ce compte', - 'Débit interdit sur ce compte', - 'Virement interdit sur compte clos', - "L'intitulé du virement ne peut contenir le ou les caractères suivants", - 'La date ne peut être inférieure à la date du jour. Veuillez la corriger', - ] + messages = [ + 'Le montant du virement doit être positif, veuillez le modifier', + 'Montant maximum autorisé au débit pour ce compte', + 'Dépassement du montant journalier autorisé', + 'Le solde de votre compte est insuffisant', + 'Nom prénom du bénéficiaire différent du titulaire. Utilisez un compte courant', + "Pour effectuer cette opération, vous devez passer par l’intermédiaire d’un compte courant", + 'Montant maximum autorisé au crédit pour ce compte', + 'Débit interdit sur ce compte', + 'Virement interdit sur compte clos', + 'Virement interdit sur compte inexistant', + "L'intitulé du virement ne peut contenir le ou les caractères suivants", + 'La date ne peut être inférieure à la date du jour. Veuillez la corriger', + 'Opération non conforme, les virements entre comptes Epargne ne sont pas autorisés.', + 'Opération non conforme: entre titulaires différents, seul un compte courant peut être débité.', + ] for message in messages: if message in content: @@ -1803,6 +1821,8 @@ class VerifCodePage(LoggedPage, HTMLPage): ) for error in errors: if error: + # don't reload state + self.browser.need_clear_storage = True raise AddRecipientBankError(message=error) action_needed = CleanText('//p[contains(text(), "Carte de CLÉS PERSONNELLES révoquée")]')(self.doc) @@ -1815,23 +1835,32 @@ class VerifCodePage(LoggedPage, HTMLPage): return v def get_question(self): - s = CleanText('//label[@for="txtCle"]')(self.doc) - img_hash_md5 = hashlib.md5(self.browser.open(Attr('//label[@for="txtCle"]/img', 'src')(self.doc)).content).hexdigest() - key_case = self.get_key_case(img_hash_md5) - assert key_case, "Hash %s not found, matching key case is available on session folder." % img_hash_md5 - return s[:25] + ' %s' % key_case + s[25:] + question = Regexp(CleanText('//div/p[input]'), r'(Veuillez .*):')(self.doc) + return question def post_code(self, key): - form = self.get_form(id='frm') - form['code'] = key - form['valChx.x'] = '1' - form['valChx.y'] = '1' + form = self.get_form('//form[contains(@action, "verif_code")]') + form['[t:xsd%3astring;]Data_KeyInput'] = key + + # we don't know the card id + # by default all users have only one card + # but to be sure, let's get it dynamically + do_validate = [k for k in form.keys() if '_FID_DoValidate_cardId' in k] + assert len(do_validate) == 1, 'There should be only one card.' + form[do_validate[0]] = '' + + activate = [k for k in form.keys() if '_FID_GoCardAction_action' in k] + assert len(activate) == 1, 'There should be only one card.' + del form[activate[0]] + form.submit() def handle_error(self): error_msg = CleanText('//div[@class="blocmsg info"]/p')(self.doc) # the card was not activated yet if 'veuillez activer votre carte' in error_msg: + # don't reload state + self.browser.need_clear_storage = True raise AddRecipientBankError(message=error_msg) @@ -1974,7 +2003,7 @@ class RevolvingLoansList(LoggedPage, HTMLPage): def obj_rate(self): if not self.async_load: - return MyDecimal(Regexp(CleanText('.//td[2]'), r'.* (\d*,\d*)%'))(self) + return MyDecimal(Regexp(CleanText('.//td[2]'), r'.* (\d*,\d*)%', default=NotAvailable))(self) class ErrorPage(HTMLPage): diff --git a/modules/fortuneo/browser.py b/modules/fortuneo/browser.py index 876f92ed0073e5b6a8c63a767319279ec0052992..4fe0209b47c2ecb2199ae0da62e70db7d96eb882 100644 --- a/modules/fortuneo/browser.py +++ b/modules/fortuneo/browser.py @@ -105,6 +105,7 @@ class Fortuneo(LoginBrowser, StatesMixin): self.page.login(self.username, self.password) if self.login_page.is_here(): + self.page.check_is_blocked() raise BrowserIncorrectPassword() self.location('/fr/prive/default.jsp?ANav=1') diff --git a/modules/fortuneo/pages/accounts_list.py b/modules/fortuneo/pages/accounts_list.py index 30dd0002cae7c1603db9f23663aaf9f35c0b38ce..0dfc2ff231f3d04a49c7140bc209a6f095e160a5 100644 --- a/modules/fortuneo/pages/accounts_list.py +++ b/modules/fortuneo/pages/accounts_list.py @@ -127,6 +127,7 @@ class PeaHistoryPage(LoggedPage, HTMLPage): return False form['dateDebut'] = (date.today() - relativedelta(years=2)).strftime('%d/%m/%Y') form['nbResultats'] = '100' + form['typeOperation'] = '01' form.submit() return True diff --git a/modules/fortuneo/pages/login.py b/modules/fortuneo/pages/login.py index 3b177a59bea48cf0693ddfdf20e70cef8aca0b09..ec10ef688738d8b5e8b03286284616050b8ad8df 100644 --- a/modules/fortuneo/pages/login.py +++ b/modules/fortuneo/pages/login.py @@ -17,10 +17,11 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . +from __future__ import unicode_literals from weboob.browser.pages import HTMLPage from weboob.browser.filters.standard import CleanText -from weboob.exceptions import BrowserUnavailable +from weboob.exceptions import BrowserUnavailable, ActionNeeded class LoginPage(HTMLPage): @@ -35,6 +36,11 @@ class LoginPage(HTMLPage): form['passwd'] = passwd form.submit() + def check_is_blocked(self): + error_message = CleanText('//div[@id="acces_client"]//p[@class="container error"]/label')(self.doc) + if 'Votre accès est désormais bloqué' in error_message: + raise ActionNeeded(error_message) + class UnavailablePage(HTMLPage): def on_load(self): diff --git a/modules/ganassurances/browser.py b/modules/ganassurances/browser.py index 25638009e668483a8ccdc1b2594de11282d00b74..9ed56ec21651a11e0a2587dec4e3ce09912b9112 100644 --- a/modules/ganassurances/browser.py +++ b/modules/ganassurances/browser.py @@ -17,129 +17,16 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . -import re +from __future__ import unicode_literals -from weboob.browser import LoginBrowser, URL, need_login -from weboob.exceptions import BrowserIncorrectPassword -from weboob.capabilities.bank import Account -from weboob.capabilities.base import empty +from weboob.browser import AbstractBrowser -from .pages import LoginPage, AccountsPage, TransactionsPage, AVAccountPage, AVHistoryPage, FormPage, IbanPage, AvJPage +class GanAssurancesBrowser(AbstractBrowser): + PARENT = 'ganpatrimoine' + PARENT_ATTR = 'package.browser.GanPatrimoineBrowser' -__all__ = ['GanAssurancesBrowser'] - - -class GanAssurancesBrowser(LoginBrowser): - BASEURL = 'https://espaceclient.ganassurances.fr' - - login = URL(r'https://authentification.(?P.*).fr/cas/login', LoginPage) - iban = URL(r'/wps/myportal/!ut/(.*)/\?paramNumCpt=(.*)', IbanPage) - accounts = URL(r'/wps/myportal/TableauDeBord', AccountsPage) - transactions = URL(r'/wps/myportal/!ut', TransactionsPage) - av_account_form = URL(r'/wps/myportal/assurancevie/', FormPage) - av_account = URL(r'https://secure-rivage.ganassurances.fr/contratVie.rivage.syntheseContratEparUc.gsi', - r'/front/vie/epargne/contrat/(.*)', AVAccountPage) - av_history = URL(r'https://secure-rivage.(?P.*).fr/contratVie.rivage.mesOperations.gsi', AVHistoryPage) - av_secondary = URL(r'/api/ecli/vie/contrats/(?P.*)', AvJPage) - - def __init__(self, *args, **kwargs): - super(GanAssurancesBrowser, self).__init__(*args, **kwargs) - self.website = 'ganassurances' - self.domain = 'ganass' - - def do_login(self): - login_url = 'https://espaceclient.%s.fr/login-%s' % (self.website, self.domain) - self.login.go(website=self.website, params={'service': login_url}) - self.page.login(self.username, self.password) - - if self.login.is_here(): - error_msg = self.page.get_error() - if error_msg and "LOGIN_ERREUR_MOT_PASSE_INVALIDE" in error_msg: - raise BrowserIncorrectPassword() - assert False, 'Unhandled error at login: %s' % error_msg - - # For life asssurance accounts, to get balance we use the link from the account. - # And to get history (or other) we need to use the link again but the link works only once. - # So we get balance only for iter_account to not use the new link each time. - @need_login - def get_accounts_list(self, balance=True, need_iban=False): - accounts = [] - self.accounts.stay_or_go() - for account in self.page.get_list(): - if account.type == Account.TYPE_LIFE_INSURANCE and balance: - assert empty(account.balance) - self.location(account._link) - if self.av_account_form.is_here(): - self.page.av_account_form() - account.balance, account.currency = self.page.get_av_balance() - # New page where some AV are stored - elif "front/vie/" in account._link: - link = re.search(r'contrat\/(.+)-Groupama', account._link) - if link: - self.av_secondary.go(id_contrat=link.group(1)) - account.balance, account.currency = self.page.get_av_balance() - - self.accounts.stay_or_go() - if account.balance or not balance: - if account.type != Account.TYPE_LIFE_INSURANCE and need_iban: - self.location(account._link) - if self.transactions.is_here() and self.page.has_iban(): - self.page.go_iban() - account.iban = self.page.get_iban() - accounts.append(account) - return accounts - - def _get_history(self, account): - if "front/vie" in account._link: - return [] - accounts = self.get_accounts_list(balance=False) - for a in accounts: - if a.id == account.id: - self.location(a._link) - if a.type == Account.TYPE_LIFE_INSURANCE: - if not self.page.av_account_form(): - self.logger.warning('history form not found for %s', account) - return [] - self.av_history.go(website=self.website) - return self.page.get_av_history() - assert self.transactions.is_here() - return self.page.get_history(accid=account.id) - return [] - - # Duplicate line in case of arbitration because the site has only one line for the 2 transactions (debit and credit on the same line) - def get_history(self, account): - for tr in self._get_history(account): - yield tr - if getattr(tr, '_arbitration', False): - tr = tr.copy() - tr.amount = -tr.amount - yield tr - - def get_coming(self, account): - if account.type == Account.TYPE_LIFE_INSURANCE: - return [] - for a in self.get_accounts_list(): - if a.id == account.id: - self.location(a._link) - assert self.transactions.is_here() - link = self.page.get_coming_link() - if link is not None: - self.location(self.page.get_coming_link()) - assert self.transactions.is_here() - return self.page.get_history(accid=account.id) - return [] - - def get_investment(self, account): - if account.type != Account.TYPE_LIFE_INSURANCE: - return [] - for a in self.get_accounts_list(balance=False): - if a.id == account.id: - # There isn't any invest on AV having front/vie - # in theirs url - if "front/vie/" not in account._link: - self.location(a._link) - self.page.av_account_form() - if self.av_account.is_here(): - return self.page.get_av_investments() - return [] + def __init__(self, website, *args, **kwargs): + super(GanAssurancesBrowser, self).__init__(website, *args, **kwargs) + self.website = website + self.BASEURL = 'https://espaceclient.%s.fr' % website diff --git a/modules/ganassurances/module.py b/modules/ganassurances/module.py index 8b2bf530b52e93bbb16e6d09a39c0ce32e4f5bb2..9a3f2f93efa741e63e5d0da0456f484e789d7715 100644 --- a/modules/ganassurances/module.py +++ b/modules/ganassurances/module.py @@ -17,13 +17,11 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . - from __future__ import unicode_literals -from weboob.capabilities.bank import CapBankWealth, AccountNotFound -from weboob.capabilities.base import find_object -from weboob.tools.backend import Module, BackendConfig -from weboob.tools.value import Value, ValueBackendPassword +from weboob.capabilities.bank import CapBank +from weboob.tools.backend import AbstractModule, BackendConfig +from weboob.tools.value import ValueBackendPassword from .browser import GanAssurancesBrowser @@ -31,7 +29,7 @@ from .browser import GanAssurancesBrowser __all__ = ['GanAssurancesModule'] -class GanAssurancesModule(Module, CapBankWealth): +class GanAssurancesModule(AbstractModule, CapBank): NAME = 'ganassurances' MAINTAINER = 'Romain Bignon' EMAIL = 'romain@weboob.org' @@ -39,28 +37,18 @@ class GanAssurancesModule(Module, CapBankWealth): DESCRIPTION = 'Gan Assurances' LICENSE = 'LGPLv3+' CONFIG = BackendConfig( - Value('login', label='Numéro client'), + ValueBackendPassword('login', label='Numéro client', masked=False), ValueBackendPassword('password', label="Code d'accès") ) + + PARENT = 'ganpatrimoine' BROWSER = GanAssurancesBrowser + def create_default_browser(self): return self.create_browser( + 'ganassurances', self.config['login'].get(), - self.config['password'].get() + self.config['password'].get(), + weboob=self.weboob ) - - def iter_accounts(self): - return self.browser.get_accounts_list(need_iban=True) - - def get_account(self, _id): - return find_object(self.browser.get_accounts_list(need_iban=True), id=_id, error=AccountNotFound) - - def iter_history(self, account): - return self.browser.get_history(account) - - def iter_coming(self, account): - return self.browser.get_coming(account) - - def iter_investment(self, account): - return self.browser.get_investment(account) diff --git a/modules/ganassurances/pages.py b/modules/ganassurances/pages.py deleted file mode 100644 index edd8b6f7d850f5de5c966c5c8667419eefa88aea..0000000000000000000000000000000000000000 --- a/modules/ganassurances/pages.py +++ /dev/null @@ -1,304 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2012 Romain Bignon -# -# This file is part of a weboob module. -# -# This weboob module is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This weboob module is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this weboob module. If not, see . - - -from __future__ import unicode_literals - -import re -import requests -import ast - -from decimal import Decimal - -from weboob.browser.pages import HTMLPage, pagination, LoggedPage, FormNotFound, JsonPage -from weboob.browser.elements import method, TableElement, ItemElement -from weboob.browser.filters.standard import Env, CleanDecimal, CleanText, Date, Regexp, Eval, Field -from weboob.browser.filters.html import Attr, Link, TableCell -from weboob.browser.filters.javascript import JSVar -from weboob.capabilities.bank import Account, Investment -from weboob.capabilities.base import NotAvailable -from weboob.tools.capabilities.bank.transactions import FrenchTransaction -from weboob.tools.capabilities.bank.investments import is_isin_valid -from weboob.browser.filters.json import Dict - - -class LoginPage(HTMLPage): - def login(self, login, passwd): - tab = re.search(r'clavierAChristian = (\[[\d,\s]*\])', self.text).group(1) - number_list = ast.literal_eval(tab) - key_map = {} - for i, number in enumerate(number_list): - if number < 10: - key_map[number] = chr(ord('A') + i) - pass_string = ''.join(key_map[int(n)] for n in passwd) - form = self.get_form(name='loginForm') - form['username'] = login - form['password'] = pass_string - form.submit() - - def get_error(self): - return CleanText('//div[@id="msg"]')(self.doc) - - -class AccountsPage(LoggedPage, HTMLPage): - ACCOUNT_TYPES = { - 'Solde des comptes bancaires - Groupama Banque': Account.TYPE_CHECKING, - 'Solde des comptes bancaires': Account.TYPE_CHECKING, - 'Epargne bancaire constituée - Groupama Banque': Account.TYPE_SAVINGS, - 'Epargne bancaire constituée': Account.TYPE_SAVINGS, - 'Mes crédits': Account.TYPE_LOAN, - 'Assurance Vie': Account.TYPE_LIFE_INSURANCE, - 'Certificats Mutualistes': Account.TYPE_SAVINGS, - } - - ACCOUNT_TYPES2 = { - 'plan epargne actions': Account.TYPE_PEA, - } - - def get_list(self): - account_type = Account.TYPE_UNKNOWN - accounts = [] - - for tr in self.doc.xpath('//div[@class="finance"]/form/table[@class="ecli"]/tr'): - if tr.attrib.get('class', '') == 'entete': - account_type = self.ACCOUNT_TYPES.get(tr.find('th').text.strip(), Account.TYPE_UNKNOWN) - continue - - tds = tr.findall('td') - a = tds[0].find('a') - - # Skip accounts that can't be accessed - if a is None: - continue - - balance = tds[-1].text.strip() - - account = Account() - account.label = ' '.join([txt.strip() for txt in tds[0].itertext()]) - account.label = re.sub(r'[\s\xa0\u2022]+', ' ', account.label).strip() - - # take "N° (FOO123 456)" but "N° (FOO123) MR. BAR" - account.id = re.search(r'N° (\w+( \d+)*)', account.label).group(1).replace(' ', '') - account.number = account.id - account.type = account_type - - for patt, type in self.ACCOUNT_TYPES2.items(): - if patt in account.label.lower(): - account.type = type - break - - if balance: - account.balance = Decimal(FrenchTransaction.clean_amount(balance)) - account.currency = account.get_currency(balance) - - if 'onclick' in a.attrib: - m = re.search(r"javascript:submitForm\(([\w]+),'([^']+)'\);", a.attrib['onclick']) - if not m: - self.logger.warning('Unable to find link for %r', account.label) - account._link = None - else: - account._link = m.group(2) - else: - account._link = a.attrib['href'].strip() - - if accounts and accounts[-1].label == account.label and account.type == Account.TYPE_PEA: - self.logger.warning('%s seems to be a duplicate of %s, skipping', account, accounts[-1]) - continue - accounts.append(account) - return accounts - - -class Transaction(FrenchTransaction): - PATTERNS = [ - (re.compile(r'^Facture (?P
\d{2})/(?P\d{2})-(?P.*) carte .*'), FrenchTransaction.TYPE_CARD), - (re.compile(r'^(Prlv( de)?|Ech(éance|\.)) (?P.*)'), FrenchTransaction.TYPE_ORDER), - (re.compile(r'^(Vir|VIR)( de)? (?P.*)'), FrenchTransaction.TYPE_TRANSFER), - (re.compile(r'^CHEQUE.*? (N° \w+)?$'), FrenchTransaction.TYPE_CHECK), - (re.compile(r'^Cotis(ation)? (?P.*)'), FrenchTransaction.TYPE_BANK), - (re.compile(r'(?PInt .*)'), FrenchTransaction.TYPE_BANK), - (re.compile(r'^SOUSCRIPTION|REINVESTISSEMENT'), FrenchTransaction.TYPE_DEPOSIT), - ] - - -class TransactionsPage(HTMLPage): - logged = True - - @pagination - @method - class get_history(Transaction.TransactionsElement): - head_xpath = '//table[@id="releve_operation"]//tr/th' - item_xpath = '//table[@id="releve_operation"]//tr' - - col_date = ['Date opé', 'Date', 'Date d\'opé', 'Date opération'] - col_vdate = ['Date valeur'] - col_credit = ['Crédit', 'Montant', 'Valeur'] - col_debit = ['Débit'] - - def next_page(self): - url = Attr('//a[contains(text(), "Page suivante")]', 'onclick', default=None)(self) - if url: - m = re.search(r'\'([^\']+).*([\d]+)', url) - return requests.Request("POST", m.group(1), - data={ - 'numCompte': Env('accid')(self), - 'vue': "ReleveOperations", - 'tri': "DateOperation", - 'sens': "DESC", - 'page': m.group(2), - 'nb_element': "25"} - ) - - class item(Transaction.TransactionElement): - def condition(self): - return len(self.el.xpath('./td')) > 3 - - def get_coming_link(self): - try: - a = self.doc.getroot().cssselect('div#sous_nav ul li a.bt_sans_off')[0] - except IndexError: - return None - return re.sub(r'[\s]+', '', a.attrib['href']) - - def has_iban(self): - return self.doc.xpath('//a[@class="rib"]') - - def go_iban(self): - js_event = Attr("//a[@class='rib']", 'onclick')(self.doc) - m = re.search(r'envoyer(.*);', js_event) - iban_params = ast.literal_eval(m.group(1)) - self.browser.location("{}?paramNumCpt={}".format(iban_params[1], iban_params[0])) - - -class IbanPage(LoggedPage, HTMLPage): - - def get_iban(self): - return CleanText('(//b[contains(text(), "IBAN")])[1]/../text()')(self.doc) - - -class AVAccountPage(LoggedPage, HTMLPage): - """ - Get balance - - :return: decimal balance, currency - :rtype: tuple - """ - def get_av_balance(self): - balance_xpath = '//p[contains(text(), "Épargne constituée")]/span' - balance = CleanDecimal(balance_xpath)(self.doc) - currency = Account.get_currency(CleanText(balance_xpath)(self.doc)) - return balance, currency - - @method - class get_av_investments(TableElement): - item_xpath = '//table[@id="repartition_epargne3"]/tr[position() > 1]' - head_xpath = '//table[@id="repartition_epargne3"]/tr/th[position() > 1]' - - col_quantity = 'Nombre d’unités de compte' - col_unitvalue = "Valeur de l’unité de compte" - col_valuation = 'Épargne constituée en euros' - col_portfolio_share = 'Répartition %' - - class item(ItemElement): - klass = Investment - - def condition(self): - return (CleanText('./th')(self) != 'Total épargne constituée') and ('Détail' not in Field('label')(self)) - - obj_label = CleanText('./th') - obj_quantity = CleanDecimal(TableCell('quantity'), default=NotAvailable) - obj_unitvalue = CleanDecimal(TableCell('unitvalue'), default=NotAvailable) - obj_valuation = CleanDecimal(TableCell('valuation'), default=NotAvailable) - obj_portfolio_share = Eval(lambda x: x / 100, CleanDecimal(TableCell('portfolio_share'))) - - def obj_code(self): - code = Regexp(Link('./th/a', default=''), r'isin=(\w+)|/(\w+)\.pdf', default=NotAvailable)(self) - return code if is_isin_valid(code) else NotAvailable - - def obj_code_type(self): - return Investment.CODE_TYPE_ISIN if is_isin_valid(Field('code')(self)) else NotAvailable - - -class AvJPage(LoggedPage, JsonPage): - def get_av_balance(self): - balance = CleanDecimal(Dict('montant'))(self.doc) - currency = "EUR" - return balance, currency - - -class AVHistoryPage(LoggedPage, HTMLPage): - @method - class get_av_history(TableElement): - item_xpath = '//table[@id="enteteTableSupports"]/tbody/tr' - head_xpath = '//table[@id="enteteTableSupports"]/thead/tr/th' - - col_date = 'Date' - col_label = 'Type de mouvement' - col_debit = 'Montant Désinvesti' - col_credit = ['Montant investi', 'Montant Net Perçu'] - # There is several types of life insurances, so multiple columns - col_credit2 = ['Montant Brut Versé'] - - class item(ItemElement): - klass = Transaction - - def condition(self): - return CleanText(TableCell('date'))(self) != 'en cours' - - obj_label = CleanText(TableCell('label')) - obj_type = Transaction.TYPE_BANK - obj_date = Date(CleanText(TableCell('date')), dayfirst=True) - obj__arbitration = False - - def obj_amount(self): - credit = CleanDecimal(TableCell('credit'), default=Decimal(0))(self) - # Different types of life insurances, use different columns. - if TableCell('debit', default=None)(self): - debit = CleanDecimal(TableCell('debit'), default=Decimal(0))(self) - # In case of financial arbitration, both columns are equal - if credit and debit: - assert credit == debit - self.obj._arbitration = True - return credit - else: - return credit - abs(debit) - else: - credit2 = CleanDecimal(TableCell('credit2'), default=Decimal(0))(self) - assert not (credit and credit2) - return credit + credit2 - - -class FormPage(LoggedPage, HTMLPage): - def get_av_balance(self): - balance_xpath = '//p[contains(text(), "montant de votre épargne")]' - balance = CleanDecimal(Regexp(CleanText(balance_xpath), r'est de ([\s\d,]+)', default=NotAvailable), - replace_dots=True, default=NotAvailable)(self.doc) - currency = Account.get_currency(CleanText(balance_xpath)(self.doc)) - return balance, currency - - def av_account_form(self): - try: - form = self.get_form(id="formGoToRivage") - form['gfr_numeroContrat'] = JSVar(var='numContrat').filter(CleanText('//script[contains(text(), "var numContrat")]')(self.doc)) - form['gfr_data'] = JSVar(var='pCryptage').filter(CleanText('//script[contains(text(), "var pCryptage")]')(self.doc)) - form['gfr_adrSite'] = 'https://espaceclient.%s.fr' % self.browser.website - form.url = 'https://secure-rivage.%s.fr/contratVie.rivage.syntheseContratEparUc.gsi' % self.browser.website - form.submit() - return True - except FormNotFound: - return False diff --git a/modules/groupamaes/module.py b/modules/groupamaes/module.py index 260e5fa696e25f2b2eb85890dd9c1ab55ba3c06d..75954d242207d5d28a33cc3ae8a620dacaf14a6a 100644 --- a/modules/groupamaes/module.py +++ b/modules/groupamaes/module.py @@ -17,10 +17,11 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . -from weboob.capabilities.bank import CapBankPockets, AccountNotFound +from __future__ import unicode_literals + +from weboob.capabilities.bank import CapBankPockets from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword -from weboob.capabilities.base import find_object from .browser import GroupamaesBrowser @@ -30,16 +31,18 @@ __all__ = ['GroupamaesModule'] class GroupamaesModule(Module, CapBankPockets): NAME = 'groupamaes' - DESCRIPTION = u"Groupama Épargne Salariale" - MAINTAINER = u'Bezleputh' + DESCRIPTION = 'Groupama Épargne Salariale' + MAINTAINER = 'Bezleputh' EMAIL = 'carton_ben@yahoo.fr' LICENSE = 'LGPLv3+' VERSION = '1.6' BROWSER = GroupamaesBrowser - CONFIG = BackendConfig(ValueBackendPassword('login', label='Identifiant', regexp='\d{8,}', masked=False), - ValueBackendPassword('password', label='Mot de passe')) + CONFIG = BackendConfig( + ValueBackendPassword('login', label='Identifiant', regexp=r'\d{8,}', masked=False), + ValueBackendPassword('password', label='Mot de passe') + ) def create_default_browser(self): return self.create_browser( @@ -47,10 +50,8 @@ class GroupamaesModule(Module, CapBankPockets): self.config['password'].get(), 'https://www.gestion-epargne-salariale.fr', 'groupama-es/', - weboob=self.weboob) - - def get_account(self, _id): - return find_object(self.browser.iter_accounts(), id=_id, error=AccountNotFound) + weboob=self.weboob + ) def iter_accounts(self): return self.browser.iter_accounts() diff --git a/modules/hsbchk/browser.py b/modules/hsbchk/browser.py old mode 100755 new mode 100644 diff --git a/modules/hsbchk/module.py b/modules/hsbchk/module.py old mode 100755 new mode 100644 diff --git a/modules/hsbchk/pages/account_pages.py b/modules/hsbchk/pages/account_pages.py old mode 100755 new mode 100644 diff --git a/modules/hsbchk/pages/login.py b/modules/hsbchk/pages/login.py old mode 100755 new mode 100644 diff --git a/modules/hsbchk/sbrowser.py b/modules/hsbchk/sbrowser.py old mode 100755 new mode 100644 diff --git a/modules/humanis/module.py b/modules/humanis/module.py index 0cf4af0c771d76351c4159c4ba525f48acde735b..2954b160bafeefdd5789f7a123d2e40a05916de9 100644 --- a/modules/humanis/module.py +++ b/modules/humanis/module.py @@ -17,11 +17,11 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . +from __future__ import unicode_literals from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword -from weboob.capabilities.bank import CapBankPockets, AccountNotFound -from weboob.capabilities.base import find_object +from weboob.capabilities.bank import CapBankPockets from .browser import HumanisBrowser @@ -31,13 +31,15 @@ __all__ = ['HumanisModule'] class HumanisModule(Module, CapBankPockets): NAME = 'humanis' - DESCRIPTION = u'Humanis Épargne Salariale' - MAINTAINER = u'Jean Walrave' - EMAIL = 'jwalrave@budget-insight.com' + DESCRIPTION = 'Humanis Épargne Salariale' + MAINTAINER = 'Quentin Defenouillère' + EMAIL = 'quentin.defenouillere@budget-insight.com' LICENSE = 'LGPLv3+' VERSION = '1.6' - CONFIG = BackendConfig(ValueBackendPassword('login', label=u'Code d\'accès', masked=False), - ValueBackendPassword('password', label='Mot de passe')) + CONFIG = BackendConfig( + ValueBackendPassword('login', label='Code d\'accès', masked=False), + ValueBackendPassword('password', label='Mot de passe') + ) BROWSER = HumanisBrowser @@ -50,9 +52,6 @@ class HumanisModule(Module, CapBankPockets): weboob=self.weboob ) - def get_account(self, _id): - return find_object(self.browser.iter_accounts(), id=_id, error=AccountNotFound) - def iter_accounts(self): return self.browser.iter_accounts() diff --git a/modules/ing/api/accounts_page.py b/modules/ing/api/accounts_page.py index adacb451ba78b87bbcb31766b5ab32b5a17dacfe..8aa2180cd28062eea36cefd9f6c1226c36e87931 100644 --- a/modules/ing/api/accounts_page.py +++ b/modules/ing/api/accounts_page.py @@ -27,7 +27,7 @@ from weboob.browser.filters.json import Dict from weboob.browser.filters.standard import ( CleanText, CleanDecimal, Date, Eval, Lower, Format, Field, Map, Upper, ) -from weboob.capabilities.bank import Account +from weboob.capabilities.bank import Account, AccountOwnership from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.capabilities.base import NotAvailable @@ -75,6 +75,18 @@ class AccountsPage(LoggedPage, JsonPage): return -CleanDecimal(Dict('ledgerBalance'))(self) return CleanDecimal(Dict('ledgerBalance'))(self) + def obj_ownership(self): + ownership = Dict('ownership/code', default=None)(self) + role = Dict('role/label', default=None)(self) + + if ownership == 'JOINT': + return AccountOwnership.CO_OWNER + elif ownership == 'SINGLE': + if role == 'Titulaire': + return AccountOwnership.OWNER + elif role == 'Procuration': + return AccountOwnership.ATTORNEY + class HistoryPage(LoggedPage, JsonPage): def is_empty_page(self): diff --git a/modules/ing/api/login.py b/modules/ing/api/login.py index b186e4b141f89c0653f648b904146a3fd1fb6bc3..64540a8688c4620817461285396b59495f162b86 100644 --- a/modules/ing/api/login.py +++ b/modules/ing/api/login.py @@ -43,41 +43,9 @@ class INGVirtKeyboard(SimpleVirtualKeyboard): 'limit_pixel': 200 } - # for matching_symbols_coords, indexes are cases place like this - # --- --- --- --- --- - # |0| |1| |2| |3| |4| - # --- --- --- --- --- - # --- --- --- --- --- - # |5| |6| |7| |8| |9| - # --- --- --- --- --- - matching_symbols_coords = { - '0': (3, 3, 93, 91), - '1': (99, 3, 189, 91), - '2': (196, 3, 286, 91), - '3': (293, 3, 383, 91), - '4': (390, 3, 480, 91), - '5': (3, 98, 93, 186), - '6': (99, 98, 189, 186), - '7': (196, 98, 286, 186), - '8': (293, 98, 383, 186), - '9': (390, 98, 480, 186), - } - - symbols = { - '0': ('7b4989b431e631ec79df5d71aecb1a47', 'e2522e1f7476ad6430219a73b10799b0', 'f7db285c5c742c3a348e332c0e9f7f3e',), - '1': ('9f1b03aa9a6f9789714c38eb90a43a11', '86bc0e7e1173472928e746db874b38c3',), - '2': ('3a7d1ba32f4326a02f717f71262ba02b', 'afc2a00289ba9e362c4e9333c14a574a',), - '3': ('203bfd122f474eb9c5c278eeda01bed4', 'c1daa556a1eff1fd18817dbef39792f8',), - '4': ('c09b323e5a80a195d9cb0c3000f3d7ec', 'f020eaf7cdffefec065d3b2801ed73e2', '5e194b0aae3b8f02ebbf9cdec5c37239',), - '5': ('1749dc3f2e302cd3562a0558755ab030', 'b64163e3f5f7d83ff1baad8c4d1bc37b',), - '6': ('0888a7dc9085fcf09d56363ac253a54a', 'e269686d10f95678caf995de6834f74b', '8c505dad47cf6029921fca5fb4b0bc8d',), - '7': ('75aaa903b8277b82c458c3540208a009', 'e97b0c0e01d77dd480b8a5f5c138a268',), - '8': ('f5fa36d16f55b72ba988eb87fa1ed753', '118a52a6a480b5db5eabb0ea26196db3',), - '9': ('62f91d10650583cb6146d25bb9ac161d', 'fd81675aa1c26cbf5bb6c9f1bcdbbdf9',), - } - - def __init__(self, file, cols, rows, browser): - # use matching_symbols_coords because margins between tiles are not equals + def __init__(self, file, cols, rows, browser, matching_symbols_coords=None, symbols=None): + self.symbols = symbols or {} + self.matching_symbols_coords = matching_symbols_coords or {} super(INGVirtKeyboard, self).__init__(file=file, cols=cols, rows=rows, matching_symbols_coords=self.matching_symbols_coords, browser=browser) def process_tiles(self): @@ -131,18 +99,53 @@ class INGVirtKeyboard(SimpleVirtualKeyboard): class LoginPage(JsonPage): + # for matching_symbols_coords, indexes are cases place like this + # --- --- --- --- --- + # |0| |1| |2| |3| |4| + # --- --- --- --- --- + # --- --- --- --- --- + # |5| |6| |7| |8| |9| + # --- --- --- --- --- + matching_symbols_coords = { + '0': (3, 3, 93, 91), + '1': (99, 3, 189, 91), + '2': (196, 3, 286, 91), + '3': (293, 3, 383, 91), + '4': (390, 3, 480, 91), + '5': (3, 98, 93, 186), + '6': (99, 98, 189, 186), + '7': (196, 98, 286, 186), + '8': (293, 98, 383, 186), + '9': (390, 98, 480, 186), + } + + vk_symbols = { + '0': ('7b4989b431e631ec79df5d71aecb1a47', 'e2522e1f7476ad6430219a73b10799b0', 'f7db285c5c742c3a348e332c0e9f7f3e',), + '1': ('9f1b03aa9a6f9789714c38eb90a43a11', '86bc0e7e1173472928e746db874b38c3',), + '2': ('3a7d1ba32f4326a02f717f71262ba02b', 'afc2a00289ba9e362c4e9333c14a574a',), + '3': ('203bfd122f474eb9c5c278eeda01bed4', 'c1daa556a1eff1fd18817dbef39792f8',), + '4': ('c09b323e5a80a195d9cb0c3000f3d7ec', 'f020eaf7cdffefec065d3b2801ed73e2', '5e194b0aae3b8f02ebbf9cdec5c37239',), + '5': ('1749dc3f2e302cd3562a0558755ab030', 'b64163e3f5f7d83ff1baad8c4d1bc37b',), + '6': ('0888a7dc9085fcf09d56363ac253a54a', 'e269686d10f95678caf995de6834f74b', '8c505dad47cf6029921fca5fb4b0bc8d',), + '7': ('75aaa903b8277b82c458c3540208a009', 'e97b0c0e01d77dd480b8a5f5c138a268',), + '8': ('f5fa36d16f55b72ba988eb87fa1ed753', '118a52a6a480b5db5eabb0ea26196db3',), + '9': ('62f91d10650583cb6146d25bb9ac161d', 'fd81675aa1c26cbf5bb6c9f1bcdbbdf9',), + } + @property def is_logged(self): return 'firstName' in self.doc - def get_password_coord(self, img, password): - assert 'pinPositions' in self.doc, 'Virtualkeyboard position has failed' - assert 'keyPadUrl' in self.doc, 'Virtualkeyboard image url is missing' - + def init_vk(self, img, password): pin_position = Dict('pinPositions')(self.doc) image = BytesIO(img) - vk = INGVirtKeyboard(image, cols=5, rows=2, browser=self.browser) + vk = INGVirtKeyboard(image, cols=5, rows=2, browser=self.browser, matching_symbols_coords=self.matching_symbols_coords, symbols=self.vk_symbols) password_random_coords = vk.password_tiles_coord(password) # pin positions (website side) start at 1, our positions start at 0 return [password_random_coords[index - 1] for index in pin_position] + + def get_password_coord(self, img, password): + assert 'pinPositions' in self.doc, 'Virtualkeyboard position has failed' + assert 'keyPadUrl' in self.doc, 'Virtualkeyboard image url is missing' + return self.init_vk(img, password) diff --git a/modules/ing/api_browser.py b/modules/ing/api_browser.py index 3c7a3248091a828089167273c9f5333ee2196fd5..4776f17f03ee74c3037a65e07475bc2a92768b6e 100644 --- a/modules/ing/api_browser.py +++ b/modules/ing/api_browser.py @@ -263,6 +263,7 @@ class IngAPIBrowser(LoginBrowser, StatesMixin): if web_acc.id[-4:] == api_acc.number[-4:]: web_acc._uid = api_acc.id web_acc.coming = api_acc.coming + web_acc.ownership = api_acc.ownership yield web_acc break else: diff --git a/modules/ing/web/titre.py b/modules/ing/web/titre.py index c0792b5d7bffe5f52ededb9e9c9267e1cbd136e6..0db033e6648cd8445764f33615df569b640c4f1d 100644 --- a/modules/ing/web/titre.py +++ b/modules/ing/web/titre.py @@ -29,8 +29,7 @@ from weboob.browser.elements import ListElement, TableElement, ItemElement, meth from weboob.browser.filters.standard import CleanDecimal, CleanText, Date, Regexp, Env from weboob.browser.filters.html import Link, Attr, TableCell from weboob.tools.capabilities.bank.transactions import FrenchTransaction -from weboob.tools.capabilities.bank.investments import create_french_liquidity -from weboob.tools.compat import unicode +from weboob.tools.capabilities.bank.investments import create_french_liquidity, IsinCode class NetissimaPage(HTMLPage): @@ -43,8 +42,9 @@ class Transaction(FrenchTransaction): class TitreValuePage(LoggedPage, HTMLPage): def get_isin(self): - isin = self.doc.xpath('//div[@id="headFiche"]//span[@id="test3"]/text()') - return unicode(isin[0].split(' - ')[0].strip()) if isin else NotAvailable + # redirection page with a url which contains the ISIN + # example: https://bourse.ing.fr/fr/marche/euronext-paris/