From 461c08ff831376b431426fd627d523cb4da75872 Mon Sep 17 00:00:00 2001 From: Romain Bignon Date: Sat, 10 Nov 2018 13:25:23 +0100 Subject: [PATCH] backport master modules fixes --- .../afer/compat/weboob_capabilities_bank.py | 5 + modules/afer/pages.py | 6 +- modules/allocine/browser.py | 21 +- modules/allocine/module.py | 9 +- modules/amazon/browser.py | 8 +- modules/amazon/pages.py | 5 +- modules/amazonstorecard/browser.py | 28 +- modules/amazonstorecard/module.py | 15 +- modules/amazonstorecard/pages.py | 25 +- modules/ameli/pages.py | 4 +- modules/amundi/browser.py | 46 +- .../amundi/compat/weboob_capabilities_bank.py | 5 + modules/amundi/module.py | 48 +- modules/amundi/pages.py | 37 +- .../apivie/compat/weboob_capabilities_bank.py | 5 + modules/axabanque/browser.py | 29 +- .../compat/weboob_capabilities_bank.py | 5 + ...oob_tools_capabilities_bank_investments.py | 86 +++ modules/axabanque/module.py | 25 +- modules/axabanque/pages/bank.py | 21 +- .../pages/compat/weboob_capabilities_bank.py | 45 ++ modules/axabanque/pages/transfer.py | 8 +- modules/axabanque/pages/wealth.py | 14 + modules/banquepopulaire/browser.py | 32 +- .../compat/weboob_capabilities_bank.py | 5 + ...oob_tools_capabilities_bank_investments.py | 86 +++ modules/banquepopulaire/module.py | 5 + modules/banquepopulaire/pages.py | 38 +- .../compat/weboob_capabilities_bank.py | 5 + modules/barclays/pages.py | 17 +- modules/bforbank/browser.py | 17 +- .../compat/weboob_capabilities_bank.py | 5 + ...oob_tools_capabilities_bank_investments.py | 86 +++ modules/bforbank/pages.py | 12 + .../binck/compat/weboob_capabilities_bank.py | 5 + modules/biplan/browser.py | 2 +- modules/bnpcards/corporate/pages.py | 9 +- .../bnporc/compat/weboob_capabilities_bank.py | 5 + modules/bnporc/enterprise/browser.py | 66 ++- modules/bnporc/enterprise/pages.py | 82 ++- modules/bnporc/pp/browser.py | 51 +- .../pp/compat/weboob_capabilities_bank.py | 45 ++ modules/bnporc/pp/pages.py | 99 ++-- modules/boursorama/browser.py | 2 +- .../compat/weboob_capabilities_bank.py | 5 + ...oob_tools_capabilities_bank_investments.py | 86 +++ modules/boursorama/pages.py | 56 +- modules/bouygues/pages.py | 13 +- modules/bp/browser.py | 15 +- modules/bp/compat/weboob_capabilities_bank.py | 5 + modules/bp/pages/accounthistory.py | 4 + modules/bp/pages/accountlist.py | 8 +- modules/bp/pages/base.py | 1 - modules/bp/pages/compat/__init__.py | 0 .../pages/compat/weboob_capabilities_bank.py | 45 ++ modules/bp/pages/login.py | 2 +- modules/bp/pages/subscription.py | 5 +- modules/bp/pages/transfer.py | 9 +- modules/bred/bred/browser.py | 2 +- modules/bred/bred/pages.py | 11 +- .../bred/compat/weboob_capabilities_bank.py | 5 + modules/caels/browser.py | 6 + .../caels/compat/weboob_capabilities_bank.py | 5 + modules/caels/module.py | 4 +- modules/caels/pages.py | 58 +++ modules/caissedepargne/browser.py | 190 +++++-- modules/caissedepargne/cenet/browser.py | 15 +- .../compat/weboob_capabilities_bank.py | 5 + ...oob_tools_capabilities_bank_investments.py | 86 +++ modules/caissedepargne/linebourse_browser.py | 1 + modules/caissedepargne/pages.py | 168 ++++-- .../compat/weboob_capabilities_bank.py | 5 + .../cices/compat/weboob_capabilities_bank.py | 5 + modules/citibank/browser.py | 66 ++- modules/citibank/parser.py | 30 +- .../cmes/compat/weboob_capabilities_bank.py | 5 + .../cmso/compat/weboob_capabilities_bank.py | 5 + modules/cmso/par/browser.py | 6 +- modules/cmso/pro/pages.py | 2 +- .../cragr/compat/weboob_capabilities_bank.py | 5 + modules/cragr/web/browser.py | 47 +- .../web/compat/weboob_capabilities_bank.py | 45 ++ ...oob_tools_capabilities_bank_investments.py | 86 +++ modules/cragr/web/pages.py | 34 +- .../creditcooperatif/caisseepargne_browser.py | 7 + modules/creditcooperatif/cenet_browser.py | 4 + .../creditcooperatif/linebourse_browser.py | 25 + modules/creditdunord/browser.py | 133 ++--- .../compat/weboob_capabilities_bank.py | 5 + modules/creditdunord/module.py | 16 +- modules/creditdunord/pages.py | 195 ++++--- .../compat/weboob_capabilities_bank.py | 5 + modules/creditmutuel/browser.py | 291 +++++++---- .../compat/weboob_capabilities_bank.py | 5 + ...oob_tools_capabilities_bank_investments.py | 86 +++ modules/creditmutuel/module.py | 14 +- modules/creditmutuel/pages.py | 491 ++++++++++++++---- modules/edf/par/browser.py | 16 +- modules/edf/par/pages.py | 18 +- modules/edf/pro/browser.py | 28 +- modules/edf/pro/pages.py | 33 +- modules/fortuneo/browser.py | 26 +- .../compat/weboob_capabilities_bank.py | 5 + modules/fortuneo/module.py | 6 +- modules/fortuneo/pages/accounts_list.py | 60 ++- modules/fortuneo/pages/compat/__init__.py | 0 .../pages/compat/weboob_capabilities_bank.py | 45 ++ ...oob_tools_capabilities_bank_investments.py | 86 +++ modules/fortuneo/pages/transfer.py | 24 +- modules/gmf/browser.py | 4 +- .../gmf/compat/weboob_capabilities_bank.py | 5 + modules/gmf/pages.py | 23 +- .../compat/weboob_capabilities_bank.py | 5 + modules/groupamaes/browser.py | 2 +- .../compat/weboob_capabilities_bank.py | 5 + modules/hsbc/browser.py | 64 ++- .../hsbc/compat/weboob_capabilities_bank.py | 5 + modules/hsbc/module.py | 7 +- modules/hsbc/pages/account_pages.py | 178 +++++-- modules/hsbc/pages/investments.py | 2 + modules/ideel/browser.py | 25 +- modules/ideel/module.py | 3 +- modules/infomaniak/module.py | 2 +- modules/ing/browser.py | 24 +- .../ing/compat/weboob_capabilities_bank.py | 5 + modules/ing/pages/accounts_list.py | 18 +- modules/ing/pages/compat/__init__.py | 0 ...oob_tools_capabilities_bank_investments.py | 86 +++ modules/ing/pages/titre.py | 8 +- modules/ing/pages/transfer.py | 6 +- modules/lcl/browser.py | 119 +++-- .../lcl/compat/weboob_capabilities_bank.py | 5 + .../lcl/compat/weboob_capabilities_bill.py | 259 +++++++++ ...oob_tools_capabilities_bank_investments.py | 86 +++ modules/lcl/module.py | 6 +- modules/lcl/pages.py | 173 +++--- modules/linebourse/api/__init__.py | 0 modules/linebourse/api/pages.py | 106 ++++ modules/linebourse/browser.py | 71 ++- modules/linebourse/pages.py | 2 +- modules/marmiton/browser.py | 2 +- modules/marmiton/pages.py | 2 +- modules/materielnet/browser.py | 29 +- modules/materielnet/module.py | 6 +- modules/materielnet/pages.py | 84 +-- modules/myhabit/browser.py | 27 +- modules/myhabit/module.py | 3 +- modules/n26/browser.py | 2 +- .../nalo/compat/weboob_capabilities_bank.py | 5 + modules/orange/browser.py | 16 +- modules/paypal/browser.py | 1 + modules/peertube/__init__.py | 26 + modules/peertube/browser.py | 78 +++ modules/peertube/module.py | 65 +++ modules/phpbb/browser.py | 11 +- modules/phpbb/module.py | 13 +- modules/phpbb/pages/forum.py | 2 +- modules/piratebay/test.py | 13 +- .../compat/weboob_capabilities_bank.py | 5 + modules/rmll/pages.py | 12 +- .../s2e/compat/weboob_capabilities_bank.py | 5 + modules/societegenerale/browser.py | 48 +- .../compat/weboob_capabilities_bank.py | 5 + .../compat/weboob_capabilities_base.py | 22 + ...oob_tools_capabilities_bank_investments.py | 86 +++ modules/societegenerale/module.py | 18 +- .../societegenerale/pages/accounts_list.py | 28 +- .../pages/compat/weboob_capabilities_bank.py | 45 ++ modules/societegenerale/pages/transfer.py | 10 +- modules/societegenerale/sgpe/browser.py | 23 +- .../societegenerale/sgpe/compat/__init__.py | 0 .../sgpe/compat/weboob_capabilities_bank.py | 45 ++ modules/societegenerale/sgpe/json_pages.py | 34 +- modules/societegenerale/sgpe/pages.py | 29 +- .../compat/weboob_capabilities_bank.py | 5 + modules/sueurdemetal/browser.py | 5 +- modules/sueurdemetal/module.py | 3 +- modules/suravenir/browser.py | 10 +- .../compat/weboob_capabilities_bank.py | 5 + modules/vicsec/browser.py | 30 +- modules/vicsec/module.py | 3 +- modules/vicseccard/browser.py | 37 +- modules/vicseccard/module.py | 3 +- modules/vimeo/pages.py | 1 + modules/vlille/browser.py | 2 +- modules/wellsfargo/browser.py | 40 +- modules/wellsfargo/pages.py | 41 +- modules/wellsfargo/parsers.py | 11 +- .../yomoni/compat/weboob_capabilities_bank.py | 5 + 189 files changed, 4823 insertions(+), 1431 deletions(-) create mode 100644 modules/axabanque/compat/weboob_tools_capabilities_bank_investments.py create mode 100644 modules/axabanque/pages/compat/weboob_capabilities_bank.py create mode 100644 modules/banquepopulaire/compat/weboob_tools_capabilities_bank_investments.py create mode 100644 modules/bforbank/compat/weboob_tools_capabilities_bank_investments.py create mode 100644 modules/bnporc/pp/compat/weboob_capabilities_bank.py create mode 100644 modules/boursorama/compat/weboob_tools_capabilities_bank_investments.py create mode 100644 modules/bp/pages/compat/__init__.py create mode 100644 modules/bp/pages/compat/weboob_capabilities_bank.py create mode 100644 modules/caels/pages.py create mode 100644 modules/caissedepargne/compat/weboob_tools_capabilities_bank_investments.py create mode 100644 modules/cragr/web/compat/weboob_capabilities_bank.py create mode 100644 modules/cragr/web/compat/weboob_tools_capabilities_bank_investments.py create mode 100644 modules/creditcooperatif/linebourse_browser.py create mode 100644 modules/creditmutuel/compat/weboob_tools_capabilities_bank_investments.py create mode 100644 modules/fortuneo/pages/compat/__init__.py create mode 100644 modules/fortuneo/pages/compat/weboob_capabilities_bank.py create mode 100644 modules/fortuneo/pages/compat/weboob_tools_capabilities_bank_investments.py create mode 100644 modules/ing/pages/compat/__init__.py create mode 100644 modules/ing/pages/compat/weboob_tools_capabilities_bank_investments.py create mode 100644 modules/lcl/compat/weboob_capabilities_bill.py create mode 100644 modules/lcl/compat/weboob_tools_capabilities_bank_investments.py create mode 100644 modules/linebourse/api/__init__.py create mode 100644 modules/linebourse/api/pages.py create mode 100644 modules/peertube/__init__.py create mode 100644 modules/peertube/browser.py create mode 100644 modules/peertube/module.py create mode 100644 modules/societegenerale/compat/weboob_capabilities_base.py create mode 100644 modules/societegenerale/compat/weboob_tools_capabilities_bank_investments.py create mode 100644 modules/societegenerale/pages/compat/weboob_capabilities_bank.py create mode 100644 modules/societegenerale/sgpe/compat/__init__.py create mode 100644 modules/societegenerale/sgpe/compat/weboob_capabilities_bank.py diff --git a/modules/afer/compat/weboob_capabilities_bank.py b/modules/afer/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/afer/compat/weboob_capabilities_bank.py +++ b/modules/afer/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/afer/pages.py b/modules/afer/pages.py index bdfefc3e24..86e5f28714 100644 --- a/modules/afer/pages.py +++ b/modules/afer/pages.py @@ -28,7 +28,7 @@ from weboob.browser.pages import HTMLPage, LoggedPage, pagination from weboob.capabilities.bank import Account, Investment, Transaction from weboob.capabilities.base import NotAvailable -from weboob.exceptions import BrowserUnavailable +from weboob.exceptions import BrowserUnavailable, ActionNeeded class LoginPage(HTMLPage): @@ -51,6 +51,10 @@ class IndexPage(LoggedPage, HTMLPage): def on_load(self): HTMLPage.on_load(self) + msg = CleanText('//div[has-class("form-input-label")]', default='')(self.doc) + if "prendre connaissance des nouvelles conditions" in msg: + raise ActionNeeded(msg) + # website sometime crash if self.doc.xpath(u'//div[@id="divError"]/span[contains(text(),"Une erreur est survenue")]'): raise BrowserUnavailable() diff --git a/modules/allocine/browser.py b/modules/allocine/browser.py index 70ccd6e75c..47895acb0d 100644 --- a/modules/allocine/browser.py +++ b/modules/allocine/browser.py @@ -17,19 +17,20 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . -from weboob.capabilities.calendar import BaseCalendarEvent, TRANSP, STATUS, CATEGORIES -from weboob.capabilities.collection import Collection -from weboob.capabilities.video import BaseVideo -from weboob.capabilities.image import Thumbnail -from weboob.capabilities.base import NotAvailable, NotLoaded, find_object -from weboob.capabilities.cinema import Movie, Person -from weboob.browser.browsers import APIBrowser -from weboob.browser.profiles import Android -from weboob.tools.compat import urlencode import base64 import hashlib -from datetime import datetime, date, timedelta import time +from datetime import date, datetime, timedelta + +from weboob.browser.browsers import APIBrowser +from weboob.browser.profiles import Android +from weboob.capabilities.base import NotAvailable, NotLoaded, find_object +from weboob.capabilities.calendar import CATEGORIES, STATUS, TRANSP, BaseCalendarEvent +from weboob.capabilities.cinema import Movie, Person +from weboob.capabilities.collection import Collection +from weboob.capabilities.image import Thumbnail +from weboob.capabilities.video import BaseVideo +from weboob.tools.compat import unicode, urlencode __all__ = ['AllocineBrowser'] diff --git a/modules/allocine/module.py b/modules/allocine/module.py index 58fd19309a..de5618c46b 100644 --- a/modules/allocine/module.py +++ b/modules/allocine/module.py @@ -19,11 +19,12 @@ import re from weboob.capabilities.base import UserError -from weboob.capabilities.calendar import CapCalendarEvent, CATEGORIES, BaseCalendarEvent -from weboob.capabilities.video import CapVideo, BaseVideo -from weboob.capabilities.collection import CapCollection, CollectionNotFound, Collection -from weboob.capabilities.cinema import CapCinema, Person, Movie +from weboob.capabilities.calendar import CATEGORIES, BaseCalendarEvent, CapCalendarEvent +from weboob.capabilities.cinema import CapCinema, Movie, Person +from weboob.capabilities.collection import CapCollection, Collection, CollectionNotFound +from weboob.capabilities.video import BaseVideo, CapVideo from weboob.tools.backend import Module +from weboob.tools.compat import unicode from .browser import AllocineBrowser diff --git a/modules/amazon/browser.py b/modules/amazon/browser.py index 8499396445..574930f2a2 100644 --- a/modules/amazon/browser.py +++ b/modules/amazon/browser.py @@ -41,7 +41,7 @@ class AmazonBrowser(LoginBrowser, StatesMixin): L_SUBSCRIBER = 'Nom : (.*) Modifier E-mail' login = URL(r'/ap/signin(.*)', LoginPage) - home = URL(r'/$', HomePage) + home = URL(r'/$', r'/\?language=\w+$', HomePage) panel = URL('/gp/css/homepage.html/ref=nav_youraccount_ya', PanelPage) subscriptions = URL(r'/ap/cnep(.*)', SubscriptionsPage) documents = URL(r'/gp/your-account/order-history\?opt=ab&digitalOrders=1(.*)&orderFilter=year-(?P.*)', @@ -170,7 +170,11 @@ def to_english(self, language): def iter_subscription(self): self.location(self.panel.go().get_sub_link()) - if not self.subscriptions.is_here(): + if self.home.is_here(): + if self.page.get_login_link(): + self.is_login() + self.location(self.page.get_panel_link()) + elif not self.subscriptions.is_here(): self.is_login() yield self.page.get_item() diff --git a/modules/amazon/pages.py b/modules/amazon/pages.py index 7455d59d27..47129b54d8 100644 --- a/modules/amazon/pages.py +++ b/modules/amazon/pages.py @@ -35,6 +35,9 @@ class HomePage(HTMLPage): def get_login_link(self): return self.doc.xpath('//a[./span[contains(., "%s")]]/@href' % self.browser.L_SIGNIN)[0] + def get_panel_link(self): + return Link('//a[contains(@href, "homepage.html") and has-class(@nav-link)]')(self.doc) + class PanelPage(LoggedPage, HTMLPage): def get_sub_link(self): @@ -125,7 +128,7 @@ class item(ItemElement): obj_id = Format('%s_%s', Env('subid'), Field('_simple_id')) obj__pre_url = Format('/gp/shared-cs/ajax/invoice/invoice.html?orderId=%s&relatedRequestId=%s&isADriveSubscription=&isHFC=', Field('_simple_id'), Env('request_id')) - obj_url = Async('details') & Link('//a[contains(@href, "download")]') + obj_url = Async('details') & Link('//a[contains(@href, "download")]|//a[contains(@href, "generated_invoices")]') obj_format = 'pdf' obj_label = Format('Facture %s', Field('_simple_id')) obj_type = 'bill' diff --git a/modules/amazonstorecard/browser.py b/modules/amazonstorecard/browser.py index 3367e87e1f..620242e58e 100644 --- a/modules/amazonstorecard/browser.py +++ b/modules/amazonstorecard/browser.py @@ -18,18 +18,17 @@ # along with weboob. If not, see . -from weboob.capabilities.bank import AccountNotFound -from weboob.browser import LoginBrowser, URL, need_login -from weboob.exceptions import BrowserIncorrectPassword -from weboob.tools.compat import unquote import json import os +from subprocess import STDOUT, CalledProcessError, check_output from tempfile import mkstemp -from subprocess import check_output, STDOUT, CalledProcessError -from .pages import SomePage, StatementsPage, StatementPage, SummaryPage, \ - ActivityPage +from weboob.browser import URL, LoginBrowser, need_login +from weboob.capabilities.bank import AccountNotFound +from weboob.exceptions import BrowserIncorrectPassword +from weboob.tools.compat import unquote +from .pages import ActivityPage, SomePage, StatementPage, StatementsPage, SummaryPage __all__ = ['AmazonStoreCard'] @@ -66,24 +65,24 @@ def do_login(self): 'agent': self.session.headers['User-Agent']}) os.close(scrf) os.close(cookf) - for i in xrange(self.MAX_RETRIES): + for i in range(self.MAX_RETRIES): try: check_output(["phantomjs", scrn], stderr=STDOUT) break except CalledProcessError as error: - pass + last_error = error else: - raise error + raise last_error with open(cookn) as cookf: cookies = json.loads(cookf.read()) os.remove(scrn) os.remove(cookn) self.session.cookies.clear() for c in cookies: - for k in ['expiry', 'expires', 'httponly']: - c.pop(k, None) - c['value'] = unquote(c['value']) - self.session.cookies.set(**c) + for k in ['expiry', 'expires', 'httponly']: + c.pop(k, None) + c['value'] = unquote(c['value']) + self.session.cookies.set(**c) if not self.summary.go().logged: raise BrowserIncorrectPassword() @@ -106,6 +105,7 @@ def iter_history(self, account): for t in s.iter_transactions(): yield t + LOGIN_JS = u'''\ var TIMEOUT = %(timeout)s*1000; // milliseconds var page = require('webpage').create(); diff --git a/modules/amazonstorecard/module.py b/modules/amazonstorecard/module.py index 01187471e3..6794daf4c3 100644 --- a/modules/amazonstorecard/module.py +++ b/modules/amazonstorecard/module.py @@ -19,12 +19,11 @@ from weboob.capabilities.bank import CapBank -from weboob.tools.backend import Module, BackendConfig +from weboob.tools.backend import BackendConfig, Module from weboob.tools.value import ValueBackendPassword from .browser import AmazonStoreCard - __all__ = ['AmazonStoreCardModule'] @@ -39,16 +38,16 @@ class AmazonStoreCardModule(Module, CapBank): ValueBackendPassword('username', label='User ID', masked=False), ValueBackendPassword('password', label='Password'), ValueBackendPassword('phone', - label='Phone to send verification code to', masked=False), + label='Phone to send verification code to', masked=False), ValueBackendPassword('code_file', - label='File to read the verification code from', masked=False)) + label='File to read the verification code from', masked=False)) BROWSER = AmazonStoreCard def create_default_browser(self): - return self.create_browser(username = self.config['username'].get(), - password = self.config['password'].get(), - phone = self.config['phone'].get(), - code_file = self.config['code_file'].get()) + return self.create_browser(username=self.config['username'].get(), + password=self.config['password'].get(), + phone=self.config['phone'].get(), + code_file=self.config['code_file'].get()) def iter_accounts(self): return self.browser.iter_accounts() diff --git a/modules/amazonstorecard/pages.py b/modules/amazonstorecard/pages.py index 3c38726b43..17f9aa792e 100644 --- a/modules/amazonstorecard/pages.py +++ b/modules/amazonstorecard/pages.py @@ -26,10 +26,18 @@ from weboob.tools.pdf import decompress_pdf from weboob.tools.tokenizer import ReTokenizer from datetime import datetime, timedelta +from weboob.tools.compat import unicode import re import json +try: + cmp = cmp +except NameError: + def cmp(x, y): + return (x > y) - (x < y) + + class SomePage(HTMLPage): @property def logged(self): @@ -39,7 +47,7 @@ def logged(self): class SummaryPage(SomePage): def account(self): label = u' '.join(self.doc.xpath( - '//div[contains(@class,"myCreditCardDetails")]')[0]\ + '//div[contains(@class,"myCreditCardDetails")]')[0] .text_content().split()) balance = self.amount(u'Balance') cardlimit = self.doc.xpath( @@ -69,8 +77,8 @@ def account(self): def amount(self, name): return u''.join(self.doc.xpath( - u'//li[text()[.="%s"]]/../li[1]'%name)[0].text_content().split())\ - .replace(u'\xb7',u'.').replace(u'*',u'') + u'//li[text()[.="%s"]]/../li[1]' % name)[0].text_content().split())\ + .replace(u'\xb7', u'.').replace(u'*', u'') class ActivityPage(SomePage): @@ -98,19 +106,20 @@ def cmp_records(rec1, rec2): def parse_date(recdate): return datetime.strptime(recdate, u'%B %d, %Y') + class StatementsPage(SomePage): def iter_statements(self): jss = self.doc.xpath(u'//a/@onclick[contains(.,"eBillViewPDFAction")]') for js in jss: url = re.match("window.open\('([^']*).*\)", js).group(1) - for i in xrange(self.browser.MAX_RETRIES): + for i in range(self.browser.MAX_RETRIES): try: self.browser.location(url) break except ServerError as e: - pass + last_error = e else: - raise e + raise last_error yield self.browser.page @@ -163,7 +172,7 @@ def read_transaction(self, pos, date_from, date_to): pos, amount_layout = self.read_layout_td(pos) pos, amount = self.read_amount(pos) if tdate is None or pdate is None \ - or desc is None or amount is None or amount == 0: + or desc is None or amount is None or amount == 0: return startPos, None else: tdate = closest_date(tdate, date_from, date_to) @@ -208,7 +217,7 @@ def read_closing_date(self): def read_text(self, pos): t = self._tok.tok(pos) - #TODO: handle PDF encodings properly. + # TODO: handle PDF encodings properly. return (pos+1, unicode(t.value(), errors='ignore')) \ if t.is_text() else (pos, None) diff --git a/modules/ameli/pages.py b/modules/ameli/pages.py index 1118dbda27..ccba3c4550 100644 --- a/modules/ameli/pages.py +++ b/modules/ameli/pages.py @@ -85,7 +85,7 @@ class AccountPage(AmeliBasePage): def iter_subscription_list(self): names_list = self.doc.xpath('//span[@class="NomEtPrenomLabel"]') fullname = CleanText(newlines=True).filter(names_list[0]) - number = re.sub(r'[^\d]+', '', CleanText('//span[@class="blocNumSecu"]', replace=[(' ','')])(self.doc)) + number = re.sub(r'[^\d]+', '', CleanText('//span[@class="blocNumSecu"]', replace=[(' ', '')])(self.doc)) sub = Subscription(number) sub._id = number sub.label = fullname @@ -118,7 +118,7 @@ def iter_documents(self, sub): try: elt.xpath('.//a[contains(@id,"lienPDFReleve")]')[0] except IndexError: - continue + continue date_str = elt.xpath('.//span[contains(@id,"moisEnCours")]')[0].text month_str = date_str.split()[0] date = datetime.strptime(re.sub(month_str, str(FRENCH_MONTHS.index(month_str) + 1), date_str), "%m %Y").date() diff --git a/modules/amundi/browser.py b/modules/amundi/browser.py index 222cb8d31d..a10dac38f4 100644 --- a/modules/amundi/browser.py +++ b/modules/amundi/browser.py @@ -17,8 +17,6 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . - -import ssl from .pages import LoginPage, AccountsPage, AccountHistoryPage from weboob.browser import URL, LoginBrowser, need_login from weboob.tools.json import json @@ -29,35 +27,18 @@ class AmundiBrowser(LoginBrowser): TIMEOUT = 120.0 - login = URL('/psf/authenticate', LoginPage) - authorize = URL('/psf/authorize', LoginPage) - accounts = URL('/psf/api/individu/positionFonds\?flagUrlFicheFonds=true&inclurePositionVide=false', AccountsPage) - account_history = URL('/psf/api/individu/operations\?valeurExterne=false&filtreStatutModeExclusion=false&statut=CPTA', AccountHistoryPage) - - def __init__(self, website, *args, **kwargs): - self.BASEURL = website - - super(AmundiBrowser, self).__init__(*args, **kwargs) - - def prepare_request(self, req): - """ - Amundi uses TLS v1.0. - """ - preq = super(AmundiBrowser, self).prepare_request(req) - conn = self.session.adapters['https://'].get_connection(preq.url) - conn.ssl_version = ssl.PROTOCOL_TLSv1 - return preq + login = URL(r'authenticate', LoginPage) + authorize = URL(r'authorize', LoginPage) + accounts = URL(r'api/individu/positionFonds\?flagUrlFicheFonds=true&inclurePositionVide=false', AccountsPage) + account_history = URL(r'api/individu/operations\?valeurExterne=false&filtreStatutModeExclusion=false&statut=CPTA', AccountHistoryPage) def do_login(self): """ Attempt to log in. Note: this method does nothing if we are already logged in. """ - assert isinstance(self.username, basestring) - assert isinstance(self.password, basestring) - try: - self.login.go(data=json.dumps({'username' : self.username, 'password' : self.password}), \ + self.login.go(data=json.dumps({'username': self.username, 'password': self.password}), headers={'Content-Type': 'application/json;charset=UTF-8'}) self.token = self.authorize.go().get_token() except ClientError: @@ -65,13 +46,22 @@ def do_login(self): @need_login def iter_accounts(self): - return self.accounts.go(headers={'X-noee-authorization': ('noeprd %s' % self.token)}).iter_accounts() + return (self.accounts.go(headers={'X-noee-authorization': ('noeprd %s' % self.token)}) + .iter_accounts()) @need_login def iter_investments(self, account): - return self.accounts.go(headers={'X-noee-authorization': ('noeprd %s' % self.token)})\ - .iter_investments(account_id=account.id) + return (self.accounts.go(headers={'X-noee-authorization': ('noeprd %s' % self.token)}) + .iter_investments(account_id=account.id)) @need_login def iter_history(self, account): - return self.account_history.go(headers={'X-noee-authorization': ('noeprd %s' % self.token)}).iter_history(account=account) + return (self.account_history.go(headers={'X-noee-authorization': ('noeprd %s' % self.token)}) + .iter_history(account=account)) + + +class EEAmundi(AmundiBrowser): + BASEURL = 'https://www.amundi-ee.com/psf/' + +class TCAmundi(AmundiBrowser): + BASEURL = 'https://epargnants.amundi-tc.com/psf/' diff --git a/modules/amundi/compat/weboob_capabilities_bank.py b/modules/amundi/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/amundi/compat/weboob_capabilities_bank.py +++ b/modules/amundi/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/amundi/module.py b/modules/amundi/module.py index d7e5666ac5..e4143c9a99 100644 --- a/modules/amundi/module.py +++ b/modules/amundi/module.py @@ -23,7 +23,7 @@ from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword, Value -from .browser import AmundiBrowser +from .browser import EEAmundi, TCAmundi __all__ = ['AmundiModule'] @@ -35,59 +35,27 @@ class AmundiModule(Module, CapBankWealth): EMAIL = 'james.galt.bi@gmail.com' LICENSE = 'AGPLv3+' VERSION = '1.3' - CONFIG = BackendConfig(ValueBackendPassword('login', label='Identifiant', regexp='\d+', masked=False), - ValueBackendPassword('password', label=u"Mot de passe", regexp='\d+'), + CONFIG = BackendConfig(ValueBackendPassword('login', label='Identifiant', regexp=r'\d+', masked=False), + ValueBackendPassword('password', label=u"Mot de passe", regexp=r'\d+'), Value('website', label='Type de compte', default='ee', - choices={'ee': 'Amundi Epargne Entreprise', - 'tc': 'Amundi Tenue de Compte'})) - - BROWSER = AmundiBrowser + choices={'ee': 'Amundi Epargne Entreprise', + 'tc': 'Amundi Tenue de Compte'})) def create_default_browser(self): - w = {'ee': 'https://www.amundi-ee.com', 'tc': 'https://epargnants.amundi-tc.com'} - return self.create_browser(w[self.config['website'].get()], self.config['login'].get(), self.config['password'].get()) + b = {'ee': EEAmundi, 'tc': TCAmundi} + self.BROWSER = b[self.config['website'].get()] + return self.create_browser(self.config['login'].get(), self.config['password'].get()) def get_account(self, id): - """ - Get an account from its ID. - - :param id: ID of the account - :type id: :class:`str` - :rtype: :class:`Account` - :raises: :class:`AccountNotFound` - """ return find_object(self.iter_accounts(), id=id, error=AccountNotFound) - def iter_accounts(self): - """ - Iter accounts. - - :rtype: iter[:class:`Account`] - """ return self.browser.iter_accounts() - def iter_investment(self, account): - """ - Iter investment of a market account - - :param account: account to get investments - :type account: :class:`Account` - :rtype: iter[:class:`Investment`] - :raises: :class:`AccountNotFound` - """ for inv in self.browser.iter_investments(account): if inv.valuation != 0: yield inv def iter_history(self, account): - """ - Iter history of transactions on a specific account. - - :param account: account to get history - :type account: :class:`Account` - :rtype: iter[:class:`Transaction`] - :raises: :class:`AccountNotFound` - """ return self.browser.iter_history(account) diff --git a/modules/amundi/pages.py b/modules/amundi/pages.py index d28dec1d27..2424af2e79 100644 --- a/modules/amundi/pages.py +++ b/modules/amundi/pages.py @@ -17,6 +17,8 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals + from datetime import datetime from weboob.browser.elements import ItemElement, method, DictElement @@ -26,6 +28,7 @@ from weboob.capabilities.bank import Account, Investment, Transaction from weboob.capabilities.base import NotAvailable from weboob.exceptions import NoAccountsException +from weboob.tools.capabilities.bank.investments import is_isin_valid class LoginPage(JsonPage): @@ -59,7 +62,7 @@ def obj_number(self): # just the id is a kind of company id so it can be unique on a backend but not unique on multiple backends return '%s_%s' % (Field('id')(self), self.page.browser.username) - obj_currency = u"EUR" + obj_currency = 'EUR' def obj_type(self): return self.page.ACCOUNT_TYPES.get(Dict('typeDispositif')(self), Account.TYPE_LIFE_INSURANCE) @@ -67,7 +70,7 @@ def obj_type(self): def obj_label(self): try: return Dict('libelleDispositif')(self).encode('iso-8859-2').decode('utf8') - except (UnicodeEncodeError, UnicodeDecodeError): + except UnicodeError: try: return Dict('libelleDispositif')(self).encode('latin1').decode('utf8') except UnicodeDecodeError: @@ -92,12 +95,17 @@ class item(ItemElement): obj_code = Dict('codeIsin', default=NotAvailable) obj_vdate = Date(Dict('dtVl')) + def obj_code_type(self): + if is_isin_valid(Field('code')(self)): + return Investment.CODE_TYPE_ISIN + return NotAvailable + class AccountHistoryPage(LoggedPage, JsonPage): def belongs(self, instructions, account): for ins in instructions: - if 'nomDispositif' in ins and 'codeDispositif' in ins and '%s%s' % (ins['nomDispositif'], ins['codeDispositif']) == \ - '%s%s' % (account.label, account.id): + if 'nomDispositif' in ins and 'codeDispositif' in ins and '%s%s' % ( + ins['nomDispositif'], ins['codeDispositif']) == '%s%s' % (account.label, account.id): return True return False @@ -105,8 +113,9 @@ def get_amount(self, instructions, account): amount = 0 for ins in instructions: - if 'nomDispositif' in ins and 'montantNet' in ins and 'codeDispositif' in ins and '%s%s' % (ins['nomDispositif'], ins['codeDispositif']) == \ - '%s%s' % (account.label, account.id): + if ('nomDispositif' in ins and 'montantNet' in ins and 'codeDispositif' in ins + and '%s%s' % (ins['nomDispositif'], ins['codeDispositif']) + == '%s%s' % (account.label, account.id)): amount += ins['montantNet'] return CleanDecimal().filter(amount) @@ -119,21 +128,7 @@ def iter_history(self, account): tr.amount = self.get_amount(hist['instructions'], account) tr.rdate = datetime.strptime(hist['dateComptabilisation'].split('T')[0], '%Y-%m-%d') tr.date = tr.rdate - tr.label = hist['libelleOperation'] if 'libelleOperation' in hist else hist['libelleCommunication'] + tr.label = hist.get('libelleOperation') or hist['libelleCommunication'] tr.type = Transaction.TYPE_UNKNOWN - # Bypassed because we don't have the ISIN code - # tr.investments = [] - # for ins in hist['instructions']: - # inv = Investment() - # inv.code = NotAvailable - # inv.label = ins['nomFonds'] - # inv.description = ' '.join([ins['type'], ins['nomDispositif']]) - # inv.vdate = datetime.strptime(ins.get('dateVlReel', ins.get('dateVlExecution')).split('T')[ - # 0], '%Y-%m-%d') - # inv.valuation = Decimal(ins['montantNet']) - # inv.quantity = Decimal(ins['nombreDeParts']) - # inv.unitprice = inv.unitvalue = Decimal(ins['vlReel']) - # tr.investments.append(inv) - yield tr diff --git a/modules/apivie/compat/weboob_capabilities_bank.py b/modules/apivie/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/apivie/compat/weboob_capabilities_bank.py +++ b/modules/apivie/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/axabanque/browser.py b/modules/axabanque/browser.py index 6b3a44b427..995960f41d 100644 --- a/modules/axabanque/browser.py +++ b/modules/axabanque/browser.py @@ -24,7 +24,7 @@ from dateutil.relativedelta import relativedelta import re -from weboob.browser import LoginBrowser, URL, need_login, StatesMixin +from weboob.browser.browsers import LoginBrowser, URL, need_login, StatesMixin from weboob.browser.exceptions import ClientError, HTTPNotFound from weboob.capabilities.base import NotAvailable from weboob.capabilities.bill import Subscription @@ -32,6 +32,7 @@ from weboob.exceptions import BrowserIncorrectPassword, ActionNeeded from weboob.tools.value import Value from weboob.tools.capabilities.bank.transactions import sorted_transactions +from .compat.weboob_tools_capabilities_bank_investments import create_french_liquidity from .pages.login import ( KeyboardPage, LoginPage, ChangepasswordPage, PredisconnectedPage, DeniedPage, @@ -39,9 +40,9 @@ ) from .pages.bank import ( AccountsPage as BankAccountsPage, CBTransactionsPage, TransactionsPage, - UnavailablePage, IbanPage, LifeInsuranceIframe, BoursePage, + UnavailablePage, IbanPage, LifeInsuranceIframe, BoursePage, BankProfilePage, ) -from .pages.wealth import AccountsPage as WealthAccountsPage, InvestmentPage, HistoryPage +from .pages.wealth import AccountsPage as WealthAccountsPage, InvestmentPage, HistoryPage, ProfilePage from .pages.transfer import ( RecipientsPage, AddRecipientPage, ValidateTransferPage, RegisterTransferPage, ConfirmTransferPage, RecipientConfirmationPage, @@ -135,6 +136,7 @@ class AXABanque(AXABrowser, StatesMixin): 'webapp/axabanque/jsp/virementSepa/saisieVirementSepa.faces', RegisterTransferPage) confirm_transfer = URL('/webapp/axabanque/jsp/virementSepa/confirmationVirementSepa.faces', ConfirmTransferPage) + profile_page = URL('/transactionnel/client/coordonnees.html', BankProfilePage) reload_state = None @@ -174,8 +176,8 @@ def iter_accounts(self): ids.add(a.id) - #The url giving life insurrance investments seems to be temporary. - #That's why we have to get them now + # The url giving life insurrance investments seems to be temporary. + # That's why we have to get them now if a.type == a.TYPE_LIFE_INSURANCE: self.cache['invs'][a.id] = list(self.open(a._url).page.iter_investment()) args = a._args @@ -186,7 +188,7 @@ def iter_accounts(self): 'codeFamille': args['paramCodeFamille'], 'codeProduit': args['paramCodeProduit'], 'codeSousProduit': args['paramCodeSousProduit'] - } + } try: r = self.open('/webapp/axabanque/popupPDF', params=iban_params) a.iban = r.page.get_iban() @@ -256,7 +258,7 @@ def iter_investment(self, account): self.transactions.go() if account._acctype == 'bank' and account.type in (Account.TYPE_PEA, Account.TYPE_MARKET): if 'Liquidités' in account.label: - return [self.page.get_liquidity_investment(account)] + return [create_french_liquidity(account.balance)] account = self.get_netfinca_account(account) self.location(account._market_link) @@ -290,7 +292,7 @@ def iter_history(self, account): self.location(acc._market_link) self.bourse_history.go() - if not 'Liquidités' in account.label: + if 'Liquidités' not in account.label: self.page.go_history_filter(cash_filter="market") else: self.page.go_history_filter(cash_filter="liquidity") @@ -488,6 +490,11 @@ def iter_documents(self, subscription): def download_document(self, url): raise NotImplementedError() + @need_login + def get_profile(self): + self.profile_page.go() + return self.page.get_profile() + class AXAAssurance(AXABrowser): BASEURL = 'https://espaceclient.axa.fr' @@ -500,6 +507,7 @@ class AXAAssurance(AXABrowser): download = URL('/content/ecc-popin-cards/technical/detailed/document.downloadPdf.html', '/content/ecc-popin-cards/technical/detailed/document/_jcr_content/', DownloadPage) + profile = URL(r'/content/ecc-popin-cards/transverse/userprofile.content-inner.html\?_=\d+', ProfilePage) def __init__(self, *args, **kwargs): super(AXAAssurance, self).__init__(*args, **kwargs) @@ -578,3 +586,8 @@ def download_document(self, url): self.location(url) self.page.create_document() return self.page.content + + @need_login + def get_profile(self): + self.profile.go() + return self.page.get_profile() diff --git a/modules/axabanque/compat/weboob_capabilities_bank.py b/modules/axabanque/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/axabanque/compat/weboob_capabilities_bank.py +++ b/modules/axabanque/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/axabanque/compat/weboob_tools_capabilities_bank_investments.py b/modules/axabanque/compat/weboob_tools_capabilities_bank_investments.py new file mode 100644 index 0000000000..2b68ff9e00 --- /dev/null +++ b/modules/axabanque/compat/weboob_tools_capabilities_bank_investments.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Jonathan Schmidt +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.tools.compat import basestring +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.bank import Investment + +def is_isin_valid(isin): + """ + Méthode générale + Table de conversion des lettres en chiffres + A=10 B=11 C=12 D=13 E=14 F=15 G=16 H=17 I=18 + J=19 K=20 L=21 M=22 N=23 O=24 P=25 Q=26 R=27 + S=28 T=29 U=30 V=31 W=32 X=33 Y=34 Z=35 + + 1 - Mettre de côté la clé, qui servira de référence à la fin de la vérification. + 2 - Convertir toutes les lettres en nombres via la table de conversion ci-contre. Si le nombre obtenu est supérieur ou égal à 10, prendre les deux chiffres du nombre séparément (exemple : 27 devient 2 et 7). + 3 - Pour chaque chiffre, multiplier sa valeur par deux si sa position est impaire en partant de la droite. Si le nombre obtenu est supérieur ou égal à 10, garder les deux chiffres du nombre séparément (exemple : 14 devient 1 et 4). + 4 - Faire la somme de tous les chiffres. + 5 - Soustraire cette somme de la dizaine supérieure ou égale la plus proche (exemples : si la somme vaut 22, la dizaine « supérieure ou égale » est 30, et la clé vaut donc 8 ; si la somme vaut 30, la dizaine « supérieure ou égale » est 30, et la clé vaut 0 ; si la somme vaut 31, la dizaine « supérieure ou égale » est 40, et la clé vaut 9). + 6 - Comparer la valeur obtenue à la clé mise initialement de côté. + + Étapes 1 et 2 : + F R 0 0 0 3 5 0 0 0 0 (+ 8 : clé) + 15 27 0 0 0 3 5 0 0 0 0 + + Étape 3 : le traitement se fait sur des chiffres + 1 5 2 7 0 0 0 3 5 0 0 0 0 + I P I P I P I P I P I P I : position en partant de la droite (P = Pair, I = Impair) + 2 1 2 1 2 1 2 1 2 1 2 1 2 : coefficient multiplicateur + + 2 5 4 7 0 0 0 3 10 0 0 0 0 : résultat + + Étape 4 : + 2 + 5 + 4 + 7 + 0 + 0 + 0 + 3 + (1 + 0)+ 0 + 0 + 0 + 0 = 22 + + Étapes 5 et 6 : 30 - 22 = 8 (valeur de la clé) + """ + + if not isinstance(isin, basestring): + return False + if not re.match(r'^[A-Z]{2}[A-Z0-9]{9}\d$', isin): + return False + + isin_in_digits = ''.join(str(ord(x) - ord('A') + 10) if not x.isdigit() else x for x in isin[:-1]) + key = isin[-1:] + result = '' + for k, val in enumerate(isin_in_digits[::-1], start=1): + if k % 2 == 0: + result = ''.join((result, val)) + else: + result = ''.join((result, str(int(val)*2))) + return str(sum(int(x) for x in result) + int(key))[-1] == '0' + + +def create_french_liquidity(valuation): + """ + Automatically fills a liquidity investment with label, code and code_type. + """ + liquidity = Investment() + liquidity.label = "Liquidités" + liquidity.code = "XX-liquidity" + liquidity.code_type = NotAvailable + liquidity.valuation = valuation + return liquidity + diff --git a/modules/axabanque/module.py b/modules/axabanque/module.py index b2d6dc682a..c7fe6418b3 100644 --- a/modules/axabanque/module.py +++ b/modules/axabanque/module.py @@ -21,6 +21,7 @@ from .compat.weboob_capabilities_bank import CapBankWealth, CapBankTransferAddRecipient, AccountNotFound, RecipientNotFound from weboob.capabilities.base import find_object, NotAvailable from weboob.capabilities.bank import Account, TransferInvalidLabel +from weboob.capabilities.profile import CapProfile from weboob.capabilities.bill import CapDocument, Subscription, Document, DocumentNotFound, SubscriptionNotFound from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword @@ -31,7 +32,7 @@ __all__ = ['AXABanqueModule'] -class AXABanqueModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapDocument): +class AXABanqueModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapDocument, CapProfile): NAME = 'axabanque' MAINTAINER = u'Romain Bignon' EMAIL = 'romain@weboob.org' @@ -81,9 +82,10 @@ def init_transfer(self, transfer, **params): raise TransferInvalidLabel() self.logger.info('Going to do a new transfer') - if transfer.account_iban: - account = find_object(self.iter_accounts(), iban=transfer.account_iban, error=AccountNotFound) - else: + + # origin account iban can be NotAvailable + account = find_object(self.iter_accounts(), iban=transfer.account_iban) + if not account: account = find_object(self.iter_accounts(), id=transfer.account_id, error=AccountNotFound) if transfer.recipient_iban: @@ -106,7 +108,17 @@ def transfer_check_label(self, old, new): def transfer_check_account_id(self, old, new): old = old[:11] - return super(AXABanqueModule, self).transfer_check_label(old, new) + return old == new + + def transfer_check_account_iban(self, old, new): + # Skip origin account iban check and force origin account iban + if new is NotAvailable: + self.logger.warning( + 'Origin account iban check (%s) is not possible because iban is currently not available', + old, + ) + return True + return old == new def iter_subscription(self): return self.browser.get_subscription_list() @@ -139,3 +151,6 @@ def iter_resources(self, objs, split_path): if Subscription in objs: self._restrict_level(split_path) return self.iter_subscription() + + def get_profile(self): + return self.browser.get_profile() diff --git a/modules/axabanque/pages/bank.py b/modules/axabanque/pages/bank.py index 59cc7c20fc..d50a13f550 100644 --- a/modules/axabanque/pages/bank.py +++ b/modules/axabanque/pages/bank.py @@ -29,10 +29,12 @@ from weboob.browser.filters.html import Attr, Link, TableCell from weboob.capabilities.bank import Account, Investment from weboob.capabilities.base import NotAvailable +from weboob.capabilities.profile import Person from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.tools.compat import unicode + def MyDecimal(*args, **kwargs): kwargs.update(replace_dots=True, default=NotAvailable) return CleanDecimal(*args, **kwargs) @@ -324,13 +326,6 @@ def obj_code(self): def condition(self): return CleanText(TableCell('valuation'))(self) - def get_liquidity_investment(self, account): - inv = Investment() - inv.label = u'Liquidités' - inv.code = 'XX-liquidity' - inv.valuation = account.balance - return inv - def more_history(self): link = None for a in self.doc.xpath('.//a'): @@ -568,3 +563,15 @@ def go_history_filter(self, cash_filter='all'): # We can't go above 2 years form['beginDayfilter'] = (datetime.strptime(form['endDayfilter'], '%d/%m/%Y') - timedelta(days=730)).strftime('%d/%m/%Y') form.submit() + + +class BankProfilePage(LoggedPage, HTMLPage): + @method + class get_profile(ItemElement): + klass = Person + + obj_email = CleanText('//form[@id="idCoordonneePersonnelle"]//table//strong[contains(text(), "e-mail")]/parent::td', children=False) + + obj_phone = CleanText('//form[@id="idCoordonneePersonnelle"]//table//strong[contains(text(), "mobile")]/parent::td', children=False) + + obj_address = Regexp(CleanText('//form[@id="idCoordonneePersonnelle"]//table//strong[contains(text(), "adresse fiscale")]/parent::td', children=False), '^(.*?)\/') diff --git a/modules/axabanque/pages/compat/weboob_capabilities_bank.py b/modules/axabanque/pages/compat/weboob_capabilities_bank.py new file mode 100644 index 0000000000..141097d781 --- /dev/null +++ b/modules/axabanque/pages/compat/weboob_capabilities_bank.py @@ -0,0 +1,45 @@ + +import weboob.capabilities.bank as OLD + +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) + + +__all__ = OLD.__all__ + + +class CapBankWealth(CapBank): + pass + + +class CapBankPockets(CapBank): + pass + + +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + +class CapBankTransfer(OLD.CapBankTransfer): + def transfer_check_label(self, old, new): + from unidecode import unidecode + + return unidecode(old) == unidecode(new) + + +class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipient): + pass + + +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + +Account.TYPE_MORTGAGE = 17 +Account.TYPE_CONSUMER_CREDIT = 18 +Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/axabanque/pages/transfer.py b/modules/axabanque/pages/transfer.py index a8f7e75658..a4851c1d40 100644 --- a/modules/axabanque/pages/transfer.py +++ b/modules/axabanque/pages/transfer.py @@ -31,8 +31,8 @@ from weboob.browser.filters.standard import ( CleanText, Date, Regexp, CleanDecimal, Currency, Format, Field, ) -from weboob.capabilities.bank import ( - Recipient, Transfer, TransferBankError, AddRecipientError, RecipientNotFound, +from .compat.weboob_capabilities_bank import ( + Recipient, Transfer, TransferBankError, AddRecipientBankError, RecipientNotFound, ) from .compat.weboob_tools_captcha_virtkeyboard import SimpleVirtualKeyboard from weboob.capabilities.base import find_object, NotAvailable @@ -120,11 +120,11 @@ def on_load(self): ) if self.doc.xpath('//input[@class="erreur_champs"]'): - raise AddRecipientError(message="Le code entré est incorrect.") + raise AddRecipientBankError(message="Le code entré est incorrect.") for error_msg in errors_msg: if error_msg: - raise AddRecipientError(message=error_msg) + raise AddRecipientBankError(message=error_msg) def continue_new_recipient(self): continue_new_recipient_btn_id = CleanText('//input[@class="btn_continuer"]/@id')(self.doc) diff --git a/modules/axabanque/pages/wealth.py b/modules/axabanque/pages/wealth.py index 83f6c41762..8fee376aa2 100644 --- a/modules/axabanque/pages/wealth.py +++ b/modules/axabanque/pages/wealth.py @@ -28,6 +28,7 @@ ) from weboob.browser.filters.html import Attr, Link, TableCell from weboob.capabilities.bank import Account, Investment +from weboob.capabilities.profile import Person from weboob.capabilities.base import NotAvailable, NotLoaded from weboob.tools.capabilities.bank.transactions import FrenchTransaction @@ -214,3 +215,16 @@ def obj_investments(self): for inv in investments: inv.vdate = Field('date')(self) return investments + + +class ProfilePage(LoggedPage, HTMLPage): + def get_profile(self): + form = self.get_form(xpath='//div[@class="popin-card"]') + + profile = Person() + + profile.name = '%s %s' % (form['party.first_name'], form['party.preferred_last_name']) + profile.address = '%s %s %s' % (form['mailing_address.street_line'], form['mailing_address.zip_postal_code'], form['mailing_address.locality']) + profile.email = CleanText('//label[@class="email-editable"]')(self.doc) + profile.phone = CleanText('//div[@class="info-title colorized phone-disabled"]//label', children=False)(self.doc) + return profile diff --git a/modules/banquepopulaire/browser.py b/modules/banquepopulaire/browser.py index ab5374f94d..130eac96c2 100644 --- a/modules/banquepopulaire/browser.py +++ b/modules/banquepopulaire/browser.py @@ -26,9 +26,10 @@ from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable from weboob.browser.exceptions import HTTPNotFound, ServerError -from weboob.browser import LoginBrowser, URL, need_login -from weboob.capabilities.bank import Account, Investment +from weboob.browser.browsers import LoginBrowser, URL, need_login +from weboob.capabilities.bank import Account from weboob.capabilities.base import NotAvailable, find_object +from .compat.weboob_tools_capabilities_bank_investments import create_french_liquidity from .pages import ( LoggedOut, @@ -129,6 +130,7 @@ class BanquePopulaire(LoginBrowser): error_page = URL(r'https://[^/]+/cyber/internet/ContinueTask.do', r'https://[^/]+/_layouts/error.aspx', + r'https://[^/]+/portailinternet/_layouts/Ibp.Cyi.Administration/RedirectPageError.aspx', ErrorPage) unavailable_page = URL(r'https://[^/]+/s3f-web/.*', @@ -139,7 +141,7 @@ class BanquePopulaire(LoginBrowser): redirect_page = URL(r'https://[^/]+/portailinternet/_layouts/Ibp.Cyi.Layouts/RedirectSegment.aspx.*', RedirectPage) home_page = URL(r'https://[^/]+/portailinternet/Catalogue/Segments/.*.aspx(\?vary=(?P.*))?', r'https://[^/]+/portailinternet/Pages/.*.aspx\?vary=(?P.*)', - r'https://[^/]+/portailinternet/Pages/default.aspx', + r'https://[^/]+/portailinternet/Pages/[dD]efault.aspx', r'https://[^/]+/portailinternet/Transactionnel/Pages/CyberIntegrationPage.aspx', r'https://[^/]+/cyber/internet/ShowPortal.do\?token=.*', HomePage) @@ -169,6 +171,11 @@ class BanquePopulaire(LoginBrowser): def __init__(self, website, *args, **kwargs): self.BASEURL = 'https://%s' % website + # this url is required because the creditmaritime abstract uses an other url + if 'cmgo.creditmaritime' in self.BASEURL: + self.redirect_url = 'https://www.icgauth.creditmaritime.groupe.banquepopulaire.fr/dacsrest/api/v1u0/transaction/' + else: + self.redirect_url = 'https://www.icgauth.banquepopulaire.fr/dacsrest/api/v1u0/transaction/' self.token = None self.weboob = kwargs['weboob'] super(BanquePopulaire, self).__init__(*args, **kwargs) @@ -193,7 +200,6 @@ def do_login(self): if self.home_page.is_here(): return self.page.login(self.username, self.password) - if self.login_page.is_here(): raise BrowserIncorrectPassword() @@ -278,10 +284,9 @@ def get_account(self, id): return find_object(self.get_accounts_list(False), id=id) def set_gocardless_transaction_details(self, transaction): - # This method is not called for the moment, in order to prevent crash during get_history() # Setting references for a GoCardless transaction data = self.page.get_params() - data['validationStrategy'] = 'NV' + data['validationStrategy'] = self.page.get_gocardless_strategy_param(transaction) data['dialogActionPerformed'] = 'DETAIL_ECRITURE' attribute_key, attribute_value = self.page.get_transaction_table_id(transaction._ref) data[attribute_key] = attribute_value @@ -343,11 +348,9 @@ def get_history(self, account, coming=False): transaction_list = self.page.get_history(account, coming) for tr in transaction_list: - # Add information about GoCardless: - # This method does not work 100% and potentially causes crash in get_history. - # Therefore, for now we decided not to call it while iterating over transactions. - #if 'GoCardless' in tr.label and tr._has_link: - #self.set_gocardless_transaction_details(tr) + # Add information about GoCardless + if 'GoCardless' in tr.label and tr._has_link: + self.set_gocardless_transaction_details(tr) yield tr next_params = self.page.get_next_params() @@ -413,12 +416,7 @@ def get_investment(self, account): # Add "Liquidities" investment if the account is a "Compte titres PEA": if account.type == Account.TYPE_PEA and account.id.startswith('CPT'): - inv = Investment() - inv.label = 'Liquidités' - inv.code = 'XX-liquidity' - inv.code_type = NotAvailable - inv.valuation = account.balance - self.investments[account.id] = [inv] + self.investments[account.id] = [create_french_liquidity(account.balance)] return self.investments[account.id] if account.id in self.investments.keys() and self.investments[account.id] is False: diff --git a/modules/banquepopulaire/compat/weboob_capabilities_bank.py b/modules/banquepopulaire/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/banquepopulaire/compat/weboob_capabilities_bank.py +++ b/modules/banquepopulaire/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/banquepopulaire/compat/weboob_tools_capabilities_bank_investments.py b/modules/banquepopulaire/compat/weboob_tools_capabilities_bank_investments.py new file mode 100644 index 0000000000..2b68ff9e00 --- /dev/null +++ b/modules/banquepopulaire/compat/weboob_tools_capabilities_bank_investments.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Jonathan Schmidt +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.tools.compat import basestring +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.bank import Investment + +def is_isin_valid(isin): + """ + Méthode générale + Table de conversion des lettres en chiffres + A=10 B=11 C=12 D=13 E=14 F=15 G=16 H=17 I=18 + J=19 K=20 L=21 M=22 N=23 O=24 P=25 Q=26 R=27 + S=28 T=29 U=30 V=31 W=32 X=33 Y=34 Z=35 + + 1 - Mettre de côté la clé, qui servira de référence à la fin de la vérification. + 2 - Convertir toutes les lettres en nombres via la table de conversion ci-contre. Si le nombre obtenu est supérieur ou égal à 10, prendre les deux chiffres du nombre séparément (exemple : 27 devient 2 et 7). + 3 - Pour chaque chiffre, multiplier sa valeur par deux si sa position est impaire en partant de la droite. Si le nombre obtenu est supérieur ou égal à 10, garder les deux chiffres du nombre séparément (exemple : 14 devient 1 et 4). + 4 - Faire la somme de tous les chiffres. + 5 - Soustraire cette somme de la dizaine supérieure ou égale la plus proche (exemples : si la somme vaut 22, la dizaine « supérieure ou égale » est 30, et la clé vaut donc 8 ; si la somme vaut 30, la dizaine « supérieure ou égale » est 30, et la clé vaut 0 ; si la somme vaut 31, la dizaine « supérieure ou égale » est 40, et la clé vaut 9). + 6 - Comparer la valeur obtenue à la clé mise initialement de côté. + + Étapes 1 et 2 : + F R 0 0 0 3 5 0 0 0 0 (+ 8 : clé) + 15 27 0 0 0 3 5 0 0 0 0 + + Étape 3 : le traitement se fait sur des chiffres + 1 5 2 7 0 0 0 3 5 0 0 0 0 + I P I P I P I P I P I P I : position en partant de la droite (P = Pair, I = Impair) + 2 1 2 1 2 1 2 1 2 1 2 1 2 : coefficient multiplicateur + + 2 5 4 7 0 0 0 3 10 0 0 0 0 : résultat + + Étape 4 : + 2 + 5 + 4 + 7 + 0 + 0 + 0 + 3 + (1 + 0)+ 0 + 0 + 0 + 0 = 22 + + Étapes 5 et 6 : 30 - 22 = 8 (valeur de la clé) + """ + + if not isinstance(isin, basestring): + return False + if not re.match(r'^[A-Z]{2}[A-Z0-9]{9}\d$', isin): + return False + + isin_in_digits = ''.join(str(ord(x) - ord('A') + 10) if not x.isdigit() else x for x in isin[:-1]) + key = isin[-1:] + result = '' + for k, val in enumerate(isin_in_digits[::-1], start=1): + if k % 2 == 0: + result = ''.join((result, val)) + else: + result = ''.join((result, str(int(val)*2))) + return str(sum(int(x) for x in result) + int(key))[-1] == '0' + + +def create_french_liquidity(valuation): + """ + Automatically fills a liquidity investment with label, code and code_type. + """ + liquidity = Investment() + liquidity.label = "Liquidités" + liquidity.code = "XX-liquidity" + liquidity.code_type = NotAvailable + liquidity.valuation = valuation + return liquidity + diff --git a/modules/banquepopulaire/module.py b/modules/banquepopulaire/module.py index 54ec323b61..098dfca182 100644 --- a/modules/banquepopulaire/module.py +++ b/modules/banquepopulaire/module.py @@ -45,6 +45,7 @@ class BanquePopulaireModule(Module, CapBankWealth, CapContact, CapProfile): 'www.ibps.bpalc.banquepopulaire.fr' : u'Alsace Lorraine Champagne', 'www.ibps.bpaca.banquepopulaire.fr': u'Aquitaine Centre atlantique', 'www.ibps.atlantique.banquepopulaire.fr': u'Atlantique', + 'www.ibps.bpgo.banquepopulaire.fr': u'Grand Ouest', 'www.ibps.loirelyonnais.banquepopulaire.fr': u'Auvergne Rhône Alpes', 'www.ibps.bpaura.banquepopulaire.fr': u'Auvergne Rhône Alpes', 'www.ibps.banquedesavoie.banquepopulaire.fr': u'Banque de Savoie', @@ -74,6 +75,10 @@ def create_default_browser(self): ('loirelyonnais', 'bpaura'), ('alpes', 'bpaura'), ('massifcentral', 'bpaura'), + ('atlantique.creditmaritime', 'cmgo.creditmaritime'), + ('bretagnenormandie.cmm', 'cmgo'), + ('atlantique.banquepopulaire', 'bpgo.banquepopulaire'), + ('ouest.banquepopulaire', 'bpgo.banquepopulaire'), ] website = reduce(lambda a, kv: a.replace(*kv), repls, self.config['website'].get()) return self.create_browser(website, diff --git a/modules/banquepopulaire/pages.py b/modules/banquepopulaire/pages.py index f6b75fa29c..741b327cf2 100644 --- a/modules/banquepopulaire/pages.py +++ b/modules/banquepopulaire/pages.py @@ -257,6 +257,8 @@ class ErrorPage(LoggedPage, MyHTMLPage): def on_load(self): if CleanText('//script[contains(text(), "momentanément indisponible")]')(self.doc): raise BrowserUnavailable(u"Le service est momentanément indisponible") + elif CleanText('//h1[contains(text(), "Cette page est indisponible")]')(self.doc): + raise BrowserUnavailable('Cette page est indisponible') return super(ErrorPage, self).on_load() def get_token(self): @@ -308,7 +310,7 @@ class Login2Page(LoginPage): def request_url(self): transactionID = self.params['transactionID'] assert transactionID - return 'https://www.icgauth.banquepopulaire.fr/dacswebssoissuer/api/v1u0/transaction/%s' % transactionID + return self.browser.redirect_url + transactionID def on_load(self): if not self.browser.no_login: @@ -369,7 +371,7 @@ def is_here(self): doc = json.loads(self.response.text) if 'response' in doc: return doc['response']['status'] == 'AUTHENTICATION_SUCCESS' and 'saml2_post' in doc['response'] - except json.decoder.JSONDecodeError: + except ValueError: # not a json page # so it should be Login2Page return False @@ -444,7 +446,7 @@ class AccountsPage(LoggedPage, MyHTMLPage): u'Liste complète de mon épargne': Account.TYPE_SAVINGS, u'Mes comptes': Account.TYPE_CHECKING, u'Comptes en euros': Account.TYPE_CHECKING, - 'Mes comptes en devises': Account.TYPE_CHECKING, + u'Mes comptes en devises': Account.TYPE_CHECKING, u'Liste complète de mes comptes': Account.TYPE_CHECKING, u'Mes emprunts': Account.TYPE_LOAN, u'Liste complète de mes emprunts': Account.TYPE_LOAN, @@ -457,6 +459,7 @@ class AccountsPage(LoggedPage, MyHTMLPage): PATTERN = [(re.compile('.*Titres Pea.*'), Account.TYPE_PEA), (re.compile(".*Plan D'epargne En Actions.*"), Account.TYPE_PEA), + (re.compile(".*Compte Especes Pea.*"), Account.TYPE_PEA), (re.compile('.*Plan Epargne Retraite.*'), Account.TYPE_PERP), (re.compile('.*Titres.*'), Account.TYPE_MARKET), (re.compile('.*Selection Vie.*'),Account.TYPE_LIFE_INSURANCE), @@ -538,11 +541,8 @@ def iter_accounts(self, next_pages): balance_text = u''.join([txt.strip() for txt in tds[3].itertext()]) balance = FrenchTransaction.clean_amount(balance_text) account.balance = Decimal(balance or '0.0') + account.currency = currency or Account.get_currency(balance_text) - if currency is None: - account.currency = Account.get_currency(balance_text) - else: - account.currency = currency if account.type == account.TYPE_LOAN: account.balance = - abs(account.balance) @@ -741,6 +741,8 @@ def get_account_history(self): t = Transaction() + # get the column index of the link to access transaction details + # (only used for GoCardLess transactions so far) t._has_link = bool(tds[self.COL_DEBIT].findall('a') or tds[self.COL_CREDIT].findall('a')) # XXX We currently take the *value* date, but it will probably @@ -756,6 +758,7 @@ def get_account_history(self): t.parse(date, re.sub(r'[ ]+', ' ', raw), vdate) t.set_amount(credit, debit) + t._amount_type = 'debit' if t.amount == debit else 'credit' # Strip the balance displayed in transaction labels t.label = re.sub('solde en valeur : .*', '', t.label) @@ -842,6 +845,27 @@ def get_transaction_table_id(self, ref): return key, value + def get_gocardless_strategy_param(self, transaction): + # A form is filled and send with javascript + # the 'validationStrategy' parameter value only depends on the column + # index in which the link lies + # + # To get more details about how things are done, see the following javascript functions: + #- attachTableRowEvents (atre) + #- attachActiveSelectionEventsOnRow + #- astr + #- updateSelection (uds) + #- selectActionButton (sab) + #- a script element embedded in the html page (search for "tcl5", "tcl6") + + assert transaction._has_link + + if transaction._amount_type == 'debit': + return 'AV' + elif transaction._amount_type == 'credit': + return 'NV' + + class NatixisChoicePage(LoggedPage, HTMLPage): def on_load(self): message = CleanText('//span[@class="rf-msgs-sum"]', default='')(self.doc) diff --git a/modules/barclays/compat/weboob_capabilities_bank.py b/modules/barclays/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/barclays/compat/weboob_capabilities_bank.py +++ b/modules/barclays/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/barclays/pages.py b/modules/barclays/pages.py index 5e3bc6d692..de5e410f85 100644 --- a/modules/barclays/pages.py +++ b/modules/barclays/pages.py @@ -107,7 +107,8 @@ class AccountsPage(StatefulPage): 'Engagement/Crédits': Account.TYPE_LOAN, } ACCOUNT_EXTRA_TYPES = {'BMOOVIE': Account.TYPE_LIFE_INSURANCE, - 'B. GESTION VIE': Account.TYPE_LIFE_INSURANCE + 'B. GESTION VIE': Account.TYPE_LIFE_INSURANCE, + 'E VIE MILLEIS': Account.TYPE_LIFE_INSURANCE, } ACCOUNT_TYPE_TO_STR = {Account.TYPE_MARKET: 'TTR', Account.TYPE_CARD: 'CRT' @@ -165,9 +166,6 @@ def obj__multiple_type(self): return False - def validate(self, el): - return Field('type')(self) != Account.TYPE_UNKNOWN - class Transaction(FrenchTransaction): PATTERNS = [ @@ -282,6 +280,10 @@ def get_space_attrs(self, space): @method class iter_investments(TableElement): + + def condition(self): + return not self.xpath('//h1[text()="Aucune position"]') + head_xpath = '//table[@id="C4__TBL_Equity"]/thead/tr/th' item_xpath = '//table[@id="C4__TBL_Equity"]/tbody/tr' @@ -313,9 +315,6 @@ def obj_code(self): obj_portfolio_share = Eval(lambda x: x / 100, MyDecimal(TableCell('portfolio_share'))) obj_code_type = Investment.CODE_TYPE_ISIN - def condition(self): - return CleanDecimal(TableCell('quantity'), default=None)(self) is not None - class LifeInsuranceAccountPage(AbstractAccountPage): def is_here(self): @@ -358,6 +357,10 @@ def obj_date(self): @method class iter_investments(TableElement): + + def condition(self): + return not self.xpath('//h1[text()="Aucune position"]') + head_xpath = '//table[@class="table-support"]/thead/tr/th' item_xpath = '//table[@class="table-support"]/tbody/tr' diff --git a/modules/bforbank/browser.py b/modules/bforbank/browser.py index 0985c1a725..16e00901b3 100644 --- a/modules/bforbank/browser.py +++ b/modules/bforbank/browser.py @@ -18,17 +18,18 @@ # along with weboob. If not, see . import datetime from dateutil.relativedelta import relativedelta - from weboob.exceptions import BrowserIncorrectPassword -from weboob.browser import LoginBrowser, URL, need_login -from weboob.capabilities.bank import Account, AccountNotFound, Investment +from weboob.browser.browsers import LoginBrowser, URL, need_login +from weboob.capabilities.bank import Account, AccountNotFound from weboob.capabilities.base import empty from weboob.tools.capabilities.bank.transactions import sorted_transactions +from weboob.tools.decorators import retry +from .compat.weboob_tools_capabilities_bank_investments import create_french_liquidity from .pages import ( LoginPage, ErrorPage, AccountsPage, HistoryPage, LoanHistoryPage, RibPage, LifeInsuranceList, LifeInsuranceIframe, LifeInsuranceRedir, - BoursePage, CardHistoryPage, CardPage, UserValidationPage, + BoursePage, CardHistoryPage, CardPage, UserValidationPage, BourseActionNeeded, ) from .spirica_browser import SpiricaBrowser @@ -59,6 +60,7 @@ class BforbankBrowser(LoginBrowser): ErrorPage) bourse_login = URL(r'/espace-client/titres/debranchementCaTitre/(?P\d+)') + bourse_action_needed = URL('https://bourse.bforbank.com/netfinca-titres/*', BourseActionNeeded) bourse = URL('https://bourse.bforbank.com/netfinca-titres/servlet/com.netfinca.frontcr.synthesis.HomeSynthesis', 'https://bourse.bforbank.com/netfinca-titres/servlet/com.netfinca.frontcr.account.*', BoursePage) @@ -198,6 +200,7 @@ def goto_lifeinsurance(self, account): self.location('https://client.bforbank.com/espace-client/assuranceVie') self.lifeinsurance_list.go() + @retry(AccountNotFound, tries=5) def goto_spirica(self, account): assert account.type == Account.TYPE_LIFE_INSURANCE self.goto_lifeinsurance(account) @@ -274,11 +277,7 @@ def iter_investment(self, account): # _especes is set during BoursePage accounts parsing. BoursePage # inherits from lcl module BoursePage if bourse_account._especes: - i = Investment() - i.valuation = bourse_account._especes - i.code = u"XX-liquidity" - i.label = u"Liquidités" - invs.append(i) + invs.append(create_french_liquidity(bourse_account._especes)) self.leave_espace_bourse() diff --git a/modules/bforbank/compat/weboob_capabilities_bank.py b/modules/bforbank/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/bforbank/compat/weboob_capabilities_bank.py +++ b/modules/bforbank/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/bforbank/compat/weboob_tools_capabilities_bank_investments.py b/modules/bforbank/compat/weboob_tools_capabilities_bank_investments.py new file mode 100644 index 0000000000..2b68ff9e00 --- /dev/null +++ b/modules/bforbank/compat/weboob_tools_capabilities_bank_investments.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Jonathan Schmidt +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.tools.compat import basestring +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.bank import Investment + +def is_isin_valid(isin): + """ + Méthode générale + Table de conversion des lettres en chiffres + A=10 B=11 C=12 D=13 E=14 F=15 G=16 H=17 I=18 + J=19 K=20 L=21 M=22 N=23 O=24 P=25 Q=26 R=27 + S=28 T=29 U=30 V=31 W=32 X=33 Y=34 Z=35 + + 1 - Mettre de côté la clé, qui servira de référence à la fin de la vérification. + 2 - Convertir toutes les lettres en nombres via la table de conversion ci-contre. Si le nombre obtenu est supérieur ou égal à 10, prendre les deux chiffres du nombre séparément (exemple : 27 devient 2 et 7). + 3 - Pour chaque chiffre, multiplier sa valeur par deux si sa position est impaire en partant de la droite. Si le nombre obtenu est supérieur ou égal à 10, garder les deux chiffres du nombre séparément (exemple : 14 devient 1 et 4). + 4 - Faire la somme de tous les chiffres. + 5 - Soustraire cette somme de la dizaine supérieure ou égale la plus proche (exemples : si la somme vaut 22, la dizaine « supérieure ou égale » est 30, et la clé vaut donc 8 ; si la somme vaut 30, la dizaine « supérieure ou égale » est 30, et la clé vaut 0 ; si la somme vaut 31, la dizaine « supérieure ou égale » est 40, et la clé vaut 9). + 6 - Comparer la valeur obtenue à la clé mise initialement de côté. + + Étapes 1 et 2 : + F R 0 0 0 3 5 0 0 0 0 (+ 8 : clé) + 15 27 0 0 0 3 5 0 0 0 0 + + Étape 3 : le traitement se fait sur des chiffres + 1 5 2 7 0 0 0 3 5 0 0 0 0 + I P I P I P I P I P I P I : position en partant de la droite (P = Pair, I = Impair) + 2 1 2 1 2 1 2 1 2 1 2 1 2 : coefficient multiplicateur + + 2 5 4 7 0 0 0 3 10 0 0 0 0 : résultat + + Étape 4 : + 2 + 5 + 4 + 7 + 0 + 0 + 0 + 3 + (1 + 0)+ 0 + 0 + 0 + 0 = 22 + + Étapes 5 et 6 : 30 - 22 = 8 (valeur de la clé) + """ + + if not isinstance(isin, basestring): + return False + if not re.match(r'^[A-Z]{2}[A-Z0-9]{9}\d$', isin): + return False + + isin_in_digits = ''.join(str(ord(x) - ord('A') + 10) if not x.isdigit() else x for x in isin[:-1]) + key = isin[-1:] + result = '' + for k, val in enumerate(isin_in_digits[::-1], start=1): + if k % 2 == 0: + result = ''.join((result, val)) + else: + result = ''.join((result, str(int(val)*2))) + return str(sum(int(x) for x in result) + int(key))[-1] == '0' + + +def create_french_liquidity(valuation): + """ + Automatically fills a liquidity investment with label, code and code_type. + """ + liquidity = Investment() + liquidity.label = "Liquidités" + liquidity.code = "XX-liquidity" + liquidity.code_type = NotAvailable + liquidity.valuation = valuation + return liquidity + diff --git a/modules/bforbank/pages.py b/modules/bforbank/pages.py index c95f3723fe..cd5f636b8d 100644 --- a/modules/bforbank/pages.py +++ b/modules/bforbank/pages.py @@ -359,6 +359,18 @@ def get_redir(self): return match.group(1) +class BourseActionNeeded(LoggedPage, HTMLPage): + ENCODING = 'latin-1' + XPATH = "//div[contains(text(), 'Création ou modification de votre mot de passe trading')]" + + def is_here(self): + return CleanText(self.XPATH)(self.doc) + + def on_load(self): + error = CleanText(self.XPATH)(self.doc) + raise ActionNeeded(error) + + class BoursePage(AbstractPage): PARENT = 'lcl' PARENT_URL = 'bourse' diff --git a/modules/binck/compat/weboob_capabilities_bank.py b/modules/binck/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/binck/compat/weboob_capabilities_bank.py +++ b/modules/binck/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/biplan/browser.py b/modules/biplan/browser.py index 259bd8265e..15ff308391 100644 --- a/modules/biplan/browser.py +++ b/modules/biplan/browser.py @@ -26,7 +26,7 @@ class BiplanBrowser(PagesBrowser): - BASEURL = 'http://www.lebiplan.org' + BASEURL = 'https://www.lebiplan.org' program_page = URL('/fr/biplan-prog-(?P<_category>.*).php', ProgramPage) event_page = URL('/(?P<_id>.*).html', EventPage) diff --git a/modules/bnpcards/corporate/pages.py b/modules/bnpcards/corporate/pages.py index f350153d6b..a4cdf9a71e 100644 --- a/modules/bnpcards/corporate/pages.py +++ b/modules/bnpcards/corporate/pages.py @@ -31,11 +31,10 @@ __all__ = ['LoginPage', 'ErrorPage', 'AccountsPage', 'TransactionsPage'] - class LoginPage(HTMLPage): def login(self, username, password): form = self.get_form(name='connecterForm') - form['type'] = '2' # Gestionnaire + form['type'] = '2' # Gestionnaire form['login'] = username form['pwd'] = password[:8] form.url = '/ce_internet_public/seConnecter.event.do' @@ -53,7 +52,9 @@ class iter_accounts(ListElement): class item(ItemElement): klass = Account - obj_id = Format('%s%s', CleanText('./td[2]', replace=[(' ', '')]), CleanText('./td[3]')) + obj_id = Format('%s%s%s', CleanText('./td[2]', replace=[(' ', '')]), CleanText('./td[3]'), + CleanText('./td[1]', replace=[(' ', '')])) + obj__owner = CleanText('./td[1]') obj_number = CleanText('./td[2]', replace=[(' ', '')]) obj_label = CleanText('./td[1]') @@ -128,7 +129,7 @@ def obj_amount(self): return CleanDecimal(replace_dots=False).filter(self.el.xpath('./td[5]')) - CleanDecimal(replace_dots=False).filter(self.el.xpath('./td[6]')) def is_not_sorted(self, order='down'): - translate = {'down':'bas','up':'haut'} + translate = {'down': 'bas', 'up': 'haut'} return len(self.doc.xpath('//table[@id="ContentTable_datas"]/thead/tr/th[1]//img[contains(@src, "tri_%s_on")]' % translate[order])) == 0 def sort(self, order='down'): diff --git a/modules/bnporc/compat/weboob_capabilities_bank.py b/modules/bnporc/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/bnporc/compat/weboob_capabilities_bank.py +++ b/modules/bnporc/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/bnporc/enterprise/browser.py b/modules/bnporc/enterprise/browser.py index cd6a8b944e..e7b752d27b 100644 --- a/modules/bnporc/enterprise/browser.py +++ b/modules/bnporc/enterprise/browser.py @@ -19,14 +19,13 @@ from __future__ import unicode_literals -import re - from datetime import datetime from dateutil.rrule import rrule, MONTHLY from dateutil.relativedelta import relativedelta from weboob.browser import LoginBrowser, need_login +from weboob.capabilities.base import find_object from weboob.capabilities.bank import Account from weboob.exceptions import BrowserIncorrectPassword, BrowserForbidden from weboob.browser.url import URL @@ -34,7 +33,7 @@ from .pages import ( LoginPage, AuthPage, AccountsPage, AccountHistoryViewPage, AccountHistoryPage, - ActionNeededPage, TransactionPage, TokenPage, InvestPage + ActionNeededPage, TransactionPage, MarketPage, InvestPage ) @@ -61,8 +60,8 @@ class BNPEnterprise(LoginBrowser): transaction_detail = URL(r'/NCCPresentationWeb/e21/getOptBDDF.do', TransactionPage) invest = URL(r'/opcvm/lister-composition/afficher.do', InvestPage) - # the token page is used only if there are several type market accounts - token_inv = URL(r'/opcvm/lister-portefeuilles/afficher.do', TokenPage) + # The Market page is used only if there are several market accounts + market = URL(r'/opcvm/lister-portefeuilles/afficher.do', MarketPage) renew_pass = URL('/sommaire/PseRedirectPasswordConnect', ActionNeededPage) @@ -72,7 +71,7 @@ def __init__(self, *args, **kwargs): def do_login(self): self.login.go() - if self.login.is_here() is False: + if not self.login.is_here(): return data = {} @@ -89,22 +88,26 @@ def do_login(self): @need_login def get_accounts_list(self): accounts = [] - try: - self.token_inv.go() - if self.token_inv.is_here(): - marketaccount = self.page.market_search() - else: - # redirected to invest page - # it means that there is only 1 market account among the ones showed - marketaccount = [self.page.get_market_account_label()] - except BrowserForbidden: - marketaccount = [] - + # Fetch checking accounts: for account in self.accounts.stay_or_go().iter_accounts(): - label_tmp = re.search(r'(\d{4,})', account.label) - if (label_tmp and label_tmp.group(0) in marketaccount) or account.label in marketaccount: - account.type = Account.TYPE_MARKET accounts.append(account) + # Fetch market accounts: + try: + self.market.go() + if self.market.is_here(): + for market_account in self.page.iter_market_accounts(): + market_account.parent = find_object(accounts, label=market_account._parent) + accounts.append(market_account) + + elif self.invest.is_here(): + # Redirected to invest page, meaning there is only 1 market account. + # We thus create an Account object for this unique market account. + account = self.page.get_unique_market_account() + account.parent = find_object(accounts, label=account._parent) + accounts.append(account) + + except BrowserForbidden: + pass return accounts @@ -116,6 +119,9 @@ def get_account(self, _id): @need_login def iter_history(self, account): + # There is no available history for market accounts + if account.type == Account.TYPE_MARKET: + return [] return self._iter_history_base(account) def _iter_history_base(self, account): @@ -144,24 +150,30 @@ def _iter_history_base(self, account): @need_login def iter_coming_operations(self, account): + # There is no available coming operation for market accounts + if account.type == Account.TYPE_MARKET: + return [] + self.account_coming_view.go(identifiant=account.iban) self.account_coming.go(identifiant=account.iban) return self.page.iter_coming() @need_login def iter_investment(self, account): - if account.type == Account.TYPE_MARKET: - self.token_inv.go() - if self.token_inv.is_here(): + if account.type != Account.TYPE_MARKET: + return + + self.market.go() + # If there is more than one market account, we must fetch the account params: + if not account._unique: + if self.market.is_here(): token = self.page.get_token() id_invest = self.page.get_id(label=account.label) data = {"numeroCompte": id_invest, "_csrf": token} self.location('/opcvm/lister-composition/redirect-afficher.do', data=data) - else: - self.invest.go() - for tr in self.page.iter_investment(): - yield tr + for inv in self.page.iter_investment(): + yield inv @need_login def get_profile(self): diff --git a/modules/bnporc/enterprise/pages.py b/modules/bnporc/enterprise/pages.py index 297b20efdd..a02aca40f0 100644 --- a/modules/bnporc/enterprise/pages.py +++ b/modules/bnporc/enterprise/pages.py @@ -30,7 +30,7 @@ from weboob.browser.elements import DictElement, ItemElement, method, TableElement from weboob.browser.filters.standard import ( CleanText, CleanDecimal, Date, Regexp, Format, Eval, BrowserURL, Field, - Async, + Async, Currency, ) from weboob.capabilities.bank import Transaction, Account, Investment from weboob.capabilities.profile import Person @@ -105,8 +105,10 @@ def on_load(self): class AccountsPage(LoggedPage, JsonPage): - TYPES = {u'Compte chèque': Account.TYPE_CHECKING, - u'Compte à vue': Account.TYPE_CHECKING} + TYPES = { + 'Compte chèque': Account.TYPE_CHECKING, + 'Compte à vue': Account.TYPE_CHECKING, + } @method class iter_accounts(DictElement): @@ -333,7 +335,43 @@ def get_redacted_card(self): return self.doc['carteNum'] -class TokenPage(LoggedPage, HTMLPage): +class MarketPage(LoggedPage, HTMLPage): + TYPES = { + 'comptes de titres': Account.TYPE_MARKET, + } + + @method + class iter_market_accounts(TableElement): + def condition(self): + return not self.el.xpath('//table[@id="table-portefeuille"]//tr/td[contains(text(), "Aucun portefeuille à afficher")]') + + item_xpath = '//table[@id="table-portefeuille"]/tbody[@class="main-content"]/tr' + head_xpath = '//table[@id="table-portefeuille"]/thead/tr/th/label' + + col_label = 'Portefeuille-titres' + col_balance = re.compile('Valorisation') + col__parent = re.compile('Compte courant') + + class item(ItemElement): + klass = Account + + # Market accounts have no IBAN so we use the account number and specify + # "MARKET" at the end to differentiate from its parent account. + obj_id = Format('%sMARKET', Regexp(CleanText(TableCell('label')), r'\*(.*)\*', default=None)) + obj_label = CleanText(TableCell('label')) + obj_balance = CleanDecimal(TableCell('balance'), replace_dots=True) + obj_currency = Currency(TableCell('balance')) + obj__parent = CleanText(TableCell('_parent')) + obj_coming = NotAvailable + obj_iban = NotAvailable + obj__unique = False + + def obj_type(self): + for key, type in self.page.TYPES.items(): + if key in CleanText(TableCell('label'))(self).lower(): + return type + return Account.TYPE_UNKNOWN + def get_token(self): return Attr('//meta[@name="_csrf"]', 'content')(self.doc) @@ -343,18 +381,23 @@ def get_id(self, label): if id_simple in CleanText(options)(self.doc): return CleanText(options.xpath('./@value'))(self) - def market_search(self): - marketaccount = [] - for account in self.doc.xpath('//div[@class="filterbox-content hide"]//select[@id="numero-compte-titre"]//option'): - account = CleanText(account)(self.doc) - temp = re.search(r'[0-9]+', account) - if temp != None: - marketaccount.append(temp.group(0)) - return marketaccount +class InvestPage(LoggedPage, HTMLPage): + @method + class get_unique_market_account(ItemElement): + klass = Account + + # Market accounts have no IBAN so we use the account number and specify + # "MARKET" at the end to differentiate it from its parent account. + obj_id = Format('%sMARKET', Regexp(CleanText('//div[@class="head"]/h2'), r'\*(.*)\*', default=None)) + obj_label = CleanText('//div[@class="head"]/h2') + obj_balance = CleanDecimal('//div[@id="apercu-val"]/h1', replace_dots=True) + obj_currency = Currency('//div[@id="apercu-val"]/h1') + obj_type = Account.TYPE_MARKET + obj__parent = CleanText('//h3/span[span[@class="info-cheque"]]', children=False) + obj__unique = True -class InvestPage(LoggedPage, HTMLPage): @method class iter_investment(TableElement): item_xpath = '//table[@class="csv-data-container hide"]//tr' @@ -367,6 +410,12 @@ class iter_investment(TableElement): col_valuation = 'Valorisation' col_diff = '+/- value' + """ + Note: Pagination is not handled yet for investments, if we find a + customer with more than 10 invests we might have to handle clicking + on the button to get 50 invests per page or check if there is a link. + """ + class item(ItemElement): klass = Investment @@ -379,8 +428,5 @@ class item(ItemElement): obj_code_type = lambda self: Investment.CODE_TYPE_ISIN if Field('code')(self) is not NotAvailable else NotAvailable def obj_code(self): - chaine = CleanText(TableCell('label'))(self) - return re.search(r'(\w+) - ', chaine).group(0)[:-3] - - def get_market_account_label(self): - return CleanText('//h3/span[span]', children=False)(self.doc) + string = CleanText(TableCell('label'))(self) + return re.search(r'(\w+) - ', string).group(0)[:-3] diff --git a/modules/bnporc/pp/browser.py b/modules/bnporc/pp/browser.py index cbf29d2f61..3242611064 100644 --- a/modules/bnporc/pp/browser.py +++ b/modules/bnporc/pp/browser.py @@ -40,9 +40,9 @@ LoginPage, AccountsPage, AccountsIBANPage, HistoryPage, TransferInitPage, ConnectionThresholdPage, LifeInsurancesPage, LifeInsurancesHistoryPage, LifeInsurancesDetailPage, NatioVieProPage, CapitalisationPage, - MarketListPage, MarketPage, MarketHistoryPage, MarketSynPage, + MarketListPage, MarketPage, MarketHistoryPage, MarketSynPage, BNPKeyboard, RecipientsPage, ValidateTransferPage, RegisterTransferPage, AdvisorPage, - AddRecipPage, ActivateRecipPage, ProfilePage, ListDetailCardPage, + AddRecipPage, ActivateRecipPage, ProfilePage, ListDetailCardPage, ListErrorPage, ) @@ -75,6 +75,8 @@ class BNPParibasBrowser(JsonBrowserMixin, LoginBrowser): 'SEEA-pa01/devServer/seeaserver', 'https://mabanqueprivee.bnpparibas.net/fr/espace-prive/comptes-et-contrats\?u=%2FSEEA-pa01%2FdevServer%2Fseeaserver', LoginPage) + + list_error_page = URL('https://mabanque.bnpparibas/rsc/contrib/document/properties/identification-fr-part-V1.json', ListErrorPage) con_threshold = URL('/fr/connexion/100-connexions', '/fr/connexion/mot-de-passe-expire', '/fr/espace-prive/100-connexions.*', @@ -129,6 +131,23 @@ def do_login(self): if self.login.is_here(): self.page.login(self.username, self.password) + def change_pass(self, oldpass, newpass): + res = self.open('/identification-wspl-pres/grille?accessible=false') + url = '/identification-wspl-pres/grille/%s' % res.json()['data']['idGrille'] + keyboard = self.open(url) + vk = BNPKeyboard(self, keyboard) + data = {} + data['codeAppli'] = 'PORTAIL' + data['idGrille'] = res.json()['data']['idGrille'] + data['typeGrille'] = res.json()['data']['typeGrille'] + data['confirmNouveauPassword'] = vk.get_string_code(newpass) + data['nouveauPassword'] = vk.get_string_code(newpass) + data['passwordActuel'] = vk.get_string_code(oldpass) + response = self.location('/mcs-wspl/rpc/modifiercodesecret', data=data) + if response.json().get('messageIden').lower() == 'nouveau mot de passe invalide': + return False + return True + @need_login def get_profile(self): self.profile.go(data=JSON({})) @@ -174,13 +193,20 @@ def get_accounts_list(self): # Fetching capitalisation contracts from the "Assurances Vie" space (some are not in the BNP API): params = self.natio_vie_pro.go().get_params() - self.capitalisation_page.go(params=params) - if self.capitalisation_page.is_here() and self.page.has_contracts(): - for account in self.page.iter_capitalisation(): - # Life Insurance accounts may appear BOTH in the API and the "Assurances Vie" domain, - # It is better to keep the API version since it contains the unitvalue: - if account.number not in [a.number for a in self.accounts_list]: - self.accounts_list.append(account) + try: + self.capitalisation_page.go(params=params) + except ServerError: + self.logger.warning("An Internal Server Error occurred") + else: + if self.capitalisation_page.is_here() and self.page.has_contracts(): + for account in self.page.iter_capitalisation(): + # Life Insurance accounts may appear BOTH in the API and the "Assurances Vie" domain, + # It is better to keep the API version since it contains the unitvalue: + if account.number not in [a.number for a in self.accounts_list]: + self.logger.warning("We found an account that only appears on the old BNP website.") + self.accounts_list.append(account) + else: + self.logger.warning("This account was skipped because it already appears in the API.") return iter(self.accounts_list) @@ -201,7 +227,7 @@ def iter_history(self, account, coming=False): try: self.market_list.go(data=JSON({})) except ServerError: - self.logger.warning("An Internal Server Error occured") + self.logger.warning("An Internal Server Error occurred") return iter([]) for market_acc in self.page.get_list(): if account.number[-4:] == market_acc['securityAccountNumber'][-4:]: @@ -260,7 +286,6 @@ def iter_investment(self, account): # Going to the "Assurances Vie" page natiovie_params = self.natio_vie_pro.go().get_params() self.capitalisation_page.go(params=natiovie_params) - # Fetching the form to get the contract investments: capitalisation_params = self.page.get_params(account) self.capitalisation_page.go(params=capitalisation_params) @@ -277,7 +302,7 @@ def iter_investment(self, account): try: self.market_list.go(data=JSON({})) except ServerError: - self.logger.warning("An Internal Server Error occured") + self.logger.warning("An Internal Server Error occurred") return iter([]) for market_acc in self.page.get_list(): if account.number[-4:] == market_acc['securityAccountNumber'][-4:] and not account.iban: @@ -287,7 +312,7 @@ def iter_investment(self, account): "securityAccountNumber": market_acc['securityAccountNumber'], })) except ServerError: - self.logger.warning("An Internal Server Error occured") + self.logger.warning("An Internal Server Error occurred") break return self.page.iter_investments() diff --git a/modules/bnporc/pp/compat/weboob_capabilities_bank.py b/modules/bnporc/pp/compat/weboob_capabilities_bank.py new file mode 100644 index 0000000000..141097d781 --- /dev/null +++ b/modules/bnporc/pp/compat/weboob_capabilities_bank.py @@ -0,0 +1,45 @@ + +import weboob.capabilities.bank as OLD + +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) + + +__all__ = OLD.__all__ + + +class CapBankWealth(CapBank): + pass + + +class CapBankPockets(CapBank): + pass + + +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + +class CapBankTransfer(OLD.CapBankTransfer): + def transfer_check_label(self, old, new): + from unidecode import unidecode + + return unidecode(old) == unidecode(new) + + +class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipient): + pass + + +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + +Account.TYPE_MORTGAGE = 17 +Account.TYPE_CONSUMER_CREDIT = 18 +Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/bnporc/pp/pages.py b/modules/bnporc/pp/pages.py index 439be56989..e92d52f5fe 100644 --- a/modules/bnporc/pp/pages.py +++ b/modules/bnporc/pp/pages.py @@ -32,7 +32,10 @@ from weboob.browser.filters.html import TableCell from weboob.browser.pages import JsonPage, LoggedPage, HTMLPage from weboob.capabilities import NotAvailable -from weboob.capabilities.bank import Account, Investment, Recipient, Transfer, TransferError, TransferBankError, AddRecipientError +from .compat.weboob_capabilities_bank import ( + Account, Investment, Recipient, Transfer, TransferError, TransferBankError, + AddRecipientBankError, +) from weboob.capabilities.contact import Advisor from .compat.weboob_capabilities_profile import Person, ProfileMissing from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable, BrowserPasswordExpired, ActionNeeded @@ -42,25 +45,12 @@ from weboob.tools.date import parse_french_date from weboob.tools.capabilities.bank.investments import is_isin_valid from weboob.tools.compat import unquote_plus +from weboob.tools.html import html2text class ConnectionThresholdPage(HTMLPage): - def change_pass(self, oldpass, newpass): - res = self.browser.open('/identification-wspl-pres/grille?accessible=false') - url = '/identification-wspl-pres/grille/%s' % res.json()['data']['idGrille'] - keyboard = self.browser.open(url) - vk = BNPKeyboard(self, keyboard) - data = {} - data['codeAppli'] = 'PORTAIL' - data['idGrille'] = res.json()['data']['idGrille'] - data['typeGrille'] = res.json()['data']['typeGrille'] - data['confirmNouveauPassword'] = vk.get_string_code(newpass) - data['nouveauPassword'] = vk.get_string_code(newpass) - data['passwordActuel'] = vk.get_string_code(oldpass) - response = self.browser.location('/mcs-wspl/rpc/modifiercodesecret', data=data) - if response.json().get('messageIden').lower() == 'nouveau mot de passe invalide': - return False - return True + NOT_REUSABLE_PASSWORDS_COUNT = 3 + """BNP disallows to reuse one of the three last used passwords.""" def make_date(self, yy, m, d): current = datetime.now().year @@ -109,20 +99,35 @@ def looks_legit(self, password): return True def on_load(self): + msg = CleanText('//div[@class="confirmation"]//span[span]')(self.doc) + + self.logger.warning('Password expired.') if not self.looks_legit(self.browser.password): # we may not be able to restore the password, so reject it - msg = CleanText('//div[@class="confirmation"]//span[span]')(self.doc) + self.logger.warning('Unable to restore it, it is not legit.') raise BrowserPasswordExpired(msg) - new_pass = ''.join([str((int(l) + 1) % 10) for l in self.browser.password]) - self.logger.warning('Password expired. Renewing it. Temporary password is %s', new_pass) - if not self.change_pass(self.browser.password, new_pass): - self.logger.warning('New temp password is rejected, giving up') - raise BrowserPasswordExpired() - - if not self.change_pass(new_pass, self.browser.password): + new_passwords = [] + for i in range(self.NOT_REUSABLE_PASSWORDS_COUNT): + new_pass = ''.join([str((int(l) + i + 1) % 10) for l in self.browser.password]) + if not self.looks_legit(new_pass): + self.logger.warning('One of rotating password is not legit') + raise BrowserPasswordExpired(msg) + new_passwords.append(new_pass) + + current_password = self.browser.password + for new_pass in new_passwords: + self.logger.warning('Renewing with temp password') + if not self.browser.change_pass(current_password, new_pass): + self.logger.warning('New temp password is rejected, giving up') + raise BrowserPasswordExpired(msg) + current_password = new_pass + + if not self.browser.change_pass(current_password, self.browser.password): self.logger.error('Could not restore old password!') + self.logger.warning('Old password restored.') + def cast(x, typ, default=None): try: @@ -145,11 +150,20 @@ class BNPKeyboard(GridVirtKeyboard): '8': 'c60b723b3d95a46416b34c2cbefba3ed', '9': 'a13b8c3617a7bf854590833ddfb97f1f'} - def __init__(self, page, image): + def __init__(self, browser, image): symbols = list('%02d' % x for x in range(1, 11)) super(BNPKeyboard, self).__init__(symbols, 5, 2, BytesIO(image.content), self.color, convert='RGB') - self.check_symbols(self.symbols, page.browser.responses_dirname) + self.check_symbols(self.symbols, browser.responses_dirname) + + +class ListErrorPage(JsonPage): + def get_error_message(self, error): + key = 'app.identification.erreur.' + str(error) + try: + return html2text(self.doc[key]) + except KeyError: + return None class LoginPage(JsonPage): @@ -183,25 +197,28 @@ def on_load(self): error = cast(self.get('errorCode'), int, 0) # you can find api documentation on errors here : https://mabanque.bnpparibas/rsc/contrib/document/properties/identification-fr-part-V1.json if error: - codes = [201, 21510, 203, 202] - msg = self.get('message') - if error in codes: + error_page = self.browser.list_error_page.open() + msg = error_page.get_error_message(error) + if not msg: + msg = self.get('message') + + wrongpass_codes = [201, 21510, 203, 202, 1001, 7] + actionNeeded_codes = [21501, 3, 4, 50] + if error in wrongpass_codes: raise BrowserIncorrectPassword(msg) - elif error == 1001: - # json says "Erreur lors de l'authentification Code retour : 1001 Code retour : 1001" - # but js message from "getErrorMessage" says "veuillez contacter votre conseiller"... - raise BrowserIncorrectPassword() - elif error == 21501: # "Rendez-vous sur le site de BNP Paribas pour gérer vos comptes" - raise ActionNeeded(msg) elif error == 21: # "Ce service est momentanément indisponible. Veuillez renouveler votre demande ultérieurement." -> In reality, account is blocked because of too much wrongpass raise ActionNeeded(u"Compte bloqué") - - self.logger.debug('Unexpected error at login: "%s" (code=%s)' % (msg, error)) + elif error in actionNeeded_codes: + raise ActionNeeded(msg) + elif error == 207: + raise BrowserUnavailable(msg) + else: + assert False, 'Unexpected error at login: "%s" (code=%s)' % (msg, error) def login(self, username, password): url = '/identification-wspl-pres/grille/%s' % self.get('data.grille.idGrille') keyboard = self.browser.open(url) - vk = BNPKeyboard(self, keyboard) + vk = BNPKeyboard(self.browser, keyboard) target = self.browser.BASEURL + 'SEEA-pa01/devServer/seeaserver' user_agent = self.browser.session.headers.get('User-Agent') or '' @@ -209,7 +226,7 @@ def login(self, username, password): idTelematique=username, password=vk.get_string_code(password), clientele=user_agent) - # XXX useless ? + # XXX useless? csrf = self.generate_token() response = self.browser.location(target, data={'AUTH': auth, 'CSRF': csrf}) @@ -791,7 +808,7 @@ class AddRecipPage(BNPPage): def on_load(self): code = cast(self.get('codeRetour'), int) if code: - raise AddRecipientError(message=self.get('message')) + raise AddRecipientBankError(message=self.get('message')) def get_recipient(self, recipient): r = Recipient() diff --git a/modules/boursorama/browser.py b/modules/boursorama/browser.py index c04a1829ee..1d3c19e859 100644 --- a/modules/boursorama/browser.py +++ b/modules/boursorama/browser.py @@ -246,7 +246,7 @@ def get_debit_date(self, debit_date): def get_closing_date(self): for i, j in zip(self.deferred_card_calendar, self.deferred_card_calendar[1:]): - if i[0] < date.today() <= j[0]: + if i[1] < date.today() <= j[1]: return i[0] def get_card_transactions(self, account, coming): diff --git a/modules/boursorama/compat/weboob_capabilities_bank.py b/modules/boursorama/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/boursorama/compat/weboob_capabilities_bank.py +++ b/modules/boursorama/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/boursorama/compat/weboob_tools_capabilities_bank_investments.py b/modules/boursorama/compat/weboob_tools_capabilities_bank_investments.py new file mode 100644 index 0000000000..2b68ff9e00 --- /dev/null +++ b/modules/boursorama/compat/weboob_tools_capabilities_bank_investments.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Jonathan Schmidt +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.tools.compat import basestring +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.bank import Investment + +def is_isin_valid(isin): + """ + Méthode générale + Table de conversion des lettres en chiffres + A=10 B=11 C=12 D=13 E=14 F=15 G=16 H=17 I=18 + J=19 K=20 L=21 M=22 N=23 O=24 P=25 Q=26 R=27 + S=28 T=29 U=30 V=31 W=32 X=33 Y=34 Z=35 + + 1 - Mettre de côté la clé, qui servira de référence à la fin de la vérification. + 2 - Convertir toutes les lettres en nombres via la table de conversion ci-contre. Si le nombre obtenu est supérieur ou égal à 10, prendre les deux chiffres du nombre séparément (exemple : 27 devient 2 et 7). + 3 - Pour chaque chiffre, multiplier sa valeur par deux si sa position est impaire en partant de la droite. Si le nombre obtenu est supérieur ou égal à 10, garder les deux chiffres du nombre séparément (exemple : 14 devient 1 et 4). + 4 - Faire la somme de tous les chiffres. + 5 - Soustraire cette somme de la dizaine supérieure ou égale la plus proche (exemples : si la somme vaut 22, la dizaine « supérieure ou égale » est 30, et la clé vaut donc 8 ; si la somme vaut 30, la dizaine « supérieure ou égale » est 30, et la clé vaut 0 ; si la somme vaut 31, la dizaine « supérieure ou égale » est 40, et la clé vaut 9). + 6 - Comparer la valeur obtenue à la clé mise initialement de côté. + + Étapes 1 et 2 : + F R 0 0 0 3 5 0 0 0 0 (+ 8 : clé) + 15 27 0 0 0 3 5 0 0 0 0 + + Étape 3 : le traitement se fait sur des chiffres + 1 5 2 7 0 0 0 3 5 0 0 0 0 + I P I P I P I P I P I P I : position en partant de la droite (P = Pair, I = Impair) + 2 1 2 1 2 1 2 1 2 1 2 1 2 : coefficient multiplicateur + + 2 5 4 7 0 0 0 3 10 0 0 0 0 : résultat + + Étape 4 : + 2 + 5 + 4 + 7 + 0 + 0 + 0 + 3 + (1 + 0)+ 0 + 0 + 0 + 0 = 22 + + Étapes 5 et 6 : 30 - 22 = 8 (valeur de la clé) + """ + + if not isinstance(isin, basestring): + return False + if not re.match(r'^[A-Z]{2}[A-Z0-9]{9}\d$', isin): + return False + + isin_in_digits = ''.join(str(ord(x) - ord('A') + 10) if not x.isdigit() else x for x in isin[:-1]) + key = isin[-1:] + result = '' + for k, val in enumerate(isin_in_digits[::-1], start=1): + if k % 2 == 0: + result = ''.join((result, val)) + else: + result = ''.join((result, str(int(val)*2))) + return str(sum(int(x) for x in result) + int(key))[-1] == '0' + + +def create_french_liquidity(valuation): + """ + Automatically fills a liquidity investment with label, code and code_type. + """ + liquidity = Investment() + liquidity.label = "Liquidités" + liquidity.code = "XX-liquidity" + liquidity.code_type = NotAvailable + liquidity.valuation = valuation + return liquidity + diff --git a/modules/boursorama/pages.py b/modules/boursorama/pages.py index 79f15916de..f97fabb95a 100644 --- a/modules/boursorama/pages.py +++ b/modules/boursorama/pages.py @@ -35,10 +35,11 @@ ) from weboob.browser.filters.json import Dict from weboob.browser.filters.html import Attr, Link, TableCell, AbsoluteLink -from weboob.capabilities.bank import ( +from .compat.weboob_capabilities_bank import ( Account, Investment, Recipient, Transfer, AccountNotFound, - AddRecipientError, TransferInvalidAmount, + AddRecipientBankError, TransferInvalidAmount, ) +from .compat.weboob_tools_capabilities_bank_investments import create_french_liquidity from weboob.capabilities.base import NotAvailable, empty, Currency from weboob.capabilities.profile import Person from weboob.tools.capabilities.bank.transactions import FrenchTransaction @@ -366,7 +367,8 @@ class HistoryPage(LoggedPage, HTMLPage): class iter_history(ListElement): item_xpath = '//ul[has-class("list__movement")]/li[div and not(contains(@class, "summary")) \ and not(contains(@class, "graph")) \ - and not (contains(@class, "separator"))]' + and not (contains(@class, "separator")) \ + and not (contains(@class, "list__movement__line--deffered")) ]' class item(ItemElement): klass = Transaction @@ -577,11 +579,7 @@ def obj_unitvalue(self): def iter_investment(self): valuation = CleanDecimal('//li[h4[contains(text(), "Solde Espèces")]]/h3', replace_dots=True, default=None)(self.doc) if valuation: - inv = Investment() - inv.code = u"XX-liquidity" - inv.label = u"Liquidités" - inv.valuation = valuation - yield inv + yield create_french_liquidity(valuation) for inv in self.get_investment(): yield inv @@ -812,31 +810,35 @@ class get_profile(ItemElement): class CardsNumberPage(LoggedPage, HTMLPage): def populate_cards_number(self, cards): - # Checking account ID used to match credit cards - # Label useful when several cards on the same checking account - labels_ids = [ - ( - CleanText('.', replace=[('DEBIT DIFFERE ', '')])(o), - re.search(r'/limite/(\w+)/', o.attrib['href']).group(1) - ) - for o in self.doc.xpath('//select//option') + """ + Checking account ID used to match credit cards. + Label useful when several cards on the same checking account. + The card_details list contains tuples including the card number, + the hash of the card's parent account, and the name of the card. + """ + card_details = [ + (CleanText('.//span')(o), CleanText('./@data-account-key')(o), CleanText('.//p')(o)) + for o in self.doc.xpath('//div[contains(@class, "zoom-carousel__item text-center credit-card-carousel__item")]') ] - # remove cards whose number ends with **** (means it is not activated yet) - labels_ids = [ - label_id for label_id in labels_ids - if not label_id[0].endswith('****') + # Remove cards whose number ends with **** (means it is not activated yet) + card_details = [ + (number, account_hash, name) for (number, account_hash, name) in card_details + if not number.endswith('****') ] for card in cards: - match = [ - label for label, account_id in labels_ids - if card.label in label and account_id in card.url - ] + match = [(number, account_hash, name) for (number, account_hash, name) in card_details if account_hash in card.url] + + if len(match) > 1: + # Several cards matched the same bank account, so we now try + # to match them using the name of the card holder: + card_username = card.label.split('-')[1].lstrip() + match = [(number, account_hash, name) for (number, account_hash, name) in match if name.startswith(card_username)] + assert len(match) <= 1, "only one card should be matched, or zero if the card is not yet activated" if len(match) == 1 : - card_label = match[0] - card.number = re.search('(\d{4}\*{8}(\d{4}|\*{4}))', card_label).group(1) + card.number = match[0][0] class HomePage(LoggedPage, HTMLPage): @@ -983,7 +985,7 @@ def on_load(self): err = CleanText('//div[@class="form-errors"]', default=None)(self.doc) if err: - raise AddRecipientError(message=err) + raise AddRecipientBankError(message=err) def _is_form(self, **kwargs): try: diff --git a/modules/bouygues/pages.py b/modules/bouygues/pages.py index 965ea584b3..a172b00350 100644 --- a/modules/bouygues/pages.py +++ b/modules/bouygues/pages.py @@ -22,7 +22,7 @@ from datetime import datetime, timedelta from weboob.capabilities.messages import CantSendMessage -from weboob.exceptions import BrowserIncorrectPassword +from weboob.exceptions import BrowserIncorrectPassword, ParseError from weboob.capabilities.base import NotLoaded from weboob.capabilities.bill import Bill, Subscription @@ -138,7 +138,16 @@ class item(ItemElement): klass = Bill obj_id = Format('%s_%s', Env('subid'), Dict('idFacture')) - obj_url = Format('https://api.bouyguestelecom.fr%s', Dict('_links/facturePDF/href')) + + def obj_url(self): + try: + link = Dict('_links/facturePDF/href')(self) + except ParseError: + # yes, sometimes it's just a misspelling word, but just sometimes... + link = Dict('_links/facturePDFDF/href')(self) + + return 'https://api.bouyguestelecom.fr%s' % link + obj_date = Env('date') obj_duedate = Env('duedate') obj_format = 'pdf' diff --git a/modules/bp/browser.py b/modules/bp/browser.py index 58d2923704..3033866d3f 100644 --- a/modules/bp/browser.py +++ b/modules/bp/browser.py @@ -251,6 +251,16 @@ def get_accounts_list(self): for loan in self.page.iter_revolving_loans(): loan.currency = account.currency accounts.append(loan) + page.go() + + elif account.type == Account.TYPE_PERP: + # PERP balances must be fetched from the details page, + # otherwise we just scrape the "Rente annuelle estimée": + balance = self.open(account.url).page.get_balance() + if balance is not None: + account.balance = balance + accounts.append(account) + else: accounts.append(account) if account.type == Account.TYPE_CHECKING and account._has_cards: @@ -311,13 +321,16 @@ def get_history(self, account): Account.TYPE_MARKET: self.par_account_savings_and_invests_history }.get(account.type) - if history is not None: + if history is not None and account.label != 'COMPTE ATTENTE': history.go(accountId=account.id) # TODO be smarter by avoid fetching all, sorting all and returning all if only coming were desired if hasattr(self.page, 'iter_transactions') and self.page.has_transactions(): return self.page.iter_transactions() + elif account.type == Account.TYPE_PERP and self.retirement_hist.is_here(): + return self.page.get_history() + return [] diff --git a/modules/bp/compat/weboob_capabilities_bank.py b/modules/bp/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/bp/compat/weboob_capabilities_bank.py +++ b/modules/bp/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/bp/pages/accounthistory.py b/modules/bp/pages/accounthistory.py index 6f3961bee4..fd6b1bc3e4 100644 --- a/modules/bp/pages/accounthistory.py +++ b/modules/bp/pages/accounthistory.py @@ -254,6 +254,10 @@ def on_load(self): if link: self.browser.location(link) + def get_balance(self): + return CleanDecimal(default=None, replace_dots=True).filter( + self.doc.xpath('//dt[span[text()="Total des versements bruts"]]/following::dd[1]/span/strong/text()')) + class InvestTable(TableElement): col_label = 'Support' diff --git a/modules/bp/pages/accountlist.py b/modules/bp/pages/accountlist.py index c4d00961d9..4160eb0e1c 100644 --- a/modules/bp/pages/accountlist.py +++ b/modules/bp/pages/accountlist.py @@ -104,8 +104,8 @@ def obj_type(self): 'comptes? titres? et pea': Account.TYPE_MARKET, 'compte-titres': Account.TYPE_MARKET, 'assurances? vie': Account.TYPE_LIFE_INSURANCE, - u'prêt': Account.TYPE_LOAN, - u'crédits?': Account.TYPE_LOAN, + 'prêt': Account.TYPE_LOAN, + 'crédits?': Account.TYPE_LOAN, 'plan d\'epargne en actions': Account.TYPE_PEA, 'comptes? attente': Account.TYPE_CHECKING, 'perp': Account.TYPE_PERP, @@ -222,8 +222,8 @@ def obj_id(self): def obj_label(self): cell = TableCell('label', default=None)(self) if cell: - return CleanText(cell, default=NotAvailable)(self) - return CleanText('//form[contains(@action, "detaillerOffre") or contains(@action, "detaillerPretPartenaireListe-encoursPrets.ea")]/div[@class="bloc Tmargin"]/h2[@class="title-level2"]')(self) + return CleanText(cell, default=NotAvailable)(self).upper() + return CleanText('//form[contains(@action, "detaillerOffre") or contains(@action, "detaillerPretPartenaireListe-encoursPrets.ea")]/div[@class="bloc Tmargin"]/h2[@class="title-level2"]')(self).upper() def obj_balance(self): if CleanText(TableCell('balance'))(self) != u'Remboursé intégralement': diff --git a/modules/bp/pages/base.py b/modules/bp/pages/base.py index 84f6a832b1..8f316a89df 100644 --- a/modules/bp/pages/base.py +++ b/modules/bp/pages/base.py @@ -21,7 +21,6 @@ from weboob.browser.pages import HTMLPage class MyHTMLPage(HTMLPage): - ENCODING = 'iso-8859-1' def on_load(self): deconnexion = self.doc.xpath('//iframe[contains(@id, "deconnexion")] | //p[@class="txt" and contains(text(), "Session expir")]') diff --git a/modules/bp/pages/compat/__init__.py b/modules/bp/pages/compat/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/modules/bp/pages/compat/weboob_capabilities_bank.py b/modules/bp/pages/compat/weboob_capabilities_bank.py new file mode 100644 index 0000000000..141097d781 --- /dev/null +++ b/modules/bp/pages/compat/weboob_capabilities_bank.py @@ -0,0 +1,45 @@ + +import weboob.capabilities.bank as OLD + +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) + + +__all__ = OLD.__all__ + + +class CapBankWealth(CapBank): + pass + + +class CapBankPockets(CapBank): + pass + + +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + +class CapBankTransfer(OLD.CapBankTransfer): + def transfer_check_label(self, old, new): + from unidecode import unidecode + + return unidecode(old) == unidecode(new) + + +class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipient): + pass + + +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + +Account.TYPE_MORTGAGE = 17 +Account.TYPE_CONSUMER_CREDIT = 18 +Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/bp/pages/login.py b/modules/bp/pages/login.py index 3a3ce789f2..05fad66038 100644 --- a/modules/bp/pages/login.py +++ b/modules/bp/pages/login.py @@ -95,7 +95,7 @@ def login(self, login, pwd): form = self.get_form(name='formAccesCompte') form['password'] = vk.get_string_code(pwd) - form['username'] = login.encode(self.ENCODING) + form['username'] = login form.submit() diff --git a/modules/bp/pages/subscription.py b/modules/bp/pages/subscription.py index 49715354dd..69ab822ca5 100644 --- a/modules/bp/pages/subscription.py +++ b/modules/bp/pages/subscription.py @@ -29,9 +29,6 @@ class SubscriptionPage(LoggedPage, HTMLPage): - # encoding is wrong on the page - ENCODING='ISO-8859-1' - # because of freaking JS from hell STATEMENT_TYPES = ('RCE', 'RPT', 'RCO') @@ -118,7 +115,7 @@ class item(ItemElement): klass = Subscription obj_label = CleanText('.') - obj_id = Regexp(Field('label'), r'\w - (\w+)') + obj_id = Regexp(Field('label'), r'\w? ?- (\w+)') obj_subscriber = Env('subscriber') obj__number = Attr('.', 'value') diff --git a/modules/bp/pages/transfer.py b/modules/bp/pages/transfer.py index a1bbc1d384..0312424300 100644 --- a/modules/bp/pages/transfer.py +++ b/modules/bp/pages/transfer.py @@ -20,8 +20,9 @@ from datetime import datetime -from weboob.capabilities.bank import ( - TransferError, TransferBankError, Transfer, TransferStep, NotAvailable, Recipient, AccountNotFound, AddRecipientError +from .compat.weboob_capabilities_bank import ( + TransferError, TransferBankError, Transfer, TransferStep, NotAvailable, Recipient, + AccountNotFound, AddRecipientBankError ) from weboob.capabilities.base import find_object from weboob.browser.pages import LoggedPage @@ -196,7 +197,7 @@ def choose_country(self, recipient, is_bp_account): # if this is present, we can't add recipient currently more_security_needed = self.doc.xpath(u'//iframe[@title="Gestion de compte par Internet"]') if more_security_needed: - raise AddRecipientError(message=u"Pour activer le service Certicode, nous vous invitons à vous rapprocher de votre Conseiller en Bureau de Poste.") + raise AddRecipientBankError(message=u"Pour activer le service Certicode, nous vous invitons à vous rapprocher de votre Conseiller en Bureau de Poste.") form = self.get_form(name='SaisiePaysBeneficiaireVirement') form['compteLBP'] = str(is_bp_account).lower() @@ -233,7 +234,7 @@ def on_load(self): error_msg = CleanText('//h2[contains(text(), "Compte rendu")]/following-sibling::p')(self.doc) if error_msg: - raise AddRecipientError(message=error_msg) + raise AddRecipientBankError(message=error_msg) def set_browser_form(self): form = self.get_form(name='SaisieOTP') diff --git a/modules/bred/bred/browser.py b/modules/bred/bred/browser.py index 07f8599b4c..47818ab48e 100644 --- a/modules/bred/bred/browser.py +++ b/modules/bred/bred/browser.py @@ -198,7 +198,7 @@ def get_history(self, account, coming=False): self.logger.debug('stopping coming after %s', t) return - next_page = bool(transactions) + next_page = len(transactions) == 50 offset += 50 # This assert supposedly prevents infinite loops, diff --git a/modules/bred/bred/pages.py b/modules/bred/bred/pages.py index 509af92611..eb5bb2d775 100644 --- a/modules/bred/bred/pages.py +++ b/modules/bred/bred/pages.py @@ -240,8 +240,15 @@ def iter_history(self, account, operation_list, seen, today, coming): if t.type == Transaction.TYPE_CARD and account.type == Account.TYPE_CARD: t.type = Transaction.TYPE_DEFERRED_CARD - transactions.append(t) - + if abs(t.rdate.year - t.date.year) < 2: + transactions.append(t) + else: + self.logger.warning( + 'rdate(%s) and date(%s) of the transaction %s are far far away, skip it', + t.rdate, + t.date, + t.label + ) return transactions diff --git a/modules/bred/compat/weboob_capabilities_bank.py b/modules/bred/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/bred/compat/weboob_capabilities_bank.py +++ b/modules/bred/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/caels/browser.py b/modules/caels/browser.py index 266e4f0e37..e77af0e673 100644 --- a/modules/caels/browser.py +++ b/modules/caels/browser.py @@ -24,3 +24,9 @@ class CAELSBrowser(AbstractBrowser): PARENT = 'amundi' PARENT_ATTR = 'package.browser.AmundiBrowser' + BASEURL = "https://www.ca-els.com/psf/" + + + def __init__(self, *args, **kwargs): + super(CAELSBrowser, self).__init__(*args, **kwargs) + self.weboob = kwargs.get('weboob') diff --git a/modules/caels/compat/weboob_capabilities_bank.py b/modules/caels/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/caels/compat/weboob_capabilities_bank.py +++ b/modules/caels/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/caels/module.py b/modules/caels/module.py index da111a2810..1274d40972 100644 --- a/modules/caels/module.py +++ b/modules/caels/module.py @@ -43,15 +43,13 @@ class CaelsModule(Module, CapBankWealth): BROWSER = CAELSBrowser def create_default_browser(self): - return self.create_browser("https://www.ca-els.com/", - self.config['login'].get(), + return self.create_browser(self.config['login'].get(), self.config['password'].get(), weboob=self.weboob) def get_account(self, id): return find_object(self.iter_accounts(), id=id, error=AccountNotFound) - def iter_accounts(self): return self.browser.iter_accounts() diff --git a/modules/caels/pages.py b/modules/caels/pages.py new file mode 100644 index 0000000000..5d01743490 --- /dev/null +++ b/modules/caels/pages.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Quentin Defenouillere +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.browser.elements import ItemElement, method, DictElement +from weboob.browser.filters.standard import CleanDecimal, Date, Field, Env +from weboob.browser.filters.json import Dict +from weboob.browser.pages import AbstractPage +from weboob.capabilities.bank import Investment +from weboob.capabilities.base import NotAvailable +from weboob.tools.capabilities.bank.investments import is_isin_valid + + +class AccountsPage(AbstractPage): + PARENT = 'amundi' + PARENT_URL = 'accounts' + + @method + class iter_investments(DictElement): + def find_elements(self): + for psds in Dict('listPositionsSalarieFondsDto')(self): + for psd in psds.get('positionsSalarieDispositifDto'): + if psd.get('codeDispositif') == Env('account_id')(self): + return psd.get('positionsSalarieFondsDto') + return {} + + class item(ItemElement): + klass = Investment + + obj_label = Dict('libelleFonds') + obj_unitvalue = Dict('vl') & CleanDecimal + obj_quantity = Dict('nbParts') & CleanDecimal + obj_valuation = Dict('mtBrut') & CleanDecimal + obj_code = Dict('codeIsin', default=NotAvailable) + obj_vdate = Date(Dict('dtVl')) + # The "diff" is only present on the CAELS website but not its parent (amundi): + obj_diff = Dict('mtPMV') & CleanDecimal + + def obj_code_type(self): + if is_isin_valid(Field('code')(self)): + return Investment.CODE_TYPE_ISIN + return NotAvailable diff --git a/modules/caissedepargne/browser.py b/modules/caissedepargne/browser.py index 8298fd3cd3..86dda7134a 100644 --- a/modules/caissedepargne/browser.py +++ b/modules/caissedepargne/browser.py @@ -29,12 +29,13 @@ from weboob.browser.browsers import LoginBrowser, need_login, StatesMixin from .compat.weboob_browser_switch import SiteSwitch from weboob.browser.url import URL -from weboob.capabilities.bank import Account, AddRecipientStep, Recipient, TransferBankError, Transaction, Investment +from weboob.capabilities.bank import Account, AddRecipientStep, Recipient, TransferBankError, Transaction from weboob.capabilities.base import NotAvailable from weboob.capabilities.profile import Profile from weboob.browser.exceptions import BrowserHTTPNotFound, ClientError from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable from weboob.tools.capabilities.bank.transactions import sorted_transactions, FrenchTransaction +from .compat.weboob_tools_capabilities_bank_investments import create_french_liquidity from weboob.tools.compat import urljoin from weboob.tools.value import Value from weboob.tools.decorators import retry @@ -46,7 +47,7 @@ ProTransferSummaryPage, ProAddRecipientOtpPage, ProAddRecipientPage, SmsPage, SmsPageOption, SmsRequest, AuthentPage, RecipientPage, CanceledAuth, CaissedepargneKeyboard, TransactionsDetailsPage, LoadingPage, ConsLoanPage, MeasurePage, NatixisLIHis, NatixisLIInv, NatixisRedirectPage, - SubscriptionPage, + SubscriptionPage, CreditCooperatifMarketPage, ) from .linebourse_browser import LinebourseBrowser @@ -60,6 +61,8 @@ class CaisseEpargne(LoginBrowser, StatesMixin): STATE_DURATION = 5 HISTORY_MAX_PAGE = 200 + LINEBOURSE_BROWSER = LinebourseBrowser + login = URL('/authentification/manage\?step=identification&identifiant=(?P.*)', 'https://.*/login.aspx', LoginPage) account_login = URL('/authentification/manage\?step=account&identifiant=(?P.*)&account=(?P.*)', LoginPage) @@ -86,6 +89,7 @@ class CaisseEpargne(LoginBrowser, StatesMixin): market = URL('https://.*/Pages/Bourse.*', 'https://www.caisse-epargne.offrebourse.com/ReroutageSJR', 'https://www.caisse-epargne.offrebourse.com/Portefeuille.*', MarketPage) + creditcooperatif_market = URL('https://www.offrebourse.com/.*', CreditCooperatifMarketPage) # just to catch the landing page of the Credit Cooperatif's Linebourse natixis_redirect = URL(r'/NaAssuranceRedirect/NaAssuranceRedirect.aspx', r'https://www.espace-assurances.caisse-epargne.fr/espaceinternet-ce/views/common/routage-itce.xhtml\?windowId=automatedEntryPoint', NatixisRedirectPage) @@ -103,6 +107,26 @@ class CaisseEpargne(LoginBrowser, StatesMixin): request_sms = URL('https://www.icgauth.caisse-epargne.fr/dacsrest/api/v1u0/transaction/(?P)', SmsRequest) __states__ = ('BASEURL', 'multi_type', 'typeAccount', 'is_cenet_website', 'recipient_form', 'is_send_sms') + # Accounts managed in life insurance space (not in linebourse) + + insurance_accounts = ('AIKIDO', + 'ASSURECUREUIL', + 'ECUREUIL PROJET', + 'GARANTIE RETRAITE EU', + 'INITIATIVES PLUS', + 'INITIATIVES TRANSMIS', + 'LIVRET ASSURANCE VIE', + 'OCEOR EVOLUTION', + 'PATRIMONIO CRESCENTE', + 'PEP TRANSMISSION', + 'PERP', + 'PERSPECTIVES ECUREUI', + 'POINTS RETRAITE ECUR', + 'RICOCHET', + 'SOLUTION PERP', + 'TENDANCES', + 'YOGA', ) + def __init__(self, nuser, *args, **kwargs): self.BASEURL = kwargs.pop('domain', self.BASEURL) if not self.BASEURL.startswith('https://'): @@ -112,17 +136,30 @@ def __init__(self, nuser, *args, **kwargs): self.multi_type = False self.accounts = None self.loans = None - self.typeAccount = 'WE' + self.typeAccount = None + self.inexttype = 0 # keep track of index in the connection type's list self.nuser = nuser self.recipient_form = None self.is_send_sms = None self.weboob = kwargs['weboob'] + self.market_url = kwargs.pop( + 'market_url', + 'https://www.caisse-epargne.offrebourse.com', + ) + super(CaisseEpargne, self).__init__(*args, **kwargs) dirname = self.responses_dirname if dirname: dirname += '/bourse' - self.linebourse = LinebourseBrowser('https://www.caisse-epargne.offrebourse.com', logger=self.logger, responses_dirname=dirname, weboob=self.weboob, proxy=self.PROXIES) + + self.linebourse = self.LINEBOURSE_BROWSER( + self.market_url, + logger=self.logger, + responses_dirname=dirname, + weboob=self.weboob, + proxy=self.PROXIES, + ) def deleteCTX(self): # For connection to offrebourse and natixis, we need to delete duplicate of CTX cookie @@ -152,30 +189,67 @@ def do_login(self): Attempt to log in. Note: this method does nothing if we are already logged in. """ + # Among the parameters used during the login step, there is + # a connection type (called typeAccount) that can take the + # following values: + # WE: espace particulier + # WP: espace pro + # WM: personnes protégées + # EU: Cenet + # + # A connection can have one connection type as well as many of + # them. There is an issue when there is many connection types: + # the connection type to use can't be guessed in advance, we + # have to test all of them until the login step is successful + # (sometimes all connection type can be used for the login, sometimes + # only one will work). + # + # For simplicity's sake, we try each connection type from first to + # last (they are returned in a list by the first request) + # + # Examples of connection types combination that have been seen so far: + # [WE] + # [WP] + # [WE, WP] + # [WE, WP, WM] + # [WP, WM] + # [EU] + # [EU, WE] (EU tends to come first when present) + if not self.username or not self.password: raise BrowserIncorrectPassword() - # Reset domain to log on pro website if first login attempt failed on personal website. - if self.multi_type: - self.BASEURL = 'https://www.caisse-epargne.fr' - self.typeAccount = 'WP' - + # Retrieve the list of types: can contain a single type or more + # - when there is a single type: all the information are available + # - when there are several types: an additional request is needed data = self.login.go(login=self.username).get_response() if data is None: raise BrowserIncorrectPassword() - if "authMode" in data and data['authMode'] == 'redirect': - raise SiteSwitch('cenet') - - if len(data['account']) > 1: + if len(data.get('account', [])) > 1: + # Additional request when there is more than one connection type + # to "choose" from the list of connection types self.multi_type = True + + if self.inexttype < len(data['account']): + self.typeAccount = data['account'][self.inexttype] + else: + assert False, 'should have logged in with at least one connection type' + self.inexttype += 1 + data = self.account_login.go(login=self.username, accountType=self.typeAccount).get_response() assert data is not None + if data.get('authMode', '') == 'redirect': # the connection type EU could also be used as a criteria + raise SiteSwitch('cenet') + typeAccount = data['account'][0] + if self.multi_type: + assert typeAccount == self.typeAccount + idTokenClavier = data['keyboard']['Id'] vk = CaissedepargneKeyboard(data['keyboard']['ImageClavier'], data['keyboard']['Num']['string']) newCodeConf = vk.get_string_code(self.password) @@ -185,10 +259,10 @@ def do_login(self): 'newCodeConf': newCodeConf, 'auth_mode': 'ajax', 'nuusager': self.nuser.encode('utf-8'), - 'codconf': self.password, + 'codconf': '', # must be present though empty 'typeAccount': typeAccount, 'step': 'authentification', - 'ctx': 'typsrv=WE', + 'ctx': 'typsrv={}'.format(typeAccount), 'clavierSecurise': '1', 'nuabbd': self.username } @@ -205,8 +279,8 @@ def do_login(self): assert response is not None if not response['action']: - if not self.typeAccount == 'WP' and self.multi_type: - # If we haven't test PRO espace we check before raising wrong pass + if self.multi_type: + # try to log in with the next connection type's value self.do_login() return raise BrowserIncorrectPassword(response['error']) @@ -266,6 +340,11 @@ def get_measure_accounts_list(self): return self.accounts + def update_linebourse_token(self): + assert self.linebourse is not None, "linebourse browser should already exist" + self.linebourse.session.cookies.update(self.session.cookies) + self.linebourse.session.headers['X-XSRF-TOKEN'] = self.session.cookies['XSRF-TOKEN'] + @need_login @retry(ClientError, tries=3) def get_accounts_list(self): @@ -295,14 +374,27 @@ def get_accounts_list(self): self.page.submit() - if self.page.is_error(): - continue + # For Caisse d'Epargne's connections + if self.url.startswith('https://www.caisse-epargne.offrebourse.com') : + if self.page.is_error(): + continue - self.garbage.go() + self.garbage.go() - if self.garbage.is_here(): - continue - self.page.get_valuation_diff(account) + if self.garbage.is_here(): + continue + + self.page.get_valuation_diff(account) + + # For CreditCooperatif's connections + elif self.url.startswith('https://www.offrebourse.com'): + self.update_linebourse_token() + self.linebourse.handle_cgu() + page = self.linebourse.go_portfolio() + assert self.linebourse.portfolio.is_here() + account.valuation_diff = page.get_valuation_diff() + else: + assert False, "new domain that hasn't been seen so far ?" # Some accounts have no available balance or label and cause issues # in the backend so we must exclude them from the accounts list: @@ -325,7 +417,9 @@ def get_loans_list(self): if self.home.is_here(): if not self.page.is_access_error(): self.page.go_loan_list() - self.loans = list(self.page.get_loan_list()) + self.loans = list(self.page.get_real_estate_loans()) + self.page.go_loan_list() + self.loans.extend(self.page.get_loan_list()) for _ in range(3): try: @@ -411,21 +505,23 @@ def _get_history_invests(self, account): self.page.go_history(account._info) - if account.type == Account.TYPE_LIFE_INSURANCE: + if account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_PERP): if "MILLEVIE" in account.label: self.page.go_life_insurance(account) label = account.label.split()[-1] self.natixis_life_ins_his.go(id1=label[:3], id2=label[3:5], id3=account.id) return sorted_transactions(self.page.get_history()) - if account.label.startswith('NUANCES '): + if account.label.startswith('NUANCES ') or account.label in self.insurance_accounts: self.page.go_life_insurance(account) + if 'JSESSIONID' in self.session.cookies: + # To access the life insurance space, we need to delete the JSESSIONID cookie to avoid an expired session + del self.session.cookies['JSESSIONID'] try: - if not self.market.is_here() and not self.message.is_here(): + if not self.life_insurance.is_here() and not self.message.is_here(): # life insurance website is not always available raise BrowserUnavailable() - self.page.submit() self.location('https://www.extranet2.caisse-epargne.fr%s' % self.page.get_cons_histo()) @@ -448,8 +544,14 @@ def get_history(self, account): self.page.go_history(account._info) if "Bourse" in self.url: self.page.submit() - self.linebourse.session.cookies.update(self.session.cookies) - return self.linebourse.iter_history(re.sub('[^0-9]', '', account.id)) + + if self.url.startswith('https://www.caisse-epargne.offrebourse.com'): + self.linebourse.session.cookies.update(self.session.cookies) + return self.linebourse.iter_history(re.sub('[^0-9]', '', account.id)) + elif self.url.startswith('https://www.offrebourse.com'): + self.update_linebourse_token() + return self.linebourse.iter_history() + return self._get_history(account._info) @need_login @@ -473,11 +575,7 @@ def get_investment(self, account): raise NotImplementedError() if account.type == Account.TYPE_PEA and account.label == 'PEA NUMERAIRE': - liquidity = Investment() - liquidity.label = 'Liquidités' - liquidity.code = 'XX-liquidity' - liquidity.valuation = account.balance - yield liquidity + yield create_french_liquidity(account.balance) return if self.home.is_here(): @@ -491,6 +589,14 @@ def get_investment(self, account): if not self.market.is_here(): return self.page.submit() + + # For Credit Cooperatif's connections + if self.url.startswith('https://www.offrebourse.com'): + self.update_linebourse_token() + for investment in self.linebourse.iter_investments(): + yield investment + return + if self.page.is_error(): return self.location('https://www.caisse-epargne.offrebourse.com/Portefeuille') @@ -557,8 +663,20 @@ def iter_recipients(self, origin_account): self.pre_transfer(origin_account) except TransferBankError: return [] - if self.page.transfer_unavailable() or self.page.need_auth() or not self.page.can_transfer(origin_account): + + go_transfer_errors = ( + # redirected to home page because: + # - need to relogin, see `self.page.need_auth()` + # - need more security, see `self.page.transfer_unavailable()` + # - transfer is not available for this connection, see `self.page.go_transfer_via_history()` + # TransferPage inherit from IndexPage so self.home.is_here() is true, check page type to avoid this problem + type(self.page) is IndexPage, + # check if origin_account have recipients + self.transfer.is_here() and not self.page.can_transfer(origin_account), + ) + if any(go_transfer_errors): return [] + return self.page.iter_recipients(account_id=origin_account.id) def pre_transfer(self, account): diff --git a/modules/caissedepargne/cenet/browser.py b/modules/caissedepargne/cenet/browser.py index ee2ecff68e..e42aee2dfc 100644 --- a/modules/caissedepargne/cenet/browser.py +++ b/modules/caissedepargne/cenet/browser.py @@ -17,6 +17,8 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals + import json from weboob.browser import LoginBrowser, need_login, StatesMixin @@ -40,12 +42,13 @@ class CenetBrowser(LoginBrowser, StatesMixin): BASEURL = "https://www.cenet.caisse-epargne.fr" + STATE_DURATION = 5 login = URL(r'https://(?P[^/]+)/authentification/manage\?step=identification&identifiant=(?P.*)', r'https://.*/authentification/manage\?step=identification&identifiant=.*', r'https://.*/login.aspx', LoginPage) - account_login = URL('/authentification/manage\?step=account&identifiant=(?P.*)&account=(?P.*)', LoginPage) + account_login = URL('https://(?P[^/]+)/authentification/manage\?step=account&identifiant=(?P.*)&account=(?P.*)', LoginPage) cenet_vk = URL('https://www.cenet.caisse-epargne.fr/Web/Api/ApiAuthentification.asmx/ChargerClavierVirtuel') cenet_home = URL('/Default.aspx$', CenetHomePage) cenet_accounts = URL('/Web/Api/ApiComptes.asmx/ChargerSyntheseComptes', CenetAccountsPage) @@ -79,8 +82,18 @@ def __init__(self, nuser, *args, **kwargs): def do_login(self): data = self.login.go(login=self.username, domain=self.login_domain).get_response() + if len(data['account']) > 1: + # additional request where there is more than one + # connection type (called typeAccount) + # TODO: test all connection type values if needed + account_type = data['account'][0] + self.account_login.go(login=self.username, accountType=account_type, domain=self.login_domain) + data = self.page.get_response() + if data is None: raise BrowserIncorrectPassword() + elif not self.nuser: + raise BrowserIncorrectPassword("Erreur: Numéro d'utilisateur requis.") assert "authMode" in data and data['authMode'] == 'redirect', 'should not be on the cenet website' diff --git a/modules/caissedepargne/compat/weboob_capabilities_bank.py b/modules/caissedepargne/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/caissedepargne/compat/weboob_capabilities_bank.py +++ b/modules/caissedepargne/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/caissedepargne/compat/weboob_tools_capabilities_bank_investments.py b/modules/caissedepargne/compat/weboob_tools_capabilities_bank_investments.py new file mode 100644 index 0000000000..2b68ff9e00 --- /dev/null +++ b/modules/caissedepargne/compat/weboob_tools_capabilities_bank_investments.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Jonathan Schmidt +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.tools.compat import basestring +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.bank import Investment + +def is_isin_valid(isin): + """ + Méthode générale + Table de conversion des lettres en chiffres + A=10 B=11 C=12 D=13 E=14 F=15 G=16 H=17 I=18 + J=19 K=20 L=21 M=22 N=23 O=24 P=25 Q=26 R=27 + S=28 T=29 U=30 V=31 W=32 X=33 Y=34 Z=35 + + 1 - Mettre de côté la clé, qui servira de référence à la fin de la vérification. + 2 - Convertir toutes les lettres en nombres via la table de conversion ci-contre. Si le nombre obtenu est supérieur ou égal à 10, prendre les deux chiffres du nombre séparément (exemple : 27 devient 2 et 7). + 3 - Pour chaque chiffre, multiplier sa valeur par deux si sa position est impaire en partant de la droite. Si le nombre obtenu est supérieur ou égal à 10, garder les deux chiffres du nombre séparément (exemple : 14 devient 1 et 4). + 4 - Faire la somme de tous les chiffres. + 5 - Soustraire cette somme de la dizaine supérieure ou égale la plus proche (exemples : si la somme vaut 22, la dizaine « supérieure ou égale » est 30, et la clé vaut donc 8 ; si la somme vaut 30, la dizaine « supérieure ou égale » est 30, et la clé vaut 0 ; si la somme vaut 31, la dizaine « supérieure ou égale » est 40, et la clé vaut 9). + 6 - Comparer la valeur obtenue à la clé mise initialement de côté. + + Étapes 1 et 2 : + F R 0 0 0 3 5 0 0 0 0 (+ 8 : clé) + 15 27 0 0 0 3 5 0 0 0 0 + + Étape 3 : le traitement se fait sur des chiffres + 1 5 2 7 0 0 0 3 5 0 0 0 0 + I P I P I P I P I P I P I : position en partant de la droite (P = Pair, I = Impair) + 2 1 2 1 2 1 2 1 2 1 2 1 2 : coefficient multiplicateur + + 2 5 4 7 0 0 0 3 10 0 0 0 0 : résultat + + Étape 4 : + 2 + 5 + 4 + 7 + 0 + 0 + 0 + 3 + (1 + 0)+ 0 + 0 + 0 + 0 = 22 + + Étapes 5 et 6 : 30 - 22 = 8 (valeur de la clé) + """ + + if not isinstance(isin, basestring): + return False + if not re.match(r'^[A-Z]{2}[A-Z0-9]{9}\d$', isin): + return False + + isin_in_digits = ''.join(str(ord(x) - ord('A') + 10) if not x.isdigit() else x for x in isin[:-1]) + key = isin[-1:] + result = '' + for k, val in enumerate(isin_in_digits[::-1], start=1): + if k % 2 == 0: + result = ''.join((result, val)) + else: + result = ''.join((result, str(int(val)*2))) + return str(sum(int(x) for x in result) + int(key))[-1] == '0' + + +def create_french_liquidity(valuation): + """ + Automatically fills a liquidity investment with label, code and code_type. + """ + liquidity = Investment() + liquidity.label = "Liquidités" + liquidity.code = "XX-liquidity" + liquidity.code_type = NotAvailable + liquidity.valuation = valuation + return liquidity + diff --git a/modules/caissedepargne/linebourse_browser.py b/modules/caissedepargne/linebourse_browser.py index c878ebbadb..89cd8db190 100644 --- a/modules/caissedepargne/linebourse_browser.py +++ b/modules/caissedepargne/linebourse_browser.py @@ -22,3 +22,4 @@ class LinebourseBrowser(AbstractBrowser): PARENT = 'linebourse' + PARENT_ATTR = 'package.browser.LinebourseBrowser' diff --git a/modules/caissedepargne/pages.py b/modules/caissedepargne/pages.py index 517b449074..5159dd4fcf 100644 --- a/modules/caissedepargne/pages.py +++ b/modules/caissedepargne/pages.py @@ -30,12 +30,15 @@ from weboob.browser.pages import LoggedPage, HTMLPage, JsonPage, pagination, FormNotFound from weboob.browser.elements import ItemElement, method, ListElement, TableElement, SkipItem, DictElement -from weboob.browser.filters.standard import Date, CleanDecimal, Regexp, CleanText, Env, Upper, Field, Eval, Format +from weboob.browser.filters.standard import Date, CleanDecimal, Regexp, CleanText, Env, Upper, Field, Eval, Format, Currency from weboob.browser.filters.html import Link, Attr, TableCell from weboob.capabilities import NotAvailable -from weboob.capabilities.bank import Account, Investment, Recipient, TransferError, TransferBankError, Transfer,\ - AddRecipientError, Loan +from .compat.weboob_capabilities_bank import ( + Account, Investment, Recipient, TransferError, TransferBankError, Transfer, + AddRecipientBankError, Loan, +) from weboob.capabilities.bill import Subscription, Document +from weboob.tools.capabilities.bank.investments import is_isin_valid from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.tools.capabilities.bank.iban import is_rib_valid, rib2iban, is_iban_valid from weboob.tools.captcha.virtkeyboard import GridVirtKeyboard @@ -43,9 +46,17 @@ from weboob.exceptions import NoAccountsException, BrowserUnavailable from weboob.browser.filters.json import Dict +def MyDecimal(*args, **kwargs): + kwargs.update(replace_dots=True) + return CleanDecimal(*args, **kwargs) + +class MyTableCell(TableCell): + def __init__(self, *names, **kwargs): + super(MyTableCell, self).__init__(*names, **kwargs) + self.td = './tr[%s]/td' def fix_form(form): - keys = ['MM$HISTORIQUE_COMPTE$btnCumul','Cartridge$imgbtnMessagerie','MM$m_CH$ButtonImageFondMessagerie', + keys = ['MM$HISTORIQUE_COMPTE$btnCumul', 'Cartridge$imgbtnMessagerie', 'MM$m_CH$ButtonImageFondMessagerie', 'MM$m_CH$ButtonImageMessagerie'] for name in keys: form.pop(name, None) @@ -171,6 +182,12 @@ class IndexPage(LoggedPage, HTMLPage): u'LIVRET DEV.DURABLE': Account.TYPE_SAVINGS, u'LDD Solidaire': Account.TYPE_SAVINGS, u'LIVRET A': Account.TYPE_SAVINGS, + u'LIVRET JEUNE': Account.TYPE_SAVINGS, + u'LIVRET GRAND PRIX': Account.TYPE_SAVINGS, + u'LEP': Account.TYPE_SAVINGS, + u'LEL': Account.TYPE_SAVINGS, + u'CPT PARTS SOCIALES': Account.TYPE_SAVINGS, + u'PEL 16 2013': Account.TYPE_SAVINGS, u'Titres': Account.TYPE_MARKET, u'Compte titres': Account.TYPE_MARKET, u'Mes crédits immobiliers': Account.TYPE_LOAN, @@ -239,7 +256,7 @@ def _get_account_info(self, a, accounts): else: info['type'] = link info['id'] = info['_id'] = id.group(1) - if info['type'] in ('SYNTHESE_ASSURANCE_CNP','SYNTHESE_EPARGNE', 'ASSURANCE_VIE'): + if info['type'] in ('SYNTHESE_ASSURANCE_CNP', 'SYNTHESE_EPARGNE', 'ASSURANCE_VIE'): info['acc_type'] = Account.TYPE_LIFE_INSURANCE if info['type'] in ('BOURSE', 'COMPTE_TITRE'): info['acc_type'] = Account.TYPE_MARKET @@ -258,6 +275,8 @@ def _add_account(self, accounts, link, label, account_type, balance): account._info = info account.label = label account.type = self.ACCOUNT_TYPES.get(label, info['acc_type'] if 'acc_type' in info else account_type) + if 'PERP' in account.label: + account.type = Account.TYPE_PERP balance = balance or self.get_balance(account) account.balance = Decimal(FrenchTransaction.clean_amount(balance)) if balance and balance is not NotAvailable else NotAvailable @@ -277,7 +296,7 @@ def _add_account(self, accounts, link, label, account_type, balance): accounts[account.id] = account def get_balance(self, account): - if account.type != Account.TYPE_LIFE_INSURANCE: + if account.type not in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_PERP): return NotAvailable page = self.go_history(account._info).page balance = page.doc.xpath('.//tr[td[ends-with(@id,"NumContrat")]/a[contains(text(),$id)]]/td[@class="somme"]', id=account.id) @@ -359,7 +378,7 @@ def get_list(self): def is_access_error(self): error_message = u"Vous n'êtes pas autorisé à accéder à cette fonction" if error_message in CleanText('//div[@class="MessageErreur"]')(self.doc): - return True + return True return False @@ -397,45 +416,76 @@ def get_loan_list(self): title = table.getprevious() if title is None: continue - account_type = self.ACCOUNT_TYPES.get(CleanText('.')(title), Account.TYPE_UNKNOWN) - for tr in table.xpath('./table/tbody/tr[contains(@id,"MM_SYNTHESE_CREDITS") and contains(@id,"IdTrGlobal")]'): - tds = tr.findall('td') - if len(tds) == 0 : - continue - for i in tds[0].xpath('.//a/strong'): - label = i.text.strip() - break - if len(tds) == 3 and Decimal(FrenchTransaction.clean_amount(CleanText('.')(tds[-2]))) and any(cls in Attr('.', 'id')(tr) for cls in ['dgImmo', 'dgConso']) == False: - # in case of Consumer credit or revolving credit, we substract avalaible amount with max amout - # to get what was spend - balance = Decimal(FrenchTransaction.clean_amount(CleanText('.')(tds[-2]))) - Decimal(FrenchTransaction.clean_amount(CleanText('.')(tds[-1]))) - else: - balance = Decimal(FrenchTransaction.clean_amount(CleanText('.')(tds[-1]))) - account = Loan() - account.id = label.split(' ')[-1] - account.label = unicode(label) - account.type = account_type - account.balance = -abs(balance) - account.currency = account.get_currency(CleanText('.')(tds[-1])) - account._card_links = [] - if "immobiliers" in CleanText('.')(title): - xp = './/div[contains(@id, "IdDivDetail")]/table/tbody/tr[contains(@id, "%s")]/td' - account.maturity_date = Date(CleanText(xp % 'IdDerniereEcheance'), dayfirst=True, default=NotAvailable)(tr) - account.total_amount = CleanDecimal(CleanText(xp % 'IdCapitalEmprunte'), replace_dots=True, default=NotAvailable)(tr) - account.subscription_date = Date(CleanText(xp % 'IdDateOuverture'), dayfirst=True, default=NotAvailable)(tr) - account.next_payment_date = Date(CleanText(xp % 'IdDateProchaineEcheance'), dayfirst=True, default=NotAvailable)(tr) - account.rate = CleanDecimal(CleanText(xp % 'IdTaux'), replace_dots=True, default=NotAvailable)(tr) - account.next_payment_amount = CleanDecimal(CleanText(xp % 'IdMontantEcheance'), replace_dots=True, default=NotAvailable)(tr) - elif "renouvelables" in CleanText('.')(title): - self.go_loans_conso(tr) - d = self.browser.loans_conso() - if d: - account.total_amount = d['contrat']['creditMaxAutorise'] - account.available_amount = d['situationCredit']['disponible'] - account.next_payment_amount = d['situationCredit']['mensualiteEnCours'] - accounts[account.id] = account + if "immobiliers" not in CleanText('.')(title): + account_type = self.ACCOUNT_TYPES.get(CleanText('.')(title), Account.TYPE_UNKNOWN) + for tr in table.xpath('./table/tbody/tr[contains(@id,"MM_SYNTHESE_CREDITS") and contains(@id,"IdTrGlobal")]'): + tds = tr.findall('td') + if len(tds) == 0 : + continue + for i in tds[0].xpath('.//a/strong'): + label = i.text.strip() + break + if len(tds) == 3 and Decimal(FrenchTransaction.clean_amount(CleanText('.')(tds[-2]))) and any(cls in Attr('.', 'id')(tr) for cls in ['dgImmo', 'dgConso']) == False: + # in case of Consumer credit or revolving credit, we substract avalaible amount with max amout + # to get what was spend + balance = Decimal(FrenchTransaction.clean_amount(CleanText('.')(tds[-2]))) - Decimal(FrenchTransaction.clean_amount(CleanText('.')(tds[-1]))) + else: + balance = Decimal(FrenchTransaction.clean_amount(CleanText('.')(tds[-1]))) + account = Loan() + account.id = label.split(' ')[-1] + account.label = unicode(label) + account.type = account_type + account.balance = -abs(balance) + account.currency = account.get_currency(CleanText('.')(tds[-1])) + account._card_links = [] + + if "renouvelables" in CleanText('.')(title): + self.go_loans_conso(tr) + d = self.browser.loans_conso() + if d: + account.total_amount = float_to_decimal(d['contrat']['creditMaxAutorise']) + account.available_amount = float_to_decimal(d['situationCredit']['disponible']) + account.next_payment_amount = float_to_decimal(d['situationCredit']['mensualiteEnCours']) + accounts[account.id] = account return accounts.values() + @method + class get_real_estate_loans(ListElement): + # beware the html response is slightly different from what can be seen with the browser + # because of some JS most likely: use the native HTML response to build the xpath + item_xpath = '//h3[contains(text(), "immobiliers")]//following-sibling::div[@class="panel"][1]//div[@id[starts-with(.,"MM_SYNTHESE_CREDITS")] and contains(@id, "IdDivDetail")]' + + class iter_account(TableElement): + item_xpath = './table[@class="static"][1]/tbody' + head_xpath = './table[@class="static"][1]/tbody/tr/th' + + col_total_amount = u'Capital Emprunté' + col_rate = u'Taux d’intérêt nominal' + col_balance = u'Capital Restant Dû' + col_last_payment_date = u'Dernière échéance' + col_next_payment_amount = u'Montant prochaine échéance' + col_next_payment_date = u'Prochaine échéance' + + def parse(self, el): + self.env['id'] = CleanText("./h2")(el).split()[-1] + self.env['label'] = CleanText("./h2")(el) + + class item(ItemElement): + + klass = Loan + + obj_id = Env('id') + obj_label = Env('label') + obj_type = Loan.TYPE_LOAN + obj_total_amount = MyDecimal(MyTableCell("total_amount")) + obj_rate = Eval(lambda x: x / 100, MyDecimal(MyTableCell("rate", default=NotAvailable), default=NotAvailable)) + obj_balance = MyDecimal(MyTableCell("balance"), sign=lambda x: -1) + obj_currency = Currency(MyTableCell("balance")) + obj_last_payment_date = Date(CleanText(MyTableCell("last_payment_date"))) + obj_next_payment_amount = MyDecimal(MyTableCell("next_payment_amount")) + obj_next_payment_date = Date(CleanText(MyTableCell("next_payment_date"))) + + def go_list(self): form = self.get_form(name='main') @@ -519,7 +569,7 @@ def get_form_to_detail(self, transaction): m = re.match('.*\("(.*)", "(DETAIL_OP&[\d]+).*\)\)', transaction._link) # go to detailcard page form = self.get_form(name='main') - form['__EVENTTARGET'] = m.group(1) + form['__EVENTTARGET'] = m.group(1) form['__EVENTARGUMENT'] = m.group(2) fix_form(form) return form @@ -625,12 +675,19 @@ def go_life_insurance(self, account): fix_form(form) form.submit() + def transfer_link(self): + return self.doc.xpath(u'//a[span[contains(text(), "Effectuer un virement")]] | //a[contains(text(), "Réaliser un virement")]') + def go_transfer_via_history(self, account): self.go_history(account._info) - self.browser.page.go_transfer(account) + + # check that transfer is available for the connection before try to go on transfer page + # otherwise website will continually crash + if self.transfer_link(): + self.browser.page.go_transfer(account) def go_transfer(self, account): - link = self.doc.xpath(u'//a[span[contains(text(), "Effectuer un virement")]] | //a[contains(text(), "Réaliser un virement")]') + link = self.transfer_link() if len(link) == 0: return self.go_transfer_via_history(account) else: @@ -674,6 +731,7 @@ def on_load(self): form = self.get_form(id="REROUTAGE") form.submit() + class NatixisRedirectPage(LoggedPage, HTMLPage): def on_load(self): try: @@ -711,6 +769,7 @@ def iter_investment(self): inv = Investment() inv.label = CleanText('.')(tbody.xpath('./tr[1]/td[1]/a/span')[0]) inv.code = CleanText('.')(tbody.xpath('./tr[1]/td[1]/a')[0]).split(' - ')[1] + inv.code_type = Investment.CODE_TYPE_ISIN if is_isin_valid(inv.code) else NotAvailable inv.quantity = self.parse_decimal(tbody.xpath('./tr[2]/td[2]')[0]) inv.unitvalue = self.parse_decimal(tbody.xpath('./tr[2]/td[3]')[0]) inv.unitprice = self.parse_decimal(tbody.xpath('./tr[2]/td[5]')[0]) @@ -758,6 +817,7 @@ def iter_investment(self): inv = Investment() libelle = CleanText('.')(tr.xpath('./td[1]')[0]).split(' ') inv.label, inv.code = self.split_label_code(libelle) + inv.code_type = Investment.CODE_TYPE_ISIN if is_isin_valid(inv.code) else NotAvailable inv.quantity = self.parse_decimal(tr.xpath('./td[2]')[0]) inv.unitvalue = self.parse_decimal(tr.xpath('./td[3]')[0]) date = CleanText('.')(tr.xpath('./td[4]')[0]) @@ -1093,7 +1153,6 @@ def go_add_recipient(self): form.submit() - class CanceledAuth(Exception): pass @@ -1124,7 +1183,7 @@ class SmsPage(LoggedPage, HTMLPage): def on_load(self): error = CleanText('//p[@class="warning_trials_before"]')(self.doc) if error: - raise AddRecipientError('Wrongcode, ' + error) + raise AddRecipientBankError(message='Wrongcode, ' + error) def get_prompt_text(self): return CleanText(u'//td[@class="auth_info_prompt"]')(self.doc) @@ -1162,7 +1221,7 @@ class RecipientPage(LoggedPage, HTMLPage): def on_load(self): error = CleanText('//span[@id="MM_LblMessagePopinError"]')(self.doc) if error: - raise AddRecipientError(message=error) + raise AddRecipientBankError(message=error) def is_here(self): return bool(CleanText(u'//h2[contains(text(), "Ajouter un compte bénéficiaire")] |\ @@ -1186,7 +1245,7 @@ class ProAddRecipientOtpPage(IndexPage): def on_load(self): error = CleanText('//div[@id="MM_m_CH_ValidationSummary" and @class="MessageErreur"]')(self.doc) if error: - raise AddRecipientError('Wrongcode, ' + error) + raise AddRecipientBankError(message='Wrongcode, ' + error) def is_here(self): return self.need_auth() and self.doc.xpath('//span[@id="MM_ANR_WS_AUTHENT_ANR_WS_AUTHENT_SAISIE_lblProcedure1"]') @@ -1315,3 +1374,10 @@ def download_document(self, document): form['__EVENTTARGET'] = document.url form['MM$COMPTE_EDOCUMENTS$ctrlEDocumentsConsultationDocument$eventId'] = document._event_id return form.submit() + + +class CreditCooperatifMarketPage(LoggedPage, HTMLPage): + # Stay logged when landing on the new Linebourse + # (which is used by Credit Cooperatif's connections) + # The parsing is done in linebourse.api.pages + pass diff --git a/modules/carrefourbanque/compat/weboob_capabilities_bank.py b/modules/carrefourbanque/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/carrefourbanque/compat/weboob_capabilities_bank.py +++ b/modules/carrefourbanque/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/cices/compat/weboob_capabilities_bank.py b/modules/cices/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/cices/compat/weboob_capabilities_bank.py +++ b/modules/cices/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/citibank/browser.py b/modules/citibank/browser.py index 0ba26ef8b7..508fd7bf4c 100644 --- a/modules/citibank/browser.py +++ b/modules/citibank/browser.py @@ -17,31 +17,29 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals -from weboob.browser import LoginBrowser, URL, need_login +import re +from datetime import datetime +from time import sleep + +from weboob.browser import URL, LoginBrowser, need_login from weboob.browser.pages import HTMLPage, JsonPage, RawPage from weboob.capabilities.bank import Account, AccountNotFound, Transaction from weboob.exceptions import BrowserIncorrectPassword -from weboob.tools.capabilities.bank.transactions import \ - AmericanTransaction as AmTr +from weboob.tools.capabilities.bank.transactions import AmericanTransaction as AmTr from weboob.tools.js import Javascript from .parser import StatementParser, clean_label -import re -from datetime import datetime - -from time import sleep - - __all__ = ['Citibank'] class SomePage(HTMLPage): @property def logged(self): - return bool(self.doc.xpath(u'//a[text()="Sign Off"]') + - self.doc.xpath(u'//div[@id="CardsLoadingDiv"]')) + return bool(self.doc.xpath('//a[text()="Sign Off"]') + + self.doc.xpath('//div[@id="CardsLoadingDiv"]')) class IndexPage(SomePage): @@ -82,7 +80,7 @@ def account(self): account.id = account.label[-4:] for bal in details['accountBalances']: label, value = bal['label'], (bal['value'] or ['0'])[0] - if label == u'Current Balance:': + if label == 'Current Balance:': if value[0] == '(' and value[-1] == ')': value = value[1:-1] sign = 1 @@ -90,17 +88,16 @@ def account(self): sign = -1 account.currency = Account.get_currency(value) account.balance = sign * AmTr.decimal_amount(value) - elif label == u'Total Revolving Credit Line:': + elif label == 'Total Revolving Credit Line:': account.cardlimit = AmTr.decimal_amount(value) - elif label.startswith(u'Minimum Payment Due'): + elif label.startswith('Minimum Payment Due'): d = re.match(r'.*(..-..-....):$', label).group(1) account.paydate = datetime.strptime(d, '%m-%d-%Y') account.paymin = AmTr.decimal_amount(value) return account def transactions(self): - return sorted(self.unsorted_trans(), - lambda a, b: cmp(a.date, b.date), reverse=True) + return sorted(self.unsorted_trans(), key=lambda t: t.date, reverse=True) def unsorted_trans(self): for jnl in self.doc['accountDetailsAndActivity']['accountActivity'] \ @@ -110,10 +107,10 @@ def unsorted_trans(self): amount = jnl['columns'][3]['activityColumn'][0] xdescs = dict((x['label'], x['value'][0]) for x in jnl['extendedDescriptions']) - pdate = xdescs[u'Posted Date :'] - ref = xdescs.get(u'Reference Number:') or u'' + pdate = xdescs['Posted Date :'] + ref = xdescs.get('Reference Number:') or '' - if amount.startswith(u'(') and amount.endswith(u')'): + if amount.startswith('(') and amount.endswith(')'): amount = AmTr.decimal_amount(amount[1:-1]) else: amount = -AmTr.decimal_amount(amount) @@ -132,8 +129,8 @@ def unsorted_trans(self): class StatementsPage(SomePage): def dates(self): return [x[:10] for x in self.doc.xpath( - u'//select[@id="currentStatementsDate"]/option/@value') - if re.match(u'^\d\d\d\d-\d\d-\d\d All$', x)] + '//select[@id="currentStatementsDate"]/option/@value') + if re.match(r'^\d\d\d\d-\d\d-\d\d All$', x)] class StatementPage(RawPage): @@ -147,8 +144,7 @@ def is_sane(self): return self._parser.read_first_date_range() is not None def transactions(self): - return sorted(self._parser.read_transactions(), - cmp=lambda t1, t2: cmp(t2.date, t1.date)) + return sorted(self._parser.read_transactions(), key=lambda t: t.date) class Citibank(LoginBrowser): @@ -171,20 +167,16 @@ class Citibank(LoginBrowser): TIMEOUT = 30.0 index = URL(r'/US/JPS/portal/Index.do', IndexPage) signon = URL(r'/US/JSO/signon/ProcessUsernameSignon.do', SomePage) - accdetailhtml = URL(u'/US/NCPS/accountdetailactivity/flow.action.*$', - SomePage) - accounts = URL(r'/US/REST/accountsPanel' - r'/getCustomerAccounts.jws\?ttc=(?P.*)$', + accdetailhtml = URL(r'/US/NCPS/accountdetailactivity/flow.action.*$', SomePage) + accounts = URL(r'/US/REST/accountsPanel/getCustomerAccounts.jws\?ttc=(?P.*)$', AccountsPage) accdetails = URL(r'/US/REST/accountDetailsActivity' - r'/getAccountDetailsActivity.jws\?accountID=(?P.*)$', - AccDetailsPage) + r'/getAccountDetailsActivity.jws\?accountID=(?P.*)$', AccDetailsPage) statements = URL(r'/US/NCSC/doccenter/flow.action\?TTC=1079&' - 'accountID=(?P.*)$', StatementsPage) + r'accountID=(?P.*)$', StatementsPage) statement = URL(r'/US/REST/doccenterresource/downloadStatementsPdf.jws\?' - r'selectedIndex=0&date=(?P....-..-..)&' - r'downloadFormat=pdf', StatementPage) - unknown = URL('/.*$', SomePage) + r'selectedIndex=0&date=(?P....-..-..)&downloadFormat=pdf', StatementPage) + unknown = URL(r'/.*$', SomePage) def get_account(self, id_): innerId = self.to_accounts().inner_ids_dict().get(id_) @@ -217,8 +209,8 @@ def to_statement(self, date): # Sometimes the website returns non-PDF file. # It recovers if we repeat whole browsing sequence all the way # from home page up to the statement. - MAX_DELAY=10 - for i in xrange(self.MAX_RETRIES): + MAX_DELAY = 10 + for i in range(self.MAX_RETRIES): if self.to_page(self.statement, date=date).is_sane(): return self.page sleep(min(MAX_DELAY, 1 << i)) @@ -233,7 +225,7 @@ def to_page(self, url, **data): def do_login(self): self.session.cookies.clear() - data = dict([('username', self.username), ('password', self.password)]+ - self.index.go().extra()) + data = dict([('username', self.username), ('password', self.password)] + + self.index.go().extra()) if not self.signon.go(data=data).logged: raise BrowserIncorrectPassword() diff --git a/modules/citibank/parser.py b/modules/citibank/parser.py index ae2f5b92d4..c97690ad39 100644 --- a/modules/citibank/parser.py +++ b/modules/citibank/parser.py @@ -17,16 +17,18 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals + +import datetime +import re + from weboob.capabilities.bank import Transaction -from weboob.tools.capabilities.bank.transactions import \ - AmericanTransaction as AmTr +from weboob.tools.capabilities.bank.transactions import AmericanTransaction as AmTr +from weboob.tools.compat import unicode from weboob.tools.date import closest_date from weboob.tools.pdf import decompress_pdf from weboob.tools.tokenizer import ReTokenizer -import datetime -import re - def clean_label(text): """ @@ -35,10 +37,10 @@ def clean_label(text): need to make labels from both sources look the same. """ for pattern in [r' \d+\.\d+ +POUND STERLING', - u'Subject to Foreign Fee', - u'Description']: - text = re.sub(pattern, u'', text, re.UNICODE) - return re.sub(r' +', u' ', text.strip().upper(), re.UNICODE) + 'Subject to Foreign Fee', + 'Description']: + text = re.sub(pattern, '', text, re.UNICODE) + return re.sub(r' +', ' ', text.strip().upper(), re.UNICODE) def formatted(read_func): @@ -54,7 +56,7 @@ def wrapped(self, pos): pos, data = read_func(self, pos) pos, et = self.read_layout_et(pos) if ws is None or bt is None or tf is None \ - or tm is None or data is None or et is None: + or tm is None or data is None or et is None: return startPos, None else: return pos, data @@ -192,13 +194,13 @@ def __getattr__(self, name): @formatted def read_date(self, pos): def parse_date(v): - for year in [1900, 1904]: # try leap and non-leap years + for year in [1900, 1904]: # try leap and non-leap years fullstr = '%s/%i' % (v, year) try: return datetime.datetime.strptime(fullstr, '%m/%d/%Y') except ValueError as e: - pass - raise e + last_error = e + raise last_error return self._tok.simple_read('date', pos, parse_date) @@ -217,7 +219,7 @@ def read_text(self, pos): lambda v: unicode(v, errors='ignore')) pos, et = self.read_layout_et(pos) if ws is None or bt is None or tf is None \ - or tm is None or text is None or et is None: + or tm is None or text is None or et is None: return startPos, None else: return pos, (text, tm) diff --git a/modules/cmes/compat/weboob_capabilities_bank.py b/modules/cmes/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/cmes/compat/weboob_capabilities_bank.py +++ b/modules/cmes/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/cmso/compat/weboob_capabilities_bank.py b/modules/cmso/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/cmso/compat/weboob_capabilities_bank.py +++ b/modules/cmso/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/cmso/par/browser.py b/modules/cmso/par/browser.py index d3d5987d24..846374cdd3 100644 --- a/modules/cmso/par/browser.py +++ b/modules/cmso/par/browser.py @@ -57,6 +57,7 @@ def wrapper(browser, *args, **kwargs): try: ret = cb() except exc_check as exc: + browser.headers = None browser.do_login() browser.logger.info('%s raised, retrying', exc) continue @@ -177,8 +178,9 @@ def iter_accounts(self): for key in page.get_keys(): for a in page.iter_savings(key=key, numbers=numbers): if a._index in seen: - self.logger.warning('skipping %s because it seems to be a duplicate of %s', a, seen[a._index]) - continue + acc = seen[a._index] + self.accounts_list.remove(acc) + self.logger.warning('replace %s because it seems to be a duplicate of %s', seen[a._index], a) self.accounts_list.append(a) # Then, get loans diff --git a/modules/cmso/pro/pages.py b/modules/cmso/pro/pages.py index b1ac8629e2..85eb8185a2 100644 --- a/modules/cmso/pro/pages.py +++ b/modules/cmso/pro/pages.py @@ -239,7 +239,7 @@ def condition(self): class HistoryPage(CMSOPage): def get_date_range_list(self): - return self.doc.xpath('//select[@name="date"]/option/@value') + return [d for d in self.doc.xpath('//select[@name="date"]/option/@value') if d] @pagination @method diff --git a/modules/cragr/compat/weboob_capabilities_bank.py b/modules/cragr/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/cragr/compat/weboob_capabilities_bank.py +++ b/modules/cragr/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/cragr/web/browser.py b/modules/cragr/web/browser.py index 173e2fddc4..9f1a17c19d 100644 --- a/modules/cragr/web/browser.py +++ b/modules/cragr/web/browser.py @@ -24,21 +24,22 @@ from html2text import unescape from datetime import date, datetime, timedelta -from weboob.capabilities.bank import ( - Account, AddRecipientStep, AddRecipientError, RecipientInvalidLabel, - Recipient, AccountNotFound, Investment, +from .compat.weboob_capabilities_bank import ( + Account, AddRecipientStep, AddRecipientBankError, RecipientInvalidLabel, + Recipient, AccountNotFound, ) from weboob.capabilities.base import NotLoaded, find_object from .compat.weboob_capabilities_profile import ProfileMissing from weboob.browser.browsers import LoginBrowser, URL, need_login, StatesMixin from weboob.browser.pages import FormNotFound -from weboob.exceptions import BrowserIncorrectPassword +from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable from weboob.tools.date import ChaoticDateGuesser, LinearDateGuesser from weboob.exceptions import BrowserHTTPError, ActionNeeded from weboob.browser.filters.standard import CleanText from weboob.tools.value import Value from weboob.tools.compat import urlparse, urljoin, basestring from weboob.tools.capabilities.bank.iban import is_iban_valid +from .compat.weboob_tools_capabilities_bank_investments import create_french_liquidity from .pages import ( HomePage, LoginPage, LoginErrorPage, AccountsPage, @@ -323,10 +324,14 @@ def get_list(self): self.location(updated_account._form.request) if (updated_account.url or updated_account._form) and not self.no_fixed_deposit_page.is_here(): - iban_url = self.page.get_iban_url() - if iban_url: - self.location(iban_url) - account.iban = self.page.get_iban() + # crashes because we get an UnavailablePage instead of TransactionsPage + if self.transactions.is_here(): + iban_url = self.page.get_iban_url() + if iban_url: + self.location(iban_url) + if self.unavailable_page.is_here(): + raise BrowserUnavailable() + account.iban = self.page.get_iban() # credit cards # reseting location in case of pagination @@ -374,10 +379,14 @@ def market_accounts_matching(self, accounts_list, market_accounts_list): account = find_object(accounts_list, id=market_account.id) if account: account.label = market_account.label or account.label - # Update accounts balance only for TYPE_MARKET because PEA accounts are fused - # with their related DAV PEA here, therefore the balance includes liquidities: if account.type == Account.TYPE_MARKET: account.balance = market_account.balance or account.balance + # Some PEA accounts are only present on the Market page but the balance includes + # liquidities from the DAV PEA, so we need another request to fetch the balance: + elif account.type == Account.TYPE_PEA: + url = 'https://www.cabourse.credit-agricole.fr/netfinca-titres/servlet/com.netfinca.frontcr.account.WalletVal?nump=%s:%s' + self.location(url % (account.id, self.code_caisse)) + account.balance = self.page.get_pea_balance() @need_login def get_history(self, account): @@ -446,11 +455,7 @@ def iter_investment(self, account): self.go_perimeter(account._perimeter) if account.type == Account.TYPE_PEA and account._liquidity_url: - liquidity_inv = Investment() - liquidity_inv.label = account.label - liquidity_inv.code = u'XX-liquidity' - liquidity_inv.valuation = account.balance - yield liquidity_inv + yield create_french_liquidity(account.balance) return if account.type in (Account.TYPE_MARKET, Account.TYPE_PEA): @@ -468,6 +473,10 @@ def iter_investment(self, account): return if self.lifeinsurance.is_here(): self.page.go_on_detail(account.id) + # We scrape the non-invested part as liquidities: + if self.page.has_liquidities(): + valuation = self.page.get_liquidities() + yield create_french_liquidity(valuation) for inv in self.page.iter_investment(): yield inv @@ -695,7 +704,8 @@ def new_recipient(self, recipient, **params): raise RecipientInvalidLabel('Recipient label contains invalid characters') if 'sms_code' in params and not re.match(r'^[a-z0-9]{6}$', params['sms_code'], re.I): - raise AddRecipientError('SMS verification code is invalid') + # check before send sms code because it can crash website if code is invalid + raise AddRecipientBankError("SMS code %s is invalid" % params['sms_code']) self.transfer_init_page.go(sag=self.sag) assert self.transfer_init_page.is_here() @@ -726,7 +736,7 @@ def new_recipient(self, recipient, **params): err = hasattr(self.page, 'get_sms_error') and self.page.get_sms_error() if err: - raise AddRecipientError(message=err) + raise AddRecipientBankError(message=err) self.page.submit_recipient(recipient.label, recipient.iban) self.page.confirm_recipient() @@ -738,6 +748,5 @@ def new_recipient(self, recipient, **params): assert self.transfer_init_page.is_here() res = self.page.find_recipient(recipient.iban) - if res is None: - raise AddRecipientError('Recipient could not be found') + assert res, 'Recipient with iban %s could not be found' % recipient.iban return res diff --git a/modules/cragr/web/compat/weboob_capabilities_bank.py b/modules/cragr/web/compat/weboob_capabilities_bank.py new file mode 100644 index 0000000000..141097d781 --- /dev/null +++ b/modules/cragr/web/compat/weboob_capabilities_bank.py @@ -0,0 +1,45 @@ + +import weboob.capabilities.bank as OLD + +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) + + +__all__ = OLD.__all__ + + +class CapBankWealth(CapBank): + pass + + +class CapBankPockets(CapBank): + pass + + +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + +class CapBankTransfer(OLD.CapBankTransfer): + def transfer_check_label(self, old, new): + from unidecode import unidecode + + return unidecode(old) == unidecode(new) + + +class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipient): + pass + + +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + +Account.TYPE_MORTGAGE = 17 +Account.TYPE_CONSUMER_CREDIT = 18 +Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/cragr/web/compat/weboob_tools_capabilities_bank_investments.py b/modules/cragr/web/compat/weboob_tools_capabilities_bank_investments.py new file mode 100644 index 0000000000..2b68ff9e00 --- /dev/null +++ b/modules/cragr/web/compat/weboob_tools_capabilities_bank_investments.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Jonathan Schmidt +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.tools.compat import basestring +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.bank import Investment + +def is_isin_valid(isin): + """ + Méthode générale + Table de conversion des lettres en chiffres + A=10 B=11 C=12 D=13 E=14 F=15 G=16 H=17 I=18 + J=19 K=20 L=21 M=22 N=23 O=24 P=25 Q=26 R=27 + S=28 T=29 U=30 V=31 W=32 X=33 Y=34 Z=35 + + 1 - Mettre de côté la clé, qui servira de référence à la fin de la vérification. + 2 - Convertir toutes les lettres en nombres via la table de conversion ci-contre. Si le nombre obtenu est supérieur ou égal à 10, prendre les deux chiffres du nombre séparément (exemple : 27 devient 2 et 7). + 3 - Pour chaque chiffre, multiplier sa valeur par deux si sa position est impaire en partant de la droite. Si le nombre obtenu est supérieur ou égal à 10, garder les deux chiffres du nombre séparément (exemple : 14 devient 1 et 4). + 4 - Faire la somme de tous les chiffres. + 5 - Soustraire cette somme de la dizaine supérieure ou égale la plus proche (exemples : si la somme vaut 22, la dizaine « supérieure ou égale » est 30, et la clé vaut donc 8 ; si la somme vaut 30, la dizaine « supérieure ou égale » est 30, et la clé vaut 0 ; si la somme vaut 31, la dizaine « supérieure ou égale » est 40, et la clé vaut 9). + 6 - Comparer la valeur obtenue à la clé mise initialement de côté. + + Étapes 1 et 2 : + F R 0 0 0 3 5 0 0 0 0 (+ 8 : clé) + 15 27 0 0 0 3 5 0 0 0 0 + + Étape 3 : le traitement se fait sur des chiffres + 1 5 2 7 0 0 0 3 5 0 0 0 0 + I P I P I P I P I P I P I : position en partant de la droite (P = Pair, I = Impair) + 2 1 2 1 2 1 2 1 2 1 2 1 2 : coefficient multiplicateur + + 2 5 4 7 0 0 0 3 10 0 0 0 0 : résultat + + Étape 4 : + 2 + 5 + 4 + 7 + 0 + 0 + 0 + 3 + (1 + 0)+ 0 + 0 + 0 + 0 = 22 + + Étapes 5 et 6 : 30 - 22 = 8 (valeur de la clé) + """ + + if not isinstance(isin, basestring): + return False + if not re.match(r'^[A-Z]{2}[A-Z0-9]{9}\d$', isin): + return False + + isin_in_digits = ''.join(str(ord(x) - ord('A') + 10) if not x.isdigit() else x for x in isin[:-1]) + key = isin[-1:] + result = '' + for k, val in enumerate(isin_in_digits[::-1], start=1): + if k % 2 == 0: + result = ''.join((result, val)) + else: + result = ''.join((result, str(int(val)*2))) + return str(sum(int(x) for x in result) + int(key))[-1] == '0' + + +def create_french_liquidity(valuation): + """ + Automatically fills a liquidity investment with label, code and code_type. + """ + liquidity = Investment() + liquidity.label = "Liquidités" + liquidity.code = "XX-liquidity" + liquidity.code_type = NotAvailable + liquidity.valuation = valuation + return liquidity + diff --git a/modules/cragr/web/pages.py b/modules/cragr/web/pages.py index 76000ed707..0890dac326 100644 --- a/modules/cragr/web/pages.py +++ b/modules/cragr/web/pages.py @@ -26,9 +26,9 @@ from weboob.browser.pages import HTMLPage, FormNotFound, pagination from weboob.capabilities import NotAvailable from weboob.capabilities.base import Currency -from weboob.capabilities.bank import ( +from .compat.weboob_capabilities_bank import ( Account, Investment, Recipient, Transfer, TransferError, TransferBankError, - AddRecipientError, Loan + AddRecipientBankError, Loan ) from weboob.capabilities.contact import Advisor from weboob.capabilities.profile import Profile @@ -687,7 +687,7 @@ def condition(self): return False label = Field('label')(self) - if label.startswith('RENT') or label.startswith('RENV'): + if any(label.startswith(prefix) for prefix in ('RENT', 'RENV', 'RACC')): # RENTE account doesn't seem to have any balance or currency return False @@ -978,6 +978,10 @@ def parse_decimal(self, value): return CleanDecimal().filter(value) return MyDecimal().filter(value) + # This method is used to fetch the PEA balance without liquidities present on the DAV PEA: + def get_pea_balance(self): + return CleanDecimal('//tr[td[text()="Valorisation titres"]]/td/span', replace_dots=True)(self.doc) + class MarketHomePage(MarketPage): COL_ID_LABEL = 1 @@ -1024,6 +1028,13 @@ def go_on_detail(self, account_id): form.submit() self.browser.location('https://assurance-personnes.credit-agricole.fr/filiale/entreeBam?sessionSAG=%s&stbpg=pagePU&act=SEPPU&stbzn=bnt&actCrt=SEPPU' % self.browser.sag) + def has_liquidities(self): + return bool(self.doc.xpath('//span[contains(text(), "de versement restant")]')) + + def get_liquidities(self): + return CleanDecimal(Regexp(CleanText('//span[contains(text(), "de versement restant")]'), + "dont (.*) de versement restant"), replace_dots=True)(self.doc) + def iter_investment(self): for line in self.doc.xpath('//table[@summary and count(descendant::td) > 1]/tbody/tr'): @@ -1138,7 +1149,7 @@ def iter_investment(self): inv.code = NotAvailable inv.quantity = self.parse_decimal(cells[self.COL_QUANTITY].text_content()) - if len(cells) == 5: + if 5 <= len(cells) <= 7: inv.unitvalue = self.parse_decimal(cells[2].text_content()) inv.valuation = self.parse_decimal(cells[3].text_content()) inv.portfolio_share = self.parse_decimal(cells[4].text_content())/100 @@ -1213,7 +1224,8 @@ def submit_accounts(self, account_id, recipient_id, amount, currency): if len(emitters) != 1: raise TransferError('Could not find emitter %r' % account_id) recipients = [rcpt for rcpt in self.iter_recipients() if rcpt.id and rcpt.id == recipient_id] - if len(recipients) != 1: + # for recipient with same IBAN, first matched recipient is the default value + if len(recipients) < 1: raise TransferError('Could not find recipient %r' % recipient_id) form = self.get_form(name='frm_fwk') @@ -1247,7 +1259,7 @@ def submit_recipient(self, label, iban): try: form = self.get_form(name='frm_fwk') except FormNotFound: - raise AddRecipientError('An error occurred before sending recipient') + assert False, 'An error occurred before sending recipient' form['NOM_BENEF'] = label for i in range(9): @@ -1337,7 +1349,7 @@ def submit_confirm(self): def on_load(self): super(TransferPage, self).on_load() # warning: the "service indisponible" message (not catched here) is not a real BrowserUnavailable - err = CleanText('//div[has-class("blc-choix-erreur")]//p', default='')(self.doc) + err = CleanText('//form//div[has-class("blc-choix-erreur")]//p', default='')(self.doc) if err: raise TransferBankError(message=err) @@ -1357,7 +1369,7 @@ def check_recipient_error(self): msg = CleanText('//tr[@bgcolor="#C74545"]', default='')(self.doc) # there is no id, class or anything... if msg: - raise AddRecipientError(message=msg) + raise AddRecipientBankError(message=msg) class RecipientMiscPage(RecipientAddingMixin, CollectePageMixin, MyLoggedPage, BasePage): @@ -1379,7 +1391,7 @@ def confirm_recipient(self): try: form = self.get_form(name='frm_fwk') except FormNotFound: - raise AddRecipientError('An error occurred before finishing adding recipient') + assert False, 'An error occurred before finishing adding recipient' form['fwkaction'] = 'ConfirmerAjout' form['fwkcodeaction'] = 'Executer' @@ -1388,7 +1400,7 @@ def confirm_recipient(self): def check_recipient_error(self): msg = CleanText('//tr[@bgcolor="#C74545"]', default='')(self.doc) # there is no id, class or anything... if msg: - raise AddRecipientError(message=msg) + raise AddRecipientBankError(message=msg) def get_iban_col(self): for index, td in enumerate(self.doc.xpath('//table[starts-with(@summary,"Nom et IBAN")]//th')): @@ -1446,7 +1458,7 @@ def on_load(self): # if the otp is incorrect error_msg = CleanText('//div[has-class("blc-choix-erreur")]//span')(self.doc) if error_msg: - raise AddRecipientError(message=error_msg) + raise AddRecipientBankError(message=error_msg) def send_sms(self): # when a code is still pending diff --git a/modules/creditcooperatif/caisseepargne_browser.py b/modules/creditcooperatif/caisseepargne_browser.py index 3759bcae66..07f124613c 100644 --- a/modules/creditcooperatif/caisseepargne_browser.py +++ b/modules/creditcooperatif/caisseepargne_browser.py @@ -18,6 +18,7 @@ # along with weboob. If not, see . from weboob.browser import AbstractBrowser +from .linebourse_browser import LinebourseAPIBrowser __all__ = ['CaisseEpargneBrowser'] @@ -26,3 +27,9 @@ class CaisseEpargneBrowser(AbstractBrowser): PARENT = 'caissedepargne' PARENT_ATTR = 'package.browser.CaisseEpargne' + + LINEBOURSE_BROWSER = LinebourseAPIBrowser + + def __init__(self, nuser, *args, **kwargs): + kwargs['market_url'] = 'https://www.offrebourse.com' + super(CaisseEpargneBrowser, self).__init__(nuser, *args, **kwargs) diff --git a/modules/creditcooperatif/cenet_browser.py b/modules/creditcooperatif/cenet_browser.py index 6eb6f8bd9a..042724be84 100644 --- a/modules/creditcooperatif/cenet_browser.py +++ b/modules/creditcooperatif/cenet_browser.py @@ -27,3 +27,7 @@ class CenetBrowser(AbstractBrowser): PARENT = 'caissedepargne' PARENT_ATTR = 'package.cenet.browser.CenetBrowser' BASEURL = 'https://www.espaceclient.credit-cooperatif.coop/' + + def __init__(self, nuser, *args, **kwargs): + kwargs['domain'] = 'www.credit-cooperatif.coop' + super(CenetBrowser, self).__init__(nuser, *args, **kwargs) diff --git a/modules/creditcooperatif/linebourse_browser.py b/modules/creditcooperatif/linebourse_browser.py new file mode 100644 index 0000000000..eedbc2f572 --- /dev/null +++ b/modules/creditcooperatif/linebourse_browser.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Fong Ngo +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from weboob.browser import AbstractBrowser + + +class LinebourseAPIBrowser(AbstractBrowser): + PARENT = 'linebourse' + PARENT_ATTR = 'package.browser.LinebourseAPIBrowser' diff --git a/modules/creditdunord/browser.py b/modules/creditdunord/browser.py index b316da64cf..c893504a84 100644 --- a/modules/creditdunord/browser.py +++ b/modules/creditdunord/browser.py @@ -17,38 +17,42 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . - -import re +from __future__ import unicode_literals from weboob.browser import LoginBrowser, URL, need_login from weboob.exceptions import BrowserIncorrectPassword, BrowserPasswordExpired from weboob.capabilities.bank import Account, Investment from weboob.capabilities.base import find_object - -from .pages import LoginPage, AccountsPage, ProAccountsPage, TransactionsPage, \ - ProTransactionsPage, IbanPage, RedirectPage, AVPage - +from .pages import ( + LoginPage, ProfilePage, AccountTypePage, AccountsPage, ProAccountsPage, + TransactionsPage, IbanPage, RedirectPage, EntryPage, AVPage, ProIbanPage, + ProTransactionsPage, LabelsPage, +) class CreditDuNordBrowser(LoginBrowser): ENCODING = 'UTF-8' + BASEURL = "https://www.credit-du-nord.fr/" login = URL('$', - '/.*\?.*_pageLabel=page_erreur_connexion', LoginPage) - redirect = URL('/swm/redirectCDN.html', RedirectPage) - av = URL('/vos-comptes/particuliers/V1_transactional_portal_page_', AVPage) - accounts = URL('/vos-comptes/particuliers', - '/vos-comptes/particuliers/transac_tableau_de_bord', AccountsPage) - transactions = URL('/vos-comptes/.*/transac/particuliers', TransactionsPage) - proaccounts = URL('/vos-comptes/(professionnels|entreprises)', ProAccountsPage) - protransactions = URL('/vos-comptes/.*/transac/(professionnels|entreprises)', ProTransactionsPage) - loans = URL('/vos-comptes/professionnels/credit_en_cours', ProAccountsPage) + '/.*\?.*_pageLabel=page_erreur_connexion', + LoginPage) + redirect = URL('/swm/redirectCDN.html', RedirectPage) + entrypage = URL('/icd/zco/#zco', EntryPage) + multitype_av = URL('/vos-comptes/IPT/appmanager/transac/professionnels\?_nfpb=true&_eventName=onRestart&_pageLabel=synthese_contrats_assurance_vie', AVPage) + loans = URL('/vos-comptes/IPT/appmanager/transac/(?P.*)\?_nfpb=true&_eventName=onRestart&_pageLabel=(?P(creditPersoImmobilier|credit__en_cours|credit_en_cours))', ProAccountsPage) + proaccounts = URL('/vos-comptes/IPT/appmanager/transac/(professionnels|entreprises)\?_nfpb=true&_eventName=onRestart&_pageLabel=(?P(transac_tableau_de_bord|page__synthese_v1|page_synthese_v1))', ProAccountsPage) + accounts = URL('/vos-comptes/IPT/appmanager/transac/(?P.*)\?_nfpb=true&_eventName=onRestart&_pageLabel=(?P(transac_tableau_de_bord|page__synthese_v1|page_synthese_v1))', AccountsPage) + multitype_iban = URL('/vos-comptes/IPT/appmanager/transac/professionnels\?_nfpb=true&_eventName=onRestart&_pageLabel=impression_rib', ProIbanPage) + transactions = URL('/vos-comptes/IPT/appmanager/transac/particuliers\?_nfpb=true(.*)', TransactionsPage) + protransactions = URL('/vos-comptes/(.*)/transac/(professionnels|entreprises)', ProTransactionsPage) iban = URL('/vos-comptes/IPT/cdnProxyResource/transacClippe/RIB_impress.asp', IbanPage) + account_type_page = URL("/icd/zco/public-data/public-ws-menuespaceperso.json", AccountTypePage) + labels_page = URL("/icd/zco/public-data/ws-menu.json", LabelsPage) + profile_page = URL("/icd/zco/data/user.json", ProfilePage) - account_type = 'particuliers' - - def __init__(self, website, *args, **kwargs): + def __init__(self, *args, **kwargs): + self.weboob = kwargs['weboob'] super(CreditDuNordBrowser, self).__init__(*args, **kwargs) - self.BASEURL = "https://%s" % website def is_logged(self): return self.page is not None and not self.login.is_here() and \ @@ -56,59 +60,64 @@ def is_logged(self): def home(self): if self.is_logged(): - self.location('/vos-comptes/%s' % self.account_type, params=self.strid) - self.location(self.page.doc.xpath(u'//a[contains(text(), "Synthèse")]')[0].attrib['href']) + self.location("/icd/zco/") + self.accounts.go(account_type=self.account_type) else: self.do_login() def do_login(self): self.login.go().login(self.username, self.password) - if self.accounts.is_here(): expired_error = self.page.get_password_expired() if expired_error: raise BrowserPasswordExpired(expired_error) if self.login.is_here(): - raise BrowserIncorrectPassword(self.page.get_error()) + error = self.page.get_error() + if error: + raise BrowserIncorrectPassword(error) + else: + # in case we are still on login without error message + # we'll check what's happening. + assert False, "Still on login page." if not self.is_logged(): raise BrowserIncorrectPassword() - self.strid = {"strid": self.page.get_strid()} - m = re.match('https://[^/]+/vos-comptes/(\w+).*', self.url) - if m: - self.account_type = m.group(1) - - @need_login def _iter_accounts(self): - self.home() - self.location(self.page.get_av_link()) - if self.av.is_here(): + self.loans.go(account_type=self.account_type, loans_page_label=self.loans_page_label) + for a in self.page.get_list(): + yield a + self.accounts.go(account_type=self.account_type, accounts_page_label=self.accounts_page_label) + self.multitype_av.go() + if self.multitype_av.is_here(): for a in self.page.get_av_accounts(): self.location(a._link, data=a._args) self.location(a._link.replace("_attente", "_detail_contrat_rep"), data=a._args) self.page.fill_diff_currency(a) yield a - self.home() - for a in self.page.get_list(): - yield a - self.loans.go() + self.accounts.go(account_type=self.account_type, accounts_page_label=self.accounts_page_label) for a in self.page.get_list(): yield a @need_login - def get_accounts_list(self): - accounts = list(self._iter_accounts()) + def get_pages_labels(self): + self.labels_page.go() + return self.page.get_labels() - self.page.iban_page() + @need_login + def get_accounts_list(self): + self.accounts_page_label, self.loans_page_label = self.get_pages_labels() + self.account_type_page.go() + self.account_type = self.page.get_account_type() + accounts = list(self._iter_accounts()) + self.multitype_iban.go() link = self.page.iban_go() - if self.page.has_iban(): - for a in [a for a in accounts if a._acc_nb]: - self.location(link + a._acc_nb) - a.iban = self.page.get_iban() + for a in [a for a in accounts if a._acc_nb]: + self.location(link + a._acc_nb) + a.iban = self.page.get_iban() return accounts @@ -116,15 +125,18 @@ def get_account(self, id): account_list = self.get_accounts_list() return find_object(account_list, id=id) + @need_login + def get_account_for_history(self, id): + account_list = list(self._iter_accounts()) + return find_object(account_list, id=id) + @need_login def iter_transactions(self, link, args, acc_type): if args is None: return - while args is not None: self.location(link, data=args) - assert self.transactions.is_here() - + assert (self.transactions.is_here() or self.protransactions.is_here()) for tr in self.page.get_history(acc_type): yield tr @@ -132,26 +144,19 @@ def iter_transactions(self, link, args, acc_type): @need_login def get_history(self, account, coming=False): - if coming and account.type is not Account.TYPE_CARD or account.type is Account.TYPE_LOAN: - return [] - - self.location('/vos-comptes/%s' % self.account_type, params=self.strid) - transactions = [] + if coming and account.type != Account.TYPE_CARD or account.type == Account.TYPE_LOAN: + return for tr in self.iter_transactions(account._link, account._args, account.type): - transactions.append(tr) - return transactions + yield tr @need_login def get_investment(self, account): - investments = [] - - if u'LIQUIDIT' in account.label: + if 'LIQUIDIT' in account.label: inv = Investment() - inv.code = u'XX-Liquidity' - inv.label = u'Liquidité' + inv.code = 'XX-Liquidity' + inv.label = 'Liquidité' inv.valuation = account.balance - investments.append(inv) - return investments + return [inv] if not account._inv: return [] @@ -159,15 +164,15 @@ def get_investment(self, account): if account.type in (Account.TYPE_MARKET, Account.TYPE_PEA): self.location(account._link, data=account._args) if self.page.can_iter_investments(): - investments = [i for i in self.page.get_market_investment()] + return self.page.get_market_investment() elif (account.type == Account.TYPE_LIFE_INSURANCE): self.location(account._link, data=account._args) self.location(account._link.replace("_attente", "_detail_contrat_rep"), data=account._args) if self.page.can_iter_investments(): - investments = [i for i in self.page.get_deposit_investment()] - return investments + return self.page.get_deposit_investment() + return [] @need_login def get_profile(self): - self.home() + self.profile_page.go() return self.page.get_profile() diff --git a/modules/creditdunord/compat/weboob_capabilities_bank.py b/modules/creditdunord/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/creditdunord/compat/weboob_capabilities_bank.py +++ b/modules/creditdunord/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/creditdunord/module.py b/modules/creditdunord/module.py index bbb5f5d44a..e6d56096ec 100644 --- a/modules/creditdunord/module.py +++ b/modules/creditdunord/module.py @@ -17,6 +17,8 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals + from collections import OrderedDict from .compat.weboob_capabilities_bank import CapBankWealth, AccountNotFound @@ -55,7 +57,8 @@ class CreditDuNordModule(Module, CapBankWealth, CapProfile): def create_default_browser(self): return self.create_browser(self.config['website'].get(), self.config['login'].get(), - self.config['password'].get()) + self.config['password'].get(), + weboob=self.weboob) def iter_accounts(self): for account in self.browser.get_accounts_list(): @@ -65,11 +68,16 @@ def get_account(self, _id): account = self.browser.get_account(_id) if account: return account - else: - raise AccountNotFound() + raise AccountNotFound() + + def get_account_for_history(self, _id): + account = self.browser.get_account_for_history(_id) + if account: + return account + raise AccountNotFound() def iter_history(self, account): - account = self.get_account(account.id) + account = self.get_account_for_history(account.id) for tr in self.browser.get_history(account): if not tr._is_coming: yield tr diff --git a/modules/creditdunord/pages.py b/modules/creditdunord/pages.py index 722fb48f26..e3993a2563 100755 --- a/modules/creditdunord/pages.py +++ b/modules/creditdunord/pages.py @@ -19,17 +19,18 @@ from __future__ import unicode_literals -import re import ast from decimal import Decimal from io import BytesIO from datetime import date as da from lxml import html +import re -from weboob.browser.pages import HTMLPage, LoggedPage +from weboob.browser.pages import HTMLPage, LoggedPage, JsonPage from weboob.browser.elements import method, ItemElement, TableElement -from weboob.browser.filters.standard import CleanText, Date, CleanDecimal, Regexp, Field +from weboob.browser.filters.standard import CleanText, Date, CleanDecimal, Regexp, Format, Field +from weboob.browser.filters.json import Dict from weboob.browser.filters.html import Attr, TableCell from weboob.exceptions import ActionNeeded, BrowserIncorrectPassword, BrowserUnavailable, BrowserPasswordExpired from weboob.capabilities.bank import Account, Investment @@ -46,7 +47,6 @@ def MyDecimal(*args, **kwargs): kwargs.update(replace_dots=True, default=NotAvailable) return CleanDecimal(*args, **kwargs) - def MyStrip(x, xpath='.'): if isinstance(x, unicode): return CleanText(xpath)(html.fromstring("

%s

" % x)) @@ -102,8 +102,11 @@ def on_load(self): self.browser.location(re.search(r'href="([^"]+)"', script.text).group(1)) -class LoginPage(HTMLPage): +class EntryPage(HTMLPage): + pass + +class LoginPage(HTMLPage): VIRTUALKEYBOARD = CDNVirtKeyboard def login(self, username, password): @@ -127,7 +130,6 @@ def vk_login(self, username, password): 'cryptocvcs': crypto, 'vk_op': 'auth', } - self.browser.location('/swm/redirectCDN.html', data=data) def classic_login(self, username, password): @@ -144,44 +146,77 @@ def classic_login(self, username, password): 'pwAuth': 'Authentification+mot+de+passe', 'username': username.encode(self.browser.ENCODING), } - self.browser.location('/saga/authentification', data=data) def get_error(self): return CleanText('//b[has-class("x-attentionErreurLigneHaut")]', default="")(self.doc) +class AccountTypePage(LoggedPage, JsonPage): + def get_account_type(self): + account_type = CleanText(Dict('donnees/id'))(self.doc) + if account_type == "menu_espace_perso_part": + return "particuliers" + elif account_type == "menu_espace_perso_pro": + return "professionnels" + elif account_type == "menu_espace_perso_ent": + return "entreprises" + + +class LabelsPage(LoggedPage, JsonPage): + def get_labels(self): + if not Dict('donnees')(self.doc) and Dict('commun/statut', default='')(self.doc) == 'nok': + # Dict('commun/statut') is only `GDPR` so we don't pass specific message. + raise ActionNeeded() + + synthesis_labels = ["Synthèse"] + loan_labels = ["Crédits en cours", "Crédits perso et immo", "Crédits"] + for element in Dict('donnees/0/submenu')(self.doc): + if CleanText(Dict('label'))(element) in synthesis_labels: + synthesis_label = CleanText(Dict('link'))(element).split("/")[-1] + if CleanText(Dict('label'))(element) in loan_labels: + loan_label = CleanText(Dict('link'))(element).split("/")[-1] + return (synthesis_label, loan_label) + + +class ProfilePage(LoggedPage, JsonPage): + def get_profile(self): + profile = Profile() + profile.name = Format('%s %s', CleanText(Dict('donnees/nom')), CleanText(Dict('donnees/prenom'), default=''))(self.doc) + return profile + + class CDNBasePage(HTMLPage): - def get_from_js(self, pattern, end_pattern, is_list=False): - """ - find a pattern in any javascript text - """ - for script in self.doc.xpath('//script'): - txt = script.text - if txt is None: - continue + def get_from_js(self, pattern, end_pattern, is_list=False): + """ + find a pattern in any javascript text + """ + for script in self.doc.xpath('//script'): + txt = script.text + if txt is None: + continue - start = txt.find(pattern) - if start < 0: - continue + start = txt.find(pattern) + if start < 0: + continue - values = [] - while start >= 0: - start += len(pattern) - end = txt.find(end_pattern, start) - values.append(txt[start:end]) + values = [] + while start >= 0: + start += len(pattern) + end = txt.find(end_pattern, start) + values.append(txt[start:end]) - if not is_list: - break + if not is_list: + break - start = txt.find(pattern, end) - return ','.join(values) + start = txt.find(pattern, end) + return ','.join(values) - def get_execution(self): - return self.get_from_js("name: 'execution', value: '", "'") + def get_execution(self): + return self.get_from_js("name: 'execution', value: '", "'") - def iban_go(self): - return '%s%s' % ('/vos-comptes/IPT/cdnProxyResource', self.get_from_js('C_PROXY.StaticResourceClientTranslation( "', '"')) + def iban_go(self): + return '%s%s' % ('/vos-comptes/IPT/cdnProxyResource', self.get_from_js('C_PROXY.StaticResourceClientTranslation( "', '"')) class AccountsPage(LoggedPage, CDNBasePage): @@ -206,6 +241,7 @@ class AccountsPage(LoggedPage, CDNBasePage): u'PLAN ÉPARGNE': Account.TYPE_SAVINGS, u'ASS.VIE': Account.TYPE_LIFE_INSURANCE, u'ÉTOILE AVANCE': Account.TYPE_LOAN, + u'ETOILE AVANCE': Account.TYPE_LOAN, u'PRÊT': Account.TYPE_LOAN, u'CREDIT': Account.TYPE_LOAN, u'FACILINVEST': Account.TYPE_LOAN, @@ -213,6 +249,14 @@ class AccountsPage(LoggedPage, CDNBasePage): u'COMPTE A TERME': Account.TYPE_DEPOSIT, } + def make__args_dict(self, line): + return {'_eventId': 'clicDetailCompte', + '_ipc_eventValue': '', + '_ipc_fireEvent': '', + 'execution': self.get_execution(), + 'idCompteClique': line[self.COL_ID], + } + def get_password_expired(self): error = CleanText('//div[@class="x-attentionErreur"]/b')(self.doc) if "vous devez modifier votre code confidentiel à la première connexion" in error: @@ -222,7 +266,6 @@ def get_account_type(self, label): for pattern, actype in sorted(self.TYPES.items()): if label.startswith(pattern) or label.endswith(pattern): return actype - return Account.TYPE_UNKNOWN def get_history_link(self): @@ -231,15 +274,6 @@ def get_history_link(self): def get_av_link(self): return self.doc.xpath('//a[contains(text(), "Consultation")]')[0].attrib['href'] - def make__args_dict(self, line): - return {'_eventId': 'clicDetailCompte', - '_ipc_eventValue': '', - '_ipc_fireEvent': '', - 'deviseAffichee': 'DEVISE', - 'execution': self.get_execution(), - 'idCompteClique': line[self.COL_ID], - } - def get_list(self): accounts = [] previous_account = None @@ -317,6 +351,10 @@ def get_strid(self): return re.search(r'(\d{4,})', Attr('//form[@name="changePageForm"]', 'action')(self.doc)).group(0) +class ProIbanPage(CDNBasePage): + pass + + class AVPage(LoggedPage, CDNBasePage): COL_LABEL = 0 COL_BALANCE = 3 @@ -331,7 +369,6 @@ def get_params(self, text): l.append(sub) for i, key in enumerate(self.ARGS): args[key] = l[self.ARGS.index(key)] - return url, args def get_av_accounts(self): @@ -343,17 +380,27 @@ def get_av_accounts(self): continue a = Account() + + # get acc_nb like on accounts page + a._acc_nb = Regexp( + CleanText('//div[@id="v1-cadre"]//b[contains(text(), "Compte N")]', replace=[(' ', '')]), + r'(\d+)' + )(self.doc)[5:] + a.label = CleanText('.')(cols[self.COL_LABEL]) a.type = Account.TYPE_LIFE_INSURANCE a.balance = MyDecimal('.')(cols[self.COL_BALANCE]) a.currency = a.get_currency(CleanText('.')(head_cols[self.COL_BALANCE])) a._link, a._args = self.get_params(cols[self.COL_LABEL].find('span/a').attrib['href']) - a.id = a._args['IndiceSupport'] + a._args['NumPolice'] - a._acc_nb = None + a.id = '%s%s%s' % (a._acc_nb, a._args['IndiceSupport'], a._args['NumPolice']) a._inv = True yield a +class PartAVPage(AVPage): + pass + + class ProAccountsPage(AccountsPage): COL_ID = 0 COL_BALANCE = 1 @@ -421,10 +468,13 @@ def get_list(self): a._link, a._args = None, None a._acc_nb = cols[self.COL_ID].xpath('.//span[@class="right-underline"] | .//span[@class="right"]')[0].text.replace(' ', '').strip() + a.id = a._acc_nb if hasattr(a, '_args') and a._args: - a.id = '%s%s%s' % (a._acc_nb, a._args['IndiceCompte'], a._args['Indiceclassement']) - else: - a.id = a._acc_nb + if a._args['IndiceCompte'].isdigit(): + a.id = '%s%s' % (a.id, a._args['IndiceCompte']) + if a._args['Indiceclassement'].isdigit(): + a.id = '%s%s' % (a.id, a._args['Indiceclassement']) + # This account can be multiple life insurance accounts if (any(a.label.startswith(lab) for lab in ['ASS.VIE-BONS CAPI-SCPI-DIVERS', 'BONS CAPI-SCPI-DIVERS']) or (u'Aucun d\\351tail correspondant pour ce compte' in tr.xpath('.//a/@href')[0]) @@ -598,8 +648,7 @@ def get_market_investment(self): COL_VALUATION = 4 COL_PERF = 5 - - for table in self.doc.xpath('//table[@class="datas-large"]'): + for table in self.doc.xpath('//div[not(@id="PortefeuilleCV")]/table[@class="datas"]'): for tr in table.xpath('.//tr[not(@class="entete")]'): cols = tr.findall('td') if len(cols) < 7: @@ -621,41 +670,33 @@ def get_market_investment(self): @method class get_deposit_investment(TableElement): - item_xpath = '//table[@class="datas"]/tr[not(@class="entete")]' - head_xpath = '//table[@class="datas"]/tr[(@class="entete")]/td/b' + item_xpath = '//table[@class="datas"]//tr[position()>1]' + head_xpath = '//table[@class="datas"]//tr[@class="entete"]/td/b' - col_label = re.compile('Libellé') - col_quantity = 'Quantité' - col_valuation = 'Montant (EUR)' - col_unitvalue = re.compile('Valeur liquidative') + col_label = u'Libellé' + col_quantity = u'Quantité' + col_unitvalue = re.compile(u"Valeur liquidative") + col_valuation = re.compile(u"Montant") class item(ItemElement): klass = Investment - - def obj_label(self): - return CleanText('./a')(TableCell("label")(self)[0]) or CleanText(TableCell("label"))(self) - - def obj_code(self): - link_label = CleanText('./a')(TableCell("label")(self)[0]) - if link_label: - return CleanText('./text()', default=NotAvailable)(TableCell("label")(self)[0]) - - obj_quantity = MyDecimal(TableCell("quantity")) - - obj_unitvalue = MyDecimal(TableCell("unitvalue")) - + obj_label = CleanText(TableCell('label')) + obj_quantity = MyDecimal(CleanText(TableCell('quantity'))) + obj_valuation = MyDecimal(TableCell('valuation')) + obj_unitvalue = MyDecimal(TableCell('unitvalue')) def obj_vdate(self): - if Field('unitvalue')(self): - return Date().filter(TableCell("unitvalue")(self)[0].xpath("./text()")[1]) - return Date(dayfirst=True, default=NotAvailable).filter(Regexp(CleanText('(//tr[td[span|b[contains(text(), "Estimation du contrat")]]]/td[2]/span)[2]'), \ - '(\d{2})/(\d{2})/(\d{4})', '\\3-\\2-\\1', default=NotAvailable)(self.page.doc)) - - obj_valuation = MyDecimal(TableCell("valuation")) - + if Field('unitvalue') is NotAvailable: + vdate = Date(dayfirst=True, default=NotAvailable)\ + .filter(Regexp(CleanText('.'), '(\d{2})/(\d{2})/(\d{4})', '\\3-\\2-\\1', default=NotAvailable)(TableCell('unitvalue')(self))) or \ + Date(dayfirst=True, default=NotAvailable)\ + .filter(Regexp(CleanText('//tr[td[span[b[contains(text(), "Estimation du contrat")]]]]/td[2]'), + '(\d{2})/(\d{2})/(\d{4})', '\\3-\\2-\\1', default=NotAvailable)(TableCell('unitvalue')(self))) + return vdate def fill_diff_currency(self, account): - valuation_diff = CleanText(u'//td[span[contains(text(), "dont +/- value : ")]]//b', default=None)(self.doc) - if valuation_diff: + valuation_diff = CleanText(u'//td[span[contains(text(), "dont +/- value : ")]]//b', default=None)(self.doc) + #NC == Non communiqué + if valuation_diff and "NC" not in valuation_diff: account.valuation_diff = MyDecimal().filter(valuation_diff) account.currency = account.get_currency(valuation_diff) diff --git a/modules/creditdunordpee/compat/weboob_capabilities_bank.py b/modules/creditdunordpee/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/creditdunordpee/compat/weboob_capabilities_bank.py +++ b/modules/creditdunordpee/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/creditmutuel/browser.py b/modules/creditmutuel/browser.py index d639dcb7b5..27633c2257 100644 --- a/modules/creditmutuel/browser.py +++ b/modules/creditmutuel/browser.py @@ -19,9 +19,11 @@ from __future__ import unicode_literals +import re from datetime import datetime from dateutil.relativedelta import relativedelta from itertools import groupby +from operator import attrgetter from weboob.tools.compat import basestring from weboob.tools.value import Value @@ -32,9 +34,11 @@ from weboob.browser.pages import FormNotFound from weboob.browser.exceptions import ClientError, ServerError from .compat.weboob_exceptions import BrowserIncorrectPassword, AuthMethodNotImplemented, BrowserUnavailable -from weboob.capabilities.bank import Account, AddRecipientStep, AddRecipientError, Recipient, Investment +from weboob.capabilities.bank import Account, AddRecipientStep, Recipient +from .compat.weboob_tools_capabilities_bank_investments import create_french_liquidity from weboob.capabilities import NotAvailable from weboob.tools.compat import urlparse +from weboob.capabilities.base import find_object from .pages import ( LoginPage, LoginErrorPage, AccountsPage, UserSpacePage, @@ -44,7 +48,7 @@ LIAccountsPage, CardsActivityPage, CardsListPage, CardsOpePage, NewAccountsPage, InternalTransferPage, ExternalTransferPage, RevolvingLoanDetails, RevolvingLoansList, - ErrorPage, SubscriptionPage, + ErrorPage, SubscriptionPage, CardsHistAvailable, CardPage2 ) @@ -58,75 +62,79 @@ class CreditMutuelBrowser(LoginBrowser, StatesMixin): BASEURL = 'https://www.creditmutuel.fr' login = URL('/fr/authentification.html', - '/(?P.*)fr/$', - '/(?P.*)fr/banques/accueil.html', - '/(?P.*)fr/banques/particuliers/index.html', + r'/(?P.*)fr/$', + r'/(?P.*)fr/banques/accueil.html', + r'/(?P.*)fr/banques/particuliers/index.html', LoginPage) - login_error = URL('/(?P.*)fr/identification/default.cgi', LoginErrorPage) - accounts = URL('/(?P.*)fr/banque/situation_financiere.cgi', - '/(?P.*)fr/banque/situation_financiere.html', + login_error = URL(r'/(?P.*)fr/identification/default.cgi', LoginErrorPage) + accounts = URL(r'/(?P.*)fr/banque/situation_financiere.cgi', + r'/(?P.*)fr/banque/situation_financiere.html', AccountsPage) revolving_loan_list = URL(r'/(?P.*)fr/banque/CR/arrivee.asp\?fam=CR.*', RevolvingLoansList) revolving_loan_details = URL(r'/(?P.*)fr/banque/CR/cam9_vis_lstcpt.asp.*', RevolvingLoanDetails) - user_space = URL('/(?P.*)fr/banque/espace_personnel.aspx', - '/(?P.*)fr/banque/accueil.cgi', - '/(?P.*)fr/banque/DELG_Gestion', - '/(?P.*)fr/banque/paci_engine/static_content_manager.aspx', + user_space = URL(r'/(?P.*)fr/banque/espace_personnel.aspx', + r'/(?P.*)fr/banque/accueil.cgi', + r'/(?P.*)fr/banque/DELG_Gestion', + r'/(?P.*)fr/banque/paci_engine/static_content_manager.aspx', UserSpacePage) - card = URL('/(?P.*)fr/banque/operations_carte.cgi.*', - '/(?P.*)fr/banque/mouvements.html\?webid=.*cardmonth=\d+$', - '/(?P.*)fr/banque/mouvements.html.*webid=.*cardmonth=\d+.*cardid=', + card = URL(r'/(?P.*)fr/banque/operations_carte.cgi.*', + r'/(?P.*)fr/banque/mouvements.html\?webid=.*cardmonth=\d+$', + r'/(?P.*)fr/banque/mouvements.html.*webid=.*cardmonth=\d+.*cardid=', CardPage) - operations = URL('/(?P.*)fr/banque/mouvements.cgi.*', - '/(?P.*)fr/banque/mouvements.html.*', - '/(?P.*)fr/banque/nr/nr_devbooster.aspx.*', + operations = URL(r'/(?P.*)fr/banque/mouvements.cgi.*', + r'/(?P.*)fr/banque/mouvements.html.*', + r'/(?P.*)fr/banque/nr/nr_devbooster.aspx.*', r'(?P.*)fr/banque/CRP8_GESTPMONT.aspx\?webid=.*&trnref=.*&contract=\d+&cardid=.*&cardmonth=\d+', OperationsPage) - coming = URL('/(?P.*)fr/banque/mvts_instance.cgi.*', ComingPage) - info = URL('/(?P.*)fr/banque/BAD.*', EmptyPage) - change_pass = URL('/(?P.*)fr/validation/change_password.cgi', + coming = URL(r'/(?P.*)fr/banque/mvts_instance.cgi.*', ComingPage) + info = URL(r'/(?P.*)fr/banque/BAD.*', EmptyPage) + change_pass = URL(r'/(?P.*)fr/validation/change_password.cgi', '/fr/services/change_password.html', ChangePasswordPage) - verify_pass = URL('/(?P.*)fr/validation/verif_code.cgi.*', VerifCodePage) - new_home = URL('/(?P.*)fr/banque/pageaccueil.html', - '/(?P.*)banque/welcome_pack.html', NewHomePage) - empty = URL('/(?P.*)fr/banques/index.html', - '/(?P.*)fr/banque/paci_beware_of_phishing.*', - '/(?P.*)fr/validation/(?!change_password|verif_code|image_case|infos).*', + verify_pass = URL(r'/(?P.*)fr/validation/verif_code.cgi.*', VerifCodePage) + new_home = URL(r'/(?P.*)fr/banque/pageaccueil.html', + r'/(?P.*)banque/welcome_pack.html', NewHomePage) + empty = URL(r'/(?P.*)fr/banques/index.html', + r'/(?P.*)fr/banque/paci_beware_of_phishing.*', + r'/(?P.*)fr/validation/(?!change_password|verif_code|image_case|infos).*', EmptyPage) - por = URL('/(?P.*)fr/banque/POR_ValoToute.aspx', - '/(?P.*)fr/banque/POR_SyntheseLst.aspx', + por = URL(r'/(?P.*)fr/banque/POR_ValoToute.aspx', + r'/(?P.*)fr/banque/POR_SyntheseLst.aspx', PorPage) - li = URL('/(?P.*)fr/assurances/profilass.aspx\?domaine=epargne', - '/(?P.*)fr/assurances/(consultations?/)?WI_ASS.*', - '/(?P.*)fr/assurances/WI_ASS', + li = URL(r'/(?P.*)fr/assurances/profilass.aspx\?domaine=epargne', + r'/(?P.*)fr/assurances/(consultations?/)?WI_ASS.*', + r'/(?P.*)fr/assurances/WI_ASS', '/fr/assurances/', LIAccountsPage) - iban = URL('/(?P.*)fr/banque/rib.cgi', IbanPage) + iban = URL(r'/(?P.*)fr/banque/rib.cgi', IbanPage) - new_accounts = URL('/(?P.*)fr/banque/comptes-et-contrats.html', NewAccountsPage) - new_operations = URL('/(?P.*)fr/banque/mouvements.cgi', - '/fr/banque/nr/nr_devbooster.aspx.*', - '/(?P.*)fr/banque/RE/aiguille(liste)?.asp', + new_accounts = URL(r'/(?P.*)fr/banque/comptes-et-contrats.html', NewAccountsPage) + new_operations = URL(r'/(?P.*)fr/banque/mouvements.cgi', + r'/fr/banque/nr/nr_devbooster.aspx.*', + r'/(?P.*)fr/banque/RE/aiguille(liste)?.asp', '/fr/banque/mouvements.html', - '/(?P.*)fr/banque/consultation/operations', OperationsPage) + r'/(?P.*)fr/banque/consultation/operations', OperationsPage) - advisor = URL('/(?P.*)fr/banques/contact/trouver-une-agence/(?P.*)', - '/(?P.*)fr/infoclient/', + advisor = URL(r'/(?P.*)fr/banques/contact/trouver-une-agence/(?P.*)', + r'/(?P.*)fr/infoclient/', r'/(?P.*)fr/banques/accueil/menu-droite/Details.aspx\?banque=.*', AdvisorPage) - redirect = URL('/(?P.*)fr/banque/paci_engine/static_content_manager.aspx', RedirectPage) + redirect = URL(r'/(?P.*)fr/banque/paci_engine/static_content_manager.aspx', RedirectPage) - cards_activity = URL('/(?P.*)fr/banque/pro/ENC_liste_tiers.aspx', CardsActivityPage) - cards_list = URL('/(?P.*)fr/banque/pro/ENC_liste_ctr.*', - '/(?P.*)fr/banque/pro/ENC_detail_ctr', CardsListPage) - cards_ope = URL('/(?P.*)fr/banque/pro/ENC_liste_oper', CardsOpePage) + cards_activity = URL(r'/(?P.*)fr/banque/pro/ENC_liste_tiers.aspx', CardsActivityPage) + cards_list = URL(r'/(?P.*)fr/banque/pro/ENC_liste_ctr.*', + r'/(?P.*)fr/banque/pro/ENC_detail_ctr', CardsListPage) + cards_ope = URL(r'/(?P.*)fr/banque/pro/ENC_liste_oper', CardsOpePage) + cards_ope2 = URL('/(?P.*)fr/banque/CRP8_SCIM_DEPCAR.aspx', CardPage2) - internal_transfer = URL('/(?P.*)fr/banque/virements/vplw_vi.html', InternalTransferPage) - external_transfer = URL('/(?P.*)fr/banque/virements/vplw_vee.html', ExternalTransferPage) - recipients_list = URL('/(?P.*)fr/banque/virements/vplw_bl.html', RecipientsListPage) - error = URL('/(?P.*)validation/infos.cgi', ErrorPage) + cards_hist_available = URL('/(?P.*)fr/banque/SCIM_default.aspx\?_tabi=C&_stack=SCIM_ListeActivityStep%3a%3a&_pid=ListeCartes&_fid=ChangeList&Data_ServiceListDatas_CurrentType=MyCards', CardsHistAvailable) + cards_hist_available2 = URL('/(?P.*)fr/banque/SCIM_default.aspx', CardsHistAvailable) - subscription = URL('/(?P.*)fr/banque/MMU2_LstDoc.aspx', SubscriptionPage) + internal_transfer = URL(r'/(?P.*)fr/banque/virements/vplw_vi.html', InternalTransferPage) + external_transfer = URL(r'/(?P.*)fr/banque/virements/vplw_vee.html', ExternalTransferPage) + recipients_list = URL(r'/(?P.*)fr/banque/virements/vplw_bl.html', RecipientsListPage) + error = URL(r'/(?P.*)validation/infos.cgi', ErrorPage) + + subscription = URL(r'/(?P.*)fr/banque/MMU2_LstDoc.aspx', SubscriptionPage) currentSubBank = None is_new_website = None @@ -172,8 +180,21 @@ def get_accounts_list(self): if not self.accounts_list: if self.currentSubBank is None: self.getCurrentSubBank() + self.accounts_list = [] self.revolving_accounts = [] + self.unavailablecards = [] + self.cards_histo_available = {} + + # For some cards the validity information is only availaible on these 2 links + self.cards_hist_available.go(subbank=self.currentSubBank) + if self.cards_hist_available.is_here(): + self.unavailablecards.extend(self.page.get_unavailable_cards()) + self.cards_histo_available.update(self.page.get_cards_list()) + + if self.cards_hist_available2.is_here(): + self.unavailablecards.extend(self.page.get_unavailable_cards()) + self.cards_histo_available.update(self.page.get_cards_list()) for acc in self.revolving_loan_list.stay_or_go(subbank=self.currentSubBank).iter_accounts(): self.accounts_list.append(acc) @@ -185,39 +206,47 @@ def get_accounts_list(self): [self.page] if self.is_new_website else [] for company in companies: page = self.open(company).page if isinstance(company, basestring) else company - self.accounts_list.extend([card for card in page.iter_cards()]) + self.accounts_list.extend(page.iter_cards()) + # Populate accounts from old website if not self.is_new_website: - for a in self.accounts.stay_or_go(subbank=self.currentSubBank).iter_accounts(): - self.accounts_list.append(a) + self.accounts.stay_or_go(subbank=self.currentSubBank) + self.accounts_list.extend(self.page.iter_accounts()) self.iban.go(subbank=self.currentSubBank).fill_iban(self.accounts_list) self.por.go(subbank=self.currentSubBank).add_por_accounts(self.accounts_list) + # Populate accounts from new website else: - for a in self.new_accounts.stay_or_go(subbank=self.currentSubBank).iter_accounts(): - self.accounts_list.append(a) + self.new_accounts.stay_or_go(subbank=self.currentSubBank) + self.accounts_list.extend(self.page.iter_accounts()) self.iban.go(subbank=self.currentSubBank).fill_iban(self.accounts_list) self.por.go(subbank=self.currentSubBank).add_por_accounts(self.accounts_list) - for acc in self.li.go(subbank=self.currentSubBank).iter_li_accounts(): - self.accounts_list.append(acc) + self.li.go(subbank=self.currentSubBank) + self.accounts_list.extend(self.page.iter_li_accounts()) + for acc in self.accounts_list: + if acc.id[:16] in self.cards_histo_available: + # ex of ID card : 000000xxxxxx0000 + acc.parent = find_object(self.accounts_list, id=self.cards_histo_available[acc.id[:16]][2]) excluded_label = ['etalis', 'valorisation totale'] self.accounts_list = [acc for acc in self.accounts_list if not any(w in acc.label.lower() for w in excluded_label)] return self.accounts_list - def get_account(self, id): - assert isinstance(id, basestring) + def get_account(self, _id): + assert isinstance(_id, basestring) for a in self.get_accounts_list(): - if a.id == id: + if a.id == _id: return a def getCurrentSubBank(self): # the account list and history urls depend on the sub bank of the user paths = urlparse(self.url).path.lstrip('/').split('/') self.currentSubBank = paths[0] + "/" if paths[0] != "fr" else "" + if self.currentSubBank and paths[0] == 'banqueprivee' and paths[1] == 'mabanque': + self.currentSubBank = 'banqueprivee/mabanque/' if self.currentSubBank and paths[1] == "decouverte": self.currentSubBank += paths[1] + "/" if paths[0] in ["fr", "mabanque"]: @@ -236,28 +265,32 @@ def list_operations(self, page, account): else: self.page = page - # on some savings accounts, the page lands on the contract tab, and we want the situation + # On some savings accounts, the page lands on the contract tab, and we want the situation if account.type == Account.TYPE_SAVINGS and "Capital Expansion" in account.label: self.page.go_on_history_tab() - # getting about 6 months history on new website + # Getting about 6 months history on new website if self.is_new_website and self.page: try: - for x in range(0, 2): + # Submit search form two times, at first empty, then filled based on available fields + for x in range(2): form = self.page.get_form(id="I1:fm", submit='//input[@name="_FID_DoActivateSearch"]') if x == 1: form.update({ - [k for k in form.keys() if "DateStart" in k][0]: (datetime.now() - relativedelta(months=7)).strftime('%d/%m/%Y'), - [k for k in form.keys() if "DateEnd" in k][0]: datetime.now().strftime('%d/%m/%Y') + next(k for k in form.keys() if "DateStart" in k): (datetime.now() - relativedelta(months=7)).strftime('%d/%m/%Y'), + next(k for k in form.keys() if "DateEnd" in k): datetime.now().strftime('%d/%m/%Y') }) - [form.pop(k, None) for k in form.keys() if "_FID_Do" in k and "DoSearch" not in k] + for k in form.keys(): + if "_FID_Do" in k and "DoSearch" not in k: + form.pop(k, None) form.submit() - except (IndexError, FormNotFound): - pass + # IndexError when form xpath returns [], StopIteration if next called on empty iterable + except (IndexError, StopIteration, FormNotFound): + self.logger.warning('Could not get history on new website') while self.page: try: - #submit form if their is more transactions to fetch + # Submit form if their is more transactions to fetch form = self.page.get_form(id="I1:fm") if self.page.doc.xpath('boolean(//a[@class="ei_loadmorebtn"])'): form['_FID_DoLoadMoreTransactions'] = "" @@ -266,7 +299,7 @@ def list_operations(self, page, account): break except (IndexError, FormNotFound): break - #sometime the browser can't go further + # Sometimes the browser can't go further except ClientError as exc: if exc.response.status_code == 413: break @@ -281,12 +314,15 @@ def list_operations(self, page, account): return self.pagination(lambda: self.page.get_history()) def get_monthly_transactions(self, trs): - groups = [list(g) for k, g in groupby(sorted(trs, key=lambda tr: tr.date), lambda tr: tr.date)] + date_getter = attrgetter('date') + groups = [list(g) for k, g in groupby(sorted(trs, key=date_getter), date_getter)] trs = [] for group in groups: + if group[0].date > datetime.today().date(): + continue tr = FrenchTransaction() - tr.raw = tr.label = u"RELEVE CARTE %s" % group[0].date - tr.amount = -sum([t.amount for t in group]) + tr.raw = tr.label = "RELEVE CARTE %s" % group[0].date + tr.amount = -sum(t.amount for t in group) tr.date = tr.rdate = tr.vdate = group[0].date tr.type = FrenchTransaction.TYPE_CARD_SUMMARY tr._is_coming = False @@ -300,8 +336,71 @@ def get_history(self, account): if not account._link_id: raise NotImplementedError() + if len(account.id) >= 16 and account.id[:16] in self.cards_histo_available: + # Check if '000000xxxxxx0000' card have an annual history + account._link_id = self.cards_histo_available[account.id[:16]][0] + self.location(account._link_id) + # The history of the card is available for 1 year with 1 month per page + # Here we catch all the url needed to be the more compatible with the catch of merged subtransactions + urlstogo = self.page.get_links() + self.location(account._link_id) + monthly_tr = [] + for url in urlstogo: + self.location(url) + if 'GoMonthPrecedent' not in url: + history = self.page.get_history() + self.tr_date = self.page.get_date() + if self.page.has_more_operations(): + for i in range(1, 100): + # Arbitrary range; it's the number of click needed to access to the full history of the month (stop with the next break) + data = { + '_FID_DoAddElem': '', + '_wxf2_cc': 'fr-FR', + '_wxf2_pmode': 'Normal', + '_wxf2_pseq': i, + '_wxf2_ptarget': 'C:P:updPan', + 'Data_ServiceListDatas_CurrentOtherCardThirdPartyNumber': '', + 'Data_ServiceListDatas_CurrentType': 'MyCards', + } + if 'fid=GoMonth&mois=' in self.url: + m = re.search(r'fid=GoMonth&mois=(\d+)', self.url) + if m: + m = m.group(1) + self.location('CRP8_SCIM_DEPCAR.aspx?_tabi=C&a__itaret=as=SCIM_ListeActivityStep\%3a\%3a\%2fSCIM_ListeRouter%3a%3a&a__mncret=SCIM_LST&a__ecpid=EID2011&_stack=_remote::moiSelectionner={},moiAfficher=firstHalf,typeDepense=T&_pid=SCIM_DEPCAR_Details'.format(m), data=data) + else: + self.location(self.url, data=data) + + if not self.page.has_more_operations_xml(): + history = self.page.iter_history_xml() + # We are now with an XML page with all the transactions of the months + break + else: + history = self.page.get_history() + + merged_amount = 0 + monthly_tr = [] + for tr in history: + if tr._regroup: + self.location(tr._regroup) + for tr2 in self.page.get_tr_merged(): + tr2._is_coming = tr._is_coming + tr2.date = self.tr_date + transactions.append(tr2) + merged_amount += tr2.amount + else: + transactions.append(tr) + monthly_tr.append(tr) + + monthly_summary = self.get_monthly_transactions(monthly_tr) + if monthly_summary: + monthly_summary[0].amount = abs(monthly_summary[0].amount - merged_amount) + transactions.extend(monthly_summary) + monthly_summary = [] + + return sorted_transactions(transactions) + # need to refresh the months select - if account._link_id.startswith('ENC_liste_oper'): + elif account._link_id.startswith('ENC_liste_oper'): self.location(account._pre_link) if not hasattr(account, '_card_pages'): @@ -314,11 +413,15 @@ def get_history(self, account): transactions.append(tr) differed_date = None - cards = [page.select_card(account._card_number) for page in account._card_pages] if hasattr(account, '_card_pages') else \ - account._card_links if hasattr(account, '_card_links') else [] + cards = ([page.select_card(account._card_number) for page in account._card_pages] + if hasattr(account, '_card_pages') + else account._card_links if hasattr(account, '_card_links') else []) for card in cards: card_trs = [] for tr in self.list_operations(card, account): + if tr._to_delete: + # Delete main transaction when subtransactions exist + continue if hasattr(tr, '_differed_date') and (not differed_date or tr._differed_date < differed_date): differed_date = tr._differed_date if tr.date >= datetime.now(): @@ -332,9 +435,9 @@ def get_history(self, account): if differed_date is not None: # set deleted for card_summary for tr in transactions: - tr.deleted = tr.type == FrenchTransaction.TYPE_CARD_SUMMARY and \ - differed_date.month <= tr.date.month and \ - not hasattr(tr, '_is_manualsum') + tr.deleted = (tr.type == FrenchTransaction.TYPE_CARD_SUMMARY + and differed_date.month <= tr.date.month + and not hasattr(tr, '_is_manualsum')) transactions = sorted_transactions(transactions) return transactions @@ -351,11 +454,9 @@ def get_investment(self, account): self.location(account._link_inv) return self.page.iter_investment() if account.type is Account.TYPE_PEA: - liquidity_inv = Investment() - liquidity_inv.label = account.label - liquidity_inv.code = u'XX-liquidity' - liquidity_inv.valuation = account.balance - return [liquidity_inv] + liquidities = create_french_liquidity(account.balance) + liquidities.label = account.label + return [liquidities] return iter([]) @need_login @@ -380,7 +481,7 @@ def iter_recipients(self, origin_account): @need_login def init_transfer(self, account, to, amount, reason=None): - if to.category != u'Interne': + if to.category != 'Interne': self.external_transfer.go(subbank=self.currentSubBank) else: self.internal_transfer.go(subbank=self.currentSubBank) @@ -436,13 +537,13 @@ def get_recipient_object(self, recipient): r.category = recipient.category # On credit mutuel recipients are immediatly available. r.enabled_at = datetime.now().replace(microsecond=0) - r.currency = u'EUR' + r.currency = 'EUR' r.bank_name = NotAvailable return r def continue_new_recipient(self, recipient, **params): - if u'Clé' in params: - self.page.post_code(params[u'Clé']) + if 'Clé' in params: + self.page.post_code(params['Clé']) self.page.add_recipient(recipient) if self.page.bic_needed(): self.page.ask_bic(self.get_recipient_object(recipient)) @@ -471,7 +572,7 @@ def post_with_bic(self, recipient, **params): for k, v in self.form.items(): if k != 'url': data[k] = v - data['[t:dbt%3astring;x(11)]data_input_BIC'] = params[u'Bic'] + data['[t:dbt%3astring;x(11)]data_input_BIC'] = params['Bic'] self.location(self.form['url'], data=data) self.page.ask_sms(self.get_recipient_object(recipient)) @@ -483,16 +584,18 @@ def new_recipient(self, recipient, **params): return self.post_with_bic(recipient, **params) if 'code' in params: return self.end_new_recipient(recipient, **params) - if u'Clé' in params: + if 'Clé' in params: return self.continue_new_recipient(recipient, **params) + self.recipients_list.go(subbank=self.currentSubBank) if self.page.has_list(): - if recipient.category not in self.page.get_recipients_list(): - raise AddRecipientError('Recipient category is not on the website available list.') + assert recipient.category in self.page.get_recipients_list(), \ + 'Recipient category is not on the website available list.' self.page.go_list(recipient.category) + self.page.go_to_add() if self.verify_pass.is_here(): - raise AddRecipientStep(self.get_recipient_object(recipient), Value(u'Clé', label=self.page.get_question())) + raise AddRecipientStep(self.get_recipient_object(recipient), Value('Clé', label=self.page.get_question())) else: return self.continue_new_recipient(recipient, **params) @@ -507,7 +610,7 @@ def iter_subscriptions(self): def iter_documents(self, subscription): if self.currentSubBank is None: self.getCurrentSubBank() - self.subscription.go(subbank=self.currentSubBank, params={'typ':'doc'}) + self.subscription.go(subbank=self.currentSubBank, params={'typ': 'doc'}) security_limit = 10 diff --git a/modules/creditmutuel/compat/weboob_capabilities_bank.py b/modules/creditmutuel/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/creditmutuel/compat/weboob_capabilities_bank.py +++ b/modules/creditmutuel/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/creditmutuel/compat/weboob_tools_capabilities_bank_investments.py b/modules/creditmutuel/compat/weboob_tools_capabilities_bank_investments.py new file mode 100644 index 0000000000..2b68ff9e00 --- /dev/null +++ b/modules/creditmutuel/compat/weboob_tools_capabilities_bank_investments.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Jonathan Schmidt +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.tools.compat import basestring +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.bank import Investment + +def is_isin_valid(isin): + """ + Méthode générale + Table de conversion des lettres en chiffres + A=10 B=11 C=12 D=13 E=14 F=15 G=16 H=17 I=18 + J=19 K=20 L=21 M=22 N=23 O=24 P=25 Q=26 R=27 + S=28 T=29 U=30 V=31 W=32 X=33 Y=34 Z=35 + + 1 - Mettre de côté la clé, qui servira de référence à la fin de la vérification. + 2 - Convertir toutes les lettres en nombres via la table de conversion ci-contre. Si le nombre obtenu est supérieur ou égal à 10, prendre les deux chiffres du nombre séparément (exemple : 27 devient 2 et 7). + 3 - Pour chaque chiffre, multiplier sa valeur par deux si sa position est impaire en partant de la droite. Si le nombre obtenu est supérieur ou égal à 10, garder les deux chiffres du nombre séparément (exemple : 14 devient 1 et 4). + 4 - Faire la somme de tous les chiffres. + 5 - Soustraire cette somme de la dizaine supérieure ou égale la plus proche (exemples : si la somme vaut 22, la dizaine « supérieure ou égale » est 30, et la clé vaut donc 8 ; si la somme vaut 30, la dizaine « supérieure ou égale » est 30, et la clé vaut 0 ; si la somme vaut 31, la dizaine « supérieure ou égale » est 40, et la clé vaut 9). + 6 - Comparer la valeur obtenue à la clé mise initialement de côté. + + Étapes 1 et 2 : + F R 0 0 0 3 5 0 0 0 0 (+ 8 : clé) + 15 27 0 0 0 3 5 0 0 0 0 + + Étape 3 : le traitement se fait sur des chiffres + 1 5 2 7 0 0 0 3 5 0 0 0 0 + I P I P I P I P I P I P I : position en partant de la droite (P = Pair, I = Impair) + 2 1 2 1 2 1 2 1 2 1 2 1 2 : coefficient multiplicateur + + 2 5 4 7 0 0 0 3 10 0 0 0 0 : résultat + + Étape 4 : + 2 + 5 + 4 + 7 + 0 + 0 + 0 + 3 + (1 + 0)+ 0 + 0 + 0 + 0 = 22 + + Étapes 5 et 6 : 30 - 22 = 8 (valeur de la clé) + """ + + if not isinstance(isin, basestring): + return False + if not re.match(r'^[A-Z]{2}[A-Z0-9]{9}\d$', isin): + return False + + isin_in_digits = ''.join(str(ord(x) - ord('A') + 10) if not x.isdigit() else x for x in isin[:-1]) + key = isin[-1:] + result = '' + for k, val in enumerate(isin_in_digits[::-1], start=1): + if k % 2 == 0: + result = ''.join((result, val)) + else: + result = ''.join((result, str(int(val)*2))) + return str(sum(int(x) for x in result) + int(key))[-1] == '0' + + +def create_french_liquidity(valuation): + """ + Automatically fills a liquidity investment with label, code and code_type. + """ + liquidity = Investment() + liquidity.label = "Liquidités" + liquidity.code = "XX-liquidity" + liquidity.code_type = NotAvailable + liquidity.valuation = valuation + return liquidity + diff --git a/modules/creditmutuel/module.py b/modules/creditmutuel/module.py index 7afa78f8ae..2b3aba7f8e 100644 --- a/modules/creditmutuel/module.py +++ b/modules/creditmutuel/module.py @@ -22,8 +22,10 @@ from decimal import Decimal from weboob.capabilities.base import find_object, NotAvailable -from .compat.weboob_capabilities_bank import CapBankWealth, CapBankTransferAddRecipient, AccountNotFound, RecipientNotFound, \ - Account, TransferError +from .compat.weboob_capabilities_bank import ( + CapBankWealth, CapBankTransferAddRecipient, AccountNotFound, RecipientNotFound, + Account, TransferError +) from weboob.capabilities.contact import CapContact from weboob.capabilities.profile import CapProfile from weboob.capabilities.bill import ( @@ -110,10 +112,12 @@ def init_transfer(self, transfer, **params): try: assert account.id.isdigit() - # quantize to show 2 decimals. + except AssertionError: + raise TransferError('Account id is invalid') + try: # quantize to show 2 decimals. amount = Decimal(transfer.amount).quantize(Decimal(10) ** -2) - except (AssertionError, ValueError): - raise TransferError('something went wrong') + except ValueError: + raise TransferError('Transfer amount is invalid') # drop characters that can crash website transfer.label = transfer.label.encode('cp1252', errors="ignore").decode('cp1252') diff --git a/modules/creditmutuel/pages.py b/modules/creditmutuel/pages.py index 8e0d506f38..0a5a9f7bad 100644 --- a/modules/creditmutuel/pages.py +++ b/modules/creditmutuel/pages.py @@ -28,10 +28,11 @@ from random import randint from collections import OrderedDict -from weboob.browser.pages import HTMLPage, FormNotFound, LoggedPage, pagination +from weboob.browser.pages import HTMLPage, FormNotFound, LoggedPage, pagination, XMLPage from weboob.browser.elements import ListElement, ItemElement, SkipItem, method, TableElement -from weboob.browser.filters.standard import Filter, Env, CleanText, CleanDecimal, Field, \ - Regexp, Async, AsyncLoad, Date, Format, Type, Currency +from weboob.browser.filters.standard import ( + Filter, Env, CleanText, CleanDecimal, Field, Regexp, Async, AsyncLoad, Date, Format, Type, Currency, +) from .compat.weboob_browser_filters_html import Link, Attr, TableCell, ColumnNotFound from .compat.weboob_exceptions import ( BrowserIncorrectPassword, ParseError, NoAccountsException, ActionNeeded, BrowserUnavailable, @@ -39,8 +40,10 @@ ) from weboob.capabilities import NotAvailable from weboob.capabilities.base import empty, find_object -from weboob.capabilities.bank import Account, Investment, Recipient, TransferError, TransferBankError, \ - Transfer, AddRecipientError, AddRecipientStep, Loan +from .compat.weboob_capabilities_bank import ( + Account, Investment, Recipient, TransferError, TransferBankError, + Transfer, AddRecipientBankError, AddRecipientStep, Loan, +) from weboob.capabilities.contact import Advisor from weboob.capabilities.profile import Profile from weboob.tools.capabilities.bank.iban import is_iban_valid @@ -114,7 +117,7 @@ def on_load(self): class UserSpacePage(LoggedPage, HTMLPage): def on_load(self): if self.doc.xpath('//form[@id="GoValider"]'): - raise ActionNeeded(u"Le site du contrat Banque à Distance a besoin d'informations supplémentaires") + raise ActionNeeded("Le site du contrat Banque à Distance a besoin d'informations supplémentaires") super(UserSpacePage, self).on_load() @@ -160,7 +163,7 @@ class item_account_generic(ItemElement): ("Plan D'Epargne", Account.TYPE_SAVINGS), ('Tonic Croissance', Account.TYPE_SAVINGS), ('Capital Expansion', Account.TYPE_SAVINGS), - ('\xc9pargne', Account.TYPE_SAVINGS), + ('Épargne', Account.TYPE_SAVINGS), ('Compte Garantie Titres', Account.TYPE_MARKET), ]) @@ -261,34 +264,35 @@ def parse(self, el): # classical account id_xpath = './td[1]/a/node()[contains(@class, "doux")]' - id = CleanText(id_xpath, replace=[(' ', '')])(el) - if not id: + _id = CleanText(id_xpath, replace=[(' ', '')])(el) + if not _id: if 'rib' in p: - id = p['rib'][0] + _id = p['rib'][0] else: - id = p['webid'][0] + _id = p['webid'][0] self.env['_is_webid'] = True if self.is_revolving(Field('label')(self)): page = self.page.browser.open(link).page if isinstance(page, RevolvingLoansList): - # some revolving loans are listed on an other page. On the accountList, there is just a link for this page - # that's why we don't handle it here + # some revolving loans are listed on an other page. On the accountList, there is + # just a link for this page, that's why we don't handle it here raise SkipItem() # Handle cards - if id in self.parent.objects: + if _id in self.parent.objects: if not page: page = self.page.browser.open(link).page # Handle real balances - coming = page.find_amount(u"Opérations à venir") if page else None - accounting = page.find_amount(u"Solde comptable") if page else None + coming = page.find_amount("Opérations à venir") if page else None + accounting = page.find_amount("Solde comptable") if page else None # on old website we want card's history in account's history if not page.browser.is_new_website: - account = self.parent.objects[id] + account = self.parent.objects[_id] if not account.coming: account.coming = Decimal('0.0') - date = parse_french_date(Regexp(Field('label'), 'Fin (.+) (\d{4})', '01 \\1 \\2')(self)) + relativedelta(day=31) + # Get end of month + date = parse_french_date(Regexp(Field('label'), r'Fin (.+) (\d{4})', '01 \\1 \\2')(self)) + relativedelta(day=31) if date > datetime.now() - relativedelta(day=1): account.coming += balance account._card_links.append(link) @@ -297,7 +301,10 @@ def parse(self, el): single_card_xpath = '//span[has-class("_c1 fg _c1")]' card_xpath = multiple_cards_xpath + ' | ' + single_card_xpath for elem in page.doc.xpath(card_xpath): - card_id = Regexp(CleanText('.', symbols=' '), '([\dx]{16})')(elem) + card_id = Regexp(CleanText('.', symbols=' '), r'([\dx]{16})')(elem) + if card_id in self.page.browser.unavailablecards: + raise SkipItem() + if any(card_id in a.id for a in page.browser.accounts_list): continue @@ -306,9 +313,9 @@ def parse(self, el): card.id = card._card_number = card_id card._link_id = link card._is_inv = card._is_webid = False - card.parent = self.parent.objects[id] + card.parent = self.parent.objects[_id] - pattern = 'Carte\s(\w+).*\d{4}\s([A-Za-z\s]+)(.*)' + pattern = r'Carte\s(\w+).*\d{4}\s([A-Za-z\s]+)(.*)' m = re.search(pattern, CleanText('.')(elem)) card.label = "%s %s %s" % (m.group(1), card_id, m.group(2)) card.balance = Decimal('0.0') @@ -316,14 +323,14 @@ def parse(self, el): card._card_pages = [page] card.coming = Decimal('0.0') #handling the case were the month is the coming one. There won't be next_month here. - date = parse_french_date(Regexp(Field('label'), 'Fin (.+) (\d{4})', '01 \\1 \\2')(self)) + relativedelta(day=31) + date = parse_french_date(Regexp(Field('label'), r'Fin (.+) (\d{4})', '01 \\1 \\2')(self)) + relativedelta(day=31) if date > datetime.now() - relativedelta(day=1): card.coming = CleanDecimal(replace_dots=True).filter(m.group(3)) next_month = Link('./following-sibling::tr[contains(@class, "encours")][1]/td[1]//a', default=None)(self) if next_month: card_page = page.browser.open(next_month).page for e in card_page.doc.xpath(card_xpath): - if card.id == Regexp(CleanText('.', symbols=' '), '([\dx]{16})')(e): + if card.id == Regexp(CleanText('.', symbols=' '), r'([\dx]{16})')(e): m = re.search(pattern, CleanText('.')(e)) card._card_pages.append(card_page) card.coming += CleanDecimal(replace_dots=True).filter(m.group(3)) @@ -333,7 +340,7 @@ def parse(self, el): raise SkipItem() - self.env['id'] = id + self.env['id'] = _id if accounting is not None and accounting + (coming or Decimal('0')) != balance: self.page.logger.warning('%s + %s != %s' % (accounting, coming, balance)) @@ -345,8 +352,9 @@ def parse(self, el): self.env['coming'] = coming or NotAvailable def is_revolving(self, label): - return any(revolving_loan_label in label - for revolving_loan_label in item_account_generic.REVOLVING_LOAN_LABELS) or label.lower() in self.page.browser.revolving_accounts + return (any(revolving_loan_label in label + for revolving_loan_label in item_account_generic.REVOLVING_LOAN_LABELS) + or label.lower() in self.page.browser.revolving_accounts) class AccountsPage(LoggedPage, HTMLPage): @@ -364,8 +372,8 @@ class iter_accounts(ListElement): class item_account(item_account_generic): def condition(self): - type = Field('type')(self) - return item_account_generic.condition(self) and type != Account.TYPE_LOAN + _type = Field('type')(self) + return item_account_generic.condition(self) and _type != Account.TYPE_LOAN class item_loan(item_account_generic): klass = Loan @@ -378,7 +386,7 @@ class item_loan(item_account_generic): obj_nb_payments_left = Async('details') & Type(CleanText( '//div[@id="F4:expContent"]/table/tbody/tr[2]/td[2]/text()'), type=int, default=NotAvailable) obj_subscription_date = Async('details') & MyDate(Regexp(CleanText( - '//*[@id="F4:expContent"]/table/tbody/tr[1]/th[1]'), ' (\d{2}/\d{2}/\d{4})', default=NotAvailable)) + '//*[@id="F4:expContent"]/table/tbody/tr[1]/th[1]'), r' (\d{2}/\d{2}/\d{4})', default=NotAvailable)) obj_maturity_date = Async('details') & MyDate( CleanText('//div[@id="F4:expContent"]/table/tbody/tr[4]/td[2]')) @@ -387,11 +395,11 @@ class item_loan(item_account_generic): CleanText('//div[@id="F4:expContent"]/table/tbody/tr[3]/td[1]')) obj_last_payment_amount = Async('details') & MyDecimal('//td[@id="F2_0.T12"]') - obj_last_payment_date = Async('details') & \ - MyDate(CleanText('//div[@id="F8:expContent"]/table/tbody/tr[1]/td[1]')) + obj_last_payment_date = (Async('details') & + MyDate(CleanText('//div[@id="F8:expContent"]/table/tbody/tr[1]/td[1]'))) def condition(self): - type = Field('type')(self) + _type = Field('type')(self) label = Field('label')(self) details_link = Link('.//a', default=None)(self) @@ -400,7 +408,8 @@ def condition(self): if re.search(r'Le\sMobile\s+([0-9]{2}\s?){5}', label): return False - if details_link and item_account_generic.condition and type == Account.TYPE_LOAN and not self.is_revolving(label): + if (details_link and item_account_generic.condition and _type == Account.TYPE_LOAN + and not self.is_revolving(label)): details = self.page.browser.open(details_link) if details.page and not 'cloturé' in CleanText('//form[@id="P:F"]//div[@class="blocmsg info"]//p')(details.page.doc): return True @@ -418,9 +427,9 @@ def obj_used_amount(self): return -Field('balance')(self) def condition(self): - type = Field('type')(self) + _type = Field('type')(self) label = Field('label')(self) - return (item_account_generic.condition(self) and type == Account.TYPE_LOAN + return (item_account_generic.condition(self) and _type == Account.TYPE_LOAN and self.is_revolving(label)) def get_advisor_link(self): @@ -448,7 +457,8 @@ def get_agency(self): class get_advisor(ItemElement): klass = Advisor - obj_name = Regexp(CleanText('//script[contains(text(), "Espace Conseiller")]'), 'consname.+?([\w\s]+)') + obj_name = Regexp(CleanText('//script[contains(text(), "Espace Conseiller")]'), + r'consname.+?([\w\s]+)') @method class get_profile(ItemElement): @@ -486,7 +496,7 @@ def next_page(self): return self.next_month() text = CleanText.clean(form.el) - m = re.search('(\d+)/(\d+)', text or '', flags=re.MULTILINE) + m = re.search(r'(\d+)/(\d+)', text or '', flags=re.MULTILINE) if not m: return self.next_month() @@ -532,7 +542,7 @@ def next_page(self): form = self.page.get_form('//form[contains(@id, "frmStarcLstCtrPag")]') form['imgCtrPagSui.x'] = randint(1, 29) form['imgCtrPagSui.y'] = randint(1, 17) - m = re.search('(\d+)/(\d+)', CleanText('.')(form.el)) + m = re.search(r'(\d+)/(\d+)', CleanText('.')(form.el)) if m and int(m.group(1)) < int(m.group(2)): return form.request except FormNotFound: @@ -543,11 +553,13 @@ class item(ItemElement): load_details = Field('_link_id') & AsyncLoad - obj_number = Field('_link_id') & Regexp(pattern='ctr=(\d+)') + obj_number = Field('_link_id') & Regexp(pattern=r'ctr=(\d+)') obj__card_number = Env('id', default="") obj_id = Format('%s%s', Env('id', default=""), Field('number')) - obj_label = Format('%s %s %s', CleanText(TableCell('card')), Env('id', default=""), CleanText(TableCell('owner'))) - obj_coming = CleanDecimal('./td[@class="i d" or @class="p d"][2]', replace_dots=True, default=NotAvailable) + obj_label = Format('%s %s %s', CleanText(TableCell('card')), Env('id', default=""), + CleanText(TableCell('owner'))) + obj_coming = CleanDecimal('./td[@class="i d" or @class="p d"][2]', replace_dots=True, + default=NotAvailable) obj_balance = Decimal('0.00') obj_currency = FrenchTransaction.Currency(CleanText('./td[small][1]')) obj_type = Account.TYPE_CARD @@ -581,6 +593,9 @@ def parse(self, el): self.handle_attr(attr, getattr(self, 'obj_%s' % attr)) setattr(card, attr, getattr(self.obj, attr)) + if _id in self.page.browser.cards_histo_available: + card.coming = self.page.browser.cards_histo_available[_id][1] + card._card_number = _id card.id = _id + card.number card.label = card.label.replace(' ', ' %s ' % _id) @@ -603,18 +618,18 @@ def parse(self, el): class Transaction(FrenchTransaction): - PATTERNS = [(re.compile('^VIR(EMENT)? (?P.*)'), FrenchTransaction.TYPE_TRANSFER), - (re.compile('^(PRLV|Plt|PRELEVEMENT) (?P.*)'), FrenchTransaction.TYPE_ORDER), - (re.compile('^(?P.*) CARTE \d+ PAIEMENT CB\s+(?P
\d{2})(?P\d{2}) ?(.*)$'), + PATTERNS = [(re.compile(r'^VIR(EMENT)? (?P.*)'), FrenchTransaction.TYPE_TRANSFER), + (re.compile(r'^(PRLV|Plt|PRELEVEMENT) (?P.*)'), FrenchTransaction.TYPE_ORDER), + (re.compile(r'^(?P.*) CARTE \d+ PAIEMENT CB\s+(?P
\d{2})(?P\d{2}) ?(.*)$'), FrenchTransaction.TYPE_CARD), - (re.compile('^PAIEMENT PSC\s+(?P
\d{2})(?P\d{2}) (?P.*) CARTE \d+ ?(.*)$'), + (re.compile(r'^PAIEMENT PSC\s+(?P
\d{2})(?P\d{2}) (?P.*) CARTE \d+ ?(.*)$'), FrenchTransaction.TYPE_CARD), - (re.compile('^(?PRELEVE CARTE.*)'), FrenchTransaction.TYPE_CARD_SUMMARY), - (re.compile('^RETRAIT DAB (?P
\d{2})(?P\d{2}) (?P.*) CARTE [\*\d]+'), + (re.compile(r'^(?PRELEVE CARTE.*)'), FrenchTransaction.TYPE_CARD_SUMMARY), + (re.compile(r'^RETRAIT DAB (?P
\d{2})(?P\d{2}) (?P.*) CARTE [\*\d]+'), FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile('^CHEQUE( (?P.*))?$'), FrenchTransaction.TYPE_CHECK), - (re.compile('^(F )?COTIS\.? (?P.*)'), FrenchTransaction.TYPE_BANK), - (re.compile('^(REMISE|REM CHQ) (?P.*)'), FrenchTransaction.TYPE_DEPOSIT), + (re.compile(r'^CHEQUE( (?P.*))?$'), FrenchTransaction.TYPE_CHECK), + (re.compile(r'^(F )?COTIS\.? (?P.*)'), FrenchTransaction.TYPE_BANK), + (re.compile(r'^(REMISE|REM CHQ) (?P.*)'), FrenchTransaction.TYPE_DEPOSIT), ] _is_coming = False @@ -642,11 +657,12 @@ def __call__(self, item): # hideifscript: Date de valeur XX/XX/XXXX # fd: Avis d'opéré # survey to add other regx - parts = (re.sub('Détail|Date de valeur\s+:\s+\d{2}/\d{2}(/\d{4})?', '', txt.strip()) for txt in el.itertext() if len(txt.strip()) > 0) - # Removing empty strings: - parts = [s for s in parts if s] + parts = (re.sub(r'Détail|Date de valeur\s+:\s+\d{2}/\d{2}(/\d{4})?', '', txt.strip()) + for txt in el.itertext() if txt.strip()) + # Removing empty strings + parts = list(filter(bool, parts)) # To simplify categorization of CB, reverse order of parts to separate - # location and institution. + # location and institution detail = "Cliquer pour déplier ou plier le détail de l'opération" if detail in parts: parts.remove(detail) @@ -708,6 +724,7 @@ class item(Transaction.TransactionElement): obj__gross_amount = CleanDecimal(Env('amount'), replace_dots=True) obj_commission = CleanDecimal(Format('-%s', Env('commission')), replace_dots=True, default=NotAvailable) + obj__to_delete = False def obj_amount(self): commission = Field('commission')(self) @@ -717,14 +734,16 @@ def obj_amount(self): return (abs(gross) - abs(commission)).copy_sign(gross) def parse(self, el): - self.env['date'] = Date(Regexp(CleanText('//td[contains(text(), "Total prélevé")]'), ' (\d{2}/\d{2}/\d{4})', \ - default=NotAvailable), default=NotAvailable)(self) + self.env['date'] = Date(Regexp(CleanText('//td[contains(text(), "Total prélevé")]'), + r' (\d{2}/\d{2}/\d{4})', default=NotAvailable), + default=NotAvailable)(self) if not self.env['date']: try: - d = CleanText('//select[@id="moi"]/option[@selected]')(self) or \ - re.search('pour le mois de (.*)', ''.join(w.strip() for w in self.page.doc.xpath('//div[@class="a_blocongfond"]/text()'))).group(1) + d = (CleanText('//select[@id="moi"]/option[@selected]')(self) + or re.search(r'pour le mois de (.*)', ''.join(w.strip() for w in + self.page.doc.xpath('//div[@class="a_blocongfond"]/text()'))).group(1)) except AttributeError: - d = Regexp(CleanText('//p[has-class("restriction")]'), 'pour le mois de ((?:\w+\s+){2})', flags=re.UNICODE)(self) + d = Regexp(CleanText('//p[has-class("restriction")]'), r'pour le mois de ((?:\w+\s+){2})', flags=re.UNICODE)(self) self.env['date'] = (parse_french_date('%s %s' % ('1', d)) + relativedelta(day=31)).date() self.env['_is_coming'] = date.today() < self.env['date'] amount = CleanText(TableCell('amount'))(self).split('dont frais') @@ -738,7 +757,7 @@ class get_history(Pagination, Transaction.TransactionsElement): head_xpath = '//table[has-class("liste")]//thead//tr/th/text()' item_xpath = '//table[has-class("liste")]//tbody/tr' - col_date = u"Date de l'annonce" + col_date = "Date de l'annonce" class item(Transaction.TransactionElement): obj__is_coming = True @@ -747,7 +766,7 @@ class item(Transaction.TransactionElement): class CardPage(OperationsPage, LoggedPage): def select_card(self, card_number): for option in self.doc.xpath('//select[@name="Data_SelectedCardItemKey"]/option'): - card_id = Regexp(CleanText('.', symbols=' '), '([\dx]+)')(option) + card_id = Regexp(CleanText('.', symbols=' '), r'([\dx]+)')(option) if card_id != card_number: continue if Attr('.', 'selected', default=None)(option): @@ -766,10 +785,20 @@ class list_cards(ListElement): class item(ItemElement): def __iter__(self): + # Here we handle the subtransactions card_link = self.el.get('href') + d = re.search(r'cardmonth=(\d+)', self.page.url) + if d: + year = int(d.group(1)[:4]) + month = int(d.group(1)[4:]) + debit_date = date(year, month, 1) + relativedelta(day=31) + page = self.page.browser.location(card_link).page for op in page.get_history(): + op.date = debit_date + op.type = FrenchTransaction.TYPE_DEFERRED_CARD + op._to_delete = False yield op class list_history(Transaction.TransactionsElement): @@ -787,7 +816,7 @@ def parse(self, el): if not label: return try: - label = re.findall('(\d+ [^ ]+ \d+)', label)[-1] + label = re.findall(r'(\d+ [^ ]+ \d+)', label)[-1] except IndexError: return # use the trick of relativedelta to get the last day of month. @@ -805,15 +834,21 @@ class item(Transaction.TransactionElement): obj_original_currency = Env('original_currency') obj__differed_date = Env('differed_date') + def obj__to_delete(self): + return bool(CleanText('.//a[contains(text(), "Regroupement")]')(self)) + def parse(self, el): try: - self.env['raw'] = "%s %s" % (CleanText().filter(TableCell('commerce')(self)[0].text), CleanText().filter(TableCell('ville')(self)[0].text)) + self.env['raw'] = "%s %s" % (CleanText().filter(TableCell('commerce')(self)[0].text), + CleanText().filter(TableCell('ville')(self)[0].text)) except (ColumnNotFound, AttributeError): self.env['raw'] = "%s" % (CleanText().filter(TableCell('commerce')(self)[0].text)) - self.env['type'] = Transaction.TYPE_DEFERRED_CARD \ - if CleanText('//a[contains(text(), "Prélevé fin")]', default=None) else Transaction.TYPE_CARD - self.env['differed_date'] = parse_french_date(Regexp(CleanText('//*[contains(text(), "Achats")]'), 'au[\s]+(.*)')(self)).date() + self.env['type'] = (Transaction.TYPE_DEFERRED_CARD + if CleanText('//a[contains(text(), "Prélevé fin")]', default=None) + else Transaction.TYPE_CARD) + self.env['differed_date'] = parse_french_date(Regexp(CleanText('//*[contains(text(), "Achats")]'), + r'au[\s]+(.*)')(self)).date() amount = TableCell('credit')(self)[0] if self.page.browser.is_new_website: if not len(amount.xpath('./div')): @@ -826,10 +861,211 @@ def parse(self, el): except IndexError: original_amount = None self.env['amount'] = CleanDecimal(replace_dots=True).filter(amount.text) - self.env['original_amount'] = CleanDecimal(replace_dots=True).filter(original_amount) \ - if original_amount is not None else NotAvailable - self.env['original_currency'] = Account.get_currency(original_amount[1:-1]) \ - if original_amount is not None else NotAvailable + self.env['original_amount'] = (CleanDecimal(replace_dots=True).filter(original_amount) + if original_amount is not None else NotAvailable) + self.env['original_currency'] = (Account.get_currency(original_amount[1:-1]) + if original_amount is not None else NotAvailable) + + +class CardPage2(CardPage, HTMLPage, XMLPage): + def build_doc(self, content): + if b'= 4 + + obj_raw = Transaction.Raw(Format("%s %s", CleanText(TableCell('commerce')), CleanText(TableCell('ville')))) + obj_rdate = Field('vdate') + + def obj_type(self): + if not 'RELEVE' in CleanText('//td[contains(., "Aucun mouvement")]')(self): + return Transaction.TYPE_DEFERRED_CARD + return Transaction.TYPE_CARD_SUMMARY + + def obj_original_amount(self): + m = re.search(r'(([\s-]\d+)+,\d+)', CleanText(TableCell('commerce'))(self)) + if m and not 'FRAIS' in CleanText(TableCell('commerce'))(self): + return Decimal(m.group(1).replace(',', '.').replace(' ', '')).quantize(Decimal('0.01')) + return NotAvailable + + def obj_original_currency(self): + m = re.search(r'(\d+,\d+) (\w+)', CleanText(TableCell('commerce'))(self)) + if Field('original_amount')(self) and m: + return m.group(2) + + def obj_date(self): + debit_date = CleanText('//a[@id="C:L4"]')(self) + if "fin" in debit_date: + return self.page.browser.tr_date + m = re.search(r'(\d{2}/\d{2}/\d{4})', debit_date) + if m: + return Date().filter(re.search(r'(\d{2}/\d{2}/\d{4})', debit_date).group(1)) + + def obj__is_coming(self): + debit_date = CleanText('//a[@id="C:L4"]')(self) + if "fin" in debit_date: + return True + if Date().filter(re.search(r'(\d{2}/\d{2}/\d{4})', debit_date).group(1)) > datetime.date(datetime.today()): + return True + return False + + def obj__regroup(self): + if "Regroupement" in CleanText('./td')(self): + return Link('./td/span/a')(self) + + @method + class get_tr_merged(ListElement): + class list_history(Transaction.TransactionsElement): + head_xpath = '//table[@class="liste"]//thead/tr/th' + item_xpath = '//table[@class="liste"]/tbody/tr' + + col_operation= u'Opération' + + def condition(self): + return not CleanText('//td[contains(., "Aucun mouvement")]', default=False)(self) + + class item(Transaction.TransactionElement): + def condition(self): + return len(self.el.xpath('./td')) >= 4 + + obj_label = CleanText(TableCell('operation')) + + def obj_type(self): + if not 'RELEVE' in Field('raw')(self): + return Transaction.TYPE_DEFERRED_CARD + return Transaction.TYPE_CARD_SUMMARY + + def has_more_operations(self): + xp = CleanText(self.doc.xpath('//div[@class="ei_blocpaginb"]/a'))(self) + if xp == 'Suite des opérations': + return True + return False + + def has_more_operations_xml(self): + if self.doc.xpath('//input') and Attr('//input', 'value')(self.doc) == 'Suite des opérations': + return True + return False + + @method + class iter_history_xml(ListElement): + class list_history(Transaction.TransactionsElement): + head_xpath = '//thead/tr/th' + item_xpath = '//tbody/tr' + + col_commerce = 'Commerce' + col_ville = 'Ville' + + class item(Transaction.TransactionElement): + obj_raw = Transaction.Raw(Format("%s %s", CleanText(TableCell('commerce')), CleanText(TableCell('ville')))) + obj_rdate = Field('vdate') + + def obj_type(self): + if not 'RELEVE' in CleanText('//td[contains(., "Aucun mouvement")]')(self): + return Transaction.TYPE_DEFERRED_CARD + return Transaction.TYPE_CARD_SUMMARY + + def obj_original_amount(self): + m = re.search(r'(([\s-]\d+)+,\d+)', CleanText(TableCell('commerce'))(self)) + if m and not 'FRAIS' in CleanText(TableCell('commerce'))(self): + return Decimal(m.group(1).replace(',', '.').replace(' ', '')).quantize(Decimal('0.01')) + return NotAvailable + + def obj_original_currency(self): + m = re.search(r'(\d+,\d+) (\w+)', CleanText(TableCell('commerce'))(self)) + if Field('original_amount')(self) and m: + return m.group(2) + + def obj__regroup(self): + if "Regroupement" in CleanText('./td')(self): + return Link('./td/span/a')(self) + + def obj_date(self): + return self.page.browser.tr_date + + def obj__is_coming(self): + if Field('date')(self) > datetime.date(datetime.today()): + return True + return False + + def get_date(self): + debit_date = CleanText(self.doc.xpath('//a[@id="C:L4"]'))(self) + if "fin" in debit_date: + m = re.search(r'fid=GoMonth&mois=(\d+)', self.browser.url) + y = re.search(r'annee=(\d+)', self.browser.url) + if m and y: + return date(int(y.group(1)), int(m.group(1)), 1) + relativedelta(day=31) + + m = re.search(r'(\d{2}/\d{2}/\d{4})', debit_date) + if m: + return Date().filter(re.search(r'(\d{2}/\d{2}/\d{4})', debit_date).group(1)) + + def get_links(self): + links = [] + + for link in self.doc.xpath('//div[@class="restriction"]/ul[1]/li'): + if link.xpath('./span/span/b'): + break + tmp_link = Link(link.xpath('./span/span/a'))(self) + if 'GoMonthPrecedent' in tmp_link: + secondpage = tmp_link + continue + m = re.search(r'fid=GoMonth&mois=(\d+)', tmp_link) + # To go to the page during the iter_history you need to have the good value from the precedent page + assert m, "It's not the URL expected" + m = int(m.group(1)) + m=m+1 if m!= 12 else 1 + url = re.sub(r'(?<=amoiSelectionner%3d)\d+', str(m), tmp_link) + links.append(url) + + links.reverse() + # Just for visiting the urls in a chronological way + m = re.search(r'fid=GoMonth&mois=(\d+)', links[0]) + y = re.search(r'annee=(\d+)', links[0]) + # We need to get a stable coming month instead of "fin du mois" + if m and y: + coming_date = date(int(y.group(1)), int(m.group(1)), 1) + relativedelta(months=+1) + + add_first = re.sub(r'(?<=amoiSelectionner%3d)\d+', str(coming_date.month), links[0]) + add_first = re.sub(r'(?<=GoMonth&mois=)\d+', str(coming_date.month), add_first) + add_first = re.sub(r'(?<=\&annee=)\d+', str(coming_date.year), add_first) + links.insert(0, add_first) + m = re.search(r'fid=GoMonth&mois=(\d+)', links[-1]).group(1) + links.append(re.sub(r'(?<=amoiSelectionner%3d)\d+', str(m), secondpage)) + + links2 = [] + page2 = self.browser.open(secondpage).page + for link in page2.doc.xpath('//div[@class="restriction"]/ul[1]/li'): + if link.xpath('./span/span/a'): + tmp_link = Link(link.xpath('./span/span/a'))(self) + if 'GoMonthSuivant' in tmp_link: + break + m = re.search(r'fid=GoMonth&mois=(\d+)', tmp_link) + assert m, "It's not the URL expected" + m = int(m.group(1)) + m=m+1 if m!= 12 else 1 + url = re.sub(r'(?<=amoiSelectionner%3d)\d+', str(m), tmp_link) + links2.append(url) + + links2.reverse() + links.extend(links2) + return links class LIAccountsPage(LoggedPage, HTMLPage): @@ -889,8 +1125,10 @@ class item(ItemElement): klass = Investment obj_label = CleanText(TableCell('label')) - obj_unitprice = CleanDecimal(TableCell('unitprice', default=NotAvailable), default=NotAvailable, replace_dots=True) - obj_vdate = Date(CleanText(TableCell('vdate'), replace=[('-', '')]), default=NotAvailable, dayfirst=True) + obj_unitprice = CleanDecimal(TableCell('unitprice', default=NotAvailable), + default=NotAvailable, replace_dots=True) + obj_vdate = Date(CleanText(TableCell('vdate'), replace=[('-', '')]), + default=NotAvailable, dayfirst=True) obj_unitvalue = CleanDecimal(TableCell('unitvalue'), default=NotAvailable, replace_dots=True) obj_quantity = CleanDecimal(TableCell('quantity'), default=NotAvailable, replace_dots=True) obj_valuation = CleanDecimal(TableCell('valuation'), default=Decimal(0), replace_dots=True) @@ -899,8 +1137,7 @@ def obj_code(self): link = Link(TableCell('label')(self)[0].xpath('./a'), default=NotAvailable)(self) if not link: return NotAvailable - return Regexp(pattern='isin=([A-Z\d]+)&?', default=NotAvailable).filter(link) - + return Regexp(pattern=r'isin=([A-Z\d]+)&?', default=NotAvailable).filter(link) class PorPage(LoggedPage, HTMLPage): @@ -933,7 +1170,7 @@ def add_por_accounts(self, accounts): if acc.id == '9999': # fake account continue - acc.label = unicode(re.sub("\d", '', ele.text).strip()) + acc.label = unicode(re.sub(r'\d', '', ele.text).strip()) acc._link_id = None acc.type = self.get_type(acc.label) acc._is_inv = True @@ -943,9 +1180,11 @@ def add_por_accounts(self, accounts): def fill(self, acc): self.send_form(acc) ele = self.browser.page.doc.xpath('.//table[has-class("fiche bourse")]')[0] - balance = CleanDecimal(ele.xpath('.//td[contains(@id, "Valorisation")]'), default=Decimal(0), replace_dots=True)(ele) + balance = CleanDecimal(ele.xpath('.//td[contains(@id, "Valorisation")]'), + default=Decimal(0), replace_dots=True)(ele) acc.balance = balance + acc.balance if acc.balance else balance - acc.valuation_diff = CleanDecimal(ele.xpath('.//td[contains(@id, "Variation")]'), default=Decimal(0), replace_dots=True)(ele) + acc.valuation_diff = CleanDecimal(ele.xpath('.//td[contains(@id, "Variation")]'), + default=Decimal(0), replace_dots=True)(ele) if balance: acc.currency = Currency('.//td[contains(@id, "Valorisation")]')(ele) else: @@ -957,13 +1196,13 @@ def fill(self, acc): # # Solution: remove the date text_content = CleanText('.')(ele) - date_pattern = r"\d{2}/\d{2}/\d{4}" + date_pattern = r'\d{2}/\d{2}/\d{4}' no_date = re.sub(date_pattern, '', text_content) acc.currency = Currency().filter(no_date) def send_form(self, account): form = self.get_form(name="frmMere") - form['POR_SyntheseEntete1$esdselLstPor'] = re.sub('\D', '', account.id) + form['POR_SyntheseEntete1$esdselLstPor'] = re.sub(r'\D', '', account.id) form.submit() @method @@ -982,7 +1221,7 @@ class item(ItemElement): klass = Investment obj_label = CleanText(TableCell('label'), default=NotAvailable) - obj_code = CleanText('.//td[1]/a/@title') & Regexp(pattern='^([^ ]+)') + obj_code = CleanText('.//td[1]/a/@title') & Regexp(pattern=r'^([^ ]+)') obj_quantity = CleanDecimal(TableCell('quantity'), default=Decimal(0), replace_dots=True) obj_unitprice = CleanDecimal(TableCell('unitprice'), default=Decimal(0), replace_dots=True) obj_valuation = CleanDecimal(TableCell('valuation'), default=Decimal(0), replace_dots=True) @@ -997,7 +1236,9 @@ def obj_unitvalue(self): def obj_vdate(self): td = TableCell('unitvalue')(self)[0] - return Date(Regexp(Attr('./img', 'title', default=''), r'Cours au : (\d{2}/\d{2}/\d{4})\b', default=None), dayfirst=True, default=NotAvailable)(td) + return Date(Regexp(Attr('./img', 'title', default=''), + r'Cours au : (\d{2}/\d{2}/\d{4})\b', default=None), + dayfirst=True, default=NotAvailable)(td) class IbanPage(LoggedPage, HTMLPage): @@ -1073,7 +1314,8 @@ class item(MyRecipient): obj_category = 'Interne' def obj_iban(self): - l = [a for a in self.page.browser.get_accounts_list() if Field('id')(self) in a.id and empty(a.valuation_diff)] + l = [a for a in self.page.browser.get_accounts_list() + if Field('id')(self) in a.id and empty(a.valuation_diff)] assert len(l) == 1 return l[0].iban @@ -1094,8 +1336,12 @@ def get_from_account_index(self, account): def get_to_account_index(self, account): return self.get_account_index(self.RECIPIENT_STRING, account) + def get_transfer_form(self): + # internal and external transfer form are differents + return self.get_form(id='P:F', submit='//input[@type="submit" and contains(@value, "Valider")]') + def prepare_transfer(self, account, to, amount, reason): - form = self.get_form(id='P:F', submit='//input[@type="submit" and contains(@value, "Valider")]') + form = self.get_transfer_form() form['data_input_indiceCompteADebiter'] = self.get_from_account_index(account.id) form[self.RECIPIENT_STRING] = self.get_to_account_index(to.id) form['[t:dbt%3adouble;]data_input_montant_value_0_'] = str(amount).replace('.', ',') @@ -1115,7 +1361,8 @@ def check_errors(self): 'Nom prénom du bénéficiaire différent du titulaire. Utilisez un compte courant', "Pour effectuer cette opération, vous devez passer par l’intermédiaire d’un compte courant", 'Montant maximum autorisé au crédit pour ce compte', - 'Débit interdit sur ce compte'] + 'Débit interdit sur ce compte', + 'Virement interdit sur compte clos',] for message in messages: if message in content: @@ -1130,16 +1377,20 @@ def check_success(self): raise TransferError('The expected message "%s" was not found.' % self.READY_FOR_TRANSFER_MSG) def check_data_consistency(self, account_id, recipient_id, amount, reason): - assert account_id in CleanText('//div[div[p[contains(text(), "Compte à débiter")]]]', replace=[(' ', '')])(self.doc) - assert recipient_id in CleanText('//div[div[p[contains(text(), "%s")]]]' % self.SUMMARY_RECIPIENT_TITLE, replace=[(' ', '')])(self.doc) + assert account_id in CleanText('//div[div[p[contains(text(), "Compte à débiter")]]]', + replace=[(' ', '')])(self.doc) + assert recipient_id in CleanText('//div[div[p[contains(text(), "%s")]]]' % self.SUMMARY_RECIPIENT_TITLE, + replace=[(' ', '')])(self.doc) - exec_date = Date(Regexp(CleanText('//table[@summary]/tbody/tr[th[contains(text(), "Date")]]/td'), '(\d{2}/\d{2}/\d{4})'), dayfirst=True)(self.doc) + exec_date = Date(Regexp(CleanText('//table[@summary]/tbody/tr[th[contains(text(), "Date")]]/td'), + r'(\d{2}/\d{2}/\d{4})'), dayfirst=True)(self.doc) assert exec_date == datetime.today().date() - r_amount = CleanDecimal('//table[@summary]/tbody/tr[th[contains(text(), "Montant")]]/td', replace_dots=True)(self.doc) + r_amount = CleanDecimal('//table[@summary]/tbody/tr[th[contains(text(), "Montant")]]/td', + replace_dots=True)(self.doc) assert r_amount == Decimal(amount) currency = FrenchTransaction.Currency('//table[@summary]/tbody/tr[th[contains(text(), "Montant")]]/td')(self.doc) if reason is not None: - assert reason.upper().strip()[:22] in CleanText('//table[@summary]/tbody/tr[th[contains(text(), "Intitulé pour le compte à débiter")]]/td')(self.doc) + assert reason.upper()[:22].strip() in CleanText('//table[@summary]/tbody/tr[th[contains(text(), "Intitulé pour le compte à débiter")]]/td')(self.doc) return exec_date, r_amount, currency def handle_response(self, account, recipient, amount, reason): @@ -1221,7 +1472,7 @@ def iter_categories(self): yield {'name': CleanText('.')(option), 'index': option.attrib['value']} def go_on_category(self, category_index): - form = self.get_form(id='P:F', submit='//input[@type="submit" and @value="Nom"]') + form = self.get_form(id='P2:F', submit='//input[@type="submit" and @value="Nom"]') form['data_input_indiceMarqueurListe'] = category_index form.submit() @@ -1254,6 +1505,11 @@ def obj_iban(self): def parse(self, el): self.env['origin_account']._external_recipients.add(Field('id')(self)) + def get_transfer_form(self): + # internal and external transfer form are differents + if self.IS_PRO_PAGE: + return self.get_form(id='P2:F', submit='//input[@type="submit" and contains(@value, "Valider")]') + return self.get_form(id='P1:F', submit='//input[@type="submit" and contains(@value, "Valider")]') class VerifCodePage(LoggedPage, HTMLPage): HASHES = { @@ -1324,9 +1580,13 @@ class VerifCodePage(LoggedPage, HTMLPage): } def on_load(self): - error = CleanText('//p[contains(text(), "Clé invalide !")] | //p[contains(text(), "Vous n\'avez pas saisi de clé!")]')(self.doc) - if error: - raise AddRecipientError(message=error) + errors = ( + CleanText('//p[contains(text(), "Clé invalide !")] | //p[contains(text(), "Vous n\'avez pas saisi de clé!")]')(self.doc), + CleanText('//p[contains(text(), "Vous n\'êtes pas inscrit") and a[text()="service d\'identification renforcée"]]')(self.doc), + ) + for error in errors: + if error: + raise AddRecipientBankError(message=error) action_needed = CleanText('//p[contains(text(), "Carte de CLÉS PERSONNELLES révoquée")]')(self.doc) if action_needed: @@ -1362,10 +1622,12 @@ def on_load(self): if error and error != 'Veuillez renseigner le BIC ou les coordonnées de la banque': # don't reload state if it fails because it's not supported by the website self.browser.need_clear_storage = True - raise AddRecipientError(message=error) + raise AddRecipientBankError(message=error) - app_validation = self.doc.xpath('//strong[contains(text(), "Démarrez votre application mobile Crédit Mutuel")]') + app_validation = self.doc.xpath('//strong[contains(text(), "Démarrez votre application mobile")]') if app_validation: + # don't reload state if it fails because it's not supported by the website + self.browser.need_clear_storage = True raise AuthMethodNotImplemented("La confirmation par validation sur votre application mobile n'est pas supportée") def has_list(self): @@ -1392,7 +1654,7 @@ def get_add_recipient_form(self, recipient): # Needed because it requires that \xe9 is encoded %E9 instead of %C3%A9 try: - del form[u'data_pilotageAffichage_habilitéSaisieInternationale'] + del form['data_pilotageAffichage_habilitéSaisieInternationale'] except KeyError: pass else: @@ -1425,9 +1687,10 @@ def ask_sms(self, recipient): form = self.get_form(id='P:F') self.set_browser_form(form) raise AddRecipientStep(recipient, Value('code', label=txt)) + # don't reload state if it fails because it's not supported by the website self.browser.need_clear_storage = True - raise AddRecipientError('Was expecting a page where sms code is asked') + assert False, 'Was expecting a page where sms code is asked' class RevolvingLoansList(LoggedPage, HTMLPage): @method @@ -1519,7 +1782,8 @@ def condition(self): # Some documents may have the same date, name and label; only parts of the PDF href may change, # so we must pick a unique ID including the href to avoid document duplicates: - obj_id = Format('%s_%s_%s', Env('sub_id'), CleanText(TableCell('date'), replace=[('/', '')]), Regexp(Field('url'), r'NOM=(.*)&RFL=')) + obj_id = Format('%s_%s_%s', Env('sub_id'), CleanText(TableCell('date'), replace=[('/', '')]), + Regexp(Field('url'), r'NOM=(.*)&RFL=')) obj_label = Format('%s %s', CleanText(TableCell('url')), CleanText(TableCell('date'))) obj_date = Date(CleanText(TableCell('date')), dayfirst=True) obj_format = 'pdf' @@ -1544,3 +1808,34 @@ def is_last_page(self): if re.search(r'(\d\/\d)', CleanText('//div[has-class("blocpaginb")]', symbols=' ')(self.doc)): return True return False + + +class CardsHistAvailable(LoggedPage, HTMLPage): + def get_unavailable_cards(self): + cards = [] + for card in self.doc.xpath('//li[@class="item"]'): + if CleanText(card.xpath('.//div[1]/p'))(self) != 'Active': + m = re.search(r'\d{4} \d{2}XX XXXX \d{4}', CleanText(card.xpath('.//span'))(self)) + if m: + cards.append(m.group(0).replace(' ', '').replace('X', 'x')) + return cards + + def get_cards_list(self): + cards = {} + for card in self.doc.xpath('//li[@class="item"]'): + if not card.xpath('.//tbody/tr/td/span'): + # No coming/balance values and history link are available + continue + m = re.search(r'\d{4} \d{2}XX XXXX \d{4}', CleanText(card.xpath('.//span'))(self)) + assert m, 'Id card is not present' + id_card = m.group(0).replace(' ', '').replace('X', 'x') + + link = Link(card.xpath('.//a[contains(@id,"C:more-card")]'))(self) + coming_xpath = card.xpath('.//tbody/tr/td/span') + coming = CleanDecimal(coming_xpath[0], replace_dots=True)(self) + if len(coming_xpath) > 1: + coming += CleanDecimal(coming_xpath[1], replace_dots=True)(self) + parent_id = re.search(r'\d+', CleanText(card.xpath('./div/div/div/p'), replace=[(' ', '')])(self)).group(0)[-16:] or None + cards[id_card] = [link, coming, parent_id] + + return cards diff --git a/modules/edf/par/browser.py b/modules/edf/par/browser.py index bd685cef80..b26ecc0e2b 100644 --- a/modules/edf/par/browser.py +++ b/modules/edf/par/browser.py @@ -40,11 +40,13 @@ class EdfBrowser(LoginBrowser): not_connected = URL('/fr/accueil/connexion/mon-espace-client.html', UnLoggedPage) connected = URL('/fr/accueil/espace-client/tableau-de-bord.html', WelcomePage) profil = URL('/services/rest/authenticate/getListContracts', ProfilPage) - csrf_token = URL('/services/rest/init/initPage\?_=(?P.*)', ProfilPage) + csrf_token = URL(r'/services/rest/init/initPage\?_=(?P.*)', ProfilPage) documents = URL('/services/rest/edoc/getMyDocuments', DocumentsPage) bills = URL('/services/rest/edoc/getBillsDocuments', DocumentsPage) bill_informations = URL('/services/rest/document/dataUserDocumentGetX', DocumentsPage) - bill_download = URL('/services/rest/document/getDocumentGetXByData\?csrfToken=(?P.*)&dn=(?P.*)&pn=(?P.*)&di=(?P.*)&bn=(?P.*)&an=(?P.*)') + bill_download = URL(r'/services/rest/document/getDocumentGetXByData' + r'\?csrfToken=(?P.*)&dn=(?P.*)&pn=(?P.*)' + r'&di=(?P.*)&bn=(?P.*)&an=(?P.*)') profile = URL('/services/rest/context/getCustomerContext', ProfilePage) def __init__(self, config, *args, **kwargs): @@ -113,7 +115,10 @@ def iter_documents(self, subscription): def download_document(self, document): token = self.get_csrf_token() - bills_informations = self.bill_informations.go(headers={'Content-Type': 'application/json;charset=UTF-8', 'Accept': 'application/json, text/plain, */*'}, data=json.dumps({ + bills_informations = self.bill_informations.go(headers={ + 'Content-Type': 'application/json;charset=UTF-8', + 'Accept': 'application/json, text/plain, */*'}, + data=json.dumps({ 'bpNumber': document._bp, 'csrfToken': token, 'docId': document._doc_number, @@ -122,9 +127,8 @@ def download_document(self, document): 'parNumber': document._par_number })).get_bills_informations() - return self.bill_download.go(csrf_token=token, - dn='FACTURE', pn=document._par_number, \ - di=document._doc_number, bn=bills_informations.get('bpNumber'), \ + return self.bill_download.go(csrf_token=token, dn='FACTURE', pn=document._par_number, + di=document._doc_number, bn=bills_informations.get('bpNumber'), an=bills_informations.get('numAcc')).content @need_login diff --git a/modules/edf/par/pages.py b/modules/edf/par/pages.py index 8fd94a972f..46283c4f20 100644 --- a/modules/edf/par/pages.py +++ b/modules/edf/par/pages.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals from datetime import datetime from decimal import Decimal @@ -67,7 +68,8 @@ class iter_subscriptions(DictElement): class item(ItemElement): klass = Subscription - obj_subscriber = Format('%s %s', Dict('bp/identity/firstName'), Dict('bp/identity/lastName')) + obj_subscriber = Format('%s %s', Dict('bp/identity/firstName'), + Dict('bp/identity/lastName')) obj_id = Dict('number') obj_label = obj_id @@ -90,12 +92,13 @@ class item(ItemElement): klass = Bill obj_id = Format('%s_%s', Env('subid'), Dict('documentNumber')) - obj_date = Date(Eval(lambda t: datetime.fromtimestamp(int(t)/1000).strftime('%Y-%m-%d'), Dict('creationDate'))) - obj_format = u"pdf" + obj_date = Date(Eval(lambda t: datetime.fromtimestamp(int(t) / 1000) \ + .strftime('%Y-%m-%d'), Dict('creationDate'))) + obj_format = 'pdf' obj_label = Format('Facture %s', Dict('documentNumber')) - obj_type = u"bill" + obj_type = 'bill' obj_price = Env('price') - obj_currency = u'EUR' + obj_currency = 'EUR' obj_vat = NotAvailable obj__doc_number = Dict('documentNumber') obj__par_number = Dict('parNumber') @@ -121,8 +124,9 @@ def get_profile(self): data = self.doc['bp'] p = Profile() - p.address = '%s %s %s %s' % (data['streetNumber'], data['streetName'], data['postCode'], data['city']) - p.name = '%s %s' % (data['lastName'], data['firstName']) + p.address = '%s %s %s %s' % (data['streetNumber'], data['streetName'], + data['postCode'], data['city']) + p.name = '%s %s %s' % (data['civility'], data['lastName'], data['firstName']) p.phone = data['mobilePhoneNumber'] or data['fixPhoneNumber'] p.email = data['mail'] diff --git a/modules/edf/pro/browser.py b/modules/edf/pro/browser.py index 4c77fc9859..3b44507a39 100644 --- a/modules/edf/pro/browser.py +++ b/modules/edf/pro/browser.py @@ -25,11 +25,11 @@ from weboob.browser import LoginBrowser, URL, need_login from weboob.capabilities.base import NotAvailable -from weboob.exceptions import BrowserIncorrectPassword, ActionNeeded +from weboob.exceptions import BrowserIncorrectPassword, ActionNeeded, BrowserUnavailable from weboob.browser.exceptions import ServerError, ClientError from .pages import ( - LoginPage, HomePage, AuthPage, LireSitePage, + LoginPage, HomePage, AuthPage, ErrorPage, LireSitePage, SubscriptionsPage, BillsPage, DocumentsPage, ProfilePage, ) @@ -40,6 +40,7 @@ class EdfproBrowser(LoginBrowser): login = URL('/openam/json/authenticate', LoginPage) auth = URL('/openam/UI/Login.*', '/ice/rest/aiguillagemp/redirect', AuthPage) + error = URL(r'/page_erreur/', ErrorPage) home = URL('/ice/content/ice-pmse/homepage.html', HomePage) liresite = URL(r'/rest/homepagemp/liresite', LireSitePage) contracts = URL('/rest/contratmp/consultercontrats', SubscriptionsPage) @@ -56,7 +57,8 @@ def __init__(self, config, *args, **kwargs): super(EdfproBrowser, self).__init__(*args, **kwargs) def do_login(self): - login_data = self.login.go('/openam/json/authenticate', method='POST').get_json_data(self.username, self.password) + self.login.go('/openam/json/authenticate', method='POST') + login_data = self.page.get_data(self.username, self.password) try: self.login.go(data=json.dumps(login_data), headers={'Content-Type': 'application/json'}) except ClientError as e: @@ -66,20 +68,24 @@ def do_login(self): self.location(self.absurl('/rest/aiguillagemp/redirect'), allow_redirects=True) if self.auth.is_here() and self.page.response.status_code != 303: - raise BrowserIncorrectPassword + raise BrowserIncorrectPassword() + + if self.error.is_here(): + raise BrowserUnavailable(self.page.get_message()) self.session.headers['Content-Type'] = 'application/json;charset=UTF-8' self.session.headers['X-XSRF-TOKEN'] = self.session.cookies['XSRF-TOKEN'] @need_login def get_subscription_list(self): - self.liresite.go(data=json.dumps({"numPremierSitePage":0,"pageSize":100000,"idTdg":None,"critereFiltre":[],"critereTri":[]})) + self.liresite.go(data=json.dumps({"numPremierSitePage": 0, "pageSize": 100000, "idTdg": None, + "critereFiltre": [], "critereTri": []})) id_site_list = self.page.get_id_site_list() if not id_site_list: raise ActionNeeded("Vous ne disposez d'aucun contrat actif relatif à vos sites") if "subs" not in self.cache.keys(): - self.contracts.go(data=json.dumps({'refDevisOMList':[], 'refDevisOHList': id_site_list})) + self.contracts.go(data=json.dumps({'refDevisOMList': [], 'refDevisOHList': id_site_list})) self.cache['subs'] = [s for s in self.page.get_subscriptions()] return self.cache['subs'] @@ -104,15 +110,17 @@ def iter_documents(self, subscription): def download_document(self, document): if document.url is not NotAvailable: try: - self.bills.go(data=json.dumps({'date': int(document.date.strftime('%s')), \ - 'iDFelix': document._account_billing, 'numFacture': document._bill_number})) + self.bills.go(data=json.dumps({'date': int(document.date.strftime('%s')), + 'iDFelix': document._account_billing, + 'numFacture': document._bill_number})) - return self.open('%s/rest/facturemp/telechargerfichier?fname=%s' % (self.BASEURL, self.page.get_bill_name())).content + return self.open('%s/rest/facturemp/telechargerfichier?fname=%s' % ( + self.BASEURL, self.page.get_bill_name())).content except ServerError: return NotAvailable @need_login def get_profile(self): - self.profile.go(json={'idSpcInterlocuteur':''}) + self.profile.go(json={'idSpcInterlocuteur': ''}) return self.page.get_profile() diff --git a/modules/edf/pro/pages.py b/modules/edf/pro/pages.py index 44360e2e6c..029c78e35d 100644 --- a/modules/edf/pro/pages.py +++ b/modules/edf/pro/pages.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals from datetime import date @@ -31,7 +32,7 @@ class LoginPage(JsonPage): - def get_json_data(self, login, password): + def get_data(self, login, password): login_data = self.doc login_data['callbacks'][0]['input'][0]['value'] = login login_data['callbacks'][1]['input'][0]['value'] = password @@ -42,22 +43,29 @@ class AuthPage(RawPage): pass +class ErrorPage(HTMLPage): + def get_message(self): + return CleanText('//div[@id="div_text"]/h1 | //div[@id="div_text"]/p')(self.doc) + + class HomePage(LoggedPage, HTMLPage): pass -class LireSitePage(LoggedPage, JsonPage): +class JsonCguPage(JsonPage): + def build_doc(self, text): + if text == 'REDIRECT_CGU': # JSON can always be decoded in UTF-8 so testing text is fine + raise ActionNeeded("Vous devez accepter les conditions générales d'utilisation.") + return super(JsonCguPage, self).build_doc(text) + + +class LireSitePage(LoggedPage, JsonCguPage): # id site is not about website but geographical site def get_id_site_list(self): return [site['idSite'] for site in self.doc['site']] -class SubscriptionsPage(LoggedPage, JsonPage): - def build_doc(self, text): - if self.content == 'REDIRECT_CGU': - raise ActionNeeded(u"Vous devez accepter les conditions générales d'utilisation sur le site de votre banque.") - return super(SubscriptionsPage, self).build_doc(text) - +class SubscriptionsPage(LoggedPage, JsonCguPage): @method class get_subscriptions(DictElement): item_xpath = 'listeContrat' @@ -70,7 +78,8 @@ class item(ItemElement): obj_label = CleanText(CleanHTML(Dict('nomOffreModele'))) def obj_subscriber(self): - return ('%s %s' % (Dict('prenomIntPrinc')(self).lower(), Dict('nomIntPrinc')(self).lower())).title() + return ('%s %s' % (Dict('prenomIntPrinc')(self).lower(), + Dict('nomIntPrinc')(self).lower())).title() class BillsPage(LoggedPage, JsonPage): @@ -87,11 +96,11 @@ def get_documents(self): doc.id = document['numFactureLabel'] doc.date = date.fromtimestamp(int(document['dateEmission'] / 1000)) - doc.format = u'PDF' + doc.format = 'PDF' doc.label = 'Facture %s' % document['numFactureLabel'] - doc.type = u'bill' + doc.type = 'bill' doc.price = CleanDecimal().filter(document['montantTTC']) - doc.currency = u'€' + doc.currency = '€' doc._account_billing = document['compteFacturation'] doc._bill_number = document['numFacture'] diff --git a/modules/fortuneo/browser.py b/modules/fortuneo/browser.py index d39ffbbe04..07410acd7a 100644 --- a/modules/fortuneo/browser.py +++ b/modules/fortuneo/browser.py @@ -25,14 +25,14 @@ from datetime import datetime, timedelta from weboob.browser.browsers import LoginBrowser, URL, need_login, StatesMixin -from .compat.weboob_exceptions import AuthMethodNotImplemented, BrowserIncorrectPassword +from .compat.weboob_exceptions import AuthMethodNotImplemented, BrowserIncorrectPassword, ActionNeeded from weboob.capabilities.bank import Account, AddRecipientStep, Recipient from weboob.tools.capabilities.bank.transactions import sorted_transactions from weboob.tools.value import Value from .pages.login import LoginPage, UnavailablePage from .pages.accounts_list import ( - AccountsList, AccountHistoryPage, CardHistoryPage, InvestmentHistoryPage, PeaHistoryPage, LoanPage, + AccountsList, AccountHistoryPage, CardHistoryPage, InvestmentHistoryPage, PeaHistoryPage, LoanPage, ProfilePage, ProfilePageCSV, SecurityPage, ) from .pages.transfer import ( RegisterTransferPage, ValidateTransferPage, ConfirmTransferPage, RecipientsPage, RecipientSMSPage @@ -62,6 +62,7 @@ class Fortuneo(LoginBrowser, StatesMixin): invest_history = URL(r'.*/prive/mes-comptes/assurance-vie/.*', InvestmentHistoryPage) loan_contract = URL(r'/fr/prive/mes-comptes/credit-immo/contrat-credit-immo/contrat-pret-immobilier.jsp.*', LoanPage) unavailable = URL(r'/customError/indispo.html', UnavailablePage) + security_page = URL(r'/fr/prive/identification-carte-securite-forte.jsp.*', SecurityPage) # transfer recipients = URL( @@ -84,6 +85,9 @@ class Fortuneo(LoginBrowser, StatesMixin): r'/fr/prive/mes-comptes/compte-courant/.*/confirmer-saisie-virement.jsp', ConfirmTransferPage) + profile = URL(r'/fr/prive/informations-client.jsp', ProfilePage) + profile_csv = URL(r'/PdfStruts\?*', ProfilePageCSV) + need_reload_state = None __states__ = ['need_reload_state', 'add_recipient_form'] @@ -149,6 +153,9 @@ def get_coming(self, account): def get_accounts_list(self): self.accounts_page.go() + # Note: if you want to debug process_action_needed() here, + # you must first set self.action_needed_processed to False + # otherwise it might not enter the "if" loop here below. if not self.action_needed_processed: self.process_action_needed() @@ -161,6 +168,13 @@ def process_action_needed(self): if url: self.location(self.absurl(url, base=True)) # beware, the landing page might vary according to the referer page. So far I didn't figure out how the landing page is chosen. + if self.security_page.is_here(): + # Some connections require reinforced security and we cannot bypass the OTP in order + # to get to the account information. Users have to provide a phone number in order to + # validate an OTP, so we must raise an ActionNeeded with the appropriate message. + raise ActionNeeded('Cette opération sensible doit être validée par un code sécurité envoyé par SMS ou serveur vocal. ' + 'Veuillez contacter le Service Clients pour renseigner vos coordonnées téléphoniques.') + # if there are skippable CGUs, skip them if self.accounts_page.is_here() and self.page.has_action_needed(): # Look for the request in the event listener registered to the button @@ -255,3 +269,11 @@ def execute_transfer(self, transfer): self.page.validate_transfer() self.page.confirm_transfer() return self.page.transfer_confirmation(transfer) + + @need_login + def get_profile(self): + self.profile.go() + csv_link = self.page.get_csv_link() + if csv_link: + self.location(csv_link) + return self.page.get_profile() diff --git a/modules/fortuneo/compat/weboob_capabilities_bank.py b/modules/fortuneo/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/fortuneo/compat/weboob_capabilities_bank.py +++ b/modules/fortuneo/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/fortuneo/module.py b/modules/fortuneo/module.py index 9aab657e64..fcba423389 100644 --- a/modules/fortuneo/module.py +++ b/modules/fortuneo/module.py @@ -23,6 +23,7 @@ CapBankWealth, CapBankTransferAddRecipient, AccountNotFound, RecipientNotFound, TransferInvalidLabel, Account, ) +from weboob.capabilities.profile import CapProfile from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword @@ -32,7 +33,7 @@ __all__ = ['FortuneoModule'] -class FortuneoModule(Module, CapBankWealth, CapBankTransferAddRecipient): +class FortuneoModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapProfile): NAME = 'fortuneo' MAINTAINER = u'Gilles-Alexandre Quenot' EMAIL = 'gilles.quenot@gmail.com' @@ -67,6 +68,9 @@ def iter_coming(self, account): def iter_investment(self, account): return self.browser.get_investments(account) + def get_profile(self): + return self.browser.get_profile() + def iter_transfer_recipients(self, origin_account): if isinstance(origin_account, Account): origin_account = origin_account.id diff --git a/modules/fortuneo/pages/accounts_list.py b/modules/fortuneo/pages/accounts_list.py index 9c0bed7d54..0514264462 100644 --- a/modules/fortuneo/pages/accounts_list.py +++ b/modules/fortuneo/pages/accounts_list.py @@ -20,19 +20,24 @@ from __future__ import unicode_literals import re +import sys from time import sleep from datetime import date from dateutil.relativedelta import relativedelta from lxml.html import etree +from weboob.browser.elements import method, ItemElement from weboob.browser.filters.html import Link, Attr from weboob.browser.filters.standard import CleanText, CleanDecimal, RawText, Regexp, Date from weboob.capabilities import NotAvailable from weboob.capabilities.bank import Account, Investment, Loan -from weboob.browser.pages import HTMLPage, LoggedPage, FormNotFound +from weboob.capabilities.profile import Person +from weboob.browser.pages import HTMLPage, LoggedPage, FormNotFound, CsvPage from weboob.tools.capabilities.bank.transactions import FrenchTransaction +from .compat.weboob_tools_capabilities_bank_investments import create_french_liquidity from weboob.tools.json import json +from weboob.tools.date import parse_french_date from weboob.exceptions import ActionNeeded, BrowserUnavailable from weboob.tools.capabilities.bank.investments import is_isin_valid @@ -96,12 +101,9 @@ def get_investments(self, account): inv.code_type = Investment.CODE_TYPE_ISIN yield inv - if not account.type == account.TYPE_MARKET: - inv = Investment() - inv.code = "XX-liquidity" - inv.label = "Liquidités" - inv.valuation = CleanDecimal(None, True).filter(self.doc.xpath('//*[@id="valorisation_compte"]/table/tr[3]/td[2]')) - yield inv + if account.type != account.TYPE_MARKET: + valuation = CleanDecimal(None, True).filter(self.doc.xpath('//*[@id="valorisation_compte"]/table/tr[3]/td[2]')) + yield create_french_liquidity(valuation) def parse_decimal(self, string, replace_dots): string = CleanText(None).filter(string) @@ -519,3 +521,47 @@ def get_subscription_date(self): def get_maturity_date(self): return Date(CleanText(u'//p[@id="c_dateFin"]//strong'), dayfirst=True)(self.doc) + + +class ProfilePage(LoggedPage, HTMLPage): + def get_csv_link(self): + return Link('//div[@id="bloc_telecharger"]//a[@id="telecharger_donnees"]', default=NotAvailable)(self.doc) + + @method + class get_profile(ItemElement): + klass = Person + + obj_phone = Regexp(CleanText('//div[@id="consultationform_telephones"]/p[@id="c_numeroPortable"]'), '([\d\*]+)', default=None) + obj_email = CleanText('//div[@id="modification_email"]//p[@id="c_email_actuel"]/span') + obj_address = CleanText('//div[@id="consultationform_adresse_domicile"]/div[@class="container"]//span') + obj_job = CleanText('//div[@id="consultationform_informations_complementaires"]/p[@id="c_profession"]/span') + obj_job_activity_area = CleanText('//div[@id="consultationform_informations_complementaires"]/p[@id="c_secteurActivite"]/span') + obj_company_name = CleanText('//div[@id="consultationform_informations_complementaires"]/p[@id="c_employeur"]/span') + + +class ProfilePageCSV(LoggedPage, CsvPage): + ENCODING = 'latin_1' + if sys.version_info.major > 2: + FMTPARAMS = {'delimiter': ';'} + else: + FMTPARAMS = {'delimiter': b';'} + + def get_profile(self): + d = {el[0]: el[1] for el in self.doc} + profile = Person() + profile.name = '%s %s' % (d['Nom'], d['Prénom']) + profile.birth_date = parse_french_date(d['Date de naissance']).date() + profile.address = '%s %s %s' % (d['Adresse de correspondance'], d['Code postal résidence fiscale'], d['Ville adresse de correspondance']) + profile.country = d['Pays adresse de correspondance'] + profile.email = d['Adresse e-mail'] + profile.phone = d.get('Téléphone portable') + profile.job_activity_area = d.get('Secteur d\'activité') + profile.job = d.get('Situation professionnelle') + profile.company_name = d.get('Employeur') + profile.family_situation = d.get('Situation familiale') + return profile + + +class SecurityPage(LoggedPage, HTMLPage): + pass + diff --git a/modules/fortuneo/pages/compat/__init__.py b/modules/fortuneo/pages/compat/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/modules/fortuneo/pages/compat/weboob_capabilities_bank.py b/modules/fortuneo/pages/compat/weboob_capabilities_bank.py new file mode 100644 index 0000000000..141097d781 --- /dev/null +++ b/modules/fortuneo/pages/compat/weboob_capabilities_bank.py @@ -0,0 +1,45 @@ + +import weboob.capabilities.bank as OLD + +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) + + +__all__ = OLD.__all__ + + +class CapBankWealth(CapBank): + pass + + +class CapBankPockets(CapBank): + pass + + +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + +class CapBankTransfer(OLD.CapBankTransfer): + def transfer_check_label(self, old, new): + from unidecode import unidecode + + return unidecode(old) == unidecode(new) + + +class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipient): + pass + + +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + +Account.TYPE_MORTGAGE = 17 +Account.TYPE_CONSUMER_CREDIT = 18 +Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/fortuneo/pages/compat/weboob_tools_capabilities_bank_investments.py b/modules/fortuneo/pages/compat/weboob_tools_capabilities_bank_investments.py new file mode 100644 index 0000000000..2b68ff9e00 --- /dev/null +++ b/modules/fortuneo/pages/compat/weboob_tools_capabilities_bank_investments.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Jonathan Schmidt +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.tools.compat import basestring +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.bank import Investment + +def is_isin_valid(isin): + """ + Méthode générale + Table de conversion des lettres en chiffres + A=10 B=11 C=12 D=13 E=14 F=15 G=16 H=17 I=18 + J=19 K=20 L=21 M=22 N=23 O=24 P=25 Q=26 R=27 + S=28 T=29 U=30 V=31 W=32 X=33 Y=34 Z=35 + + 1 - Mettre de côté la clé, qui servira de référence à la fin de la vérification. + 2 - Convertir toutes les lettres en nombres via la table de conversion ci-contre. Si le nombre obtenu est supérieur ou égal à 10, prendre les deux chiffres du nombre séparément (exemple : 27 devient 2 et 7). + 3 - Pour chaque chiffre, multiplier sa valeur par deux si sa position est impaire en partant de la droite. Si le nombre obtenu est supérieur ou égal à 10, garder les deux chiffres du nombre séparément (exemple : 14 devient 1 et 4). + 4 - Faire la somme de tous les chiffres. + 5 - Soustraire cette somme de la dizaine supérieure ou égale la plus proche (exemples : si la somme vaut 22, la dizaine « supérieure ou égale » est 30, et la clé vaut donc 8 ; si la somme vaut 30, la dizaine « supérieure ou égale » est 30, et la clé vaut 0 ; si la somme vaut 31, la dizaine « supérieure ou égale » est 40, et la clé vaut 9). + 6 - Comparer la valeur obtenue à la clé mise initialement de côté. + + Étapes 1 et 2 : + F R 0 0 0 3 5 0 0 0 0 (+ 8 : clé) + 15 27 0 0 0 3 5 0 0 0 0 + + Étape 3 : le traitement se fait sur des chiffres + 1 5 2 7 0 0 0 3 5 0 0 0 0 + I P I P I P I P I P I P I : position en partant de la droite (P = Pair, I = Impair) + 2 1 2 1 2 1 2 1 2 1 2 1 2 : coefficient multiplicateur + + 2 5 4 7 0 0 0 3 10 0 0 0 0 : résultat + + Étape 4 : + 2 + 5 + 4 + 7 + 0 + 0 + 0 + 3 + (1 + 0)+ 0 + 0 + 0 + 0 = 22 + + Étapes 5 et 6 : 30 - 22 = 8 (valeur de la clé) + """ + + if not isinstance(isin, basestring): + return False + if not re.match(r'^[A-Z]{2}[A-Z0-9]{9}\d$', isin): + return False + + isin_in_digits = ''.join(str(ord(x) - ord('A') + 10) if not x.isdigit() else x for x in isin[:-1]) + key = isin[-1:] + result = '' + for k, val in enumerate(isin_in_digits[::-1], start=1): + if k % 2 == 0: + result = ''.join((result, val)) + else: + result = ''.join((result, str(int(val)*2))) + return str(sum(int(x) for x in result) + int(key))[-1] == '0' + + +def create_french_liquidity(valuation): + """ + Automatically fills a liquidity investment with label, code and code_type. + """ + liquidity = Investment() + liquidity.label = "Liquidités" + liquidity.code = "XX-liquidity" + liquidity.code_type = NotAvailable + liquidity.valuation = valuation + return liquidity + diff --git a/modules/fortuneo/pages/transfer.py b/modules/fortuneo/pages/transfer.py index 9e8c5193ce..9cbd23587d 100644 --- a/modules/fortuneo/pages/transfer.py +++ b/modules/fortuneo/pages/transfer.py @@ -23,11 +23,11 @@ from datetime import date, timedelta from weboob.browser.pages import HTMLPage, PartialHTMLPage, LoggedPage -from weboob.browser.elements import method, ListElement, ItemElement +from weboob.browser.elements import method, ListElement, ItemElement, SkipItem from weboob.browser.filters.standard import ( CleanText, Date, Regexp, CleanDecimal, Currency, Field, Env, ) -from weboob.capabilities.bank import Recipient, Transfer, TransferBankError, AddRecipientError +from .compat.weboob_capabilities_bank import Recipient, Transfer, TransferBankError, AddRecipientBankError from weboob.capabilities.base import NotAvailable @@ -43,16 +43,18 @@ def condition(self): class item(ItemElement): klass = Recipient - def obj_id(self): - iban = CleanText('./td[6]', replace=[(' ', '')])(self) - iban = re.search(r'(?<=IBAN:)(\w+)BIC', iban).group(1) - return iban - def obj_label(self): if Field('_custom_label')(self): return '{} - {}'.format(Field('_recipient_name')(self), Field('_custom_label')(self)) return Field('_recipient_name')(self) + def obj_id(self): + iban = CleanText('./td[6]', replace=[(' ', '')])(self) + iban_number = re.search(r'(?<=IBAN:)(\w+)BIC', iban) + if iban_number: + return iban_number.group(1) + raise SkipItem('There is no IBAN for the recipient %s' % Field('label')(self)) + obj__recipient_name = CleanText('./td[2]') obj__custom_label = CleanText('./td[4]') obj_iban = NotAvailable @@ -70,7 +72,7 @@ def check_external_iban_form(self, recipient): def check_recipient_iban(self): if not CleanText('//input[@name="codeBic"]/@value')(self.doc): - raise AddRecipientError('Recipient already exist or invalid iban') + raise AddRecipientBankError(message="Le bénéficiaire est déjà présent ou bien l'iban est incorrect") def fill_recipient_form(self, recipient) : form = self.get_form(id='CompteExterneActionForm') @@ -89,8 +91,8 @@ def get_new_recipient(self, recipient): recipient_xpath + '/li[contains(text(), "Nom du titulaire")]', replace=[(' ', '')] ), r'(?<=Nomdutitulaire:)(\w+)')(self.doc) rcpt.iban = Regexp(CleanText( - recipient_xpath + '/li[contains(text(), "IBAN")]', replace=[(' ', '')] - ), r'(?<=IBAN:)([A-Z]{2}\d+)')(self.doc) + recipient_xpath + '/li[contains(text(), "IBAN")]' + ), r'IBAN : ([A-Za-z]{2}[\dA-Za-z]+)')(self.doc) rcpt.id = rcpt.iban rcpt.category = 'Externe' rcpt.enabled_at = date.today() + timedelta(1) @@ -104,7 +106,7 @@ def get_send_code_form(self): class RecipientSMSPage(LoggedPage, PartialHTMLPage): def on_load(self): if not self.doc.xpath('//input[@id="otp"]') and not self.doc.xpath('//div[@class="confirmationAjoutCompteExterne"]'): - raise AddRecipientError(CleanText('//div[@id="aidesecuforte"]/p[contains("Nous vous invitons")]')(self.doc)) + raise AddRecipientBankError(message=CleanText('//div[@id="aidesecuforte"]/p[contains("Nous vous invitons")]')(self.doc)) def build_doc(self, content): content = '
' + content.decode('latin-1') + '
' diff --git a/modules/gmf/browser.py b/modules/gmf/browser.py index 859e463178..221bc15b29 100644 --- a/modules/gmf/browser.py +++ b/modules/gmf/browser.py @@ -24,7 +24,7 @@ from .pages import ( LoginPage, AccountsPage, TransactionsInvestmentsPage, AllTransactionsPage, - DocumentsSignaturePage, + DocumentsSignaturePage, RedirectToUserAgreementPage, UserAgreementPage, ) @@ -32,6 +32,8 @@ class GmfBrowser(LoginBrowser): BASEURL = 'https://espace-assure.gmf.fr' login = URL(r'/public/pages/securite/IC2.faces', LoginPage) + redirect_to_user_agreement = URL('^$', RedirectToUserAgreementPage) + user_agreement = URL(r'restreint/pages/securite/IC9.faces', UserAgreementPage) accounts = URL(r'/pointentree/client/homepage', AccountsPage) transactions_investments = URL(r'/pointentree/contratvie/detailsContrats', TransactionsInvestmentsPage) all_transactions = URL(r'/pages/contratvie/detailscontrats/.*\.faces', AllTransactionsPage) diff --git a/modules/gmf/compat/weboob_capabilities_bank.py b/modules/gmf/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/gmf/compat/weboob_capabilities_bank.py +++ b/modules/gmf/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/gmf/pages.py b/modules/gmf/pages.py index 22a641ba10..1d251b9b19 100644 --- a/modules/gmf/pages.py +++ b/modules/gmf/pages.py @@ -63,7 +63,8 @@ class item(ItemElement): obj_label = CleanText('.//div[@class="type-contrat"]//h2') obj_type = Account.TYPE_LIFE_INSURANCE obj_balance = CleanDecimal(CleanText('.//div[@class="col-right"]', children=False), replace_dots=True, default=NotAvailable) - obj_currency = Currency(CleanText(u'.//div[@class="col-right"]', children=False, replace=[("Au", "")])) + obj_currency = Currency(CleanText('.//div[@class="col-right"]', children=False, + replace=[("Au", "")])) def get_detail_page_parameters(self, account): """ @@ -74,7 +75,8 @@ def get_detail_page_parameters(self, account): # parameter 1 el = self.doc.xpath('//div[@class="infos-contrat"][descendant::strong[contains(text(), $contract_id)]]/parent::div//div[@class="zone-detail"]//span/a', contract_id=account.id) assert len(el) == 1 - parameter = Regexp(Attr('.', 'onclick'), r".*,\{'(.*)':'(.*)'\},.*\);return false", '\\1 \\2')(el[0]).split(' ') + parameter = Regexp(Attr('.', 'onclick'), r".*,\{'(.*)':'(.*)'\},.*\);return false", + '\\1 \\2')(el[0]).split(' ') data.append((parameter[0], parameter[1])) form = self.get_form(id='j_idt254') @@ -163,8 +165,10 @@ class AllTransactionsPage(LoggedPage, XMLPage, HTMLPage, TransactionsParser): def build_doc(self, content): # HTML embedded in XML: parse XML first then extract the html xml = XMLPage.build_doc(self, content) - transactions_html = xml.xpath('//partial-response/changes/update[1]')[0].text.encode(encoding=self.encoding) - investments_html = xml.xpath('//partial-response/changes/update[2]')[0].text.encode(encoding=self.encoding) + transactions_html = (xml.xpath('//partial-response/changes/update[1]')[0].text + .encode(encoding=self.encoding)) + investments_html = (xml.xpath('//partial-response/changes/update[2]')[0].text + .encode(encoding=self.encoding)) html = transactions_html + investments_html return HTMLPage.build_doc(self, html) @@ -173,3 +177,14 @@ class DocumentsSignaturePage(LoggedPage, HTMLPage): def on_load(self): if self.doc.xpath('//span[contains(text(), "VO(S) DOCUMENT(S) A SIGNER")]'): raise ActionNeeded(CleanText('//div[@class="block"]/p[contains(text(), "Vous avez un ou plusieurs document(s) à signer")]')(self.doc)) + + +class RedirectToUserAgreementPage(LoggedPage, HTMLPage): + MAX_REFRESH = 0 + + +class UserAgreementPage(LoggedPage, HTMLPage): + def on_load(self): + message = CleanText('//fieldset//legend|//fieldset//label')(self.doc) + if 'conditions générales' in message: + raise ActionNeeded(message) diff --git a/modules/groupama/compat/weboob_capabilities_bank.py b/modules/groupama/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/groupama/compat/weboob_capabilities_bank.py +++ b/modules/groupama/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/groupamaes/browser.py b/modules/groupamaes/browser.py index 619e40df0d..8c548dd8e4 100644 --- a/modules/groupamaes/browser.py +++ b/modules/groupamaes/browser.py @@ -67,4 +67,4 @@ def iter_investment(self, account): @need_login def iter_pocket(self, account): - return self.groupamaes_pocket.go(page='&_pid=SituationParPlan&_fid=GoPositionsDetaillee').iter_pocket(account.label) + return self.groupamaes_pocket.go(page='&_pid=SituationParPlan&_fid=GoPositionsDetaillee').iter_pocket(account.label) diff --git a/modules/groupamaes/compat/weboob_capabilities_bank.py b/modules/groupamaes/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/groupamaes/compat/weboob_capabilities_bank.py +++ b/modules/groupamaes/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/hsbc/browser.py b/modules/hsbc/browser.py index a5284791c7..4f3255192e 100644 --- a/modules/hsbc/browser.py +++ b/modules/hsbc/browser.py @@ -35,7 +35,7 @@ from .pages.account_pages import ( AccountsPage, CBOperationPage, CPTOperationPage, LoginPage, AppGonePage, RibPage, - UnavailablePage, OtherPage, FrameContainer + UnavailablePage, OtherPage, FrameContainer, ProfilePage, ) from .pages.life_insurances import ( LifeInsurancesPage, LifeInsurancePortal, LifeInsuranceMain, LifeInsuranceUseless, @@ -79,6 +79,7 @@ class HSBC(LoginBrowser): rib = URL(r'/cgi-bin/emcgi', RibPage) accounts = URL(r'/cgi-bin/emcgi', AccountsPage) life_insurance_useless = URL(r'/cgi-bin/emcgi', LifeInsuranceUseless) + profile = URL(r'/cgi-bin/emcgi', ProfilePage) unavailable = URL(r'/cgi-bin/emcgi', UnavailablePage) frame_page = URL(r'/cgi-bin/emcgi', r'https://clients.hsbc.fr/cgi-bin/emcgi', FrameContainer) @@ -166,20 +167,6 @@ def do_login(self): self.location(home_url) - @need_login - def get_accounts_list(self): - if not self.accounts_list: - self.update_accounts_list() - for a in self.accounts_list.values(): - # Get parent of card account - if a.type == Account.TYPE_CARD: - card_page = self.open(a.url).page - parent_id = card_page.get_parent_id() - a.parent = find_object(self.accounts_list.values(), id=parent_id) - if a.parent and not a.currency: - a.currency = a.parent.currency - yield a - def go_post(self, url, data=None): # most of HSBC accounts links are actually handled by js code # which convert a GET query string to POST data. @@ -190,6 +177,33 @@ def go_post(self, url, data=None): url = url[:url.find('?')] self.location(url, data=q) + @need_login + def get_accounts_list(self): + if not self.accounts_list: + self.update_accounts_list() + + # go on cards page if there are cards accounts + for a in self.accounts_list.values(): + if a.type == Account.TYPE_CARD: + self.location(a.url) + break + + # get all couples (card, parent) on cards page + all_card_and_parent = [] + if self.cbPage.is_here(): + all_card_and_parent = self.page.get_all_parent_id() + self.go_post(self.js_url, data={'debr': 'COMPTES_PAN'}) + + # update cards parent and currency + for a in self.accounts_list.values(): + if a.type == Account.TYPE_CARD: + for card in all_card_and_parent: + if a.id in card[0].replace(' ', ''): + a.parent = find_object(self.accounts_list.values(), id=card[1]) + if a.parent and not a.currency: + a.currency = a.parent.currency + yield a + @need_login def update_accounts_list(self, iban=True): if self.accounts.is_here(): @@ -198,7 +212,7 @@ def update_accounts_list(self, iban=True): data = {'debr': 'COMPTES_PAN'} self.go_post(self.js_url, data=data) - for a in self.page.iter_accounts(): + for a in self.page.iter_spaces_account(): try: self.accounts_list[a.id].url = a.url except KeyError: @@ -257,10 +271,10 @@ def get_history(self, account, coming=False, retry_li=True): if account.url is None: return [] - if account.url.startswith('javascript') or '&Crd=' in account.url: + if account.url.startswith('javascript') or '&Crd=' in account.url or account.type == Account.TYPE_LOAN: raise NotImplementedError() - if account.type == Account.TYPE_LIFE_INSURANCE: + if account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_CAPITALISATION): if coming is True: return [] @@ -309,6 +323,12 @@ def get_history(self, account, coming=False, retry_li=True): if self.page is None: return [] + # for 'fusion' space + if hasattr(account, '_is_form') and account._is_form: + # go on accounts page to get account form + self.go_post(self.js_url, data={'debr': 'COMPTES_PAN'}) + self.page.go_history_page(account) + if self.cbPage.is_here(): guesser = LinearDateGuesser(date_max_bump=timedelta(45)) history = list(self.page.get_history(date_guesser=guesser)) @@ -336,7 +356,7 @@ def _get_history(self): yield tr def get_investments(self, account, retry_li=True): - if account.type == Account.TYPE_LIFE_INSURANCE: + if account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_CAPITALISATION): return self.get_life_investments(account, retry_li=retry_li) elif account.type == Account.TYPE_PEA: return self.get_pea_investments(account) @@ -449,3 +469,9 @@ def _go_to_wealth_accounts(self): self.PEA_LISTING['liquidities'] = list(helper.retrieve_liquidity()) self.PEA_LISTING['investments'] = list(helper.retrieve_invests()) self.connection.go() + + @need_login + def get_profile(self): + data = {'debr': 'PARAM'} + self.go_post(self.js_url, data=data) + return self.page.get_profile() diff --git a/modules/hsbc/compat/weboob_capabilities_bank.py b/modules/hsbc/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/hsbc/compat/weboob_capabilities_bank.py +++ b/modules/hsbc/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/hsbc/module.py b/modules/hsbc/module.py index 0a6984cdc2..1124ddcf0c 100644 --- a/modules/hsbc/module.py +++ b/modules/hsbc/module.py @@ -22,14 +22,14 @@ from weboob.capabilities.base import find_object from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword, Value - +from weboob.capabilities.profile import CapProfile from .browser import HSBC __all__ = ['HSBCModule'] -class HSBCModule(Module, CapBankWealth): +class HSBCModule(Module, CapBankWealth, CapProfile): NAME = 'hsbc' MAINTAINER = u'Romain Bignon' EMAIL = 'romain@weboob.org' @@ -64,3 +64,6 @@ def iter_investment(self, account): def iter_coming(self, account): for tr in self.browser.get_history(account, coming=True): yield tr + + def get_profile(self): + return self.browser.get_profile() diff --git a/modules/hsbc/pages/account_pages.py b/modules/hsbc/pages/account_pages.py index 03e1b1c17e..f129080835 100644 --- a/modules/hsbc/pages/account_pages.py +++ b/modules/hsbc/pages/account_pages.py @@ -29,18 +29,18 @@ from weboob.browser.elements import ListElement, ItemElement, method from weboob.browser.pages import HTMLPage, pagination from weboob.browser.filters.standard import ( - Filter, Env, CleanText, CleanDecimal, Field, DateGuesser, Regexp + Filter, Env, CleanText, CleanDecimal, Field, DateGuesser, Regexp, Currency, ) from weboob.browser.filters.html import AbsoluteLink, TableCell from weboob.browser.filters.javascript import JSVar - +from weboob.capabilities.profile import Person from .landing_pages import GenericLandingPage class Transaction(FrenchTransaction): PATTERNS = [(re.compile(r'^VIR(EMENT)? (?P.*)'), FrenchTransaction.TYPE_TRANSFER), (re.compile(r'^PRLV (?P.*)'), FrenchTransaction.TYPE_ORDER), - (re.compile(r'^CB (?P.*?)\s+(?P
\d+)/(?P[01]\d)\s*(?P.*)'), + (re.compile(r'^CB (?P.*?)\s+(?P
\d+)/(?P[01]\d)\s+(?P.*)'), FrenchTransaction.TYPE_CARD), (re.compile(r'^DAB (?P
\d{2})/(?P\d{2}) ((?P\d{2})H(?P\d{2}) )?(?P.*?)( CB N°.*)?$'), FrenchTransaction.TYPE_WITHDRAWAL), @@ -80,9 +80,73 @@ def on_load(self): raise BrowserUnavailable() +class AccountsType(Filter): + PATTERNS = [ + ('c.aff', Account.TYPE_CHECKING), + ('pea', Account.TYPE_PEA), + ('invest', Account.TYPE_MARKET), + ('ptf', Account.TYPE_MARKET), + ('ldd', Account.TYPE_SAVINGS), + ('cel', Account.TYPE_SAVINGS), + ('pel', Account.TYPE_SAVINGS), + ('livret', Account.TYPE_SAVINGS), + ('livjeu', Account.TYPE_SAVINGS), + ('compte', Account.TYPE_CHECKING), + ('cpte', Account.TYPE_CHECKING), + ('scpi', Account.TYPE_MARKET), + ('account', Account.TYPE_CHECKING), + ('pret', Account.TYPE_LOAN), + ('vie', Account.TYPE_LIFE_INSURANCE), + ('strategie patr.', Account.TYPE_LIFE_INSURANCE), + ('essentiel', Account.TYPE_LIFE_INSURANCE), + ('elysee', Account.TYPE_LIFE_INSURANCE), + ('abondance', Account.TYPE_LIFE_INSURANCE), + ('ely. retraite', Account.TYPE_LIFE_INSURANCE), + ('lae option assurance', Account.TYPE_LIFE_INSURANCE), + ('carte ', Account.TYPE_CARD), + ('business ', Account.TYPE_CARD), + ('plan assur. innovat.', Account.TYPE_LIFE_INSURANCE), + ('hsbc evol pat transf', Account.TYPE_LIFE_INSURANCE), + ('hsbc evol pat capi', Account.TYPE_CAPITALISATION), + ('bourse libre', Account.TYPE_MARKET), + ('plurival', Account.TYPE_LIFE_INSURANCE), + ] + + def filter(self, label): + label = label.lower() + for pattern, type in self.PATTERNS: + if pattern in label: + return type + return Account.TYPE_UNKNOWN + + +class Label(Filter): + def filter(self, text): + return text.lstrip(' 0123456789').title() + + class AccountsPage(GenericLandingPage): is_here = '//h1[text()="Synthèse"]' + def iter_spaces_account(self): + if self.doc.xpath('//p[text()="HSBC Fusion"]'): + space = 'fusion' + else: + space = 'default' + + accounts = { + 'fusion': self.iter_fusion_accounts, + 'default': self.iter_accounts, + } + return accounts[space]() + + def go_history_page(self, account): + for acc in self.doc.xpath('//div[@onclick]'): + # label contains account number, it's enough to check if it's the right account + if account.label == Label(CleanText('.//p[@class="title"]'))(acc): + form_id = CleanText('.//form/@id')(acc) + return self.get_form(id=form_id).submit() + @method class iter_accounts(ListElement): item_xpath = '//tr' @@ -94,54 +158,13 @@ class item(ItemElement): def condition(self): return len(self.el.xpath('./td')) > 2 - class Label(Filter): - def filter(self, text): - return text.lstrip(' 0123456789').title() - - class Type(Filter): - PATTERNS = [ - ('c.aff', Account.TYPE_CHECKING), - ('pea', Account.TYPE_PEA), - ('invest', Account.TYPE_MARKET), - ('ptf', Account.TYPE_MARKET), - ('ldd', Account.TYPE_SAVINGS), - ('cel', Account.TYPE_SAVINGS), - ('pel', Account.TYPE_SAVINGS), - ('livret', Account.TYPE_SAVINGS), - ('livjeu', Account.TYPE_SAVINGS), - ('compte', Account.TYPE_CHECKING), - ('cpte', Account.TYPE_CHECKING), - ('scpi', Account.TYPE_MARKET), - ('account', Account.TYPE_CHECKING), - ('pret', Account.TYPE_LOAN), - ('vie', Account.TYPE_LIFE_INSURANCE), - ('strategie patr.', Account.TYPE_LIFE_INSURANCE), - ('essentiel', Account.TYPE_LIFE_INSURANCE), - ('elysee', Account.TYPE_LIFE_INSURANCE), - ('abondance', Account.TYPE_LIFE_INSURANCE), - ('ely. retraite', Account.TYPE_LIFE_INSURANCE), - ('lae option assurance', Account.TYPE_LIFE_INSURANCE), - ('carte ', Account.TYPE_CARD), - ('plan assur. innovat.', Account.TYPE_LIFE_INSURANCE), - ('hsbc evol pat transf', Account.TYPE_LIFE_INSURANCE), - ('bourse libre', Account.TYPE_MARKET), - ('plurival', Account.TYPE_LIFE_INSURANCE), - ] - - def filter(self, label): - label = label.lower() - for pattern, type in self.PATTERNS: - if pattern in label: - return type - return Account.TYPE_UNKNOWN - obj_label = Label(CleanText('./td[1]/a')) obj_coming = Env('coming') obj_currency = FrenchTransaction.Currency('./td[2]') obj_url = CleanText(AbsoluteLink('./td[1]/a'), replace=[('\n', '')]) - obj_type = Type(Field('label')) + obj_type = AccountsType(Field('label')) obj_coming = NotAvailable @property @@ -161,6 +184,43 @@ def obj_id(self): return CleanText(replace=[('.', ''), (' ', '')]).filter(self.el.xpath('./td[2]')) + ".INVEST" return CleanText(replace=[('.', ''), (' ', '')]).filter(self.el.xpath('./td[2]')) + @method + class iter_fusion_accounts(ListElement): + def find_elements(self): + all_xpaths = ( + '//div[@id="All" and @class="tabcontent"]/div', + '//div[@class="formGroup"]/div' + ) + for xpath in all_xpaths: + ret = self.xpath(xpath) + if ret: + return ret + else: + assert False, 'Accounts are not well handled' + + class iter_accounts_tables(ListElement): + item_xpath = './div[@onclick]' + + class item(ItemElement): + klass = Account + + obj_label = Label(CleanText('.//p[@class="title"]')) + obj_balance = CleanDecimal(CleanText('.//p[@class="balance"]'), replace_dots=True) + obj_currency = Currency(CleanText('.//p[@class="balance"]')) + obj_type = AccountsType(Field('label')) + obj_url = CleanText('.//form/@action') + obj__is_form = bool(CleanText('.//form/@id')) + + @property + def obj_id(self): + account_id = CleanText('.//p[@class="title"]/span', replace=[('.', ''), (' ', '')])(self) + # Investment account and main account can have the same id + # so we had account type in case of Investment to prevent conflict + # and also the same problem with scpi accounts. + if Field('type')(self) == Account.TYPE_MARKET: + return account_id + ".INVEST" + return account_id + class RibPage(GenericLandingPage): def is_here(self): @@ -219,8 +279,19 @@ def obj_date(self): def get_parent_id(self): # The parent id is in the details of the card - m = re.search(r'Solde du compte (.*)', CleanText('//div[@class="RecentTransactions"]/h2')(self.doc)) - return m.group(1) + return Regexp(CleanText('//h2[contains(text(), "Solde du compte")]'), r'Solde du compte (.*)')(self.doc) + + def get_all_parent_id(self): + all_parent_id = [] + all_card = [CleanText('.')(card) for card in self.doc.xpath('//select[@name="choix_carte"]/option')] + + for index, card in enumerate(all_card): + form = self.get_form(name='FORM_LIB_CARTE') + form['index_carte'] = index + form['choix_carte'] = card + all_parent_id.append((card, form.submit().page.get_parent_id())) + + return all_parent_id class CPTOperationPage(GenericLandingPage): @@ -328,3 +399,16 @@ def on_load(self): for msg, exc in self.ERROR_CLASSES: for tag in self.doc.xpath('//p[@class="debit"]//strong[text()[contains(.,$msg)]]', msg=msg): raise exc(CleanText('.')(tag)) + + +class ProfilePage(OtherPage): + # Warning: this page contains a div_err and displays "Service indisponible" even if it is not... + # but we can still see the data we need + is_here = '//h1[contains(text(), "mes données")]' + + @method + class get_profile(ItemElement): + klass = Person + + obj_name = CleanText('//div[@id="div_adr_P1"]//p/label[contains(text(), "Nom")]/parent::p/strong') + obj_address = CleanText('//div[@id="div_adr_P1"]//p/label[contains(text(), "Adresse")]/parent::p/strong') diff --git a/modules/hsbc/pages/investments.py b/modules/hsbc/pages/investments.py index 7cd1e7e5ed..67a0e81a7a 100644 --- a/modules/hsbc/pages/investments.py +++ b/modules/hsbc/pages/investments.py @@ -280,6 +280,8 @@ def retrieve_invests(self): def retrieve_liquidity(self): self.retrieve_products(kind='liquidity_list') + if self.browser.retrieve_useless_page.is_here(): + return [] assert isinstance(self.browser.page, RetrieveLiquidityPage) return self.browser.page.iter_liquidity() diff --git a/modules/ideel/browser.py b/modules/ideel/browser.py index 307ee4c7b7..4c19aa03f7 100644 --- a/modules/ideel/browser.py +++ b/modules/ideel/browser.py @@ -18,19 +18,18 @@ # along with weboob. If not, see . -from weboob.tools.capabilities.bank.transactions import \ - AmericanTransaction as AmTr -from weboob.browser import LoginBrowser, URL, need_login -from weboob.browser.pages import HTMLPage -from weboob.capabilities.base import Currency -from weboob.capabilities.shop import Order, Item, Payment, OrderNotFound -from weboob.exceptions import BrowserIncorrectPassword - import re -from decimal import Decimal from datetime import datetime -from itertools import takewhile, count +from decimal import Decimal +from itertools import count, takewhile +from weboob.browser import URL, LoginBrowser, need_login +from weboob.browser.pages import HTMLPage +from weboob.capabilities.base import Currency +from weboob.capabilities.shop import Item, Order, OrderNotFound, Payment +from weboob.exceptions import BrowserIncorrectPassword +from weboob.tools.capabilities.bank.transactions import AmericanTransaction as AmTr +from weboob.tools.compat import unicode __all__ = ['Ideel'] @@ -55,7 +54,7 @@ def exists(self): def iter_orders(self): return (tr.xpath('td[1]/a/text()')[0][1:] - for tr in self.doc.xpath('//table[@id="order_history"]/tbody/tr')) + for tr in self.doc.xpath('//table[@id="order_history"]/tbody/tr')) class OrderPage(IdeelPage): @@ -78,7 +77,7 @@ def items(self): url = tr.xpath('*//div[@class="item_img"]//@src')[0] onclk = tr.xpath('*//div[@class="item_img"]//@onclick') if onclk: - url=re.match(r'window.open\(\'([^\']*)\'.*', onclk[0]).group(1) + url = re.match(r'window.open\(\'([^\']*)\'.*', onclk[0]).group(1) if url.startswith('/'): url = self.browser.BASEURL + url price = tr.xpath('td[@class="items_price"]/span/text()')[0] @@ -123,7 +122,7 @@ def discount(self): TAGS = ['coupon_discount_amount', 'promo_discount_amount', 'total_rewards', 'applied_credit'] return -sum(AmTr.decimal_amount(x[1:][:-1]) for tag in TAGS - for x in self.doc.xpath('//span[@id="%s"]/text()' % tag)) + for x in self.doc.xpath('//span[@id="%s"]/text()' % tag)) def total(self): return AmTr.decimal_amount(self.doc.xpath( diff --git a/modules/ideel/module.py b/modules/ideel/module.py index 9bbaa9ae12..3d28d51aa1 100644 --- a/modules/ideel/module.py +++ b/modules/ideel/module.py @@ -19,13 +19,14 @@ from weboob.capabilities.shop import CapShop -from weboob.tools.backend import Module, BackendConfig +from weboob.tools.backend import BackendConfig, Module from weboob.tools.value import ValueBackendPassword from .browser import Ideel __all__ = ['IdeelModule'] + class IdeelModule(Module, CapShop): NAME = 'ideel' MAINTAINER = u'Oleg Plakhotniuk' diff --git a/modules/infomaniak/module.py b/modules/infomaniak/module.py index 8e4de385d9..c11c3e1157 100644 --- a/modules/infomaniak/module.py +++ b/modules/infomaniak/module.py @@ -38,7 +38,7 @@ class InfomaniakModule(Module, CapDocument): EMAIL = 'dev@indigo.re' LICENSE = 'AGPLv3+' VERSION = '1.3' - CONFIG = BackendConfig(ValueBackendPassword('login', label='Email', masked=False), + CONFIG = BackendConfig(ValueBackendPassword('login', label='Email de connexion', masked=False), ValueBackendPassword('password', label='Mot de passe')) BROWSER = InfomaniakBrowser diff --git a/modules/ing/browser.py b/modules/ing/browser.py index 70dde32b1b..0808565444 100644 --- a/modules/ing/browser.py +++ b/modules/ing/browser.py @@ -20,11 +20,12 @@ import hashlib import time +import json from requests.exceptions import SSLError from weboob.browser import LoginBrowser, URL, need_login -from weboob.exceptions import BrowserIncorrectPassword +from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable from weboob.browser.exceptions import ServerError from weboob.capabilities.bank import Account, AccountNotFound from weboob.capabilities.base import find_object, NotAvailable @@ -218,6 +219,12 @@ def get_accounts_on_space(self, space, get_iban=True): accounts_list.append(loan) yield loan + def get_coming_balance(self, account): + if account.type == Account.TYPE_CHECKING: + self.go_account_page(account) + return self.page.get_coming_balance() + return NotAvailable + @need_login @start_with_main_site def get_accounts_list(self, space=None, get_iban=True): @@ -228,17 +235,20 @@ def get_accounts_list(self, space=None, get_iban=True): if space: for acc in self.get_accounts_on_space(space, get_iban=get_iban): + acc.coming = self.get_coming_balance(acc) yield acc elif self.multispace: for space in self.multispace: for acc in self.get_accounts_on_space(space, get_iban=get_iban): + acc.coming = self.get_coming_balance(acc) yield acc else: for acc in self.page.get_list(): acc._space = None if get_iban: self.get_iban(acc) + acc.coming = self.get_coming_balance(acc) yield acc for loan in self.iter_detailed_loans(): @@ -261,8 +271,16 @@ def iter_detailed_loans(self): self.accountspage.go(data=data) self.loantokenpage.go(data=data) - self.loandetailpage.go().getdetails(loan) - + try: + self.loandetailpage.go() + + except ServerError as exception: + json_error = json.loads(exception.response.text) + if json_error['error']['code'] == "INTERNAL_ERROR": + raise BrowserUnavailable(json_error['error']['message']) + raise + else: + self.page.getdetails(loan) yield loan self.return_from_loan_site() diff --git a/modules/ing/compat/weboob_capabilities_bank.py b/modules/ing/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/ing/compat/weboob_capabilities_bank.py +++ b/modules/ing/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/ing/pages/accounts_list.py b/modules/ing/pages/accounts_list.py index 9967545b16..665160fc0c 100644 --- a/modules/ing/pages/accounts_list.py +++ b/modules/ing/pages/accounts_list.py @@ -39,7 +39,9 @@ class Transaction(FrenchTransaction): - PATTERNS = [(re.compile(u'^retrait (?P.*)'), FrenchTransaction.TYPE_WITHDRAWAL), + PATTERNS = [(re.compile(u'^retrait dab (?P
\d{2})/(?P\d{2})/(?P\d{4}) (?P.*)'), FrenchTransaction.TYPE_WITHDRAWAL), + # Withdrawal in foreign currencies will look like "retrait 123 currency" + (re.compile(u'^retrait (?P.*)'), FrenchTransaction.TYPE_WITHDRAWAL), (re.compile(u'^carte (?P
\d{2})/(?P\d{2})/(?P\d{4}) (?P.*)'), FrenchTransaction.TYPE_CARD), (re.compile(u'^virement (sepa )?(emis vers|recu|emis)? (?P.*)'), FrenchTransaction.TYPE_TRANSFER), (re.compile(u'^remise cheque(?P.*)'), FrenchTransaction.TYPE_DEPOSIT), @@ -171,7 +173,6 @@ class item(ItemElement): obj_label = CleanText('./span[@class="title"]') obj_id = AddPref(Field('_id'), Field('label')) obj_type = AddType(Field('label')) - obj_coming = NotAvailable obj__jid = Attr('//input[@name="javax.faces.ViewState"]', 'value') def obj_balance(self): @@ -189,15 +190,14 @@ class item(ItemElement): klass = Loan obj_currency = u'EUR' - obj_label = CleanText('./span[@class="title"]') + obj_label = CleanText('.//span[@class="title"]') obj_id = AddPref(Field('_id'), Field('label')) obj_type = AddType(Field('label')) - obj_coming = NotAvailable obj__jid = Attr('//input[@name="javax.faces.ViewState"]', 'value') - obj__id = CleanText('./span[@class="account-number"]') + obj__id = CleanText('.//span[@class="account-number"]') def obj_balance(self): - balance = CleanDecimal('./span[@class="solde"]/label', replace_dots=True)(self) + balance = CleanDecimal('.//div/span[@class="solde"]/label', replace_dots=True)(self) return -abs(balance) class generic_transactions(ListElement): @@ -323,6 +323,12 @@ def load_space_page(self): self.fillup_form(form, r"\),\{(.*)\},'", on_click) form.submit() + def get_coming_balance(self): + return CleanDecimal('//div[@class="previsionnel"]/div[@class="solde_value"]//label', + replace_dots=True, + default=NotAvailable)(self.doc) + + class IbanPage(LoggedPage, HTMLPage): def get_iban(self): iban = CleanText('//tr[td[1]//text()="IBAN"]/td[2]')(self.doc).strip().replace(' ', '') diff --git a/modules/ing/pages/compat/__init__.py b/modules/ing/pages/compat/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/modules/ing/pages/compat/weboob_tools_capabilities_bank_investments.py b/modules/ing/pages/compat/weboob_tools_capabilities_bank_investments.py new file mode 100644 index 0000000000..2b68ff9e00 --- /dev/null +++ b/modules/ing/pages/compat/weboob_tools_capabilities_bank_investments.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Jonathan Schmidt +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.tools.compat import basestring +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.bank import Investment + +def is_isin_valid(isin): + """ + Méthode générale + Table de conversion des lettres en chiffres + A=10 B=11 C=12 D=13 E=14 F=15 G=16 H=17 I=18 + J=19 K=20 L=21 M=22 N=23 O=24 P=25 Q=26 R=27 + S=28 T=29 U=30 V=31 W=32 X=33 Y=34 Z=35 + + 1 - Mettre de côté la clé, qui servira de référence à la fin de la vérification. + 2 - Convertir toutes les lettres en nombres via la table de conversion ci-contre. Si le nombre obtenu est supérieur ou égal à 10, prendre les deux chiffres du nombre séparément (exemple : 27 devient 2 et 7). + 3 - Pour chaque chiffre, multiplier sa valeur par deux si sa position est impaire en partant de la droite. Si le nombre obtenu est supérieur ou égal à 10, garder les deux chiffres du nombre séparément (exemple : 14 devient 1 et 4). + 4 - Faire la somme de tous les chiffres. + 5 - Soustraire cette somme de la dizaine supérieure ou égale la plus proche (exemples : si la somme vaut 22, la dizaine « supérieure ou égale » est 30, et la clé vaut donc 8 ; si la somme vaut 30, la dizaine « supérieure ou égale » est 30, et la clé vaut 0 ; si la somme vaut 31, la dizaine « supérieure ou égale » est 40, et la clé vaut 9). + 6 - Comparer la valeur obtenue à la clé mise initialement de côté. + + Étapes 1 et 2 : + F R 0 0 0 3 5 0 0 0 0 (+ 8 : clé) + 15 27 0 0 0 3 5 0 0 0 0 + + Étape 3 : le traitement se fait sur des chiffres + 1 5 2 7 0 0 0 3 5 0 0 0 0 + I P I P I P I P I P I P I : position en partant de la droite (P = Pair, I = Impair) + 2 1 2 1 2 1 2 1 2 1 2 1 2 : coefficient multiplicateur + + 2 5 4 7 0 0 0 3 10 0 0 0 0 : résultat + + Étape 4 : + 2 + 5 + 4 + 7 + 0 + 0 + 0 + 3 + (1 + 0)+ 0 + 0 + 0 + 0 = 22 + + Étapes 5 et 6 : 30 - 22 = 8 (valeur de la clé) + """ + + if not isinstance(isin, basestring): + return False + if not re.match(r'^[A-Z]{2}[A-Z0-9]{9}\d$', isin): + return False + + isin_in_digits = ''.join(str(ord(x) - ord('A') + 10) if not x.isdigit() else x for x in isin[:-1]) + key = isin[-1:] + result = '' + for k, val in enumerate(isin_in_digits[::-1], start=1): + if k % 2 == 0: + result = ''.join((result, val)) + else: + result = ''.join((result, str(int(val)*2))) + return str(sum(int(x) for x in result) + int(key))[-1] == '0' + + +def create_french_liquidity(valuation): + """ + Automatically fills a liquidity investment with label, code and code_type. + """ + liquidity = Investment() + liquidity.label = "Liquidités" + liquidity.code = "XX-liquidity" + liquidity.code_type = NotAvailable + liquidity.valuation = valuation + return liquidity + diff --git a/modules/ing/pages/titre.py b/modules/ing/pages/titre.py index 8ed4189bf1..cc576886cd 100644 --- a/modules/ing/pages/titre.py +++ b/modules/ing/pages/titre.py @@ -29,6 +29,7 @@ from weboob.browser.filters.standard import CleanDecimal, CleanText, Date, Regexp, Env from weboob.browser.filters.html import Link, Attr, TableCell from weboob.tools.capabilities.bank.transactions import FrenchTransaction +from .compat.weboob_tools_capabilities_bank_investments import create_french_liquidity from weboob.tools.compat import unicode class NetissimaPage(HTMLPage): @@ -113,11 +114,8 @@ def iter_investments(self, account): # There is no investment on life insurance in the process to be created. if len(message.split('&')) >= 4: # We also have to get the liquidity as an investment. - invest = Investment() - invest.label = "Liquidités" - invest.code = "XX-liquidity" - invest.valuation = CleanDecimal(None, True).filter(message.split('&')[3].replace('euro;{','').strip()) - invests.append(invest) + valuation = CleanDecimal(None, True).filter(message.split('&')[3].replace('euro;{','').strip()) + invests.append(create_french_liquidity(valuation)) for invest in invests: yield invest diff --git a/modules/ing/pages/transfer.py b/modules/ing/pages/transfer.py index 65126b828a..034bd08599 100644 --- a/modules/ing/pages/transfer.py +++ b/modules/ing/pages/transfer.py @@ -187,7 +187,7 @@ def recap(self, origin, recipient, transfer): error = CleanText(u'//div[@id="transfer_form:moveMoneyDetailsBody"]//span[@class="error"]', default=None)(self.doc) or \ CleanText(u'//p[contains(text(), "Nous sommes désolés. Le solde de votre compte ne doit pas être inférieur au montant de votre découvert autorisé. Veuillez saisir un montant inférieur.")]', default=None)(self.doc) if error: - raise TransferInvalidAmount(message=error) + raise TransferInvalidAmount(message=error) t = Transfer() t.label = transfer.label @@ -219,7 +219,7 @@ def recap(self, origin, recipient, transfer): t.recipient_id = recipient.id t.exec_date = parse_french_date(CleanText('//p[has-class("exec-date")]', children=False, - replace=[('le', ''), (u'exécuté', ''), ('demain', ''), ('(', ''), (')', ''), - ("aujourd'hui", '')])(self.doc)).date() + replace=[('le', ''), (u'exécuté', ''), ('demain', ''), ('(', ''), (')', ''), + ("aujourd'hui", '')])(self.doc)).date() return t diff --git a/modules/lcl/browser.py b/modules/lcl/browser.py index 22e8938a01..c9716a69d6 100644 --- a/modules/lcl/browser.py +++ b/modules/lcl/browser.py @@ -18,15 +18,17 @@ # along with weboob. If not, see . +from __future__ import unicode_literals + from datetime import datetime, timedelta, date from functools import wraps -from weboob.exceptions import BrowserIncorrectPassword -from weboob.browser import LoginBrowser, URL, need_login, StatesMixin +from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable +from weboob.browser.browsers import LoginBrowser, URL, need_login, StatesMixin from weboob.browser.exceptions import ServerError -from weboob.browser.pages import FormNotFound from weboob.capabilities.base import NotAvailable -from weboob.capabilities.bank import Account, Investment, AddRecipientError, AddRecipientStep, Recipient +from .compat.weboob_capabilities_bank import Account, AddRecipientBankError, AddRecipientStep, Recipient +from .compat.weboob_tools_capabilities_bank_investments import create_french_liquidity from weboob.tools.compat import basestring, urlsplit, parse_qsl, unicode from weboob.tools.value import Value @@ -36,7 +38,7 @@ HomePage, LoansPage, TransferPage, AddRecipientPage, \ RecipientPage, RecipConfirmPage, SmsPage, RecipRecapPage, \ LoansProPage, Form2Page, DocumentsPage, ClientPage, SendTokenPage, \ - CaliePage, ProfilePage, DepositPage + CaliePage, ProfilePage, DepositPage, AVHistoryPage, AVInvestmentsPage __all__ = ['LCLBrowser', 'LCLProBrowser', 'ELCLBrowser'] @@ -84,12 +86,17 @@ class LCLBrowser(LoginBrowser, StatesMixin): form2 = URL(r'/outil/UWVI/Routage/', Form2Page) send_token = URL('/outil/UWVI/AssuranceVie/envoyerJeton', SendTokenPage) - calie = URL('https://www.my-calie.fr/FO.HoldersWebSite/Disclaimer/Disclaimer.aspx.*', CaliePage) + calie = URL('https://www.my-calie.fr/FO.HoldersWebSite/Disclaimer/Disclaimer.aspx.*', + 'https://www.my-calie.fr/FO.HoldersWebSite/Contract/ContractDetails.aspx.*', + 'https://www.my-calie.fr/FO.HoldersWebSite/Contract/ContractOperations.aspx.*', CaliePage) assurancevie = URL('/outil/UWVI/AssuranceVie/accesSynthese', '/outil/UWVI/AssuranceVie/accesDetail.*', AVPage) - avdetail = URL('https://assurance-vie-et-prevoyance.secure.lcl.fr.*', AVDetailPage) + + avdetail = URL('https://assurance-vie-et-prevoyance.secure.lcl.fr/consultation/epargne', AVDetailPage) + av_history = URL('https://assurance-vie-et-prevoyance.secure.lcl.fr/rest/assurance/historique', AVHistoryPage) + av_investments = URL('https://assurance-vie-et-prevoyance.secure.lcl.fr/rest/detailEpargne/contrat', AVInvestmentsPage) loans = URL('/outil/UWCR/SynthesePar/', LoansPage) loans_pro = URL('/outil/UWCR/SynthesePro/', LoansProPage) @@ -112,6 +119,8 @@ class LCLBrowser(LoginBrowser, StatesMixin): __states__ = ('contracts', 'current_contract',) + IDENTIFIANT_ROUTING = 'CLI' + def __init__(self, *args, **kwargs): super(LCLBrowser, self).__init__(*args, **kwargs) self.accounts_list = None @@ -294,31 +303,44 @@ def get_history(self, account): self.location(account._link_id) except ServerError: return + if self.login.is_here(): + # Website crashed and we are disconnected. + raise BrowserUnavailable() for tr in self.page.get_operations(): yield tr for tr in self.get_cb_operations(account, 1): yield tr - elif account.type == Account.TYPE_LIFE_INSURANCE and account._form: + + elif account.type == Account.TYPE_LIFE_INSURANCE: + if not account._form: + self.logger.warning('This account is limited, there is no available history.') + return + self.assurancevie.stay_or_go() - account._form.submit() + # The website often returns an error so we try again: + # "L’accès au service est momentanément indisponible." + try: + account._form.submit() + except BrowserUnavailable: + self.logger.warning("Service unavailable, we submit the form again.") + self.assurancevie.stay_or_go() + account._form.submit() + if self.calie.is_here(): - # come back to syntese + # Get back to Synthèse self.assurancevie.go() return - - # certain users will get a message : "Ne détenant pas de compte dépôt + # Some users will get a message : "Ne détenant pas de compte dépôt # chez LCL, l'accès à ce service vous est indisponible." if self.form2.is_here() and self.page.assurancevie_hist_not_available(): return - try: - self.page.get_details(account, "OHIPU") - except FormNotFound: - assert self.page.is_restricted() - self.logger.warning('restricted access to account %s', account) - else: - for tr in self.page.iter_history(): - yield tr + self.avdetail.go() + self.av_history.go() + for tr in self.page.iter_history(): + yield tr + + self.avdetail.go() self.page.come_back() @go_contract @@ -351,34 +373,46 @@ def get_cb_operations(self, account, month=0): @go_contract @need_login def get_investment(self, account): - if account.type == Account.TYPE_LIFE_INSURANCE and account._form: + if account.type == Account.TYPE_LIFE_INSURANCE: + if not account._form: + self.logger.warning('This account is limited, there is no available investment.') + return + self.assurancevie.stay_or_go() - account._form.submit() + # The website often returns an error so we try again: + # "L’accès au service est momentanément indisponible." + try: + account._form.submit() + except BrowserUnavailable: + self.logger.warning("Service unavailable, we submit the form again.") + self.assurancevie.stay_or_go() + account._form.submit() if self.calie.is_here(): - # come back to syntese + # Get back to Synthèse self.assurancevie.go() return + # Some users will get a message : "Ne détenant pas de compte dépôt + # chez LCL, l'accès à ce service vous est indisponible." + if self.form2.is_here() and self.page.assurancevie_hist_not_available(): + return + + self.avdetail.go() + self.av_investments.go() + + for inv in self.page.iter_investment(): + yield inv + + self.avdetail.go() + self.page.come_back() - if self.page.is_restricted(): - self.logger.warning('restricted access to account %s', account) - else: - for inv in self.page.iter_investment(): - yield inv - if self.avdetail.is_here(): - self.page.come_back() elif hasattr(account, '_market_link') and account._market_link: self.connexion_bourse() for inv in self.location(account._market_link).page.iter_investment(): yield inv self.deconnexion_bourse() elif account.id in self.get_bourse_accounts_ids(): - inv = Investment() - inv.id = account.id - inv.code = 'XX-Liquidity' - inv.label = "Liquidités" - inv.valuation = account.balance - yield inv + yield create_french_liquidity(account.balance) def locate_browser(self, state): if state['url'] == 'https://particuliers.secure.lcl.fr/outil/UWBE/Creation/creationConfirmation': @@ -390,7 +424,7 @@ def locate_browser(self, state): def send_code(self, recipient, **params): res = self.open('/outil/UWBE/Otp/getValidationCodeOtp?codeOtp=%s' % params['code']) if res.text == 'false': - raise AddRecipientError(message='Mauvais code sms.') + raise AddRecipientBankError(message='Mauvais code sms.') self.recip_recap.go().check_values(recipient.iban, recipient.label) return self.get_recipient_object(recipient.iban, recipient.label) @@ -412,7 +446,7 @@ def new_recipient(self, recipient, **params): return self.send_code(recipient, **params) if recipient.iban[:2] not in ('FR', 'MC'): - raise AddRecipientError(message=u"LCL n'accepte que les iban commençant par MC ou FR.") + raise AddRecipientBankError(message=u"LCL n'accepte que les iban commençant par MC ou FR.") for _ in range(2): self.add_recip.go() @@ -420,13 +454,12 @@ def new_recipient(self, recipient, **params): break if self.no_perm.is_here() and self.page.get_error_msg(): - raise AddRecipientError(message=self.page.get_error_msg()) - elif not self.add_recip.is_here(): - raise AddRecipientError('Navigation failed: not on add_recip.') + raise AddRecipientBankError(message=self.page.get_error_msg()) + + assert self.add_recip.is_here(), 'Navigation failed: not on add_recip' self.page.validate(recipient.iban, recipient.label) - if not self.recip_confirm.is_here(): - raise AddRecipientError('Navigation failed: not on recip_confirm.') + assert self.recip_confirm.is_here(), 'Navigation failed: not on recip_confirm' self.page.check_values(recipient.iban, recipient.label) # Send sms to user. diff --git a/modules/lcl/compat/weboob_capabilities_bank.py b/modules/lcl/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/lcl/compat/weboob_capabilities_bank.py +++ b/modules/lcl/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/lcl/compat/weboob_capabilities_bill.py b/modules/lcl/compat/weboob_capabilities_bill.py new file mode 100644 index 0000000000..09a2fd7504 --- /dev/null +++ b/modules/lcl/compat/weboob_capabilities_bill.py @@ -0,0 +1,259 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012 Romain Bignon, Florent Fourcot +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.capabilities.base import BaseObject, StringField, DecimalField, BoolField, UserError, Currency, Field +from weboob.capabilities.date import DateField +from weboob.capabilities.collection import CapCollection + + +__all__ = ['SubscriptionNotFound', 'DocumentNotFound', 'DocumentTypes', 'Detail', 'Document', 'Bill', 'Subscription', 'CapDocument'] + + +from weboob.capabilities.bill import SubscriptionNotFound as _SubscriptionNotFound +class SubscriptionNotFound(_SubscriptionNotFound): + """ + Raised when a subscription is not found. + """ + + def __init__(self, msg='Subscription not found'): + super(SubscriptionNotFound, self).__init__(msg) + + +from weboob.capabilities.bill import DocumentNotFound as _DocumentNotFound +class DocumentNotFound(_DocumentNotFound): + """ + Raised when a document is not found. + """ + + def __init__(self, msg='Document not found'): + super(DocumentNotFound, self).__init__(msg) + + +class DocumentTypes(object): + RIB = u'RIB' + STATEMENT = u'statement' + CONTRACT = u'contract' + NOTICE = u'notice' + REPORT = u'report' + BILL = u'bill' + OTHER = u'other' + + +from weboob.capabilities.bill import Detail as _Detail +class Detail(_Detail): + """ + Detail of a subscription + """ + label = StringField('label of the detail line') + infos = StringField('information') + datetime = DateField('date information') + price = DecimalField('Total price, taxes included') + vat = DecimalField('Value added Tax') + currency = StringField('Currency', default=None) + quantity = DecimalField('Number of units consumed') + unit = StringField('Unit of the consumption') + + +from weboob.capabilities.bill import Document as _Document +class Document(_Document): + """ + Document. + """ + date = DateField('The day the document has been sent to the subscriber') + format = StringField('file format of the document') + label = StringField('label of document') + type = StringField('type of document') + transactions = Field('List of transaction ID related to the document', list, default=[]) + + +from weboob.capabilities.bill import Bill as _Bill +class Bill(_Bill): + """ + Bill. + """ + price = DecimalField('Price to pay') + currency = StringField('Currency', default=None) + vat = DecimalField('VAT included in the price') + duedate = DateField('The day the bill must be paid') + startdate = DateField('The first day the bill applies to') + finishdate = DateField('The last day the bill applies to') + income = BoolField('Boolean to set if bill is income or invoice', default=False) + + def __init__(self, *args, **kwargs): + super(Bill, self).__init__(*args, **kwargs) + self.type = u'bill' + + +from weboob.capabilities.bill import Subscription as _Subscription +class Subscription(_Subscription): + """ + Subscription to a service. + """ + label = StringField('label of subscription') + subscriber = StringField('Subscriber name or identifier (for companies)') + validity = DateField('End validity date of the subscription (if any)') + renewdate = DateField('Reset date of consumption, for time based suscription (monthly, yearly, etc)') + + +from weboob.capabilities.bill import CapDocument as _CapDocument +class CapDocument(_CapDocument): + accepted_doc_types = () + """ + Tuple of document types handled by the module (:class:`DocumentTypes`) + """ + + def iter_subscription(self): + """ + Iter subscriptions. + + :rtype: iter[:class:`Subscription`] + """ + raise NotImplementedError() + + def get_subscription(self, _id): + """ + Get a subscription. + + :param _id: ID of subscription + :rtype: :class:`Subscription` + :raises: :class:`SubscriptionNotFound` + """ + raise NotImplementedError() + + def iter_documents_history(self, subscription): + """ + Iter history of a subscription. + + :param subscription: subscription to get history + :type subscription: :class:`Subscription` + :rtype: iter[:class:`Detail`] + """ + raise NotImplementedError() + + def iter_bills_history(self, subscription): + """ + Iter history of a subscription. + + :param subscription: subscription to get history + :type subscription: :class:`Subscription` + :rtype: iter[:class:`Detail`] + """ + return self.iter_documents_history(subscription) + + def get_document(self, id): + """ + Get a document. + + :param id: ID of document + :rtype: :class:`Document` + :raises: :class:`DocumentNotFound` + """ + raise NotImplementedError() + + def download_document(self, id): + """ + Download a document. + + :param id: ID of document + :rtype: bytes + :raises: :class:`DocumentNotFound` + """ + raise NotImplementedError() + + def download_document_pdf(self, id): + """ + Download a document, convert it to PDF if it isn't the document format. + + :param id: ID of document + :rtype: bytes + :raises: :class:`DocumentNotFound` + """ + if not isinstance(id, Document): + id = self.get_document(id) + + if id.format == 'pdf': + return self.download_document(id) + else: + raise NotImplementedError() + + def iter_documents(self, subscription): + """ + Iter documents. + + :param subscription: subscription to get documents + :type subscription: :class:`Subscription` + :rtype: iter[:class:`Document`] + """ + raise NotImplementedError() + + def iter_documents_by_types(self, subscription, accepted_types): + """ + Iter documents with specific types. + + :param subscription: subscription to get documents + :type subscription: :class:`Subscription` + :param accepted_types: list of document types that should be returned + :type accepted_types: [:class:`DocumentTypes`] + :rtype: iter[:class:`Document`] + """ + accepted_types = frozenset(accepted_types) + for document in self.iter_documents(subscription): + if document.type in accepted_types: + yield document + + def iter_bills(self, subscription): + """ + Iter bills. + + :param subscription: subscription to get bills + :type subscription: :class:`Subscription` + :rtype: iter[:class:`Bill`] + """ + documents = self.iter_documents(subscription) + return [doc for doc in documents if doc.type == "bill"] + + def get_details(self, subscription): + """ + Get details of a subscription. + + :param subscription: subscription to get details + :type subscription: :class:`Subscription` + :rtype: iter[:class:`Detail`] + """ + raise NotImplementedError() + + def get_balance(self, subscription): + """ + Get the balance of a subscription. + + :param subscription: subscription to get balance + :type subscription: :class:`Subscription` + :rtype: class:`Detail` + """ + raise NotImplementedError() + + def iter_resources(self, objs, split_path): + """ + Iter resources. Will return :func:`iter_subscriptions`. + """ + if Subscription in objs: + self._restrict_level(split_path) + return self.iter_subscription() + diff --git a/modules/lcl/compat/weboob_tools_capabilities_bank_investments.py b/modules/lcl/compat/weboob_tools_capabilities_bank_investments.py new file mode 100644 index 0000000000..2b68ff9e00 --- /dev/null +++ b/modules/lcl/compat/weboob_tools_capabilities_bank_investments.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Jonathan Schmidt +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.tools.compat import basestring +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.bank import Investment + +def is_isin_valid(isin): + """ + Méthode générale + Table de conversion des lettres en chiffres + A=10 B=11 C=12 D=13 E=14 F=15 G=16 H=17 I=18 + J=19 K=20 L=21 M=22 N=23 O=24 P=25 Q=26 R=27 + S=28 T=29 U=30 V=31 W=32 X=33 Y=34 Z=35 + + 1 - Mettre de côté la clé, qui servira de référence à la fin de la vérification. + 2 - Convertir toutes les lettres en nombres via la table de conversion ci-contre. Si le nombre obtenu est supérieur ou égal à 10, prendre les deux chiffres du nombre séparément (exemple : 27 devient 2 et 7). + 3 - Pour chaque chiffre, multiplier sa valeur par deux si sa position est impaire en partant de la droite. Si le nombre obtenu est supérieur ou égal à 10, garder les deux chiffres du nombre séparément (exemple : 14 devient 1 et 4). + 4 - Faire la somme de tous les chiffres. + 5 - Soustraire cette somme de la dizaine supérieure ou égale la plus proche (exemples : si la somme vaut 22, la dizaine « supérieure ou égale » est 30, et la clé vaut donc 8 ; si la somme vaut 30, la dizaine « supérieure ou égale » est 30, et la clé vaut 0 ; si la somme vaut 31, la dizaine « supérieure ou égale » est 40, et la clé vaut 9). + 6 - Comparer la valeur obtenue à la clé mise initialement de côté. + + Étapes 1 et 2 : + F R 0 0 0 3 5 0 0 0 0 (+ 8 : clé) + 15 27 0 0 0 3 5 0 0 0 0 + + Étape 3 : le traitement se fait sur des chiffres + 1 5 2 7 0 0 0 3 5 0 0 0 0 + I P I P I P I P I P I P I : position en partant de la droite (P = Pair, I = Impair) + 2 1 2 1 2 1 2 1 2 1 2 1 2 : coefficient multiplicateur + + 2 5 4 7 0 0 0 3 10 0 0 0 0 : résultat + + Étape 4 : + 2 + 5 + 4 + 7 + 0 + 0 + 0 + 3 + (1 + 0)+ 0 + 0 + 0 + 0 = 22 + + Étapes 5 et 6 : 30 - 22 = 8 (valeur de la clé) + """ + + if not isinstance(isin, basestring): + return False + if not re.match(r'^[A-Z]{2}[A-Z0-9]{9}\d$', isin): + return False + + isin_in_digits = ''.join(str(ord(x) - ord('A') + 10) if not x.isdigit() else x for x in isin[:-1]) + key = isin[-1:] + result = '' + for k, val in enumerate(isin_in_digits[::-1], start=1): + if k % 2 == 0: + result = ''.join((result, val)) + else: + result = ''.join((result, str(int(val)*2))) + return str(sum(int(x) for x in result) + int(key))[-1] == '0' + + +def create_french_liquidity(valuation): + """ + Automatically fills a liquidity investment with label, code and code_type. + """ + liquidity = Investment() + liquidity.label = "Liquidités" + liquidity.code = "XX-liquidity" + liquidity.code_type = NotAvailable + liquidity.valuation = valuation + return liquidity + diff --git a/modules/lcl/module.py b/modules/lcl/module.py index 684b9ee0cd..12590172bc 100644 --- a/modules/lcl/module.py +++ b/modules/lcl/module.py @@ -24,8 +24,8 @@ from .compat.weboob_capabilities_bank import CapBankWealth, CapBankTransferAddRecipient, AccountNotFound, \ RecipientNotFound, TransferError, Account -from weboob.capabilities.bill import CapDocument, Subscription, SubscriptionNotFound, \ - Document, DocumentNotFound +from .compat.weboob_capabilities_bill import CapDocument, Subscription, SubscriptionNotFound, \ + Document, DocumentNotFound, DocumentTypes from weboob.capabilities.contact import CapContact from weboob.capabilities.profile import CapProfile from weboob.tools.backend import Module, BackendConfig @@ -70,6 +70,8 @@ class LCLModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapContact, 'elcl': 'e.LCL'})) BROWSER = LCLBrowser + accepted_doc_types = (DocumentTypes.STATEMENT, DocumentTypes.NOTICE, DocumentTypes.REPORT, DocumentTypes.OTHER) + def create_default_browser(self): # assume all `website` option choices are defined here browsers = {'par': LCLBrowser, diff --git a/modules/lcl/pages.py b/modules/lcl/pages.py index f919aac9d2..637a8f2bcf 100644 --- a/modules/lcl/pages.py +++ b/modules/lcl/pages.py @@ -31,20 +31,19 @@ from weboob.capabilities.base import find_object, Currency from weboob.capabilities.bank import ( Account, Investment, Recipient, TransferError, TransferBankError, Transfer, - AddRecipientError, ) -from weboob.capabilities.bill import Document, Subscription +from .compat.weboob_capabilities_bill import Document, Subscription, DocumentTypes from .compat.weboob_capabilities_profile import Person, ProfileMissing from weboob.capabilities.contact import Advisor -from weboob.browser.elements import method, ListElement, TableElement, ItemElement, SkipItem +from weboob.browser.elements import method, ListElement, TableElement, ItemElement, SkipItem, DictElement from weboob.exceptions import ParseError from weboob.browser.exceptions import ServerError -from weboob.browser.pages import LoggedPage, HTMLPage, FormNotFound, pagination +from weboob.browser.pages import LoggedPage, HTMLPage, JsonPage, FormNotFound, pagination from weboob.browser.filters.html import Attr, Link, TableCell, AttributeNotFound from weboob.browser.filters.standard import ( - CleanText, Field, Regexp, Format, Date, CleanDecimal, Map, AsyncLoad, Async, Env, - Eval, Slugify, + CleanText, Field, Regexp, Format, Date, CleanDecimal, Map, AsyncLoad, Async, Env, Slugify, BrowserURL, ) +from weboob.browser.filters.json import Dict from weboob.exceptions import BrowserUnavailable, BrowserIncorrectPassword from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.tools.captcha.virtkeyboard import MappedVirtKeyboard, VirtKeyboardError @@ -252,6 +251,7 @@ def condition(self): '007': Account.TYPE_SAVINGS, '012': Account.TYPE_SAVINGS, '023': Account.TYPE_CHECKING, + '036': Account.TYPE_SAVINGS, '046': Account.TYPE_SAVINGS, '047': Account.TYPE_SAVINGS, '049': Account.TYPE_SAVINGS, @@ -433,7 +433,6 @@ def obj_type(self): match = pattern.match(type) if match: return _type - break return Transaction.TYPE_UNKNOWN def condition(self): @@ -455,6 +454,9 @@ def validate(self, obj): else: obj.label = '%s %s' % (obj.raw, raw) obj.raw = '%s %s' % (obj.raw, raw) + m = re.search(r'\d+,\d+COM (\d+,\d+)', raw) + if m: + obj.commission = -CleanDecimal(replace_dots=True).filter(m.group(1)) elif not obj.raw: # Empty transaction label obj.raw = obj.label = Async('details', CleanText(u'//td[contains(text(), "Nature de l\'opération")]/following-sibling::*[1]'))(self) @@ -718,7 +720,9 @@ def obj_id(self): self.page.browser.open(self.page.url) # redirection to lifeinsurances accounts and comeback on Lcl original website page = self.obj__form().submit().page - account_id = page.get_account_id() + # Getting the account details from the JSON containing the account information: + details_page = self.page.browser.open(BrowserURL('av_investments')(self)).page + account_id = Dict('situationAdministrativeEpargne/idcntcar')(details_page.doc) page.come_back() return account_id @@ -768,22 +772,13 @@ def assurancevie_hist_not_available(self): msg = "Ne détenant pas de compte dépôt chez LCL, l'accès à ce service vous est indisponible" return msg in CleanText('//div[@id="attTxt"]')(self.doc) - def is_restricted(self): - return self.assurancevie_hist_not_available() - def on_load(self): if self.assurancevie_hist_not_available(): return error = CleanText('//div[@id="attTxt"]/text()[1]')(self.doc) if "L’accès au service est momentanément indisponible" in error: raise BrowserUnavailable(error) - - form = self.get_form(name="formulaire") - cName = self.get_from_js('.cName.value = "', '";') - if cName: - form['cName'] = cName - form['cValue'] = self.get_from_js('.cValue.value = "', '";') - form['cMaxAge'] = '-1' + form = self.get_form() return form.submit() @@ -795,22 +790,6 @@ def on_load(self): class AVDetailPage(LoggedPage, LCLBasePage): - def get_account_id(self): - return Regexp(CleanText('//div[@class="libelletitrepage"]/h1'), r"N° (\w+)")(self.doc) - - def sub(self): - form = self.get_form(name="formulaire") - cName = self.get_from_js('.cName.value = "', '";') - if cName: - form['cName'] = cName - form['cValue'] = self.get_from_js('.cValue.value = "', '";') - form['cMaxAge'] = '-1' - return form.submit() - - def submit_simple(self): - form = self.get_form(name="formulaire") - return form.submit() - def come_back(self): session = self.get_from_js('idSessionSag = "', '"') params = {} @@ -822,78 +801,70 @@ def come_back(self): params['stbzn'] = 'bnc' return self.browser.location('https://assurance-vie-et-prevoyance.secure.lcl.fr/filiale/entreeBam', params=params) - def get_details(self, account, act=None): - form = self.get_form(id="frm_fwk") - form.submit() - if act is not None: - self.browser.location("entreeBam?sessionSAG=%s&act=%s" % (form['sessionSAG'], act)) - - def is_restricted(self): - msg = CleanText('//div[has-class("titre_libelle_erreur")]')(self.doc) - return msg in ( - "Pour ce type d’opération, nous vous conseillons de vous rapprocher de votre conseiller afin de bénéficier du conseil personnalisé le mieux adapté à vos objectifs.", - "Vous n’avez pas accès à l’espace assurances de personnes.", - ) +class AVHistoryPage(LoggedPage, JsonPage): @method - class iter_investment(TableElement): - head_xpath = '//table[@class="table"][ends-with(@id,"CD_UCT")]/thead//th' - item_xpath = '//table[@class="table"][ends-with(@id,"CD_UCT")]/tbody/tr' - - col_label = 'Le(s) support(s) financier(s) de votre contrat' - col_unitvalue = 'Valeur de la part' - col_vdate = 'En date du :' - col_quantity = 'Nombre de parts' - col_valuation = 'Total' - col_portfolio_share = u'Répartition' + class iter_history(DictElement): + item_xpath = 'listeOperations' class item(ItemElement): - klass = Investment + klass = Transaction - obj_label = CleanText(TableCell('label')) + obj_label = CleanText(Dict('lcope')) + obj_amount = CleanDecimal(Dict('mtope')) + obj_type = Transaction.TYPE_BANK + obj_investments = NotAvailable - def obj_code(self): - td = TableCell('label')(self)[0] - return Attr('.//a', 'id', default=NotAvailable)(td) + # The 'idope' key contains a string such as "70_ABC666ABC 2018-03-182018-03-16-20.55.27.960852" + # 70= N° transaction, 6660666= N° account, 2018-03-18= date and 2018-03-16=rdate. + # We thus use "70_ABC666ABC" for the transaction ID. - obj_code_type = Investment.CODE_TYPE_ISIN + obj_id = Regexp(CleanText(Dict('idope')), '(\d+_[\dA-Z]+)') - def obj_quantity(self): - return MyDecimal(TableCell('quantity'))(self) or NotAvailable + def obj__dates(self): + raw = CleanText(Dict('idope'))(self) + m = re.findall('\d{4}-\d{2}-\d{2}', raw) + # We must verify that the two dates are correctly fetched + assert len(m) == 2 + return m - def obj_unitvalue(self): - return MyDecimal(TableCell('unitvalue'))(self) or NotAvailable + def obj_date(self): + return Date().filter(Field('_dates')(self)[0]) - obj_valuation = MyDecimal(TableCell('valuation')) - obj_portfolio_share = Eval(lambda x: x / 100, CleanDecimal(TableCell('portfolio_share'), replace_dots=True)) + def obj_rdate(self): + return Date().filter(Field('_dates')(self)[1]) - obj_vdate = Date(CleanText(TableCell('vdate')), dayfirst=True, default=NotAvailable) - @pagination +class AVInvestmentsPage(LoggedPage, JsonPage): @method - class iter_history(TableElement): - item_xpath = '//table[@class="table"]/tbody/tr' - head_xpath = '//table[@class="table"]/thead/tr/th' - - col_date = 'Date d\'effet' - col_label = u'Opération(s)' - col_amount = 'Montant' - - def next_page(self): - if Link('//a[@class="pictoSuivant"]', default=None)(self): - form = self.page.get_form(id="frm_fwk") - form['fwkaction'] = "precSuivDet" - form['fwkcodeaction'] = "Executer" - form['ACTION_CHOISIE'] = "suivant" - return requests.Request("POST", form.url, data=dict(form)) + class iter_investment(DictElement): + item_xpath = 'listeSupports/support' class item(ItemElement): - klass = Transaction + klass = Investment - obj_label = CleanText(TableCell('label')) - obj_type = Transaction.TYPE_BANK - obj_date = Date(CleanText(TableCell('date')), dayfirst=True) - obj_amount = MyDecimal(TableCell('amount')) + obj_label = CleanText(Dict('lcspt')) + obj_valuation = CleanDecimal(Dict('mtvalspt')) + obj_code = CleanText(Dict('cdsptisn'), default=NotAvailable) + obj_unitvalue = CleanDecimal(Dict('mtliqpaaspt'), default=NotAvailable) + obj_quantity = CleanDecimal(Dict('qtpaaspt'), default=NotAvailable) + obj_diff = CleanDecimal(Dict('mtpmvspt'), default=NotAvailable) + + def obj_portfolio_share(self): + ptf = CleanDecimal(Dict('txrpaspt'), default=NotAvailable)(self) + ptf /= 100 + return ptf + + def obj_vdate(self): + time = Dict('dvspt')(self) + if not time: + return NotAvailable + return datetime.fromtimestamp(time//1000) + + def obj_code_type(self): + if is_isin_valid(Field('code')(self)): + return Investment.CODE_TYPE_ISIN + return NotAvailable class RibPage(LoggedPage, LCLBasePage): @@ -1074,14 +1045,8 @@ def validate(self, iban, label): class CheckValuesPage(LoggedPage, HTMLPage): def check_values(self, iban, label): values = CleanText('//form[contains(@action, "/outil")]')(self.doc) - try: - assert iban in values - except AssertionError: - raise AddRecipientError('iban not found in values') - try: - assert label in values - except AssertionError: - raise AddRecipientError('recipient label not found in values') + assert iban in values, 'Iban (%s) not found in values: %s' % (iban, values) + assert label in values, 'Recipient label (%s) not found in values: %s' % (label, values) class DocumentsPage(LoggedPage, HTMLPage): @@ -1098,7 +1063,7 @@ def do_search_request(self): @method class get_list(TableElement): head_xpath = '//table[@class="dematTab"]/thead/tr/th' - item_xpath = u'//table[@class="dematTab"]/tbody/tr[./td[@class="dematTab-firstCell" and contains(., "Relevés")]]' + item_xpath = u'//table[@class="dematTab"]/tbody/tr[./td[@class="dematTab-firstCell"]]' col_label = 'Nature de document' col_id = 'Type de document' @@ -1116,6 +1081,16 @@ class item(ItemElement): def obj_url(self): return Link(TableCell('url')(self)[0].xpath('./a'))(self) + def obj_type(self): + if 'Relevé' in Field('label')(self): + return DocumentTypes.STATEMENT + elif 'Bourse' in Field('label')(self): + return DocumentTypes.REPORT + elif ('information' in Field('id')(self)) or ('avis' in Field('id')(self)): + return DocumentTypes.NOTICE + else: + return DocumentTypes.OTHER + class ClientPage(LoggedPage, HTMLPage): @method diff --git a/modules/linebourse/api/__init__.py b/modules/linebourse/api/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/modules/linebourse/api/pages.py b/modules/linebourse/api/pages.py new file mode 100644 index 0000000000..15a616a0ce --- /dev/null +++ b/modules/linebourse/api/pages.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Fong Ngo +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +from weboob.browser.elements import method, DictElement, ItemElement +from weboob.browser.filters.json import Dict +from weboob.browser.filters.standard import ( + CleanText, Date, CleanDecimal, Eval, Field, Env, Regexp, +) +from weboob.browser.pages import JsonPage, HTMLPage, LoggedPage +from weboob.capabilities.bank import Investment +from weboob.capabilities.base import NotAvailable +from weboob.exceptions import ActionNeeded +from weboob.tools.capabilities.bank.investments import is_isin_valid + + +class AccountPage(LoggedPage, JsonPage): + def get_ncontrat(self): + return self.doc['identifiantContratCrypte'] + + +class PortfolioPage(LoggedPage, JsonPage): + def get_valuation_diff(self): + return CleanDecimal(Dict('totalPlv'))(self.doc) # Plv = plus-value + + def get_date(self): + return Date(Regexp(Dict('dateValo'), r'(\d{2})(\d{2})(\d{2})', '\\3\\2\\1'), dayfirst=True)(self.doc) + + @method + class iter_investments(DictElement): + item_xpath = 'listeSegmentation/*' # all categories are fetched: obligations, actions, OPC + + class item(ItemElement): + klass = Investment + + obj_label = Dict('libval') + obj_code = Dict('codval') + obj_code_type = Eval( + lambda x: Investment.CODE_TYPE_ISIN if is_isin_valid(x) else NotAvailable, + Field('code') + ) + obj_quantity = CleanDecimal(Dict('qttit')) + obj_unitprice = CleanDecimal(Dict('pam')) + obj_unitvalue = CleanDecimal(Dict('crs')) + obj_valuation = CleanDecimal(Dict('mnt')) + obj_vdate = Env('date') + obj_portfolio_share = Eval(lambda x: x / 100, CleanDecimal(Dict('pourcentageActif'))) + + def parse(self, el): + symbol = Dict('signePlv')(self) + assert symbol in ('+', '-'), 'should be either positive or negative' + self.env['sign'] = 1 if symbol == '+' else -1 + + def obj_diff(self): + return CleanDecimal(Dict('plv'), sign=lambda x: Env('sign')(self))(self) + + def obj_diff_percent(self): + return CleanDecimal(Dict('plvPourcentage'), sign=lambda x: Env('sign')(self))(self) + + +class ConfigurationPage(LoggedPage, JsonPage): + def is_first_connexion(self): + return self.doc['premiereConnexion'] # either True or False + + def get_contract_number(self): + return self.doc['idCompteActif'] + + +class NewWebsiteFirstConnectionPage(LoggedPage, JsonPage): + def build_doc(self, content): + content = JsonPage.build_doc(self, content) + if 'data' in content: + # The value contains HTML + # Must be encoded into str because HTMLPage.build_doc() uses BytesIO + # which expects bytes + html_page = HTMLPage(self.browser, self.response) + return html_page.build_doc(content['data'].encode(self.encoding)) + return content + + def has_first_connection_cgu(self): + # New Espace bourse: user is asked to read some documents during first connection + message = CleanText('//p[contains(text(), "prendre connaissance")]')(self.doc) + if message: + raise ActionNeeded(message) + + +class HistoryAPIPage(LoggedPage, JsonPage): + def has_history(self): + return bool(self.doc['data']['nbTotalValeurs']) diff --git a/modules/linebourse/browser.py b/modules/linebourse/browser.py index 5a341b8b11..3f44b30ae0 100644 --- a/modules/linebourse/browser.py +++ b/modules/linebourse/browser.py @@ -19,15 +19,27 @@ from __future__ import unicode_literals +import time +import re + from weboob.browser import LoginBrowser, URL from weboob.exceptions import BrowserUnavailable from .pages import ( MessagePage, InvestmentPage, HistoryPage, BrokenPage, - MainPage, FirstConnectionPage + MainPage, FirstConnectionPage, +) + +from .api.pages import ( + PortfolioPage, NewWebsiteFirstConnectionPage, ConfigurationPage, + HistoryAPIPage, ) +def get_timestamp(): + return '{}'.format(int(time.time() * 1000)) # in milliseconds + + class LinebourseBrowser(LoginBrowser): BASEURL = 'https://www.linebourse.fr' @@ -36,8 +48,7 @@ class LinebourseBrowser(LoginBrowser): invest = URL(r'/Portefeuille$', r'/Portefeuille\?compte=(?P[^&]+)', InvestmentPage) message = URL(r'/DetailMessage.*', MessagePage) history = URL(r'/HistoriqueOperations', - r'/HistoriqueOperations\?compte=(?P[^&]+)&devise=EUR&modeTri=7&sensTri=-1&periode=(?P\d+)', - HistoryPage) + r'/HistoriqueOperations\?compte=(?P[^&]+)&devise=EUR&modeTri=7&sensTri=-1&periode=(?P\d+)', HistoryPage) useless = URL(r'/ReroutageSJR', MessagePage) broken = URL(r'.*/timeout.html$', BrokenPage) @@ -84,3 +95,57 @@ def iter_history(self, account_id): self.history.go(id=self.page.get_compte(account_id), period=period) for tr in self.page.iter_history(): yield tr + + +class LinebourseAPIBrowser(LoginBrowser): + BASEURL = 'https://www.offrebourse.com' + + new_website_first = URL(r'/rest/premiereConnexion', NewWebsiteFirstConnectionPage) + config = URL(r'/rest/configuration', ConfigurationPage) + + # The API works with an encrypted id_contract that starts with 'CRY' + portfolio = URL(r'/rest/portefeuille/(?PCRY[\w\d]+)/vide/true/false', PortfolioPage) + history = URL(r'/rest/historiqueOperations/(?PCRY[\w\d]+)/0/7/1', HistoryAPIPage) # TODO: not sure if last 3 path levels can be hardcoded + + def __init__(self, baseurl, *args, **kwargs): + self.BASEURL = baseurl + self.id_contract = None # encrypted contract number used to browse between pages + + super(LinebourseAPIBrowser, self).__init__(username='', password='', *args, **kwargs) + + def go_portfolio(self): + self.config.go() + self.id_contract = self.page.get_contract_number() + return self.portfolio.go(id_contract=self.id_contract) + + def handle_cgu(self): + # the website uses an additional timestamp as query parameter + # for this GET request but it turns out it is not mandatory + self.config.go() + assert self.config.is_here() + + if self.page.is_first_connexion(): + self.new_website_first.go() + assert self.new_website_first.is_here() + + self.page.has_first_connection_cgu() + + def iter_investments(self): + self.go_portfolio() + assert self.portfolio.is_here() + date = self.page.get_date() + return self.page.iter_investments(date=date) + + def iter_history(self): + assert re.match(r'CRY[\w\d]+', self.id_contract) + self.history.go( + id_contract=self.id_contract, + params={'_': get_timestamp()}, # timestamp is necessary + ) + assert self.history.is_here() + + # Didn't find a connection with transactions + # TODO: implement corresponding pages + if self.page.has_history(): + assert False, 'please implement iter_history' + return [] diff --git a/modules/linebourse/pages.py b/modules/linebourse/pages.py index 02ec81fbbe..0587885140 100644 --- a/modules/linebourse/pages.py +++ b/modules/linebourse/pages.py @@ -131,7 +131,7 @@ def condition(self): obj_quantity = MyDecimal(TableCell('quantity'), default=NotAvailable) obj_unitvalue = MyDecimal(TableCell('unitvalue'), default=NotAvailable) obj_unitprice = MyDecimal(TableCell('unitprice'), default=NotAvailable) - obj_valuation = MyDecimal(TableCell('valuation')) + obj_valuation = MyDecimal(TableCell('valuation'), default=NotAvailable) obj_portfolio_share = Eval(lambda x: x / 100 if x else NotAvailable, MyDecimal(TableCell('portfolio_share'), default=NotAvailable)) obj_diff = MyDecimal(TableCell('diff', default=NotAvailable), default=NotAvailable) obj_code_type = Investment.CODE_TYPE_ISIN diff --git a/modules/marmiton/browser.py b/modules/marmiton/browser.py index 4857b6aba2..8c47440d11 100644 --- a/modules/marmiton/browser.py +++ b/modules/marmiton/browser.py @@ -27,7 +27,7 @@ class MarmitonBrowser(PagesBrowser): - BASEURL = 'http://www.marmiton.org' + BASEURL = 'https://www.marmiton.org' search = URL('/recettes/recherche.aspx\?aqt=(?P.*)&start=(?P\d*)', '/recettes/recherche.aspx\?aqt=.*', ResultsPage) diff --git a/modules/marmiton/pages.py b/modules/marmiton/pages.py index beb6d35e69..944494c7ad 100644 --- a/modules/marmiton/pages.py +++ b/modules/marmiton/pages.py @@ -33,7 +33,7 @@ class ResultsPage(HTMLPage): @pagination @method class iter_recipes(ListElement): - item_xpath = "//div[@class='recipe-results ']/a" + item_xpath = "//a[@class='recipe-card-link']" def next_page(self): return CleanText('//nav/ul/li[@class="next-page"]/a/@href', default="")(self) diff --git a/modules/materielnet/browser.py b/modules/materielnet/browser.py index 01dc764580..c81e142c5a 100644 --- a/modules/materielnet/browser.py +++ b/modules/materielnet/browser.py @@ -21,25 +21,31 @@ from weboob.browser import LoginBrowser, URL, need_login from weboob.exceptions import BrowserIncorrectPassword -from .pages import LoginPage, CaptchaPage, ProfilPage, DocumentsPage +from .pages import LoginPage, CaptchaPage, ProfilPage, DocumentsPage, DocumentsDetailsPage class MaterielnetBrowser(LoginBrowser): - BASEURL = 'https://www.materiel.net' + BASEURL = 'https://secure.materiel.net' - login = URL('/pm/client/login.html', LoginPage) + login = URL(r'https://www.materiel.net/form/login', + r'/Login/PartialPublicLogin', LoginPage) captcha = URL('/pm/client/captcha.html', CaptchaPage) - profil = URL('/pm/client/compte.html', ProfilPage) - documents = URL('/pm/client/commande.html\?page=(?P.*)', - '/pm/client/commande.html', DocumentsPage) + profil = URL(r'/Account/InformationsSection', ProfilPage) + documents = URL(r'/Orders/PartialCompletedOrdersHeader', DocumentsPage) + document_details = URL(r'/Orders/PartialCompletedOrderContent', DocumentsDetailsPage) def do_login(self): self.login.go() - self.page.login(self.username, self.password) - if self.login.is_here() or self.captcha.is_here(): - raise BrowserIncorrectPassword(self.page.get_error()) + if self.captcha.is_here(): + BrowserIncorrectPassword() + + if self.login.is_here(): + error = self.page.get_error() + # when everything is good we land on this page + if error: + raise BrowserIncorrectPassword(error) @need_login def get_subscription_list(self): @@ -47,4 +53,7 @@ def get_subscription_list(self): @need_login def iter_documents(self, subscription): - return self.documents.stay_or_go(page=1).get_documents() + json_response = self.location('/Orders/CompletedOrdersPeriodSelection').json() + + for data in json_response: + return self.documents.go(data=data).get_documents() diff --git a/modules/materielnet/module.py b/modules/materielnet/module.py index 35bec35ec5..4d7d066b00 100644 --- a/modules/materielnet/module.py +++ b/modules/materielnet/module.py @@ -21,7 +21,7 @@ from weboob.capabilities.bill import CapDocument, Subscription, Document, SubscriptionNotFound, DocumentNotFound from weboob.capabilities.base import find_object, NotAvailable from weboob.tools.backend import Module, BackendConfig -from weboob.tools.value import ValueBackendPassword, Value +from weboob.tools.value import ValueBackendPassword from .browser import MaterielnetBrowser @@ -36,8 +36,8 @@ class MaterielnetModule(Module, CapDocument): EMAIL = 'elambert@budget-insight.com' LICENSE = 'AGPLv3+' VERSION = '1.3' - CONFIG = BackendConfig(Value('login', label='Identifiant'), - ValueBackendPassword('password', label='Mot de passe')) + CONFIG = BackendConfig(ValueBackendPassword('login', label='Email', regex='.+@.+'), + ValueBackendPassword('password', label='Mot de passe')) BROWSER = MaterielnetBrowser diff --git a/modules/materielnet/pages.py b/modules/materielnet/pages.py index 26b8b1e94f..ffe08a87e7 100644 --- a/modules/materielnet/pages.py +++ b/modules/materielnet/pages.py @@ -17,25 +17,40 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals import re -from weboob.browser.pages import HTMLPage, LoggedPage, pagination -from weboob.browser.filters.standard import CleanText, CleanDecimal, Env, Format, Date, Async, AsyncLoad -from weboob.browser.elements import ListElement, ItemElement, TableElement, method -from weboob.browser.filters.html import Attr, Link, TableCell +from weboob.browser.pages import HTMLPage, LoggedPage, PartialHTMLPage +from weboob.browser.filters.standard import CleanText, CleanDecimal, Env, Format, Date, Async, Filter, Regexp, Field +from weboob.browser.elements import ListElement, ItemElement, method +from weboob.browser.filters.html import Attr, Link from weboob.capabilities.bill import Bill, Subscription from weboob.capabilities.base import NotAvailable +from weboob.exceptions import BrowserIncorrectPassword -class LoginPage(HTMLPage): + +class LoginPage(PartialHTMLPage): def login(self, login, password): - form = self.get_form('//form[@class="login-form"]') - form['identifier'] = login - form['credentials'] = password + maxlength = Attr('//input[@id="Email"]', 'data-val-maxlength-max')(self.doc) + regex = Attr('//input[@id="Email"]', 'data-val-regex-pattern')(self.doc) + # their regex is: ^([\w\-+\.]+)@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.)|(([\w-]+\.)+))([a-zA-Z]{2,15}|[0-9]{1,3})(\]?)$ + # but it is not very good, we escape - inside [] to avoid bad character range Exception + regex = regex.replace('[\w-+\.]', '[\w\-+\.]') + + if len(login) > maxlength: # actually it's 60 char + raise BrowserIncorrectPassword(Attr('//input[@id="Email"]', 'data-val-maxlength')(self.doc)) + + if not re.match(regex, login): + raise BrowserIncorrectPassword(Attr('//input[@id="Email"]', 'data-val-regex')(self.doc)) + + form = self.get_form(id='loginForm') + form['Email'] = login + form['Password'] = password form.submit() def get_error(self): - return CleanText('//div[@class="ValidatorError"]')(self.doc) + return CleanText('//div[contains(@class, "error")]')(self.doc) class CaptchaPage(HTMLPage): @@ -49,7 +64,7 @@ class get_list(ListElement): class item(ItemElement): klass = Subscription - obj_subscriber = Format('%s %s', Attr('//input[@id="prenom"]', 'value'), Attr('//input[@id="nom"]', 'value')) + obj_subscriber = Format('%s %s', Attr('//input[@id="FirstName"]', 'value'), Attr('//input[@id="LastName"]', 'value')) obj_id = Env('subid') obj_label = obj_id @@ -57,38 +72,35 @@ def parse(self, el): self.env['subid'] = self.page.browser.username -class DocumentsPage(LoggedPage, HTMLPage): - @pagination +class MyAsyncLoad(Filter): + def __call__(self, item): + link = self.select(self.selector, item) + data = {'X-Requested-With': 'XMLHttpRequest'} + return item.page.browser.async_open(link, data=data) if link else None + + +class DocumentsPage(LoggedPage, PartialHTMLPage): @method - class get_documents(TableElement): - item_xpath = '//div[@id="ListCmd"]//table//tr[position() > 1]' - head_xpath = '//div[@id="ListCmd"]//table//tr//th' - - col_id = u'Référence' - col_date = u'Date' - col_price = u'Montant' - - def next_page(self): - m = re.search('([^*]+page=)([^*]+)', self.page.url) - if m: - page = int(m.group(2)) + 1 - if self.el.xpath('//a[contains(@href, "commande.html?page=' + str(page) + '")]'): - next_page = u"%s%s" % (m.group(1), page) - return next_page + class get_documents(ListElement): + item_xpath = '//div[@class="historic-table"]' class item(ItemElement): klass = Bill - load_details = Attr('./td/a', 'href') & AsyncLoad + load_details = Link('.//a') & MyAsyncLoad - obj_id = Format('%s_%s', Env('email'), CleanDecimal(TableCell('id'))) - obj_url = Async('details') & Link('//a[contains(@href, "facture")]', default=NotAvailable) - obj_date = Date(CleanText(TableCell('date'))) - obj_format = u"pdf" - obj_label = Async('details') & CleanText('//table/tr/td[@class="Prod"]') - obj_type = u"bill" - obj_price = CleanDecimal(TableCell('price'), replace_dots=True) - obj_currency = u'EUR' + obj_id = Format('%s_%s', Env('email'), Field('label')) + obj_url = Async('details') & Link('//a', default=NotAvailable) + obj_date = Date(CleanText('./div[contains(@class, "date")]'), dayfirst=True) + obj_format = 'pdf' + obj_label = Regexp(CleanText('./div[contains(@class, "ref")]'), r' (.*)') + obj_type = 'bill' + obj_price = CleanDecimal(CleanText('./div[contains(@class, "price")]'), replace_dots=(' ', '€')) + obj_currency = 'EUR' def parse(self, el): self.env['email'] = self.page.browser.username + + +class DocumentsDetailsPage(LoggedPage, PartialHTMLPage): + pass diff --git a/modules/myhabit/browser.py b/modules/myhabit/browser.py index 20d26d5577..dfcc2283c9 100644 --- a/modules/myhabit/browser.py +++ b/modules/myhabit/browser.py @@ -18,19 +18,18 @@ # along with weboob. If not, see . +from datetime import datetime +from decimal import Decimal + from requests.exceptions import Timeout -from weboob.tools.capabilities.bank.transactions import \ - AmericanTransaction as AmTr -from weboob.browser import LoginBrowser, URL, need_login +from weboob.browser import URL, LoginBrowser, need_login from weboob.browser.pages import HTMLPage from weboob.capabilities.base import Currency -from weboob.capabilities.shop import OrderNotFound, Order, Payment, Item +from weboob.capabilities.shop import Item, Order, OrderNotFound, Payment from weboob.exceptions import BrowserIncorrectPassword - -from datetime import datetime -from decimal import Decimal - +from weboob.tools.capabilities.bank.transactions import AmericanTransaction as AmTr +from weboob.tools.compat import unicode __all__ = ['MyHabit'] @@ -108,12 +107,12 @@ def items(self): def order_date(self): date = self.doc.xpath(u'//span[text()="Order Placed:"]' - '/following-sibling::span[1]/text()')[0].strip() + u'/following-sibling::span[1]/text()')[0].strip() return datetime.strptime(date, '%b %d, %Y') def order_number(self): return self.doc.xpath(u'//span[text()="MyHabit Order Number:"]' - '/following-sibling::span[1]/text()')[0].strip() + u'/following-sibling::span[1]/text()')[0].strip() def order_amount(self, which): return AmTr.decimal_amount((self.doc.xpath( @@ -187,7 +186,7 @@ def iter_items(self, order): @need_login def to_history(self): - for i in xrange(self.MAX_RETRIES): + for i in range(self.MAX_RETRIES): if self.history.is_here() and self.page.is_sane(): return self.page self.history.go() @@ -197,9 +196,9 @@ def do_login(self): raise BrowserIncorrectPassword() def location(self, *args, **kwargs): - for i in xrange(self.MAX_RETRIES): + for i in range(self.MAX_RETRIES): try: return super(MyHabit, self).location(*args, **kwargs) except Timeout as e: - pass - raise e + last_error = e + raise last_error diff --git a/modules/myhabit/module.py b/modules/myhabit/module.py index f7eb47fd85..616b0450fe 100644 --- a/modules/myhabit/module.py +++ b/modules/myhabit/module.py @@ -19,13 +19,14 @@ from weboob.capabilities.shop import CapShop -from weboob.tools.backend import Module, BackendConfig +from weboob.tools.backend import BackendConfig, Module from weboob.tools.value import ValueBackendPassword from .browser import MyHabit __all__ = ['MyHabitModule'] + class MyHabitModule(Module, CapShop): NAME = 'myhabit' MAINTAINER = u'Oleg Plakhotniuk' diff --git a/modules/n26/browser.py b/modules/n26/browser.py index d160e45296..3d91db7d6a 100644 --- a/modules/n26/browser.py +++ b/modules/n26/browser.py @@ -143,7 +143,7 @@ def _internal_get_transactions(self, categories, filter_func): for t in transactions: - if not filter_func(t): + if not filter_func(t) or t["amount"] == 0: continue new = Transaction() diff --git a/modules/nalo/compat/weboob_capabilities_bank.py b/modules/nalo/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/nalo/compat/weboob_capabilities_bank.py +++ b/modules/nalo/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/orange/browser.py b/modules/orange/browser.py index bd514e4efa..0bb944c0de 100644 --- a/modules/orange/browser.py +++ b/modules/orange/browser.py @@ -32,7 +32,7 @@ class OrangeBillBrowser(LoginBrowser): - BASEURL = 'https://espaceclientv3.orange.fr/' + BASEURL = 'https://espaceclientv3.orange.fr' loginpage = URL('https://login.orange.fr/\?service=sosh&return_url=https://www.sosh.fr/', 'https://login.orange.fr/front/login', LoginPage) @@ -73,6 +73,13 @@ def get_nb_remaining_free_sms(self): def post_message(self, message, sender): raise NotImplementedError() + def _iter_subscriptions_by_type(self, name, _type): + self.location('https://espaceclientv3.orange.fr/?page=gt-home-page&%s' % _type) + self.subscriptions.go() + for sub in self.page.iter_subscription(): + sub.subscriber = name + yield sub + @need_login def get_subscription_list(self): profile = self.profile.go().get_profile() @@ -92,10 +99,9 @@ def get_subscription_list(self): return # if nb_sub is 0, we continue, because we can get them in next url - self.location('https://espaceclientv3.orange.fr/?page=gt-home-page&sosh') - self.subscriptions.go() - for sub in self.page.iter_subscription(): - sub.subscriber = profile.name + for sub in self._iter_subscriptions_by_type(profile.name, 'sosh'): + yield sub + for sub in self._iter_subscriptions_by_type(profile.name, 'orange'): yield sub @need_login diff --git a/modules/paypal/browser.py b/modules/paypal/browser.py index 57a58d8cd8..79f27bea90 100644 --- a/modules/paypal/browser.py +++ b/modules/paypal/browser.py @@ -79,6 +79,7 @@ def __init__(self, *args, **kwargs): super(Paypal, self).__init__(*args, **kwargs) def do_login(self): + raise BrowserUnavailable() assert isinstance(self.username, basestring) assert isinstance(self.password, basestring) diff --git a/modules/peertube/__init__.py b/modules/peertube/__init__.py new file mode 100644 index 0000000000..e070b900fe --- /dev/null +++ b/modules/peertube/__init__.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + + +from .module import PeertubeModule + + +__all__ = ['PeertubeModule'] diff --git a/modules/peertube/browser.py b/modules/peertube/browser.py new file mode 100644 index 0000000000..25e483ac7f --- /dev/null +++ b/modules/peertube/browser.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +from weboob.browser.browsers import APIBrowser +from weboob.capabilities.video import BaseVideo +from weboob.capabilities.image import Thumbnail +from weboob.capabilities.file import LICENSES + + +class PeertubeBrowser(APIBrowser): + # source: server/initializers/constants.ts + SITE_LICENSES = { + 1: LICENSES.CCBY, + 2: LICENSES.CCBYSA, + 3: LICENSES.CCBYND, + 4: LICENSES.CCBYNC, + 5: LICENSES.CCBYNCSA, + 6: LICENSES.CCBYNCND, + 7: LICENSES.PD, + } + + def __init__(self, baseurl, *args, **kwargs): + super(PeertubeBrowser, self).__init__(*args, **kwargs) + self.BASEURL = baseurl + + def search_videos(self, pattern, sortby): + j = self.request('/api/v1/search/videos?count=10&sort=-match', params={ + 'search': pattern, + 'start': 0, + }) + + for item in j['data']: + video = BaseVideo() + self._parse_video(video, item) + yield video + + def get_video(self, id, video=None): + item = self.request('/api/v1/videos/%s' % id) + + if not video: + video = BaseVideo() + + self._parse_video(video, item) + + video._torrent = item['files'][0]['magnetUri'] + video.url = item['files'][0]['fileUrl'] + video.ext = video.url.rsplit('.', 1)[-1] + video.size = item['files'][0]['size'] + + return video + + def _parse_video(self, video, item): + video.id = item['uuid'] + video.nsfw = item['nsfw'] + video.title = item['name'] + video.description = item['description'] + video.author = item['account']['name'] + video.duration = item['duration'] + video.license = self.SITE_LICENSES[item['licence']['id']] + video.thumbnail = Thumbnail(self.absurl(item['thumbnailPath'])) diff --git a/modules/peertube/module.py b/modules/peertube/module.py new file mode 100644 index 0000000000..20498a9569 --- /dev/null +++ b/modules/peertube/module.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +from weboob.tools.backend import Module, BackendConfig +from weboob.tools.value import Value +from weboob.capabilities.video import CapVideo, BaseVideo + +from .browser import PeertubeBrowser + + +__all__ = ['PeertubeModule'] + + +class PeertubeModule(Module, CapVideo): + NAME = 'peertube' + DESCRIPTION = 'Peertube' + MAINTAINER = 'Vincent A' + EMAIL = 'dev@indigo.re' + LICENSE = 'AGPLv3+' + VERSION = '1.3' + + CONFIG = BackendConfig( + Value('url', label='Base URL of the PeerTube instance'), + ) + + BROWSER = PeertubeBrowser + + def create_default_browser(self): + return self.create_browser(self.config['url'].get()) + + def get_video(self, id): + return self.browser.get_video(id) + + def search_videos(self, pattern, sortby=CapVideo.SEARCH_RELEVANCE, nsfw=False): + for video in self.browser.search_videos(pattern, sortby): + if nsfw or not video.nsfw: + yield video + + def fill_video(self, obj, fields): + if set(('url', 'size')) & set(fields): + self.browser.get_video(obj.id, obj) + if 'thumbnail' in fields and obj.thumbnail: + obj.thumbnail.data = self.browser.open(obj.thumbnail.url).content + + OBJECTS = { + BaseVideo: fill_video, + } diff --git a/modules/phpbb/browser.py b/modules/phpbb/browser.py index fc1dbc8849..927011e4d2 100644 --- a/modules/phpbb/browser.py +++ b/modules/phpbb/browser.py @@ -20,15 +20,14 @@ import re -from weboob.browser import LoginBrowser, need_login, URL -from weboob.exceptions import BrowserIncorrectPassword +from weboob.browser import URL, LoginBrowser, need_login from weboob.capabilities.messages import CantSendMessage +from weboob.exceptions import BrowserIncorrectPassword +from .pages.forum import ForumPage, PostingPage, TopicPage from .pages.index import LoginPage -from .pages.forum import ForumPage, TopicPage, PostingPage from .tools import id2url, url2id - __all__ = ['PhpBB'] @@ -56,7 +55,7 @@ def do_login(self): data = {'login': 'Connexion', 'username': self.username, 'password': self.password, - } + } self.location('ucp.php?mode=login', data=data) if not self.page.logged: @@ -158,7 +157,7 @@ def post_answer(self, forum_id, topic_id, title, content): raise CantSendMessage('Please enter a title formatted like that:\n\t"[FORUM] SUBJECT"\n\n%s' % forums_prompt) forum_id = None - for k,v in forums.iteritems(): + for k, v in forums.items(): if v.lower() == m.group(1).lower(): forum_id = k break diff --git a/modules/phpbb/module.py b/modules/phpbb/module.py index c4ea6e6861..4e022d8ace 100644 --- a/modules/phpbb/module.py +++ b/modules/phpbb/module.py @@ -18,15 +18,14 @@ # along with weboob. If not, see . -from weboob.tools.backend import Module, BackendConfig -from weboob.tools.newsfeed import Newsfeed -from weboob.tools.value import Value, ValueInt, ValueBackendPassword +from weboob.capabilities.messages import CantSendMessage, CapMessages, CapMessagesPost, Message, Thread +from weboob.tools.backend import BackendConfig, Module from weboob.tools.misc import limit -from weboob.capabilities.messages import CapMessages, CapMessagesPost, Message, Thread, CantSendMessage +from weboob.tools.newsfeed import Newsfeed +from weboob.tools.value import Value, ValueBackendPassword, ValueInt from .browser import PhpBB -from .tools import rssid, url2id, id2url, id2topic - +from .tools import id2topic, id2url, rssid, url2id __all__ = ['PhpBBModule'] @@ -42,7 +41,7 @@ class PhpBBModule(Module, CapMessages, CapMessagesPost): Value('username', label='Username', default=''), ValueBackendPassword('password', label='Password', default=''), ValueInt('thread_unread_messages', label='Limit number of unread messages to retrieve for a thread', default=500) - ) + ) STORAGE = {'seen': {}} BROWSER = PhpBB diff --git a/modules/phpbb/pages/forum.py b/modules/phpbb/pages/forum.py index 2703dddbf5..5a8c0573ad 100644 --- a/modules/phpbb/pages/forum.py +++ b/modules/phpbb/pages/forum.py @@ -29,7 +29,7 @@ class Link(object): (FORUM, - TOPIC) = xrange(2) + TOPIC) = range(2) def __init__(self, type, url): self.type = type diff --git a/modules/piratebay/test.py b/modules/piratebay/test.py index 7e966bf8b0..e31cb35740 100644 --- a/modules/piratebay/test.py +++ b/modules/piratebay/test.py @@ -17,20 +17,21 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . -from weboob.tools.test import BackendTest -from weboob.capabilities.torrent import MagnetOnly - from random import choice +from weboob.capabilities.torrent import MagnetOnly +from weboob.tools.compat import basestring +from weboob.tools.test import BackendTest + class PiratebayTest(BackendTest): MODULE = 'piratebay' def test_torrent(self): # try something popular so we sometimes get a magnet-only torrent - l = list(self.backend.iter_torrents('ubuntu linux')) - if len(l): - torrent = choice(l) + torrents = list(self.backend.iter_torrents('ubuntu linux')) + if len(torrents): + torrent = choice(torrents) full_torrent = self.backend.get_torrent(torrent.id) assert torrent.name assert full_torrent.name == torrent.name diff --git a/modules/pradoepargne/compat/weboob_capabilities_bank.py b/modules/pradoepargne/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/pradoepargne/compat/weboob_capabilities_bank.py +++ b/modules/pradoepargne/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/rmll/pages.py b/modules/rmll/pages.py index bc814da7e0..fe06110161 100644 --- a/modules/rmll/pages.py +++ b/modules/rmll/pages.py @@ -18,19 +18,19 @@ # along with weboob. If not, see . import re +from datetime import timedelta -from weboob.browser.pages import HTMLPage, JsonPage from weboob.browser.elements import ItemElement, ListElement, method -from weboob.browser.filters.standard import Regexp, Format, CleanText, Duration, DateTime, Filter -from weboob.browser.filters.html import Link, XPath, CleanHTML +from weboob.browser.filters.html import CleanHTML, Link, XPath from weboob.browser.filters.json import Dict - +from weboob.browser.filters.standard import CleanText, DateTime, Duration, Filter, Format, Regexp +from weboob.browser.pages import HTMLPage, JsonPage from weboob.capabilities import NotLoaded -from weboob.capabilities.image import Thumbnail from weboob.capabilities.collection import Collection +from weboob.capabilities.image import Thumbnail +from weboob.tools.compat import unicode from .video import RmllVideo -from datetime import timedelta BASE_URL = 'http://video.rmll.info' diff --git a/modules/s2e/compat/weboob_capabilities_bank.py b/modules/s2e/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/s2e/compat/weboob_capabilities_bank.py +++ b/modules/s2e/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/societegenerale/browser.py b/modules/societegenerale/browser.py index e275b69c61..f15e84ef46 100644 --- a/modules/societegenerale/browser.py +++ b/modules/societegenerale/browser.py @@ -17,20 +17,24 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals + from datetime import datetime from dateutil.relativedelta import relativedelta from weboob.browser.browsers import LoginBrowser, URL, need_login, StatesMixin from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable, ActionNeeded -from weboob.capabilities.bank import Account, AddRecipientError +from weboob.capabilities.bank import Account, TransferBankError from weboob.capabilities.base import find_object, NotAvailable from weboob.browser.exceptions import BrowserHTTPNotFound from .compat.weboob_capabilities_profile import ProfileMissing +from .compat.weboob_tools_capabilities_bank_investments import create_french_liquidity from .pages.accounts_list import ( AccountsList, AccountHistory, CardsList, LifeInsurance, LifeInsuranceHistory, LifeInsuranceInvest, LifeInsuranceInvest2, Market, UnavailableServicePage, - ListRibPage, AdvisorPage, HTMLProfilePage, XMLProfilePage, LoansPage, IbanPage, + ListRibPage, AdvisorPage, HTMLProfilePage, XMLProfilePage, LoansPage, IbanPage, ComingPage, + NewLandingPage, ) from .pages.transfer import RecipientsPage, TransferPage, AddRecipientPage, RecipientJson from .pages.login import LoginPage, BadLoginPage, ReinitPasswordPage, ActionNeededPage, ErrorPage @@ -47,11 +51,13 @@ class SocieteGenerale(LoginBrowser, StatesMixin): login = URL('https://particuliers.societegenerale.fr/index.html', LoginPage) action_needed = URL('/com/icd-web/forms/cct-index.html', '/com/icd-web/gdpr/gdpr-recueil-consentements.html', + '/com/icd-web/forms/kyc-index.html', ActionNeededPage) bad_login = URL('\/acces/authlgn.html', '/error403.html', BadLoginPage) reinit = URL('/acces/changecodeobligatoire.html', ReinitPasswordPage) iban_page = URL(r'/lgn/url\.html\?dup', IbanPage) accounts = URL('/restitution/cns_listeprestation.html', AccountsList) + coming_page = URL('/restitution/cns_listeEncours.xml', ComingPage) cards_list = URL('/restitution/cns_listeCartes.*.html', CardsList) account_history = URL('/restitution/cns_detail.*\.html', '/lgn/url.html', AccountHistory) market = URL('/brs/cct/comti20.html', Market) @@ -76,6 +82,8 @@ class SocieteGenerale(LoginBrowser, StatesMixin): bank_statement_search = URL(r'/restitution/rce_recherche.html\?noRedirect=1', r'/restitution/rce_recherche_resultat.html', BankStatementPage) + new_landing = URL(r'/com/icd-web/cbo/index.html', NewLandingPage) + error = URL('https://static.societegenerale.fr/pri/erreur.html', ErrorPage) accounts_list = None @@ -90,7 +98,7 @@ def load_state(self, state): super(SocieteGenerale, self).load_state(state) def do_login(self): - if not self.password.isdigit() or len(self.password) != 6: + if not self.password.isdigit() or len(self.password) not in (6, 7): raise BrowserIncorrectPassword() if not self.username.isdigit() or len(self.username) < 8: raise BrowserIncorrectPassword() @@ -125,7 +133,14 @@ def do_login(self): def get_accounts_list(self): if self.accounts_list is None: self.accounts.stay_or_go() + # the link is not on the new landing page, navigating manually + if self.new_landing.is_here(): + self.logger.info('Falling back on old accounts consulting page.') + self.location('/restitution/cns_listeprestation.html?NoRedirect=true') self.accounts_list = self.page.get_list() + # Coming amount is on another page, whose url must be retrieved on the main page + self.location(self.page.get_coming_url()) + self.page.set_coming(self.accounts_list) self.list_rib.go() if self.list_rib.is_here(): # Caching rib url, so we don't have to go back and forth for each account @@ -202,6 +217,14 @@ def iter_investment(self, account): # Other Life Insurance pages: self.life_insurance_invest.go() + elif account.type == Account.TYPE_PEA: + # Scraping liquidities for "PEA Espèces" accounts + self.location(account._link_id) + valuation = self.page.get_liquidities() + if valuation != NotAvailable: + yield create_french_liquidity(valuation) + return + else: self.logger.warning('This account is not supported') return @@ -215,7 +238,11 @@ def get_advisor(self): @need_login def iter_recipients(self, account): - if not self.transfer.go().is_able_to_transfer(account): + try: + self.transfer.go() + except TransferBankError: + return + if not self.page.is_able_to_transfer(account): return for recipient in self.page.iter_recipients(account_id=account.id): yield recipient @@ -243,12 +270,13 @@ def end_sms_recipient(self, recipient, **params): def end_oob_recipient(self, recipient, **params): r = self.open('https://particuliers.secure.societegenerale.fr/sec/oob_polling.json', data={'n10_id_transaction': self.id_transaction}) - if r.page.doc['donnees']['transaction_status'] == 'available': - data = [('context', self.context), ('b64_jeton_transaction', self.context), ('dup', self.dup), ('n10_id_transaction', self.id_transaction), ('oob_op', 'sign')] - self.add_recipient.go(data=data, headers={'Referer': 'https://particuliers.secure.societegenerale.fr/lgn/url.html'}) - return self.page.get_recipient_object(recipient) - else: - raise AddRecipientError('transaction_status is %s' % r.page.doc['donnees']['transaction_status']) + assert r.page.doc['donnees']['transaction_status'] in ('available', 'in_progress'), \ + 'transaction_status is %s' % r.page.doc['donnees']['transaction_status'] + + data = [('context', self.context), ('b64_jeton_transaction', self.context), + ('dup', self.dup), ('n10_id_transaction', self.id_transaction), ('oob_op', 'sign')] + self.add_recipient.go(data=data, headers={'Referer': 'https://particuliers.secure.societegenerale.fr/lgn/url.html'}) + return self.page.get_recipient_object(recipient) @need_login def new_recipient(self, recipient, **params): diff --git a/modules/societegenerale/compat/weboob_capabilities_bank.py b/modules/societegenerale/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/societegenerale/compat/weboob_capabilities_bank.py +++ b/modules/societegenerale/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/societegenerale/compat/weboob_capabilities_base.py b/modules/societegenerale/compat/weboob_capabilities_base.py new file mode 100644 index 0000000000..df23e45a76 --- /dev/null +++ b/modules/societegenerale/compat/weboob_capabilities_base.py @@ -0,0 +1,22 @@ + +import weboob.capabilities.base as OLD + +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) + + +__all__ = OLD.__all__ + + +def strict_find_object(mylist, error=None, **kwargs): + """ + Tools to return an object with the matching parameters in kwargs. + Parameters with empty value are skipped + """ + kwargs = {k: v for k, v in kwargs.items() if not empty(v)} + if kwargs: + return find_object(mylist, error=error, **kwargs) + + if error is not None: + raise error() diff --git a/modules/societegenerale/compat/weboob_tools_capabilities_bank_investments.py b/modules/societegenerale/compat/weboob_tools_capabilities_bank_investments.py new file mode 100644 index 0000000000..2b68ff9e00 --- /dev/null +++ b/modules/societegenerale/compat/weboob_tools_capabilities_bank_investments.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Jonathan Schmidt +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.tools.compat import basestring +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.bank import Investment + +def is_isin_valid(isin): + """ + Méthode générale + Table de conversion des lettres en chiffres + A=10 B=11 C=12 D=13 E=14 F=15 G=16 H=17 I=18 + J=19 K=20 L=21 M=22 N=23 O=24 P=25 Q=26 R=27 + S=28 T=29 U=30 V=31 W=32 X=33 Y=34 Z=35 + + 1 - Mettre de côté la clé, qui servira de référence à la fin de la vérification. + 2 - Convertir toutes les lettres en nombres via la table de conversion ci-contre. Si le nombre obtenu est supérieur ou égal à 10, prendre les deux chiffres du nombre séparément (exemple : 27 devient 2 et 7). + 3 - Pour chaque chiffre, multiplier sa valeur par deux si sa position est impaire en partant de la droite. Si le nombre obtenu est supérieur ou égal à 10, garder les deux chiffres du nombre séparément (exemple : 14 devient 1 et 4). + 4 - Faire la somme de tous les chiffres. + 5 - Soustraire cette somme de la dizaine supérieure ou égale la plus proche (exemples : si la somme vaut 22, la dizaine « supérieure ou égale » est 30, et la clé vaut donc 8 ; si la somme vaut 30, la dizaine « supérieure ou égale » est 30, et la clé vaut 0 ; si la somme vaut 31, la dizaine « supérieure ou égale » est 40, et la clé vaut 9). + 6 - Comparer la valeur obtenue à la clé mise initialement de côté. + + Étapes 1 et 2 : + F R 0 0 0 3 5 0 0 0 0 (+ 8 : clé) + 15 27 0 0 0 3 5 0 0 0 0 + + Étape 3 : le traitement se fait sur des chiffres + 1 5 2 7 0 0 0 3 5 0 0 0 0 + I P I P I P I P I P I P I : position en partant de la droite (P = Pair, I = Impair) + 2 1 2 1 2 1 2 1 2 1 2 1 2 : coefficient multiplicateur + + 2 5 4 7 0 0 0 3 10 0 0 0 0 : résultat + + Étape 4 : + 2 + 5 + 4 + 7 + 0 + 0 + 0 + 3 + (1 + 0)+ 0 + 0 + 0 + 0 = 22 + + Étapes 5 et 6 : 30 - 22 = 8 (valeur de la clé) + """ + + if not isinstance(isin, basestring): + return False + if not re.match(r'^[A-Z]{2}[A-Z0-9]{9}\d$', isin): + return False + + isin_in_digits = ''.join(str(ord(x) - ord('A') + 10) if not x.isdigit() else x for x in isin[:-1]) + key = isin[-1:] + result = '' + for k, val in enumerate(isin_in_digits[::-1], start=1): + if k % 2 == 0: + result = ''.join((result, val)) + else: + result = ''.join((result, str(int(val)*2))) + return str(sum(int(x) for x in result) + int(key))[-1] == '0' + + +def create_french_liquidity(valuation): + """ + Automatically fills a liquidity investment with label, code and code_type. + """ + liquidity = Investment() + liquidity.label = "Liquidités" + liquidity.code = "XX-liquidity" + liquidity.code_type = NotAvailable + liquidity.valuation = valuation + return liquidity + diff --git a/modules/societegenerale/module.py b/modules/societegenerale/module.py index 82d63c4f9b..ff4f1bfd27 100644 --- a/modules/societegenerale/module.py +++ b/modules/societegenerale/module.py @@ -35,7 +35,7 @@ from weboob.tools.capabilities.bank.transactions import sorted_transactions from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import Value, ValueBackendPassword -from weboob.capabilities.base import find_object, NotAvailable +from .compat.weboob_capabilities_base import find_object, NotAvailable, strict_find_object from .browser import SocieteGenerale from .sgpe.browser import SGEnterpriseBrowser, SGProfessionalBrowser @@ -117,18 +117,16 @@ def init_transfer(self, transfer, **params): raise NotImplementedError() transfer.label = ' '.join(w for w in re.sub('[^0-9a-zA-Z ]+', '', transfer.label).split()) self.logger.info('Going to do a new transfer') - if transfer.account_iban: - account = find_object(self.iter_accounts(), iban=transfer.account_iban, error=AccountNotFound) - else: - account = find_object(self.iter_accounts(), id=transfer.account_id, error=AccountNotFound) - if transfer.recipient_iban: - recipient = find_object(self.iter_transfer_recipients(account.id), iban=transfer.recipient_iban, error=RecipientNotFound) - else: - recipient = find_object(self.iter_transfer_recipients(account.id), id=transfer.recipient_id, error=RecipientNotFound) + account = strict_find_object(self.iter_accounts(), iban=transfer.account_iban) + if not account: + account = strict_find_object(self.iter_accounts(), id=transfer.account_id, error=AccountNotFound) - transfer.amount = transfer.amount.quantize(Decimal('.01')) + recipient = strict_find_object(self.iter_transfer_recipients(account.id), iban=transfer.recipient_iban) + if not recipient: + recipient = strict_find_object(self.iter_transfer_recipients(account.id), id=transfer.recipient_id, error=RecipientNotFound) + transfer.amount = transfer.amount.quantize(Decimal('.01')) return self.browser.init_transfer(account, recipient, transfer) def execute_transfer(self, transfer, **params): diff --git a/modules/societegenerale/pages/accounts_list.py b/modules/societegenerale/pages/accounts_list.py index 12346b9094..6610b53f47 100644 --- a/modules/societegenerale/pages/accounts_list.py +++ b/modules/societegenerale/pages/accounts_list.py @@ -42,7 +42,6 @@ from .base import BasePage - def MyDecimal(*args, **kwargs): kwargs.update(replace_dots=True, default=NotAvailable) return CleanDecimal(*args, **kwargs) @@ -77,6 +76,13 @@ class AccountsList(LoggedPage, BasePage): 'Avance Patrimoniale': Account.TYPE_LOAN, } + def get_coming_url(self): + for script in self.doc.xpath('//script'): + s_content = CleanText('.')(script) + if "var url_encours" in s_content: + break + return re.search(r'url_encours=\"(.+)\"; ', s_content).group(1) + def get_list(self): err = CleanText('//span[@class="error_msg"]', default='')(self.doc) if err == 'Vous ne disposez pas de compte consultable.': @@ -147,12 +153,11 @@ def check_valid_url(url): # Layout with several cards line = CleanText('//table//div[contains(text(), "Liste des cartes")]', replace=[(' ', '')])(page.doc) - if line: - parent_id = re.search(r'(\d+)', line).group() - # Layout with only one card + m = re.search(r'(\d+)', line) + if m: + parent_id = m.group() else: parent_id = CleanText('//div[contains(text(), "Numéro de compte débité")]/following::div[1]', replace=[(' ', '')])(page.doc) - account.parent = find_object(accounts_list, id=parent_id) if account.type == Account.TYPE_UNKNOWN: @@ -163,6 +168,12 @@ def check_valid_url(url): return accounts_list +class ComingPage(LoggedPage, XMLPage): + def set_coming(self, accounts_list): + for a in accounts_list: + a.coming = CleanDecimal('//EnCours[contains(@id, "%s")]' % a.id, replace_dots=True, default=NotAvailable)(self.doc) + + class IbanPage(LoggedPage, NotTransferBasePage): def is_here(self): if self.is_transfer_here(): @@ -344,6 +355,9 @@ def _iter_transactions(self, doc): yield t + def get_liquidities(self): + return CleanDecimal('//td[contains(@headers, "solde")]', replace_dots=True)(self.doc) + class Invest(object): def create_investment(self, cells): @@ -685,3 +699,7 @@ class UnavailableServicePage(LoggedPage, HTMLPage): def on_load(self): if self.doc.xpath('//div[contains(@class, "erreur_404_content")]'): raise BrowserUnavailable() + + +class NewLandingPage(LoggedPage, HTMLPage): + pass diff --git a/modules/societegenerale/pages/compat/weboob_capabilities_bank.py b/modules/societegenerale/pages/compat/weboob_capabilities_bank.py new file mode 100644 index 0000000000..141097d781 --- /dev/null +++ b/modules/societegenerale/pages/compat/weboob_capabilities_bank.py @@ -0,0 +1,45 @@ + +import weboob.capabilities.bank as OLD + +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) + + +__all__ = OLD.__all__ + + +class CapBankWealth(CapBank): + pass + + +class CapBankPockets(CapBank): + pass + + +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + +class CapBankTransfer(OLD.CapBankTransfer): + def transfer_check_label(self, old, new): + from unidecode import unidecode + + return unidecode(old) == unidecode(new) + + +class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipient): + pass + + +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + +Account.TYPE_MORTGAGE = 17 +Account.TYPE_CONSUMER_CREDIT = 18 +Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/societegenerale/pages/transfer.py b/modules/societegenerale/pages/transfer.py index 0121c7b85e..541c4e2337 100644 --- a/modules/societegenerale/pages/transfer.py +++ b/modules/societegenerale/pages/transfer.py @@ -26,9 +26,9 @@ from weboob.browser.pages import LoggedPage, JsonPage, FormNotFound from weboob.browser.elements import method, ListElement, ItemElement, SkipItem -from weboob.capabilities.bank import ( +from .compat.weboob_capabilities_bank import ( Recipient, TransferBankError, TransferInvalidCurrency, Transfer, - AddRecipientError, AddRecipientStep, + AddRecipientBankError, AddRecipientStep, ) from weboob.capabilities.base import find_object, NotAvailable, empty from weboob.browser.filters.standard import CleanText, Regexp, CleanDecimal, \ @@ -318,7 +318,7 @@ class AddRecipientPage(LoggedPage, BasePage): def on_load(self): error_msg = CleanText(u'//span[@class="error_msg"]')(self.doc) if error_msg: - raise AddRecipientError(message=error_msg) + raise AddRecipientBankError(message=error_msg) def is_here(self): return bool(CleanText(u'//h3[contains(text(), "Ajouter un compte bénéficiaire de virement")]')(self.doc)) or \ @@ -348,7 +348,7 @@ def double_auth(self, recipient): try: form = self.get_form(id='formCache') except FormNotFound: - raise AddRecipientError('form not found') + assert False, 'Double auth form not found' self.browser.context = form['context'] self.browser.dup = form['dup'] @@ -377,7 +377,7 @@ def double_auth(self, recipient): self.browser.id_transaction = r.page.doc['donnees']['id-transaction'] raise AddRecipientStep(recipient, ValueBool('pass', label=u'Valider cette opération sur votre applicaton société générale')) else: - raise AddRecipientError('sign process unknown') + assert False, 'Sign process unknown: %s' % r.page.doc['donnees']['sign_proc'] def get_recipient_object(self, recipient, get_info=False): r = Recipient() diff --git a/modules/societegenerale/sgpe/browser.py b/modules/societegenerale/sgpe/browser.py index 2cae330a1c..81e26a10c6 100644 --- a/modules/societegenerale/sgpe/browser.py +++ b/modules/societegenerale/sgpe/browser.py @@ -27,8 +27,8 @@ from weboob.browser.exceptions import ClientError from weboob.exceptions import BrowserIncorrectPassword from weboob.capabilities.base import find_object -from weboob.capabilities.bank import ( - AccountNotFound, RecipientNotFound, AddRecipientStep, AddRecipientError, +from .compat.weboob_capabilities_bank import ( + AccountNotFound, RecipientNotFound, AddRecipientStep, AddRecipientBankError, Recipient, TransferBankError, ) from weboob.tools.value import Value @@ -36,6 +36,7 @@ from .pages import ( LoginPage, CardsPage, CardHistoryPage, IncorrectLoginPage, ProfileProPage, ProfileEntPage, ChangePassPage, SubscriptionPage, + ErrorPage, ) from .json_pages import AccountsJsonPage, BalancesJsonPage, HistoryJsonPage, BankStatementPage from .transfer_pages import ( @@ -118,7 +119,6 @@ def get_profile(self): class SGEnterpriseBrowser(SGPEBrowser): BASEURL = 'https://entreprises.secure.societegenerale.fr' - LOGIN_FORM = 'auth' MENUID = 'BANREL' CERTHASH = '2231d5ddb97d2950d5e6fc4d986c23be4cd231c31ad530942343a8fdcc44bb99' @@ -145,10 +145,10 @@ def get_accounts_list(self): @need_login def iter_history(self, account): value = self.history.go(data={'cl500_compte': account._id, 'cl200_typeReleve': 'valeur'}).get_value() - transactions = [] - transactions.extend(self.history.go(data={'cl500_compte': account._id, 'cl200_typeReleve': value}).iter_history(value=value)) - transactions.extend(self.location('/icd/syd-front/data/syd-intraday-chargerDetail.json', data={'cl500_compte': account._id}).page.iter_history()) - return iter(transactions) + for tr in self.history.go(data={'cl500_compte': account._id, 'cl200_typeReleve': value}).iter_history(value=value): + yield tr + for tr in self.location('/icd/syd-front/data/syd-intraday-chargerDetail.json', data={'cl500_compte': account._id}).page.iter_history(): + yield tr @need_login def iter_subscription(self): @@ -176,7 +176,6 @@ def iter_documents(self, subscription): class SGProfessionalBrowser(SGEnterpriseBrowser, StatesMixin): BASEURL = 'https://professionnels.secure.societegenerale.fr' - LOGIN_FORM = 'auth_reco' MENUID = 'SBOREL' CERTHASH = '9f5232c9b2283814976608bfd5bba9d8030247f44c8493d8d205e574ea75148e' STATE_DURATION = 5 @@ -204,6 +203,8 @@ class SGProfessionalBrowser(SGEnterpriseBrowser, StatesMixin): bank_statement_menu = URL('/icd/syd-front/data/syd-rce-accederDepuisMenu.json', BankStatementPage) bank_statement_search = URL('/icd/syd-front/data/syd-rce-lancerRecherche.json', BankStatementPage) + error_page = URL('https://static.societegenerale.fr/pro/erreur.html', ErrorPage) + date_max = None date_min = None @@ -359,16 +360,14 @@ def new_recipient(self, recipient, **params): @need_login def validate_rcpt_with_sms(self, code): - if not self.new_rcpt_validate_form: - raise AddRecipientError() - + assert self.new_rcpt_validate_form, 'There should have recipient validate form in states' self.new_rcpt_validate_form.update({'code': code}) try: self.confirm_new_recipient.go(data=self.new_rcpt_validate_form) except ClientError as e: assert e.response.status_code == 403, \ 'Something went wrong in add recipient, response status code is %s' % e.response.status_code - raise AddRecipientError(message='Le code entré est incorrect.') + raise AddRecipientBankError(message='Le code entré est incorrect.') @need_login def iter_recipients(self, origin_account): diff --git a/modules/societegenerale/sgpe/compat/__init__.py b/modules/societegenerale/sgpe/compat/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/modules/societegenerale/sgpe/compat/weboob_capabilities_bank.py b/modules/societegenerale/sgpe/compat/weboob_capabilities_bank.py new file mode 100644 index 0000000000..141097d781 --- /dev/null +++ b/modules/societegenerale/sgpe/compat/weboob_capabilities_bank.py @@ -0,0 +1,45 @@ + +import weboob.capabilities.bank as OLD + +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) + + +__all__ = OLD.__all__ + + +class CapBankWealth(CapBank): + pass + + +class CapBankPockets(CapBank): + pass + + +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + +class CapBankTransfer(OLD.CapBankTransfer): + def transfer_check_label(self, old, new): + from unidecode import unidecode + + return unidecode(old) == unidecode(new) + + +class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipient): + pass + + +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + +Account.TYPE_MORTGAGE = 17 +Account.TYPE_CONSUMER_CREDIT = 18 +Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/societegenerale/sgpe/json_pages.py b/modules/societegenerale/sgpe/json_pages.py index 20df079fa3..992e67110a 100644 --- a/modules/societegenerale/sgpe/json_pages.py +++ b/modules/societegenerale/sgpe/json_pages.py @@ -28,7 +28,7 @@ from weboob.capabilities import NotAvailable from weboob.capabilities.bank import Account from weboob.capabilities.bill import Document, Subscription -from weboob.exceptions import BrowserUnavailable +from weboob.exceptions import BrowserUnavailable, BrowserIncorrectPassword from weboob.tools.capabilities.bank.iban import is_iban_valid from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.tools.compat import quote_plus @@ -89,7 +89,10 @@ def get_error(self): class BalancesJsonPage(LoggedPage, JsonPage): def on_load(self): if self.doc['commun']['statut'] == 'NOK': - raise BrowserUnavailable(self.doc['commun']['raison']) + reason = self.doc['commun']['raison'] + if reason == 'SYD-COMPTES-UNAUTHORIZED-ACCESS': + raise BrowserIncorrectPassword("Vous n'avez pas l'autorisation de consulter : {}".format(reason)) + raise BrowserUnavailable(reason) def populate_balances(self, accounts): for account in accounts: @@ -138,9 +141,30 @@ class item(ItemElement): obj_rdate = Date(Dict('date', default=None), dayfirst=True, default=NotAvailable) obj_date = Date(Dict('dVl', default=None), dayfirst=True, default=NotAvailable) obj__coming = False - obj_raw = Transaction.Raw(Format('%s %s %s', Dict('l1'), Dict('l2'), Dict('l3'))) - # We have l4 and l5 too most of the time, but it seems to be unimportant and would make label too long. - #tr.label = ' '.join([' '.join(transaction[l].strip().split()) for l in ['l1', 'l2', 'l3']]) + + # Label is split into l1, l2, l3, l4, l5. + # l5 is needed for transfer label, for example: + # 'l1': "000001 VIR EUROPEEN EMIS NET" + # 'l2': "POUR: XXXXXXXXXXXXX" + # 'l3': "REF: XXXXXXXXXXXXXX" + # 'l4': "REMISE: XXXXXX TRANSFER LABEL" + # 'l5': "MOTIF: TRANSFER LABEL" + obj_raw = Transaction.Raw(Format( + '%s %s %s %s %s', + Dict('l1'), + Dict('l2'), + Dict('l3'), + Dict('l4'), + Dict('l5'), + )) + + # keep the 3 first rows for transaction label + obj_label = Transaction.Raw(Format( + '%s %s %s', + Dict('l1'), + Dict('l2'), + Dict('l3'), + )) def obj_amount(self): return CleanDecimal(Dict('c', default=None), replace_dots=True, default=None)(self) or \ diff --git a/modules/societegenerale/sgpe/pages.py b/modules/societegenerale/sgpe/pages.py index b63442692e..82f8b6d897 100644 --- a/modules/societegenerale/sgpe/pages.py +++ b/modules/societegenerale/sgpe/pages.py @@ -33,7 +33,7 @@ from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.capabilities.profile import Profile, Person from weboob.capabilities.bill import Document, Subscription -from weboob.exceptions import ActionNeeded, BrowserIncorrectPassword +from weboob.exceptions import ActionNeeded, BrowserIncorrectPassword, BrowserUnavailable from weboob.tools.json import json from weboob.capabilities.base import NotAvailable @@ -42,7 +42,7 @@ class Transaction(FrenchTransaction): - PATTERNS = [(re.compile(r'^CARTE \w+ RETRAIT DAB.* (?P
\d{2})/(?P\d{2})( (?P\d+)H(?P\d+))? (?P.*)'), + PATTERNS = [(re.compile(r'^CARTE \w+ RETRAIT DAB.*? (?P
\d{2})/(?P\d{2})( (?P\d+)H(?P\d+))? (?P.*)'), FrenchTransaction.TYPE_WITHDRAWAL), (re.compile(r'^CARTE \w+ (?P
\d{2})/(?P\d{2})( A (?P\d+)H(?P\d+))? RETRAIT DAB (?P.*)'), FrenchTransaction.TYPE_WITHDRAWAL), @@ -122,17 +122,13 @@ def get_authentication_data(self): def login(self, login, password): authentication_data = self.get_authentication_data() - form = self.get_form(name=self.browser.LOGIN_FORM) - form['user_id'] = login - form['codsec'] = authentication_data['img'].get_codes(password[:6]) - form['cryptocvcs'] = authentication_data['infos']['crypto'] - form['vk_op'] = 'auth' - form.url = '/authent.html' - try: - form.pop('button') - except KeyError: - pass - form.submit() + data = { + 'user_id': login, + 'codsec': authentication_data['img'].get_codes(password[:6]), + 'cryptocvcs': authentication_data['infos']['crypto'], + 'vk_op': 'auth', + } + self.browser.location(self.browser.absurl('/authent.html'), data=data) class CardsPage(LoggedPage, SGPEPage): @@ -256,3 +252,10 @@ class IncorrectLoginPage(SGPEPage): def on_load(self): if self.doc.xpath('//div[@class="ngo_mu_message" and contains(text(), "saisies sont incorrectes")]'): raise BrowserIncorrectPassword(CleanText('//div[@class="ngo_mu_message"]')(self.doc)) + + +class ErrorPage(SGPEPage): + def on_load(self): + if self.doc.xpath('//div[@class="ngo_mu_message" and contains(text(), "momentanément indisponible")]'): + # Warning: it could occurs because of wrongpass, user have to change password + raise BrowserUnavailable(CleanText('//div[@class="ngo_mu_message"]')(self.doc)) diff --git a/modules/spirica/compat/weboob_capabilities_bank.py b/modules/spirica/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/spirica/compat/weboob_capabilities_bank.py +++ b/modules/spirica/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/sueurdemetal/browser.py b/modules/sueurdemetal/browser.py index 7b31e2ccbc..3f76558277 100644 --- a/modules/sueurdemetal/browser.py +++ b/modules/sueurdemetal/browser.py @@ -19,12 +19,11 @@ from __future__ import unicode_literals -from weboob.browser import PagesBrowser, URL +from weboob.browser import URL, PagesBrowser from weboob.tools.json import json from .pages import ConcertListPage, ConcertPage, NoEvent - __all__ = ['SueurDeMetalBrowser'] @@ -64,7 +63,7 @@ def get_concert(self, id): def build_cities(self): if self.cities: - return + return self.deps.go() for dept in self.page.get_depts(): self.jlocation(self.cities.build, data={ diff --git a/modules/sueurdemetal/module.py b/modules/sueurdemetal/module.py index d080cdb670..204ee96e94 100644 --- a/modules/sueurdemetal/module.py +++ b/modules/sueurdemetal/module.py @@ -18,12 +18,11 @@ # along with weboob. If not, see . +from weboob.capabilities.calendar import CATEGORIES, BaseCalendarEvent, CapCalendarEvent, Query from weboob.tools.backend import Module -from weboob.capabilities.calendar import CapCalendarEvent, BaseCalendarEvent, CATEGORIES, Query from .browser import SueurDeMetalBrowser - __all__ = ['SueurDeMetalModule'] diff --git a/modules/suravenir/browser.py b/modules/suravenir/browser.py index 8f725f7618..88bbcf64d1 100644 --- a/modules/suravenir/browser.py +++ b/modules/suravenir/browser.py @@ -30,10 +30,10 @@ class Suravenir(LoginBrowser): BASEURL = 'https://www.previ-direct.com/' - broker_to_instance = { 'assurancevie.com' : 'Q4n1', - 'linxea' : 'S9o6', - 'mes-placements.fr': '5yKs', - 'epargnissimo' : 'FtA1'} + broker_to_instance = {'assurancevie.com' : 'Q4n1', + 'linxea' : 'S9o6', + 'mes-placements.fr': '5yKs', + 'epargnissimo' : 'FtA1'} login_page = URL('/web/eclient-(?P.*)', LoginPage) accounts_page = URL('/group/eclient-(?P.*)/home$', AccountsList) @@ -65,7 +65,7 @@ def get_accounts_list(self): return self.page.get_contracts() def get_URI_for_account(self, account): - return '%s&_portletespaceClientmonCompte_WAR_portletespaceclient_INSTANCE_%s_tabName=detailsContrat.tabulateur.tabulation' % (account._detail_link, self.broker_to_instance[self.broker]) + return '%s&_portletespaceClientmonCompte_WAR_portletespaceclient_INSTANCE_%s_tabName=detailsContrat.tabulateur.tabulation' % (account._detail_link, self.broker_to_instance[self.broker]) @need_login def iter_investments(self, account): diff --git a/modules/suravenir/compat/weboob_capabilities_bank.py b/modules/suravenir/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/suravenir/compat/weboob_capabilities_bank.py +++ b/modules/suravenir/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/vicsec/browser.py b/modules/vicsec/browser.py index bc39d9ea5e..31e366a8da 100644 --- a/modules/vicsec/browser.py +++ b/modules/vicsec/browser.py @@ -18,19 +18,18 @@ # along with weboob. If not, see . -from weboob.tools.capabilities.bank.transactions import \ - AmericanTransaction as AmTr -from weboob.browser import LoginBrowser, URL, need_login -from weboob.browser.pages import HTMLPage -from weboob.capabilities.base import Currency -from weboob.capabilities.shop import Order, Item, Payment -from weboob.exceptions import BrowserIncorrectPassword - +import re from datetime import datetime from decimal import Decimal from itertools import chain -import re +from weboob.browser import URL, LoginBrowser, need_login +from weboob.browser.pages import HTMLPage +from weboob.capabilities.base import Currency +from weboob.capabilities.shop import Item, Order, Payment +from weboob.exceptions import BrowserIncorrectPassword +from weboob.tools.capabilities.bank.transactions import AmericanTransaction as AmTr +from weboob.tools.compat import unicode __all__ = ['VicSec'] @@ -83,7 +82,7 @@ def order(self): def payments(self): for tr in self.doc.xpath('//tbody[@class="payment-summary"]' - '//th[text()="Payment Summary"]/../../../tbody/tr'): + '//th[text()="Payment Summary"]/../../../tbody/tr'): method = tr.xpath('td[1]/text()')[0] amnode = tr.xpath('td[2]')[0] amsign = -1 if amnode.xpath('em') else 1 @@ -99,8 +98,8 @@ def items(self): for tr in self.doc.xpath('//tbody[@class="order-items"]/tr'): label = tr.xpath('*//h1')[0].text_content().strip() price = AmTr.decimal_amount(re.match(r'^\s*([^\s]+)(\s+.*)?', - tr.xpath('*//div[@class="price"]')[0].text_content(), - re.DOTALL).group(1)) + tr.xpath('*//div[@class="price"]')[0].text_content(), + re.DOTALL).group(1)) url = 'http:' + tr.xpath('*//img/@src')[0] item = Item() item.label = unicode(label) @@ -124,9 +123,8 @@ def shipping(self): return self.payment_part(u'Shipping & Handling') def order_info(self, which): - info = self.doc.xpath('//p[@class="orderinfo"]' - )[0].text_content() - return re.match(u'.*%s:\\s+([^\\s]+)\\s'%which,info,re.DOTALL).group(1) + info = self.doc.xpath('//p[@class="orderinfo"]')[0].text_content() + return re.match(u'.*%s:\\s+([^\\s]+)\\s' % which, info, re.DOTALL).group(1) def discount(self): # Sometimes subtotal doesn't add up with items. @@ -143,7 +141,7 @@ def payment_part(self, which): # The numbers notation on VS is super wierd. # Sometimes negative amounts are represented by element. for node in self.doc.xpath('//tbody[@class="payment-summary"]' - '//td[contains(text(),"%s")]/../td[2]' % which): + '//td[contains(text(),"%s")]/../td[2]' % which): strv = node.text_content().strip() v = Decimal(0) if strv == u'FREE' else AmTr.decimal_amount(strv) return -v if node.xpath('em') and v > 0 else v diff --git a/modules/vicsec/module.py b/modules/vicsec/module.py index 8f37fe2d05..fd89306ccc 100644 --- a/modules/vicsec/module.py +++ b/modules/vicsec/module.py @@ -19,12 +19,11 @@ from weboob.capabilities.shop import CapShop -from weboob.tools.backend import Module, BackendConfig +from weboob.tools.backend import BackendConfig, Module from weboob.tools.value import ValueBackendPassword from .browser import VicSec - __all__ = ['VicSecModule'] diff --git a/modules/vicseccard/browser.py b/modules/vicseccard/browser.py index ee759a5255..0361209b02 100644 --- a/modules/vicseccard/browser.py +++ b/modules/vicseccard/browser.py @@ -17,17 +17,17 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . -from requests.exceptions import Timeout, ConnectionError +from datetime import datetime + +from requests.exceptions import ConnectionError, Timeout -from weboob.capabilities.bank import AccountNotFound, Account, Transaction -from weboob.tools.capabilities.bank.transactions import \ - AmericanTransaction as AmTr -from weboob.browser import LoginBrowser, URL, need_login +from weboob.browser import URL, LoginBrowser, need_login from weboob.browser.exceptions import ServerError from weboob.browser.pages import HTMLPage +from weboob.capabilities.bank import Account, AccountNotFound, Transaction from weboob.exceptions import BrowserIncorrectPassword -from datetime import datetime - +from weboob.tools.capabilities.bank.transactions import AmericanTransaction as AmTr +from weboob.tools.compat import unicode __all__ = ['VicSecCard'] @@ -52,13 +52,14 @@ def login(self, username, password): class HomePage(SomePage): def account(self): id_ = self.doc.xpath(u'//strong[contains(text(),' - '"Credit Card account ending in")]/text()')[0].strip()[-4:] - balance = self.doc.xpath(u'//span[@class="description" and text()="Current Balance"]/../span[@class="total"]/text()')[0].strip() + u'"Credit Card account ending in")]/text()')[0].strip()[-4:] + balance = self.doc.xpath( + u'//span[@class="description" and text()="Current Balance"]/../span[@class="total"]/text()')[0].strip() cardlimit = self.doc.xpath(u'//span[contains(text(),"Credit limit")]' - '/text()')[0].split()[-1] + u'/text()')[0].split()[-1] paymin = self.doc.xpath(u'//section[@id=" account_summary"]' - '//strong[text()="Minimum Payment Due"]/../../span[2]/text()' - )[0].strip() + u'//strong[text()="Minimum Payment Due"]/../../span[2]/text()' + )[0].strip() a = Account() a.id = id_ a.label = u'ACCOUNT ENDING IN %s' % id_ @@ -67,9 +68,9 @@ def account(self): a.type = Account.TYPE_CARD a.cardlimit = AmTr.decimal_amount(cardlimit) a.paymin = AmTr.decimal_amount(paymin) - #TODO: Add paydate. - #Oleg: I don't have an account with scheduled payment. - # Need to wait for a while... + # TODO: Add paydate. + # Oleg: I don't have an account with scheduled payment. + # Need to wait for a while... return a @@ -121,9 +122,9 @@ def do_login(self): raise BrowserIncorrectPassword() def location(self, *args, **kwargs): - for i in xrange(self.MAX_RETRIES): + for i in range(self.MAX_RETRIES): try: return super(VicSecCard, self).location(*args, **kwargs) except (ServerError, Timeout, ConnectionError) as e: - pass - raise e + last_error = e + raise last_error diff --git a/modules/vicseccard/module.py b/modules/vicseccard/module.py index f3fa74c8cd..19c992d152 100644 --- a/modules/vicseccard/module.py +++ b/modules/vicseccard/module.py @@ -18,12 +18,11 @@ # along with weboob. If not, see . from weboob.capabilities.bank import CapBank -from weboob.tools.backend import Module, BackendConfig +from weboob.tools.backend import BackendConfig, Module from weboob.tools.value import ValueBackendPassword from .browser import VicSecCard - __all__ = ['VicSecCardModule'] diff --git a/modules/vimeo/pages.py b/modules/vimeo/pages.py index 77854cc4d9..fd3dfbb2e9 100644 --- a/modules/vimeo/pages.py +++ b/modules/vimeo/pages.py @@ -149,6 +149,7 @@ class VimeoItem(ItemElement): obj_date = DateTime(CleanText('./upload_date')) obj__is_hd = CleanText('./@is_hd') obj_duration = VimeoDuration(CleanText('./duration')) + obj_ext = u'mp4' def obj_thumbnail(self): t = CleanText('./thumbnails/thumbnail[1]', default='')(self) diff --git a/modules/vlille/browser.py b/modules/vlille/browser.py index 43d4e82d4d..9297a53cf3 100644 --- a/modules/vlille/browser.py +++ b/modules/vlille/browser.py @@ -29,7 +29,7 @@ class VlilleBrowser(PagesBrowser): BASEURL = 'https://www.transpole.fr' - list_page = URL('/cms/institutionnel/vlille-carto/', ListStationsPage) + list_page = URL('/cms/vlille/les-stations-cartographies/', ListStationsPage) def get_station_list(self): return self.list_page.go().get_station_list() diff --git a/modules/wellsfargo/browser.py b/modules/wellsfargo/browser.py index 0834d0a694..0d31e511d5 100644 --- a/modules/wellsfargo/browser.py +++ b/modules/wellsfargo/browser.py @@ -18,23 +18,20 @@ # along with weboob. If not, see . -from weboob.capabilities.bank import AccountNotFound -from weboob.browser import LoginBrowser, URL, need_login -from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable -from weboob.tools.compat import unquote - -import ssl import json import os -from tempfile import mkstemp +import ssl from subprocess import STDOUT, CalledProcessError, check_output +from tempfile import mkstemp -from .pages import LoginProceedPage, LoginRedirectPage, \ - SummaryPage, ActivityCashPage, ActivityCardPage, \ - DocumentsPage, StatementPage, StatementsPage, \ - StatementsEmbeddedPage, LoggedInPage, CodeRequestPage, \ - CodeSubmitPage +from weboob.browser import URL, LoginBrowser, need_login +from weboob.capabilities.bank import AccountNotFound +from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable +from weboob.tools.compat import unquote +from .pages import (ActivityCardPage, ActivityCashPage, CodeRequestPage, CodeSubmitPage, DocumentsPage, LoggedInPage, + LoginProceedPage, LoginRedirectPage, StatementPage, StatementsEmbeddedPage, StatementsPage, + SummaryPage) __all__ = ['WellsFargo'] @@ -63,14 +60,14 @@ class WellsFargo(LoginBrowser): documents = URL('https://connect.secure.wellsfargo.com' '/accounts/start\?.+$', DocumentsPage) statements_embedded = URL('https://connect.secure.wellsfargo.com' - '/accounts/start\?.+$', StatementsEmbeddedPage) + '/accounts/start\?.+$', StatementsEmbeddedPage) statements = URL('https://connect.secure.wellsfargo.com' '/accounts/documents/statement/list.+$', StatementsPage) statement = URL('https://connect.secure.wellsfargo.com' '/accounts/documents/retrieve/.+$', StatementPage) - unknown = URL('/.*$', LoggedInPage) # E.g. random advertisement pages. + unknown = URL('/.*$', LoggedInPage) # e.g. random advertisement pages. def __init__(self, question1, answer1, question2, answer2, question3, answer3, phone_last4, code_file, *args, **kwargs): @@ -90,7 +87,7 @@ def do_login(self): which uses DOM. For now the easiest option seems to be to run it in PhantomJs. ''' - for i in xrange(self.MAX_RETRIES): + for i in range(self.MAX_RETRIES): scrf, scrn = mkstemp('.js') cookf, cookn = mkstemp('.json') os.write(scrf, LOGIN_JS % { @@ -118,10 +115,10 @@ def do_login(self): os.remove(cookn) self.session.cookies.clear() for c in cookies: - for k in ['expiry', 'expires', 'httponly']: - c.pop(k, None) - c['value'] = unquote(c['value']) - self.session.cookies.set(**c) + for k in ['expiry', 'expires', 'httponly']: + c.pop(k, None) + c['value'] = unquote(c['value']) + self.session.cookies.set(**c) self.summary.go() if self.page.logged: break @@ -190,7 +187,7 @@ def to_activity(self, id_=None): @need_login def to_statements(self, id_=None, year=None): if not self.statements.is_here() \ - and not self.statements_embedded.is_here(): + and not self.statements_embedded.is_here(): self.to_summary() self.page.to_documents() if self.documents.is_here(): @@ -209,7 +206,7 @@ def to_statements(self, id_=None, year=None): @need_login def to_statement(self, uri): - for i in xrange(self.MAX_RETRIES): + for i in range(self.MAX_RETRIES): self.location(uri) if self.statement.is_here(): break @@ -242,6 +239,7 @@ def iter_history(self, account): for trans in self.page.iter_transactions(): yield trans + LOGIN_JS = u'''\ var page = require('webpage').create(); diff --git a/modules/wellsfargo/pages.py b/modules/wellsfargo/pages.py index 0d327c41db..02884c534c 100644 --- a/modules/wellsfargo/pages.py +++ b/modules/wellsfargo/pages.py @@ -17,20 +17,29 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . -from weboob.capabilities.bank import Account, Transaction -from weboob.tools.capabilities.bank.transactions import \ - AmericanTransaction as AmTr -from weboob.browser.pages import HTMLPage, LoggedPage, RawPage -from decimal import Decimal -from requests.cookies import morsel_to_cookie -from .parsers import StatementParser, clean_label -from time import sleep +import Cookie +import datetime import itertools import json -import re import os -import datetime -import Cookie +import re +from decimal import Decimal +from time import sleep + +from requests.cookies import morsel_to_cookie + +from weboob.browser.pages import HTMLPage, LoggedPage, RawPage +from weboob.capabilities.bank import Account, Transaction +from weboob.tools.capabilities.bank.transactions import AmericanTransaction as AmTr + +from .parsers import StatementParser, clean_label + + +try: + cmp = cmp +except NameError: + def cmp(x, y): + return (x > y) - (x < y) class LoginProceedPage(LoggedPage, HTMLPage): @@ -98,7 +107,7 @@ def submit_code(self): except IOError: sleep(1) os.remove(self.browser.code_file) - self.browser.logger.info('The code %s has been successfully read'%code) + self.browser.logger.info('The code %s has been successfully read' % code) form = self.get_form(name='otp') form['passcode'] = [code] del form['cancelBtn'] @@ -122,7 +131,7 @@ def to_documents(self): class AccountPage(object): def account_id(self, name=None): if name: - return name[-4:] # Last 4 digits of "BLAH XXXXXXX1234" + return name[-4:] # Last 4 digits of "BLAH XXXXXXX1234" else: return self.account_id(self.account_name()) @@ -391,7 +400,7 @@ def is_here(self): def get_embedded_data(self): scr = self.doc.xpath(self.SCRIPT_XPATH)[0] data = json.loads('\n'.join(scr.split('\n')[2:-2]).replace( - "'appendTo'",'"appendTo"')) + "'appendTo'", '"appendTo"')) return json.loads(data['data']) def parser(self): @@ -401,8 +410,8 @@ def parser(self): class WfJsonPage(LoggedPage, RawPage): def __init__(self, *args, **kwArgs): RawPage.__init__(self, *args, **kwArgs) - clean = self.doc.replace('"/*WellFargoProprietary%','') \ - .replace('%WellFargoProprietary*/"','').decode('string_escape') + clean = self.doc.replace('"/*WellFargoProprietary%', '') \ + .replace('%WellFargoProprietary*/"', '').decode('string_escape') self.doc = json.loads(clean) diff --git a/modules/wellsfargo/parsers.py b/modules/wellsfargo/parsers.py index 19e7159f8c..ad229caf39 100644 --- a/modules/wellsfargo/parsers.py +++ b/modules/wellsfargo/parsers.py @@ -23,6 +23,7 @@ from weboob.tools.date import closest_date from weboob.tools.pdf import decompress_pdf from weboob.tools.tokenizer import ReTokenizer +from weboob.tools.compat import unicode import re import datetime @@ -124,7 +125,7 @@ def read_card_transaction(self, pos, date_from, date_to): range_plus = (0, INDENT_CHARGES)) if tdate is None or pdate_layout is None or pdate is None \ - or ref_layout is None or ref is None or desc is None or amount is None: + or ref_layout is None or ref is None or desc is None or amount is None: return startPos, None else: tdate = closest_date(tdate, date_from, date_to) @@ -262,19 +263,19 @@ def read_star_2(self, pos): def read_date(self, pos): def parse_date(v): - for year in [1900, 1904]: # try leap and non-leap years + for year in [1900, 1904]: # try leap and non-leap years fullstr = '%s/%i' % (v, year) try: return datetime.datetime.strptime(fullstr, '%m/%d/%Y') except ValueError as e: - pass - raise e + last_error = e + raise last_error return self._tok.simple_read('date', pos, parse_date) def read_text(self, pos): t = self._tok.tok(pos) - #TODO: handle PDF encodings properly. + # TODO: handle PDF encodings properly. return (pos+1, unicode(t.value(), errors='ignore')) \ if t.is_text() else (pos, None) diff --git a/modules/yomoni/compat/weboob_capabilities_bank.py b/modules/yomoni/compat/weboob_capabilities_bank.py index 404e8d30ed..141097d781 100644 --- a/modules/yomoni/compat/weboob_capabilities_bank.py +++ b/modules/yomoni/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 -- GitLab