diff --git a/modules/afer/compat/weboob_capabilities_bank.py b/modules/afer/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/afer/compat/weboob_capabilities_bank.py +++ b/modules/afer/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/afer/pages.py b/modules/afer/pages.py index bdfefc3e24c9e9e482b47cb18e0642b18746c437..86e5f287142f09cc6ccba26be6efbc381511c5e9 100644 --- a/modules/afer/pages.py +++ b/modules/afer/pages.py @@ -28,7 +28,7 @@ from weboob.browser.pages import HTMLPage, LoggedPage, pagination from weboob.capabilities.bank import Account, Investment, Transaction from weboob.capabilities.base import NotAvailable -from weboob.exceptions import BrowserUnavailable +from weboob.exceptions import BrowserUnavailable, ActionNeeded class LoginPage(HTMLPage): @@ -51,6 +51,10 @@ class IndexPage(LoggedPage, HTMLPage): def on_load(self): HTMLPage.on_load(self) + msg = CleanText('//div[has-class("form-input-label")]', default='')(self.doc) + if "prendre connaissance des nouvelles conditions" in msg: + raise ActionNeeded(msg) + # website sometime crash if self.doc.xpath(u'//div[@id="divError"]/span[contains(text(),"Une erreur est survenue")]'): raise BrowserUnavailable() diff --git a/modules/allocine/browser.py b/modules/allocine/browser.py index 70ccd6e75c5705ac35e36bfb8b205d9a5731f983..47895acb0d54be86d5854968aeea5cb5f662c222 100644 --- a/modules/allocine/browser.py +++ b/modules/allocine/browser.py @@ -17,19 +17,20 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . -from weboob.capabilities.calendar import BaseCalendarEvent, TRANSP, STATUS, CATEGORIES -from weboob.capabilities.collection import Collection -from weboob.capabilities.video import BaseVideo -from weboob.capabilities.image import Thumbnail -from weboob.capabilities.base import NotAvailable, NotLoaded, find_object -from weboob.capabilities.cinema import Movie, Person -from weboob.browser.browsers import APIBrowser -from weboob.browser.profiles import Android -from weboob.tools.compat import urlencode import base64 import hashlib -from datetime import datetime, date, timedelta import time +from datetime import date, datetime, timedelta + +from weboob.browser.browsers import APIBrowser +from weboob.browser.profiles import Android +from weboob.capabilities.base import NotAvailable, NotLoaded, find_object +from weboob.capabilities.calendar import CATEGORIES, STATUS, TRANSP, BaseCalendarEvent +from weboob.capabilities.cinema import Movie, Person +from weboob.capabilities.collection import Collection +from weboob.capabilities.image import Thumbnail +from weboob.capabilities.video import BaseVideo +from weboob.tools.compat import unicode, urlencode __all__ = ['AllocineBrowser'] diff --git a/modules/allocine/module.py b/modules/allocine/module.py index 58fd19309a2cdeecc6f2c5e0ec94eaa17de853e8..de5618c46bb417b7b42dc653fd71ba9f1dad05e2 100644 --- a/modules/allocine/module.py +++ b/modules/allocine/module.py @@ -19,11 +19,12 @@ import re from weboob.capabilities.base import UserError -from weboob.capabilities.calendar import CapCalendarEvent, CATEGORIES, BaseCalendarEvent -from weboob.capabilities.video import CapVideo, BaseVideo -from weboob.capabilities.collection import CapCollection, CollectionNotFound, Collection -from weboob.capabilities.cinema import CapCinema, Person, Movie +from weboob.capabilities.calendar import CATEGORIES, BaseCalendarEvent, CapCalendarEvent +from weboob.capabilities.cinema import CapCinema, Movie, Person +from weboob.capabilities.collection import CapCollection, Collection, CollectionNotFound +from weboob.capabilities.video import BaseVideo, CapVideo from weboob.tools.backend import Module +from weboob.tools.compat import unicode from .browser import AllocineBrowser diff --git a/modules/amazon/browser.py b/modules/amazon/browser.py index 849939644521cc5058395d30a4ef50f508315d7a..574930f2a22d30f040c35dee798229caf7510eb7 100644 --- a/modules/amazon/browser.py +++ b/modules/amazon/browser.py @@ -41,7 +41,7 @@ class AmazonBrowser(LoginBrowser, StatesMixin): L_SUBSCRIBER = 'Nom : (.*) Modifier E-mail' login = URL(r'/ap/signin(.*)', LoginPage) - home = URL(r'/$', HomePage) + home = URL(r'/$', r'/\?language=\w+$', HomePage) panel = URL('/gp/css/homepage.html/ref=nav_youraccount_ya', PanelPage) subscriptions = URL(r'/ap/cnep(.*)', SubscriptionsPage) documents = URL(r'/gp/your-account/order-history\?opt=ab&digitalOrders=1(.*)&orderFilter=year-(?P.*)', @@ -170,7 +170,11 @@ def to_english(self, language): def iter_subscription(self): self.location(self.panel.go().get_sub_link()) - if not self.subscriptions.is_here(): + if self.home.is_here(): + if self.page.get_login_link(): + self.is_login() + self.location(self.page.get_panel_link()) + elif not self.subscriptions.is_here(): self.is_login() yield self.page.get_item() diff --git a/modules/amazon/pages.py b/modules/amazon/pages.py index 7455d59d27ebae3f69c64abe4291bbff1a424a85..47129b54d88eed4b5a716d26d61d92dea36759ce 100644 --- a/modules/amazon/pages.py +++ b/modules/amazon/pages.py @@ -35,6 +35,9 @@ class HomePage(HTMLPage): def get_login_link(self): return self.doc.xpath('//a[./span[contains(., "%s")]]/@href' % self.browser.L_SIGNIN)[0] + def get_panel_link(self): + return Link('//a[contains(@href, "homepage.html") and has-class(@nav-link)]')(self.doc) + class PanelPage(LoggedPage, HTMLPage): def get_sub_link(self): @@ -125,7 +128,7 @@ class item(ItemElement): obj_id = Format('%s_%s', Env('subid'), Field('_simple_id')) obj__pre_url = Format('/gp/shared-cs/ajax/invoice/invoice.html?orderId=%s&relatedRequestId=%s&isADriveSubscription=&isHFC=', Field('_simple_id'), Env('request_id')) - obj_url = Async('details') & Link('//a[contains(@href, "download")]') + obj_url = Async('details') & Link('//a[contains(@href, "download")]|//a[contains(@href, "generated_invoices")]') obj_format = 'pdf' obj_label = Format('Facture %s', Field('_simple_id')) obj_type = 'bill' diff --git a/modules/amazonstorecard/browser.py b/modules/amazonstorecard/browser.py index 3367e87e1fa66be795645c1f4697a9a696a70bd3..620242e58edcf3f8b92e48bdc0b8f3063c883176 100644 --- a/modules/amazonstorecard/browser.py +++ b/modules/amazonstorecard/browser.py @@ -18,18 +18,17 @@ # along with weboob. If not, see . -from weboob.capabilities.bank import AccountNotFound -from weboob.browser import LoginBrowser, URL, need_login -from weboob.exceptions import BrowserIncorrectPassword -from weboob.tools.compat import unquote import json import os +from subprocess import STDOUT, CalledProcessError, check_output from tempfile import mkstemp -from subprocess import check_output, STDOUT, CalledProcessError -from .pages import SomePage, StatementsPage, StatementPage, SummaryPage, \ - ActivityPage +from weboob.browser import URL, LoginBrowser, need_login +from weboob.capabilities.bank import AccountNotFound +from weboob.exceptions import BrowserIncorrectPassword +from weboob.tools.compat import unquote +from .pages import ActivityPage, SomePage, StatementPage, StatementsPage, SummaryPage __all__ = ['AmazonStoreCard'] @@ -66,24 +65,24 @@ def do_login(self): 'agent': self.session.headers['User-Agent']}) os.close(scrf) os.close(cookf) - for i in xrange(self.MAX_RETRIES): + for i in range(self.MAX_RETRIES): try: check_output(["phantomjs", scrn], stderr=STDOUT) break except CalledProcessError as error: - pass + last_error = error else: - raise error + raise last_error with open(cookn) as cookf: cookies = json.loads(cookf.read()) os.remove(scrn) os.remove(cookn) self.session.cookies.clear() for c in cookies: - for k in ['expiry', 'expires', 'httponly']: - c.pop(k, None) - c['value'] = unquote(c['value']) - self.session.cookies.set(**c) + for k in ['expiry', 'expires', 'httponly']: + c.pop(k, None) + c['value'] = unquote(c['value']) + self.session.cookies.set(**c) if not self.summary.go().logged: raise BrowserIncorrectPassword() @@ -106,6 +105,7 @@ def iter_history(self, account): for t in s.iter_transactions(): yield t + LOGIN_JS = u'''\ var TIMEOUT = %(timeout)s*1000; // milliseconds var page = require('webpage').create(); diff --git a/modules/amazonstorecard/module.py b/modules/amazonstorecard/module.py index 01187471e36c0e4de6456920bcf50487e797d65c..6794daf4c37e850099f2af556bec14fb49a96f7b 100644 --- a/modules/amazonstorecard/module.py +++ b/modules/amazonstorecard/module.py @@ -19,12 +19,11 @@ from weboob.capabilities.bank import CapBank -from weboob.tools.backend import Module, BackendConfig +from weboob.tools.backend import BackendConfig, Module from weboob.tools.value import ValueBackendPassword from .browser import AmazonStoreCard - __all__ = ['AmazonStoreCardModule'] @@ -39,16 +38,16 @@ class AmazonStoreCardModule(Module, CapBank): ValueBackendPassword('username', label='User ID', masked=False), ValueBackendPassword('password', label='Password'), ValueBackendPassword('phone', - label='Phone to send verification code to', masked=False), + label='Phone to send verification code to', masked=False), ValueBackendPassword('code_file', - label='File to read the verification code from', masked=False)) + label='File to read the verification code from', masked=False)) BROWSER = AmazonStoreCard def create_default_browser(self): - return self.create_browser(username = self.config['username'].get(), - password = self.config['password'].get(), - phone = self.config['phone'].get(), - code_file = self.config['code_file'].get()) + return self.create_browser(username=self.config['username'].get(), + password=self.config['password'].get(), + phone=self.config['phone'].get(), + code_file=self.config['code_file'].get()) def iter_accounts(self): return self.browser.iter_accounts() diff --git a/modules/amazonstorecard/pages.py b/modules/amazonstorecard/pages.py index 3c38726b43618682e1e01bf8de9b2db7a6211e53..17f9aa792e23cfb17d79b32bcdc59f54f57d8baf 100644 --- a/modules/amazonstorecard/pages.py +++ b/modules/amazonstorecard/pages.py @@ -26,10 +26,18 @@ from weboob.tools.pdf import decompress_pdf from weboob.tools.tokenizer import ReTokenizer from datetime import datetime, timedelta +from weboob.tools.compat import unicode import re import json +try: + cmp = cmp +except NameError: + def cmp(x, y): + return (x > y) - (x < y) + + class SomePage(HTMLPage): @property def logged(self): @@ -39,7 +47,7 @@ def logged(self): class SummaryPage(SomePage): def account(self): label = u' '.join(self.doc.xpath( - '//div[contains(@class,"myCreditCardDetails")]')[0]\ + '//div[contains(@class,"myCreditCardDetails")]')[0] .text_content().split()) balance = self.amount(u'Balance') cardlimit = self.doc.xpath( @@ -69,8 +77,8 @@ def account(self): def amount(self, name): return u''.join(self.doc.xpath( - u'//li[text()[.="%s"]]/../li[1]'%name)[0].text_content().split())\ - .replace(u'\xb7',u'.').replace(u'*',u'') + u'//li[text()[.="%s"]]/../li[1]' % name)[0].text_content().split())\ + .replace(u'\xb7', u'.').replace(u'*', u'') class ActivityPage(SomePage): @@ -98,19 +106,20 @@ def cmp_records(rec1, rec2): def parse_date(recdate): return datetime.strptime(recdate, u'%B %d, %Y') + class StatementsPage(SomePage): def iter_statements(self): jss = self.doc.xpath(u'//a/@onclick[contains(.,"eBillViewPDFAction")]') for js in jss: url = re.match("window.open\('([^']*).*\)", js).group(1) - for i in xrange(self.browser.MAX_RETRIES): + for i in range(self.browser.MAX_RETRIES): try: self.browser.location(url) break except ServerError as e: - pass + last_error = e else: - raise e + raise last_error yield self.browser.page @@ -163,7 +172,7 @@ def read_transaction(self, pos, date_from, date_to): pos, amount_layout = self.read_layout_td(pos) pos, amount = self.read_amount(pos) if tdate is None or pdate is None \ - or desc is None or amount is None or amount == 0: + or desc is None or amount is None or amount == 0: return startPos, None else: tdate = closest_date(tdate, date_from, date_to) @@ -208,7 +217,7 @@ def read_closing_date(self): def read_text(self, pos): t = self._tok.tok(pos) - #TODO: handle PDF encodings properly. + # TODO: handle PDF encodings properly. return (pos+1, unicode(t.value(), errors='ignore')) \ if t.is_text() else (pos, None) diff --git a/modules/ameli/pages.py b/modules/ameli/pages.py index 1118dbda27aae4992323782dc3772de3b0dcf27e..ccba3c455025809004b5cc0be8aa627616e7aada 100644 --- a/modules/ameli/pages.py +++ b/modules/ameli/pages.py @@ -85,7 +85,7 @@ class AccountPage(AmeliBasePage): def iter_subscription_list(self): names_list = self.doc.xpath('//span[@class="NomEtPrenomLabel"]') fullname = CleanText(newlines=True).filter(names_list[0]) - number = re.sub(r'[^\d]+', '', CleanText('//span[@class="blocNumSecu"]', replace=[(' ','')])(self.doc)) + number = re.sub(r'[^\d]+', '', CleanText('//span[@class="blocNumSecu"]', replace=[(' ', '')])(self.doc)) sub = Subscription(number) sub._id = number sub.label = fullname @@ -118,7 +118,7 @@ def iter_documents(self, sub): try: elt.xpath('.//a[contains(@id,"lienPDFReleve")]')[0] except IndexError: - continue + continue date_str = elt.xpath('.//span[contains(@id,"moisEnCours")]')[0].text month_str = date_str.split()[0] date = datetime.strptime(re.sub(month_str, str(FRENCH_MONTHS.index(month_str) + 1), date_str), "%m %Y").date() diff --git a/modules/amundi/browser.py b/modules/amundi/browser.py index 222cb8d31dda2b47d0cec2f4b79ebfffcd9f7c9c..a10dac38f4c3d5193b57ea6c36aa8acd296f3d67 100644 --- a/modules/amundi/browser.py +++ b/modules/amundi/browser.py @@ -17,8 +17,6 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . - -import ssl from .pages import LoginPage, AccountsPage, AccountHistoryPage from weboob.browser import URL, LoginBrowser, need_login from weboob.tools.json import json @@ -29,35 +27,18 @@ class AmundiBrowser(LoginBrowser): TIMEOUT = 120.0 - login = URL('/psf/authenticate', LoginPage) - authorize = URL('/psf/authorize', LoginPage) - accounts = URL('/psf/api/individu/positionFonds\?flagUrlFicheFonds=true&inclurePositionVide=false', AccountsPage) - account_history = URL('/psf/api/individu/operations\?valeurExterne=false&filtreStatutModeExclusion=false&statut=CPTA', AccountHistoryPage) - - def __init__(self, website, *args, **kwargs): - self.BASEURL = website - - super(AmundiBrowser, self).__init__(*args, **kwargs) - - def prepare_request(self, req): - """ - Amundi uses TLS v1.0. - """ - preq = super(AmundiBrowser, self).prepare_request(req) - conn = self.session.adapters['https://'].get_connection(preq.url) - conn.ssl_version = ssl.PROTOCOL_TLSv1 - return preq + login = URL(r'authenticate', LoginPage) + authorize = URL(r'authorize', LoginPage) + accounts = URL(r'api/individu/positionFonds\?flagUrlFicheFonds=true&inclurePositionVide=false', AccountsPage) + account_history = URL(r'api/individu/operations\?valeurExterne=false&filtreStatutModeExclusion=false&statut=CPTA', AccountHistoryPage) def do_login(self): """ Attempt to log in. Note: this method does nothing if we are already logged in. """ - assert isinstance(self.username, basestring) - assert isinstance(self.password, basestring) - try: - self.login.go(data=json.dumps({'username' : self.username, 'password' : self.password}), \ + self.login.go(data=json.dumps({'username': self.username, 'password': self.password}), headers={'Content-Type': 'application/json;charset=UTF-8'}) self.token = self.authorize.go().get_token() except ClientError: @@ -65,13 +46,22 @@ def do_login(self): @need_login def iter_accounts(self): - return self.accounts.go(headers={'X-noee-authorization': ('noeprd %s' % self.token)}).iter_accounts() + return (self.accounts.go(headers={'X-noee-authorization': ('noeprd %s' % self.token)}) + .iter_accounts()) @need_login def iter_investments(self, account): - return self.accounts.go(headers={'X-noee-authorization': ('noeprd %s' % self.token)})\ - .iter_investments(account_id=account.id) + return (self.accounts.go(headers={'X-noee-authorization': ('noeprd %s' % self.token)}) + .iter_investments(account_id=account.id)) @need_login def iter_history(self, account): - return self.account_history.go(headers={'X-noee-authorization': ('noeprd %s' % self.token)}).iter_history(account=account) + return (self.account_history.go(headers={'X-noee-authorization': ('noeprd %s' % self.token)}) + .iter_history(account=account)) + + +class EEAmundi(AmundiBrowser): + BASEURL = 'https://www.amundi-ee.com/psf/' + +class TCAmundi(AmundiBrowser): + BASEURL = 'https://epargnants.amundi-tc.com/psf/' diff --git a/modules/amundi/compat/weboob_capabilities_bank.py b/modules/amundi/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/amundi/compat/weboob_capabilities_bank.py +++ b/modules/amundi/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/amundi/module.py b/modules/amundi/module.py index d7e5666ac5e082793ab9af6d0932d25722a4954e..e4143c9a999915c953a8e69f4d5201476af5e383 100644 --- a/modules/amundi/module.py +++ b/modules/amundi/module.py @@ -23,7 +23,7 @@ from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword, Value -from .browser import AmundiBrowser +from .browser import EEAmundi, TCAmundi __all__ = ['AmundiModule'] @@ -35,59 +35,27 @@ class AmundiModule(Module, CapBankWealth): EMAIL = 'james.galt.bi@gmail.com' LICENSE = 'AGPLv3+' VERSION = '1.3' - CONFIG = BackendConfig(ValueBackendPassword('login', label='Identifiant', regexp='\d+', masked=False), - ValueBackendPassword('password', label=u"Mot de passe", regexp='\d+'), + CONFIG = BackendConfig(ValueBackendPassword('login', label='Identifiant', regexp=r'\d+', masked=False), + ValueBackendPassword('password', label=u"Mot de passe", regexp=r'\d+'), Value('website', label='Type de compte', default='ee', - choices={'ee': 'Amundi Epargne Entreprise', - 'tc': 'Amundi Tenue de Compte'})) - - BROWSER = AmundiBrowser + choices={'ee': 'Amundi Epargne Entreprise', + 'tc': 'Amundi Tenue de Compte'})) def create_default_browser(self): - w = {'ee': 'https://www.amundi-ee.com', 'tc': 'https://epargnants.amundi-tc.com'} - return self.create_browser(w[self.config['website'].get()], self.config['login'].get(), self.config['password'].get()) + b = {'ee': EEAmundi, 'tc': TCAmundi} + self.BROWSER = b[self.config['website'].get()] + return self.create_browser(self.config['login'].get(), self.config['password'].get()) def get_account(self, id): - """ - Get an account from its ID. - - :param id: ID of the account - :type id: :class:`str` - :rtype: :class:`Account` - :raises: :class:`AccountNotFound` - """ return find_object(self.iter_accounts(), id=id, error=AccountNotFound) - def iter_accounts(self): - """ - Iter accounts. - - :rtype: iter[:class:`Account`] - """ return self.browser.iter_accounts() - def iter_investment(self, account): - """ - Iter investment of a market account - - :param account: account to get investments - :type account: :class:`Account` - :rtype: iter[:class:`Investment`] - :raises: :class:`AccountNotFound` - """ for inv in self.browser.iter_investments(account): if inv.valuation != 0: yield inv def iter_history(self, account): - """ - Iter history of transactions on a specific account. - - :param account: account to get history - :type account: :class:`Account` - :rtype: iter[:class:`Transaction`] - :raises: :class:`AccountNotFound` - """ return self.browser.iter_history(account) diff --git a/modules/amundi/pages.py b/modules/amundi/pages.py index d28dec1d27f2bf8f91b9123b32e823fd0e39aa6a..2424af2e798b7ab7209b6e04b7d63dc3e2e1bb4b 100644 --- a/modules/amundi/pages.py +++ b/modules/amundi/pages.py @@ -17,6 +17,8 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals + from datetime import datetime from weboob.browser.elements import ItemElement, method, DictElement @@ -26,6 +28,7 @@ from weboob.capabilities.bank import Account, Investment, Transaction from weboob.capabilities.base import NotAvailable from weboob.exceptions import NoAccountsException +from weboob.tools.capabilities.bank.investments import is_isin_valid class LoginPage(JsonPage): @@ -59,7 +62,7 @@ def obj_number(self): # just the id is a kind of company id so it can be unique on a backend but not unique on multiple backends return '%s_%s' % (Field('id')(self), self.page.browser.username) - obj_currency = u"EUR" + obj_currency = 'EUR' def obj_type(self): return self.page.ACCOUNT_TYPES.get(Dict('typeDispositif')(self), Account.TYPE_LIFE_INSURANCE) @@ -67,7 +70,7 @@ def obj_type(self): def obj_label(self): try: return Dict('libelleDispositif')(self).encode('iso-8859-2').decode('utf8') - except (UnicodeEncodeError, UnicodeDecodeError): + except UnicodeError: try: return Dict('libelleDispositif')(self).encode('latin1').decode('utf8') except UnicodeDecodeError: @@ -92,12 +95,17 @@ class item(ItemElement): obj_code = Dict('codeIsin', default=NotAvailable) obj_vdate = Date(Dict('dtVl')) + def obj_code_type(self): + if is_isin_valid(Field('code')(self)): + return Investment.CODE_TYPE_ISIN + return NotAvailable + class AccountHistoryPage(LoggedPage, JsonPage): def belongs(self, instructions, account): for ins in instructions: - if 'nomDispositif' in ins and 'codeDispositif' in ins and '%s%s' % (ins['nomDispositif'], ins['codeDispositif']) == \ - '%s%s' % (account.label, account.id): + if 'nomDispositif' in ins and 'codeDispositif' in ins and '%s%s' % ( + ins['nomDispositif'], ins['codeDispositif']) == '%s%s' % (account.label, account.id): return True return False @@ -105,8 +113,9 @@ def get_amount(self, instructions, account): amount = 0 for ins in instructions: - if 'nomDispositif' in ins and 'montantNet' in ins and 'codeDispositif' in ins and '%s%s' % (ins['nomDispositif'], ins['codeDispositif']) == \ - '%s%s' % (account.label, account.id): + if ('nomDispositif' in ins and 'montantNet' in ins and 'codeDispositif' in ins + and '%s%s' % (ins['nomDispositif'], ins['codeDispositif']) + == '%s%s' % (account.label, account.id)): amount += ins['montantNet'] return CleanDecimal().filter(amount) @@ -119,21 +128,7 @@ def iter_history(self, account): tr.amount = self.get_amount(hist['instructions'], account) tr.rdate = datetime.strptime(hist['dateComptabilisation'].split('T')[0], '%Y-%m-%d') tr.date = tr.rdate - tr.label = hist['libelleOperation'] if 'libelleOperation' in hist else hist['libelleCommunication'] + tr.label = hist.get('libelleOperation') or hist['libelleCommunication'] tr.type = Transaction.TYPE_UNKNOWN - # Bypassed because we don't have the ISIN code - # tr.investments = [] - # for ins in hist['instructions']: - # inv = Investment() - # inv.code = NotAvailable - # inv.label = ins['nomFonds'] - # inv.description = ' '.join([ins['type'], ins['nomDispositif']]) - # inv.vdate = datetime.strptime(ins.get('dateVlReel', ins.get('dateVlExecution')).split('T')[ - # 0], '%Y-%m-%d') - # inv.valuation = Decimal(ins['montantNet']) - # inv.quantity = Decimal(ins['nombreDeParts']) - # inv.unitprice = inv.unitvalue = Decimal(ins['vlReel']) - # tr.investments.append(inv) - yield tr diff --git a/modules/apivie/compat/weboob_capabilities_bank.py b/modules/apivie/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/apivie/compat/weboob_capabilities_bank.py +++ b/modules/apivie/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/axabanque/browser.py b/modules/axabanque/browser.py index 6b3a44b42704990e4f0ef81b79b63036e24c2ccc..995960f41d1e3939f832c706cc42427d69fb235a 100644 --- a/modules/axabanque/browser.py +++ b/modules/axabanque/browser.py @@ -24,7 +24,7 @@ from dateutil.relativedelta import relativedelta import re -from weboob.browser import LoginBrowser, URL, need_login, StatesMixin +from weboob.browser.browsers import LoginBrowser, URL, need_login, StatesMixin from weboob.browser.exceptions import ClientError, HTTPNotFound from weboob.capabilities.base import NotAvailable from weboob.capabilities.bill import Subscription @@ -32,6 +32,7 @@ from weboob.exceptions import BrowserIncorrectPassword, ActionNeeded from weboob.tools.value import Value from weboob.tools.capabilities.bank.transactions import sorted_transactions +from .compat.weboob_tools_capabilities_bank_investments import create_french_liquidity from .pages.login import ( KeyboardPage, LoginPage, ChangepasswordPage, PredisconnectedPage, DeniedPage, @@ -39,9 +40,9 @@ ) from .pages.bank import ( AccountsPage as BankAccountsPage, CBTransactionsPage, TransactionsPage, - UnavailablePage, IbanPage, LifeInsuranceIframe, BoursePage, + UnavailablePage, IbanPage, LifeInsuranceIframe, BoursePage, BankProfilePage, ) -from .pages.wealth import AccountsPage as WealthAccountsPage, InvestmentPage, HistoryPage +from .pages.wealth import AccountsPage as WealthAccountsPage, InvestmentPage, HistoryPage, ProfilePage from .pages.transfer import ( RecipientsPage, AddRecipientPage, ValidateTransferPage, RegisterTransferPage, ConfirmTransferPage, RecipientConfirmationPage, @@ -135,6 +136,7 @@ class AXABanque(AXABrowser, StatesMixin): 'webapp/axabanque/jsp/virementSepa/saisieVirementSepa.faces', RegisterTransferPage) confirm_transfer = URL('/webapp/axabanque/jsp/virementSepa/confirmationVirementSepa.faces', ConfirmTransferPage) + profile_page = URL('/transactionnel/client/coordonnees.html', BankProfilePage) reload_state = None @@ -174,8 +176,8 @@ def iter_accounts(self): ids.add(a.id) - #The url giving life insurrance investments seems to be temporary. - #That's why we have to get them now + # The url giving life insurrance investments seems to be temporary. + # That's why we have to get them now if a.type == a.TYPE_LIFE_INSURANCE: self.cache['invs'][a.id] = list(self.open(a._url).page.iter_investment()) args = a._args @@ -186,7 +188,7 @@ def iter_accounts(self): 'codeFamille': args['paramCodeFamille'], 'codeProduit': args['paramCodeProduit'], 'codeSousProduit': args['paramCodeSousProduit'] - } + } try: r = self.open('/webapp/axabanque/popupPDF', params=iban_params) a.iban = r.page.get_iban() @@ -256,7 +258,7 @@ def iter_investment(self, account): self.transactions.go() if account._acctype == 'bank' and account.type in (Account.TYPE_PEA, Account.TYPE_MARKET): if 'Liquidités' in account.label: - return [self.page.get_liquidity_investment(account)] + return [create_french_liquidity(account.balance)] account = self.get_netfinca_account(account) self.location(account._market_link) @@ -290,7 +292,7 @@ def iter_history(self, account): self.location(acc._market_link) self.bourse_history.go() - if not 'Liquidités' in account.label: + if 'Liquidités' not in account.label: self.page.go_history_filter(cash_filter="market") else: self.page.go_history_filter(cash_filter="liquidity") @@ -488,6 +490,11 @@ def iter_documents(self, subscription): def download_document(self, url): raise NotImplementedError() + @need_login + def get_profile(self): + self.profile_page.go() + return self.page.get_profile() + class AXAAssurance(AXABrowser): BASEURL = 'https://espaceclient.axa.fr' @@ -500,6 +507,7 @@ class AXAAssurance(AXABrowser): download = URL('/content/ecc-popin-cards/technical/detailed/document.downloadPdf.html', '/content/ecc-popin-cards/technical/detailed/document/_jcr_content/', DownloadPage) + profile = URL(r'/content/ecc-popin-cards/transverse/userprofile.content-inner.html\?_=\d+', ProfilePage) def __init__(self, *args, **kwargs): super(AXAAssurance, self).__init__(*args, **kwargs) @@ -578,3 +586,8 @@ def download_document(self, url): self.location(url) self.page.create_document() return self.page.content + + @need_login + def get_profile(self): + self.profile.go() + return self.page.get_profile() diff --git a/modules/axabanque/compat/weboob_capabilities_bank.py b/modules/axabanque/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/axabanque/compat/weboob_capabilities_bank.py +++ b/modules/axabanque/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/axabanque/compat/weboob_tools_capabilities_bank_investments.py b/modules/axabanque/compat/weboob_tools_capabilities_bank_investments.py new file mode 100644 index 0000000000000000000000000000000000000000..2b68ff9e001c726a90445e05e0a8ef2e2f165059 --- /dev/null +++ b/modules/axabanque/compat/weboob_tools_capabilities_bank_investments.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Jonathan Schmidt +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.tools.compat import basestring +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.bank import Investment + +def is_isin_valid(isin): + """ + Méthode générale + Table de conversion des lettres en chiffres + A=10 B=11 C=12 D=13 E=14 F=15 G=16 H=17 I=18 + J=19 K=20 L=21 M=22 N=23 O=24 P=25 Q=26 R=27 + S=28 T=29 U=30 V=31 W=32 X=33 Y=34 Z=35 + + 1 - Mettre de côté la clé, qui servira de référence à la fin de la vérification. + 2 - Convertir toutes les lettres en nombres via la table de conversion ci-contre. Si le nombre obtenu est supérieur ou égal à 10, prendre les deux chiffres du nombre séparément (exemple : 27 devient 2 et 7). + 3 - Pour chaque chiffre, multiplier sa valeur par deux si sa position est impaire en partant de la droite. Si le nombre obtenu est supérieur ou égal à 10, garder les deux chiffres du nombre séparément (exemple : 14 devient 1 et 4). + 4 - Faire la somme de tous les chiffres. + 5 - Soustraire cette somme de la dizaine supérieure ou égale la plus proche (exemples : si la somme vaut 22, la dizaine « supérieure ou égale » est 30, et la clé vaut donc 8 ; si la somme vaut 30, la dizaine « supérieure ou égale » est 30, et la clé vaut 0 ; si la somme vaut 31, la dizaine « supérieure ou égale » est 40, et la clé vaut 9). + 6 - Comparer la valeur obtenue à la clé mise initialement de côté. + + Étapes 1 et 2 : + F R 0 0 0 3 5 0 0 0 0 (+ 8 : clé) + 15 27 0 0 0 3 5 0 0 0 0 + + Étape 3 : le traitement se fait sur des chiffres + 1 5 2 7 0 0 0 3 5 0 0 0 0 + I P I P I P I P I P I P I : position en partant de la droite (P = Pair, I = Impair) + 2 1 2 1 2 1 2 1 2 1 2 1 2 : coefficient multiplicateur + + 2 5 4 7 0 0 0 3 10 0 0 0 0 : résultat + + Étape 4 : + 2 + 5 + 4 + 7 + 0 + 0 + 0 + 3 + (1 + 0)+ 0 + 0 + 0 + 0 = 22 + + Étapes 5 et 6 : 30 - 22 = 8 (valeur de la clé) + """ + + if not isinstance(isin, basestring): + return False + if not re.match(r'^[A-Z]{2}[A-Z0-9]{9}\d$', isin): + return False + + isin_in_digits = ''.join(str(ord(x) - ord('A') + 10) if not x.isdigit() else x for x in isin[:-1]) + key = isin[-1:] + result = '' + for k, val in enumerate(isin_in_digits[::-1], start=1): + if k % 2 == 0: + result = ''.join((result, val)) + else: + result = ''.join((result, str(int(val)*2))) + return str(sum(int(x) for x in result) + int(key))[-1] == '0' + + +def create_french_liquidity(valuation): + """ + Automatically fills a liquidity investment with label, code and code_type. + """ + liquidity = Investment() + liquidity.label = "Liquidités" + liquidity.code = "XX-liquidity" + liquidity.code_type = NotAvailable + liquidity.valuation = valuation + return liquidity + diff --git a/modules/axabanque/module.py b/modules/axabanque/module.py index b2d6dc682a5cd1846e29f15e3ce6debc57737197..c7fe6418b3204fe579b266e9569e099b4ffa67cf 100644 --- a/modules/axabanque/module.py +++ b/modules/axabanque/module.py @@ -21,6 +21,7 @@ from .compat.weboob_capabilities_bank import CapBankWealth, CapBankTransferAddRecipient, AccountNotFound, RecipientNotFound from weboob.capabilities.base import find_object, NotAvailable from weboob.capabilities.bank import Account, TransferInvalidLabel +from weboob.capabilities.profile import CapProfile from weboob.capabilities.bill import CapDocument, Subscription, Document, DocumentNotFound, SubscriptionNotFound from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword @@ -31,7 +32,7 @@ __all__ = ['AXABanqueModule'] -class AXABanqueModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapDocument): +class AXABanqueModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapDocument, CapProfile): NAME = 'axabanque' MAINTAINER = u'Romain Bignon' EMAIL = 'romain@weboob.org' @@ -81,9 +82,10 @@ def init_transfer(self, transfer, **params): raise TransferInvalidLabel() self.logger.info('Going to do a new transfer') - if transfer.account_iban: - account = find_object(self.iter_accounts(), iban=transfer.account_iban, error=AccountNotFound) - else: + + # origin account iban can be NotAvailable + account = find_object(self.iter_accounts(), iban=transfer.account_iban) + if not account: account = find_object(self.iter_accounts(), id=transfer.account_id, error=AccountNotFound) if transfer.recipient_iban: @@ -106,7 +108,17 @@ def transfer_check_label(self, old, new): def transfer_check_account_id(self, old, new): old = old[:11] - return super(AXABanqueModule, self).transfer_check_label(old, new) + return old == new + + def transfer_check_account_iban(self, old, new): + # Skip origin account iban check and force origin account iban + if new is NotAvailable: + self.logger.warning( + 'Origin account iban check (%s) is not possible because iban is currently not available', + old, + ) + return True + return old == new def iter_subscription(self): return self.browser.get_subscription_list() @@ -139,3 +151,6 @@ def iter_resources(self, objs, split_path): if Subscription in objs: self._restrict_level(split_path) return self.iter_subscription() + + def get_profile(self): + return self.browser.get_profile() diff --git a/modules/axabanque/pages/bank.py b/modules/axabanque/pages/bank.py index 59cc7c20fc0ca438ac528335f20d0c80051e8c58..d50a13f5507b7dab1271791c315f78793e7294a8 100644 --- a/modules/axabanque/pages/bank.py +++ b/modules/axabanque/pages/bank.py @@ -29,10 +29,12 @@ from weboob.browser.filters.html import Attr, Link, TableCell from weboob.capabilities.bank import Account, Investment from weboob.capabilities.base import NotAvailable +from weboob.capabilities.profile import Person from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.tools.compat import unicode + def MyDecimal(*args, **kwargs): kwargs.update(replace_dots=True, default=NotAvailable) return CleanDecimal(*args, **kwargs) @@ -324,13 +326,6 @@ def obj_code(self): def condition(self): return CleanText(TableCell('valuation'))(self) - def get_liquidity_investment(self, account): - inv = Investment() - inv.label = u'Liquidités' - inv.code = 'XX-liquidity' - inv.valuation = account.balance - return inv - def more_history(self): link = None for a in self.doc.xpath('.//a'): @@ -568,3 +563,15 @@ def go_history_filter(self, cash_filter='all'): # We can't go above 2 years form['beginDayfilter'] = (datetime.strptime(form['endDayfilter'], '%d/%m/%Y') - timedelta(days=730)).strftime('%d/%m/%Y') form.submit() + + +class BankProfilePage(LoggedPage, HTMLPage): + @method + class get_profile(ItemElement): + klass = Person + + obj_email = CleanText('//form[@id="idCoordonneePersonnelle"]//table//strong[contains(text(), "e-mail")]/parent::td', children=False) + + obj_phone = CleanText('//form[@id="idCoordonneePersonnelle"]//table//strong[contains(text(), "mobile")]/parent::td', children=False) + + obj_address = Regexp(CleanText('//form[@id="idCoordonneePersonnelle"]//table//strong[contains(text(), "adresse fiscale")]/parent::td', children=False), '^(.*?)\/') diff --git a/modules/axabanque/pages/compat/weboob_capabilities_bank.py b/modules/axabanque/pages/compat/weboob_capabilities_bank.py new file mode 100644 index 0000000000000000000000000000000000000000..141097d781b97a933881efe1a6851a102454051e --- /dev/null +++ b/modules/axabanque/pages/compat/weboob_capabilities_bank.py @@ -0,0 +1,45 @@ + +import weboob.capabilities.bank as OLD + +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) + + +__all__ = OLD.__all__ + + +class CapBankWealth(CapBank): + pass + + +class CapBankPockets(CapBank): + pass + + +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + +class CapBankTransfer(OLD.CapBankTransfer): + def transfer_check_label(self, old, new): + from unidecode import unidecode + + return unidecode(old) == unidecode(new) + + +class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipient): + pass + + +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + +Account.TYPE_MORTGAGE = 17 +Account.TYPE_CONSUMER_CREDIT = 18 +Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/axabanque/pages/transfer.py b/modules/axabanque/pages/transfer.py index a8f7e756582b67940a8190d1dd4aa15787bfc639..a4851c1d40771af77f0226b79acce4832f1611c7 100644 --- a/modules/axabanque/pages/transfer.py +++ b/modules/axabanque/pages/transfer.py @@ -31,8 +31,8 @@ from weboob.browser.filters.standard import ( CleanText, Date, Regexp, CleanDecimal, Currency, Format, Field, ) -from weboob.capabilities.bank import ( - Recipient, Transfer, TransferBankError, AddRecipientError, RecipientNotFound, +from .compat.weboob_capabilities_bank import ( + Recipient, Transfer, TransferBankError, AddRecipientBankError, RecipientNotFound, ) from .compat.weboob_tools_captcha_virtkeyboard import SimpleVirtualKeyboard from weboob.capabilities.base import find_object, NotAvailable @@ -120,11 +120,11 @@ def on_load(self): ) if self.doc.xpath('//input[@class="erreur_champs"]'): - raise AddRecipientError(message="Le code entré est incorrect.") + raise AddRecipientBankError(message="Le code entré est incorrect.") for error_msg in errors_msg: if error_msg: - raise AddRecipientError(message=error_msg) + raise AddRecipientBankError(message=error_msg) def continue_new_recipient(self): continue_new_recipient_btn_id = CleanText('//input[@class="btn_continuer"]/@id')(self.doc) diff --git a/modules/axabanque/pages/wealth.py b/modules/axabanque/pages/wealth.py index 83f6c4176272a955075ba5754291d6f807322578..8fee376aa2837b05059a0120bddf1236e22c9fac 100644 --- a/modules/axabanque/pages/wealth.py +++ b/modules/axabanque/pages/wealth.py @@ -28,6 +28,7 @@ ) from weboob.browser.filters.html import Attr, Link, TableCell from weboob.capabilities.bank import Account, Investment +from weboob.capabilities.profile import Person from weboob.capabilities.base import NotAvailable, NotLoaded from weboob.tools.capabilities.bank.transactions import FrenchTransaction @@ -214,3 +215,16 @@ def obj_investments(self): for inv in investments: inv.vdate = Field('date')(self) return investments + + +class ProfilePage(LoggedPage, HTMLPage): + def get_profile(self): + form = self.get_form(xpath='//div[@class="popin-card"]') + + profile = Person() + + profile.name = '%s %s' % (form['party.first_name'], form['party.preferred_last_name']) + profile.address = '%s %s %s' % (form['mailing_address.street_line'], form['mailing_address.zip_postal_code'], form['mailing_address.locality']) + profile.email = CleanText('//label[@class="email-editable"]')(self.doc) + profile.phone = CleanText('//div[@class="info-title colorized phone-disabled"]//label', children=False)(self.doc) + return profile diff --git a/modules/banquepopulaire/browser.py b/modules/banquepopulaire/browser.py index ab5374f94d6703024b22a369425745d1660b0bad..130eac96c2dfd32acc0d0f4cad5621bc5e2e9a0e 100644 --- a/modules/banquepopulaire/browser.py +++ b/modules/banquepopulaire/browser.py @@ -26,9 +26,10 @@ from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable from weboob.browser.exceptions import HTTPNotFound, ServerError -from weboob.browser import LoginBrowser, URL, need_login -from weboob.capabilities.bank import Account, Investment +from weboob.browser.browsers import LoginBrowser, URL, need_login +from weboob.capabilities.bank import Account from weboob.capabilities.base import NotAvailable, find_object +from .compat.weboob_tools_capabilities_bank_investments import create_french_liquidity from .pages import ( LoggedOut, @@ -129,6 +130,7 @@ class BanquePopulaire(LoginBrowser): error_page = URL(r'https://[^/]+/cyber/internet/ContinueTask.do', r'https://[^/]+/_layouts/error.aspx', + r'https://[^/]+/portailinternet/_layouts/Ibp.Cyi.Administration/RedirectPageError.aspx', ErrorPage) unavailable_page = URL(r'https://[^/]+/s3f-web/.*', @@ -139,7 +141,7 @@ class BanquePopulaire(LoginBrowser): redirect_page = URL(r'https://[^/]+/portailinternet/_layouts/Ibp.Cyi.Layouts/RedirectSegment.aspx.*', RedirectPage) home_page = URL(r'https://[^/]+/portailinternet/Catalogue/Segments/.*.aspx(\?vary=(?P.*))?', r'https://[^/]+/portailinternet/Pages/.*.aspx\?vary=(?P.*)', - r'https://[^/]+/portailinternet/Pages/default.aspx', + r'https://[^/]+/portailinternet/Pages/[dD]efault.aspx', r'https://[^/]+/portailinternet/Transactionnel/Pages/CyberIntegrationPage.aspx', r'https://[^/]+/cyber/internet/ShowPortal.do\?token=.*', HomePage) @@ -169,6 +171,11 @@ class BanquePopulaire(LoginBrowser): def __init__(self, website, *args, **kwargs): self.BASEURL = 'https://%s' % website + # this url is required because the creditmaritime abstract uses an other url + if 'cmgo.creditmaritime' in self.BASEURL: + self.redirect_url = 'https://www.icgauth.creditmaritime.groupe.banquepopulaire.fr/dacsrest/api/v1u0/transaction/' + else: + self.redirect_url = 'https://www.icgauth.banquepopulaire.fr/dacsrest/api/v1u0/transaction/' self.token = None self.weboob = kwargs['weboob'] super(BanquePopulaire, self).__init__(*args, **kwargs) @@ -193,7 +200,6 @@ def do_login(self): if self.home_page.is_here(): return self.page.login(self.username, self.password) - if self.login_page.is_here(): raise BrowserIncorrectPassword() @@ -278,10 +284,9 @@ def get_account(self, id): return find_object(self.get_accounts_list(False), id=id) def set_gocardless_transaction_details(self, transaction): - # This method is not called for the moment, in order to prevent crash during get_history() # Setting references for a GoCardless transaction data = self.page.get_params() - data['validationStrategy'] = 'NV' + data['validationStrategy'] = self.page.get_gocardless_strategy_param(transaction) data['dialogActionPerformed'] = 'DETAIL_ECRITURE' attribute_key, attribute_value = self.page.get_transaction_table_id(transaction._ref) data[attribute_key] = attribute_value @@ -343,11 +348,9 @@ def get_history(self, account, coming=False): transaction_list = self.page.get_history(account, coming) for tr in transaction_list: - # Add information about GoCardless: - # This method does not work 100% and potentially causes crash in get_history. - # Therefore, for now we decided not to call it while iterating over transactions. - #if 'GoCardless' in tr.label and tr._has_link: - #self.set_gocardless_transaction_details(tr) + # Add information about GoCardless + if 'GoCardless' in tr.label and tr._has_link: + self.set_gocardless_transaction_details(tr) yield tr next_params = self.page.get_next_params() @@ -413,12 +416,7 @@ def get_investment(self, account): # Add "Liquidities" investment if the account is a "Compte titres PEA": if account.type == Account.TYPE_PEA and account.id.startswith('CPT'): - inv = Investment() - inv.label = 'Liquidités' - inv.code = 'XX-liquidity' - inv.code_type = NotAvailable - inv.valuation = account.balance - self.investments[account.id] = [inv] + self.investments[account.id] = [create_french_liquidity(account.balance)] return self.investments[account.id] if account.id in self.investments.keys() and self.investments[account.id] is False: diff --git a/modules/banquepopulaire/compat/weboob_capabilities_bank.py b/modules/banquepopulaire/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/banquepopulaire/compat/weboob_capabilities_bank.py +++ b/modules/banquepopulaire/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/banquepopulaire/compat/weboob_tools_capabilities_bank_investments.py b/modules/banquepopulaire/compat/weboob_tools_capabilities_bank_investments.py new file mode 100644 index 0000000000000000000000000000000000000000..2b68ff9e001c726a90445e05e0a8ef2e2f165059 --- /dev/null +++ b/modules/banquepopulaire/compat/weboob_tools_capabilities_bank_investments.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Jonathan Schmidt +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.tools.compat import basestring +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.bank import Investment + +def is_isin_valid(isin): + """ + Méthode générale + Table de conversion des lettres en chiffres + A=10 B=11 C=12 D=13 E=14 F=15 G=16 H=17 I=18 + J=19 K=20 L=21 M=22 N=23 O=24 P=25 Q=26 R=27 + S=28 T=29 U=30 V=31 W=32 X=33 Y=34 Z=35 + + 1 - Mettre de côté la clé, qui servira de référence à la fin de la vérification. + 2 - Convertir toutes les lettres en nombres via la table de conversion ci-contre. Si le nombre obtenu est supérieur ou égal à 10, prendre les deux chiffres du nombre séparément (exemple : 27 devient 2 et 7). + 3 - Pour chaque chiffre, multiplier sa valeur par deux si sa position est impaire en partant de la droite. Si le nombre obtenu est supérieur ou égal à 10, garder les deux chiffres du nombre séparément (exemple : 14 devient 1 et 4). + 4 - Faire la somme de tous les chiffres. + 5 - Soustraire cette somme de la dizaine supérieure ou égale la plus proche (exemples : si la somme vaut 22, la dizaine « supérieure ou égale » est 30, et la clé vaut donc 8 ; si la somme vaut 30, la dizaine « supérieure ou égale » est 30, et la clé vaut 0 ; si la somme vaut 31, la dizaine « supérieure ou égale » est 40, et la clé vaut 9). + 6 - Comparer la valeur obtenue à la clé mise initialement de côté. + + Étapes 1 et 2 : + F R 0 0 0 3 5 0 0 0 0 (+ 8 : clé) + 15 27 0 0 0 3 5 0 0 0 0 + + Étape 3 : le traitement se fait sur des chiffres + 1 5 2 7 0 0 0 3 5 0 0 0 0 + I P I P I P I P I P I P I : position en partant de la droite (P = Pair, I = Impair) + 2 1 2 1 2 1 2 1 2 1 2 1 2 : coefficient multiplicateur + + 2 5 4 7 0 0 0 3 10 0 0 0 0 : résultat + + Étape 4 : + 2 + 5 + 4 + 7 + 0 + 0 + 0 + 3 + (1 + 0)+ 0 + 0 + 0 + 0 = 22 + + Étapes 5 et 6 : 30 - 22 = 8 (valeur de la clé) + """ + + if not isinstance(isin, basestring): + return False + if not re.match(r'^[A-Z]{2}[A-Z0-9]{9}\d$', isin): + return False + + isin_in_digits = ''.join(str(ord(x) - ord('A') + 10) if not x.isdigit() else x for x in isin[:-1]) + key = isin[-1:] + result = '' + for k, val in enumerate(isin_in_digits[::-1], start=1): + if k % 2 == 0: + result = ''.join((result, val)) + else: + result = ''.join((result, str(int(val)*2))) + return str(sum(int(x) for x in result) + int(key))[-1] == '0' + + +def create_french_liquidity(valuation): + """ + Automatically fills a liquidity investment with label, code and code_type. + """ + liquidity = Investment() + liquidity.label = "Liquidités" + liquidity.code = "XX-liquidity" + liquidity.code_type = NotAvailable + liquidity.valuation = valuation + return liquidity + diff --git a/modules/banquepopulaire/module.py b/modules/banquepopulaire/module.py index 54ec323b616ed3de5305a0f16fbca8a5469e2cad..098dfca182b49d0084ed226fc7aedeff923d7d26 100644 --- a/modules/banquepopulaire/module.py +++ b/modules/banquepopulaire/module.py @@ -45,6 +45,7 @@ class BanquePopulaireModule(Module, CapBankWealth, CapContact, CapProfile): 'www.ibps.bpalc.banquepopulaire.fr' : u'Alsace Lorraine Champagne', 'www.ibps.bpaca.banquepopulaire.fr': u'Aquitaine Centre atlantique', 'www.ibps.atlantique.banquepopulaire.fr': u'Atlantique', + 'www.ibps.bpgo.banquepopulaire.fr': u'Grand Ouest', 'www.ibps.loirelyonnais.banquepopulaire.fr': u'Auvergne Rhône Alpes', 'www.ibps.bpaura.banquepopulaire.fr': u'Auvergne Rhône Alpes', 'www.ibps.banquedesavoie.banquepopulaire.fr': u'Banque de Savoie', @@ -74,6 +75,10 @@ def create_default_browser(self): ('loirelyonnais', 'bpaura'), ('alpes', 'bpaura'), ('massifcentral', 'bpaura'), + ('atlantique.creditmaritime', 'cmgo.creditmaritime'), + ('bretagnenormandie.cmm', 'cmgo'), + ('atlantique.banquepopulaire', 'bpgo.banquepopulaire'), + ('ouest.banquepopulaire', 'bpgo.banquepopulaire'), ] website = reduce(lambda a, kv: a.replace(*kv), repls, self.config['website'].get()) return self.create_browser(website, diff --git a/modules/banquepopulaire/pages.py b/modules/banquepopulaire/pages.py index f6b75fa29c623ec406aeadb2e60055271b995f5c..741b327cf202f46cce177fa4ebd6caddd4a6e813 100644 --- a/modules/banquepopulaire/pages.py +++ b/modules/banquepopulaire/pages.py @@ -257,6 +257,8 @@ class ErrorPage(LoggedPage, MyHTMLPage): def on_load(self): if CleanText('//script[contains(text(), "momentanément indisponible")]')(self.doc): raise BrowserUnavailable(u"Le service est momentanément indisponible") + elif CleanText('//h1[contains(text(), "Cette page est indisponible")]')(self.doc): + raise BrowserUnavailable('Cette page est indisponible') return super(ErrorPage, self).on_load() def get_token(self): @@ -308,7 +310,7 @@ class Login2Page(LoginPage): def request_url(self): transactionID = self.params['transactionID'] assert transactionID - return 'https://www.icgauth.banquepopulaire.fr/dacswebssoissuer/api/v1u0/transaction/%s' % transactionID + return self.browser.redirect_url + transactionID def on_load(self): if not self.browser.no_login: @@ -369,7 +371,7 @@ def is_here(self): doc = json.loads(self.response.text) if 'response' in doc: return doc['response']['status'] == 'AUTHENTICATION_SUCCESS' and 'saml2_post' in doc['response'] - except json.decoder.JSONDecodeError: + except ValueError: # not a json page # so it should be Login2Page return False @@ -444,7 +446,7 @@ class AccountsPage(LoggedPage, MyHTMLPage): u'Liste complète de mon épargne': Account.TYPE_SAVINGS, u'Mes comptes': Account.TYPE_CHECKING, u'Comptes en euros': Account.TYPE_CHECKING, - 'Mes comptes en devises': Account.TYPE_CHECKING, + u'Mes comptes en devises': Account.TYPE_CHECKING, u'Liste complète de mes comptes': Account.TYPE_CHECKING, u'Mes emprunts': Account.TYPE_LOAN, u'Liste complète de mes emprunts': Account.TYPE_LOAN, @@ -457,6 +459,7 @@ class AccountsPage(LoggedPage, MyHTMLPage): PATTERN = [(re.compile('.*Titres Pea.*'), Account.TYPE_PEA), (re.compile(".*Plan D'epargne En Actions.*"), Account.TYPE_PEA), + (re.compile(".*Compte Especes Pea.*"), Account.TYPE_PEA), (re.compile('.*Plan Epargne Retraite.*'), Account.TYPE_PERP), (re.compile('.*Titres.*'), Account.TYPE_MARKET), (re.compile('.*Selection Vie.*'),Account.TYPE_LIFE_INSURANCE), @@ -538,11 +541,8 @@ def iter_accounts(self, next_pages): balance_text = u''.join([txt.strip() for txt in tds[3].itertext()]) balance = FrenchTransaction.clean_amount(balance_text) account.balance = Decimal(balance or '0.0') + account.currency = currency or Account.get_currency(balance_text) - if currency is None: - account.currency = Account.get_currency(balance_text) - else: - account.currency = currency if account.type == account.TYPE_LOAN: account.balance = - abs(account.balance) @@ -741,6 +741,8 @@ def get_account_history(self): t = Transaction() + # get the column index of the link to access transaction details + # (only used for GoCardLess transactions so far) t._has_link = bool(tds[self.COL_DEBIT].findall('a') or tds[self.COL_CREDIT].findall('a')) # XXX We currently take the *value* date, but it will probably @@ -756,6 +758,7 @@ def get_account_history(self): t.parse(date, re.sub(r'[ ]+', ' ', raw), vdate) t.set_amount(credit, debit) + t._amount_type = 'debit' if t.amount == debit else 'credit' # Strip the balance displayed in transaction labels t.label = re.sub('solde en valeur : .*', '', t.label) @@ -842,6 +845,27 @@ def get_transaction_table_id(self, ref): return key, value + def get_gocardless_strategy_param(self, transaction): + # A form is filled and send with javascript + # the 'validationStrategy' parameter value only depends on the column + # index in which the link lies + # + # To get more details about how things are done, see the following javascript functions: + #- attachTableRowEvents (atre) + #- attachActiveSelectionEventsOnRow + #- astr + #- updateSelection (uds) + #- selectActionButton (sab) + #- a script element embedded in the html page (search for "tcl5", "tcl6") + + assert transaction._has_link + + if transaction._amount_type == 'debit': + return 'AV' + elif transaction._amount_type == 'credit': + return 'NV' + + class NatixisChoicePage(LoggedPage, HTMLPage): def on_load(self): message = CleanText('//span[@class="rf-msgs-sum"]', default='')(self.doc) diff --git a/modules/barclays/compat/weboob_capabilities_bank.py b/modules/barclays/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/barclays/compat/weboob_capabilities_bank.py +++ b/modules/barclays/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/barclays/pages.py b/modules/barclays/pages.py index 5e3bc6d692a5031dd30f5e485fcae7c697d97edd..de5e410f85f9979fc09947168576514bbc73622c 100644 --- a/modules/barclays/pages.py +++ b/modules/barclays/pages.py @@ -107,7 +107,8 @@ class AccountsPage(StatefulPage): 'Engagement/Crédits': Account.TYPE_LOAN, } ACCOUNT_EXTRA_TYPES = {'BMOOVIE': Account.TYPE_LIFE_INSURANCE, - 'B. GESTION VIE': Account.TYPE_LIFE_INSURANCE + 'B. GESTION VIE': Account.TYPE_LIFE_INSURANCE, + 'E VIE MILLEIS': Account.TYPE_LIFE_INSURANCE, } ACCOUNT_TYPE_TO_STR = {Account.TYPE_MARKET: 'TTR', Account.TYPE_CARD: 'CRT' @@ -165,9 +166,6 @@ def obj__multiple_type(self): return False - def validate(self, el): - return Field('type')(self) != Account.TYPE_UNKNOWN - class Transaction(FrenchTransaction): PATTERNS = [ @@ -282,6 +280,10 @@ def get_space_attrs(self, space): @method class iter_investments(TableElement): + + def condition(self): + return not self.xpath('//h1[text()="Aucune position"]') + head_xpath = '//table[@id="C4__TBL_Equity"]/thead/tr/th' item_xpath = '//table[@id="C4__TBL_Equity"]/tbody/tr' @@ -313,9 +315,6 @@ def obj_code(self): obj_portfolio_share = Eval(lambda x: x / 100, MyDecimal(TableCell('portfolio_share'))) obj_code_type = Investment.CODE_TYPE_ISIN - def condition(self): - return CleanDecimal(TableCell('quantity'), default=None)(self) is not None - class LifeInsuranceAccountPage(AbstractAccountPage): def is_here(self): @@ -358,6 +357,10 @@ def obj_date(self): @method class iter_investments(TableElement): + + def condition(self): + return not self.xpath('//h1[text()="Aucune position"]') + head_xpath = '//table[@class="table-support"]/thead/tr/th' item_xpath = '//table[@class="table-support"]/tbody/tr' diff --git a/modules/bforbank/browser.py b/modules/bforbank/browser.py index 0985c1a725187849234aed685e59d9b2058fefd0..16e00901b3f7b1450840aa76e00a40f80d51edc9 100644 --- a/modules/bforbank/browser.py +++ b/modules/bforbank/browser.py @@ -18,17 +18,18 @@ # along with weboob. If not, see . import datetime from dateutil.relativedelta import relativedelta - from weboob.exceptions import BrowserIncorrectPassword -from weboob.browser import LoginBrowser, URL, need_login -from weboob.capabilities.bank import Account, AccountNotFound, Investment +from weboob.browser.browsers import LoginBrowser, URL, need_login +from weboob.capabilities.bank import Account, AccountNotFound from weboob.capabilities.base import empty from weboob.tools.capabilities.bank.transactions import sorted_transactions +from weboob.tools.decorators import retry +from .compat.weboob_tools_capabilities_bank_investments import create_french_liquidity from .pages import ( LoginPage, ErrorPage, AccountsPage, HistoryPage, LoanHistoryPage, RibPage, LifeInsuranceList, LifeInsuranceIframe, LifeInsuranceRedir, - BoursePage, CardHistoryPage, CardPage, UserValidationPage, + BoursePage, CardHistoryPage, CardPage, UserValidationPage, BourseActionNeeded, ) from .spirica_browser import SpiricaBrowser @@ -59,6 +60,7 @@ class BforbankBrowser(LoginBrowser): ErrorPage) bourse_login = URL(r'/espace-client/titres/debranchementCaTitre/(?P\d+)') + bourse_action_needed = URL('https://bourse.bforbank.com/netfinca-titres/*', BourseActionNeeded) bourse = URL('https://bourse.bforbank.com/netfinca-titres/servlet/com.netfinca.frontcr.synthesis.HomeSynthesis', 'https://bourse.bforbank.com/netfinca-titres/servlet/com.netfinca.frontcr.account.*', BoursePage) @@ -198,6 +200,7 @@ def goto_lifeinsurance(self, account): self.location('https://client.bforbank.com/espace-client/assuranceVie') self.lifeinsurance_list.go() + @retry(AccountNotFound, tries=5) def goto_spirica(self, account): assert account.type == Account.TYPE_LIFE_INSURANCE self.goto_lifeinsurance(account) @@ -274,11 +277,7 @@ def iter_investment(self, account): # _especes is set during BoursePage accounts parsing. BoursePage # inherits from lcl module BoursePage if bourse_account._especes: - i = Investment() - i.valuation = bourse_account._especes - i.code = u"XX-liquidity" - i.label = u"Liquidités" - invs.append(i) + invs.append(create_french_liquidity(bourse_account._especes)) self.leave_espace_bourse() diff --git a/modules/bforbank/compat/weboob_capabilities_bank.py b/modules/bforbank/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/bforbank/compat/weboob_capabilities_bank.py +++ b/modules/bforbank/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/bforbank/compat/weboob_tools_capabilities_bank_investments.py b/modules/bforbank/compat/weboob_tools_capabilities_bank_investments.py new file mode 100644 index 0000000000000000000000000000000000000000..2b68ff9e001c726a90445e05e0a8ef2e2f165059 --- /dev/null +++ b/modules/bforbank/compat/weboob_tools_capabilities_bank_investments.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Jonathan Schmidt +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.tools.compat import basestring +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.bank import Investment + +def is_isin_valid(isin): + """ + Méthode générale + Table de conversion des lettres en chiffres + A=10 B=11 C=12 D=13 E=14 F=15 G=16 H=17 I=18 + J=19 K=20 L=21 M=22 N=23 O=24 P=25 Q=26 R=27 + S=28 T=29 U=30 V=31 W=32 X=33 Y=34 Z=35 + + 1 - Mettre de côté la clé, qui servira de référence à la fin de la vérification. + 2 - Convertir toutes les lettres en nombres via la table de conversion ci-contre. Si le nombre obtenu est supérieur ou égal à 10, prendre les deux chiffres du nombre séparément (exemple : 27 devient 2 et 7). + 3 - Pour chaque chiffre, multiplier sa valeur par deux si sa position est impaire en partant de la droite. Si le nombre obtenu est supérieur ou égal à 10, garder les deux chiffres du nombre séparément (exemple : 14 devient 1 et 4). + 4 - Faire la somme de tous les chiffres. + 5 - Soustraire cette somme de la dizaine supérieure ou égale la plus proche (exemples : si la somme vaut 22, la dizaine « supérieure ou égale » est 30, et la clé vaut donc 8 ; si la somme vaut 30, la dizaine « supérieure ou égale » est 30, et la clé vaut 0 ; si la somme vaut 31, la dizaine « supérieure ou égale » est 40, et la clé vaut 9). + 6 - Comparer la valeur obtenue à la clé mise initialement de côté. + + Étapes 1 et 2 : + F R 0 0 0 3 5 0 0 0 0 (+ 8 : clé) + 15 27 0 0 0 3 5 0 0 0 0 + + Étape 3 : le traitement se fait sur des chiffres + 1 5 2 7 0 0 0 3 5 0 0 0 0 + I P I P I P I P I P I P I : position en partant de la droite (P = Pair, I = Impair) + 2 1 2 1 2 1 2 1 2 1 2 1 2 : coefficient multiplicateur + + 2 5 4 7 0 0 0 3 10 0 0 0 0 : résultat + + Étape 4 : + 2 + 5 + 4 + 7 + 0 + 0 + 0 + 3 + (1 + 0)+ 0 + 0 + 0 + 0 = 22 + + Étapes 5 et 6 : 30 - 22 = 8 (valeur de la clé) + """ + + if not isinstance(isin, basestring): + return False + if not re.match(r'^[A-Z]{2}[A-Z0-9]{9}\d$', isin): + return False + + isin_in_digits = ''.join(str(ord(x) - ord('A') + 10) if not x.isdigit() else x for x in isin[:-1]) + key = isin[-1:] + result = '' + for k, val in enumerate(isin_in_digits[::-1], start=1): + if k % 2 == 0: + result = ''.join((result, val)) + else: + result = ''.join((result, str(int(val)*2))) + return str(sum(int(x) for x in result) + int(key))[-1] == '0' + + +def create_french_liquidity(valuation): + """ + Automatically fills a liquidity investment with label, code and code_type. + """ + liquidity = Investment() + liquidity.label = "Liquidités" + liquidity.code = "XX-liquidity" + liquidity.code_type = NotAvailable + liquidity.valuation = valuation + return liquidity + diff --git a/modules/bforbank/pages.py b/modules/bforbank/pages.py index c95f3723fed24aef2727b821dab4eb489079d8d8..cd5f636b8d12ff3325820c774871ebba37d58ed2 100644 --- a/modules/bforbank/pages.py +++ b/modules/bforbank/pages.py @@ -359,6 +359,18 @@ def get_redir(self): return match.group(1) +class BourseActionNeeded(LoggedPage, HTMLPage): + ENCODING = 'latin-1' + XPATH = "//div[contains(text(), 'Création ou modification de votre mot de passe trading')]" + + def is_here(self): + return CleanText(self.XPATH)(self.doc) + + def on_load(self): + error = CleanText(self.XPATH)(self.doc) + raise ActionNeeded(error) + + class BoursePage(AbstractPage): PARENT = 'lcl' PARENT_URL = 'bourse' diff --git a/modules/binck/compat/weboob_capabilities_bank.py b/modules/binck/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/binck/compat/weboob_capabilities_bank.py +++ b/modules/binck/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/biplan/browser.py b/modules/biplan/browser.py index 259bd8265e44ba0d4a7a12ffc623fe95809dd7a8..15ff3083919dc20d12098000e0e289ea0cead7e0 100644 --- a/modules/biplan/browser.py +++ b/modules/biplan/browser.py @@ -26,7 +26,7 @@ class BiplanBrowser(PagesBrowser): - BASEURL = 'http://www.lebiplan.org' + BASEURL = 'https://www.lebiplan.org' program_page = URL('/fr/biplan-prog-(?P<_category>.*).php', ProgramPage) event_page = URL('/(?P<_id>.*).html', EventPage) diff --git a/modules/bnpcards/corporate/pages.py b/modules/bnpcards/corporate/pages.py index f350153d6bf825422d3858d76a2da6b32cd0bf6b..a4cdf9a71e2e236f110b2f27907ac4ddb106e4ba 100644 --- a/modules/bnpcards/corporate/pages.py +++ b/modules/bnpcards/corporate/pages.py @@ -31,11 +31,10 @@ __all__ = ['LoginPage', 'ErrorPage', 'AccountsPage', 'TransactionsPage'] - class LoginPage(HTMLPage): def login(self, username, password): form = self.get_form(name='connecterForm') - form['type'] = '2' # Gestionnaire + form['type'] = '2' # Gestionnaire form['login'] = username form['pwd'] = password[:8] form.url = '/ce_internet_public/seConnecter.event.do' @@ -53,7 +52,9 @@ class iter_accounts(ListElement): class item(ItemElement): klass = Account - obj_id = Format('%s%s', CleanText('./td[2]', replace=[(' ', '')]), CleanText('./td[3]')) + obj_id = Format('%s%s%s', CleanText('./td[2]', replace=[(' ', '')]), CleanText('./td[3]'), + CleanText('./td[1]', replace=[(' ', '')])) + obj__owner = CleanText('./td[1]') obj_number = CleanText('./td[2]', replace=[(' ', '')]) obj_label = CleanText('./td[1]') @@ -128,7 +129,7 @@ def obj_amount(self): return CleanDecimal(replace_dots=False).filter(self.el.xpath('./td[5]')) - CleanDecimal(replace_dots=False).filter(self.el.xpath('./td[6]')) def is_not_sorted(self, order='down'): - translate = {'down':'bas','up':'haut'} + translate = {'down': 'bas', 'up': 'haut'} return len(self.doc.xpath('//table[@id="ContentTable_datas"]/thead/tr/th[1]//img[contains(@src, "tri_%s_on")]' % translate[order])) == 0 def sort(self, order='down'): diff --git a/modules/bnporc/compat/weboob_capabilities_bank.py b/modules/bnporc/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/bnporc/compat/weboob_capabilities_bank.py +++ b/modules/bnporc/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/bnporc/enterprise/browser.py b/modules/bnporc/enterprise/browser.py index cd6a8b944e0a62f52a05d62604edcdd03c649cfe..e7b752d27bd02924caf0ea363d8ee277819bbea1 100644 --- a/modules/bnporc/enterprise/browser.py +++ b/modules/bnporc/enterprise/browser.py @@ -19,14 +19,13 @@ from __future__ import unicode_literals -import re - from datetime import datetime from dateutil.rrule import rrule, MONTHLY from dateutil.relativedelta import relativedelta from weboob.browser import LoginBrowser, need_login +from weboob.capabilities.base import find_object from weboob.capabilities.bank import Account from weboob.exceptions import BrowserIncorrectPassword, BrowserForbidden from weboob.browser.url import URL @@ -34,7 +33,7 @@ from .pages import ( LoginPage, AuthPage, AccountsPage, AccountHistoryViewPage, AccountHistoryPage, - ActionNeededPage, TransactionPage, TokenPage, InvestPage + ActionNeededPage, TransactionPage, MarketPage, InvestPage ) @@ -61,8 +60,8 @@ class BNPEnterprise(LoginBrowser): transaction_detail = URL(r'/NCCPresentationWeb/e21/getOptBDDF.do', TransactionPage) invest = URL(r'/opcvm/lister-composition/afficher.do', InvestPage) - # the token page is used only if there are several type market accounts - token_inv = URL(r'/opcvm/lister-portefeuilles/afficher.do', TokenPage) + # The Market page is used only if there are several market accounts + market = URL(r'/opcvm/lister-portefeuilles/afficher.do', MarketPage) renew_pass = URL('/sommaire/PseRedirectPasswordConnect', ActionNeededPage) @@ -72,7 +71,7 @@ def __init__(self, *args, **kwargs): def do_login(self): self.login.go() - if self.login.is_here() is False: + if not self.login.is_here(): return data = {} @@ -89,22 +88,26 @@ def do_login(self): @need_login def get_accounts_list(self): accounts = [] - try: - self.token_inv.go() - if self.token_inv.is_here(): - marketaccount = self.page.market_search() - else: - # redirected to invest page - # it means that there is only 1 market account among the ones showed - marketaccount = [self.page.get_market_account_label()] - except BrowserForbidden: - marketaccount = [] - + # Fetch checking accounts: for account in self.accounts.stay_or_go().iter_accounts(): - label_tmp = re.search(r'(\d{4,})', account.label) - if (label_tmp and label_tmp.group(0) in marketaccount) or account.label in marketaccount: - account.type = Account.TYPE_MARKET accounts.append(account) + # Fetch market accounts: + try: + self.market.go() + if self.market.is_here(): + for market_account in self.page.iter_market_accounts(): + market_account.parent = find_object(accounts, label=market_account._parent) + accounts.append(market_account) + + elif self.invest.is_here(): + # Redirected to invest page, meaning there is only 1 market account. + # We thus create an Account object for this unique market account. + account = self.page.get_unique_market_account() + account.parent = find_object(accounts, label=account._parent) + accounts.append(account) + + except BrowserForbidden: + pass return accounts @@ -116,6 +119,9 @@ def get_account(self, _id): @need_login def iter_history(self, account): + # There is no available history for market accounts + if account.type == Account.TYPE_MARKET: + return [] return self._iter_history_base(account) def _iter_history_base(self, account): @@ -144,24 +150,30 @@ def _iter_history_base(self, account): @need_login def iter_coming_operations(self, account): + # There is no available coming operation for market accounts + if account.type == Account.TYPE_MARKET: + return [] + self.account_coming_view.go(identifiant=account.iban) self.account_coming.go(identifiant=account.iban) return self.page.iter_coming() @need_login def iter_investment(self, account): - if account.type == Account.TYPE_MARKET: - self.token_inv.go() - if self.token_inv.is_here(): + if account.type != Account.TYPE_MARKET: + return + + self.market.go() + # If there is more than one market account, we must fetch the account params: + if not account._unique: + if self.market.is_here(): token = self.page.get_token() id_invest = self.page.get_id(label=account.label) data = {"numeroCompte": id_invest, "_csrf": token} self.location('/opcvm/lister-composition/redirect-afficher.do', data=data) - else: - self.invest.go() - for tr in self.page.iter_investment(): - yield tr + for inv in self.page.iter_investment(): + yield inv @need_login def get_profile(self): diff --git a/modules/bnporc/enterprise/pages.py b/modules/bnporc/enterprise/pages.py index 297b20efddf91ba866e53937ce35ce2e0532d834..a02aca40f0b688935d747f9054c1b1b9e66ded18 100644 --- a/modules/bnporc/enterprise/pages.py +++ b/modules/bnporc/enterprise/pages.py @@ -30,7 +30,7 @@ from weboob.browser.elements import DictElement, ItemElement, method, TableElement from weboob.browser.filters.standard import ( CleanText, CleanDecimal, Date, Regexp, Format, Eval, BrowserURL, Field, - Async, + Async, Currency, ) from weboob.capabilities.bank import Transaction, Account, Investment from weboob.capabilities.profile import Person @@ -105,8 +105,10 @@ def on_load(self): class AccountsPage(LoggedPage, JsonPage): - TYPES = {u'Compte chèque': Account.TYPE_CHECKING, - u'Compte à vue': Account.TYPE_CHECKING} + TYPES = { + 'Compte chèque': Account.TYPE_CHECKING, + 'Compte à vue': Account.TYPE_CHECKING, + } @method class iter_accounts(DictElement): @@ -333,7 +335,43 @@ def get_redacted_card(self): return self.doc['carteNum'] -class TokenPage(LoggedPage, HTMLPage): +class MarketPage(LoggedPage, HTMLPage): + TYPES = { + 'comptes de titres': Account.TYPE_MARKET, + } + + @method + class iter_market_accounts(TableElement): + def condition(self): + return not self.el.xpath('//table[@id="table-portefeuille"]//tr/td[contains(text(), "Aucun portefeuille à afficher")]') + + item_xpath = '//table[@id="table-portefeuille"]/tbody[@class="main-content"]/tr' + head_xpath = '//table[@id="table-portefeuille"]/thead/tr/th/label' + + col_label = 'Portefeuille-titres' + col_balance = re.compile('Valorisation') + col__parent = re.compile('Compte courant') + + class item(ItemElement): + klass = Account + + # Market accounts have no IBAN so we use the account number and specify + # "MARKET" at the end to differentiate from its parent account. + obj_id = Format('%sMARKET', Regexp(CleanText(TableCell('label')), r'\*(.*)\*', default=None)) + obj_label = CleanText(TableCell('label')) + obj_balance = CleanDecimal(TableCell('balance'), replace_dots=True) + obj_currency = Currency(TableCell('balance')) + obj__parent = CleanText(TableCell('_parent')) + obj_coming = NotAvailable + obj_iban = NotAvailable + obj__unique = False + + def obj_type(self): + for key, type in self.page.TYPES.items(): + if key in CleanText(TableCell('label'))(self).lower(): + return type + return Account.TYPE_UNKNOWN + def get_token(self): return Attr('//meta[@name="_csrf"]', 'content')(self.doc) @@ -343,18 +381,23 @@ def get_id(self, label): if id_simple in CleanText(options)(self.doc): return CleanText(options.xpath('./@value'))(self) - def market_search(self): - marketaccount = [] - for account in self.doc.xpath('//div[@class="filterbox-content hide"]//select[@id="numero-compte-titre"]//option'): - account = CleanText(account)(self.doc) - temp = re.search(r'[0-9]+', account) - if temp != None: - marketaccount.append(temp.group(0)) - return marketaccount +class InvestPage(LoggedPage, HTMLPage): + @method + class get_unique_market_account(ItemElement): + klass = Account + + # Market accounts have no IBAN so we use the account number and specify + # "MARKET" at the end to differentiate it from its parent account. + obj_id = Format('%sMARKET', Regexp(CleanText('//div[@class="head"]/h2'), r'\*(.*)\*', default=None)) + obj_label = CleanText('//div[@class="head"]/h2') + obj_balance = CleanDecimal('//div[@id="apercu-val"]/h1', replace_dots=True) + obj_currency = Currency('//div[@id="apercu-val"]/h1') + obj_type = Account.TYPE_MARKET + obj__parent = CleanText('//h3/span[span[@class="info-cheque"]]', children=False) + obj__unique = True -class InvestPage(LoggedPage, HTMLPage): @method class iter_investment(TableElement): item_xpath = '//table[@class="csv-data-container hide"]//tr' @@ -367,6 +410,12 @@ class iter_investment(TableElement): col_valuation = 'Valorisation' col_diff = '+/- value' + """ + Note: Pagination is not handled yet for investments, if we find a + customer with more than 10 invests we might have to handle clicking + on the button to get 50 invests per page or check if there is a link. + """ + class item(ItemElement): klass = Investment @@ -379,8 +428,5 @@ class item(ItemElement): obj_code_type = lambda self: Investment.CODE_TYPE_ISIN if Field('code')(self) is not NotAvailable else NotAvailable def obj_code(self): - chaine = CleanText(TableCell('label'))(self) - return re.search(r'(\w+) - ', chaine).group(0)[:-3] - - def get_market_account_label(self): - return CleanText('//h3/span[span]', children=False)(self.doc) + string = CleanText(TableCell('label'))(self) + return re.search(r'(\w+) - ', string).group(0)[:-3] diff --git a/modules/bnporc/pp/browser.py b/modules/bnporc/pp/browser.py index cbf29d2f613ded7d263e1f078a77c6f1970b11bf..3242611064955de8dc81610a4bdf138e78c62619 100644 --- a/modules/bnporc/pp/browser.py +++ b/modules/bnporc/pp/browser.py @@ -40,9 +40,9 @@ LoginPage, AccountsPage, AccountsIBANPage, HistoryPage, TransferInitPage, ConnectionThresholdPage, LifeInsurancesPage, LifeInsurancesHistoryPage, LifeInsurancesDetailPage, NatioVieProPage, CapitalisationPage, - MarketListPage, MarketPage, MarketHistoryPage, MarketSynPage, + MarketListPage, MarketPage, MarketHistoryPage, MarketSynPage, BNPKeyboard, RecipientsPage, ValidateTransferPage, RegisterTransferPage, AdvisorPage, - AddRecipPage, ActivateRecipPage, ProfilePage, ListDetailCardPage, + AddRecipPage, ActivateRecipPage, ProfilePage, ListDetailCardPage, ListErrorPage, ) @@ -75,6 +75,8 @@ class BNPParibasBrowser(JsonBrowserMixin, LoginBrowser): 'SEEA-pa01/devServer/seeaserver', 'https://mabanqueprivee.bnpparibas.net/fr/espace-prive/comptes-et-contrats\?u=%2FSEEA-pa01%2FdevServer%2Fseeaserver', LoginPage) + + list_error_page = URL('https://mabanque.bnpparibas/rsc/contrib/document/properties/identification-fr-part-V1.json', ListErrorPage) con_threshold = URL('/fr/connexion/100-connexions', '/fr/connexion/mot-de-passe-expire', '/fr/espace-prive/100-connexions.*', @@ -129,6 +131,23 @@ def do_login(self): if self.login.is_here(): self.page.login(self.username, self.password) + def change_pass(self, oldpass, newpass): + res = self.open('/identification-wspl-pres/grille?accessible=false') + url = '/identification-wspl-pres/grille/%s' % res.json()['data']['idGrille'] + keyboard = self.open(url) + vk = BNPKeyboard(self, keyboard) + data = {} + data['codeAppli'] = 'PORTAIL' + data['idGrille'] = res.json()['data']['idGrille'] + data['typeGrille'] = res.json()['data']['typeGrille'] + data['confirmNouveauPassword'] = vk.get_string_code(newpass) + data['nouveauPassword'] = vk.get_string_code(newpass) + data['passwordActuel'] = vk.get_string_code(oldpass) + response = self.location('/mcs-wspl/rpc/modifiercodesecret', data=data) + if response.json().get('messageIden').lower() == 'nouveau mot de passe invalide': + return False + return True + @need_login def get_profile(self): self.profile.go(data=JSON({})) @@ -174,13 +193,20 @@ def get_accounts_list(self): # Fetching capitalisation contracts from the "Assurances Vie" space (some are not in the BNP API): params = self.natio_vie_pro.go().get_params() - self.capitalisation_page.go(params=params) - if self.capitalisation_page.is_here() and self.page.has_contracts(): - for account in self.page.iter_capitalisation(): - # Life Insurance accounts may appear BOTH in the API and the "Assurances Vie" domain, - # It is better to keep the API version since it contains the unitvalue: - if account.number not in [a.number for a in self.accounts_list]: - self.accounts_list.append(account) + try: + self.capitalisation_page.go(params=params) + except ServerError: + self.logger.warning("An Internal Server Error occurred") + else: + if self.capitalisation_page.is_here() and self.page.has_contracts(): + for account in self.page.iter_capitalisation(): + # Life Insurance accounts may appear BOTH in the API and the "Assurances Vie" domain, + # It is better to keep the API version since it contains the unitvalue: + if account.number not in [a.number for a in self.accounts_list]: + self.logger.warning("We found an account that only appears on the old BNP website.") + self.accounts_list.append(account) + else: + self.logger.warning("This account was skipped because it already appears in the API.") return iter(self.accounts_list) @@ -201,7 +227,7 @@ def iter_history(self, account, coming=False): try: self.market_list.go(data=JSON({})) except ServerError: - self.logger.warning("An Internal Server Error occured") + self.logger.warning("An Internal Server Error occurred") return iter([]) for market_acc in self.page.get_list(): if account.number[-4:] == market_acc['securityAccountNumber'][-4:]: @@ -260,7 +286,6 @@ def iter_investment(self, account): # Going to the "Assurances Vie" page natiovie_params = self.natio_vie_pro.go().get_params() self.capitalisation_page.go(params=natiovie_params) - # Fetching the form to get the contract investments: capitalisation_params = self.page.get_params(account) self.capitalisation_page.go(params=capitalisation_params) @@ -277,7 +302,7 @@ def iter_investment(self, account): try: self.market_list.go(data=JSON({})) except ServerError: - self.logger.warning("An Internal Server Error occured") + self.logger.warning("An Internal Server Error occurred") return iter([]) for market_acc in self.page.get_list(): if account.number[-4:] == market_acc['securityAccountNumber'][-4:] and not account.iban: @@ -287,7 +312,7 @@ def iter_investment(self, account): "securityAccountNumber": market_acc['securityAccountNumber'], })) except ServerError: - self.logger.warning("An Internal Server Error occured") + self.logger.warning("An Internal Server Error occurred") break return self.page.iter_investments() diff --git a/modules/bnporc/pp/compat/weboob_capabilities_bank.py b/modules/bnporc/pp/compat/weboob_capabilities_bank.py new file mode 100644 index 0000000000000000000000000000000000000000..141097d781b97a933881efe1a6851a102454051e --- /dev/null +++ b/modules/bnporc/pp/compat/weboob_capabilities_bank.py @@ -0,0 +1,45 @@ + +import weboob.capabilities.bank as OLD + +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) + + +__all__ = OLD.__all__ + + +class CapBankWealth(CapBank): + pass + + +class CapBankPockets(CapBank): + pass + + +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + +class CapBankTransfer(OLD.CapBankTransfer): + def transfer_check_label(self, old, new): + from unidecode import unidecode + + return unidecode(old) == unidecode(new) + + +class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipient): + pass + + +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + +Account.TYPE_MORTGAGE = 17 +Account.TYPE_CONSUMER_CREDIT = 18 +Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/bnporc/pp/pages.py b/modules/bnporc/pp/pages.py index 439be56989618022452253335eff8d58412b988d..e92d52f5fef055e9da6560b1e596d719a5f12572 100644 --- a/modules/bnporc/pp/pages.py +++ b/modules/bnporc/pp/pages.py @@ -32,7 +32,10 @@ from weboob.browser.filters.html import TableCell from weboob.browser.pages import JsonPage, LoggedPage, HTMLPage from weboob.capabilities import NotAvailable -from weboob.capabilities.bank import Account, Investment, Recipient, Transfer, TransferError, TransferBankError, AddRecipientError +from .compat.weboob_capabilities_bank import ( + Account, Investment, Recipient, Transfer, TransferError, TransferBankError, + AddRecipientBankError, +) from weboob.capabilities.contact import Advisor from .compat.weboob_capabilities_profile import Person, ProfileMissing from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable, BrowserPasswordExpired, ActionNeeded @@ -42,25 +45,12 @@ from weboob.tools.date import parse_french_date from weboob.tools.capabilities.bank.investments import is_isin_valid from weboob.tools.compat import unquote_plus +from weboob.tools.html import html2text class ConnectionThresholdPage(HTMLPage): - def change_pass(self, oldpass, newpass): - res = self.browser.open('/identification-wspl-pres/grille?accessible=false') - url = '/identification-wspl-pres/grille/%s' % res.json()['data']['idGrille'] - keyboard = self.browser.open(url) - vk = BNPKeyboard(self, keyboard) - data = {} - data['codeAppli'] = 'PORTAIL' - data['idGrille'] = res.json()['data']['idGrille'] - data['typeGrille'] = res.json()['data']['typeGrille'] - data['confirmNouveauPassword'] = vk.get_string_code(newpass) - data['nouveauPassword'] = vk.get_string_code(newpass) - data['passwordActuel'] = vk.get_string_code(oldpass) - response = self.browser.location('/mcs-wspl/rpc/modifiercodesecret', data=data) - if response.json().get('messageIden').lower() == 'nouveau mot de passe invalide': - return False - return True + NOT_REUSABLE_PASSWORDS_COUNT = 3 + """BNP disallows to reuse one of the three last used passwords.""" def make_date(self, yy, m, d): current = datetime.now().year @@ -109,20 +99,35 @@ def looks_legit(self, password): return True def on_load(self): + msg = CleanText('//div[@class="confirmation"]//span[span]')(self.doc) + + self.logger.warning('Password expired.') if not self.looks_legit(self.browser.password): # we may not be able to restore the password, so reject it - msg = CleanText('//div[@class="confirmation"]//span[span]')(self.doc) + self.logger.warning('Unable to restore it, it is not legit.') raise BrowserPasswordExpired(msg) - new_pass = ''.join([str((int(l) + 1) % 10) for l in self.browser.password]) - self.logger.warning('Password expired. Renewing it. Temporary password is %s', new_pass) - if not self.change_pass(self.browser.password, new_pass): - self.logger.warning('New temp password is rejected, giving up') - raise BrowserPasswordExpired() - - if not self.change_pass(new_pass, self.browser.password): + new_passwords = [] + for i in range(self.NOT_REUSABLE_PASSWORDS_COUNT): + new_pass = ''.join([str((int(l) + i + 1) % 10) for l in self.browser.password]) + if not self.looks_legit(new_pass): + self.logger.warning('One of rotating password is not legit') + raise BrowserPasswordExpired(msg) + new_passwords.append(new_pass) + + current_password = self.browser.password + for new_pass in new_passwords: + self.logger.warning('Renewing with temp password') + if not self.browser.change_pass(current_password, new_pass): + self.logger.warning('New temp password is rejected, giving up') + raise BrowserPasswordExpired(msg) + current_password = new_pass + + if not self.browser.change_pass(current_password, self.browser.password): self.logger.error('Could not restore old password!') + self.logger.warning('Old password restored.') + def cast(x, typ, default=None): try: @@ -145,11 +150,20 @@ class BNPKeyboard(GridVirtKeyboard): '8': 'c60b723b3d95a46416b34c2cbefba3ed', '9': 'a13b8c3617a7bf854590833ddfb97f1f'} - def __init__(self, page, image): + def __init__(self, browser, image): symbols = list('%02d' % x for x in range(1, 11)) super(BNPKeyboard, self).__init__(symbols, 5, 2, BytesIO(image.content), self.color, convert='RGB') - self.check_symbols(self.symbols, page.browser.responses_dirname) + self.check_symbols(self.symbols, browser.responses_dirname) + + +class ListErrorPage(JsonPage): + def get_error_message(self, error): + key = 'app.identification.erreur.' + str(error) + try: + return html2text(self.doc[key]) + except KeyError: + return None class LoginPage(JsonPage): @@ -183,25 +197,28 @@ def on_load(self): error = cast(self.get('errorCode'), int, 0) # you can find api documentation on errors here : https://mabanque.bnpparibas/rsc/contrib/document/properties/identification-fr-part-V1.json if error: - codes = [201, 21510, 203, 202] - msg = self.get('message') - if error in codes: + error_page = self.browser.list_error_page.open() + msg = error_page.get_error_message(error) + if not msg: + msg = self.get('message') + + wrongpass_codes = [201, 21510, 203, 202, 1001, 7] + actionNeeded_codes = [21501, 3, 4, 50] + if error in wrongpass_codes: raise BrowserIncorrectPassword(msg) - elif error == 1001: - # json says "Erreur lors de l'authentification Code retour : 1001 Code retour : 1001" - # but js message from "getErrorMessage" says "veuillez contacter votre conseiller"... - raise BrowserIncorrectPassword() - elif error == 21501: # "Rendez-vous sur le site de BNP Paribas pour gérer vos comptes" - raise ActionNeeded(msg) elif error == 21: # "Ce service est momentanément indisponible. Veuillez renouveler votre demande ultérieurement." -> In reality, account is blocked because of too much wrongpass raise ActionNeeded(u"Compte bloqué") - - self.logger.debug('Unexpected error at login: "%s" (code=%s)' % (msg, error)) + elif error in actionNeeded_codes: + raise ActionNeeded(msg) + elif error == 207: + raise BrowserUnavailable(msg) + else: + assert False, 'Unexpected error at login: "%s" (code=%s)' % (msg, error) def login(self, username, password): url = '/identification-wspl-pres/grille/%s' % self.get('data.grille.idGrille') keyboard = self.browser.open(url) - vk = BNPKeyboard(self, keyboard) + vk = BNPKeyboard(self.browser, keyboard) target = self.browser.BASEURL + 'SEEA-pa01/devServer/seeaserver' user_agent = self.browser.session.headers.get('User-Agent') or '' @@ -209,7 +226,7 @@ def login(self, username, password): idTelematique=username, password=vk.get_string_code(password), clientele=user_agent) - # XXX useless ? + # XXX useless? csrf = self.generate_token() response = self.browser.location(target, data={'AUTH': auth, 'CSRF': csrf}) @@ -791,7 +808,7 @@ class AddRecipPage(BNPPage): def on_load(self): code = cast(self.get('codeRetour'), int) if code: - raise AddRecipientError(message=self.get('message')) + raise AddRecipientBankError(message=self.get('message')) def get_recipient(self, recipient): r = Recipient() diff --git a/modules/boursorama/browser.py b/modules/boursorama/browser.py index c04a1829ee757124de3a43a9c5e4cfb0766005de..1d3c19e859bc3fa7534d4ceb468889ca986e0533 100644 --- a/modules/boursorama/browser.py +++ b/modules/boursorama/browser.py @@ -246,7 +246,7 @@ def get_debit_date(self, debit_date): def get_closing_date(self): for i, j in zip(self.deferred_card_calendar, self.deferred_card_calendar[1:]): - if i[0] < date.today() <= j[0]: + if i[1] < date.today() <= j[1]: return i[0] def get_card_transactions(self, account, coming): diff --git a/modules/boursorama/compat/weboob_capabilities_bank.py b/modules/boursorama/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/boursorama/compat/weboob_capabilities_bank.py +++ b/modules/boursorama/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/boursorama/compat/weboob_tools_capabilities_bank_investments.py b/modules/boursorama/compat/weboob_tools_capabilities_bank_investments.py new file mode 100644 index 0000000000000000000000000000000000000000..2b68ff9e001c726a90445e05e0a8ef2e2f165059 --- /dev/null +++ b/modules/boursorama/compat/weboob_tools_capabilities_bank_investments.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Jonathan Schmidt +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.tools.compat import basestring +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.bank import Investment + +def is_isin_valid(isin): + """ + Méthode générale + Table de conversion des lettres en chiffres + A=10 B=11 C=12 D=13 E=14 F=15 G=16 H=17 I=18 + J=19 K=20 L=21 M=22 N=23 O=24 P=25 Q=26 R=27 + S=28 T=29 U=30 V=31 W=32 X=33 Y=34 Z=35 + + 1 - Mettre de côté la clé, qui servira de référence à la fin de la vérification. + 2 - Convertir toutes les lettres en nombres via la table de conversion ci-contre. Si le nombre obtenu est supérieur ou égal à 10, prendre les deux chiffres du nombre séparément (exemple : 27 devient 2 et 7). + 3 - Pour chaque chiffre, multiplier sa valeur par deux si sa position est impaire en partant de la droite. Si le nombre obtenu est supérieur ou égal à 10, garder les deux chiffres du nombre séparément (exemple : 14 devient 1 et 4). + 4 - Faire la somme de tous les chiffres. + 5 - Soustraire cette somme de la dizaine supérieure ou égale la plus proche (exemples : si la somme vaut 22, la dizaine « supérieure ou égale » est 30, et la clé vaut donc 8 ; si la somme vaut 30, la dizaine « supérieure ou égale » est 30, et la clé vaut 0 ; si la somme vaut 31, la dizaine « supérieure ou égale » est 40, et la clé vaut 9). + 6 - Comparer la valeur obtenue à la clé mise initialement de côté. + + Étapes 1 et 2 : + F R 0 0 0 3 5 0 0 0 0 (+ 8 : clé) + 15 27 0 0 0 3 5 0 0 0 0 + + Étape 3 : le traitement se fait sur des chiffres + 1 5 2 7 0 0 0 3 5 0 0 0 0 + I P I P I P I P I P I P I : position en partant de la droite (P = Pair, I = Impair) + 2 1 2 1 2 1 2 1 2 1 2 1 2 : coefficient multiplicateur + + 2 5 4 7 0 0 0 3 10 0 0 0 0 : résultat + + Étape 4 : + 2 + 5 + 4 + 7 + 0 + 0 + 0 + 3 + (1 + 0)+ 0 + 0 + 0 + 0 = 22 + + Étapes 5 et 6 : 30 - 22 = 8 (valeur de la clé) + """ + + if not isinstance(isin, basestring): + return False + if not re.match(r'^[A-Z]{2}[A-Z0-9]{9}\d$', isin): + return False + + isin_in_digits = ''.join(str(ord(x) - ord('A') + 10) if not x.isdigit() else x for x in isin[:-1]) + key = isin[-1:] + result = '' + for k, val in enumerate(isin_in_digits[::-1], start=1): + if k % 2 == 0: + result = ''.join((result, val)) + else: + result = ''.join((result, str(int(val)*2))) + return str(sum(int(x) for x in result) + int(key))[-1] == '0' + + +def create_french_liquidity(valuation): + """ + Automatically fills a liquidity investment with label, code and code_type. + """ + liquidity = Investment() + liquidity.label = "Liquidités" + liquidity.code = "XX-liquidity" + liquidity.code_type = NotAvailable + liquidity.valuation = valuation + return liquidity + diff --git a/modules/boursorama/pages.py b/modules/boursorama/pages.py index 79f15916deecf6c75768edbd8d918693abb2567c..f97fabb95aa03b910255029301acf77c399a5b0e 100644 --- a/modules/boursorama/pages.py +++ b/modules/boursorama/pages.py @@ -35,10 +35,11 @@ ) from weboob.browser.filters.json import Dict from weboob.browser.filters.html import Attr, Link, TableCell, AbsoluteLink -from weboob.capabilities.bank import ( +from .compat.weboob_capabilities_bank import ( Account, Investment, Recipient, Transfer, AccountNotFound, - AddRecipientError, TransferInvalidAmount, + AddRecipientBankError, TransferInvalidAmount, ) +from .compat.weboob_tools_capabilities_bank_investments import create_french_liquidity from weboob.capabilities.base import NotAvailable, empty, Currency from weboob.capabilities.profile import Person from weboob.tools.capabilities.bank.transactions import FrenchTransaction @@ -366,7 +367,8 @@ class HistoryPage(LoggedPage, HTMLPage): class iter_history(ListElement): item_xpath = '//ul[has-class("list__movement")]/li[div and not(contains(@class, "summary")) \ and not(contains(@class, "graph")) \ - and not (contains(@class, "separator"))]' + and not (contains(@class, "separator")) \ + and not (contains(@class, "list__movement__line--deffered")) ]' class item(ItemElement): klass = Transaction @@ -577,11 +579,7 @@ def obj_unitvalue(self): def iter_investment(self): valuation = CleanDecimal('//li[h4[contains(text(), "Solde Espèces")]]/h3', replace_dots=True, default=None)(self.doc) if valuation: - inv = Investment() - inv.code = u"XX-liquidity" - inv.label = u"Liquidités" - inv.valuation = valuation - yield inv + yield create_french_liquidity(valuation) for inv in self.get_investment(): yield inv @@ -812,31 +810,35 @@ class get_profile(ItemElement): class CardsNumberPage(LoggedPage, HTMLPage): def populate_cards_number(self, cards): - # Checking account ID used to match credit cards - # Label useful when several cards on the same checking account - labels_ids = [ - ( - CleanText('.', replace=[('DEBIT DIFFERE ', '')])(o), - re.search(r'/limite/(\w+)/', o.attrib['href']).group(1) - ) - for o in self.doc.xpath('//select//option') + """ + Checking account ID used to match credit cards. + Label useful when several cards on the same checking account. + The card_details list contains tuples including the card number, + the hash of the card's parent account, and the name of the card. + """ + card_details = [ + (CleanText('.//span')(o), CleanText('./@data-account-key')(o), CleanText('.//p')(o)) + for o in self.doc.xpath('//div[contains(@class, "zoom-carousel__item text-center credit-card-carousel__item")]') ] - # remove cards whose number ends with **** (means it is not activated yet) - labels_ids = [ - label_id for label_id in labels_ids - if not label_id[0].endswith('****') + # Remove cards whose number ends with **** (means it is not activated yet) + card_details = [ + (number, account_hash, name) for (number, account_hash, name) in card_details + if not number.endswith('****') ] for card in cards: - match = [ - label for label, account_id in labels_ids - if card.label in label and account_id in card.url - ] + match = [(number, account_hash, name) for (number, account_hash, name) in card_details if account_hash in card.url] + + if len(match) > 1: + # Several cards matched the same bank account, so we now try + # to match them using the name of the card holder: + card_username = card.label.split('-')[1].lstrip() + match = [(number, account_hash, name) for (number, account_hash, name) in match if name.startswith(card_username)] + assert len(match) <= 1, "only one card should be matched, or zero if the card is not yet activated" if len(match) == 1 : - card_label = match[0] - card.number = re.search('(\d{4}\*{8}(\d{4}|\*{4}))', card_label).group(1) + card.number = match[0][0] class HomePage(LoggedPage, HTMLPage): @@ -983,7 +985,7 @@ def on_load(self): err = CleanText('//div[@class="form-errors"]', default=None)(self.doc) if err: - raise AddRecipientError(message=err) + raise AddRecipientBankError(message=err) def _is_form(self, **kwargs): try: diff --git a/modules/bouygues/pages.py b/modules/bouygues/pages.py index 965ea584b3419b3ba7f872c5aa004b2bca32656c..a172b003501d90cfdddf27c43d4db64795b180e9 100644 --- a/modules/bouygues/pages.py +++ b/modules/bouygues/pages.py @@ -22,7 +22,7 @@ from datetime import datetime, timedelta from weboob.capabilities.messages import CantSendMessage -from weboob.exceptions import BrowserIncorrectPassword +from weboob.exceptions import BrowserIncorrectPassword, ParseError from weboob.capabilities.base import NotLoaded from weboob.capabilities.bill import Bill, Subscription @@ -138,7 +138,16 @@ class item(ItemElement): klass = Bill obj_id = Format('%s_%s', Env('subid'), Dict('idFacture')) - obj_url = Format('https://api.bouyguestelecom.fr%s', Dict('_links/facturePDF/href')) + + def obj_url(self): + try: + link = Dict('_links/facturePDF/href')(self) + except ParseError: + # yes, sometimes it's just a misspelling word, but just sometimes... + link = Dict('_links/facturePDFDF/href')(self) + + return 'https://api.bouyguestelecom.fr%s' % link + obj_date = Env('date') obj_duedate = Env('duedate') obj_format = 'pdf' diff --git a/modules/bp/browser.py b/modules/bp/browser.py index 58d292370486d726b402b2a3e1d2cb370e1d9240..3033866d3f67349fd18b67251e5b47377b19c3cf 100644 --- a/modules/bp/browser.py +++ b/modules/bp/browser.py @@ -251,6 +251,16 @@ def get_accounts_list(self): for loan in self.page.iter_revolving_loans(): loan.currency = account.currency accounts.append(loan) + page.go() + + elif account.type == Account.TYPE_PERP: + # PERP balances must be fetched from the details page, + # otherwise we just scrape the "Rente annuelle estimée": + balance = self.open(account.url).page.get_balance() + if balance is not None: + account.balance = balance + accounts.append(account) + else: accounts.append(account) if account.type == Account.TYPE_CHECKING and account._has_cards: @@ -311,13 +321,16 @@ def get_history(self, account): Account.TYPE_MARKET: self.par_account_savings_and_invests_history }.get(account.type) - if history is not None: + if history is not None and account.label != 'COMPTE ATTENTE': history.go(accountId=account.id) # TODO be smarter by avoid fetching all, sorting all and returning all if only coming were desired if hasattr(self.page, 'iter_transactions') and self.page.has_transactions(): return self.page.iter_transactions() + elif account.type == Account.TYPE_PERP and self.retirement_hist.is_here(): + return self.page.get_history() + return [] diff --git a/modules/bp/compat/weboob_capabilities_bank.py b/modules/bp/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/bp/compat/weboob_capabilities_bank.py +++ b/modules/bp/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/bp/pages/accounthistory.py b/modules/bp/pages/accounthistory.py index 6f3961bee4e56f0aebcca96029a11b5ee81cd4e9..fd6b1bc3e4880c7de766ba8056650cfce4539afc 100644 --- a/modules/bp/pages/accounthistory.py +++ b/modules/bp/pages/accounthistory.py @@ -254,6 +254,10 @@ def on_load(self): if link: self.browser.location(link) + def get_balance(self): + return CleanDecimal(default=None, replace_dots=True).filter( + self.doc.xpath('//dt[span[text()="Total des versements bruts"]]/following::dd[1]/span/strong/text()')) + class InvestTable(TableElement): col_label = 'Support' diff --git a/modules/bp/pages/accountlist.py b/modules/bp/pages/accountlist.py index c4d00961d970f6f58f92ee48f05d0aa4105a9974..4160eb0e1c9ca3c4c8f409022f32168eec99f7be 100644 --- a/modules/bp/pages/accountlist.py +++ b/modules/bp/pages/accountlist.py @@ -104,8 +104,8 @@ def obj_type(self): 'comptes? titres? et pea': Account.TYPE_MARKET, 'compte-titres': Account.TYPE_MARKET, 'assurances? vie': Account.TYPE_LIFE_INSURANCE, - u'prêt': Account.TYPE_LOAN, - u'crédits?': Account.TYPE_LOAN, + 'prêt': Account.TYPE_LOAN, + 'crédits?': Account.TYPE_LOAN, 'plan d\'epargne en actions': Account.TYPE_PEA, 'comptes? attente': Account.TYPE_CHECKING, 'perp': Account.TYPE_PERP, @@ -222,8 +222,8 @@ def obj_id(self): def obj_label(self): cell = TableCell('label', default=None)(self) if cell: - return CleanText(cell, default=NotAvailable)(self) - return CleanText('//form[contains(@action, "detaillerOffre") or contains(@action, "detaillerPretPartenaireListe-encoursPrets.ea")]/div[@class="bloc Tmargin"]/h2[@class="title-level2"]')(self) + return CleanText(cell, default=NotAvailable)(self).upper() + return CleanText('//form[contains(@action, "detaillerOffre") or contains(@action, "detaillerPretPartenaireListe-encoursPrets.ea")]/div[@class="bloc Tmargin"]/h2[@class="title-level2"]')(self).upper() def obj_balance(self): if CleanText(TableCell('balance'))(self) != u'Remboursé intégralement': diff --git a/modules/bp/pages/base.py b/modules/bp/pages/base.py index 84f6a832b17494b2985c08bfead120e91a47612d..8f316a89df34822a4209ca42e383d6ddccf500ac 100644 --- a/modules/bp/pages/base.py +++ b/modules/bp/pages/base.py @@ -21,7 +21,6 @@ from weboob.browser.pages import HTMLPage class MyHTMLPage(HTMLPage): - ENCODING = 'iso-8859-1' def on_load(self): deconnexion = self.doc.xpath('//iframe[contains(@id, "deconnexion")] | //p[@class="txt" and contains(text(), "Session expir")]') diff --git a/modules/bp/pages/compat/__init__.py b/modules/bp/pages/compat/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/modules/bp/pages/compat/weboob_capabilities_bank.py b/modules/bp/pages/compat/weboob_capabilities_bank.py new file mode 100644 index 0000000000000000000000000000000000000000..141097d781b97a933881efe1a6851a102454051e --- /dev/null +++ b/modules/bp/pages/compat/weboob_capabilities_bank.py @@ -0,0 +1,45 @@ + +import weboob.capabilities.bank as OLD + +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) + + +__all__ = OLD.__all__ + + +class CapBankWealth(CapBank): + pass + + +class CapBankPockets(CapBank): + pass + + +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + +class CapBankTransfer(OLD.CapBankTransfer): + def transfer_check_label(self, old, new): + from unidecode import unidecode + + return unidecode(old) == unidecode(new) + + +class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipient): + pass + + +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + +Account.TYPE_MORTGAGE = 17 +Account.TYPE_CONSUMER_CREDIT = 18 +Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/bp/pages/login.py b/modules/bp/pages/login.py index 3a3ce789f2be1433c4b5d6de95d0bd8ee9722062..05fad660386872f39db193eac0b6cfdd6435f762 100644 --- a/modules/bp/pages/login.py +++ b/modules/bp/pages/login.py @@ -95,7 +95,7 @@ def login(self, login, pwd): form = self.get_form(name='formAccesCompte') form['password'] = vk.get_string_code(pwd) - form['username'] = login.encode(self.ENCODING) + form['username'] = login form.submit() diff --git a/modules/bp/pages/subscription.py b/modules/bp/pages/subscription.py index 49715354ddad236eb366e117c1ad743ec2f87fd4..69ab822ca505bd8c133ede867892c0f33e19699e 100644 --- a/modules/bp/pages/subscription.py +++ b/modules/bp/pages/subscription.py @@ -29,9 +29,6 @@ class SubscriptionPage(LoggedPage, HTMLPage): - # encoding is wrong on the page - ENCODING='ISO-8859-1' - # because of freaking JS from hell STATEMENT_TYPES = ('RCE', 'RPT', 'RCO') @@ -118,7 +115,7 @@ class item(ItemElement): klass = Subscription obj_label = CleanText('.') - obj_id = Regexp(Field('label'), r'\w - (\w+)') + obj_id = Regexp(Field('label'), r'\w? ?- (\w+)') obj_subscriber = Env('subscriber') obj__number = Attr('.', 'value') diff --git a/modules/bp/pages/transfer.py b/modules/bp/pages/transfer.py index a1bbc1d38491daf09800143934ee0269cbc33862..031242430017e0ef0fcd1eaf6400ba011c0b283f 100644 --- a/modules/bp/pages/transfer.py +++ b/modules/bp/pages/transfer.py @@ -20,8 +20,9 @@ from datetime import datetime -from weboob.capabilities.bank import ( - TransferError, TransferBankError, Transfer, TransferStep, NotAvailable, Recipient, AccountNotFound, AddRecipientError +from .compat.weboob_capabilities_bank import ( + TransferError, TransferBankError, Transfer, TransferStep, NotAvailable, Recipient, + AccountNotFound, AddRecipientBankError ) from weboob.capabilities.base import find_object from weboob.browser.pages import LoggedPage @@ -196,7 +197,7 @@ def choose_country(self, recipient, is_bp_account): # if this is present, we can't add recipient currently more_security_needed = self.doc.xpath(u'//iframe[@title="Gestion de compte par Internet"]') if more_security_needed: - raise AddRecipientError(message=u"Pour activer le service Certicode, nous vous invitons à vous rapprocher de votre Conseiller en Bureau de Poste.") + raise AddRecipientBankError(message=u"Pour activer le service Certicode, nous vous invitons à vous rapprocher de votre Conseiller en Bureau de Poste.") form = self.get_form(name='SaisiePaysBeneficiaireVirement') form['compteLBP'] = str(is_bp_account).lower() @@ -233,7 +234,7 @@ def on_load(self): error_msg = CleanText('//h2[contains(text(), "Compte rendu")]/following-sibling::p')(self.doc) if error_msg: - raise AddRecipientError(message=error_msg) + raise AddRecipientBankError(message=error_msg) def set_browser_form(self): form = self.get_form(name='SaisieOTP') diff --git a/modules/bred/bred/browser.py b/modules/bred/bred/browser.py index 07f8599b4c37196c2780c5ab87f354d1c8310b90..47818ab48e11b915527a4e5164472c37d86caedf 100644 --- a/modules/bred/bred/browser.py +++ b/modules/bred/bred/browser.py @@ -198,7 +198,7 @@ def get_history(self, account, coming=False): self.logger.debug('stopping coming after %s', t) return - next_page = bool(transactions) + next_page = len(transactions) == 50 offset += 50 # This assert supposedly prevents infinite loops, diff --git a/modules/bred/bred/pages.py b/modules/bred/bred/pages.py index 509af9261134f2a7164264d68ad97db0cb03d0df..eb5bb2d775d07c09248824107cc671cbf73e8d77 100644 --- a/modules/bred/bred/pages.py +++ b/modules/bred/bred/pages.py @@ -240,8 +240,15 @@ def iter_history(self, account, operation_list, seen, today, coming): if t.type == Transaction.TYPE_CARD and account.type == Account.TYPE_CARD: t.type = Transaction.TYPE_DEFERRED_CARD - transactions.append(t) - + if abs(t.rdate.year - t.date.year) < 2: + transactions.append(t) + else: + self.logger.warning( + 'rdate(%s) and date(%s) of the transaction %s are far far away, skip it', + t.rdate, + t.date, + t.label + ) return transactions diff --git a/modules/bred/compat/weboob_capabilities_bank.py b/modules/bred/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/bred/compat/weboob_capabilities_bank.py +++ b/modules/bred/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/caels/browser.py b/modules/caels/browser.py index 266e4f0e3785e6ffd34d336bfb5e060e32b6cd9c..e77af0e673e6a20d1e8df0f9c629cad1005f71ea 100644 --- a/modules/caels/browser.py +++ b/modules/caels/browser.py @@ -24,3 +24,9 @@ class CAELSBrowser(AbstractBrowser): PARENT = 'amundi' PARENT_ATTR = 'package.browser.AmundiBrowser' + BASEURL = "https://www.ca-els.com/psf/" + + + def __init__(self, *args, **kwargs): + super(CAELSBrowser, self).__init__(*args, **kwargs) + self.weboob = kwargs.get('weboob') diff --git a/modules/caels/compat/weboob_capabilities_bank.py b/modules/caels/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/caels/compat/weboob_capabilities_bank.py +++ b/modules/caels/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/caels/module.py b/modules/caels/module.py index da111a28105046a2fdf8f0707ecdd395a6eb59f1..1274d40972944e4c94eaaf57a16e571dc95c1720 100644 --- a/modules/caels/module.py +++ b/modules/caels/module.py @@ -43,15 +43,13 @@ class CaelsModule(Module, CapBankWealth): BROWSER = CAELSBrowser def create_default_browser(self): - return self.create_browser("https://www.ca-els.com/", - self.config['login'].get(), + return self.create_browser(self.config['login'].get(), self.config['password'].get(), weboob=self.weboob) def get_account(self, id): return find_object(self.iter_accounts(), id=id, error=AccountNotFound) - def iter_accounts(self): return self.browser.iter_accounts() diff --git a/modules/caels/pages.py b/modules/caels/pages.py new file mode 100644 index 0000000000000000000000000000000000000000..5d017434904e7241b745b8581cfac28cfbc614a2 --- /dev/null +++ b/modules/caels/pages.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Quentin Defenouillere +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.browser.elements import ItemElement, method, DictElement +from weboob.browser.filters.standard import CleanDecimal, Date, Field, Env +from weboob.browser.filters.json import Dict +from weboob.browser.pages import AbstractPage +from weboob.capabilities.bank import Investment +from weboob.capabilities.base import NotAvailable +from weboob.tools.capabilities.bank.investments import is_isin_valid + + +class AccountsPage(AbstractPage): + PARENT = 'amundi' + PARENT_URL = 'accounts' + + @method + class iter_investments(DictElement): + def find_elements(self): + for psds in Dict('listPositionsSalarieFondsDto')(self): + for psd in psds.get('positionsSalarieDispositifDto'): + if psd.get('codeDispositif') == Env('account_id')(self): + return psd.get('positionsSalarieFondsDto') + return {} + + class item(ItemElement): + klass = Investment + + obj_label = Dict('libelleFonds') + obj_unitvalue = Dict('vl') & CleanDecimal + obj_quantity = Dict('nbParts') & CleanDecimal + obj_valuation = Dict('mtBrut') & CleanDecimal + obj_code = Dict('codeIsin', default=NotAvailable) + obj_vdate = Date(Dict('dtVl')) + # The "diff" is only present on the CAELS website but not its parent (amundi): + obj_diff = Dict('mtPMV') & CleanDecimal + + def obj_code_type(self): + if is_isin_valid(Field('code')(self)): + return Investment.CODE_TYPE_ISIN + return NotAvailable diff --git a/modules/caissedepargne/browser.py b/modules/caissedepargne/browser.py index 8298fd3cd36787e802b4a44d81d5b5aeb12d15ab..86dda7134a3afc30dc5ba4b3ca4f83f55657c3df 100644 --- a/modules/caissedepargne/browser.py +++ b/modules/caissedepargne/browser.py @@ -29,12 +29,13 @@ from weboob.browser.browsers import LoginBrowser, need_login, StatesMixin from .compat.weboob_browser_switch import SiteSwitch from weboob.browser.url import URL -from weboob.capabilities.bank import Account, AddRecipientStep, Recipient, TransferBankError, Transaction, Investment +from weboob.capabilities.bank import Account, AddRecipientStep, Recipient, TransferBankError, Transaction from weboob.capabilities.base import NotAvailable from weboob.capabilities.profile import Profile from weboob.browser.exceptions import BrowserHTTPNotFound, ClientError from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable from weboob.tools.capabilities.bank.transactions import sorted_transactions, FrenchTransaction +from .compat.weboob_tools_capabilities_bank_investments import create_french_liquidity from weboob.tools.compat import urljoin from weboob.tools.value import Value from weboob.tools.decorators import retry @@ -46,7 +47,7 @@ ProTransferSummaryPage, ProAddRecipientOtpPage, ProAddRecipientPage, SmsPage, SmsPageOption, SmsRequest, AuthentPage, RecipientPage, CanceledAuth, CaissedepargneKeyboard, TransactionsDetailsPage, LoadingPage, ConsLoanPage, MeasurePage, NatixisLIHis, NatixisLIInv, NatixisRedirectPage, - SubscriptionPage, + SubscriptionPage, CreditCooperatifMarketPage, ) from .linebourse_browser import LinebourseBrowser @@ -60,6 +61,8 @@ class CaisseEpargne(LoginBrowser, StatesMixin): STATE_DURATION = 5 HISTORY_MAX_PAGE = 200 + LINEBOURSE_BROWSER = LinebourseBrowser + login = URL('/authentification/manage\?step=identification&identifiant=(?P.*)', 'https://.*/login.aspx', LoginPage) account_login = URL('/authentification/manage\?step=account&identifiant=(?P.*)&account=(?P.*)', LoginPage) @@ -86,6 +89,7 @@ class CaisseEpargne(LoginBrowser, StatesMixin): market = URL('https://.*/Pages/Bourse.*', 'https://www.caisse-epargne.offrebourse.com/ReroutageSJR', 'https://www.caisse-epargne.offrebourse.com/Portefeuille.*', MarketPage) + creditcooperatif_market = URL('https://www.offrebourse.com/.*', CreditCooperatifMarketPage) # just to catch the landing page of the Credit Cooperatif's Linebourse natixis_redirect = URL(r'/NaAssuranceRedirect/NaAssuranceRedirect.aspx', r'https://www.espace-assurances.caisse-epargne.fr/espaceinternet-ce/views/common/routage-itce.xhtml\?windowId=automatedEntryPoint', NatixisRedirectPage) @@ -103,6 +107,26 @@ class CaisseEpargne(LoginBrowser, StatesMixin): request_sms = URL('https://www.icgauth.caisse-epargne.fr/dacsrest/api/v1u0/transaction/(?P)', SmsRequest) __states__ = ('BASEURL', 'multi_type', 'typeAccount', 'is_cenet_website', 'recipient_form', 'is_send_sms') + # Accounts managed in life insurance space (not in linebourse) + + insurance_accounts = ('AIKIDO', + 'ASSURECUREUIL', + 'ECUREUIL PROJET', + 'GARANTIE RETRAITE EU', + 'INITIATIVES PLUS', + 'INITIATIVES TRANSMIS', + 'LIVRET ASSURANCE VIE', + 'OCEOR EVOLUTION', + 'PATRIMONIO CRESCENTE', + 'PEP TRANSMISSION', + 'PERP', + 'PERSPECTIVES ECUREUI', + 'POINTS RETRAITE ECUR', + 'RICOCHET', + 'SOLUTION PERP', + 'TENDANCES', + 'YOGA', ) + def __init__(self, nuser, *args, **kwargs): self.BASEURL = kwargs.pop('domain', self.BASEURL) if not self.BASEURL.startswith('https://'): @@ -112,17 +136,30 @@ def __init__(self, nuser, *args, **kwargs): self.multi_type = False self.accounts = None self.loans = None - self.typeAccount = 'WE' + self.typeAccount = None + self.inexttype = 0 # keep track of index in the connection type's list self.nuser = nuser self.recipient_form = None self.is_send_sms = None self.weboob = kwargs['weboob'] + self.market_url = kwargs.pop( + 'market_url', + 'https://www.caisse-epargne.offrebourse.com', + ) + super(CaisseEpargne, self).__init__(*args, **kwargs) dirname = self.responses_dirname if dirname: dirname += '/bourse' - self.linebourse = LinebourseBrowser('https://www.caisse-epargne.offrebourse.com', logger=self.logger, responses_dirname=dirname, weboob=self.weboob, proxy=self.PROXIES) + + self.linebourse = self.LINEBOURSE_BROWSER( + self.market_url, + logger=self.logger, + responses_dirname=dirname, + weboob=self.weboob, + proxy=self.PROXIES, + ) def deleteCTX(self): # For connection to offrebourse and natixis, we need to delete duplicate of CTX cookie @@ -152,30 +189,67 @@ def do_login(self): Attempt to log in. Note: this method does nothing if we are already logged in. """ + # Among the parameters used during the login step, there is + # a connection type (called typeAccount) that can take the + # following values: + # WE: espace particulier + # WP: espace pro + # WM: personnes protégées + # EU: Cenet + # + # A connection can have one connection type as well as many of + # them. There is an issue when there is many connection types: + # the connection type to use can't be guessed in advance, we + # have to test all of them until the login step is successful + # (sometimes all connection type can be used for the login, sometimes + # only one will work). + # + # For simplicity's sake, we try each connection type from first to + # last (they are returned in a list by the first request) + # + # Examples of connection types combination that have been seen so far: + # [WE] + # [WP] + # [WE, WP] + # [WE, WP, WM] + # [WP, WM] + # [EU] + # [EU, WE] (EU tends to come first when present) + if not self.username or not self.password: raise BrowserIncorrectPassword() - # Reset domain to log on pro website if first login attempt failed on personal website. - if self.multi_type: - self.BASEURL = 'https://www.caisse-epargne.fr' - self.typeAccount = 'WP' - + # Retrieve the list of types: can contain a single type or more + # - when there is a single type: all the information are available + # - when there are several types: an additional request is needed data = self.login.go(login=self.username).get_response() if data is None: raise BrowserIncorrectPassword() - if "authMode" in data and data['authMode'] == 'redirect': - raise SiteSwitch('cenet') - - if len(data['account']) > 1: + if len(data.get('account', [])) > 1: + # Additional request when there is more than one connection type + # to "choose" from the list of connection types self.multi_type = True + + if self.inexttype < len(data['account']): + self.typeAccount = data['account'][self.inexttype] + else: + assert False, 'should have logged in with at least one connection type' + self.inexttype += 1 + data = self.account_login.go(login=self.username, accountType=self.typeAccount).get_response() assert data is not None + if data.get('authMode', '') == 'redirect': # the connection type EU could also be used as a criteria + raise SiteSwitch('cenet') + typeAccount = data['account'][0] + if self.multi_type: + assert typeAccount == self.typeAccount + idTokenClavier = data['keyboard']['Id'] vk = CaissedepargneKeyboard(data['keyboard']['ImageClavier'], data['keyboard']['Num']['string']) newCodeConf = vk.get_string_code(self.password) @@ -185,10 +259,10 @@ def do_login(self): 'newCodeConf': newCodeConf, 'auth_mode': 'ajax', 'nuusager': self.nuser.encode('utf-8'), - 'codconf': self.password, + 'codconf': '', # must be present though empty 'typeAccount': typeAccount, 'step': 'authentification', - 'ctx': 'typsrv=WE', + 'ctx': 'typsrv={}'.format(typeAccount), 'clavierSecurise': '1', 'nuabbd': self.username } @@ -205,8 +279,8 @@ def do_login(self): assert response is not None if not response['action']: - if not self.typeAccount == 'WP' and self.multi_type: - # If we haven't test PRO espace we check before raising wrong pass + if self.multi_type: + # try to log in with the next connection type's value self.do_login() return raise BrowserIncorrectPassword(response['error']) @@ -266,6 +340,11 @@ def get_measure_accounts_list(self): return self.accounts + def update_linebourse_token(self): + assert self.linebourse is not None, "linebourse browser should already exist" + self.linebourse.session.cookies.update(self.session.cookies) + self.linebourse.session.headers['X-XSRF-TOKEN'] = self.session.cookies['XSRF-TOKEN'] + @need_login @retry(ClientError, tries=3) def get_accounts_list(self): @@ -295,14 +374,27 @@ def get_accounts_list(self): self.page.submit() - if self.page.is_error(): - continue + # For Caisse d'Epargne's connections + if self.url.startswith('https://www.caisse-epargne.offrebourse.com') : + if self.page.is_error(): + continue - self.garbage.go() + self.garbage.go() - if self.garbage.is_here(): - continue - self.page.get_valuation_diff(account) + if self.garbage.is_here(): + continue + + self.page.get_valuation_diff(account) + + # For CreditCooperatif's connections + elif self.url.startswith('https://www.offrebourse.com'): + self.update_linebourse_token() + self.linebourse.handle_cgu() + page = self.linebourse.go_portfolio() + assert self.linebourse.portfolio.is_here() + account.valuation_diff = page.get_valuation_diff() + else: + assert False, "new domain that hasn't been seen so far ?" # Some accounts have no available balance or label and cause issues # in the backend so we must exclude them from the accounts list: @@ -325,7 +417,9 @@ def get_loans_list(self): if self.home.is_here(): if not self.page.is_access_error(): self.page.go_loan_list() - self.loans = list(self.page.get_loan_list()) + self.loans = list(self.page.get_real_estate_loans()) + self.page.go_loan_list() + self.loans.extend(self.page.get_loan_list()) for _ in range(3): try: @@ -411,21 +505,23 @@ def _get_history_invests(self, account): self.page.go_history(account._info) - if account.type == Account.TYPE_LIFE_INSURANCE: + if account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_PERP): if "MILLEVIE" in account.label: self.page.go_life_insurance(account) label = account.label.split()[-1] self.natixis_life_ins_his.go(id1=label[:3], id2=label[3:5], id3=account.id) return sorted_transactions(self.page.get_history()) - if account.label.startswith('NUANCES '): + if account.label.startswith('NUANCES ') or account.label in self.insurance_accounts: self.page.go_life_insurance(account) + if 'JSESSIONID' in self.session.cookies: + # To access the life insurance space, we need to delete the JSESSIONID cookie to avoid an expired session + del self.session.cookies['JSESSIONID'] try: - if not self.market.is_here() and not self.message.is_here(): + if not self.life_insurance.is_here() and not self.message.is_here(): # life insurance website is not always available raise BrowserUnavailable() - self.page.submit() self.location('https://www.extranet2.caisse-epargne.fr%s' % self.page.get_cons_histo()) @@ -448,8 +544,14 @@ def get_history(self, account): self.page.go_history(account._info) if "Bourse" in self.url: self.page.submit() - self.linebourse.session.cookies.update(self.session.cookies) - return self.linebourse.iter_history(re.sub('[^0-9]', '', account.id)) + + if self.url.startswith('https://www.caisse-epargne.offrebourse.com'): + self.linebourse.session.cookies.update(self.session.cookies) + return self.linebourse.iter_history(re.sub('[^0-9]', '', account.id)) + elif self.url.startswith('https://www.offrebourse.com'): + self.update_linebourse_token() + return self.linebourse.iter_history() + return self._get_history(account._info) @need_login @@ -473,11 +575,7 @@ def get_investment(self, account): raise NotImplementedError() if account.type == Account.TYPE_PEA and account.label == 'PEA NUMERAIRE': - liquidity = Investment() - liquidity.label = 'Liquidités' - liquidity.code = 'XX-liquidity' - liquidity.valuation = account.balance - yield liquidity + yield create_french_liquidity(account.balance) return if self.home.is_here(): @@ -491,6 +589,14 @@ def get_investment(self, account): if not self.market.is_here(): return self.page.submit() + + # For Credit Cooperatif's connections + if self.url.startswith('https://www.offrebourse.com'): + self.update_linebourse_token() + for investment in self.linebourse.iter_investments(): + yield investment + return + if self.page.is_error(): return self.location('https://www.caisse-epargne.offrebourse.com/Portefeuille') @@ -557,8 +663,20 @@ def iter_recipients(self, origin_account): self.pre_transfer(origin_account) except TransferBankError: return [] - if self.page.transfer_unavailable() or self.page.need_auth() or not self.page.can_transfer(origin_account): + + go_transfer_errors = ( + # redirected to home page because: + # - need to relogin, see `self.page.need_auth()` + # - need more security, see `self.page.transfer_unavailable()` + # - transfer is not available for this connection, see `self.page.go_transfer_via_history()` + # TransferPage inherit from IndexPage so self.home.is_here() is true, check page type to avoid this problem + type(self.page) is IndexPage, + # check if origin_account have recipients + self.transfer.is_here() and not self.page.can_transfer(origin_account), + ) + if any(go_transfer_errors): return [] + return self.page.iter_recipients(account_id=origin_account.id) def pre_transfer(self, account): diff --git a/modules/caissedepargne/cenet/browser.py b/modules/caissedepargne/cenet/browser.py index ee2ecff68ef67369a2d6b08f6b0301621cf83e7b..e42aee2dfced7f526c399d5935be3f88c69b8ad4 100644 --- a/modules/caissedepargne/cenet/browser.py +++ b/modules/caissedepargne/cenet/browser.py @@ -17,6 +17,8 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals + import json from weboob.browser import LoginBrowser, need_login, StatesMixin @@ -40,12 +42,13 @@ class CenetBrowser(LoginBrowser, StatesMixin): BASEURL = "https://www.cenet.caisse-epargne.fr" + STATE_DURATION = 5 login = URL(r'https://(?P[^/]+)/authentification/manage\?step=identification&identifiant=(?P.*)', r'https://.*/authentification/manage\?step=identification&identifiant=.*', r'https://.*/login.aspx', LoginPage) - account_login = URL('/authentification/manage\?step=account&identifiant=(?P.*)&account=(?P.*)', LoginPage) + account_login = URL('https://(?P[^/]+)/authentification/manage\?step=account&identifiant=(?P.*)&account=(?P.*)', LoginPage) cenet_vk = URL('https://www.cenet.caisse-epargne.fr/Web/Api/ApiAuthentification.asmx/ChargerClavierVirtuel') cenet_home = URL('/Default.aspx$', CenetHomePage) cenet_accounts = URL('/Web/Api/ApiComptes.asmx/ChargerSyntheseComptes', CenetAccountsPage) @@ -79,8 +82,18 @@ def __init__(self, nuser, *args, **kwargs): def do_login(self): data = self.login.go(login=self.username, domain=self.login_domain).get_response() + if len(data['account']) > 1: + # additional request where there is more than one + # connection type (called typeAccount) + # TODO: test all connection type values if needed + account_type = data['account'][0] + self.account_login.go(login=self.username, accountType=account_type, domain=self.login_domain) + data = self.page.get_response() + if data is None: raise BrowserIncorrectPassword() + elif not self.nuser: + raise BrowserIncorrectPassword("Erreur: Numéro d'utilisateur requis.") assert "authMode" in data and data['authMode'] == 'redirect', 'should not be on the cenet website' diff --git a/modules/caissedepargne/compat/weboob_capabilities_bank.py b/modules/caissedepargne/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/caissedepargne/compat/weboob_capabilities_bank.py +++ b/modules/caissedepargne/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/caissedepargne/compat/weboob_tools_capabilities_bank_investments.py b/modules/caissedepargne/compat/weboob_tools_capabilities_bank_investments.py new file mode 100644 index 0000000000000000000000000000000000000000..2b68ff9e001c726a90445e05e0a8ef2e2f165059 --- /dev/null +++ b/modules/caissedepargne/compat/weboob_tools_capabilities_bank_investments.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Jonathan Schmidt +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.tools.compat import basestring +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.bank import Investment + +def is_isin_valid(isin): + """ + Méthode générale + Table de conversion des lettres en chiffres + A=10 B=11 C=12 D=13 E=14 F=15 G=16 H=17 I=18 + J=19 K=20 L=21 M=22 N=23 O=24 P=25 Q=26 R=27 + S=28 T=29 U=30 V=31 W=32 X=33 Y=34 Z=35 + + 1 - Mettre de côté la clé, qui servira de référence à la fin de la vérification. + 2 - Convertir toutes les lettres en nombres via la table de conversion ci-contre. Si le nombre obtenu est supérieur ou égal à 10, prendre les deux chiffres du nombre séparément (exemple : 27 devient 2 et 7). + 3 - Pour chaque chiffre, multiplier sa valeur par deux si sa position est impaire en partant de la droite. Si le nombre obtenu est supérieur ou égal à 10, garder les deux chiffres du nombre séparément (exemple : 14 devient 1 et 4). + 4 - Faire la somme de tous les chiffres. + 5 - Soustraire cette somme de la dizaine supérieure ou égale la plus proche (exemples : si la somme vaut 22, la dizaine « supérieure ou égale » est 30, et la clé vaut donc 8 ; si la somme vaut 30, la dizaine « supérieure ou égale » est 30, et la clé vaut 0 ; si la somme vaut 31, la dizaine « supérieure ou égale » est 40, et la clé vaut 9). + 6 - Comparer la valeur obtenue à la clé mise initialement de côté. + + Étapes 1 et 2 : + F R 0 0 0 3 5 0 0 0 0 (+ 8 : clé) + 15 27 0 0 0 3 5 0 0 0 0 + + Étape 3 : le traitement se fait sur des chiffres + 1 5 2 7 0 0 0 3 5 0 0 0 0 + I P I P I P I P I P I P I : position en partant de la droite (P = Pair, I = Impair) + 2 1 2 1 2 1 2 1 2 1 2 1 2 : coefficient multiplicateur + + 2 5 4 7 0 0 0 3 10 0 0 0 0 : résultat + + Étape 4 : + 2 + 5 + 4 + 7 + 0 + 0 + 0 + 3 + (1 + 0)+ 0 + 0 + 0 + 0 = 22 + + Étapes 5 et 6 : 30 - 22 = 8 (valeur de la clé) + """ + + if not isinstance(isin, basestring): + return False + if not re.match(r'^[A-Z]{2}[A-Z0-9]{9}\d$', isin): + return False + + isin_in_digits = ''.join(str(ord(x) - ord('A') + 10) if not x.isdigit() else x for x in isin[:-1]) + key = isin[-1:] + result = '' + for k, val in enumerate(isin_in_digits[::-1], start=1): + if k % 2 == 0: + result = ''.join((result, val)) + else: + result = ''.join((result, str(int(val)*2))) + return str(sum(int(x) for x in result) + int(key))[-1] == '0' + + +def create_french_liquidity(valuation): + """ + Automatically fills a liquidity investment with label, code and code_type. + """ + liquidity = Investment() + liquidity.label = "Liquidités" + liquidity.code = "XX-liquidity" + liquidity.code_type = NotAvailable + liquidity.valuation = valuation + return liquidity + diff --git a/modules/caissedepargne/linebourse_browser.py b/modules/caissedepargne/linebourse_browser.py index c878ebbadb6580115f213f259bd70ad5b6ea0067..89cd8db190a1468612f84fa38038ab97011ad86b 100644 --- a/modules/caissedepargne/linebourse_browser.py +++ b/modules/caissedepargne/linebourse_browser.py @@ -22,3 +22,4 @@ class LinebourseBrowser(AbstractBrowser): PARENT = 'linebourse' + PARENT_ATTR = 'package.browser.LinebourseBrowser' diff --git a/modules/caissedepargne/pages.py b/modules/caissedepargne/pages.py index 517b4490748b0f0c562f9d95d886f2d87b60545a..5159dd4fcfa129f1d3b99eb5acdaafaf3d8bfbe9 100644 --- a/modules/caissedepargne/pages.py +++ b/modules/caissedepargne/pages.py @@ -30,12 +30,15 @@ from weboob.browser.pages import LoggedPage, HTMLPage, JsonPage, pagination, FormNotFound from weboob.browser.elements import ItemElement, method, ListElement, TableElement, SkipItem, DictElement -from weboob.browser.filters.standard import Date, CleanDecimal, Regexp, CleanText, Env, Upper, Field, Eval, Format +from weboob.browser.filters.standard import Date, CleanDecimal, Regexp, CleanText, Env, Upper, Field, Eval, Format, Currency from weboob.browser.filters.html import Link, Attr, TableCell from weboob.capabilities import NotAvailable -from weboob.capabilities.bank import Account, Investment, Recipient, TransferError, TransferBankError, Transfer,\ - AddRecipientError, Loan +from .compat.weboob_capabilities_bank import ( + Account, Investment, Recipient, TransferError, TransferBankError, Transfer, + AddRecipientBankError, Loan, +) from weboob.capabilities.bill import Subscription, Document +from weboob.tools.capabilities.bank.investments import is_isin_valid from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.tools.capabilities.bank.iban import is_rib_valid, rib2iban, is_iban_valid from weboob.tools.captcha.virtkeyboard import GridVirtKeyboard @@ -43,9 +46,17 @@ from weboob.exceptions import NoAccountsException, BrowserUnavailable from weboob.browser.filters.json import Dict +def MyDecimal(*args, **kwargs): + kwargs.update(replace_dots=True) + return CleanDecimal(*args, **kwargs) + +class MyTableCell(TableCell): + def __init__(self, *names, **kwargs): + super(MyTableCell, self).__init__(*names, **kwargs) + self.td = './tr[%s]/td' def fix_form(form): - keys = ['MM$HISTORIQUE_COMPTE$btnCumul','Cartridge$imgbtnMessagerie','MM$m_CH$ButtonImageFondMessagerie', + keys = ['MM$HISTORIQUE_COMPTE$btnCumul', 'Cartridge$imgbtnMessagerie', 'MM$m_CH$ButtonImageFondMessagerie', 'MM$m_CH$ButtonImageMessagerie'] for name in keys: form.pop(name, None) @@ -171,6 +182,12 @@ class IndexPage(LoggedPage, HTMLPage): u'LIVRET DEV.DURABLE': Account.TYPE_SAVINGS, u'LDD Solidaire': Account.TYPE_SAVINGS, u'LIVRET A': Account.TYPE_SAVINGS, + u'LIVRET JEUNE': Account.TYPE_SAVINGS, + u'LIVRET GRAND PRIX': Account.TYPE_SAVINGS, + u'LEP': Account.TYPE_SAVINGS, + u'LEL': Account.TYPE_SAVINGS, + u'CPT PARTS SOCIALES': Account.TYPE_SAVINGS, + u'PEL 16 2013': Account.TYPE_SAVINGS, u'Titres': Account.TYPE_MARKET, u'Compte titres': Account.TYPE_MARKET, u'Mes crédits immobiliers': Account.TYPE_LOAN, @@ -239,7 +256,7 @@ def _get_account_info(self, a, accounts): else: info['type'] = link info['id'] = info['_id'] = id.group(1) - if info['type'] in ('SYNTHESE_ASSURANCE_CNP','SYNTHESE_EPARGNE', 'ASSURANCE_VIE'): + if info['type'] in ('SYNTHESE_ASSURANCE_CNP', 'SYNTHESE_EPARGNE', 'ASSURANCE_VIE'): info['acc_type'] = Account.TYPE_LIFE_INSURANCE if info['type'] in ('BOURSE', 'COMPTE_TITRE'): info['acc_type'] = Account.TYPE_MARKET @@ -258,6 +275,8 @@ def _add_account(self, accounts, link, label, account_type, balance): account._info = info account.label = label account.type = self.ACCOUNT_TYPES.get(label, info['acc_type'] if 'acc_type' in info else account_type) + if 'PERP' in account.label: + account.type = Account.TYPE_PERP balance = balance or self.get_balance(account) account.balance = Decimal(FrenchTransaction.clean_amount(balance)) if balance and balance is not NotAvailable else NotAvailable @@ -277,7 +296,7 @@ def _add_account(self, accounts, link, label, account_type, balance): accounts[account.id] = account def get_balance(self, account): - if account.type != Account.TYPE_LIFE_INSURANCE: + if account.type not in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_PERP): return NotAvailable page = self.go_history(account._info).page balance = page.doc.xpath('.//tr[td[ends-with(@id,"NumContrat")]/a[contains(text(),$id)]]/td[@class="somme"]', id=account.id) @@ -359,7 +378,7 @@ def get_list(self): def is_access_error(self): error_message = u"Vous n'êtes pas autorisé à accéder à cette fonction" if error_message in CleanText('//div[@class="MessageErreur"]')(self.doc): - return True + return True return False @@ -397,45 +416,76 @@ def get_loan_list(self): title = table.getprevious() if title is None: continue - account_type = self.ACCOUNT_TYPES.get(CleanText('.')(title), Account.TYPE_UNKNOWN) - for tr in table.xpath('./table/tbody/tr[contains(@id,"MM_SYNTHESE_CREDITS") and contains(@id,"IdTrGlobal")]'): - tds = tr.findall('td') - if len(tds) == 0 : - continue - for i in tds[0].xpath('.//a/strong'): - label = i.text.strip() - break - if len(tds) == 3 and Decimal(FrenchTransaction.clean_amount(CleanText('.')(tds[-2]))) and any(cls in Attr('.', 'id')(tr) for cls in ['dgImmo', 'dgConso']) == False: - # in case of Consumer credit or revolving credit, we substract avalaible amount with max amout - # to get what was spend - balance = Decimal(FrenchTransaction.clean_amount(CleanText('.')(tds[-2]))) - Decimal(FrenchTransaction.clean_amount(CleanText('.')(tds[-1]))) - else: - balance = Decimal(FrenchTransaction.clean_amount(CleanText('.')(tds[-1]))) - account = Loan() - account.id = label.split(' ')[-1] - account.label = unicode(label) - account.type = account_type - account.balance = -abs(balance) - account.currency = account.get_currency(CleanText('.')(tds[-1])) - account._card_links = [] - if "immobiliers" in CleanText('.')(title): - xp = './/div[contains(@id, "IdDivDetail")]/table/tbody/tr[contains(@id, "%s")]/td' - account.maturity_date = Date(CleanText(xp % 'IdDerniereEcheance'), dayfirst=True, default=NotAvailable)(tr) - account.total_amount = CleanDecimal(CleanText(xp % 'IdCapitalEmprunte'), replace_dots=True, default=NotAvailable)(tr) - account.subscription_date = Date(CleanText(xp % 'IdDateOuverture'), dayfirst=True, default=NotAvailable)(tr) - account.next_payment_date = Date(CleanText(xp % 'IdDateProchaineEcheance'), dayfirst=True, default=NotAvailable)(tr) - account.rate = CleanDecimal(CleanText(xp % 'IdTaux'), replace_dots=True, default=NotAvailable)(tr) - account.next_payment_amount = CleanDecimal(CleanText(xp % 'IdMontantEcheance'), replace_dots=True, default=NotAvailable)(tr) - elif "renouvelables" in CleanText('.')(title): - self.go_loans_conso(tr) - d = self.browser.loans_conso() - if d: - account.total_amount = d['contrat']['creditMaxAutorise'] - account.available_amount = d['situationCredit']['disponible'] - account.next_payment_amount = d['situationCredit']['mensualiteEnCours'] - accounts[account.id] = account + if "immobiliers" not in CleanText('.')(title): + account_type = self.ACCOUNT_TYPES.get(CleanText('.')(title), Account.TYPE_UNKNOWN) + for tr in table.xpath('./table/tbody/tr[contains(@id,"MM_SYNTHESE_CREDITS") and contains(@id,"IdTrGlobal")]'): + tds = tr.findall('td') + if len(tds) == 0 : + continue + for i in tds[0].xpath('.//a/strong'): + label = i.text.strip() + break + if len(tds) == 3 and Decimal(FrenchTransaction.clean_amount(CleanText('.')(tds[-2]))) and any(cls in Attr('.', 'id')(tr) for cls in ['dgImmo', 'dgConso']) == False: + # in case of Consumer credit or revolving credit, we substract avalaible amount with max amout + # to get what was spend + balance = Decimal(FrenchTransaction.clean_amount(CleanText('.')(tds[-2]))) - Decimal(FrenchTransaction.clean_amount(CleanText('.')(tds[-1]))) + else: + balance = Decimal(FrenchTransaction.clean_amount(CleanText('.')(tds[-1]))) + account = Loan() + account.id = label.split(' ')[-1] + account.label = unicode(label) + account.type = account_type + account.balance = -abs(balance) + account.currency = account.get_currency(CleanText('.')(tds[-1])) + account._card_links = [] + + if "renouvelables" in CleanText('.')(title): + self.go_loans_conso(tr) + d = self.browser.loans_conso() + if d: + account.total_amount = float_to_decimal(d['contrat']['creditMaxAutorise']) + account.available_amount = float_to_decimal(d['situationCredit']['disponible']) + account.next_payment_amount = float_to_decimal(d['situationCredit']['mensualiteEnCours']) + accounts[account.id] = account return accounts.values() + @method + class get_real_estate_loans(ListElement): + # beware the html response is slightly different from what can be seen with the browser + # because of some JS most likely: use the native HTML response to build the xpath + item_xpath = '//h3[contains(text(), "immobiliers")]//following-sibling::div[@class="panel"][1]//div[@id[starts-with(.,"MM_SYNTHESE_CREDITS")] and contains(@id, "IdDivDetail")]' + + class iter_account(TableElement): + item_xpath = './table[@class="static"][1]/tbody' + head_xpath = './table[@class="static"][1]/tbody/tr/th' + + col_total_amount = u'Capital Emprunté' + col_rate = u'Taux d’intérêt nominal' + col_balance = u'Capital Restant Dû' + col_last_payment_date = u'Dernière échéance' + col_next_payment_amount = u'Montant prochaine échéance' + col_next_payment_date = u'Prochaine échéance' + + def parse(self, el): + self.env['id'] = CleanText("./h2")(el).split()[-1] + self.env['label'] = CleanText("./h2")(el) + + class item(ItemElement): + + klass = Loan + + obj_id = Env('id') + obj_label = Env('label') + obj_type = Loan.TYPE_LOAN + obj_total_amount = MyDecimal(MyTableCell("total_amount")) + obj_rate = Eval(lambda x: x / 100, MyDecimal(MyTableCell("rate", default=NotAvailable), default=NotAvailable)) + obj_balance = MyDecimal(MyTableCell("balance"), sign=lambda x: -1) + obj_currency = Currency(MyTableCell("balance")) + obj_last_payment_date = Date(CleanText(MyTableCell("last_payment_date"))) + obj_next_payment_amount = MyDecimal(MyTableCell("next_payment_amount")) + obj_next_payment_date = Date(CleanText(MyTableCell("next_payment_date"))) + + def go_list(self): form = self.get_form(name='main') @@ -519,7 +569,7 @@ def get_form_to_detail(self, transaction): m = re.match('.*\("(.*)", "(DETAIL_OP&[\d]+).*\)\)', transaction._link) # go to detailcard page form = self.get_form(name='main') - form['__EVENTTARGET'] = m.group(1) + form['__EVENTTARGET'] = m.group(1) form['__EVENTARGUMENT'] = m.group(2) fix_form(form) return form @@ -625,12 +675,19 @@ def go_life_insurance(self, account): fix_form(form) form.submit() + def transfer_link(self): + return self.doc.xpath(u'//a[span[contains(text(), "Effectuer un virement")]] | //a[contains(text(), "Réaliser un virement")]') + def go_transfer_via_history(self, account): self.go_history(account._info) - self.browser.page.go_transfer(account) + + # check that transfer is available for the connection before try to go on transfer page + # otherwise website will continually crash + if self.transfer_link(): + self.browser.page.go_transfer(account) def go_transfer(self, account): - link = self.doc.xpath(u'//a[span[contains(text(), "Effectuer un virement")]] | //a[contains(text(), "Réaliser un virement")]') + link = self.transfer_link() if len(link) == 0: return self.go_transfer_via_history(account) else: @@ -674,6 +731,7 @@ def on_load(self): form = self.get_form(id="REROUTAGE") form.submit() + class NatixisRedirectPage(LoggedPage, HTMLPage): def on_load(self): try: @@ -711,6 +769,7 @@ def iter_investment(self): inv = Investment() inv.label = CleanText('.')(tbody.xpath('./tr[1]/td[1]/a/span')[0]) inv.code = CleanText('.')(tbody.xpath('./tr[1]/td[1]/a')[0]).split(' - ')[1] + inv.code_type = Investment.CODE_TYPE_ISIN if is_isin_valid(inv.code) else NotAvailable inv.quantity = self.parse_decimal(tbody.xpath('./tr[2]/td[2]')[0]) inv.unitvalue = self.parse_decimal(tbody.xpath('./tr[2]/td[3]')[0]) inv.unitprice = self.parse_decimal(tbody.xpath('./tr[2]/td[5]')[0]) @@ -758,6 +817,7 @@ def iter_investment(self): inv = Investment() libelle = CleanText('.')(tr.xpath('./td[1]')[0]).split(' ') inv.label, inv.code = self.split_label_code(libelle) + inv.code_type = Investment.CODE_TYPE_ISIN if is_isin_valid(inv.code) else NotAvailable inv.quantity = self.parse_decimal(tr.xpath('./td[2]')[0]) inv.unitvalue = self.parse_decimal(tr.xpath('./td[3]')[0]) date = CleanText('.')(tr.xpath('./td[4]')[0]) @@ -1093,7 +1153,6 @@ def go_add_recipient(self): form.submit() - class CanceledAuth(Exception): pass @@ -1124,7 +1183,7 @@ class SmsPage(LoggedPage, HTMLPage): def on_load(self): error = CleanText('//p[@class="warning_trials_before"]')(self.doc) if error: - raise AddRecipientError('Wrongcode, ' + error) + raise AddRecipientBankError(message='Wrongcode, ' + error) def get_prompt_text(self): return CleanText(u'//td[@class="auth_info_prompt"]')(self.doc) @@ -1162,7 +1221,7 @@ class RecipientPage(LoggedPage, HTMLPage): def on_load(self): error = CleanText('//span[@id="MM_LblMessagePopinError"]')(self.doc) if error: - raise AddRecipientError(message=error) + raise AddRecipientBankError(message=error) def is_here(self): return bool(CleanText(u'//h2[contains(text(), "Ajouter un compte bénéficiaire")] |\ @@ -1186,7 +1245,7 @@ class ProAddRecipientOtpPage(IndexPage): def on_load(self): error = CleanText('//div[@id="MM_m_CH_ValidationSummary" and @class="MessageErreur"]')(self.doc) if error: - raise AddRecipientError('Wrongcode, ' + error) + raise AddRecipientBankError(message='Wrongcode, ' + error) def is_here(self): return self.need_auth() and self.doc.xpath('//span[@id="MM_ANR_WS_AUTHENT_ANR_WS_AUTHENT_SAISIE_lblProcedure1"]') @@ -1315,3 +1374,10 @@ def download_document(self, document): form['__EVENTTARGET'] = document.url form['MM$COMPTE_EDOCUMENTS$ctrlEDocumentsConsultationDocument$eventId'] = document._event_id return form.submit() + + +class CreditCooperatifMarketPage(LoggedPage, HTMLPage): + # Stay logged when landing on the new Linebourse + # (which is used by Credit Cooperatif's connections) + # The parsing is done in linebourse.api.pages + pass diff --git a/modules/carrefourbanque/compat/weboob_capabilities_bank.py b/modules/carrefourbanque/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/carrefourbanque/compat/weboob_capabilities_bank.py +++ b/modules/carrefourbanque/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/cices/compat/weboob_capabilities_bank.py b/modules/cices/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/cices/compat/weboob_capabilities_bank.py +++ b/modules/cices/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/citibank/browser.py b/modules/citibank/browser.py index 0ba26ef8b7d0a3f2142114bd0c3abec6a914893e..508fd7bf4c3a8d361f19ca0d04eeda9bebf2972e 100644 --- a/modules/citibank/browser.py +++ b/modules/citibank/browser.py @@ -17,31 +17,29 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals -from weboob.browser import LoginBrowser, URL, need_login +import re +from datetime import datetime +from time import sleep + +from weboob.browser import URL, LoginBrowser, need_login from weboob.browser.pages import HTMLPage, JsonPage, RawPage from weboob.capabilities.bank import Account, AccountNotFound, Transaction from weboob.exceptions import BrowserIncorrectPassword -from weboob.tools.capabilities.bank.transactions import \ - AmericanTransaction as AmTr +from weboob.tools.capabilities.bank.transactions import AmericanTransaction as AmTr from weboob.tools.js import Javascript from .parser import StatementParser, clean_label -import re -from datetime import datetime - -from time import sleep - - __all__ = ['Citibank'] class SomePage(HTMLPage): @property def logged(self): - return bool(self.doc.xpath(u'//a[text()="Sign Off"]') + - self.doc.xpath(u'//div[@id="CardsLoadingDiv"]')) + return bool(self.doc.xpath('//a[text()="Sign Off"]') + + self.doc.xpath('//div[@id="CardsLoadingDiv"]')) class IndexPage(SomePage): @@ -82,7 +80,7 @@ def account(self): account.id = account.label[-4:] for bal in details['accountBalances']: label, value = bal['label'], (bal['value'] or ['0'])[0] - if label == u'Current Balance:': + if label == 'Current Balance:': if value[0] == '(' and value[-1] == ')': value = value[1:-1] sign = 1 @@ -90,17 +88,16 @@ def account(self): sign = -1 account.currency = Account.get_currency(value) account.balance = sign * AmTr.decimal_amount(value) - elif label == u'Total Revolving Credit Line:': + elif label == 'Total Revolving Credit Line:': account.cardlimit = AmTr.decimal_amount(value) - elif label.startswith(u'Minimum Payment Due'): + elif label.startswith('Minimum Payment Due'): d = re.match(r'.*(..-..-....):$', label).group(1) account.paydate = datetime.strptime(d, '%m-%d-%Y') account.paymin = AmTr.decimal_amount(value) return account def transactions(self): - return sorted(self.unsorted_trans(), - lambda a, b: cmp(a.date, b.date), reverse=True) + return sorted(self.unsorted_trans(), key=lambda t: t.date, reverse=True) def unsorted_trans(self): for jnl in self.doc['accountDetailsAndActivity']['accountActivity'] \ @@ -110,10 +107,10 @@ def unsorted_trans(self): amount = jnl['columns'][3]['activityColumn'][0] xdescs = dict((x['label'], x['value'][0]) for x in jnl['extendedDescriptions']) - pdate = xdescs[u'Posted Date :'] - ref = xdescs.get(u'Reference Number:') or u'' + pdate = xdescs['Posted Date :'] + ref = xdescs.get('Reference Number:') or '' - if amount.startswith(u'(') and amount.endswith(u')'): + if amount.startswith('(') and amount.endswith(')'): amount = AmTr.decimal_amount(amount[1:-1]) else: amount = -AmTr.decimal_amount(amount) @@ -132,8 +129,8 @@ def unsorted_trans(self): class StatementsPage(SomePage): def dates(self): return [x[:10] for x in self.doc.xpath( - u'//select[@id="currentStatementsDate"]/option/@value') - if re.match(u'^\d\d\d\d-\d\d-\d\d All$', x)] + '//select[@id="currentStatementsDate"]/option/@value') + if re.match(r'^\d\d\d\d-\d\d-\d\d All$', x)] class StatementPage(RawPage): @@ -147,8 +144,7 @@ def is_sane(self): return self._parser.read_first_date_range() is not None def transactions(self): - return sorted(self._parser.read_transactions(), - cmp=lambda t1, t2: cmp(t2.date, t1.date)) + return sorted(self._parser.read_transactions(), key=lambda t: t.date) class Citibank(LoginBrowser): @@ -171,20 +167,16 @@ class Citibank(LoginBrowser): TIMEOUT = 30.0 index = URL(r'/US/JPS/portal/Index.do', IndexPage) signon = URL(r'/US/JSO/signon/ProcessUsernameSignon.do', SomePage) - accdetailhtml = URL(u'/US/NCPS/accountdetailactivity/flow.action.*$', - SomePage) - accounts = URL(r'/US/REST/accountsPanel' - r'/getCustomerAccounts.jws\?ttc=(?P.*)$', + accdetailhtml = URL(r'/US/NCPS/accountdetailactivity/flow.action.*$', SomePage) + accounts = URL(r'/US/REST/accountsPanel/getCustomerAccounts.jws\?ttc=(?P.*)$', AccountsPage) accdetails = URL(r'/US/REST/accountDetailsActivity' - r'/getAccountDetailsActivity.jws\?accountID=(?P.*)$', - AccDetailsPage) + r'/getAccountDetailsActivity.jws\?accountID=(?P.*)$', AccDetailsPage) statements = URL(r'/US/NCSC/doccenter/flow.action\?TTC=1079&' - 'accountID=(?P.*)$', StatementsPage) + r'accountID=(?P.*)$', StatementsPage) statement = URL(r'/US/REST/doccenterresource/downloadStatementsPdf.jws\?' - r'selectedIndex=0&date=(?P....-..-..)&' - r'downloadFormat=pdf', StatementPage) - unknown = URL('/.*$', SomePage) + r'selectedIndex=0&date=(?P....-..-..)&downloadFormat=pdf', StatementPage) + unknown = URL(r'/.*$', SomePage) def get_account(self, id_): innerId = self.to_accounts().inner_ids_dict().get(id_) @@ -217,8 +209,8 @@ def to_statement(self, date): # Sometimes the website returns non-PDF file. # It recovers if we repeat whole browsing sequence all the way # from home page up to the statement. - MAX_DELAY=10 - for i in xrange(self.MAX_RETRIES): + MAX_DELAY = 10 + for i in range(self.MAX_RETRIES): if self.to_page(self.statement, date=date).is_sane(): return self.page sleep(min(MAX_DELAY, 1 << i)) @@ -233,7 +225,7 @@ def to_page(self, url, **data): def do_login(self): self.session.cookies.clear() - data = dict([('username', self.username), ('password', self.password)]+ - self.index.go().extra()) + data = dict([('username', self.username), ('password', self.password)] + + self.index.go().extra()) if not self.signon.go(data=data).logged: raise BrowserIncorrectPassword() diff --git a/modules/citibank/parser.py b/modules/citibank/parser.py index ae2f5b92d4a4c80a7924a66d8315d96b0a86e498..c97690ad3991b802400e38c25cf23067dfe5e0e6 100644 --- a/modules/citibank/parser.py +++ b/modules/citibank/parser.py @@ -17,16 +17,18 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals + +import datetime +import re + from weboob.capabilities.bank import Transaction -from weboob.tools.capabilities.bank.transactions import \ - AmericanTransaction as AmTr +from weboob.tools.capabilities.bank.transactions import AmericanTransaction as AmTr +from weboob.tools.compat import unicode from weboob.tools.date import closest_date from weboob.tools.pdf import decompress_pdf from weboob.tools.tokenizer import ReTokenizer -import datetime -import re - def clean_label(text): """ @@ -35,10 +37,10 @@ def clean_label(text): need to make labels from both sources look the same. """ for pattern in [r' \d+\.\d+ +POUND STERLING', - u'Subject to Foreign Fee', - u'Description']: - text = re.sub(pattern, u'', text, re.UNICODE) - return re.sub(r' +', u' ', text.strip().upper(), re.UNICODE) + 'Subject to Foreign Fee', + 'Description']: + text = re.sub(pattern, '', text, re.UNICODE) + return re.sub(r' +', ' ', text.strip().upper(), re.UNICODE) def formatted(read_func): @@ -54,7 +56,7 @@ def wrapped(self, pos): pos, data = read_func(self, pos) pos, et = self.read_layout_et(pos) if ws is None or bt is None or tf is None \ - or tm is None or data is None or et is None: + or tm is None or data is None or et is None: return startPos, None else: return pos, data @@ -192,13 +194,13 @@ def __getattr__(self, name): @formatted def read_date(self, pos): def parse_date(v): - for year in [1900, 1904]: # try leap and non-leap years + for year in [1900, 1904]: # try leap and non-leap years fullstr = '%s/%i' % (v, year) try: return datetime.datetime.strptime(fullstr, '%m/%d/%Y') except ValueError as e: - pass - raise e + last_error = e + raise last_error return self._tok.simple_read('date', pos, parse_date) @@ -217,7 +219,7 @@ def read_text(self, pos): lambda v: unicode(v, errors='ignore')) pos, et = self.read_layout_et(pos) if ws is None or bt is None or tf is None \ - or tm is None or text is None or et is None: + or tm is None or text is None or et is None: return startPos, None else: return pos, (text, tm) diff --git a/modules/cmes/compat/weboob_capabilities_bank.py b/modules/cmes/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/cmes/compat/weboob_capabilities_bank.py +++ b/modules/cmes/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/cmso/compat/weboob_capabilities_bank.py b/modules/cmso/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/cmso/compat/weboob_capabilities_bank.py +++ b/modules/cmso/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/cmso/par/browser.py b/modules/cmso/par/browser.py index d3d5987d245afaf1d5461fdb6c7da6c40686d6e1..846374cdd31e048e49763e84b54183ce851c79f2 100644 --- a/modules/cmso/par/browser.py +++ b/modules/cmso/par/browser.py @@ -57,6 +57,7 @@ def wrapper(browser, *args, **kwargs): try: ret = cb() except exc_check as exc: + browser.headers = None browser.do_login() browser.logger.info('%s raised, retrying', exc) continue @@ -177,8 +178,9 @@ def iter_accounts(self): for key in page.get_keys(): for a in page.iter_savings(key=key, numbers=numbers): if a._index in seen: - self.logger.warning('skipping %s because it seems to be a duplicate of %s', a, seen[a._index]) - continue + acc = seen[a._index] + self.accounts_list.remove(acc) + self.logger.warning('replace %s because it seems to be a duplicate of %s', seen[a._index], a) self.accounts_list.append(a) # Then, get loans diff --git a/modules/cmso/pro/pages.py b/modules/cmso/pro/pages.py index b1ac8629e2a3ebaad554f77dd88629462825c44a..85eb8185a2cde7a1fc1677d00b4d20b0934f8be4 100644 --- a/modules/cmso/pro/pages.py +++ b/modules/cmso/pro/pages.py @@ -239,7 +239,7 @@ def condition(self): class HistoryPage(CMSOPage): def get_date_range_list(self): - return self.doc.xpath('//select[@name="date"]/option/@value') + return [d for d in self.doc.xpath('//select[@name="date"]/option/@value') if d] @pagination @method diff --git a/modules/cragr/compat/weboob_capabilities_bank.py b/modules/cragr/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/cragr/compat/weboob_capabilities_bank.py +++ b/modules/cragr/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/cragr/web/browser.py b/modules/cragr/web/browser.py index 173e2fddc4019c5062e3190217fc18c5f3e08270..9f1a17c19d9a9dfd7a60c0e7b3e567c1be2e2387 100644 --- a/modules/cragr/web/browser.py +++ b/modules/cragr/web/browser.py @@ -24,21 +24,22 @@ from html2text import unescape from datetime import date, datetime, timedelta -from weboob.capabilities.bank import ( - Account, AddRecipientStep, AddRecipientError, RecipientInvalidLabel, - Recipient, AccountNotFound, Investment, +from .compat.weboob_capabilities_bank import ( + Account, AddRecipientStep, AddRecipientBankError, RecipientInvalidLabel, + Recipient, AccountNotFound, ) from weboob.capabilities.base import NotLoaded, find_object from .compat.weboob_capabilities_profile import ProfileMissing from weboob.browser.browsers import LoginBrowser, URL, need_login, StatesMixin from weboob.browser.pages import FormNotFound -from weboob.exceptions import BrowserIncorrectPassword +from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable from weboob.tools.date import ChaoticDateGuesser, LinearDateGuesser from weboob.exceptions import BrowserHTTPError, ActionNeeded from weboob.browser.filters.standard import CleanText from weboob.tools.value import Value from weboob.tools.compat import urlparse, urljoin, basestring from weboob.tools.capabilities.bank.iban import is_iban_valid +from .compat.weboob_tools_capabilities_bank_investments import create_french_liquidity from .pages import ( HomePage, LoginPage, LoginErrorPage, AccountsPage, @@ -323,10 +324,14 @@ def get_list(self): self.location(updated_account._form.request) if (updated_account.url or updated_account._form) and not self.no_fixed_deposit_page.is_here(): - iban_url = self.page.get_iban_url() - if iban_url: - self.location(iban_url) - account.iban = self.page.get_iban() + # crashes because we get an UnavailablePage instead of TransactionsPage + if self.transactions.is_here(): + iban_url = self.page.get_iban_url() + if iban_url: + self.location(iban_url) + if self.unavailable_page.is_here(): + raise BrowserUnavailable() + account.iban = self.page.get_iban() # credit cards # reseting location in case of pagination @@ -374,10 +379,14 @@ def market_accounts_matching(self, accounts_list, market_accounts_list): account = find_object(accounts_list, id=market_account.id) if account: account.label = market_account.label or account.label - # Update accounts balance only for TYPE_MARKET because PEA accounts are fused - # with their related DAV PEA here, therefore the balance includes liquidities: if account.type == Account.TYPE_MARKET: account.balance = market_account.balance or account.balance + # Some PEA accounts are only present on the Market page but the balance includes + # liquidities from the DAV PEA, so we need another request to fetch the balance: + elif account.type == Account.TYPE_PEA: + url = 'https://www.cabourse.credit-agricole.fr/netfinca-titres/servlet/com.netfinca.frontcr.account.WalletVal?nump=%s:%s' + self.location(url % (account.id, self.code_caisse)) + account.balance = self.page.get_pea_balance() @need_login def get_history(self, account): @@ -446,11 +455,7 @@ def iter_investment(self, account): self.go_perimeter(account._perimeter) if account.type == Account.TYPE_PEA and account._liquidity_url: - liquidity_inv = Investment() - liquidity_inv.label = account.label - liquidity_inv.code = u'XX-liquidity' - liquidity_inv.valuation = account.balance - yield liquidity_inv + yield create_french_liquidity(account.balance) return if account.type in (Account.TYPE_MARKET, Account.TYPE_PEA): @@ -468,6 +473,10 @@ def iter_investment(self, account): return if self.lifeinsurance.is_here(): self.page.go_on_detail(account.id) + # We scrape the non-invested part as liquidities: + if self.page.has_liquidities(): + valuation = self.page.get_liquidities() + yield create_french_liquidity(valuation) for inv in self.page.iter_investment(): yield inv @@ -695,7 +704,8 @@ def new_recipient(self, recipient, **params): raise RecipientInvalidLabel('Recipient label contains invalid characters') if 'sms_code' in params and not re.match(r'^[a-z0-9]{6}$', params['sms_code'], re.I): - raise AddRecipientError('SMS verification code is invalid') + # check before send sms code because it can crash website if code is invalid + raise AddRecipientBankError("SMS code %s is invalid" % params['sms_code']) self.transfer_init_page.go(sag=self.sag) assert self.transfer_init_page.is_here() @@ -726,7 +736,7 @@ def new_recipient(self, recipient, **params): err = hasattr(self.page, 'get_sms_error') and self.page.get_sms_error() if err: - raise AddRecipientError(message=err) + raise AddRecipientBankError(message=err) self.page.submit_recipient(recipient.label, recipient.iban) self.page.confirm_recipient() @@ -738,6 +748,5 @@ def new_recipient(self, recipient, **params): assert self.transfer_init_page.is_here() res = self.page.find_recipient(recipient.iban) - if res is None: - raise AddRecipientError('Recipient could not be found') + assert res, 'Recipient with iban %s could not be found' % recipient.iban return res diff --git a/modules/cragr/web/compat/weboob_capabilities_bank.py b/modules/cragr/web/compat/weboob_capabilities_bank.py new file mode 100644 index 0000000000000000000000000000000000000000..141097d781b97a933881efe1a6851a102454051e --- /dev/null +++ b/modules/cragr/web/compat/weboob_capabilities_bank.py @@ -0,0 +1,45 @@ + +import weboob.capabilities.bank as OLD + +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) + + +__all__ = OLD.__all__ + + +class CapBankWealth(CapBank): + pass + + +class CapBankPockets(CapBank): + pass + + +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + +class CapBankTransfer(OLD.CapBankTransfer): + def transfer_check_label(self, old, new): + from unidecode import unidecode + + return unidecode(old) == unidecode(new) + + +class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipient): + pass + + +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + +Account.TYPE_MORTGAGE = 17 +Account.TYPE_CONSUMER_CREDIT = 18 +Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/cragr/web/compat/weboob_tools_capabilities_bank_investments.py b/modules/cragr/web/compat/weboob_tools_capabilities_bank_investments.py new file mode 100644 index 0000000000000000000000000000000000000000..2b68ff9e001c726a90445e05e0a8ef2e2f165059 --- /dev/null +++ b/modules/cragr/web/compat/weboob_tools_capabilities_bank_investments.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Jonathan Schmidt +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.tools.compat import basestring +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.bank import Investment + +def is_isin_valid(isin): + """ + Méthode générale + Table de conversion des lettres en chiffres + A=10 B=11 C=12 D=13 E=14 F=15 G=16 H=17 I=18 + J=19 K=20 L=21 M=22 N=23 O=24 P=25 Q=26 R=27 + S=28 T=29 U=30 V=31 W=32 X=33 Y=34 Z=35 + + 1 - Mettre de côté la clé, qui servira de référence à la fin de la vérification. + 2 - Convertir toutes les lettres en nombres via la table de conversion ci-contre. Si le nombre obtenu est supérieur ou égal à 10, prendre les deux chiffres du nombre séparément (exemple : 27 devient 2 et 7). + 3 - Pour chaque chiffre, multiplier sa valeur par deux si sa position est impaire en partant de la droite. Si le nombre obtenu est supérieur ou égal à 10, garder les deux chiffres du nombre séparément (exemple : 14 devient 1 et 4). + 4 - Faire la somme de tous les chiffres. + 5 - Soustraire cette somme de la dizaine supérieure ou égale la plus proche (exemples : si la somme vaut 22, la dizaine « supérieure ou égale » est 30, et la clé vaut donc 8 ; si la somme vaut 30, la dizaine « supérieure ou égale » est 30, et la clé vaut 0 ; si la somme vaut 31, la dizaine « supérieure ou égale » est 40, et la clé vaut 9). + 6 - Comparer la valeur obtenue à la clé mise initialement de côté. + + Étapes 1 et 2 : + F R 0 0 0 3 5 0 0 0 0 (+ 8 : clé) + 15 27 0 0 0 3 5 0 0 0 0 + + Étape 3 : le traitement se fait sur des chiffres + 1 5 2 7 0 0 0 3 5 0 0 0 0 + I P I P I P I P I P I P I : position en partant de la droite (P = Pair, I = Impair) + 2 1 2 1 2 1 2 1 2 1 2 1 2 : coefficient multiplicateur + + 2 5 4 7 0 0 0 3 10 0 0 0 0 : résultat + + Étape 4 : + 2 + 5 + 4 + 7 + 0 + 0 + 0 + 3 + (1 + 0)+ 0 + 0 + 0 + 0 = 22 + + Étapes 5 et 6 : 30 - 22 = 8 (valeur de la clé) + """ + + if not isinstance(isin, basestring): + return False + if not re.match(r'^[A-Z]{2}[A-Z0-9]{9}\d$', isin): + return False + + isin_in_digits = ''.join(str(ord(x) - ord('A') + 10) if not x.isdigit() else x for x in isin[:-1]) + key = isin[-1:] + result = '' + for k, val in enumerate(isin_in_digits[::-1], start=1): + if k % 2 == 0: + result = ''.join((result, val)) + else: + result = ''.join((result, str(int(val)*2))) + return str(sum(int(x) for x in result) + int(key))[-1] == '0' + + +def create_french_liquidity(valuation): + """ + Automatically fills a liquidity investment with label, code and code_type. + """ + liquidity = Investment() + liquidity.label = "Liquidités" + liquidity.code = "XX-liquidity" + liquidity.code_type = NotAvailable + liquidity.valuation = valuation + return liquidity + diff --git a/modules/cragr/web/pages.py b/modules/cragr/web/pages.py index 76000ed7072efa9660e7f91ed6a1b4cac9c24e04..0890dac326de4648e0b9250b525968091cb68db0 100644 --- a/modules/cragr/web/pages.py +++ b/modules/cragr/web/pages.py @@ -26,9 +26,9 @@ from weboob.browser.pages import HTMLPage, FormNotFound, pagination from weboob.capabilities import NotAvailable from weboob.capabilities.base import Currency -from weboob.capabilities.bank import ( +from .compat.weboob_capabilities_bank import ( Account, Investment, Recipient, Transfer, TransferError, TransferBankError, - AddRecipientError, Loan + AddRecipientBankError, Loan ) from weboob.capabilities.contact import Advisor from weboob.capabilities.profile import Profile @@ -687,7 +687,7 @@ def condition(self): return False label = Field('label')(self) - if label.startswith('RENT') or label.startswith('RENV'): + if any(label.startswith(prefix) for prefix in ('RENT', 'RENV', 'RACC')): # RENTE account doesn't seem to have any balance or currency return False @@ -978,6 +978,10 @@ def parse_decimal(self, value): return CleanDecimal().filter(value) return MyDecimal().filter(value) + # This method is used to fetch the PEA balance without liquidities present on the DAV PEA: + def get_pea_balance(self): + return CleanDecimal('//tr[td[text()="Valorisation titres"]]/td/span', replace_dots=True)(self.doc) + class MarketHomePage(MarketPage): COL_ID_LABEL = 1 @@ -1024,6 +1028,13 @@ def go_on_detail(self, account_id): form.submit() self.browser.location('https://assurance-personnes.credit-agricole.fr/filiale/entreeBam?sessionSAG=%s&stbpg=pagePU&act=SEPPU&stbzn=bnt&actCrt=SEPPU' % self.browser.sag) + def has_liquidities(self): + return bool(self.doc.xpath('//span[contains(text(), "de versement restant")]')) + + def get_liquidities(self): + return CleanDecimal(Regexp(CleanText('//span[contains(text(), "de versement restant")]'), + "dont (.*) de versement restant"), replace_dots=True)(self.doc) + def iter_investment(self): for line in self.doc.xpath('//table[@summary and count(descendant::td) > 1]/tbody/tr'): @@ -1138,7 +1149,7 @@ def iter_investment(self): inv.code = NotAvailable inv.quantity = self.parse_decimal(cells[self.COL_QUANTITY].text_content()) - if len(cells) == 5: + if 5 <= len(cells) <= 7: inv.unitvalue = self.parse_decimal(cells[2].text_content()) inv.valuation = self.parse_decimal(cells[3].text_content()) inv.portfolio_share = self.parse_decimal(cells[4].text_content())/100 @@ -1213,7 +1224,8 @@ def submit_accounts(self, account_id, recipient_id, amount, currency): if len(emitters) != 1: raise TransferError('Could not find emitter %r' % account_id) recipients = [rcpt for rcpt in self.iter_recipients() if rcpt.id and rcpt.id == recipient_id] - if len(recipients) != 1: + # for recipient with same IBAN, first matched recipient is the default value + if len(recipients) < 1: raise TransferError('Could not find recipient %r' % recipient_id) form = self.get_form(name='frm_fwk') @@ -1247,7 +1259,7 @@ def submit_recipient(self, label, iban): try: form = self.get_form(name='frm_fwk') except FormNotFound: - raise AddRecipientError('An error occurred before sending recipient') + assert False, 'An error occurred before sending recipient' form['NOM_BENEF'] = label for i in range(9): @@ -1337,7 +1349,7 @@ def submit_confirm(self): def on_load(self): super(TransferPage, self).on_load() # warning: the "service indisponible" message (not catched here) is not a real BrowserUnavailable - err = CleanText('//div[has-class("blc-choix-erreur")]//p', default='')(self.doc) + err = CleanText('//form//div[has-class("blc-choix-erreur")]//p', default='')(self.doc) if err: raise TransferBankError(message=err) @@ -1357,7 +1369,7 @@ def check_recipient_error(self): msg = CleanText('//tr[@bgcolor="#C74545"]', default='')(self.doc) # there is no id, class or anything... if msg: - raise AddRecipientError(message=msg) + raise AddRecipientBankError(message=msg) class RecipientMiscPage(RecipientAddingMixin, CollectePageMixin, MyLoggedPage, BasePage): @@ -1379,7 +1391,7 @@ def confirm_recipient(self): try: form = self.get_form(name='frm_fwk') except FormNotFound: - raise AddRecipientError('An error occurred before finishing adding recipient') + assert False, 'An error occurred before finishing adding recipient' form['fwkaction'] = 'ConfirmerAjout' form['fwkcodeaction'] = 'Executer' @@ -1388,7 +1400,7 @@ def confirm_recipient(self): def check_recipient_error(self): msg = CleanText('//tr[@bgcolor="#C74545"]', default='')(self.doc) # there is no id, class or anything... if msg: - raise AddRecipientError(message=msg) + raise AddRecipientBankError(message=msg) def get_iban_col(self): for index, td in enumerate(self.doc.xpath('//table[starts-with(@summary,"Nom et IBAN")]//th')): @@ -1446,7 +1458,7 @@ def on_load(self): # if the otp is incorrect error_msg = CleanText('//div[has-class("blc-choix-erreur")]//span')(self.doc) if error_msg: - raise AddRecipientError(message=error_msg) + raise AddRecipientBankError(message=error_msg) def send_sms(self): # when a code is still pending diff --git a/modules/creditcooperatif/caisseepargne_browser.py b/modules/creditcooperatif/caisseepargne_browser.py index 3759bcae669484ce66457dcde017d8b9c8bf2640..07f124613c3e0bbc6a992145eb20dd3bf63190ae 100644 --- a/modules/creditcooperatif/caisseepargne_browser.py +++ b/modules/creditcooperatif/caisseepargne_browser.py @@ -18,6 +18,7 @@ # along with weboob. If not, see . from weboob.browser import AbstractBrowser +from .linebourse_browser import LinebourseAPIBrowser __all__ = ['CaisseEpargneBrowser'] @@ -26,3 +27,9 @@ class CaisseEpargneBrowser(AbstractBrowser): PARENT = 'caissedepargne' PARENT_ATTR = 'package.browser.CaisseEpargne' + + LINEBOURSE_BROWSER = LinebourseAPIBrowser + + def __init__(self, nuser, *args, **kwargs): + kwargs['market_url'] = 'https://www.offrebourse.com' + super(CaisseEpargneBrowser, self).__init__(nuser, *args, **kwargs) diff --git a/modules/creditcooperatif/cenet_browser.py b/modules/creditcooperatif/cenet_browser.py index 6eb6f8bd9a1dbcb10ca8a91d9e61dcbbba872199..042724be84da6bae3372202655a90486bce2f183 100644 --- a/modules/creditcooperatif/cenet_browser.py +++ b/modules/creditcooperatif/cenet_browser.py @@ -27,3 +27,7 @@ class CenetBrowser(AbstractBrowser): PARENT = 'caissedepargne' PARENT_ATTR = 'package.cenet.browser.CenetBrowser' BASEURL = 'https://www.espaceclient.credit-cooperatif.coop/' + + def __init__(self, nuser, *args, **kwargs): + kwargs['domain'] = 'www.credit-cooperatif.coop' + super(CenetBrowser, self).__init__(nuser, *args, **kwargs) diff --git a/modules/creditcooperatif/linebourse_browser.py b/modules/creditcooperatif/linebourse_browser.py new file mode 100644 index 0000000000000000000000000000000000000000..eedbc2f57259db39860065c4b379bb4eec621ad6 --- /dev/null +++ b/modules/creditcooperatif/linebourse_browser.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Fong Ngo +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from weboob.browser import AbstractBrowser + + +class LinebourseAPIBrowser(AbstractBrowser): + PARENT = 'linebourse' + PARENT_ATTR = 'package.browser.LinebourseAPIBrowser' diff --git a/modules/creditdunord/browser.py b/modules/creditdunord/browser.py index b316da64cf093939b880c12b45d74550cb6c1546..c893504a849d2f74b7d99e7f3557b887dfbefc8d 100644 --- a/modules/creditdunord/browser.py +++ b/modules/creditdunord/browser.py @@ -17,38 +17,42 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . - -import re +from __future__ import unicode_literals from weboob.browser import LoginBrowser, URL, need_login from weboob.exceptions import BrowserIncorrectPassword, BrowserPasswordExpired from weboob.capabilities.bank import Account, Investment from weboob.capabilities.base import find_object - -from .pages import LoginPage, AccountsPage, ProAccountsPage, TransactionsPage, \ - ProTransactionsPage, IbanPage, RedirectPage, AVPage - +from .pages import ( + LoginPage, ProfilePage, AccountTypePage, AccountsPage, ProAccountsPage, + TransactionsPage, IbanPage, RedirectPage, EntryPage, AVPage, ProIbanPage, + ProTransactionsPage, LabelsPage, +) class CreditDuNordBrowser(LoginBrowser): ENCODING = 'UTF-8' + BASEURL = "https://www.credit-du-nord.fr/" login = URL('$', - '/.*\?.*_pageLabel=page_erreur_connexion', LoginPage) - redirect = URL('/swm/redirectCDN.html', RedirectPage) - av = URL('/vos-comptes/particuliers/V1_transactional_portal_page_', AVPage) - accounts = URL('/vos-comptes/particuliers', - '/vos-comptes/particuliers/transac_tableau_de_bord', AccountsPage) - transactions = URL('/vos-comptes/.*/transac/particuliers', TransactionsPage) - proaccounts = URL('/vos-comptes/(professionnels|entreprises)', ProAccountsPage) - protransactions = URL('/vos-comptes/.*/transac/(professionnels|entreprises)', ProTransactionsPage) - loans = URL('/vos-comptes/professionnels/credit_en_cours', ProAccountsPage) + '/.*\?.*_pageLabel=page_erreur_connexion', + LoginPage) + redirect = URL('/swm/redirectCDN.html', RedirectPage) + entrypage = URL('/icd/zco/#zco', EntryPage) + multitype_av = URL('/vos-comptes/IPT/appmanager/transac/professionnels\?_nfpb=true&_eventName=onRestart&_pageLabel=synthese_contrats_assurance_vie', AVPage) + loans = URL('/vos-comptes/IPT/appmanager/transac/(?P.*)\?_nfpb=true&_eventName=onRestart&_pageLabel=(?P(creditPersoImmobilier|credit__en_cours|credit_en_cours))', ProAccountsPage) + proaccounts = URL('/vos-comptes/IPT/appmanager/transac/(professionnels|entreprises)\?_nfpb=true&_eventName=onRestart&_pageLabel=(?P(transac_tableau_de_bord|page__synthese_v1|page_synthese_v1))', ProAccountsPage) + accounts = URL('/vos-comptes/IPT/appmanager/transac/(?P.*)\?_nfpb=true&_eventName=onRestart&_pageLabel=(?P(transac_tableau_de_bord|page__synthese_v1|page_synthese_v1))', AccountsPage) + multitype_iban = URL('/vos-comptes/IPT/appmanager/transac/professionnels\?_nfpb=true&_eventName=onRestart&_pageLabel=impression_rib', ProIbanPage) + transactions = URL('/vos-comptes/IPT/appmanager/transac/particuliers\?_nfpb=true(.*)', TransactionsPage) + protransactions = URL('/vos-comptes/(.*)/transac/(professionnels|entreprises)', ProTransactionsPage) iban = URL('/vos-comptes/IPT/cdnProxyResource/transacClippe/RIB_impress.asp', IbanPage) + account_type_page = URL("/icd/zco/public-data/public-ws-menuespaceperso.json", AccountTypePage) + labels_page = URL("/icd/zco/public-data/ws-menu.json", LabelsPage) + profile_page = URL("/icd/zco/data/user.json", ProfilePage) - account_type = 'particuliers' - - def __init__(self, website, *args, **kwargs): + def __init__(self, *args, **kwargs): + self.weboob = kwargs['weboob'] super(CreditDuNordBrowser, self).__init__(*args, **kwargs) - self.BASEURL = "https://%s" % website def is_logged(self): return self.page is not None and not self.login.is_here() and \ @@ -56,59 +60,64 @@ def is_logged(self): def home(self): if self.is_logged(): - self.location('/vos-comptes/%s' % self.account_type, params=self.strid) - self.location(self.page.doc.xpath(u'//a[contains(text(), "Synthèse")]')[0].attrib['href']) + self.location("/icd/zco/") + self.accounts.go(account_type=self.account_type) else: self.do_login() def do_login(self): self.login.go().login(self.username, self.password) - if self.accounts.is_here(): expired_error = self.page.get_password_expired() if expired_error: raise BrowserPasswordExpired(expired_error) if self.login.is_here(): - raise BrowserIncorrectPassword(self.page.get_error()) + error = self.page.get_error() + if error: + raise BrowserIncorrectPassword(error) + else: + # in case we are still on login without error message + # we'll check what's happening. + assert False, "Still on login page." if not self.is_logged(): raise BrowserIncorrectPassword() - self.strid = {"strid": self.page.get_strid()} - m = re.match('https://[^/]+/vos-comptes/(\w+).*', self.url) - if m: - self.account_type = m.group(1) - - @need_login def _iter_accounts(self): - self.home() - self.location(self.page.get_av_link()) - if self.av.is_here(): + self.loans.go(account_type=self.account_type, loans_page_label=self.loans_page_label) + for a in self.page.get_list(): + yield a + self.accounts.go(account_type=self.account_type, accounts_page_label=self.accounts_page_label) + self.multitype_av.go() + if self.multitype_av.is_here(): for a in self.page.get_av_accounts(): self.location(a._link, data=a._args) self.location(a._link.replace("_attente", "_detail_contrat_rep"), data=a._args) self.page.fill_diff_currency(a) yield a - self.home() - for a in self.page.get_list(): - yield a - self.loans.go() + self.accounts.go(account_type=self.account_type, accounts_page_label=self.accounts_page_label) for a in self.page.get_list(): yield a @need_login - def get_accounts_list(self): - accounts = list(self._iter_accounts()) + def get_pages_labels(self): + self.labels_page.go() + return self.page.get_labels() - self.page.iban_page() + @need_login + def get_accounts_list(self): + self.accounts_page_label, self.loans_page_label = self.get_pages_labels() + self.account_type_page.go() + self.account_type = self.page.get_account_type() + accounts = list(self._iter_accounts()) + self.multitype_iban.go() link = self.page.iban_go() - if self.page.has_iban(): - for a in [a for a in accounts if a._acc_nb]: - self.location(link + a._acc_nb) - a.iban = self.page.get_iban() + for a in [a for a in accounts if a._acc_nb]: + self.location(link + a._acc_nb) + a.iban = self.page.get_iban() return accounts @@ -116,15 +125,18 @@ def get_account(self, id): account_list = self.get_accounts_list() return find_object(account_list, id=id) + @need_login + def get_account_for_history(self, id): + account_list = list(self._iter_accounts()) + return find_object(account_list, id=id) + @need_login def iter_transactions(self, link, args, acc_type): if args is None: return - while args is not None: self.location(link, data=args) - assert self.transactions.is_here() - + assert (self.transactions.is_here() or self.protransactions.is_here()) for tr in self.page.get_history(acc_type): yield tr @@ -132,26 +144,19 @@ def iter_transactions(self, link, args, acc_type): @need_login def get_history(self, account, coming=False): - if coming and account.type is not Account.TYPE_CARD or account.type is Account.TYPE_LOAN: - return [] - - self.location('/vos-comptes/%s' % self.account_type, params=self.strid) - transactions = [] + if coming and account.type != Account.TYPE_CARD or account.type == Account.TYPE_LOAN: + return for tr in self.iter_transactions(account._link, account._args, account.type): - transactions.append(tr) - return transactions + yield tr @need_login def get_investment(self, account): - investments = [] - - if u'LIQUIDIT' in account.label: + if 'LIQUIDIT' in account.label: inv = Investment() - inv.code = u'XX-Liquidity' - inv.label = u'Liquidité' + inv.code = 'XX-Liquidity' + inv.label = 'Liquidité' inv.valuation = account.balance - investments.append(inv) - return investments + return [inv] if not account._inv: return [] @@ -159,15 +164,15 @@ def get_investment(self, account): if account.type in (Account.TYPE_MARKET, Account.TYPE_PEA): self.location(account._link, data=account._args) if self.page.can_iter_investments(): - investments = [i for i in self.page.get_market_investment()] + return self.page.get_market_investment() elif (account.type == Account.TYPE_LIFE_INSURANCE): self.location(account._link, data=account._args) self.location(account._link.replace("_attente", "_detail_contrat_rep"), data=account._args) if self.page.can_iter_investments(): - investments = [i for i in self.page.get_deposit_investment()] - return investments + return self.page.get_deposit_investment() + return [] @need_login def get_profile(self): - self.home() + self.profile_page.go() return self.page.get_profile() diff --git a/modules/creditdunord/compat/weboob_capabilities_bank.py b/modules/creditdunord/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/creditdunord/compat/weboob_capabilities_bank.py +++ b/modules/creditdunord/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/creditdunord/module.py b/modules/creditdunord/module.py index bbb5f5d44a18ff04ba5f11b2c74286cfd2b5559f..e6d56096ece52c966a72764dfbc542b4b85dc691 100644 --- a/modules/creditdunord/module.py +++ b/modules/creditdunord/module.py @@ -17,6 +17,8 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals + from collections import OrderedDict from .compat.weboob_capabilities_bank import CapBankWealth, AccountNotFound @@ -55,7 +57,8 @@ class CreditDuNordModule(Module, CapBankWealth, CapProfile): def create_default_browser(self): return self.create_browser(self.config['website'].get(), self.config['login'].get(), - self.config['password'].get()) + self.config['password'].get(), + weboob=self.weboob) def iter_accounts(self): for account in self.browser.get_accounts_list(): @@ -65,11 +68,16 @@ def get_account(self, _id): account = self.browser.get_account(_id) if account: return account - else: - raise AccountNotFound() + raise AccountNotFound() + + def get_account_for_history(self, _id): + account = self.browser.get_account_for_history(_id) + if account: + return account + raise AccountNotFound() def iter_history(self, account): - account = self.get_account(account.id) + account = self.get_account_for_history(account.id) for tr in self.browser.get_history(account): if not tr._is_coming: yield tr diff --git a/modules/creditdunord/pages.py b/modules/creditdunord/pages.py index 722fb48f26612a19cd6b8ab1c0213ba33ae14a3f..e3993a25634cef4d5d78f5efac23900d9e57e6d2 100755 --- a/modules/creditdunord/pages.py +++ b/modules/creditdunord/pages.py @@ -19,17 +19,18 @@ from __future__ import unicode_literals -import re import ast from decimal import Decimal from io import BytesIO from datetime import date as da from lxml import html +import re -from weboob.browser.pages import HTMLPage, LoggedPage +from weboob.browser.pages import HTMLPage, LoggedPage, JsonPage from weboob.browser.elements import method, ItemElement, TableElement -from weboob.browser.filters.standard import CleanText, Date, CleanDecimal, Regexp, Field +from weboob.browser.filters.standard import CleanText, Date, CleanDecimal, Regexp, Format, Field +from weboob.browser.filters.json import Dict from weboob.browser.filters.html import Attr, TableCell from weboob.exceptions import ActionNeeded, BrowserIncorrectPassword, BrowserUnavailable, BrowserPasswordExpired from weboob.capabilities.bank import Account, Investment @@ -46,7 +47,6 @@ def MyDecimal(*args, **kwargs): kwargs.update(replace_dots=True, default=NotAvailable) return CleanDecimal(*args, **kwargs) - def MyStrip(x, xpath='.'): if isinstance(x, unicode): return CleanText(xpath)(html.fromstring("

%s

" % x)) @@ -102,8 +102,11 @@ def on_load(self): self.browser.location(re.search(r'href="([^"]+)"', script.text).group(1)) -class LoginPage(HTMLPage): +class EntryPage(HTMLPage): + pass + +class LoginPage(HTMLPage): VIRTUALKEYBOARD = CDNVirtKeyboard def login(self, username, password): @@ -127,7 +130,6 @@ def vk_login(self, username, password): 'cryptocvcs': crypto, 'vk_op': 'auth', } - self.browser.location('/swm/redirectCDN.html', data=data) def classic_login(self, username, password): @@ -144,44 +146,77 @@ def classic_login(self, username, password): 'pwAuth': 'Authentification+mot+de+passe', 'username': username.encode(self.browser.ENCODING), } - self.browser.location('/saga/authentification', data=data) def get_error(self): return CleanText('//b[has-class("x-attentionErreurLigneHaut")]', default="")(self.doc) +class AccountTypePage(LoggedPage, JsonPage): + def get_account_type(self): + account_type = CleanText(Dict('donnees/id'))(self.doc) + if account_type == "menu_espace_perso_part": + return "particuliers" + elif account_type == "menu_espace_perso_pro": + return "professionnels" + elif account_type == "menu_espace_perso_ent": + return "entreprises" + + +class LabelsPage(LoggedPage, JsonPage): + def get_labels(self): + if not Dict('donnees')(self.doc) and Dict('commun/statut', default='')(self.doc) == 'nok': + # Dict('commun/statut') is only `GDPR` so we don't pass specific message. + raise ActionNeeded() + + synthesis_labels = ["Synthèse"] + loan_labels = ["Crédits en cours", "Crédits perso et immo", "Crédits"] + for element in Dict('donnees/0/submenu')(self.doc): + if CleanText(Dict('label'))(element) in synthesis_labels: + synthesis_label = CleanText(Dict('link'))(element).split("/")[-1] + if CleanText(Dict('label'))(element) in loan_labels: + loan_label = CleanText(Dict('link'))(element).split("/")[-1] + return (synthesis_label, loan_label) + + +class ProfilePage(LoggedPage, JsonPage): + def get_profile(self): + profile = Profile() + profile.name = Format('%s %s', CleanText(Dict('donnees/nom')), CleanText(Dict('donnees/prenom'), default=''))(self.doc) + return profile + + class CDNBasePage(HTMLPage): - def get_from_js(self, pattern, end_pattern, is_list=False): - """ - find a pattern in any javascript text - """ - for script in self.doc.xpath('//script'): - txt = script.text - if txt is None: - continue + def get_from_js(self, pattern, end_pattern, is_list=False): + """ + find a pattern in any javascript text + """ + for script in self.doc.xpath('//script'): + txt = script.text + if txt is None: + continue - start = txt.find(pattern) - if start < 0: - continue + start = txt.find(pattern) + if start < 0: + continue - values = [] - while start >= 0: - start += len(pattern) - end = txt.find(end_pattern, start) - values.append(txt[start:end]) + values = [] + while start >= 0: + start += len(pattern) + end = txt.find(end_pattern, start) + values.append(txt[start:end]) - if not is_list: - break + if not is_list: + break - start = txt.find(pattern, end) - return ','.join(values) + start = txt.find(pattern, end) + return ','.join(values) - def get_execution(self): - return self.get_from_js("name: 'execution', value: '", "'") + def get_execution(self): + return self.get_from_js("name: 'execution', value: '", "'") - def iban_go(self): - return '%s%s' % ('/vos-comptes/IPT/cdnProxyResource', self.get_from_js('C_PROXY.StaticResourceClientTranslation( "', '"')) + def iban_go(self): + return '%s%s' % ('/vos-comptes/IPT/cdnProxyResource', self.get_from_js('C_PROXY.StaticResourceClientTranslation( "', '"')) class AccountsPage(LoggedPage, CDNBasePage): @@ -206,6 +241,7 @@ class AccountsPage(LoggedPage, CDNBasePage): u'PLAN ÉPARGNE': Account.TYPE_SAVINGS, u'ASS.VIE': Account.TYPE_LIFE_INSURANCE, u'ÉTOILE AVANCE': Account.TYPE_LOAN, + u'ETOILE AVANCE': Account.TYPE_LOAN, u'PRÊT': Account.TYPE_LOAN, u'CREDIT': Account.TYPE_LOAN, u'FACILINVEST': Account.TYPE_LOAN, @@ -213,6 +249,14 @@ class AccountsPage(LoggedPage, CDNBasePage): u'COMPTE A TERME': Account.TYPE_DEPOSIT, } + def make__args_dict(self, line): + return {'_eventId': 'clicDetailCompte', + '_ipc_eventValue': '', + '_ipc_fireEvent': '', + 'execution': self.get_execution(), + 'idCompteClique': line[self.COL_ID], + } + def get_password_expired(self): error = CleanText('//div[@class="x-attentionErreur"]/b')(self.doc) if "vous devez modifier votre code confidentiel à la première connexion" in error: @@ -222,7 +266,6 @@ def get_account_type(self, label): for pattern, actype in sorted(self.TYPES.items()): if label.startswith(pattern) or label.endswith(pattern): return actype - return Account.TYPE_UNKNOWN def get_history_link(self): @@ -231,15 +274,6 @@ def get_history_link(self): def get_av_link(self): return self.doc.xpath('//a[contains(text(), "Consultation")]')[0].attrib['href'] - def make__args_dict(self, line): - return {'_eventId': 'clicDetailCompte', - '_ipc_eventValue': '', - '_ipc_fireEvent': '', - 'deviseAffichee': 'DEVISE', - 'execution': self.get_execution(), - 'idCompteClique': line[self.COL_ID], - } - def get_list(self): accounts = [] previous_account = None @@ -317,6 +351,10 @@ def get_strid(self): return re.search(r'(\d{4,})', Attr('//form[@name="changePageForm"]', 'action')(self.doc)).group(0) +class ProIbanPage(CDNBasePage): + pass + + class AVPage(LoggedPage, CDNBasePage): COL_LABEL = 0 COL_BALANCE = 3 @@ -331,7 +369,6 @@ def get_params(self, text): l.append(sub) for i, key in enumerate(self.ARGS): args[key] = l[self.ARGS.index(key)] - return url, args def get_av_accounts(self): @@ -343,17 +380,27 @@ def get_av_accounts(self): continue a = Account() + + # get acc_nb like on accounts page + a._acc_nb = Regexp( + CleanText('//div[@id="v1-cadre"]//b[contains(text(), "Compte N")]', replace=[(' ', '')]), + r'(\d+)' + )(self.doc)[5:] + a.label = CleanText('.')(cols[self.COL_LABEL]) a.type = Account.TYPE_LIFE_INSURANCE a.balance = MyDecimal('.')(cols[self.COL_BALANCE]) a.currency = a.get_currency(CleanText('.')(head_cols[self.COL_BALANCE])) a._link, a._args = self.get_params(cols[self.COL_LABEL].find('span/a').attrib['href']) - a.id = a._args['IndiceSupport'] + a._args['NumPolice'] - a._acc_nb = None + a.id = '%s%s%s' % (a._acc_nb, a._args['IndiceSupport'], a._args['NumPolice']) a._inv = True yield a +class PartAVPage(AVPage): + pass + + class ProAccountsPage(AccountsPage): COL_ID = 0 COL_BALANCE = 1 @@ -421,10 +468,13 @@ def get_list(self): a._link, a._args = None, None a._acc_nb = cols[self.COL_ID].xpath('.//span[@class="right-underline"] | .//span[@class="right"]')[0].text.replace(' ', '').strip() + a.id = a._acc_nb if hasattr(a, '_args') and a._args: - a.id = '%s%s%s' % (a._acc_nb, a._args['IndiceCompte'], a._args['Indiceclassement']) - else: - a.id = a._acc_nb + if a._args['IndiceCompte'].isdigit(): + a.id = '%s%s' % (a.id, a._args['IndiceCompte']) + if a._args['Indiceclassement'].isdigit(): + a.id = '%s%s' % (a.id, a._args['Indiceclassement']) + # This account can be multiple life insurance accounts if (any(a.label.startswith(lab) for lab in ['ASS.VIE-BONS CAPI-SCPI-DIVERS', 'BONS CAPI-SCPI-DIVERS']) or (u'Aucun d\\351tail correspondant pour ce compte' in tr.xpath('.//a/@href')[0]) @@ -598,8 +648,7 @@ def get_market_investment(self): COL_VALUATION = 4 COL_PERF = 5 - - for table in self.doc.xpath('//table[@class="datas-large"]'): + for table in self.doc.xpath('//div[not(@id="PortefeuilleCV")]/table[@class="datas"]'): for tr in table.xpath('.//tr[not(@class="entete")]'): cols = tr.findall('td') if len(cols) < 7: @@ -621,41 +670,33 @@ def get_market_investment(self): @method class get_deposit_investment(TableElement): - item_xpath = '//table[@class="datas"]/tr[not(@class="entete")]' - head_xpath = '//table[@class="datas"]/tr[(@class="entete")]/td/b' + item_xpath = '//table[@class="datas"]//tr[position()>1]' + head_xpath = '//table[@class="datas"]//tr[@class="entete"]/td/b' - col_label = re.compile('Libellé') - col_quantity = 'Quantité' - col_valuation = 'Montant (EUR)' - col_unitvalue = re.compile('Valeur liquidative') + col_label = u'Libellé' + col_quantity = u'Quantité' + col_unitvalue = re.compile(u"Valeur liquidative") + col_valuation = re.compile(u"Montant") class item(ItemElement): klass = Investment - - def obj_label(self): - return CleanText('./a')(TableCell("label")(self)[0]) or CleanText(TableCell("label"))(self) - - def obj_code(self): - link_label = CleanText('./a')(TableCell("label")(self)[0]) - if link_label: - return CleanText('./text()', default=NotAvailable)(TableCell("label")(self)[0]) - - obj_quantity = MyDecimal(TableCell("quantity")) - - obj_unitvalue = MyDecimal(TableCell("unitvalue")) - + obj_label = CleanText(TableCell('label')) + obj_quantity = MyDecimal(CleanText(TableCell('quantity'))) + obj_valuation = MyDecimal(TableCell('valuation')) + obj_unitvalue = MyDecimal(TableCell('unitvalue')) def obj_vdate(self): - if Field('unitvalue')(self): - return Date().filter(TableCell("unitvalue")(self)[0].xpath("./text()")[1]) - return Date(dayfirst=True, default=NotAvailable).filter(Regexp(CleanText('(//tr[td[span|b[contains(text(), "Estimation du contrat")]]]/td[2]/span)[2]'), \ - '(\d{2})/(\d{2})/(\d{4})', '\\3-\\2-\\1', default=NotAvailable)(self.page.doc)) - - obj_valuation = MyDecimal(TableCell("valuation")) - + if Field('unitvalue') is NotAvailable: + vdate = Date(dayfirst=True, default=NotAvailable)\ + .filter(Regexp(CleanText('.'), '(\d{2})/(\d{2})/(\d{4})', '\\3-\\2-\\1', default=NotAvailable)(TableCell('unitvalue')(self))) or \ + Date(dayfirst=True, default=NotAvailable)\ + .filter(Regexp(CleanText('//tr[td[span[b[contains(text(), "Estimation du contrat")]]]]/td[2]'), + '(\d{2})/(\d{2})/(\d{4})', '\\3-\\2-\\1', default=NotAvailable)(TableCell('unitvalue')(self))) + return vdate def fill_diff_currency(self, account): - valuation_diff = CleanText(u'//td[span[contains(text(), "dont +/- value : ")]]//b', default=None)(self.doc) - if valuation_diff: + valuation_diff = CleanText(u'//td[span[contains(text(), "dont +/- value : ")]]//b', default=None)(self.doc) + #NC == Non communiqué + if valuation_diff and "NC" not in valuation_diff: account.valuation_diff = MyDecimal().filter(valuation_diff) account.currency = account.get_currency(valuation_diff) diff --git a/modules/creditdunordpee/compat/weboob_capabilities_bank.py b/modules/creditdunordpee/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/creditdunordpee/compat/weboob_capabilities_bank.py +++ b/modules/creditdunordpee/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/creditmutuel/browser.py b/modules/creditmutuel/browser.py index d639dcb7b59565c22c9ca11d06e390fd35071cf9..27633c225764dd809fdaaa9e7736d1300fc33755 100644 --- a/modules/creditmutuel/browser.py +++ b/modules/creditmutuel/browser.py @@ -19,9 +19,11 @@ from __future__ import unicode_literals +import re from datetime import datetime from dateutil.relativedelta import relativedelta from itertools import groupby +from operator import attrgetter from weboob.tools.compat import basestring from weboob.tools.value import Value @@ -32,9 +34,11 @@ from weboob.browser.pages import FormNotFound from weboob.browser.exceptions import ClientError, ServerError from .compat.weboob_exceptions import BrowserIncorrectPassword, AuthMethodNotImplemented, BrowserUnavailable -from weboob.capabilities.bank import Account, AddRecipientStep, AddRecipientError, Recipient, Investment +from weboob.capabilities.bank import Account, AddRecipientStep, Recipient +from .compat.weboob_tools_capabilities_bank_investments import create_french_liquidity from weboob.capabilities import NotAvailable from weboob.tools.compat import urlparse +from weboob.capabilities.base import find_object from .pages import ( LoginPage, LoginErrorPage, AccountsPage, UserSpacePage, @@ -44,7 +48,7 @@ LIAccountsPage, CardsActivityPage, CardsListPage, CardsOpePage, NewAccountsPage, InternalTransferPage, ExternalTransferPage, RevolvingLoanDetails, RevolvingLoansList, - ErrorPage, SubscriptionPage, + ErrorPage, SubscriptionPage, CardsHistAvailable, CardPage2 ) @@ -58,75 +62,79 @@ class CreditMutuelBrowser(LoginBrowser, StatesMixin): BASEURL = 'https://www.creditmutuel.fr' login = URL('/fr/authentification.html', - '/(?P.*)fr/$', - '/(?P.*)fr/banques/accueil.html', - '/(?P.*)fr/banques/particuliers/index.html', + r'/(?P.*)fr/$', + r'/(?P.*)fr/banques/accueil.html', + r'/(?P.*)fr/banques/particuliers/index.html', LoginPage) - login_error = URL('/(?P.*)fr/identification/default.cgi', LoginErrorPage) - accounts = URL('/(?P.*)fr/banque/situation_financiere.cgi', - '/(?P.*)fr/banque/situation_financiere.html', + login_error = URL(r'/(?P.*)fr/identification/default.cgi', LoginErrorPage) + accounts = URL(r'/(?P.*)fr/banque/situation_financiere.cgi', + r'/(?P.*)fr/banque/situation_financiere.html', AccountsPage) revolving_loan_list = URL(r'/(?P.*)fr/banque/CR/arrivee.asp\?fam=CR.*', RevolvingLoansList) revolving_loan_details = URL(r'/(?P.*)fr/banque/CR/cam9_vis_lstcpt.asp.*', RevolvingLoanDetails) - user_space = URL('/(?P.*)fr/banque/espace_personnel.aspx', - '/(?P.*)fr/banque/accueil.cgi', - '/(?P.*)fr/banque/DELG_Gestion', - '/(?P.*)fr/banque/paci_engine/static_content_manager.aspx', + user_space = URL(r'/(?P.*)fr/banque/espace_personnel.aspx', + r'/(?P.*)fr/banque/accueil.cgi', + r'/(?P.*)fr/banque/DELG_Gestion', + r'/(?P.*)fr/banque/paci_engine/static_content_manager.aspx', UserSpacePage) - card = URL('/(?P.*)fr/banque/operations_carte.cgi.*', - '/(?P.*)fr/banque/mouvements.html\?webid=.*cardmonth=\d+$', - '/(?P.*)fr/banque/mouvements.html.*webid=.*cardmonth=\d+.*cardid=', + card = URL(r'/(?P.*)fr/banque/operations_carte.cgi.*', + r'/(?P.*)fr/banque/mouvements.html\?webid=.*cardmonth=\d+$', + r'/(?P.*)fr/banque/mouvements.html.*webid=.*cardmonth=\d+.*cardid=', CardPage) - operations = URL('/(?P.*)fr/banque/mouvements.cgi.*', - '/(?P.*)fr/banque/mouvements.html.*', - '/(?P.*)fr/banque/nr/nr_devbooster.aspx.*', + operations = URL(r'/(?P.*)fr/banque/mouvements.cgi.*', + r'/(?P.*)fr/banque/mouvements.html.*', + r'/(?P.*)fr/banque/nr/nr_devbooster.aspx.*', r'(?P.*)fr/banque/CRP8_GESTPMONT.aspx\?webid=.*&trnref=.*&contract=\d+&cardid=.*&cardmonth=\d+', OperationsPage) - coming = URL('/(?P.*)fr/banque/mvts_instance.cgi.*', ComingPage) - info = URL('/(?P.*)fr/banque/BAD.*', EmptyPage) - change_pass = URL('/(?P.*)fr/validation/change_password.cgi', + coming = URL(r'/(?P.*)fr/banque/mvts_instance.cgi.*', ComingPage) + info = URL(r'/(?P.*)fr/banque/BAD.*', EmptyPage) + change_pass = URL(r'/(?P.*)fr/validation/change_password.cgi', '/fr/services/change_password.html', ChangePasswordPage) - verify_pass = URL('/(?P.*)fr/validation/verif_code.cgi.*', VerifCodePage) - new_home = URL('/(?P.*)fr/banque/pageaccueil.html', - '/(?P.*)banque/welcome_pack.html', NewHomePage) - empty = URL('/(?P.*)fr/banques/index.html', - '/(?P.*)fr/banque/paci_beware_of_phishing.*', - '/(?P.*)fr/validation/(?!change_password|verif_code|image_case|infos).*', + verify_pass = URL(r'/(?P.*)fr/validation/verif_code.cgi.*', VerifCodePage) + new_home = URL(r'/(?P.*)fr/banque/pageaccueil.html', + r'/(?P.*)banque/welcome_pack.html', NewHomePage) + empty = URL(r'/(?P.*)fr/banques/index.html', + r'/(?P.*)fr/banque/paci_beware_of_phishing.*', + r'/(?P.*)fr/validation/(?!change_password|verif_code|image_case|infos).*', EmptyPage) - por = URL('/(?P.*)fr/banque/POR_ValoToute.aspx', - '/(?P.*)fr/banque/POR_SyntheseLst.aspx', + por = URL(r'/(?P.*)fr/banque/POR_ValoToute.aspx', + r'/(?P.*)fr/banque/POR_SyntheseLst.aspx', PorPage) - li = URL('/(?P.*)fr/assurances/profilass.aspx\?domaine=epargne', - '/(?P.*)fr/assurances/(consultations?/)?WI_ASS.*', - '/(?P.*)fr/assurances/WI_ASS', + li = URL(r'/(?P.*)fr/assurances/profilass.aspx\?domaine=epargne', + r'/(?P.*)fr/assurances/(consultations?/)?WI_ASS.*', + r'/(?P.*)fr/assurances/WI_ASS', '/fr/assurances/', LIAccountsPage) - iban = URL('/(?P.*)fr/banque/rib.cgi', IbanPage) + iban = URL(r'/(?P.*)fr/banque/rib.cgi', IbanPage) - new_accounts = URL('/(?P.*)fr/banque/comptes-et-contrats.html', NewAccountsPage) - new_operations = URL('/(?P.*)fr/banque/mouvements.cgi', - '/fr/banque/nr/nr_devbooster.aspx.*', - '/(?P.*)fr/banque/RE/aiguille(liste)?.asp', + new_accounts = URL(r'/(?P.*)fr/banque/comptes-et-contrats.html', NewAccountsPage) + new_operations = URL(r'/(?P.*)fr/banque/mouvements.cgi', + r'/fr/banque/nr/nr_devbooster.aspx.*', + r'/(?P.*)fr/banque/RE/aiguille(liste)?.asp', '/fr/banque/mouvements.html', - '/(?P.*)fr/banque/consultation/operations', OperationsPage) + r'/(?P.*)fr/banque/consultation/operations', OperationsPage) - advisor = URL('/(?P.*)fr/banques/contact/trouver-une-agence/(?P.*)', - '/(?P.*)fr/infoclient/', + advisor = URL(r'/(?P.*)fr/banques/contact/trouver-une-agence/(?P.*)', + r'/(?P.*)fr/infoclient/', r'/(?P.*)fr/banques/accueil/menu-droite/Details.aspx\?banque=.*', AdvisorPage) - redirect = URL('/(?P.*)fr/banque/paci_engine/static_content_manager.aspx', RedirectPage) + redirect = URL(r'/(?P.*)fr/banque/paci_engine/static_content_manager.aspx', RedirectPage) - cards_activity = URL('/(?P.*)fr/banque/pro/ENC_liste_tiers.aspx', CardsActivityPage) - cards_list = URL('/(?P.*)fr/banque/pro/ENC_liste_ctr.*', - '/(?P.*)fr/banque/pro/ENC_detail_ctr', CardsListPage) - cards_ope = URL('/(?P.*)fr/banque/pro/ENC_liste_oper', CardsOpePage) + cards_activity = URL(r'/(?P.*)fr/banque/pro/ENC_liste_tiers.aspx', CardsActivityPage) + cards_list = URL(r'/(?P.*)fr/banque/pro/ENC_liste_ctr.*', + r'/(?P.*)fr/banque/pro/ENC_detail_ctr', CardsListPage) + cards_ope = URL(r'/(?P.*)fr/banque/pro/ENC_liste_oper', CardsOpePage) + cards_ope2 = URL('/(?P.*)fr/banque/CRP8_SCIM_DEPCAR.aspx', CardPage2) - internal_transfer = URL('/(?P.*)fr/banque/virements/vplw_vi.html', InternalTransferPage) - external_transfer = URL('/(?P.*)fr/banque/virements/vplw_vee.html', ExternalTransferPage) - recipients_list = URL('/(?P.*)fr/banque/virements/vplw_bl.html', RecipientsListPage) - error = URL('/(?P.*)validation/infos.cgi', ErrorPage) + cards_hist_available = URL('/(?P.*)fr/banque/SCIM_default.aspx\?_tabi=C&_stack=SCIM_ListeActivityStep%3a%3a&_pid=ListeCartes&_fid=ChangeList&Data_ServiceListDatas_CurrentType=MyCards', CardsHistAvailable) + cards_hist_available2 = URL('/(?P.*)fr/banque/SCIM_default.aspx', CardsHistAvailable) - subscription = URL('/(?P.*)fr/banque/MMU2_LstDoc.aspx', SubscriptionPage) + internal_transfer = URL(r'/(?P.*)fr/banque/virements/vplw_vi.html', InternalTransferPage) + external_transfer = URL(r'/(?P.*)fr/banque/virements/vplw_vee.html', ExternalTransferPage) + recipients_list = URL(r'/(?P.*)fr/banque/virements/vplw_bl.html', RecipientsListPage) + error = URL(r'/(?P.*)validation/infos.cgi', ErrorPage) + + subscription = URL(r'/(?P.*)fr/banque/MMU2_LstDoc.aspx', SubscriptionPage) currentSubBank = None is_new_website = None @@ -172,8 +180,21 @@ def get_accounts_list(self): if not self.accounts_list: if self.currentSubBank is None: self.getCurrentSubBank() + self.accounts_list = [] self.revolving_accounts = [] + self.unavailablecards = [] + self.cards_histo_available = {} + + # For some cards the validity information is only availaible on these 2 links + self.cards_hist_available.go(subbank=self.currentSubBank) + if self.cards_hist_available.is_here(): + self.unavailablecards.extend(self.page.get_unavailable_cards()) + self.cards_histo_available.update(self.page.get_cards_list()) + + if self.cards_hist_available2.is_here(): + self.unavailablecards.extend(self.page.get_unavailable_cards()) + self.cards_histo_available.update(self.page.get_cards_list()) for acc in self.revolving_loan_list.stay_or_go(subbank=self.currentSubBank).iter_accounts(): self.accounts_list.append(acc) @@ -185,39 +206,47 @@ def get_accounts_list(self): [self.page] if self.is_new_website else [] for company in companies: page = self.open(company).page if isinstance(company, basestring) else company - self.accounts_list.extend([card for card in page.iter_cards()]) + self.accounts_list.extend(page.iter_cards()) + # Populate accounts from old website if not self.is_new_website: - for a in self.accounts.stay_or_go(subbank=self.currentSubBank).iter_accounts(): - self.accounts_list.append(a) + self.accounts.stay_or_go(subbank=self.currentSubBank) + self.accounts_list.extend(self.page.iter_accounts()) self.iban.go(subbank=self.currentSubBank).fill_iban(self.accounts_list) self.por.go(subbank=self.currentSubBank).add_por_accounts(self.accounts_list) + # Populate accounts from new website else: - for a in self.new_accounts.stay_or_go(subbank=self.currentSubBank).iter_accounts(): - self.accounts_list.append(a) + self.new_accounts.stay_or_go(subbank=self.currentSubBank) + self.accounts_list.extend(self.page.iter_accounts()) self.iban.go(subbank=self.currentSubBank).fill_iban(self.accounts_list) self.por.go(subbank=self.currentSubBank).add_por_accounts(self.accounts_list) - for acc in self.li.go(subbank=self.currentSubBank).iter_li_accounts(): - self.accounts_list.append(acc) + self.li.go(subbank=self.currentSubBank) + self.accounts_list.extend(self.page.iter_li_accounts()) + for acc in self.accounts_list: + if acc.id[:16] in self.cards_histo_available: + # ex of ID card : 000000xxxxxx0000 + acc.parent = find_object(self.accounts_list, id=self.cards_histo_available[acc.id[:16]][2]) excluded_label = ['etalis', 'valorisation totale'] self.accounts_list = [acc for acc in self.accounts_list if not any(w in acc.label.lower() for w in excluded_label)] return self.accounts_list - def get_account(self, id): - assert isinstance(id, basestring) + def get_account(self, _id): + assert isinstance(_id, basestring) for a in self.get_accounts_list(): - if a.id == id: + if a.id == _id: return a def getCurrentSubBank(self): # the account list and history urls depend on the sub bank of the user paths = urlparse(self.url).path.lstrip('/').split('/') self.currentSubBank = paths[0] + "/" if paths[0] != "fr" else "" + if self.currentSubBank and paths[0] == 'banqueprivee' and paths[1] == 'mabanque': + self.currentSubBank = 'banqueprivee/mabanque/' if self.currentSubBank and paths[1] == "decouverte": self.currentSubBank += paths[1] + "/" if paths[0] in ["fr", "mabanque"]: @@ -236,28 +265,32 @@ def list_operations(self, page, account): else: self.page = page - # on some savings accounts, the page lands on the contract tab, and we want the situation + # On some savings accounts, the page lands on the contract tab, and we want the situation if account.type == Account.TYPE_SAVINGS and "Capital Expansion" in account.label: self.page.go_on_history_tab() - # getting about 6 months history on new website + # Getting about 6 months history on new website if self.is_new_website and self.page: try: - for x in range(0, 2): + # Submit search form two times, at first empty, then filled based on available fields + for x in range(2): form = self.page.get_form(id="I1:fm", submit='//input[@name="_FID_DoActivateSearch"]') if x == 1: form.update({ - [k for k in form.keys() if "DateStart" in k][0]: (datetime.now() - relativedelta(months=7)).strftime('%d/%m/%Y'), - [k for k in form.keys() if "DateEnd" in k][0]: datetime.now().strftime('%d/%m/%Y') + next(k for k in form.keys() if "DateStart" in k): (datetime.now() - relativedelta(months=7)).strftime('%d/%m/%Y'), + next(k for k in form.keys() if "DateEnd" in k): datetime.now().strftime('%d/%m/%Y') }) - [form.pop(k, None) for k in form.keys() if "_FID_Do" in k and "DoSearch" not in k] + for k in form.keys(): + if "_FID_Do" in k and "DoSearch" not in k: + form.pop(k, None) form.submit() - except (IndexError, FormNotFound): - pass + # IndexError when form xpath returns [], StopIteration if next called on empty iterable + except (IndexError, StopIteration, FormNotFound): + self.logger.warning('Could not get history on new website') while self.page: try: - #submit form if their is more transactions to fetch + # Submit form if their is more transactions to fetch form = self.page.get_form(id="I1:fm") if self.page.doc.xpath('boolean(//a[@class="ei_loadmorebtn"])'): form['_FID_DoLoadMoreTransactions'] = "" @@ -266,7 +299,7 @@ def list_operations(self, page, account): break except (IndexError, FormNotFound): break - #sometime the browser can't go further + # Sometimes the browser can't go further except ClientError as exc: if exc.response.status_code == 413: break @@ -281,12 +314,15 @@ def list_operations(self, page, account): return self.pagination(lambda: self.page.get_history()) def get_monthly_transactions(self, trs): - groups = [list(g) for k, g in groupby(sorted(trs, key=lambda tr: tr.date), lambda tr: tr.date)] + date_getter = attrgetter('date') + groups = [list(g) for k, g in groupby(sorted(trs, key=date_getter), date_getter)] trs = [] for group in groups: + if group[0].date > datetime.today().date(): + continue tr = FrenchTransaction() - tr.raw = tr.label = u"RELEVE CARTE %s" % group[0].date - tr.amount = -sum([t.amount for t in group]) + tr.raw = tr.label = "RELEVE CARTE %s" % group[0].date + tr.amount = -sum(t.amount for t in group) tr.date = tr.rdate = tr.vdate = group[0].date tr.type = FrenchTransaction.TYPE_CARD_SUMMARY tr._is_coming = False @@ -300,8 +336,71 @@ def get_history(self, account): if not account._link_id: raise NotImplementedError() + if len(account.id) >= 16 and account.id[:16] in self.cards_histo_available: + # Check if '000000xxxxxx0000' card have an annual history + account._link_id = self.cards_histo_available[account.id[:16]][0] + self.location(account._link_id) + # The history of the card is available for 1 year with 1 month per page + # Here we catch all the url needed to be the more compatible with the catch of merged subtransactions + urlstogo = self.page.get_links() + self.location(account._link_id) + monthly_tr = [] + for url in urlstogo: + self.location(url) + if 'GoMonthPrecedent' not in url: + history = self.page.get_history() + self.tr_date = self.page.get_date() + if self.page.has_more_operations(): + for i in range(1, 100): + # Arbitrary range; it's the number of click needed to access to the full history of the month (stop with the next break) + data = { + '_FID_DoAddElem': '', + '_wxf2_cc': 'fr-FR', + '_wxf2_pmode': 'Normal', + '_wxf2_pseq': i, + '_wxf2_ptarget': 'C:P:updPan', + 'Data_ServiceListDatas_CurrentOtherCardThirdPartyNumber': '', + 'Data_ServiceListDatas_CurrentType': 'MyCards', + } + if 'fid=GoMonth&mois=' in self.url: + m = re.search(r'fid=GoMonth&mois=(\d+)', self.url) + if m: + m = m.group(1) + self.location('CRP8_SCIM_DEPCAR.aspx?_tabi=C&a__itaret=as=SCIM_ListeActivityStep\%3a\%3a\%2fSCIM_ListeRouter%3a%3a&a__mncret=SCIM_LST&a__ecpid=EID2011&_stack=_remote::moiSelectionner={},moiAfficher=firstHalf,typeDepense=T&_pid=SCIM_DEPCAR_Details'.format(m), data=data) + else: + self.location(self.url, data=data) + + if not self.page.has_more_operations_xml(): + history = self.page.iter_history_xml() + # We are now with an XML page with all the transactions of the months + break + else: + history = self.page.get_history() + + merged_amount = 0 + monthly_tr = [] + for tr in history: + if tr._regroup: + self.location(tr._regroup) + for tr2 in self.page.get_tr_merged(): + tr2._is_coming = tr._is_coming + tr2.date = self.tr_date + transactions.append(tr2) + merged_amount += tr2.amount + else: + transactions.append(tr) + monthly_tr.append(tr) + + monthly_summary = self.get_monthly_transactions(monthly_tr) + if monthly_summary: + monthly_summary[0].amount = abs(monthly_summary[0].amount - merged_amount) + transactions.extend(monthly_summary) + monthly_summary = [] + + return sorted_transactions(transactions) + # need to refresh the months select - if account._link_id.startswith('ENC_liste_oper'): + elif account._link_id.startswith('ENC_liste_oper'): self.location(account._pre_link) if not hasattr(account, '_card_pages'): @@ -314,11 +413,15 @@ def get_history(self, account): transactions.append(tr) differed_date = None - cards = [page.select_card(account._card_number) for page in account._card_pages] if hasattr(account, '_card_pages') else \ - account._card_links if hasattr(account, '_card_links') else [] + cards = ([page.select_card(account._card_number) for page in account._card_pages] + if hasattr(account, '_card_pages') + else account._card_links if hasattr(account, '_card_links') else []) for card in cards: card_trs = [] for tr in self.list_operations(card, account): + if tr._to_delete: + # Delete main transaction when subtransactions exist + continue if hasattr(tr, '_differed_date') and (not differed_date or tr._differed_date < differed_date): differed_date = tr._differed_date if tr.date >= datetime.now(): @@ -332,9 +435,9 @@ def get_history(self, account): if differed_date is not None: # set deleted for card_summary for tr in transactions: - tr.deleted = tr.type == FrenchTransaction.TYPE_CARD_SUMMARY and \ - differed_date.month <= tr.date.month and \ - not hasattr(tr, '_is_manualsum') + tr.deleted = (tr.type == FrenchTransaction.TYPE_CARD_SUMMARY + and differed_date.month <= tr.date.month + and not hasattr(tr, '_is_manualsum')) transactions = sorted_transactions(transactions) return transactions @@ -351,11 +454,9 @@ def get_investment(self, account): self.location(account._link_inv) return self.page.iter_investment() if account.type is Account.TYPE_PEA: - liquidity_inv = Investment() - liquidity_inv.label = account.label - liquidity_inv.code = u'XX-liquidity' - liquidity_inv.valuation = account.balance - return [liquidity_inv] + liquidities = create_french_liquidity(account.balance) + liquidities.label = account.label + return [liquidities] return iter([]) @need_login @@ -380,7 +481,7 @@ def iter_recipients(self, origin_account): @need_login def init_transfer(self, account, to, amount, reason=None): - if to.category != u'Interne': + if to.category != 'Interne': self.external_transfer.go(subbank=self.currentSubBank) else: self.internal_transfer.go(subbank=self.currentSubBank) @@ -436,13 +537,13 @@ def get_recipient_object(self, recipient): r.category = recipient.category # On credit mutuel recipients are immediatly available. r.enabled_at = datetime.now().replace(microsecond=0) - r.currency = u'EUR' + r.currency = 'EUR' r.bank_name = NotAvailable return r def continue_new_recipient(self, recipient, **params): - if u'Clé' in params: - self.page.post_code(params[u'Clé']) + if 'Clé' in params: + self.page.post_code(params['Clé']) self.page.add_recipient(recipient) if self.page.bic_needed(): self.page.ask_bic(self.get_recipient_object(recipient)) @@ -471,7 +572,7 @@ def post_with_bic(self, recipient, **params): for k, v in self.form.items(): if k != 'url': data[k] = v - data['[t:dbt%3astring;x(11)]data_input_BIC'] = params[u'Bic'] + data['[t:dbt%3astring;x(11)]data_input_BIC'] = params['Bic'] self.location(self.form['url'], data=data) self.page.ask_sms(self.get_recipient_object(recipient)) @@ -483,16 +584,18 @@ def new_recipient(self, recipient, **params): return self.post_with_bic(recipient, **params) if 'code' in params: return self.end_new_recipient(recipient, **params) - if u'Clé' in params: + if 'Clé' in params: return self.continue_new_recipient(recipient, **params) + self.recipients_list.go(subbank=self.currentSubBank) if self.page.has_list(): - if recipient.category not in self.page.get_recipients_list(): - raise AddRecipientError('Recipient category is not on the website available list.') + assert recipient.category in self.page.get_recipients_list(), \ + 'Recipient category is not on the website available list.' self.page.go_list(recipient.category) + self.page.go_to_add() if self.verify_pass.is_here(): - raise AddRecipientStep(self.get_recipient_object(recipient), Value(u'Clé', label=self.page.get_question())) + raise AddRecipientStep(self.get_recipient_object(recipient), Value('Clé', label=self.page.get_question())) else: return self.continue_new_recipient(recipient, **params) @@ -507,7 +610,7 @@ def iter_subscriptions(self): def iter_documents(self, subscription): if self.currentSubBank is None: self.getCurrentSubBank() - self.subscription.go(subbank=self.currentSubBank, params={'typ':'doc'}) + self.subscription.go(subbank=self.currentSubBank, params={'typ': 'doc'}) security_limit = 10 diff --git a/modules/creditmutuel/compat/weboob_capabilities_bank.py b/modules/creditmutuel/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/creditmutuel/compat/weboob_capabilities_bank.py +++ b/modules/creditmutuel/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/creditmutuel/compat/weboob_tools_capabilities_bank_investments.py b/modules/creditmutuel/compat/weboob_tools_capabilities_bank_investments.py new file mode 100644 index 0000000000000000000000000000000000000000..2b68ff9e001c726a90445e05e0a8ef2e2f165059 --- /dev/null +++ b/modules/creditmutuel/compat/weboob_tools_capabilities_bank_investments.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Jonathan Schmidt +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.tools.compat import basestring +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.bank import Investment + +def is_isin_valid(isin): + """ + Méthode générale + Table de conversion des lettres en chiffres + A=10 B=11 C=12 D=13 E=14 F=15 G=16 H=17 I=18 + J=19 K=20 L=21 M=22 N=23 O=24 P=25 Q=26 R=27 + S=28 T=29 U=30 V=31 W=32 X=33 Y=34 Z=35 + + 1 - Mettre de côté la clé, qui servira de référence à la fin de la vérification. + 2 - Convertir toutes les lettres en nombres via la table de conversion ci-contre. Si le nombre obtenu est supérieur ou égal à 10, prendre les deux chiffres du nombre séparément (exemple : 27 devient 2 et 7). + 3 - Pour chaque chiffre, multiplier sa valeur par deux si sa position est impaire en partant de la droite. Si le nombre obtenu est supérieur ou égal à 10, garder les deux chiffres du nombre séparément (exemple : 14 devient 1 et 4). + 4 - Faire la somme de tous les chiffres. + 5 - Soustraire cette somme de la dizaine supérieure ou égale la plus proche (exemples : si la somme vaut 22, la dizaine « supérieure ou égale » est 30, et la clé vaut donc 8 ; si la somme vaut 30, la dizaine « supérieure ou égale » est 30, et la clé vaut 0 ; si la somme vaut 31, la dizaine « supérieure ou égale » est 40, et la clé vaut 9). + 6 - Comparer la valeur obtenue à la clé mise initialement de côté. + + Étapes 1 et 2 : + F R 0 0 0 3 5 0 0 0 0 (+ 8 : clé) + 15 27 0 0 0 3 5 0 0 0 0 + + Étape 3 : le traitement se fait sur des chiffres + 1 5 2 7 0 0 0 3 5 0 0 0 0 + I P I P I P I P I P I P I : position en partant de la droite (P = Pair, I = Impair) + 2 1 2 1 2 1 2 1 2 1 2 1 2 : coefficient multiplicateur + + 2 5 4 7 0 0 0 3 10 0 0 0 0 : résultat + + Étape 4 : + 2 + 5 + 4 + 7 + 0 + 0 + 0 + 3 + (1 + 0)+ 0 + 0 + 0 + 0 = 22 + + Étapes 5 et 6 : 30 - 22 = 8 (valeur de la clé) + """ + + if not isinstance(isin, basestring): + return False + if not re.match(r'^[A-Z]{2}[A-Z0-9]{9}\d$', isin): + return False + + isin_in_digits = ''.join(str(ord(x) - ord('A') + 10) if not x.isdigit() else x for x in isin[:-1]) + key = isin[-1:] + result = '' + for k, val in enumerate(isin_in_digits[::-1], start=1): + if k % 2 == 0: + result = ''.join((result, val)) + else: + result = ''.join((result, str(int(val)*2))) + return str(sum(int(x) for x in result) + int(key))[-1] == '0' + + +def create_french_liquidity(valuation): + """ + Automatically fills a liquidity investment with label, code and code_type. + """ + liquidity = Investment() + liquidity.label = "Liquidités" + liquidity.code = "XX-liquidity" + liquidity.code_type = NotAvailable + liquidity.valuation = valuation + return liquidity + diff --git a/modules/creditmutuel/module.py b/modules/creditmutuel/module.py index 7afa78f8ae90ae3dd1bfab6f5388c080300fa6ae..2b3aba7f8e17c3b80c336aaff445df3a52182925 100644 --- a/modules/creditmutuel/module.py +++ b/modules/creditmutuel/module.py @@ -22,8 +22,10 @@ from decimal import Decimal from weboob.capabilities.base import find_object, NotAvailable -from .compat.weboob_capabilities_bank import CapBankWealth, CapBankTransferAddRecipient, AccountNotFound, RecipientNotFound, \ - Account, TransferError +from .compat.weboob_capabilities_bank import ( + CapBankWealth, CapBankTransferAddRecipient, AccountNotFound, RecipientNotFound, + Account, TransferError +) from weboob.capabilities.contact import CapContact from weboob.capabilities.profile import CapProfile from weboob.capabilities.bill import ( @@ -110,10 +112,12 @@ def init_transfer(self, transfer, **params): try: assert account.id.isdigit() - # quantize to show 2 decimals. + except AssertionError: + raise TransferError('Account id is invalid') + try: # quantize to show 2 decimals. amount = Decimal(transfer.amount).quantize(Decimal(10) ** -2) - except (AssertionError, ValueError): - raise TransferError('something went wrong') + except ValueError: + raise TransferError('Transfer amount is invalid') # drop characters that can crash website transfer.label = transfer.label.encode('cp1252', errors="ignore").decode('cp1252') diff --git a/modules/creditmutuel/pages.py b/modules/creditmutuel/pages.py index 8e0d506f38e8bf3362ae5432fa7dd11388e778e6..0a5a9f7badfff707de016c0b497b129599dc5644 100644 --- a/modules/creditmutuel/pages.py +++ b/modules/creditmutuel/pages.py @@ -28,10 +28,11 @@ from random import randint from collections import OrderedDict -from weboob.browser.pages import HTMLPage, FormNotFound, LoggedPage, pagination +from weboob.browser.pages import HTMLPage, FormNotFound, LoggedPage, pagination, XMLPage from weboob.browser.elements import ListElement, ItemElement, SkipItem, method, TableElement -from weboob.browser.filters.standard import Filter, Env, CleanText, CleanDecimal, Field, \ - Regexp, Async, AsyncLoad, Date, Format, Type, Currency +from weboob.browser.filters.standard import ( + Filter, Env, CleanText, CleanDecimal, Field, Regexp, Async, AsyncLoad, Date, Format, Type, Currency, +) from .compat.weboob_browser_filters_html import Link, Attr, TableCell, ColumnNotFound from .compat.weboob_exceptions import ( BrowserIncorrectPassword, ParseError, NoAccountsException, ActionNeeded, BrowserUnavailable, @@ -39,8 +40,10 @@ ) from weboob.capabilities import NotAvailable from weboob.capabilities.base import empty, find_object -from weboob.capabilities.bank import Account, Investment, Recipient, TransferError, TransferBankError, \ - Transfer, AddRecipientError, AddRecipientStep, Loan +from .compat.weboob_capabilities_bank import ( + Account, Investment, Recipient, TransferError, TransferBankError, + Transfer, AddRecipientBankError, AddRecipientStep, Loan, +) from weboob.capabilities.contact import Advisor from weboob.capabilities.profile import Profile from weboob.tools.capabilities.bank.iban import is_iban_valid @@ -114,7 +117,7 @@ def on_load(self): class UserSpacePage(LoggedPage, HTMLPage): def on_load(self): if self.doc.xpath('//form[@id="GoValider"]'): - raise ActionNeeded(u"Le site du contrat Banque à Distance a besoin d'informations supplémentaires") + raise ActionNeeded("Le site du contrat Banque à Distance a besoin d'informations supplémentaires") super(UserSpacePage, self).on_load() @@ -160,7 +163,7 @@ class item_account_generic(ItemElement): ("Plan D'Epargne", Account.TYPE_SAVINGS), ('Tonic Croissance', Account.TYPE_SAVINGS), ('Capital Expansion', Account.TYPE_SAVINGS), - ('\xc9pargne', Account.TYPE_SAVINGS), + ('Épargne', Account.TYPE_SAVINGS), ('Compte Garantie Titres', Account.TYPE_MARKET), ]) @@ -261,34 +264,35 @@ def parse(self, el): # classical account id_xpath = './td[1]/a/node()[contains(@class, "doux")]' - id = CleanText(id_xpath, replace=[(' ', '')])(el) - if not id: + _id = CleanText(id_xpath, replace=[(' ', '')])(el) + if not _id: if 'rib' in p: - id = p['rib'][0] + _id = p['rib'][0] else: - id = p['webid'][0] + _id = p['webid'][0] self.env['_is_webid'] = True if self.is_revolving(Field('label')(self)): page = self.page.browser.open(link).page if isinstance(page, RevolvingLoansList): - # some revolving loans are listed on an other page. On the accountList, there is just a link for this page - # that's why we don't handle it here + # some revolving loans are listed on an other page. On the accountList, there is + # just a link for this page, that's why we don't handle it here raise SkipItem() # Handle cards - if id in self.parent.objects: + if _id in self.parent.objects: if not page: page = self.page.browser.open(link).page # Handle real balances - coming = page.find_amount(u"Opérations à venir") if page else None - accounting = page.find_amount(u"Solde comptable") if page else None + coming = page.find_amount("Opérations à venir") if page else None + accounting = page.find_amount("Solde comptable") if page else None # on old website we want card's history in account's history if not page.browser.is_new_website: - account = self.parent.objects[id] + account = self.parent.objects[_id] if not account.coming: account.coming = Decimal('0.0') - date = parse_french_date(Regexp(Field('label'), 'Fin (.+) (\d{4})', '01 \\1 \\2')(self)) + relativedelta(day=31) + # Get end of month + date = parse_french_date(Regexp(Field('label'), r'Fin (.+) (\d{4})', '01 \\1 \\2')(self)) + relativedelta(day=31) if date > datetime.now() - relativedelta(day=1): account.coming += balance account._card_links.append(link) @@ -297,7 +301,10 @@ def parse(self, el): single_card_xpath = '//span[has-class("_c1 fg _c1")]' card_xpath = multiple_cards_xpath + ' | ' + single_card_xpath for elem in page.doc.xpath(card_xpath): - card_id = Regexp(CleanText('.', symbols=' '), '([\dx]{16})')(elem) + card_id = Regexp(CleanText('.', symbols=' '), r'([\dx]{16})')(elem) + if card_id in self.page.browser.unavailablecards: + raise SkipItem() + if any(card_id in a.id for a in page.browser.accounts_list): continue @@ -306,9 +313,9 @@ def parse(self, el): card.id = card._card_number = card_id card._link_id = link card._is_inv = card._is_webid = False - card.parent = self.parent.objects[id] + card.parent = self.parent.objects[_id] - pattern = 'Carte\s(\w+).*\d{4}\s([A-Za-z\s]+)(.*)' + pattern = r'Carte\s(\w+).*\d{4}\s([A-Za-z\s]+)(.*)' m = re.search(pattern, CleanText('.')(elem)) card.label = "%s %s %s" % (m.group(1), card_id, m.group(2)) card.balance = Decimal('0.0') @@ -316,14 +323,14 @@ def parse(self, el): card._card_pages = [page] card.coming = Decimal('0.0') #handling the case were the month is the coming one. There won't be next_month here. - date = parse_french_date(Regexp(Field('label'), 'Fin (.+) (\d{4})', '01 \\1 \\2')(self)) + relativedelta(day=31) + date = parse_french_date(Regexp(Field('label'), r'Fin (.+) (\d{4})', '01 \\1 \\2')(self)) + relativedelta(day=31) if date > datetime.now() - relativedelta(day=1): card.coming = CleanDecimal(replace_dots=True).filter(m.group(3)) next_month = Link('./following-sibling::tr[contains(@class, "encours")][1]/td[1]//a', default=None)(self) if next_month: card_page = page.browser.open(next_month).page for e in card_page.doc.xpath(card_xpath): - if card.id == Regexp(CleanText('.', symbols=' '), '([\dx]{16})')(e): + if card.id == Regexp(CleanText('.', symbols=' '), r'([\dx]{16})')(e): m = re.search(pattern, CleanText('.')(e)) card._card_pages.append(card_page) card.coming += CleanDecimal(replace_dots=True).filter(m.group(3)) @@ -333,7 +340,7 @@ def parse(self, el): raise SkipItem() - self.env['id'] = id + self.env['id'] = _id if accounting is not None and accounting + (coming or Decimal('0')) != balance: self.page.logger.warning('%s + %s != %s' % (accounting, coming, balance)) @@ -345,8 +352,9 @@ def parse(self, el): self.env['coming'] = coming or NotAvailable def is_revolving(self, label): - return any(revolving_loan_label in label - for revolving_loan_label in item_account_generic.REVOLVING_LOAN_LABELS) or label.lower() in self.page.browser.revolving_accounts + return (any(revolving_loan_label in label + for revolving_loan_label in item_account_generic.REVOLVING_LOAN_LABELS) + or label.lower() in self.page.browser.revolving_accounts) class AccountsPage(LoggedPage, HTMLPage): @@ -364,8 +372,8 @@ class iter_accounts(ListElement): class item_account(item_account_generic): def condition(self): - type = Field('type')(self) - return item_account_generic.condition(self) and type != Account.TYPE_LOAN + _type = Field('type')(self) + return item_account_generic.condition(self) and _type != Account.TYPE_LOAN class item_loan(item_account_generic): klass = Loan @@ -378,7 +386,7 @@ class item_loan(item_account_generic): obj_nb_payments_left = Async('details') & Type(CleanText( '//div[@id="F4:expContent"]/table/tbody/tr[2]/td[2]/text()'), type=int, default=NotAvailable) obj_subscription_date = Async('details') & MyDate(Regexp(CleanText( - '//*[@id="F4:expContent"]/table/tbody/tr[1]/th[1]'), ' (\d{2}/\d{2}/\d{4})', default=NotAvailable)) + '//*[@id="F4:expContent"]/table/tbody/tr[1]/th[1]'), r' (\d{2}/\d{2}/\d{4})', default=NotAvailable)) obj_maturity_date = Async('details') & MyDate( CleanText('//div[@id="F4:expContent"]/table/tbody/tr[4]/td[2]')) @@ -387,11 +395,11 @@ class item_loan(item_account_generic): CleanText('//div[@id="F4:expContent"]/table/tbody/tr[3]/td[1]')) obj_last_payment_amount = Async('details') & MyDecimal('//td[@id="F2_0.T12"]') - obj_last_payment_date = Async('details') & \ - MyDate(CleanText('//div[@id="F8:expContent"]/table/tbody/tr[1]/td[1]')) + obj_last_payment_date = (Async('details') & + MyDate(CleanText('//div[@id="F8:expContent"]/table/tbody/tr[1]/td[1]'))) def condition(self): - type = Field('type')(self) + _type = Field('type')(self) label = Field('label')(self) details_link = Link('.//a', default=None)(self) @@ -400,7 +408,8 @@ def condition(self): if re.search(r'Le\sMobile\s+([0-9]{2}\s?){5}', label): return False - if details_link and item_account_generic.condition and type == Account.TYPE_LOAN and not self.is_revolving(label): + if (details_link and item_account_generic.condition and _type == Account.TYPE_LOAN + and not self.is_revolving(label)): details = self.page.browser.open(details_link) if details.page and not 'cloturé' in CleanText('//form[@id="P:F"]//div[@class="blocmsg info"]//p')(details.page.doc): return True @@ -418,9 +427,9 @@ def obj_used_amount(self): return -Field('balance')(self) def condition(self): - type = Field('type')(self) + _type = Field('type')(self) label = Field('label')(self) - return (item_account_generic.condition(self) and type == Account.TYPE_LOAN + return (item_account_generic.condition(self) and _type == Account.TYPE_LOAN and self.is_revolving(label)) def get_advisor_link(self): @@ -448,7 +457,8 @@ def get_agency(self): class get_advisor(ItemElement): klass = Advisor - obj_name = Regexp(CleanText('//script[contains(text(), "Espace Conseiller")]'), 'consname.+?([\w\s]+)') + obj_name = Regexp(CleanText('//script[contains(text(), "Espace Conseiller")]'), + r'consname.+?([\w\s]+)') @method class get_profile(ItemElement): @@ -486,7 +496,7 @@ def next_page(self): return self.next_month() text = CleanText.clean(form.el) - m = re.search('(\d+)/(\d+)', text or '', flags=re.MULTILINE) + m = re.search(r'(\d+)/(\d+)', text or '', flags=re.MULTILINE) if not m: return self.next_month() @@ -532,7 +542,7 @@ def next_page(self): form = self.page.get_form('//form[contains(@id, "frmStarcLstCtrPag")]') form['imgCtrPagSui.x'] = randint(1, 29) form['imgCtrPagSui.y'] = randint(1, 17) - m = re.search('(\d+)/(\d+)', CleanText('.')(form.el)) + m = re.search(r'(\d+)/(\d+)', CleanText('.')(form.el)) if m and int(m.group(1)) < int(m.group(2)): return form.request except FormNotFound: @@ -543,11 +553,13 @@ class item(ItemElement): load_details = Field('_link_id') & AsyncLoad - obj_number = Field('_link_id') & Regexp(pattern='ctr=(\d+)') + obj_number = Field('_link_id') & Regexp(pattern=r'ctr=(\d+)') obj__card_number = Env('id', default="") obj_id = Format('%s%s', Env('id', default=""), Field('number')) - obj_label = Format('%s %s %s', CleanText(TableCell('card')), Env('id', default=""), CleanText(TableCell('owner'))) - obj_coming = CleanDecimal('./td[@class="i d" or @class="p d"][2]', replace_dots=True, default=NotAvailable) + obj_label = Format('%s %s %s', CleanText(TableCell('card')), Env('id', default=""), + CleanText(TableCell('owner'))) + obj_coming = CleanDecimal('./td[@class="i d" or @class="p d"][2]', replace_dots=True, + default=NotAvailable) obj_balance = Decimal('0.00') obj_currency = FrenchTransaction.Currency(CleanText('./td[small][1]')) obj_type = Account.TYPE_CARD @@ -581,6 +593,9 @@ def parse(self, el): self.handle_attr(attr, getattr(self, 'obj_%s' % attr)) setattr(card, attr, getattr(self.obj, attr)) + if _id in self.page.browser.cards_histo_available: + card.coming = self.page.browser.cards_histo_available[_id][1] + card._card_number = _id card.id = _id + card.number card.label = card.label.replace(' ', ' %s ' % _id) @@ -603,18 +618,18 @@ def parse(self, el): class Transaction(FrenchTransaction): - PATTERNS = [(re.compile('^VIR(EMENT)? (?P.*)'), FrenchTransaction.TYPE_TRANSFER), - (re.compile('^(PRLV|Plt|PRELEVEMENT) (?P.*)'), FrenchTransaction.TYPE_ORDER), - (re.compile('^(?P.*) CARTE \d+ PAIEMENT CB\s+(?P
\d{2})(?P\d{2}) ?(.*)$'), + PATTERNS = [(re.compile(r'^VIR(EMENT)? (?P.*)'), FrenchTransaction.TYPE_TRANSFER), + (re.compile(r'^(PRLV|Plt|PRELEVEMENT) (?P.*)'), FrenchTransaction.TYPE_ORDER), + (re.compile(r'^(?P.*) CARTE \d+ PAIEMENT CB\s+(?P
\d{2})(?P\d{2}) ?(.*)$'), FrenchTransaction.TYPE_CARD), - (re.compile('^PAIEMENT PSC\s+(?P
\d{2})(?P\d{2}) (?P.*) CARTE \d+ ?(.*)$'), + (re.compile(r'^PAIEMENT PSC\s+(?P
\d{2})(?P\d{2}) (?P.*) CARTE \d+ ?(.*)$'), FrenchTransaction.TYPE_CARD), - (re.compile('^(?PRELEVE CARTE.*)'), FrenchTransaction.TYPE_CARD_SUMMARY), - (re.compile('^RETRAIT DAB (?P
\d{2})(?P\d{2}) (?P.*) CARTE [\*\d]+'), + (re.compile(r'^(?PRELEVE CARTE.*)'), FrenchTransaction.TYPE_CARD_SUMMARY), + (re.compile(r'^RETRAIT DAB (?P
\d{2})(?P\d{2}) (?P.*) CARTE [\*\d]+'), FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile('^CHEQUE( (?P.*))?$'), FrenchTransaction.TYPE_CHECK), - (re.compile('^(F )?COTIS\.? (?P.*)'), FrenchTransaction.TYPE_BANK), - (re.compile('^(REMISE|REM CHQ) (?P.*)'), FrenchTransaction.TYPE_DEPOSIT), + (re.compile(r'^CHEQUE( (?P.*))?$'), FrenchTransaction.TYPE_CHECK), + (re.compile(r'^(F )?COTIS\.? (?P.*)'), FrenchTransaction.TYPE_BANK), + (re.compile(r'^(REMISE|REM CHQ) (?P.*)'), FrenchTransaction.TYPE_DEPOSIT), ] _is_coming = False @@ -642,11 +657,12 @@ def __call__(self, item): # hideifscript: Date de valeur XX/XX/XXXX # fd: Avis d'opéré # survey to add other regx - parts = (re.sub('Détail|Date de valeur\s+:\s+\d{2}/\d{2}(/\d{4})?', '', txt.strip()) for txt in el.itertext() if len(txt.strip()) > 0) - # Removing empty strings: - parts = [s for s in parts if s] + parts = (re.sub(r'Détail|Date de valeur\s+:\s+\d{2}/\d{2}(/\d{4})?', '', txt.strip()) + for txt in el.itertext() if txt.strip()) + # Removing empty strings + parts = list(filter(bool, parts)) # To simplify categorization of CB, reverse order of parts to separate - # location and institution. + # location and institution detail = "Cliquer pour déplier ou plier le détail de l'opération" if detail in parts: parts.remove(detail) @@ -708,6 +724,7 @@ class item(Transaction.TransactionElement): obj__gross_amount = CleanDecimal(Env('amount'), replace_dots=True) obj_commission = CleanDecimal(Format('-%s', Env('commission')), replace_dots=True, default=NotAvailable) + obj__to_delete = False def obj_amount(self): commission = Field('commission')(self) @@ -717,14 +734,16 @@ def obj_amount(self): return (abs(gross) - abs(commission)).copy_sign(gross) def parse(self, el): - self.env['date'] = Date(Regexp(CleanText('//td[contains(text(), "Total prélevé")]'), ' (\d{2}/\d{2}/\d{4})', \ - default=NotAvailable), default=NotAvailable)(self) + self.env['date'] = Date(Regexp(CleanText('//td[contains(text(), "Total prélevé")]'), + r' (\d{2}/\d{2}/\d{4})', default=NotAvailable), + default=NotAvailable)(self) if not self.env['date']: try: - d = CleanText('//select[@id="moi"]/option[@selected]')(self) or \ - re.search('pour le mois de (.*)', ''.join(w.strip() for w in self.page.doc.xpath('//div[@class="a_blocongfond"]/text()'))).group(1) + d = (CleanText('//select[@id="moi"]/option[@selected]')(self) + or re.search(r'pour le mois de (.*)', ''.join(w.strip() for w in + self.page.doc.xpath('//div[@class="a_blocongfond"]/text()'))).group(1)) except AttributeError: - d = Regexp(CleanText('//p[has-class("restriction")]'), 'pour le mois de ((?:\w+\s+){2})', flags=re.UNICODE)(self) + d = Regexp(CleanText('//p[has-class("restriction")]'), r'pour le mois de ((?:\w+\s+){2})', flags=re.UNICODE)(self) self.env['date'] = (parse_french_date('%s %s' % ('1', d)) + relativedelta(day=31)).date() self.env['_is_coming'] = date.today() < self.env['date'] amount = CleanText(TableCell('amount'))(self).split('dont frais') @@ -738,7 +757,7 @@ class get_history(Pagination, Transaction.TransactionsElement): head_xpath = '//table[has-class("liste")]//thead//tr/th/text()' item_xpath = '//table[has-class("liste")]//tbody/tr' - col_date = u"Date de l'annonce" + col_date = "Date de l'annonce" class item(Transaction.TransactionElement): obj__is_coming = True @@ -747,7 +766,7 @@ class item(Transaction.TransactionElement): class CardPage(OperationsPage, LoggedPage): def select_card(self, card_number): for option in self.doc.xpath('//select[@name="Data_SelectedCardItemKey"]/option'): - card_id = Regexp(CleanText('.', symbols=' '), '([\dx]+)')(option) + card_id = Regexp(CleanText('.', symbols=' '), r'([\dx]+)')(option) if card_id != card_number: continue if Attr('.', 'selected', default=None)(option): @@ -766,10 +785,20 @@ class list_cards(ListElement): class item(ItemElement): def __iter__(self): + # Here we handle the subtransactions card_link = self.el.get('href') + d = re.search(r'cardmonth=(\d+)', self.page.url) + if d: + year = int(d.group(1)[:4]) + month = int(d.group(1)[4:]) + debit_date = date(year, month, 1) + relativedelta(day=31) + page = self.page.browser.location(card_link).page for op in page.get_history(): + op.date = debit_date + op.type = FrenchTransaction.TYPE_DEFERRED_CARD + op._to_delete = False yield op class list_history(Transaction.TransactionsElement): @@ -787,7 +816,7 @@ def parse(self, el): if not label: return try: - label = re.findall('(\d+ [^ ]+ \d+)', label)[-1] + label = re.findall(r'(\d+ [^ ]+ \d+)', label)[-1] except IndexError: return # use the trick of relativedelta to get the last day of month. @@ -805,15 +834,21 @@ class item(Transaction.TransactionElement): obj_original_currency = Env('original_currency') obj__differed_date = Env('differed_date') + def obj__to_delete(self): + return bool(CleanText('.//a[contains(text(), "Regroupement")]')(self)) + def parse(self, el): try: - self.env['raw'] = "%s %s" % (CleanText().filter(TableCell('commerce')(self)[0].text), CleanText().filter(TableCell('ville')(self)[0].text)) + self.env['raw'] = "%s %s" % (CleanText().filter(TableCell('commerce')(self)[0].text), + CleanText().filter(TableCell('ville')(self)[0].text)) except (ColumnNotFound, AttributeError): self.env['raw'] = "%s" % (CleanText().filter(TableCell('commerce')(self)[0].text)) - self.env['type'] = Transaction.TYPE_DEFERRED_CARD \ - if CleanText('//a[contains(text(), "Prélevé fin")]', default=None) else Transaction.TYPE_CARD - self.env['differed_date'] = parse_french_date(Regexp(CleanText('//*[contains(text(), "Achats")]'), 'au[\s]+(.*)')(self)).date() + self.env['type'] = (Transaction.TYPE_DEFERRED_CARD + if CleanText('//a[contains(text(), "Prélevé fin")]', default=None) + else Transaction.TYPE_CARD) + self.env['differed_date'] = parse_french_date(Regexp(CleanText('//*[contains(text(), "Achats")]'), + r'au[\s]+(.*)')(self)).date() amount = TableCell('credit')(self)[0] if self.page.browser.is_new_website: if not len(amount.xpath('./div')): @@ -826,10 +861,211 @@ def parse(self, el): except IndexError: original_amount = None self.env['amount'] = CleanDecimal(replace_dots=True).filter(amount.text) - self.env['original_amount'] = CleanDecimal(replace_dots=True).filter(original_amount) \ - if original_amount is not None else NotAvailable - self.env['original_currency'] = Account.get_currency(original_amount[1:-1]) \ - if original_amount is not None else NotAvailable + self.env['original_amount'] = (CleanDecimal(replace_dots=True).filter(original_amount) + if original_amount is not None else NotAvailable) + self.env['original_currency'] = (Account.get_currency(original_amount[1:-1]) + if original_amount is not None else NotAvailable) + + +class CardPage2(CardPage, HTMLPage, XMLPage): + def build_doc(self, content): + if b'= 4 + + obj_raw = Transaction.Raw(Format("%s %s", CleanText(TableCell('commerce')), CleanText(TableCell('ville')))) + obj_rdate = Field('vdate') + + def obj_type(self): + if not 'RELEVE' in CleanText('//td[contains(., "Aucun mouvement")]')(self): + return Transaction.TYPE_DEFERRED_CARD + return Transaction.TYPE_CARD_SUMMARY + + def obj_original_amount(self): + m = re.search(r'(([\s-]\d+)+,\d+)', CleanText(TableCell('commerce'))(self)) + if m and not 'FRAIS' in CleanText(TableCell('commerce'))(self): + return Decimal(m.group(1).replace(',', '.').replace(' ', '')).quantize(Decimal('0.01')) + return NotAvailable + + def obj_original_currency(self): + m = re.search(r'(\d+,\d+) (\w+)', CleanText(TableCell('commerce'))(self)) + if Field('original_amount')(self) and m: + return m.group(2) + + def obj_date(self): + debit_date = CleanText('//a[@id="C:L4"]')(self) + if "fin" in debit_date: + return self.page.browser.tr_date + m = re.search(r'(\d{2}/\d{2}/\d{4})', debit_date) + if m: + return Date().filter(re.search(r'(\d{2}/\d{2}/\d{4})', debit_date).group(1)) + + def obj__is_coming(self): + debit_date = CleanText('//a[@id="C:L4"]')(self) + if "fin" in debit_date: + return True + if Date().filter(re.search(r'(\d{2}/\d{2}/\d{4})', debit_date).group(1)) > datetime.date(datetime.today()): + return True + return False + + def obj__regroup(self): + if "Regroupement" in CleanText('./td')(self): + return Link('./td/span/a')(self) + + @method + class get_tr_merged(ListElement): + class list_history(Transaction.TransactionsElement): + head_xpath = '//table[@class="liste"]//thead/tr/th' + item_xpath = '//table[@class="liste"]/tbody/tr' + + col_operation= u'Opération' + + def condition(self): + return not CleanText('//td[contains(., "Aucun mouvement")]', default=False)(self) + + class item(Transaction.TransactionElement): + def condition(self): + return len(self.el.xpath('./td')) >= 4 + + obj_label = CleanText(TableCell('operation')) + + def obj_type(self): + if not 'RELEVE' in Field('raw')(self): + return Transaction.TYPE_DEFERRED_CARD + return Transaction.TYPE_CARD_SUMMARY + + def has_more_operations(self): + xp = CleanText(self.doc.xpath('//div[@class="ei_blocpaginb"]/a'))(self) + if xp == 'Suite des opérations': + return True + return False + + def has_more_operations_xml(self): + if self.doc.xpath('//input') and Attr('//input', 'value')(self.doc) == 'Suite des opérations': + return True + return False + + @method + class iter_history_xml(ListElement): + class list_history(Transaction.TransactionsElement): + head_xpath = '//thead/tr/th' + item_xpath = '//tbody/tr' + + col_commerce = 'Commerce' + col_ville = 'Ville' + + class item(Transaction.TransactionElement): + obj_raw = Transaction.Raw(Format("%s %s", CleanText(TableCell('commerce')), CleanText(TableCell('ville')))) + obj_rdate = Field('vdate') + + def obj_type(self): + if not 'RELEVE' in CleanText('//td[contains(., "Aucun mouvement")]')(self): + return Transaction.TYPE_DEFERRED_CARD + return Transaction.TYPE_CARD_SUMMARY + + def obj_original_amount(self): + m = re.search(r'(([\s-]\d+)+,\d+)', CleanText(TableCell('commerce'))(self)) + if m and not 'FRAIS' in CleanText(TableCell('commerce'))(self): + return Decimal(m.group(1).replace(',', '.').replace(' ', '')).quantize(Decimal('0.01')) + return NotAvailable + + def obj_original_currency(self): + m = re.search(r'(\d+,\d+) (\w+)', CleanText(TableCell('commerce'))(self)) + if Field('original_amount')(self) and m: + return m.group(2) + + def obj__regroup(self): + if "Regroupement" in CleanText('./td')(self): + return Link('./td/span/a')(self) + + def obj_date(self): + return self.page.browser.tr_date + + def obj__is_coming(self): + if Field('date')(self) > datetime.date(datetime.today()): + return True + return False + + def get_date(self): + debit_date = CleanText(self.doc.xpath('//a[@id="C:L4"]'))(self) + if "fin" in debit_date: + m = re.search(r'fid=GoMonth&mois=(\d+)', self.browser.url) + y = re.search(r'annee=(\d+)', self.browser.url) + if m and y: + return date(int(y.group(1)), int(m.group(1)), 1) + relativedelta(day=31) + + m = re.search(r'(\d{2}/\d{2}/\d{4})', debit_date) + if m: + return Date().filter(re.search(r'(\d{2}/\d{2}/\d{4})', debit_date).group(1)) + + def get_links(self): + links = [] + + for link in self.doc.xpath('//div[@class="restriction"]/ul[1]/li'): + if link.xpath('./span/span/b'): + break + tmp_link = Link(link.xpath('./span/span/a'))(self) + if 'GoMonthPrecedent' in tmp_link: + secondpage = tmp_link + continue + m = re.search(r'fid=GoMonth&mois=(\d+)', tmp_link) + # To go to the page during the iter_history you need to have the good value from the precedent page + assert m, "It's not the URL expected" + m = int(m.group(1)) + m=m+1 if m!= 12 else 1 + url = re.sub(r'(?<=amoiSelectionner%3d)\d+', str(m), tmp_link) + links.append(url) + + links.reverse() + # Just for visiting the urls in a chronological way + m = re.search(r'fid=GoMonth&mois=(\d+)', links[0]) + y = re.search(r'annee=(\d+)', links[0]) + # We need to get a stable coming month instead of "fin du mois" + if m and y: + coming_date = date(int(y.group(1)), int(m.group(1)), 1) + relativedelta(months=+1) + + add_first = re.sub(r'(?<=amoiSelectionner%3d)\d+', str(coming_date.month), links[0]) + add_first = re.sub(r'(?<=GoMonth&mois=)\d+', str(coming_date.month), add_first) + add_first = re.sub(r'(?<=\&annee=)\d+', str(coming_date.year), add_first) + links.insert(0, add_first) + m = re.search(r'fid=GoMonth&mois=(\d+)', links[-1]).group(1) + links.append(re.sub(r'(?<=amoiSelectionner%3d)\d+', str(m), secondpage)) + + links2 = [] + page2 = self.browser.open(secondpage).page + for link in page2.doc.xpath('//div[@class="restriction"]/ul[1]/li'): + if link.xpath('./span/span/a'): + tmp_link = Link(link.xpath('./span/span/a'))(self) + if 'GoMonthSuivant' in tmp_link: + break + m = re.search(r'fid=GoMonth&mois=(\d+)', tmp_link) + assert m, "It's not the URL expected" + m = int(m.group(1)) + m=m+1 if m!= 12 else 1 + url = re.sub(r'(?<=amoiSelectionner%3d)\d+', str(m), tmp_link) + links2.append(url) + + links2.reverse() + links.extend(links2) + return links class LIAccountsPage(LoggedPage, HTMLPage): @@ -889,8 +1125,10 @@ class item(ItemElement): klass = Investment obj_label = CleanText(TableCell('label')) - obj_unitprice = CleanDecimal(TableCell('unitprice', default=NotAvailable), default=NotAvailable, replace_dots=True) - obj_vdate = Date(CleanText(TableCell('vdate'), replace=[('-', '')]), default=NotAvailable, dayfirst=True) + obj_unitprice = CleanDecimal(TableCell('unitprice', default=NotAvailable), + default=NotAvailable, replace_dots=True) + obj_vdate = Date(CleanText(TableCell('vdate'), replace=[('-', '')]), + default=NotAvailable, dayfirst=True) obj_unitvalue = CleanDecimal(TableCell('unitvalue'), default=NotAvailable, replace_dots=True) obj_quantity = CleanDecimal(TableCell('quantity'), default=NotAvailable, replace_dots=True) obj_valuation = CleanDecimal(TableCell('valuation'), default=Decimal(0), replace_dots=True) @@ -899,8 +1137,7 @@ def obj_code(self): link = Link(TableCell('label')(self)[0].xpath('./a'), default=NotAvailable)(self) if not link: return NotAvailable - return Regexp(pattern='isin=([A-Z\d]+)&?', default=NotAvailable).filter(link) - + return Regexp(pattern=r'isin=([A-Z\d]+)&?', default=NotAvailable).filter(link) class PorPage(LoggedPage, HTMLPage): @@ -933,7 +1170,7 @@ def add_por_accounts(self, accounts): if acc.id == '9999': # fake account continue - acc.label = unicode(re.sub("\d", '', ele.text).strip()) + acc.label = unicode(re.sub(r'\d', '', ele.text).strip()) acc._link_id = None acc.type = self.get_type(acc.label) acc._is_inv = True @@ -943,9 +1180,11 @@ def add_por_accounts(self, accounts): def fill(self, acc): self.send_form(acc) ele = self.browser.page.doc.xpath('.//table[has-class("fiche bourse")]')[0] - balance = CleanDecimal(ele.xpath('.//td[contains(@id, "Valorisation")]'), default=Decimal(0), replace_dots=True)(ele) + balance = CleanDecimal(ele.xpath('.//td[contains(@id, "Valorisation")]'), + default=Decimal(0), replace_dots=True)(ele) acc.balance = balance + acc.balance if acc.balance else balance - acc.valuation_diff = CleanDecimal(ele.xpath('.//td[contains(@id, "Variation")]'), default=Decimal(0), replace_dots=True)(ele) + acc.valuation_diff = CleanDecimal(ele.xpath('.//td[contains(@id, "Variation")]'), + default=Decimal(0), replace_dots=True)(ele) if balance: acc.currency = Currency('.//td[contains(@id, "Valorisation")]')(ele) else: @@ -957,13 +1196,13 @@ def fill(self, acc): # # Solution: remove the date text_content = CleanText('.')(ele) - date_pattern = r"\d{2}/\d{2}/\d{4}" + date_pattern = r'\d{2}/\d{2}/\d{4}' no_date = re.sub(date_pattern, '', text_content) acc.currency = Currency().filter(no_date) def send_form(self, account): form = self.get_form(name="frmMere") - form['POR_SyntheseEntete1$esdselLstPor'] = re.sub('\D', '', account.id) + form['POR_SyntheseEntete1$esdselLstPor'] = re.sub(r'\D', '', account.id) form.submit() @method @@ -982,7 +1221,7 @@ class item(ItemElement): klass = Investment obj_label = CleanText(TableCell('label'), default=NotAvailable) - obj_code = CleanText('.//td[1]/a/@title') & Regexp(pattern='^([^ ]+)') + obj_code = CleanText('.//td[1]/a/@title') & Regexp(pattern=r'^([^ ]+)') obj_quantity = CleanDecimal(TableCell('quantity'), default=Decimal(0), replace_dots=True) obj_unitprice = CleanDecimal(TableCell('unitprice'), default=Decimal(0), replace_dots=True) obj_valuation = CleanDecimal(TableCell('valuation'), default=Decimal(0), replace_dots=True) @@ -997,7 +1236,9 @@ def obj_unitvalue(self): def obj_vdate(self): td = TableCell('unitvalue')(self)[0] - return Date(Regexp(Attr('./img', 'title', default=''), r'Cours au : (\d{2}/\d{2}/\d{4})\b', default=None), dayfirst=True, default=NotAvailable)(td) + return Date(Regexp(Attr('./img', 'title', default=''), + r'Cours au : (\d{2}/\d{2}/\d{4})\b', default=None), + dayfirst=True, default=NotAvailable)(td) class IbanPage(LoggedPage, HTMLPage): @@ -1073,7 +1314,8 @@ class item(MyRecipient): obj_category = 'Interne' def obj_iban(self): - l = [a for a in self.page.browser.get_accounts_list() if Field('id')(self) in a.id and empty(a.valuation_diff)] + l = [a for a in self.page.browser.get_accounts_list() + if Field('id')(self) in a.id and empty(a.valuation_diff)] assert len(l) == 1 return l[0].iban @@ -1094,8 +1336,12 @@ def get_from_account_index(self, account): def get_to_account_index(self, account): return self.get_account_index(self.RECIPIENT_STRING, account) + def get_transfer_form(self): + # internal and external transfer form are differents + return self.get_form(id='P:F', submit='//input[@type="submit" and contains(@value, "Valider")]') + def prepare_transfer(self, account, to, amount, reason): - form = self.get_form(id='P:F', submit='//input[@type="submit" and contains(@value, "Valider")]') + form = self.get_transfer_form() form['data_input_indiceCompteADebiter'] = self.get_from_account_index(account.id) form[self.RECIPIENT_STRING] = self.get_to_account_index(to.id) form['[t:dbt%3adouble;]data_input_montant_value_0_'] = str(amount).replace('.', ',') @@ -1115,7 +1361,8 @@ def check_errors(self): 'Nom prénom du bénéficiaire différent du titulaire. Utilisez un compte courant', "Pour effectuer cette opération, vous devez passer par l’intermédiaire d’un compte courant", 'Montant maximum autorisé au crédit pour ce compte', - 'Débit interdit sur ce compte'] + 'Débit interdit sur ce compte', + 'Virement interdit sur compte clos',] for message in messages: if message in content: @@ -1130,16 +1377,20 @@ def check_success(self): raise TransferError('The expected message "%s" was not found.' % self.READY_FOR_TRANSFER_MSG) def check_data_consistency(self, account_id, recipient_id, amount, reason): - assert account_id in CleanText('//div[div[p[contains(text(), "Compte à débiter")]]]', replace=[(' ', '')])(self.doc) - assert recipient_id in CleanText('//div[div[p[contains(text(), "%s")]]]' % self.SUMMARY_RECIPIENT_TITLE, replace=[(' ', '')])(self.doc) + assert account_id in CleanText('//div[div[p[contains(text(), "Compte à débiter")]]]', + replace=[(' ', '')])(self.doc) + assert recipient_id in CleanText('//div[div[p[contains(text(), "%s")]]]' % self.SUMMARY_RECIPIENT_TITLE, + replace=[(' ', '')])(self.doc) - exec_date = Date(Regexp(CleanText('//table[@summary]/tbody/tr[th[contains(text(), "Date")]]/td'), '(\d{2}/\d{2}/\d{4})'), dayfirst=True)(self.doc) + exec_date = Date(Regexp(CleanText('//table[@summary]/tbody/tr[th[contains(text(), "Date")]]/td'), + r'(\d{2}/\d{2}/\d{4})'), dayfirst=True)(self.doc) assert exec_date == datetime.today().date() - r_amount = CleanDecimal('//table[@summary]/tbody/tr[th[contains(text(), "Montant")]]/td', replace_dots=True)(self.doc) + r_amount = CleanDecimal('//table[@summary]/tbody/tr[th[contains(text(), "Montant")]]/td', + replace_dots=True)(self.doc) assert r_amount == Decimal(amount) currency = FrenchTransaction.Currency('//table[@summary]/tbody/tr[th[contains(text(), "Montant")]]/td')(self.doc) if reason is not None: - assert reason.upper().strip()[:22] in CleanText('//table[@summary]/tbody/tr[th[contains(text(), "Intitulé pour le compte à débiter")]]/td')(self.doc) + assert reason.upper()[:22].strip() in CleanText('//table[@summary]/tbody/tr[th[contains(text(), "Intitulé pour le compte à débiter")]]/td')(self.doc) return exec_date, r_amount, currency def handle_response(self, account, recipient, amount, reason): @@ -1221,7 +1472,7 @@ def iter_categories(self): yield {'name': CleanText('.')(option), 'index': option.attrib['value']} def go_on_category(self, category_index): - form = self.get_form(id='P:F', submit='//input[@type="submit" and @value="Nom"]') + form = self.get_form(id='P2:F', submit='//input[@type="submit" and @value="Nom"]') form['data_input_indiceMarqueurListe'] = category_index form.submit() @@ -1254,6 +1505,11 @@ def obj_iban(self): def parse(self, el): self.env['origin_account']._external_recipients.add(Field('id')(self)) + def get_transfer_form(self): + # internal and external transfer form are differents + if self.IS_PRO_PAGE: + return self.get_form(id='P2:F', submit='//input[@type="submit" and contains(@value, "Valider")]') + return self.get_form(id='P1:F', submit='//input[@type="submit" and contains(@value, "Valider")]') class VerifCodePage(LoggedPage, HTMLPage): HASHES = { @@ -1324,9 +1580,13 @@ class VerifCodePage(LoggedPage, HTMLPage): } def on_load(self): - error = CleanText('//p[contains(text(), "Clé invalide !")] | //p[contains(text(), "Vous n\'avez pas saisi de clé!")]')(self.doc) - if error: - raise AddRecipientError(message=error) + errors = ( + CleanText('//p[contains(text(), "Clé invalide !")] | //p[contains(text(), "Vous n\'avez pas saisi de clé!")]')(self.doc), + CleanText('//p[contains(text(), "Vous n\'êtes pas inscrit") and a[text()="service d\'identification renforcée"]]')(self.doc), + ) + for error in errors: + if error: + raise AddRecipientBankError(message=error) action_needed = CleanText('//p[contains(text(), "Carte de CLÉS PERSONNELLES révoquée")]')(self.doc) if action_needed: @@ -1362,10 +1622,12 @@ def on_load(self): if error and error != 'Veuillez renseigner le BIC ou les coordonnées de la banque': # don't reload state if it fails because it's not supported by the website self.browser.need_clear_storage = True - raise AddRecipientError(message=error) + raise AddRecipientBankError(message=error) - app_validation = self.doc.xpath('//strong[contains(text(), "Démarrez votre application mobile Crédit Mutuel")]') + app_validation = self.doc.xpath('//strong[contains(text(), "Démarrez votre application mobile")]') if app_validation: + # don't reload state if it fails because it's not supported by the website + self.browser.need_clear_storage = True raise AuthMethodNotImplemented("La confirmation par validation sur votre application mobile n'est pas supportée") def has_list(self): @@ -1392,7 +1654,7 @@ def get_add_recipient_form(self, recipient): # Needed because it requires that \xe9 is encoded %E9 instead of %C3%A9 try: - del form[u'data_pilotageAffichage_habilitéSaisieInternationale'] + del form['data_pilotageAffichage_habilitéSaisieInternationale'] except KeyError: pass else: @@ -1425,9 +1687,10 @@ def ask_sms(self, recipient): form = self.get_form(id='P:F') self.set_browser_form(form) raise AddRecipientStep(recipient, Value('code', label=txt)) + # don't reload state if it fails because it's not supported by the website self.browser.need_clear_storage = True - raise AddRecipientError('Was expecting a page where sms code is asked') + assert False, 'Was expecting a page where sms code is asked' class RevolvingLoansList(LoggedPage, HTMLPage): @method @@ -1519,7 +1782,8 @@ def condition(self): # Some documents may have the same date, name and label; only parts of the PDF href may change, # so we must pick a unique ID including the href to avoid document duplicates: - obj_id = Format('%s_%s_%s', Env('sub_id'), CleanText(TableCell('date'), replace=[('/', '')]), Regexp(Field('url'), r'NOM=(.*)&RFL=')) + obj_id = Format('%s_%s_%s', Env('sub_id'), CleanText(TableCell('date'), replace=[('/', '')]), + Regexp(Field('url'), r'NOM=(.*)&RFL=')) obj_label = Format('%s %s', CleanText(TableCell('url')), CleanText(TableCell('date'))) obj_date = Date(CleanText(TableCell('date')), dayfirst=True) obj_format = 'pdf' @@ -1544,3 +1808,34 @@ def is_last_page(self): if re.search(r'(\d\/\d)', CleanText('//div[has-class("blocpaginb")]', symbols=' ')(self.doc)): return True return False + + +class CardsHistAvailable(LoggedPage, HTMLPage): + def get_unavailable_cards(self): + cards = [] + for card in self.doc.xpath('//li[@class="item"]'): + if CleanText(card.xpath('.//div[1]/p'))(self) != 'Active': + m = re.search(r'\d{4} \d{2}XX XXXX \d{4}', CleanText(card.xpath('.//span'))(self)) + if m: + cards.append(m.group(0).replace(' ', '').replace('X', 'x')) + return cards + + def get_cards_list(self): + cards = {} + for card in self.doc.xpath('//li[@class="item"]'): + if not card.xpath('.//tbody/tr/td/span'): + # No coming/balance values and history link are available + continue + m = re.search(r'\d{4} \d{2}XX XXXX \d{4}', CleanText(card.xpath('.//span'))(self)) + assert m, 'Id card is not present' + id_card = m.group(0).replace(' ', '').replace('X', 'x') + + link = Link(card.xpath('.//a[contains(@id,"C:more-card")]'))(self) + coming_xpath = card.xpath('.//tbody/tr/td/span') + coming = CleanDecimal(coming_xpath[0], replace_dots=True)(self) + if len(coming_xpath) > 1: + coming += CleanDecimal(coming_xpath[1], replace_dots=True)(self) + parent_id = re.search(r'\d+', CleanText(card.xpath('./div/div/div/p'), replace=[(' ', '')])(self)).group(0)[-16:] or None + cards[id_card] = [link, coming, parent_id] + + return cards diff --git a/modules/edf/par/browser.py b/modules/edf/par/browser.py index bd685cef80d53c3df4b201ff4f52d34303d6cad3..b26ecc0e2b90172b896e7eafbbc4b13ce387c8a5 100644 --- a/modules/edf/par/browser.py +++ b/modules/edf/par/browser.py @@ -40,11 +40,13 @@ class EdfBrowser(LoginBrowser): not_connected = URL('/fr/accueil/connexion/mon-espace-client.html', UnLoggedPage) connected = URL('/fr/accueil/espace-client/tableau-de-bord.html', WelcomePage) profil = URL('/services/rest/authenticate/getListContracts', ProfilPage) - csrf_token = URL('/services/rest/init/initPage\?_=(?P.*)', ProfilPage) + csrf_token = URL(r'/services/rest/init/initPage\?_=(?P.*)', ProfilPage) documents = URL('/services/rest/edoc/getMyDocuments', DocumentsPage) bills = URL('/services/rest/edoc/getBillsDocuments', DocumentsPage) bill_informations = URL('/services/rest/document/dataUserDocumentGetX', DocumentsPage) - bill_download = URL('/services/rest/document/getDocumentGetXByData\?csrfToken=(?P.*)&dn=(?P.*)&pn=(?P.*)&di=(?P.*)&bn=(?P.*)&an=(?P.*)') + bill_download = URL(r'/services/rest/document/getDocumentGetXByData' + r'\?csrfToken=(?P.*)&dn=(?P.*)&pn=(?P.*)' + r'&di=(?P.*)&bn=(?P.*)&an=(?P.*)') profile = URL('/services/rest/context/getCustomerContext', ProfilePage) def __init__(self, config, *args, **kwargs): @@ -113,7 +115,10 @@ def iter_documents(self, subscription): def download_document(self, document): token = self.get_csrf_token() - bills_informations = self.bill_informations.go(headers={'Content-Type': 'application/json;charset=UTF-8', 'Accept': 'application/json, text/plain, */*'}, data=json.dumps({ + bills_informations = self.bill_informations.go(headers={ + 'Content-Type': 'application/json;charset=UTF-8', + 'Accept': 'application/json, text/plain, */*'}, + data=json.dumps({ 'bpNumber': document._bp, 'csrfToken': token, 'docId': document._doc_number, @@ -122,9 +127,8 @@ def download_document(self, document): 'parNumber': document._par_number })).get_bills_informations() - return self.bill_download.go(csrf_token=token, - dn='FACTURE', pn=document._par_number, \ - di=document._doc_number, bn=bills_informations.get('bpNumber'), \ + return self.bill_download.go(csrf_token=token, dn='FACTURE', pn=document._par_number, + di=document._doc_number, bn=bills_informations.get('bpNumber'), an=bills_informations.get('numAcc')).content @need_login diff --git a/modules/edf/par/pages.py b/modules/edf/par/pages.py index 8fd94a972f5e36ff7cb3749122ade51190ce520d..46283c4f20a1934653efe28e84d9f2dd7554b04a 100644 --- a/modules/edf/par/pages.py +++ b/modules/edf/par/pages.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals from datetime import datetime from decimal import Decimal @@ -67,7 +68,8 @@ class iter_subscriptions(DictElement): class item(ItemElement): klass = Subscription - obj_subscriber = Format('%s %s', Dict('bp/identity/firstName'), Dict('bp/identity/lastName')) + obj_subscriber = Format('%s %s', Dict('bp/identity/firstName'), + Dict('bp/identity/lastName')) obj_id = Dict('number') obj_label = obj_id @@ -90,12 +92,13 @@ class item(ItemElement): klass = Bill obj_id = Format('%s_%s', Env('subid'), Dict('documentNumber')) - obj_date = Date(Eval(lambda t: datetime.fromtimestamp(int(t)/1000).strftime('%Y-%m-%d'), Dict('creationDate'))) - obj_format = u"pdf" + obj_date = Date(Eval(lambda t: datetime.fromtimestamp(int(t) / 1000) \ + .strftime('%Y-%m-%d'), Dict('creationDate'))) + obj_format = 'pdf' obj_label = Format('Facture %s', Dict('documentNumber')) - obj_type = u"bill" + obj_type = 'bill' obj_price = Env('price') - obj_currency = u'EUR' + obj_currency = 'EUR' obj_vat = NotAvailable obj__doc_number = Dict('documentNumber') obj__par_number = Dict('parNumber') @@ -121,8 +124,9 @@ def get_profile(self): data = self.doc['bp'] p = Profile() - p.address = '%s %s %s %s' % (data['streetNumber'], data['streetName'], data['postCode'], data['city']) - p.name = '%s %s' % (data['lastName'], data['firstName']) + p.address = '%s %s %s %s' % (data['streetNumber'], data['streetName'], + data['postCode'], data['city']) + p.name = '%s %s %s' % (data['civility'], data['lastName'], data['firstName']) p.phone = data['mobilePhoneNumber'] or data['fixPhoneNumber'] p.email = data['mail'] diff --git a/modules/edf/pro/browser.py b/modules/edf/pro/browser.py index 4c77fc98591d7fae47437cf96f0db333f25ca949..3b44507a39eff686e1a59fd83f69bc4b13dd4d49 100644 --- a/modules/edf/pro/browser.py +++ b/modules/edf/pro/browser.py @@ -25,11 +25,11 @@ from weboob.browser import LoginBrowser, URL, need_login from weboob.capabilities.base import NotAvailable -from weboob.exceptions import BrowserIncorrectPassword, ActionNeeded +from weboob.exceptions import BrowserIncorrectPassword, ActionNeeded, BrowserUnavailable from weboob.browser.exceptions import ServerError, ClientError from .pages import ( - LoginPage, HomePage, AuthPage, LireSitePage, + LoginPage, HomePage, AuthPage, ErrorPage, LireSitePage, SubscriptionsPage, BillsPage, DocumentsPage, ProfilePage, ) @@ -40,6 +40,7 @@ class EdfproBrowser(LoginBrowser): login = URL('/openam/json/authenticate', LoginPage) auth = URL('/openam/UI/Login.*', '/ice/rest/aiguillagemp/redirect', AuthPage) + error = URL(r'/page_erreur/', ErrorPage) home = URL('/ice/content/ice-pmse/homepage.html', HomePage) liresite = URL(r'/rest/homepagemp/liresite', LireSitePage) contracts = URL('/rest/contratmp/consultercontrats', SubscriptionsPage) @@ -56,7 +57,8 @@ def __init__(self, config, *args, **kwargs): super(EdfproBrowser, self).__init__(*args, **kwargs) def do_login(self): - login_data = self.login.go('/openam/json/authenticate', method='POST').get_json_data(self.username, self.password) + self.login.go('/openam/json/authenticate', method='POST') + login_data = self.page.get_data(self.username, self.password) try: self.login.go(data=json.dumps(login_data), headers={'Content-Type': 'application/json'}) except ClientError as e: @@ -66,20 +68,24 @@ def do_login(self): self.location(self.absurl('/rest/aiguillagemp/redirect'), allow_redirects=True) if self.auth.is_here() and self.page.response.status_code != 303: - raise BrowserIncorrectPassword + raise BrowserIncorrectPassword() + + if self.error.is_here(): + raise BrowserUnavailable(self.page.get_message()) self.session.headers['Content-Type'] = 'application/json;charset=UTF-8' self.session.headers['X-XSRF-TOKEN'] = self.session.cookies['XSRF-TOKEN'] @need_login def get_subscription_list(self): - self.liresite.go(data=json.dumps({"numPremierSitePage":0,"pageSize":100000,"idTdg":None,"critereFiltre":[],"critereTri":[]})) + self.liresite.go(data=json.dumps({"numPremierSitePage": 0, "pageSize": 100000, "idTdg": None, + "critereFiltre": [], "critereTri": []})) id_site_list = self.page.get_id_site_list() if not id_site_list: raise ActionNeeded("Vous ne disposez d'aucun contrat actif relatif à vos sites") if "subs" not in self.cache.keys(): - self.contracts.go(data=json.dumps({'refDevisOMList':[], 'refDevisOHList': id_site_list})) + self.contracts.go(data=json.dumps({'refDevisOMList': [], 'refDevisOHList': id_site_list})) self.cache['subs'] = [s for s in self.page.get_subscriptions()] return self.cache['subs'] @@ -104,15 +110,17 @@ def iter_documents(self, subscription): def download_document(self, document): if document.url is not NotAvailable: try: - self.bills.go(data=json.dumps({'date': int(document.date.strftime('%s')), \ - 'iDFelix': document._account_billing, 'numFacture': document._bill_number})) + self.bills.go(data=json.dumps({'date': int(document.date.strftime('%s')), + 'iDFelix': document._account_billing, + 'numFacture': document._bill_number})) - return self.open('%s/rest/facturemp/telechargerfichier?fname=%s' % (self.BASEURL, self.page.get_bill_name())).content + return self.open('%s/rest/facturemp/telechargerfichier?fname=%s' % ( + self.BASEURL, self.page.get_bill_name())).content except ServerError: return NotAvailable @need_login def get_profile(self): - self.profile.go(json={'idSpcInterlocuteur':''}) + self.profile.go(json={'idSpcInterlocuteur': ''}) return self.page.get_profile() diff --git a/modules/edf/pro/pages.py b/modules/edf/pro/pages.py index 44360e2e6c3f3f9a70d0e0f004021934f4deafb3..029c78e35de75bad8f3214b94e86ceabdae0fe3c 100644 --- a/modules/edf/pro/pages.py +++ b/modules/edf/pro/pages.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals from datetime import date @@ -31,7 +32,7 @@ class LoginPage(JsonPage): - def get_json_data(self, login, password): + def get_data(self, login, password): login_data = self.doc login_data['callbacks'][0]['input'][0]['value'] = login login_data['callbacks'][1]['input'][0]['value'] = password @@ -42,22 +43,29 @@ class AuthPage(RawPage): pass +class ErrorPage(HTMLPage): + def get_message(self): + return CleanText('//div[@id="div_text"]/h1 | //div[@id="div_text"]/p')(self.doc) + + class HomePage(LoggedPage, HTMLPage): pass -class LireSitePage(LoggedPage, JsonPage): +class JsonCguPage(JsonPage): + def build_doc(self, text): + if text == 'REDIRECT_CGU': # JSON can always be decoded in UTF-8 so testing text is fine + raise ActionNeeded("Vous devez accepter les conditions générales d'utilisation.") + return super(JsonCguPage, self).build_doc(text) + + +class LireSitePage(LoggedPage, JsonCguPage): # id site is not about website but geographical site def get_id_site_list(self): return [site['idSite'] for site in self.doc['site']] -class SubscriptionsPage(LoggedPage, JsonPage): - def build_doc(self, text): - if self.content == 'REDIRECT_CGU': - raise ActionNeeded(u"Vous devez accepter les conditions générales d'utilisation sur le site de votre banque.") - return super(SubscriptionsPage, self).build_doc(text) - +class SubscriptionsPage(LoggedPage, JsonCguPage): @method class get_subscriptions(DictElement): item_xpath = 'listeContrat' @@ -70,7 +78,8 @@ class item(ItemElement): obj_label = CleanText(CleanHTML(Dict('nomOffreModele'))) def obj_subscriber(self): - return ('%s %s' % (Dict('prenomIntPrinc')(self).lower(), Dict('nomIntPrinc')(self).lower())).title() + return ('%s %s' % (Dict('prenomIntPrinc')(self).lower(), + Dict('nomIntPrinc')(self).lower())).title() class BillsPage(LoggedPage, JsonPage): @@ -87,11 +96,11 @@ def get_documents(self): doc.id = document['numFactureLabel'] doc.date = date.fromtimestamp(int(document['dateEmission'] / 1000)) - doc.format = u'PDF' + doc.format = 'PDF' doc.label = 'Facture %s' % document['numFactureLabel'] - doc.type = u'bill' + doc.type = 'bill' doc.price = CleanDecimal().filter(document['montantTTC']) - doc.currency = u'€' + doc.currency = '€' doc._account_billing = document['compteFacturation'] doc._bill_number = document['numFacture'] diff --git a/modules/fortuneo/browser.py b/modules/fortuneo/browser.py index d39ffbbe04879a68a32f8529b29e32add08b876f..07410acd7ae452d6a96231aec8ed2b5105f1d3d5 100644 --- a/modules/fortuneo/browser.py +++ b/modules/fortuneo/browser.py @@ -25,14 +25,14 @@ from datetime import datetime, timedelta from weboob.browser.browsers import LoginBrowser, URL, need_login, StatesMixin -from .compat.weboob_exceptions import AuthMethodNotImplemented, BrowserIncorrectPassword +from .compat.weboob_exceptions import AuthMethodNotImplemented, BrowserIncorrectPassword, ActionNeeded from weboob.capabilities.bank import Account, AddRecipientStep, Recipient from weboob.tools.capabilities.bank.transactions import sorted_transactions from weboob.tools.value import Value from .pages.login import LoginPage, UnavailablePage from .pages.accounts_list import ( - AccountsList, AccountHistoryPage, CardHistoryPage, InvestmentHistoryPage, PeaHistoryPage, LoanPage, + AccountsList, AccountHistoryPage, CardHistoryPage, InvestmentHistoryPage, PeaHistoryPage, LoanPage, ProfilePage, ProfilePageCSV, SecurityPage, ) from .pages.transfer import ( RegisterTransferPage, ValidateTransferPage, ConfirmTransferPage, RecipientsPage, RecipientSMSPage @@ -62,6 +62,7 @@ class Fortuneo(LoginBrowser, StatesMixin): invest_history = URL(r'.*/prive/mes-comptes/assurance-vie/.*', InvestmentHistoryPage) loan_contract = URL(r'/fr/prive/mes-comptes/credit-immo/contrat-credit-immo/contrat-pret-immobilier.jsp.*', LoanPage) unavailable = URL(r'/customError/indispo.html', UnavailablePage) + security_page = URL(r'/fr/prive/identification-carte-securite-forte.jsp.*', SecurityPage) # transfer recipients = URL( @@ -84,6 +85,9 @@ class Fortuneo(LoginBrowser, StatesMixin): r'/fr/prive/mes-comptes/compte-courant/.*/confirmer-saisie-virement.jsp', ConfirmTransferPage) + profile = URL(r'/fr/prive/informations-client.jsp', ProfilePage) + profile_csv = URL(r'/PdfStruts\?*', ProfilePageCSV) + need_reload_state = None __states__ = ['need_reload_state', 'add_recipient_form'] @@ -149,6 +153,9 @@ def get_coming(self, account): def get_accounts_list(self): self.accounts_page.go() + # Note: if you want to debug process_action_needed() here, + # you must first set self.action_needed_processed to False + # otherwise it might not enter the "if" loop here below. if not self.action_needed_processed: self.process_action_needed() @@ -161,6 +168,13 @@ def process_action_needed(self): if url: self.location(self.absurl(url, base=True)) # beware, the landing page might vary according to the referer page. So far I didn't figure out how the landing page is chosen. + if self.security_page.is_here(): + # Some connections require reinforced security and we cannot bypass the OTP in order + # to get to the account information. Users have to provide a phone number in order to + # validate an OTP, so we must raise an ActionNeeded with the appropriate message. + raise ActionNeeded('Cette opération sensible doit être validée par un code sécurité envoyé par SMS ou serveur vocal. ' + 'Veuillez contacter le Service Clients pour renseigner vos coordonnées téléphoniques.') + # if there are skippable CGUs, skip them if self.accounts_page.is_here() and self.page.has_action_needed(): # Look for the request in the event listener registered to the button @@ -255,3 +269,11 @@ def execute_transfer(self, transfer): self.page.validate_transfer() self.page.confirm_transfer() return self.page.transfer_confirmation(transfer) + + @need_login + def get_profile(self): + self.profile.go() + csv_link = self.page.get_csv_link() + if csv_link: + self.location(csv_link) + return self.page.get_profile() diff --git a/modules/fortuneo/compat/weboob_capabilities_bank.py b/modules/fortuneo/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/fortuneo/compat/weboob_capabilities_bank.py +++ b/modules/fortuneo/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/fortuneo/module.py b/modules/fortuneo/module.py index 9aab657e641b5ad778f6e6f947f8900df58953db..fcba42338984fa38075a0c95d3f4ee8fea61b4f6 100644 --- a/modules/fortuneo/module.py +++ b/modules/fortuneo/module.py @@ -23,6 +23,7 @@ CapBankWealth, CapBankTransferAddRecipient, AccountNotFound, RecipientNotFound, TransferInvalidLabel, Account, ) +from weboob.capabilities.profile import CapProfile from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword @@ -32,7 +33,7 @@ __all__ = ['FortuneoModule'] -class FortuneoModule(Module, CapBankWealth, CapBankTransferAddRecipient): +class FortuneoModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapProfile): NAME = 'fortuneo' MAINTAINER = u'Gilles-Alexandre Quenot' EMAIL = 'gilles.quenot@gmail.com' @@ -67,6 +68,9 @@ def iter_coming(self, account): def iter_investment(self, account): return self.browser.get_investments(account) + def get_profile(self): + return self.browser.get_profile() + def iter_transfer_recipients(self, origin_account): if isinstance(origin_account, Account): origin_account = origin_account.id diff --git a/modules/fortuneo/pages/accounts_list.py b/modules/fortuneo/pages/accounts_list.py index 9c0bed7d54ba2d9e3addd37427d2e378cc2dbef8..051426446245ef7b80dc85022989473b1bbbffef 100644 --- a/modules/fortuneo/pages/accounts_list.py +++ b/modules/fortuneo/pages/accounts_list.py @@ -20,19 +20,24 @@ from __future__ import unicode_literals import re +import sys from time import sleep from datetime import date from dateutil.relativedelta import relativedelta from lxml.html import etree +from weboob.browser.elements import method, ItemElement from weboob.browser.filters.html import Link, Attr from weboob.browser.filters.standard import CleanText, CleanDecimal, RawText, Regexp, Date from weboob.capabilities import NotAvailable from weboob.capabilities.bank import Account, Investment, Loan -from weboob.browser.pages import HTMLPage, LoggedPage, FormNotFound +from weboob.capabilities.profile import Person +from weboob.browser.pages import HTMLPage, LoggedPage, FormNotFound, CsvPage from weboob.tools.capabilities.bank.transactions import FrenchTransaction +from .compat.weboob_tools_capabilities_bank_investments import create_french_liquidity from weboob.tools.json import json +from weboob.tools.date import parse_french_date from weboob.exceptions import ActionNeeded, BrowserUnavailable from weboob.tools.capabilities.bank.investments import is_isin_valid @@ -96,12 +101,9 @@ def get_investments(self, account): inv.code_type = Investment.CODE_TYPE_ISIN yield inv - if not account.type == account.TYPE_MARKET: - inv = Investment() - inv.code = "XX-liquidity" - inv.label = "Liquidités" - inv.valuation = CleanDecimal(None, True).filter(self.doc.xpath('//*[@id="valorisation_compte"]/table/tr[3]/td[2]')) - yield inv + if account.type != account.TYPE_MARKET: + valuation = CleanDecimal(None, True).filter(self.doc.xpath('//*[@id="valorisation_compte"]/table/tr[3]/td[2]')) + yield create_french_liquidity(valuation) def parse_decimal(self, string, replace_dots): string = CleanText(None).filter(string) @@ -519,3 +521,47 @@ def get_subscription_date(self): def get_maturity_date(self): return Date(CleanText(u'//p[@id="c_dateFin"]//strong'), dayfirst=True)(self.doc) + + +class ProfilePage(LoggedPage, HTMLPage): + def get_csv_link(self): + return Link('//div[@id="bloc_telecharger"]//a[@id="telecharger_donnees"]', default=NotAvailable)(self.doc) + + @method + class get_profile(ItemElement): + klass = Person + + obj_phone = Regexp(CleanText('//div[@id="consultationform_telephones"]/p[@id="c_numeroPortable"]'), '([\d\*]+)', default=None) + obj_email = CleanText('//div[@id="modification_email"]//p[@id="c_email_actuel"]/span') + obj_address = CleanText('//div[@id="consultationform_adresse_domicile"]/div[@class="container"]//span') + obj_job = CleanText('//div[@id="consultationform_informations_complementaires"]/p[@id="c_profession"]/span') + obj_job_activity_area = CleanText('//div[@id="consultationform_informations_complementaires"]/p[@id="c_secteurActivite"]/span') + obj_company_name = CleanText('//div[@id="consultationform_informations_complementaires"]/p[@id="c_employeur"]/span') + + +class ProfilePageCSV(LoggedPage, CsvPage): + ENCODING = 'latin_1' + if sys.version_info.major > 2: + FMTPARAMS = {'delimiter': ';'} + else: + FMTPARAMS = {'delimiter': b';'} + + def get_profile(self): + d = {el[0]: el[1] for el in self.doc} + profile = Person() + profile.name = '%s %s' % (d['Nom'], d['Prénom']) + profile.birth_date = parse_french_date(d['Date de naissance']).date() + profile.address = '%s %s %s' % (d['Adresse de correspondance'], d['Code postal résidence fiscale'], d['Ville adresse de correspondance']) + profile.country = d['Pays adresse de correspondance'] + profile.email = d['Adresse e-mail'] + profile.phone = d.get('Téléphone portable') + profile.job_activity_area = d.get('Secteur d\'activité') + profile.job = d.get('Situation professionnelle') + profile.company_name = d.get('Employeur') + profile.family_situation = d.get('Situation familiale') + return profile + + +class SecurityPage(LoggedPage, HTMLPage): + pass + diff --git a/modules/fortuneo/pages/compat/__init__.py b/modules/fortuneo/pages/compat/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/modules/fortuneo/pages/compat/weboob_capabilities_bank.py b/modules/fortuneo/pages/compat/weboob_capabilities_bank.py new file mode 100644 index 0000000000000000000000000000000000000000..141097d781b97a933881efe1a6851a102454051e --- /dev/null +++ b/modules/fortuneo/pages/compat/weboob_capabilities_bank.py @@ -0,0 +1,45 @@ + +import weboob.capabilities.bank as OLD + +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) + + +__all__ = OLD.__all__ + + +class CapBankWealth(CapBank): + pass + + +class CapBankPockets(CapBank): + pass + + +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + +class CapBankTransfer(OLD.CapBankTransfer): + def transfer_check_label(self, old, new): + from unidecode import unidecode + + return unidecode(old) == unidecode(new) + + +class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipient): + pass + + +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + +Account.TYPE_MORTGAGE = 17 +Account.TYPE_CONSUMER_CREDIT = 18 +Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/fortuneo/pages/compat/weboob_tools_capabilities_bank_investments.py b/modules/fortuneo/pages/compat/weboob_tools_capabilities_bank_investments.py new file mode 100644 index 0000000000000000000000000000000000000000..2b68ff9e001c726a90445e05e0a8ef2e2f165059 --- /dev/null +++ b/modules/fortuneo/pages/compat/weboob_tools_capabilities_bank_investments.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Jonathan Schmidt +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.tools.compat import basestring +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.bank import Investment + +def is_isin_valid(isin): + """ + Méthode générale + Table de conversion des lettres en chiffres + A=10 B=11 C=12 D=13 E=14 F=15 G=16 H=17 I=18 + J=19 K=20 L=21 M=22 N=23 O=24 P=25 Q=26 R=27 + S=28 T=29 U=30 V=31 W=32 X=33 Y=34 Z=35 + + 1 - Mettre de côté la clé, qui servira de référence à la fin de la vérification. + 2 - Convertir toutes les lettres en nombres via la table de conversion ci-contre. Si le nombre obtenu est supérieur ou égal à 10, prendre les deux chiffres du nombre séparément (exemple : 27 devient 2 et 7). + 3 - Pour chaque chiffre, multiplier sa valeur par deux si sa position est impaire en partant de la droite. Si le nombre obtenu est supérieur ou égal à 10, garder les deux chiffres du nombre séparément (exemple : 14 devient 1 et 4). + 4 - Faire la somme de tous les chiffres. + 5 - Soustraire cette somme de la dizaine supérieure ou égale la plus proche (exemples : si la somme vaut 22, la dizaine « supérieure ou égale » est 30, et la clé vaut donc 8 ; si la somme vaut 30, la dizaine « supérieure ou égale » est 30, et la clé vaut 0 ; si la somme vaut 31, la dizaine « supérieure ou égale » est 40, et la clé vaut 9). + 6 - Comparer la valeur obtenue à la clé mise initialement de côté. + + Étapes 1 et 2 : + F R 0 0 0 3 5 0 0 0 0 (+ 8 : clé) + 15 27 0 0 0 3 5 0 0 0 0 + + Étape 3 : le traitement se fait sur des chiffres + 1 5 2 7 0 0 0 3 5 0 0 0 0 + I P I P I P I P I P I P I : position en partant de la droite (P = Pair, I = Impair) + 2 1 2 1 2 1 2 1 2 1 2 1 2 : coefficient multiplicateur + + 2 5 4 7 0 0 0 3 10 0 0 0 0 : résultat + + Étape 4 : + 2 + 5 + 4 + 7 + 0 + 0 + 0 + 3 + (1 + 0)+ 0 + 0 + 0 + 0 = 22 + + Étapes 5 et 6 : 30 - 22 = 8 (valeur de la clé) + """ + + if not isinstance(isin, basestring): + return False + if not re.match(r'^[A-Z]{2}[A-Z0-9]{9}\d$', isin): + return False + + isin_in_digits = ''.join(str(ord(x) - ord('A') + 10) if not x.isdigit() else x for x in isin[:-1]) + key = isin[-1:] + result = '' + for k, val in enumerate(isin_in_digits[::-1], start=1): + if k % 2 == 0: + result = ''.join((result, val)) + else: + result = ''.join((result, str(int(val)*2))) + return str(sum(int(x) for x in result) + int(key))[-1] == '0' + + +def create_french_liquidity(valuation): + """ + Automatically fills a liquidity investment with label, code and code_type. + """ + liquidity = Investment() + liquidity.label = "Liquidités" + liquidity.code = "XX-liquidity" + liquidity.code_type = NotAvailable + liquidity.valuation = valuation + return liquidity + diff --git a/modules/fortuneo/pages/transfer.py b/modules/fortuneo/pages/transfer.py index 9e8c5193cedb586d3374e2783ee62918e2a752b1..9cbd23587dd748cdb7746af3051488e7936d76fd 100644 --- a/modules/fortuneo/pages/transfer.py +++ b/modules/fortuneo/pages/transfer.py @@ -23,11 +23,11 @@ from datetime import date, timedelta from weboob.browser.pages import HTMLPage, PartialHTMLPage, LoggedPage -from weboob.browser.elements import method, ListElement, ItemElement +from weboob.browser.elements import method, ListElement, ItemElement, SkipItem from weboob.browser.filters.standard import ( CleanText, Date, Regexp, CleanDecimal, Currency, Field, Env, ) -from weboob.capabilities.bank import Recipient, Transfer, TransferBankError, AddRecipientError +from .compat.weboob_capabilities_bank import Recipient, Transfer, TransferBankError, AddRecipientBankError from weboob.capabilities.base import NotAvailable @@ -43,16 +43,18 @@ def condition(self): class item(ItemElement): klass = Recipient - def obj_id(self): - iban = CleanText('./td[6]', replace=[(' ', '')])(self) - iban = re.search(r'(?<=IBAN:)(\w+)BIC', iban).group(1) - return iban - def obj_label(self): if Field('_custom_label')(self): return '{} - {}'.format(Field('_recipient_name')(self), Field('_custom_label')(self)) return Field('_recipient_name')(self) + def obj_id(self): + iban = CleanText('./td[6]', replace=[(' ', '')])(self) + iban_number = re.search(r'(?<=IBAN:)(\w+)BIC', iban) + if iban_number: + return iban_number.group(1) + raise SkipItem('There is no IBAN for the recipient %s' % Field('label')(self)) + obj__recipient_name = CleanText('./td[2]') obj__custom_label = CleanText('./td[4]') obj_iban = NotAvailable @@ -70,7 +72,7 @@ def check_external_iban_form(self, recipient): def check_recipient_iban(self): if not CleanText('//input[@name="codeBic"]/@value')(self.doc): - raise AddRecipientError('Recipient already exist or invalid iban') + raise AddRecipientBankError(message="Le bénéficiaire est déjà présent ou bien l'iban est incorrect") def fill_recipient_form(self, recipient) : form = self.get_form(id='CompteExterneActionForm') @@ -89,8 +91,8 @@ def get_new_recipient(self, recipient): recipient_xpath + '/li[contains(text(), "Nom du titulaire")]', replace=[(' ', '')] ), r'(?<=Nomdutitulaire:)(\w+)')(self.doc) rcpt.iban = Regexp(CleanText( - recipient_xpath + '/li[contains(text(), "IBAN")]', replace=[(' ', '')] - ), r'(?<=IBAN:)([A-Z]{2}\d+)')(self.doc) + recipient_xpath + '/li[contains(text(), "IBAN")]' + ), r'IBAN : ([A-Za-z]{2}[\dA-Za-z]+)')(self.doc) rcpt.id = rcpt.iban rcpt.category = 'Externe' rcpt.enabled_at = date.today() + timedelta(1) @@ -104,7 +106,7 @@ def get_send_code_form(self): class RecipientSMSPage(LoggedPage, PartialHTMLPage): def on_load(self): if not self.doc.xpath('//input[@id="otp"]') and not self.doc.xpath('//div[@class="confirmationAjoutCompteExterne"]'): - raise AddRecipientError(CleanText('//div[@id="aidesecuforte"]/p[contains("Nous vous invitons")]')(self.doc)) + raise AddRecipientBankError(message=CleanText('//div[@id="aidesecuforte"]/p[contains("Nous vous invitons")]')(self.doc)) def build_doc(self, content): content = '
' + content.decode('latin-1') + '
' diff --git a/modules/gmf/browser.py b/modules/gmf/browser.py index 859e46317803510ca78e8f7d3401fca7e06fa131..221bc15b29fb298e29362a9eb11993d0a8966367 100644 --- a/modules/gmf/browser.py +++ b/modules/gmf/browser.py @@ -24,7 +24,7 @@ from .pages import ( LoginPage, AccountsPage, TransactionsInvestmentsPage, AllTransactionsPage, - DocumentsSignaturePage, + DocumentsSignaturePage, RedirectToUserAgreementPage, UserAgreementPage, ) @@ -32,6 +32,8 @@ class GmfBrowser(LoginBrowser): BASEURL = 'https://espace-assure.gmf.fr' login = URL(r'/public/pages/securite/IC2.faces', LoginPage) + redirect_to_user_agreement = URL('^$', RedirectToUserAgreementPage) + user_agreement = URL(r'restreint/pages/securite/IC9.faces', UserAgreementPage) accounts = URL(r'/pointentree/client/homepage', AccountsPage) transactions_investments = URL(r'/pointentree/contratvie/detailsContrats', TransactionsInvestmentsPage) all_transactions = URL(r'/pages/contratvie/detailscontrats/.*\.faces', AllTransactionsPage) diff --git a/modules/gmf/compat/weboob_capabilities_bank.py b/modules/gmf/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/gmf/compat/weboob_capabilities_bank.py +++ b/modules/gmf/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/gmf/pages.py b/modules/gmf/pages.py index 22a641ba10c7a73bedc25910466f34df9c888943..1d251b9b19f80fd98fe97e44ead48d35a59c588d 100644 --- a/modules/gmf/pages.py +++ b/modules/gmf/pages.py @@ -63,7 +63,8 @@ class item(ItemElement): obj_label = CleanText('.//div[@class="type-contrat"]//h2') obj_type = Account.TYPE_LIFE_INSURANCE obj_balance = CleanDecimal(CleanText('.//div[@class="col-right"]', children=False), replace_dots=True, default=NotAvailable) - obj_currency = Currency(CleanText(u'.//div[@class="col-right"]', children=False, replace=[("Au", "")])) + obj_currency = Currency(CleanText('.//div[@class="col-right"]', children=False, + replace=[("Au", "")])) def get_detail_page_parameters(self, account): """ @@ -74,7 +75,8 @@ def get_detail_page_parameters(self, account): # parameter 1 el = self.doc.xpath('//div[@class="infos-contrat"][descendant::strong[contains(text(), $contract_id)]]/parent::div//div[@class="zone-detail"]//span/a', contract_id=account.id) assert len(el) == 1 - parameter = Regexp(Attr('.', 'onclick'), r".*,\{'(.*)':'(.*)'\},.*\);return false", '\\1 \\2')(el[0]).split(' ') + parameter = Regexp(Attr('.', 'onclick'), r".*,\{'(.*)':'(.*)'\},.*\);return false", + '\\1 \\2')(el[0]).split(' ') data.append((parameter[0], parameter[1])) form = self.get_form(id='j_idt254') @@ -163,8 +165,10 @@ class AllTransactionsPage(LoggedPage, XMLPage, HTMLPage, TransactionsParser): def build_doc(self, content): # HTML embedded in XML: parse XML first then extract the html xml = XMLPage.build_doc(self, content) - transactions_html = xml.xpath('//partial-response/changes/update[1]')[0].text.encode(encoding=self.encoding) - investments_html = xml.xpath('//partial-response/changes/update[2]')[0].text.encode(encoding=self.encoding) + transactions_html = (xml.xpath('//partial-response/changes/update[1]')[0].text + .encode(encoding=self.encoding)) + investments_html = (xml.xpath('//partial-response/changes/update[2]')[0].text + .encode(encoding=self.encoding)) html = transactions_html + investments_html return HTMLPage.build_doc(self, html) @@ -173,3 +177,14 @@ class DocumentsSignaturePage(LoggedPage, HTMLPage): def on_load(self): if self.doc.xpath('//span[contains(text(), "VO(S) DOCUMENT(S) A SIGNER")]'): raise ActionNeeded(CleanText('//div[@class="block"]/p[contains(text(), "Vous avez un ou plusieurs document(s) à signer")]')(self.doc)) + + +class RedirectToUserAgreementPage(LoggedPage, HTMLPage): + MAX_REFRESH = 0 + + +class UserAgreementPage(LoggedPage, HTMLPage): + def on_load(self): + message = CleanText('//fieldset//legend|//fieldset//label')(self.doc) + if 'conditions générales' in message: + raise ActionNeeded(message) diff --git a/modules/groupama/compat/weboob_capabilities_bank.py b/modules/groupama/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/groupama/compat/weboob_capabilities_bank.py +++ b/modules/groupama/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/groupamaes/browser.py b/modules/groupamaes/browser.py index 619e40df0d384795507a79d6ff0a6a63344a379e..8c548dd8e45d89472c235ed37a9a85dabbc9d641 100644 --- a/modules/groupamaes/browser.py +++ b/modules/groupamaes/browser.py @@ -67,4 +67,4 @@ def iter_investment(self, account): @need_login def iter_pocket(self, account): - return self.groupamaes_pocket.go(page='&_pid=SituationParPlan&_fid=GoPositionsDetaillee').iter_pocket(account.label) + return self.groupamaes_pocket.go(page='&_pid=SituationParPlan&_fid=GoPositionsDetaillee').iter_pocket(account.label) diff --git a/modules/groupamaes/compat/weboob_capabilities_bank.py b/modules/groupamaes/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/groupamaes/compat/weboob_capabilities_bank.py +++ b/modules/groupamaes/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/hsbc/browser.py b/modules/hsbc/browser.py index a5284791c76289f6f8b0bb81da6820ba362db657..4f3255192e3be74b93562e64a0c5932cdd59745a 100644 --- a/modules/hsbc/browser.py +++ b/modules/hsbc/browser.py @@ -35,7 +35,7 @@ from .pages.account_pages import ( AccountsPage, CBOperationPage, CPTOperationPage, LoginPage, AppGonePage, RibPage, - UnavailablePage, OtherPage, FrameContainer + UnavailablePage, OtherPage, FrameContainer, ProfilePage, ) from .pages.life_insurances import ( LifeInsurancesPage, LifeInsurancePortal, LifeInsuranceMain, LifeInsuranceUseless, @@ -79,6 +79,7 @@ class HSBC(LoginBrowser): rib = URL(r'/cgi-bin/emcgi', RibPage) accounts = URL(r'/cgi-bin/emcgi', AccountsPage) life_insurance_useless = URL(r'/cgi-bin/emcgi', LifeInsuranceUseless) + profile = URL(r'/cgi-bin/emcgi', ProfilePage) unavailable = URL(r'/cgi-bin/emcgi', UnavailablePage) frame_page = URL(r'/cgi-bin/emcgi', r'https://clients.hsbc.fr/cgi-bin/emcgi', FrameContainer) @@ -166,20 +167,6 @@ def do_login(self): self.location(home_url) - @need_login - def get_accounts_list(self): - if not self.accounts_list: - self.update_accounts_list() - for a in self.accounts_list.values(): - # Get parent of card account - if a.type == Account.TYPE_CARD: - card_page = self.open(a.url).page - parent_id = card_page.get_parent_id() - a.parent = find_object(self.accounts_list.values(), id=parent_id) - if a.parent and not a.currency: - a.currency = a.parent.currency - yield a - def go_post(self, url, data=None): # most of HSBC accounts links are actually handled by js code # which convert a GET query string to POST data. @@ -190,6 +177,33 @@ def go_post(self, url, data=None): url = url[:url.find('?')] self.location(url, data=q) + @need_login + def get_accounts_list(self): + if not self.accounts_list: + self.update_accounts_list() + + # go on cards page if there are cards accounts + for a in self.accounts_list.values(): + if a.type == Account.TYPE_CARD: + self.location(a.url) + break + + # get all couples (card, parent) on cards page + all_card_and_parent = [] + if self.cbPage.is_here(): + all_card_and_parent = self.page.get_all_parent_id() + self.go_post(self.js_url, data={'debr': 'COMPTES_PAN'}) + + # update cards parent and currency + for a in self.accounts_list.values(): + if a.type == Account.TYPE_CARD: + for card in all_card_and_parent: + if a.id in card[0].replace(' ', ''): + a.parent = find_object(self.accounts_list.values(), id=card[1]) + if a.parent and not a.currency: + a.currency = a.parent.currency + yield a + @need_login def update_accounts_list(self, iban=True): if self.accounts.is_here(): @@ -198,7 +212,7 @@ def update_accounts_list(self, iban=True): data = {'debr': 'COMPTES_PAN'} self.go_post(self.js_url, data=data) - for a in self.page.iter_accounts(): + for a in self.page.iter_spaces_account(): try: self.accounts_list[a.id].url = a.url except KeyError: @@ -257,10 +271,10 @@ def get_history(self, account, coming=False, retry_li=True): if account.url is None: return [] - if account.url.startswith('javascript') or '&Crd=' in account.url: + if account.url.startswith('javascript') or '&Crd=' in account.url or account.type == Account.TYPE_LOAN: raise NotImplementedError() - if account.type == Account.TYPE_LIFE_INSURANCE: + if account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_CAPITALISATION): if coming is True: return [] @@ -309,6 +323,12 @@ def get_history(self, account, coming=False, retry_li=True): if self.page is None: return [] + # for 'fusion' space + if hasattr(account, '_is_form') and account._is_form: + # go on accounts page to get account form + self.go_post(self.js_url, data={'debr': 'COMPTES_PAN'}) + self.page.go_history_page(account) + if self.cbPage.is_here(): guesser = LinearDateGuesser(date_max_bump=timedelta(45)) history = list(self.page.get_history(date_guesser=guesser)) @@ -336,7 +356,7 @@ def _get_history(self): yield tr def get_investments(self, account, retry_li=True): - if account.type == Account.TYPE_LIFE_INSURANCE: + if account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_CAPITALISATION): return self.get_life_investments(account, retry_li=retry_li) elif account.type == Account.TYPE_PEA: return self.get_pea_investments(account) @@ -449,3 +469,9 @@ def _go_to_wealth_accounts(self): self.PEA_LISTING['liquidities'] = list(helper.retrieve_liquidity()) self.PEA_LISTING['investments'] = list(helper.retrieve_invests()) self.connection.go() + + @need_login + def get_profile(self): + data = {'debr': 'PARAM'} + self.go_post(self.js_url, data=data) + return self.page.get_profile() diff --git a/modules/hsbc/compat/weboob_capabilities_bank.py b/modules/hsbc/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/hsbc/compat/weboob_capabilities_bank.py +++ b/modules/hsbc/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/hsbc/module.py b/modules/hsbc/module.py index 0a6984cdc2b3401c1b47ce5a068ef52e2e7c309b..1124ddcf0c52028c03f91d4e91f16360d7e98165 100644 --- a/modules/hsbc/module.py +++ b/modules/hsbc/module.py @@ -22,14 +22,14 @@ from weboob.capabilities.base import find_object from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword, Value - +from weboob.capabilities.profile import CapProfile from .browser import HSBC __all__ = ['HSBCModule'] -class HSBCModule(Module, CapBankWealth): +class HSBCModule(Module, CapBankWealth, CapProfile): NAME = 'hsbc' MAINTAINER = u'Romain Bignon' EMAIL = 'romain@weboob.org' @@ -64,3 +64,6 @@ def iter_investment(self, account): def iter_coming(self, account): for tr in self.browser.get_history(account, coming=True): yield tr + + def get_profile(self): + return self.browser.get_profile() diff --git a/modules/hsbc/pages/account_pages.py b/modules/hsbc/pages/account_pages.py index 03e1b1c17e573f97a3f0c924e4c7066abfb4108d..f129080835bafea840c50e2ffe834c8e58482f1b 100644 --- a/modules/hsbc/pages/account_pages.py +++ b/modules/hsbc/pages/account_pages.py @@ -29,18 +29,18 @@ from weboob.browser.elements import ListElement, ItemElement, method from weboob.browser.pages import HTMLPage, pagination from weboob.browser.filters.standard import ( - Filter, Env, CleanText, CleanDecimal, Field, DateGuesser, Regexp + Filter, Env, CleanText, CleanDecimal, Field, DateGuesser, Regexp, Currency, ) from weboob.browser.filters.html import AbsoluteLink, TableCell from weboob.browser.filters.javascript import JSVar - +from weboob.capabilities.profile import Person from .landing_pages import GenericLandingPage class Transaction(FrenchTransaction): PATTERNS = [(re.compile(r'^VIR(EMENT)? (?P.*)'), FrenchTransaction.TYPE_TRANSFER), (re.compile(r'^PRLV (?P.*)'), FrenchTransaction.TYPE_ORDER), - (re.compile(r'^CB (?P.*?)\s+(?P
\d+)/(?P[01]\d)\s*(?P.*)'), + (re.compile(r'^CB (?P.*?)\s+(?P
\d+)/(?P[01]\d)\s+(?P.*)'), FrenchTransaction.TYPE_CARD), (re.compile(r'^DAB (?P
\d{2})/(?P\d{2}) ((?P\d{2})H(?P\d{2}) )?(?P.*?)( CB N°.*)?$'), FrenchTransaction.TYPE_WITHDRAWAL), @@ -80,9 +80,73 @@ def on_load(self): raise BrowserUnavailable() +class AccountsType(Filter): + PATTERNS = [ + ('c.aff', Account.TYPE_CHECKING), + ('pea', Account.TYPE_PEA), + ('invest', Account.TYPE_MARKET), + ('ptf', Account.TYPE_MARKET), + ('ldd', Account.TYPE_SAVINGS), + ('cel', Account.TYPE_SAVINGS), + ('pel', Account.TYPE_SAVINGS), + ('livret', Account.TYPE_SAVINGS), + ('livjeu', Account.TYPE_SAVINGS), + ('compte', Account.TYPE_CHECKING), + ('cpte', Account.TYPE_CHECKING), + ('scpi', Account.TYPE_MARKET), + ('account', Account.TYPE_CHECKING), + ('pret', Account.TYPE_LOAN), + ('vie', Account.TYPE_LIFE_INSURANCE), + ('strategie patr.', Account.TYPE_LIFE_INSURANCE), + ('essentiel', Account.TYPE_LIFE_INSURANCE), + ('elysee', Account.TYPE_LIFE_INSURANCE), + ('abondance', Account.TYPE_LIFE_INSURANCE), + ('ely. retraite', Account.TYPE_LIFE_INSURANCE), + ('lae option assurance', Account.TYPE_LIFE_INSURANCE), + ('carte ', Account.TYPE_CARD), + ('business ', Account.TYPE_CARD), + ('plan assur. innovat.', Account.TYPE_LIFE_INSURANCE), + ('hsbc evol pat transf', Account.TYPE_LIFE_INSURANCE), + ('hsbc evol pat capi', Account.TYPE_CAPITALISATION), + ('bourse libre', Account.TYPE_MARKET), + ('plurival', Account.TYPE_LIFE_INSURANCE), + ] + + def filter(self, label): + label = label.lower() + for pattern, type in self.PATTERNS: + if pattern in label: + return type + return Account.TYPE_UNKNOWN + + +class Label(Filter): + def filter(self, text): + return text.lstrip(' 0123456789').title() + + class AccountsPage(GenericLandingPage): is_here = '//h1[text()="Synthèse"]' + def iter_spaces_account(self): + if self.doc.xpath('//p[text()="HSBC Fusion"]'): + space = 'fusion' + else: + space = 'default' + + accounts = { + 'fusion': self.iter_fusion_accounts, + 'default': self.iter_accounts, + } + return accounts[space]() + + def go_history_page(self, account): + for acc in self.doc.xpath('//div[@onclick]'): + # label contains account number, it's enough to check if it's the right account + if account.label == Label(CleanText('.//p[@class="title"]'))(acc): + form_id = CleanText('.//form/@id')(acc) + return self.get_form(id=form_id).submit() + @method class iter_accounts(ListElement): item_xpath = '//tr' @@ -94,54 +158,13 @@ class item(ItemElement): def condition(self): return len(self.el.xpath('./td')) > 2 - class Label(Filter): - def filter(self, text): - return text.lstrip(' 0123456789').title() - - class Type(Filter): - PATTERNS = [ - ('c.aff', Account.TYPE_CHECKING), - ('pea', Account.TYPE_PEA), - ('invest', Account.TYPE_MARKET), - ('ptf', Account.TYPE_MARKET), - ('ldd', Account.TYPE_SAVINGS), - ('cel', Account.TYPE_SAVINGS), - ('pel', Account.TYPE_SAVINGS), - ('livret', Account.TYPE_SAVINGS), - ('livjeu', Account.TYPE_SAVINGS), - ('compte', Account.TYPE_CHECKING), - ('cpte', Account.TYPE_CHECKING), - ('scpi', Account.TYPE_MARKET), - ('account', Account.TYPE_CHECKING), - ('pret', Account.TYPE_LOAN), - ('vie', Account.TYPE_LIFE_INSURANCE), - ('strategie patr.', Account.TYPE_LIFE_INSURANCE), - ('essentiel', Account.TYPE_LIFE_INSURANCE), - ('elysee', Account.TYPE_LIFE_INSURANCE), - ('abondance', Account.TYPE_LIFE_INSURANCE), - ('ely. retraite', Account.TYPE_LIFE_INSURANCE), - ('lae option assurance', Account.TYPE_LIFE_INSURANCE), - ('carte ', Account.TYPE_CARD), - ('plan assur. innovat.', Account.TYPE_LIFE_INSURANCE), - ('hsbc evol pat transf', Account.TYPE_LIFE_INSURANCE), - ('bourse libre', Account.TYPE_MARKET), - ('plurival', Account.TYPE_LIFE_INSURANCE), - ] - - def filter(self, label): - label = label.lower() - for pattern, type in self.PATTERNS: - if pattern in label: - return type - return Account.TYPE_UNKNOWN - obj_label = Label(CleanText('./td[1]/a')) obj_coming = Env('coming') obj_currency = FrenchTransaction.Currency('./td[2]') obj_url = CleanText(AbsoluteLink('./td[1]/a'), replace=[('\n', '')]) - obj_type = Type(Field('label')) + obj_type = AccountsType(Field('label')) obj_coming = NotAvailable @property @@ -161,6 +184,43 @@ def obj_id(self): return CleanText(replace=[('.', ''), (' ', '')]).filter(self.el.xpath('./td[2]')) + ".INVEST" return CleanText(replace=[('.', ''), (' ', '')]).filter(self.el.xpath('./td[2]')) + @method + class iter_fusion_accounts(ListElement): + def find_elements(self): + all_xpaths = ( + '//div[@id="All" and @class="tabcontent"]/div', + '//div[@class="formGroup"]/div' + ) + for xpath in all_xpaths: + ret = self.xpath(xpath) + if ret: + return ret + else: + assert False, 'Accounts are not well handled' + + class iter_accounts_tables(ListElement): + item_xpath = './div[@onclick]' + + class item(ItemElement): + klass = Account + + obj_label = Label(CleanText('.//p[@class="title"]')) + obj_balance = CleanDecimal(CleanText('.//p[@class="balance"]'), replace_dots=True) + obj_currency = Currency(CleanText('.//p[@class="balance"]')) + obj_type = AccountsType(Field('label')) + obj_url = CleanText('.//form/@action') + obj__is_form = bool(CleanText('.//form/@id')) + + @property + def obj_id(self): + account_id = CleanText('.//p[@class="title"]/span', replace=[('.', ''), (' ', '')])(self) + # Investment account and main account can have the same id + # so we had account type in case of Investment to prevent conflict + # and also the same problem with scpi accounts. + if Field('type')(self) == Account.TYPE_MARKET: + return account_id + ".INVEST" + return account_id + class RibPage(GenericLandingPage): def is_here(self): @@ -219,8 +279,19 @@ def obj_date(self): def get_parent_id(self): # The parent id is in the details of the card - m = re.search(r'Solde du compte (.*)', CleanText('//div[@class="RecentTransactions"]/h2')(self.doc)) - return m.group(1) + return Regexp(CleanText('//h2[contains(text(), "Solde du compte")]'), r'Solde du compte (.*)')(self.doc) + + def get_all_parent_id(self): + all_parent_id = [] + all_card = [CleanText('.')(card) for card in self.doc.xpath('//select[@name="choix_carte"]/option')] + + for index, card in enumerate(all_card): + form = self.get_form(name='FORM_LIB_CARTE') + form['index_carte'] = index + form['choix_carte'] = card + all_parent_id.append((card, form.submit().page.get_parent_id())) + + return all_parent_id class CPTOperationPage(GenericLandingPage): @@ -328,3 +399,16 @@ def on_load(self): for msg, exc in self.ERROR_CLASSES: for tag in self.doc.xpath('//p[@class="debit"]//strong[text()[contains(.,$msg)]]', msg=msg): raise exc(CleanText('.')(tag)) + + +class ProfilePage(OtherPage): + # Warning: this page contains a div_err and displays "Service indisponible" even if it is not... + # but we can still see the data we need + is_here = '//h1[contains(text(), "mes données")]' + + @method + class get_profile(ItemElement): + klass = Person + + obj_name = CleanText('//div[@id="div_adr_P1"]//p/label[contains(text(), "Nom")]/parent::p/strong') + obj_address = CleanText('//div[@id="div_adr_P1"]//p/label[contains(text(), "Adresse")]/parent::p/strong') diff --git a/modules/hsbc/pages/investments.py b/modules/hsbc/pages/investments.py index 7cd1e7e5edeb3a1312d920ba0323148b60e74d11..67a0e81a7a42e0fe0b338552c4dd03ce390962be 100644 --- a/modules/hsbc/pages/investments.py +++ b/modules/hsbc/pages/investments.py @@ -280,6 +280,8 @@ def retrieve_invests(self): def retrieve_liquidity(self): self.retrieve_products(kind='liquidity_list') + if self.browser.retrieve_useless_page.is_here(): + return [] assert isinstance(self.browser.page, RetrieveLiquidityPage) return self.browser.page.iter_liquidity() diff --git a/modules/ideel/browser.py b/modules/ideel/browser.py index 307ee4c7b720a092d468a5380d10e874b5c90b08..4c19aa03f73b375fddac72c00dbfe533f84ad318 100644 --- a/modules/ideel/browser.py +++ b/modules/ideel/browser.py @@ -18,19 +18,18 @@ # along with weboob. If not, see . -from weboob.tools.capabilities.bank.transactions import \ - AmericanTransaction as AmTr -from weboob.browser import LoginBrowser, URL, need_login -from weboob.browser.pages import HTMLPage -from weboob.capabilities.base import Currency -from weboob.capabilities.shop import Order, Item, Payment, OrderNotFound -from weboob.exceptions import BrowserIncorrectPassword - import re -from decimal import Decimal from datetime import datetime -from itertools import takewhile, count +from decimal import Decimal +from itertools import count, takewhile +from weboob.browser import URL, LoginBrowser, need_login +from weboob.browser.pages import HTMLPage +from weboob.capabilities.base import Currency +from weboob.capabilities.shop import Item, Order, OrderNotFound, Payment +from weboob.exceptions import BrowserIncorrectPassword +from weboob.tools.capabilities.bank.transactions import AmericanTransaction as AmTr +from weboob.tools.compat import unicode __all__ = ['Ideel'] @@ -55,7 +54,7 @@ def exists(self): def iter_orders(self): return (tr.xpath('td[1]/a/text()')[0][1:] - for tr in self.doc.xpath('//table[@id="order_history"]/tbody/tr')) + for tr in self.doc.xpath('//table[@id="order_history"]/tbody/tr')) class OrderPage(IdeelPage): @@ -78,7 +77,7 @@ def items(self): url = tr.xpath('*//div[@class="item_img"]//@src')[0] onclk = tr.xpath('*//div[@class="item_img"]//@onclick') if onclk: - url=re.match(r'window.open\(\'([^\']*)\'.*', onclk[0]).group(1) + url = re.match(r'window.open\(\'([^\']*)\'.*', onclk[0]).group(1) if url.startswith('/'): url = self.browser.BASEURL + url price = tr.xpath('td[@class="items_price"]/span/text()')[0] @@ -123,7 +122,7 @@ def discount(self): TAGS = ['coupon_discount_amount', 'promo_discount_amount', 'total_rewards', 'applied_credit'] return -sum(AmTr.decimal_amount(x[1:][:-1]) for tag in TAGS - for x in self.doc.xpath('//span[@id="%s"]/text()' % tag)) + for x in self.doc.xpath('//span[@id="%s"]/text()' % tag)) def total(self): return AmTr.decimal_amount(self.doc.xpath( diff --git a/modules/ideel/module.py b/modules/ideel/module.py index 9bbaa9ae12577a07b2a27434316843177a39c2f2..3d28d51aa18b92f435f14cdc20446650b1bd69d4 100644 --- a/modules/ideel/module.py +++ b/modules/ideel/module.py @@ -19,13 +19,14 @@ from weboob.capabilities.shop import CapShop -from weboob.tools.backend import Module, BackendConfig +from weboob.tools.backend import BackendConfig, Module from weboob.tools.value import ValueBackendPassword from .browser import Ideel __all__ = ['IdeelModule'] + class IdeelModule(Module, CapShop): NAME = 'ideel' MAINTAINER = u'Oleg Plakhotniuk' diff --git a/modules/infomaniak/module.py b/modules/infomaniak/module.py index 8e4de385d9220c5c5b05cd5ee765fcb231cf368f..c11c3e11579123965a4dfaf6b019e473b378c371 100644 --- a/modules/infomaniak/module.py +++ b/modules/infomaniak/module.py @@ -38,7 +38,7 @@ class InfomaniakModule(Module, CapDocument): EMAIL = 'dev@indigo.re' LICENSE = 'AGPLv3+' VERSION = '1.3' - CONFIG = BackendConfig(ValueBackendPassword('login', label='Email', masked=False), + CONFIG = BackendConfig(ValueBackendPassword('login', label='Email de connexion', masked=False), ValueBackendPassword('password', label='Mot de passe')) BROWSER = InfomaniakBrowser diff --git a/modules/ing/browser.py b/modules/ing/browser.py index 70dde32b1b62d7f5b0c083f586b18b1a9f984f7e..08085654446a54601f11edf9ac60068711f9ce74 100644 --- a/modules/ing/browser.py +++ b/modules/ing/browser.py @@ -20,11 +20,12 @@ import hashlib import time +import json from requests.exceptions import SSLError from weboob.browser import LoginBrowser, URL, need_login -from weboob.exceptions import BrowserIncorrectPassword +from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable from weboob.browser.exceptions import ServerError from weboob.capabilities.bank import Account, AccountNotFound from weboob.capabilities.base import find_object, NotAvailable @@ -218,6 +219,12 @@ def get_accounts_on_space(self, space, get_iban=True): accounts_list.append(loan) yield loan + def get_coming_balance(self, account): + if account.type == Account.TYPE_CHECKING: + self.go_account_page(account) + return self.page.get_coming_balance() + return NotAvailable + @need_login @start_with_main_site def get_accounts_list(self, space=None, get_iban=True): @@ -228,17 +235,20 @@ def get_accounts_list(self, space=None, get_iban=True): if space: for acc in self.get_accounts_on_space(space, get_iban=get_iban): + acc.coming = self.get_coming_balance(acc) yield acc elif self.multispace: for space in self.multispace: for acc in self.get_accounts_on_space(space, get_iban=get_iban): + acc.coming = self.get_coming_balance(acc) yield acc else: for acc in self.page.get_list(): acc._space = None if get_iban: self.get_iban(acc) + acc.coming = self.get_coming_balance(acc) yield acc for loan in self.iter_detailed_loans(): @@ -261,8 +271,16 @@ def iter_detailed_loans(self): self.accountspage.go(data=data) self.loantokenpage.go(data=data) - self.loandetailpage.go().getdetails(loan) - + try: + self.loandetailpage.go() + + except ServerError as exception: + json_error = json.loads(exception.response.text) + if json_error['error']['code'] == "INTERNAL_ERROR": + raise BrowserUnavailable(json_error['error']['message']) + raise + else: + self.page.getdetails(loan) yield loan self.return_from_loan_site() diff --git a/modules/ing/compat/weboob_capabilities_bank.py b/modules/ing/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/ing/compat/weboob_capabilities_bank.py +++ b/modules/ing/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/ing/pages/accounts_list.py b/modules/ing/pages/accounts_list.py index 9967545b16d902617aa417cdb635617d174e81e8..665160fc0cccee027d13d70901d0c83114ef7870 100644 --- a/modules/ing/pages/accounts_list.py +++ b/modules/ing/pages/accounts_list.py @@ -39,7 +39,9 @@ class Transaction(FrenchTransaction): - PATTERNS = [(re.compile(u'^retrait (?P.*)'), FrenchTransaction.TYPE_WITHDRAWAL), + PATTERNS = [(re.compile(u'^retrait dab (?P
\d{2})/(?P\d{2})/(?P\d{4}) (?P.*)'), FrenchTransaction.TYPE_WITHDRAWAL), + # Withdrawal in foreign currencies will look like "retrait 123 currency" + (re.compile(u'^retrait (?P.*)'), FrenchTransaction.TYPE_WITHDRAWAL), (re.compile(u'^carte (?P
\d{2})/(?P\d{2})/(?P\d{4}) (?P.*)'), FrenchTransaction.TYPE_CARD), (re.compile(u'^virement (sepa )?(emis vers|recu|emis)? (?P.*)'), FrenchTransaction.TYPE_TRANSFER), (re.compile(u'^remise cheque(?P.*)'), FrenchTransaction.TYPE_DEPOSIT), @@ -171,7 +173,6 @@ class item(ItemElement): obj_label = CleanText('./span[@class="title"]') obj_id = AddPref(Field('_id'), Field('label')) obj_type = AddType(Field('label')) - obj_coming = NotAvailable obj__jid = Attr('//input[@name="javax.faces.ViewState"]', 'value') def obj_balance(self): @@ -189,15 +190,14 @@ class item(ItemElement): klass = Loan obj_currency = u'EUR' - obj_label = CleanText('./span[@class="title"]') + obj_label = CleanText('.//span[@class="title"]') obj_id = AddPref(Field('_id'), Field('label')) obj_type = AddType(Field('label')) - obj_coming = NotAvailable obj__jid = Attr('//input[@name="javax.faces.ViewState"]', 'value') - obj__id = CleanText('./span[@class="account-number"]') + obj__id = CleanText('.//span[@class="account-number"]') def obj_balance(self): - balance = CleanDecimal('./span[@class="solde"]/label', replace_dots=True)(self) + balance = CleanDecimal('.//div/span[@class="solde"]/label', replace_dots=True)(self) return -abs(balance) class generic_transactions(ListElement): @@ -323,6 +323,12 @@ def load_space_page(self): self.fillup_form(form, r"\),\{(.*)\},'", on_click) form.submit() + def get_coming_balance(self): + return CleanDecimal('//div[@class="previsionnel"]/div[@class="solde_value"]//label', + replace_dots=True, + default=NotAvailable)(self.doc) + + class IbanPage(LoggedPage, HTMLPage): def get_iban(self): iban = CleanText('//tr[td[1]//text()="IBAN"]/td[2]')(self.doc).strip().replace(' ', '') diff --git a/modules/ing/pages/compat/__init__.py b/modules/ing/pages/compat/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/modules/ing/pages/compat/weboob_tools_capabilities_bank_investments.py b/modules/ing/pages/compat/weboob_tools_capabilities_bank_investments.py new file mode 100644 index 0000000000000000000000000000000000000000..2b68ff9e001c726a90445e05e0a8ef2e2f165059 --- /dev/null +++ b/modules/ing/pages/compat/weboob_tools_capabilities_bank_investments.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Jonathan Schmidt +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.tools.compat import basestring +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.bank import Investment + +def is_isin_valid(isin): + """ + Méthode générale + Table de conversion des lettres en chiffres + A=10 B=11 C=12 D=13 E=14 F=15 G=16 H=17 I=18 + J=19 K=20 L=21 M=22 N=23 O=24 P=25 Q=26 R=27 + S=28 T=29 U=30 V=31 W=32 X=33 Y=34 Z=35 + + 1 - Mettre de côté la clé, qui servira de référence à la fin de la vérification. + 2 - Convertir toutes les lettres en nombres via la table de conversion ci-contre. Si le nombre obtenu est supérieur ou égal à 10, prendre les deux chiffres du nombre séparément (exemple : 27 devient 2 et 7). + 3 - Pour chaque chiffre, multiplier sa valeur par deux si sa position est impaire en partant de la droite. Si le nombre obtenu est supérieur ou égal à 10, garder les deux chiffres du nombre séparément (exemple : 14 devient 1 et 4). + 4 - Faire la somme de tous les chiffres. + 5 - Soustraire cette somme de la dizaine supérieure ou égale la plus proche (exemples : si la somme vaut 22, la dizaine « supérieure ou égale » est 30, et la clé vaut donc 8 ; si la somme vaut 30, la dizaine « supérieure ou égale » est 30, et la clé vaut 0 ; si la somme vaut 31, la dizaine « supérieure ou égale » est 40, et la clé vaut 9). + 6 - Comparer la valeur obtenue à la clé mise initialement de côté. + + Étapes 1 et 2 : + F R 0 0 0 3 5 0 0 0 0 (+ 8 : clé) + 15 27 0 0 0 3 5 0 0 0 0 + + Étape 3 : le traitement se fait sur des chiffres + 1 5 2 7 0 0 0 3 5 0 0 0 0 + I P I P I P I P I P I P I : position en partant de la droite (P = Pair, I = Impair) + 2 1 2 1 2 1 2 1 2 1 2 1 2 : coefficient multiplicateur + + 2 5 4 7 0 0 0 3 10 0 0 0 0 : résultat + + Étape 4 : + 2 + 5 + 4 + 7 + 0 + 0 + 0 + 3 + (1 + 0)+ 0 + 0 + 0 + 0 = 22 + + Étapes 5 et 6 : 30 - 22 = 8 (valeur de la clé) + """ + + if not isinstance(isin, basestring): + return False + if not re.match(r'^[A-Z]{2}[A-Z0-9]{9}\d$', isin): + return False + + isin_in_digits = ''.join(str(ord(x) - ord('A') + 10) if not x.isdigit() else x for x in isin[:-1]) + key = isin[-1:] + result = '' + for k, val in enumerate(isin_in_digits[::-1], start=1): + if k % 2 == 0: + result = ''.join((result, val)) + else: + result = ''.join((result, str(int(val)*2))) + return str(sum(int(x) for x in result) + int(key))[-1] == '0' + + +def create_french_liquidity(valuation): + """ + Automatically fills a liquidity investment with label, code and code_type. + """ + liquidity = Investment() + liquidity.label = "Liquidités" + liquidity.code = "XX-liquidity" + liquidity.code_type = NotAvailable + liquidity.valuation = valuation + return liquidity + diff --git a/modules/ing/pages/titre.py b/modules/ing/pages/titre.py index 8ed4189bf100b57cf59b9c1253a4f1884e56595c..cc576886cd946858ed1e5bbf3bbc11888c93daff 100644 --- a/modules/ing/pages/titre.py +++ b/modules/ing/pages/titre.py @@ -29,6 +29,7 @@ from weboob.browser.filters.standard import CleanDecimal, CleanText, Date, Regexp, Env from weboob.browser.filters.html import Link, Attr, TableCell from weboob.tools.capabilities.bank.transactions import FrenchTransaction +from .compat.weboob_tools_capabilities_bank_investments import create_french_liquidity from weboob.tools.compat import unicode class NetissimaPage(HTMLPage): @@ -113,11 +114,8 @@ def iter_investments(self, account): # There is no investment on life insurance in the process to be created. if len(message.split('&')) >= 4: # We also have to get the liquidity as an investment. - invest = Investment() - invest.label = "Liquidités" - invest.code = "XX-liquidity" - invest.valuation = CleanDecimal(None, True).filter(message.split('&')[3].replace('euro;{','').strip()) - invests.append(invest) + valuation = CleanDecimal(None, True).filter(message.split('&')[3].replace('euro;{','').strip()) + invests.append(create_french_liquidity(valuation)) for invest in invests: yield invest diff --git a/modules/ing/pages/transfer.py b/modules/ing/pages/transfer.py index 65126b828a7d411153fc29d6ae8e8ead2fa8a257..034bd085997c3930c70caf5f3e804201b8546a50 100644 --- a/modules/ing/pages/transfer.py +++ b/modules/ing/pages/transfer.py @@ -187,7 +187,7 @@ def recap(self, origin, recipient, transfer): error = CleanText(u'//div[@id="transfer_form:moveMoneyDetailsBody"]//span[@class="error"]', default=None)(self.doc) or \ CleanText(u'//p[contains(text(), "Nous sommes désolés. Le solde de votre compte ne doit pas être inférieur au montant de votre découvert autorisé. Veuillez saisir un montant inférieur.")]', default=None)(self.doc) if error: - raise TransferInvalidAmount(message=error) + raise TransferInvalidAmount(message=error) t = Transfer() t.label = transfer.label @@ -219,7 +219,7 @@ def recap(self, origin, recipient, transfer): t.recipient_id = recipient.id t.exec_date = parse_french_date(CleanText('//p[has-class("exec-date")]', children=False, - replace=[('le', ''), (u'exécuté', ''), ('demain', ''), ('(', ''), (')', ''), - ("aujourd'hui", '')])(self.doc)).date() + replace=[('le', ''), (u'exécuté', ''), ('demain', ''), ('(', ''), (')', ''), + ("aujourd'hui", '')])(self.doc)).date() return t diff --git a/modules/lcl/browser.py b/modules/lcl/browser.py index 22e8938a01d17d61dee6e001978e33d09da45c5f..c9716a69d6acb1c7c35b7a5aa0febae5d5a018ac 100644 --- a/modules/lcl/browser.py +++ b/modules/lcl/browser.py @@ -18,15 +18,17 @@ # along with weboob. If not, see . +from __future__ import unicode_literals + from datetime import datetime, timedelta, date from functools import wraps -from weboob.exceptions import BrowserIncorrectPassword -from weboob.browser import LoginBrowser, URL, need_login, StatesMixin +from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable +from weboob.browser.browsers import LoginBrowser, URL, need_login, StatesMixin from weboob.browser.exceptions import ServerError -from weboob.browser.pages import FormNotFound from weboob.capabilities.base import NotAvailable -from weboob.capabilities.bank import Account, Investment, AddRecipientError, AddRecipientStep, Recipient +from .compat.weboob_capabilities_bank import Account, AddRecipientBankError, AddRecipientStep, Recipient +from .compat.weboob_tools_capabilities_bank_investments import create_french_liquidity from weboob.tools.compat import basestring, urlsplit, parse_qsl, unicode from weboob.tools.value import Value @@ -36,7 +38,7 @@ HomePage, LoansPage, TransferPage, AddRecipientPage, \ RecipientPage, RecipConfirmPage, SmsPage, RecipRecapPage, \ LoansProPage, Form2Page, DocumentsPage, ClientPage, SendTokenPage, \ - CaliePage, ProfilePage, DepositPage + CaliePage, ProfilePage, DepositPage, AVHistoryPage, AVInvestmentsPage __all__ = ['LCLBrowser', 'LCLProBrowser', 'ELCLBrowser'] @@ -84,12 +86,17 @@ class LCLBrowser(LoginBrowser, StatesMixin): form2 = URL(r'/outil/UWVI/Routage/', Form2Page) send_token = URL('/outil/UWVI/AssuranceVie/envoyerJeton', SendTokenPage) - calie = URL('https://www.my-calie.fr/FO.HoldersWebSite/Disclaimer/Disclaimer.aspx.*', CaliePage) + calie = URL('https://www.my-calie.fr/FO.HoldersWebSite/Disclaimer/Disclaimer.aspx.*', + 'https://www.my-calie.fr/FO.HoldersWebSite/Contract/ContractDetails.aspx.*', + 'https://www.my-calie.fr/FO.HoldersWebSite/Contract/ContractOperations.aspx.*', CaliePage) assurancevie = URL('/outil/UWVI/AssuranceVie/accesSynthese', '/outil/UWVI/AssuranceVie/accesDetail.*', AVPage) - avdetail = URL('https://assurance-vie-et-prevoyance.secure.lcl.fr.*', AVDetailPage) + + avdetail = URL('https://assurance-vie-et-prevoyance.secure.lcl.fr/consultation/epargne', AVDetailPage) + av_history = URL('https://assurance-vie-et-prevoyance.secure.lcl.fr/rest/assurance/historique', AVHistoryPage) + av_investments = URL('https://assurance-vie-et-prevoyance.secure.lcl.fr/rest/detailEpargne/contrat', AVInvestmentsPage) loans = URL('/outil/UWCR/SynthesePar/', LoansPage) loans_pro = URL('/outil/UWCR/SynthesePro/', LoansProPage) @@ -112,6 +119,8 @@ class LCLBrowser(LoginBrowser, StatesMixin): __states__ = ('contracts', 'current_contract',) + IDENTIFIANT_ROUTING = 'CLI' + def __init__(self, *args, **kwargs): super(LCLBrowser, self).__init__(*args, **kwargs) self.accounts_list = None @@ -294,31 +303,44 @@ def get_history(self, account): self.location(account._link_id) except ServerError: return + if self.login.is_here(): + # Website crashed and we are disconnected. + raise BrowserUnavailable() for tr in self.page.get_operations(): yield tr for tr in self.get_cb_operations(account, 1): yield tr - elif account.type == Account.TYPE_LIFE_INSURANCE and account._form: + + elif account.type == Account.TYPE_LIFE_INSURANCE: + if not account._form: + self.logger.warning('This account is limited, there is no available history.') + return + self.assurancevie.stay_or_go() - account._form.submit() + # The website often returns an error so we try again: + # "L’accès au service est momentanément indisponible." + try: + account._form.submit() + except BrowserUnavailable: + self.logger.warning("Service unavailable, we submit the form again.") + self.assurancevie.stay_or_go() + account._form.submit() + if self.calie.is_here(): - # come back to syntese + # Get back to Synthèse self.assurancevie.go() return - - # certain users will get a message : "Ne détenant pas de compte dépôt + # Some users will get a message : "Ne détenant pas de compte dépôt # chez LCL, l'accès à ce service vous est indisponible." if self.form2.is_here() and self.page.assurancevie_hist_not_available(): return - try: - self.page.get_details(account, "OHIPU") - except FormNotFound: - assert self.page.is_restricted() - self.logger.warning('restricted access to account %s', account) - else: - for tr in self.page.iter_history(): - yield tr + self.avdetail.go() + self.av_history.go() + for tr in self.page.iter_history(): + yield tr + + self.avdetail.go() self.page.come_back() @go_contract @@ -351,34 +373,46 @@ def get_cb_operations(self, account, month=0): @go_contract @need_login def get_investment(self, account): - if account.type == Account.TYPE_LIFE_INSURANCE and account._form: + if account.type == Account.TYPE_LIFE_INSURANCE: + if not account._form: + self.logger.warning('This account is limited, there is no available investment.') + return + self.assurancevie.stay_or_go() - account._form.submit() + # The website often returns an error so we try again: + # "L’accès au service est momentanément indisponible." + try: + account._form.submit() + except BrowserUnavailable: + self.logger.warning("Service unavailable, we submit the form again.") + self.assurancevie.stay_or_go() + account._form.submit() if self.calie.is_here(): - # come back to syntese + # Get back to Synthèse self.assurancevie.go() return + # Some users will get a message : "Ne détenant pas de compte dépôt + # chez LCL, l'accès à ce service vous est indisponible." + if self.form2.is_here() and self.page.assurancevie_hist_not_available(): + return + + self.avdetail.go() + self.av_investments.go() + + for inv in self.page.iter_investment(): + yield inv + + self.avdetail.go() + self.page.come_back() - if self.page.is_restricted(): - self.logger.warning('restricted access to account %s', account) - else: - for inv in self.page.iter_investment(): - yield inv - if self.avdetail.is_here(): - self.page.come_back() elif hasattr(account, '_market_link') and account._market_link: self.connexion_bourse() for inv in self.location(account._market_link).page.iter_investment(): yield inv self.deconnexion_bourse() elif account.id in self.get_bourse_accounts_ids(): - inv = Investment() - inv.id = account.id - inv.code = 'XX-Liquidity' - inv.label = "Liquidités" - inv.valuation = account.balance - yield inv + yield create_french_liquidity(account.balance) def locate_browser(self, state): if state['url'] == 'https://particuliers.secure.lcl.fr/outil/UWBE/Creation/creationConfirmation': @@ -390,7 +424,7 @@ def locate_browser(self, state): def send_code(self, recipient, **params): res = self.open('/outil/UWBE/Otp/getValidationCodeOtp?codeOtp=%s' % params['code']) if res.text == 'false': - raise AddRecipientError(message='Mauvais code sms.') + raise AddRecipientBankError(message='Mauvais code sms.') self.recip_recap.go().check_values(recipient.iban, recipient.label) return self.get_recipient_object(recipient.iban, recipient.label) @@ -412,7 +446,7 @@ def new_recipient(self, recipient, **params): return self.send_code(recipient, **params) if recipient.iban[:2] not in ('FR', 'MC'): - raise AddRecipientError(message=u"LCL n'accepte que les iban commençant par MC ou FR.") + raise AddRecipientBankError(message=u"LCL n'accepte que les iban commençant par MC ou FR.") for _ in range(2): self.add_recip.go() @@ -420,13 +454,12 @@ def new_recipient(self, recipient, **params): break if self.no_perm.is_here() and self.page.get_error_msg(): - raise AddRecipientError(message=self.page.get_error_msg()) - elif not self.add_recip.is_here(): - raise AddRecipientError('Navigation failed: not on add_recip.') + raise AddRecipientBankError(message=self.page.get_error_msg()) + + assert self.add_recip.is_here(), 'Navigation failed: not on add_recip' self.page.validate(recipient.iban, recipient.label) - if not self.recip_confirm.is_here(): - raise AddRecipientError('Navigation failed: not on recip_confirm.') + assert self.recip_confirm.is_here(), 'Navigation failed: not on recip_confirm' self.page.check_values(recipient.iban, recipient.label) # Send sms to user. diff --git a/modules/lcl/compat/weboob_capabilities_bank.py b/modules/lcl/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/lcl/compat/weboob_capabilities_bank.py +++ b/modules/lcl/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/lcl/compat/weboob_capabilities_bill.py b/modules/lcl/compat/weboob_capabilities_bill.py new file mode 100644 index 0000000000000000000000000000000000000000..09a2fd7504b01e0c9d622483bc2ebf600e1a2429 --- /dev/null +++ b/modules/lcl/compat/weboob_capabilities_bill.py @@ -0,0 +1,259 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012 Romain Bignon, Florent Fourcot +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + + +from weboob.capabilities.base import BaseObject, StringField, DecimalField, BoolField, UserError, Currency, Field +from weboob.capabilities.date import DateField +from weboob.capabilities.collection import CapCollection + + +__all__ = ['SubscriptionNotFound', 'DocumentNotFound', 'DocumentTypes', 'Detail', 'Document', 'Bill', 'Subscription', 'CapDocument'] + + +from weboob.capabilities.bill import SubscriptionNotFound as _SubscriptionNotFound +class SubscriptionNotFound(_SubscriptionNotFound): + """ + Raised when a subscription is not found. + """ + + def __init__(self, msg='Subscription not found'): + super(SubscriptionNotFound, self).__init__(msg) + + +from weboob.capabilities.bill import DocumentNotFound as _DocumentNotFound +class DocumentNotFound(_DocumentNotFound): + """ + Raised when a document is not found. + """ + + def __init__(self, msg='Document not found'): + super(DocumentNotFound, self).__init__(msg) + + +class DocumentTypes(object): + RIB = u'RIB' + STATEMENT = u'statement' + CONTRACT = u'contract' + NOTICE = u'notice' + REPORT = u'report' + BILL = u'bill' + OTHER = u'other' + + +from weboob.capabilities.bill import Detail as _Detail +class Detail(_Detail): + """ + Detail of a subscription + """ + label = StringField('label of the detail line') + infos = StringField('information') + datetime = DateField('date information') + price = DecimalField('Total price, taxes included') + vat = DecimalField('Value added Tax') + currency = StringField('Currency', default=None) + quantity = DecimalField('Number of units consumed') + unit = StringField('Unit of the consumption') + + +from weboob.capabilities.bill import Document as _Document +class Document(_Document): + """ + Document. + """ + date = DateField('The day the document has been sent to the subscriber') + format = StringField('file format of the document') + label = StringField('label of document') + type = StringField('type of document') + transactions = Field('List of transaction ID related to the document', list, default=[]) + + +from weboob.capabilities.bill import Bill as _Bill +class Bill(_Bill): + """ + Bill. + """ + price = DecimalField('Price to pay') + currency = StringField('Currency', default=None) + vat = DecimalField('VAT included in the price') + duedate = DateField('The day the bill must be paid') + startdate = DateField('The first day the bill applies to') + finishdate = DateField('The last day the bill applies to') + income = BoolField('Boolean to set if bill is income or invoice', default=False) + + def __init__(self, *args, **kwargs): + super(Bill, self).__init__(*args, **kwargs) + self.type = u'bill' + + +from weboob.capabilities.bill import Subscription as _Subscription +class Subscription(_Subscription): + """ + Subscription to a service. + """ + label = StringField('label of subscription') + subscriber = StringField('Subscriber name or identifier (for companies)') + validity = DateField('End validity date of the subscription (if any)') + renewdate = DateField('Reset date of consumption, for time based suscription (monthly, yearly, etc)') + + +from weboob.capabilities.bill import CapDocument as _CapDocument +class CapDocument(_CapDocument): + accepted_doc_types = () + """ + Tuple of document types handled by the module (:class:`DocumentTypes`) + """ + + def iter_subscription(self): + """ + Iter subscriptions. + + :rtype: iter[:class:`Subscription`] + """ + raise NotImplementedError() + + def get_subscription(self, _id): + """ + Get a subscription. + + :param _id: ID of subscription + :rtype: :class:`Subscription` + :raises: :class:`SubscriptionNotFound` + """ + raise NotImplementedError() + + def iter_documents_history(self, subscription): + """ + Iter history of a subscription. + + :param subscription: subscription to get history + :type subscription: :class:`Subscription` + :rtype: iter[:class:`Detail`] + """ + raise NotImplementedError() + + def iter_bills_history(self, subscription): + """ + Iter history of a subscription. + + :param subscription: subscription to get history + :type subscription: :class:`Subscription` + :rtype: iter[:class:`Detail`] + """ + return self.iter_documents_history(subscription) + + def get_document(self, id): + """ + Get a document. + + :param id: ID of document + :rtype: :class:`Document` + :raises: :class:`DocumentNotFound` + """ + raise NotImplementedError() + + def download_document(self, id): + """ + Download a document. + + :param id: ID of document + :rtype: bytes + :raises: :class:`DocumentNotFound` + """ + raise NotImplementedError() + + def download_document_pdf(self, id): + """ + Download a document, convert it to PDF if it isn't the document format. + + :param id: ID of document + :rtype: bytes + :raises: :class:`DocumentNotFound` + """ + if not isinstance(id, Document): + id = self.get_document(id) + + if id.format == 'pdf': + return self.download_document(id) + else: + raise NotImplementedError() + + def iter_documents(self, subscription): + """ + Iter documents. + + :param subscription: subscription to get documents + :type subscription: :class:`Subscription` + :rtype: iter[:class:`Document`] + """ + raise NotImplementedError() + + def iter_documents_by_types(self, subscription, accepted_types): + """ + Iter documents with specific types. + + :param subscription: subscription to get documents + :type subscription: :class:`Subscription` + :param accepted_types: list of document types that should be returned + :type accepted_types: [:class:`DocumentTypes`] + :rtype: iter[:class:`Document`] + """ + accepted_types = frozenset(accepted_types) + for document in self.iter_documents(subscription): + if document.type in accepted_types: + yield document + + def iter_bills(self, subscription): + """ + Iter bills. + + :param subscription: subscription to get bills + :type subscription: :class:`Subscription` + :rtype: iter[:class:`Bill`] + """ + documents = self.iter_documents(subscription) + return [doc for doc in documents if doc.type == "bill"] + + def get_details(self, subscription): + """ + Get details of a subscription. + + :param subscription: subscription to get details + :type subscription: :class:`Subscription` + :rtype: iter[:class:`Detail`] + """ + raise NotImplementedError() + + def get_balance(self, subscription): + """ + Get the balance of a subscription. + + :param subscription: subscription to get balance + :type subscription: :class:`Subscription` + :rtype: class:`Detail` + """ + raise NotImplementedError() + + def iter_resources(self, objs, split_path): + """ + Iter resources. Will return :func:`iter_subscriptions`. + """ + if Subscription in objs: + self._restrict_level(split_path) + return self.iter_subscription() + diff --git a/modules/lcl/compat/weboob_tools_capabilities_bank_investments.py b/modules/lcl/compat/weboob_tools_capabilities_bank_investments.py new file mode 100644 index 0000000000000000000000000000000000000000..2b68ff9e001c726a90445e05e0a8ef2e2f165059 --- /dev/null +++ b/modules/lcl/compat/weboob_tools_capabilities_bank_investments.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Jonathan Schmidt +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.tools.compat import basestring +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.bank import Investment + +def is_isin_valid(isin): + """ + Méthode générale + Table de conversion des lettres en chiffres + A=10 B=11 C=12 D=13 E=14 F=15 G=16 H=17 I=18 + J=19 K=20 L=21 M=22 N=23 O=24 P=25 Q=26 R=27 + S=28 T=29 U=30 V=31 W=32 X=33 Y=34 Z=35 + + 1 - Mettre de côté la clé, qui servira de référence à la fin de la vérification. + 2 - Convertir toutes les lettres en nombres via la table de conversion ci-contre. Si le nombre obtenu est supérieur ou égal à 10, prendre les deux chiffres du nombre séparément (exemple : 27 devient 2 et 7). + 3 - Pour chaque chiffre, multiplier sa valeur par deux si sa position est impaire en partant de la droite. Si le nombre obtenu est supérieur ou égal à 10, garder les deux chiffres du nombre séparément (exemple : 14 devient 1 et 4). + 4 - Faire la somme de tous les chiffres. + 5 - Soustraire cette somme de la dizaine supérieure ou égale la plus proche (exemples : si la somme vaut 22, la dizaine « supérieure ou égale » est 30, et la clé vaut donc 8 ; si la somme vaut 30, la dizaine « supérieure ou égale » est 30, et la clé vaut 0 ; si la somme vaut 31, la dizaine « supérieure ou égale » est 40, et la clé vaut 9). + 6 - Comparer la valeur obtenue à la clé mise initialement de côté. + + Étapes 1 et 2 : + F R 0 0 0 3 5 0 0 0 0 (+ 8 : clé) + 15 27 0 0 0 3 5 0 0 0 0 + + Étape 3 : le traitement se fait sur des chiffres + 1 5 2 7 0 0 0 3 5 0 0 0 0 + I P I P I P I P I P I P I : position en partant de la droite (P = Pair, I = Impair) + 2 1 2 1 2 1 2 1 2 1 2 1 2 : coefficient multiplicateur + + 2 5 4 7 0 0 0 3 10 0 0 0 0 : résultat + + Étape 4 : + 2 + 5 + 4 + 7 + 0 + 0 + 0 + 3 + (1 + 0)+ 0 + 0 + 0 + 0 = 22 + + Étapes 5 et 6 : 30 - 22 = 8 (valeur de la clé) + """ + + if not isinstance(isin, basestring): + return False + if not re.match(r'^[A-Z]{2}[A-Z0-9]{9}\d$', isin): + return False + + isin_in_digits = ''.join(str(ord(x) - ord('A') + 10) if not x.isdigit() else x for x in isin[:-1]) + key = isin[-1:] + result = '' + for k, val in enumerate(isin_in_digits[::-1], start=1): + if k % 2 == 0: + result = ''.join((result, val)) + else: + result = ''.join((result, str(int(val)*2))) + return str(sum(int(x) for x in result) + int(key))[-1] == '0' + + +def create_french_liquidity(valuation): + """ + Automatically fills a liquidity investment with label, code and code_type. + """ + liquidity = Investment() + liquidity.label = "Liquidités" + liquidity.code = "XX-liquidity" + liquidity.code_type = NotAvailable + liquidity.valuation = valuation + return liquidity + diff --git a/modules/lcl/module.py b/modules/lcl/module.py index 684b9ee0cd238ae301e036e3e287e9e2d8e88914..12590172bc87b0b00e353d38a14f17c82f27dc81 100644 --- a/modules/lcl/module.py +++ b/modules/lcl/module.py @@ -24,8 +24,8 @@ from .compat.weboob_capabilities_bank import CapBankWealth, CapBankTransferAddRecipient, AccountNotFound, \ RecipientNotFound, TransferError, Account -from weboob.capabilities.bill import CapDocument, Subscription, SubscriptionNotFound, \ - Document, DocumentNotFound +from .compat.weboob_capabilities_bill import CapDocument, Subscription, SubscriptionNotFound, \ + Document, DocumentNotFound, DocumentTypes from weboob.capabilities.contact import CapContact from weboob.capabilities.profile import CapProfile from weboob.tools.backend import Module, BackendConfig @@ -70,6 +70,8 @@ class LCLModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapContact, 'elcl': 'e.LCL'})) BROWSER = LCLBrowser + accepted_doc_types = (DocumentTypes.STATEMENT, DocumentTypes.NOTICE, DocumentTypes.REPORT, DocumentTypes.OTHER) + def create_default_browser(self): # assume all `website` option choices are defined here browsers = {'par': LCLBrowser, diff --git a/modules/lcl/pages.py b/modules/lcl/pages.py index f919aac9d2c0748f4c7d003ccb23ac8e4a031e0c..637a8f2bcf5c9bdce64ad6e68a46242802c47a8d 100644 --- a/modules/lcl/pages.py +++ b/modules/lcl/pages.py @@ -31,20 +31,19 @@ from weboob.capabilities.base import find_object, Currency from weboob.capabilities.bank import ( Account, Investment, Recipient, TransferError, TransferBankError, Transfer, - AddRecipientError, ) -from weboob.capabilities.bill import Document, Subscription +from .compat.weboob_capabilities_bill import Document, Subscription, DocumentTypes from .compat.weboob_capabilities_profile import Person, ProfileMissing from weboob.capabilities.contact import Advisor -from weboob.browser.elements import method, ListElement, TableElement, ItemElement, SkipItem +from weboob.browser.elements import method, ListElement, TableElement, ItemElement, SkipItem, DictElement from weboob.exceptions import ParseError from weboob.browser.exceptions import ServerError -from weboob.browser.pages import LoggedPage, HTMLPage, FormNotFound, pagination +from weboob.browser.pages import LoggedPage, HTMLPage, JsonPage, FormNotFound, pagination from weboob.browser.filters.html import Attr, Link, TableCell, AttributeNotFound from weboob.browser.filters.standard import ( - CleanText, Field, Regexp, Format, Date, CleanDecimal, Map, AsyncLoad, Async, Env, - Eval, Slugify, + CleanText, Field, Regexp, Format, Date, CleanDecimal, Map, AsyncLoad, Async, Env, Slugify, BrowserURL, ) +from weboob.browser.filters.json import Dict from weboob.exceptions import BrowserUnavailable, BrowserIncorrectPassword from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.tools.captcha.virtkeyboard import MappedVirtKeyboard, VirtKeyboardError @@ -252,6 +251,7 @@ def condition(self): '007': Account.TYPE_SAVINGS, '012': Account.TYPE_SAVINGS, '023': Account.TYPE_CHECKING, + '036': Account.TYPE_SAVINGS, '046': Account.TYPE_SAVINGS, '047': Account.TYPE_SAVINGS, '049': Account.TYPE_SAVINGS, @@ -433,7 +433,6 @@ def obj_type(self): match = pattern.match(type) if match: return _type - break return Transaction.TYPE_UNKNOWN def condition(self): @@ -455,6 +454,9 @@ def validate(self, obj): else: obj.label = '%s %s' % (obj.raw, raw) obj.raw = '%s %s' % (obj.raw, raw) + m = re.search(r'\d+,\d+COM (\d+,\d+)', raw) + if m: + obj.commission = -CleanDecimal(replace_dots=True).filter(m.group(1)) elif not obj.raw: # Empty transaction label obj.raw = obj.label = Async('details', CleanText(u'//td[contains(text(), "Nature de l\'opération")]/following-sibling::*[1]'))(self) @@ -718,7 +720,9 @@ def obj_id(self): self.page.browser.open(self.page.url) # redirection to lifeinsurances accounts and comeback on Lcl original website page = self.obj__form().submit().page - account_id = page.get_account_id() + # Getting the account details from the JSON containing the account information: + details_page = self.page.browser.open(BrowserURL('av_investments')(self)).page + account_id = Dict('situationAdministrativeEpargne/idcntcar')(details_page.doc) page.come_back() return account_id @@ -768,22 +772,13 @@ def assurancevie_hist_not_available(self): msg = "Ne détenant pas de compte dépôt chez LCL, l'accès à ce service vous est indisponible" return msg in CleanText('//div[@id="attTxt"]')(self.doc) - def is_restricted(self): - return self.assurancevie_hist_not_available() - def on_load(self): if self.assurancevie_hist_not_available(): return error = CleanText('//div[@id="attTxt"]/text()[1]')(self.doc) if "L’accès au service est momentanément indisponible" in error: raise BrowserUnavailable(error) - - form = self.get_form(name="formulaire") - cName = self.get_from_js('.cName.value = "', '";') - if cName: - form['cName'] = cName - form['cValue'] = self.get_from_js('.cValue.value = "', '";') - form['cMaxAge'] = '-1' + form = self.get_form() return form.submit() @@ -795,22 +790,6 @@ def on_load(self): class AVDetailPage(LoggedPage, LCLBasePage): - def get_account_id(self): - return Regexp(CleanText('//div[@class="libelletitrepage"]/h1'), r"N° (\w+)")(self.doc) - - def sub(self): - form = self.get_form(name="formulaire") - cName = self.get_from_js('.cName.value = "', '";') - if cName: - form['cName'] = cName - form['cValue'] = self.get_from_js('.cValue.value = "', '";') - form['cMaxAge'] = '-1' - return form.submit() - - def submit_simple(self): - form = self.get_form(name="formulaire") - return form.submit() - def come_back(self): session = self.get_from_js('idSessionSag = "', '"') params = {} @@ -822,78 +801,70 @@ def come_back(self): params['stbzn'] = 'bnc' return self.browser.location('https://assurance-vie-et-prevoyance.secure.lcl.fr/filiale/entreeBam', params=params) - def get_details(self, account, act=None): - form = self.get_form(id="frm_fwk") - form.submit() - if act is not None: - self.browser.location("entreeBam?sessionSAG=%s&act=%s" % (form['sessionSAG'], act)) - - def is_restricted(self): - msg = CleanText('//div[has-class("titre_libelle_erreur")]')(self.doc) - return msg in ( - "Pour ce type d’opération, nous vous conseillons de vous rapprocher de votre conseiller afin de bénéficier du conseil personnalisé le mieux adapté à vos objectifs.", - "Vous n’avez pas accès à l’espace assurances de personnes.", - ) +class AVHistoryPage(LoggedPage, JsonPage): @method - class iter_investment(TableElement): - head_xpath = '//table[@class="table"][ends-with(@id,"CD_UCT")]/thead//th' - item_xpath = '//table[@class="table"][ends-with(@id,"CD_UCT")]/tbody/tr' - - col_label = 'Le(s) support(s) financier(s) de votre contrat' - col_unitvalue = 'Valeur de la part' - col_vdate = 'En date du :' - col_quantity = 'Nombre de parts' - col_valuation = 'Total' - col_portfolio_share = u'Répartition' + class iter_history(DictElement): + item_xpath = 'listeOperations' class item(ItemElement): - klass = Investment + klass = Transaction - obj_label = CleanText(TableCell('label')) + obj_label = CleanText(Dict('lcope')) + obj_amount = CleanDecimal(Dict('mtope')) + obj_type = Transaction.TYPE_BANK + obj_investments = NotAvailable - def obj_code(self): - td = TableCell('label')(self)[0] - return Attr('.//a', 'id', default=NotAvailable)(td) + # The 'idope' key contains a string such as "70_ABC666ABC 2018-03-182018-03-16-20.55.27.960852" + # 70= N° transaction, 6660666= N° account, 2018-03-18= date and 2018-03-16=rdate. + # We thus use "70_ABC666ABC" for the transaction ID. - obj_code_type = Investment.CODE_TYPE_ISIN + obj_id = Regexp(CleanText(Dict('idope')), '(\d+_[\dA-Z]+)') - def obj_quantity(self): - return MyDecimal(TableCell('quantity'))(self) or NotAvailable + def obj__dates(self): + raw = CleanText(Dict('idope'))(self) + m = re.findall('\d{4}-\d{2}-\d{2}', raw) + # We must verify that the two dates are correctly fetched + assert len(m) == 2 + return m - def obj_unitvalue(self): - return MyDecimal(TableCell('unitvalue'))(self) or NotAvailable + def obj_date(self): + return Date().filter(Field('_dates')(self)[0]) - obj_valuation = MyDecimal(TableCell('valuation')) - obj_portfolio_share = Eval(lambda x: x / 100, CleanDecimal(TableCell('portfolio_share'), replace_dots=True)) + def obj_rdate(self): + return Date().filter(Field('_dates')(self)[1]) - obj_vdate = Date(CleanText(TableCell('vdate')), dayfirst=True, default=NotAvailable) - @pagination +class AVInvestmentsPage(LoggedPage, JsonPage): @method - class iter_history(TableElement): - item_xpath = '//table[@class="table"]/tbody/tr' - head_xpath = '//table[@class="table"]/thead/tr/th' - - col_date = 'Date d\'effet' - col_label = u'Opération(s)' - col_amount = 'Montant' - - def next_page(self): - if Link('//a[@class="pictoSuivant"]', default=None)(self): - form = self.page.get_form(id="frm_fwk") - form['fwkaction'] = "precSuivDet" - form['fwkcodeaction'] = "Executer" - form['ACTION_CHOISIE'] = "suivant" - return requests.Request("POST", form.url, data=dict(form)) + class iter_investment(DictElement): + item_xpath = 'listeSupports/support' class item(ItemElement): - klass = Transaction + klass = Investment - obj_label = CleanText(TableCell('label')) - obj_type = Transaction.TYPE_BANK - obj_date = Date(CleanText(TableCell('date')), dayfirst=True) - obj_amount = MyDecimal(TableCell('amount')) + obj_label = CleanText(Dict('lcspt')) + obj_valuation = CleanDecimal(Dict('mtvalspt')) + obj_code = CleanText(Dict('cdsptisn'), default=NotAvailable) + obj_unitvalue = CleanDecimal(Dict('mtliqpaaspt'), default=NotAvailable) + obj_quantity = CleanDecimal(Dict('qtpaaspt'), default=NotAvailable) + obj_diff = CleanDecimal(Dict('mtpmvspt'), default=NotAvailable) + + def obj_portfolio_share(self): + ptf = CleanDecimal(Dict('txrpaspt'), default=NotAvailable)(self) + ptf /= 100 + return ptf + + def obj_vdate(self): + time = Dict('dvspt')(self) + if not time: + return NotAvailable + return datetime.fromtimestamp(time//1000) + + def obj_code_type(self): + if is_isin_valid(Field('code')(self)): + return Investment.CODE_TYPE_ISIN + return NotAvailable class RibPage(LoggedPage, LCLBasePage): @@ -1074,14 +1045,8 @@ def validate(self, iban, label): class CheckValuesPage(LoggedPage, HTMLPage): def check_values(self, iban, label): values = CleanText('//form[contains(@action, "/outil")]')(self.doc) - try: - assert iban in values - except AssertionError: - raise AddRecipientError('iban not found in values') - try: - assert label in values - except AssertionError: - raise AddRecipientError('recipient label not found in values') + assert iban in values, 'Iban (%s) not found in values: %s' % (iban, values) + assert label in values, 'Recipient label (%s) not found in values: %s' % (label, values) class DocumentsPage(LoggedPage, HTMLPage): @@ -1098,7 +1063,7 @@ def do_search_request(self): @method class get_list(TableElement): head_xpath = '//table[@class="dematTab"]/thead/tr/th' - item_xpath = u'//table[@class="dematTab"]/tbody/tr[./td[@class="dematTab-firstCell" and contains(., "Relevés")]]' + item_xpath = u'//table[@class="dematTab"]/tbody/tr[./td[@class="dematTab-firstCell"]]' col_label = 'Nature de document' col_id = 'Type de document' @@ -1116,6 +1081,16 @@ class item(ItemElement): def obj_url(self): return Link(TableCell('url')(self)[0].xpath('./a'))(self) + def obj_type(self): + if 'Relevé' in Field('label')(self): + return DocumentTypes.STATEMENT + elif 'Bourse' in Field('label')(self): + return DocumentTypes.REPORT + elif ('information' in Field('id')(self)) or ('avis' in Field('id')(self)): + return DocumentTypes.NOTICE + else: + return DocumentTypes.OTHER + class ClientPage(LoggedPage, HTMLPage): @method diff --git a/modules/linebourse/api/__init__.py b/modules/linebourse/api/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/modules/linebourse/api/pages.py b/modules/linebourse/api/pages.py new file mode 100644 index 0000000000000000000000000000000000000000..15a616a0ce57beb2d06c31f7115d54468bebdb32 --- /dev/null +++ b/modules/linebourse/api/pages.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Fong Ngo +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +from weboob.browser.elements import method, DictElement, ItemElement +from weboob.browser.filters.json import Dict +from weboob.browser.filters.standard import ( + CleanText, Date, CleanDecimal, Eval, Field, Env, Regexp, +) +from weboob.browser.pages import JsonPage, HTMLPage, LoggedPage +from weboob.capabilities.bank import Investment +from weboob.capabilities.base import NotAvailable +from weboob.exceptions import ActionNeeded +from weboob.tools.capabilities.bank.investments import is_isin_valid + + +class AccountPage(LoggedPage, JsonPage): + def get_ncontrat(self): + return self.doc['identifiantContratCrypte'] + + +class PortfolioPage(LoggedPage, JsonPage): + def get_valuation_diff(self): + return CleanDecimal(Dict('totalPlv'))(self.doc) # Plv = plus-value + + def get_date(self): + return Date(Regexp(Dict('dateValo'), r'(\d{2})(\d{2})(\d{2})', '\\3\\2\\1'), dayfirst=True)(self.doc) + + @method + class iter_investments(DictElement): + item_xpath = 'listeSegmentation/*' # all categories are fetched: obligations, actions, OPC + + class item(ItemElement): + klass = Investment + + obj_label = Dict('libval') + obj_code = Dict('codval') + obj_code_type = Eval( + lambda x: Investment.CODE_TYPE_ISIN if is_isin_valid(x) else NotAvailable, + Field('code') + ) + obj_quantity = CleanDecimal(Dict('qttit')) + obj_unitprice = CleanDecimal(Dict('pam')) + obj_unitvalue = CleanDecimal(Dict('crs')) + obj_valuation = CleanDecimal(Dict('mnt')) + obj_vdate = Env('date') + obj_portfolio_share = Eval(lambda x: x / 100, CleanDecimal(Dict('pourcentageActif'))) + + def parse(self, el): + symbol = Dict('signePlv')(self) + assert symbol in ('+', '-'), 'should be either positive or negative' + self.env['sign'] = 1 if symbol == '+' else -1 + + def obj_diff(self): + return CleanDecimal(Dict('plv'), sign=lambda x: Env('sign')(self))(self) + + def obj_diff_percent(self): + return CleanDecimal(Dict('plvPourcentage'), sign=lambda x: Env('sign')(self))(self) + + +class ConfigurationPage(LoggedPage, JsonPage): + def is_first_connexion(self): + return self.doc['premiereConnexion'] # either True or False + + def get_contract_number(self): + return self.doc['idCompteActif'] + + +class NewWebsiteFirstConnectionPage(LoggedPage, JsonPage): + def build_doc(self, content): + content = JsonPage.build_doc(self, content) + if 'data' in content: + # The value contains HTML + # Must be encoded into str because HTMLPage.build_doc() uses BytesIO + # which expects bytes + html_page = HTMLPage(self.browser, self.response) + return html_page.build_doc(content['data'].encode(self.encoding)) + return content + + def has_first_connection_cgu(self): + # New Espace bourse: user is asked to read some documents during first connection + message = CleanText('//p[contains(text(), "prendre connaissance")]')(self.doc) + if message: + raise ActionNeeded(message) + + +class HistoryAPIPage(LoggedPage, JsonPage): + def has_history(self): + return bool(self.doc['data']['nbTotalValeurs']) diff --git a/modules/linebourse/browser.py b/modules/linebourse/browser.py index 5a341b8b117b61c9a38e04e384e3b6ac9d889a40..3f44b30ae041c788fce31fcaacd66f16cce315e4 100644 --- a/modules/linebourse/browser.py +++ b/modules/linebourse/browser.py @@ -19,15 +19,27 @@ from __future__ import unicode_literals +import time +import re + from weboob.browser import LoginBrowser, URL from weboob.exceptions import BrowserUnavailable from .pages import ( MessagePage, InvestmentPage, HistoryPage, BrokenPage, - MainPage, FirstConnectionPage + MainPage, FirstConnectionPage, +) + +from .api.pages import ( + PortfolioPage, NewWebsiteFirstConnectionPage, ConfigurationPage, + HistoryAPIPage, ) +def get_timestamp(): + return '{}'.format(int(time.time() * 1000)) # in milliseconds + + class LinebourseBrowser(LoginBrowser): BASEURL = 'https://www.linebourse.fr' @@ -36,8 +48,7 @@ class LinebourseBrowser(LoginBrowser): invest = URL(r'/Portefeuille$', r'/Portefeuille\?compte=(?P[^&]+)', InvestmentPage) message = URL(r'/DetailMessage.*', MessagePage) history = URL(r'/HistoriqueOperations', - r'/HistoriqueOperations\?compte=(?P[^&]+)&devise=EUR&modeTri=7&sensTri=-1&periode=(?P\d+)', - HistoryPage) + r'/HistoriqueOperations\?compte=(?P[^&]+)&devise=EUR&modeTri=7&sensTri=-1&periode=(?P\d+)', HistoryPage) useless = URL(r'/ReroutageSJR', MessagePage) broken = URL(r'.*/timeout.html$', BrokenPage) @@ -84,3 +95,57 @@ def iter_history(self, account_id): self.history.go(id=self.page.get_compte(account_id), period=period) for tr in self.page.iter_history(): yield tr + + +class LinebourseAPIBrowser(LoginBrowser): + BASEURL = 'https://www.offrebourse.com' + + new_website_first = URL(r'/rest/premiereConnexion', NewWebsiteFirstConnectionPage) + config = URL(r'/rest/configuration', ConfigurationPage) + + # The API works with an encrypted id_contract that starts with 'CRY' + portfolio = URL(r'/rest/portefeuille/(?PCRY[\w\d]+)/vide/true/false', PortfolioPage) + history = URL(r'/rest/historiqueOperations/(?PCRY[\w\d]+)/0/7/1', HistoryAPIPage) # TODO: not sure if last 3 path levels can be hardcoded + + def __init__(self, baseurl, *args, **kwargs): + self.BASEURL = baseurl + self.id_contract = None # encrypted contract number used to browse between pages + + super(LinebourseAPIBrowser, self).__init__(username='', password='', *args, **kwargs) + + def go_portfolio(self): + self.config.go() + self.id_contract = self.page.get_contract_number() + return self.portfolio.go(id_contract=self.id_contract) + + def handle_cgu(self): + # the website uses an additional timestamp as query parameter + # for this GET request but it turns out it is not mandatory + self.config.go() + assert self.config.is_here() + + if self.page.is_first_connexion(): + self.new_website_first.go() + assert self.new_website_first.is_here() + + self.page.has_first_connection_cgu() + + def iter_investments(self): + self.go_portfolio() + assert self.portfolio.is_here() + date = self.page.get_date() + return self.page.iter_investments(date=date) + + def iter_history(self): + assert re.match(r'CRY[\w\d]+', self.id_contract) + self.history.go( + id_contract=self.id_contract, + params={'_': get_timestamp()}, # timestamp is necessary + ) + assert self.history.is_here() + + # Didn't find a connection with transactions + # TODO: implement corresponding pages + if self.page.has_history(): + assert False, 'please implement iter_history' + return [] diff --git a/modules/linebourse/pages.py b/modules/linebourse/pages.py index 02ec81fbbe14fda3ff3e5836fa3a254727d23001..058788514049f23f0ea91f65f6f1ebaee4e6caed 100644 --- a/modules/linebourse/pages.py +++ b/modules/linebourse/pages.py @@ -131,7 +131,7 @@ def condition(self): obj_quantity = MyDecimal(TableCell('quantity'), default=NotAvailable) obj_unitvalue = MyDecimal(TableCell('unitvalue'), default=NotAvailable) obj_unitprice = MyDecimal(TableCell('unitprice'), default=NotAvailable) - obj_valuation = MyDecimal(TableCell('valuation')) + obj_valuation = MyDecimal(TableCell('valuation'), default=NotAvailable) obj_portfolio_share = Eval(lambda x: x / 100 if x else NotAvailable, MyDecimal(TableCell('portfolio_share'), default=NotAvailable)) obj_diff = MyDecimal(TableCell('diff', default=NotAvailable), default=NotAvailable) obj_code_type = Investment.CODE_TYPE_ISIN diff --git a/modules/marmiton/browser.py b/modules/marmiton/browser.py index 4857b6aba25b057e5b48d2ee5cd2513f5102eb3f..8c47440d114a907d627d38c48e243e77b2c6f7e9 100644 --- a/modules/marmiton/browser.py +++ b/modules/marmiton/browser.py @@ -27,7 +27,7 @@ class MarmitonBrowser(PagesBrowser): - BASEURL = 'http://www.marmiton.org' + BASEURL = 'https://www.marmiton.org' search = URL('/recettes/recherche.aspx\?aqt=(?P.*)&start=(?P\d*)', '/recettes/recherche.aspx\?aqt=.*', ResultsPage) diff --git a/modules/marmiton/pages.py b/modules/marmiton/pages.py index beb6d35e6925aff9cad7f4a0501c02520ef5d406..944494c7ad116063444c007b78efa6f662799328 100644 --- a/modules/marmiton/pages.py +++ b/modules/marmiton/pages.py @@ -33,7 +33,7 @@ class ResultsPage(HTMLPage): @pagination @method class iter_recipes(ListElement): - item_xpath = "//div[@class='recipe-results ']/a" + item_xpath = "//a[@class='recipe-card-link']" def next_page(self): return CleanText('//nav/ul/li[@class="next-page"]/a/@href', default="")(self) diff --git a/modules/materielnet/browser.py b/modules/materielnet/browser.py index 01dc764580a63ab8b87b295565f72c1fd3a85d6a..c81e142c5a6c378c0894d5e4e74ae357988f1444 100644 --- a/modules/materielnet/browser.py +++ b/modules/materielnet/browser.py @@ -21,25 +21,31 @@ from weboob.browser import LoginBrowser, URL, need_login from weboob.exceptions import BrowserIncorrectPassword -from .pages import LoginPage, CaptchaPage, ProfilPage, DocumentsPage +from .pages import LoginPage, CaptchaPage, ProfilPage, DocumentsPage, DocumentsDetailsPage class MaterielnetBrowser(LoginBrowser): - BASEURL = 'https://www.materiel.net' + BASEURL = 'https://secure.materiel.net' - login = URL('/pm/client/login.html', LoginPage) + login = URL(r'https://www.materiel.net/form/login', + r'/Login/PartialPublicLogin', LoginPage) captcha = URL('/pm/client/captcha.html', CaptchaPage) - profil = URL('/pm/client/compte.html', ProfilPage) - documents = URL('/pm/client/commande.html\?page=(?P.*)', - '/pm/client/commande.html', DocumentsPage) + profil = URL(r'/Account/InformationsSection', ProfilPage) + documents = URL(r'/Orders/PartialCompletedOrdersHeader', DocumentsPage) + document_details = URL(r'/Orders/PartialCompletedOrderContent', DocumentsDetailsPage) def do_login(self): self.login.go() - self.page.login(self.username, self.password) - if self.login.is_here() or self.captcha.is_here(): - raise BrowserIncorrectPassword(self.page.get_error()) + if self.captcha.is_here(): + BrowserIncorrectPassword() + + if self.login.is_here(): + error = self.page.get_error() + # when everything is good we land on this page + if error: + raise BrowserIncorrectPassword(error) @need_login def get_subscription_list(self): @@ -47,4 +53,7 @@ def get_subscription_list(self): @need_login def iter_documents(self, subscription): - return self.documents.stay_or_go(page=1).get_documents() + json_response = self.location('/Orders/CompletedOrdersPeriodSelection').json() + + for data in json_response: + return self.documents.go(data=data).get_documents() diff --git a/modules/materielnet/module.py b/modules/materielnet/module.py index 35bec35ec53ff5ddd54aa716d49cf535904fe5fd..4d7d066b00158872d6a4d8efbe21412d087e9cc6 100644 --- a/modules/materielnet/module.py +++ b/modules/materielnet/module.py @@ -21,7 +21,7 @@ from weboob.capabilities.bill import CapDocument, Subscription, Document, SubscriptionNotFound, DocumentNotFound from weboob.capabilities.base import find_object, NotAvailable from weboob.tools.backend import Module, BackendConfig -from weboob.tools.value import ValueBackendPassword, Value +from weboob.tools.value import ValueBackendPassword from .browser import MaterielnetBrowser @@ -36,8 +36,8 @@ class MaterielnetModule(Module, CapDocument): EMAIL = 'elambert@budget-insight.com' LICENSE = 'AGPLv3+' VERSION = '1.3' - CONFIG = BackendConfig(Value('login', label='Identifiant'), - ValueBackendPassword('password', label='Mot de passe')) + CONFIG = BackendConfig(ValueBackendPassword('login', label='Email', regex='.+@.+'), + ValueBackendPassword('password', label='Mot de passe')) BROWSER = MaterielnetBrowser diff --git a/modules/materielnet/pages.py b/modules/materielnet/pages.py index 26b8b1e94f72b1f1095e6eb66d255668e8d41baf..ffe08a87e7128d89866704833ced69f8eb0b2149 100644 --- a/modules/materielnet/pages.py +++ b/modules/materielnet/pages.py @@ -17,25 +17,40 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals import re -from weboob.browser.pages import HTMLPage, LoggedPage, pagination -from weboob.browser.filters.standard import CleanText, CleanDecimal, Env, Format, Date, Async, AsyncLoad -from weboob.browser.elements import ListElement, ItemElement, TableElement, method -from weboob.browser.filters.html import Attr, Link, TableCell +from weboob.browser.pages import HTMLPage, LoggedPage, PartialHTMLPage +from weboob.browser.filters.standard import CleanText, CleanDecimal, Env, Format, Date, Async, Filter, Regexp, Field +from weboob.browser.elements import ListElement, ItemElement, method +from weboob.browser.filters.html import Attr, Link from weboob.capabilities.bill import Bill, Subscription from weboob.capabilities.base import NotAvailable +from weboob.exceptions import BrowserIncorrectPassword -class LoginPage(HTMLPage): + +class LoginPage(PartialHTMLPage): def login(self, login, password): - form = self.get_form('//form[@class="login-form"]') - form['identifier'] = login - form['credentials'] = password + maxlength = Attr('//input[@id="Email"]', 'data-val-maxlength-max')(self.doc) + regex = Attr('//input[@id="Email"]', 'data-val-regex-pattern')(self.doc) + # their regex is: ^([\w\-+\.]+)@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.)|(([\w-]+\.)+))([a-zA-Z]{2,15}|[0-9]{1,3})(\]?)$ + # but it is not very good, we escape - inside [] to avoid bad character range Exception + regex = regex.replace('[\w-+\.]', '[\w\-+\.]') + + if len(login) > maxlength: # actually it's 60 char + raise BrowserIncorrectPassword(Attr('//input[@id="Email"]', 'data-val-maxlength')(self.doc)) + + if not re.match(regex, login): + raise BrowserIncorrectPassword(Attr('//input[@id="Email"]', 'data-val-regex')(self.doc)) + + form = self.get_form(id='loginForm') + form['Email'] = login + form['Password'] = password form.submit() def get_error(self): - return CleanText('//div[@class="ValidatorError"]')(self.doc) + return CleanText('//div[contains(@class, "error")]')(self.doc) class CaptchaPage(HTMLPage): @@ -49,7 +64,7 @@ class get_list(ListElement): class item(ItemElement): klass = Subscription - obj_subscriber = Format('%s %s', Attr('//input[@id="prenom"]', 'value'), Attr('//input[@id="nom"]', 'value')) + obj_subscriber = Format('%s %s', Attr('//input[@id="FirstName"]', 'value'), Attr('//input[@id="LastName"]', 'value')) obj_id = Env('subid') obj_label = obj_id @@ -57,38 +72,35 @@ def parse(self, el): self.env['subid'] = self.page.browser.username -class DocumentsPage(LoggedPage, HTMLPage): - @pagination +class MyAsyncLoad(Filter): + def __call__(self, item): + link = self.select(self.selector, item) + data = {'X-Requested-With': 'XMLHttpRequest'} + return item.page.browser.async_open(link, data=data) if link else None + + +class DocumentsPage(LoggedPage, PartialHTMLPage): @method - class get_documents(TableElement): - item_xpath = '//div[@id="ListCmd"]//table//tr[position() > 1]' - head_xpath = '//div[@id="ListCmd"]//table//tr//th' - - col_id = u'Référence' - col_date = u'Date' - col_price = u'Montant' - - def next_page(self): - m = re.search('([^*]+page=)([^*]+)', self.page.url) - if m: - page = int(m.group(2)) + 1 - if self.el.xpath('//a[contains(@href, "commande.html?page=' + str(page) + '")]'): - next_page = u"%s%s" % (m.group(1), page) - return next_page + class get_documents(ListElement): + item_xpath = '//div[@class="historic-table"]' class item(ItemElement): klass = Bill - load_details = Attr('./td/a', 'href') & AsyncLoad + load_details = Link('.//a') & MyAsyncLoad - obj_id = Format('%s_%s', Env('email'), CleanDecimal(TableCell('id'))) - obj_url = Async('details') & Link('//a[contains(@href, "facture")]', default=NotAvailable) - obj_date = Date(CleanText(TableCell('date'))) - obj_format = u"pdf" - obj_label = Async('details') & CleanText('//table/tr/td[@class="Prod"]') - obj_type = u"bill" - obj_price = CleanDecimal(TableCell('price'), replace_dots=True) - obj_currency = u'EUR' + obj_id = Format('%s_%s', Env('email'), Field('label')) + obj_url = Async('details') & Link('//a', default=NotAvailable) + obj_date = Date(CleanText('./div[contains(@class, "date")]'), dayfirst=True) + obj_format = 'pdf' + obj_label = Regexp(CleanText('./div[contains(@class, "ref")]'), r' (.*)') + obj_type = 'bill' + obj_price = CleanDecimal(CleanText('./div[contains(@class, "price")]'), replace_dots=(' ', '€')) + obj_currency = 'EUR' def parse(self, el): self.env['email'] = self.page.browser.username + + +class DocumentsDetailsPage(LoggedPage, PartialHTMLPage): + pass diff --git a/modules/myhabit/browser.py b/modules/myhabit/browser.py index 20d26d557742f218a3ec37196d9186e5c1bcc6d3..dfcc2283c96fbf6d1bd276891f56168b02d18a00 100644 --- a/modules/myhabit/browser.py +++ b/modules/myhabit/browser.py @@ -18,19 +18,18 @@ # along with weboob. If not, see . +from datetime import datetime +from decimal import Decimal + from requests.exceptions import Timeout -from weboob.tools.capabilities.bank.transactions import \ - AmericanTransaction as AmTr -from weboob.browser import LoginBrowser, URL, need_login +from weboob.browser import URL, LoginBrowser, need_login from weboob.browser.pages import HTMLPage from weboob.capabilities.base import Currency -from weboob.capabilities.shop import OrderNotFound, Order, Payment, Item +from weboob.capabilities.shop import Item, Order, OrderNotFound, Payment from weboob.exceptions import BrowserIncorrectPassword - -from datetime import datetime -from decimal import Decimal - +from weboob.tools.capabilities.bank.transactions import AmericanTransaction as AmTr +from weboob.tools.compat import unicode __all__ = ['MyHabit'] @@ -108,12 +107,12 @@ def items(self): def order_date(self): date = self.doc.xpath(u'//span[text()="Order Placed:"]' - '/following-sibling::span[1]/text()')[0].strip() + u'/following-sibling::span[1]/text()')[0].strip() return datetime.strptime(date, '%b %d, %Y') def order_number(self): return self.doc.xpath(u'//span[text()="MyHabit Order Number:"]' - '/following-sibling::span[1]/text()')[0].strip() + u'/following-sibling::span[1]/text()')[0].strip() def order_amount(self, which): return AmTr.decimal_amount((self.doc.xpath( @@ -187,7 +186,7 @@ def iter_items(self, order): @need_login def to_history(self): - for i in xrange(self.MAX_RETRIES): + for i in range(self.MAX_RETRIES): if self.history.is_here() and self.page.is_sane(): return self.page self.history.go() @@ -197,9 +196,9 @@ def do_login(self): raise BrowserIncorrectPassword() def location(self, *args, **kwargs): - for i in xrange(self.MAX_RETRIES): + for i in range(self.MAX_RETRIES): try: return super(MyHabit, self).location(*args, **kwargs) except Timeout as e: - pass - raise e + last_error = e + raise last_error diff --git a/modules/myhabit/module.py b/modules/myhabit/module.py index f7eb47fd853d464bdfffbc1b6ddf90be24418e23..616b0450fea675e6a7f302691b47246e4d08f0e4 100644 --- a/modules/myhabit/module.py +++ b/modules/myhabit/module.py @@ -19,13 +19,14 @@ from weboob.capabilities.shop import CapShop -from weboob.tools.backend import Module, BackendConfig +from weboob.tools.backend import BackendConfig, Module from weboob.tools.value import ValueBackendPassword from .browser import MyHabit __all__ = ['MyHabitModule'] + class MyHabitModule(Module, CapShop): NAME = 'myhabit' MAINTAINER = u'Oleg Plakhotniuk' diff --git a/modules/n26/browser.py b/modules/n26/browser.py index d160e45296d786bf1e8b18afc52262b28e7ef19a..3d91db7d6a8af49a2869fed948277ff2966bdbdc 100644 --- a/modules/n26/browser.py +++ b/modules/n26/browser.py @@ -143,7 +143,7 @@ def _internal_get_transactions(self, categories, filter_func): for t in transactions: - if not filter_func(t): + if not filter_func(t) or t["amount"] == 0: continue new = Transaction() diff --git a/modules/nalo/compat/weboob_capabilities_bank.py b/modules/nalo/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/nalo/compat/weboob_capabilities_bank.py +++ b/modules/nalo/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/orange/browser.py b/modules/orange/browser.py index bd514e4efadc42a73f9e0d27b03388df2f22ca25..0bb944c0de63c1c4ae3589b813e6d856f2e7e596 100644 --- a/modules/orange/browser.py +++ b/modules/orange/browser.py @@ -32,7 +32,7 @@ class OrangeBillBrowser(LoginBrowser): - BASEURL = 'https://espaceclientv3.orange.fr/' + BASEURL = 'https://espaceclientv3.orange.fr' loginpage = URL('https://login.orange.fr/\?service=sosh&return_url=https://www.sosh.fr/', 'https://login.orange.fr/front/login', LoginPage) @@ -73,6 +73,13 @@ def get_nb_remaining_free_sms(self): def post_message(self, message, sender): raise NotImplementedError() + def _iter_subscriptions_by_type(self, name, _type): + self.location('https://espaceclientv3.orange.fr/?page=gt-home-page&%s' % _type) + self.subscriptions.go() + for sub in self.page.iter_subscription(): + sub.subscriber = name + yield sub + @need_login def get_subscription_list(self): profile = self.profile.go().get_profile() @@ -92,10 +99,9 @@ def get_subscription_list(self): return # if nb_sub is 0, we continue, because we can get them in next url - self.location('https://espaceclientv3.orange.fr/?page=gt-home-page&sosh') - self.subscriptions.go() - for sub in self.page.iter_subscription(): - sub.subscriber = profile.name + for sub in self._iter_subscriptions_by_type(profile.name, 'sosh'): + yield sub + for sub in self._iter_subscriptions_by_type(profile.name, 'orange'): yield sub @need_login diff --git a/modules/paypal/browser.py b/modules/paypal/browser.py index 57a58d8cd870aaac1c37f507f0ca358ecc1259a4..79f27bea90b763715e516d8b2a2730df8ce2514c 100644 --- a/modules/paypal/browser.py +++ b/modules/paypal/browser.py @@ -79,6 +79,7 @@ def __init__(self, *args, **kwargs): super(Paypal, self).__init__(*args, **kwargs) def do_login(self): + raise BrowserUnavailable() assert isinstance(self.username, basestring) assert isinstance(self.password, basestring) diff --git a/modules/peertube/__init__.py b/modules/peertube/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e070b900fe1dba80ac81521c0fc8dfe8c33cf4ab --- /dev/null +++ b/modules/peertube/__init__.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + + +from .module import PeertubeModule + + +__all__ = ['PeertubeModule'] diff --git a/modules/peertube/browser.py b/modules/peertube/browser.py new file mode 100644 index 0000000000000000000000000000000000000000..25e483ac7ffb5c69f5826487e51ab64f1c1f69e4 --- /dev/null +++ b/modules/peertube/browser.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +from weboob.browser.browsers import APIBrowser +from weboob.capabilities.video import BaseVideo +from weboob.capabilities.image import Thumbnail +from weboob.capabilities.file import LICENSES + + +class PeertubeBrowser(APIBrowser): + # source: server/initializers/constants.ts + SITE_LICENSES = { + 1: LICENSES.CCBY, + 2: LICENSES.CCBYSA, + 3: LICENSES.CCBYND, + 4: LICENSES.CCBYNC, + 5: LICENSES.CCBYNCSA, + 6: LICENSES.CCBYNCND, + 7: LICENSES.PD, + } + + def __init__(self, baseurl, *args, **kwargs): + super(PeertubeBrowser, self).__init__(*args, **kwargs) + self.BASEURL = baseurl + + def search_videos(self, pattern, sortby): + j = self.request('/api/v1/search/videos?count=10&sort=-match', params={ + 'search': pattern, + 'start': 0, + }) + + for item in j['data']: + video = BaseVideo() + self._parse_video(video, item) + yield video + + def get_video(self, id, video=None): + item = self.request('/api/v1/videos/%s' % id) + + if not video: + video = BaseVideo() + + self._parse_video(video, item) + + video._torrent = item['files'][0]['magnetUri'] + video.url = item['files'][0]['fileUrl'] + video.ext = video.url.rsplit('.', 1)[-1] + video.size = item['files'][0]['size'] + + return video + + def _parse_video(self, video, item): + video.id = item['uuid'] + video.nsfw = item['nsfw'] + video.title = item['name'] + video.description = item['description'] + video.author = item['account']['name'] + video.duration = item['duration'] + video.license = self.SITE_LICENSES[item['licence']['id']] + video.thumbnail = Thumbnail(self.absurl(item['thumbnailPath'])) diff --git a/modules/peertube/module.py b/modules/peertube/module.py new file mode 100644 index 0000000000000000000000000000000000000000..20498a9569fdb390abeb6ccdcd43cfd770e49c48 --- /dev/null +++ b/modules/peertube/module.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +from weboob.tools.backend import Module, BackendConfig +from weboob.tools.value import Value +from weboob.capabilities.video import CapVideo, BaseVideo + +from .browser import PeertubeBrowser + + +__all__ = ['PeertubeModule'] + + +class PeertubeModule(Module, CapVideo): + NAME = 'peertube' + DESCRIPTION = 'Peertube' + MAINTAINER = 'Vincent A' + EMAIL = 'dev@indigo.re' + LICENSE = 'AGPLv3+' + VERSION = '1.3' + + CONFIG = BackendConfig( + Value('url', label='Base URL of the PeerTube instance'), + ) + + BROWSER = PeertubeBrowser + + def create_default_browser(self): + return self.create_browser(self.config['url'].get()) + + def get_video(self, id): + return self.browser.get_video(id) + + def search_videos(self, pattern, sortby=CapVideo.SEARCH_RELEVANCE, nsfw=False): + for video in self.browser.search_videos(pattern, sortby): + if nsfw or not video.nsfw: + yield video + + def fill_video(self, obj, fields): + if set(('url', 'size')) & set(fields): + self.browser.get_video(obj.id, obj) + if 'thumbnail' in fields and obj.thumbnail: + obj.thumbnail.data = self.browser.open(obj.thumbnail.url).content + + OBJECTS = { + BaseVideo: fill_video, + } diff --git a/modules/phpbb/browser.py b/modules/phpbb/browser.py index fc1dbc8849b4720ecc92ad92cb8a649617cfefde..927011e4d2c72984ded6b6562cb0a06bdcdac762 100644 --- a/modules/phpbb/browser.py +++ b/modules/phpbb/browser.py @@ -20,15 +20,14 @@ import re -from weboob.browser import LoginBrowser, need_login, URL -from weboob.exceptions import BrowserIncorrectPassword +from weboob.browser import URL, LoginBrowser, need_login from weboob.capabilities.messages import CantSendMessage +from weboob.exceptions import BrowserIncorrectPassword +from .pages.forum import ForumPage, PostingPage, TopicPage from .pages.index import LoginPage -from .pages.forum import ForumPage, TopicPage, PostingPage from .tools import id2url, url2id - __all__ = ['PhpBB'] @@ -56,7 +55,7 @@ def do_login(self): data = {'login': 'Connexion', 'username': self.username, 'password': self.password, - } + } self.location('ucp.php?mode=login', data=data) if not self.page.logged: @@ -158,7 +157,7 @@ def post_answer(self, forum_id, topic_id, title, content): raise CantSendMessage('Please enter a title formatted like that:\n\t"[FORUM] SUBJECT"\n\n%s' % forums_prompt) forum_id = None - for k,v in forums.iteritems(): + for k, v in forums.items(): if v.lower() == m.group(1).lower(): forum_id = k break diff --git a/modules/phpbb/module.py b/modules/phpbb/module.py index c4ea6e6861b10832eeb5f061bbbd32e726bd10b6..4e022d8acec5b2d71a8dca5721bf7bc0d9fa47ed 100644 --- a/modules/phpbb/module.py +++ b/modules/phpbb/module.py @@ -18,15 +18,14 @@ # along with weboob. If not, see . -from weboob.tools.backend import Module, BackendConfig -from weboob.tools.newsfeed import Newsfeed -from weboob.tools.value import Value, ValueInt, ValueBackendPassword +from weboob.capabilities.messages import CantSendMessage, CapMessages, CapMessagesPost, Message, Thread +from weboob.tools.backend import BackendConfig, Module from weboob.tools.misc import limit -from weboob.capabilities.messages import CapMessages, CapMessagesPost, Message, Thread, CantSendMessage +from weboob.tools.newsfeed import Newsfeed +from weboob.tools.value import Value, ValueBackendPassword, ValueInt from .browser import PhpBB -from .tools import rssid, url2id, id2url, id2topic - +from .tools import id2topic, id2url, rssid, url2id __all__ = ['PhpBBModule'] @@ -42,7 +41,7 @@ class PhpBBModule(Module, CapMessages, CapMessagesPost): Value('username', label='Username', default=''), ValueBackendPassword('password', label='Password', default=''), ValueInt('thread_unread_messages', label='Limit number of unread messages to retrieve for a thread', default=500) - ) + ) STORAGE = {'seen': {}} BROWSER = PhpBB diff --git a/modules/phpbb/pages/forum.py b/modules/phpbb/pages/forum.py index 2703dddbf5d9aa2eba637630a4399971d49a4e66..5a8c0573ad2c0433241095565858766d79f194ed 100644 --- a/modules/phpbb/pages/forum.py +++ b/modules/phpbb/pages/forum.py @@ -29,7 +29,7 @@ class Link(object): (FORUM, - TOPIC) = xrange(2) + TOPIC) = range(2) def __init__(self, type, url): self.type = type diff --git a/modules/piratebay/test.py b/modules/piratebay/test.py index 7e966bf8b0622e78160a1f982ec4ead1a4971951..e31cb35740ea260f67e811336f4b606ab9fa2641 100644 --- a/modules/piratebay/test.py +++ b/modules/piratebay/test.py @@ -17,20 +17,21 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . -from weboob.tools.test import BackendTest -from weboob.capabilities.torrent import MagnetOnly - from random import choice +from weboob.capabilities.torrent import MagnetOnly +from weboob.tools.compat import basestring +from weboob.tools.test import BackendTest + class PiratebayTest(BackendTest): MODULE = 'piratebay' def test_torrent(self): # try something popular so we sometimes get a magnet-only torrent - l = list(self.backend.iter_torrents('ubuntu linux')) - if len(l): - torrent = choice(l) + torrents = list(self.backend.iter_torrents('ubuntu linux')) + if len(torrents): + torrent = choice(torrents) full_torrent = self.backend.get_torrent(torrent.id) assert torrent.name assert full_torrent.name == torrent.name diff --git a/modules/pradoepargne/compat/weboob_capabilities_bank.py b/modules/pradoepargne/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/pradoepargne/compat/weboob_capabilities_bank.py +++ b/modules/pradoepargne/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/rmll/pages.py b/modules/rmll/pages.py index bc814da7e0a268a6a3b658ff566fcda192123e36..fe06110161fe6ba5d8c300749662436842441331 100644 --- a/modules/rmll/pages.py +++ b/modules/rmll/pages.py @@ -18,19 +18,19 @@ # along with weboob. If not, see . import re +from datetime import timedelta -from weboob.browser.pages import HTMLPage, JsonPage from weboob.browser.elements import ItemElement, ListElement, method -from weboob.browser.filters.standard import Regexp, Format, CleanText, Duration, DateTime, Filter -from weboob.browser.filters.html import Link, XPath, CleanHTML +from weboob.browser.filters.html import CleanHTML, Link, XPath from weboob.browser.filters.json import Dict - +from weboob.browser.filters.standard import CleanText, DateTime, Duration, Filter, Format, Regexp +from weboob.browser.pages import HTMLPage, JsonPage from weboob.capabilities import NotLoaded -from weboob.capabilities.image import Thumbnail from weboob.capabilities.collection import Collection +from weboob.capabilities.image import Thumbnail +from weboob.tools.compat import unicode from .video import RmllVideo -from datetime import timedelta BASE_URL = 'http://video.rmll.info' diff --git a/modules/s2e/compat/weboob_capabilities_bank.py b/modules/s2e/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/s2e/compat/weboob_capabilities_bank.py +++ b/modules/s2e/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/societegenerale/browser.py b/modules/societegenerale/browser.py index e275b69c618af7cccad06b7878fd67139df6a6e6..f15e84ef4602c6be1a142b57142fa997fac7e29e 100644 --- a/modules/societegenerale/browser.py +++ b/modules/societegenerale/browser.py @@ -17,20 +17,24 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals + from datetime import datetime from dateutil.relativedelta import relativedelta from weboob.browser.browsers import LoginBrowser, URL, need_login, StatesMixin from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable, ActionNeeded -from weboob.capabilities.bank import Account, AddRecipientError +from weboob.capabilities.bank import Account, TransferBankError from weboob.capabilities.base import find_object, NotAvailable from weboob.browser.exceptions import BrowserHTTPNotFound from .compat.weboob_capabilities_profile import ProfileMissing +from .compat.weboob_tools_capabilities_bank_investments import create_french_liquidity from .pages.accounts_list import ( AccountsList, AccountHistory, CardsList, LifeInsurance, LifeInsuranceHistory, LifeInsuranceInvest, LifeInsuranceInvest2, Market, UnavailableServicePage, - ListRibPage, AdvisorPage, HTMLProfilePage, XMLProfilePage, LoansPage, IbanPage, + ListRibPage, AdvisorPage, HTMLProfilePage, XMLProfilePage, LoansPage, IbanPage, ComingPage, + NewLandingPage, ) from .pages.transfer import RecipientsPage, TransferPage, AddRecipientPage, RecipientJson from .pages.login import LoginPage, BadLoginPage, ReinitPasswordPage, ActionNeededPage, ErrorPage @@ -47,11 +51,13 @@ class SocieteGenerale(LoginBrowser, StatesMixin): login = URL('https://particuliers.societegenerale.fr/index.html', LoginPage) action_needed = URL('/com/icd-web/forms/cct-index.html', '/com/icd-web/gdpr/gdpr-recueil-consentements.html', + '/com/icd-web/forms/kyc-index.html', ActionNeededPage) bad_login = URL('\/acces/authlgn.html', '/error403.html', BadLoginPage) reinit = URL('/acces/changecodeobligatoire.html', ReinitPasswordPage) iban_page = URL(r'/lgn/url\.html\?dup', IbanPage) accounts = URL('/restitution/cns_listeprestation.html', AccountsList) + coming_page = URL('/restitution/cns_listeEncours.xml', ComingPage) cards_list = URL('/restitution/cns_listeCartes.*.html', CardsList) account_history = URL('/restitution/cns_detail.*\.html', '/lgn/url.html', AccountHistory) market = URL('/brs/cct/comti20.html', Market) @@ -76,6 +82,8 @@ class SocieteGenerale(LoginBrowser, StatesMixin): bank_statement_search = URL(r'/restitution/rce_recherche.html\?noRedirect=1', r'/restitution/rce_recherche_resultat.html', BankStatementPage) + new_landing = URL(r'/com/icd-web/cbo/index.html', NewLandingPage) + error = URL('https://static.societegenerale.fr/pri/erreur.html', ErrorPage) accounts_list = None @@ -90,7 +98,7 @@ def load_state(self, state): super(SocieteGenerale, self).load_state(state) def do_login(self): - if not self.password.isdigit() or len(self.password) != 6: + if not self.password.isdigit() or len(self.password) not in (6, 7): raise BrowserIncorrectPassword() if not self.username.isdigit() or len(self.username) < 8: raise BrowserIncorrectPassword() @@ -125,7 +133,14 @@ def do_login(self): def get_accounts_list(self): if self.accounts_list is None: self.accounts.stay_or_go() + # the link is not on the new landing page, navigating manually + if self.new_landing.is_here(): + self.logger.info('Falling back on old accounts consulting page.') + self.location('/restitution/cns_listeprestation.html?NoRedirect=true') self.accounts_list = self.page.get_list() + # Coming amount is on another page, whose url must be retrieved on the main page + self.location(self.page.get_coming_url()) + self.page.set_coming(self.accounts_list) self.list_rib.go() if self.list_rib.is_here(): # Caching rib url, so we don't have to go back and forth for each account @@ -202,6 +217,14 @@ def iter_investment(self, account): # Other Life Insurance pages: self.life_insurance_invest.go() + elif account.type == Account.TYPE_PEA: + # Scraping liquidities for "PEA Espèces" accounts + self.location(account._link_id) + valuation = self.page.get_liquidities() + if valuation != NotAvailable: + yield create_french_liquidity(valuation) + return + else: self.logger.warning('This account is not supported') return @@ -215,7 +238,11 @@ def get_advisor(self): @need_login def iter_recipients(self, account): - if not self.transfer.go().is_able_to_transfer(account): + try: + self.transfer.go() + except TransferBankError: + return + if not self.page.is_able_to_transfer(account): return for recipient in self.page.iter_recipients(account_id=account.id): yield recipient @@ -243,12 +270,13 @@ def end_sms_recipient(self, recipient, **params): def end_oob_recipient(self, recipient, **params): r = self.open('https://particuliers.secure.societegenerale.fr/sec/oob_polling.json', data={'n10_id_transaction': self.id_transaction}) - if r.page.doc['donnees']['transaction_status'] == 'available': - data = [('context', self.context), ('b64_jeton_transaction', self.context), ('dup', self.dup), ('n10_id_transaction', self.id_transaction), ('oob_op', 'sign')] - self.add_recipient.go(data=data, headers={'Referer': 'https://particuliers.secure.societegenerale.fr/lgn/url.html'}) - return self.page.get_recipient_object(recipient) - else: - raise AddRecipientError('transaction_status is %s' % r.page.doc['donnees']['transaction_status']) + assert r.page.doc['donnees']['transaction_status'] in ('available', 'in_progress'), \ + 'transaction_status is %s' % r.page.doc['donnees']['transaction_status'] + + data = [('context', self.context), ('b64_jeton_transaction', self.context), + ('dup', self.dup), ('n10_id_transaction', self.id_transaction), ('oob_op', 'sign')] + self.add_recipient.go(data=data, headers={'Referer': 'https://particuliers.secure.societegenerale.fr/lgn/url.html'}) + return self.page.get_recipient_object(recipient) @need_login def new_recipient(self, recipient, **params): diff --git a/modules/societegenerale/compat/weboob_capabilities_bank.py b/modules/societegenerale/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/societegenerale/compat/weboob_capabilities_bank.py +++ b/modules/societegenerale/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/societegenerale/compat/weboob_capabilities_base.py b/modules/societegenerale/compat/weboob_capabilities_base.py new file mode 100644 index 0000000000000000000000000000000000000000..df23e45a76007314cac6e7ebd2c8cfb41a948152 --- /dev/null +++ b/modules/societegenerale/compat/weboob_capabilities_base.py @@ -0,0 +1,22 @@ + +import weboob.capabilities.base as OLD + +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) + + +__all__ = OLD.__all__ + + +def strict_find_object(mylist, error=None, **kwargs): + """ + Tools to return an object with the matching parameters in kwargs. + Parameters with empty value are skipped + """ + kwargs = {k: v for k, v in kwargs.items() if not empty(v)} + if kwargs: + return find_object(mylist, error=error, **kwargs) + + if error is not None: + raise error() diff --git a/modules/societegenerale/compat/weboob_tools_capabilities_bank_investments.py b/modules/societegenerale/compat/weboob_tools_capabilities_bank_investments.py new file mode 100644 index 0000000000000000000000000000000000000000..2b68ff9e001c726a90445e05e0a8ef2e2f165059 --- /dev/null +++ b/modules/societegenerale/compat/weboob_tools_capabilities_bank_investments.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Jonathan Schmidt +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.tools.compat import basestring +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.bank import Investment + +def is_isin_valid(isin): + """ + Méthode générale + Table de conversion des lettres en chiffres + A=10 B=11 C=12 D=13 E=14 F=15 G=16 H=17 I=18 + J=19 K=20 L=21 M=22 N=23 O=24 P=25 Q=26 R=27 + S=28 T=29 U=30 V=31 W=32 X=33 Y=34 Z=35 + + 1 - Mettre de côté la clé, qui servira de référence à la fin de la vérification. + 2 - Convertir toutes les lettres en nombres via la table de conversion ci-contre. Si le nombre obtenu est supérieur ou égal à 10, prendre les deux chiffres du nombre séparément (exemple : 27 devient 2 et 7). + 3 - Pour chaque chiffre, multiplier sa valeur par deux si sa position est impaire en partant de la droite. Si le nombre obtenu est supérieur ou égal à 10, garder les deux chiffres du nombre séparément (exemple : 14 devient 1 et 4). + 4 - Faire la somme de tous les chiffres. + 5 - Soustraire cette somme de la dizaine supérieure ou égale la plus proche (exemples : si la somme vaut 22, la dizaine « supérieure ou égale » est 30, et la clé vaut donc 8 ; si la somme vaut 30, la dizaine « supérieure ou égale » est 30, et la clé vaut 0 ; si la somme vaut 31, la dizaine « supérieure ou égale » est 40, et la clé vaut 9). + 6 - Comparer la valeur obtenue à la clé mise initialement de côté. + + Étapes 1 et 2 : + F R 0 0 0 3 5 0 0 0 0 (+ 8 : clé) + 15 27 0 0 0 3 5 0 0 0 0 + + Étape 3 : le traitement se fait sur des chiffres + 1 5 2 7 0 0 0 3 5 0 0 0 0 + I P I P I P I P I P I P I : position en partant de la droite (P = Pair, I = Impair) + 2 1 2 1 2 1 2 1 2 1 2 1 2 : coefficient multiplicateur + + 2 5 4 7 0 0 0 3 10 0 0 0 0 : résultat + + Étape 4 : + 2 + 5 + 4 + 7 + 0 + 0 + 0 + 3 + (1 + 0)+ 0 + 0 + 0 + 0 = 22 + + Étapes 5 et 6 : 30 - 22 = 8 (valeur de la clé) + """ + + if not isinstance(isin, basestring): + return False + if not re.match(r'^[A-Z]{2}[A-Z0-9]{9}\d$', isin): + return False + + isin_in_digits = ''.join(str(ord(x) - ord('A') + 10) if not x.isdigit() else x for x in isin[:-1]) + key = isin[-1:] + result = '' + for k, val in enumerate(isin_in_digits[::-1], start=1): + if k % 2 == 0: + result = ''.join((result, val)) + else: + result = ''.join((result, str(int(val)*2))) + return str(sum(int(x) for x in result) + int(key))[-1] == '0' + + +def create_french_liquidity(valuation): + """ + Automatically fills a liquidity investment with label, code and code_type. + """ + liquidity = Investment() + liquidity.label = "Liquidités" + liquidity.code = "XX-liquidity" + liquidity.code_type = NotAvailable + liquidity.valuation = valuation + return liquidity + diff --git a/modules/societegenerale/module.py b/modules/societegenerale/module.py index 82d63c4f9b7dbcb3653984fe40a96989c2689f45..ff4f1bfd270099805ee281f669f9d6adb270b3e6 100644 --- a/modules/societegenerale/module.py +++ b/modules/societegenerale/module.py @@ -35,7 +35,7 @@ from weboob.tools.capabilities.bank.transactions import sorted_transactions from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import Value, ValueBackendPassword -from weboob.capabilities.base import find_object, NotAvailable +from .compat.weboob_capabilities_base import find_object, NotAvailable, strict_find_object from .browser import SocieteGenerale from .sgpe.browser import SGEnterpriseBrowser, SGProfessionalBrowser @@ -117,18 +117,16 @@ def init_transfer(self, transfer, **params): raise NotImplementedError() transfer.label = ' '.join(w for w in re.sub('[^0-9a-zA-Z ]+', '', transfer.label).split()) self.logger.info('Going to do a new transfer') - if transfer.account_iban: - account = find_object(self.iter_accounts(), iban=transfer.account_iban, error=AccountNotFound) - else: - account = find_object(self.iter_accounts(), id=transfer.account_id, error=AccountNotFound) - if transfer.recipient_iban: - recipient = find_object(self.iter_transfer_recipients(account.id), iban=transfer.recipient_iban, error=RecipientNotFound) - else: - recipient = find_object(self.iter_transfer_recipients(account.id), id=transfer.recipient_id, error=RecipientNotFound) + account = strict_find_object(self.iter_accounts(), iban=transfer.account_iban) + if not account: + account = strict_find_object(self.iter_accounts(), id=transfer.account_id, error=AccountNotFound) - transfer.amount = transfer.amount.quantize(Decimal('.01')) + recipient = strict_find_object(self.iter_transfer_recipients(account.id), iban=transfer.recipient_iban) + if not recipient: + recipient = strict_find_object(self.iter_transfer_recipients(account.id), id=transfer.recipient_id, error=RecipientNotFound) + transfer.amount = transfer.amount.quantize(Decimal('.01')) return self.browser.init_transfer(account, recipient, transfer) def execute_transfer(self, transfer, **params): diff --git a/modules/societegenerale/pages/accounts_list.py b/modules/societegenerale/pages/accounts_list.py index 12346b9094065b2ed2ce0f5622eb90abac991bb0..6610b53f47d35f8dbc4240d5f0cf839fd007126a 100644 --- a/modules/societegenerale/pages/accounts_list.py +++ b/modules/societegenerale/pages/accounts_list.py @@ -42,7 +42,6 @@ from .base import BasePage - def MyDecimal(*args, **kwargs): kwargs.update(replace_dots=True, default=NotAvailable) return CleanDecimal(*args, **kwargs) @@ -77,6 +76,13 @@ class AccountsList(LoggedPage, BasePage): 'Avance Patrimoniale': Account.TYPE_LOAN, } + def get_coming_url(self): + for script in self.doc.xpath('//script'): + s_content = CleanText('.')(script) + if "var url_encours" in s_content: + break + return re.search(r'url_encours=\"(.+)\"; ', s_content).group(1) + def get_list(self): err = CleanText('//span[@class="error_msg"]', default='')(self.doc) if err == 'Vous ne disposez pas de compte consultable.': @@ -147,12 +153,11 @@ def check_valid_url(url): # Layout with several cards line = CleanText('//table//div[contains(text(), "Liste des cartes")]', replace=[(' ', '')])(page.doc) - if line: - parent_id = re.search(r'(\d+)', line).group() - # Layout with only one card + m = re.search(r'(\d+)', line) + if m: + parent_id = m.group() else: parent_id = CleanText('//div[contains(text(), "Numéro de compte débité")]/following::div[1]', replace=[(' ', '')])(page.doc) - account.parent = find_object(accounts_list, id=parent_id) if account.type == Account.TYPE_UNKNOWN: @@ -163,6 +168,12 @@ def check_valid_url(url): return accounts_list +class ComingPage(LoggedPage, XMLPage): + def set_coming(self, accounts_list): + for a in accounts_list: + a.coming = CleanDecimal('//EnCours[contains(@id, "%s")]' % a.id, replace_dots=True, default=NotAvailable)(self.doc) + + class IbanPage(LoggedPage, NotTransferBasePage): def is_here(self): if self.is_transfer_here(): @@ -344,6 +355,9 @@ def _iter_transactions(self, doc): yield t + def get_liquidities(self): + return CleanDecimal('//td[contains(@headers, "solde")]', replace_dots=True)(self.doc) + class Invest(object): def create_investment(self, cells): @@ -685,3 +699,7 @@ class UnavailableServicePage(LoggedPage, HTMLPage): def on_load(self): if self.doc.xpath('//div[contains(@class, "erreur_404_content")]'): raise BrowserUnavailable() + + +class NewLandingPage(LoggedPage, HTMLPage): + pass diff --git a/modules/societegenerale/pages/compat/weboob_capabilities_bank.py b/modules/societegenerale/pages/compat/weboob_capabilities_bank.py new file mode 100644 index 0000000000000000000000000000000000000000..141097d781b97a933881efe1a6851a102454051e --- /dev/null +++ b/modules/societegenerale/pages/compat/weboob_capabilities_bank.py @@ -0,0 +1,45 @@ + +import weboob.capabilities.bank as OLD + +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) + + +__all__ = OLD.__all__ + + +class CapBankWealth(CapBank): + pass + + +class CapBankPockets(CapBank): + pass + + +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + +class CapBankTransfer(OLD.CapBankTransfer): + def transfer_check_label(self, old, new): + from unidecode import unidecode + + return unidecode(old) == unidecode(new) + + +class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipient): + pass + + +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + +Account.TYPE_MORTGAGE = 17 +Account.TYPE_CONSUMER_CREDIT = 18 +Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/societegenerale/pages/transfer.py b/modules/societegenerale/pages/transfer.py index 0121c7b85edeaf5b8ad573ad9579a8badadaf2a0..541c4e23371fbaf189034f0e9a99745d1d4dd8e1 100644 --- a/modules/societegenerale/pages/transfer.py +++ b/modules/societegenerale/pages/transfer.py @@ -26,9 +26,9 @@ from weboob.browser.pages import LoggedPage, JsonPage, FormNotFound from weboob.browser.elements import method, ListElement, ItemElement, SkipItem -from weboob.capabilities.bank import ( +from .compat.weboob_capabilities_bank import ( Recipient, TransferBankError, TransferInvalidCurrency, Transfer, - AddRecipientError, AddRecipientStep, + AddRecipientBankError, AddRecipientStep, ) from weboob.capabilities.base import find_object, NotAvailable, empty from weboob.browser.filters.standard import CleanText, Regexp, CleanDecimal, \ @@ -318,7 +318,7 @@ class AddRecipientPage(LoggedPage, BasePage): def on_load(self): error_msg = CleanText(u'//span[@class="error_msg"]')(self.doc) if error_msg: - raise AddRecipientError(message=error_msg) + raise AddRecipientBankError(message=error_msg) def is_here(self): return bool(CleanText(u'//h3[contains(text(), "Ajouter un compte bénéficiaire de virement")]')(self.doc)) or \ @@ -348,7 +348,7 @@ def double_auth(self, recipient): try: form = self.get_form(id='formCache') except FormNotFound: - raise AddRecipientError('form not found') + assert False, 'Double auth form not found' self.browser.context = form['context'] self.browser.dup = form['dup'] @@ -377,7 +377,7 @@ def double_auth(self, recipient): self.browser.id_transaction = r.page.doc['donnees']['id-transaction'] raise AddRecipientStep(recipient, ValueBool('pass', label=u'Valider cette opération sur votre applicaton société générale')) else: - raise AddRecipientError('sign process unknown') + assert False, 'Sign process unknown: %s' % r.page.doc['donnees']['sign_proc'] def get_recipient_object(self, recipient, get_info=False): r = Recipient() diff --git a/modules/societegenerale/sgpe/browser.py b/modules/societegenerale/sgpe/browser.py index 2cae330a1c982c410377d12e16e7da79b6a385df..81e26a10c64587e04aa833d9016481b0f6de20be 100644 --- a/modules/societegenerale/sgpe/browser.py +++ b/modules/societegenerale/sgpe/browser.py @@ -27,8 +27,8 @@ from weboob.browser.exceptions import ClientError from weboob.exceptions import BrowserIncorrectPassword from weboob.capabilities.base import find_object -from weboob.capabilities.bank import ( - AccountNotFound, RecipientNotFound, AddRecipientStep, AddRecipientError, +from .compat.weboob_capabilities_bank import ( + AccountNotFound, RecipientNotFound, AddRecipientStep, AddRecipientBankError, Recipient, TransferBankError, ) from weboob.tools.value import Value @@ -36,6 +36,7 @@ from .pages import ( LoginPage, CardsPage, CardHistoryPage, IncorrectLoginPage, ProfileProPage, ProfileEntPage, ChangePassPage, SubscriptionPage, + ErrorPage, ) from .json_pages import AccountsJsonPage, BalancesJsonPage, HistoryJsonPage, BankStatementPage from .transfer_pages import ( @@ -118,7 +119,6 @@ def get_profile(self): class SGEnterpriseBrowser(SGPEBrowser): BASEURL = 'https://entreprises.secure.societegenerale.fr' - LOGIN_FORM = 'auth' MENUID = 'BANREL' CERTHASH = '2231d5ddb97d2950d5e6fc4d986c23be4cd231c31ad530942343a8fdcc44bb99' @@ -145,10 +145,10 @@ def get_accounts_list(self): @need_login def iter_history(self, account): value = self.history.go(data={'cl500_compte': account._id, 'cl200_typeReleve': 'valeur'}).get_value() - transactions = [] - transactions.extend(self.history.go(data={'cl500_compte': account._id, 'cl200_typeReleve': value}).iter_history(value=value)) - transactions.extend(self.location('/icd/syd-front/data/syd-intraday-chargerDetail.json', data={'cl500_compte': account._id}).page.iter_history()) - return iter(transactions) + for tr in self.history.go(data={'cl500_compte': account._id, 'cl200_typeReleve': value}).iter_history(value=value): + yield tr + for tr in self.location('/icd/syd-front/data/syd-intraday-chargerDetail.json', data={'cl500_compte': account._id}).page.iter_history(): + yield tr @need_login def iter_subscription(self): @@ -176,7 +176,6 @@ def iter_documents(self, subscription): class SGProfessionalBrowser(SGEnterpriseBrowser, StatesMixin): BASEURL = 'https://professionnels.secure.societegenerale.fr' - LOGIN_FORM = 'auth_reco' MENUID = 'SBOREL' CERTHASH = '9f5232c9b2283814976608bfd5bba9d8030247f44c8493d8d205e574ea75148e' STATE_DURATION = 5 @@ -204,6 +203,8 @@ class SGProfessionalBrowser(SGEnterpriseBrowser, StatesMixin): bank_statement_menu = URL('/icd/syd-front/data/syd-rce-accederDepuisMenu.json', BankStatementPage) bank_statement_search = URL('/icd/syd-front/data/syd-rce-lancerRecherche.json', BankStatementPage) + error_page = URL('https://static.societegenerale.fr/pro/erreur.html', ErrorPage) + date_max = None date_min = None @@ -359,16 +360,14 @@ def new_recipient(self, recipient, **params): @need_login def validate_rcpt_with_sms(self, code): - if not self.new_rcpt_validate_form: - raise AddRecipientError() - + assert self.new_rcpt_validate_form, 'There should have recipient validate form in states' self.new_rcpt_validate_form.update({'code': code}) try: self.confirm_new_recipient.go(data=self.new_rcpt_validate_form) except ClientError as e: assert e.response.status_code == 403, \ 'Something went wrong in add recipient, response status code is %s' % e.response.status_code - raise AddRecipientError(message='Le code entré est incorrect.') + raise AddRecipientBankError(message='Le code entré est incorrect.') @need_login def iter_recipients(self, origin_account): diff --git a/modules/societegenerale/sgpe/compat/__init__.py b/modules/societegenerale/sgpe/compat/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/modules/societegenerale/sgpe/compat/weboob_capabilities_bank.py b/modules/societegenerale/sgpe/compat/weboob_capabilities_bank.py new file mode 100644 index 0000000000000000000000000000000000000000..141097d781b97a933881efe1a6851a102454051e --- /dev/null +++ b/modules/societegenerale/sgpe/compat/weboob_capabilities_bank.py @@ -0,0 +1,45 @@ + +import weboob.capabilities.bank as OLD + +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) + + +__all__ = OLD.__all__ + + +class CapBankWealth(CapBank): + pass + + +class CapBankPockets(CapBank): + pass + + +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + +class CapBankTransfer(OLD.CapBankTransfer): + def transfer_check_label(self, old, new): + from unidecode import unidecode + + return unidecode(old) == unidecode(new) + + +class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipient): + pass + + +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + +Account.TYPE_MORTGAGE = 17 +Account.TYPE_CONSUMER_CREDIT = 18 +Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/societegenerale/sgpe/json_pages.py b/modules/societegenerale/sgpe/json_pages.py index 20df079fa393352f1a84e5ea8e174857e22a4741..992e67110a97a295778e3c84b8467cff50cd39a0 100644 --- a/modules/societegenerale/sgpe/json_pages.py +++ b/modules/societegenerale/sgpe/json_pages.py @@ -28,7 +28,7 @@ from weboob.capabilities import NotAvailable from weboob.capabilities.bank import Account from weboob.capabilities.bill import Document, Subscription -from weboob.exceptions import BrowserUnavailable +from weboob.exceptions import BrowserUnavailable, BrowserIncorrectPassword from weboob.tools.capabilities.bank.iban import is_iban_valid from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.tools.compat import quote_plus @@ -89,7 +89,10 @@ def get_error(self): class BalancesJsonPage(LoggedPage, JsonPage): def on_load(self): if self.doc['commun']['statut'] == 'NOK': - raise BrowserUnavailable(self.doc['commun']['raison']) + reason = self.doc['commun']['raison'] + if reason == 'SYD-COMPTES-UNAUTHORIZED-ACCESS': + raise BrowserIncorrectPassword("Vous n'avez pas l'autorisation de consulter : {}".format(reason)) + raise BrowserUnavailable(reason) def populate_balances(self, accounts): for account in accounts: @@ -138,9 +141,30 @@ class item(ItemElement): obj_rdate = Date(Dict('date', default=None), dayfirst=True, default=NotAvailable) obj_date = Date(Dict('dVl', default=None), dayfirst=True, default=NotAvailable) obj__coming = False - obj_raw = Transaction.Raw(Format('%s %s %s', Dict('l1'), Dict('l2'), Dict('l3'))) - # We have l4 and l5 too most of the time, but it seems to be unimportant and would make label too long. - #tr.label = ' '.join([' '.join(transaction[l].strip().split()) for l in ['l1', 'l2', 'l3']]) + + # Label is split into l1, l2, l3, l4, l5. + # l5 is needed for transfer label, for example: + # 'l1': "000001 VIR EUROPEEN EMIS NET" + # 'l2': "POUR: XXXXXXXXXXXXX" + # 'l3': "REF: XXXXXXXXXXXXXX" + # 'l4': "REMISE: XXXXXX TRANSFER LABEL" + # 'l5': "MOTIF: TRANSFER LABEL" + obj_raw = Transaction.Raw(Format( + '%s %s %s %s %s', + Dict('l1'), + Dict('l2'), + Dict('l3'), + Dict('l4'), + Dict('l5'), + )) + + # keep the 3 first rows for transaction label + obj_label = Transaction.Raw(Format( + '%s %s %s', + Dict('l1'), + Dict('l2'), + Dict('l3'), + )) def obj_amount(self): return CleanDecimal(Dict('c', default=None), replace_dots=True, default=None)(self) or \ diff --git a/modules/societegenerale/sgpe/pages.py b/modules/societegenerale/sgpe/pages.py index b63442692e0b138e84858715844bf1295a55792a..82f8b6d8970ce9fb74876a06f785fc9179ebacc5 100644 --- a/modules/societegenerale/sgpe/pages.py +++ b/modules/societegenerale/sgpe/pages.py @@ -33,7 +33,7 @@ from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.capabilities.profile import Profile, Person from weboob.capabilities.bill import Document, Subscription -from weboob.exceptions import ActionNeeded, BrowserIncorrectPassword +from weboob.exceptions import ActionNeeded, BrowserIncorrectPassword, BrowserUnavailable from weboob.tools.json import json from weboob.capabilities.base import NotAvailable @@ -42,7 +42,7 @@ class Transaction(FrenchTransaction): - PATTERNS = [(re.compile(r'^CARTE \w+ RETRAIT DAB.* (?P
\d{2})/(?P\d{2})( (?P\d+)H(?P\d+))? (?P.*)'), + PATTERNS = [(re.compile(r'^CARTE \w+ RETRAIT DAB.*? (?P
\d{2})/(?P\d{2})( (?P\d+)H(?P\d+))? (?P.*)'), FrenchTransaction.TYPE_WITHDRAWAL), (re.compile(r'^CARTE \w+ (?P
\d{2})/(?P\d{2})( A (?P\d+)H(?P\d+))? RETRAIT DAB (?P.*)'), FrenchTransaction.TYPE_WITHDRAWAL), @@ -122,17 +122,13 @@ def get_authentication_data(self): def login(self, login, password): authentication_data = self.get_authentication_data() - form = self.get_form(name=self.browser.LOGIN_FORM) - form['user_id'] = login - form['codsec'] = authentication_data['img'].get_codes(password[:6]) - form['cryptocvcs'] = authentication_data['infos']['crypto'] - form['vk_op'] = 'auth' - form.url = '/authent.html' - try: - form.pop('button') - except KeyError: - pass - form.submit() + data = { + 'user_id': login, + 'codsec': authentication_data['img'].get_codes(password[:6]), + 'cryptocvcs': authentication_data['infos']['crypto'], + 'vk_op': 'auth', + } + self.browser.location(self.browser.absurl('/authent.html'), data=data) class CardsPage(LoggedPage, SGPEPage): @@ -256,3 +252,10 @@ class IncorrectLoginPage(SGPEPage): def on_load(self): if self.doc.xpath('//div[@class="ngo_mu_message" and contains(text(), "saisies sont incorrectes")]'): raise BrowserIncorrectPassword(CleanText('//div[@class="ngo_mu_message"]')(self.doc)) + + +class ErrorPage(SGPEPage): + def on_load(self): + if self.doc.xpath('//div[@class="ngo_mu_message" and contains(text(), "momentanément indisponible")]'): + # Warning: it could occurs because of wrongpass, user have to change password + raise BrowserUnavailable(CleanText('//div[@class="ngo_mu_message"]')(self.doc)) diff --git a/modules/spirica/compat/weboob_capabilities_bank.py b/modules/spirica/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/spirica/compat/weboob_capabilities_bank.py +++ b/modules/spirica/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/sueurdemetal/browser.py b/modules/sueurdemetal/browser.py index 7b31e2ccbcabfb3ff4ac0af545e64c727f63a840..3f765582776aaff37879a585777ceae3bd3337bc 100644 --- a/modules/sueurdemetal/browser.py +++ b/modules/sueurdemetal/browser.py @@ -19,12 +19,11 @@ from __future__ import unicode_literals -from weboob.browser import PagesBrowser, URL +from weboob.browser import URL, PagesBrowser from weboob.tools.json import json from .pages import ConcertListPage, ConcertPage, NoEvent - __all__ = ['SueurDeMetalBrowser'] @@ -64,7 +63,7 @@ def get_concert(self, id): def build_cities(self): if self.cities: - return + return self.deps.go() for dept in self.page.get_depts(): self.jlocation(self.cities.build, data={ diff --git a/modules/sueurdemetal/module.py b/modules/sueurdemetal/module.py index d080cdb67097715dd24143896047614d55e91e10..204ee96e94207e9ea74c587be5cfb9dbe26cc52c 100644 --- a/modules/sueurdemetal/module.py +++ b/modules/sueurdemetal/module.py @@ -18,12 +18,11 @@ # along with weboob. If not, see . +from weboob.capabilities.calendar import CATEGORIES, BaseCalendarEvent, CapCalendarEvent, Query from weboob.tools.backend import Module -from weboob.capabilities.calendar import CapCalendarEvent, BaseCalendarEvent, CATEGORIES, Query from .browser import SueurDeMetalBrowser - __all__ = ['SueurDeMetalModule'] diff --git a/modules/suravenir/browser.py b/modules/suravenir/browser.py index 8f725f7618fe27887c9059109a8fe45a2ec0f19f..88bbcf64d1386e22e610871b5b521a9b2d25bf68 100644 --- a/modules/suravenir/browser.py +++ b/modules/suravenir/browser.py @@ -30,10 +30,10 @@ class Suravenir(LoginBrowser): BASEURL = 'https://www.previ-direct.com/' - broker_to_instance = { 'assurancevie.com' : 'Q4n1', - 'linxea' : 'S9o6', - 'mes-placements.fr': '5yKs', - 'epargnissimo' : 'FtA1'} + broker_to_instance = {'assurancevie.com' : 'Q4n1', + 'linxea' : 'S9o6', + 'mes-placements.fr': '5yKs', + 'epargnissimo' : 'FtA1'} login_page = URL('/web/eclient-(?P.*)', LoginPage) accounts_page = URL('/group/eclient-(?P.*)/home$', AccountsList) @@ -65,7 +65,7 @@ def get_accounts_list(self): return self.page.get_contracts() def get_URI_for_account(self, account): - return '%s&_portletespaceClientmonCompte_WAR_portletespaceclient_INSTANCE_%s_tabName=detailsContrat.tabulateur.tabulation' % (account._detail_link, self.broker_to_instance[self.broker]) + return '%s&_portletespaceClientmonCompte_WAR_portletespaceclient_INSTANCE_%s_tabName=detailsContrat.tabulateur.tabulation' % (account._detail_link, self.broker_to_instance[self.broker]) @need_login def iter_investments(self, account): diff --git a/modules/suravenir/compat/weboob_capabilities_bank.py b/modules/suravenir/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/suravenir/compat/weboob_capabilities_bank.py +++ b/modules/suravenir/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/vicsec/browser.py b/modules/vicsec/browser.py index bc39d9ea5e282e1db72279e7d4825c9c7dce6646..31e366a8da8ca66b35abb2f3934f7ce43142cc3e 100644 --- a/modules/vicsec/browser.py +++ b/modules/vicsec/browser.py @@ -18,19 +18,18 @@ # along with weboob. If not, see . -from weboob.tools.capabilities.bank.transactions import \ - AmericanTransaction as AmTr -from weboob.browser import LoginBrowser, URL, need_login -from weboob.browser.pages import HTMLPage -from weboob.capabilities.base import Currency -from weboob.capabilities.shop import Order, Item, Payment -from weboob.exceptions import BrowserIncorrectPassword - +import re from datetime import datetime from decimal import Decimal from itertools import chain -import re +from weboob.browser import URL, LoginBrowser, need_login +from weboob.browser.pages import HTMLPage +from weboob.capabilities.base import Currency +from weboob.capabilities.shop import Item, Order, Payment +from weboob.exceptions import BrowserIncorrectPassword +from weboob.tools.capabilities.bank.transactions import AmericanTransaction as AmTr +from weboob.tools.compat import unicode __all__ = ['VicSec'] @@ -83,7 +82,7 @@ def order(self): def payments(self): for tr in self.doc.xpath('//tbody[@class="payment-summary"]' - '//th[text()="Payment Summary"]/../../../tbody/tr'): + '//th[text()="Payment Summary"]/../../../tbody/tr'): method = tr.xpath('td[1]/text()')[0] amnode = tr.xpath('td[2]')[0] amsign = -1 if amnode.xpath('em') else 1 @@ -99,8 +98,8 @@ def items(self): for tr in self.doc.xpath('//tbody[@class="order-items"]/tr'): label = tr.xpath('*//h1')[0].text_content().strip() price = AmTr.decimal_amount(re.match(r'^\s*([^\s]+)(\s+.*)?', - tr.xpath('*//div[@class="price"]')[0].text_content(), - re.DOTALL).group(1)) + tr.xpath('*//div[@class="price"]')[0].text_content(), + re.DOTALL).group(1)) url = 'http:' + tr.xpath('*//img/@src')[0] item = Item() item.label = unicode(label) @@ -124,9 +123,8 @@ def shipping(self): return self.payment_part(u'Shipping & Handling') def order_info(self, which): - info = self.doc.xpath('//p[@class="orderinfo"]' - )[0].text_content() - return re.match(u'.*%s:\\s+([^\\s]+)\\s'%which,info,re.DOTALL).group(1) + info = self.doc.xpath('//p[@class="orderinfo"]')[0].text_content() + return re.match(u'.*%s:\\s+([^\\s]+)\\s' % which, info, re.DOTALL).group(1) def discount(self): # Sometimes subtotal doesn't add up with items. @@ -143,7 +141,7 @@ def payment_part(self, which): # The numbers notation on VS is super wierd. # Sometimes negative amounts are represented by element. for node in self.doc.xpath('//tbody[@class="payment-summary"]' - '//td[contains(text(),"%s")]/../td[2]' % which): + '//td[contains(text(),"%s")]/../td[2]' % which): strv = node.text_content().strip() v = Decimal(0) if strv == u'FREE' else AmTr.decimal_amount(strv) return -v if node.xpath('em') and v > 0 else v diff --git a/modules/vicsec/module.py b/modules/vicsec/module.py index 8f37fe2d053090eeb4b0468e5ca873d3b58441cc..fd89306ccc479cdd3f6796445f2ded6ebe659feb 100644 --- a/modules/vicsec/module.py +++ b/modules/vicsec/module.py @@ -19,12 +19,11 @@ from weboob.capabilities.shop import CapShop -from weboob.tools.backend import Module, BackendConfig +from weboob.tools.backend import BackendConfig, Module from weboob.tools.value import ValueBackendPassword from .browser import VicSec - __all__ = ['VicSecModule'] diff --git a/modules/vicseccard/browser.py b/modules/vicseccard/browser.py index ee759a5255df8754838f947d84ca001c2214f702..0361209b027a7230d3bccab12c9e05bb382d6aa9 100644 --- a/modules/vicseccard/browser.py +++ b/modules/vicseccard/browser.py @@ -17,17 +17,17 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . -from requests.exceptions import Timeout, ConnectionError +from datetime import datetime + +from requests.exceptions import ConnectionError, Timeout -from weboob.capabilities.bank import AccountNotFound, Account, Transaction -from weboob.tools.capabilities.bank.transactions import \ - AmericanTransaction as AmTr -from weboob.browser import LoginBrowser, URL, need_login +from weboob.browser import URL, LoginBrowser, need_login from weboob.browser.exceptions import ServerError from weboob.browser.pages import HTMLPage +from weboob.capabilities.bank import Account, AccountNotFound, Transaction from weboob.exceptions import BrowserIncorrectPassword -from datetime import datetime - +from weboob.tools.capabilities.bank.transactions import AmericanTransaction as AmTr +from weboob.tools.compat import unicode __all__ = ['VicSecCard'] @@ -52,13 +52,14 @@ def login(self, username, password): class HomePage(SomePage): def account(self): id_ = self.doc.xpath(u'//strong[contains(text(),' - '"Credit Card account ending in")]/text()')[0].strip()[-4:] - balance = self.doc.xpath(u'//span[@class="description" and text()="Current Balance"]/../span[@class="total"]/text()')[0].strip() + u'"Credit Card account ending in")]/text()')[0].strip()[-4:] + balance = self.doc.xpath( + u'//span[@class="description" and text()="Current Balance"]/../span[@class="total"]/text()')[0].strip() cardlimit = self.doc.xpath(u'//span[contains(text(),"Credit limit")]' - '/text()')[0].split()[-1] + u'/text()')[0].split()[-1] paymin = self.doc.xpath(u'//section[@id=" account_summary"]' - '//strong[text()="Minimum Payment Due"]/../../span[2]/text()' - )[0].strip() + u'//strong[text()="Minimum Payment Due"]/../../span[2]/text()' + )[0].strip() a = Account() a.id = id_ a.label = u'ACCOUNT ENDING IN %s' % id_ @@ -67,9 +68,9 @@ def account(self): a.type = Account.TYPE_CARD a.cardlimit = AmTr.decimal_amount(cardlimit) a.paymin = AmTr.decimal_amount(paymin) - #TODO: Add paydate. - #Oleg: I don't have an account with scheduled payment. - # Need to wait for a while... + # TODO: Add paydate. + # Oleg: I don't have an account with scheduled payment. + # Need to wait for a while... return a @@ -121,9 +122,9 @@ def do_login(self): raise BrowserIncorrectPassword() def location(self, *args, **kwargs): - for i in xrange(self.MAX_RETRIES): + for i in range(self.MAX_RETRIES): try: return super(VicSecCard, self).location(*args, **kwargs) except (ServerError, Timeout, ConnectionError) as e: - pass - raise e + last_error = e + raise last_error diff --git a/modules/vicseccard/module.py b/modules/vicseccard/module.py index f3fa74c8cd5194e91fd064c9bb2ceea5557c457e..19c992d152ce10cfb978b1a266ebdc6b28e27886 100644 --- a/modules/vicseccard/module.py +++ b/modules/vicseccard/module.py @@ -18,12 +18,11 @@ # along with weboob. If not, see . from weboob.capabilities.bank import CapBank -from weboob.tools.backend import Module, BackendConfig +from weboob.tools.backend import BackendConfig, Module from weboob.tools.value import ValueBackendPassword from .browser import VicSecCard - __all__ = ['VicSecCardModule'] diff --git a/modules/vimeo/pages.py b/modules/vimeo/pages.py index 77854cc4d973dddf2dd0fcab95b03737f23e5058..fd3dfbb2e910ba299e7c85d5357279ae9013bd29 100644 --- a/modules/vimeo/pages.py +++ b/modules/vimeo/pages.py @@ -149,6 +149,7 @@ class VimeoItem(ItemElement): obj_date = DateTime(CleanText('./upload_date')) obj__is_hd = CleanText('./@is_hd') obj_duration = VimeoDuration(CleanText('./duration')) + obj_ext = u'mp4' def obj_thumbnail(self): t = CleanText('./thumbnails/thumbnail[1]', default='')(self) diff --git a/modules/vlille/browser.py b/modules/vlille/browser.py index 43d4e82d4d59bc48645a7aa1705ba7a941480747..9297a53cf33d295581b0915b1d7d754f11579112 100644 --- a/modules/vlille/browser.py +++ b/modules/vlille/browser.py @@ -29,7 +29,7 @@ class VlilleBrowser(PagesBrowser): BASEURL = 'https://www.transpole.fr' - list_page = URL('/cms/institutionnel/vlille-carto/', ListStationsPage) + list_page = URL('/cms/vlille/les-stations-cartographies/', ListStationsPage) def get_station_list(self): return self.list_page.go().get_station_list() diff --git a/modules/wellsfargo/browser.py b/modules/wellsfargo/browser.py index 0834d0a694f2c1a7d48e1b8461596524e9a9a769..0d31e511d55a7f69cfd342ab2f0a2f00a0e142ff 100644 --- a/modules/wellsfargo/browser.py +++ b/modules/wellsfargo/browser.py @@ -18,23 +18,20 @@ # along with weboob. If not, see . -from weboob.capabilities.bank import AccountNotFound -from weboob.browser import LoginBrowser, URL, need_login -from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable -from weboob.tools.compat import unquote - -import ssl import json import os -from tempfile import mkstemp +import ssl from subprocess import STDOUT, CalledProcessError, check_output +from tempfile import mkstemp -from .pages import LoginProceedPage, LoginRedirectPage, \ - SummaryPage, ActivityCashPage, ActivityCardPage, \ - DocumentsPage, StatementPage, StatementsPage, \ - StatementsEmbeddedPage, LoggedInPage, CodeRequestPage, \ - CodeSubmitPage +from weboob.browser import URL, LoginBrowser, need_login +from weboob.capabilities.bank import AccountNotFound +from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable +from weboob.tools.compat import unquote +from .pages import (ActivityCardPage, ActivityCashPage, CodeRequestPage, CodeSubmitPage, DocumentsPage, LoggedInPage, + LoginProceedPage, LoginRedirectPage, StatementPage, StatementsEmbeddedPage, StatementsPage, + SummaryPage) __all__ = ['WellsFargo'] @@ -63,14 +60,14 @@ class WellsFargo(LoginBrowser): documents = URL('https://connect.secure.wellsfargo.com' '/accounts/start\?.+$', DocumentsPage) statements_embedded = URL('https://connect.secure.wellsfargo.com' - '/accounts/start\?.+$', StatementsEmbeddedPage) + '/accounts/start\?.+$', StatementsEmbeddedPage) statements = URL('https://connect.secure.wellsfargo.com' '/accounts/documents/statement/list.+$', StatementsPage) statement = URL('https://connect.secure.wellsfargo.com' '/accounts/documents/retrieve/.+$', StatementPage) - unknown = URL('/.*$', LoggedInPage) # E.g. random advertisement pages. + unknown = URL('/.*$', LoggedInPage) # e.g. random advertisement pages. def __init__(self, question1, answer1, question2, answer2, question3, answer3, phone_last4, code_file, *args, **kwargs): @@ -90,7 +87,7 @@ def do_login(self): which uses DOM. For now the easiest option seems to be to run it in PhantomJs. ''' - for i in xrange(self.MAX_RETRIES): + for i in range(self.MAX_RETRIES): scrf, scrn = mkstemp('.js') cookf, cookn = mkstemp('.json') os.write(scrf, LOGIN_JS % { @@ -118,10 +115,10 @@ def do_login(self): os.remove(cookn) self.session.cookies.clear() for c in cookies: - for k in ['expiry', 'expires', 'httponly']: - c.pop(k, None) - c['value'] = unquote(c['value']) - self.session.cookies.set(**c) + for k in ['expiry', 'expires', 'httponly']: + c.pop(k, None) + c['value'] = unquote(c['value']) + self.session.cookies.set(**c) self.summary.go() if self.page.logged: break @@ -190,7 +187,7 @@ def to_activity(self, id_=None): @need_login def to_statements(self, id_=None, year=None): if not self.statements.is_here() \ - and not self.statements_embedded.is_here(): + and not self.statements_embedded.is_here(): self.to_summary() self.page.to_documents() if self.documents.is_here(): @@ -209,7 +206,7 @@ def to_statements(self, id_=None, year=None): @need_login def to_statement(self, uri): - for i in xrange(self.MAX_RETRIES): + for i in range(self.MAX_RETRIES): self.location(uri) if self.statement.is_here(): break @@ -242,6 +239,7 @@ def iter_history(self, account): for trans in self.page.iter_transactions(): yield trans + LOGIN_JS = u'''\ var page = require('webpage').create(); diff --git a/modules/wellsfargo/pages.py b/modules/wellsfargo/pages.py index 0d327c41db7a9ab531fe90bc7a617fd5084cbc8b..02884c534c31939b390e36c0fb3ff7f6788cd685 100644 --- a/modules/wellsfargo/pages.py +++ b/modules/wellsfargo/pages.py @@ -17,20 +17,29 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . -from weboob.capabilities.bank import Account, Transaction -from weboob.tools.capabilities.bank.transactions import \ - AmericanTransaction as AmTr -from weboob.browser.pages import HTMLPage, LoggedPage, RawPage -from decimal import Decimal -from requests.cookies import morsel_to_cookie -from .parsers import StatementParser, clean_label -from time import sleep +import Cookie +import datetime import itertools import json -import re import os -import datetime -import Cookie +import re +from decimal import Decimal +from time import sleep + +from requests.cookies import morsel_to_cookie + +from weboob.browser.pages import HTMLPage, LoggedPage, RawPage +from weboob.capabilities.bank import Account, Transaction +from weboob.tools.capabilities.bank.transactions import AmericanTransaction as AmTr + +from .parsers import StatementParser, clean_label + + +try: + cmp = cmp +except NameError: + def cmp(x, y): + return (x > y) - (x < y) class LoginProceedPage(LoggedPage, HTMLPage): @@ -98,7 +107,7 @@ def submit_code(self): except IOError: sleep(1) os.remove(self.browser.code_file) - self.browser.logger.info('The code %s has been successfully read'%code) + self.browser.logger.info('The code %s has been successfully read' % code) form = self.get_form(name='otp') form['passcode'] = [code] del form['cancelBtn'] @@ -122,7 +131,7 @@ def to_documents(self): class AccountPage(object): def account_id(self, name=None): if name: - return name[-4:] # Last 4 digits of "BLAH XXXXXXX1234" + return name[-4:] # Last 4 digits of "BLAH XXXXXXX1234" else: return self.account_id(self.account_name()) @@ -391,7 +400,7 @@ def is_here(self): def get_embedded_data(self): scr = self.doc.xpath(self.SCRIPT_XPATH)[0] data = json.loads('\n'.join(scr.split('\n')[2:-2]).replace( - "'appendTo'",'"appendTo"')) + "'appendTo'", '"appendTo"')) return json.loads(data['data']) def parser(self): @@ -401,8 +410,8 @@ def parser(self): class WfJsonPage(LoggedPage, RawPage): def __init__(self, *args, **kwArgs): RawPage.__init__(self, *args, **kwArgs) - clean = self.doc.replace('"/*WellFargoProprietary%','') \ - .replace('%WellFargoProprietary*/"','').decode('string_escape') + clean = self.doc.replace('"/*WellFargoProprietary%', '') \ + .replace('%WellFargoProprietary*/"', '').decode('string_escape') self.doc = json.loads(clean) diff --git a/modules/wellsfargo/parsers.py b/modules/wellsfargo/parsers.py index 19e7159f8ca91a4b65485eca82d67f44335ca910..ad229caf397b324390d59de8b6aa06f55103bd28 100644 --- a/modules/wellsfargo/parsers.py +++ b/modules/wellsfargo/parsers.py @@ -23,6 +23,7 @@ from weboob.tools.date import closest_date from weboob.tools.pdf import decompress_pdf from weboob.tools.tokenizer import ReTokenizer +from weboob.tools.compat import unicode import re import datetime @@ -124,7 +125,7 @@ def read_card_transaction(self, pos, date_from, date_to): range_plus = (0, INDENT_CHARGES)) if tdate is None or pdate_layout is None or pdate is None \ - or ref_layout is None or ref is None or desc is None or amount is None: + or ref_layout is None or ref is None or desc is None or amount is None: return startPos, None else: tdate = closest_date(tdate, date_from, date_to) @@ -262,19 +263,19 @@ def read_star_2(self, pos): def read_date(self, pos): def parse_date(v): - for year in [1900, 1904]: # try leap and non-leap years + for year in [1900, 1904]: # try leap and non-leap years fullstr = '%s/%i' % (v, year) try: return datetime.datetime.strptime(fullstr, '%m/%d/%Y') except ValueError as e: - pass - raise e + last_error = e + raise last_error return self._tok.simple_read('date', pos, parse_date) def read_text(self, pos): t = self._tok.tok(pos) - #TODO: handle PDF encodings properly. + # TODO: handle PDF encodings properly. return (pos+1, unicode(t.value(), errors='ignore')) \ if t.is_text() else (pos, None) diff --git a/modules/yomoni/compat/weboob_capabilities_bank.py b/modules/yomoni/compat/weboob_capabilities_bank.py index 404e8d30ed28683c8a27f183d1e670c10509ce56..141097d781b97a933881efe1a6851a102454051e 100644 --- a/modules/yomoni/compat/weboob_capabilities_bank.py +++ b/modules/yomoni/compat/weboob_capabilities_bank.py @@ -35,6 +35,11 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie pass +class AddRecipientBankError(AddRecipientError): + code = 'bankMessage' + + + Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19