From dcb73fa4846935e935c278c53b61e041ec5010af Mon Sep 17 00:00:00 2001 From: Romain Bignon Date: Thu, 6 Dec 2018 09:21:03 +0100 Subject: [PATCH] backport fixes from unstable --- modules/banquepopulaire/pages.py | 2 +- modules/barclays/pages.py | 13 +-- modules/bnporc/pp/browser.py | 6 +- modules/bnporc/pp/pages.py | 9 +- modules/boursorama/browser.py | 3 +- modules/boursorama/pages.py | 10 +-- modules/bp/browser.py | 4 +- modules/caissedepargne/browser.py | 5 ++ modules/caissedepargne/pages.py | 2 +- modules/cmso/pro/browser.py | 14 +-- modules/cmso/pro/pages.py | 3 - modules/cragr/web/pages.py | 6 ++ modules/creditmutuel/pages.py | 12 ++- modules/fortuneo/pages/transfer.py | 16 ++-- modules/hsbc/browser.py | 21 ++++- modules/hsbc/pages/account_pages.py | 32 ++++++- modules/hsbc/pages/investments.py | 5 +- modules/lcl/module.py | 2 +- modules/linebourse/browser.py | 19 +++- modules/linebourse/compat/__init__.py | 0 ...oob_tools_capabilities_bank_investments.py | 86 +++++++++++++++++++ modules/linebourse/pages.py | 11 ++- .../sgpe/compat/weboob_exceptions.py | 46 ++++++++++ modules/societegenerale/sgpe/json_pages.py | 5 +- 24 files changed, 273 insertions(+), 59 deletions(-) create mode 100644 modules/linebourse/compat/__init__.py create mode 100644 modules/linebourse/compat/weboob_tools_capabilities_bank_investments.py create mode 100644 modules/societegenerale/sgpe/compat/weboob_exceptions.py diff --git a/modules/banquepopulaire/pages.py b/modules/banquepopulaire/pages.py index 774fe27466..4f9e422f66 100644 --- a/modules/banquepopulaire/pages.py +++ b/modules/banquepopulaire/pages.py @@ -278,7 +278,7 @@ def on_load(self): if "est indisponible" in h1: raise BrowserUnavailable(h1) body = CleanText(".")(self.doc) - if "An unexpected error has occurred." in body: + if "An unexpected error has occurred." in body or "Une erreur s'est produite" in body: raise BrowserUnavailable(body) a = Link('//a[@class="btn"][1]', default=None)(self.doc) diff --git a/modules/barclays/pages.py b/modules/barclays/pages.py index de5e410f85..61ee0273b7 100644 --- a/modules/barclays/pages.py +++ b/modules/barclays/pages.py @@ -394,12 +394,13 @@ def has_iban(self): return False def do_account_attachment(self, accounts): - caccount_aid = Regexp(CleanText('//span[@id="C4__QUE_B160DC66D26AA39615599"]'), r'-(.*?)-')(self.doc) - - for account in accounts: - if account.id == re.sub(r'\s', '', caccount_aid): - return account - + caccount_aid = CleanText('//span[@id="C4__QUE_B160DC66D26AA39615599"]')(self.doc) + m = re.search('-(.*?)-', caccount_aid) + if m: + regex = m.group(1) + for account in accounts: + if account.id == re.sub(r'\s', '', regex): + return account return NotAvailable def has_history(self): diff --git a/modules/bnporc/pp/browser.py b/modules/bnporc/pp/browser.py index a9b0cd13ac..3e181dcf84 100644 --- a/modules/bnporc/pp/browser.py +++ b/modules/bnporc/pp/browser.py @@ -118,7 +118,7 @@ def __init__(self, config, *args, **kwargs): super(BNPParibasBrowser, self).__init__(config['login'].get(), config['password'].get(), *args, **kwargs) self.accounts_list = None self.card_to_transaction_type = {} - self.rotating_password = config['rotating_password'] + self.rotating_password = config['rotating_password'].get() @retry(ConnectionError, tries=3) def open(self, *args, **kwargs): @@ -372,10 +372,8 @@ def init_transfer(self, account, recipient, amount, reason, exec_date): @need_login def send_code(self, recipient, **params): - # depending on whether recipient is a weboob or a budgea backend object. - _id = recipient.webid if hasattr(recipient, 'webid') else recipient.id data = {} - data['idBeneficiaire'] = _id + data['idBeneficiaire'] = recipient.id data['typeActivation'] = 1 data['codeActivation'] = params['code'] return self.activate_recip.go(data=json.dumps(data), headers={'Content-Type': 'application/json'}).get_recipient(recipient) diff --git a/modules/bnporc/pp/pages.py b/modules/bnporc/pp/pages.py index 5ce42ff02e..27852a2664 100644 --- a/modules/bnporc/pp/pages.py +++ b/modules/bnporc/pp/pages.py @@ -205,15 +205,16 @@ def on_load(self): if not msg: msg = self.get('message') - wrongpass_codes = [201, 21510, 203, 202, 1001, 7] + wrongpass_codes = [201, 21510, 203, 202, 7] actionNeeded_codes = [21501, 3, 4, 50] + websiteUnavailable_codes = [207, 1001] if error in wrongpass_codes: raise BrowserIncorrectPassword(msg) elif error == 21: # "Ce service est momentanément indisponible. Veuillez renouveler votre demande ultérieurement." -> In reality, account is blocked because of too much wrongpass raise ActionNeeded(u"Compte bloqué") elif error in actionNeeded_codes: raise ActionNeeded(msg) - elif error == 207: + elif error in websiteUnavailable_codes: raise BrowserUnavailable(msg) else: assert False, 'Unexpected error at login: "%s" (code=%s)' % (msg, error) @@ -461,7 +462,7 @@ def handle_response(self, transfer): transfer_data = self.doc['data']['enregistrementVirement'] transfer.id = transfer_data['reference'] - assert transfer.exec_date == parse_french_date(self.doc['data']['enregistrementVirement']['dateExecution']).date() + transfer.exec_date = parse_french_date(self.doc['data']['enregistrementVirement']['dateExecution']).date() # Timestamp at which the bank registered the transfer register_date = re.sub(' 24:', ' 00:', self.doc['data']['enregistrementVirement']['dateEnregistrement']) transfer._register_date = parse_french_date(register_date) @@ -828,7 +829,7 @@ class ActivateRecipPage(AddRecipPage): def get_recipient(self, recipient): r = Recipient() r.iban = recipient.iban - r.id = recipient.webid if hasattr(recipient, 'webid') else recipient.id + r.id = recipient.id r.label = recipient.label r.category = u'Externe' r.enabled_at = datetime.now().replace(microsecond=0) + timedelta(days=5) diff --git a/modules/boursorama/browser.py b/modules/boursorama/browser.py index 7d2dc3707e..04a715e824 100644 --- a/modules/boursorama/browser.py +++ b/modules/boursorama/browser.py @@ -89,6 +89,7 @@ class BoursoramaBrowser(RetryLoginBrowser, StatesMixin): incident = URL('/compte/cav/(?P.*)/mes-incidents.*', IncidentPage) transfer_accounts = URL(r'/compte/(?P[^/]+)/(?P\w+)/virements/nouveau/(?P\w+)/1', + r'/compte/(?P[^/]+)/(?P\w+)/virements/nouveau$', TransferAccounts) recipients_page = URL(r'/compte/(?P[^/]+)/(?P\w+)/virements/$', r'/compte/(?P[^/]+)/(?P\w+)/virements/nouveau/(?P\w+)/2', @@ -357,7 +358,7 @@ def get_regular_transactions(self, account, coming): @retry_on_logout() @need_login def get_history(self, account, coming=False): - if account.type is Account.TYPE_LOAN or '/compte/derive' in account.url: + if account.type in (Account.TYPE_LOAN, Account.TYPE_CONSUMER_CREDIT) or '/compte/derive' in account.url: return [] if account.type is Account.TYPE_SAVINGS and u"PLAN D'\xc9PARGNE POPULAIRE" in account.label: return [] diff --git a/modules/boursorama/pages.py b/modules/boursorama/pages.py index 8c08e0451e..a28b722db6 100644 --- a/modules/boursorama/pages.py +++ b/modules/boursorama/pages.py @@ -242,7 +242,7 @@ class item(ItemElement): def condition(self): # Ignore externally aggregated accounts and insurances: - return not self.is_external() and not any(x in Field('url')(self) for x in ('automobile', 'assurance/protection', 'assurance/comptes')) + return not self.is_external() and not any(x in Field('url')(self) for x in ('automobile', 'assurance/protection', 'assurance/comptes', 'assurance/famille')) obj_label = CleanText('.//a[has-class("account--name")] | .//div[has-class("account--name")]') @@ -343,13 +343,13 @@ class get_loan(ItemElement): obj_label = CleanText('//h2[contains(@class, "page-title__account")]//div[@class="account-edit-label"]/span') obj_total_amount = CleanDecimal('//p[contains(text(), "Montant emprunt")]/span', replace_dots=True) obj_currency = CleanCurrency('//p[contains(text(), "Montant emprunt")]/span') - obj_duration = CleanDecimal('//p[contains(text(), "Nombre prévisionnel d\'échéances restantes")]/span') + obj_duration = CleanDecimal('//p[contains(text(), "Nombre prévisionnel d\'échéances restantes")]/span', default=NotAvailable) obj_rate = CleanDecimal('//p[contains(text(), "Taux nominal en vigueur du prêt")]/span') - obj_nb_payments_left = CleanDecimal('//p[contains(text(), "Nombre prévisionnel d\'échéances restantes")]/span') - obj_next_payment_amount = CleanDecimal('//p[contains(text(), "Montant de la prochaine échéance")]/span', replace_dots=True) + obj_nb_payments_left = CleanDecimal('//p[contains(text(), "Nombre prévisionnel d\'échéances restantes")]/span', default=NotAvailable) + obj_next_payment_amount = CleanDecimal('//p[contains(text(), "Montant de la prochaine échéance")]/span', replace_dots=True, default=NotAvailable) obj_nb_payments_total = CleanDecimal('//p[contains(text(), "Nombre d\'écheances totales") or contains(text(), "Nombre total d\'échéances")]/span') obj_subscription_date = Date(CleanText('//p[contains(text(), "Date de départ du prêt")]/span'), parse_func=parse_french_date) - obj_maturity_date = Date(CleanText('//p[contains(text(), "Date prévisionnelle d\'échéance finale")]/span'), parse_func=parse_french_date) + obj_maturity_date = Date(CleanText('//p[contains(text(), "Date prévisionnelle d\'échéance finale")]/span'), parse_func=parse_french_date, default=NotAvailable) def obj_balance(self): balance = CleanDecimal('//p[contains(text(), "Capital restant dû")]/span', replace_dots=True)(self) diff --git a/modules/bp/browser.py b/modules/bp/browser.py index 3033866d3f..73361b9fdf 100644 --- a/modules/bp/browser.py +++ b/modules/bp/browser.py @@ -398,7 +398,9 @@ def iter_investment(self, account): if account.type in (account.TYPE_PEA, account.TYPE_MARKET): self.go_linebourse(account) - return self.linebourse.iter_investment(account.id) + investments = list(self.linebourse.iter_investment(account.id)) + investments.append(self.linebourse.get_liquidity(account.id)) + return investments if account.type != Account.TYPE_LIFE_INSURANCE: return iter([]) diff --git a/modules/caissedepargne/browser.py b/modules/caissedepargne/browser.py index fc2e589396..046b219453 100644 --- a/modules/caissedepargne/browser.py +++ b/modules/caissedepargne/browser.py @@ -246,6 +246,11 @@ def do_login(self): self.multi_type = True if self.inexttype < len(data['account']): + if data['account'][self.inexttype] == 'EU' and not self.nuser: + # when EU is present and not alone, it tends to come first + # if nuser is unset though, user probably doesn't want 'EU' + self.inexttype += 1 + self.typeAccount = data['account'][self.inexttype] else: assert False, 'should have logged in with at least one connection type' diff --git a/modules/caissedepargne/pages.py b/modules/caissedepargne/pages.py index 683c4bf537..4039385d4f 100644 --- a/modules/caissedepargne/pages.py +++ b/modules/caissedepargne/pages.py @@ -996,7 +996,7 @@ def get_origin_account_value(self, account): def get_recipient_value(self, recipient): if recipient.category == u'Externe': recipient_value = [Attr('.', 'value')(o) for o in self.doc.xpath(self.RECIPIENT_XPATH) if - Regexp(CleanText('.'), ' - (.*) -', default=NotAvailable)(o) == recipient.iban] + Regexp(CleanText('.'), '.* - ([A-Za-z0-9]*) -', default=NotAvailable)(o) == recipient.iban] elif recipient.category == u'Interne': recipient_value = [Attr('.', 'value')(o) for o in self.doc.xpath(self.RECIPIENT_XPATH) if Regexp(CleanText('.'), '- (\d+)', default=NotAvailable)(o) and Regexp(CleanText('.'), '- (\d+)', default=NotAvailable)(o) in recipient.id] diff --git a/modules/cmso/pro/browser.py b/modules/cmso/pro/browser.py index 9d50d66039..534ba0e22c 100644 --- a/modules/cmso/pro/browser.py +++ b/modules/cmso/pro/browser.py @@ -33,7 +33,7 @@ from .pages import ( LoginPage, PasswordCreationPage, AccountsPage, HistoryPage, ChoiceLinkPage, SubscriptionPage, InvestmentPage, - InvestmentAccountPage, UselessPage, TokenPage, SSODomiPage, AuthCheckUser, SecurityCheckUser, + InvestmentAccountPage, UselessPage, TokenPage, SSODomiPage, AuthCheckUser, ) from ..par.pages import ProfilePage @@ -57,7 +57,6 @@ class CmsoProBrowser(LoginBrowser): tokens = URL('/domiweb/prive/espacesegment/selectionnerAbonnement/3-selectionnerAbonnement.act', TokenPage) ssoDomiweb = URL('https://pro.(?P[\w.]+)/domiapi/oauth/json/ssoDomiwebEmbedded', SSODomiPage) auth_checkuser = URL('https://pro.(?P[\w.]+)/auth/checkuser', AuthCheckUser) - security_checkuser = URL('https://pro.(?P[\w.]+)/securityapi/checkuser', SecurityCheckUser) def __init__(self, website, *args, **kwargs): super(CmsoProBrowser, self).__init__(*args, **kwargs) @@ -148,17 +147,6 @@ def go_on_area(self, area): self.location(area) self.location('/domiweb/accueil.jsp') self.auth_checkuser.go(website=self.website) - self.security_checkuser.go( - website=self.website, - json={'appOrigin': 'domiweb', 'espaceApplication': 'PRO'}, - headers={'Authentication': 'Bearer %s' % self.token, - 'Authorization': 'Bearer %s' % self.csrf, - 'X-Csrf-Token': self.csrf, - 'Accept': 'application/json', - 'X-REFERER-TOKEN': 'RWDPRO', - 'X-ARKEA-EFS': self.arkea, - 'ADRIM': 'isAjax:true', - }) @need_login def iter_accounts(self): diff --git a/modules/cmso/pro/pages.py b/modules/cmso/pro/pages.py index 85eb8185a2..145bb7f620 100644 --- a/modules/cmso/pro/pages.py +++ b/modules/cmso/pro/pages.py @@ -292,6 +292,3 @@ def on_load(self): class AuthCheckUser(HTMLPage): pass - -class SecurityCheckUser(JsonPage): - pass diff --git a/modules/cragr/web/pages.py b/modules/cragr/web/pages.py index fc60058607..c309eebe18 100644 --- a/modules/cragr/web/pages.py +++ b/modules/cragr/web/pages.py @@ -206,6 +206,7 @@ class AccountsPage(MyLoggedPage, BasePage): u'EKO' : Account.TYPE_CHECKING, u'DAV NANTI': Account.TYPE_SAVINGS, u'LIV A': Account.TYPE_SAVINGS, + u'LIV A ASS': Account.TYPE_SAVINGS, u'LDD': Account.TYPE_SAVINGS, u'PEL': Account.TYPE_SAVINGS, u'CEL': Account.TYPE_SAVINGS, @@ -989,6 +990,11 @@ class MarketHomePage(MarketPage): COL_ID_LABEL = 1 COL_VALUATION = 5 + def on_load(self): + action_needed_msg = CleanText('//div[contains(text(), "Afin de finaliser le paramétrage de votre environnement Bourse")]', replace=[('Enregistrer', '')])(self.doc) + if action_needed_msg: + raise ActionNeeded(action_needed_msg) + @method class get_list(TableElement): item_xpath = '//table[has-class("tableau_comptes_details")]//tr[td[2]]' diff --git a/modules/creditmutuel/pages.py b/modules/creditmutuel/pages.py index d1df2f2507..b20ec79ab4 100644 --- a/modules/creditmutuel/pages.py +++ b/modules/creditmutuel/pages.py @@ -613,8 +613,11 @@ def parse(self, el): page = page.browser.open(Link('//form//a[text()="Contrat"]', default=None)(page.doc)).page xpath = '//table[has-class("liste")]/tbody/tr' active_card = CleanText('%s[td[text()="Active"]][1]/td[2]' % xpath, replace=[(' ', '')], default=None)(page.doc) + for cards in page.doc.xpath(xpath): + if CleanText(cards.xpath('./td[1]'))(self) != 'Active': + self.page.browser.unavailablecards.append(CleanText(cards.xpath('./td[2]'), replace=[(' ', '')])(self)) - if not active_card or len(page.doc.xpath(xpath)) != 1: + if not active_card and len(page.doc.xpath(xpath)) != 1: raise SkipItem() self.env['id'] = active_card or CleanText('%s[1]/td[2]' % xpath, replace=[(' ', '')])(page.doc) @@ -1835,9 +1838,14 @@ def is_last_page(self): class NewCardsListPage(LoggedPage, HTMLPage): + @pagination @method class iter_accounts(ListElement): item_xpath = '//li[@class="item"]' + def next_page(self): + other_cards = self.el.xpath('//span/a[contains(text(), "Autres cartes")]') + if other_cards: + return Link(other_cards)(self) class item(ItemElement): klass = Account @@ -1899,6 +1907,8 @@ def parse(self, el): raise SkipItem() elif doc.xpath('//div/p[contains(text(), "Vous n\'avez pas l\'autorisation")]'): self.logger.warning("The user can't reach this page") + elif doc.xpath('//td[contains(text(), "Problème technique")]'): + raise BrowserUnavailable(CleanText(doc.xpath('//td[contains(text(), "Problème technique")]'))(self)) else: assert False, 'xpath for card type information could have changed' diff --git a/modules/fortuneo/pages/transfer.py b/modules/fortuneo/pages/transfer.py index e80c8f21e1..a2307542a3 100644 --- a/modules/fortuneo/pages/transfer.py +++ b/modules/fortuneo/pages/transfer.py @@ -183,16 +183,22 @@ def fill_transfer_form(self, account, recipient, amount, label, exec_date): form['typeDeVirement'] = 'VI' form['dateDeVirement'] = exec_date.strftime('%d/%m/%Y') form['montantVirement'] = amount - form['libelleVirementSaisie'] = label + form['libelleVirementSaisie'] = label.encode(self.encoding, errors='xmlcharrefreplace').decode(self.encoding) form.submit() class ValidateTransferPage(LoggedPage, HTMLPage): def on_load(self): - if self.doc.xpath('//form[@id="SaisieVirementForm"]/p[has-class("error")]'): - raise TransferBankError(CleanText( - '//form[@id="SaisieVirementForm"]/p[has-class("error")]/label' - )(self.doc)) + errors_msg = ( + CleanText('//form[@id="SaisieVirementForm"]/p[has-class("error")]/label')(self.doc), + CleanText('//div[@id="error" and @class="erreur_texte"]/p[contains(text(), "n\'est pas autorisé")]')(self.doc), + ) + for error in errors_msg: + if error: + raise TransferBankError(error) + + other_error_msg = self.doc.xpath('//div[@id="error" and @class="erreur_texte"]') + assert not other_error_msg, 'Error "other_error_msg" is not handled yet' def check_transfer_data(self, transfer_data): for t_data in transfer_data: diff --git a/modules/hsbc/browser.py b/modules/hsbc/browser.py index b3e8d1a3ff..41f4f1e34f 100644 --- a/modules/hsbc/browser.py +++ b/modules/hsbc/browser.py @@ -36,7 +36,7 @@ from .pages.account_pages import ( AccountsPage, OwnersListPage, CBOperationPage, CPTOperationPage, LoginPage, - AppGonePage, RibPage, UnavailablePage, OtherPage, FrameContainer, ProfilePage, + AppGonePage, RibPage, UnavailablePage, OtherPage, FrameContainer, ProfilePage, ScpiHisPage ) from .pages.life_insurances import ( LifeInsurancesPage, LifeInsurancePortal, LifeInsuranceMain, LifeInsuranceUseless, @@ -58,6 +58,7 @@ class HSBC(LoginBrowser): app_gone = False scpi_investment_page = URL(r'https://www.hsbc.fr/1/[0-9]/.*', ScpiInvestmentPage) + scpi_his_page = URL(r'https://www.hsbc.fr/1/[0-9]/.*', ScpiHisPage) connection = URL(r'https://www.hsbc.fr/1/2/hsbc-france/particuliers/connexion', LoginPage) login = URL(r'https://www.hsbc.fr/1/*', LoginPage) cptPage = URL(r'/cgi-bin/emcgi.*\&Cpt=.*', @@ -157,11 +158,12 @@ def do_login(self): if self.login.is_here(): self.page.useless_form() - # This shitty website has 2 baseurl with only one difference: the 's' at the end of 'client' + # This wonderful website has 2 baseurl with only one difference: the 's' at the end of 'client' new_base_url = 'https://clients.hsbc.fr/' if new_base_url in self.url: self.BASEURL = new_base_url + home_url = None if self.frame_page.is_here(): home_url = self.page.get_frame() self.js_url = self.page.get_js_url() @@ -324,6 +326,21 @@ def get_history(self, account, coming=False, retry_li=True): if account.url.startswith('javascript') or '&Crd=' in account.url or account.type == Account.TYPE_LOAN: raise NotImplementedError() + if account.type == Account.TYPE_MARKET and not 'BOURSE_INV' in account.url: + # Clean account url + m = re.search(r"'(.*)'", account.url) + if m: + account_url = m.group(1) + else: + account_url = account.url + # Need to be on accounts page to go on scpi page + self.accounts.go() + # Go on scpi page + self.location(account_url) + self.location(self.page.go_scpi_his_detail_page()) + + return self.page.iter_history() + if account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_CAPITALISATION): if coming is True: return [] diff --git a/modules/hsbc/pages/account_pages.py b/modules/hsbc/pages/account_pages.py index 089ba1ce94..fd822841a4 100644 --- a/modules/hsbc/pages/account_pages.py +++ b/modules/hsbc/pages/account_pages.py @@ -26,10 +26,10 @@ from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.tools.compat import urljoin from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable, ActionNeeded -from weboob.browser.elements import ListElement, ItemElement, method -from weboob.browser.pages import HTMLPage, pagination +from weboob.browser.elements import ListElement, ItemElement, method, TableElement +from weboob.browser.pages import HTMLPage, pagination, LoggedPage from weboob.browser.filters.standard import ( - Filter, Env, CleanText, CleanDecimal, Field, DateGuesser, Regexp, Currency, + Filter, Env, CleanText, CleanDecimal, Field, DateGuesser, Regexp, Currency, Format, Date ) from weboob.browser.filters.html import AbsoluteLink, TableCell from weboob.browser.filters.javascript import JSVar @@ -93,12 +93,13 @@ class AccountsType(Filter): (r'livret', Account.TYPE_SAVINGS), (r'livjeu', Account.TYPE_SAVINGS), (r'csljun', Account.TYPE_SAVINGS), + (r'ldds', Account.TYPE_SAVINGS), (r'compte', Account.TYPE_CHECKING), (r'cpte', Account.TYPE_CHECKING), (r'scpi', Account.TYPE_MARKET), (r'account', Account.TYPE_CHECKING), (r'\bpret\b', Account.TYPE_LOAN), - (r'\bvie\b', Account.TYPE_LIFE_INSURANCE), + (r'\bvie2?\b', Account.TYPE_LIFE_INSURANCE), (r'strategie patr.', Account.TYPE_LIFE_INSURANCE), (r'essentiel', Account.TYPE_LIFE_INSURANCE), (r'elysee', Account.TYPE_LIFE_INSURANCE), @@ -112,6 +113,7 @@ class AccountsType(Filter): (r'hsbc evol pat capi', Account.TYPE_CAPITALISATION), (r'bourse libre', Account.TYPE_MARKET), (r'plurival', Account.TYPE_LIFE_INSURANCE), + (r'europep', Account.TYPE_LIFE_INSURANCE), ] def filter(self, label): @@ -431,3 +433,25 @@ class get_profile(ItemElement): obj_name = CleanText('//div[@id="div_adr_P1"]//p/label[contains(text(), "Nom")]/parent::p/strong') obj_address = CleanText('//div[@id="div_adr_P1"]//p/label[contains(text(), "Adresse")]/parent::p/strong') + + +class ScpiHisPage(LoggedPage, HTMLPage): + def is_here(self): + return self.doc.xpath('//h3[contains(text(), "HISTORIQUE DES MOUVEMENTS")]') + + @method + class iter_history(TableElement): + item_xpath = '//table[@class="csTable"]//tbody//tr' + head_xpath = '//table[@class="csTable"]//thead//th/a' + + col_date = 'Date' + col_amount = 'Montant brut (en €)' + col_operation = 'Opération' + col_nature = 'Nature' + + class item(ItemElement): + klass = Transaction + + obj_label = Format('%s - %s', CleanText(TableCell('operation')), CleanText(TableCell('nature'))) + obj_rdate = Date(CleanText(TableCell('date')), dayfirst=True) + obj_amount = CleanDecimal(TableCell('amount'), sign=lambda x: -1, replace_dots=True) diff --git a/modules/hsbc/pages/investments.py b/modules/hsbc/pages/investments.py index b1e382d17d..63147b3d73 100644 --- a/modules/hsbc/pages/investments.py +++ b/modules/hsbc/pages/investments.py @@ -16,7 +16,7 @@ from weboob.browser.filters.standard import ( CleanText, CleanDecimal, Regexp, Currency, Field, Env, ) -from weboob.browser.filters.html import TableCell +from weboob.browser.filters.html import TableCell, Link from weboob.browser.filters.json import Dict from weboob.browser.filters.javascript import JSVar from weboob.exceptions import BrowserUnavailable @@ -524,6 +524,9 @@ def go_more_scpi_detail_page(self): assert len(detail_page) == 1 self.browser.location('https://www.hsbc.fr' + CleanText('./@href')(detail_page[0])) + def go_scpi_his_detail_page(self): + return Link('//div/a[contains(text(), "Historique de vos mouvements de parts")]')(self.doc) + @method class iter_scpi_investment(TableElement): item_xpath = '//table[@class="csTable"]//tbody//tr' diff --git a/modules/lcl/module.py b/modules/lcl/module.py index 12590172bc..aa50c51d14 100644 --- a/modules/lcl/module.py +++ b/modules/lcl/module.py @@ -145,7 +145,7 @@ def execute_transfer(self, transfer, **params): return self.browser.execute_transfer(transfer) def transfer_check_label(self, old, new): - old = re.sub(r"[/<\?='!]", '', old).strip() + old = re.sub(r"[/<\?='!\+]", '', old).strip() old = old.encode('latin-1', errors='replace').decode('latin-1') return super(LCLModule, self).transfer_check_label(old, new) diff --git a/modules/linebourse/browser.py b/modules/linebourse/browser.py index 3f44b30ae0..44d5edca0c 100644 --- a/modules/linebourse/browser.py +++ b/modules/linebourse/browser.py @@ -72,7 +72,24 @@ def iter_investment(self, account_id): assert self.invest.is_here() if not self.page.is_on_right_portfolio(account_id): self.invest.go(id=self.page.get_compte(account_id)) - return self.page.iter_investment() + return self.page.iter_investments() + + # Method used only by bp module + def get_liquidity(self, account_id): + self.main.go() + self.invest.go() + if self.message.is_here(): + self.page.submit() + self.invest.go() + + if self.broken.is_here(): + return iter([]) + + assert self.invest.is_here() + if not self.page.is_on_right_portfolio(account_id): + self.invest.go(id=self.page.get_compte(account_id)) + + return self.page.get_liquidity() def iter_history(self, account_id): self.main.go() diff --git a/modules/linebourse/compat/__init__.py b/modules/linebourse/compat/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/modules/linebourse/compat/weboob_tools_capabilities_bank_investments.py b/modules/linebourse/compat/weboob_tools_capabilities_bank_investments.py new file mode 100644 index 0000000000..2b68ff9e00 --- /dev/null +++ b/modules/linebourse/compat/weboob_tools_capabilities_bank_investments.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Jonathan Schmidt +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.tools.compat import basestring +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.bank import Investment + +def is_isin_valid(isin): + """ + Méthode générale + Table de conversion des lettres en chiffres + A=10 B=11 C=12 D=13 E=14 F=15 G=16 H=17 I=18 + J=19 K=20 L=21 M=22 N=23 O=24 P=25 Q=26 R=27 + S=28 T=29 U=30 V=31 W=32 X=33 Y=34 Z=35 + + 1 - Mettre de côté la clé, qui servira de référence à la fin de la vérification. + 2 - Convertir toutes les lettres en nombres via la table de conversion ci-contre. Si le nombre obtenu est supérieur ou égal à 10, prendre les deux chiffres du nombre séparément (exemple : 27 devient 2 et 7). + 3 - Pour chaque chiffre, multiplier sa valeur par deux si sa position est impaire en partant de la droite. Si le nombre obtenu est supérieur ou égal à 10, garder les deux chiffres du nombre séparément (exemple : 14 devient 1 et 4). + 4 - Faire la somme de tous les chiffres. + 5 - Soustraire cette somme de la dizaine supérieure ou égale la plus proche (exemples : si la somme vaut 22, la dizaine « supérieure ou égale » est 30, et la clé vaut donc 8 ; si la somme vaut 30, la dizaine « supérieure ou égale » est 30, et la clé vaut 0 ; si la somme vaut 31, la dizaine « supérieure ou égale » est 40, et la clé vaut 9). + 6 - Comparer la valeur obtenue à la clé mise initialement de côté. + + Étapes 1 et 2 : + F R 0 0 0 3 5 0 0 0 0 (+ 8 : clé) + 15 27 0 0 0 3 5 0 0 0 0 + + Étape 3 : le traitement se fait sur des chiffres + 1 5 2 7 0 0 0 3 5 0 0 0 0 + I P I P I P I P I P I P I : position en partant de la droite (P = Pair, I = Impair) + 2 1 2 1 2 1 2 1 2 1 2 1 2 : coefficient multiplicateur + + 2 5 4 7 0 0 0 3 10 0 0 0 0 : résultat + + Étape 4 : + 2 + 5 + 4 + 7 + 0 + 0 + 0 + 3 + (1 + 0)+ 0 + 0 + 0 + 0 = 22 + + Étapes 5 et 6 : 30 - 22 = 8 (valeur de la clé) + """ + + if not isinstance(isin, basestring): + return False + if not re.match(r'^[A-Z]{2}[A-Z0-9]{9}\d$', isin): + return False + + isin_in_digits = ''.join(str(ord(x) - ord('A') + 10) if not x.isdigit() else x for x in isin[:-1]) + key = isin[-1:] + result = '' + for k, val in enumerate(isin_in_digits[::-1], start=1): + if k % 2 == 0: + result = ''.join((result, val)) + else: + result = ''.join((result, str(int(val)*2))) + return str(sum(int(x) for x in result) + int(key))[-1] == '0' + + +def create_french_liquidity(valuation): + """ + Automatically fills a liquidity investment with label, code and code_type. + """ + liquidity = Investment() + liquidity.label = "Liquidités" + liquidity.code = "XX-liquidity" + liquidity.code_type = NotAvailable + liquidity.valuation = valuation + return liquidity + diff --git a/modules/linebourse/pages.py b/modules/linebourse/pages.py index 0587885140..fe5d40731f 100644 --- a/modules/linebourse/pages.py +++ b/modules/linebourse/pages.py @@ -30,6 +30,7 @@ from weboob.capabilities.base import NotAvailable from weboob.capabilities.bank import Investment from weboob.tools.capabilities.bank.transactions import FrenchTransaction as Transaction +from .compat.weboob_tools_capabilities_bank_investments import create_french_liquidity from weboob.tools.compat import quote_plus from weboob.exceptions import ActionNeeded @@ -110,7 +111,7 @@ def obj_investments(self): class InvestmentPage(AccountPage): @method - class get_investment(TableElement): + class iter_investments(TableElement): col_label = 'Valeur' col_quantity = u'Quantité' col_valuation = u'Valorisation EUR' @@ -139,9 +140,11 @@ def condition(self): obj_label = CleanText(Regexp(CleanText('./preceding-sibling::tr/td[1]'), '(.*)- .*')) obj_code = Regexp(CleanText('./preceding-sibling::tr/td[1]'), '- (.*)') - def iter_investment(self): - for inv in self.get_investment(): - yield inv + # Only used by bp modules since others quality websites provide another account with the liquidities + def get_liquidity(self): + liquidity = CleanDecimal('//table//tr[@class="titreAvant"]/td[contains(text(), "Liquidit")]/following-sibling::td', replace_dots=True)(self.doc) + if liquidity: + return create_french_liquidity(liquidity) class MessagePage(LoggedPage, HTMLPage): diff --git a/modules/societegenerale/sgpe/compat/weboob_exceptions.py b/modules/societegenerale/sgpe/compat/weboob_exceptions.py new file mode 100644 index 0000000000..e1912dfbf1 --- /dev/null +++ b/modules/societegenerale/sgpe/compat/weboob_exceptions.py @@ -0,0 +1,46 @@ + +from weboob.exceptions import * + + +class AuthMethodNotImplemented(Exception): + pass + + +class CaptchaQuestion(Exception): + """Site requires solving a CAPTCHA (base class)""" + # could be improved to pass the name of the backendconfig key + + def __init__(self, type=None, **kwargs): + super(CaptchaQuestion, self).__init__("The site requires solving a captcha") + self.type = type + for key, value in kwargs.items(): + setattr(self, key, value) + + +class ImageCaptchaQuestion(CaptchaQuestion): + type = 'image_captcha' + + image_data = None + + def __init__(self, image_data): + super(ImageCaptchaQuestion, self).__init__(self.type, image_data=image_data) + + +class NocaptchaQuestion(CaptchaQuestion): + type = 'g_recaptcha' + + website_key = None + website_url = None + + def __init__(self, website_key, website_url): + super(NocaptchaQuestion, self).__init__(self.type, website_key=website_key, website_url=website_url) + + +class RecaptchaQuestion(CaptchaQuestion): + type = 'g_recaptcha' + + website_key = None + website_url = None + + def __init__(self, website_key, website_url): + super(RecaptchaQuestion, self).__init__(self.type, website_key=website_key, website_url=website_url) diff --git a/modules/societegenerale/sgpe/json_pages.py b/modules/societegenerale/sgpe/json_pages.py index b06b284c90..0b9f461046 100644 --- a/modules/societegenerale/sgpe/json_pages.py +++ b/modules/societegenerale/sgpe/json_pages.py @@ -31,8 +31,9 @@ from weboob.capabilities import NotAvailable from weboob.capabilities.bank import Account from weboob.capabilities.bill import Document, Subscription -from weboob.exceptions import ( +from .compat.weboob_exceptions import ( BrowserUnavailable, NoAccountsException, BrowserIncorrectPassword, BrowserPasswordExpired, + AuthMethodNotImplemented, ) from weboob.tools.capabilities.bank.iban import is_iban_valid from weboob.tools.capabilities.bank.transactions import FrenchTransaction @@ -66,6 +67,8 @@ def on_load(self): raise BrowserIncorrectPassword('Vos identifiants sont incorrects') elif reason == 'chgt_mdp_oblig': raise BrowserPasswordExpired('Veuillez renouveler votre mot de passe') + elif reason == 'oob_insc_oblig': + raise AuthMethodNotImplemented("L'authentification par Secure Access n'est pas prise en charge") raise BrowserUnavailable(reason) @method -- GitLab