From 22355a1ca7a3e5007f89878dffb3cd24cba5b7b6 Mon Sep 17 00:00:00 2001 From: Vincent A Date: Wed, 28 Oct 2020 18:48:13 +0100 Subject: [PATCH] backport master modules fixes --- modules/amazon/browser.py | 21 +- modules/amazon/pages.py | 6 +- modules/axabanque/pages/login.py | 20 +- modules/banquepopulaire/browser.py | 7 +- modules/becm/browser.py | 4 +- modules/becm/pages.py | 10 + modules/bforbank/browser.py | 1 + modules/bforbank/pages.py | 1 + modules/bnporc/enterprise/pages.py | 2 +- modules/bnporc/pp/browser.py | 2 +- modules/bnporc/pp/pages.py | 1 + modules/boursorama/browser.py | 174 +++++-- modules/boursorama/module.py | 12 + modules/boursorama/pages.py | 204 ++++++-- modules/bp/module.py | 1 + modules/bred/bred/browser.py | 25 +- modules/bred/module.py | 15 +- modules/caissedepargne/browser.py | 8 +- modules/caissedepargne/pages.py | 13 +- modules/cic/browser.py | 4 +- modules/cmes/browser.py | 17 +- modules/cmes/pages.py | 18 +- modules/cmso/par/pages.py | 12 +- modules/cmso/pro/browser.py | 4 +- modules/cragr/browser.py | 2 +- modules/cragr/pages.py | 3 + modules/creditmutuel/browser.py | 14 +- modules/creditmutuel/pages.py | 3 +- modules/hsbc/browser.py | 52 +- .../hsbc/compat/weboob_browser_browsers.py | 59 +++ modules/hsbc/module.py | 6 +- modules/hsbc/pages/account_pages.py | 8 + modules/hsbc/pages/investments.py | 3 +- modules/ing/api/transfer_page.py | 2 +- modules/ing/test.py | 5 +- modules/lcl/browser.py | 113 ++++- modules/lcl/compat/weboob_browser_browsers.py | 59 +++ modules/lcl/module.py | 10 +- modules/lcl/pages.py | 68 ++- modules/myedenred/browser.py | 21 +- modules/myedenred/pages.py | 17 +- modules/orange/browser.py | 41 +- modules/orange/module.py | 6 +- modules/orange/pages/captcha.py | 150 ++++++ modules/orange/pages/captcha_symbols.py | 445 ++++++++++++++++++ modules/orange/pages/login.py | 4 - modules/pagesjaunes/browser.py | 9 +- modules/pagesjaunes/module.py | 1 - modules/pagesjaunes/pages.py | 58 ++- modules/s2e/browser.py | 1 + modules/s2e/pages.py | 4 +- modules/societegenerale/browser.py | 14 +- modules/societegenerale/pages/login.py | 2 +- modules/societegenerale/pages/subscription.py | 1 + modules/societegenerale/sgpe/browser.py | 2 +- modules/societegenerale/sgpe/json_pages.py | 1 + modules/societegenerale/sgpe/pages.py | 2 +- modules/swile/browser.py | 6 +- modules/yomoni/browser.py | 14 +- 59 files changed, 1537 insertions(+), 251 deletions(-) create mode 100644 modules/hsbc/compat/weboob_browser_browsers.py create mode 100644 modules/lcl/compat/weboob_browser_browsers.py create mode 100644 modules/orange/pages/captcha.py create mode 100644 modules/orange/pages/captcha_symbols.py diff --git a/modules/amazon/browser.py b/modules/amazon/browser.py index 55eb6b2414..00d564c01e 100644 --- a/modules/amazon/browser.py +++ b/modules/amazon/browser.py @@ -144,7 +144,9 @@ def handle_captcha(self, captcha): def check_app_validation(self): # client has 60 seconds to unlock this page - timeout = time.time() + 60.00 + # the resend link will appear from 60 seconds is why there are 2 additional seconds, it's to have a margin + timeout = time.time() + 62.00 + second_try = True while time.time() < timeout: link = self.page.get_link_app_validation() self.location(link) @@ -152,6 +154,13 @@ def check_app_validation(self): time.sleep(2) else: return + + if time.time() >= timeout and second_try: + # second try because 60 seconds is short, the second try is longger + second_try = False + timeout = time.time() + 70.00 + self.page.resend_link() + else: raise AppValidationExpired() @@ -169,6 +178,8 @@ def do_login(self): if self.config['resume'].get(): self.check_app_validation() + # we are logged + return if self.security.is_here(): self.handle_security() @@ -192,8 +203,11 @@ def do_login(self): raise WrongCaptchaResponse(msg) else: assert False, msg - else: - return + + if self.approval_page.is_here(): + # if we have captcha and app validation + msg_validation = self.page.get_msg_app_validation() + raise AppValidation(msg_validation) # Change language so everything is handled the same way self.to_english(self.LANGUAGE) @@ -210,6 +224,7 @@ def do_login(self): self.page.login(self.username, self.password) if self.approval_page.is_here(): + # if we don't have captcha and we have app validation msg_validation = self.page.get_msg_app_validation() raise AppValidation(msg_validation) diff --git a/modules/amazon/pages.py b/modules/amazon/pages.py index 29cc97638e..9e13e5d261 100644 --- a/modules/amazon/pages.py +++ b/modules/amazon/pages.py @@ -102,7 +102,11 @@ def get_msg_app_validation(self): return msg(self.doc) def get_link_app_validation(self): - return Link('//a[contains(text(), "Click here to refresh the page")]')(self.doc) + return Link('//a[@id="resend-approval-link"]')(self.doc) + + def resend_link(self): + form = self.get_form(id='resend-approval-form') + form.submit() class LanguagePage(HTMLPage): diff --git a/modules/axabanque/pages/login.py b/modules/axabanque/pages/login.py index 28eaa5eb21..91cb9c0f02 100644 --- a/modules/axabanque/pages/login.py +++ b/modules/axabanque/pages/login.py @@ -33,16 +33,16 @@ class MyVirtKeyboard(VirtKeyboard): color = (255, 255, 255) symbols = { - '0': '7c19886349f1b8f41d9876bbb4182786', - '1': '7825fb0dade1227999abd21ab44529a6', - '2': '94790a9747373a540995f132c0d46686', - '3': '237154eb1838b2d995e789c4b97b1454', - '4': 'a6fd31cb646e5fd0c9c6c4bfb5467ede', - '5': '5c7823607874fbc7cd6cdd058f9c05c7', - '6': '5eb962c5f38be89e17b2c2acc4d61a94', - '7': '8c926a882094ce769579786b50bb7a69', - '8': '1d9c6b845dc4f85dc56426bbf23faa80', - '9': 'f817f2a21497fc32438b07fd15beedbc', + '0': '962575659eb1bb72b15f856c0358c644', + '1': '36ccbc0fff397cef567b5be362127484', + '2': 'b823b3078cbfa1707ecbf8b9a92dea44', + '3': 'f4790e47f878eba58ef93cfd6956726b', + '4': '577620e004518fb057cc704842d59245', + '5': '01e11c7a7092f2f7ec119b78be605923', + '6': '0b7b051871b6bf4e2c91282cfcae09bc', + '7': '920313cbddda9934447d8f1daa71e76b', + '8': 'bdb35b451e6de3fb7221f50669fe52fb', + '9': '1ee5fdfd7877ec9e0a957e14db5d29e6', } coords = { diff --git a/modules/banquepopulaire/browser.py b/modules/banquepopulaire/browser.py index bc451d51fc..ad3b4a1fb4 100644 --- a/modules/banquepopulaire/browser.py +++ b/modules/banquepopulaire/browser.py @@ -202,7 +202,7 @@ class BanquePopulaire(LoginBrowser): advisor = URL(r'https://[^/]+/cyber/internet/StartTask.do\?taskInfoOID=accueil.*', r'https://[^/]+/cyber/internet/StartTask.do\?taskInfoOID=contacter.*', AdvisorPage) - basic_token_page = URL(r'/SRVATE/context/mde/1.1.5', BasicTokenPage) + basic_token_page = URL(r'https://(?P.[\w\.]+)/SRVATE/context/mde/1.1.5', BasicTokenPage) subscriber_page = URL(r'https://[^/]+/api-bp/wapi/2.0/abonnes/current/mes-documents-electroniques', SubscriberPage) subscription_page = URL(r'https://[^/]+/api-bp/wapi/2.0/abonnes/current/contrats', SubscriptionsPage) documents_page = URL(r'/api-bp/wapi/2.0/abonnes/current/documents/recherche-avancee', DocumentsPage) @@ -937,7 +937,10 @@ def get_advisor(self): @need_login def iter_subscriptions(self): - self.location('/SRVATE/context/mde/1.1.5') + # specify the website url in order to avoid 404 errors. + # 404 errors occur when the baseurl is a website we have + # been redirected to, like natixis or linebourse + self.basic_token_page.go(website=self.website) headers = {'Authorization': 'Basic %s' % self.page.get_basic_token()} response = self.location('/as-bp/as/2.0/tokens', method='POST', headers=headers) self.documents_headers = {'Authorization': 'Bearer %s' % response.json()['access_token']} diff --git a/modules/becm/browser.py b/modules/becm/browser.py index 85f9b65509..ce453789ba 100644 --- a/modules/becm/browser.py +++ b/modules/becm/browser.py @@ -24,7 +24,7 @@ from .compat.weboob_browser_url import URL from .compat.weboob_browser_browsers import need_login -from .pages import AdvisorPage, LoginPage +from .pages import AdvisorPage, LoginPage, DecoupledStatePage, CancelDecoupled __all__ = ['BECMBrowser'] @@ -38,6 +38,8 @@ class BECMBrowser(AbstractBrowser): login = URL('/fr/authentification.html', LoginPage) advisor = URL('/fr/banques/Details.aspx\?banque=.*', AdvisorPage) + decoupled_state = URL(r'/(?P.*)fr/otp/SOSD_OTP_GetTransactionState.htm', DecoupledStatePage) + cancel_decoupled = URL(r'/(?P.*)fr/otp/SOSD_OTP_CancelTransaction.htm', CancelDecoupled) @need_login def get_advisor(self): diff --git a/modules/becm/pages.py b/modules/becm/pages.py index 35ca70bb13..b0f64fdada 100644 --- a/modules/becm/pages.py +++ b/modules/becm/pages.py @@ -40,3 +40,13 @@ class update_advisor(ItemElement): obj_address = Format('%s %s %s', CleanText('//table//*[@itemprop="streetAddress"]'), CleanText('//table//*[@itemprop="postalCode"]'), CleanText('//table//*[@itemprop="addressLocality"]')) + + +class DecoupledStatePage(AbstractPage): + PARENT = 'creditmutuel' + PARENT_URL = 'decoupled_state' + + +class CancelDecoupled(AbstractPage): + PARENT = 'creditmutuel' + PARENT_URL = 'cancel_decoupled' diff --git a/modules/bforbank/browser.py b/modules/bforbank/browser.py index a3b1e601f6..500c7a779f 100644 --- a/modules/bforbank/browser.py +++ b/modules/bforbank/browser.py @@ -22,6 +22,7 @@ import datetime from dateutil.relativedelta import relativedelta + from weboob.exceptions import BrowserIncorrectPassword, ActionNeeded from weboob.browser.browsers import LoginBrowser, URL, need_login from .compat.weboob_capabilities_bank import Account, AccountNotFound diff --git a/modules/bforbank/pages.py b/modules/bforbank/pages.py index 70c678cf8a..f0ffb29825 100644 --- a/modules/bforbank/pages.py +++ b/modules/bforbank/pages.py @@ -28,6 +28,7 @@ import re from PIL import Image + from weboob.exceptions import ActionNeeded from .compat.weboob_browser_pages import LoggedPage, HTMLPage, pagination, AbstractPage, JsonPage from weboob.browser.elements import method, ListElement, ItemElement, TableElement diff --git a/modules/bnporc/enterprise/pages.py b/modules/bnporc/enterprise/pages.py index 9fae79c93b..e924a117d4 100644 --- a/modules/bnporc/enterprise/pages.py +++ b/modules/bnporc/enterprise/pages.py @@ -22,11 +22,11 @@ from __future__ import unicode_literals import re - from datetime import datetime from io import BytesIO import dateutil.parser + from .compat.weboob_browser_pages import LoggedPage, HTMLPage, JsonPage from weboob.browser.filters.json import Dict from weboob.browser.filters.html import TableCell, Attr diff --git a/modules/bnporc/pp/browser.py b/modules/bnporc/pp/browser.py index deac70086e..756406e582 100644 --- a/modules/bnporc/pp/browser.py +++ b/modules/bnporc/pp/browser.py @@ -56,9 +56,9 @@ AddRecipPage, ActivateRecipPage, ProfilePage, ListDetailCardPage, ListErrorPage, UselessPage, TransferAssertionError, LoanDetailsPage, TransfersPage, OTPPage, ) - from .document_pages import DocumentsPage, DocumentsResearchPage, TitulairePage, RIBPage + __all__ = ['BNPPartPro', 'HelloBank'] diff --git a/modules/bnporc/pp/pages.py b/modules/bnporc/pp/pages.py index 2d5e529c75..dbc41f0670 100644 --- a/modules/bnporc/pp/pages.py +++ b/modules/bnporc/pp/pages.py @@ -27,6 +27,7 @@ from random import randint from decimal import Decimal from datetime import datetime, timedelta + import lxml.html as html from requests.exceptions import ConnectionError diff --git a/modules/boursorama/browser.py b/modules/boursorama/browser.py index 9b8c92cdc1..6ea72b8c20 100644 --- a/modules/boursorama/browser.py +++ b/modules/boursorama/browser.py @@ -21,15 +21,19 @@ from __future__ import unicode_literals -import requests - from datetime import date, datetime +import re + from dateutil.relativedelta import relativedelta +import requests from weboob.browser.retry import login_method, retry_on_logout, RetryLoginBrowser from .compat.weboob_browser_browsers import need_login, TwoFactorBrowser from .compat.weboob_browser_url import URL -from weboob.exceptions import BrowserIncorrectPassword, BrowserHTTPNotFound, NoAccountsException, BrowserUnavailable +from weboob.exceptions import ( + BrowserIncorrectPassword, BrowserHTTPNotFound, NoAccountsException, + BrowserUnavailable, ActionNeeded, +) from weboob.browser.exceptions import LoggedOut, ClientError from .compat.weboob_capabilities_bank import ( Account, AccountNotFound, TransferError, TransferInvalidAmount, @@ -48,10 +52,10 @@ VirtKeyboardPage, AccountsPage, AsvPage, HistoryPage, AuthenticationPage, MarketPage, LoanPage, SavingMarketPage, ErrorPage, IncidentPage, IbanPage, ProfilePage, ExpertPage, CardsNumberPage, CalendarPage, HomePage, PEPPage, - TransferAccounts, TransferRecipients, TransferCharac, TransferConfirm, TransferSent, + TransferAccounts, TransferRecipients, TransferCharacteristics, TransferConfirm, TransferSent, AddRecipientPage, StatusPage, CardHistoryPage, CardCalendarPage, CurrencyListPage, CurrencyConvertPage, - AccountsErrorPage, NoAccountPage, TransferMainPage, PasswordPage, NewTransferRecipients, - NewTransferAccounts, CardSumDetailPage, + AccountsErrorPage, NoAccountPage, TransferMainPage, PasswordPage, NewTransferWizard, + NewTransferConfirm, NewTransferSent, CardSumDetailPage, ) from .transfer_pages import TransferListPage, TransferInfoPage @@ -123,18 +127,9 @@ class BoursoramaBrowser(RetryLoginBrowser, TwoFactorBrowser): r'/compte/(?P[^/]+)/(?P\w+)/virements/nouveau/(?P\w+)/2', TransferRecipients ) - new_transfer_accounts = URL( - r'/compte/(?P[^/]+)/(?P\w+)/virements/immediat/nouveau/?$', - r'/compte/(?P[^/]+)/(?P\w+)/virements/immediat/nouveau/(?P\w+)/1', - NewTransferAccounts - ) - new_recipients_page = URL( - r'/compte/(?P[^/]+)/(?P\w+)/virements/immediat/nouveau/(?P\w+)/2', - NewTransferRecipients - ) - transfer_charac = URL( + transfer_characteristics = URL( r'/compte/(?P[^/]+)/(?P\w+)/virements/nouveau/(?P\w+)/3', - TransferCharac + TransferCharacteristics ) transfer_confirm = URL( r'/compte/(?P[^/]+)/(?P\w+)/virements/nouveau/(?P\w+)/4', @@ -144,6 +139,23 @@ class BoursoramaBrowser(RetryLoginBrowser, TwoFactorBrowser): r'/compte/(?P[^/]+)/(?P\w+)/virements/nouveau/(?P\w+)/5', TransferSent ) + # transfer_type should be one of : "immediat", "programme" + new_transfer_wizard = URL( + r'/compte/(?P[^/]+)/(?P\w+)/virements/(?Pimmediat|programme)/nouveau/?$', + r'/compte/(?P[^/]+)/(?P\w+)/virements/immediat/nouveau/(?P\w+)/(?P[1-6])$', + r'/compte/(?P[^/]+)/(?P\w+)/virements/programme/nouveau/(?P\w+)/(?P[1-7])$', + NewTransferWizard + ) + new_transfer_confirm = URL( + r'/compte/(?P[^/]+)/(?P\w+)/virements/immediat/nouveau/(?P\w+)/7$', + r'/compte/(?P[^/]+)/(?P\w+)/virements/programme/nouveau/(?P\w+)/8$', + NewTransferConfirm + ) + new_transfer_sent = URL( + r'/compte/(?P[^/]+)/(?P\w+)/virements/immediat/nouveau/(?P\w+)/9$', + r'/compte/(?P[^/]+)/(?P\w+)/virements/programme/nouveau/(?P\w+)/10$', + NewTransferSent + ) rcpt_page = URL( r'/compte/(?P[^/]+)/(?P\w+)/virements/comptes-externes/nouveau/(?P\w+)/\d', AddRecipientPage @@ -257,17 +269,28 @@ def init_login(self): error = self.page.get_error() assert error, 'Should not be on login page without error message' - wrongpass_messages = ( - 'Identifiant ou mot de passe invalide', - "Erreur d'authentification", - "Cette valeur n'est pas valide", - "votre identifiant ou votre mot de passe n'est pas valide", + is_wrongpass = re.search( + "Identifiant ou mot de passe invalide" + + "|Erreur d'authentification" + + "|Cette valeur n'est pas valide" + + "|votre identifiant ou votre mot de passe n'est pas valide", + error ) - if 'vous pouvez actuellement rencontrer des difficultés pour accéder à votre Espace Client' in error: + is_website_unavailable = re.search( + "vous pouvez actuellement rencontrer des difficultés pour accéder à votre Espace Client" + + "|Une erreur est survenue. Veuillez réessayer ultérieurement" + + "|Oups, Il semble qu'une erreur soit survenue de notre côté", + error + ) + + if is_website_unavailable: raise BrowserUnavailable() - elif any(msg in error for msg in wrongpass_messages): + elif is_wrongpass: raise BrowserIncorrectPassword(error) + elif "pour changer votre mot de passe" in error: + # this popup appears after few wrongpass errors and requires a password change + raise ActionNeeded(error) raise AssertionError('Unhandled error message : "%s"' % error) @@ -413,6 +436,10 @@ def get_account(self, id): return a return None + def get_opening_date(self, account_url): + self.location(account_url) + return self.page.fetch_opening_date() + def get_debit_date(self, debit_date): for i, j in zip(self.deferred_card_calendar, self.deferred_card_calendar[1:]): if i[0] < debit_date <= j[0]: @@ -573,7 +600,7 @@ def get_advisor(self): advisor.phone = u"0146094949" return iter([advisor]) - def go_recipients_list(self, account_url, account_id): + def go_recipients_list(self, account_url, account_id, for_scheduled=False): # url transfer preparation url = urlsplit(account_url) parts = [part for part in url.path.split('/') if part] @@ -582,20 +609,33 @@ def go_recipients_list(self, account_url, account_id): account_type = parts[1] # cav, ord, epargne ... account_webid = parts[-1] - self.transfer_main_page.go(acc_type=account_type, webid=account_webid) # may raise a BrowserHTTPNotFound + # may raise a BrowserHTTPNotFound + self.transfer_main_page.go(acc_type=account_type, webid=account_webid) # can check all account available transfer option if self.transfer_main_page.is_here(): self.transfer_accounts.go(acc_type=account_type, webid=account_webid) if self.transfer_accounts.is_here(): - self.page.submit_account(account_id) # may raise AccountNotFound + # may raise AccountNotFound + self.page.submit_account(account_id) elif self.transfer_main_page.is_here(): - self.new_transfer_accounts.go(acc_type=account_type, webid=account_webid) - self.page.submit_account(account_id) # may raise AccountNotFound + if for_scheduled: + transfer_type = 'programme' + else: + transfer_type = 'immediat' + self.new_transfer_wizard.go( + acc_type=account_type, + webid=account_webid, + transfer_type=transfer_type + ) + # may raise AccountNotFound + self.page.submit_account(account_id) + + return account_type, account_webid @need_login - def iter_transfer_recipients(self, account): + def iter_transfer_recipients(self, account, for_scheduled=False): if account.type in (Account.TYPE_LOAN, Account.TYPE_LIFE_INSURANCE): return [] @@ -606,18 +646,20 @@ def iter_transfer_recipients(self, account): assert account.url, 'Account should have an url to access its recipients' try: - self.go_recipients_list(account.url, account.id) + self.go_recipients_list(account.url, account.id, for_scheduled) except (BrowserHTTPNotFound, AccountNotFound): return [] assert ( self.recipients_page.is_here() - or self.new_recipients_page.is_here() + or self.new_transfer_wizard.is_here() ), 'Should be on recipients page' return self.page.iter_recipients() def check_basic_transfer(self, transfer): + if transfer.date_type == TransferDateType.PERIODIC: + raise NotImplementedError('Periodic transfer is not implemented') if transfer.amount <= 0: raise TransferInvalidAmount('transfer amount must be positive') if transfer.recipient_id == transfer.account_id: @@ -627,13 +669,25 @@ def check_basic_transfer(self, transfer): @need_login def init_transfer(self, transfer, **kwargs): + # Transfer_date_type is set and used only for the new transfer wizard flow + # the support for the old transfer wizard is left untouched as much as possible + # until it can be removed. + transfer_date_type = transfer.date_type + if empty(transfer_date_type): + if not empty(transfer.exec_date) and transfer.exec_date > date.today(): + transfer_date_type = TransferDateType.DEFERRED + else: + transfer_date_type = TransferDateType.FIRST_OPEN_DAY + + is_scheduled = (transfer_date_type in [TransferDateType.DEFERRED, TransferDateType.PERIODIC]) + self.check_basic_transfer(transfer) account = self.get_account(transfer.account_id) if not account: raise AccountNotFound() - recipients = list(self.iter_transfer_recipients(account)) + recipients = list(self.iter_transfer_recipients(account, is_scheduled)) if not recipients: raise TransferInvalidEmitter('The account cannot emit transfers') @@ -642,20 +696,37 @@ def init_transfer(self, transfer, **kwargs): raise TransferInvalidRecipient('The recipient cannot be used with the emitter account') assert len(recipients) == 1 - if self.new_recipients_page.is_here(): - raise NotImplementedError('The new transfer pages are not yet implemented') - self.page.submit_recipient(recipients[0]._tempid) - assert self.transfer_charac.is_here() - self.page.submit_info(transfer.amount, transfer.label, transfer.exec_date) - assert self.transfer_confirm.is_here() + if self.transfer_characteristics.is_here(): + # Old transfer interface of Boursorama + self.page.submit_info(transfer.amount, transfer.label, transfer.exec_date) + assert self.transfer_confirm.is_here() - if self.page.need_refresh(): - # In some case we are not yet in the transfer_charac page, you need to refresh the page - self.location(self.url) - assert not self.page.need_refresh() - ret = self.page.get_transfer() + if self.page.need_refresh(): + # In some case we are not yet in the transfer_characteristics page, you need to refresh the page + self.location(self.url) + assert not self.page.need_refresh() + ret = self.page.get_transfer() + + else: + # New transfer interface + assert self.new_transfer_wizard.is_here() + self.page.submit_amount(transfer.amount) + + assert self.new_transfer_wizard.is_here() + if is_scheduled: + self.page.submit_programme_date_type(transfer_date_type) + + self.page.submit_info(transfer.label, transfer_date_type, transfer.exec_date) + + assert self.new_transfer_confirm.is_here() + transfer_error = self.page.get_errors() + if transfer_error: + raise TransferBankError(message=transfer_error) + ret = self.page.get_transfer() + + ## Last checks to ensure that the confirmation matches what was expected # at this stage, the site doesn't show the real ids/ibans, we can only guess if recipients[0].label != ret.recipient_label: @@ -674,7 +745,8 @@ def init_transfer(self, transfer, **kwargs): ret.recipient_iban = recipients[0].iban if account.label != ret.account_label: - raise TransferError('Account label changed during transfer') + raise TransferError('Account label changed during transfer (from "%s" to "%s")' + % (account.label, ret.account_label)) ret.account_id = account.id ret.account_iban = account.iban @@ -683,11 +755,11 @@ def init_transfer(self, transfer, **kwargs): @need_login def execute_transfer(self, transfer, **kwargs): - assert self.transfer_confirm.is_here() + assert self.transfer_confirm.is_here() or self.new_transfer_confirm.is_here() self.page.submit() - assert self.transfer_sent.is_here() - transfer_error = self.page.get_transfer_error() + assert self.transfer_sent.is_here() or self.new_transfer_sent.is_here() + transfer_error = self.page.get_errors() if transfer_error: raise TransferBankError(message=transfer_error) @@ -711,7 +783,7 @@ def init_new_recipient(self, recipient): target = account.url + '/' + suffix self.location(target) - assert self.page.is_charac(), 'Not on the page to add recipients.' + assert self.page.is_characteristics(), 'Not on the page to add recipients.' # fill recipient form self.page.submit_recipient(recipient) @@ -731,7 +803,7 @@ def init_new_recipient(self, recipient): raise AddRecipientStep(recipient, Value('otp_sms', label='Veuillez saisir le code recu par sms')) # if the add recipient is restarted after the sms has been confirmed recently, the sms step is not presented again - return self.rcpt_after_sms() + return self.rcpt_after_sms(recipient, account.url) def new_recipient(self, recipient, **kwargs): # step 2 of new_recipient @@ -868,4 +940,6 @@ def iter_emitters(self): # It seems that if we give a wrong acc_type and webid to the transfer page # we are redirected to a page where we can choose the emitter account self.transfer_accounts.go(acc_type='temp', webid='temp') + if self.transfer_main_page.is_here(): + self.new_transfer_wizard.go(acc_type='temp', webid='temp', transfer_type='immediat') return self.page.iter_emitters() diff --git a/modules/boursorama/module.py b/modules/boursorama/module.py index 3de1ddb95d..056a64d253 100644 --- a/modules/boursorama/module.py +++ b/modules/boursorama/module.py @@ -131,3 +131,15 @@ def get_rate(self, currency_from, currency_to): def iter_emitters(self): return self.browser.iter_emitters() + + def fill_account(self, account, fields): + if ( + 'opening_date' in fields + and account.type == Account.TYPE_LIFE_INSURANCE + and '/compte/derive' not in account.url + ): + account.opening_date = self.browser.get_opening_date(account.url) + + OBJECTS = { + Account: fill_account, + } diff --git a/modules/boursorama/pages.py b/modules/boursorama/pages.py index cde06968ca..2558a0ac7b 100644 --- a/modules/boursorama/pages.py +++ b/modules/boursorama/pages.py @@ -42,22 +42,21 @@ from weboob.browser.filters.json import Dict from weboob.browser.filters.html import Attr, Link, TableCell from .compat.weboob_capabilities_bank import ( - Account as BaseAccount, Recipient, Transfer, AccountNotFound, + Account as BaseAccount, Recipient, Transfer, TransferDateType, AccountNotFound, AddRecipientBankError, TransferInvalidAmount, Loan, AccountOwnership, - Emitter, + Emitter, TransferBankError, ) from .compat.weboob_capabilities_wealth import ( Investment, MarketOrder, MarketOrderType, MarketOrderDirection, MarketOrderPayment, ) -from weboob.tools.capabilities.bank.investments import create_french_liquidity from weboob.capabilities.base import NotAvailable, Currency, find_object, empty from weboob.capabilities.profile import Person -from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.tools.capabilities.bank.iban import is_iban_valid -from weboob.tools.capabilities.bank.investments import IsinCode, IsinType -from .compat.weboob_tools_value import Value -from weboob.tools.date import parse_french_date +from weboob.tools.capabilities.bank.investments import IsinCode, IsinType, create_french_liquidity +from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.tools.compat import urljoin, urlencode, urlparse, range +from weboob.tools.date import parse_french_date +from .compat.weboob_tools_value import Value from weboob.exceptions import ( BrowserQuestion, BrowserIncorrectPassword, BrowserHTTPNotFound, BrowserUnavailable, ActionNeeded, @@ -88,8 +87,7 @@ def get_iban(self): ): return NotAvailable return CleanText( - '//div[strong[contains(text(),"IBAN")]]/div[contains(@class, "definition")]', - replace=[(' ', '')] + '//div[strong[contains(text(),"IBAN")]]/div[contains(@class, "definition")]', replace=[(' ', '')] )(self.doc) @@ -1095,6 +1093,13 @@ class item(Myitem): def obj_label(self): return CleanText('.//strong/a')(self) or CleanText('.//strong', children=False)(self) + def fetch_opening_date(self): + return Date( + CleanText('//div[contains(text(), "ouverture fiscale")]//strong'), + dayfirst=True, + default=NotAvailable + )(self.doc) + class ErrorPage(HTMLPage): def on_load(self): @@ -1194,6 +1199,10 @@ class TransferMainPage(LoggedPage, HTMLPage): class TransferAccounts(LoggedPage, HTMLPage): + def on_load(self): + super(TransferAccounts, self).on_load() + self.logger.warning('CANARY Boursorama: Usage detected of an old interface transfer web page') + @method class iter_accounts(ListElement): item_xpath = '//a[has-class("next-step")][@data-value]' @@ -1279,7 +1288,43 @@ def submit_recipient(self, tempid): form.submit() -class NewTransferRecipients(LoggedPage, HTMLPage): +class NewTransferWizard(LoggedPage, HTMLPage): + def get_errors(self): + return CleanText('//form//div[@class="form-errors"]//li')(self.doc) + + # STEP 1 - Select account + def submit_account(self, account_id): + no_account_msg = CleanText('//div[contains(@class, "alert--warning")]')(self.doc) + if 'Vous ne possédez pas de compte éligible au virement' in no_account_msg: + raise AccountNotFound() + elif no_account_msg: + raise AssertionError('Unhandled error message when trying to select an account for a new transfer: "%s"' + % no_account_msg) + + form = self.get_form() + debit_account = CleanText( + '//input[./following-sibling::div/span/span[contains(text(), "%s")]]/@value' % account_id + )(self.doc) + if not debit_account: + raise AccountNotFound() + + form['DebitAccount[debit]'] = debit_account + form.submit() + + @method + class iter_emitters(ListElement): + item_xpath = '//ul[has-class("c-info-box")]/li[has-class("c-info-box__item")]' + + class item(ItemElement): + klass = Emitter + + obj_id = CleanText('.//span[@class="c-info-box__account-sub-label"]/span') + obj_label = CleanText('.//span[@class="c-info-box__account-label"]') + obj_currency = CleanCurrency('.//span[has-class("c-info-box__account-balance")]') + obj_balance = CleanDecimal.French('.//span[has-class("c-info-box__account-balance")]') + obj__bourso_id = Attr('.//div[has-class("c-info-box__content")]', 'data-value') + + # STEP 2 - Select recipient (or to create a new recipient) @method class iter_recipients(ListElement): item_xpath = '//div[contains(@id, "panel-")]//div[contains(@class, "panel__body")]//label' @@ -1292,11 +1337,11 @@ class item(ItemElement): replace=[(' ', '')], ) - obj_label = Regexp( + obj_label = CleanText(Regexp( CleanText('.//span[contains(@class, "account-label")]'), r'([^-]+)', '\\1', - ) + )) def obj_category(self): text = CleanText( @@ -1319,27 +1364,128 @@ def obj_enabled_at(self): obj__tempid = Attr('./input', 'value') + def submit_recipient(self, tempid): + form = self.get_form(name='CreditAccount') + # newBeneficiary values: + # 0 = Existing recipient; 1 = New recipient + form['CreditAccount[newBeneficiary]'] = 0 + form['CreditAccount[credit]'] = tempid -class NewTransferAccounts(LoggedPage, HTMLPage): - def submit_account(self, account_id): - no_account_msg = CleanText('//div[contains(@class, "alert--warning")]')(self.doc) - if 'Vous ne possédez pas de compte éligible au virement' in no_account_msg: - raise AccountNotFound() - elif no_account_msg: - raise AssertionError('Unhandled error message : "%s"' % no_account_msg) + form.submit() - form = self.get_form() - debit_account = CleanText( - '//input[./following-sibling::div/span/span[contains(text(), "%s")]]/@value' % account_id - )(self.doc) - if not debit_account: - raise AccountNotFound() + # STEP 3 - + # If using existing recipient: select the amount + # If new beneficiary: select if new recipient is own account or third party one + # For the moment, we only support the transfer to an existing recipient + def submit_amount(self, amount): + error_msg = self.get_errors() + if error_msg: + raise TransferBankError(message=error_msg) + + form = self.get_form(name='Amount') + str_amount = str(amount.quantize(Decimal('0.00'))).replace('.', ',') + form['Amount[amount]'] = str_amount + form.submit() + + # STEP 4 - SKIPPED - Fill new beneficiary info + + # STEP 5 - SKIPPED - select the amount after new beneficiary + + # STEP 6 for "programme" - To select deferred or periodic + def submit_programme_date_type(self, transfer_date_type): + error_msg = self.get_errors() + if error_msg: + raise TransferBankError(message=error_msg) + + assert transfer_date_type == TransferDateType.DEFERRED, "periodic transfer not supported" + + form = self.get_form(name='Scheduling') + # SchedulingType: 2=Deffered ; 3=Periodic + form['Scheduling[schedulingType]'] = '2' + form.submit() + + # STEP 6 for "immediate" - Enter label and scheduling type + # STEP 7 for "programme" - Enter label and scheduled date + def submit_info(self, label, transfer_date_type, exec_date=None): + error_msg = self.get_errors() + if error_msg: + raise TransferBankError(message=error_msg) + + form = self.get_form(name='Characteristics') + + form['Characteristics[label]'] = label + if transfer_date_type == TransferDateType.INSTANT: + form['Characteristics[schedulingType]'] = '1' + elif transfer_date_type == TransferDateType.FIRST_OPEN_DAY: + # It looks like that no schedulingType is sent in the "Ponctual" + # ie FIRST_OPEN_DAY case + form['Characteristics[schedulingType]'] = None + elif transfer_date_type == TransferDateType.DEFERRED: + if empty(exec_date): + exec_date = datetime.date.today() + form['Characteristics[scheduledDate]'] = exec_date.strftime('%d/%m/%Y') + else: + raise AssertionError("Periodic transfer is not supported") - form['DebitAccount[debit]'] = debit_account form.submit() -class TransferCharac(LoggedPage, HTMLPage): +class NewTransferConfirm(LoggedPage, HTMLPage): + # STEP 7 for "immediate" - Confirmation page + # STEP 8 for "programme" + def get_errors(self): + return CleanText('//form//div[@class="form-errors"]//li')(self.doc) + + @method + class get_transfer(ItemElement): + klass = Transfer + + XPATH_TMPL = '//form[@name="Confirm"]//tr[has-class("definition-list__row")][th[contains(text(),"%s")]]/td[1]' + + mapping_date_type = { + 'Ponctuel': TransferDateType.FIRST_OPEN_DAY, + 'Instantané': TransferDateType.INSTANT, + 'Différé': TransferDateType.DEFERRED, + 'Permanent': TransferDateType.PERIODIC, + } + + obj_account_label = CleanText(XPATH_TMPL % 'Compte à débiter') + obj_recipient_label = CleanText(XPATH_TMPL % 'Compte à créditer') + obj_amount = CleanDecimal.French(XPATH_TMPL % 'Montant en euro') + obj_currency = CleanCurrency(XPATH_TMPL % 'Montant en euro') + obj_label = CleanText(XPATH_TMPL % 'Motif visible par le bénéficiaire') + obj_date_type = Map( + CleanText(XPATH_TMPL % 'Type de virement'), + mapping_date_type, + ) + + def obj_exec_date(self): + type_ = Field('date_type')(self) + if type_ in [TransferDateType.INSTANT, TransferDateType.FIRST_OPEN_DAY]: + return datetime.date.today() + elif type_ == TransferDateType.DEFERRED: + return Date( + CleanText(self.XPATH_TMPL % "Date d'envoi"), + parse_func=parse_french_date, + )(self) + + def submit(self): + error_msg = self.get_errors() + if error_msg: + raise TransferBankError(message=error_msg) + + form = self.get_form(name='Confirm') + form.submit() + + +class NewTransferSent(LoggedPage, HTMLPage): + # STEP 9 for immediat - Confirmation de virement. + # STEP 10 for "programme" + def get_errors(self): + return CleanText('//div[@class="form-errors"]//li')(self.doc) + + +class TransferCharacteristics(LoggedPage, HTMLPage): def get_option(self, select, text): for opt in select.xpath('option'): if opt.text_content() == text: @@ -1403,7 +1549,7 @@ def submit(self): class TransferSent(LoggedPage, HTMLPage): - def get_transfer_error(self): + def get_errors(self): return CleanText('//form[@name="Confirm"]/div[@class="form-errors"]//li')(self.doc) @@ -1422,7 +1568,7 @@ def _is_form(self, **kwargs): return False return True - def is_charac(self): + def is_characteristics(self): return self._is_form(name='externalAccountsPrepareType') def submit_recipient(self, recipient): diff --git a/modules/bp/module.py b/modules/bp/module.py index 57c27b32b3..cb1782e122 100644 --- a/modules/bp/module.py +++ b/modules/bp/module.py @@ -21,6 +21,7 @@ from decimal import Decimal from datetime import timedelta + from .compat.weboob_capabilities_bank import CapBankTransferAddRecipient, Account, AccountNotFound, RecipientNotFound from .compat.weboob_capabilities_wealth import CapBankWealth from weboob.capabilities.contact import CapContact diff --git a/modules/bred/bred/browser.py b/modules/bred/bred/browser.py index da9966562e..77b30de47d 100644 --- a/modules/bred/bred/browser.py +++ b/modules/bred/bred/browser.py @@ -367,7 +367,16 @@ def get_profile(self): self.profile.go() profile = self.page.get_profile() - self.emails.go() + try: + self.emails.go() + except ClientError as e: + if e.response.status_code == 403: + msg = e.response.json().get('erreur', {}).get('libelle', '') + if msg == "Cette fonctionnalité n'est pas disponible avec votre compte.": + # We cannot get the mails, return the profile now. + return profile + raise + self.page.set_email(profile=profile) return profile @@ -382,9 +391,17 @@ def fill_account(self, account, fields): def iter_transfer_recipients(self, account): self.get_and_update_bred_token() - self.emitters_list.go(json={ - 'typeVirement': 'C', - }) + try: + self.emitters_list.go(json={ + 'typeVirement': 'C', + }) + except ClientError as e: + if e.response.status_code == 403: + msg = e.response.json().get('erreur', {}).get('libelle', '') + if msg == "Cette fonctionnalité n'est pas disponible avec votre compte.": + # Means the account cannot emit transfers + return + raise if not self.page.can_account_emit_transfer(account.id): return diff --git a/modules/bred/module.py b/modules/bred/module.py index 6cf3a190ce..5bae076247 100644 --- a/modules/bred/module.py +++ b/modules/bred/module.py @@ -58,8 +58,11 @@ class BredModule(Module, CapBankWealth, CapProfile, CapBankTransferAddRecipient) 'dispobank': DispoBankBrowser, } + def get_website(self): + return self.config['website'].get() + def create_default_browser(self): - self.BROWSER = self.BROWSERS[self.config['website'].get()] + self.BROWSER = self.BROWSERS[self.get_website()] return self.create_browser( self.config['accnum'].get().replace(' ', '').zfill(11), @@ -90,7 +93,7 @@ def get_profile(self): return self.browser.get_profile() def fill_account(self, account, fields): - if self.config['website'].get() != 'bred': + if self.get_website() != 'bred': return self.browser.fill_account(account, fields) @@ -100,7 +103,7 @@ def fill_account(self, account, fields): } def iter_transfer_recipients(self, account): - if self.config['website'].get() != 'bred': + if self.get_website() != 'bred': raise NotImplementedError() if not isinstance(account, Account): @@ -109,7 +112,7 @@ def iter_transfer_recipients(self, account): return self.browser.iter_transfer_recipients(account) def new_recipient(self, recipient, **params): - if self.config['website'].get() != 'bred': + if self.get_website() != 'bred': raise NotImplementedError() recipient.label = recipient.label[:32].strip() @@ -122,7 +125,7 @@ def new_recipient(self, recipient, **params): return self.browser.new_recipient(recipient, **params) def init_transfer(self, transfer, **params): - if self.config['website'].get() != 'bred': + if self.get_website() != 'bred': raise NotImplementedError() transfer.label = transfer.label[:140].strip() @@ -144,6 +147,6 @@ def init_transfer(self, transfer, **params): return self.browser.init_transfer(transfer, account, recipient, **params) def execute_transfer(self, transfer, **params): - if self.config['website'].get() != 'bred': + if self.get_website() != 'bred': raise NotImplementedError() return self.browser.execute_transfer(transfer, **params) diff --git a/modules/caissedepargne/browser.py b/modules/caissedepargne/browser.py index 7d11f97dda..81e7e54232 100644 --- a/modules/caissedepargne/browser.py +++ b/modules/caissedepargne/browser.py @@ -74,7 +74,6 @@ AppValidationPage, ) from .transfer_pages import CheckingPage, TransferListPage - from .linebourse_browser import LinebourseAPIBrowser @@ -772,7 +771,12 @@ def do_new_login(self, data): 'bpcesta': json.dumps(bpcesta, separators=(',', ':')), } if self.nuser: - params['login_hint'] += ' %s' % self.nuser + if len(self.username) == 10: + # We must fill with the missing 0 expected by the caissedepargne server + # Some clues are given in js file + params['login_hint'] += self.nuser.zfill(6) + else: + params['login_hint'] += ' %s' % self.nuser self.authorize.go(params=params) self.page.send_form() diff --git a/modules/caissedepargne/pages.py b/modules/caissedepargne/pages.py index 1332516876..1a9644972c 100644 --- a/modules/caissedepargne/pages.py +++ b/modules/caissedepargne/pages.py @@ -25,11 +25,12 @@ import re from base64 import b64decode from collections import OrderedDict -from PIL import Image, ImageFilter from io import BytesIO from decimal import Decimal from datetime import datetime + from lxml import html +from PIL import Image, ImageFilter from .compat.weboob_browser_pages import ( LoggedPage, HTMLPage, JsonPage, pagination, @@ -485,10 +486,12 @@ def need_auth(self): return bool(CleanText('//span[contains(text(), "Authentification non rejouable")]')(self.doc)) def check_no_loans(self): - return ( - not bool(CleanText('//table[@class="menu"]//div[contains(., "Crédits")]')(self.doc)) - and not bool(CleanText('//table[@class="header-navigation_main"]//a[contains(., "Crédits")]')(self.doc)) - ) + return not any(( + CleanText('//table[@class="menu"]//div[contains(., "Crédits")]')(self.doc), + CleanText( + '//table[@class="header-navigation_main"]//a[contains(@href, "CRESYNT0")]' + )(self.doc), + )) def check_measure_accounts(self): return not CleanText( diff --git a/modules/cic/browser.py b/modules/cic/browser.py index 018669f043..a874aafe72 100644 --- a/modules/cic/browser.py +++ b/modules/cic/browser.py @@ -40,5 +40,5 @@ class CICBrowser(AbstractBrowser): LoginPage ) - decoupled_state = URL(r'/fr/otp/SOSD_OTP_GetTransactionState.htm', DecoupledStatePage) - cancel_decoupled = URL(r'/fr/otp/SOSD_OTP_CancelTransaction.htm', CancelDecoupled) + decoupled_state = URL(r'/(?P.*)fr/otp/SOSD_OTP_GetTransactionState.htm', DecoupledStatePage) + cancel_decoupled = URL(r'/(?P.*)fr/otp/SOSD_OTP_CancelTransaction.htm', CancelDecoupled) diff --git a/modules/cmes/browser.py b/modules/cmes/browser.py index 94ba874bad..b5e3df0ec9 100644 --- a/modules/cmes/browser.py +++ b/modules/cmes/browser.py @@ -103,16 +103,21 @@ def iter_accounts(self): return self.page.iter_accounts() + def go_investment(self, form, inv_param): + form[inv_param] = '' + form.submit() + @need_login def iter_investment(self, account): if 'compte courant bloqué' in account.label.lower(): # CCB accounts have Pockets but no Investments return self.accounts.stay_or_go(subsite=self.subsite, client_space=self.client_space) + form = self.page.get_investment_form() for inv in self.page.iter_investments(account=account): - if inv._url: - # Go to the investment details to get employee savings attributes - self.location(inv._url) + # Go to the investment details to get employee savings attributes + self.go_investment(form, inv._form_param) + if self.investments.is_here(): asset_management_url = self.page.get_asset_management_url() # Fetch SRRI, asset category & recommended period @@ -126,7 +131,7 @@ def iter_investment(self, account): self.page.fill_investment(obj=inv) # We need to return to the investment page - self.location(inv._url) + self.go_investment(form, inv._form_param) else: performances = {} # Get 1-year performance @@ -151,13 +156,13 @@ def iter_investment(self, account): performances[3] = self.page.get_performance() inv.performance_history = performances - # Fetch investment quantity on the 'Mes Avoirs' tab + # Fetch investment quantity on the 'Mes Avoirs'/'Mon épargne' tab self.page.go_investment_details() inv.quantity = self.page.get_quantity() self.page.go_back() - else: self.logger.info('No available details for investment %s.', inv.label) + self.accounts.stay_or_go(subsite=self.subsite, client_space=self.client_space) yield inv @need_login diff --git a/modules/cmes/pages.py b/modules/cmes/pages.py index 84fae45e3d..bb06afdbf4 100644 --- a/modules/cmes/pages.py +++ b/modules/cmes/pages.py @@ -116,18 +116,22 @@ def iter_invest_rows(self, account): row.xpath('//div[contains(@id, "dv::s::%s")]' % id_diff[0].rsplit(':', 1)[0])[0] if id_diff else None, ) + def get_investment_form(self): + form = self.get_form(id='I0:P5:F') + # Each investment uses the same form with a different submit input. + # We remove all relevant inputs and will add the one we want manually as we submit the form. + keys_to_remove = [key for key in form if key.startswith('_FID_')] + for key in keys_to_remove: + form.pop(key) + return form + def iter_investments(self, account): for row, elem_repartition, elem_pocket, elem_diff in self.iter_invest_rows(account=account): inv = Investment() inv._account = account inv._el_pocket = elem_pocket inv.label = CleanText('.//td[1]')(row) - _url = Link('.//td[1]/a', default=None)(row) - if _url: - inv._url = self.absurl(_url) - else: - # If _url is None, self.absurl returns the BASEURL, so we need to set the value manually. - inv._url = None + inv._form_param = CleanText('.//td[1]/input/@name')(row) inv.valuation = MyDecimal('.//td[2]')(row) # On all Cmes children the row shows percentages and the popup shows absolute values in currency. @@ -214,7 +218,7 @@ def get_performance(self): return Eval(lambda x: x/100, CleanDecimal.French('//p[contains(@class, "plusvalue--value")]'))(self.doc) def go_investment_details(self): - investment_details_url = Link('//a[text()="Mes avoirs"]')(self.doc) + investment_details_url = Link('//a[text()="Mes avoirs" or text()="Mon épargne"]')(self.doc) self.browser.location(investment_details_url) diff --git a/modules/cmso/par/pages.py b/modules/cmso/par/pages.py index c8fd41b893..c44b396e18 100644 --- a/modules/cmso/par/pages.py +++ b/modules/cmso/par/pages.py @@ -22,13 +22,13 @@ from __future__ import unicode_literals import re -import requests import json -import datetime as dt +import datetime from hashlib import md5 - from collections import OrderedDict +import requests + from weboob.exceptions import BrowserUnavailable from .compat.weboob_browser_pages import HTMLPage, JsonPage, RawPage, LoggedPage, pagination from weboob.browser.elements import DictElement, ItemElement, TableElement, SkipItem, method @@ -366,14 +366,14 @@ def obj_maturity_date(self): # Key not always available, when revolving credit not yet consummed timestamp = Dict('dateFin', default=None)(self) if timestamp: - return dt.date.fromtimestamp(timestamp / 1000) + return datetime.date.fromtimestamp(timestamp / 1000) return NotAvailable def obj_next_payment_date(self): # Key not always available, when revolving credit not yet consummed timestamp = Dict('dateProchaineEcheance', default=None)(self) if timestamp: - return dt.date.fromtimestamp(timestamp / 1000) + return datetime.date.fromtimestamp(timestamp / 1000) return NotAvailable def obj_balance(self): @@ -448,7 +448,7 @@ class item(ItemElement): class FromTimestamp(Filter): def filter(self, timestamp): try: - return dt.date.fromtimestamp(int(timestamp[:-3])) + return datetime.date.fromtimestamp(int(timestamp[:-3])) except TypeError: return self.default_or_raise(ParseError('Element %r not found' % self.selector)) diff --git a/modules/cmso/pro/browser.py b/modules/cmso/pro/browser.py index 096d04e1f2..cfa7c75c8a 100644 --- a/modules/cmso/pro/browser.py +++ b/modules/cmso/pro/browser.py @@ -22,9 +22,10 @@ from __future__ import unicode_literals import datetime -from dateutil.relativedelta import relativedelta import re +from dateutil.relativedelta import relativedelta + from weboob.tools.capabilities.bank.transactions import sorted_transactions from weboob.capabilities.base import find_object from .compat.weboob_capabilities_bank import Account @@ -38,7 +39,6 @@ LoginPage, PasswordCreationPage, AccountsPage, HistoryPage, SubscriptionPage, InvestmentPage, InvestmentAccountPage, UselessPage, SSODomiPage, AuthCheckUser, ErrorPage, ) - from ..par.pages import ProfilePage diff --git a/modules/cragr/browser.py b/modules/cragr/browser.py index f6f4abadda..e27c1a7988 100644 --- a/modules/cragr/browser.py +++ b/modules/cragr/browser.py @@ -58,9 +58,9 @@ VerifyNewRecipientPage, ValidateNewRecipientPage, CheckSmsPage, EndNewRecipientPage, ) - from .netfinca_browser import NetfincaBrowser + __all__ = ['CreditAgricoleBrowser'] diff --git a/modules/cragr/pages.py b/modules/cragr/pages.py index 380560a211..61e4cd123e 100644 --- a/modules/cragr/pages.py +++ b/modules/cragr/pages.py @@ -26,6 +26,7 @@ import json import dateutil + from .compat.weboob_browser_pages import HTMLPage, JsonPage, LoggedPage from weboob.exceptions import ActionNeeded from weboob.capabilities import NotAvailable @@ -247,6 +248,8 @@ class ContractsPage(LoggedPage, HTMLPage): 'OPTA': Account.TYPE_LIFE_INSURANCE, # Optalissime 'RENV VITAL': Account.TYPE_LIFE_INSURANCE, # Rente viagère Vitalité 'ANAE': Account.TYPE_LIFE_INSURANCE, + 'PAT STH': Account.TYPE_LIFE_INSURANCE, # Patrimoine ST Honoré + 'PRSH2': Account.TYPE_LIFE_INSURANCE, # Prestige ST Honoré 2 'ATOUT LIB': Account.TYPE_REVOLVING_CREDIT, 'PACC': Account.TYPE_CONSUMER_CREDIT, # 'PAC' = 'Prêt à consommer' 'PACP': Account.TYPE_CONSUMER_CREDIT, diff --git a/modules/creditmutuel/browser.py b/modules/creditmutuel/browser.py index 2ffb847237..903120c5c2 100644 --- a/modules/creditmutuel/browser.py +++ b/modules/creditmutuel/browser.py @@ -87,8 +87,8 @@ class CreditMutuelBrowser(TwoFactorBrowser): twofa_unabled_page = URL(r'/(?P.*)fr/banque/validation.aspx', TwoFAUnabledPage) mobile_confirmation = URL(r'/(?P.*)fr/banque/validation.aspx', MobileConfirmationPage) safetrans_page = URL(r'/(?P.*)fr/banque/validation.aspx', SafeTransPage) - decoupled_state = URL(r'/fr/banque/async/otp/SOSD_OTP_GetTransactionState.htm', DecoupledStatePage) - cancel_decoupled = URL(r'/fr/banque/async/otp/SOSD_OTP_CancelTransaction.htm', CancelDecoupled) + decoupled_state = URL(r'/(?P.*)fr/banque/async/otp/SOSD_OTP_GetTransactionState.htm', DecoupledStatePage) + cancel_decoupled = URL(r'/(?P.*)fr/banque/async/otp/SOSD_OTP_CancelTransaction.htm', CancelDecoupled) otp_validation_page = URL(r'/(?P.*)fr/banque/validation.aspx', OtpValidationPage) otp_blocked_error_page = URL(r'/(?P.*)fr/banque/validation.aspx', OtpBlockedErrorPage) fiscality = URL(r'/(?P.*)fr/banque/residencefiscale.aspx', FiscalityConfirmationPage) @@ -299,10 +299,10 @@ def poll_decoupled(self, transactionId): """ # 15' on website, we don't wait that much, but leave sufficient time for the user timeout = time.time() + 600.00 # 15' on webview, need not to wait that much + data = {'transactionId': transactionId} while time.time() < timeout: - data = {'transactionId': transactionId} - self.decoupled_state.go(data=data) + self.decoupled_state.go(data=data, subbank=self.currentSubBank) decoupled_state = self.page.get_decoupled_state() if decoupled_state == 'VALIDATED': @@ -312,10 +312,10 @@ def poll_decoupled(self, transactionId): raise AppValidationCancelled() assert decoupled_state == 'PENDING', 'Unhandled polling state: "%s"' % decoupled_state - time.sleep(5) # every second on wbesite, need to slow that down + time.sleep(5) # every second on website, need to slow that down # manually cancel polling before website max duration for it - self.cancel_decoupled.go(data=data) + self.cancel_decoupled.go(data=data, subbank=self.currentSubBank) raise AppValidationExpired() def handle_polling(self): @@ -368,6 +368,8 @@ def check_redirections(self): self.location(location, allow_redirects=False) def check_auth_methods(self): + self.getCurrentSubBank() + if self.mobile_confirmation.is_here(): self.page.check_bypass() if self.mobile_confirmation.is_here(): diff --git a/modules/creditmutuel/pages.py b/modules/creditmutuel/pages.py index 56453262f2..a55d32b385 100644 --- a/modules/creditmutuel/pages.py +++ b/modules/creditmutuel/pages.py @@ -231,10 +231,11 @@ class CancelDecoupled(HTMLPage): # and might be empty of text while used in a redirection class OtpValidationPage(PartialHTMLPage): def is_here(self): - return 'envoyé par SMS' in CleanText('//div[contains(@id, "OTPDeliveryChannelText")]')(self.doc) + return 'code de confirmation vient de vous être envoyé par' in CleanText('//div[contains(@id, "OTPDeliveryChannelText")]')(self.doc) def get_message(self): # Ex: 'Un code de confirmation vient de vous être envoyé par SMS au 06 XX XX X1 23, le jeudi 26 décembre 2019 à 18:12:56.' + # can be 'par SMS', 'par appel téléphonique', or 'par email' return Regexp(CleanText('//div[contains(@id, "OTPDeliveryChannelText")]'), r'(.+\d{2}), le')(self.doc) def get_error_message(self): diff --git a/modules/hsbc/browser.py b/modules/hsbc/browser.py index 8b6966bd3b..5b2b9f831e 100644 --- a/modules/hsbc/browser.py +++ b/modules/hsbc/browser.py @@ -22,16 +22,19 @@ from __future__ import unicode_literals import re +from collections import OrderedDict from datetime import timedelta, date + from lxml.etree import XMLSyntaxError -from collections import OrderedDict from weboob.tools.date import LinearDateGuesser from .compat.weboob_capabilities_bank import Account, AccountNotFound, AccountOwnership from weboob.tools.capabilities.bank.transactions import sorted_transactions, keep_only_card_transactions from weboob.tools.compat import parse_qsl, urlparse -from weboob.exceptions import ActionNeeded, BrowserIncorrectPassword, BrowserUnavailable -from weboob.browser.browsers import LoginBrowser, URL, need_login +from .compat.weboob_tools_value import Value +from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable, BrowserQuestion +from weboob.browser.browsers import URL, need_login +from .compat.weboob_browser_browsers import TwoFactorBrowser from weboob.browser.exceptions import HTTPNotFound from weboob.capabilities.base import find_object @@ -53,8 +56,9 @@ __all__ = ['HSBC'] -class HSBC(LoginBrowser): +class HSBC(TwoFactorBrowser): BASEURL = 'https://client.hsbc.fr' + HAS_CREDENTIALS_ONLY = True app_gone = False @@ -140,8 +144,9 @@ class HSBC(LoginBrowser): # catch-all other_page = URL(r'/cgi-bin/emcgi', OtherPage) - def __init__(self, username, password, secret, *args, **kwargs): - super(HSBC, self).__init__(username, password, *args, **kwargs) + def __init__(self, config, username, password, secret, *args, **kwargs): + self.config = config + super(HSBC, self).__init__(config, username, password, *args, **kwargs) self.accounts_dict = OrderedDict() self.unique_accounts_dict = dict() self.secret = secret @@ -149,11 +154,22 @@ def __init__(self, username, password, secret, *args, **kwargs): self.owners_url_list = [] self.web_space = None self.home_url = None + self.AUTHENTICATION_METHODS = { + 'otp': self.handle_otp, + } def load_state(self, state): + # when the otp is being handled, we want to keep the same session + if self.config['otp'].get(): + return super(HSBC, self).load_state(state) return - def do_login(self): + def handle_otp(self): + otp = self.config['otp'].get() + self.page.login_with_secure_key(self.secret, otp) + self.end_login() + + def init_login(self): self.session.cookies.clear() self.app_gone = False @@ -166,14 +182,26 @@ def do_login(self): self.connection2.go() self.page.login(self.username) - + # The handling of 2FA is unusual. When authenticating, the user has the choice to use an OTP or his password + # when the sca is required, the link to log on the website without otp is not available. That's how we know + # this is the only available authentication method. no_secure_key_link = self.page.get_no_secure_key_link() - if not no_secure_key_link and self.page.is_secure_key(): - raise ActionNeeded("Vous devez réaliser l'authentification forte sur le portail internet avec Secure Key") - - self.location(no_secure_key_link) + # to test the sca, just invert the following if condition, authentication using an otp is always available + if no_secure_key_link: + self.location(no_secure_key_link) + else: + self.check_interactive() + raise BrowserQuestion( + Value( + 'otp', + label='''Veuillez entrer un code à usage unique à générer depuis votre application HSBC (bouton "Générer un code à usage unique" sur la page de login de l'application)''', + ) + ) self.page.login_w_secure(self.password, self.secret) + self.end_login() + + def end_login(self): for _ in range(3): if self.login.is_here(): self.page.useless_form() diff --git a/modules/hsbc/compat/weboob_browser_browsers.py b/modules/hsbc/compat/weboob_browser_browsers.py new file mode 100644 index 0000000000..2c6f68e296 --- /dev/null +++ b/modules/hsbc/compat/weboob_browser_browsers.py @@ -0,0 +1,59 @@ +import weboob.browser.browsers as OLD + +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) + +try: + __all__ = OLD.__all__ +except AttributeError: + pass + + +# backport of GET/POST guessing 04681719ca54a061d2b4b35feabf484c80543cf1 +def build_request(self, url, referrer=None, data_encoding=None, **kwargs): + """ + Does the same job as open(), but returns a Request without + submitting it. + This allows further customization to the Request. + """ + if isinstance(url, requests.Request): + req = url + url = req.url + else: + req = requests.Request(url=url, **kwargs) + + # guess method + if req.method is None: + # 'data' and 'json' (even if empty) are (always?) passed to build_request + # and None is their default. For a Request object, the defaults are different. + # Request.json is None and Request.data == [] by default. + # Could they break unexpectedly? + if ( + req.data or kwargs.get('data') is not None + or req.json or kwargs.get('json') is not None + ): + req.method = 'POST' + else: + req.method = 'GET' + + # convert unicode strings to proper encoding + if isinstance(req.data, unicode) and data_encoding: + req.data = req.data.encode(data_encoding) + if isinstance(req.data, dict) and data_encoding: + req.data = OrderedDict([(k, v.encode(data_encoding) if isinstance(v, unicode) else v) + for k, v in req.data.items()]) + + if referrer is None: + referrer = self.get_referrer(self.url, url) + if referrer: + # Yes, it is a misspelling. + req.headers.setdefault('Referer', referrer) + + return req + + +OLD.Browser.build_request = build_request + + +del OLD diff --git a/modules/hsbc/module.py b/modules/hsbc/module.py index 43f16e6b74..d5eb2ae73e 100644 --- a/modules/hsbc/module.py +++ b/modules/hsbc/module.py @@ -25,8 +25,9 @@ from .compat.weboob_capabilities_wealth import CapBankWealth from weboob.capabilities.base import find_object from weboob.tools.backend import Module, BackendConfig -from .compat.weboob_tools_value import ValueBackendPassword +from .compat.weboob_tools_value import ValueBackendPassword, ValueTransient from weboob.capabilities.profile import CapProfile + from .browser import HSBC @@ -44,11 +45,14 @@ class HSBCModule(Module, CapBankWealth, CapProfile): ValueBackendPassword('login', label='Identifiant', masked=False), ValueBackendPassword('password', label='Mot de passe'), ValueBackendPassword('secret', label=u'Réponse secrète'), + ValueTransient('otp'), + ValueTransient('request_information'), ) BROWSER = HSBC def create_default_browser(self): return self.create_browser( + self.config, self.config['login'].get(), self.config['password'].get(), self.config['secret'].get() diff --git a/modules/hsbc/pages/account_pages.py b/modules/hsbc/pages/account_pages.py index f9d1e2c1e0..4bc00ad9fd 100644 --- a/modules/hsbc/pages/account_pages.py +++ b/modules/hsbc/pages/account_pages.py @@ -37,6 +37,7 @@ from weboob.exceptions import ActionNeeded, BrowserIncorrectPassword, BrowserUnavailable from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.tools.compat import urljoin + from .landing_pages import GenericLandingPage @@ -536,6 +537,7 @@ def on_load(self): 'Please enter valid credentials for memorable answer and password.', 'Please enter a valid Username.', 'mot de passe invalide', + 'Log on error', # wrong otp ] ): raise BrowserIncorrectPassword(error_msg) @@ -582,6 +584,12 @@ def login_w_secure(self, password, secret): form['password'] = split_pass form.submit() + def login_with_secure_key(self, secret, otp): + form = self.get_form(nr=0) + form['memorableAnswer'] = secret + form['idv_OtpCredential'] = otp + form.submit() + def useless_form(self): form = self.get_form(nr=0) # There is space added at the end of the url diff --git a/modules/hsbc/pages/investments.py b/modules/hsbc/pages/investments.py index 463bbface3..92938eb97d 100644 --- a/modules/hsbc/pages/investments.py +++ b/modules/hsbc/pages/investments.py @@ -25,11 +25,10 @@ import json import time -from weboob.capabilities import NotAvailable +from weboob.capabilities.base import NotAvailable from .compat.weboob_capabilities_bank import Account from .compat.weboob_capabilities_wealth import Investment from weboob.tools.capabilities.bank.investments import is_isin_valid - from weboob.browser.elements import ItemElement, TableElement, DictElement, method from .compat.weboob_browser_pages import HTMLPage, JsonPage, LoggedPage from .compat.weboob_browser_filters_standard import ( diff --git a/modules/ing/api/transfer_page.py b/modules/ing/api/transfer_page.py index aa6f539cfb..0f578299d8 100644 --- a/modules/ing/api/transfer_page.py +++ b/modules/ing/api/transfer_page.py @@ -24,10 +24,10 @@ import random from datetime import datetime from io import BytesIO + from PIL import Image, ImageFilter from weboob.tools.captcha.virtkeyboard import SimpleVirtualKeyboard - from .compat.weboob_browser_pages import LoggedPage, JsonPage from weboob.browser.elements import method, DictElement, ItemElement from weboob.browser.filters.json import Dict diff --git a/modules/ing/test.py b/modules/ing/test.py index 7d278b4752..6dfe8e9e37 100644 --- a/modules/ing/test.py +++ b/modules/ing/test.py @@ -19,11 +19,12 @@ # flake8: compatible -from weboob.tools.test import BackendTest -from .compat.weboob_capabilities_bank import Account, Transaction from datetime import timedelta import random +from weboob.tools.test import BackendTest +from .compat.weboob_capabilities_bank import Account, Transaction + class INGTest(BackendTest): MODULE = 'ing' diff --git a/modules/lcl/browser.py b/modules/lcl/browser.py index 195c41dd2b..0a7eef2866 100644 --- a/modules/lcl/browser.py +++ b/modules/lcl/browser.py @@ -27,12 +27,15 @@ from functools import wraps from dateutil.relativedelta import relativedelta + from weboob.exceptions import ( BrowserIncorrectPassword, BrowserUnavailable, - AuthMethodNotImplemented, ActionNeeded, + AuthMethodNotImplemented, BrowserQuestion, + AppValidation, AppValidationCancelled, AppValidationExpired, ) -from weboob.browser.browsers import LoginBrowser, URL, need_login, StatesMixin -from weboob.browser.exceptions import ServerError +from weboob.browser.browsers import URL, need_login +from .compat.weboob_browser_browsers import TwoFactorBrowser +from weboob.browser.exceptions import ServerError, ClientError from weboob.capabilities.base import NotAvailable from .compat.weboob_capabilities_bank import ( Account, AddRecipientBankError, AddRecipientStep, Recipient, AccountOwnerType, @@ -47,10 +50,10 @@ from .pages import ( LoginPage, AccountsPage, AccountHistoryPage, ContractsPage, ContractsChoicePage, BoursePage, AVPage, AVDetailPage, DiscPage, NoPermissionPage, RibPage, HomePage, LoansPage, TransferPage, - AddRecipientPage, RecipientPage, RecipConfirmPage, SmsPage, RecipRecapPage, LoansProPage, + AddRecipientPage, RecipientPage, SmsPage, RecipConfirmPage, RecipRecapPage, LoansProPage, Form2Page, DocumentsPage, ClientPage, SendTokenPage, CaliePage, ProfilePage, DepositPage, AVHistoryPage, AVInvestmentsPage, CardsPage, AVListPage, CalieContractsPage, RedirectPage, - MarketOrdersPage, AVNotAuthorized, AVReroute, + MarketOrdersPage, AVNotAuthorized, AVReroute, TwoFAPage, AuthentStatusPage, FinalizeTwoFAPage, ) @@ -58,9 +61,10 @@ # Browser -class LCLBrowser(LoginBrowser, StatesMixin): +class LCLBrowser(TwoFactorBrowser): BASEURL = 'https://particuliers.secure.lcl.fr' STATE_DURATION = 15 + HAS_CREDENTIALS_ONLY = True login = URL( r'/outil/UAUT\?from=/outil/UWHO/Accueil/', @@ -185,20 +189,40 @@ class LCLBrowser(LoginBrowser, StatesMixin): DepositPage ) + # StrongAuth + authent_status_page = URL( + r'https://afafc.lcl.fr//wsafafc/api/v1/authentications/(?P.*)/status\?_=(?P\w+)', + AuthentStatusPage + ) + twofa_page = URL(r'/outil/UWAF/AuthentForteDesktop/authenticate', TwoFAPage) + finalize_twofa_page = URL( + r'/outil/UWAF/AuthentForteDesktop/finalisation', + FinalizeTwoFAPage + ) + __states__ = ('contracts', 'current_contract', 'parsed_contracts') IDENTIFIANT_ROUTING = 'CLI' - def __init__(self, *args, **kwargs): - super(LCLBrowser, self).__init__(*args, **kwargs) + def __init__(self, config, *args, **kwargs): + self.config = config + kwargs['username'] = self.config['login'].get() + kwargs['password'] = self.config['password'].get() + + super(LCLBrowser, self).__init__(config, *args, **kwargs) self.accounts_list = None self.current_contract = None self.contracts = [] self.parsed_contracts = False self.owner_type = AccountOwnerType.PRIVATE + self.AUTHENTICATION_METHODS = { + 'resume': self.handle_polling, + 'code': self.handle_sms, + } + def load_state(self, state): - if 'envoiCodeOtp' in state.get('url', ''): + if 'CodeOtp' in state.get('url', ''): state.pop('url') super(LCLBrowser, self).load_state(state) @@ -209,7 +233,7 @@ def load_state(self, state): if self.current_contract: self.current_contract = unicode(self.current_contract) - def do_login(self): + def init_login(self): assert isinstance(self.username, basestring) assert isinstance(self.password, basestring) @@ -231,9 +255,9 @@ def do_login(self): if self.response.status_code == 302: if 'AuthentForteDesktop' in self.response.headers['location']: # If we follow the redirection we will get a 2fa - # The 2fa validation is crossbrowser, for now we raise an ActionNeeded - # TODO Handle SMS and appvalidation - raise ActionNeeded("Veuillez vous identifier sur le site web LCL depuis votre navigateur habituel afin de réaliser l'authentification forte") + # The 2fa validation is crossbrowser + self.check_interactive() + self.two_factor_authentication() else: # If we're not redirected to 2fa page, it's likely to be the home page and we're logged in self.location(self.response.headers['location']) @@ -251,6 +275,64 @@ def do_login(self): self.accounts.stay_or_go() + def two_factor_authentication(self): + self.twofa_page.go() + authent_mechanism = self.page.get_authent_mechanism() + if authent_mechanism == 'otp_sms': + phone = self.page.get_phone_attributes() + + # Send sms to user. + self.location( + '/outil/UWAF/Otp/envoiCodeOtp', + params={'telChoisi': phone['attr_id'], '_': int(round(time.time() * 1000))} + ) + + self.page.check_otp_error() + raise BrowserQuestion( + Value( + 'code', + label="Veuillez saisir le code qui vient d'être envoyé sur le numéro %s" % phone['number'] + ) + ) + elif authent_mechanism == 'app_validation': + msg = self.page.get_app_validation_msg() + raise AppValidation( + msg or 'Veuillez valider votre connexion depuis votre application mobile LCL' + ) + else: + raise AssertionError("Strong authentication '%s' not handled" % authent_mechanism) + + def handle_polling(self): + match = re.search(r'var requestId = "([^"]+)"', self.page.text) + assert match, "request id not found in the javascript" + request_id = match.group(1) + + timeout = time.time() + 300 # 5 minutes + while time.time() < timeout: + try: + status = self.authent_status_page.go( + request_id=request_id, + timestamp=int(round(time.time() * 1000)) # current timestamp with millisecond + ).get_status() + except ClientError as e: + if e.response.status_code == 400 and e.response.json()['codeError'] == "FCT_UID_UNKNOWN": + raise AppValidationExpired('La validation par application a expirée.') + raise + if status == "VALID": + self.finalize_twofa_page.go(params={'status': 'VALID'}) + break + elif status == "CANCELLED": + raise AppValidationCancelled() + + # on the website, the request is made every 5 seconds + time.sleep(5) + else: + raise AppValidationExpired('La validation par application a expirée.') + + def handle_sms(self): + self.location('/outil/UWAF/Otp/validationCodeOtp?codeOtp=%s' % self.code) + self.page.check_otp_error(otp_sent=True) + @need_login def connexion_bourse(self): self.location('/outil/UWBO/AccesBourse/temporisationCar?codeTicker=TICKERBOURSECLI') @@ -701,7 +783,7 @@ def iter_market_orders(self, account): def send_code(self, recipient, **params): self.location('/outil/UWAF/Otp/validationCodeOtp?codeOtp=%s' % params['code']) - self.page.check_error(otp_sent=True) + self.page.check_recip_error(otp_sent=True) self.recip_recap.go() error = self.page.get_error() @@ -711,7 +793,6 @@ def send_code(self, recipient, **params): self.page.check_values(recipient.iban, recipient.label) return self.get_recipient_object(recipient.iban, recipient.label) - @need_login def get_recipient_object(self, iban, label): r = Recipient() r.iban = iban @@ -754,7 +835,7 @@ def init_new_recipient(self, recipient, **params): ('_', int(round(time.time() * 1000))), ] self.location('/outil/UWAF/Otp/envoiCodeOtp', params=data) - self.page.check_error() + self.page.check_recip_error() raise AddRecipientStep( self.get_recipient_object(recipient.iban, recipient.label), Value('code', label='Saisissez le code.') diff --git a/modules/lcl/compat/weboob_browser_browsers.py b/modules/lcl/compat/weboob_browser_browsers.py new file mode 100644 index 0000000000..2c6f68e296 --- /dev/null +++ b/modules/lcl/compat/weboob_browser_browsers.py @@ -0,0 +1,59 @@ +import weboob.browser.browsers as OLD + +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) + +try: + __all__ = OLD.__all__ +except AttributeError: + pass + + +# backport of GET/POST guessing 04681719ca54a061d2b4b35feabf484c80543cf1 +def build_request(self, url, referrer=None, data_encoding=None, **kwargs): + """ + Does the same job as open(), but returns a Request without + submitting it. + This allows further customization to the Request. + """ + if isinstance(url, requests.Request): + req = url + url = req.url + else: + req = requests.Request(url=url, **kwargs) + + # guess method + if req.method is None: + # 'data' and 'json' (even if empty) are (always?) passed to build_request + # and None is their default. For a Request object, the defaults are different. + # Request.json is None and Request.data == [] by default. + # Could they break unexpectedly? + if ( + req.data or kwargs.get('data') is not None + or req.json or kwargs.get('json') is not None + ): + req.method = 'POST' + else: + req.method = 'GET' + + # convert unicode strings to proper encoding + if isinstance(req.data, unicode) and data_encoding: + req.data = req.data.encode(data_encoding) + if isinstance(req.data, dict) and data_encoding: + req.data = OrderedDict([(k, v.encode(data_encoding) if isinstance(v, unicode) else v) + for k, v in req.data.items()]) + + if referrer is None: + referrer = self.get_referrer(self.url, url) + if referrer: + # Yes, it is a misspelling. + req.headers.setdefault('Referer', referrer) + + return req + + +OLD.Browser.build_request = build_request + + +del OLD diff --git a/modules/lcl/module.py b/modules/lcl/module.py index 2c53c88097..18da2acd93 100644 --- a/modules/lcl/module.py +++ b/modules/lcl/module.py @@ -36,7 +36,7 @@ from weboob.capabilities.profile import CapProfile from weboob.tools.backend import Module, BackendConfig from weboob.tools.capabilities.bank.transactions import sorted_transactions -from .compat.weboob_tools_value import ValueBackendPassword, Value +from .compat.weboob_tools_value import ValueBackendPassword, Value, ValueTransient from weboob.capabilities.base import ( find_object, strict_find_object, NotAvailable, empty, ) @@ -82,7 +82,10 @@ class LCLModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapContact, 'esp': 'Espace Pro', }, aliases={'elcl': 'par'} - ) + ), + ValueTransient('resume'), + ValueTransient('request_information'), + ValueTransient('code', regexp=r'^\d{6}$'), ) BROWSER = LCLBrowser @@ -104,8 +107,7 @@ def create_default_browser(self): ) return self.create_browser( - self.config['login'].get(), - self.config['password'].get() + self.config, ) def iter_accounts(self): diff --git a/modules/lcl/pages.py b/modules/lcl/pages.py index 24f859fbe1..5e703f1aae 100644 --- a/modules/lcl/pages.py +++ b/modules/lcl/pages.py @@ -21,7 +21,6 @@ from __future__ import unicode_literals, division import re -import requests import base64 import math import random @@ -30,6 +29,7 @@ from datetime import datetime, timedelta from dateutil.relativedelta import relativedelta +import requests from weboob.capabilities.base import empty, find_object, NotAvailable from .compat.weboob_capabilities_bank import ( @@ -1682,7 +1682,7 @@ def validate(self, iban, label): form.submit() -class CheckValuesPage(LoggedPage, HTMLPage): +class CheckValuesPage(HTMLPage): def get_error(self): return CleanText('//div[@id="attTxt"]/p')(self.doc) @@ -1762,16 +1762,61 @@ class get_item(ItemElement): obj_subscriber = CleanText('//li[@id="nomClient"]', replace=[('M', ''), ('Mme', '')]) -class RecipConfirmPage(CheckValuesPage): - pass +class RecipConfirmPage(LoggedPage, CheckValuesPage): + def is_here(self): + return CleanText( + '//div[@id="componentContainer"]//div[contains(text(), "Compte bénéficiaire de virement à ajouter à votre contrat")]', + default=None + )(self.doc) + + +class TwoFAPage(CheckValuesPage): + def is_here(self): + return CleanText( + '''//div[@id="componentContainer"]//h1[contains(text(), "BIENVENUE SUR L'ESPACE DE CONNEXION")]''', + default=None + )(self.doc) + + def get_phone_attributes(self): + # The number which begin by 06 or 07 is not always referred as MOBILE number + # this function parse the html tag of the phone number which begins with 06 or 07 + # to determine the canal attributed by the website, it can be MOBILE or FIXE + phone = {} + for phone_tag in self.doc.xpath('//div[@class="choixTel"]//div[@class="selectTel"]'): + phone['attr_id'] = Attr('.', 'id')(phone_tag) + phone['number'] = CleanText('.//a[@id="fixIpad"]')(phone_tag) + if phone['number'].startswith('06') or phone['number'].startswith('07'): + # Let's take the first mobile phone + # If no mobile phone is available, we take last phone found (ex: 01) + break + assert phone['attr_id'], 'no phone found for 2FA' + canal = re.match('envoi(Fixe|Mobile)', phone['attr_id']) + assert canal, 'Canal unknown %s' % phone['attr_id'] + phone['attr_id'] = canal.group(1).upper() + return phone + + def get_app_validation_msg(self): + return CleanText( + '//form[@id="formNoSend"]//div[@id="polling"]//div[contains(text(), "application")]' + )(self.doc) class RecipientPage(LoggedPage, HTMLPage): pass -class SmsPage(LoggedPage, HTMLPage): - def check_error(self, otp_sent=False): +class SmsPage(HTMLPage): + def check_otp_error(self, otp_sent=False): + # This page contains only 'true' or 'false' + result = CleanText('.')(self.doc) == 'true' + + if not result and otp_sent: + raise BrowserIncorrectPassword( + "Le code saisi ne correspond pas à celui qui vient de vous être envoyé par téléphone. Vérifiez votre code et saisissez-le à nouveau." + ) + assert result, 'Something went wrong during login sent otp sms' + + def check_recip_error(self, otp_sent=False): # This page contains only 'true' or 'false' result = CleanText('.')(self.doc) == 'true' @@ -1780,7 +1825,7 @@ def check_error(self, otp_sent=False): assert result, 'Something went wrong during add new recipient sent otp sms' -class RecipRecapPage(CheckValuesPage): +class RecipRecapPage(LoggedPage, CheckValuesPage): pass @@ -1831,3 +1876,12 @@ def obj_ownership(self): def set_deposit_account_id(self, account): account.id = CleanText('//td[contains(text(), "N° contrat")]/following::td[1]//b')(self.doc) + + +class AuthentStatusPage(JsonPage): + def get_status(self): + return self.doc['status'] + + +class FinalizeTwoFAPage(HTMLPage): + pass diff --git a/modules/myedenred/browser.py b/modules/myedenred/browser.py index 3330d6b520..bd114dc665 100644 --- a/modules/myedenred/browser.py +++ b/modules/myedenred/browser.py @@ -25,10 +25,15 @@ from weboob.browser import URL, PagesBrowser from weboob.browser.browsers import OAuth2PKCEMixin -from weboob.exceptions import BrowserIncorrectPassword, NocaptchaQuestion, WrongCaptchaResponse -from weboob.browser.exceptions import ServerError, ClientError +from weboob.exceptions import BrowserIncorrectPassword, NocaptchaQuestion, WrongCaptchaResponse, ActionNeeded +from weboob.browser.exceptions import ServerError, ClientError, BrowserUnavailable +from weboob.tools.decorators import retry -from .pages import LoginPage, AccountsPage, TransactionsPage, JsParamsPage, JsUserPage, HomePage +from .pages import ( + LoginPage, AccountsPage, TransactionsPage, + JsParamsPage, JsUserPage, HomePage, + AuthorizePage, +) def need_login(func): @@ -44,7 +49,6 @@ def wrapper(self, *args, **kwargs): class MyedenredBrowser(OAuth2PKCEMixin, PagesBrowser): BASEURL = 'https://app-container.eu.edenred.io' - AUTHORIZATION_URI = 'https://sso.eu.edenred.io/connect/authorize' ACCESS_TOKEN_URI = 'https://sso.eu.edenred.io/connect/token' redirect_uri = 'https://www.myedenred.fr/connect' @@ -58,6 +62,7 @@ class MyedenredBrowser(OAuth2PKCEMixin, PagesBrowser): ) params_js = URL(r'https://www.myedenred.fr/js/parameters.(?P\w+).js', JsParamsPage) connexion_js = URL(r'https://myedenred.fr/js/connexion.(?P\w+).js', JsUserPage) + authorize = URL(r'https://sso.eu.edenred.io/connect/authorize', AuthorizePage) def __init__(self, config, *args, **kwargs): super(MyedenredBrowser, self).__init__(*args, **kwargs) @@ -68,6 +73,7 @@ def __init__(self, config, *args, **kwargs): self._fetch_auth_parameters() + @retry(BrowserUnavailable) def _fetch_auth_parameters(self): self.home.go() params_random_str = self.page.get_href_randomstring('parameters') @@ -104,10 +110,11 @@ def build_authorization_parameters(self): } return params + @retry(BrowserUnavailable) def request_authorization(self): self.session.cookies.clear() - self.location(self.build_authorization_uri()) + self.authorize.go(params=self.build_authorization_parameters()) website_key = self.page.get_recaptcha_site_key() if not self.config['captcha_response'].get() and website_key: @@ -122,9 +129,11 @@ def request_authorization(self): if self.login.is_here(): message = self.page.get_error_message() if 'Couple Email' in message: - raise BrowserIncorrectPassword() + raise BrowserIncorrectPassword(message) elif 'validation du captcha' in message: raise WrongCaptchaResponse() + elif 'compte est verrouillé' in message: + raise ActionNeeded(message) raise AssertionError('Unhandled error at login: "%s".' % message) self.auth_uri = self.url diff --git a/modules/myedenred/pages.py b/modules/myedenred/pages.py index 22e5aacc89..88df68d963 100644 --- a/modules/myedenred/pages.py +++ b/modules/myedenred/pages.py @@ -31,6 +31,7 @@ CleanText, CleanDecimal, Currency, Field, Eval, Date, Regexp, ) +from weboob.browser.exceptions import BrowserUnavailable from weboob.browser.filters.html import Attr from weboob.browser.filters.json import Dict from .compat.weboob_capabilities_bank import Account, Transaction @@ -38,7 +39,13 @@ from weboob.tools.json import json -class HomePage(HTMLPage): +class RejectableHTMLPage(HTMLPage): + def on_load(self): + if CleanText('//title[text() = "Request Rejected"]')(self.doc): + raise BrowserUnavailable('Last request was rejected') + + +class HomePage(RejectableHTMLPage): def get_href_randomstring(self, filename): # The filename has a random string like `3eacdd2f` that changes often # (at least once a week). @@ -53,7 +60,7 @@ def get_href_randomstring(self, filename): return href.group(1) -class JsParamsPage(RawPage): +class JsParamsPage(RejectableHTMLPage): def get_json_content(self): json_data = re.search(r"JSON\.parse\('(.*)'\)", self.text) return json.loads(json_data.group(1)) @@ -78,7 +85,11 @@ def get_recaptcha_site_key(self): return Attr('//button[contains(@class, "g-recaptcha")]', 'data-sitekey', default=False)(self.doc) def get_error_message(self): - return CleanText('//div[@role="alert"]//li')(self.doc) + return CleanText('//div[@class="login-page"]/div[@role="alert"]//li')(self.doc) + + +class AuthorizePage(RejectableHTMLPage): + pass class AccountsPage(LoggedPage, JsonPage): diff --git a/modules/orange/browser.py b/modules/orange/browser.py index 1706f904b0..e366e6865f 100644 --- a/modules/orange/browser.py +++ b/modules/orange/browser.py @@ -19,12 +19,16 @@ from __future__ import unicode_literals +import random +from time import sleep + from requests.exceptions import ConnectTimeout from weboob.browser import LoginBrowser, URL, need_login, StatesMixin from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable, ActionNeeded, BrowserPasswordExpired from .pages import LoginPage, BillsPage -from .pages.login import ManageCGI, HomePage, PasswordPage, PortalPage, CaptchaPage +from .pages.captcha import OrangeCaptchaHandler, CaptchaPage +from .pages.login import ManageCGI, HomePage, PasswordPage, PortalPage from .pages.bills import ( SubscriptionsPage, SubscriptionsApiPage, BillsApiProPage, BillsApiParPage, ContractsPage, ContractsApiPage @@ -47,9 +51,10 @@ class OrangeBillBrowser(LoginBrowser, StatesMixin): home_page = URL(r'https://businesslounge.orange.fr/?$', HomePage) portal_page = URL(r'https://www.orange.fr/portail', PortalPage) - loginpage = URL( + login_page = URL( r'https://login.orange.fr/\?service=sosh&return_url=https://www.sosh.fr/', r'https://login.orange.fr/front/login', + r'https://login.orange.fr/$', LoginPage, ) password_page = URL(r'https://login.orange.fr/front/password', PasswordPage) @@ -99,9 +104,9 @@ def do_login(self): assert isinstance(self.username, basestring) assert isinstance(self.password, basestring) try: - self.loginpage.go() + self.login_page.go() if self.captcha_page.is_here(): - raise BrowserUnavailable() + self._handle_captcha() data = self.page.do_login_and_get_token(self.username, self.password) self.password_page.go(json=data) @@ -126,6 +131,23 @@ def get_nb_remaining_free_sms(self): def post_message(self, message, sender): raise NotImplementedError() + def _handle_captcha(self): + data_captcha = self.page.get_captcha_data() + + if not data_captcha: + raise BrowserUnavailable() + + images = self.page.download_images(data_captcha) + # captcha resolution takes about 50 milliseconds + self.captcha_handler = OrangeCaptchaHandler(self.logger, data_captcha['indications'], images) + captcha_response = self.captcha_handler.get_captcha_response() + + # we need to wait a little bit, because we are human after all^^ + waiting = random.randint(5000, 9000)/1000 + sleep(waiting) + body = {'value': captcha_response} + self.location('https://login.orange.fr/front/captcha', json=body) + def _iter_subscriptions_by_type(self, name, _type): self.location('https://espaceclientv3.orange.fr/?page=gt-home-page&%s' % _type) self.subscriptions.go() @@ -241,3 +263,14 @@ def get_profile(self): if not self.profile_par.is_here(): self.profile_pro.go() return self.page.get_profile() + + @retry(ServerError, delay=10) + @need_login + def download_document(self, document): + # sometimes the site sends us a server error when downloading the document. + # it is necessary to try again. + + if document._is_v2: + # get 404 without this header + return self.open(document.url, headers={'x-orange-caller-id': 'ECQ'}).content + return self.open(document.url).content diff --git a/modules/orange/module.py b/modules/orange/module.py index d423bf1bb5..bcc9bb623c 100644 --- a/modules/orange/module.py +++ b/modules/orange/module.py @@ -78,11 +78,7 @@ def download_document(self, document): document = self.get_document(document) if document.url is NotAvailable: return - - if document._is_v2: - # get 404 without this header - return self.browser.open(document.url, headers={'x-orange-caller-id': 'ECQ'}).content - return self.browser.open(document.url).content + return self.browser.download_document(document) def get_profile(self): return self.browser.get_profile() diff --git a/modules/orange/pages/captcha.py b/modules/orange/pages/captcha.py new file mode 100644 index 0000000000..84b2ad07ae --- /dev/null +++ b/modules/orange/pages/captcha.py @@ -0,0 +1,150 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012-2020 Budget Insight + +import json +import re + +from io import BytesIO +from PIL import Image + +from .compat.weboob_browser_pages import HTMLPage +from weboob.exceptions import BrowserUnavailable + +from .captcha_symbols import CAPTCHA_SYMBOLS + + +class OrangeCaptchaHandler(object): + symbols = CAPTCHA_SYMBOLS + + def __init__(self, logger, indications, images): + self.logger = logger + self.indications = indications # it contains 6 elements actually + self.fingerprints = {} + + for value, image_data in images.items(): + symbol = self.get_symbol_from_image_data(image_data) + best_indication, best_index = self.get_best_indication(symbol) + self.fingerprints[value] = { + 'value': value, + 'symbol': symbol, + 'best_indication': best_indication, + 'best_index': best_index, + } + + @staticmethod + def get_symbol_from_image_data(image_data): + img = Image.open(BytesIO(image_data)) + img = img.convert('RGB') + + small_image = img.resize((10, 10)) + + matrix = small_image.load() + symbol = "" + + for y in range(0, 10): + for x in range(0, 10): + (r, g, b) = matrix[x, y] + # If the pixel is "white" enough + if r + g + b > 384: + symbol += "1" + else: + symbol += "0" + + return symbol + + @staticmethod + def _get_similar_index(ref, value): + cpt = 0 + for r, v in zip(ref, value): + if r == v: + cpt += 1 + + return cpt + + def get_best_indication(self, symbol): + # indication is 'avion', 'chat', 'fleur', etc... + # this function tries to find which one is the good one based on his symbol + best_index = 0 + best_indication = None + for key, symbols in self.symbols.items(): + for sym in symbols: + index = self._get_similar_index(sym, symbol) + if index > best_index: + best_index = index + best_indication = key + + return best_indication, best_index + + def get_captcha_response(self): + captcha_response = [] + cache = {} # because we can have several time same indication in list: 'avion', 'fleur', 'avion', ... + + for indication in self.indications: + best_index = 0 + best_indication = None + good_key = None + + if indication in cache.keys(): + good_key = cache[indication]['value'] + best_indication = cache[indication]['best_indication'] + best_index = cache[indication]['best_index'] + else: + for key, value in self.fingerprints.items(): + if value['best_indication'] == indication and value['best_index'] > best_index: + cache[indication] = value + best_index = value['best_index'] + best_indication = value['best_indication'] + good_key = key + + if not good_key: + # we have failed to detect which key is the good one + raise BrowserUnavailable() + + if best_index < 90: + # index is always in [0:100], but when < 90 there is a strong chance image is not what we think + # we probably have failed to identify it correctly, maybe because we don't know it + # IF image isn't known + # add his symbol + # ELSE + # improve matching algorithm to have a better matching, + # but in that case DO NOT FORGET to rebuild all symbols + self.logger.error('best_indication: %s best_index: %d', best_indication, best_index) + raise BrowserUnavailable() + elif best_index < 95: + # there is a small chance image is not what we think it is, but not sure at all + # take the chance anyway + self.logger.warning('best_indication: %s best_index: %d', best_indication, best_index) + + captcha_response.append(good_key) + + return captcha_response + + +class CaptchaPage(HTMLPage): + def get_captcha_data(self): + scripts = self.doc.xpath('//script[contains(text(), "captchaOptions")]') + if not scripts: + return + + script = scripts[0] + value = re.search(r'config: (.*),', script.text).group(1) + data = json.loads(value) + + urls = {} + for row in data['rows']: + for col in row: + url = 'https:' + col['data'] + urls[col['value']] = url + + return { + 'indications': data['indications'], + 'urls': urls, + } + + def download_images(self, data_captcha): + images = {} + for key, url in data_captcha['urls'].items(): + images[key] = self.browser.open(url).content + + return images diff --git a/modules/orange/pages/captcha_symbols.py b/modules/orange/pages/captcha_symbols.py new file mode 100644 index 0000000000..b209e8232e --- /dev/null +++ b/modules/orange/pages/captcha_symbols.py @@ -0,0 +1,445 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012-2020 Budget Insight + +# orange can display a custom captcha +# it asks us to click on 6 images in a specific order that is given +# captcha contains 9 images, each one of them has some noise +# there are 17 kind of image so far, each type contains around 20~30 images all in 75*75 +# this file contains a custom hash for each image that is computed like this: +# +# noise of image has been removed (when enough version of a same image have been got to clean them with statistics) +# image have been reduced from 75*75 to 10*10 +# for each pixel: +# if withe enough: (r+g+b > 384) # 384 = (256*3/2) +# put 1 +# else: +# put 0 + +# CAUTION +# * it may miss some image +# * image marked with noise didn't have been cleaned, but should still works, +# (maybe with a very slightly high risk of failure, but still very low) +# * if you change matching algorithm (r+g+b > 384), theses symbols have to be changed as well + + +# I'M NOT A FUCKING ROBOT !!! +CAPTCHA_SYMBOLS = { + 'avion': [ + '1111111111111111111111111111001111111111111101111111101111000000000001110100000111000000001111111010', + '0000000000000010000000011011000111001111010000011100001100010110011101011011111100001111100001111110', + '1000000000111100000011110000001111010001101001100011110001001111011110111111011111111111111111111101', + '0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', + '1111111111111111111111101101110011001111111000011101111111110011111111000101111110001110110000000000', + '0111111111111111111011111111101111111101111101110011111101001111110000111111000000110000000000000000', + '1111111111111111111111111111111101111111111010000011011110011110001101110000000001111011011111111111', + '0000000000000000000000000000000000000000000001100000000001110000000011000000000000000000000000000000', + '1111111111111111111111111111111110110111000000100000000001000000000001000000000100000000110000000000', + '0110000000111110000011111000001111100000111010110011111110001111111111111111111111111111111111100000', + '0000000000000000000000000100000000110000000100100000000110000000001100000000100000000000000000000000', + '1111111111111111111111111111111110111110110011110111011101111100000011111111111111111111111111111111', + '0000000000000000000000000000000000011000000000100011111000111110111000000011000000000000000000000000', + '0000111111000011111100001111110000111111000000111100000000010000100100000000001000000000010000001000', + '1111111111111111111111111111111111111111111001111100000000001111101111111111111111111111110111111110', + '1111111111100111111111001011111110001111111100011110100001111100001111010001111100001110101111111111', # noise + '1111111111111111111111111111111111111111111111111100010110001111111111000000010000000100001000000000', + '1111111111111111111111111111111111111111111111110100000000001111111000010000001000000000000000000000', + '1111111111111111111111111111111111110111111111011111111011111000000111111101111111111111111111111111', + '1111111111111111111111111111111111111111111111111111111011111111001111111111111100000000000000000000', + '1111111111111111111111101111111111000011111011111111111011111111111111111111111111111111111111111111', + '0000000000000000000000100000000001000000000000000000001000000000000000000000000000000000000000000000', + '1111111111111111111111111111111111111111111101111111100100011000000111111110111100111011110111111111', + '1111111111111111111111111111111111111111111011111111110011111100001111111111111111111111111111111111', + '0000111111011111111111111111111111111111100011111111111001111111111000000000000000000000000000000000', + '1111111111111111111111111111111111111111111111111111111111111111101111000000000011110100111111111111', + '1111111111111111111111111111111111011111011101111111100001111111011001111111111111111111111111111111', + '1111111111111111111111111111111111111111111011011111110111101110001101111111111111111111111111111111', + '1111111110111111111111111111111111111011110000011111111000000000000000000000000000000000000000000000' + ], + 'bateau': [ + '1111111111111000011101000000111000000001100000000111100000001110000001110000000011100000001110000000', + '1111111111111111111111111111111111111111111100001010000000011000000001000000000000000000001000000000', + '1100000000111111000011111111001111111111111110111111111011111111111111111111111100111110000001101000', + '1111111111111111111111111110001111100000101000000011111001001101011000111111001011111111111111111111', + '0000000000110000000011111111111111111111111111111111110111011111101101111100110000000000000000000000', + '1111111111111111011111111001111111101111111111101111111110111111111011111110110101000110000000000010', + '0000000000000000000000000000001110100000111101000011011010000000010000000000000000000000000000000000', + '1111111111111111111111111111111111111111111111111100001100010000000000000000000100000000000000000000', + '1111111111111111111111111111111110000011111000000111110000011111100011111110001111111000001111110111', + '1111111111111111111111111111111111111111111111111111111111111111111111000000000010000000000111111111', # noise + '0000000010000000011000000000000001000000011110100000011011000001110011000000011100000000000000000000', + '1111111111111111111111111111111111111111111111111111111111111111111111111101111111100000111010000011', + '1111111111111111111111111111111111111111111111110111101111010010101110100000000000000000000000010000', + '1111111111111111111111110001111111111111000110010101100000010000000001000000000100000000111000000001', + '1101111111111111111111111111111011111111110011111111110111111110000011111100000011110000001111100000', + '1111111111111111111111111111111111111111111101011111111101111111010111100001101110000000000000011100', + '1111111111111111111110000111110000000111100110000010011111001100001111001000001100011111111111111111' + ], + 'camion': [ + '0000000000000000000000000000000000100000101010001010111111100011111000000001100011110100001111111110', + '1111111111111111111111101011111100000011110000001110000000011100000001100000000110000000110000000000', + '1111111111111111010010100011111100001011110000011111100001100000001101110000001111111111111111111111', + '1111111111001000011100000000110000100111000110001100000010100000000110000000111100000000110000000011', + '1111111111111011111111000000111000000011000000001111000000101000000010110000000011111111100000000011', + '1111111111110000001111100000111110000011111000001110000101111001000011000000001100000000000000000000', + '1111111111111111111111111111111111111111111000111111110111110000000001000000010000000000010000000001', + '0000000000000000000000000000000001000000000100100000000111110000001111000000000000000000000000000000', + '0000100010000011001000001100110000000100000100000001010010100001001000001111110000000000000000000000', + '1111111111111111111111111111111001010111001101000100000010010000000000000000000000000000000000000000', + '1111111111111111111111111111110101110000000000000000111111100010000000000000000000000000000000000000', + '1111111111111111111111110001111011100111000010001100000000001000000000000100000001101000000010011000', + '1111100000101111110010000000000000000000110000111001100011100000000100000000000000000000000000000000', + '0000000000000000000000000000001110001001111110111111111011110110000010000000000000000000000000000000', + '1111111111111111111111101110100111100000111100000011000000010000000000000010010000001000010000000000', + '1111111111111111111111111111111111011111111000011111111001111111111111111100001100000000000000000000', + '1111001111111110111111111011111111001111111001111111000000000010000000010001000000000000000010000000', + '1111110100111111111111111100011111111101111111110110001001010010001001000000000000000000000000000000', + '1111111111110000111111100111111110100111111000001101000000001000100010100001111111100000011111111111', + '1111111111111111111100111100010001011100011011100000101010000001000000000000000100011000000000001110', + '0000000000000100001010000000111110001111110001111111000011111100000001100000011111101111111011111111', + '1111001111111111110010110101101110000111011000001110000000011100000000111000100011111111110111111001' + ], + 'chat': [ + '1111111111111100111111110011111111001111111000111111100001111110000111111000011111100011111111101111', + '0000001000000011100000010010000000011000000001000000000100000000110000000010100000001100000000110000', + '0000000000000000000000000000000000010100000000000000011000000000100000100000100011000000101111001001', + '1111111111111101111111111011111111111101111111101111110001111111001111111101111111111111110001011111', + '0000000000110000000011000100001101000000110111100011011101011000100101110000000111000011111101001101', + '1111111111111111011111100011111110010111111011011111100011111110000011111000001111100000111110000001', + '1111100111111110111011111011111111001101101000100011010101001111000000111100110010111111001111111111', + '0000000000000000100000001000000000100001000011000000011100000001110000000111000000011110100001111000', # noise + '0000000000000011011011111111101111110000011111100011111110001111110000110011000010000000000000000000', + '1111000111111110011111111001111111110011111111000111111110011111111001111110010011100001111111110011', + '1000011111000011111100000111110110011111010110111110110011110001001001001111011110110111111111111111', + '1111111111101100111110101011111101111111110111111111110011111111111111111110011111111111111111111111', + '1111111111111111111111111111111111111111111101111111110011111111001101111111010111111111111111111111', + '0000000000000000000000000000001000000000100000000011110000001110000010111010000001000000000000000000', + '1111110111111111011111001111011110001000111001101110000000101000000111111100101111111001111111111111', + '1111111111111111111111000011111100001111110000111111010110111000011010100000010011100010001111111111', # noise + '0000001110000000111100100111110100011011000000110100000111010000000011000000000000000000001001000000', + '1111111111111101111111100001111100000011110000001111111010111111111111111111111011111111111110000011', + '1111111111111111111111111101111000001111000011111100000011110110011111000000011100000011110000001111', + '0000000000000000001000111000000000111110000111110000011111100011111110001001001000000000000000000000', + '0100000011111000001100100010010110110110010000101110101000011110000000110000010011100110101110010100', + '1111111111111111111111111111111111101111110001111111100011111111111111111111111111111111111111111111', + '0000000000000000000000000000000010000000000010000000001100000011111000000101100000000000000000000000', + '0001001110000000100100100010000000000010100101000000000010001001000001000000000110001011000001100000', + '1111111111100010111100001011110000001111000001111100010111110001111111001111111100110111111011111111', + '1111111010111111111010001100001100111010000010010010111000001111111100111101000010111110001101100000', + '1111111111111111111111111111001110111100110001000111001100011101110001111111000111011110001111111111', + '1111111111111111111110101110111011111011101111111110111000111101011111111101100110010000001011000001', + '1111001110111001111111101101111101111111101111010010000101000000111111000001111101000111110100111111' + ], + 'chaussure': [ + '1111111111111110111111111001111111000111111000111111000001111100001111110001111111000111111111111111', + '1111111111111111111111110001111111111111111111111111110101111111101111111111111111111111111111000111', + '1111111111111111101111111110111111110100111111111111111111111111111111111111111111111111111111111111', + '1111111111111011111111101111111110111111111101111111111111111111100111111111111111111111111111111111', + '1111111111111111111111110111111100000011100000001111001100011000011000100000111111000001111111111111', + '1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111110001000011', + '1111111111111111111111111111111111111111111111111111111111011111110000111000000000000001001100011100', + '1101101101101100000010001000011000000001100010000100000010111001111111111111111111111111111101101111', + '1111111111111111111111111111111111111111111111111111111111110000011111100000000110110000001011110011', + '1111111111110011111111101101111110110011111010001111001100111110110011111010001111111000111111111111', + '1111111111111111111111111111111111111111111100111111110011111110001111111000111111111111111111111111', + '1101101111000110000000111101010001110111111111011111011101110110011000000000011000000000000000000000', + '1111111111111110111111101001111110100011110001001111000101011100111111111011111111111111111111111111', + '1111111111111111111111111111110010011111000100011100000000010000000001111111111111111111111111111111', + '1111111111110100011111100001111110000111111010011111100000111111110011111111001111111101111111111111', + '0000000000000000000000000000000000000000000000000001000000000000000000001100011000110000100000000000', + '1111111111111111111111111111111110111111101111111110011111111100111111111111111111011110111111100100', + '1111111111101111111100110011110011000111001100111100110001110001010111000100001100110000111111111111' + ], + 'cheval': [ + '1111111101111111010111111110001011110000111111000000000000000000000000000000000000000000000000000000', + '1111111011111100001111111010110011000100011100010000110001000110001110011000001011111110100111111000', + '1111111111111110111111111101111111011111111111101111101101111111110011011110001000111110000001110000', + '1111100101111110011111100101110000000011000000001100000100110000010101000011101100011110110011111001', + '1111111111111111111111111110011100000001101000011111001101111100000000001000000000000100001011001000', + '1111111111111111111111111110010101100011100000011110000001110101111111111111111111111111111111111111', + '0011111010111101010011011111110000111111000001111100000110101011111011111111101111111101111111110011', # noise + '1011011110110001110111000111111100000110110000000011100000001100000000000000000001000000001111100000', + '1001011111000111111100011001011110000000111000001100000000110000000011000001101100000111110000011110', + '1111111111111110001111110000111111000011111000001100100000000010101101000000010000000000000000000100', + '1111101010000000000000010000001111011111111000111111011111111111110001111100000111111111100010010000', + '0010001000010010101011001000111100100001011010000011101000001100000000110010000010001000011000100000', + '1111110001111111111111111111101111111111111111110111111111111111101111111101111011100111001111011100', + '1111111111111111101111111100111111100101110000011100000010000000000000000000000000000000000000000000', + '1111111111111111111011111100101110001001101111110111101011101111101110000000001011110111101101000100', + '1111111111111111111110001111111110011111111111111111100101111111111111000001111111100000000000000000', + '0000000000010010000001101100001111110000111101110011111111101011011100100001110000000000000000000000', + '0000000000010000000001100000000011000100001111110000111110000100001000000000000000000000000000000000', + '1111111111111111111111110111111110011111111001111111000111111100111111110111111011111110001111111100', + '1111111011111110001110010000111000001000100001100010000100000000000000001000000000000000000000000000', + '1101100000111000001011000000001000000000110000000001000000000100000000010001000001001100100011000000', + '1111111111111111100011111100001111100000101110001011100000100000000000000101111111111111111111111111', + '0100000000010011011100110100000010011111000001000100000000000000111000000110000011011111010111110011', + '1111111111111011111111000111111000001111100000011100000010110000000011000011001110011111111000011111', + '0001101000000100110100100111010011110100001111101001111110000111000101100001111000100001001111110111', + '0001000000100000000001000100000000000100010000010110111111111011011110100111110111111110101101111111', + '0000000000111111111111111111111111111011111100001111110000010000000101000000011100001000000000000000' + ], + 'chien': [ + '1111111111111111111111011111111110111111111111111111111111111111111111111111111111111101111111111111', + '0001100000111110010011101011000111111100001111110010000111101100111110000011111000000100000011111111', + '1111111000111010001000100011111110101111000011000001111000011111110001111111111111111111111111111111', + '0000000011000000010000000001000100000001100001000110000011111000011111100111010110011111011000011101', + '1111111111111111111111101101111111101111111111111111111111111111111011111110111111111111111111111111', + '1100001000010111100000010001000001111110000001111100100010110000000001000001111100000111110000011111', + '1110011111110110111100011110110111001111101110001111111000111111000001111100000011111000001101101111', + '1111111111110000111111100011111001001111111000111111100000111111000011111000110111011001111111111111', # noise + '1111111111110111001111001000111110100111111010011111110101111111111111111001111111111111111111111111', + '1111111111111100111111000001111100000111100000001110001001111000101111110010011111000011111110000101', + '1111111111111111111111011111111100111111110001111111011101111111111011111111111100110111010111011100', # noise + '0000000000000001100000000010000000001000000001110000111110000011111000001101000000110000000000000000', + '1110000011110011001110000000110000010001000101000110000011001101000110000000111000000001010010001100', + '0001111111000100111111110111111111001111110000011111000011101101001110110011000010011111011110010101', + '1111111111111111111110011111111100000111110000100111101001111111110111111111011100001101110000000000', # noise + '0000000000000000000000111100000111010000001110000000111000000011000000001110000001111000000111010000', + '0000000000000000000000011110000011011000000111000000001000000001100000001111000011111100000111111000', + '1111111111111000001111000000011000000001100011000110001010011100000001111101111111111111011101111000', + '0000000000000000000011101111011100000011111000011111111111111111111111111111111111111111111111111111', + '0011111010000010000000100001111001010011110111110100000111101100100110101000110000110111110001100011', + '0111001000000001111000001111111111101111111111111111111111111111111111111110011111111111111111111111', # noise + '1111111111111111111111000001110000000011001000000000001000000000010000000011100000000110000000001000', + '1111111111111111101000000000000011111111101101111110111001111000000011101100011101000011110000011110', + '1111101111111000111111100111011111001111111000011100010001100000000001001010000010110010000000100000', + '1111111111111111111111111111111110100110111011101011101110111111001111111111111111111111101111111110', + '1111111111111100001111000000011000000000100000000010001000101010000011111000001111000000011100000001', + '0000000000000000000000111110100010100000000111000000010101000001000010000110010000011111000001111000' + ], + 'éléphant': [ + '1111111111110110111111010001111101000011110000000011111000001111100000111110000011111010001111101001', + '0000000000000000010000000001000000000000000000010010000000001100000000110000000011000000001001110010', + '1111111111110011011110000101011100010011001000000000000000000000000000111000111111110011111111111111', + '0111100001000111100111000110001110011000111000000011001000001100101011111000100111001000010100111111', + '1111111111111111111111100001111100000000000000000001100000000000000000100000000001000000001111111111', + '0000000000000010010100111111000000011110010011111001011010100000101010000010110010011000010000111110', + '0110111010000011000101000000000110000010001100010010110001111111000101101110011101111111110111111111', + '1110000100111000110011011000001100100100100001100000000000000011010000111010001100110000110000000011', + '0000001000000000000000010000000010111110111011000010000000011000000001111010110111101000000000100000', + '1111111111100000000110010000000000000000010000000000100000010000000001000000000000000000000000010100', + '0000111101010000110100010000011000000010000101000000100000000101110000010010110000111100000000100000', + '0000000000000001000000000010000000010000000000000000000000000000000000000001000000000000000000000000', + '0000000000000000000011100000100100000011000000000100000110010000011100010001000010001001001111111110', + '1111111000001111100000111011110111000000000100010011111100001101100100110111001101001100101111100000', + '0000000000000000000000000000000000000000000000000000000000000000000000001000001000100000010000000000', + '0100101111000011000001000000000000111100000001110000010110000001000001100001000000000100000000010000', + '0000111111000111111100011111100011101110001110111001010011100000000110001110010000000001100001000010', + '0000000011000000001100000011110000001110000000111100001101110000111111010001100000010011110001000000', + '0000000000000000000000000000000000000000100000000101000001001010000100100000000000110000000000100000' + ], + 'fleur': [ + '1111111111111111111111111111111111111111111111111111111111111111101111111111111111111111011111111111', + '0000000000000000010001100001110100010000000000000000001100110000000000000000010100000011000000001001', # noise + '1111111111111111111111111111111111110111110000001111110001011111001111111111111111111111111111111111', + '1011111010101111000010001111011000011110111000111111100111111111011001111111111111111111111111111111', + '1111101111111110111101111101111111110100111110001111110010001011100101001111111111111100101111100000', + '0000010000000011100000111110100101111110001111111101110111100111111100001111110000011000100000000000', + '1111111111111111111111111101111111110011111111010111100000011100000111111110110111111111111111111111', + '0000000000000011000000010100000001111100000110110000010010000001001000000011000000001100000000000000', + '0000000000000001000000111100000001111100000111110000111101000011111100001011110100001000010000000001', + '0000000000000000011000000001100000000110000000111000000011100000001100010000100000111110000000010000', + '1111111111111111111111111111111111000111011001011111000011011110000110111111111111111000011110000101', # noise + '0000010000111110000011111111100111111100001101111000011111100001111000000101100000100010000000000000', + '1111111111111111111111111111111111111111111111111111111111111111110111111111111111111111111111111111', + '1100011110100110110000001101101001111111000111111000011111101111111010101111100010111100000110000000', + '1111111111111111111111110101111111100111111111111111111111111111111111111111111111111111111111111111', + '1100000000010000000000001000000000111000001110100000111100000001111100000110000000000000000000000000', + '1110111111111111111111111111111111111111111111110000111111000010110010111001101010110111110000001101', + '0000000000000000000000001110000000111000001111110000011011100001111100000011110000001111000000000000', + '1111110001111110011110111000111000000011100000001110000000101100000010001000000010010000001001100100', # noise + '1111111111111111111100111111111110111111110111111111101111111111101111111010111111000011111111101111', + '1111111111111111111111111111111111111111101111111111011111111101111111111111111111111111111111111111', + '0000011001000111111000111111110111111110011111111011100111100111111110011111110000111110000000100000', + '1111111111111111111111111111111111111111110001011110000010011000000111011000111111011011111111011111', + '1111111111111111111111111111111111000111111100001111110001111111110111111101111111010010100111000000', + '0000000100000000000010000000010000001000000000101100000010010100000000101000001110101000111101101100', + '0000000000000000000000001010000000110110010000000111110000001100000001110000001010001110110000000001', + '0000000000000000000000111100000011111000001001100000111100000001110000000000000000000000000000000000', # noise + '1111111111111100001111101000111010001101010000011101110001011111011101100010001110110011111000111111' + ], + 'girafe': [ + '0011111011100111100111111010000000000000000000000000000000000000000000000000000000000000000000000000', + '1111111111111111111110111111100011011100000000011100000010000000000000000000000000000000000000000000', + '0111100000000000000000000000000000000000000001000000000000000001000000000000000000000000000000000000', + '1111111111111101111111110111111000000111110111111111111100111101010011111110110111110001011110000010', + '1111111111111111111111111110011111101000000000010100000000010000100000000000000000000110000000001000', + '1111111111111111011111111111111111101111111001111111000111111000111001001100000000111111111111111111', + '1111111111111111101111111111110111100111000000000011100111110000010001110111111101011111111111111111', + '1111111111011111111111111011110111110000000100000000000000000000000000001100000000000000001011101111', + '1111100101001100000000011000000000000000010010000011111001111111100111111111111111111110110101111111', + '1111110111110111111111100100111110010111111000011111100000111100000111110010000111001110011000111011', + '1111111111111111111111000111111001011111000100000100000000000001000000010100000001000000000000000000', + '1111111111111111111111100111111011000010000010000000000000000011100111001110011011111111111111111111', + '1111111111110111111111101111111111001111111110011111111000011111100011111110100111111011011111111111', + '0101111111111101000110101111111111011111111101111011101111110000011111001000010011010001001101110110', + '0000000000101011110101001001100011001100001001000000010110000001000000000100100000100111001110111100', + '0111000000010100001000010001001001001000000000000001000001000011101010000000101000000000100000000001', # noise + '1111111111111111111111110111111111011111110001111110100011111000000101000111110100011111110011111111', + '1111111111110111111101111111110000000110011000000000000000000000001000111111100011111110000000111111' + ], + 'lion': [ + '1111111111111111101111101111111100010111101011111110010101111100010111110000011011110011011111111001', + '0011111111111111111111111111110110111001110001000110000000011011101010100001111100010111110001111111', + '0100001000100110000000000110001000000000000110111000111110011011111101111000000000011000000000000000', + '0000000000011111100000001010100001100010010110001001100000010100001101010000000110001000011000000011', + '0000000010000000000000001111100000000111000010000100000000010000000011000000000100000000010000000001', + '0011111000000000000000000100000000010000000001110000011110000000101100000010000000000000000000000000', + '1111111111111111111111110011111111010011111111101111111110011111011101111110100011111100001100111111', + '1111111111111000011111000000011111010000111111100011111010001111000000111110000011111100001110000000', + '0000000000000000000000000000000000000000000000000000000000000001000000001110000000000001000000000000', + '0000000000000000000000000001000001110000000100000000011101000000110000011110100000111101000000111000', + '1111011111100111111100011100110001111011000000011100000001110100001011011011101100001100100000000000', + '0111000110111000001100000100011000011001000010000100000100010000010000000000000110000110001000000000', + '1001110100010100101111011000110011100000001110000000111000000010000000000000000000000000000000010000', + '0010010000000101000010100111110111110111101100111111000011111110001111111111111111011111111011111111', + '1111011111111010011110000011110001110111000000010100000010000000001001000000101100111100001001111111', + '0011101111001000001100000000010000000001111000000111000001000100000100000001000000011101000000000000', + '1111110001111110000111101100111110000001110100001011010000001110000000110000001011100000001111001100', + '0001000000001101000000100000000001101100001111010000111001000011111010011111100011111010001111001000', + '0110110010011111100000011111111011001111111111011101100101110010010111001001011000000000000000011100' + ], + 'moto': [ + '1111111111111011111111000111110000000001100000000000000001100000000000000000000000000000000000000000', + '0000000000000000000000000000000001110000110010111110000000111000000001001001000000000000000000000000', + '0111011111100100010100000111110000011110000011010100000000100001100001000010000011111111101111111000', + '1111111111111111111111111111111111100111101000111101000001111101000101011100000000111100001111001111', + '0000000011011111100111000100011111101100110100001111111000111010000111000000111100000111110001111111', # noise + '1111111111111111111111101111111111111111110010010011101111111100010000111111000001100000000000000000', + '1111111111110010111100110110010010100001000000110100011011001010111100100111111100000001000100110000', + '0000000000000000000000000000000000000000000001000000000000000000000000000000000010000110001111111011', + '1100000110110000011011000011011110011101000000010011001000000000000100000000110100010001100000000100', + '0000000000000000000000000000000000010000000001000000000010000111000000000000000100000001000000000000', + '1111111111111111111111110111111111001111111101111101110010000001000000111010011011000000101100000100', + '1100000000111111001111111111111101111111110110111100010000100000000000000000000011000000011111111111', + '0000000000000000000010011111110010000000100001100000000011010101000011000000010010101011111111111111', + '1000010100111111111111001111111100000000110000000010000000001100000000100000000010001001111111111111', + '1111111111111111111111101111111110111111111001010011100000000000000000000000000000000000000000000000', + '0000000000000000000000000000000000000000000000100000100011000001000000000000000000010000000000000000', # noise + '1111110111111110011111110101111111001001111100010011111100011110001000110010000000000000000000000000', + '1100100111001111011111111110111111101000001101000011100001101011010111000000000000000000000000010000', + '0111111111001110100000100011011110000011101000001010000000110000000000000000000000000000000000000000', + '0000000001000000000000011000000000000000000000010000000000000000000011000000000100000000000000000000', + '1111111111111111111111111111111010001111101010011101010011110100000011000000000000111111111111111111', + '1000000000000011111111111000000000010000001000000010101100000000001000000000001111000000011111111111', + '1111111111111101111111100111111111011111110001101111001101111100011111100101111111010111111101000000', + '0000000000001000100011100110111110101111101010111111011111110000000010000001111000000000000000000000', + '1111111111111111111111100111110110010111010011110000100000000000010000110000001111111111011111111111' + ], + 'oiseau': [ + '1111100111011111111110111111100101111110000000111100000011000000001000000000000000000000000000000001', + '0000000000000000000000010000000011000000001100000000000000000100000000001000000000000000001010000000', + '1111111111111111111111111110010001100001001010011110000010010000110100000011000000001011100000000011', + '0000000000000000010000000100000000011000000000000000000000000100000000100000000000000000000000000000', + '1111111111111111111111111111111111110011111111001111111011111110111111101111111111111111111111111111', + '0000000000000000011000000011100000011100000000100000111110000000111100000000000001100000000001110000', + '1111111111111111001111111101111111111111111111111111111011111111101111111110111111111001001111001001', + '0000000000000000000000000000000000000000000001110000000110000110001100111111000000000000000000000000', + '0000000000000000000000010000000010010000100010000000000000000000100000111110000100010011100000000111', + '1111111111111111111111111111111111111111110111111110011110111011011111111111111011111111111000111111', + '1111111111111111000111111100111111100011111100011111100001111100000111111110111111111111111111111111', + '1111111111111111111111111011111111101111111111111111111111111111111111111111101111111001001111101001', + '1111111111111111111111111111111111111111111111111111111111111111110011101111011110111111111011111111', + '0000000000000000010000000001100000000111100000011000000000011000001000000001111101000011110111100100', + '1111111111110000011110000100110000000001000000010100000001110000000111000000111101000011110001101111', + '0000000000000001000000000100000001100000001110000000111000000001100000000000000011000000001111100000', + '1011111111101111111111011111111101111100111011101010001001100000001110100000001001000000000000000100', + '1110011111000110100010111000001000011110011111111000111110001111111000111111110011111111000111111111', + '0000000000000010000000011000000001100000000111110000011111000000100001000000100000011000001000000000', + '1000011111100011111110001111111000011111000000011110010111111100001111111000011111100001111110001111', + '1111111111111110011111111111111111100111111110011111100001111111001111111110111011110111111111011111', + '1100000000011001001110010000000000111100100000011001011001110010001111000000000100011110011111101111', + '0000000000111111100001001111100101111011111111100110111110010001111000011111101100011110110010001101', + '0000000011000000001100000001110000111111000111111100111111110010001111000001111100011011110010000000', + '1111111111111110111111111010111111111111111110101111111010111111110111111111001111111101101111110111', + '0000000000000000000000000110010000110000000001000100010100000000000000000000000000000000000000000000', + '0000001000111100100011111000111111110011111110001111101001111110000111100001111111011111111111111111', + '0111111111101111011111111011111111111111111111111111111111101111010110100011100010001111111000011111', + '1111111111011001111100100111110001111111001111111100001111110001111111000000111100000011110000011111' + ], + 'papillon': [ + '0000000000000000000000000000000000000111001000011101000000010100000000010000000000100000000000000000', + '0111111101100001001100110100011000000111001100001100100000110100001011011110111101111111110001111110', # noise + '1111111111111110001111101000011100010011100000001100000010010000000001000001001100000111111000000011', + '1111111111111111111111111110111110110001110000100111000000111110110111111011111111111111111111111111', + '1111111111111111111111111111111001110001101000010110000000111100000111111111011111111111111111111111', + '0010011111011001111110000111111110110111001110111100111111110011101111010110111110000001110000011111', + '1111111111100111110110001100011000010001110000001111000000111100100011111011011111111111111111111111', + '0000111111100110110010111010011001000110000000001010100010001000000101000000110111000001111110011000', + '1110001111111000100111101010010000011111000001111100011111111111111111000111111100011111111111111111', # noise + '1111111111011111111100011111100000111000000000000010000000001000000000100010000111011110011111111111', + '0000000000000000000000000000000000000000000000000000011010000000000000000100000000001100001110000000', + '1111111111111111111110011111111000111111101011111110100000011110101001110111011111110011111111111111', + '1111111111111111111111111111111000110000100000000111000000011100010001111011101111111111111111111111', + '0000000000000000000000000000000101001010011110111101100011000000111000000111100000000101000000000000', + '0000001000010000000000100010000000011010100001111001110011001111100100001110000011101100000010000000', + '0000000000000000000010100000000100000010001000011000011011000001001000010100000000100001000000000000', + '0100000000000000010111110001001111101100001101110100000110110000000010000000000101000110000000000011', + '1111111111110111111111100111111110111111110000100111101001111111111111111111111111111111111111111111', + '1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111' + ], + 'poisson': [ + '0000000000000000000000011000000010101101010110111111111111111111111111111111111111111111111111111111', + '1111111111111111111111111111111111000111111000011111111111111111111111111111111111111111111111111111', + '0000000000000000000000000000000001111100000011010000000000000000001000000000000000000000000000000000', + '0101000000010011111110000011100000000100011001000001100011000000000000000000000000000000000100000000', + '1111111111111111111111110001111100000011111000011111101111111110111111111111111111111111111111111111', + '0000000000000000000000000001000000000010000001111000000110000000100000000000000000000000000000000000', + '1111111111111111111110001111110000001111000000011111011001111111111111111111111111111111111111111111', + '1000000000100000000000000100000011110000011111100001111110000011000000000100000000000000000000000000', + '0000000000000000000000000000000011100000001000000001010000000111100000000001100000000000000000000000', + '1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111', + '1111111111111011111111110011111110001111111010011111111001111111101111111111011111111111111111111111', + '0000000000000000000000000000000011110000000111000000011100000001100110001000000000001000000000100000', + '0000111111000111011000111100100001100000111001111011000110011101000010110000000110000001000001011111', + '1111111111111111111111111111111101011111110101111110010011111101101111111011011111111111111111111111', + '0000000000000000000000000000000001110000001011000001110101110110011110000111000000000000000000000000', + '0000000000000000000000010000000011100000011011000000010000000000000000000000011000000001110000000001', + '0000000000000000000011000000000000000000111110000011111110001111111110111111111011111111100000000000', + '0000000100001010010000000000000000111000000111001000111110010011100000010110001000000000100000000000', + '0000000000000000000000000000000000000000001110000001111000000000000000000000000000000000000000000000', + '0000000000000000000000000111000001011100001011111001101100000000000000111000000000000000001000000000' + ], + 'théière': [ + '0111111111011111111101100011111100000011100000001110000000011000000011100000011100000001110000000111', + '1111111111111111111111110111110010010110101000101010000000011110000011111010001111111111111111111111', + '0000000000000000000000000000000101000000000001111000101111110001100000100000000000000000000000000000', + '1111111111111111111111111111111111111111111111111111111111111110111111100000111111000011111111111111', + '1111111111111011101111111111011111111101110110110111001000011110000011110000001111100000111111101111', + '1111111111111111111111111111111111111111111111111111111111111111111111111111111111111001111111111111', + '1111111111111111111111111111111111111111111111111111111111011111111011111111001111111100111111111111', + '1111111111111111111111111111111111101111111001000101010000011100000001100000000111000000111111101111', + '1111111111111111111111110011011010000011111000001111100001111111111111111111111111111111111111111111', + '1111111111111111111111111111111111101100100100011011010000111111000011111111111111111111111111111111', + '1111111111111111111111111111111111111111111111111110111111110110111111111111111110011111111111001111', + '1111111111111111111111111111111111000111110000011110000000111000000001111000000111111100011111111011', + '1100100000000000000000000100000001000001010110110000000000010000000001110000000111000000011100000000', + '0000000001000000000100000111110001000110000100000100000000110000000001000000001100000101110000110111', + '0001000000000110000000110000000000000000000011000011111111111111111111111011111101111100111111111111', + '1111111111111111111111111111111111111111111111111111111111111111101111111111111111111111111111111111', + '1110111111111111111111111011111111111101111111111111111110111111101111111100111011110001111111101111', + '1111111111111111111111111111111111111111111111111101111111111110111111111000011111110101111111111111' + ], + 'violon': [ + '0000000000000100000000001000000000000100000100000100000100000000010000000110000000011100000011101000', + '1111111111101111111111011111111110111111101011111111000011111000000010110000011111000000111111111111', + '1111111111111111111111101111111101111011000001111110000011101100000000111001111111111111101111111111', + '0000000000000000000000000000000000000000000010000000101000000000000000010000000000000000000000000000', + '0000000000000000000000000001000000010000000000000000000000000000000000000000000000000000000000000000', + '1111111111111111111111111111101111111100111110000011111000001110100001100111111101111011111111111111', + '1111111111111111111111111111110001011111111110001111111000001111100000111111111011111111101111111111', # noise + '1110000000111000000011100000101110000111111000110111100011111000011111000001111100000110110000011011', + '0000000000000000000000000100000001010000000011000000010000000011100000000100000000000000000000000000', + '1111110111111111110011101000001000000010100000000111100000001100000000100000000000000000010011110101', + '1111111100111111110111111110001111111000111110100001010010000100000000100000000001100000000000000000', + '1111100011111100000000000000000000100000000000000000000000010000000111100000011111010011111111101111', + '0000000000000000000000000000000000110000000101000001100000000111100000011011000001101000000000000000', + '0000000111000000011100100001110000000111000000011100000001110010000111000010000000000000000000000000', + '1111010000011101010001110100000011100000001010000000001100000001100000000010000000000000000000000000', + '1111111111111111101111111100011110101000110100000011000010011110011111111111111111111111111111111111', + '1111111111111100111101111111110111111111011111111101111001110110000111011100111001110011010110000100', + '0000000000010000000000101100000001100000000100000000001000100000010000000000100000000000000000000000', + '1111111111100111111110101111111111000011111110000111111000001111100100111111110011111111001111111110' + ] +} diff --git a/modules/orange/pages/login.py b/modules/orange/pages/login.py index 542d082f7d..2f3d181f69 100644 --- a/modules/orange/pages/login.py +++ b/modules/orange/pages/login.py @@ -82,7 +82,3 @@ def get_error_message(self): class PortalPage(LoggedPage, RawPage): pass - - -class CaptchaPage(RawPage): - pass diff --git a/modules/pagesjaunes/browser.py b/modules/pagesjaunes/browser.py index 19fac9ce25..4a4955dc54 100644 --- a/modules/pagesjaunes/browser.py +++ b/modules/pagesjaunes/browser.py @@ -30,8 +30,10 @@ class PagesjaunesBrowser(PagesBrowser): BASEURL = 'https://www.pagesjaunes.fr' - search = URL('/recherche/(?P[a-z0-9-]+)/(?P[a-z0-9-]+)', ResultsPage) - company = URL('/pros/\d+', PlacePage) + search = URL( + r'/annuaire/chercherlespros\?quoiqui=(?P[a-z0-9-]+)&ou=(?P[a-z0-9-]+)&page=(?P\d+)', + ResultsPage) + company = URL(r'/pros/\d+', PlacePage) def simplify(self, name): return re.sub(r'[^a-z0-9-]+', '-', name.lower()) @@ -40,11 +42,10 @@ def search_contacts(self, query): assert query.name assert query.city - self.search.go(city=self.simplify(query.city), pattern=self.simplify(query.name)) + self.search.go(city=self.simplify(query.city), pattern=self.simplify(query.name), page=1) return self.page.iter_contacts() def fill_hours(self, contact): self.location(contact.url) contact.opening = OpeningHours() contact.opening.rules = list(self.page.iter_hours()) - diff --git a/modules/pagesjaunes/module.py b/modules/pagesjaunes/module.py index 86af75c197..b77d9eb7c0 100644 --- a/modules/pagesjaunes/module.py +++ b/modules/pagesjaunes/module.py @@ -49,4 +49,3 @@ def fill_contact(self, obj, fields): OBJECTS = { Place: fill_contact, } - diff --git a/modules/pagesjaunes/pages.py b/modules/pagesjaunes/pages.py index e7ab332670..fa314c1693 100644 --- a/modules/pagesjaunes/pages.py +++ b/modules/pagesjaunes/pages.py @@ -24,52 +24,72 @@ from dateutil import rrule from weboob.browser.elements import method, ListElement, ItemElement -from .compat.weboob_browser_filters_standard import CleanText, Regexp -from weboob.browser.filters.html import AbsoluteLink, HasElement -from .compat.weboob_browser_pages import HTMLPage +from .compat.weboob_browser_filters_standard import CleanText, Regexp, Field, Env, BrowserURL +from weboob.browser.filters.html import AbsoluteLink, HasElement, XPath +from .compat.weboob_browser_pages import HTMLPage, pagination from weboob.capabilities.base import NotLoaded, NotAvailable from weboob.capabilities.contact import Place, OpeningRule class ResultsPage(HTMLPage): + @pagination @method class iter_contacts(ListElement): - item_xpath = '//section[@id="listResults"]/article' + item_xpath = '//section[@id="listResults"]/ul/li' + + def next_page(self): + if XPath('//div/@class="pagination"', default=False)(self): + next_page = int(Env('page')(self)) + 1 + return BrowserURL('search', + city=Env('city'), + pattern=Env('pattern'), + page=next_page)(self) class item(ItemElement): klass = Place obj_name = CleanText('.//a[has-class("denomination-links")]') obj_address = CleanText('.//a[has-class("adresse")]') - obj_phone = Regexp( - CleanText( - './/div[has-class("tel-zone")][span[contains(text(),"Tél")]]//strong[@class="num"]', - replace=[(' ', '')]), r'^0(\d{9})$', r'+33\1') - obj_url = AbsoluteLink('.//a[has-class("denomination-links")]') + + def obj_phone(self): + tel = [] + for _ in XPath( + './/div[has-class("tel-zone")][span[contains(text(),"Tél")]]//strong[@class="num"]')(self): + tel.append(Regexp(CleanText('.', replace=[(' ', '')]), r'^0(\d{9})$', r'+33\1')(_)) + + return " / ".join(tel) + + def obj_url(self): + if CleanText('.//a[has-class("denomination-links")]/@href', replace=[('#', '')])(self): + return AbsoluteLink('.//a[has-class("denomination-links")]')(self) + return NotAvailable + obj_opening = HasElement('.//span[text()="Horaires"]', NotLoaded, NotAvailable) class PlacePage(HTMLPage): @method class iter_hours(ListElement): - item_xpath = '//ul[@class="liste-horaires-principaux"]/li[@class="horaire-ouvert"]' + item_xpath = '//div[@id="infos-horaires"]/ul/li[@class="horaire-ouvert"]' class item(ItemElement): klass = OpeningRule def obj_dates(self): - wday = CleanText('./span')(self) - wday = ['lundi', 'mardi', 'mercredi', 'jeudi', 'vendredi', 'samedi', 'dimanche'].index(wday) + wday = CleanText('./p')(self) + wday = ['lundi', 'mardi', 'mercredi', 'jeudi', 'vendredi', 'samedi', 'dimanche'].index(wday.lower()) assert wday >= 0 - return rrule.rrule(rrule.DAILY, byweekday=wday) + return rrule.rrule(rrule.DAILY, byweekday=wday, count=1) def obj_times(self): times = [] - for sub in self.el.xpath('.//li[@itemprop]'): - t = CleanText('./@content')(sub) - m = re.match(r'\w{2} (\d{2}):(\d{2})-(\d{2}):(\d{2})$', t) - m = [int(x) for x in m.groups()] - times.append((time(m[0], m[1]), time(m[2], m[3]))) + for sub in XPath('.//li')(self): + t = CleanText('.')(sub) + m = re.match(r'(\d{2})h(\d{2}) - (\d{2})h(\d{2})$', t) + if m: + m = [int(x) for x in m.groups()] + times.append((time(m[0], m[1]), time(m[2], m[3]))) return times - obj_is_open = True + def obj_is_open(self): + return len(Field('times')(self)) > 0 diff --git a/modules/s2e/browser.py b/modules/s2e/browser.py index adc6b92e92..fd37779edc 100644 --- a/modules/s2e/browser.py +++ b/modules/s2e/browser.py @@ -22,6 +22,7 @@ from __future__ import unicode_literals import re + from requests.exceptions import ConnectionError from urllib3.exceptions import ReadTimeoutError diff --git a/modules/s2e/pages.py b/modules/s2e/pages.py index 7a08bc8581..0925d2017f 100644 --- a/modules/s2e/pages.py +++ b/modules/s2e/pages.py @@ -22,11 +22,11 @@ from __future__ import unicode_literals import re - -import requests from io import BytesIO from decimal import Decimal + from lxml import objectify +import requests from .compat.weboob_browser_pages import ( HTMLPage, XMLPage, RawPage, LoggedPage, pagination, diff --git a/modules/societegenerale/browser.py b/modules/societegenerale/browser.py index 15c9406ec6..518fd277d8 100644 --- a/modules/societegenerale/browser.py +++ b/modules/societegenerale/browser.py @@ -22,9 +22,9 @@ from __future__ import unicode_literals import time - from datetime import datetime from decimal import Decimal + from dateutil.relativedelta import relativedelta from weboob.browser.browsers import URL, need_login @@ -202,11 +202,21 @@ def check_password(self): def check_login_reason(self): reason = self.page.get_reason() + if reason is not None: + self.logger.info('Bad login for reason: %s', reason) # logger to catch and survey different cases if reason == 'echec_authent': raise BrowserIncorrectPassword() elif reason in ('acces_bloq', 'acces_susp', 'pas_acces_bad', ): - raise ActionNeeded() + # 'reason' doesn't bear a user-friendly message, so + # those messages were collected from the website since they are JavaScript-forged + action_needed_messages = { + 'acces_bloq': '''Suite à trois saisies erronées de vos codes, l'accès à vos comptes est bloqué jusqu'à demain pour des raisons de sécurité.''', + 'acces_susp': '''Votre accès est suspendu. Vous n'êtes pas autorisé à accéder à l'application.''', + # yes, same message + 'pas_acces_bad': '''Votre accès est suspendu. Vous n'êtes pas autorisé à accéder à l'application.''', + } + raise ActionNeeded(action_needed_messages[reason]) elif reason == 'err_tech': # there is message "Service momentanément indisponible. Veuillez réessayer." # in SG website in that case ... diff --git a/modules/societegenerale/pages/login.py b/modules/societegenerale/pages/login.py index e2d2ae40b9..14b0c577b0 100644 --- a/modules/societegenerale/pages/login.py +++ b/modules/societegenerale/pages/login.py @@ -23,8 +23,8 @@ from base64 import b64decode from logging import error import re -from weboob.tools.json import json +from weboob.tools.json import json from weboob.exceptions import BrowserUnavailable, BrowserPasswordExpired, ActionNeeded from .compat.weboob_browser_pages import HTMLPage, JsonPage from .compat.weboob_browser_filters_standard import CleanText diff --git a/modules/societegenerale/pages/subscription.py b/modules/societegenerale/pages/subscription.py index c74cee1f71..c84e919112 100644 --- a/modules/societegenerale/pages/subscription.py +++ b/modules/societegenerale/pages/subscription.py @@ -24,6 +24,7 @@ import re from dateutil.relativedelta import relativedelta + from weboob.capabilities.bill import Document, Subscription, DocumentTypes from weboob.browser.elements import TableElement, ItemElement, method from .compat.weboob_browser_filters_standard import CleanText, Regexp, Date, Format, Field diff --git a/modules/societegenerale/sgpe/browser.py b/modules/societegenerale/sgpe/browser.py index 9ae1cafc99..e81c8c083a 100644 --- a/modules/societegenerale/sgpe/browser.py +++ b/modules/societegenerale/sgpe/browser.py @@ -24,6 +24,7 @@ from datetime import date from dateutil.relativedelta import relativedelta + from .compat.weboob_browser_browsers import LoginBrowser, need_login from .compat.weboob_browser_url import URL from weboob.browser.exceptions import ClientError @@ -48,7 +49,6 @@ EasyTransferPage, RecipientsJsonPage, TransferPage, SignTransferPage, TransferDatesPage, AddRecipientPage, AddRecipientStepPage, ConfirmRecipientPage, ) - from ..browser import SocieteGenerale as SocieteGeneraleParBrowser diff --git a/modules/societegenerale/sgpe/json_pages.py b/modules/societegenerale/sgpe/json_pages.py index 7a2506df9b..e5d507c311 100644 --- a/modules/societegenerale/sgpe/json_pages.py +++ b/modules/societegenerale/sgpe/json_pages.py @@ -24,6 +24,7 @@ from datetime import datetime import requests + from .compat.weboob_browser_pages import LoggedPage, JsonPage, pagination from weboob.browser.elements import ItemElement, method, DictElement from .compat.weboob_browser_filters_standard import ( diff --git a/modules/societegenerale/sgpe/pages.py b/modules/societegenerale/sgpe/pages.py index fe4bb0599a..8cecb36ac1 100644 --- a/modules/societegenerale/sgpe/pages.py +++ b/modules/societegenerale/sgpe/pages.py @@ -192,7 +192,7 @@ def login(self, login, password): authentication_data = self.get_authentication_data(login, password) authentication_data.update({ 'top_code_etoile': 0, - 'top_ident': 0, + 'top_ident': 1, 'cible': 300, }) self.browser.location( diff --git a/modules/swile/browser.py b/modules/swile/browser.py index 0586beb282..a92e7315c9 100644 --- a/modules/swile/browser.py +++ b/modules/swile/browser.py @@ -30,7 +30,7 @@ ) from weboob.capabilities.base import empty from weboob.browser.filters.json import Dict -from weboob.browser.exceptions import ClientError +from weboob.browser.exceptions import ClientError, BrowserTooManyRequests from weboob.exceptions import BrowserIncorrectPassword, NocaptchaQuestion from .compat.weboob_browser_browsers import APIBrowser, OAuth2Mixin from .compat.weboob_capabilities_bank import Account, Transaction @@ -69,14 +69,16 @@ def request_authorization(self): self.credentials['recaptcha'] = self.config['captcha_response'].get() self.location(self.ACCESS_TOKEN_URI, data=self.credentials) except ClientError as e: - json = e.response.json() # if the captcha's response is not completed the error is # 426 Client Error: Upgrade Required if e.response.status_code == 426 and not self.config['captcha_response'].get(): raise NocaptchaQuestion(website_url='https://app.swile.co/signin', website_key='6LceI-EUAAAAACrBsmKCmllNdk1-H5U7G7NOTzmj') if e.response.status_code == 401: + json = e.response.json() message = json['error_description'] raise BrowserIncorrectPassword(message) + if e.response.status_code == 429: + raise BrowserTooManyRequests() raise e self.update_token(self.response.json()) diff --git a/modules/yomoni/browser.py b/modules/yomoni/browser.py index 68986dda5f..527ee5ffa6 100644 --- a/modules/yomoni/browser.py +++ b/modules/yomoni/browser.py @@ -27,7 +27,7 @@ from .compat.weboob_browser_browsers import APIBrowser from weboob.browser.exceptions import ClientError -from .compat.weboob_browser_filters_standard import CleanDecimal, Date, Coalesce +from .compat.weboob_browser_filters_standard import CleanDecimal, Date, Coalesce, MapIn from weboob.browser.filters.html import ReplaceEntities from weboob.exceptions import BrowserIncorrectPassword, ActionNeeded from .compat.weboob_capabilities_bank import Account, Transaction @@ -49,6 +49,12 @@ def wrapper(self, *args, **kwargs): class YomoniBrowser(APIBrowser): BASEURL = 'https://yomoni.fr' + ACCOUNT_TYPES = { + 'assurance vie': Account.TYPE_LIFE_INSURANCE, + 'compte titre': Account.TYPE_MARKET, + 'pea': Account.TYPE_PEA, + } + def __init__(self, username, password, *args, **kwargs): super(YomoniBrowser, self).__init__(*args, **kwargs) self.username = username @@ -115,11 +121,9 @@ def iter_accounts(self): a = Account() a.id = "".join(me['numeroContrat'].split()) a.number = me['numeroContrat'] + a.opening_date = Date(default=NotAvailable).filter(me.get('dateAdhesion')) a.label = " ".join(me['supportEpargne'].split("_")) - a.type = Account.TYPE_LIFE_INSURANCE if "assurance vie" in a.label.lower() else \ - Account.TYPE_MARKET if "compte titre" in a.label.lower() else \ - Account.TYPE_PEA if "pea" in a.label.lower() else \ - Account.TYPE_UNKNOWN + a.type = MapIn(self, self.ACCOUNT_TYPES, Account.TYPE_UNKNOWN).filter(a.label.lower()) a.balance = CleanDecimal().filter(me['solde']) a.currency = u'EUR' # performanceEuro, montantEuro everywhere in Yomoni JSON a.iban = me['ibancompteTitre'] or NotAvailable -- GitLab