diff --git a/modules/amazon/browser.py b/modules/amazon/browser.py index e8273154d89a815ac61f3f225d01d76f7f6bea88..3a790d9ed85bfdafd569d498d336a9a98d013722 100644 --- a/modules/amazon/browser.py +++ b/modules/amazon/browser.py @@ -26,14 +26,15 @@ from woob.exceptions import ( BrowserIncorrectPassword, BrowserUnavailable, ImageCaptchaQuestion, BrowserQuestion, WrongCaptchaResponse, NeedInteractiveFor2FA, BrowserPasswordExpired, - AppValidation, AppValidationExpired, + AppValidation, AppValidationExpired, AppValidationCancelled, ) from woob.tools.value import Value from woob.browser.browsers import ClientError from .pages import ( LoginPage, SubscriptionsPage, DocumentsPage, DownloadDocumentPage, HomePage, - PanelPage, SecurityPage, LanguagePage, HistoryPage, PasswordExpired, ApprovalPage, + SecurityPage, LanguagePage, HistoryPage, PasswordExpired, ApprovalPage, PollingPage, + ResetPasswordPage, ) @@ -54,17 +55,22 @@ class AmazonBrowser(LoginBrowser, StatesMixin): WRONG_CAPTCHA_RESPONSE = "Saisissez les caractères tels qu'ils apparaissent sur l'image." login = URL(r'/ap/signin(.*)', LoginPage) - home = URL(r'/$', r'/\?language=\w+$', HomePage) - panel = URL('/gp/css/homepage.html/ref=nav_youraccount_ya', PanelPage) - subscriptions = URL(r'/ap/cnep(.*)', SubscriptionsPage) - history = URL(r'/gp/your-account/order-history\?ref_=ya_d_c_yo', HistoryPage) + home = URL(r'/$', r'/\?language=.+$', HomePage) + subscriptions = URL(r'/gp/profile', SubscriptionsPage) + history = URL( + r'/gp/your-account/order-history\?ref_=ya_d_c_yo', + r'/gp/css/order-history\?', + HistoryPage, + ) documents = URL( r'/gp/your-account/order-history\?opt=ab&digitalOrders=1(.*)&orderFilter=year-(?P.*)', r'/gp/your-account/order-history', DocumentsPage, ) download_doc = URL(r'/gp/shared-cs/ajax/invoice/invoice.html', DownloadDocumentPage) - approval_page = URL(r'/ap/cvf/approval', ApprovalPage) + approval_page = URL(r'/ap/cvf/approval\?', ApprovalPage) + reset_password_page = URL(r'/ap/forgotpassword/reverification', ResetPasswordPage) + poll_page = URL(r'/ap/cvf/approval/poll', PollingPage) security = URL( r'/ap/dcq', r'/ap/cvf/', @@ -76,7 +82,9 @@ class AmazonBrowser(LoginBrowser, StatesMixin): __states__ = ('otp_form', 'otp_url', 'otp_style', 'otp_headers') - STATE_DURATION = 10 + # According to the cookies we are fine for 1 year after the last sync. + # If we reset the state every 10 minutes we'll get a in-app validation after 10 minutes + STATE_DURATION = 60 * 24 * 365 otp_form = None otp_url = None @@ -156,27 +164,39 @@ def handle_captcha(self, captcha): raise ImageCaptchaQuestion(image) def check_app_validation(self): - # client has 60 seconds to unlock this page - # the resend link will appear from 60 seconds is why there are 2 additional seconds, it's to have a margin - timeout = time.time() + 62.00 - second_try = True + # 25' on website, we don't wait that much, but leave sufficient time for the user + timeout = time.time() + 600.00 + app_validation_link = self.page.get_link_app_validation() + polling_request = self.page.get_polling_request() + approval_status = '' while time.time() < timeout: - link = self.page.get_link_app_validation() - self.location(link) - if self.approval_page.is_here(): - time.sleep(2) - else: - return + self.location(polling_request) + approval_status = self.page.get_approval_status() - if time.time() >= timeout and second_try: - # second try because 60 seconds is short, the second try is longger - second_try = False - timeout = time.time() + 70.00 - self.page.resend_link() + if approval_status != 'TransactionPending': + break + + # poll every 5 seconds on website + time.sleep(5) else: raise AppValidationExpired() + if approval_status in ['TransactionCompleted', 'TransactionResponded']: + self.location(app_validation_link) + + if self.reset_password_page.is_here(): + raise AppValidationCancelled() + + if self.approval_page.is_here(): + raise AssertionError('The validation was not effective for an unknown reason.') + + elif approval_status == 'TransactionCompletionTimeout': + raise AppValidationExpired() + + else: + raise AssertionError('Unknown transaction status: %s' % approval_status) + def do_login(self): if self.config['pin_code'].get(): # Resolve pin_code @@ -279,18 +299,10 @@ def to_english(self, language): @need_login def iter_subscription(self): - self.location(self.panel.go().get_sub_link()) - - if self.home.is_here(): - if self.page.get_login_link(): - self.is_login() - self.location(self.page.get_panel_link()) - elif not self.subscriptions.is_here(): - self.is_login() + self.subscriptions.go() - # goes back to the subscription page as you may be redirected to the documents page if not self.subscriptions.is_here(): - self.location(self.panel.go().get_sub_link()) + self.is_login() yield self.page.get_item() diff --git a/modules/amazon/pages.py b/modules/amazon/pages.py index 47395ecb3fd8de47118bec7f0479ef3ef3213999..d3a87b21fd50e4a0251ce7aa3aa2e3b0af8c677a 100644 --- a/modules/amazon/pages.py +++ b/modules/amazon/pages.py @@ -23,28 +23,24 @@ from woob.browser.elements import ItemElement, ListElement, method from woob.browser.filters.html import Link, Attr from woob.browser.filters.standard import ( - CleanText, CleanDecimal, Env, Regexp, Format, - Field, Currency, RegexpError, Date, Async, AsyncLoad, + CleanText, CleanDecimal, Env, Regexp, Format, RawText, + Field, Currency, Date, Async, AsyncLoad, Coalesce, ) from woob.capabilities.bill import DocumentTypes, Bill, Subscription from woob.capabilities.base import NotAvailable +from woob.tools.json import json from woob.tools.date import parse_french_date class HomePage(HTMLPage): def get_login_link(self): - return self.doc.xpath('//a[./span[contains(., "%s")]]/@href' % self.browser.L_SIGNIN)[0] + return Attr('//a[@data-nav-role="signin"]', 'href')(self.doc) def get_panel_link(self): return Link('//a[contains(@href, "homepage.html") and has-class(@nav-link)]')(self.doc) -class PanelPage(LoggedPage, HTMLPage): - def get_sub_link(self): - return CleanText('//a[@class="ya-card__whole-card-link" and contains(@href, "cnep")]/@href')(self.doc) - - class SecurityPage(HTMLPage): def get_otp_type(self): if self.doc.xpath('//form[@id="auth-select-device-form"]'): @@ -108,18 +104,30 @@ def has_form_select_device(self): class ApprovalPage(HTMLPage, LoggedPage): def get_msg_app_validation(self): - msg = CleanText('//span[has-class("transaction-approval-word-break")]') + msg = CleanText('//div[has-class("a-spacing-large")]/span[has-class("transaction-approval-word-break")]') sending_address = CleanText('//div[@class="a-row"][1]') - msg = Format('%s %s', msg, sending_address) - return msg(self.doc) + return Format('%s %s', msg, sending_address)(self.doc) def get_link_app_validation(self): - return Link('//a[@id="resend-approval-link"]')(self.doc) + return Attr('//input[@name="openid.return_to"]', 'value')(self.doc) def resend_link(self): form = self.get_form(id='resend-approval-form') form.submit() + def get_polling_request(self): + form = self.get_form(id="pollingForm") + return form.request + + +class PollingPage(HTMLPage): + def get_approval_status(self): + return Attr('//input[@name="transactionApprovalStatus"]', 'value', default=None)(self.doc) + + +class ResetPasswordPage(HTMLPage): + pass + class LanguagePage(HTMLPage): pass @@ -165,14 +173,15 @@ class SubscriptionsPage(LoggedPage, HTMLPage): class get_item(ItemElement): klass = Subscription - def obj_subscriber(self): - try: - return Regexp(CleanText('//div[contains(@class, "a-fixed-right-grid-col")]'), self.page.browser.L_SUBSCRIBER)(self) - except RegexpError: - return self.page.browser.username - obj_id = 'amazon' + def obj_subscriber(self): + profile_data = json.loads(Regexp( + RawText('//script[contains(text(), "window.CustomerProfileRootProps")]'), + r'window.CustomerProfileRootProps = ({.+});', + )(self)) + return profile_data.get('nameHeaderData', {}).get('name', NotAvailable) + def obj_label(self): return self.page.browser.username @@ -219,7 +228,7 @@ def obj_date(self): Date(CleanText('.//div[has-class("a-span2") and not(has-class("recipient"))]/div[2]'), parse_func=parse_french_date, dayfirst=True, default=NotAvailable), )(self) - def obj_price(self): + def obj_total_price(self): # Some orders, audiobooks for example, are paid using "audio credits", they have no price or currency currency = Env('currency')(self) return CleanDecimal( @@ -245,10 +254,13 @@ def obj_url(self): url = Coalesce( Link('//a[contains(@href, "download")]|//a[contains(@href, "generated_invoices")]', default=NotAvailable), Link('//a[contains(text(), "Récapitulatif de commande")]', default=NotAvailable), + default=NotAvailable )(async_page.doc) return url def obj_format(self): + if not Field('url')(self): + return NotAvailable if 'summary' in Field('url')(self): return 'html' return 'pdf' diff --git a/modules/ameli/browser.py b/modules/ameli/browser.py index 079d02d3ff771ef5c8b030aff59f6de95f1f70a3..3fd5a8f9cbc5241d2e73955c97fceda1830c65c1 100644 --- a/modules/ameli/browser.py +++ b/modules/ameli/browser.py @@ -17,14 +17,18 @@ # You should have received a copy of the GNU Lesser General Public License # along with this woob module. If not, see . +# flake8: compatible + from __future__ import unicode_literals +import re from datetime import date from time import time + from dateutil.relativedelta import relativedelta from woob.browser import LoginBrowser, URL, need_login -from woob.exceptions import ActionNeeded +from woob.exceptions import ActionNeeded, BrowserIncorrectPassword, BrowserUnavailable from woob.tools.capabilities.bill.documents import merge_iterators from .pages import ( @@ -38,10 +42,23 @@ class AmeliBrowser(LoginBrowser): BASEURL = 'https://assure.ameli.fr' error_page = URL(r'/vu/INDISPO_COMPTE_ASSURES.html', ErrorPage) - login_page = URL(r'/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&connexioncompte_2actionEvt=afficher.*', LoginPage) - redirect_page = URL(r'/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&.*validationconnexioncompte.*', RedirectPage) - cgu_page = URL(r'/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&_pageLabel=as_conditions_generales_page.*', CguPage) - subscription_page = URL(r'/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&_pageLabel=as_info_perso_page.*', SubscriptionPage) + login_page = URL( + r'/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&connexioncompte_2actionEvt=afficher.*', + r'/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&.*validationconnexioncompte.*', + LoginPage + ) + redirect_page = URL( + r'/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&.*validationconnexioncompte.*', + RedirectPage + ) + cgu_page = URL( + r'/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&_pageLabel=as_conditions_generales_page.*', + CguPage + ) + subscription_page = URL( + r'/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&_pageLabel=as_info_perso_page.*', + SubscriptionPage + ) documents_details_page = URL(r'/PortailAS/paiements.do', DocumentsDetailsPage) documents_first_summary_page = URL( r'PortailAS/appmanager/PortailAS/assure\?_nfpb=true&_pageLabel=as_releve_mensuel_paiement_page', @@ -59,6 +76,19 @@ def do_login(self): _ct = self.ct_page.open(method='POST', headers={'FETCH-CSRF-TOKEN': '1'}).get_ct_value() self.page.login(self.username, self.password, _ct) + if self.login_page.is_here(): + err_msg = self.page.get_error_message() + wrongpass_regex = re.compile( + 'numéro de sécurité sociale et le code personnel' + + '|compte ameli verrouillé' + ) + if wrongpass_regex.search(err_msg): + raise BrowserIncorrectPassword(err_msg) + raise AssertionError('Unhandled error at login %s' % err_msg) + + if self.error_page.is_here(): + raise BrowserUnavailable(self.page.get_error_message()) + if self.cgu_page.is_here(): raise ActionNeeded(self.page.get_cgu_message()) @@ -84,7 +114,7 @@ def _iter_details_documents(self, subscription): 'afficherRS': 'false', 'afficherReleves': 'false', 'afficherRentes': 'false', - 'idNoCache': int(time()*1000) + 'idNoCache': int(time() * 1000), } # website tell us details documents are available for 6 months @@ -105,8 +135,10 @@ def _iter_summary_documents(self, subscription): for doc in self.page.iter_documents(subid=subscription.id): yield doc - @need_login def iter_documents(self, subscription): - for doc in merge_iterators(self._iter_details_documents(subscription), self._iter_summary_documents(subscription)): + for doc in merge_iterators( + self._iter_details_documents(subscription), + self._iter_summary_documents(subscription) + ): yield doc diff --git a/modules/ameli/module.py b/modules/ameli/module.py index 453ad894d180ea45e7bcd2e0a90a51dcc041ae55..0564bf44a2ad3a72c7248ff9ed1591cfd1fe581d 100644 --- a/modules/ameli/module.py +++ b/modules/ameli/module.py @@ -17,12 +17,16 @@ # You should have received a copy of the GNU Lesser General Public License # along with this woob module. If not, see . +# flake8: compatible from __future__ import unicode_literals from woob.capabilities.base import find_object from woob.tools.backend import Module, BackendConfig -from woob.capabilities.bill import CapDocument, Document, DocumentTypes, SubscriptionNotFound, DocumentNotFound, Subscription +from woob.capabilities.bill import ( + CapDocument, Document, DocumentNotFound, DocumentTypes, + Subscription, SubscriptionNotFound, +) from woob.tools.value import ValueBackendPassword from .browser import AmeliBrowser diff --git a/modules/ameli/pages.py b/modules/ameli/pages.py index 0fc21a78018bb9608495c14205eef4ef63df9a1d..be4e9962ce373394b83763f878b50cfa6f07f8b7 100644 --- a/modules/ameli/pages.py +++ b/modules/ameli/pages.py @@ -17,24 +17,28 @@ # You should have received a copy of the GNU Lesser General Public License # along with this woob module. If not, see . +# flake8: compatible + from __future__ import unicode_literals import re - from hashlib import sha1 from woob.browser.elements import method, ListElement, ItemElement, DictElement from woob.browser.filters.html import Link -from woob.browser.filters.standard import CleanText, Regexp, CleanDecimal, Currency, Field, Env, Format +from woob.browser.filters.standard import CleanText, Coalesce, Regexp, CleanDecimal, Currency, Field, Env, Format from woob.browser.filters.json import Dict from woob.browser.pages import LoggedPage, HTMLPage, PartialHTMLPage, RawPage, JsonPage from woob.capabilities.bill import Subscription, Bill, Document, DocumentTypes -from woob.exceptions import BrowserUnavailable, BrowserIncorrectPassword +from woob.tools.compat import html_unescape from woob.tools.date import parse_french_date from woob.tools.json import json class LoginPage(HTMLPage): + def is_here(self): + return self.doc.xpath('//form[contains(@id, "CompteForm")]') + def login(self, username, password, _ct): form = self.get_form(id='connexioncompte_2connexionCompteForm') form['connexioncompte_2numSecuriteSociale'] = username @@ -42,6 +46,13 @@ def login(self, username, password, _ct): form['_ct'] = _ct form.submit() + def get_error_message(self): + return Coalesce( + CleanText('//div[@id="loginPage"]//div[has-class("zone-alerte") and not(has-class("hidden"))]/span'), + CleanText('//div[@class="centrepage compte_bloque"]//p[@class="msg_erreur"]'), + default=None + )(self.doc) + class CtPage(RawPage): # the page contains only _ct value @@ -53,19 +64,6 @@ class RedirectPage(LoggedPage, HTMLPage): REFRESH_MAX = 0 REFRESH_XPATH = '//meta[@http-equiv="refresh"]' - def on_load(self): - if not self.doc.xpath('//meta[@http-equiv="refresh"]'): - error_message = self.get_error_message() - if 'Le numéro de sécurité sociale et le code personnel' in error_message: - raise BrowserIncorrectPassword(error_message) - raise AssertionError(error_message) - super(RedirectPage, self).on_load() - - def get_error_message(self): - return CleanText( - '//div[@id="loginPage"]//div[has-class("zone-alerte") and not(has-class("hidden"))]/span' - )(self.doc) - class CguPage(LoggedPage, HTMLPage): def get_cgu_message(self): @@ -73,9 +71,8 @@ def get_cgu_message(self): class ErrorPage(HTMLPage): - def on_load(self): - msg = CleanText('//div[@id="backgroundId"]//p')(self.doc) - raise BrowserUnavailable(msg) + def get_error_message(self): + return html_unescape(CleanText('//div[@class="mobile"]/p')(self.doc)) class SubscriptionPage(LoggedPage, HTMLPage): @@ -115,7 +112,9 @@ def obj_id(self): obj_format = 'pdf' def obj_date(self): - year = Regexp(CleanText('./preceding-sibling::li[@class="rowdate"]//span[@class="mois"]'), r'(\d+)')(self) + year = Regexp( + CleanText('./preceding-sibling::li[@class="rowdate"]//span[@class="mois"]'), r'(\d+)' + )(self) day_month = CleanText('.//div[has-class("col-date")]/span')(self) return parse_french_date(day_month + ' ' + year) diff --git a/modules/americanexpress/browser.py b/modules/americanexpress/browser.py index c3144f90640ee6c81a98e2521b4cd21abab4407d..b888ae70d07589e6fa4d360811e74030efc43a8e 100644 --- a/modules/americanexpress/browser.py +++ b/modules/americanexpress/browser.py @@ -20,31 +20,45 @@ from __future__ import unicode_literals import datetime -from uuid import uuid4 +import uuid from dateutil.parser import parse as parse_date from collections import OrderedDict -from woob.exceptions import BrowserIncorrectPassword, ActionNeeded, BrowserUnavailable -from woob.browser.browsers import LoginBrowser, need_login -from woob.browser.exceptions import HTTPNotFound, ServerError +from woob.exceptions import ( + BrowserIncorrectPassword, ActionNeeded, BrowserUnavailable, + AuthMethodNotImplemented, BrowserQuestion, +) +from woob.browser.browsers import TwoFactorBrowser, need_login +from woob.browser.exceptions import HTTPNotFound, ServerError, ClientError from woob.browser.url import URL -from woob.tools.compat import urlencode +from woob.tools.compat import urljoin, urlencode, quote +from woob.tools.value import Value from .pages import ( AccountsPage, JsonBalances, JsonPeriods, JsonHistory, JsonBalances2, CurrencyPage, LoginPage, NoCardPage, - NotFoundPage, JsDataPage, HomeLoginPage, + NotFoundPage, HomeLoginPage, + ReadAuthChallengePage, UpdateAuthTokenPage, ) +from .fingerprint import FingerprintPage + __all__ = ['AmericanExpressBrowser'] -class AmericanExpressBrowser(LoginBrowser): +class AmericanExpressBrowser(TwoFactorBrowser): BASEURL = 'https://global.americanexpress.com' + TWOFA_BASEURL = r'https://functions.americanexpress.com' home_login = URL(r'/login\?inav=fr_utility_logout', HomeLoginPage) login = URL(r'/myca/logon/emea/action/login', LoginPage) + fingerprint = URL(r'https://www.cdn-path.com/cc.js\?=&sid=ee490b8fb9a4d570&tid=(?P.*)&namespace=inauth', FingerprintPage) + + read_auth_challenges = URL(TWOFA_BASEURL + r'/ReadAuthenticationChallenges.v1', ReadAuthChallengePage) + create_otp_uri = URL(TWOFA_BASEURL + r'/CreateOneTimePasscodeDelivery.v1') + update_auth_token = URL(TWOFA_BASEURL + r'/UpdateAuthenticationTokenWithChallenge.v1', UpdateAuthTokenPage) + create_2fa_uri = URL(TWOFA_BASEURL + r'/CreateTwoFactorAuthenticationForUser.v1') accounts = URL(r'/api/servicing/v1/member', AccountsPage) json_balances = URL(r'/api/servicing/v1/financials/balances', JsonBalances) @@ -60,8 +74,6 @@ class AmericanExpressBrowser(LoginBrowser): json_periods = URL(r'/api/servicing/v1/financials/statement_periods', JsonPeriods) currency_page = URL(r'https://www.aexp-static.com/cdaas/axp-app/modules/axp-balance-summary/4.7.0/(?P\w\w-\w\w)/axp-balance-summary.json', CurrencyPage) - js_data = URL(r'/myca/logon/us/docs/javascript/gatekeeper/gtkp_aa.js', JsDataPage) - no_card = URL(r'https://www.americanexpress.com/us/content/no-card/', r'https://www.americanexpress.com/us/no-card/', NoCardPage) @@ -72,52 +84,76 @@ class AmericanExpressBrowser(LoginBrowser): 'PRELEVEMENT AUTOMATIQUE ENREGISTRE-MERCI', ] + HAS_CREDENTIALS_ONLY = True + def __init__(self, *args, **kwargs): super(AmericanExpressBrowser, self).__init__(*args, **kwargs) - def get_version(self): - self.js_data.go() - return self.page.get_version() + # State to keep during OTP + self.authentication_action_id = None + self.application_id = None + self.account_token = None + self.mfa_id = None + self.auth_trusted = None + + self.__states__ += ( + 'authentication_action_id', + 'application_id', + 'account_token', + 'mfa_id', + 'auth_trusted', + ) - def do_login(self): + self.AUTHENTICATION_METHODS = { + 'otp': self.handle_otp, + } + + def init_login(self): self.home_login.go() + now = datetime.datetime.utcnow() + transaction_id = 'LOGIN-%s' % uuid.uuid4() # Randomly generated in js + + self.register_transaction_id(transaction_id, now) + data = { 'request_type': 'login', + 'Face': 'fr_FR', + 'Logon': 'Logon', + 'version': 4, + 'inauth_profile_transaction_id': transaction_id, + 'DestPage': urljoin(self.BASEURL,'dashboard'), 'UserID': self.username, 'Password': self.password, - 'Logon': 'Logon', + 'channel': 'Web', 'REMEMBERME': 'on', - 'Face': 'fr_FR', - 'DestPage': self.BASEURL + '/dashboard', - 'inauth_profile_transaction_id': 'USLOGON-%s' % str(uuid4()), + 'b_hour': now.hour, + 'b_minute': now.minute, + 'b_second': now.second, + 'b_dayNumber': now.day, + 'b_month': now.month, + 'b_year': now.year, + 'b_timeZone': '0', + 'devicePrint': self.make_device_print(), } - # we have to overwrite `Content-Length` and `Cookie` to get all - # headers in alphabetical order or they will be added at the end - # when doing request, also we add every headers needed on website - # to try to exactly match what's done or we could get a LGON011 error - self.session.headers.update({ + self.send_login_request(data) + + def send_login_request(self, data): + # Match the headers on website to prevent LGON011 error + headers_for_login = { 'Accept': '*/*', - 'Accept-Encoding': 'gzip: deflate: br', - 'Accept-Language': 'en-US,en;q=0.9,fr;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', 'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8', - 'Content-Length': str(len(urlencode(data))), - 'Cookie': '; '.join('%s=%s' % (k, v) for k, v in self.session.cookies.get_dict().items()), - 'Origin': self.BASEURL, - 'Referer': self.BASEURL + '/login?inav=fr_utility_logout', - 'Sec-Fetch-Dest': 'empty', - 'Sec-Fetch-Mode': 'cors', - 'Sec-Fetch-Site': 'same-origin', - }) + 'Origin': 'https://www.americanexpress.com', + 'Host': 'global.americanexpress.com', - self.session.headers = OrderedDict(sorted(self.session.headers.items())) - del self.session.headers['Upgrade-Insecure-Requests'] - - self.login.go(data=data) + # Setting headers to None to remove them from the request + 'Referer': None, + 'Upgrade-Insecure-Requests': None, + } - # set back headers - self.set_profile(self.PROFILE) + self.login.go(data=data, headers=headers_for_login) if self.page.get_status_code() != 0: error_code = self.page.get_error_code() @@ -141,15 +177,233 @@ def do_login(self): # - headers not in the right order # - headers with value that doesn't match the one from website # - headers missing - # what's next ? - assert False, 'Error code "LGON011" (msg:"%s")' % message + # - IP blacklisted + # What's next ? + raise AssertionError('Error code "LGON011" (msg:"%s")' % message) + elif error_code == 'LGON013': + self.raise_otp() else: - assert False, 'Error code "%s" (msg:"%s") not handled' % (error_code, message) + raise AssertionError('Error code "%s" (msg:"%s") not handled' % (error_code, message)) + + def prepare_request(self, req): + # Get all headers in alphabetical order to prevent LGON011 error + prep = super(AmericanExpressBrowser, self).prepare_request(req) + prep.headers = OrderedDict(sorted(prep.headers.items(), key=lambda i: i[0].lower())) + return prep + + def locate_browser(self, url): + # For some reason, it looks like we cannot reconnect from storage. + pass + + def clear_init_cookies(self): + # Keep the device-id to prevent an SCA + for cookie in self.session.cookies: + if cookie.name == "device-id": + device = cookie + break + else: + device = None + self.session.cookies.clear() + if device: + self.session.cookies.set_cookie(device) + + def register_transaction_id(self, transaction_id, now): + self.fingerprint.go(transaction_id=transaction_id) + payload = self.page.make_payload_for_s2(transaction_id, now) + self.open('https://www.cdn-path.com/s2', method="POST", + params={ + 't': self.page.get_t(), + 'x': 1, # Not seen change yet + 'sid': 'ee490b8fb9a4d570', # Not seen change yet + 'tid': transaction_id, + }, + files = { + '_f': payload, + }, + headers = { + 'Accept-Encoding': 'gzip, deflate, br', + 'Host': 'www.cdn-path.com', + 'Origin': 'https://www.americanexpress.com', + 'Referer': 'https://www.americanexpress.com/', + 'Pragma': 'no-cache', + 'TE': 'Trailers', + }, + ) + + def make_device_print(self): + d = OrderedDict() + d['version'] = "3.4.0.0_1" + d['pm_fpua'] = self.session.headers['User-Agent'] + '|5.0 (X11)|Linux x86_64' + d['pm_fpsc'] = '24|1650|498|498' + d['pm_fptw'] = '' + d['pm_fptz'] = 0 + d['pm_fpln'] = 'lang=en-US|syslang=|userlang=' + d['pm_fpjv'] = 0 + d['pm_fpco'] = 1 + d['pm_fpasw'] = '' + d['pm_fpan'] = "Netscape" + d['pm_fpacn'] = "Mozilla" + d['pm_fpol'] = 'true' + d['pm_fposp'] = '' + d['pm_fpup'] = '' + d['pm_fpsaw'] = '1920' + d['pm_fpspd'] = '24' + d['pm_fpsbd'] = '' + d['pm_fpsdx'] = '' + d['pm_fpsdy'] = '' + d['pm_fpslx'] = '' + d['pm_fpsly'] = '' + d['pm_fpsfse'] = '' + d['pm_fpsui'] = '' + d['pm_os'] = 'Linux' + d['pm_brmjv'] = 78 + d['pm_br'] = 'Firefox' + d['pm_inpt'] = '' + d['pm_expt'] = '' + return ( + urlencode(d,quote_via=quote) # using quote to prevent encoding space as + + # The next four character are not quoted by quote + .replace('~', "%7E") + .replace('-', "%2D") + .replace('_', "%5F") + .replace('.', "%2E") + + # These replace are to remove the & and = included by urlencode + .replace('=', "%3D") + .replace('&', "%26") + ) + + def raise_otp(self): + self.check_interactive() + + reauth = self.page.get_reauth() + self.authentication_action_id = reauth["actionId"] + self.application_id = reauth["applicationId"] + self.mfa_id = reauth["mfaId"] + self.auth_trusted = reauth["trust"] + + if not self.auth_trusted: + self.logger.warning( + "We are not trusted. There could be a problem with the fingerprinting of cc.js" + ) + + read_auth_challenges_payload = [{ + "authenticationActionId": self.authentication_action_id, + "applicationId": self.application_id, + "locale": self.locale, + }] + self.read_auth_challenges.go(json=read_auth_challenges_payload) + + challenge = self.page.get_challenge() + assert challenge == "OTP", "We don't know how to handle '%s' challenge." % challenge + + self.account_token = self.page.get_account_token() + methods = self.page.get_otp_methods() + delivery_payload, message = self.make_otp_delivery_payload(methods) + + self.create_otp_uri.go(json=delivery_payload) + raise BrowserQuestion( + Value('otp', label=message) + ) + + def make_otp_delivery_payload(self, methods): + known_methods = ["SMS", "EMAIL"] # This is also our preference order. + methods = {m["deliveryMethod"]: m for m in methods} + + chosen_method = None + + # Select the 2FA method for this authentification. + # Search for them in the order of known_methods. + for known_method in known_methods: + chosen_method = methods.get(known_method) + if chosen_method: + break + + if chosen_method is None: + assert methods != {}, "Received no challenge option" + raise AuthMethodNotImplemented(', '.join(methods.keys())) + + delivery_method = chosen_method["deliveryMethod"] + delivery_payload = [{ + "authenticationActionId": self.authentication_action_id, + "applicationId": self.application_id, + "accountToken": self.account_token, + "locale": self.locale, + "deliveryMethod": delivery_method, + "channelType": chosen_method["channelType"], + "channelEncryptedValue": chosen_method["channelEncryptedValue"], + }] + + display_value = chosen_method["channelDisplayValue"] + if delivery_method == "EMAIL": + message = "Veuillez entrer le code d’authentification qui vous a été envoyé à l'adresse courriel %s." % display_value + else: + message = "Veuillez entrer le code d’authentification qui vous a été envoyé au %s." % display_value + + return delivery_payload, message + + def handle_otp(self): + update_auth_token_payload = [{ + "authenticationActionId": self.authentication_action_id, + "applicationId": self.application_id, + "accountToken": self.account_token, + "locale": self.locale, + "fieldName": "OTP", + "fieldValue": self.otp, + }] + try: + self.update_auth_token.go(json=update_auth_token_payload) + pending_challenge = self.page.get_pending_challenges() + except ClientError as e: + self.drop_2fa_state() + if e.response.status_code == 400 and "UEVE008" in e.response.text: + # {"description":"Invalid Claim: Data does not match SOR","errorCode":"UEVE008"} + raise BrowserIncorrectPassword("Mauvais code lors de l'authentification forte.") + raise + + if pending_challenge != "": + self.drop_2fa_state() + raise AssertionError("Multiple challenge not handled by the module yet.") + + self.enrol_device() + self.tfa_login() + self.drop_2fa_state() + + def drop_2fa_state(self): + self.account_token = None + self.application_id = None + self.authentication_action_id = None + self.mfa_id = None + self.auth_trusted = None + + def enrol_device(self): + if self.auth_trusted: + enrol_payload = [{ + "locale": self.locale, + "trust": self.auth_trusted, + "deviceName":"Accès Budget Insight pour agrégation", + }] + self.create_2fa_uri.go(json=enrol_payload) + else: + self.logger.info("Cannot enrol when we are not trusted.") + + def tfa_login(self): + data = { + 'request_type': "login", + 'Face': 'fr_FR', + 'Logon': 'Logon', + 'version': 4, + 'mfaId': self.mfa_id, + } + self.send_login_request(data) + + @property + def locale(self): + return self.session.cookies.get_dict(domain=".americanexpress.com")['axplocale'] @need_login def iter_accounts(self): - loc = self.session.cookies.get_dict(domain=".americanexpress.com")['axplocale'].lower() - self.currency_page.go(locale=loc) + self.currency_page.go(locale=self.locale.lower()) currency = self.page.get_currency() self.accounts.go() diff --git a/modules/americanexpress/compat/__init__.py b/modules/americanexpress/compat/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/modules/americanexpress/fingerprint.py b/modules/americanexpress/fingerprint.py new file mode 100644 index 0000000000000000000000000000000000000000..24d8bf1733deb4c1151e9896afe8aeb239c786c1 --- /dev/null +++ b/modules/americanexpress/fingerprint.py @@ -0,0 +1,510 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2021 Martin Lavoie +# +# This file is part of a woob module. +# +# This woob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This woob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this woob module. If not, see . + +import re +import json +from itertools import cycle +from base64 import b64encode, b64decode + +from woob.browser.pages import RawPage + + +def xor4(plaintext): + """ + function za from cc.js + + Apply one of 4 transformations to + every character of the text. + + The transformations are + (XOR 89), (XOR 231), (XOR 225) and (XOR 55). + """ + key = [89, 231, 225, 55] + for p, c in zip(plaintext, cycle(key)): + r = ord(p) ^ c + yield chr(r) + + +def utf8_encode(pt): + return ''.join(pt).encode('utf8') + + +def encode(content): + """ + Takes a dictionnary and returns a base64 string representation. + function Q in cc.js + """ + json_content = json.dumps(content, separators=(',',':')) + xor_content = xor4(json_content) + utf8_content = utf8_encode(xor_content) + return b64encode(utf8_content).decode('ascii') + + +def decode(cyphertext): + """ + Inverse of encode + """ + utf8_content = b64decode(cyphertext).decode('utf8') + unxor_content = ''.join(xor4(utf8_content)) + return json.loads(unxor_content) + + +class FingerprintPage(RawPage): + def get_t(self): + return self.get_I()[:24] + + def get_I(self): + """ + Random bits encoded in base64 in the javascript code + It contained the the I variable in cc.js + """ + regex = r'run[^,]*="([a-zA-Z0-9+=/]+)"' + match = re.search(regex, self.text, re.DOTALL) + assert match, "Could not find the secret I" + return match[1] + + def make_payload_for_s2(self, tid, now): + cookie = self.browser.session.cookies['_cc-x'] + user_agent = self.browser.session.headers['User-Agent'] + return encode(self.make_payload(tid, cookie, user_agent, now)) + + def make_payload(self, tid, cookie_cc, user_agent, now): + """ + Create a payload for the s2 verification. + The original code in is cc.js where it has the name run. + + How to update the dictionary: + - Locate the s2 request to cdn-path.com while + doing an authentification in the browser. + - Pass as argument to the decode function + the long random line in the body of the request. + - Update the dictionary with the result. + - Put back the computed value for the fields that + we support. + + In case of problem with decode, you could check if + the magic key of xor4 has changed. If not, then + there is no easy path for you. + + How to update the dictionary (harder way): + - Launch Chromium + - Toggle the breakpoint "Script First Statement" in + "Event Listener Breakpoints/Script/Script First Statement" + - Do an authentification + - Continue through the breakpoints until you get + the script cc.js + - Find a location where you can see the new dictionary. + Keywords: + run : name of the function constructing the dictionary. + stringify : function used to transform the dictionary + into JSON. The data should pass through it. + + If you need to take a deep dive into the cc.js script, + I recommend using the following method. + + How to update the dictionary (hardest way): + - Locate the cc.js request to cdn-path.com while + doing an authentification in the browser. + Make a copy of this file on your computer. + - Find the javascript code that made the request to cc.js. + You should find multiple occurence of the code + `_cc.push` just before the request. + The values pushed are the command to be executed + when the javascript code (cc.js) will load. + - Make an empty html page with a script adding the + same command and loading your local copy of the code. + That will allow to step through the code easily. + - (Optional) Remove or comment the line in your + local copy of cc.js that is sending the s2 request. + That way you can debug without hiting the server. + """ + + time_local = now.strftime('%-m/%-d/%Y, %-I:%-M:%-S %p') + time_string = now.strftime('%a %b %d %Y %I:%M:%S GMT+0000 (UTC)') + unix_epoch = round(now.timestamp() * 1000) + + return { + "sid": "ee490b8fb9a4d570", + "tid": tid, + "_t": self.get_I(), + "cf_flags": 135732211, + "cdfr": True, + "cookie-_cc": cookie_cc, + "timing-sc": 0, + "time-unix-epoch-ms": unix_epoch, + "time-local": time_local, + "time-string": time_string, + "time-tz-offset-minutes": 0, + "time-tz-has-dst": "false", + "time-tz-dst-active": "false", + "time-tz-std-offset": 0, + "time-tz-fixed-locale-string": "3/6/2014, 7:58:39 AM", + "timing-ti": 1, + "dom-local-tag": cookie_cc, + "timing-ls": 0, + "dom-session-tag": cookie_cc, + "timing-ss": 0, + "navigator.appVersion": "5.0 (X11)", + "navigator.appName": "Netscape", + "navigator.product": "Gecko", + "navigator.buildID": "20181001000000", + "navigator.platform": "Linux x86_64", + "navigator.language": "en-US", + "navigator.oscpu": "Linux x86_64", + "navigator.userAgent": user_agent, + "navigator.cookieEnabled": "true", + "navigator.appCodeName": "Mozilla", + "navigator.productSub": "20100101", + "timing-no": 0, + "navigator.hardwareConcurrency": "8", + "touchEnabled": False, + "navigator.automationEnabled": False, + "navigator.doNotTrack": "1", + "window.screen.pixelDepth": "24", + "window.screen.height": "1080", + "window.screen.colorDepth": "24", + "window.menubar.visible": "true", + "window.devicePixelRatio": "1", + "window.history.length": "3", + "window.screen.width": "1920", + "timing-wo": 0, + "window.screen.availHeight": "1080", + "window.screen.orientation.type": "landscape-primary", + "window.screen.orientation.angle": "0", + "window.screen.darkMode.enabled": False, + "timing-do": 0, + "plugin-suffixes": "", + "plugin-mimes": "", + "timing-np": 0, + "timing-iepl": 0, + "canvas-print-100-999": "8ac8e8cd487550a157647ee7b84032290b33ca0b", + "canvas-print-detailed-100-999": "904566a766a494df6ae8a327276267e1e38b0738", + "timing-cp": 40, + "timing-gief": 0, + "js-errors": [ + "InvalidCharacterError: String contains an invalid character", + "InvalidCharacterError: String contains an invalid character" + ], + "font-Times New Roman CYR": False, + "font-Arial CYR": False, + "font-Courier New CYR": False, + "font-宋体": False, + "font-Arial Cyr": False, + "font-Times New Roman Cyr": False, + "font-Courier New Cyr": False, + "font-华文细黑": False, + "font-儷黑 Pro": False, + "font-WP CyrillicB": False, + "font-WP CyrillicA": False, + "font-궁서체": False, + "font-細明體": False, + "font-小塚明朝 Pr6N B": False, + "font-宋体-PUA": False, + "font-方正流行体繁体": False, + "font-汉仪娃娃篆简": False, + "font-돋움": False, + "font-GaramondNo4CyrTCYLig": False, + "font-HelveticaInseratCyr Upright": False, + "font-HelveticaCyr Upright": False, + "font-TL Help Cyrillic": False, + "font-가는안상수체": False, + "font-TLCyrillic2": False, + "font-AGRevueCyr-Roman": False, + "font-AGOptimaCyr": False, + "font-HelveticaInseratCyrillicUpright": False, + "font-HelveticaCyrillicUpright": False, + "font-HelveticaCyrillic": False, + "font-CyrillicRibbon": False, + "font-CyrillicHover": False, + "font-文鼎POP-4": False, + "font-方正中倩简体": False, + "font-创艺简中圆": False, + "font-Zrnic Cyr": False, + "font-Zipper1 Cyr": False, + "font-Xorx_windy Cyr": False, + "font-Xorx_Toothy Cyr": False, + "font-소야솔9": False, + "font-Цветные эмодзи Apple": False, + "font-Chinese Generic1": False, + "font-Korean Generic1": False, + "font-Bullets 5(Korean)": True, + "font-UkrainianFuturisExtra": False, + "font-VNI-Viettay": False, + "font-UkrainianCompact": False, + "font-UkrainianBrushScript": False, + "font-TiffanyUkraine": False, + "font-Baltica_Russian-ITV": False, + "font-Vietnamese font": False, + "font-Unicorn Ukrainian": False, + "font-UkrainianTimesET": False, + "font-UkrainianCourier": False, + "font-Tiff-HeavyUkraine": False, + "font-䡵湧䱡渠䅲瑤敳楧渠㈰〲‭⁁汬⁲楧桴猠牥獥牶敤⹔桵⁰桡瀠噎周畦慰〲†乯牭慬ㄮ〠䍯摥⁖义⁦潲⁗楮摯睳周畦慰〲乯牭慬HungLan Artdesign - http://www.vietcomic.comVNI-Thufap2 Normalv2.0 Code VNI for WindowsVNI-Thufap2 Normal\u0002": True, + "font-Vietnam": False, + "font-Bwviet": False, + "font-Soviet": False, + "font-Soviet Expanded": False, + "font-Soviet Bold": False, + "font-Russian": False, + "font-UVN Han Viet": False, + "font-UkrainianAcademy": False, + "font-Symbol": True, + "font-Verdana": False, + "font-Webdings": False, + "font-Arial": True, + "font-Georgia": False, + "font-Courier New": True, + "font-Trebuchet MS": False, + "font-Times New Roman": True, + "font-Impact": False, + "font-Comic Sans MS": False, + "font-Wingdings": False, + "font-Tahoma": False, + "font-Microsoft Sans Serif": False, + "font-Arial Black": False, + "font-Plantagenet Cherokee": False, + "font-Arial Narrow": True, + "font-Wingdings 2": True, + "font-Wingdings 3": True, + "font-Arial Unicode MS": False, + "font-Papyrus": False, + "font-Calibri": False, + "font-Cambria": False, + "font-Consolas": False, + "font-Candara": False, + "font-Franklin Gothic Medium": False, + "font-Corbel": False, + "font-Constantia": False, + "font-Marlett": False, + "font-Lucida Console": False, + "font-Lucida Sans Unicode": False, + "font-MS Mincho": False, + "font-Arial Rounded MT Bold": False, + "font-Palatino Linotype": False, + "font-Batang": False, + "font-MS Gothic": False, + "font-PMingLiU": False, + "font-SimSun": False, + "font-MS PGothic": False, + "font-MS PMincho": False, + "font-Gulim": False, + "font-Cambria Math": False, + "font-Garamond": False, + "font-Bookman Old Style": False, + "font-Book Antiqua": False, + "font-Century Gothic": False, + "font-Monotype Corsiva": False, + "font-Courier": False, + "font-Meiryo": False, + "font-Century": False, + "font-MT Extra": False, + "font-MS Reference Sans Serif": False, + "font-MS Reference Specialty": False, + "font-Mistral": False, + "font-Bookshelf Symbol 7": True, + "font-Lucida Bright": False, + "font-Cooper Black": False, + "font-Modern No. 20": True, + "font-Bernard MT Condensed": False, + "font-Bell MT": False, + "font-Baskerville Old Face": False, + "font-Bauhaus 93": True, + "font-Britannic Bold": False, + "font-Wide Latin": False, + "font-Playbill": False, + "font-Harrington": False, + "font-Onyx": False, + "font-Footlight MT Light": False, + "font-Stencil": False, + "font-Colonna MT": False, + "font-Matura MT Script Capitals": False, + "font-Copperplate Gothic Bold": False, + "font-Copperplate Gothic Light": False, + "font-Edwardian Script ITC": False, + "font-Rockwell": False, + "font-Curlz MT": False, + "font-Engravers MT": False, + "font-Rockwell Extra Bold": False, + "font-Haettenschweiler": False, + "font-MingLiU": False, + "font-Mongolian Baiti": False, + "font-Microsoft Yi Baiti": False, + "font-Microsoft Himalaya": False, + "font-SimHei": False, + "font-SimSun-ExtB": False, + "font-PMingLiU-ExtB": False, + "font-MingLiU-ExtB": False, + "font-MingLiU_HKSCS-ExtB": False, + "font-MingLiU_HKSCS": False, + "font-Gabriola": False, + "font-Goudy Old Style": False, + "font-Calisto MT": False, + "font-Imprint MT Shadow": False, + "font-Gill Sans Ultra Bold": False, + "font-Century Schoolbook": False, + "font-Gloucester MT Extra Condensed": False, + "font-Perpetua": False, + "font-Franklin Gothic Book": False, + "font-Brush Script MT": False, + "font-Microsoft Tai Le": False, + "font-Gill Sans MT": False, + "font-Tw Cen MT": False, + "font-Lucida Handwriting": False, + "font-Lucida Sans": False, + "font-Segoe UI": False, + "font-Lucida Fax": False, + "font-MV Boli": False, + "font-Sylfaen": False, + "font-Estrangelo Edessa": False, + "font-Mangal": True, + "font-Gautami": False, + "font-Tunga": False, + "font-Shruti": False, + "font-Raavi": False, + "font-Latha": False, + "font-Lucida Calligraphy": False, + "font-Lucida Sans Typewriter": False, + "font-Kartika": False, + "font-Vrinda": False, + "font-Perpetua Titling MT": False, + "font-Cordia New": True, + "font-Angsana New": True, + "font-IrisUPC": False, + "font-CordiaUPC": True, + "font-FreesiaUPC": False, + "font-Miriam": True, + "font-Traditional Arabic": False, + "font-Miriam Fixed": True, + "font-JasmineUPC": False, + "font-KodchiangUPC": False, + "font-LilyUPC": False, + "font-Levenim MT": True, + "font-EucrosiaUPC": False, + "font-DilleniaUPC": False, + "font-Rod": False, + "font-Narkisim": False, + "font-FrankRuehl": True, + "font-David": True, + "font-Andalus": False, + "font-Browallia New": True, + "font-AngsanaUPC": True, + "font-BrowalliaUPC": True, + "font-MS UI Gothic": False, + "font-Aharoni": True, + "font-Simplified Arabic Fixed": False, + "font-Simplified Arabic": False, + "font-GulimChe": False, + "font-Dotum": False, + "font-DotumChe": False, + "font-GungsuhChe": False, + "font-Gungsuh": False, + "font-BatangChe": False, + "font-Meiryo UI": False, + "font-NSimSun": False, + "font-Segoe Script": False, + "font-Segoe Print": False, + "font-DaunPenh": False, + "font-Kalinga": False, + "font-Iskoola Pota": False, + "font-Euphemia": False, + "font-DokChampa": False, + "font-Nyala": False, + "font-MoolBoran": False, + "font-Leelawadee": False, + "font-Gisha": False, + "font-Microsoft Uighur": False, + "font-Arabic Typesetting": False, + "font-Malgun Gothic": False, + "font-Microsoft JhengHei": False, + "font-DFKai-SB": False, + "font-Microsoft YaHei": False, + "font-FangSong": False, + "font-KaiTi": False, + "font-Helvetica": False, + "font-Segoe UI Light": False, + "font-Segoe UI Semibold": False, + "font-Andale Mono": False, + "font-Palatino": False, + "font-Geneva": False, + "font-Monaco": False, + "font-Lucida Grande": False, + "font-Gill Sans": False, + "font-Helvetica Neue": False, + "font-Baskerville": False, + "font-Hoefler Text": False, + "font-Thonburi": False, + "font-Herculanum": False, + "font-Apple Chancery": False, + "font-Didot": False, + "font-Zapf Dingbats": False, + "font-Apple Symbols": False, + "font-Copperplate": False, + "font-American Typewriter": False, + "font-Zapfino": False, + "font-Cochin": False, + "font-Chalkboard": False, + "font-Sathu": False, + "font-Osaka": False, + "font-BiauKai": False, + "font-Segoe UI Symbol": False, + "font-Aparajita": False, + "font-Krungthep": False, + "font-Ebrima": False, + "font-Silom": False, + "font-Kokila": False, + "font-Shonar Bangla": False, + "font-Sakkal Majalla": False, + "font-Microsoft PhagsPa": False, + "font-Microsoft New Tai Lue": False, + "font-Khmer UI": False, + "font-Vijaya": False, + "font-Utsaah": False, + "font-Charcoal CY": False, + "font-Ayuthaya": False, + "font-InaiMathi": False, + "font-Euphemia UCAS": False, + "font-Vani": False, + "font-Lao UI": False, + "font-GB18030 Bitmap": False, + "font-KufiStandardGK": False, + "font-Geeza Pro": False, + "font-Chalkduster": False, + "font-Tempus Sans ITC": False, + "font-Kristen ITC": False, + "font-Apple Braille": False, + "font-Juice ITC": False, + "font-STHeiti": False, + "font-LiHei Pro": False, + "font-DecoType Naskh": False, + "font-New Peninim MT": True, + "font-Nadeem": False, + "font-Mshtakan": False, + "font-Gujarati MT": False, + "font-Devanagari MT": False, + "font-Arial Hebrew": False, + "font-Corsiva Hebrew": False, + "font-Baghdad": False, + "font-STFangsong": False, + "timing-kf": 45, + "webgl-supported": False, + "timing-wgl": 0, + "timing-sync-collection": 90, + "timing-generation": 2, + "timing-wr": 177 + } diff --git a/modules/americanexpress/module.py b/modules/americanexpress/module.py index 5564af8deb0ae15712002d958835acc87a62e2d0..665bb00ac65394b8d032d702f704ad739e54c889 100644 --- a/modules/americanexpress/module.py +++ b/modules/americanexpress/module.py @@ -20,7 +20,7 @@ from woob.capabilities.bank import CapBank from woob.tools.backend import Module, BackendConfig -from woob.tools.value import ValueBackendPassword +from woob.tools.value import ValueBackendPassword, ValueTransient from .browser import AmericanExpressBrowser @@ -37,12 +37,15 @@ class AmericanExpressModule(Module, CapBank): LICENSE = 'LGPLv3+' CONFIG = BackendConfig( ValueBackendPassword('login', label='Code utilisateur', masked=False), - ValueBackendPassword('password', label='Mot de passe') + ValueBackendPassword('password', label='Mot de passe'), + ValueTransient('request_information'), + ValueTransient('otp', regexp=r'^\d{6}$'), ) BROWSER = AmericanExpressBrowser def create_default_browser(self): return self.create_browser( + self.config, self.config['login'].get(), self.config['password'].get() ) diff --git a/modules/americanexpress/pages.py b/modules/americanexpress/pages.py index 7b5ffdaae04d4fd4dd149eee52979ce30c326215..0b202c8012218f7de3686fb30a9ebe15bbd1646e 100644 --- a/modules/americanexpress/pages.py +++ b/modules/americanexpress/pages.py @@ -20,9 +20,8 @@ from __future__ import unicode_literals from decimal import Decimal -import re -from woob.browser.pages import LoggedPage, JsonPage, HTMLPage, RawPage +from woob.browser.pages import LoggedPage, JsonPage, HTMLPage from woob.browser.elements import ItemElement, DictElement, method from woob.browser.filters.standard import ( Date, Eval, Env, CleanText, Field, CleanDecimal, Format, @@ -77,6 +76,7 @@ def get_error_code(self): # - LGON005 = Account blocked # - LGON008 = ? # - LGON010 = Browser unavailable + # - LGON013 = SCA return CleanText(Dict('errorCode'))(self.doc) def get_error_message(self): @@ -88,6 +88,27 @@ def get_error_message(self): def get_redirect_url(self): return CleanText(Dict('redirectUrl'))(self.doc) + def get_reauth(self): + return Dict('reauth')(self.doc) + + +class ReadAuthChallengePage(JsonPage): + def get_challenge(self): + return Dict("challenge")(self.doc) + + def get_account_token(self): + identity_data = Dict("identityData")(self.doc) + assert len(identity_data) == 1, "How can we have multiple identity_data?" + return identity_data[0]["identityValue"] + + def get_otp_methods(self): + return Dict("tenuredChannels")(self.doc) + + +class UpdateAuthTokenPage(JsonPage): + def get_pending_challenges(self): + return Dict('pendingChallenges')(self.doc) + class AccountsPage(LoggedPage, JsonPage): @method @@ -213,10 +234,3 @@ def obj_original_amount(self): return original_amount obj__ref = Dict('identifier') - - -class JsDataPage(RawPage): - def get_version(self): - version = re.search(r'"(\d\.[\d\._]+)"', self.text) - assert version, 'Could not match version number in javascript' - return version.group(1) diff --git a/modules/april/module.py b/modules/april/module.py index a584be4adc0c134dd3692fa5ed3a10409ec4218e..d988e7e39c1ead28332d71acfa40b71d2795dc54 100644 --- a/modules/april/module.py +++ b/modules/april/module.py @@ -48,7 +48,7 @@ class AprilModule(Module, CapDocument, CapProfile): MAINTAINER = "Ludovic LANGE" EMAIL = "llange@users.noreply.github.com" LICENSE = "LGPLv3+" - VERSION = "3.0" + VERSION = '3.0' BROWSER = AprilBrowser diff --git a/modules/banquepopulaire/browser.py b/modules/banquepopulaire/browser.py index d05e57659d70077a54ca0a4a70ddedf25cfd6be1..215d5c51c98e49293b7f7c66a64f229a6c8e748d 100644 --- a/modules/banquepopulaire/browser.py +++ b/modules/banquepopulaire/browser.py @@ -50,7 +50,7 @@ NewLoginPage, JsFilePage, AuthorizePage, LoginTokensPage, VkImagePage, AuthenticationMethodPage, AuthenticationStepPage, CaissedepargneVirtKeyboard, AccountsNextPage, GenericAccountsPage, InfoTokensPage, NatixisUnavailablePage, - RedirectErrorPage, + RedirectErrorPage, BPCEPage, ) from .document_pages import BasicTokenPage, SubscriberPage, SubscriptionsPage, DocumentsPage from .linebourse_browser import LinebourseAPIBrowser @@ -115,6 +115,7 @@ def wrapper(browser, *args, **kwargs): class BanquePopulaire(LoginBrowser): + first_login_page = URL(r'/$') login_page = URL(r'https://[^/]+/auth/UI/Login.*', LoginPage) new_login = URL(r'https://[^/]+/.*se-connecter/sso', NewLoginPage) js_file = URL(r'https://[^/]+/.*se-connecter/main-.*.js$', JsFilePage) @@ -206,6 +207,7 @@ class BanquePopulaire(LoginBrowser): ) redirect_page = URL(r'https://[^/]+/portailinternet/_layouts/Ibp.Cyi.Layouts/RedirectSegment.aspx.*', RedirectPage) + bpce_page = URL(r'https://[^/]+/cyber/ibp/ate/portal/internet89C3Portal.jsp', BPCEPage) redirect_error_page = URL( r'https://[^/]+/portailinternet/?$', @@ -218,6 +220,7 @@ class BanquePopulaire(LoginBrowser): r'https://[^/]+/portailinternet/Pages/[dD]efault.aspx', r'https://[^/]+/portailinternet/Transactionnel/Pages/CyberIntegrationPage.aspx', r'https://[^/]+/cyber/internet/ShowPortal.do\?token=.*', + r'https://[^/]+/cyber/internet/ShowPortal.do\?taskInfoOID=.*', HomePage ) @@ -326,7 +329,7 @@ def follow_back_button_if_any(self, params=None, actions=None): @no_need_login def do_login(self): try: - self.location(self.BASEURL) + self.first_login_page.go() except (ClientError, HTTPNotFound) as e: if e.response.status_code in (403, 404): # Sometimes the website makes some redirections that leads diff --git a/modules/banquepopulaire/pages.py b/modules/banquepopulaire/pages.py index ed1043cfca848e0a127edb59fdb71bc7d8f63268..76297eda8209d9db066da934a5f7c6aa33e0480c 100644 --- a/modules/banquepopulaire/pages.py +++ b/modules/banquepopulaire/pages.py @@ -203,6 +203,10 @@ def build_doc(self, data, *args, **kwargs): return super(MyHTMLPage, self).build_doc(data, *args, **kwargs) +class BPCEPage(LoggedPage, MyHTMLPage): + pass + + class RedirectPage(LoggedPage, MyHTMLPage): ENCODING = None @@ -852,7 +856,7 @@ def iter_accounts(self, next_pages, accounts_parsed=None, next_with_params=None) account.balance = Decimal(balance or '0.0') account.currency = currency or Account.get_currency(balance_text) - if account.type == account.TYPE_LOAN: + if account.type in (Account.TYPE_LOAN, Account.TYPE_REVOLVING_CREDIT): account.balance = - abs(account.balance) account._prev_debit = None diff --git a/modules/bforbank/pages.py b/modules/bforbank/pages.py index b6964192dfcb5503508d2caadd3ffbc2e87a3579..f7d20ba2d95e8295c442af7eab92ad03d89f35f9 100644 --- a/modules/bforbank/pages.py +++ b/modules/bforbank/pages.py @@ -166,7 +166,6 @@ class item(ItemElement): ) obj_number = obj_id obj_label = CleanText('./td//div[contains(@class, "-synthese-title")]') - obj_balance = MyDecimal('./td//div[contains(@class, "-synthese-num")]', replace_dots=True) obj_currency = FrenchTransaction.Currency('./td//div[contains(@class, "-synthese-num")]') obj_type = Map(Regexp(Field('label'), r'^([^ ]*)'), TYPE, default=Account.TYPE_UNKNOWN) @@ -175,6 +174,13 @@ def obj_url(self): obj__card_balance = CleanDecimal('./td//div[@class="synthese-encours"][last()]/div[2]', default=None) + def obj_balance(self): + if Field('type')(self) == Account.TYPE_LOAN: + sign = '-' + else: + sign = None + return MyDecimal('./td//div[contains(@class, "-synthese-num")]', replace_dots=True, sign=sign)(self) + def condition(self): return not len(self.el.xpath('./td[@class="chart"]')) diff --git a/modules/bnporc/pp/browser.py b/modules/bnporc/pp/browser.py index 9b539f38dc9c903dc240a450f99e1ee9b47b5019..2bef7b52e900d4a794db56b1d61705a3ebf91887 100644 --- a/modules/bnporc/pp/browser.py +++ b/modules/bnporc/pp/browser.py @@ -38,14 +38,15 @@ from woob.tools.decorators import retry from woob.tools.capabilities.bank.bank_transfer import sorted_transfers from woob.tools.capabilities.bank.transactions import sorted_transactions -from woob.browser.exceptions import ServerError +from woob.browser.exceptions import ServerError, ClientError from woob.browser.elements import DataError from woob.exceptions import ( BrowserIncorrectPassword, BrowserUnavailable, AppValidation, - AppValidationExpired, ActionNeeded, + AppValidationExpired, ActionNeeded, BrowserUserBanned, BrowserPasswordExpired, ) from woob.tools.value import Value from woob.tools.capabilities.bank.investments import create_french_liquidity +from woob.browser.filters.standard import QueryValue from .pages import ( LoginPage, AccountsPage, AccountsIBANPage, HistoryPage, TransferInitPage, @@ -55,7 +56,7 @@ RecipientsPage, ValidateTransferPage, RegisterTransferPage, AdvisorPage, AddRecipPage, ActivateRecipPage, ProfilePage, ListDetailCardPage, ListErrorPage, UselessPage, TransferAssertionError, LoanDetailsPage, TransfersPage, OTPPage, - UnavailablePage, + UnavailablePage, InitLoginPage, FinalizeLoginPage, ) from .document_pages import DocumentsPage, TitulairePage, RIBPage @@ -65,31 +66,42 @@ class BNPParibasBrowser(LoginBrowser, StatesMixin): TIMEOUT = 30.0 + init_login = URL( + r'https://connexion-mabanque.bnpparibas/oidc/authorize', + InitLoginPage + ) login = URL( - r'identification-wspl-pres/identification\?acceptRedirection=true×tamp=(?P\d+)', - r'SEEA-pa01/devServer/seeaserver', - r'https://mabanqueprivee.bnpparibas.net/fr/espace-prive/comptes-et-contrats\?u=%2FSEEA-pa01%2FdevServer%2Fseeaserver', + r'https://connexion-mabanque.bnpparibas/login', LoginPage ) + finalize_login = URL( + r'SEEA-pa01/devServer/seeaserver', + FinalizeLoginPage + ) + + errors_list = URL( + r'/rsc/contrib/identification/src/zonespubliables/mabanque-part/fr/identification-fr-part-CAS.json' + ) + list_error_page = URL( r'https://mabanque.bnpparibas/rsc/contrib/document/properties/identification-fr-part-V1.json', ListErrorPage ) useless_page = URL(r'/fr/connexion/comptes-et-contrats', UselessPage) - otp = URL(r'/fr/espace-prive/authentification-forte-anr', OTPPage) + otp = URL( + r'/fr/espace-prive/authentification-forte-anr', + r'https://.*/fr/secure/authentification-forte', # We can be redirected on other baseurl + OTPPage + ) con_threshold = URL( - r'/fr/connexion/100-connexions', + r'https://.*/100-connexion', r'/fr/connexion/mot-de-passe-expire', - r'/fr/espace-prive/100-connexions.*', - r'/fr/espace-pro/100-connexions-pro.*', r'/fr/espace-pro/changer-son-mot-de-passe', - r'/fr/espace-client/100-connexions', r'/fr/espace-prive/mot-de-passe-expire', r'/fr/client/mdp-expire', - r'/fr/client/100-connexion', ConnectionThresholdPage ) unavailable_page = URL( @@ -138,6 +150,8 @@ class BNPParibasBrowser(LoginBrowser, StatesMixin): profile = URL(r'/kyc-wspl/rest/informationsClient', ProfilePage) list_detail_card = URL(r'/udcarte-wspl/rest/listeDetailCartes', ListDetailCardPage) + DIST_ID = None + STATE_DURATION = 10 __states__ = ('rcpt_transfer_id',) @@ -158,19 +172,66 @@ def do_login(self): if not (self.username.isdigit() and self.password.isdigit()): raise BrowserIncorrectPassword() - timestamp = int(time.time() * 1e3) - # If a previous login session is still valid, we will be redirected with a - # 302 http status code. Otherwise, the page content will be returned directly. - # We have to avoid following redirects as there is a bug with bnpparibas - # website that could enter in a redirect loop if we try to go to the page - # more than once with an active session. + try: + self.init_login.go( + params={ + 'client_id': '0e0fe16f-4e44-4138-9c46-fdf077d56087', + 'scope': 'openid bnpp_mabanque ikpi', + 'response_type': 'code', + 'redirect_uri': 'https://mabanque.bnpparibas/fr/connexion', + 'ui': 'classic part', + 'ui_locales': 'fr', + 'wcm_referer': 'mabanque.bnpparibas/', + } + ) + self.page.login(self.username, self.password) + except ClientError as e: + # We have to call the page manually with the response + # in order to get the error message + message = LoginPage(self, e.response).get_error() + + # Get dynamically error messages + rep = self.errors_list.open() + + error_message = rep.json().get(message).replace('
', ' ') + + if message in ('authenticationFailure.ClientNotFoundException201', 'authenticationFailure.SecretErrorException201'): + raise BrowserIncorrectPassword(error_message) + if message in ('authenticationFailure.CurrentS1DelayException3', 'authenticationFailure.CurrentS2DelayException4'): + raise BrowserUserBanned(error_message) + raise AssertionError('Unhandled error at login: %s: %s' % (message, error_message)) + + code = QueryValue(None, 'code').filter(self.url) + + auth = ( + '%sBNPPOIDC_CAS' + + '%s0e0fe16f-4e44-4138-9c46-fdf077d56087' + + 'https://mabanque.bnpparibas/fr/connexion' + ) + self.location( - self.login.build(timestamp=timestamp), + self.BASEURL + 'SEEA-pa01/devServer/seeaserver', + data={ + 'AUTH': auth % (self.DIST_ID, code), + }, allow_redirects=False, ) - if self.login.is_here(): - self.page.login(self.username, self.password) + # We must check each request one by one to check if an otp will be sent after the redirections + for _ in range(6): + next_location = self.response.headers.get('location') + if not next_location: + break + # This is temporary while we handle the new change pass + if self.con_threshold.is_here(): + raise BrowserPasswordExpired('Vous avez atteint le seuil de 100 connexions avec le même code secret.') + self.location(next_location, allow_redirects=False) + if self.otp.is_here(): + raise ActionNeeded( + "Veuillez réaliser l'authentification forte depuis votre navigateur." + ) + else: + raise AssertionError('Multiple redirects, check if we are not in an infinite loop') def load_state(self, state): # reload state only for new recipient feature @@ -685,6 +746,9 @@ def iter_transfers(self, account): class BNPPartPro(BNPParibasBrowser): BASEURL_TEMPLATE = r'https://%s.bnpparibas/' BASEURL = BASEURL_TEMPLATE % 'mabanque' + # BNPNetEntrepros is supposed to be for pro accounts, but it seems that BNPNetParticulier + # works for pros as well, on the other side BNPNetEntrepros doesn't work for part + DIST_ID = 'BNPNetParticulier' def __init__(self, config=None, *args, **kwargs): self.config = config @@ -748,3 +812,4 @@ def iter_documents(self, subscription): class HelloBank(BNPParibasBrowser): BASEURL = 'https://www.hellobank.fr/' + DIST_ID = 'HelloBank' diff --git a/modules/bnporc/pp/pages.py b/modules/bnporc/pp/pages.py index 15b9fdcd42d335c9988b58e860b546a899d08d5f..3f069a0082948af1f9228f40a490ce92174b2ec8 100644 --- a/modules/bnporc/pp/pages.py +++ b/modules/bnporc/pp/pages.py @@ -21,16 +21,11 @@ from __future__ import unicode_literals -from collections import Counter import re from io import BytesIO -from random import randint from decimal import Decimal from datetime import datetime, timedelta -import lxml.html as html -from requests.exceptions import ConnectionError - from woob.browser.elements import DictElement, ListElement, TableElement, ItemElement, method from woob.browser.filters.json import Dict from woob.browser.filters.standard import ( @@ -38,7 +33,7 @@ Field, Coalesce, Map, MapIn, Env, Currency, FromTimestamp, ) from woob.browser.filters.html import TableCell -from woob.browser.pages import JsonPage, LoggedPage, HTMLPage, PartialHTMLPage +from woob.browser.pages import JsonPage, LoggedPage, HTMLPage, PartialHTMLPage, RawPage from woob.capabilities import NotAvailable from woob.capabilities.bank import ( Account, Recipient, Transfer, TransferBankError, @@ -53,9 +48,7 @@ from woob.capabilities.contact import Advisor from woob.capabilities.profile import Person, ProfileMissing from woob.exceptions import ( - BrowserIncorrectPassword, BrowserUnavailable, - BrowserPasswordExpired, ActionNeeded, - AppValidationCancelled, AppValidationExpired, + BrowserUnavailable, AppValidationCancelled, AppValidationExpired, ) from woob.tools.capabilities.bank.iban import rib2iban, rebuild_rib, is_iban_valid from woob.tools.capabilities.bank.transactions import FrenchTransaction, parse_with_patterns @@ -75,93 +68,8 @@ class TransferAssertionError(Exception): class ConnectionThresholdPage(HTMLPage): - NOT_REUSABLE_PASSWORDS_COUNT = 3 - """BNP disallows to reuse one of the three last used passwords.""" - def make_date(self, yy, m, d): - current = datetime.now().year - if yy > current - 2000: - yyyy = 1900 + yy - else: - yyyy = 2000 + yy - return datetime(yyyy, m, d) - - def looks_legit(self, password): - # the site says: - # have at least 3 different digits - if len(Counter(password)) < 3: - return False - - # not the birthdate (but we don't know it) - first, mm, end = map(int, (password[0:2], password[2:4], password[4:6])) - now = datetime.now() - try: - delta = now - self.make_date(first, mm, end) - except ValueError: - pass - else: - if 10 < delta.days / 365 < 70: - return False - - try: - delta = now - self.make_date(end, mm, first) - except ValueError: - pass - else: - if 10 < delta.days / 365 < 70: - return False - - # no sequence (more than 4 digits?) - password = list(map(int, password)) - up = 0 - down = 0 - for a, b in zip(password[:-1], password[1:]): - up += int(a + 1 == b) - down += int(a - 1 == b) - if up >= 4 or down >= 4: - return False - - return True - - def on_load(self): - msg = ( - CleanText('//div[@class="confirmation"]//span[span]')(self.doc) - or CleanText('//p[contains(text(), "Vous avez atteint la date de fin de vie de votre code secret")]')( - self.doc - ) - ) - - self.logger.warning('Password expired.') - if not self.browser.rotating_password: - raise BrowserPasswordExpired(msg) - - if not self.looks_legit(self.browser.password): - # we may not be able to restore the password, so reject it - self.logger.warning('Unable to restore it, it is not legit.') - raise BrowserPasswordExpired(msg) - - new_passwords = [] - for i in range(self.NOT_REUSABLE_PASSWORDS_COUNT): - new_pass = ''.join(str((int(char) + i + 1) % 10) for char in self.browser.password) - if not self.looks_legit(new_pass): - self.logger.warning('One of rotating password is not legit') - raise BrowserPasswordExpired(msg) - new_passwords.append(new_pass) - - current_password = self.browser.password - for new_pass in new_passwords: - self.logger.warning('Renewing with temp password') - if not self.browser.change_pass(current_password, new_pass): - self.logger.warning('New temp password is rejected, giving up') - raise BrowserPasswordExpired(msg) - current_password = new_pass - - if not self.browser.change_pass(current_password, self.browser.password): - self.logger.error('Could not restore old password!') - - self.logger.warning('Old password restored.') - - # we don't want to try to rotate password two times in a row - self.browser.rotating_password = 0 + # WIP: Temporarily remove change pass feature + pass def cast(x, typ, default=None): @@ -203,96 +111,31 @@ def get_error_message(self, error): return None -class LoginPage(JsonPage): - def is_here(self): - # If we are already logged in and we go to the page without following redirections, - # we will be redirected instead of being presented with a page content when - # everything is good and we don't have to login anymore - return self.response.status_code != 302 +class InitLoginPage(RawPage): + pass - @staticmethod - def render_template(tmpl, **values): - for k, v in values.items(): - tmpl = tmpl.replace('{{ ' + k + ' }}', v) - return tmpl - @staticmethod - def generate_token(length=11): - chars = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXTZabcdefghiklmnopqrstuvwxyz' - return ''.join((chars[randint(0, len(chars) - 1)] for _ in range(length))) +class LoginPage(HTMLPage): + def login(self, username, password): + url = Regexp(CleanText('//style[contains(text(), "grid")]'), r"url\(\"([^\"]+)\"")(self.doc) + keyboard = self.browser.open(url) + vk = BNPKeyboard(self.browser, keyboard) - def build_doc(self, text): - try: - return super(LoginPage, self).build_doc(text) - except ValueError: - # XXX When login is successful, server sends HTML instead of JSON, - # we can ignore it. - return {} + form = self.get_form(id="logincanalnet") + form['userGridPasswordCredential.username'] = username + form['userGridPasswordCredential.gridPosition'] = vk.get_string_code(password) - def on_load(self): - if self.url.startswith('https://mabanqueprivee.'): - self.browser.switch('mabanqueprivee') - - # Some kind of internal server error instead of normal wrongpass errorCode. - if self.get('errorCode') == 'INTO_FACADE ERROR: JDF_GENERIC_EXCEPTION': - raise BrowserIncorrectPassword() - - error = cast(self.get('errorCode', self.get('codeRetour')), int, 0) - # you can find api documentation on errors here : https://mabanque.bnpparibas/rsc/contrib/document/properties/identification-fr-part-V1.json - if error: - try: - # this page can be unreachable - error_page = self.browser.list_error_page.open() - msg = error_page.get_error_message(error) or self.get('message') - except ConnectionError: - msg = self.get('message') - - wrongpass_codes = [201, 21510, 203, 202, 7] - actionNeeded_codes = [21501, 3, 4, 50] - # 'codeRetour' list - # -1 : Erreur technique lors de l'accès à l'application - # -99 : Service actuellement indisponible - websiteUnavailable_codes = [207, 1000, 1001, -99, -1] - if error in wrongpass_codes: - raise BrowserIncorrectPassword(msg) - elif error == 21: # "Ce service est momentanément indisponible. Veuillez renouveler votre demande ultérieurement." -> In reality, account is blocked because of too much wrongpass - raise ActionNeeded(u"Compte bloqué") - elif error in actionNeeded_codes: - raise ActionNeeded(msg) - elif error in websiteUnavailable_codes: - raise BrowserUnavailable(msg) - else: - raise AssertionError('Unexpected error at login: "%s" (code=%s)' % (msg, error)) + form.submit() - parser = html.HTMLParser() - doc = html.parse(BytesIO(self.content), parser) - error = CleanText('//div[h1[contains(text(), "Incident en cours")]]/p')(doc) - if error: - raise BrowserUnavailable(error) + def get_error(self): + return Regexp( + CleanText('//form[@id="logincanalnet"]//script//text()'), + r"errorMessage = \[\"([^\"]+)\"\]" + )(self.doc) - def login(self, username, password): - url = '/identification-wspl-pres/grille/%s' % self.get('data.grille.idGrille') - keyboard = self.browser.open(url) - vk = BNPKeyboard(self.browser, keyboard) - target = self.browser.BASEURL + 'SEEA-pa01/devServer/seeaserver' - user_agent = self.browser.session.headers.get('User-Agent') or '' - auth = self.render_template( - self.get('data.authTemplate'), - idTelematique=username, - password=vk.get_string_code(password), - clientele=user_agent - ) - # XXX useless? - csrf = self.generate_token() - response = self.browser.location(target, data={'AUTH': auth, 'CSRF': csrf}) - - if 'authentification-forte' in response.url: - raise ActionNeeded("Veuillez réaliser l'authentification forte depuis votre navigateur.") - if response.url.startswith('https://pro.mabanque.bnpparibas'): - self.browser.switch('pro.mabanque') - if response.url.startswith('https://banqueprivee.mabanque.bnpparibas'): - self.browser.switch('banqueprivee.mabanque') +class FinalizeLoginPage(RawPage): + pass class OTPPage(HTMLPage): diff --git a/modules/bp/pages/accounthistory.py b/modules/bp/pages/accounthistory.py index eac8be31b0802302d452144362c0224d50906df7..36ae808ba9e55ca53163a97654b515f64a4dd536 100644 --- a/modules/bp/pages/accounthistory.py +++ b/modules/bp/pages/accounthistory.py @@ -451,10 +451,14 @@ def condition(self): obj_type = Account.TYPE_CARD obj_number = CleanText(Dict('numeroPanTronque'), replace=[(' ', '')]) - obj_coming = CleanDecimal.US(Dict('encoursCarte/listeOuverte/0/montantEncours')) - - def obj_currency(self): - return Currency(Dict('encoursCarte/listeOuverte/0/deviseEncours'))(self) + obj_coming = CleanDecimal.US( + Dict('encoursCarte/listeOuverte/0/montantEncours', default=None), + default=NotAvailable + ) + obj_currency = Currency( + Dict('encoursCarte/listeOuverte/0/deviseEncours', default=''), + default=NotAvailable + ) def obj_id(self): return '%s.%s' % (Env('parent_id')(self), self.obj.number) @@ -466,6 +470,8 @@ def generate_summary(self, card): for item in self.doc['cartouchesCarte']: if CleanText(Dict('numeroPanTronque'), replace=[(' ', '')])(item) != card.number: continue + if empty(card.coming): + continue tr = Transaction() # card account: positive summary amount tr.amount = abs(card.coming) diff --git a/modules/bred/bred/browser.py b/modules/bred/bred/browser.py index ce26e53489c2dcfd2f006d83a4fcd088a88db5e1..dd71ba8081a09f44ee1fcda85c3516a6f805b14b 100644 --- a/modules/bred/bred/browser.py +++ b/modules/bred/bred/browser.py @@ -218,13 +218,17 @@ def trigger_connection_twofa(self): ) def get_connection_twofa_method(self): + # The order and tests are taken from the bred website code. + # Keywords in scripts.js: showSMS showEasyOTP showOTP + methods = self.context['liste'] + if methods.get('sms'): + return 'sms' + elif methods.get('notification') and methods.get('otp'): + return 'notification' + elif methods.get('otp'): + return 'otp' + message = self.context['message'] - # Order is important: there can be 'notification' and 'otp' true at the same time, - # but it will prioritized by BRED as an AppValidation - auth_methods = ('notification', 'sms', 'otp') - for auth_method in auth_methods: - if self.context['liste'].get(auth_method): - return auth_method raise AuthMethodNotImplemented('Unhandled strong authentification method: %s' % message) def update_headers(self): diff --git a/modules/bred/module.py b/modules/bred/module.py index 8b0eeda9147204d09c5f4a941d61ece9fdac4176..eb6254c973b70ded21c12db7f59b27fb7f20de25 100644 --- a/modules/bred/module.py +++ b/modules/bred/module.py @@ -46,7 +46,7 @@ class BredModule(Module, CapBankWealth, CapProfile, CapBankTransferAddRecipient) DESCRIPTION = u'Bred' LICENSE = 'LGPLv3+' CONFIG = BackendConfig( - ValueBackendPassword('login', label='Identifiant', masked=False, regexp=r'.{,32}'), + ValueBackendPassword('login', label='Identifiant', masked=False, regexp=r'.{1,32}'), ValueBackendPassword('password', label='Mot de passe'), Value('website', label="Site d'accès", default='bred', choices={'bred': 'BRED', 'dispobank': 'DispoBank'}), diff --git a/modules/caissedepargne/browser.py b/modules/caissedepargne/browser.py index c84658c6ce474119fb9f6a7c15c5a768adb24109..0c33728df087633b08939b22e6151212abaaebfc 100644 --- a/modules/caissedepargne/browser.py +++ b/modules/caissedepargne/browser.py @@ -31,6 +31,7 @@ import sys from dateutil import parser +from requests.cookies import remove_cookie_by_name from woob.browser import LoginBrowser, need_login, StatesMixin from woob.browser.switch import SiteSwitch @@ -270,6 +271,11 @@ def do_api_pre_login(self): self.cdetab = self.page.get_cdetab() self.connection_type = self.page.get_connection_type() + def get_cdetab(self): + if not self.cdetab: + self.do_api_pre_login() # this sets cdetab + return self.cdetab + def get_connection_data(self): """ Attempt to log in. @@ -1408,9 +1414,11 @@ def _get_history_invests(self, account): # Some life insurances are not on the accounts summary self.home_tache.go(tache='EPASYNT0') self.page.go_life_insurance(account) - if 'JSESSIONID' in self.session.cookies: - # To access the life insurance space, we need to delete the JSESSIONID cookie to avoid an expired session - del self.session.cookies['JSESSIONID'] + # To access the life insurance space, we need to delete the JSESSIONID cookie + # to avoid an expired session + # There might be duplicated JSESSIONID cookies (eg with different paths), + # that's why we need to use remove_cookie_by_name() + remove_cookie_by_name(self.session.cookies, 'JSESSIONID') if self.home.is_here(): # no detail available for this account diff --git a/modules/caissedepargne/pages.py b/modules/caissedepargne/pages.py index f39c76585a6bf7ceb600f522740d11f3a959fffa..515722e210a754c49f72fa23157d678467ab4a3c 100644 --- a/modules/caissedepargne/pages.py +++ b/modules/caissedepargne/pages.py @@ -31,6 +31,7 @@ from lxml import html from PIL import Image, ImageFilter +from requests.cookies import remove_cookie_by_name from woob.browser.pages import ( LoggedPage, HTMLPage, JsonPage, pagination, @@ -902,9 +903,11 @@ def get_loan_list(self): account.ownership = AccountOwnership.OWNER if "renouvelables" in CleanText('.')(title): - if 'JSESSIONID' in self.browser.session.cookies: - # Need to delete this to access the consumer loans space (a new one will be created) - del self.browser.session.cookies['JSESSIONID'] + # To access the life insurance space, we need to delete the JSESSIONID cookie + # to avoid an expired session + # There might be duplicated JSESSIONID cookies (eg with different paths), + # that's why we use remove_cookie_by_name() + remove_cookie_by_name(self.browser.session.cookies, 'JSESSIONID') try: self.go_loans_conso(tr) except ClientError as e: @@ -1874,6 +1877,18 @@ def obj_vdate(self): obj_quantity = Eval(float_to_decimal, Dict('nombreUnitesCompte')) obj_unitvalue = Eval(float_to_decimal, Dict('valeurUniteCompte')) + def obj_diff(self): + diff = Dict('capitalGainView/pvLatentInEuroOrigin', default=None)(self) + if diff is not None: + return float_to_decimal(diff) + return NotAvailable + + def obj_diff_ratio(self): + diff_ratio_percent = Dict('capitalGainView/pvLatentInPorcentOrigin', default=None)(self) + if diff_ratio_percent is not None: + return float_to_decimal(diff_ratio_percent) / 100 + return NotAvailable + def obj_portfolio_share(self): repartition = Dict('repartition', default=None)(self) if repartition: diff --git a/modules/carrefourbanque/browser.py b/modules/carrefourbanque/browser.py index 4f1f95a30f33cafc4f5f2a2f0150a98b0313f17e..b6e1bb48d7ec0a740c4924fc23c457e5a1774b6c 100644 --- a/modules/carrefourbanque/browser.py +++ b/modules/carrefourbanque/browser.py @@ -17,6 +17,8 @@ # You should have received a copy of the GNU Lesser General Public License # along with this woob module. If not, see . +# flake8: compatible + from __future__ import absolute_import, unicode_literals import re @@ -31,8 +33,8 @@ from woob.tools.compat import basestring from .pages import ( - LoginPage, MaintenancePage, HomePage, IncapsulaResourcePage, LoanHistoryPage, CardHistoryPage, SavingHistoryPage, - LifeInvestmentsPage, LifeHistoryPage, CardHistoryJsonPage, + LoginPage, MaintenancePage, HomePage, IncapsulaResourcePage, LoanHistoryPage, CardHistoryPage, + SavingHistoryPage, LifeInvestmentsPage, LifeHistoryPage, CardHistoryJsonPage, ) @@ -66,9 +68,6 @@ def __init__(self, config, *args, **kwargs): kwargs['password'] = self.config['password'].get() super(CarrefourBanqueBrowser, self).__init__(*args, **kwargs) - def locate_browser(self, state): - pass - def do_login(self): """ Attempt to log in. @@ -109,7 +108,7 @@ def do_login(self): raise RecaptchaV2Question(website_key=website_key, website_url=website_url) else: # we got javascript page again, this shouldn't happen - assert False, "obfuscated javascript not managed" + raise AssertionError("obfuscated javascript not managed") if self.maintenance.is_here(): raise BrowserUnavailable(self.page.get_message()) @@ -133,8 +132,8 @@ def do_login(self): raise BrowserUnavailable(error) elif 'saisies ne correspondent pas à l\'identifiant' in error: raise BrowserIncorrectPassword(error) - assert False, 'Unexpected error at login: "%s"' % error - assert False, 'Unexpected error at login' + raise AssertionError('Unexpected error at login: "%s"' % error) + raise AssertionError('Unexpected error at login') if self.login.is_here(): # Check if the website asks for strong authentication with OTP @@ -191,7 +190,7 @@ def iter_history(self, account): tr = None total = 0 loop_limit = 500 - for page in range(loop_limit): + for _ in range(loop_limit): self.card_history_json.go(data={'dateRecup': previous_date, 'index': card_index}) previous_date = self.page.get_previous_date() diff --git a/modules/carrefourbanque/module.py b/modules/carrefourbanque/module.py index 3da5952ca0191877cfcf5208beb6f75a46ec329b..fc052c8a97188e376039b88d8ff2b5bbae819efc 100644 --- a/modules/carrefourbanque/module.py +++ b/modules/carrefourbanque/module.py @@ -17,6 +17,9 @@ # You should have received a copy of the GNU Lesser General Public License # along with this woob module. If not, see . +# flake8: compatible + +from __future__ import unicode_literals from woob.capabilities.base import find_object from woob.capabilities.bank import AccountNotFound @@ -32,14 +35,16 @@ class CarrefourBanqueModule(Module, CapBankWealth): NAME = 'carrefourbanque' - MAINTAINER = u'Romain Bignon' + MAINTAINER = 'Romain Bignon' EMAIL = 'romain@weboob.org' VERSION = '3.0' - DESCRIPTION = u'Carrefour Banque' + DESCRIPTION = 'Carrefour Banque' LICENSE = 'LGPLv3+' - CONFIG = BackendConfig(ValueBackendPassword('login', label=u'Votre Identifiant Internet', masked=False), - ValueBackendPassword('password', label=u"Code d'accès", regexp=u'\d+'), - Value('captcha_response', label='Captcha Response', default='', required=False)) + CONFIG = BackendConfig( + ValueBackendPassword('login', label='Votre Identifiant Internet', masked=False), + ValueBackendPassword('password', label="Code d'accès", regexp=r'\d+'), + Value('captcha_response', label='Captcha Response', default='', required=False) + ) BROWSER = CarrefourBanqueBrowser def create_default_browser(self): diff --git a/modules/carrefourbanque/pages.py b/modules/carrefourbanque/pages.py index aad20e4407ffc4622cf6fece85560d380545e4bb..c8594c1f5be1b50aa9ac522a63f77ef596836ec0 100644 --- a/modules/carrefourbanque/pages.py +++ b/modules/carrefourbanque/pages.py @@ -17,12 +17,15 @@ # You should have received a copy of the GNU Lesser General Public License # along with this woob module. If not, see . +# flake8: compatible + from __future__ import unicode_literals import re import base64 import datetime from io import BytesIO + from PIL import Image from woob.tools.json import json @@ -51,7 +54,7 @@ class CarrefourBanqueKeyboard(object): '6': '00011100111110111000011000001111110111111111001111100011110001111111110111110', '7': '11111111111111000011100001100000110000110000011000011100001100001110000110000', '8': '00111001111110110011111001111111110011110011111101100111110001111111110111110', - '9': '00110001111110110011111000111100011111111111111110000011000011011111101111100' + '9': '00110001111110110011111000111100011111111111111110000011000011011111101111100', } def __init__(self, data_code): @@ -160,9 +163,9 @@ def check_action_needed(self): # The real message contains the user's phone number, so we send a generic message. raise ActionNeeded( "Veuillez vous connecter sur le site de Carrefour Banque pour " - "recevoir un code par SMS afin d'accéder à votre Espace Client." + + "recevoir un code par SMS afin d'accéder à votre Espace Client." ) - assert False, 'Unhandled error: password submission failed and we are still on Login Page.' + raise AssertionError('Unhandled error: password submission failed and we are still on Login Page.') def get_error_message(self): return CleanText('//div[@class="messages error"]', default=None)(self.doc) @@ -198,14 +201,17 @@ class item_account_generic(ItemElement): klass = Account def obj_balance(self): - balance = CleanDecimal('.//div[contains(@class, "right_col")]//h2[1]', replace_dots=True)(self) + balance = CleanDecimal.French('.//div[contains(@class, "right_col")]//h2[1]')(self) if Field('type')(self) in (Account.TYPE_LOAN, ): return -balance return balance obj_currency = Currency('.//div[contains(@class, "right_col")]//h2[1]') obj_label = CleanText('.//div[contains(@class, "leftcol")]//h2[1]') - obj_id = Regexp(CleanText('.//div[contains(@class, "leftcol")]//p'), ":\s+([\d]+)") + obj_id = Regexp( + CleanText('.//div[contains(@class, "leftcol")]//p'), + r':\s+([\d]+)' + ) obj_number = Field('id') def obj_url(self): @@ -215,13 +221,13 @@ def obj_url(self): class iter_history_generic(Transaction.TransactionsElement): - head_xpath = u'//div[*[contains(text(), "opérations")]]/table//thead/tr/th' - item_xpath = u'//div[*[contains(text(), "opérations")]]/table/tbody/tr[td]' + head_xpath = '//div[*[contains(text(), "opérations")]]/table//thead/tr/th' + item_xpath = '//div[*[contains(text(), "opérations")]]/table/tbody/tr[td]' col_debittype = 'Mode' def next_page(self): - next_page = Link(u'//a[contains(text(), "précédentes")]', default=None)(self) + next_page = Link('//a[contains(text(), "précédentes")]', default=None)(self) if next_page: return "/%s" % next_page @@ -256,12 +262,11 @@ class item(item_account_generic): obj_type = Account.TYPE_CARD def obj_balance(self): - available = CleanDecimal( + available = CleanDecimal.French( './/p[contains(., "encours depuis le")]//preceding-sibling::h2', - default=None, - replace_dots=True + default=NotAvailable, )(self) - if available is not None: + if available: return -available # No "en cours" available: return - (total_amount - available_amount) @@ -294,10 +299,18 @@ class item(item_account_generic): obj_url = Link('.//a[contains(., "Historique des opérations")]') def obj_balance(self): - val = CleanDecimal('.//a[contains(text(), "versement")]//preceding-sibling::h2', replace_dots=True, default=NotAvailable)(self) + val = CleanDecimal.French( + './/a[contains(text(), "versement")]//preceding-sibling::h2', + default=NotAvailable + )(self) if val is not NotAvailable: return val - val = CleanDecimal(Regexp(CleanText('./div[@class="right_col_wrapper"]//h2'), r'([\d ,]+€)'), replace_dots=True)(self) + val = CleanDecimal.French( + Regexp( + CleanText('./div[@class="right_col_wrapper"]//h2'), + r'([\d ,]+€)' + ), + )(self) return val @method @@ -309,11 +322,15 @@ class item(item_account_generic): def obj_url(self): acc_number = Field('id')(self) - xpath_link = '//li[contains(., "{acc_number}")]/ul/li/a[contains(text(), "Dernieres opérations")]'.format(acc_number=acc_number) + xpath_link = ( + '//li[contains(., "{acc_number}")]/ul/li/a[contains(text(), "Dernieres opérations")]' + ).format(acc_number=acc_number) return Link(xpath_link)(self) def obj__life_investments(self): - xpath_link = '//li[contains(., "{acc_number}")]/ul/li/a[contains(text(), "Solde")]'.format(acc_number=Field('id')(self)) + xpath_link = '//li[contains(., "{acc_number}")]/ul/li/a[contains(text(), "Solde")]'.format( + acc_number=Field('id')(self) + ) return Link(xpath_link)(self) @@ -339,11 +356,11 @@ class get_investment(TableElement): item_xpath = '//table[@id="assets"]/tbody/tr[position() > 1]' head_xpath = '//table[@id="assets"]/tbody/tr[1]/td' - col_label = u'Fonds' - col_quantity = u'Nombre de parts' - col_unitvalue = u'Valeur part' - col_valuation = u'Total' - col_portfolio_share = u'Répartition' + col_label = 'Fonds' + col_quantity = 'Nombre de parts' + col_unitvalue = 'Valeur part' + col_valuation = 'Total' + col_portfolio_share = 'Répartition' class item(ItemElement): klass = Investment @@ -395,7 +412,11 @@ def on_load(self): # # this function converts the response to the good format if needed if isinstance(self.doc['tab_historique'], dict): - self.doc['tab_historique'] = sorted(self.doc['tab_historique'].values(), key=lambda x: x['timestampOperation'], reverse=True) + self.doc['tab_historique'] = sorted( + self.doc['tab_historique'].values(), + key=lambda x: x['timestampOperation'], + reverse=True + ) elif self.doc['tab_historique'] is None: # No transaction available, set value to empty dict @@ -410,7 +431,10 @@ class item(ItemElement): klass = Transaction def obj_date(self): - return datetime.datetime.strptime(CleanText(Dict('timestampOperation'))(self), "%Y-%m-%d-%H.%M.%S.%f").date() + return datetime.datetime.strptime( + CleanText(Dict('timestampOperation'))(self), + "%Y-%m-%d-%H.%M.%S.%f" + ).date() obj_rdate = Date(CleanText(Dict('date')), dayfirst=True) obj_raw = CleanText(Dict('label')) diff --git a/modules/cesu/module.py b/modules/cesu/module.py index 376716140821cf29995ad1078a7c6e0708dacd6c..f51419505d79a03c47a9eed72016e38c5e022048 100644 --- a/modules/cesu/module.py +++ b/modules/cesu/module.py @@ -44,7 +44,7 @@ class CesuModule(Module, CapDocument): MAINTAINER = "Ludovic LANGE" EMAIL = "llange@users.noreply.github.com" LICENSE = "LGPLv3+" - VERSION = "3.0" + VERSION = '3.0' CONFIG = BackendConfig( Value("username", label="User ID"), diff --git a/modules/cic/browser.py b/modules/cic/browser.py index 734cc641e6191679a96796d1c1d8538908bedebf..f4f9202aa329420d47cf554f12303591526a78a1 100644 --- a/modules/cic/browser.py +++ b/modules/cic/browser.py @@ -17,11 +17,15 @@ # You should have received a copy of the GNU Lesser General Public License # along with this woob module. If not, see . -from .pages import LoginPage, DecoupledStatePage, CancelDecoupled +# flake8: compatible + from woob.browser.browsers import AbstractBrowser from woob.browser.profiles import Wget from woob.browser.url import URL +from .pages import LoginPage, DecoupledStatePage, CancelDecoupled + + __all__ = ['CICBrowser'] diff --git a/modules/cic/module.py b/modules/cic/module.py index 89fc28d57cb2584f63ef312972342a2660bf2783..76d9803c59b55292611795b287ba8aa4fb6cbd5f 100644 --- a/modules/cic/module.py +++ b/modules/cic/module.py @@ -18,6 +18,8 @@ # You should have received a copy of the GNU Lesser General Public License # along with this woob module. If not, see . +# flake8: compatible + from woob.capabilities.bank import CapBankTransferAddRecipient from woob.capabilities.bill import CapDocument from woob.capabilities.contact import CapContact diff --git a/modules/cic/pages.py b/modules/cic/pages.py index 0a0d49685b7024ece18173daafc2f15fa060664f..e0d8aa8feca0a734a3a6c17864a50938293775d7 100644 --- a/modules/cic/pages.py +++ b/modules/cic/pages.py @@ -17,6 +17,8 @@ # You should have received a copy of the GNU Lesser General Public License # along with this woob module. If not, see . +# flake8: compatible + from __future__ import unicode_literals from woob.browser.pages import AbstractPage diff --git a/modules/cmso/par/browser.py b/modules/cmso/par/browser.py index 81ffacb1cc82d129c2305c47f7132e31a2d1806b..59327e52a8490d7885090dd0af76ec1e6c39e8f5 100644 --- a/modules/cmso/par/browser.py +++ b/modules/cmso/par/browser.py @@ -41,7 +41,7 @@ from .pages import ( LogoutPage, AccountsPage, HistoryPage, LifeinsurancePage, MarketPage, AdvisorPage, LoginPage, ProfilePage, RedirectInsurancePage, SpacesPage, - ChangeSpacePage, ConsentPage, + ChangeSpacePage, ConsentPage, AccessTokenPage, ) from .transfer_pages import TransferInfoPage, RecipientsListPage, TransferPage, AllowedRecipientsPage @@ -105,6 +105,7 @@ class CmsoParBrowser(TwoFactorBrowser): spaces = URL(r'/domiapi/oauth/json/accesAbonnement', SpacesPage) change_space = URL(r'/securityapi/changeSpace', ChangeSpacePage) consent = URL(r'/consentapi/tpp/consents', ConsentPage) + access_token = URL(r'/oauth/token', AccessTokenPage) accounts = URL(r'/domiapi/oauth/json/accounts/synthese(?P.*)', AccountsPage) history = URL(r'/domiapi/oauth/json/accounts/(?P.*)', HistoryPage) @@ -142,7 +143,6 @@ class CmsoParBrowser(TwoFactorBrowser): json_headers = {'Content-Type': 'application/json'} authorization_uri = URL(r'/oauth/authorize') - access_token_uri = URL(r'/oauth/token') authorization_codegen_uri = URL(r'/oauth/authorization-code') redirect_uri = 'https://mon.cmso.com/auth/checkuser' error_uri = 'https://mon.cmso.com/auth/errorauthn' @@ -243,10 +243,11 @@ def init_login(self): # authentication token generation data = self.get_tokengen_data(location_params['code']) - response = self.access_token_uri.go(json=data) - self.update_authentication_headers(response.json()) + self.access_token.go(json=data) + self.update_authentication_headers() self.login.go(json={'espaceApplication': 'PART'}) + self.setup_space_after_login() def handle_sms(self): data = { @@ -277,9 +278,24 @@ def handle_sms(self): 'code_verifier': self.login_verifier, } - access_token = self.access_token_uri.go(json=data).json() - self.update_authentication_headers(access_token) + self.access_token.go(json=data) + self.update_authentication_headers() self.login.go(json={'espaceApplication': 'PART'}) + self.setup_space_after_login() + + def setup_space_after_login(self): + self.spaces.go(json={'includePart': True}) + part_space = self.page.get_part_space() + if part_space is None: + # If there is no PAR space, then the PAR browser returns no account. + self.accounts_list = None + self.change_space.go(json={ + 'clientIdSource': self.arkea_client_id, + 'espaceDestination': 'PART', + 'fromMobile': False, + 'numContractDestination': part_space, + }) + self.update_authentication_headers() def get_authcode_data(self): return { @@ -302,10 +318,11 @@ def get_tpp_headers(self, data=''): # to add specific headers and be recognize by the bank return {} - def update_authentication_headers(self, params): - self.session.headers['Authorization'] = "Bearer %s" % params['access_token'] + def update_authentication_headers(self): + token = self.page.get_access_token() + self.session.headers['Authorization'] = "Bearer %s" % token self.session.headers['X-ARKEA-EFS'] = self.arkea - self.session.headers['X-Csrf-Token'] = params['access_token'] + self.session.headers['X-Csrf-Token'] = token self.session.headers['X-REFERER-TOKEN'] = 'RWDPART' def get_account(self, _id): @@ -314,7 +331,15 @@ def get_account(self, _id): @retry((ClientError, ServerError)) @need_login def iter_accounts(self): - if self.accounts_list: + if self.accounts_list is None: + # No PAR space available + return [] + elif self.accounts_list: + # In case of a retry, we need to leave the browser with a good URL and page. + # This page has been chosen because almost all other pages are either + # 1. A POST page: issue if we don't have the posting data + # 2. Broken on the PSD2 version of the website (at the time of writing) + self.redirect_insurance.go() return self.accounts_list seen = {} @@ -327,19 +352,6 @@ def iter_accounts(self): # to know if account can do transfer accounts_eligibilite_debit = self.page.get_eligibilite_debit() - self.spaces.go(json={'includePart': True}) - part_space = self.page.get_part_space() - if part_space is None: - # no par account for this connection - return [] - self.change_space.go(json={ - 'clientIdSource': self.arkea_client_id, - 'espaceDestination': 'PART', - 'fromMobile': False, - 'numContractDestination': part_space, - }) - self.session.headers['Authorization'] = 'Bearer %s' % self.page.get_access_token() - # First get all checking accounts... # We might have some savings accounts here for special cases such as mandated accounts # (e.g children's accounts) @@ -466,6 +478,19 @@ def iter_history(self, account): self._return_from_market() self.history.go(json={"index": account._index}, page="pendingListOperations") + exception_code = self.page.get_exception_code() + + if exception_code == 300: + # When this request returns an exception code, the request to get + # the details will return a ServerError(500) with message "account ID not found" + # Try a workaround of loading the account list page. + # It seems to help the server "find" the account. + self.accounts.go(json={'typeListeCompte': 'COMPTE_SOLDE_COMPTES_CHEQUES'}, type='comptes') + + self.history.go(json={"index": account._index}, page="pendingListOperations") + elif exception_code is not None: + raise AssertionError("Unknown exception_code: %s" % exception_code) + has_deferred_cards = self.page.has_deferred_cards() # 1.fetch the last 6 weeks transactions but keep only the current month ones diff --git a/modules/cmso/par/compat/__init__.py b/modules/cmso/par/compat/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/modules/cmso/par/pages.py b/modules/cmso/par/pages.py index 7045bc3bf1ac504f2bfa4df330521d18e71698da..a9319eadd21e9948dd45ed5bae45d014825a1d34 100644 --- a/modules/cmso/par/pages.py +++ b/modules/cmso/par/pages.py @@ -80,6 +80,11 @@ def get_access_token(self): return Dict('accessToken')(self.doc) +class AccessTokenPage(JsonPage): + def get_access_token(self): + return Dict('access_token')(self.doc) + + class AccountsPage(LoggedPage, JsonPage): TYPES = OrderedDict([ ('courant', Account.TYPE_CHECKING), @@ -424,6 +429,9 @@ class HistoryPage(LoggedPage, JsonPage): def has_deferred_cards(self): return Dict('pendingDeferredDebitCardList/currentMonthCardList', default=None)(self.doc) + def get_exception_code(self): + return Dict('exception/code', default=None)(self.doc) + def get_keys(self): if 'exception' in self.doc: return [] diff --git a/modules/creditmutuel/browser.py b/modules/creditmutuel/browser.py index a284cba47f8e942422d195c1cb9256b097797c49..7f583a3a3856c735882933b8455a4db1c75bb3f1 100644 --- a/modules/creditmutuel/browser.py +++ b/modules/creditmutuel/browser.py @@ -165,6 +165,7 @@ class CreditMutuelBrowser(TwoFactorBrowser): r'/(?P.*)fr/assurances/WI_ASS', r'/(?P.*)fr/assurances/SYNASSINT.aspx.*', r'/(?P.*)fr/assurances/SYNASSVIE.aspx.*', + r'/(?P.*)fr/assurances/SYNASSINTNEXT.aspx.*', '/fr/assurances/', LIAccountsPage) li_history = URL( r'/(?P.*)fr/assurances/SYNASSVIE.aspx\?_tabi=C&_pid=ValueStep&_fid=GoOnglets&Id=3', @@ -290,7 +291,13 @@ def finalize_twofa(self, twofa_data): # not present if 2FA is triggered systematically self.twofa_auth_state['value'] = cookie.value # this is a token self.twofa_auth_state['expires'] = cookie.expires # this is a timestamp - self.location(self.response.headers['Location']) + break + else: + self.logger.info("User probably has his account setup with a systematic sca") + + redirect_uri = self.response.headers.get('Location') + if redirect_uri: + self.location(redirect_uri) def poll_decoupled(self, transactionId): """ diff --git a/modules/creditmutuel/pages.py b/modules/creditmutuel/pages.py index 5da74276cf1a0e805544dacbf6a9c7060e6d0570..b1a08e9002d4d7eca5453f1514259ce49e0bc368 100644 --- a/modules/creditmutuel/pages.py +++ b/modules/creditmutuel/pages.py @@ -1420,7 +1420,10 @@ def has_accounts(self): return self.doc.xpath('//input[@name="_FID_GoBusinessSpaceLife"]') def go_accounts_list(self): - form = self.get_form(id='C:P14:F', submit='//input[@name="_FID_GoBusinessSpaceLife"]') + if 'SYNASSINTNEXT.aspx' in self.url: + form = self.get_form(id='C:P4:F', submit='//input[@name="_FID_GoBusinessSpaceLife"]') + else: + form = self.get_form(id='C:P14:F', submit='//input[@name="_FID_GoBusinessSpaceLife"]') form.submit() def has_details(self, account): @@ -2650,8 +2653,10 @@ def condition(self): # Some documents may have the same date, name and label; only parts of the PDF href may change, # so we must pick a unique ID including the href to avoid document duplicates: - obj_id = Format('%s_%s_%s', Env('sub_id'), CleanText(TableCell('date'), replace=[('/', '')]), - Regexp(Field('url'), r'guid=(.*)&sit=')) + obj_id = Format( + '%s_%s_%s', Env('sub_id'), CleanText(TableCell('date'), replace=[('/', '')]), + Regexp(Field('url'), r'_fid=.+cle=(\d+)') + ) obj_label = Format('%s %s', CleanText(TableCell('url')), CleanText(TableCell('date'))) obj_date = Date(CleanText(TableCell('date')), dayfirst=True) obj_format = 'pdf' diff --git a/modules/fortuneo/pages/accounts_list.py b/modules/fortuneo/pages/accounts_list.py index 7738f4ee697a7e38cde123cb7dfc205fad8747be..19c4cecd3e04776730c5ec0501534b0e82744c5e 100644 --- a/modules/fortuneo/pages/accounts_list.py +++ b/modules/fortuneo/pages/accounts_list.py @@ -170,11 +170,42 @@ def condition(self): obj_code = IsinCode(Regexp(Field('id'), r'^[A-Z]+[0-9]+(.*)$'), default=NotAvailable) obj_code_type = IsinType(Regexp(Field('id'), r'^[A-Z]+[0-9]+(.*)$'), default=NotAvailable) obj_quantity = CleanDecimal.French(TableCell('quantity'), default=NotAvailable) - obj_unitprice = CleanDecimal.French(TableCell('unitprice'), default=NotAvailable) - obj_unitvalue = CleanDecimal.SI(TableCell('unitvalue'), default=NotAvailable) obj_valuation = CleanDecimal.French(TableCell('valuation'), default=NotAvailable) - obj_diff = Base(TableCell('diff'), CleanDecimal.French('./text()', default=NotAvailable)) + # We check if there is a currency in the unitvalue TableCell. + # If there is, it means the unitvalue & unitprice are displayed in the original currency of the asset. + # If not, it means the unitvalue & unitprice are displayed in the account currency. + obj_original_currency = Currency(TableCell('unitvalue')) + + def obj_unitvalue(self): + if not Field('original_currency')(self): + return CleanDecimal.US(TableCell('unitvalue'), default=NotAvailable)(self) + return NotAvailable + + def obj_original_unitvalue(self): + if Field('original_currency')(self): + return CleanDecimal.US(TableCell('unitvalue'), default=NotAvailable)(self) + return NotAvailable + + def obj_unitprice(self): + if not Field('original_currency')(self): + return CleanDecimal.French(TableCell('unitprice'), default=NotAvailable)(self) + return NotAvailable + + def obj_original_unitprice(self): + if Field('original_currency')(self): + return CleanDecimal.French(TableCell('unitprice'), default=NotAvailable)(self) + return NotAvailable + + def obj_diff(self): + if not Field('original_currency')(self): + return Base(TableCell('diff'), CleanDecimal.French('./text()', default=NotAvailable))(self) + return NotAvailable + + def obj_original_diff(self): + if Field('original_currency')(self): + return Base(TableCell('diff'), CleanDecimal.French('./text()', default=NotAvailable))(self) + return NotAvailable def obj_diff_ratio(self): diff_ratio_percent = Base(TableCell('diff'), CleanDecimal.French('./span', default=None))(self) @@ -182,6 +213,12 @@ def obj_diff_ratio(self): return diff_ratio_percent / 100 return NotAvailable + def obj_portfolio_share(self): + portfolio_share = CleanDecimal.French(TableCell('portfolio_share'), default=None)(self) + if portfolio_share: + return portfolio_share / 100 + return NotAvailable + def get_liquidity(self): return CleanDecimal.French('//*[@id="valorisation_compte"]/table/tr[3]/td[2]', default=0)(self.doc) @@ -710,7 +747,7 @@ class FalseActionPage(ActionNeededPage): class LoanPage(ActionNeededPage): @method class fill_account(ItemElement): - obj_balance = CleanDecimal.French('//p[@id="c_montantRestant"]//strong') + obj_balance = CleanDecimal.French('//p[@id="c_montantRestant"]//strong', sign='-') obj_total_amount = CleanDecimal.French('(//p[@id="c_montantEmprunte"]//strong)[2]') obj_next_payment_amount = CleanDecimal.French(Regexp(CleanText('//p[@id="c_prochaineEcheance"]//strong'), r'(.*) le')) obj_next_payment_date = Date(CleanText('//p[@id="c_prochaineEcheance"]//strong/strong'), dayfirst=True) diff --git a/modules/genericnewspaper/compat/__init__.py b/modules/genericnewspaper/compat/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/modules/hsbc/compat/__init__.py b/modules/hsbc/compat/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/modules/lcl/pages.py b/modules/lcl/pages.py index 44504502ea27b55e6f34d2527d31f30afc7a6b4c..768c50bd35f9fdb9817084eda9e89bae97686f90 100644 --- a/modules/lcl/pages.py +++ b/modules/lcl/pages.py @@ -58,11 +58,6 @@ from woob.tools.capabilities.bank.investments import is_isin_valid, IsinCode -def MyDecimal(*args, **kwargs): - kwargs.update(replace_dots=True, default=Decimal(0)) - return CleanDecimal(*args, **kwargs) - - def myXOR(value, seed): s = '' for i in range(len(value)): @@ -381,8 +376,8 @@ class LoansTableElement(TableElement): flush_at_end = True col_id = re.compile('Emprunteur') - col_balance = ['Capital restant dû', re.compile('Sommes totales restant dues'), re.compile('Montant disponible')] - col_amount = 'Montant du prêt' + col_balance = ['Capital restant dû', re.compile('Sommes totales restant dues'), re.compile('Montant utilisé')] + col_amount = ['Montant du prêt', 'Montant maximum autorisé'] col_maturity = ['Montant et date de la dernière échéance prélevée', 'Date de fin de prêt'] col_next_payment = 'Montant et date de la prochaine échéance' @@ -401,8 +396,10 @@ class account(ItemElement): obj_rate = CleanDecimal.French( Regexp( CleanText('.//div[@class="tooltipContent tooltipLeft testClic"]//ul/li[2]/node()[not(self::strong)]'), - r'(.+)%' - ) + r'(.+)%', + default=NotAvailable + ), + default=NotAvailable ) obj_maturity_date = Date( Regexp( @@ -986,14 +983,17 @@ class iter_investment(ListElement): class item(ItemElement): klass = Investment + def validate(self, obj): + return not empty(obj.valuation) + obj_label = CleanText('.//td[2]/div/a') obj_code = ( CleanText('.//td[2]/div/br/following-sibling::text()') & Regexp(pattern='^([^ ]+).*', default=NotAvailable) ) - obj_quantity = MyDecimal('.//td[3]/span') - obj_diff = MyDecimal('.//td[7]/span') - obj_valuation = MyDecimal('.//td[5]') + obj_quantity = CleanDecimal.French('.//td[3]/span', default=NotAvailable) + obj_diff = CleanDecimal.French('.//td[7]/span', default=NotAvailable) + obj_valuation = CleanDecimal.French('.//td[5]', default=NotAvailable) def obj_code_type(self): code = Field('code')(self) @@ -1001,15 +1001,29 @@ def obj_code_type(self): return Investment.CODE_TYPE_ISIN return NotAvailable + def obj_original_currency(self): + # Only the unitvalue is in its original currency + currency = Currency('.//td[4]')(self) + if currency != Currency('.//td[5]')(self): + return currency + return NotAvailable + def obj_unitvalue(self): if "%" in CleanText('.//td[4]')(self) and "%" in CleanText('.//td[6]')(self): return NotAvailable - return MyDecimal('.//td[4]/text()')(self) + if Field('original_currency')(self): + return NotAvailable + return CleanDecimal.French('.//td[4]/text()', default=NotAvailable)(self) + + def obj_original_unitvalue(self): + if Field('original_currency')(self): + return CleanDecimal.French('.//td[4]/text()', default=NotAvailable)(self) + return NotAvailable def obj_unitprice(self): if "%" in CleanText('.//td[4]')(self) and "%" in CleanText('.//td[6]')(self): return NotAvailable - return MyDecimal('.//td[6]')(self) + return CleanDecimal.French('.//td[6]', default=NotAvailable)(self) @pagination @method @@ -1048,7 +1062,7 @@ def parse(self, el): i = Investment() i.label = Field('label')(self) i.code = unicode(TableCell('code')(self)[0].xpath('./text()[last()]')[0]).strip() - i.quantity = MyDecimal(TableCell('quantity'))(self) + i.quantity = CleanDecimal.French(TableCell('quantity'), default=NotAvailable)(self) i.valuation = Field('amount')(self) i.vdate = Field('date')(self) @@ -1956,9 +1970,12 @@ class get_list(TableElement): class item(OwnedItemElement): klass = Account + def validate(self, obj): + return not empty(obj.balance) + obj_type = Account.TYPE_DEPOSIT obj_label = Format('%s %s', CleanText(TableCell('name')), CleanText(TableCell('owner'))) - obj_balance = MyDecimal(TableCell('balance')) + obj_balance = CleanDecimal.French(TableCell('balance'), default=NotAvailable) obj_currency = 'EUR' obj__contract = CleanText(TableCell('name')) obj__link_index = Regexp(CleanText('.//a/@id'), r'(\d+)') diff --git a/modules/myedenred/compat/__init__.py b/modules/myedenred/compat/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/modules/oney/browser.py b/modules/oney/browser.py index 82391f261362c381dccd49cfe11fa624f2c9e2e0..e176e0b8b2e4928c28c70b7ae042c06ef16c3d64 100644 --- a/modules/oney/browser.py +++ b/modules/oney/browser.py @@ -19,19 +19,16 @@ from __future__ import unicode_literals -from datetime import date, timedelta -from dateutil.relativedelta import relativedelta -from itertools import chain +from datetime import datetime from woob.capabilities.bank import Account from woob.exceptions import BrowserIncorrectPassword, BrowserPasswordExpired from woob.browser import LoginBrowser, URL, need_login -from woob.tools.date import new_date from .pages import ( LoginPage, ClientPage, OperationsPage, ChoicePage, - CreditHome, CreditAccountPage, CreditHistory, LastHistoryPage, - ContextInitPage, SendUsernamePage, SendPasswordPage, CheckTokenPage, ClientSpacePage + ContextInitPage, SendUsernamePage, SendPasswordPage, CheckTokenPage, ClientSpacePage, + OtherDashboardPage, OAuthPage, AccountsPage, JWTTokenPage, OtherOperationsPage, ) __all__ = ['OneyBrowser'] @@ -39,6 +36,8 @@ class OneyBrowser(LoginBrowser): BASEURL = 'https://www.oney.fr' + LOGINURL = 'https://login.oney.fr' + OTHERURL = 'https://middle.mobile.oney.io' home_login = URL( r'/site/s/login/login.html', @@ -50,32 +49,42 @@ class OneyBrowser(LoginBrowser): LoginPage ) - send_username = URL(r'https://login.oney.fr/middle/authenticationflowinit', SendUsernamePage) - send_password = URL(r'https://login.oney.fr/middle/completeauthflowstep', SendPasswordPage) - context_init = URL(r'https://login.oney.fr/middle/context', ContextInitPage) + send_username = URL(LOGINURL + r'/middle/authenticationflowinit', SendUsernamePage) + send_password = URL(LOGINURL + r'/middle/completeauthflowstep', SendPasswordPage) + context_init = URL(LOGINURL + r'/middle/context', ContextInitPage) - check_token = URL(r'https://login.oney.fr/middle/check_token', CheckTokenPage) + check_token = URL(LOGINURL + r'/middle/check_token', CheckTokenPage) + # Space selection choice = URL(r'/site/s/multimarque/choixsite.html', ChoicePage) choice_portal = URL(r'/site/s/login/loginidentifiant.html') + # Oney space client = URL(r'/oney/client', ClientPage) client_space = URL(r'https://www.compte.oney.fr/espace-client/historique-facilypay', ClientSpacePage) operations = URL(r'/oney/client', OperationsPage) card_page = URL(r'/oney/client\?task=Synthese&process=SyntheseMultiCompte&indexSelectionne=(?P\d+)') - credit_home = URL(r'/site/s/detailcompte/detailcompte.html', CreditHome) - credit_info = URL(r'/site/s/detailcompte/ongletdetailcompte.html', CreditAccountPage) - credit_hist = URL(r'/site/s/detailcompte/exportoperations.html', CreditHistory) - last_hist = URL(r'/site/s/detailcompte/ongletdernieresoperations.html', LastHistoryPage) + # Other space + dashboard = URL(r'https://espaceclient.oney.fr/dashboard', OtherDashboardPage) + jwt_token = URL(OTHERURL + r'/JWTToken', JWTTokenPage) + oauth = URL(OTHERURL + r'/web/login/oauth', OAuthPage) + other_accounts = URL(OTHERURL + r'/web/dashboard', AccountsPage) + other_operations = URL(OTHERURL + r'/web/operation/operations', OtherOperationsPage) has_oney = False has_other = False card_name = None is_mail = False + pristine_params_headers = { + 'Environment': "PRD", + 'Origin': "Web", + 'IsLoggedIn': False, + } + params_headers = pristine_params_headers.copy() def do_login(self): - self.session.cookies.clear() + self.reset_session_for_new_auth() self.home_login.go(method="POST") context_token = self.page.get_context_token() @@ -130,59 +139,111 @@ def do_login(self): if self.choice.is_here(): self.other_space_url = self.page.get_redirect_other_space() self.has_other = self.has_oney = True - elif self.credit_home.is_here(): + elif self.dashboard.is_here(): self.has_other = True + self.setup_headers_other_space() elif self.client.is_here(): self.has_oney = True else: raise BrowserIncorrectPassword() - @need_login - def go_site(self, site): - if site == 'oney': - if ( - self.credit_home.is_here() or self.credit_info.is_here() - or self.credit_hist.is_here() - or self.last_hist.is_here() - ): - - self.choice.go() - assert self.choice.is_here() - if self.choice.is_here(): - # if no redirect was found in the choice_page we try the previous method. Due to a lack of example - # it might be deprecated - if self.other_space_url: - self.location(self.other_space_url) - self.client_space.go() - else: - self.choice_portal.go(data={'selectedSite': 'ONEY_HISTO'}) - - elif site == 'other': - if self.client.is_here() or self.operations.is_here(): + def setup_headers_other_space(self): + assert self.dashboard.is_here() + isaac_token = self.page.get_token() + + self.session.headers.update({ + 'Origin': "https://espaceclient.oney.fr", + }) + self.jwt_token.go(params={ + 'localTime': datetime.now().isoformat()[:-3]+ 'Z' + }) + self.update_authorization(self.page.get_token()) + + self.oauth.go(json={ + 'header': self.params_headers, + 'isaacToken': isaac_token, + }) + + self.params_headers.update(self.page.get_headers_from_json()) + + def update_authorization(self, token): + self.session.headers.update({ + 'Authorization': 'Bearer %s' % token + }) + + def reset_session_for_new_auth(self): + self.session.cookies.clear() + self.session.headers.pop('Authorization', None) + self.session.headers.pop('Origin', None) + self.params_headers = self.pristine_params_headers.copy() + + def other_space_params_headers(self): + return { + 'Headers.%s' % key: value + for key, value in self.params_headers.items() + } + + def get_referrer(self, oldurl, newurl): + if newurl.startswith(self.OTHERURL): + return "https://espaceclient.oney.fr/" + else: + return super(OneyBrowser, self).get_referrer(oldurl, newurl) + + def get_site(self): + try: + return self.page.get_site() + except AttributeError: + # That error mean that we are on an unknown page or a login page. + # These case are then handled by try_go_site + return None + + def try_go_site(self, target_site): + current_site = self.get_site() + if current_site == target_site: + return True + + if target_site == 'oney': + if not self.has_oney: + return False + + if not self.choice.is_here(): + self.do_login() + assert self.choice.is_here() + + # if no redirect was found in the choice_page we try the previous method. Due to a lack of example + # it might be deprecated + if self.other_space_url: + self.location(self.other_space_url) + self.client_space.go() + else: + self.choice_portal.go(data={'selectedSite': 'ONEY_HISTO'}) + + elif target_site == 'other': + if not self.has_other: + return False + + if not self.choice.is_here(): self.do_login() - assert self.choice.is_here() - if self.choice.is_here(): - self.choice_portal.go(data={'selectedSite': 'ONEY'}) + assert self.choice.is_here() + + self.choice_portal.go(data={'selectedSite': 'ONEY'}) + self.setup_headers_other_space() + else: + raise AssertionError('Unkown target_site: %s' % target_site) + + current_site = self.get_site() + assert current_site == target_site, 'Should be on site %s, landed on %s site instead' % (target_site, current_site) + return True @need_login - def get_accounts_list(self): + def iter_accounts(self): accounts = [] - if self.has_other: - self.go_site('other') - for acc_id in self.page.get_accounts_ids(): - self.credit_home.go(data={'numeroCompte': acc_id}) - label = self.page.get_label() - if 'prêt' in label.lower(): - acc = self.page.get_loan() - else: - self.credit_info.go() - acc = self.page.get_account() - acc.label = label - accounts.append(acc) - - if self.has_oney: - self.go_site('oney') + if self.try_go_site('other'): + self.other_accounts.go(params=self.other_space_params_headers()) + accounts.extend(self.page.iter_accounts()) + + if self.try_go_site('oney'): if self.client_space.is_here(): return accounts self.client.stay_or_go() @@ -190,81 +251,36 @@ def get_accounts_list(self): return accounts - def _build_hist_form(self, last_months=False): - form = {} - d = date.today() - - if not last_months: - # before the last two months - end = d.replace(day=1) + relativedelta(months=-1, days=-1) - form['jourDebut'] = '1' - form['moisDebut'] = '1' - form['anneeDebut'] = '2016' - form['jourFin'] = str(end.day) - form['moisFin'] = str(end.month) - form['anneeFin'] = str(end.year) - else: - # the last two months - start = d.replace(day=1) - timedelta(days=1) - form['jourDebut'] = '1' - form['moisDebut'] = str(start.month) - form['anneeDebut'] = str(start.year) - form['jourFin'] = str(d.day) - form['moisFin'] = str(d.month) - form['anneeFin'] = str(d.year) - - form['typeOpe'] = 'deux' - form['formatFichier'] = 'xls' # or pdf... great choice - return form - @need_login def iter_history(self, account): - self.go_site(account._site) + self.try_go_site(account._site) if account._site == 'oney': if account._num: self.card_page.go(acc_num=account._num) post = {'task': 'Synthese', 'process': 'SyntheseCompte', 'taskid': 'Releve'} self.operations.go(data=post) - for tr in self.page.iter_transactions(seen=set()): - yield tr - - elif account._site == 'other' and account.type != Account.TYPE_LOAN: - self.credit_home.go(data={'numeroCompte': account.id}) - self.last_hist.go() - if self.page.has_transactions(): - # transactions are missing from the xls from 2016 to today - # so two requests are needed - d = date.today() - page_before = self.credit_hist.open( - params=self._build_hist_form(last_months=True) - ) - page_today = self.credit_hist.go( - params=self._build_hist_form() - ) - - for tr in chain(page_before.iter_history(), page_today.iter_history()): - if new_date(tr.date) < d: - yield tr + return self.page.iter_transactions(seen=set()) + + elif account._site == 'other' and account.type == Account.TYPE_CHECKING: + self.other_operations.go(params=self.other_space_params_headers()) + return self.page.iter_history(guid=account._guid, is_coming=False) + else: + return [] @need_login def iter_coming(self, account): - self.go_site(account._site) + self.try_go_site(account._site) if account._site == 'oney': if account._num: self.card_page.go(acc_num=account._num) post = {'task': 'OperationRecente', 'process': 'OperationRecente', 'taskid': 'OperationRecente'} self.operations.go(data=post) - for tr in self.page.iter_transactions(seen=set()): - yield tr - - elif account._site == 'other' and account.type != Account.TYPE_LOAN: - self.credit_home.go(data={'numeroCompte': account.id}) - self.last_hist.go() - if self.page.has_transactions(): - self.credit_hist.go(params=self._build_hist_form()) - d = date.today().replace(day=1) # TODO is it the right date? - for tr in self.page.iter_history(): - if new_date(tr.date) >= d: - yield tr + return self.page.iter_transactions(seen=set()) + + elif account._site == 'other' and account.type == Account.TYPE_CHECKING: + self.other_operations.go(params=self.other_space_params_headers()) + return self.page.iter_history(guid=account._guid, is_coming=True) + else: + return [] diff --git a/modules/oney/module.py b/modules/oney/module.py index 103917ce2273fa68ce0e382205725489a6ae0605..aa8624db22e1005fd1dd095f3db290c9894870df 100644 --- a/modules/oney/module.py +++ b/modules/oney/module.py @@ -20,8 +20,7 @@ from __future__ import unicode_literals -from woob.capabilities.bank import CapBank, AccountNotFound -from woob.capabilities.base import find_object +from woob.capabilities.bank import CapBank from woob.tools.backend import Module, BackendConfig from woob.tools.value import ValueBackendPassword @@ -51,11 +50,7 @@ def create_default_browser(self): ) def iter_accounts(self): - for account in self.browser.get_accounts_list(): - yield account - - def get_account(self, _id): - return find_object(self.browser.get_accounts_list(), id=_id, error=AccountNotFound) + return self.browser.iter_accounts() def iter_history(self, account): # To prevent issues in calcul of actual balance and coming one, all diff --git a/modules/oney/pages.py b/modules/oney/pages.py index 19afb02980c656b0e6430817afc2c78dd7674b26..8956559ffbc61450ce047e5c3968cf3d3f222013 100644 --- a/modules/oney/pages.py +++ b/modules/oney/pages.py @@ -22,11 +22,19 @@ import re import requests +from datetime import date + +from decimal import Decimal + +from woob.capabilities.base import NotAvailable from woob.capabilities.bank import Account from woob.tools.capabilities.bank.transactions import FrenchTransaction, sorted_transactions -from woob.browser.pages import HTMLPage, LoggedPage, pagination, XLSPage, PartialHTMLPage, JsonPage +from woob.browser.pages import HTMLPage, LoggedPage, pagination, JsonPage from woob.browser.elements import ListElement, ItemElement, method, DictElement -from woob.browser.filters.standard import Env, CleanDecimal, CleanText, Field, Format, Currency, Date +from woob.browser.filters.standard import ( + Env, CleanDecimal, CleanText, Field, Format, + Currency, Date, QueryValue, Map, Coalesce, +) from woob.browser.filters.html import Attr from woob.browser.filters.json import Dict from woob.tools.compat import urlparse, parse_qsl @@ -90,20 +98,12 @@ def get_pages(self): yield self.browser.open('/site/s/login/loginidentifiant.html', data={'selectedSite': page_attrib}).page - -class ClientSpacePage(LoggedPage, HTMLPage): - # skip consumer credit, there is not enough information. - # If an other type of page appear handle it here - pass - - -class DetailPage(LoggedPage, HTMLPage): - - def iter_accounts(self): - return [] +class OneySpacePage(LoggedPage): + def get_site(self): + return "oney" -class ClientPage(LoggedPage, HTMLPage): +class ClientPage(OneySpacePage, HTMLPage): is_here = "//div[@id='situation']" @method @@ -133,7 +133,7 @@ def parse(self, el): self.env['balance'] = - amount_due -class OperationsPage(LoggedPage, HTMLPage): +class OperationsPage(OneySpacePage, HTMLPage): is_here = "//div[@id='releve-reserve-credit'] | //div[@id='operations-recentes'] | //select[@id='periode']" @pagination @@ -190,74 +190,120 @@ def next_page(self): return requests.Request("POST", self.page.url, data=data) -class CreditHome(LoggedPage, HTMLPage): - def get_accounts_ids(self): - ids = [] - for elem in self.doc.xpath('//li[@id="menu-n2-mesproduits"]//a/@onclick'): - regex_result = re.search(r"afficherDetailCompte\('(\d+)'\)", elem) - if not regex_result: - continue - acc_id = regex_result.group(1) - if acc_id not in ids: - ids.append(acc_id) - return ids +class ClientSpacePage(OneySpacePage, HTMLPage): + # skip consumer credit, there is not enough information. + # If an other type of page appear handle it here + pass - def get_label(self): - # 'Ma carte Alinea', 'Mon Prêt Oney', ... - return CleanText('//div[@class="conteneur"]/h1')(self.doc) - @method - class get_loan(ItemElement): - klass = Account +class OtherSpacePage(LoggedPage): + def get_site(self): + return "other" + - obj_type = Account.TYPE_LOAN - obj__site = 'other' - obj_balance = 0 - obj_label = CleanText('//div[@class="conteneur"]/h1') - obj_number = obj_id = CleanText('//td[contains(text(), "Mon numéro de compte")]/following-sibling::td', replace=[(' ', '')]) - obj_coming = CleanDecimal.US('//td[strong[contains(text(), "Montant de la")]]/following-sibling::td/strong') +class OtherSpaceJsonPage(OtherSpacePage, JsonPage): + def on_load(self): + is_success = Dict('header/isSuccess', default=None)(self.doc) + assert is_success, "a page returned that the request was not a success" + new_jwt_token = Dict('header/jwtToken/token', default=None)(self.doc) + if new_jwt_token: + self.browser.update_authorization(new_jwt_token) -class CreditAccountPage(LoggedPage, HTMLPage): + +class OAuthPage(OtherSpaceJsonPage): + def get_headers_from_json(self): + return { + 'UserId': Dict('userId')(self.doc), + 'IdentifierType': Dict('identifierType')(self.doc), + 'IsLoggedIn': Dict('content/header/isLoggedIn')(self.doc), + } + + +class JWTTokenPage(JsonPage): + def get_token(self): + return Dict('token')(self.doc) + + +class OtherDashboardPage(OtherSpacePage, HTMLPage): + def get_token(self): + return QueryValue(None, 'authentication_token').filter(self.url) + + +OtherAccountTypeMap = { + 'RCP': Account.TYPE_CHECKING, + 'PP': Account.TYPE_LOAN, +} + +class AccountsPage(OtherSpaceJsonPage): @method - class get_account(ItemElement): - klass = Account - - obj_type = Account.TYPE_CHECKING - obj__site = 'other' - obj_balance = 0 - obj_number = obj_id = CleanText('//tr[td[text()="Mon numéro de compte"]]/td[@class="droite"]', replace=[(' ', '')]) - obj_coming = CleanDecimal('//div[@id="mod-paiementcomptant"]//tr[td[contains(text(),"débité le")]]/td[@class="droite"]', sign='-', default=0) - obj_currency = Currency('//div[@id="mod-paiementcomptant"]//tr[td[starts-with(normalize-space(text()),"Montant disponible")]]/td[@class="droite"]') - - -class CreditHistory(LoggedPage, XLSPage): - # this history doesn't contain the monthly recharges, so the balance isn't consistent with the transactions? - def build_doc(self, content): - lines = super(CreditHistory, self).build_doc(content) - dict_list = list() - header = [element.strip() for element in lines[0]] - for line in lines[1:][::-1]: - dict_list.append(dict(zip(header, line))) - return dict_list + class iter_accounts(DictElement): + item_xpath = 'content/body/dashboardContracts' + + class item(ItemElement): + klass = Account + obj__site = 'other' + obj_currency = Currency(Dict('contract/currencyCode')) + obj_type = Map(Dict('contract/typeCode'), OtherAccountTypeMap, default=Account.TYPE_UNKNOWN) + obj_label = Dict('contract/displayableShortLabel') + obj_number = Dict('contract/externalReference', default=NotAvailable) + obj_id = Coalesce( + Field('number'), + Field('_guid'), + ) + obj_balance = Decimal(0) + + def obj_coming(self): + cur_type = Field('type')(self) + if cur_type == Account.TYPE_LOAN: + return CleanDecimal.SI( + Dict('depreciableAccount/installments/0/totalAmount', default=0), + sign='-' + )(self) + elif cur_type == Account.TYPE_CHECKING: + # Since it is a credit account, the amount are reversed. + return - CleanDecimal.SI( + Dict('cashAccount/cashPaymentOutstanding/amount'), + )(self) + else: + return NotAvailable + + def obj__guid(self): + for links in Dict('contract/links')(self): + if links['rel'] == "self": + return links["guid"] + return NotAvailable + + +class OtherOperationsPage(OtherSpaceJsonPage): @method class iter_history(DictElement): + item_xpath = 'content/body/transactions' + class item(ItemElement): klass = Transaction - obj_raw = Transaction.Raw(CleanText(Dict("Libellé de l'opération"))) - - def obj_amount(self): - assert not (Dict('Débit')(self) and Dict('Credit')(self)), "cannot have both debit and credit" - - if Dict('Credit')(self): - return CleanDecimal.US(Dict('Credit'))(self) - return -CleanDecimal.US(Dict('Débit'))(self) - - obj_date = Date(Dict('Date'), dayfirst=True) + def condition(self): + # The website always display the transactions for all cards at the same time. + # So we filter by account (guid). + # The date is to separate between coming and transaction. + trans_guid = Dict('contractGuid')(self) + acc_guid = Env('guid')(self) + guid_ok = trans_guid == acc_guid + + today = date.today() + trans_date = Field('date')(self) + if Env('is_coming')(self): + date_ok = trans_date >= today + else: + date_ok = trans_date < today + + return date_ok and guid_ok + obj_type = Transaction.TYPE_CARD + obj_date = Date(Dict('transaction/date')) + obj_raw = Transaction.Raw(Dict('transaction/displayableLabel')) -class LastHistoryPage(LoggedPage, PartialHTMLPage): - def has_transactions(self): - return not CleanText('//h2[contains(text(), "Vous n\'avez pas effectué d\'opération depuis votre dernier relevé de compte.")]')(self.doc) + def obj_amount(self): + return -CleanDecimal.SI(Dict('transaction/amount'))(self) diff --git a/modules/orange/browser.py b/modules/orange/browser.py index f9bd2598b86e3e68f1c16806866fdec2fa2a9c5d..62ec968c25d25aa223de111b15cc564a9ef261fd 100644 --- a/modules/orange/browser.py +++ b/modules/orange/browser.py @@ -25,6 +25,7 @@ from requests.exceptions import ConnectTimeout from woob.browser import LoginBrowser, URL, need_login, StatesMixin +from woob.capabilities import NotAvailable from woob.exceptions import ( BrowserIncorrectPassword, BrowserUnavailable, ActionNeeded, BrowserPasswordExpired, ScrapingBlocked, @@ -298,6 +299,14 @@ def iter_documents(self, subscription): return [] raise + except ClientError as e: + if e.response.status_code == 412: + # if the code is 412 the user is not the owner of the subscription and we can't get the invoices + msg = e.response.json()['error']['customerMessage']['subMessage'] + self.logger.info("no documents because: %s", msg) + return [] + raise + for b in self.page.get_bills(subid=subscription.id): documents.append(b) return iter(documents) @@ -318,8 +327,13 @@ def get_profile(self): def download_document(self, document): # sometimes the site sends us a server error when downloading the document. # it is necessary to try again. - - if document._is_v2: - # get 404 without this header - return self.open(document.url, headers={'x-orange-caller-id': 'ECQ'}).content - return self.open(document.url).content + try: + if document._is_v2: + # get 404 without this header + return self.open(document.url, headers={'x-orange-caller-id': 'ECQ'}).content + return self.open(document.url).content + except ClientError as e: + if e.response.status_code == 422: + # if the code is 422 the download of the document is currently unavailable + return NotAvailable + raise diff --git a/modules/pajemploi/module.py b/modules/pajemploi/module.py index a335049b603b147daa6cd412a3b9729a68bcaf0a..26beaa90896788839fe093dd6e44d8e4699b03f1 100644 --- a/modules/pajemploi/module.py +++ b/modules/pajemploi/module.py @@ -48,7 +48,7 @@ class PajemploiModule(Module, CapDocument): MAINTAINER = "Ludovic LANGE" EMAIL = "llange@users.noreply.github.com" LICENSE = "LGPLv3+" - VERSION = "3.0" + VERSION = '3.0' CONFIG = BackendConfig( Value("username", label="User ID"), diff --git a/modules/societegenerale/browser.py b/modules/societegenerale/browser.py index 9670453e4c494d71f7da7184eb5bdbdf935c75e0..9a061010a7a870e86d1464e53fab2e908a077fda 100644 --- a/modules/societegenerale/browser.py +++ b/modules/societegenerale/browser.py @@ -102,7 +102,7 @@ def check_login_reason(self): 'pas_acces_bad': '''Votre accès est suspendu. Vous n'êtes pas autorisé à accéder à l'application.''', } raise ActionNeeded(action_needed_messages[reason]) - elif reason == 'err_tech': + elif reason in ('err_is', 'err_tech'): # there is message "Service momentanément indisponible. Veuillez réessayer." # in SG website in that case ... raise BrowserUnavailable() @@ -478,7 +478,7 @@ def iter_history(self, account): # get history for account on old website # request to get json is not available yet, old request to get html response if any(( - account.type in (account.TYPE_LIFE_INSURANCE, account.TYPE_CAPITALISATION, account.TYPE_PERP), + account.type in (account.TYPE_LIFE_INSURANCE, account.TYPE_PERP), account.type == account.TYPE_REVOLVING_CREDIT and account._loan_type != 'PR_CONSO', account.type in (account.TYPE_REVOLVING_CREDIT, account.TYPE_SAVINGS) and not account._is_json_histo, )): @@ -534,7 +534,6 @@ def iter_coming(self, account): Account.TYPE_MARKET, Account.TYPE_PEA, Account.TYPE_LIFE_INSURANCE, - Account.TYPE_CAPITALISATION, Account.TYPE_REVOLVING_CREDIT, Account.TYPE_CONSUMER_CREDIT, Account.TYPE_PERP, @@ -581,7 +580,7 @@ def iter_coming(self, account): @need_login def iter_investment(self, account): if account.type not in ( - Account.TYPE_MARKET, Account.TYPE_LIFE_INSURANCE, Account.TYPE_CAPITALISATION, + Account.TYPE_MARKET, Account.TYPE_LIFE_INSURANCE, Account.TYPE_PEA, Account.TYPE_PERP, ): self.logger.debug('This account is not supported') diff --git a/modules/societegenerale/pages/accounts_list.py b/modules/societegenerale/pages/accounts_list.py index 0d5758ba63e17db7046c223bbcce194873a67117..1c09a2285bbb1734c57f7891ef6fb7058a90b64f 100644 --- a/modules/societegenerale/pages/accounts_list.py +++ b/modules/societegenerale/pages/accounts_list.py @@ -193,7 +193,7 @@ def condition(self): 'ASSURANCE_VIE_GENERALE': Account.TYPE_LIFE_INSURANCE, 'ASSURANCE_VIE_SOGECAP_GENERAL': Account.TYPE_LIFE_INSURANCE, 'VIE_AXA': Account.TYPE_LIFE_INSURANCE, - 'CAPI_AGF': Account.TYPE_CAPITALISATION, + 'CAPI_AGF': Account.TYPE_LIFE_INSURANCE, 'RESERVEA': Account.TYPE_REVOLVING_CREDIT, 'COMPTE_ALTERNA': Account.TYPE_REVOLVING_CREDIT, 'CREDIT_CONFIANCE': Account.TYPE_REVOLVING_CREDIT, diff --git a/modules/societegenerale/sgpe/json_pages.py b/modules/societegenerale/sgpe/json_pages.py index bcd648d9beb7d8274632c13dbe53b7f2de338460..17f312b023d47cd90db854139c2683745f6ebae2 100644 --- a/modules/societegenerale/sgpe/json_pages.py +++ b/modules/societegenerale/sgpe/json_pages.py @@ -359,6 +359,7 @@ class get_profile(ItemElement): Dict('donnees/telephoneSecurite', default=NotAvailable), Dict('donnees/telephoneMobile', default=NotAvailable), Dict('donnees/telephoneFixe', default=NotAvailable), + default=NotAvailable, ) obj_email = Coalesce( diff --git a/modules/sogecartenet/compat/__init__.py b/modules/sogecartenet/compat/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/modules/trainline/browser.py b/modules/trainline/browser.py index 476a54d46e8d6f660b18b04d9f51e598540270ae..87f2eaf59b4a4a308345e38497723d09a2155fb5 100644 --- a/modules/trainline/browser.py +++ b/modules/trainline/browser.py @@ -25,6 +25,9 @@ from woob.browser.exceptions import ClientError from .pages import SigninPage, UserPage, DocumentsPage +from .sensor_data import build_sensor_data, get_cf_date + +sensor_data = build_sensor_data() class TrainlineBrowser(LoginBrowser): @@ -39,10 +42,24 @@ def __init__(self, login, password, *args, **kwargs): self.session.headers['X-Requested-With'] = 'XMLHttpRequest' def do_login(self): - # without this additional header we get a timeout while using a proxy - self.session.headers['Proxy-Connection'] = 'keep-alive' # set some cookies self.go_home() + if self.session.cookies.get('_abck'): + # _abck cookie is special, (you can find it to some other website also) + # first we receive it with one value (which contains ~-1~ in the middle) + # and we have to send it to receive it again but with another value + # this new value is mandatory + + # update cookie _abck + self.open('/staticweb/31b06785f73ti1714cafa96c8bd3eba79') + data = { + "sensor_data": sensor_data % ( + self.session.headers['User-Agent'], + get_cf_date(), + self.session.cookies['_abck'], + ), + } + self.open('/staticweb/31b06785f73ti1714cafa96c8bd3eba79', json=data) try: self.signin.go(json={'email': self.username, 'password': self.password}) diff --git a/modules/trainline/pages.py b/modules/trainline/pages.py index 633bf61d2435f85d27cf3a5e8e8706b68375c77a..a3879baf048e1240364f567f5c9d40e10a480eb9 100644 --- a/modules/trainline/pages.py +++ b/modules/trainline/pages.py @@ -41,12 +41,14 @@ class DocumentsPage(LoggedPage, JsonPage): @method class iter_documents(DictElement): item_xpath = 'pastBookings/results' + # when the seller is ouigo we have duplicate data + ignore_duplicate = True class item(ItemElement): klass = Bill def condition(self): - return 'Paid' in Dict('order/payment/paymentState')(self) + return 'COMPLETE' in Dict('order/state')(self) obj_id = Format('%s_%s', Env('subid'), CleanText(Dict('order/id'))) obj_number = CleanText(Dict('order/friendlyOrderId')) diff --git a/modules/trainline/sensor_data.py b/modules/trainline/sensor_data.py new file mode 100644 index 0000000000000000000000000000000000000000..075d7c93b13afafed61670236250a274ef6d7516 --- /dev/null +++ b/modules/trainline/sensor_data.py @@ -0,0 +1,110 @@ + +# -*- coding: utf-8 -*- + +# Copyright(C) 2012-2021 Budget Insight + +# flake8: compatible + +from __future__ import unicode_literals + +import datetime +from math import floor + + +# These following two values are found in '/staticweb/31b06785f73ti1714cafa96c8bd3eba79.js' +# in the bmak dictionary. +PUBLIC_API_KEY = 'afSbep8yjnZUjq3aL010jO15Sawj2VZfdYK8uY90uxq' # bmak['api_public_key'] +CS = '0a46G5m17Vrp4o4c' # bmak['cs'] + +SENSOR_DATA_CORE = ( + '1.68-1,2,-94,-100,' + + '%s' + + ',uaend,11059,20100101,fr,Gecko,0,0,0,0,398206,1476349,1920,1080,1920,1080,523,938,1920,,cpen:0,' + + 'i1:0,dm:0,cwen:0,non:1,opc:0,fc:1,sc:0,wrc:1,isc:142,vib:1,bat:0,x11:0,x12:1,4853,0.70177237935' + + '0,809205738174,0,loc:-1,2,-94,-101,do_en,dm_en,t_dis-1,2,-94,-105,0,0,0,0,1112,1112,0;0,0,0,0,9' + + '03,903,0;0,-1,0,0,3759,3759,0;0,-1,0,0,3630,3630,1;0,-1,0,0,2164,0,1;-1,2,-94,-102,0,0,0,0,1112' + + ',1112,0;0,0,0,0,903,903,0;0,-1,0,0,3759,3759,1;0,-1,0,0,3630,3630,1;0,-1,0,0,2164,0,1;-1,2,-94,' + + '-108,-1,2,-94,-110,-1,2,-94,-117,-1,2,-94,-111,-1,2,-94,-109,-1,2,-94,-114,-1,2,-94,-103,-1,2,-' + + '94,-112,https://www.thetrainline.com/fr-1,2,-94,-115,1,32,32,0,0,0,0,1369,0,' + + '%d' + + ',9,17313,0,0,2885,0,0,1369,0,0,' + + '%s' + + ',37735,515,1058213677,25543097,PiZtE,75358,86-1,2,-94,-106,9,1-1,2,-94,-119,0,0,200,0,0,200,200,' + + '0,200,200,0,0,200,200,-1,2,-94,-122,0,0,0,0,1,0,0-1,2,-94,-123,-1,2,-94,-124,-1,2,-94,-126,-1,2,-' + + '94,-127,11133333331333333333-1,2,-94,-70,-1279939100;-324940575;dis;;true;true;true;-120;true;24;' + + '24;true;false;1-1,2,-94,-80,5377-1,2,-94,-116,22145328-1,2,-94,-118,93225-1,2,-94,-129,9feb9225ad' + + '5184a8162588771f441d89e1b7d80189ac0e715d0fdd97d7b0e0e4,1,0,Intel Open Source Technology Center,Me' + + 'sa DRI Intel(R) HD Graphics (Whiskey Lake 3x8 GT2) ,' + + '63b8f3d843a1aa3f6c70274ea7c720e02910278be9d61db366c6242af84e0072,28-1,2,-94,-121,;57;13;0' +) + + +def build_sensor_data(): + date0_ms = get_cf_date() + sensor_data_header = encode(CS, PUBLIC_API_KEY)[:16] + date1_hours = str(floor(get_cf_date() / 3600000)) + date2_ms = get_cf_date() + + sensor_data = ( + sensor_data_header + encode(date1_hours, sensor_data_header) + + SENSOR_DATA_CORE + ';' + str(get_cf_date() - date0_ms) + ';-1;' + str(get_cf_date() - date2_ms) + ) + + return sensor_data + + +def get_cf_date(): + return floor(datetime.datetime.now().timestamp() * 1000) + + +def encode(base_text, key): + """ + js reference: od: function(t,a) + + Given two strings: base_text and key + If 'key' is not empty, returns 'base_text' where its numeral characters [0, 9] at i-th position are modified using a cyclic shift. + The number of gross hops for the i-th shift is conditioned by the unicode value of 'key' (i % n)-th character + """ + + if not key: + return '' + + res = [] + n = len(key) + for i, char in enumerate(base_text): + # The characters corresponding to these unicode values range + # (47, 57) are ['/', '0', '1', ..., '8', '9'] + # using 'rir' function, char1 is modified only if is a number + # as minimum value is excluded + res.append(chr(hops_in_range(ord(char), 47, 57, gross_hops=ord(key[i % n])))) + + return ''.join(res) + + +def hops_in_range(initial_value, minimum, maximum, gross_hops): + """ + js reference: rir: function(t, a, e, n) + + For an input 'initial_value' satisfying the boundaries defined by minimum and maximum. + The output 'returned_value' is obtained by making a cyclic shift with (gross_hops % delta) hops + and skipping the minimum value + + For an input 'initial_value' not satisfying the boundaries constraint, the output 'returned_value' + is equal to the 'initial_value' (simple return) + + Otherwise it will always return a value in ]min, max] + + simple return min <--- delta ---> max simple return + --------------------|-------------------------------|-------------------- + perform % + """ + returned_value = initial_value + + if minimum < returned_value <= maximum: + delta = maximum - minimum + hops = gross_hops % delta + returned_value += hops + + if returned_value > maximum: + returned_value -= delta + return returned_value