modules/ameli/browser.py

@@ -24,7 +24,9 @@
 from dateutil.relativedelta import relativedelta
 
 from weboob.browser import LoginBrowser, URL, need_login
-from .pages import ErrorPage, LoginPage, SubscriptionPage, DocumentsPage
+from weboob.exceptions import ActionNeeded
+
+from .pages import ErrorPage, LoginPage, RedirectPage, CguPage, SubscriptionPage, DocumentsPage
 
 
 class AmeliBrowser(LoginBrowser):
@@ -32,6 +34,8 @@ class AmeliBrowser(LoginBrowser):
 
     error_page = URL(r'/vu/INDISPO_COMPTE_ASSURES.html', ErrorPage)
     login_page = URL(r'/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&connexioncompte_2actionEvt=afficher.*', LoginPage)
+    redirect_page = URL(r'/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&.*validationconnexioncompte.*', RedirectPage)
+    cgu_page = URL(r'/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&_pageLabel=as_conditions_generales_page.*', CguPage)
     subscription_page = URL(r'/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&_pageLabel=as_info_perso_page.*', SubscriptionPage)
     documents_page = URL(r'/PortailAS/paiements.do', DocumentsPage)
 
@@ -39,6 +43,9 @@ def do_login(self):
         self.login_page.go()
         self.page.login(self.username, self.password)
 
+        if self.cgu_page.is_here():
+            raise ActionNeeded(self.page.get_cgu_message())
+
     @need_login
     def iter_subscription(self):
         self.subscription_page.go()

modules/ameli/pages.py

@@ -38,6 +38,16 @@ def login(self, username, password):
         form.submit()
 
 
+class RedirectPage(LoggedPage, HTMLPage):
+    REFRESH_MAX = 0
+    REFRESH_XPATH = '//meta[@http-equiv="refresh"]'
+
+
+class CguPage(LoggedPage, HTMLPage):
+    def get_cgu_message(self):
+        return CleanText('//div[@class="page_nouvelles_cgus"]/p')(self.doc)
+
+
 class ErrorPage(HTMLPage):
     def on_load(self):
         msg = CleanText('//div[@id="backgroundId"]//p')(self.doc)

modules/americanexpress/browser.py

@@ -20,74 +20,31 @@
 from __future__ import unicode_literals
 
 import datetime
+from uuid import uuid4
 from dateutil.parser import parse as parse_date
+from collections import OrderedDict
 
-from weboob.exceptions import BrowserIncorrectPassword, ActionNeeded
-from weboob.browser.browsers import PagesBrowser, need_login
+from weboob.exceptions import BrowserIncorrectPassword, ActionNeeded, BrowserUnavailable
+from weboob.browser.browsers import LoginBrowser, need_login
 from weboob.browser.exceptions import HTTPNotFound, ServerError
-from weboob.browser.selenium import (
-    SeleniumBrowser, webdriver, IsHereCondition, AnyCondition,
-    SubSeleniumMixin,
-)
 from weboob.browser.url import URL
+from weboob.tools.compat import urlencode
 
 from .pages import (
     AccountsPage, JsonBalances, JsonPeriods, JsonHistory,
     JsonBalances2, CurrencyPage, LoginPage, NoCardPage,
-    NotFoundPage, LoginErrorPage, DashboardPage,
+    NotFoundPage, JsDataPage, HomeLoginPage,
 )
 
 
 __all__ = ['AmericanExpressBrowser']
 
 
-class AmericanExpressLoginBrowser(SeleniumBrowser):
+class AmericanExpressBrowser(LoginBrowser):
     BASEURL = 'https://global.americanexpress.com'
 
-    DRIVER = webdriver.Chrome
-
-    # True for Production / False for debug
-    HEADLESS = True
-
-    login = URL(r'/login', LoginPage)
-    login_error = URL(
-        r'/login',
-        r'/authentication/recovery/password',
-        LoginErrorPage
-    )
-    dashboard = URL(r'/dashboard', DashboardPage)
-
-    def __init__(self, config, *args, **kwargs):
-        super(AmericanExpressLoginBrowser, self).__init__(*args, **kwargs)
-        self.username = config['login'].get()
-        self.password = config['password'].get()
-
-    def do_login(self):
-        self.login.go()
-        self.wait_until_is_here(self.login)
-
-        self.page.login(self.username, self.password)
-
-        self.wait_until(AnyCondition(
-            IsHereCondition(self.login_error),
-            IsHereCondition(self.dashboard),
-        ))
-
-        if self.login_error.is_here():
-            error = self.page.get_error()
-            if any((
-                'The User ID or Password is incorrect' in error,
-                'Both the User ID and Password are required' in error,
-            )):
-                raise BrowserIncorrectPassword(error)
-            if 'Your account has been locked' in error:
-                raise ActionNeeded(error)
-
-            assert False, 'Unhandled error : "%s"' % error
-
-
-class AmericanExpressBrowser(PagesBrowser, SubSeleniumMixin):
-    BASEURL = 'https://global.americanexpress.com'
+    home_login = URL(r'/login\?inav=fr_utility_logout', HomeLoginPage)
+    login = URL(r'/myca/logon/emea/action/login', LoginPage)
 
     accounts = URL(r'/api/servicing/v1/member', AccountsPage)
     json_balances = URL(r'/account-data/v1/financials/balances', JsonBalances)
@@ -103,6 +60,8 @@ class AmericanExpressBrowser(PagesBrowser, SubSeleniumMixin):
     json_periods = URL(r'/account-data/v1/financials/statement_periods', JsonPeriods)
     currency_page = URL(r'https://www.aexp-static.com/cdaas/axp-app/modules/axp-balance-summary/4.7.0/(?P<locale>\w\w-\w\w)/axp-balance-summary.json', CurrencyPage)
 
+    js_data = URL(r'/myca/logon/us/docs/javascript/gatekeeper/gtkp_aa.js', JsDataPage)
+
     no_card = URL(r'https://www.americanexpress.com/us/content/no-card/',
                   r'https://www.americanexpress.com/us/no-card/', NoCardPage)
 
@@ -113,13 +72,79 @@ class AmericanExpressBrowser(PagesBrowser, SubSeleniumMixin):
         'PRELEVEMENT AUTOMATIQUE ENREGISTRE-MERCI',
     ]
 
-    SELENIUM_BROWSER = AmericanExpressLoginBrowser
-
-    def __init__(self, config, *args, **kwargs):
+    def __init__(self, *args, **kwargs):
         super(AmericanExpressBrowser, self).__init__(*args, **kwargs)
-        self.config = config
-        self.username = config['login'].get()
-        self.password = config['password'].get()
+
+    def get_version(self):
+        self.js_data.go()
+        return self.page.get_version()
+
+    def do_login(self):
+        self.home_login.go()
+
+        data = {
+            'request_type': 'login',
+            'UserID': self.username,
+            'Password': self.password,
+            'Logon': 'Logon',
+            'REMEMBERME': 'on',
+            'Face': 'fr_FR',
+            'DestPage': self.BASEURL + '/dashboard',
+            'inauth_profile_transaction_id': 'USLOGON-%s' % str(uuid4()),
+        }
+
+        # we have to overwrite `Content-Length` and `Cookie` to get all
+        # headers in alphabetical order or they will be added at the end
+        # when doing request, also we add every headers needed on website
+        # to try to exactly match what's done or we could get a LGON011 error
+        self.session.headers.update({
+            'Accept': '*/*',
+            'Accept-Encoding': 'gzip: deflate: br',
+            'Accept-Language': 'en-US,en;q=0.9,fr;q=0.8',
+            'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8',
+            'Content-Length': str(len(urlencode(data))),
+            'Cookie': '; '.join('%s=%s' % (k, v) for k, v in self.session.cookies.get_dict().items()),
+            'Origin': self.BASEURL,
+            'Referer': self.BASEURL + '/login?inav=fr_utility_logout',
+            'Sec-Fetch-Dest': 'empty',
+            'Sec-Fetch-Mode': 'cors',
+            'Sec-Fetch-Site': 'same-origin',
+        })
+
+        self.session.headers = OrderedDict(sorted(self.session.headers.items()))
+        del self.session.headers['Upgrade-Insecure-Requests']
+
+        self.login.go(data=data)
+
+        # set back headers
+        self.set_profile(self.PROFILE)
+
+        if self.page.get_status_code() != 0:
+            error_code = self.page.get_error_code()
+            message = self.page.get_error_message()
+            if any(code in error_code for code in ('LGON001', 'LGON003')):
+                raise BrowserIncorrectPassword(message)
+            elif error_code == 'LGON004':
+                # This error happens when the website needs the user to
+                # enter his card information and reset his password.
+                # There is no message returned when this error happens.
+                raise ActionNeeded()
+            elif error_code == 'LGON008':
+                # Don't know what this error means, but if we follow the redirect
+                # url it allows us to be correctly logged.
+                self.location(self.page.get_redirect_url())
+            elif error_code == 'LGON010':
+                raise BrowserUnavailable(message)
+            elif error_code == 'LGON011':
+                # this kind of error is for mystical reasons,
+                # but until now it was headers related, it could be :
+                # - headers not in the right order
+                # - headers with value that doesn't match the one from website
+                # - headers missing
+                # what's next ?
+                assert False, 'Error code "LGON011" (msg:"%s")' % message
+            else:
+                assert False, 'Error code "%s" (msg:"%s") not handled' % (error_code, message)
 
     @need_login
     def iter_accounts(self):

modules/americanexpress/module.py

@@ -37,13 +37,15 @@ class AmericanExpressModule(Module, CapBank):
     LICENSE = 'LGPLv3+'
     CONFIG = BackendConfig(
         ValueBackendPassword('login', label='Code utilisateur', masked=False),
-        ValueBackendPassword('password', label='Mot de passe'),
+        ValueBackendPassword('password', label='Mot de passe')
     )
-
     BROWSER = AmericanExpressBrowser
 
     def create_default_browser(self):
-        return self.create_browser(self.config)
+        return self.create_browser(
+            self.config['login'].get(),
+            self.config['password'].get()
+        )
 
     def iter_accounts(self):
         return self.browser.iter_accounts()

modules/americanexpress/pages.py

@@ -20,11 +20,9 @@
 from __future__ import unicode_literals
 
 from decimal import Decimal
+import re
 
-from dateutil.parser import parse as parse_date
-from selenium.webdriver.common.keys import Keys
-
-from weboob.browser.pages import LoggedPage, JsonPage, HTMLPage
+from weboob.browser.pages import LoggedPage, JsonPage, HTMLPage, RawPage
 from weboob.browser.elements import ItemElement, DictElement, method
 from weboob.browser.filters.standard import (
     Date, Eval, Env, CleanText, Field, CleanDecimal, Format,
@@ -34,9 +32,7 @@
 from weboob.capabilities.bank import Account, Transaction
 from weboob.capabilities.base import NotAvailable
 from weboob.exceptions import ActionNeeded, BrowserUnavailable
-from weboob.browser.selenium import (
-    SeleniumPage, VisibleXPath, AllCondition, NotCondition,
-)
+from dateutil.parser import parse as parse_date
 
 
 def float_to_decimal(f):
@@ -64,32 +60,33 @@ def on_load(self):
         raise BrowserUnavailable(alert_header, alert_content)
 
 
-class LoginErrorPage(SeleniumPage):
-    is_here = VisibleXPath('//div[@role="alert"]/div')
-
-    def get_error(self):
-        return CleanText('//div[@role="alert"]/div')(self.doc)
-
+class HomeLoginPage(HTMLPage):
+    pass
 
-class LoginPage(SeleniumPage):
-    is_here = AllCondition(
-        VisibleXPath('//input[contains(@id, "UserID")]'),
-        VisibleXPath('//input[contains(@id, "Password")]'),
-        VisibleXPath('//button[@id="loginSubmit"]'),
-        NotCondition(VisibleXPath('//div[@role="alert"]/div')),
-    )
 
-    def login(self, username, password):
-        el = self.driver.find_element_by_xpath('//input[contains(@id, "UserID")]')
-        el.send_keys(username)
+class LoginPage(JsonPage):
+    def get_status_code(self):
+        # - 0 = OK
+        # - 1 = Error
+        return CleanDecimal(Dict('statusCode'))(self.doc)
 
-        el = self.driver.find_element_by_xpath('//input[contains(@id, "Password")]')
-        el.send_keys(password)
-        el.send_keys(Keys.RETURN)
+    def get_error_code(self):
+        # - LGON001 = Incorrect password
+        # - LGON003 = Incorrect password: 'Le code utilisateur ou le mot de passe est erroné. Veuillez essayer de nouveau.' on website
+        # - LGON004 = Action needed
+        # - LGON005 = Account blocked
+        # - LGON008 = ?
+        # - LGON010 = Browser unavailable
+        return CleanText(Dict('errorCode'))(self.doc)
 
+    def get_error_message(self):
+        return (
+            CleanText(Dict('errorMessage'))(self.doc) or
+            CleanText(Dict('debugInfo', default=''))(self.doc)
+        )
 
-class DashboardPage(LoggedPage, SeleniumPage):
-    pass
+    def get_redirect_url(self):
+        return CleanText(Dict('redirectUrl'))(self.doc)
 
 
 class AccountsPage(LoggedPage, JsonPage):
@@ -216,3 +213,10 @@ def obj_original_amount(self):
                     return original_amount
 
             obj__ref = Dict('identifier')
+
+
+class JsDataPage(RawPage):
+    def get_version(self):
+        version = re.search(r'"(\d\.[\d\._]+)"', self.text)
+        assert version, 'Could not match version number in javascript'
+        return version.group(1)

modules/amundi/module.py

@@ -18,7 +18,7 @@
 # along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
 
 
-from weboob.capabilities.bank import CapBankPockets
+from weboob.capabilities.bank import CapBankWealth
 from weboob.tools.backend import Module, BackendConfig
 from weboob.tools.value import ValueBackendPassword, Value
 
@@ -27,7 +27,7 @@
 __all__ = ['AmundiModule']
 
 
-class AmundiModule(Module, CapBankPockets):
+class AmundiModule(Module, CapBankWealth):
     NAME = 'amundi'
     DESCRIPTION = u'Amundi'
     MAINTAINER = u'James GALT'

modules/amundi/pages.py

@@ -51,8 +51,9 @@ def get_token(self):
     'PEE': Account.TYPE_PEE,
     'PEG': Account.TYPE_PEE,
     'PEI': Account.TYPE_PEE,
-    'PERCO': Account.TYPE_PER,
-    'PERCOI': Account.TYPE_PER,
+    'HES': Account.TYPE_PEE,
+    'PERCO': Account.TYPE_PERCO,
+    'PERCOI': Account.TYPE_PERCO,
     'PER': Account.TYPE_PER,
     'RSP': Account.TYPE_RSP,
     'ART 83': Account.TYPE_ARTICLE_83,
@@ -251,12 +252,10 @@ def get_performance_history(self):
         # We do not fill the performance dictionary if no performance is available,
         # otherwise it will overwrite the data obtained from the JSON with empty values.
         perfs = {}
-        if matches.get('1 an'):
-            perfs[1] = percent_to_ratio(CleanDecimal.French(default=NotAvailable).filter(matches['1 an']))
-        if matches.get('3 ans'):
-            perfs[3] = percent_to_ratio(CleanDecimal.French(default=NotAvailable).filter(matches['3 ans']))
-        if matches.get('5 ans'):
-            perfs[5] = percent_to_ratio(CleanDecimal.French(default=NotAvailable).filter(matches['5 ans']))
+        for k, v in {1: '1 an', 3: '3 ans', 5: '5 ans'}.items():
+            if matches.get(v):
+                perfs[k] = percent_to_ratio(CleanDecimal.French(default=NotAvailable).filter(matches[v]))
+
         return perfs
 
 

modules/aviva/browser.py

@@ -23,6 +23,7 @@
 from weboob.browser import LoginBrowser, need_login
 from weboob.browser.url import BrowserParamURL
 from weboob.capabilities.base import empty, NotAvailable
+from weboob.capabilities.bank import Account
 from weboob.exceptions import BrowserIncorrectPassword, BrowserPasswordExpired, ActionNeeded, BrowserHTTPError
 from weboob.tools.capabilities.bank.transactions import sorted_transactions
 
@@ -82,6 +83,8 @@ def iter_accounts(self):
                     # and accounts with unavailable balances
                     continue
                 self.page.fill_account(obj=account)
+                if account.type == Account.TYPE_UNKNOWN:
+                    self.logger.warning('Account "%s" is untyped, please check the related type in account details.', account.label)
                 yield account
             except BrowserHTTPError:
                 self.logger.warning('Could not get the account details: account %s will be skipped', account.id)

modules/aviva/pages.py

@@ -20,7 +20,7 @@
 
 from weboob.browser.pages import HTMLPage, LoggedPage
 from weboob.browser.elements import ListElement, ItemElement, method
-from weboob.browser.filters.standard import CleanText, Capitalize, Format, Date, Regexp, CleanDecimal, Env, Field, Async, AsyncLoad
+from .compat.weboob_browser_filters_standard import CleanText, Capitalize, Format, Date, Regexp, CleanDecimal, Env, Field, Async, AsyncLoad
 from weboob.browser.filters.html import Attr, Link
 from weboob.capabilities.bank import Account, Investment, Transaction
 from weboob.capabilities.base import NotAvailable

modules/aviva/pages/account_page.py

@@ -21,9 +21,7 @@
 
 from weboob.browser.pages import LoggedPage
 from weboob.browser.elements import ListElement, ItemElement, method
-from weboob.browser.filters.standard import (
-    CleanText, Field, Map, Regexp
-)
+from weboob.browser.filters.standard import CleanText, Field
 from weboob.browser.filters.html import AbsoluteLink
 from weboob.capabilities.bank import Account
 from weboob.capabilities.base import NotAvailable
@@ -31,12 +29,6 @@
 from .detail_pages import BasePage
 
 
-ACCOUNT_TYPES = {
-    'Assurance vie': Account.TYPE_LIFE_INSURANCE,
-    'Epargne – Retraite': Account.TYPE_PERP,
-}
-
-
 class AccountsPage(LoggedPage, BasePage):
     @method
     class iter_accounts(ListElement):
@@ -49,9 +41,6 @@ class item(ItemElement):
             obj_number = Field('id')
             obj_label = CleanText('.//p[has-class("a-heading")]', default=NotAvailable)
             obj_url = AbsoluteLink('.//a[contains(text(), "Détail")]')
-            obj_type = Map(Regexp(CleanText('../../../div[contains(@class, "o-product-roundels-category")]'),
-                           r'Vérifier votre (.*) contrats', default=NotAvailable),
-                           ACCOUNT_TYPES, Account.TYPE_UNKNOWN)
 
             def condition(self):
                 # 'Prévoyance' div is for insurance contracts -- they are not bank accounts and thus are skipped

modules/aviva/pages/detail_pages.py

@@ -24,9 +24,9 @@
 from weboob.browser.elements import ListElement, ItemElement, method
 from weboob.browser.filters.standard import (
     CleanText, Title, Format, Date, Regexp, CleanDecimal, Env,
-    Currency, Field, Eval, Coalesce,
+    Currency, Field, Eval, Coalesce, MapIn, Lower,
 )
-from weboob.capabilities.bank import Investment, Transaction
+from weboob.capabilities.bank import Account, Investment, Transaction
 from weboob.capabilities.base import NotAvailable
 from weboob.exceptions import ActionNeeded, BrowserUnavailable
 from weboob.tools.compat import urljoin
@@ -59,12 +59,29 @@ def get_error(self):
         return CleanText('//h1[contains(text(), "Votre nouvel identifiant et mot de passe")]')(self.doc)
 
 
+ACCOUNT_TYPES = {
+    "assurance vie": Account.TYPE_LIFE_INSURANCE,
+    "retraite madelin": Account.TYPE_MADELIN,
+    "article 83": Account.TYPE_ARTICLE_83,
+    "plan d'epargne retraite populaire": Account.TYPE_PERP,
+    "plan epargne retraite": Account.TYPE_PER,
+}
+
 class InvestmentPage(LoggedPage, HTMLPage):
     @method
     class fill_account(ItemElement):
-        obj_balance = CleanDecimal.French('//ul[has-class("m-data-group")]//strong')
-        obj_currency = Currency('//ul[has-class("m-data-group")]//strong')
+        obj_balance = Coalesce(
+            CleanDecimal.French('//h3[contains(text(), "Valeur de rachat")]/following-sibling::p/strong', default=NotAvailable),
+            CleanDecimal.French('//h3[contains(text(), "pargne retraite")]/following-sibling::p/strong', default=NotAvailable),
+            CleanDecimal.French('//h3[contains(text(), "Capital constitutif de rente")]/following-sibling::p', default=NotAvailable),
+        )
+        obj_currency = Coalesce(
+            Currency('//h3[contains(text(), "Valeur de rachat")]/following-sibling::p/strong', default=NotAvailable),
+            Currency('//h3[contains(text(), "pargne retraite")]/following-sibling::p/strong', default=NotAvailable),
+            Currency('//h3[contains(text(), "Capital constitutif de rente")]/following-sibling::p', default=NotAvailable),
+        )
         obj_valuation_diff = CleanDecimal.French('//h3[contains(., "value latente")]/following-sibling::p[1]', default=NotAvailable)
+        obj_type = MapIn(Lower(CleanText('//h3[contains(text(), "Type de produit")]/following-sibling::p')), ACCOUNT_TYPES, Account.TYPE_UNKNOWN)
 
     def get_history_link(self):
         history_link = self.doc.xpath('//li/a[contains(text(), "Historique")]/@href')

modules/banquepopulaire/browser.py

@@ -20,6 +20,7 @@
 from __future__ import unicode_literals
 
 import re
+from uuid import uuid4
 
 from datetime import datetime
 from collections import OrderedDict
@@ -32,6 +33,7 @@
 from weboob.capabilities.bank import Account, AccountOwnership
 from weboob.capabilities.base import NotAvailable, find_object
 from weboob.tools.capabilities.bank.investments import create_french_liquidity
+from weboob.tools.compat import urlparse, parse_qs
 
 from .pages import (
     LoggedOut,
@@ -41,6 +43,8 @@
     NatixisPage, EtnaPage, NatixisInvestPage, NatixisHistoryPage, NatixisErrorPage,
     NatixisDetailsPage, NatixisChoicePage, NatixisRedirect,
     LineboursePage, AlreadyLoginPage, InvestmentPage,
+    NewLoginPage, JsFilePage, AuthorizePage, LoginTokensPage, VkImagePage,
+    AuthenticationMethodPage, AuthenticationStepPage, CaissedepargneVirtKeyboard,
 )
 
 from .document_pages import BasicTokenPage, SubscriberPage, SubscriptionsPage, DocumentsPage
@@ -102,6 +106,21 @@ def wrapper(browser, *args, **kwargs):
 
 class BanquePopulaire(LoginBrowser):
     login_page = URL(r'https://[^/]+/auth/UI/Login.*', LoginPage)
+    new_login = URL(r'https://[^/]+/se-connecter/sso', NewLoginPage)
+    js_file = URL(r'https://[^/]+/se-connecter/main-.*.js$', JsFilePage)
+    authorize = URL(r'https://www.as-ex-ath-groupe.banquepopulaire.fr/api/oauth/v2/authorize', AuthorizePage)
+    login_tokens = URL(r'https://www.as-ex-ath-groupe.banquepopulaire.fr/api/oauth/v2/consume', LoginTokensPage)
+    authentication_step = URL(
+        r'https://www.icgauth.banquepopulaire.fr/dacsrest/api/v1u0/transaction/(?P<validation_id>[^/]+)/step', AuthenticationStepPage
+    )
+    authentication_method_page = URL(
+        r'https://www.icgauth.banquepopulaire.fr/dacsrest/api/v1u0/transaction/(?P<validation_id>)',
+        AuthenticationMethodPage,
+    )
+    vk_image = URL(
+        r'https://www.icgauth.banquepopulaire.fr/dacs-rest-media/api/v1u0/medias/mappings/[a-z0-9-]+/images',
+        VkImagePage,
+    )
     index_page = URL(r'https://[^/]+/cyber/internet/Login.do', IndexPage)
     accounts_page = URL(r'https://[^/]+/cyber/internet/StartTask.do\?taskInfoOID=mesComptes.*',
                         r'https://[^/]+/cyber/internet/StartTask.do\?taskInfoOID=maSyntheseGratuite.*',
@@ -152,8 +171,11 @@ class BanquePopulaire(LoginBrowser):
                     r'https://[^/]+/cyber/internet/ShowPortal.do\?token=.*',
                     HomePage)
 
-    already_login_page = URL(r'https://[^/]+/dacswebssoissuer.*',
-                             r'https://[^/]+/WebSSO_BP/_(?P<bankid>\d+)/index.html\?transactionID=(?P<transactionID>.*)', AlreadyLoginPage)
+    already_login_page = URL(
+        r'https://[^/]+/dacswebssoissuer.*',
+        r'https://[^/]+/WebSSO_BP/_(?P<bankid>\d+)/index.html\?transactionID=(?P<transactionID>.*)',
+        AlreadyLoginPage
+    )
     login2_page = URL(r'https://[^/]+/WebSSO_BP/_(?P<bankid>\d+)/index.html\?transactionID=(?P<transactionID>.*)', Login2Page)
 
     # natixis
@@ -234,6 +256,17 @@ def do_login(self):
         if self.home_page.is_here():
             return
 
+        if self.new_login.is_here():
+            if not self.password.isnumeric():
+                # Vk from new login only accepts numeric characters
+                raise BrowserIncorrectPassword('Le mot de passe doit être composé de chiffres uniquement')
+            return self.do_new_login()
+        return self.do_old_login()
+
+    def do_old_login(self):
+        assert self.login2_page.is_here(), 'Should be on login2 page'
+        self.page.set_form_ids()
+
         try:
             self.page.login(self.username, self.password)
         except BrowserUnavailable as ex:
@@ -244,6 +277,8 @@ def do_login(self):
             if 'Cette page est indisponible' in ex.message and not self.password.isdigit():
                 raise BrowserIncorrectPassword()
             raise
+        if not self.password.isnumeric():
+            self.logger.warning('Password with non numeric chararacters still works')
 
         if self.login_page.is_here():
             raise BrowserIncorrectPassword()
@@ -252,8 +287,108 @@ def do_login(self):
             data = {'integrationMode': 'INTERNET_RESCUE'}
             self.location('/cyber/internet/Login.do', data=data)
 
+    def do_new_login(self):
+        # Same login as caissedepargne
+        url_params = parse_qs(urlparse(self.url).query)
+        cdetab = url_params['cdetab'][0]
+        continue_url = url_params['continue'][0]
+
+        main_js_file = self.page.get_main_js_file_url()
+        self.location(main_js_file)
+
+        client_id = self.page.get_client_id()
+        nonce = self.page.get_nonce()  # Hardcoded in their js...
+
+        # On the website, this sends back json because of the header
+        # 'Accept': 'applcation/json'. If we do not add this header, we
+        # instead have a form that we can directly send to complete
+        # the login.
+        self.authorize.go(
+            params={
+                'nonce': nonce,
+                'scope': '',
+                'response_type': 'id_token token',
+                'response_mode': 'form_post',
+                'cdetab': cdetab,
+                'login_hint': self.username.upper(),
+                'display': 'page',
+                'client_id': client_id,
+                'claims': '{"userinfo":{"cdetab":null,"authMethod":null,"authLevel":null},"id_token":{"auth_time":{"essential":true},"last_login":null}}',
+                'bpcesta': '{"csid":"%s","typ_app":"rest","enseigne":"bp","typ_sp":"out-band","typ_act":"auth","snid":"%s","cdetab":"%s","typ_srv":"part","phase":"1"}' % (str(uuid4()), 123456, cdetab),
+            },
+        )
+        self.page.send_form()
+
+        self.page.check_errors(feature='login')
+
+        validation_id = self.page.get_validation_id()
+        validation_unit_id = self.page.validation_unit_id
+
+        vk_info = self.page.get_authentication_method_info()
+        vk_id = vk_info['id']
+        vk_images_url = vk_info['virtualKeyboard']['externalRestMediaApiUrl']
+
+        self.location(vk_images_url)
+        images_url = self.page.get_all_images_data()
+        vk = CaissedepargneVirtKeyboard(self, images_url)
+        code = vk.get_string_code(self.password)
+
+        headers = {
+            'Referer': self.BASEURL,
+            'Accept': 'application/json, text/plain, */*',
+        }
+        self.authentication_step.go(
+            validation_id=validation_id,
+            json={
+                'validate': {
+                    validation_unit_id: [{
+                        'id': vk_id,
+                        'password': code,
+                        'type': 'PASSWORD',
+                    }],
+                },
+            },
+            headers=headers,
+        )
+
+        assert self.authentication_step.is_here()
+        self.page.check_errors(feature='login')
+
+        self.do_redirect(headers)
+
+        access_token = self.page.get_access_token()
+        expires_in = self.page.get_expires_in()
+
+        self.location(
+            continue_url,
+            params={
+                'access_token': access_token,
+                'token_type': 'Bearer',
+                'grant_type': 'implicit flow',
+                'NameId': self.username,
+                'Segment': 'part',
+                'scopes': '',
+                'expires_in': expires_in,
+            },
+        )
+
+        url_params = parse_qs(urlparse(self.url).query)
+        validation_id = url_params['transactionID'][0]
+
+        self.authentication_method_page.go(validation_id=validation_id)
+        # Need to do the redirect a second time to finish login
+        self.do_redirect(headers)
+
     ACCOUNT_URLS = ['mesComptes', 'mesComptesPRO', 'maSyntheseGratuite', 'accueilSynthese', 'equipementComplet']
 
+    def do_redirect(self, headers):
+        redirect_data = self.page.get_redirect_data()
+        self.location(
+            redirect_data['action'],
+            data={'SAMLResponse': redirect_data['samlResponse']},
+            headers=headers,
+        )
+
     @retry(BrokenPageError)
     @need_login
     def go_on_accounts_list(self):
@@ -662,9 +797,10 @@ def iter_documents(self, subscription):
                 {'typeDocument': {'code': 'EXTRAIT', 'label': 'Extrait de compte', 'type': 'referenceLogiqueDocument'}}
             ]
         }
-        self.location('/api-bp/wapi/2.0/abonnes/current/documents/recherche-avancee', json=body, headers=self.documents_headers)
+        self.documents_page.go(json=body, headers=self.documents_headers)
         return self.page.iter_documents(subid=subscription.id)
 
+    @retry(ClientError)
     def download_document(self, document):
         return self.open(document.url, headers=self.documents_headers).content
 

modules/banquepopulaire/module.py

@@ -20,7 +20,6 @@
 from __future__ import unicode_literals
 
 from collections import OrderedDict
-from functools import reduce
 
 from weboob.capabilities.bank import CapBankWealth, AccountNotFound
 from weboob.capabilities.base import find_object
@@ -70,8 +69,23 @@ class BanquePopulaireModule(Module, CapBankWealth, CapContact, CapProfile, CapDo
         'www.ibps.valdefrance.banquepopulaire.fr': 'Val de France',
         }.items(), key=lambda k_v: (k_v[1], k_v[0]))])
 
+    # Some regions have been renamed after bank cooptation
+    region_aliases = {
+        'www.ibps.alsace.banquepopulaire.fr': 'www.ibps.bpalc.banquepopulaire.fr',
+        'www.ibps.lorrainechampagne.banquepopulaire.fr': 'www.ibps.bpalc.banquepopulaire.fr',
+        'www.ibps.loirelyonnais.banquepopulaire.fr': 'www.ibps.bpaura.banquepopulaire.fr',
+        'www.ibps.alpes.banquepopulaire.fr': 'www.ibps.bpaura.banquepopulaire.fr',
+        'www.ibps.massifcentral.banquepopulaire.fr': 'www.ibps.bpaura.banquepopulaire.fr',
+        # creditmaritime atlantique now redirecting to Banque Populaire Aquitaine Centre Atlantique (new website)
+        'www.ibps.atlantique.creditmaritime.groupe.banquepopulaire.fr': 'www.ibps.bpaca.banquepopulaire.fr',
+        # creditmaritime bretagnenormandie now redirecting to Banque Populaire Grand Ouest (old website)
+        'www.ibps.bretagnenormandie.cmm.groupe.banquepopulaire.fr': 'www.ibps.cmgo.creditmaritime.groupe.banquepopulaire.fr',
+        'www.ibps.atlantique.banquepopulaire.fr': 'www.ibps.bpgo.banquepopulaire.fr',
+        'www.ibps.ouest.banquepopulaire.fr': 'www.ibps.bpgo.banquepopulaire.fr',
+    }
+
     CONFIG = BackendConfig(
-        Value('website', label='Région', choices=website_choices),
+        Value('website', label='Région', choices=website_choices, aliases=region_aliases),
         ValueBackendPassword('login', label='Identifiant', masked=False),
         ValueBackendPassword('password', label='Mot de passe')
     )
@@ -81,18 +95,7 @@ class BanquePopulaireModule(Module, CapBankWealth, CapContact, CapProfile, CapDo
     accepted_document_types = (DocumentTypes.STATEMENT,)
 
     def create_default_browser(self):
-        repls = [
-            ('alsace', 'bpalc'),
-            ('lorrainechampagne', 'bpalc'),
-            ('loirelyonnais', 'bpaura'),
-            ('alpes', 'bpaura'),
-            ('massifcentral', 'bpaura'),
-            ('atlantique.creditmaritime', 'cmgo.creditmaritime'),
-            ('bretagnenormandie.cmm', 'cmgo'),
-            ('atlantique.banquepopulaire', 'bpgo.banquepopulaire'),
-            ('ouest.banquepopulaire', 'bpgo.banquepopulaire'),
-        ]
-        website = reduce(lambda a, kv: a.replace(*kv), repls, self.config['website'].get())
+        website = self.config['website'].get()
 
         return self.create_browser(
             website,

modules/banquepopulaire/pages.py

@@ -33,7 +33,10 @@
 from weboob.browser.filters.json import Dict
 from weboob.exceptions import BrowserUnavailable, BrowserIncorrectPassword, ActionNeeded
 
-from weboob.browser.pages import HTMLPage, LoggedPage, FormNotFound, JsonPage, RawPage, XMLPage
+from weboob.browser.pages import (
+    HTMLPage, LoggedPage, FormNotFound, JsonPage, RawPage, XMLPage,
+    AbstractPage,
+)
 
 from weboob.capabilities.bank import Account, Investment
 from weboob.capabilities.profile import Person
@@ -306,6 +309,54 @@ def on_load(self):
         self.browser.location(a)
 
 
+class NewLoginPage(AbstractPage):
+    PARENT = 'caissedepargne'
+    PARENT_URL = 'new_login'
+    BROWSER_ATTR = 'package.browser.CaisseEpargne'
+
+
+class JsFilePage(AbstractPage):
+    PARENT = 'caissedepargne'
+    PARENT_URL = 'js_file'
+    BROWSER_ATTR = 'package.browser.CaisseEpargne'
+
+
+class AuthorizePage(AbstractPage):
+    PARENT = 'caissedepargne'
+    PARENT_URL = 'authorize'
+    BROWSER_ATTR = 'package.browser.CaisseEpargne'
+
+
+class LoginTokensPage(AbstractPage):
+    PARENT = 'caissedepargne'
+    PARENT_URL = 'login_tokens'
+    BROWSER_ATTR = 'package.browser.CaisseEpargne'
+
+    def get_expires_in(self):
+        return Dict('parameters/expires_in')(self.doc)
+
+
+class VkImagePage(AbstractPage):
+    PARENT = 'caissedepargne'
+    PARENT_URL = 'vk_image'
+    BROWSER_ATTR = 'package.browser.CaisseEpargne'
+
+
+class AuthenticationMethodPage(AbstractPage):
+    PARENT = 'caissedepargne'
+    PARENT_URL = 'authentication_method_page'
+    BROWSER_ATTR = 'package.browser.CaisseEpargne'
+
+    def get_redirect_data(self):
+        return Dict('response/saml2_post')(self.doc)
+
+
+class AuthenticationStepPage(AbstractPage):
+    PARENT = 'caissedepargne'
+    PARENT_URL = 'authentication_step'
+    BROWSER_ATTR = 'package.browser.CaisseEpargne'
+
+
 class LoginPage(MyHTMLPage):
     def on_load(self):
         h1 = CleanText('//h1[1]')(self.doc)
@@ -324,6 +375,39 @@ def login(self, login, passwd):
         form.submit()
 
 
+class CaissedepargneVirtKeyboard(SplitKeyboard):
+    char_to_hash = {
+        '0': '66ec79b200706e7f9c14f2b6d35dbb05',
+        '1': '529819241cce382b429b4624cb019b56',
+        '2': 'fab68678204198b794ce580015c8637f',
+        '3': '3fc5280d17cf057d1c4b58e4f442ceb8',
+        '4': ('dea8800bdd5fcaee1903a2b097fbdef0', 'e413098a4d69a92d08ccae226cea9267', '61f720966ccac6c0f4035fec55f61fe6', '2cbd19a4b01c54b82483f0a7a61c88a1'),
+        '5': 'ff1909c3b256e7ab9ed0d4805bdbc450',
+        '6': '7b014507ffb92a80f7f0534a3af39eaa',
+        '7': '7d598ff47a5607022cab932c6ad7bc5b',
+        '8': ('4ed28045e63fa30550f7889a18cdbd81', '88944bdbef2e0a49be9e0c918dd4be64'),
+        '9': 'dd6317eadb5a0c68f1938cec21b05ebe',
+    }
+    codesep = ' '
+
+    def __init__(self, browser, images):
+        code_to_filedata = {}
+        for img_item in images:
+            img_content = browser.location(img_item['uri']).content
+            img = Image.open(BytesIO(img_content))
+            img = img.filter(ImageFilter.UnsharpMask(
+                radius=2,
+                percent=150,
+                threshold=3,
+            ))
+            img = img.convert('L', dither=None)
+            img = Image.eval(img, lambda x: 0 if x < 20 else 255)
+            b = BytesIO()
+            img.save(b, format='PNG')
+            code_to_filedata[img_item['value']] = b.getvalue()
+        super(CaissedepargneVirtKeyboard, self).__init__(code_to_filedata)
+
+
 class MyVirtKeyboard(SplitKeyboard):
     char_to_hash = {
         '0': 'cce0f72c47c74a3dde57c4fdbcda1db4',
@@ -364,6 +448,7 @@ def on_load(self):
         if not self.browser.no_login:
             raise LoggedOut()
 
+    def set_form_ids(self):
         r = self.browser.open(self.request_url)
         doc = r.json()
 
@@ -554,7 +639,8 @@ class AccountsPage(LoggedPage, MyHTMLPage):
         (re.compile(r'.*Plan Epargne Retraite.*'), Account.TYPE_PERP),
         (re.compile(r'.*Titres.*'), Account.TYPE_MARKET),
         (re.compile(r'.*Selection Vie.*'), Account.TYPE_LIFE_INSURANCE),
-        (re.compile(r'^Fructi Pulse.*'), Account.TYPE_MARKET),
+        (re.compile(r'^Fructi Pulse.*'), Account.TYPE_LIFE_INSURANCE),
+        (re.compile(r'^Fructi Neo.*'), Account.TYPE_LIFE_INSURANCE),
         (re.compile(r'^(Quintessa|Solevia|Irriga|Delfea).*'), Account.TYPE_LIFE_INSURANCE),
         (re.compile(r'^Plan Epargne Enfant Mul.*'), Account.TYPE_MARKET),
         (re.compile(r'^Alc Premium'), Account.TYPE_MARKET),

modules/bnporc/module.py

@@ -79,6 +79,7 @@ class BNPorcModule(
         DocumentTypes.STATEMENT,
         DocumentTypes.REPORT,
         DocumentTypes.BILL,
+        DocumentTypes.RIB,
         DocumentTypes.OTHER,
     )
 
@@ -125,8 +126,15 @@ def iter_investment(self, account):
     def iter_transfer_recipients(self, origin_account):
         if self.config['website'].get() != 'pp':
             raise NotImplementedError()
+
         if isinstance(origin_account, Account):
-            origin_account = origin_account.id
+            emitter_account = find_object(self.iter_accounts(), id=origin_account.id)
+            if not emitter_account:
+                # account_id is different in PSD2 case
+                # search for the account with iban first to get the account_id
+                assert origin_account.iban, 'Cannot do iter_transfer_recipient, the origin account was not found'
+                emitter_account = find_object(self.iter_accounts(), iban=origin_account.iban, error=AccountNotFound)
+            origin_account = emitter_account.id
         return self.browser.iter_recipients(origin_account)
 
     def new_recipient(self, recipient, **params):
@@ -175,6 +183,10 @@ def transfer_check_recipient_id(self, old, new):
             # iternal recipients id
             return old == new
 
+    def transfer_check_account_id(self, old, new):
+        # don't check account id because in PSD2 case, account_id is different
+        return True
+
     def iter_contacts(self):
         if not hasattr(self.browser, 'get_advisor'):
             raise NotImplementedError()

modules/bnporc/pp/browser.py

@@ -24,7 +24,7 @@
 from datetime import datetime
 from dateutil.relativedelta import relativedelta
 import time
-from requests.exceptions import ConnectionError
+from requests.exceptions import ConnectionError, SSLError
 
 from weboob.browser.browsers import LoginBrowser, URL, need_login, StatesMixin
 from weboob.capabilities.base import find_object
@@ -32,7 +32,7 @@
     AccountNotFound, Account, AddRecipientStep, AddRecipientTimeout,
     TransferInvalidRecipient, Loan,
 )
-from weboob.capabilities.bill import Subscription
+from weboob.capabilities.bill import Subscription, Document, DocumentTypes
 from weboob.capabilities.profile import ProfileMissing
 from weboob.tools.decorators import retry
 from weboob.tools.capabilities.bank.transactions import sorted_transactions
@@ -52,7 +52,7 @@
     UselessPage, TransferAssertionError, LoanDetailsPage,
 )
 
-from .document_pages import DocumentsPage, DocumentsResearchPage, TitulairePage
+from .document_pages import DocumentsPage, DocumentsResearchPage, TitulairePage, RIBPage
 
 __all__ = ['BNPPartPro', 'HelloBank']
 
@@ -119,6 +119,7 @@ class BNPParibasBrowser(LoginBrowser, StatesMixin):
     titulaire = URL(r'/demat-wspl/rest/listerTitulairesDemat', TitulairePage)
     document = URL(r'/demat-wspl/rest/listerDocuments', DocumentsPage)
     document_research = URL(r'/demat-wspl/rest/rechercheCriteresDemat', DocumentsResearchPage)
+    rib_page = URL(r'/rib-wspl/rpc/restituerRIB', RIBPage)
 
     profile = URL(r'/kyc-wspl/rest/informationsClient', ProfilePage)
     list_detail_card = URL(r'/udcarte-wspl/rest/listeDetailCartes', ListDetailCardPage)
@@ -241,7 +242,15 @@ def iter_accounts(self):
                 self.capitalisation_page.go(params=params)
             except ServerError:
                 self.logger.warning("An Internal Server Error occurred")
-            else:
+            except SSLError as e:
+                self.logger.warning("SSL Error occurred : %s", e)
+                certificate_errors = (
+                    'SEC_ERROR_EXPIRED_CERTIFICATE',  # nss
+                    'certificate verify failed',  # openssl
+                )
+                if all(error not in str(e) for error in certificate_errors):
+                    raise e
+            finally:
                 if self.capitalisation_page.is_here() and self.page.has_contracts():
                     for account in self.page.iter_capitalisation():
                         # Life Insurance accounts may appear BOTH in the API and the "Assurances Vie" domain,
@@ -265,14 +274,16 @@ def iter_history(self, account, coming=False):
             return []
         if account.type == Account.TYPE_PEA and account.label.endswith('Espèces'):
             return []
-        if account.type == account.TYPE_LIFE_INSURANCE:
+        if account.type == Account.TYPE_LIFE_INSURANCE:
             return self.iter_lifeinsurance_history(account, coming)
-        elif account.type in (account.TYPE_MARKET, Account.TYPE_PEA) and not coming:
+        elif account.type in (Account.TYPE_MARKET, Account.TYPE_PEA):
+            if coming:
+                return []
             try:
                 self.market_list.go(json={}, method='POST')
             except ServerError:
                 self.logger.warning("An Internal Server Error occurred")
-                return iter([])
+                return []
             for market_acc in self.page.get_list():
                 if account.number[-4:] == market_acc['securityAccountNumber'][-4:]:
                     self.page = self.market_history.go(
@@ -281,7 +292,7 @@ def iter_history(self, account, coming=False):
                         }
                     )
                     return self.page.iter_history()
-            return iter([])
+            return []
         else:
             if not self.card_to_transaction_type:
                 self.list_detail_card.go()
@@ -298,7 +309,7 @@ def iter_history(self, account, coming=False):
             except BrowserUnavailable:
                 # old url is still used for certain connections bu we don't know which one is,
                 # so the same HistoryPage is attained by the old url in another URL object
-                data[1]['startDate'] = (datetime.now() - relativedelta(years=3)).strftime('%d%m%Y')
+                data['startDate'] = (datetime.now() - relativedelta(years=3)).strftime('%d%m%Y')
                 # old url authorizes up to 3 years of history
                 self.history_old.go(data=data)
 
@@ -535,8 +546,30 @@ def iter_threads(self):
     def get_thread(self, thread):
         raise NotImplementedError()
 
+    def _fetch_rib_document(self, subscription):
+        self.rib_page.go(
+            params={
+                'contractId': subscription.id,
+                'i18nSiteType': 'part',  # site type value doesn't seem to matter as long as it's present
+                'i18nLang': 'fr',
+                'i18nVersion': 'V1',
+            },
+        )
+        if self.rib_page.is_here() and self.page.is_rib_available():
+            d = Document()
+            d.id = subscription.id + '_RIB'
+            d.url = self.page.url
+            d.type = DocumentTypes.RIB
+            d.format = 'pdf'
+            d.label = 'RIB'
+            return d
+
     @need_login
     def iter_documents(self, subscription):
+        rib = self._fetch_rib_document(subscription)
+        if rib:
+            yield rib
+
         titulaires = self.titulaire.go().get_titulaires()
         # Calling '/demat-wspl/rest/listerDocuments' before the request on 'document'
         # is necessary when you specify an ikpi, otherwise no documents are returned

modules/bnporc/pp/document_pages.py

@@ -25,9 +25,9 @@
 
 from weboob.browser.elements import DictElement, ItemElement, method
 from weboob.browser.filters.json import Dict
-from weboob.browser.filters.standard import Format, Date, Env
-from weboob.browser.pages import JsonPage, LoggedPage
-from weboob.capabilities.bill import Document, DocumentTypes
+from weboob.browser.filters.standard import Format, Date, Env, Field
+from weboob.browser.pages import JsonPage, LoggedPage, RawPage
+from weboob.capabilities.bill import Document, Bill, DocumentTypes
 from weboob.tools.compat import urlencode
 
 patterns = {
@@ -58,7 +58,10 @@ def get_titulaires(self):
 
 
 class ItemDocument(ItemElement):
-    klass = Document
+    def build_object(self):
+        if Field('type')(self) == DocumentTypes.BILL:
+            return Bill()
+        return Document()
 
     def condition(self):
         # There is two type of json, the one with the ibancrypte in it
@@ -148,3 +151,9 @@ class iter_documents(DictElement):
 
         class item(ItemDocument):
             pass
+
+
+class RIBPage(LoggedPage, RawPage):
+    def is_rib_available(self):
+        # If the page has no content, it means no RIB can be found
+        return bool(self.content)

modules/bnppere/browser.py

@@ -19,7 +19,8 @@
 
 from __future__ import unicode_literals
 
-from weboob.browser import AbstractBrowser, LoginBrowser, URL, need_login
+from weboob.browser.browsers import AbstractBrowser, LoginBrowser, URL, need_login
+from .compat.weboob_capabilities_bank import Account, Per
 from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable, ActionNeeded
 from .pages import (
     LoginPage, ProfilePage, ErrorPage, AccountPage, AccountSwitchPage,
@@ -31,9 +32,6 @@ class BnppereBrowser(AbstractBrowser):
     PARENT = 's2e'
     PARENT_ATTR = 'package.browser.BnppereBrowser'
 
-    def get_profile(self):
-        raise NotImplementedError()
-
 
 class VisiogoBrowser(LoginBrowser):
     BASEURL = 'https://visiogo.bnpparibas.com/'
@@ -74,7 +72,16 @@ def do_login(self):
     @need_login
     def iter_accounts(self):
         self.account_page.go()
-        accounts_list = list(self.page.iter_accounts())
+        accounts_list = []
+
+        for account in self.page.iter_accounts():
+            if account.type == Account.TYPE_PER:
+                per = Per.from_dict(account.to_dict())
+                self.page.fill_per(obj=per)
+                accounts_list.append(per)
+            else:
+                accounts_list.append(account)
+
         # We need to know if there are several accounts
         # in order to handle their investments properly
         if len(accounts_list) > 1: