.gitignore

@@ -15,3 +15,4 @@ modules/modules.list
 *.DS_Store
 *.coverage*
 .vscode/
+geckodriver.log

modules/americanexpress/browser.py

@@ -20,36 +20,87 @@
 from __future__ import unicode_literals
 
 import datetime
+from dateutil.parser import parse as parse_date
 
 from weboob.exceptions import BrowserIncorrectPassword, ActionNeeded
-from weboob.browser.browsers import LoginBrowser, need_login
+from weboob.browser.browsers import PagesBrowser, need_login
 from weboob.browser.exceptions import HTTPNotFound, ServerError
+from weboob.browser.selenium import (
+    SeleniumBrowser, webdriver, IsHereCondition, AnyCondition,
+    SubSeleniumMixin,
+)
 from weboob.browser.url import URL
-from dateutil.parser import parse as parse_date
 
 from .pages import (
     AccountsPage, JsonBalances, JsonPeriods, JsonHistory,
     JsonBalances2, CurrencyPage, LoginPage, NoCardPage,
-    NotFoundPage,
+    NotFoundPage, LoginErrorPage, DashboardPage,
 )
 
 
 __all__ = ['AmericanExpressBrowser']
 
 
-class AmericanExpressBrowser(LoginBrowser):
+class AmericanExpressLoginBrowser(SeleniumBrowser):
     BASEURL = 'https://global.americanexpress.com'
 
-    login = URL(r'/myca/logon/emea/action/login', LoginPage)
+    DRIVER = webdriver.Chrome
+
+    # True for Production / False for debug
+    HEADLESS = True
+
+    login = URL(r'/login', LoginPage)
+    login_error = URL(
+        r'/login',
+        r'/authentication/recovery/password',
+        LoginErrorPage
+    )
+    dashboard = URL(r'/dashboard', DashboardPage)
+
+    def __init__(self, config, *args, **kwargs):
+        super(AmericanExpressLoginBrowser, self).__init__(*args, **kwargs)
+        self.username = config['login'].get()
+        self.password = config['password'].get()
+
+    def do_login(self):
+        self.login.go()
+        self.wait_until_is_here(self.login)
+
+        self.page.login(self.username, self.password)
+
+        self.wait_until(AnyCondition(
+            IsHereCondition(self.login_error),
+            IsHereCondition(self.dashboard),
+        ))
+
+        if self.login_error.is_here():
+            error = self.page.get_error()
+            if any((
+                'The User ID or Password is incorrect' in error,
+                'Both the User ID and Password are required' in error,
+            )):
+                raise BrowserIncorrectPassword(error)
+            if 'Your account has been locked' in error:
+                raise ActionNeeded(error)
+
+            assert False, 'Unhandled error : "%s"' % error
+
+
+class AmericanExpressBrowser(PagesBrowser, SubSeleniumMixin):
+    BASEURL = 'https://global.americanexpress.com'
 
     accounts = URL(r'/api/servicing/v1/member', AccountsPage)
-    js_balances = URL(r'/account-data/v1/financials/balances', JsonBalances)
-    js_balances2 = URL(r'/api/servicing/v1/financials/transaction_summary\?type=split_by_cardmember&statement_end_date=(?P<date>[\d-]+)', JsonBalances2)
-    js_pending = URL(r'/account-data/v1/financials/transactions\?limit=1000&offset=(?P<offset>\d+)&status=pending',
-                     JsonHistory)
-    js_posted = URL(r'/account-data/v1/financials/transactions\?limit=1000&offset=(?P<offset>\d+)&statement_end_date=(?P<end>[0-9-]+)&status=posted',
-                    JsonHistory)
-    js_periods = URL(r'/account-data/v1/financials/statement_periods', JsonPeriods)
+    json_balances = URL(r'/account-data/v1/financials/balances', JsonBalances)
+    json_balances2 = URL(r'/api/servicing/v1/financials/transaction_summary\?type=split_by_cardmember&statement_end_date=(?P<date>[\d-]+)', JsonBalances2)
+    json_pending = URL(
+        r'/account-data/v1/financials/transactions\?limit=1000&offset=(?P<offset>\d+)&status=pending',
+        JsonHistory
+    )
+    json_posted = URL(
+        r'/account-data/v1/financials/transactions\?limit=1000&offset=(?P<offset>\d+)&statement_end_date=(?P<end>[0-9-]+)&status=posted',
+        JsonHistory
+    )
+    json_periods = URL(r'/account-data/v1/financials/statement_periods', JsonPeriods)
     currency_page = URL(r'https://www.aexp-static.com/cdaas/axp-app/modules/axp-balance-summary/4.7.0/(?P<locale>\w\w-\w\w)/axp-balance-summary.json', CurrencyPage)
 
     no_card = URL(r'https://www.americanexpress.com/us/content/no-card/',
@@ -62,25 +113,13 @@ class AmericanExpressBrowser(LoginBrowser):
         'PRELEVEMENT AUTOMATIQUE ENREGISTRE-MERCI',
     ]
 
-    def __init__(self, *args, **kwargs):
-        super(AmericanExpressBrowser, self).__init__(*args, **kwargs)
-        self.cache = {}
+    SELENIUM_BROWSER = AmericanExpressLoginBrowser
 
-    def do_login(self):
-        self.login.go(data={
-            'request_type': 'login',
-            'UserID': self.username,
-            'Password': self.password,
-            'Logon': 'Logon',
-        })
-
-        if self.page.get_status_code() != 0:
-            if self.page.get_error_code() == 'LGON004':
-                # This error happens when the website needs that the user
-                # enter his card information and reset his password.
-                # There is no message returned when this error happens.
-                raise ActionNeeded()
-            raise BrowserIncorrectPassword()
+    def __init__(self, config, *args, **kwargs):
+        super(AmericanExpressBrowser, self).__init__(*args, **kwargs)
+        self.config = config
+        self.username = config['login'].get()
+        self.password = config['password'].get()
 
     @need_login
     def iter_accounts(self):
@@ -93,23 +132,23 @@ def iter_accounts(self):
         for account in account_list:
             try:
                 # for the main account
-                self.js_balances.go(headers={'account_tokens': account.id})
+                self.json_balances.go(headers={'account_tokens': account.id})
             except HTTPNotFound:
                 # for secondary accounts
-                self.js_periods.go(headers={'account_token': account._history_token})
+                self.json_periods.go(headers={'account_token': account._history_token})
                 periods = self.page.get_periods()
-                self.js_balances2.go(date=periods[1], headers={'account_tokens': account.id})
+                self.json_balances2.go(date=periods[1], headers={'account_tokens': account.id})
             self.page.fill_balances(obj=account)
             yield account
 
     @need_login
     def iter_history(self, account):
-        self.js_periods.go(headers={'account_token': account._history_token})
+        self.json_periods.go(headers={'account_token': account._history_token})
         periods = self.page.get_periods()
         today = datetime.date.today()
         # TODO handle pagination
         for p in periods:
-            self.js_posted.go(offset=0, end=p, headers={'account_token': account._history_token})
+            self.json_posted.go(offset=0, end=p, headers={'account_token': account._history_token})
             for tr in self.page.iter_history(periods=periods):
                 # As the website is very handy, passing account_token is not enough:
                 # it will return every transactions of each account, so we
@@ -124,17 +163,17 @@ def iter_coming(self, account):
         # ('Enregistrées' tab on the website)
 
         # "pending" have no vdate and debit date is in future
-        self.js_periods.go(headers={'account_token': account._history_token})
+        self.json_periods.go(headers={'account_token': account._history_token})
         periods = self.page.get_periods()
         date = parse_date(periods[0]).date()
         today = datetime.date.today()
         # when the latest period ends today we can't know the coming debit date
         if date != today:
             try:
-                self.js_pending.go(offset=0, headers={'account_token': account._history_token})
+                self.json_pending.go(offset=0, headers={'account_token': account._history_token})
             except ServerError as exc:
                 # At certain times of the month a connection might not have pendings;
-                # in that case, `js_pending.go` would throw a 502 error Bad Gateway
+                # in that case, `json_pending.go` would throw a 502 error Bad Gateway
                 error_code = exc.response.json().get('code')
                 error_message = exc.response.json().get('message')
                 self.logger.warning('No pendings page to access to, got error %s and message "%s" instead.', error_code, error_message)
@@ -146,7 +185,7 @@ def iter_coming(self, account):
 
         # "posted" have a vdate but debit date can be future or past
         for p in periods:
-            self.js_posted.go(offset=0, end=p, headers={'account_token': account._history_token})
+            self.json_posted.go(offset=0, end=p, headers={'account_token': account._history_token})
             for tr in self.page.iter_history(periods=periods):
                 if tr.date > today or not tr.date:
                     if tr._owner == account._idforJSON:

modules/americanexpress/module.py

@@ -35,13 +35,15 @@ class AmericanExpressModule(Module, CapBank):
     VERSION = '1.6'
     DESCRIPTION = u'American Express'
     LICENSE = 'LGPLv3+'
-    CONFIG = BackendConfig(ValueBackendPassword('login',    label='Code utilisateur', masked=False),
-                           ValueBackendPassword('password', label='Mot de passe'))
+    CONFIG = BackendConfig(
+        ValueBackendPassword('login', label='Code utilisateur', masked=False),
+        ValueBackendPassword('password', label='Mot de passe'),
+    )
+
     BROWSER = AmericanExpressBrowser
 
     def create_default_browser(self):
-        return self.create_browser(self.config['login'].get(),
-                                   self.config['password'].get())
+        return self.create_browser(self.config)
 
     def iter_accounts(self):
         return self.browser.iter_accounts()

modules/americanexpress/pages.py

@@ -21,14 +21,22 @@
 
 from decimal import Decimal
 
+from dateutil.parser import parse as parse_date
+from selenium.webdriver.common.keys import Keys
+
 from weboob.browser.pages import LoggedPage, JsonPage, HTMLPage
 from weboob.browser.elements import ItemElement, DictElement, method
-from weboob.browser.filters.standard import Date, Eval, Env, CleanText, Field, CleanDecimal, Format, Currency
+from weboob.browser.filters.standard import (
+    Date, Eval, Env, CleanText, Field, CleanDecimal, Format,
+    Currency,
+)
 from weboob.browser.filters.json import Dict
 from weboob.capabilities.bank import Account, Transaction
 from weboob.capabilities.base import NotAvailable
 from weboob.exceptions import ActionNeeded, BrowserUnavailable
-from dateutil.parser import parse as parse_date
+from weboob.browser.selenium import (
+    SeleniumPage, VisibleXPath, AllCondition, NotCondition,
+)
 
 
 def float_to_decimal(f):
@@ -56,15 +64,32 @@ def on_load(self):
         raise BrowserUnavailable(alert_header, alert_content)
 
 
-class LoginPage(JsonPage):
-    def get_status_code(self):
-        # - 0 = OK
-        # - 1 = Incorrect login/password
-        return CleanDecimal(Dict('statusCode'))(self.doc)
+class LoginErrorPage(SeleniumPage):
+    is_here = VisibleXPath('//div[@role="alert"]/div')
+
+    def get_error(self):
+        return CleanText('//div[@role="alert"]/div')(self.doc)
+
+
+class LoginPage(SeleniumPage):
+    is_here = AllCondition(
+        VisibleXPath('//input[contains(@id, "UserID")]'),
+        VisibleXPath('//input[contains(@id, "Password")]'),
+        VisibleXPath('//button[@id="loginSubmit"]'),
+        NotCondition(VisibleXPath('//div[@role="alert"]/div')),
+    )
+
+    def login(self, username, password):
+        el = self.driver.find_element_by_xpath('//input[contains(@id, "UserID")]')
+        el.send_keys(username)
+
+        el = self.driver.find_element_by_xpath('//input[contains(@id, "Password")]')
+        el.send_keys(password)
+        el.send_keys(Keys.RETURN)
+
 
-    def get_error_code(self):
-        # - LGON004 = ActionNeeded
-        return CleanText(Dict('errorCode'))(self.doc)
+class DashboardPage(LoggedPage, SeleniumPage):
+    pass
 
 
 class AccountsPage(LoggedPage, JsonPage):

modules/amundi/browser.py

@@ -26,7 +26,7 @@
 
 from .pages import (
     LoginPage, AccountsPage, AccountHistoryPage, AmundiInvestmentsPage, AllianzInvestmentPage,
-    EEInvestmentPage, EEInvestmentPerformancePage, EEInvestmentDetailPage, EEProductInvestmentPage,
+    EEInvestmentPage, InvestmentPerformancePage, InvestmentDetailPage, EEProductInvestmentPage,
     EresInvestmentPage, CprInvestmentPage, BNPInvestmentPage, BNPInvestmentApiPage, AxaInvestmentPage,
     EpsensInvestmentPage, EcofiInvestmentPage, SGGestionInvestmentPage, SGGestionPerformancePage,
 )
@@ -44,8 +44,8 @@ class AmundiBrowser(LoginBrowser):
     amundi_investments = URL(r'https://www.amundi.fr/fr_part/product/view', AmundiInvestmentsPage)
     # EEAmundi browser investments
     ee_investments = URL(r'https://www.amundi-ee.com/part/home_fp&partner=PACTEO_SYS', EEInvestmentPage)
-    ee_performance_details = URL(r'https://www.amundi-ee.com/psAmundiEEPart/ezjscore/call(.*)_tab_2', EEInvestmentPerformancePage)
-    ee_investment_details = URL(r'https://www.amundi-ee.com/psAmundiEEPart/ezjscore/call(.*)_tab_5', EEInvestmentDetailPage)
+    performance_details = URL(r'https://(.*)/ezjscore/call(.*)_tab_2', InvestmentPerformancePage)
+    investment_details = URL(r'https://(.*)/ezjscore/call(.*)_tab_5', InvestmentDetailPage)
     # EEAmundi product investments
     ee_product_investments = URL(r'https://www.amundi-ee.com/product', EEProductInvestmentPage)
     # Allianz GI investments
@@ -143,8 +143,7 @@ def fill_investment_details(self, inv):
             return inv
 
         # Pages with only asset category available
-        if (self.amundi_investments.is_here() or
-            self.allianz_investments.is_here() or
+        if (self.allianz_investments.is_here() or
             self.axa_investments.is_here()):
             inv.asset_category = self.page.get_asset_category()
             inv.recommended_period = NotAvailable
@@ -158,17 +157,20 @@ def fill_investment_details(self, inv):
             self.page.fill_investment(obj=inv)
 
         # Particular cases
-        elif self.ee_investments.is_here():
-            inv.recommended_period = self.page.get_recommended_period()
+        elif (self.ee_investments.is_here() or
+              self.amundi_investments.is_here()):
+            if self.ee_investments.is_here():
+                inv.recommended_period = self.page.get_recommended_period()
             details_url = self.page.get_details_url()
             performance_url = self.page.get_performance_url()
             if details_url:
                 self.location(details_url)
-                if self.ee_investment_details.is_here():
+                if self.investment_details.is_here():
+                    inv.recommended_period = inv.recommended_period or self.page.get_recommended_period()
                     inv.asset_category = self.page.get_asset_category()
             if performance_url:
                 self.location(performance_url)
-                if self.ee_performance_details.is_here():
+                if self.performance_details.is_here():
                     # The investments JSON only contains 1 & 5 years performances
                     # If we can access EEInvestmentPerformancePage, we can fetch all three
                     # values (1, 3 and 5 years), in addition the values are more accurate here.
@@ -199,6 +201,20 @@ def fill_investment_details(self, inv):
 
         return inv
 
+    @need_login
+    def iter_pockets(self, account):
+        if account.balance == 0:
+            self.logger.info('Account %s has a null balance, no pocket available.', account.label)
+            return
+
+        headers = {'X-noee-authorization': 'noeprd %s' % self.token}
+        self.accounts.go(headers=headers)
+        for investment in self.page.iter_investments(account_id=account.id):
+            for pocket in investment._pockets:
+                pocket.investment = investment
+                pocket.label = investment.label
+                yield pocket
+
     @need_login
     def iter_history(self, account):
         headers = {'X-noee-authorization': 'noeprd %s' % self.token}

modules/amundi/module.py

@@ -18,7 +18,7 @@
 # along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
 
 
-from weboob.capabilities.bank import CapBankWealth
+from weboob.capabilities.bank import CapBankPockets
 from weboob.tools.backend import Module, BackendConfig
 from weboob.tools.value import ValueBackendPassword, Value
 
@@ -27,7 +27,7 @@
 __all__ = ['AmundiModule']
 
 
-class AmundiModule(Module, CapBankWealth):
+class AmundiModule(Module, CapBankPockets):
     NAME = 'amundi'
     DESCRIPTION = u'Amundi'
     MAINTAINER = u'James GALT'
@@ -61,5 +61,8 @@ def iter_investment(self, account):
             if inv.valuation != 0:
                 yield inv
 
+    def iter_pocket(self, account):
+        return self.browser.iter_pockets(account)
+
     def iter_history(self, account):
         return self.browser.iter_history(account)

modules/amundi/pages.py

@@ -25,17 +25,23 @@
 from weboob.browser.elements import ItemElement, method, DictElement
 from weboob.browser.filters.standard import (
     CleanDecimal, Date, Field, CleanText,
-    Env, Eval, Map, Regexp, Title,
+    Env, Eval, Map, Regexp, Title, Format,
 )
 from weboob.browser.filters.html import Attr
 from weboob.browser.filters.json import Dict
 from weboob.browser.pages import LoggedPage, JsonPage, HTMLPage
-from weboob.capabilities.bank import Account, Investment, Transaction
-from weboob.capabilities.base import NotAvailable
+from weboob.capabilities.bank import Account, Investment, Transaction, Pocket
+from weboob.capabilities.base import NotAvailable, empty
 from weboob.exceptions import NoAccountsException
 from weboob.tools.capabilities.bank.investments import IsinCode, IsinType
 
 
+def percent_to_ratio(value):
+    if empty(value):
+        return NotAvailable
+    return value / 100
+
+
 class LoginPage(JsonPage):
     def get_token(self):
         return Dict('token')(self.doc)
@@ -45,8 +51,9 @@ def get_token(self):
     'PEE': Account.TYPE_PEE,
     'PEG': Account.TYPE_PEE,
     'PEI': Account.TYPE_PEE,
-    'PERCO': Account.TYPE_PERCO,
-    'PERCOI': Account.TYPE_PERCO,
+    'PERCO': Account.TYPE_PER,
+    'PERCOI': Account.TYPE_PER,
+    'PER': Account.TYPE_PER,
     'RSP': Account.TYPE_RSP,
     'ART 83': Account.TYPE_ARTICLE_83,
 }
@@ -71,14 +78,13 @@ class item(ItemElement):
 
             obj_id = CleanText(Dict('codeDispositif'))
             obj_balance = CleanDecimal(Dict('mtBrut'))
+            obj_currency = 'EUR'
+            obj_type = Map(Dict('typeDispositif'), ACCOUNT_TYPES, Account.TYPE_LIFE_INSURANCE)
 
             def obj_number(self):
                 # just the id is a kind of company id so it can be unique on a backend but not unique on multiple backends
                 return '%s_%s' % (Field('id')(self), self.page.browser.username)
 
-            obj_currency = 'EUR'
-            obj_type = Map(Dict('typeDispositif'), ACCOUNT_TYPES, Account.TYPE_LIFE_INSURANCE)
-
             def obj_label(self):
                 try:
                     return Dict('libelleDispositif')(self).encode('iso-8859-2').decode('utf8')
@@ -114,6 +120,7 @@ def condition(self):
             obj__details_url = Dict('urlFicheFonds', default=None)
             obj_code = IsinCode(Dict('codeIsin', default=NotAvailable), default=NotAvailable)
             obj_code_type = IsinType(Dict('codeIsin', default=NotAvailable))
+            obj_diff = CleanDecimal.SI(Dict('mtPMV', default=None), default=NotAvailable)
 
             def obj_srri(self):
                 srri = Dict('SRRI')(self)
@@ -132,6 +139,29 @@ def obj_performance_history(self):
                     perfs[5] = Eval(lambda x: x / 100, CleanDecimal(Dict('performanceCinqAns')))(self)
                 return perfs
 
+            # Fetch pockets for each investment:
+            class obj__pockets(DictElement):
+                item_xpath = 'positionSalarieFondsEchDto'
+
+                class item(ItemElement):
+                    klass = Pocket
+
+                    obj_condition = Env('condition')
+                    obj_availability_date = Env('availability_date')
+                    obj_amount = CleanDecimal.SI(Dict('mtBrut'))
+                    obj_quantity = CleanDecimal.SI(Dict('nbParts'))
+
+                    def parse(self, obj):
+                        availability_date = datetime.strptime(obj['dtEcheance'].split('T')[0], '%Y-%m-%d')
+                        if availability_date <= datetime.today():
+                            # In the past, already available
+                            self.env['availability_date'] = availability_date
+                            self.env['condition'] = Pocket.CONDITION_AVAILABLE
+                        else:
+                            # In the future, but we have no information on condition
+                            self.env['availability_date'] = availability_date
+                            self.env['condition'] = Pocket.CONDITION_UNKNOWN
+
 
 class AccountHistoryPage(LoggedPage, JsonPage):
     def belongs(self, instructions, account):
@@ -169,15 +199,19 @@ def iter_history(self, account):
 
 
 class AmundiInvestmentsPage(LoggedPage, HTMLPage):
-    def get_asset_category(self):
-        # Descriptions are like 'Fonds d'Investissement - (ISIN: FR001018 - Action'
-        # Fetch the last words of the description (e.g. 'Action' or 'Diversifié')
-        return Regexp(
-            CleanText('//div[@class="amundi-fund-legend"]//strong'),
-            r' ([^-]+)$',
-            default=NotAvailable
+    def get_tab_url(self, tab_id):
+        return Format(
+            '%s%d',
+            Regexp(CleanText('//script[contains(text(), "Product.init")]'), r'init\(.*?,"(.*?tab_)\d"', default=None),
+            tab_id
         )(self.doc)
 
+    def get_details_url(self):
+        return self.get_tab_url(5)
+
+    def get_performance_url(self):
+        return self.get_tab_url(2)
+
 
 class EEInvestmentPage(LoggedPage, HTMLPage):
     def get_recommended_period(self):
@@ -190,19 +224,29 @@ def get_performance_url(self):
         return Attr('//a[contains(text(), "Performances")]', 'data-href', default=None)(self.doc)
 
 
-class EEInvestmentPerformancePage(LoggedPage, HTMLPage):
+class InvestmentPerformancePage(LoggedPage, HTMLPage):
     def get_performance_history(self):
+        # The positions of the columns depend on the age of the investment fund.
+        # For example, if the fund is younger than 5 years, there will be not '5 ans' column.
+        durations = [CleanText('.')(el) for el in self.doc.xpath('//div[h2[contains(text(), "Performances glissantes")]]//th')]
+        values = [CleanText('.')(el) for el in self.doc.xpath('//div[h2[contains(text(), "Performances glissantes")]]//tr[td[text()="Fonds"]]//td')]
+        matches = dict(zip(durations, values))
+        # We do not fill the performance dictionary if no performance is available,
+        # otherwise it will overwrite the data obtained from the JSON with empty values.
         perfs = {}
-        if CleanDecimal.French('//tr[td[text()="Fonds"]]//td[position()=last()-2]', default=None)(self.doc):
-            perfs[1] = Eval(lambda x: x / 100, CleanDecimal.French('//tr[td[text()="Fonds"]]//td[position()=last()-2]'))(self.doc)
-        if CleanDecimal.French('//tr[td[text()="Fonds"]]//td[position()=last()-1]', default=None)(self.doc):
-            perfs[3] = Eval(lambda x: x / 100, CleanDecimal.French('//tr[td[text()="Fonds"]]//td[position()=last()-1]'))(self.doc)
-        if CleanDecimal.French('//tr[td[text()="Fonds"]]//td[position()=last()]', default=None)(self.doc):
-            perfs[5] = Eval(lambda x: x / 100, CleanDecimal.French('//tr[td[text()="Fonds"]]//td[position()=last()]'))(self.doc)
+        if matches.get('1 an'):
+            perfs[1] = percent_to_ratio(CleanDecimal.French(default=NotAvailable).filter(matches['1 an']))
+        if matches.get('3 ans'):
+            perfs[3] = percent_to_ratio(CleanDecimal.French(default=NotAvailable).filter(matches['3 ans']))
+        if matches.get('5 ans'):
+            perfs[5] = percent_to_ratio(CleanDecimal.French(default=NotAvailable).filter(matches['5 ans']))
         return perfs
 
 
-class EEInvestmentDetailPage(LoggedPage, HTMLPage):
+class InvestmentDetailPage(LoggedPage, HTMLPage):
+    def get_recommended_period(self):
+        return Title(CleanText('//label[contains(text(), "Durée minimum de placement")]/following-sibling::span', default=NotAvailable))(self.doc)
+
     def get_asset_category(self):
         return CleanText('//label[contains(text(), "Classe d\'actifs")]/following-sibling::span', default=NotAvailable)(self.doc)
 
@@ -246,8 +290,9 @@ class CprInvestmentPage(LoggedPage, HTMLPage):
     @method
     class fill_investment(ItemElement):
         obj_srri = CleanText('//span[@class="active"]', default=NotAvailable)
-        obj_asset_category = CleanText('//div[contains(text(), "Classe d\'actifs")]//strong', default=NotAvailable)
-        obj_recommended_period = CleanText('//div[contains(text(), "Durée recommandée")]//strong', default=NotAvailable)
+        # Text headers can be in French or in English
+        obj_asset_category = Title('//div[contains(text(), "Classe d\'actifs") or contains(text(), "Asset class")]//strong', default=NotAvailable)
+        obj_recommended_period = Title('//div[contains(text(), "Durée recommandée") or contains(text(), "Recommended duration")]//strong', default=NotAvailable)
 
 
 class BNPInvestmentPage(LoggedPage, HTMLPage):
@@ -300,5 +345,5 @@ def get_performance_url(self):
         return Attr('(//li[@role="presentation"])[1]//a', 'data-href', default=None)(self.doc)
 
 
-class SGGestionPerformancePage(EEInvestmentPerformancePage):
+class SGGestionPerformancePage(InvestmentPerformancePage):
     pass

modules/asana/browser.py

@@ -59,19 +59,18 @@ def open(self, *args, **kwargs):
             raise
 
     def _make_user(self, data):
-        u = User(data['id'], None)
+        u = User(data['gid'], None)
         if 'name' in data:
             u.name = data['name']
         return u
 
     def _make_project(self, data):
-        p = Project(str(data['id']), data['name'])
+        p = Project(str(data['gid']), data['name'])
         p.url = 'https://app.asana.com/0/%s' % p.id
         if 'members' in data:
             p.members = [self._make_user(u) for u in data['members']]
 
         p.statuses = [self.STATUS_OPEN, self.STATUS_CLOSED]
-        p._workspace = data['workspace']['id']
 
         # these fields don't exist in asana
         p.priorities = []
@@ -83,7 +82,7 @@ def _make_issue(self, data):
             # section, not task
             return None
 
-        i = Issue(str(data['id']))
+        i = Issue(str(data['gid']))
         i.url = 'https://app.asana.com/0/0/%s/f' % i.id
         i.title = data['name']
         if 'notes' in data:

modules/asana/module.py

@@ -44,7 +44,7 @@ class AsanaModule(Module, CapBugTracker):
     def create_default_browser(self):
         return self.create_browser(self.config['token'].get())
 
-    ## read-only issues and projects
+    #  read-only issues and projects
     def iter_issues(self, query):
         query = query.copy()
 
@@ -78,7 +78,7 @@ def iter_issues(self, query):
                 params['completed'] = 'true'
             else:
                 params['completed'] = 'false'
-                params['completed_since'] = 'now' # completed=false is not enough...
+                params['completed_since'] = 'now'  # completed=false is not enough...
 
         if not query.project:
             workspaces = list(self._iter_workspaces())
@@ -162,7 +162,7 @@ def get_project(self, id):
     def _iter_workspaces(self):
         return (d['id'] for d in self.browser.paginate('workspaces'))
 
-    ## writing issues
+    #  writing issues
     def create_issue(self, project):
         issue = Issue(0)
         issue._project = project
@@ -213,7 +213,7 @@ def update_issue(self, issue, update):
     def remove_issue(self, issue):
         self.browser.request('tasks/%s' % issue.id, method='DELETE')
 
-    ## filling
+    #  filling
     def fill_project(self, project, fields):
         if set(['members']) & set(fields):
             return self.get_project(project.id)

modules/axabanque/browser.py

@@ -37,15 +37,16 @@
 
 from .pages.login import (
     KeyboardPage, LoginPage, ChangepasswordPage, PredisconnectedPage, DeniedPage,
-    AccountSpaceLogin, ErrorPage,
+    AccountSpaceLogin, ErrorPage, AuthorizePage,
 )
 from .pages.bank import (
     AccountsPage as BankAccountsPage, CBTransactionsPage, TransactionsPage,
     UnavailablePage, IbanPage, LifeInsuranceIframe, BoursePage, BankProfilePage,
 )
 from .pages.wealth import (
-    AccountsPage as WealthAccountsPage, AccountDetailsPage,
-    InvestmentPage, HistoryPage, HistoryInvestmentsPage, ProfilePage,
+    AccountsPage as WealthAccountsPage, AccountDetailsPage, InvestmentPage,
+    InvestmentMonAxaPage, HistoryPage, HistoryInvestmentsPage, ProfilePage,
+    PerformanceMonAxaPage,
 )
 from .pages.transfer import (
     RecipientsPage, AddRecipientPage, ValidateTransferPage, RegisterTransferPage,
@@ -64,7 +65,7 @@ class AXABrowser(LoginBrowser):
         r'https://www.axa.fr/axa-postmaw-predisconnect.html',
         PredisconnectedPage
     )
-
+    authorize = URL(r'https://connect.axa.fr/connect/authorize', AuthorizePage)
     denied = URL(r'https://connect.axa.fr/Account/AccessDenied', DeniedPage)
     account_space_login = URL(r'https://connect.axa.fr/api/accountspace', AccountSpaceLogin)
     errors = URL(
@@ -598,7 +599,10 @@ class AXAAssurance(AXABrowser):
         r'/#',
         AccountDetailsPage
     )
+
     investment = URL(r'/content/ecc-popin-cards/savings/[^/]+/repartition', InvestmentPage)
+    investment_monaxa = URL(r'https://monaxaweb-gp.axa.fr/MonAxa/Contrat/', InvestmentMonAxaPage)
+    performance_monaxa = URL(r'https://monaxaweb-gp.axa.fr/MonAxa/ContratPerformance/', PerformanceMonAxaPage)
 
     documents_life_insurance = URL(
         r'/content/espace-client/accueil/mes-documents/situations-de-contrats-assurance-vie.content-inner.din_SAVINGS_STATEMENT.html',
@@ -625,8 +629,6 @@ class AXAAssurance(AXABrowser):
 
     def __init__(self, *args, **kwargs):
         super(AXAAssurance, self).__init__(*args, **kwargs)
-        self.cache = {}
-        self.cache['invs'] = {}
 
     def go_wealth_pages(self, account):
         self.location('/' + account.url)
@@ -634,38 +636,59 @@ def go_wealth_pages(self, account):
 
     @need_login
     def iter_accounts(self):
-        if 'accs' not in self.cache.keys():
-            self.accounts.go()
-            self.cache['accs'] = list(self.page.iter_accounts())
-        return self.cache['accs']
+        self.accounts.go()
+        return self.page.iter_accounts()
+
+    @need_login
+    def iter_investment_espaceclient(self, account):
+        invests = []
+        portfolio_page = self.page
+        detailed_view = self.page.detailed_view()
+        if detailed_view:
+            self.location(detailed_view)
+            invests.extend(self.page.iter_investment(currency=account.currency))
+        for inv in portfolio_page.iter_investment(currency=account.currency):
+            i = [i2 for i2 in invests if
+                 (i2.valuation == inv.valuation and i2.label == inv.label)]
+            assert len(i) in (0, 1)
+            if i:
+                i[0].portfolio_share = inv.portfolio_share
+            else:
+                invests.append(inv)
+        return invests
+
+    @need_login
+    def iter_investment_monaxa(self, account):
+        # Try to fetch a URL to 'monaxaweb-gp.axa.fr'
+        invests = list(self.page.iter_investment())
+
+        performance_url = self.page.get_performance_url()
+        if performance_url:
+            self.location(performance_url)
+            for inv in invests:
+                self.page.fill_investment(obj=inv)
+
+        # return to espaceclient.axa.fr
+        self.accounts.go()
+        return invests
 
     @need_login
     def iter_investment(self, account):
-        if account.id not in self.cache['invs']:
-            self.go_wealth_pages(account)
-            investment_url = self.page.get_investment_url()
-            if not investment_url:
-                # fake data, don't cache it
-                self.logger.warning('No investment URL available for account %s, investments cannot be retrieved.', account.id)
-                return []
+        self.go_wealth_pages(account)
 
+        investment_url = self.page.get_investment_url()
+        if investment_url:
             self.location(investment_url)
-            portfolio_page = self.page
-            detailed_view = self.page.detailed_view()
-            if detailed_view:
-                self.location(detailed_view)
-                self.cache['invs'][account.id] = list(self.page.iter_investment(currency=account.currency))
-            else:
-                self.cache['invs'][account.id] = []
-            for inv in portfolio_page.iter_investment(currency=account.currency):
-                i = [i2 for i2 in self.cache['invs'][account.id] if (i2.valuation == inv.valuation and i2.label == inv.label)]
-                assert len(i) in (0, 1)
-                if i:
-                    i[0].portfolio_share = inv.portfolio_share
-                else:
-                    self.cache['invs'][account.id].append(inv)
+            return self.iter_investment_espaceclient(account)
 
-        return self.cache['invs'][account.id]
+        iframe_url = self.page.get_iframe_url()
+        if iframe_url:
+            self.location(iframe_url)
+            return self.iter_investment_monaxa(account)
+
+        # No data available for this account.
+        self.logger.warning('No investment URL available for account %s, investments cannot be retrieved.', account.id)
+        return []
 
     @need_login
     def iter_history(self, account):

modules/axabanque/pages/login.py

@@ -131,3 +131,9 @@ def on_load(self):
         for error in error_msg:
             if error:
                 raise BrowserUnavailable(error)
+
+
+class AuthorizePage(HTMLPage):
+    def on_load(self):
+        form = self.get_form()
+        form.submit()

modules/axabanque/pages/wealth.py

@@ -33,6 +33,7 @@
 from weboob.capabilities.bank import Account, Investment, AccountOwnership
 from weboob.capabilities.profile import Person
 from weboob.capabilities.base import NotAvailable, NotLoaded, empty
+from weboob.tools.capabilities.bank.investments import IsinCode, IsinType
 from weboob.tools.capabilities.bank.transactions import FrenchTransaction
 
 
@@ -55,6 +56,7 @@ class item(ItemElement):
                 'perp': Account.TYPE_PERP,
                 'epargne retraite agipi pair': Account.TYPE_PERP,
                 'epargne retraite agipi far': Account.TYPE_MADELIN,
+                'epargne retraite ma retraite': Account.TYPE_PER,
                 'novial avenir': Account.TYPE_MADELIN,
                 'epargne retraite novial': Account.TYPE_LIFE_INSURANCE,
             }
@@ -152,6 +154,44 @@ def is_detail(self):
         return bool(self.doc.xpath('//th[contains(text(), "Valeur de la part")]'))
 
 
+class InvestmentMonAxaPage(LoggedPage, HTMLPage):
+    def get_performance_url(self):
+        return Link('//a[contains(text(), "Performance")]', default=None)(self.doc)
+
+    @method
+    class iter_investment(TableElement):
+        item_xpath = '//div[@id="tabVisionContrat"]/table/tbody/tr'
+        head_xpath = '//div[@id="tabVisionContrat"]/table/thead//th'
+
+        col_label = 'Nom'
+        col_code = 'ISIN'
+        col_asset_category = 'Catégorie'
+        col_valuation = 'Montant'
+        col_portfolio_share = 'Poids'
+
+        class item(ItemElement):
+            klass = Investment
+
+            obj_label = CleanText(TableCell('label'))
+            obj_code = IsinCode(TableCell('code'), default=NotAvailable)
+            obj_code_type = IsinType(TableCell('code'), default=NotAvailable)
+            obj_asset_category = CleanText(TableCell('asset_category'))
+            obj_valuation = CleanDecimal.French(TableCell('valuation'), default=NotAvailable)
+
+            def obj_portfolio_share(self):
+                share_percent = CleanDecimal.French(TableCell('portfolio_share'), default=None)(self)
+                if not empty(share_percent):
+                    return share_percent / 100
+                return NotAvailable
+
+
+class PerformanceMonAxaPage(LoggedPage, HTMLPage):
+    @method
+    class fill_investment(ItemElement):
+        obj_vdate = Date(CleanText('//span[@id="cellDateValorisation"]'), dayfirst=True, default=NotAvailable)
+        # TODO Other values (like `quantity`) may be available. They are not available for the account we have.
+
+
 class Transaction(FrenchTransaction):
     PATTERNS = [
         (re.compile(r'^(?P<text>souscription.*)'), FrenchTransaction.TYPE_DEPOSIT),
@@ -166,6 +206,9 @@ def get_account_url(self, url):
     def get_investment_url(self):
         return Attr('//div[contains(@data-analytics-label, "repartition_par_fond")]', 'data-url', default=None)(self.doc)
 
+    def get_iframe_url(self):
+        return Attr('//div[contains(@class, "iframe-quantalys")]', 'data-module-iframe-quantalys--iframe-url', default=None)(self.doc)
+
     def get_pid(self):
         return Attr('//div[@data-module="operations-movements"]', 'data-module-operations-movements--pid', default=None)(self.doc)
 

modules/becm/browser.py

@@ -23,7 +23,6 @@
 from weboob.browser.profiles import Wget
 from weboob.browser.url import URL
 from weboob.browser.browsers import need_login
-from weboob.exceptions import ActionNeeded, AuthMethodNotImplemented
 
 from .pages import AdvisorPage, LoginPage
 
@@ -40,23 +39,6 @@ class BECMBrowser(AbstractBrowser):
     login = URL('/fr/authentification.html', LoginPage)
     advisor = URL('/fr/banques/Details.aspx\?banque=.*', AdvisorPage)
 
-    def do_login(self):
-        # Clear cookies.
-        self.do_logout()
-        self.login.go()
-        if not self.page.logged:
-            self.page.login(self.username, self.password)
-            # Many "Credit Mutuel" customers tried to add their connection to BECM, but the BECM
-            # website does not return any error when you try to login with correct Crédit Mutuel
-            # credentials, therefore we must suggest them to try regular Crédit Mutuel if login fails.
-            if self.login.is_here():
-                raise ActionNeeded("La connexion au site de BECM n'a pas fonctionné avec les identifiants fournis.\
-                                    Si vous êtes client du Crédit Mutuel, veuillez réessayer en sélectionnant le module Crédit Mutuel.")
-
-        if self.verify_pass.is_here():
-            raise AuthMethodNotImplemented("L'identification renforcée avec la carte n'est pas supportée.")
-
-
     @need_login
     def get_advisor(self):
         advisor = None

modules/becm/module.py

@@ -20,8 +20,7 @@
 
 from weboob.capabilities.bank import CapBankTransferAddRecipient
 from weboob.capabilities.contact import CapContact
-from weboob.tools.backend import AbstractModule, BackendConfig
-from weboob.tools.value import ValueBackendPassword
+from weboob.tools.backend import AbstractModule
 
 from .browser import BECMBrowser
 
@@ -36,12 +35,11 @@ class BECMModule(AbstractModule, CapBankTransferAddRecipient, CapContact):
     VERSION = '1.6'
     DESCRIPTION = u'Banque Européenne Crédit Mutuel'
     LICENSE = 'LGPLv3+'
-    CONFIG = BackendConfig(ValueBackendPassword('login',    label='Identifiant', masked=False),
-                           ValueBackendPassword('password', label='Mot de passe'))
+
     BROWSER = BECMBrowser
     PARENT = 'creditmutuel'
 
     def create_default_browser(self):
-        browser = self.create_browser(self.config['login'].get(), self.config['password'].get(), weboob=self.weboob)
+        browser = self.create_browser(self.config, weboob=self.weboob)
         browser.new_accounts.urls.insert(0, "/mabanque/fr/banque/comptes-et-contrats.html")
         return browser

modules/bforbank/browser.py

@@ -18,7 +18,7 @@
 # along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
 import datetime
 from dateutil.relativedelta import relativedelta
-from weboob.exceptions import BrowserIncorrectPassword
+from weboob.exceptions import BrowserIncorrectPassword, ActionNeeded
 from weboob.browser import LoginBrowser, URL, need_login
 from weboob.capabilities.bank import Account, AccountNotFound
 from weboob.capabilities.base import empty
@@ -90,8 +90,18 @@ def do_login(self):
         self.login.stay_or_go()
         assert self.login.is_here()
         self.page.login(self.birthdate, self.username, self.password)
-        if self.error.is_here():
+        # When we try to login, the server return a json, if no error occurred
+        # `error` will be None otherwise it will be filled with the content of
+        # the error.
+        error = self.page.get_error_message()
+        if error == 'error.compte.bloque':
+            raise ActionNeeded('Compte bloqué')
+        elif error == 'error.authentification':
             raise BrowserIncorrectPassword()
+        elif error is not None:
+            assert False, 'Unexpected error at login: "%s"' % error
+        # We must go home after login otherwise do_login will be done twice.
+        self.home.go()
 
     @need_login
     def iter_accounts(self):

modules/bforbank/pages.py

@@ -27,7 +27,7 @@
 from PIL import Image
 
 from weboob.exceptions import ActionNeeded
-from weboob.browser.pages import LoggedPage, HTMLPage, pagination, AbstractPage
+from weboob.browser.pages import LoggedPage, HTMLPage, pagination, AbstractPage, JsonPage
 from weboob.browser.elements import method, ListElement, ItemElement, TableElement
 from weboob.capabilities.bank import Account, AccountOwnership
 from weboob.capabilities.profile import Person
@@ -99,8 +99,9 @@ def login(self, birthdate, username, password):
         form.submit()
 
 
-class ErrorPage(HTMLPage):
-    pass
+class ErrorPage(JsonPage):
+    def get_error_message(self):
+        return self.doc.get('errorMessage', None)
 
 
 class UserValidationPage(HTMLPage):

modules/bibliothequesparis/browser.py

@@ -64,7 +64,6 @@ def do_renew(self, id):
             'userUniqueIdentifier': '',
         }
         self.renew.go(json=post, headers=self.json_headers)
-        self.page.check_error()
 
     def search_books(self, pattern):
         max_page = 0

modules/bibliothequesparis/pages.py

@@ -21,12 +21,9 @@
 
 from datetime import datetime
 
-from weboob.browser.pages import HTMLPage, JsonPage, LoggedPage
-from weboob.browser.elements import ListElement, ItemElement, method, DictElement
-from weboob.browser.filters.standard import (
-    CleanText, Date, Regexp, Field,
-)
-from weboob.browser.filters.html import Link
+from weboob.browser.pages import JsonPage, LoggedPage
+from weboob.browser.elements import ItemElement, method, DictElement
+from weboob.browser.filters.standard import Regexp
 from weboob.browser.filters.json import Dict
 from weboob.capabilities.base import UserError
 from weboob.capabilities.library import Book
@@ -74,44 +71,10 @@ def obj_date(self):
                 return datetime.fromtimestamp(int(Regexp(Dict('WhenBack'), r'\((\d+)000')(self)) - 3600).date()
 
             obj_location = Dict('Location')
-            #obj_author = Regexp(CleanText('.//div[@class="loan-custom-result"]//p[@class="template-info"]'), '^(.*?) - ')
 
             def obj__renew_data(self):
                 return self.el
 
-    def x__init__(self, browser, response, *args, **kwargs):
-        super(LoansPage, self).__init__(browser, response, *args, **kwargs)
-        self.sub = self.sub_class(browser, response, data=self.sub_data)
-
-    @property
-    def sub_data(self):
-        if isinstance(self.doc['d'], dict):
-            return b''
-        return self.doc['d'].encode('utf-8')
-
-    class sub_class(HTMLPage):
-        data = None
-
-        def __init__(self, browser, response, data):
-            self.data = data
-            super(LoansPage.sub_class, self).__init__(browser, response)
-
-        @method
-        class get_loans(ListElement):
-            item_xpath = '//div[@id="loans-box"]//li[has-class("loan-item")]'
-
-            class item(ItemElement):
-                klass = Book
-
-                obj_url = Link('.//div[@class="loan-custom-result"]/a')
-                obj_id = Regexp(Field('url'), r'/SYRACUSE/(\d+)/')
-                obj_name = CleanText('.//h3[has-class("title")]')
-                # warning: date span may also contain "(à rendre bientôt)" along with date
-                obj_date = Date(Regexp(CleanText('.//li[has-class("dateretour")]/span[@class="loan-info-value"]'), r'(\d+/\d+/\d+)'), dayfirst=True)
-                obj_location = CleanText('.//li[has-class("localisation")]//span[@class="loan-info-value"]')
-                obj_author = Regexp(CleanText('.//div[@class="loan-custom-result"]//p[@class="template-info"]'), '^(.*?) - ')
-                obj__renew_data = CleanText('.//span[has-class("loan-data")]')
-
 
 class RenewPage(LoggedPage, JsonMixin):
     pass

modules/bnpcards/phenix/browser.py

@@ -19,6 +19,7 @@
 
 
 from weboob.exceptions import BrowserIncorrectPassword, BrowserPasswordExpired
+from weboob.browser.exceptions import ClientError
 from weboob.browser import LoginBrowser, URL, need_login
 
 from .pages import (
@@ -50,9 +51,16 @@ def __init__(self, website, *args, **kwargs):
 
     def do_login(self):
         self.login_cas.go()
-        self.page.login(self.username, self.password)
-        if not(self.page.is_logged()):
+        try:
+            self.page.login(self.username, self.password)
+        except ClientError as e:
+            if e.response.status_code == 401:
+                raise BrowserIncorrectPassword()
+            raise
+
+        if not self.page.is_logged():
             raise BrowserIncorrectPassword(self.page.get_error_message())
+
         self.dashboard.go()
         if self.password_expired.is_here():
             raise BrowserPasswordExpired(self.page.get_error_message())

modules/bnpcards/phenix/pages.py

@@ -132,6 +132,7 @@ class item(ItemElement):
             TRANSACTION_TYPES = {
                 'FACTURE CB': Transaction.TYPE_CARD,
                 'RETRAIT CB': Transaction.TYPE_WITHDRAWAL,
+                "TRANSACTION INITIALE RECUE D'AVOIR": Transaction.TYPE_PAYBACK,
             }
 
             obj_label = CleanText(Dict('Raison sociale commerçant'))