modules/afer/browser.py

 # -*- coding: utf-8 -*-
 
-# Copyright(C) 2015      James GALT
+# Copyright(C) 2012-2019  Budget Insight
 #
 # This file is part of a weboob module.
 #
@@ -19,62 +19,15 @@
 
 from __future__ import unicode_literals
 
-from random import randint
-from weboob.browser.browsers import URL, LoginBrowser, need_login
-from .compat.weboob_exceptions import BrowserIncorrectPassword, BrowserUnavailable, BrowserPasswordExpired
-from weboob.tools.compat import basestring
 
-from .pages import (
-    LoginPage, IndexPage, WrongPasswordPage, WrongWebsitePage,
-    AccountDetailPage, AccountHistoryPage, MigrationPage,
-)
+from weboob.browser import AbstractBrowser
 
 
-class AferBrowser(LoginBrowser):
+class AferBrowser(AbstractBrowser):
+    PARENT = 'aviva'
+    PARENT_ATTR = 'package.browser.AvivaBrowser'
     BASEURL = 'https://adherent.gie-afer.fr'
 
-    login = URL(r'/espaceadherent/MonCompte/Connexion$', LoginPage)
-    wrong_password = URL(r'/espaceadherent/MonCompte/Connexion\?err=6001', WrongPasswordPage)
-    wrong_website = URL(r'/espaceadherent/MonCompte/Connexion\?err=6008', WrongWebsitePage)
-    migration = URL(r'/espaceadherent/MonCompte/Migration', MigrationPage)
-    index = URL('/web/ega.nsf/listeAdhesions\?OpenForm', IndexPage)
-    account_detail = URL('/web/ega.nsf/soldeEpargne\?openForm', AccountDetailPage)
-    account_history = URL('/web/ega.nsf/generationSearchModule\?OpenAgent', AccountHistoryPage)
-    history_detail = URL('/web/ega.nsf/WOpendetailOperation\?OpenAgent', AccountHistoryPage)
-
-    def do_login(self):
-        assert isinstance(self.username, basestring)
-        assert isinstance(self.password, basestring)
-        self.login.go()
-
-        try:
-            self.page.login(self.username, self.password)
-        except BrowserUnavailable:
-            raise BrowserIncorrectPassword()
-
-        if self.migration.is_here():
-            raise BrowserPasswordExpired(self.page.get_error())
-
-        if self.wrong_password.is_here():
-            error = self.page.get_error()
-            if error:
-                raise BrowserIncorrectPassword(error)
-            assert False, 'We landed on WrongPasswordPage but no error message was fetched.'
-
-
-    @need_login
-    def iter_accounts(self):
-        self.index.stay_or_go()
-        return self.page.iter_accounts()
-
-    @need_login
-    def iter_investment(self, account):
-        self.account_detail.go(params={'nads': account.id})
-        return self.page.iter_investment()
-
-    @need_login
-    def iter_history(self, account):
-        al = randint(0, 1000)
-        data = {'cdeAdh': account.id, 'al': al, 'page': 1, 'form': 'F'}
-        self.account_history.go(data={'cdeAdh': account.id, 'al': al, 'page': 1, 'form': 'F'})
-        return self.page.iter_history(data=data)
+    def __init__(self, *args, **kwargs):
+        self.subsite = 'espaceadherent'
+        super(AferBrowser, self).__init__(*args, **kwargs)

modules/afer/module.py

@@ -19,10 +19,8 @@
 
 from __future__ import unicode_literals
 
-from weboob.capabilities.base import find_object
-from .compat.weboob_capabilities_bank import CapBankWealth, AccountNotFound
-from weboob.tools.backend import Module, BackendConfig
-from weboob.tools.value import ValueBackendPassword
+from .compat.weboob_capabilities_bank import CapBankWealth
+from weboob.tools.backend import AbstractModule
 
 from .browser import AferBrowser
 
@@ -30,38 +28,20 @@
 __all__ = ['AferModule']
 
 
-class AferModule(Module, CapBankWealth):
+class AferModule(AbstractModule, CapBankWealth):
     NAME = 'afer'
-    DESCRIPTION = u'Association française d\'épargne et de retraite'
-    MAINTAINER = u'James GALT'
-    EMAIL = 'jgalt@budget-insight.com'
+    DESCRIPTION = "Association française d'épargne et de retraite"
+    MAINTAINER = 'Quentin Defenouillère'
+    EMAIL = 'quentin.defenouillere@budget-insight.com'
     LICENSE = 'LGPLv3+'
     VERSION = '1.5'
 
+    PARENT = 'aviva'
     BROWSER = AferBrowser
-    CONFIG = BackendConfig(
-        ValueBackendPassword('login', label='Identifiant', regexp=r'.+', masked=False),
-        ValueBackendPassword('password', label="Mot de passe", regexp=r'\d{1,8}|[a-zA-Z0-9]{7,30}')
-        # TODO lose previous regex (and in backend) once users credentials migration is complete
-    )
 
     def create_default_browser(self):
-        return self.create_browser(self.config['login'].get(),
-                                   self.config['password'].get())
-
-
-    def get_account(self, id):
-        return find_object(self.iter_accounts(), id=id, error=AccountNotFound)
-
-
-    def iter_accounts(self):
-        return self.browser.iter_accounts()
-
-    def iter_coming(self, account):
-        raise NotImplementedError()
-
-    def iter_history(self, account):
-        return self.browser.iter_history(account)
-
-    def iter_investment(self, account):
-        return self.browser.iter_investment(account)
+        return self.create_browser(
+            self.config['login'].get(),
+            self.config['password'].get(),
+            weboob=self.weboob
+        )

modules/amazon/compat/weboob_exceptions.py

@@ -36,3 +36,22 @@ def __str__(self):
 
 class AppValidation(DecoupledValidation):
     pass
+
+
+class NeedInteractive(Exception):
+    pass
+
+
+class NeedInteractiveForRedirect(NeedInteractive):
+    """
+    An authentication is required to connect and credentials are not supplied
+    """
+    pass
+        
+            
+class NeedInteractiveFor2FA(NeedInteractive):
+    """
+    A 2FA is required to connect, credentials are supplied but not the second factor
+    """
+    pass
+

modules/amazonstorecard/compat/weboob_exceptions.py

@@ -36,3 +36,22 @@ def __str__(self):
 
 class AppValidation(DecoupledValidation):
     pass
+
+
+class NeedInteractive(Exception):
+    pass
+
+
+class NeedInteractiveForRedirect(NeedInteractive):
+    """
+    An authentication is required to connect and credentials are not supplied
+    """
+    pass
+        
+            
+class NeedInteractiveFor2FA(NeedInteractive):
+    """
+    A 2FA is required to connect, credentials are supplied but not the second factor
+    """
+    pass
+

modules/ameli/compat/weboob_exceptions.py

@@ -36,3 +36,22 @@ def __str__(self):
 
 class AppValidation(DecoupledValidation):
     pass
+
+
+class NeedInteractive(Exception):
+    pass
+
+
+class NeedInteractiveForRedirect(NeedInteractive):
+    """
+    An authentication is required to connect and credentials are not supplied
+    """
+    pass
+        
+            
+class NeedInteractiveFor2FA(NeedInteractive):
+    """
+    A 2FA is required to connect, credentials are supplied but not the second factor
+    """
+    pass
+

modules/amelipro/compat/weboob_exceptions.py

@@ -36,3 +36,22 @@ def __str__(self):
 
 class AppValidation(DecoupledValidation):
     pass
+
+
+class NeedInteractive(Exception):
+    pass
+
+
+class NeedInteractiveForRedirect(NeedInteractive):
+    """
+    An authentication is required to connect and credentials are not supplied
+    """
+    pass
+        
+            
+class NeedInteractiveFor2FA(NeedInteractive):
+    """
+    A 2FA is required to connect, credentials are supplied but not the second factor
+    """
+    pass
+

modules/americanexpress/compat/weboob_exceptions.py

@@ -36,3 +36,22 @@ def __str__(self):
 
 class AppValidation(DecoupledValidation):
     pass
+
+
+class NeedInteractive(Exception):
+    pass
+
+
+class NeedInteractiveForRedirect(NeedInteractive):
+    """
+    An authentication is required to connect and credentials are not supplied
+    """
+    pass
+        
+            
+class NeedInteractiveFor2FA(NeedInteractive):
+    """
+    A 2FA is required to connect, credentials are supplied but not the second factor
+    """
+    pass
+

modules/americanexpress/module.py

@@ -18,8 +18,7 @@
 # along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
 
 
-from .compat.weboob_capabilities_bank import CapBank, AccountNotFound
-from weboob.capabilities.base import find_object
+from .compat.weboob_capabilities_bank import CapBank
 from weboob.tools.backend import Module, BackendConfig
 from weboob.tools.value import ValueBackendPassword
 
@@ -47,9 +46,6 @@ def create_default_browser(self):
     def iter_accounts(self):
         return self.browser.get_accounts_list()
 
-    def get_account(self, _id):
-        return find_object(self.browser.get_accounts_list(), id=_id, error=AccountNotFound)
-
     def iter_history(self, account):
         return self.browser.iter_history(account)
 

modules/amundi/browser.py

@@ -17,11 +17,19 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
 
-from .pages import LoginPage, AccountsPage, AccountHistoryPage
+from __future__ import unicode_literals
+
 from weboob.browser.browsers import URL, LoginBrowser, need_login
 from .compat.weboob_exceptions import BrowserIncorrectPassword
-from weboob.browser.exceptions import ClientError
-from weboob.capabilities.base import empty
+from weboob.browser.exceptions import ClientError, ServerError
+from weboob.capabilities.base import empty, NotAvailable
+
+from .pages import (
+    LoginPage, AccountsPage, AccountHistoryPage, AmundiInvestmentsPage, AllianzInvestmentPage,
+    EEInvestmentPage, EEInvestmentDetailPage, EEProductInvestmentPage, EresInvestmentPage,
+    CprInvestmentPage, BNPInvestmentPage, BNPInvestmentApiPage, AxaInvestmentPage,
+    EpsensInvestmentPage, EcofiInvestmentPage,
+)
 
 
 class AmundiBrowser(LoginBrowser):
@@ -32,6 +40,29 @@ class AmundiBrowser(LoginBrowser):
     accounts = URL(r'api/individu/positionFonds\?flagUrlFicheFonds=true&inclurePositionVide=false', AccountsPage)
     account_history = URL(r'api/individu/operations\?valeurExterne=false&filtreStatutModeExclusion=false&statut=CPTA', AccountHistoryPage)
 
+    # Amundi.fr investments
+    amundi_investments = URL(r'https://www.amundi.fr/fr_part/product/view', AmundiInvestmentsPage)
+    # EEAmundi browser investments
+    ee_investments = URL(r'https://www.amundi-ee.com/part/home_fp&partner=PACTEO_SYS', EEInvestmentPage)
+    ee_investment_details = URL(r'https://www.amundi-ee.com/psAmundiEEPart/ezjscore/call', EEInvestmentDetailPage)
+    # EEAmundi product investments
+    ee_product_investments = URL(r'https://www.amundi-ee.com/product', EEProductInvestmentPage)
+    # Allianz GI investments
+    allianz_investments = URL(r'https://fr.allianzgi.com', AllianzInvestmentPage)
+    # Eres investments
+    eres_investments = URL(r'https://www.eres-group.com/eres/new_fiche_fonds.php', EresInvestmentPage)
+    # CPR asset management investments
+    cpr_investments = URL(r'https://www.cpr-am.fr/particuliers/product/view', CprInvestmentPage)
+    # BNP Paribas Epargne Retraite Entreprises
+    bnp_investments = URL(r'https://www.epargne-retraite-entreprises.bnpparibas.com/entreprises/fonds', BNPInvestmentPage)
+    bnp_investment_api = URL(r'https://www.epargne-retraite-entreprises.bnpparibas.com/api2/funds/overview/(?P<fund_id>.*)', BNPInvestmentApiPage)
+    # AXA investments
+    axa_investments = URL(r'https://(.*).axa-im.fr/fr/fund-page', AxaInvestmentPage)
+    # Epsens investments
+    epsens_investments = URL(r'https://www.epsens.com/information-financiere', EpsensInvestmentPage)
+    # Ecofi investments
+    ecofi_investments = URL(r'http://www.ecofi.fr/fr/fonds/dynamis-solidaire', EcofiInvestmentPage)
+
     def do_login(self):
         data = {
             'username': self.username,
@@ -61,9 +92,89 @@ def iter_investment(self, account):
             return
         headers = {'X-noee-authorization': 'noeprd %s' % self.token}
         self.accounts.go(headers=headers)
+
+        ignored_urls = (
+            'www.sggestion-ede.com/product',  # Going there leads to a 404
+            'www.assetmanagement.hsbc.com',  # Information not accessible
+            'www.labanquepostale-am.fr/nos-fonds',  # Nothing interesting there
+        )
+
+        handled_urls = (
+            'www.amundi.fr/fr_part',  # AmundiInvestmentsPage
+            'www.amundi-ee.com/part/home_fp',  # EEInvestmentPage
+            'www.amundi-ee.com/product',  # EEProductInvestmentPage
+            'fr.allianzgi.com/fr-fr',  # AllianzInvestmentPage
+            'www.eres-group.com/eres',  # EresInvestmentPage
+            'www.cpr-am.fr/particuliers/product',  # CprInvestmentPage
+            'www.epargne-retraite-entreprises.bnpparibas.com',  # BNPInvestmentPage
+            'axa-im.fr/fr/fund-page',  # AxaInvestmentPage
+            'www.epsens.com/information-financiere',  # EpsensInvestmentPage
+            'www.ecofi.fr/fr/fonds/dynamis-solidaire',  # EcofiInvestmentPage
+        )
+
         for inv in self.page.iter_investments(account_id=account.id):
+            if inv._details_url:
+                # Only go to known details pages to avoid logout on unhandled pages
+                if any(url in inv._details_url for url in handled_urls):
+                    self.fill_investment_details(inv)
+                else:
+                    if not any(url in inv._details_url for url in ignored_urls):
+                        # Not need to raise warning if the URL is already known and ignored
+                        self.logger.warning('Investment details on URL %s are not handled yet.', inv._details_url)
+                    inv.asset_category = NotAvailable
+                    inv.recommended_period = NotAvailable
             yield inv
 
+    @need_login
+    def fill_investment_details(self, inv):
+        # Going to investment details may lead to various websites.
+        # This method handles all the already encountered pages.
+        try:
+            self.location(inv._details_url)
+        except ServerError:
+            # Some URLs return a 500 even on the website
+            inv.asset_category = NotAvailable
+            inv.recommended_period = NotAvailable
+            return inv
+
+        # Pages with only asset category available
+        if (self.amundi_investments.is_here() or
+            self.allianz_investments.is_here() or
+            self.axa_investments.is_here()):
+            inv.asset_category = self.page.get_asset_category()
+            inv.recommended_period = NotAvailable
+
+        # Pages with asset category & recommended period
+        elif (self.eres_investments.is_here() or
+            self.cpr_investments.is_here() or
+            self.ee_product_investments.is_here() or
+            self.epsens_investments.is_here() or
+            self.ecofi_investments.is_here()):
+            self.page.fill_investment(obj=inv)
+
+        # Particular cases
+        elif self.ee_investments.is_here():
+            inv.recommended_period = self.page.get_recommended_period()
+            details_url = self.page.get_details_url()
+            if details_url:
+                self.location(details_url)
+                if self.ee_investment_details.is_here():
+                    inv.asset_category = self.page.get_asset_category()
+
+        elif self.bnp_investments.is_here():
+            # We fetch the fund ID and get the attributes directly from the BNP-ERE API
+            fund_id = self.page.get_fund_id()
+            if fund_id:
+                # Specify the 'Accept' header otherwise the server returns WSDL instead of JSON
+                self.bnp_investment_api.go(fund_id=fund_id, headers={'Accept': 'application/json'})
+                self.page.fill_investment(obj=inv)
+            else:
+                self.logger.warning('Could not fetch the fund_id for BNP investment %s.', inv.label)
+                inv.asset_category = NotAvailable
+                inv.recommended_period = NotAvailable
+
+        return inv
+
     @need_login
     def iter_history(self, account):
         headers = {'X-noee-authorization': 'noeprd %s' % self.token}
@@ -73,8 +184,10 @@ def iter_history(self, account):
 
 
 class EEAmundi(AmundiBrowser):
+    # Careful if you modify the BASEURL, also verify Amundi's Abstract modules
     BASEURL = 'https://www.amundi-ee.com/psf/'
 
 
 class TCAmundi(AmundiBrowser):
+    # Careful if you modify the BASEURL, also verify Amundi's Abstract modules
     BASEURL = 'https://epargnants.amundi-tc.com/psf/'

modules/amundi/compat/weboob_exceptions.py

@@ -36,3 +36,22 @@ def __str__(self):
 
 class AppValidation(DecoupledValidation):
     pass
+
+
+class NeedInteractive(Exception):
+    pass
+
+
+class NeedInteractiveForRedirect(NeedInteractive):
+    """
+    An authentication is required to connect and credentials are not supplied
+    """
+    pass
+        
+            
+class NeedInteractiveFor2FA(NeedInteractive):
+    """
+    A 2FA is required to connect, credentials are supplied but not the second factor
+    """
+    pass
+

modules/amundi/compat/weboob_tools_capabilities_bank_investments.py

+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2017 Jonathan Schmidt
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import unicode_literals
+
+import re
+
+from weboob.tools.compat import basestring
+from weboob.capabilities.base import NotAvailable
+from weboob.capabilities.bank import Investment
+from weboob.browser.filters.base import Filter, FilterError, debug
+
+def is_isin_valid(isin):
+    """
+    Méthode générale
+    Table de conversion des lettres en chiffres
+    A=10    B=11    C=12    D=13    E=14    F=15    G=16    H=17    I=18
+    J=19    K=20    L=21    M=22    N=23    O=24    P=25    Q=26    R=27
+    S=28    T=29    U=30    V=31    W=32    X=33    Y=34    Z=35
+
+    1 - Mettre de côté la clé, qui servira de référence à la fin de la vérification.
+    2 - Convertir toutes les lettres en nombres via la table de conversion ci-contre. Si le nombre obtenu est supérieur ou égal à 10, prendre les deux chiffres du nombre séparément (exemple : 27 devient 2 et 7).
+    3 - Pour chaque chiffre, multiplier sa valeur par deux si sa position est impaire en partant de la droite. Si le nombre obtenu est supérieur ou égal à 10, garder les deux chiffres du nombre séparément (exemple : 14 devient 1 et 4).
+    4 - Faire la somme de tous les chiffres.
+    5 - Soustraire cette somme de la dizaine supérieure ou égale la plus proche (exemples : si la somme vaut 22, la dizaine « supérieure ou égale » est 30, et la clé vaut donc 8 ; si la somme vaut 30, la dizaine « supérieure ou égale » est 30, et la clé vaut 0 ; si la somme vaut 31, la dizaine « supérieure ou égale » est 40, et la clé vaut 9).
+    6 - Comparer la valeur obtenue à la clé mise initialement de côté.
+
+    Étapes 1 et 2 :
+    F R 0 0 0 3 5 0 0 0 0 (+ 8 : clé)
+    15 27 0 0 0 3 5 0 0 0 0
+
+    Étape 3 : le traitement se fait sur des chiffres
+    1 5 2 7 0 0 0 3 5 0 0 0 0
+    I P I P I P I P I P I P I : position en partant de la droite (P = Pair, I = Impair)
+    2 1 2 1 2 1 2 1 2 1 2 1 2 : coefficient multiplicateur
+
+    2 5 4 7 0 0 0 3 10 0 0 0 0 : résultat
+
+    Étape 4 :
+    2 + 5 + 4 + 7 + 0 + 0 + 0 + 3 + (1 + 0)+ 0 + 0 + 0 + 0 = 22
+
+    Étapes 5 et 6 : 30 - 22 = 8 (valeur de la clé)
+    """
+
+    if not isinstance(isin, basestring):
+        return False
+    if not re.match(r'^[A-Z]{2}[A-Z0-9]{9}\d$', isin):
+        return False
+
+    isin_in_digits = ''.join(str(ord(x) - ord('A') + 10) if not x.isdigit() else x for x in isin[:-1])
+    key = isin[-1:]
+    result = ''
+    for k, val in enumerate(isin_in_digits[::-1], start=1):
+        if k % 2 == 0:
+            result = ''.join((result, val))
+        else:
+            result = ''.join((result, str(int(val)*2)))
+    return str(sum(int(x) for x in result) + int(key))[-1] == '0'
+
+
+def create_french_liquidity(valuation):
+    """
+    Automatically fills a liquidity investment with label, code and code_type.
+    """
+    liquidity = Investment()
+    liquidity.label = "Liquidités"
+    liquidity.code = "XX-liquidity"
+    liquidity.code_type = NotAvailable
+    liquidity.valuation = valuation
+    return liquidity
+
+
+# These filters can be used to set Investment.code
+# and Investment.code_type without having to declare
+# obj_code() and obj_code_type() methods in each module
+
+class FormatError(FilterError):
+    pass
+
+
+class IsinCode(Filter):
+    """
+    Returns the input only if it is a valid ISIN code.
+    """
+    @debug()
+    def filter(self, code):
+        if is_isin_valid(code):
+            return code
+        return self.default_or_raise(FormatError('%r is not a valid ISIN code, no default value was set.' % code))
+
+
+class IsinType(Filter):
+    """
+    Returns Investment.CODE_TYPE_ISIN if the input is a valid ISIN code.
+    """
+    @debug()
+    def filter(self, code):
+        if is_isin_valid(code):
+            return Investment.CODE_TYPE_ISIN
+        return NotAvailable
+

modules/amundi/module.py

@@ -18,8 +18,7 @@
 # along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
 
 
-from .compat.weboob_capabilities_bank import CapBankWealth, AccountNotFound
-from weboob.capabilities.base import find_object
+from .compat.weboob_capabilities_bank import CapBankWealth
 from weboob.tools.backend import Module, BackendConfig
 from weboob.tools.value import ValueBackendPassword, Value
 
@@ -46,9 +45,6 @@ def create_default_browser(self):
         self.BROWSER = b[self.config['website'].get()]
         return self.create_browser(self.config['login'].get(), self.config['password'].get())
 
-    def get_account(self, id):
-        return find_object(self.iter_accounts(), id=id, error=AccountNotFound)
-
     def iter_accounts(self):
         return self.browser.iter_accounts()
 

modules/amundi/pages.py

@@ -19,18 +19,21 @@
 
 from __future__ import unicode_literals
 
+import re
 from datetime import datetime
 
 from weboob.browser.elements import ItemElement, method, DictElement
 from .compat.weboob_browser_filters_standard import (
-    CleanDecimal, Date, Field, CleanText, Env, Eval,
+    CleanDecimal, Date, Field, CleanText,
+    Env, Eval, Map, Regexp, Title,
 )
+from weboob.browser.filters.html import Attr
 from weboob.browser.filters.json import Dict
-from weboob.browser.pages import LoggedPage, JsonPage
+from weboob.browser.pages import LoggedPage, JsonPage, HTMLPage
 from .compat.weboob_capabilities_bank import Account, Investment, Transaction
-from weboob.capabilities.base import NotAvailable, empty
+from weboob.capabilities.base import NotAvailable
 from .compat.weboob_exceptions import NoAccountsException
-from weboob.tools.capabilities.bank.investments import is_isin_valid
+from .compat.weboob_tools_capabilities_bank_investments import IsinCode, IsinType
 
 
 class LoginPage(JsonPage):
@@ -38,6 +41,14 @@ def get_token(self):
         return Dict('token')(self.doc)
 
 
+ACCOUNT_TYPES = {
+    'PEE': Account.TYPE_PEE,
+    'PEG': Account.TYPE_PEE,
+    'PEI': Account.TYPE_PEE,
+    'PERCO': Account.TYPE_PERCO,
+    'RSP': Account.TYPE_RSP,
+}
+
 class AccountsPage(LoggedPage, JsonPage):
     def get_company_name(self):
         json_list = Dict('listPositionsSalarieFondsDto')(self.doc)
@@ -45,14 +56,6 @@ def get_company_name(self):
             return json_list[0].get('nomEntreprise', NotAvailable)
         return NotAvailable
 
-    ACCOUNT_TYPES = {
-        'PEE': Account.TYPE_PEE,
-        'PEG': Account.TYPE_PEE,
-        'PEI': Account.TYPE_PEE,
-        'PERCO': Account.TYPE_PERCO,
-        'RSP': Account.TYPE_RSP,
-    }
-
     @method
     class iter_accounts(DictElement):
         def parse(self, el):
@@ -72,9 +75,7 @@ def obj_number(self):
                 return '%s_%s' % (Field('id')(self), self.page.browser.username)
 
             obj_currency = 'EUR'
-
-            def obj_type(self):
-                return self.page.ACCOUNT_TYPES.get(Dict('typeDispositif')(self), Account.TYPE_LIFE_INSURANCE)
+            obj_type = Map(Dict('typeDispositif'), ACCOUNT_TYPES, Account.TYPE_LIFE_INSURANCE)
 
             def obj_label(self):
                 try:
@@ -97,11 +98,20 @@ def find_elements(self):
         class item(ItemElement):
             klass = Investment
 
+            def condition(self):
+                # Some additional invests are present in the JSON but are not
+                # displayed on the website, besides they have no valuation,
+                # so we check the 'valuation' key before parsing them
+                return Dict('mtBrut', default=None)(self)
+
             obj_label = Dict('libelleFonds')
             obj_unitvalue = Dict('vl') & CleanDecimal
             obj_quantity = Dict('nbParts') & CleanDecimal
             obj_valuation = Dict('mtBrut') & CleanDecimal
             obj_vdate = Date(Dict('dtVl'))
+            obj__details_url = Dict('urlFicheFonds', default=None)
+            obj_code = IsinCode(Dict('codeIsin', default=NotAvailable), default=NotAvailable)
+            obj_code_type = IsinType(Dict('codeIsin', default=NotAvailable))
 
             def obj_srri(self):
                 srri = Dict('SRRI')(self)
@@ -110,25 +120,14 @@ def obj_srri(self):
                     return NotAvailable
                 return int(srri)
 
-            def obj_code(self):
-                code = Dict('codeIsin', default=NotAvailable)(self)
-                if is_isin_valid(code):
-                    return code
-                return NotAvailable
-
-            def obj_code_type(self):
-                if empty(Field('code')(self)):
-                    return NotAvailable
-                return Investment.CODE_TYPE_ISIN
-
             def obj_performance_history(self):
                 # The Amundi JSON only contains 1 year and 5 years performances.
                 # It seems that when a value is unavailable, they display '0.0' instead...
                 perfs = {}
                 if Dict('performanceUnAn', default=None)(self) not in (0.0, None):
-                    perfs[1] = Eval(lambda x: x/100, CleanDecimal(Dict('performanceUnAn')))(self)
+                    perfs[1] = Eval(lambda x: x / 100, CleanDecimal(Dict('performanceUnAn')))(self)
                 if Dict('performanceCinqAns', default=None)(self) not in (0.0, None):
-                    perfs[5] = Eval(lambda x: x/100, CleanDecimal(Dict('performanceCinqAns')))(self)
+                    perfs[5] = Eval(lambda x: x / 100, CleanDecimal(Dict('performanceCinqAns')))(self)
                 return perfs
 
 
@@ -164,5 +163,111 @@ def iter_history(self, account):
                     tr.date = tr.rdate
                     tr.label = hist.get('libelleOperation') or hist['libelleCommunication']
                     tr.type = Transaction.TYPE_UNKNOWN
-
                     yield tr
+
+
+class AmundiInvestmentsPage(LoggedPage, HTMLPage):
+    def get_asset_category(self):
+        # Descriptions are like 'Fonds d'Investissement - (ISIN: FR001018 - Action'
+        # Fetch the last words of the description (e.g. 'Action' or 'Diversifié')
+        return Regexp(
+            CleanText('//div[@class="amundi-fund-legend"]//strong'),
+            r' ([^-]+)$',
+            default=NotAvailable
+        )(self.doc)
+
+
+class EEInvestmentPage(LoggedPage, HTMLPage):
+    def get_recommended_period(self):
+        return Title(CleanText('//label[contains(text(), "Durée minimum de placement")]/following-sibling::span', default=NotAvailable))(self.doc)
+
+    def get_details_url(self):
+        return Attr('//a[contains(text(), "Caractéristiques")]', 'data-href', default=None)(self.doc)
+
+
+class EEInvestmentDetailPage(LoggedPage, HTMLPage):
+    def get_asset_category(self):
+        return CleanText('//label[contains(text(), "Classe d\'actifs")]/following-sibling::span', default=NotAvailable)(self.doc)
+
+
+class EEProductInvestmentPage(LoggedPage, HTMLPage):
+    @method
+    class fill_investment(ItemElement):
+        obj_asset_category = CleanText('//span[contains(text(), "Classe")]/following-sibling::span[@class="valeur"][1]')
+        obj_recommended_period = CleanText('//span[contains(text(), "Durée minimum")]/following-sibling::span[@class="valeur"][1]')
+
+
+class AllianzInvestmentPage(LoggedPage, HTMLPage):
+    def get_asset_category(self):
+        # The format may be a very short description, or be
+        # included between quotation marks within a paragraph
+        asset_category = CleanText('//div[contains(@class, "fund-summary")]//h3/following-sibling::div', default=NotAvailable)(self.doc)
+        m = re.search(r'« (.*) »', asset_category)
+        if m:
+            return m.group(1)
+        return asset_category
+
+
+class EresInvestmentPage(LoggedPage, HTMLPage):
+    @method
+    class fill_investment(ItemElement):
+        obj_asset_category = CleanText('//li[span[contains(text(), "Classification")]]', children=False, default=NotAvailable)
+        obj_recommended_period = CleanText('//li[span[contains(text(), "Durée")]]', children=False, default=NotAvailable)
+
+        def obj_performance_history(self):
+            perfs = {}
+            if CleanDecimal.French('(//tr[th[text()="1 an"]]/td[1])[1]', default=None)(self):
+                perfs[1] = Eval(lambda x: x / 100, CleanDecimal.French('(//tr[th[text()="1 an"]]/td[1])[1]'))(self)
+            if CleanDecimal.French('(//tr[th[text()="3 ans"]]/td[1])[1]', default=None)(self):
+                perfs[3] = Eval(lambda x: x / 100, CleanDecimal.French('(//tr[th[text()="3 ans"]]/td[1])[1]'))(self)
+            if CleanDecimal.French('(//tr[th[text()="5 ans"]]/td[1])[1]', default=None)(self):
+                perfs[5] = Eval(lambda x: x / 100, CleanDecimal.French('(//tr[th[text()="5 ans"]]/td[1])[1]'))(self)
+            return perfs
+
+
+class CprInvestmentPage(LoggedPage, HTMLPage):
+    @method
+    class fill_investment(ItemElement):
+        obj_srri = CleanText('//span[@class="active"]', default=NotAvailable)
+        obj_asset_category = CleanText('//div[contains(text(), "Classe d\'actifs")]//strong', default=NotAvailable)
+        obj_recommended_period = CleanText('//div[contains(text(), "Durée recommandée")]//strong', default=NotAvailable)
+
+
+class BNPInvestmentPage(LoggedPage, HTMLPage):
+    def get_fund_id(self):
+        return Regexp(
+            CleanText('//script[contains(text(), "GLB_ProductId")]'),
+            r'GLB_ProductId = "(\w+)',
+            default=None
+        )(self.doc)
+
+
+class BNPInvestmentApiPage(LoggedPage, JsonPage):
+    @method
+    class fill_investment(ItemElement):
+        obj_asset_category = Dict('Classification', default=NotAvailable)
+        obj_recommended_period = Dict('DureePlacement', default=NotAvailable)
+
+
+class AxaInvestmentPage(LoggedPage, HTMLPage):
+    def get_asset_category(self):
+        return Title(CleanText('//th[contains(text(), "Classe")]/following-sibling::td'))(self.doc)
+
+
+class EpsensInvestmentPage(LoggedPage, HTMLPage):
+    @method
+    class fill_investment(ItemElement):
+        obj_asset_category = CleanText('//div[div[span[contains(text(), "Classification")]]]/div[2]/span', default=NotAvailable)
+        obj_recommended_period = CleanText('//div[div[span[contains(text(), "Durée de placement")]]]/div[2]/span', default=NotAvailable)
+
+
+class EcofiInvestmentPage(LoggedPage, HTMLPage):
+    @method
+    class fill_investment(ItemElement):
+        # Recommended period is actually an image so we extract the
+        # information from its URL such as '/Horizon/Horizon_5_ans.png'
+        obj_recommended_period = Regexp(
+            CleanText(Attr('//img[contains(@src, "/Horizon/")]', 'src', default=NotAvailable), replace=[(u'_', ' ')]),
+            r'\/Horizon (.*)\.png'
+        )
+        obj_asset_category = CleanText('//div[contains(text(), "Classification")]/following-sibling::div[1]', default=NotAvailable)

modules/anticaptcha/compat/weboob_exceptions.py

@@ -36,3 +36,22 @@ def __str__(self):
 
 class AppValidation(DecoupledValidation):
     pass
+
+
+class NeedInteractive(Exception):
+    pass
+
+
+class NeedInteractiveForRedirect(NeedInteractive):
+    """
+    An authentication is required to connect and credentials are not supplied
+    """
+    pass
+        
+            
+class NeedInteractiveFor2FA(NeedInteractive):
+    """
+    A 2FA is required to connect, credentials are supplied but not the second factor
+    """
+    pass
+

modules/apivie/compat/weboob_exceptions.py

@@ -36,3 +36,22 @@ def __str__(self):
 
 class AppValidation(DecoupledValidation):
     pass
+
+
+class NeedInteractive(Exception):
+    pass
+
+
+class NeedInteractiveForRedirect(NeedInteractive):
+    """
+    An authentication is required to connect and credentials are not supplied
+    """
+    pass
+        
+            
+class NeedInteractiveFor2FA(NeedInteractive):
+    """
+    A 2FA is required to connect, credentials are supplied but not the second factor
+    """
+    pass
+

modules/asana/compat/weboob_exceptions.py

@@ -36,3 +36,22 @@ def __str__(self):
 
 class AppValidation(DecoupledValidation):
     pass
+
+
+class NeedInteractive(Exception):
+    pass
+
+
+class NeedInteractiveForRedirect(NeedInteractive):
+    """
+    An authentication is required to connect and credentials are not supplied
+    """
+    pass
+        
+            
+class NeedInteractiveFor2FA(NeedInteractive):
+    """
+    A 2FA is required to connect, credentials are supplied but not the second factor
+    """
+    pass
+

modules/aum/compat/weboob_exceptions.py

@@ -36,3 +36,22 @@ def __str__(self):
 
 class AppValidation(DecoupledValidation):
     pass
+
+
+class NeedInteractive(Exception):
+    pass
+
+
+class NeedInteractiveForRedirect(NeedInteractive):
+    """
+    An authentication is required to connect and credentials are not supplied
+    """
+    pass
+        
+            
+class NeedInteractiveFor2FA(NeedInteractive):
+    """
+    A 2FA is required to connect, credentials are supplied but not the second factor
+    """
+    pass
+

modules/aum/optim/compat/weboob_exceptions.py

@@ -36,3 +36,22 @@ def __str__(self):
 
 class AppValidation(DecoupledValidation):
     pass
+
+
+class NeedInteractive(Exception):
+    pass
+
+
+class NeedInteractiveForRedirect(NeedInteractive):
+    """
+    An authentication is required to connect and credentials are not supplied
+    """
+    pass
+        
+            
+class NeedInteractiveFor2FA(NeedInteractive):
+    """
+    A 2FA is required to connect, credentials are supplied but not the second factor
+    """
+    pass
+

modules/aviva/__init__.py

+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2012-2019  Budget Insight
+#
+# This file is part of a weboob module.
+#
+# This weboob module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This weboob module is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
+
+
+from .module import AvivaModule
+
+
+__all__ = ['AvivaModule']

modules/aviva/browser.py

+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2012-2019  Budget Insight
+#
+# This file is part of a weboob module.
+#
+# This weboob module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This weboob module is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import unicode_literals
+
+
+from weboob.browser.browsers import LoginBrowser, need_login
+from .compat.weboob_browser_url import BrowserParamURL
+from weboob.capabilities.base import empty, NotAvailable
+from .compat.weboob_exceptions import BrowserIncorrectPassword, BrowserPasswordExpired, ActionNeeded, BrowserHTTPError
+from weboob.tools.capabilities.bank.transactions import sorted_transactions
+
+from .pages.detail_pages import (
+    LoginPage, MigrationPage, InvestmentPage, HistoryPage, ActionNeededPage,
+    InvestDetailPage, PrevoyancePage, ValidationPage, InvestPerformancePage,
+)
+
+from .pages.account_page import AccountsPage
+
+
+class AvivaBrowser(LoginBrowser):
+    BASEURL = 'https://www.aviva.fr'
+
+    validation = BrowserParamURL(r'/conventions/acceptation\?backurl=/(?P<browser_subsite>[^/]+)/Accueil', ValidationPage)
+    login = BrowserParamURL(
+        r'/(?P<browser_subsite>[^/]+)/MonCompte/Connexion',
+        r'/(?P<browser_subsite>[^/]+)/conventions/acceptation',
+        LoginPage
+    )
+    migration = BrowserParamURL(r'/(?P<browser_subsite>[^/]+)/MonCompte/Migration', MigrationPage)
+    accounts = BrowserParamURL(r'/(?P<browser_subsite>[^/]+)/Accueil/Synthese-Contrats', AccountsPage)
+    investment = BrowserParamURL(r'/(?P<browser_subsite>[^/]+)/contrat/epargne/-(?P<page_id>[0-9]{10})', InvestmentPage)
+    prevoyance = BrowserParamURL(r'/(?P<browser_subsite>[^/]+)/contrat/prevoyance/-(?P<page_id>[0-9]{10})', PrevoyancePage)
+    history = BrowserParamURL(r'/(?P<browser_subsite>[^/]+)/contrat/getOperations\?param1=(?P<history_token>.*)', HistoryPage)
+    action_needed = BrowserParamURL(r'/(?P<browser_subsite>[^/]+)/coordonnees/detailspersonne\?majcontacts=true', ActionNeededPage)
+    invest_detail = BrowserParamURL(r'https://aviva-fonds.webfg.net/sheet/fund/(?P<isin>[A-Z0-9]+)', InvestDetailPage)
+    invest_performance = BrowserParamURL(r'https://aviva-fonds.webfg.net/sheet/fund-calculator', InvestPerformancePage)
+
+    def __init__(self, *args, **kwargs):
+        self.subsite = 'espaceclient'
+        super(AvivaBrowser, self).__init__(*args, **kwargs)
+
+    def do_login(self):
+        self.login.go()
+        self.page.login(self.username, self.password)
+        if self.login.is_here():
+            if 'acceptation' in self.url:
+                raise ActionNeeded("Veuillez accepter les conditions générales d'utilisation sur le site.")
+            else:
+                raise BrowserIncorrectPassword("L'identifiant ou le mot de passe est incorrect.")
+        elif self.migration.is_here():
+            # Usually landing here when customers have to renew their credentials
+            message = self.page.get_error()
+            raise BrowserPasswordExpired(message)
+
+    @need_login
+    def iter_accounts(self):
+        self.accounts.go()
+        for account in self.page.iter_accounts():
+            # Request to account details sometimes returns a 500
+            try:
+                self.location(account.url)
+                if not self.investment.is_here() or self.page.unavailable_details():
+                    # We don't scrape insurances, guarantees, health contracts
+                    # and accounts with unavailable balances
+                    continue
+                self.page.fill_account(obj=account)
+                yield account
+            except BrowserHTTPError:
+                self.logger.warning('Could not get the account details: account %s will be skipped', account.id)
+
+    @need_login
+    def iter_investment(self, account):
+        # Request to account details sometimes returns a 500
+        try:
+            self.location(account.url)
+        except BrowserHTTPError:
+            self.logger.warning('Could not get the account investments for account %s', account.id)
+            return
+        for inv in self.page.iter_investment():
+            if not empty(inv.code):
+                # Need to go first on InvestDetailPage...
+                self.invest_detail.go(isin=inv.code)
+                # ...to then request the InvestPerformancePage tab
+                self.invest_performance.go()
+                self.page.fill_investment(obj=inv)
+            else:
+                inv.unitprice = inv.diff_ratio = inv.description = NotAvailable
+            yield inv
+
+    @need_login
+    def iter_history(self, account):
+        if empty(account.url):
+            # An account should always have a link to the details
+            raise NotImplementedError()
+        try:
+            self.location(account.url)
+        except BrowserHTTPError:
+            self.logger.warning('Could not get the history for account %s', account.id)
+            return
+
+        history_link = self.page.get_history_link()
+
+        if not history_link:
+            # accounts don't always have an history_link
+            raise NotImplementedError()
+
+        self.location(history_link)
+        assert self.history.is_here()
+        result = []
+        result.extend(self.page.iter_versements())
+        result.extend(self.page.iter_arbitrages())
+        return sorted_transactions(result)
+
+    def get_subscription_list(self):
+        return []