modules/aviva/browser.py

@@ -22,7 +22,7 @@
 
 from weboob.browser import LoginBrowser, need_login
 from weboob.browser.url import BrowserParamURL
-from weboob.capabilities.base import empty, NotAvailable
+from weboob.capabilities.base import empty
 from weboob.capabilities.bank import Account
 from weboob.exceptions import (
     BrowserIncorrectPassword, BrowserPasswordExpired,
@@ -116,11 +116,11 @@ def iter_investment(self, account):
             if not empty(inv.code):
                 # Need to go first on InvestDetailPage...
                 self.invest_detail.go(isin=inv.code)
-                # ...to then request the InvestPerformancePage tab
-                self.invest_performance.go()
-                self.page.fill_investment(obj=inv)
-            else:
-                inv.unitprice = inv.diff_ratio = inv.description = NotAvailable
+                # Sometimes the page loads but there is no info
+                if not self.page.is_empty():
+                    # ...to then request the InvestPerformancePage tab
+                    self.invest_performance.go()
+                    self.page.fill_investment(obj=inv)
             yield inv
 
     @need_login

modules/aviva/pages/detail_pages.py

@@ -271,7 +271,8 @@ def on_load(self):
 
 
 class InvestDetailPage(LoggedPage, HTMLPage):
-    pass
+    def is_empty(self):
+        return not self.doc.xpath('//table')
 
 
 class InvestPerformancePage(LoggedPage, HTMLPage):

modules/banquepopulaire/browser.py

@@ -48,7 +48,7 @@
     LineboursePage, AlreadyLoginPage, InvestmentPage,
     NewLoginPage, JsFilePage, AuthorizePage, LoginTokensPage, VkImagePage,
     AuthenticationMethodPage, AuthenticationStepPage, CaissedepargneVirtKeyboard,
-    AccountsNextPage, GenericAccountsPage, InfoTokensPage,
+    AccountsNextPage, GenericAccountsPage, InfoTokensPage, NatixisUnavailablePage,
 )
 from .document_pages import BasicTokenPage, SubscriberPage, SubscriptionsPage, DocumentsPage
 from .linebourse_browser import LinebourseAPIBrowser
@@ -237,6 +237,10 @@ class BanquePopulaire(LoginBrowser):
         r'https://www.assurances.natixis.fr/etna-ihs-bp/#/equipement;codeEtab=.*\?windowId=.*',
         NatixisErrorPage
     )
+    natixis_unavailable_page = URL(
+        r'https://www.assurances.natixis.fr/espaceinternet-bp/page500.xhtml',
+        NatixisUnavailablePage
+    )
     natixis_invest = URL(
         r'https://www.assurances.natixis.fr/espaceinternet-bp/rest/v2/contratVie/load/(?P<id1>\w+)/(?P<id2>\w+)/(?P<id3>\w+)',
         NatixisInvestPage
@@ -936,6 +940,8 @@ def get_invest_history(self, account):
             except ServerError:
                 return
             else:
+                if self.natixis_unavailable_page.is_here():
+                    raise BrowserUnavailable(self.page.get_message())
                 history = list(self.page.get_history())
                 history.sort(reverse=True, key=lambda item: item.date)
                 for tr in history:

modules/banquepopulaire/pages.py

@@ -709,6 +709,7 @@ class GenericAccountsPage(LoggedPage, MyHTMLPage):
         (re.compile(r'.*Plan Epargne Retraite.*'), Account.TYPE_PERP),
         (re.compile(r'.*Titres.*'), Account.TYPE_MARKET),
         (re.compile(r'.*Selection Vie.*'), Account.TYPE_LIFE_INSURANCE),
+        (re.compile(r'.*Horizeo.*'), Account.TYPE_LIFE_INSURANCE),
         (re.compile(r'^Fructi Pulse.*'), Account.TYPE_LIFE_INSURANCE),
         (re.compile(r'^Fructi Neo.*'), Account.TYPE_LIFE_INSURANCE),
         (re.compile(r'^(Quintessa|Solevia|Irriga|Delfea|Maritime).*'), Account.TYPE_LIFE_INSURANCE),
@@ -1274,6 +1275,11 @@ class NatixisErrorPage(LoggedPage, HTMLPage):
     pass
 
 
+class NatixisUnavailablePage(LoggedPage, HTMLPage):
+    def get_message(self):
+        return CleanText('//label')(self.doc)
+
+
 class IbanPage(LoggedPage, MyHTMLPage):
     def need_to_go(self):
         return len(self.doc.xpath('//div[@class="grid"]/div/span[contains(text(), "IBAN")]')) == 0

modules/boursedirect/browser.py

@@ -29,8 +29,7 @@
 from .pages import (
     LoginPage, PasswordRenewalPage, AccountsPage, HistoryPage,
     InvestPage, MarketOrdersPage, MarketOrderDetailsPage,
-    LifeInsurancePage, IsinPage, PortfolioPage, JsRedirectPage,
-    HomePage,
+    IsinPage, PortfolioPage, JsRedirectPage, HomePage,
 )
 
 
@@ -52,11 +51,6 @@ class BoursedirectBrowser(LoginBrowser):
     invests = URL(r'/streaming/compteTempsReelCK.php\?stream=0', InvestPage)
     market_orders = URL(r'/priv/new/ordres-en-carnet.php\?ong=7&nc=(?P<nc>\d+)', MarketOrdersPage)
     market_orders_details = URL(r'/priv/new/detailOrdre.php', MarketOrderDetailsPage)
-    lifeinsurance = URL(
-        r'/priv/asVieSituationEncours.php',
-        r'/priv/encours.php\?nc=\d+&idUnique=[\dA-F-]+',
-        LifeInsurancePage
-    )
     isin_page = URL(r'/fr/marche/', IsinPage)
     js_redirect = URL(r'/priv/fiche-valeur.php', JsRedirectPage)
 
@@ -77,14 +71,6 @@ def do_login(self):
 
     @need_login
     def iter_accounts(self):
-        for account in self.iter_accounts_but_insurances():
-            yield account
-
-        self.lifeinsurance.go()
-        if self.lifeinsurance.is_here() and self.page.has_account():
-            yield self.page.get_account()
-
-    def iter_accounts_but_insurances(self):
         self.accounts.go()
         for account in self.page.iter_accounts():
             self.accounts.go(nc=account._select)
@@ -93,18 +79,12 @@ def iter_accounts_but_insurances(self):
 
     @need_login
     def iter_investment(self, account):
-        if account.type == account.TYPE_LIFE_INSURANCE:
-            self.accounts.go()
-            self.lifeinsurance.go()
-            for inv in self.page.iter_investment():
-                yield inv
-        else:
-            self.pre_invests.go(nc=account._select)
-            self.invests.go()
+        self.pre_invests.go(nc=account._select)
+        self.invests.go()
 
-            for inv in self.page.iter_investment():
-                yield inv
-            yield self.page.get_liquidity()
+        for inv in self.page.iter_investment():
+            yield inv
+        yield self.page.get_liquidity()
 
     @need_login
     def iter_market_orders(self, account):
@@ -125,9 +105,7 @@ def iter_market_orders(self, account):
 
     @need_login
     def iter_history(self, account):
-        if account.type == account.TYPE_LIFE_INSURANCE:
-            self.lifeinsurance.go()
-        elif account.type in (account.TYPE_MARKET, account.TYPE_PEA):
+        if account.type in (account.TYPE_MARKET, account.TYPE_PEA):
             self.history.go(nc=account._select)
         else:
             raise NotImplementedError()

modules/boursedirect/pages.py

@@ -406,109 +406,6 @@ def get_isin(self):
         )
 
 
-class LifeInsurancePage(BasePage):
-    def has_account(self):
-        message = CleanText('//fieldset[legend[text()="Message"]]')(self.doc)
-        if 'Vous n´avez pas de contrat. Ce service ne vous est pas accessible.' in message:
-            return False
-        return True
-
-    @method
-    class get_account(ItemElement):
-        klass = Account
-
-        obj_balance = CleanDecimal.French('''//label[text()="Valorisation de l'encours"]/following-sibling::b[1]''')
-        obj_currency = 'EUR'
-        obj_id = obj_number = CleanText('''//label[text()="N° d'adhésion"]/following-sibling::b[1]''')
-        obj_label = Format(
-            '%s (%s)',
-            CleanText('//label[text()="Nom"]/following-sibling::b[1]'),
-            CleanText('//label[text()="Produit"]/following-sibling::b[1]'),
-        )
-        obj_type = Account.TYPE_LIFE_INSURANCE
-
-    @method
-    class iter_investment(TableElement):
-        head_xpath = '//fieldset[legend[text()="Répartition de l´encours"]]/table/tr[@class="place"]/th'
-        item_xpath = '//fieldset[legend[text()="Répartition de l´encours"]]/table/tr[@class!="place"]'
-
-        col_label = 'Nom des supports'
-        col_quantity = 'Nombre de parts'
-        col_unitprice = 'Prix Moyen d´Achat'
-        col_valuation = 'Valorisation des supports'
-        col_vdate = 'Date de valorisation'
-        col_portfolio_share = '(%)'
-
-        class item(ItemElement):
-            klass = Investment
-
-            obj_label = CleanText(TableCell('label'))
-            obj_quantity = CleanDecimal.French(TableCell('quantity'), default=NotAvailable)
-            obj_unitprice = CleanDecimal.French(TableCell('unitprice'), default=NotAvailable)
-            obj_valuation = CleanDecimal.French(TableCell('valuation'), default=NotAvailable)
-            obj_vdate = Date(CleanText(TableCell('vdate')), dayfirst=True)
-            obj_portfolio_share = Eval(lambda x: x / 100, CleanDecimal.French(TableCell('portfolio_share')))
-
-            def obj_code(self):
-                # 'href', 'alt' & 'title' attributes all contain the ISIN
-                isin = Attr(TableCell('label')(self)[0], 'title', default=NotAvailable)(self)
-                return IsinCode(default=NotAvailable).filter(isin)
-
-            obj_code_type = IsinType(Field('code'), default=NotAvailable)
-
-    @method
-    class iter_history(ListElement):
-        # Historique des versements:
-        class iter_versements(ListElement):
-            item_xpath = '//fieldset[legend[text()="Historique des versements"]]/table/tr[@class!="place"]'
-
-            class item(ItemElement):
-                klass = Transaction
-
-                obj_date = Date(CleanText('.//td[3]'), dayfirst=True)
-                obj_label = Format('Versement %s', CleanText('.//td[4]'))
-                obj_amount = CleanDecimal.French('.//td[6]')
-
-        # Historique des Rachats partiels:
-        class iter_partial_repurchase(ListElement):
-            item_xpath = '//fieldset[legend[text()="Historique des Rachats partiels"]]/table/tr[@class!="place"]'
-
-            class item(ItemElement):
-                klass = Transaction
-
-                obj_date = Date(CleanText('.//td[3]'), dayfirst=True)
-                obj_label = Format('Rachat %s', CleanText('.//td[4]'))
-                obj_amount = CleanDecimal.French('.//td[5]')
-
-        # Historique des demandes d´avance:
-        class iter_advances(ListElement):
-            item_xpath = '//fieldset[legend[text()="Historique des demandes d´avance"]]/table/tr[@class!="place"]'
-
-            class item(ItemElement):
-                klass = Transaction
-
-                obj_date = Date(CleanText('.//td[3]'), dayfirst=True)
-                obj_label = Format('Demande d\'avance %s', CleanText('.//td[4]'))
-                obj_amount = CleanDecimal.French('.//td[5]')
-
-        '''
-        - We do not fetch the "Historique des arbitrages" category
-          because the transactions have no available amount.
-        - The part below will crash if the remaining table is not empty:
-          it will be the occasion to implement the scraping of these transactions.
-        '''
-        class iter_other(ListElement):
-            def parse(self, el):
-                texts = [
-                    'Sécurisation des plus values',
-                ]
-                for text in texts:
-                    assert CleanText('.')(self.page.doc.xpath(
-                        '//fieldset[legend[text()=$text]]//div[@class="noRecord"]',
-                        text=text,
-                    )[0]), '%s is not handled' % text
-
-
 class PortfolioPage(BasePage):
     # we don't do anything here, but we might land here from a SSO like ing
     pass

modules/boursorama/pages.py

@@ -397,7 +397,7 @@ def obj_url(self):
                 return urljoin(self.page.url, link)
 
             def is_external(self):
-                return '/budget/' in Field('url')(self)
+                return '/budget/' in Field('url')(self) or '/crypto/' in Field('url')(self)
 
             def obj__idparts(self):
                 return re.findall(r'[a-z\d]{32}', Field('url')(self))

modules/caissedepargne/browser.py

@@ -1956,6 +1956,8 @@ def download_document(self, document):
             self.home_tache.go(tache='EPASYNT0')
         self.page.go_subscription()
         assert self.subscription.is_here()
+        self.page.change_year(document.date.year)
+        assert self.subscription.is_here()
 
         return self.page.download_document(document).content
 

modules/caissedepargne/module.py

@@ -30,7 +30,6 @@
     CapDocument, Subscription, SubscriptionNotFound,
     Document, DocumentNotFound, DocumentTypes,
 )
-from weboob.capabilities.base import NotAvailable
 from weboob.capabilities.contact import CapContact
 from weboob.capabilities.profile import CapProfile
 from weboob.capabilities.base import find_object
@@ -58,7 +57,9 @@ class CaisseEpargneModule(Module, CapBankWealth, CapBankTransferAddRecipient, Ca
         ValueTransient('request_information'),
     )
 
-    accepted_document_types = (DocumentTypes.STATEMENT, DocumentTypes.OTHER,)
+    accepted_document_types = (
+        DocumentTypes.STATEMENT, DocumentTypes.OTHER, DocumentTypes.NOTICE,
+    )
 
     def create_default_browser(self):
         return self.create_browser(
@@ -169,9 +170,6 @@ def download_document(self, document):
         if not isinstance(document, Document):
             document = self.get_document(document)
 
-        if document.url is NotAvailable:
-            return
-
         return self.browser.download_document(document)
 
     def iter_emitters(self):

modules/caissedepargne/pages.py

@@ -2490,7 +2490,7 @@ class iter_subscription(ListElement):
         class item(ItemElement):
             klass = Subscription
 
-            obj_id = CleanDecimal('.')
+            obj_id = Regexp(CleanText('.'), r"(\d+)")
             obj_label = Regexp(CleanText('.'), r'([^\d]*) ')
             obj_subscriber = Field('label')
 
@@ -2525,10 +2525,14 @@ class item(ItemElement):
             obj_label = Format('%s %s', CleanText('./td[3]'), CleanText('./td[2]'))
             obj_date = Date(CleanText('./td[2]'), dayfirst=True)
 
-            def obj_type(self):
-                if 'Relevé' in CleanText('./td[3]')(self):
-                    return DocumentTypes.STATEMENT
-                return DocumentTypes.OTHER
+            _type_text = CleanText('./td[3]')
+
+            TYPES_PATTERNS = {
+                "Relevé": DocumentTypes.STATEMENT,
+                "Info préalable à tarification": DocumentTypes.NOTICE,
+                "Information annuelle": DocumentTypes.NOTICE,
+            }
+            obj_type = MapIn(_type_text, TYPES_PATTERNS, default=DocumentTypes.OTHER)
 
     def download_document(self, document):
         form = self.get_form(id='main')

modules/creditcooperatif/module.py

@@ -18,6 +18,7 @@
 # along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
 
 from weboob.capabilities.bank import CapBankTransferAddRecipient
+from weboob.capabilities.bill import CapDocument
 from weboob.capabilities.profile import CapProfile
 from weboob.tools.backend import AbstractModule, BackendConfig
 from weboob.tools.value import ValueBackendPassword, Value, ValueTransient
@@ -28,7 +29,7 @@
 __all__ = ['CreditCooperatifModule']
 
 
-class CreditCooperatifModule(AbstractModule, CapBankTransferAddRecipient, CapProfile):
+class CreditCooperatifModule(AbstractModule, CapBankTransferAddRecipient, CapDocument, CapProfile):
     NAME = 'creditcooperatif'
     MAINTAINER = u'Kevin Pouget'
     EMAIL = 'weboob@kevin.pouget.me'

modules/delubac/pages.py

@@ -181,11 +181,7 @@ class item(ItemElement):
             obj_date = Date(CleanText(TableCell('date')), dayfirst=True)
             obj_vdate = Date(CleanText(TableCell('vdate')), dayfirst=True)
             obj_raw = obj_label = Transaction.Raw(TableCell('raw'))
-
-            def obj_amount(self):
-                amount = re.search(r'\-*\d+\.\d+', CleanText(TableCell('amount'))(self))
-                if amount:
-                    return CleanDecimal.US(amount.group(0))(self)
+            obj_amount = CleanDecimal.US(CleanText(TableCell('amount')))
 
             def condition(self):
                 return Field('amount')(self)

modules/hsbc/browser.py

@@ -32,7 +32,7 @@
 from weboob.tools.capabilities.bank.transactions import sorted_transactions, keep_only_card_transactions
 from weboob.tools.compat import parse_qsl, urlparse
 from weboob.tools.value import Value
-from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable, BrowserQuestion
+from weboob.exceptions import ActionNeeded, BrowserIncorrectPassword, BrowserUnavailable, BrowserQuestion
 from weboob.browser import URL, need_login, TwoFactorBrowser
 from weboob.browser.exceptions import HTTPNotFound
 from weboob.capabilities.base import find_object
@@ -190,6 +190,12 @@ def init_login(self):
         if no_secure_key_link:
             self.location(no_secure_key_link)
         else:
+            error = self.page.get_error()
+            if error and 'Please click Reset Credentials' in error:
+                raise ActionNeeded(error)
+            elif error:
+                raise AssertionError('Unhandled error at login: %s' % error)
+
             self.check_interactive()
             raise BrowserQuestion(
                 Value(

modules/hsbc/pages/account_pages.py

@@ -554,6 +554,9 @@ def login(self, login):
         form['userid'] = form['__hbfruserid'] = login
         form.submit()
 
+    def get_error(self):
+        return CleanText('//div[contains(@class, "PanelMsgGroup")]')(self.doc)
+
     def get_no_secure_key_link(self):
         try:
             a = self.doc.xpath('//a[contains(text(), "Without HSBC Secure Key")]')[0]

modules/infomaniak/pages.py

@@ -77,6 +77,6 @@ class item(ItemElement):
             obj_date = Eval(datetime.fromtimestamp, Dict('created_at'))
             obj_label = Format('Facture %s', obj_number)
             obj_url = Dict('document/href')
-            obj_price = CleanDecimal(Dict('amount/amount'))
-            obj_currency = Currency(Dict('amount/currency'))
+            obj_total_price = CleanDecimal.SI(Dict('amount/amount_incl_tax'))
+            obj_currency = Currency(Dict('amount/currency_code'))
             obj_format = 'pdf'

modules/ing/api_browser.py

@@ -267,7 +267,7 @@ def iter_accounts(self):
                 continue
 
             self.go_bourse(account)
-            bourse_accounts = list(self.bourse.iter_accounts_but_insurances())
+            bourse_accounts = list(self.bourse.iter_accounts())
 
             for bourse_account in bourse_accounts:
                 # bourse number is in format 111TI11111119999EUR

modules/lcl/enterprise/browser.py

@@ -82,12 +82,18 @@ def get_history(self, account):
     def get_profile(self):
         return self.profile.go().get_profile()
 
+    @need_login
     def get_coming(self, account):
         raise NotImplementedError()
 
+    @need_login
     def get_investment(self, account):
         raise NotImplementedError()
 
+    @need_login
+    def iter_market_orders(self, account):
+        raise NotImplementedError()
+
 
 class LCLEspaceProBrowser(LCLEnterpriseBrowser):
     BASEURL = 'https://espacepro.secure.lcl.fr'

modules/societegenerale/browser.py

@@ -478,7 +478,7 @@ def iter_history(self, account):
         # get history for account on old website
         # request to get json is not available yet, old request to get html response
         if any((
-                account.type in (account.TYPE_LIFE_INSURANCE, account.TYPE_PERP),
+                account.type in (account.TYPE_LIFE_INSURANCE, account.TYPE_CAPITALISATION, account.TYPE_PERP),
                 account.type == account.TYPE_REVOLVING_CREDIT and account._loan_type != 'PR_CONSO',
                 account.type in (account.TYPE_REVOLVING_CREDIT, account.TYPE_SAVINGS) and not account._is_json_histo,
         )):
@@ -534,6 +534,7 @@ def iter_coming(self, account):
             Account.TYPE_MARKET,
             Account.TYPE_PEA,
             Account.TYPE_LIFE_INSURANCE,
+            Account.TYPE_CAPITALISATION,
             Account.TYPE_REVOLVING_CREDIT,
             Account.TYPE_CONSUMER_CREDIT,
             Account.TYPE_PERP,
@@ -580,7 +581,7 @@ def iter_coming(self, account):
     @need_login
     def iter_investment(self, account):
         if account.type not in (
-            Account.TYPE_MARKET, Account.TYPE_LIFE_INSURANCE,
+            Account.TYPE_MARKET, Account.TYPE_LIFE_INSURANCE, Account.TYPE_CAPITALISATION,
             Account.TYPE_PEA, Account.TYPE_PERP,
         ):
             self.logger.debug('This account is not supported')

modules/societegenerale/pages/accounts_list.py

@@ -192,6 +192,8 @@ def condition(self):
                 'EBENE_CAPITALISATION': Account.TYPE_LIFE_INSURANCE,
                 'ASSURANCE_VIE_GENERALE': Account.TYPE_LIFE_INSURANCE,
                 'ASSURANCE_VIE_SOGECAP_GENERAL': Account.TYPE_LIFE_INSURANCE,
+                'VIE_AXA': Account.TYPE_LIFE_INSURANCE,
+                'CAPI_AGF': Account.TYPE_CAPITALISATION,
                 'RESERVEA': Account.TYPE_REVOLVING_CREDIT,
                 'COMPTE_ALTERNA': Account.TYPE_REVOLVING_CREDIT,
                 'CREDIT_CONFIANCE': Account.TYPE_REVOLVING_CREDIT,

modules/trainline/browser.py

@@ -39,6 +39,8 @@ def __init__(self, login, password, *args, **kwargs):
         self.session.headers['X-Requested-With'] = 'XMLHttpRequest'
 
     def do_login(self):
+        # without this additional header we get a timeout while using a proxy
+        self.session.headers['Proxy-Connection'] = 'keep-alive'
         # set some cookies
         self.go_home()