modules/amazon/browser.py

@@ -18,12 +18,15 @@
 # along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
 
 from __future__ import unicode_literals
+
+import time
 from datetime import date
 
 from weboob.browser.browsers import LoginBrowser, URL, need_login, StatesMixin
 from weboob.exceptions import (
     BrowserIncorrectPassword, BrowserUnavailable, ImageCaptchaQuestion, BrowserQuestion,
-    WrongCaptchaResponse, AuthMethodNotImplemented, NeedInteractiveFor2FA, BrowserPasswordExpired,
+    WrongCaptchaResponse, AuthMethodNotImplemented, NeedInteractiveFor2FA,
+    BrowserPasswordExpired, AppValidation, AppValidationExpired,
 )
 from weboob.tools.value import Value
 from .compat.weboob_browser_browsers import ClientError
@@ -107,6 +110,10 @@ def handle_security(self):
             # many captcha, reset value
             self.config['captcha_response'] = Value(value=None)
         else:
+            msg_validation = self.page.get_msg_app_validation()
+            if 'approve the notification' in msg_validation:
+                raise AppValidation(msg_validation)
+
             otp_type = self.page.get_otp_type()
             if otp_type == '/ap/signin':
                 # this otp will be always present until user deactivate it
@@ -138,6 +145,19 @@ def handle_captcha(self, captcha):
         image = self.open(captcha[0]).content
         raise ImageCaptchaQuestion(image)
 
+    def check_app_validation(self):
+        # client has 60 seconds to unlock this page
+        timeout = time.time() + 60.00
+        while time.time() < timeout:
+            link = self.page.get_link_app_validation()
+            self.location(link)
+            if self.security.is_here():
+                time.sleep(2)
+            else:
+                return
+        else:
+            raise AppValidationExpired()
+
     def do_login(self):
         if self.config['pin_code'].get():
             # Resolve pin_code
@@ -189,6 +209,9 @@ def do_login(self):
 
         self.page.login(self.username, self.password)
 
+        if self.config['resume'].get():
+            self.check_app_validation()
+
         if self.password_expired.is_here():
             raise BrowserPasswordExpired(self.page.get_message())
 

modules/amazon/module.py

@@ -24,7 +24,7 @@
 from weboob.capabilities.base import find_object, NotAvailable
 from weboob.tools.backend import Module, BackendConfig
 from weboob.tools.compat import urljoin
-from weboob.tools.value import ValueBackendPassword, Value
+from weboob.tools.value import ValueBackendPassword, Value, ValueTransient
 from weboob.tools.pdf import html_to_pdf
 
 from .browser import AmazonBrowser
@@ -64,6 +64,7 @@ class AmazonModule(Module, CapDocument):
         Value('captcha_response', label='Captcha Response', required=False, default=''),
         Value('pin_code', label='OTP response', required=False, default=''),
         Value('request_information', label='request_information', default=None, required=False, noprompt=True),
+        ValueTransient('resume'),
     )
 
     accepted_document_types = (DocumentTypes.BILL,)

modules/amazon/pages.py

@@ -58,6 +58,14 @@ def get_otp_type(self):
         assert url in ('verify', '/ap/signin'), url
         return url
 
+    def get_msg_app_validation(self):
+        msg = CleanText('//span[contains(@class, "transaction-approval-word-break")]')(self.doc)
+        if "To complete the sign-in, approve the notification sent to" in msg:
+            return msg
+
+    def get_link_app_validation(self):
+        return Link('//a[contains(text(), "Click here to refresh the page")]')(self.doc)
+
     def get_otp_message(self):
         return CleanText('//div[@class="a-box-inner"]/p')(self.doc)
 

modules/ameli/browser.py

@@ -23,12 +23,14 @@
 from time import time
 from dateutil.relativedelta import relativedelta
 
-from weboob.browser import LoginBrowser, URL, need_login
+from weboob.browser.browsers import LoginBrowser, URL, need_login
 from weboob.exceptions import ActionNeeded
+from .compat.weboob_tools_capabilities_bill_documents import merge_iterators
 
 from .pages import (
     ErrorPage, LoginPage, RedirectPage, CguPage,
-    SubscriptionPage, DocumentsPage, CtPage,
+    SubscriptionPage, DocumentsDetailsPage, CtPage, DocumentsFirstSummaryPage,
+    DocumentsLastSummaryPage,
 )
 
 
@@ -40,7 +42,15 @@ class AmeliBrowser(LoginBrowser):
     redirect_page = URL(r'/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&.*validationconnexioncompte.*', RedirectPage)
     cgu_page = URL(r'/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&_pageLabel=as_conditions_generales_page.*', CguPage)
     subscription_page = URL(r'/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&_pageLabel=as_info_perso_page.*', SubscriptionPage)
-    documents_page = URL(r'/PortailAS/paiements.do', DocumentsPage)
+    documents_details_page = URL(r'/PortailAS/paiements.do', DocumentsDetailsPage)
+    documents_first_summary_page = URL(
+        r'PortailAS/appmanager/PortailAS/assure\?_nfpb=true&_pageLabel=as_releve_mensuel_paiement_page',
+        DocumentsFirstSummaryPage
+    )
+    documents_last_summary_page = URL(
+        r'PortailAS/portlets/relevemensuelpaiement/relevemensuelpaiement.do\?actionEvt=afficherPlusReleves',
+        DocumentsLastSummaryPage
+    )
     ct_page = URL(r'/PortailAS/JavaScriptServlet', CtPage)
 
     def do_login(self):
@@ -58,19 +68,16 @@ def iter_subscription(self):
         yield self.page.get_subscription()
 
     @need_login
-    def iter_documents(self, subscription):
+    def _iter_details_documents(self, subscription):
         end_date = date.today()
 
         start_date = end_date - relativedelta(years=1)
-        # FUN FACT, website tell us documents are available for 6 months
-        # let's suppose today is 28/05/19, website frontend limit DateDebut to 28/11/18 but we can get a little bit more
-        # by setting a previous date and get documents that are no longer available for simple user
 
         params = {
             'Beneficiaire': 'tout_selectionner',
             'DateDebut': start_date.strftime('%d/%m/%Y'),
             'DateFin': end_date.strftime('%d/%m/%Y'),
-            'actionEvt': 'afficherPaiementsComplementaires',
+            'actionEvt': 'Rechercher',
             'afficherIJ': 'false',
             'afficherInva': 'false',
             'afficherPT': 'false',
@@ -80,14 +87,26 @@ def iter_documents(self, subscription):
             'idNoCache': int(time()*1000)
         }
 
-        # the second request is stateful
-        # first value of actionEvt is afficherPaiementsComplementaires to get all payments from last 6 months
-        # (start_date 6 months in the past is needed but not enough)
-        self.documents_page.go(params=params)
-
-        # then we set Rechercher to actionEvt to filter for this subscription, within last 6 months
-        # without first request we would have filter for this subscription but within last 2 months
-        params['actionEvt'] = 'Rechercher'
-        params['Beneficiaire'] = 'tout_selectionner'
-        self.documents_page.go(params=params)
+        # website tell us details documents are available for 6 months
+        self.documents_details_page.go(params=params)
         return self.page.iter_documents(subid=subscription.id)
+
+    @need_login
+    def _iter_summary_documents(self, subscription):
+        # The monthly statements for the last 23 months are available in two parts.
+        # The first part contains the last 6 months on an HTML page.
+        self.documents_first_summary_page.go()
+        for doc in self.page.iter_documents(subid=subscription.id):
+            yield doc
+
+        # The second part is retrieved in JSON via this page which displays the next 6 months at each iteration.
+        for _ in range(3):
+            self.documents_last_summary_page.go()
+            for doc in self.page.iter_documents(subid=subscription.id):
+                yield doc
+
+
+    @need_login
+    def iter_documents(self, subscription):
+        for doc in merge_iterators(self._iter_details_documents(subscription), self._iter_summary_documents(subscription)):
+            yield doc

modules/ameli/compat/weboob_tools_capabilities_bill_documents.py

+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2020  Budget Insight
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+from collections import OrderedDict
+
+
+__all__ = ['sorted_documents', 'merge_iterators']
+
+
+def sorted_documents(iterable):
+    """Sort an iterable of documents in reverse chronological order"""
+    return sorted(iterable, reverse=True, key=lambda doc: doc.date)
+
+
+def merge_iterators(*iterables):
+    """Merge documents iterators keeping sort order.
+
+    Each iterator must already be sorted in reverse chronological order.
+    """
+
+    def keyfunc(kv):
+        return kv[1].date
+
+    its = OrderedDict((iter(it), None) for it in iterables)
+    for k in list(its):
+        try:
+            its[k] = next(k)
+        except StopIteration:
+            del its[k]
+
+    while its:
+        k, v = max(its.items(), key=keyfunc)
+        yield v
+
+        try:
+            its[k] = next(k)
+        except StopIteration:
+            del its[k]
+

modules/ameli/module.py

@@ -41,8 +41,10 @@ class AmeliModule(Module, CapDocument):
 
     BROWSER = AmeliBrowser
 
-    CONFIG = BackendConfig(ValueBackendPassword('login', label='Mon numero de sécurité sociale', regexp=r'^\d{13}$', masked=False),
-                           ValueBackendPassword('password', label='Mon code (4 à 13 chiffres)', regexp=r'^\d{4,13}', masked=True))
+    CONFIG = BackendConfig(
+        ValueBackendPassword('login', label='Mon numero de sécurité sociale', regexp=r'\d{13}', masked=False),
+        ValueBackendPassword('password', label='Mon code personnel', regexp=r'\S{8,50}', masked=True),
+    )
 
     accepted_document_types = (DocumentTypes.BILL,)
 

modules/ameli/pages.py

@@ -23,11 +23,12 @@
 
 from hashlib import sha1
 
-from weboob.browser.elements import method, ListElement, ItemElement
+from weboob.browser.elements import method, ListElement, ItemElement, DictElement
 from weboob.browser.filters.html import Link
-from .compat.weboob_browser_filters_standard import CleanText, Regexp, CleanDecimal, Currency, Field, Env
-from .compat.weboob_browser_pages import LoggedPage, HTMLPage, PartialHTMLPage, RawPage
-from weboob.capabilities.bill import Subscription, Bill
+from .compat.weboob_browser_filters_standard import CleanText, Regexp, CleanDecimal, Currency, Field, Env, Format
+from weboob.browser.filters.json import Dict
+from .compat.weboob_browser_pages import LoggedPage, HTMLPage, PartialHTMLPage, RawPage, JsonPage
+from weboob.capabilities.bill import Subscription, Bill, Document, DocumentTypes
 from weboob.exceptions import BrowserUnavailable
 from weboob.tools.date import parse_french_date
 from weboob.tools.json import json
@@ -74,12 +75,12 @@ def get_subscription(self):
         return sub
 
 
-class DocumentsPage(LoggedPage, PartialHTMLPage):
+class DocumentsDetailsPage(LoggedPage, PartialHTMLPage):
     ENCODING = 'utf-8'
 
     def build_doc(self, content):
         res = json.loads(content)
-        return super(DocumentsPage, self).build_doc(res['tableauPaiement'].encode('utf-8'))
+        return super(DocumentsDetailsPage, self).build_doc(res['tableauPaiement'].encode('utf-8'))
 
     @method
     class iter_documents(ListElement):
@@ -105,3 +106,61 @@ def obj_date(self):
                 day_month = CleanText('.//div[has-class("col-date")]/span')(self)
 
                 return parse_french_date(day_month + ' ' + year)
+
+
+class DocumentsFirstSummaryPage(LoggedPage, HTMLPage):
+
+    @method
+    class iter_documents(ListElement):
+        item_xpath = '//ul[@id="unordered_list"]//li[@class="rowdate" and .//span[@class="blocTelecharger"]]'
+
+        class item(ItemElement):
+            klass = Document
+
+            obj_type = DocumentTypes.BILL
+            obj_label = Format('%s %s', CleanText('.//span[@class="libelle"]'), CleanText('.//span[@class="mois"]'))
+            obj_url = Link('.//div[@class="col-telechargement"]//a')
+            obj_format = 'pdf'
+
+            def obj_date(self):
+                year = Regexp(CleanText('.//span[@class="mois"]'), r'(\d+)')(self)
+                month = Regexp(CleanText('.//span[@class="mois"]'), r'(\D+)')(self)
+
+                return parse_french_date(month + ' ' + year)
+
+            def obj_id(self):
+                year = Regexp(CleanText('.//span[@class="mois"]'), r'(\d+)')(self)
+                month = Regexp(CleanText('.//span[@class="mois"]'), r'(\D+)')(self)
+
+                return '%s_%s' % (Env('subid')(self), parse_french_date(month + ' ' + year).strftime('%Y%m'))
+
+
+class DocumentsLastSummaryPage(LoggedPage, JsonPage):
+
+    @method
+    class iter_documents(DictElement):
+
+        def find_elements(self):
+            for doc in self.el['listeDecomptes']:
+                if doc['montant']:
+                    yield doc
+
+        class item(ItemElement):
+            klass = Document
+
+            obj_type = DocumentTypes.BILL
+            obj_url = Dict('urlPDF')
+            obj_format = 'pdf'
+            obj_label = Format('Relevé mensuel %s', CleanText(Dict('mois')))
+
+            def obj_date(self):
+                year = Regexp(CleanText(Dict('mois')), r'(\d+)')(self)
+                month = Regexp(CleanText(Dict('mois')), r'(\D+)')(self)
+
+                return parse_french_date(month + ' ' + year)
+
+            def obj_id(self):
+                year = Regexp(CleanText(Dict('mois')), r'(\d+)')(self)
+                month = Regexp(CleanText(Dict('mois')), r'(\D+)')(self)
+
+                return '%s_%s' % (Env('subid')(self), parse_french_date(month + ' ' + year).strftime('%Y%m'))

modules/banquepopulaire/browser.py

@@ -26,8 +26,8 @@
 from datetime import datetime
 from collections import OrderedDict
 from functools import wraps
-
 from dateutil.relativedelta import relativedelta
+
 from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable
 from weboob.browser.exceptions import HTTPNotFound, ClientError, ServerError
 from weboob.browser.browsers import LoginBrowser, URL, need_login
@@ -960,11 +960,24 @@ def iter_documents(self, subscription):
             ],
             'inListeTypesDocuments': [
                 {'typeDocument': {'code': 'EXTRAIT', 'label': 'Extrait de compte', 'type': 'referenceLogiqueDocument'}},
-                {'typeDocument': {'code': 'RLVCB  ', 'label': 'Relevé Carte Bancaire', 'type': 'referenceLogiqueDocument'}}
-                # space at the end of 'RLVCB  ' is mandatory else => error 500
+                # space at the end of 'RELVCB ' is mandatory
+                {'typeDocument': {'code': 'RELVCB ', 'label': 'Relevé Carte Bancaire', 'type': 'referenceLogiqueDocument'}}
             ]
         }
-        self.documents_page.go(json=body, headers=self.documents_headers)
+        # if the syntax is not exactly the correct one we have an error 400 for card statement
+        # banquepopulaire has subdomain so the param change if we are in subdomain or not
+        # if we are in subdomain the param for card statement is 'RLVCB  '
+        # else the param is 'RELVCB '
+        try:
+            self.documents_page.go(json=body, headers=self.documents_headers)
+        except ClientError as e:
+            if e.response.status_code == 400:
+                # two spaces at the end of 'RLVCB  ' is mandatory
+                body['inListeTypesDocuments'][1] = {'typeDocument': {'code': 'RLVCB  ', 'label': 'Relevé Carte Bancaire', 'type': 'referenceLogiqueDocument'}}
+                self.documents_page.go(json=body, headers=self.documents_headers)
+            else:
+                raise
+
         return self.page.iter_documents(subid=subscription.id)
 
     @retry(ClientError)

modules/barclays/browser.py

@@ -17,6 +17,7 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
 
+# flake8: compatible
 
 from __future__ import unicode_literals
 
@@ -41,16 +42,16 @@ class Barclays(LoginBrowser):
     logout = URL('https://www.milleis.fr/deconnexion')
     milleis_ajax = URL('/BconnectDesk/ajaxservletcontroller')
 
-    login = URL('/BconnectDesk/servletcontroller',                  LoginPage)
-    accounts = URL('/BconnectDesk/servletcontroller',               AccountsPage)
-    loan_account = URL('/BconnectDesk/servletcontroller',           LoanAccountPage)
-    account = URL('/BconnectDesk/servletcontroller',                AccountPage)
-    card_account = URL('/BconnectDesk/servletcontroller',           CardPage)
-    market_account = URL('/BconnectDesk/servletcontroller',         MarketAccountPage)
+    login = URL('/BconnectDesk/servletcontroller', LoginPage)
+    accounts = URL('/BconnectDesk/servletcontroller', AccountsPage)
+    loan_account = URL('/BconnectDesk/servletcontroller', LoanAccountPage)
+    account = URL('/BconnectDesk/servletcontroller', AccountPage)
+    card_account = URL('/BconnectDesk/servletcontroller', CardPage)
+    market_account = URL('/BconnectDesk/servletcontroller', MarketAccountPage)
     life_insurance_account = URL('/BconnectDesk/servletcontroller', LifeInsuranceAccountPage)
-    revolving_account = URL('/BconnectDesk/servletcontroller',      RevolvingAccountPage)
-    actionNeededPage = URL('/BconnectDesk/servletcontroller',       ActionNeededPage)
-    iban = URL('/BconnectDesk/editique',                            IbanPDFPage)
+    revolving_account = URL('/BconnectDesk/servletcontroller', RevolvingAccountPage)
+    actionNeededPage = URL('/BconnectDesk/servletcontroller', ActionNeededPage)
+    iban = URL('/BconnectDesk/editique', IbanPDFPage)
 
     def __init__(self, secret, *args, **kwargs):
         super(Barclays, self).__init__(*args, **kwargs)
@@ -72,7 +73,8 @@ def _go_to_account(self, account, refresh=False):
         else:
             if not self.accounts.is_here():
                 self.page.go_to_menu('Comptes et contrats')
-                if not self.accounts.is_here(): # Sometime we can't go out from account page, so re-login
+                if not self.accounts.is_here():
+                    # Sometime we can't go out from account page, so re-login
                     self._relogin()
 
             self.page.go_to_account(account)
@@ -93,8 +95,8 @@ def _go_to_account_space(self, space, account):
             'controllername': 'servletcontroller',
             'disable': 'false',
             'title': 'Milleis',
-            token[0]: token[1]
-         }
+            token[0]: token[1],
+        }
 
         self.milleis_ajax.open(data=data)
         self._go_to_account(account, refresh=True)
@@ -132,7 +134,7 @@ def iter_accounts(self):
         if not self.accounts.is_here():
             self.page.go_to_menu('Comptes et contrats')
 
-        if not 'accounts' in self.cache:
+        if 'accounts' not in self.cache:
             accounts = list(self.page.iter_accounts())
             traccounts = []
 
@@ -145,7 +147,10 @@ def iter_accounts(self):
                 if account.type == Account.TYPE_CHECKING:
                     # Only checking accounts have an IBAN
                     self._go_to_account(account)
-                    account.iban = self.iban.open().get_iban() if self.page.has_iban() else NotAvailable
+                    if self.page.has_iban():
+                        account.iban = self.iban.open().get_iban()
+                    else:
+                        account.iban = NotAvailable
 
                 if account.type == Account.TYPE_LOAN:
                     self._go_to_account(account)
@@ -159,7 +164,9 @@ def iter_accounts(self):
                     if not self.page.has_history():
                         continue
 
-                    account._attached_account = self.page.do_account_attachment([a for a in accounts if a.type == Account.TYPE_CHECKING])
+                    account._attached_account = self.page.do_account_attachment([
+                        a for a in accounts if a.type == Account.TYPE_CHECKING
+                    ])
 
                 if account.type == Account.TYPE_REVOLVING_CREDIT:
                     self._go_to_account(account)
@@ -174,8 +181,10 @@ def iter_accounts(self):
             # is not specified, therefore to avoid transaction duplicates,
             # we only return transactions from the 'EUR' twin account.
             for account in self.cache['accounts']:
-                if (account.id.replace(account.currency, '') in
-                    [acc.id.replace(acc.currency, '') for acc in self.cache['accounts'] if acc.id != account.id]):
+                accounts_id_without_currency = [
+                    acc.id.replace(acc.currency, '') for acc in self.cache['accounts'] if acc.id != account.id
+                ]
+                if account.id.replace(account.currency, '') in accounts_id_without_currency:
                     account._twin = True
                 else:
                     account._twin = False
@@ -208,7 +217,9 @@ def iter_history(self, account):
         history_page = self.page
 
         if account.type != Account.TYPE_LIFE_INSURANCE:
-            for _ in range(100): # on new history page they take previous results too, so go to the last page before starts recover history
+            for _ in range(100):
+                # on new history page they take previous results too,
+                # so go to the last page before starts recover history
                 form = history_page.form_to_history_page()
 
                 if not form:
@@ -216,13 +227,14 @@ def iter_history(self, account):
 
                 try:
                     history_page = self.account.open(data=form)
-                except ConnectionError: # Sometime accounts have too much history and website crash
+                except ConnectionError:
+                    # Sometime accounts have too much history and website crash
                     # Need to relogin
                     self._relogin()
 
                     break
             else:
-                assert False, "Too many iterations"
+                raise AssertionError('Too many iterations')
 
         if history_page.has_history():
             return list(history_page.iter_history())

modules/barclays/module.py

@@ -17,14 +17,13 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
 
+# flake8: compatible
 
 from __future__ import unicode_literals
 
-from .compat.weboob_capabilities_bank import AccountNotFound
 from .compat.weboob_capabilities_wealth import CapBankWealth
 from weboob.tools.backend import Module, BackendConfig
 from weboob.tools.value import ValueBackendPassword
-from weboob.capabilities.base import find_object
 
 from .browser import Barclays
 
@@ -34,27 +33,28 @@
 
 class BarclaysModule(Module, CapBankWealth):
     NAME = 'barclays'
-    MAINTAINER = u'Jean Walrave'
+    MAINTAINER = 'Jean Walrave'
     EMAIL = 'jwalrave@budget-insight.com'
     VERSION = '2.0'
-    DESCRIPTION = u'Barclays'
+    DESCRIPTION = 'Barclays'
     LICENSE = 'LGPLv3+'
-    CONFIG = BackendConfig(ValueBackendPassword('login',    label=u"N° d'abonné", masked=False),
-                           ValueBackendPassword('password', label='Code confidentiel'),
-                           ValueBackendPassword('secret',   label='Mot secret'))
+    CONFIG = BackendConfig(
+        ValueBackendPassword('login', label="N° d'abonné", masked=False),
+        ValueBackendPassword('password', label='Code confidentiel'),
+        ValueBackendPassword('secret', label='Mot secret'),
+    )
     BROWSER = Barclays
 
     def create_default_browser(self):
-        return self.create_browser(self.config['secret'].get(),
-                                   self.config['login'].get(),
-                                   self.config['password'].get())
+        return self.create_browser(
+            self.config['secret'].get(),
+            self.config['login'].get(),
+            self.config['password'].get(),
+        )
 
     def iter_accounts(self):
         return self.browser.iter_accounts()
 
-    def get_account(self, _id):
-        return find_object(self.browser.iter_accounts(), id=_id, error=AccountNotFound)
-
     def iter_history(self, account):
         return self.browser.iter_history(account)
 

modules/bforbank/pages.py

@@ -96,7 +96,7 @@ def login(self, birthdate, username, password):
         code = vk.get_string_code(password)
         form = self.get_form()
         form['j_username'] = username
-        form['birthDate'] = birthdate
+        form['birthDate'] = birthdate.strftime('%d/%m/%Y')
         form['indexes'] = code
         form.submit()
 

modules/boursorama/browser.py

@@ -50,7 +50,8 @@
     CardsNumberPage, CalendarPage, HomePage, PEPPage,
     TransferAccounts, TransferRecipients, TransferCharac, TransferConfirm, TransferSent,
     AddRecipientPage, StatusPage, CardHistoryPage, CardCalendarPage, CurrencyListPage, CurrencyConvertPage,
-    AccountsErrorPage, NoAccountPage, TransferMainPage, PasswordPage,
+    AccountsErrorPage, NoAccountPage, TransferMainPage, PasswordPage, NewTransferRecipients,
+    NewTransferAccounts,
 )
 from .transfer_pages import TransferListPage, TransferInfoPage
 
@@ -121,6 +122,15 @@ class BoursoramaBrowser(RetryLoginBrowser, TwoFactorBrowser):
         r'/compte/(?P<type>[^/]+)/(?P<webid>\w+)/virements/nouveau/(?P<id>\w+)/2',
         TransferRecipients
     )
+    new_transfer_accounts = URL(
+        r'/compte/(?P<acc_type>[^/]+)/(?P<webid>\w+)/virements/immediat/nouveau/?$',
+        r'/compte/(?P<type>[^/]+)/(?P<webid>\w+)/virements/immediat/nouveau/(?P<id>\w+)/1',
+        NewTransferAccounts
+    )
+    new_recipients_page = URL(
+        r'/compte/(?P<type>[^/]+)/(?P<webid>\w+)/virements/immediat/nouveau/(?P<id>\w+)/2',
+        NewTransferRecipients
+    )
     transfer_charac = URL(
         r'/compte/(?P<type>[^/]+)/(?P<webid>\w+)/virements/nouveau/(?P<id>\w+)/3',
         TransferCharac
@@ -560,6 +570,9 @@ def go_recipients_list(self, account_url, account_id):
 
         if self.transfer_accounts.is_here():
             self.page.submit_account(account_id)  # may raise AccountNotFound
+        elif self.transfer_main_page.is_here():
+            self.new_transfer_accounts.go(acc_type=account_type, webid=account_webid)
+            self.page.submit_account(account_id)  # may raise AccountNotFound
 
     @need_login
     def iter_transfer_recipients(self, account):
@@ -572,7 +585,11 @@ def iter_transfer_recipients(self, account):
         except (BrowserHTTPNotFound, AccountNotFound):
             return []
 
-        assert self.recipients_page.is_here()
+        assert (
+            self.recipients_page.is_here()
+            or self.new_recipients_page.is_here()
+        ), 'Should be on recipients page'
+
         return self.page.iter_recipients()
 
     def check_basic_transfer(self, transfer):
@@ -600,6 +617,9 @@ def init_transfer(self, transfer, **kwargs):
             raise TransferInvalidRecipient('The recipient cannot be used with the emitter account')
         assert len(recipients) == 1
 
+        if self.new_recipients_page.is_here():
+            raise NotImplementedError('The new transfer pages are not yet implemented')
+
         self.page.submit_recipient(recipients[0]._tempid)
         assert self.transfer_charac.is_here()
 

modules/boursorama/pages.py

@@ -1255,6 +1255,60 @@ def submit_recipient(self, tempid):
         form.submit()
 
 
+class NewTransferRecipients(LoggedPage, HTMLPage):
+    @method
+    class iter_recipients(ListElement):
+        item_xpath = '//div[contains(@id, "panel-")]//div[contains(@class, "panel__body")]//label'
+
+        class item(ItemElement):
+            klass = Recipient
+
+            obj_id = CleanText(
+                './/span[contains(@class, "sub-label")]/span[not(contains(@class,"sub-label"))]',
+                replace=[(' ', '')],
+            )
+
+            obj_label = Regexp(
+                CleanText('.//span[contains(@class, "account-label")]'),
+                r'([^-]+)',
+                '\\1',
+            )
+
+            def obj_category(self):
+                text = CleanText(
+                    './ancestor::div[contains(@class, "panel__body")]'
+                    + '/preceding-sibling::div[contains(@class, "panel__header")]'
+                    + '//span[contains(@class, "panel__title")]'
+                )(self).lower()
+                if 'mes comptes boursorama banque' in text:
+                    return 'Interne'
+                elif any(exp in text for exp in ('comptes externes', 'comptes de tiers', 'mes bénéficiaires')):
+                    return 'Externe'
+
+            def obj_iban(self):
+                if Field('category')(self) == 'Externe':
+                    return Field('id')(self)
+                return NotAvailable
+
+            def obj_enabled_at(self):
+                return datetime.datetime.now().replace(microsecond=0)
+
+            obj__tempid = Attr('./input', 'value')
+
+
+class NewTransferAccounts(LoggedPage, HTMLPage):
+    def submit_account(self, account_id):
+        form = self.get_form()
+        debit_account = CleanText(
+            '//input[./following-sibling::div/span/span[contains(text(), "%s")]]/@value' % account_id
+        )(self.doc)
+        if not debit_account:
+            raise AccountNotFound()
+
+        form['DebitAccount[debit]'] = debit_account
+        form.submit()
+
+
 class TransferCharac(LoggedPage, HTMLPage):
     def get_option(self, select, text):
         for opt in select.xpath('option'):

modules/bp/browser.py

@@ -40,7 +40,7 @@
 from weboob.tools.decorators import retry
 from .compat.weboob_capabilities_bank import (
     Account, Recipient, AddRecipientStep, TransferStep,
-    TransferInvalidEmitter,
+    TransferInvalidEmitter, RecipientInvalidOTP,
 )
 from weboob.tools.value import Value, ValueBool
 
@@ -51,7 +51,7 @@
     ValidateCountry, ConfirmPage, RcptSummary,
     SubscriptionPage, DownloadPage, ProSubscriptionPage,
     RevolvingAttributesPage,
-    TwoFAPage, Validated2FAPage, SmsPage, DecoupledPage, HonorTransferPage, RecipientSubmitDevicePage,
+    TwoFAPage, Validated2FAPage, SmsPage, DecoupledPage, HonorTransferPage, RecipientSubmitDevicePage, RcptErrorPage,
 )
 from .pages.accounthistory import (
     LifeInsuranceInvest, LifeInsuranceHistory, LifeInsuranceHistoryInv, RetirementHistory,
@@ -303,6 +303,10 @@ class BPBrowser(LoginBrowser, StatesMixin):
         r'/voscomptes/canalXHTML/virement/mpiGestionBeneficiairesVirementsCreationBeneficiaire/validerRecapBeneficiaire-creationBeneficiaire.ea',
         ConfirmPage
     )
+    rcpt_error = URL(
+        r'/voscomptes/canalXHTML/securisation/otp/validation-securisationOTP.ea',
+        RcptErrorPage,
+    )
     rcpt_summary = URL(
         r'/voscomptes/canalXHTML/virement/mpiGestionBeneficiairesVirementsCreationBeneficiaire/finalisation-creationBeneficiaire.ea',
         RcptSummary
@@ -374,7 +378,7 @@ class BPBrowser(LoginBrowser, StatesMixin):
 
     accounts = None
 
-    __states__ = ('need_reload_state', 'sms_form')
+    __states__ = ('need_reload_state', 'sms_form', 'recipient_form')
 
     def __init__(self, config, *args, **kwargs):
         self.weboob = kwargs.pop('weboob')
@@ -407,9 +411,6 @@ def load_state(self, state):
             super(BPBrowser, self).load_state(state)
             self.need_reload_state = None
 
-        if 'recipient_form' in state and state['recipient_form'] is not None:
-            self.logged = True
-
     def deinit(self):
         super(BPBrowser, self).deinit()
         self.linebourse.deinit()
@@ -893,7 +894,7 @@ def init_new_recipient(self, recipient, is_bp_account=False, **params):
 
         # Case of SMS OTP
         self.page.set_browser_form()
-        raise AddRecipientStep(self.build_recipient(recipient), Value('code', label='Veuillez saisir votre code de validation'))
+        raise AddRecipientStep(self.build_recipient(recipient), Value('code', label='Veuillez saisir le code reçu par SMS'))
 
     def new_recipient(self, recipient, is_bp_account=False, **params):
         if params.get('resume') or self.resume:
@@ -902,11 +903,17 @@ def new_recipient(self, recipient, is_bp_account=False, **params):
 
         if 'code' in params:
             # Case of SMS OTP
-            assert self.rcpt_code.is_here()
-
             self.post_code(params['code'])
             self.recipient_form = None
-            assert self.rcpt_summary.is_here()
+
+            if self.rcpt_error.is_here():
+                error = self.page.get_error()
+                if error:
+                    if 'Votre code sécurité est incorrect' in error:
+                        raise RecipientInvalidOTP(message=error)
+                    raise AssertionError('Unhandled error message : "%s"' % error)
+
+            assert self.rcpt_summary.is_here(), 'Should be on recipient addition summary page'
             return self.build_recipient(recipient)
 
         self.init_new_recipient(recipient, is_bp_account, **params)

modules/bp/pages/__init__.py

@@ -30,6 +30,7 @@
     TransferSummary, CreateRecipient, ValidateRecipient,
     ValidateCountry, ConfirmPage, RcptSummary,
     HonorTransferPage, RecipientSubmitDevicePage,
+    RcptErrorPage,
 )
 from .subscription import SubscriptionPage, DownloadPage, ProSubscriptionPage
 
@@ -39,5 +40,5 @@
     'AccountDesactivate', 'TransferChooseAccounts', 'CompleteTransfer', 'TransferConfirm', 'TransferSummary', 'UnavailablePage',
     'CardsList', 'AccountRIB', 'Advisor', 'CreateRecipient', 'ValidateRecipient', 'ValidateCountry', 'ConfirmPage', 'RcptSummary',
     'SubscriptionPage', 'DownloadPage', 'ProSubscriptionPage', 'RevolvingAttributesPage', 'Validated2FAPage', 'TwoFAPage',
-    'SmsPage', 'DecoupledPage', 'HonorTransferPage', 'RecipientSubmitDevicePage',
+    'SmsPage', 'DecoupledPage', 'HonorTransferPage', 'RecipientSubmitDevicePage', 'RcptErrorPage',
 ]

modules/bp/pages/transfer.py

@@ -36,6 +36,7 @@
 from weboob.tools.capabilities.bank.transactions import FrenchTransaction
 from weboob.tools.capabilities.bank.iban import is_iban_valid
 from weboob.tools.value import Value
+from weboob.tools.compat import urljoin
 from weboob.exceptions import BrowserUnavailable, AuthMethodNotImplemented
 
 from .base import MyHTMLPage
@@ -391,12 +392,14 @@ def get_confirm_link(self):
         return Link('//a[@title="confirmer la creation"]')(self.doc)
 
 
-class ConfirmPage(LoggedPage, MyHTMLPage):
+class CheckErrorsPage(LoggedPage, MyHTMLPage):
     def check_errors(self):
         error_msg = CleanText('//h2[contains(text(), "Compte rendu")]/following-sibling::p')(self.doc)
         if error_msg:
             raise AddRecipientBankError(message=error_msg)
 
+
+class ConfirmPage(CheckErrorsPage):
     def get_device_choice_url(self):
         device_choice_popup_js = CleanText('//script[contains(text(), "popupChoixDevice")]')(self.doc)
         if device_choice_popup_js:
@@ -407,7 +410,15 @@ def get_device_choice_url(self):
     def set_browser_form(self):
         form = self.get_form(name='SaisieOTP')
         self.browser.recipient_form = dict((k, v) for k, v in form.items() if v)
-        self.browser.recipient_form['url'] = form.url
+        # Confirmation url is relative to the current page. We need to
+        # build it now or the relative path will fail when reloading state
+        # because we do not reload the url in it.
+        self.browser.recipient_form['url'] = urljoin(self.url, form.url)
+
+
+class RcptErrorPage(LoggedPage, MyHTMLPage):
+    def get_error(self):
+        return CleanText('//form//span[@class="warning"]')(self.doc)
 
 
 class RecipientSubmitDevicePage(LoggedPage, MyHTMLPage):
@@ -431,5 +442,5 @@ def get_app_validation_message(self):
         return app_validation_message
 
 
-class RcptSummary(LoggedPage, MyHTMLPage):
+class RcptSummary(CheckErrorsPage):
     pass

modules/bred/bred/transfer_pages.py

@@ -53,6 +53,15 @@ def get_status(self):
 
 class EmittersListPage(LoggedPage, JsonPage):
     def can_account_emit_transfer(self, account_id):
+        code = Dict('erreur/code')(self.doc)
+        if code == '90624':
+            # Not the owner of the account:
+            # Nous vous précisons que votre pouvoir ne vous permet pas
+            # d'effectuer des virements de ce type au débit du compte sélectionné.
+            return False
+        elif code != '0':
+            raise AssertionError('Unhandled code %s in transfer emitter selection' % code)
+
         for obj in Dict('content')(self.doc):
 
             for account in Dict('postes')(obj):

modules/caissedepargne/browser.py

@@ -53,6 +53,7 @@
 )
 from weboob.tools.capabilities.bank.investments import create_french_liquidity
 from weboob.tools.compat import urljoin, urlparse, parse_qsl, parse_qs, urlencode, urlunparse
+from weboob.tools.date import date
 from weboob.tools.json import json
 from weboob.tools.value import Value
 from weboob.tools.decorators import retry
@@ -1011,13 +1012,16 @@ def add_owner_accounts(self):
 
         self.accounts = list(self.page.get_list(owner_name))
 
-        # Get wealth accounts that are not on the summary page
-        self.home_tache.go(tache='EPASYNT0')
-        # If there are no wealth accounts we are redirected to the "garbage page"
-        if self.home.is_here():
-            for account in self.page.get_list(owner_name):
-                if account.id not in [acc.id for acc in self.accounts]:
-                    self.accounts.append(account)
+        try:
+            # Get wealth accounts that are not on the summary page
+            self.home_tache.go(tache='EPASYNT0')
+            # If there are no wealth accounts we are redirected to the "garbage page"
+            if self.home.is_here():
+                for account in self.page.get_list(owner_name):
+                    if account.id not in [acc.id for acc in self.accounts]:
+                        self.accounts.append(account)
+        except ServerError:
+            self.logger.warning("Could not access wealth accounts page")
 
         self.add_linebourse_accounts_data()
         self.add_card_accounts()
@@ -1711,15 +1715,24 @@ def iter_documents(self, subscription):
         self.home.go()
         if not self.has_subscription:
             self.go_documents_without_sub()
-            return self.page.iter_documents(sub_id=subscription.id, has_subscription=self.has_subscription)
-        self.home_tache.go(tache='CPTSYNT1')
-        if self.unavailable_page.is_here():
-            # some users don't have checking account
-            self.home_tache.go(tache='EPASYNT0')
-        self.page.go_subscription()
-        assert self.subscription.is_here()
+            for doc in self.page.iter_documents(sub_id=subscription.id, has_subscription=self.has_subscription):
+                yield doc
+        else:
+            today = date.today()
+
+            self.home_tache.go(tache='CPTSYNT1')
+            if self.unavailable_page.is_here():
+                # some users don't have checking account
+                self.home_tache.go(tache='EPASYNT0')
+            self.page.go_subscription()
+            # setting to have 3 years of history
+            for year in range(today.year - 2, today.year + 1):
+                self.page.change_year(year)
+
+                assert self.subscription.is_here()
 
-        return self.page.iter_documents(sub_id=subscription.id, has_subscription=self.has_subscription)
+                for doc in self.page.iter_documents(sub_id=subscription.id, has_subscription=self.has_subscription):
+                    yield doc
 
     @need_login
     def download_document(self, document):

modules/caissedepargne/pages.py

@@ -50,7 +50,7 @@
 )
 from .compat.weboob_capabilities_wealth import Investment
 from weboob.capabilities.bill import DocumentTypes, Subscription, Document
-from weboob.tools.capabilities.bank.investments import is_isin_valid
+from weboob.tools.capabilities.bank.investments import is_isin_valid, IsinCode, IsinType
 from weboob.tools.capabilities.bank.transactions import FrenchTransaction
 from weboob.tools.capabilities.bank.iban import is_rib_valid, rib2iban, is_iban_valid
 from weboob.tools.captcha.virtkeyboard import SplitKeyboard, GridVirtKeyboard
@@ -433,6 +433,16 @@ class IndexPage(LoggedPage, BasePage):
         'PEA': Account.TYPE_PEA,
     }
 
+    ACCOUNT_TYPES_LINK = {
+        'SYNTHESE_ASSURANCE_CNP': Account.TYPE_LIFE_INSURANCE,
+        'REDIR_ASS_VIE': Account.TYPE_LIFE_INSURANCE,
+        'SYNTHESE_EPARGNE': Account.TYPE_LIFE_INSURANCE,
+        'ASSURANCE_VIE': Account.TYPE_LIFE_INSURANCE,
+        'NA_WEB': Account.TYPE_LIFE_INSURANCE,
+        'BOURSE': Account.TYPE_MARKET,
+        'COMPTE_TITRE': Account.TYPE_MARKET,
+    }
+
     def on_load(self):
 
         # For now, we have to handle this because after this warning message,
@@ -519,8 +529,8 @@ def _get_account_info(self, a, accounts):
             id = re.search(r"([\d]+)", a.attrib.get('title', ''))
             if len(parts) > 1:
                 info['type'] = parts[0]
-                if info['type'] == 'REDIR_ASS_VIE':
-                    # The link format for this account type has an additional parameter
+                if info['type'] in ('REDIR_ASS_VIE', 'NA_WEB'):
+                    # The link format for these account types has an additional parameter
                     info['id'] = info['_id'] = parts[2]
                 else:
                     info['id'] = info['_id'] = parts[1]
@@ -532,12 +542,13 @@ def _get_account_info(self, a, accounts):
                         _id = list(unique_ids)[0]
                     self.find_and_replace(info, _id)
             else:
+                if id is None:
+                    return None
                 info['type'] = link
                 info['id'] = info['_id'] = id.group(1)
-            if info['type'] in ('SYNTHESE_ASSURANCE_CNP', 'REDIR_ASS_VIE', 'SYNTHESE_EPARGNE', 'ASSURANCE_VIE'):
-                info['acc_type'] = Account.TYPE_LIFE_INSURANCE
-            if info['type'] in ('BOURSE', 'COMPTE_TITRE'):
-                info['acc_type'] = Account.TYPE_MARKET
+            account_type = self.ACCOUNT_TYPES_LINK.get(info['type'])
+            if account_type:
+                info['acc_type'] = account_type
             return info
 
     def is_account_inactive(self, account_id):
@@ -1730,16 +1741,8 @@ def obj_unitvalue(self):
                     return Eval(float_to_decimal, Dict('cotation/montant/valeur'))(self)
                 return NotAvailable
 
-            def obj_code(self):
-                code = Dict('codeISIN')(self)
-                if is_isin_valid(code):
-                    return code
-                return NotAvailable
-
-            def obj_code_type(self):
-                if Field('code')(self) == NotAvailable:
-                    return NotAvailable
-                return Investment.CODE_TYPE_ISIN
+            obj_code = IsinCode(CleanText(Dict('codeIsin', default='')), default=NotAvailable)
+            obj_code_type = IsinType(CleanText(Dict('codeIsin', default='')))
 
 
 class NatixisLIHis(LoggedPage, JsonPage):
@@ -1770,7 +1773,8 @@ class item(ItemElement):
             klass = Investment
 
             obj_label = CleanText(Dict('nom'))
-            obj_code = CleanText(Dict('codeIsin'))
+            obj_code = IsinCode(CleanText(Dict('codeIsin', default='')), default=NotAvailable)
+            obj_code_type = IsinType(CleanText(Dict('codeIsin', default='')))
 
             def obj_vdate(self):
                 dt = Dict('dateValeurUniteCompte', default=None)(self)
@@ -2384,6 +2388,13 @@ def has_subscriptions(self):
         # This message appears if the customer has not activated the e-Documents yet
         return not bool(self.doc.xpath('//a[contains(text(), "Je souscris au service e-Documents")]'))
 
+    def change_year(self, year):
+        form = self.get_form(id='main')
+        form['__EVENTTARGET'] = 'MM$CONSULTATION_MULTI_UNIVERS_EDOCUMENTS$lnkbRechercherConsultationMultiUnivers'
+        form['MM$CONSULTATION_MULTI_UNIVERS_EDOCUMENTS$ddlConsultationAnnee'] = year
+
+        form.submit()
+
     @method
     class iter_subscription(ListElement):
         item_xpath = '//span[@id="MM_CONSULTATION_MULTI_UNIVERS_EDOCUMENTS_ucUniversComptes"]//h3'