pages.py

# -*- coding: utf-8 -*-

# Copyright(C) 2010-2012 Julien Veyssier
#
# This file is part of a weboob module.
#
# This weboob module is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This weboob module is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals

import re
import hashlib

from decimal import Decimal, InvalidOperation
from dateutil.relativedelta import relativedelta
from datetime import date, datetime
from random import randint
from collections import OrderedDict

from weboob.browser.pages import HTMLPage, FormNotFound, LoggedPage, pagination, XMLPage
from weboob.browser.elements import ListElement, ItemElement, SkipItem, method, TableElement
from weboob.browser.filters.standard import (
    Filter, Env, CleanText, CleanDecimal, Field, Regexp, Async, AsyncLoad, Date, Format, Type, Currency,
)
from weboob.browser.filters.html import Link, Attr, TableCell, ColumnNotFound
from weboob.exceptions import (
    BrowserIncorrectPassword, ParseError, ActionNeeded, BrowserUnavailable,
    AuthMethodNotImplemented,
)
from weboob.capabilities import NotAvailable
from weboob.capabilities.base import empty, find_object
from weboob.capabilities.bank import (
    Account, Investment, Recipient, TransferBankError,
    Transfer, AddRecipientBankError, AddRecipientStep, Loan,
)
from weboob.capabilities.contact import Advisor
from weboob.capabilities.profile import Profile
from weboob.tools.capabilities.bank.iban import is_iban_valid
from weboob.tools.capabilities.bank.transactions import FrenchTransaction
from weboob.capabilities.bill import DocumentTypes, Subscription, Document
from weboob.tools.compat import urlparse, parse_qs, urljoin, range, unicode
from weboob.tools.date import parse_french_date
from weboob.tools.value import Value


def MyDecimal(*args, **kwargs):
    kwargs.update(replace_dots=True, default=NotAvailable)
    return CleanDecimal(*args, **kwargs)

def MyDate(*args, **kwargs):
    kwargs.update(dayfirst=True, default=NotAvailable)
    return Date(*args, **kwargs)

class RedirectPage(LoggedPage, HTMLPage):
    def on_load(self):
        super(RedirectPage, self).on_load()
        link = self.doc.xpath('//a[@id="P:F_1.R2:link"]')
        if link:
            self.browser.location(link[0].attrib['href'])


class NewHomePage(LoggedPage, HTMLPage):
    def on_load(self):
        self.browser.is_new_website = True
        super(NewHomePage, self).on_load()


class LoginPage(HTMLPage):
    REFRESH_MAX = 10.0

    def on_load(self):
        error_msg_xpath = '//div[has-class("err")]//p[contains(text(), "votre mot de passe est faux")]'
        if self.doc.xpath(error_msg_xpath):
            raise BrowserIncorrectPassword(CleanText(error_msg_xpath)(self.doc))

    def login(self, login, passwd):
        form = self.get_form(xpath='//form[contains(@name, "ident")]')
        # format login/password like login/password sent by firefox or chromium browser
        form['_cm_user'] = login.encode('cp1252', errors='xmlcharrefreplace').decode('cp1252')
        form['_cm_pwd'] = passwd.encode('cp1252', errors='xmlcharrefreplace').decode('cp1252')
        form.submit()

    @property
    def logged(self):
        return self.doc.xpath('//div[@id="e_identification_ok"]')


class LoginErrorPage(HTMLPage):
    def on_load(self):
        raise BrowserIncorrectPassword(CleanText('//div[has-class("blocmsg")]')(self.doc))


class EmptyPage(LoggedPage, HTMLPage):
    REFRESH_MAX = 10.0

    def on_load(self):
        # Action needed message is like "Votre Carte de Clés Personnelles numéro 3 est révoquée."
        action_needed = CleanText('//p[contains(text(), "Votre Carte de Clés Personnelles") and contains(text(), "est révoquée")]')(self.doc)
        if action_needed:
            raise ActionNeeded(action_needed)
        maintenance = CleanText('//td[@class="ALERTE"]/p/span[contains(text(), "Dans le cadre de l\'amélioration de nos services, nous vous informons que le service est interrompu")]')(self.doc)
        if maintenance:
            raise BrowserUnavailable(maintenance)


class UserSpacePage(LoggedPage, HTMLPage):
    def on_load(self):
        if self.doc.xpath('//form[@id="GoValider"]'):
            raise ActionNeeded("Le site du contrat Banque à Distance a besoin d'informations supplémentaires")
        super(UserSpacePage, self).on_load()


class ChangePasswordPage(LoggedPage, HTMLPage):
    def on_load(self):
        raise BrowserIncorrectPassword('Please change your password')


class item_account_generic(ItemElement):
    klass = Account

    TYPES = OrderedDict([
        ('Credits Promoteurs', Account.TYPE_CHECKING),  # it doesn't fit loan's model
        ('Compte Cheque', Account.TYPE_CHECKING),
        ('Compte Courant', Account.TYPE_CHECKING),
        ('Cpte Courant', Account.TYPE_CHECKING),
        ('Contrat Personnel', Account.TYPE_CHECKING),
        ('Cc Contrat Personnel', Account.TYPE_CHECKING),
        ('C/C', Account.TYPE_CHECKING),
        ('Start', Account.TYPE_CHECKING),
        ('Comptes courants', Account.TYPE_CHECKING),
        ('Service Accueil', Account.TYPE_CHECKING),
        ('Eurocompte Serenite', Account.TYPE_CHECKING),
        ('Catip', Account.TYPE_DEPOSIT),
        ('Cic Immo', Account.TYPE_MORTGAGE),
        ('Credit', Account.TYPE_LOAN),
        ('Crédits', Account.TYPE_LOAN),
        ('Eco-Prêt', Account.TYPE_LOAN),
        ('Mcne', Account.TYPE_LOAN),
        ('Nouveau Prêt', Account.TYPE_LOAN),
        ('Pret', Account.TYPE_LOAN),
        ('Regroupement De Credits', Account.TYPE_LOAN),
        ('Nouveau Pret 0%', Account.TYPE_LOAN),
        ('Global Auto', Account.TYPE_LOAN),
        ('Passeport Credit', Account.TYPE_REVOLVING_CREDIT),
        ('Allure', Account.TYPE_REVOLVING_CREDIT),  # 'Allure Libre' or 'credit Allure'
        ('Preference', Account.TYPE_REVOLVING_CREDIT),
        ('Plan 4', Account.TYPE_REVOLVING_CREDIT),
        ('P.E.A', Account.TYPE_PEA),
        ('Pea', Account.TYPE_PEA),
        ('Compte De Liquidite Pea', Account.TYPE_PEA),
        ('Compte Epargne', Account.TYPE_SAVINGS),
        ('Etalis', Account.TYPE_SAVINGS),
        ('Ldd', Account.TYPE_SAVINGS),
        ('Livret', Account.TYPE_SAVINGS),
        ("Plan D'Epargne", Account.TYPE_SAVINGS),
        ('Tonic Croissance', Account.TYPE_SAVINGS),
        ('Capital Expansion', Account.TYPE_SAVINGS),
        ('Épargne', Account.TYPE_SAVINGS),
        ('Capital Plus', Account.TYPE_SAVINGS),
        ('Compte Garantie Titres', Account.TYPE_MARKET),
    ])

    REVOLVING_LOAN_LABELS = [
        'Passeport Credit',
        'Allure',
        'Preference',
        'Plan 4',
        'Credit En Reserve',
    ]

    def condition(self):
        if len(self.el.xpath('./td')) < 2:
            return False

        first_td = self.el.xpath('./td')[0]

        return (("i" in first_td.attrib.get('class', '') or "p" in first_td.attrib.get('class', ''))
                and (first_td.find('a') is not None or (first_td.find('.//span') is not None
                and "cartes" in first_td.findtext('.//span') and first_td.find('./div/a') is not None)))

    class Label(Filter):
        def filter(self, text):
            return text.lstrip(' 0123456789').title()

    class Type(Filter):
        def filter(self, label):
            for pattern, actype in item_account_generic.TYPES.items():
                if pattern in label:
                    return actype
            return Account.TYPE_UNKNOWN

    obj_id = Env('id')
    obj_number = Env('id')
    obj__card_number = None
    obj_label = Label(CleanText('./td[1]/a/text() | ./td[1]/a/span[@class and not(contains(@class, "doux"))] | ./td[1]/div/a[has-class("cb")]'))
    obj_coming = Env('coming')
    obj_balance = Env('balance')
    obj_currency = FrenchTransaction.Currency('./td[2] | ./td[3]')
    obj__card_links = []

    def obj__link_id(self):
        if self.is_revolving(Field('label')(self)):
            page = self.page.browser.open(Link('./td[1]//a')(self)).page
            if page and page.doc.xpath('//div[@class="fg"]/a[contains(@href, "%s")]' % Field('id')(self)):
                return urljoin(page.url, Link('//div[@class="fg"]/a')(page.doc))
        return Link('./td[1]//a')(self)

    def obj_type(self):
        t = self.Type(Field('label'))(self)
        # sometimes, using the label is not enough to infer the account's type.
        # this is a fallback that uses the account's group label
        if t == 0:
            return self.Type(CleanText('./preceding-sibling::tr/th[contains(@class, "rupture eir_tblshowth")][1]'))(self)
        return t

    obj__is_inv = False
    obj__is_webid = Env('_is_webid')

    def parse(self, el):
        accounting = None
        coming = None
        page = None
        link = el.xpath('./td[1]//a')[0].get('href', '')
        if 'POR_SyntheseLst' in link:
            raise SkipItem()

        url = urlparse(link)
        p = parse_qs(url.query)
        if 'rib' not in p and 'webid' not in p:
            raise SkipItem()

        for td in el.xpath('./td[2] | ./td[3]'):
            try:
                balance = CleanDecimal('.', replace_dots=True)(td)
            except InvalidOperation:
                continue
            else:
                break
        else:
            if 'lien_inter_sites' in link:
                raise SkipItem()
            else:
                raise ParseError('Unable to find balance for account %s' % CleanText('./td[1]/a')(el))

        self.env['_is_webid'] = False

        if "cartes" in CleanText('./td[1]')(el):
            # handle cb differed card
            if "cartes" in CleanText('./preceding-sibling::tr[1]/td[1]', replace=[(' ', '')])(el):
                # In case it's the second month of card history present, we need to ignore the first
                # one to get the attach accoount
                id_xpath = './preceding-sibling::tr[2]/td[1]/a/node()[contains(@class, "doux")]'
            else:
                # first month of history, the previous tr is the attached account
                id_xpath = './preceding-sibling::tr[1]/td[1]/a/node()[contains(@class, "doux")]'
        else:
            # classical account
            id_xpath = './td[1]/a/node()[contains(@class, "doux")]'

        _id = CleanText(id_xpath, replace=[(' ', '')])(el)
        if not _id:
            if 'rib' in p:
                _id = p['rib'][0]
            else:
                _id = p['webid'][0]
                self.env['_is_webid'] = True

        if self.is_revolving(Field('label')(self)):
            page = self.page.browser.open(link).page
            if isinstance(page, RevolvingLoansList):
                # some revolving loans are listed on an other page. On the accountList, there is
                # just a link for this page, that's why we don't handle it here
                raise SkipItem()

        # Handle cards
        if _id in self.parent.objects:
            if not page:
                page = self.page.browser.open(link).page
            # Handle real balances
            coming = page.find_amount("Opérations à venir") if page else None
            accounting = page.find_amount("Solde comptable") if page else None
            # on old website we want card's history in account's history
            if not page.browser.is_new_website:
                account = self.parent.objects[_id]
                if not account.coming:
                    account.coming = Decimal('0.0')
                # Get end of month
                date = parse_french_date(Regexp(Field('label'), r'Fin (.+) (\d{4})', '01 \\1 \\2')(self)) + relativedelta(day=31)
                if date > datetime.now() - relativedelta(day=1):
                    account.coming += balance
                account._card_links.append(link)
            else:
                multiple_cards_xpath = '//select[@name="Data_SelectedCardItemKey"]/option[contains(text(),"Carte")]'
                single_card_xpath = '//span[has-class("_c1 fg _c1")]'
                card_xpath = multiple_cards_xpath + ' | ' + single_card_xpath
                for elem in page.doc.xpath(card_xpath):
                    card_id = Regexp(CleanText('.', symbols=' '), r'([\dx]{16})')(elem)
                    is_in_accounts = any(card_id in a.id for a in page.browser.accounts_list)
                    if card_id in self.page.browser.unavailablecards or is_in_accounts:
                        continue

                    card = Account()
                    card.type = Account.TYPE_CARD
                    card.id = card._card_number = card_id
                    card._link_id = link
                    card._is_inv = card._is_webid = False
                    card.parent = self.parent.objects[_id]

                    pattern = r'Carte\s(\w+).*\d{4}\s([A-Za-z\s]+)(.*)'
                    m = re.search(pattern, CleanText('.')(elem))
                    card.label = "%s %s %s" % (m.group(1), card_id, m.group(2))
                    card.balance = Decimal('0.0')
                    card.currency = card.get_currency(m.group(3))
                    card._card_pages = [page]
                    card.coming = Decimal('0.0')
                    #handling the case were the month is the coming one. There won't be next_month here.
                    date = parse_french_date(Regexp(Field('label'), r'Fin (.+) (\d{4})', '01 \\1 \\2')(self)) + relativedelta(day=31)
                    if date > datetime.now() - relativedelta(day=1):
                        card.coming = CleanDecimal(replace_dots=True).filter(m.group(3))
                    next_month = Link('./following-sibling::tr[contains(@class, "encours")][1]/td[1]//a', default=None)(self)
                    if next_month:
                        card_page = page.browser.open(next_month).page
                        for e in card_page.doc.xpath(card_xpath):
                            if card.id == Regexp(CleanText('.', symbols=' '), r'([\dx]{16})')(e):
                                m = re.search(pattern, CleanText('.')(e))
                                card._card_pages.append(card_page)
                                card.coming += CleanDecimal(replace_dots=True).filter(m.group(3))
                                break

                    self.page.browser.accounts_list.append(card)

            raise SkipItem()

        self.env['id'] = _id

        if accounting is not None and accounting + (coming or Decimal('0')) != balance:
            self.page.logger.warning('%s + %s != %s' % (accounting, coming, balance))

        if accounting is not None:
            balance = accounting

        self.env['balance'] = balance
        self.env['coming'] = coming or NotAvailable

    def is_revolving(self, label):
        return (any(revolving_loan_label in label
                    for revolving_loan_label in item_account_generic.REVOLVING_LOAN_LABELS)
                or label.lower() in self.page.browser.revolving_accounts)


class AccountsPage(LoggedPage, HTMLPage):
    def has_no_account(self):
        return CleanText('//td[contains(text(), "Votre contrat de banque à distance ne vous donne accès à aucun compte.")]')(self.doc)

    @method
    class iter_accounts(ListElement):
        item_xpath = '//div[has-class("a_blocappli")]//tr'
        flush_at_end = True

        class item_account(item_account_generic):
            def condition(self):
                _type = Field('type')(self)
                return item_account_generic.condition(self) and _type not in (Account.TYPE_LOAN, Account.TYPE_MORTGAGE)

        class item_loan(item_account_generic):
            klass = Loan

            load_details = Link('.//a') & AsyncLoad

            obj_total_amount = Async('details') & MyDecimal('//div[@id="F4:expContent"]/table/tbody/tr[1]/td[1]/text()')
            obj_rate = Async('details') & MyDecimal('//div[@id="F4:expContent"]/table/tbody/tr[2]/td[1]')
            obj_nb_payments_left = Async('details') & Type(CleanText(
                '//div[@id="F4:expContent"]/table/tbody/tr[2]/td[2]/text()'), type=int, default=NotAvailable)
            obj_subscription_date = Async('details') & MyDate(Regexp(CleanText(
                '//*[@id="F4:expContent"]/table/tbody/tr[1]/th[1]'), r' (\d{2}/\d{2}/\d{4})', default=NotAvailable))
            obj_maturity_date = Async('details') & MyDate(
                CleanText('//div[@id="F4:expContent"]/table/tbody/tr[4]/td[2]'))

            obj_next_payment_amount = Async('details') & MyDecimal('//div[@id="F4:expContent"]/table/tbody/tr[3]/td[2]')
            obj_next_payment_date = Async('details') & MyDate(
                CleanText('//div[@id="F4:expContent"]/table/tbody/tr[3]/td[1]'))

            obj_last_payment_amount = Async('details') & MyDecimal('//td[@id="F2_0.T12"]')
            obj_last_payment_date = (Async('details') &
                MyDate(CleanText('//div[@id="F8:expContent"]/table/tbody/tr[1]/td[1]')))

            def obj__parent_id(self):
                # There are 5 numbers that we don't want before the real id
                # "12345 01200 000123456798" => "01200000123456798"
                parent_id = Async('details',
                                  Regexp(CleanText('//div[@id="F4:expContent"]/table/tbody/tr[1]/td[2]',
                                                   default=None), r'\d{5} (\d+\s\d+)')
                                  )(self)
                if parent_id:
                    return parent_id.replace(' ', '')
                return NotAvailable

            def condition(self):
                _type = Field('type')(self)
                label = Field('label')(self)
                details_link = Link('.//a', default=None)(self)

                # mobile accounts are leading to a 404 error when parsing history
                # furthermore this is not exactly a loan account
                if re.search(r'Le\sMobile\s+([0-9]{2}\s?){5}', label):
                    return False

                if (details_link and item_account_generic.condition and _type in (Account.TYPE_LOAN, Account.TYPE_MORTGAGE)
                                 and not self.is_revolving(label)):
                    details = self.page.browser.open(details_link)
                    if details.page and not 'cloturé' in CleanText('//form[@id="P:F"]//div[@class="blocmsg info"]//p')(details.page.doc):
                        return True
                return False


        class item_revolving_loan(item_account_generic):
            klass = Loan

            load_details = Link('.//a') & AsyncLoad

            obj_total_amount = Async('details') & MyDecimal('//main[@id="ei_tpl_content"]/div/div[2]/table/tbody/tr/td[3]')
            obj_type = Account.TYPE_REVOLVING_CREDIT

            def obj_used_amount(self):
                return -Field('balance')(self)

            def condition(self):
                label = Field('label')(self)
                return (
                    item_account_generic.condition(self)
                    and Field('type')(self) == Account.TYPE_REVOLVING_CREDIT
                    and self.is_revolving(label)
                )

    def get_advisor_link(self):
        return Link('//div[@id="e_conseiller"]/a', default=None)(self.doc)

    @method
    class get_advisor(ItemElement):
        klass = Advisor

        obj_name = CleanText('//div[@id="e_conseiller"]/a')

    @method
    class get_profile(ItemElement):
        klass = Profile

        obj_name = CleanText('//div[@id="e_identification_ok_content"]//strong[1]')


class NewAccountsPage(NewHomePage, AccountsPage):
    def get_agency(self):
        return Regexp(CleanText('//script[contains(text(), "lien_caisse")]', default=''),
                      r'(https://[^"]+)', default='')(self.doc)

    @method
    class get_advisor(ItemElement):
        klass = Advisor

        obj_name = Regexp(CleanText('//script[contains(text(), "Espace Conseiller")]'),
                          r'consname.+?([\w\s]+)')

    @method
    class get_profile(ItemElement):
        klass = Profile

        obj_name = CleanText('//p[contains(@class, "master_nom")]')


class AdvisorPage(LoggedPage, HTMLPage):
    @method
    class update_advisor(ItemElement):
        obj_email = CleanText('//table//*[@itemprop="email"]')
        obj_phone = CleanText('//table//*[@itemprop="telephone"]', replace=[(' ', '')])
        obj_mobile = NotAvailable
        obj_fax = CleanText('//table//*[@itemprop="faxNumber"]', replace=[(' ', '')])
        obj_agency = CleanText('//div/*[@itemprop="name"]')
        obj_address = Format('%s %s %s', CleanText('//table//*[@itemprop="streetAddress"]'),
                                         CleanText('//table//*[@itemprop="postalCode"]'),
                                         CleanText('//table//*[@itemprop="addressLocality"]'))


class CardsActivityPage(LoggedPage, HTMLPage):
    def companies_link(self):
        companies_link = []
        for tr in self.doc.xpath('//table[@summary="Liste des titulaires de contrats cartes"]//tr'):
            companies_link.append(Link(tr.xpath('.//a'))(self))
        return companies_link


class Pagination(object):
    def next_page(self):
        try:
            form = self.page.get_form('//form[@id="paginationForm" or @id="frmSTARCpag"]')
        except FormNotFound:
            return self.next_month()

        text = CleanText.clean(form.el)
        m = re.search(r'(\d+)/(\d+)', text or '', flags=re.MULTILINE)
        if not m:
            return self.next_month()

        cur = int(m.group(1))
        last = int(m.group(2))

        if cur == last:
            return self.next_month()

        form['imgOpePagSui.x'] = randint(1, 29)
        form['imgOpePagSui.y'] = randint(1, 17)

        form['page'] = str(cur + 1)
        return form.request

    def next_month(self):
        try:
            form = self.page.get_form('//form[@id="frmStarcLstOpe"]')
        except FormNotFound:
            return

        try:
            form['moi'] = self.page.doc.xpath('//select[@id="moi"]/option[@selected]/following-sibling::option')[0].attrib['value']
            form['page'] = 1
        except IndexError:
            return

        return form.request


class CardsListPage(LoggedPage, HTMLPage):
    @pagination
    @method
    class iter_cards(TableElement):
        item_xpath = '//table[has-class("liste")]/tbody/tr'
        head_xpath = '//table[has-class("liste")]/thead//tr/th'

        col_owner = 'Porteur'
        col_card = 'Carte'

        def next_page(self):
            try:
                form = self.page.get_form('//form[contains(@id, "frmStarcLstCtrPag")]')
                form['imgCtrPagSui.x'] =  randint(1, 29)
                form['imgCtrPagSui.y'] =  randint(1, 17)
                m = re.search(r'(\d+)/(\d+)', CleanText('.')(form.el))
                if m and int(m.group(1)) < int(m.group(2)):
                    return form.request
            except FormNotFound:
                return

        class item(ItemElement):
            klass = Account

            obj_number = Field('_link_id') & Regexp(pattern=r'ctr=(\d+)')
            obj__card_number = Env('id', default="")
            obj_id = Format('%s%s', Env('id', default=""), Field('number'))
            obj_label = Format('%s %s %s', CleanText(TableCell('card')), Env('id', default=""),
                               CleanText(TableCell('owner')))
            obj_coming = CleanDecimal('./td[@class="i d" or @class="p d"][2]', replace_dots=True,
                                      default=NotAvailable)
            obj_balance = Decimal('0.00')
            obj_currency = FrenchTransaction.Currency(CleanText('./td[small][1]'))
            obj_type = Account.TYPE_CARD
            obj__card_pages = Env('page')
            obj__is_inv = False
            obj__is_webid = False

            def obj__pre_link(self):
                return self.page.url

            def obj__link_id(self):
                return Link(TableCell('card')(self)[0].xpath('./a'))(self)

            def parse(self, el):
                page = self.page.browser.open(Field('_link_id')(self)).page
                self.env['page'] = [page]

                if len(page.doc.xpath('//caption[contains(text(), "débits immédiats")]')):
                    raise SkipItem()

                # Handle multi cards
                options = page.doc.xpath('//select[@id="iso"]/option')
                for option in options:
                    card = Account()
                    card_list_page = page.browser.open(Link('//form//a[text()="Contrat"]', default=None)(page.doc)).page
                    xpath = '//table[has-class("liste")]/tbody/tr'
                    active_card = CleanText('%s[td[text()="Active"]][1]/td[2]' % xpath, replace=[(' ', '')], default=None)(card_list_page.doc)
                    _id = CleanText('.', replace=[(' ', '')])(option)
                    if active_card == _id:
                        for attr in self._attrs:
                            self.handle_attr(attr, getattr(self, 'obj_%s' % attr))
                            setattr(card, attr, getattr(self.obj, attr))

                        card._card_number = _id
                        card.id = _id + card.number
                        card.label = card.label.replace('  ', ' %s ' % _id)
                        card2 = find_object(self.page.browser.cards_list, id=card.id[:16])
                        if card2:
                            card._link_id = card2._link_id
                            card._parent_id = card2._parent_id
                            card.coming = card2.coming
                            card._referer = card2._referer
                            card._secondpage = card2._secondpage
                            self.page.browser.accounts_list.remove(card2)
                        self.page.browser.accounts_list.append(card)
                        self.page.browser.cards_list.append(card)

                # Skip multi and expired cards
                if len(options) or len(page.doc.xpath('//span[@id="ERREUR"]')):
                    raise SkipItem()

                # 1 card : we have to check on another page to get id
                page = page.browser.open(Link('//form//a[text()="Contrat"]', default=None)(page.doc)).page
                xpath = '//table[has-class("liste")]/tbody/tr'
                active_card = CleanText('%s[td[text()="Active"]][1]/td[2]' % xpath, replace=[(' ', '')], default=None)(page.doc)
                for cards in page.doc.xpath(xpath):
                    if CleanText(cards.xpath('./td[1]'))(self) != 'Active':
                        self.page.browser.unavailablecards.append(CleanText(cards.xpath('./td[2]'), replace=[(' ', '')])(self))

                if not active_card and len(page.doc.xpath(xpath)) != 1:
                    raise SkipItem()

                self.env['id'] = active_card or CleanText('%s[1]/td[2]' % xpath, replace=[(' ', '')])(page.doc)


class Transaction(FrenchTransaction):
    PATTERNS = [
        (re.compile(r'^(VIR(EMENT)?|VIRT.) (?P<text>.*)'), FrenchTransaction.TYPE_TRANSFER),
        (re.compile(r'^(PRLV|Plt|PRELEVEMENT) (?P<text>.*)'), FrenchTransaction.TYPE_ORDER),
        (re.compile(r'^(?P<text>.*) (CARTE |PAYWEB)\d+ PAIEMENT CB\s+(?P<dd>\d{2})(?P<mm>\d{2}) ?(.*)$'), FrenchTransaction.TYPE_CARD),
        (re.compile(r'^PAIEMENT PSC\s+(?P<dd>\d{2})(?P<mm>\d{2}) (?P<text>.*) CARTE \d+ ?(.*)$'), FrenchTransaction.TYPE_CARD),
        (re.compile(r'^Regroupement \d+ PAIEMENTS (?P<dd>\d{2})(?P<mm>\d{2}) (?P<text>.*) CARTE \d+ ?(.*)$'), FrenchTransaction.TYPE_CARD),
        (re.compile(r'^(?P<text>RELEVE CARTE.*)'), FrenchTransaction.TYPE_CARD_SUMMARY),
        (re.compile(r'^RETRAIT DAB (?P<dd>\d{2})(?P<mm>\d{2}) (?P<text>.*) CARTE [\*\d]+'), FrenchTransaction.TYPE_WITHDRAWAL),
        (re.compile(r'^CHEQUE( (?P<text>.*))?$'), FrenchTransaction.TYPE_CHECK),
        (re.compile(r'^(F )?COTIS\.? (?P<text>.*)'), FrenchTransaction.TYPE_BANK),
        (re.compile(r'^(REMISE|REM CHQ) (?P<text>.*)'), FrenchTransaction.TYPE_DEPOSIT),
    ]

    _is_coming = False


class OperationsPage(LoggedPage, HTMLPage):
    def go_on_history_tab(self):
        try:
            # Maybe obsolete
            form = self.get_form(id='I1:fm')
        except FormNotFound:
            form = self.get_form(id='I1:P:F')
        form['_FID_DoShowListView'] = ''
        form.submit()

    @method
    class get_history(Pagination, Transaction.TransactionsElement):
        head_xpath = '//table[has-class("liste")]//thead//tr/th'
        item_xpath = '//table[has-class("liste")]//tbody/tr'

        class item(Transaction.TransactionElement):
            condition = lambda self: len(self.el.xpath('./td')) >= 3 and len(self.el.xpath('./td[@class="i g" or @class="p g" or contains(@class, "_c1")]')) > 0

            class OwnRaw(Filter):
                def __call__(self, item):
                    el = TableCell('raw')(item)[0]

                    # Remove hidden parts of labels:
                    # hideifscript: Date de valeur XX/XX/XXXX
                    # fd: Avis d'opéré
                    # survey to add other regx
                    parts = (re.sub(r'Détail|Date de valeur\s+:\s+\d{2}/\d{2}(/\d{4})?', '', txt.strip())
                             for txt in el.itertext() if txt.strip())
                    # Removing empty strings
                    parts = list(filter(bool, parts))
                    # To simplify categorization of CB, reverse order of parts to separate
                    # location and institution
                    detail = "Cliquer pour déplier ou plier le détail de l'opération"
                    if detail in parts:
                        parts.remove(detail)
                    if parts[0].startswith('PAIEMENT CB'):
                        parts.reverse()

                    return ' '.join(parts)

            obj_raw = Transaction.Raw(OwnRaw())

    def find_amount(self, title):
        try:
            td = self.doc.xpath('//th[contains(text(), $title)]/../td', title=title)[0]
        except IndexError:
            return None
        else:
            return Decimal(FrenchTransaction.clean_amount(td.text))

    def get_coming_link(self):
        try:
            a = self.doc.xpath('//a[contains(text(), "Opérations à venir")]')[0]
        except IndexError:
            return None
        else:
            return a.attrib['href']

    def has_more_operations(self):
        return bool(self.doc.xpath('//a/span[contains(text(), "Plus d\'opérations")]'))


class CardsOpePage(OperationsPage):
    def select_card(self, card_number):
        if CleanText('//select[@id="iso"]', default=None)(self.doc):
            form = self.get_form('//p[has-class("restriction")]')
            card_number = ' '.join([card_number[j*4:j*4+4] for j in range(len(card_number)//4+1)]).strip()
            form['iso'] = Attr('//option[text()="%s"]' % card_number, 'value')(self.doc)
            moi = Attr('//select[@id="moi"]/option[@selected]', 'value', default=None)(self.doc)
            if moi:
                form['moi'] = moi
            return self.browser.open(form.url, data=dict(form)).page
        return self

    @method
    class get_history(Pagination, Transaction.TransactionsElement):
        head_xpath = '//table[has-class("liste")]//thead//tr/th'
        item_xpath = '//table[has-class("liste")]/tr'

        col_city = 'Ville'
        col_original_amount = "Montant d'origine"
        col_amount = 'Montant'

        class item(Transaction.TransactionElement):
            condition = lambda self: len(self.el.xpath('./td')) >= 5

            obj_raw = obj_label = Format('%s %s', TableCell('raw') & CleanText, TableCell('city') & CleanText)
            obj_original_amount = CleanDecimal(TableCell('original_amount'), default=NotAvailable, replace_dots=True)
            obj_original_currency = FrenchTransaction.Currency(TableCell('original_amount'))
            obj_type = Transaction.TYPE_DEFERRED_CARD
            obj_rdate = obj_bdate = Transaction.Date(TableCell('date'))
            obj_date = obj_vdate = Env('date')
            obj__is_coming = Env('_is_coming')

            obj__gross_amount = CleanDecimal(Env('amount'), replace_dots=True)
            obj_commission = CleanDecimal(Format('-%s', Env('commission')), replace_dots=True, default=NotAvailable)
            obj__to_delete = False

            def obj_amount(self):
                commission = Field('commission')(self)
                gross = Field('_gross_amount')(self)
                if empty(commission):
                    return gross
                return (abs(gross) - abs(commission)).copy_sign(gross)

            def parse(self, el):
                self.env['date'] = Date(Regexp(CleanText('//td[contains(text(), "Total prélevé")]'),
                                               r' (\d{2}/\d{2}/\d{4})', default=NotAvailable),
                                               default=NotAvailable)(self)
                if not self.env['date']:
                    try:
                        d = (CleanText('//select[@id="moi"]/option[@selected]')(self)
                             or re.search(r'pour le mois de (.*)', ''.join(w.strip() for w in
                                self.page.doc.xpath('//div[@class="a_blocongfond"]/text()'))).group(1))
                    except AttributeError:
                        d = Regexp(CleanText('//p[has-class("restriction")]'), r'pour le mois de ((?:\w+\s+){2})', flags=re.UNICODE)(self)
                    self.env['date'] = (parse_french_date('%s %s' % ('1', d)) + relativedelta(day=31)).date()
                self.env['_is_coming'] = date.today() < self.env['date']
                amount = CleanText(TableCell('amount'))(self).split('dont frais')
                self.env['amount'] = amount[0]
                self.env['commission'] = amount[1] if len(amount) > 1 else NotAvailable


class ComingPage(OperationsPage, LoggedPage):
    @method
    class get_history(Pagination, Transaction.TransactionsElement):
        head_xpath = '//table[has-class("liste")]//thead//tr/th/text()'
        item_xpath = '//table[has-class("liste")]//tbody/tr'

        col_date = "Date de l'annonce"

        class item(Transaction.TransactionElement):
            obj__is_coming = True


class CardPage(OperationsPage, LoggedPage):
    def select_card(self, card_number):
        for option in self.doc.xpath('//select[@name="Data_SelectedCardItemKey"]/option'):
            card_id = Regexp(CleanText('.', symbols=' '), r'(\d+x+\d+)')(option)
            if card_id != card_number:
                continue
            if Attr('.', 'selected', default=None)(option):
                break

            try:
                # Maybe obsolete
                form = self.get_form(id="I1:fm")
            except FormNotFound:
                form = self.get_form(id='I1:P:F')
            form['_FID_DoChangeCardDetails'] = ""
            form['Data_SelectedCardItemKey'] = Attr('.', 'value')(option)
            return self.browser.open(form.url, data=dict(form)).page
        return self

    @method
    class get_history(Pagination, ListElement):
        class list_cards(ListElement):
            item_xpath = '//table[has-class("liste")]/tbody/tr/td/a'

            class item(ItemElement):
                def __iter__(self):
                    # Here we handle the subtransactions
                    card_link = self.el.get('href')
                    d = re.search(r'cardmonth=(\d+)', self.page.url)
                    if d:
                        year = int(d.group(1)[:4])
                        month = int(d.group(1)[4:])
                    debit_date = date(year, month, 1) + relativedelta(day=31)

                    page = self.page.browser.location(card_link).page

                    for op in page.get_history():
                        op.date = debit_date
                        op.type = FrenchTransaction.TYPE_DEFERRED_CARD
                        op._to_delete = False
                        yield op

        class list_history(Transaction.TransactionsElement):
            head_xpath = '//table[has-class("liste")]//thead/tr/th'
            item_xpath = '//table[has-class("liste")]/tbody/tr'

            col_commerce = 'Commerce'
            col_ville = 'Ville'

            def condition(self):
                return not CleanText('//td[contains(., "Aucun mouvement")]', default=False)(self)

            def parse(self, el):
                label = CleanText('//*[contains(text(), "Achats")]')(el)
                if not label:
                    return
                try:
                    label = re.findall(r'(\d+ [^ ]+ \d+)', label)[-1]
                except IndexError:
                    return
                # use the trick of relativedelta to get the last day of month.
                self.env['debit_date'] = (parse_french_date(label) + relativedelta(day=31)).date()

            class item(Transaction.TransactionElement):
                condition = lambda self: len(self.el.xpath('./td')) >= 4

                obj_raw = Transaction.Raw(Env('raw'))
                obj_type = Env('type')
                obj_date = Env('debit_date')
                obj_rdate = Transaction.Date(TableCell('date'))
                obj_amount = Env('amount')
                obj_original_amount = Env('original_amount')
                obj_original_currency = Env('original_currency')
                obj__differed_date = Env('differed_date')

                def obj_bdate(self):
                    if Field('type')(self) == Transaction.TYPE_DEFERRED_CARD:
                        return Field('rdate')(self)

                def obj__to_delete(self):
                    return bool(CleanText('.//a[contains(text(), "Regroupement")]')(self))

                def parse(self, el):
                    try:
                        self.env['raw'] = "%s %s" % (CleanText().filter(TableCell('commerce')(self)[0].text),
                                                     CleanText().filter(TableCell('ville')(self)[0].text))
                    except (ColumnNotFound, AttributeError):
                        self.env['raw'] = "%s" % (CleanText().filter(TableCell('commerce')(self)[0].text))

                    self.env['type'] = (Transaction.TYPE_DEFERRED_CARD
                                        if CleanText('//a[contains(text(), "Prélevé fin")]', default=None)
                                        else Transaction.TYPE_CARD)
                    self.env['differed_date'] = parse_french_date(Regexp(CleanText('//*[contains(text(), "Achats")]'),
                                                                         r'au[\s]+(.*)')(self)).date()
                    amount = TableCell('credit')(self)[0]
                    if self.page.browser.is_new_website:
                        if not len(amount.xpath('./div')):
                            amount = TableCell('debit')(self)[0]
                        original_amount = amount.xpath('./div')[1].text if len(amount.xpath('./div')) > 1 else None
                        amount = amount.xpath('./div')[0]
                    else:
                        try:
                            original_amount = amount.xpath('./span')[0].text
                        except IndexError:
                            original_amount = None
                    self.env['amount'] = CleanDecimal(replace_dots=True).filter(amount.text)
                    self.env['original_amount'] = (CleanDecimal(replace_dots=True).filter(original_amount)
                                                   if original_amount is not None else NotAvailable)
                    self.env['original_currency'] = (Account.get_currency(original_amount[1:-1])
                                                     if original_amount is not None else NotAvailable)


class CardPage2(CardPage, HTMLPage, XMLPage):
    def build_doc(self, content):
        if b'<?xml version="1.0"' in content:
            xml = XMLPage.build_doc(self, content)
            html = xml.xpath('//htmlcontent')[0].text.encode(encoding=self.encoding)
            return HTMLPage.build_doc(self, html)

        return super(CardPage2, self).build_doc(content)

    @method
    class get_history(ListElement):
        class list_history(Transaction.TransactionsElement):
            head_xpath = '//table[has-class("liste")]//thead/tr/th'
            item_xpath = '//table[has-class("liste")]/tbody/tr'

            col_commerce = 'Commerce'
            col_ville = 'Ville'

            def condition(self):
                return not CleanText('//td[contains(., "Aucun mouvement")]', default=False)(self) or not CleanText('//td[contains(., "Aucune opération")]', default=False)(self)

            class item(Transaction.TransactionElement):
                def condition(self):
                    # Withdraw transactions are also presents on the checking account
                    return len(self.el.xpath('./td')) >= 4 and not CleanText(TableCell('commerce'))(self).startswith('RETRAIT CB')

                obj_raw = Transaction.Raw(Format("%s %s", CleanText(TableCell('commerce')), CleanText(TableCell('ville'))))
                obj_rdate = obj_bdate = Field('vdate')
                obj_date = Env('date')

                def obj_type(self):
                    if not 'RELEVE' in CleanText('//td[contains(., "Aucun mouvement")]')(self):
                        return Transaction.TYPE_DEFERRED_CARD
                    return Transaction.TYPE_CARD_SUMMARY

                def obj_original_amount(self):
                    m = re.search(r'(([\s-]\d+)+,\d+)', CleanText(TableCell('commerce'))(self))
                    if m and not 'FRAIS' in CleanText(TableCell('commerce'))(self):
                        matched_text = m.group(1)
                        submatch = re.search(r'\d+-(.*)', matched_text)
                        if submatch:
                            matched_text = submatch.group(1)
                        return Decimal(matched_text.replace(',', '.').replace(' ', '')).quantize(Decimal('0.01'))
                    return NotAvailable

                def obj_original_currency(self):
                    m = re.search(r'(\d+,\d+) (\w+)', CleanText(TableCell('commerce'))(self))
                    if Field('original_amount')(self) and m:
                        return m.group(2)

                def obj__is_coming(self):
                    if Field('date')(self) > datetime.date(datetime.today()):
                        return True
                    return False

                # Some payment made on the same organization are regrouped,
                # we have to get the detail for each one later
                def obj__regroup(self):
                    if "Regroupement" in CleanText('./td')(self):
                        return Link('./td/span/a')(self)

    @method
    class get_tr_merged(ListElement):
        class list_history(Transaction.TransactionsElement):
            head_xpath = '//table[@class="liste"]//thead/tr/th'
            item_xpath = '//table[@class="liste"]/tbody/tr'

            col_operation= u'Opération'

            def condition(self):
                return not CleanText('//td[contains(., "Aucun mouvement")]', default=False)(self)

            class item(Transaction.TransactionElement):
                def condition(self):
                    return len(self.el.xpath('./td')) >= 4 and not CleanText(TableCell('operation'))(self).startswith('RETRAIT CB')

                obj_label = CleanText(TableCell('operation'))

                def obj_type(self):
                    if not 'RELEVE' in Field('raw')(self):
                        return Transaction.TYPE_DEFERRED_CARD
                    return Transaction.TYPE_CARD_SUMMARY

                def obj_bdate(self):
                    if Field('type')(self) == Transaction.TYPE_DEFERRED_CARD:
                        return Transaction.Date(TableCell('date'))(self)

    def has_more_operations(self):
        xp = CleanText(self.doc.xpath('//div[@class="ei_blocpaginb"]/a'))(self)
        if xp == 'Suite des opérations':
            return True
        return False

    def has_more_operations_xml(self):
        if self.doc.xpath('//input') and Attr('//input', 'value')(self.doc) == 'Suite des opérations':
            return True
        return False

    @method
    class iter_history_xml(ListElement):