accounts_list.py 18.8 KB
Newer Older
1 2
# -*- coding: utf-8 -*-

3
# Copyright(C) 2009-2014  Florent Fourcot, Romain Bignon
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.

20
from __future__ import unicode_literals
21

Romain Bignon's avatar
Romain Bignon committed
22
from datetime import date, timedelta
23
import datetime
24 25
import re

26
from weboob.capabilities.bank import Account, Investment, Loan
27
from weboob.capabilities.base import NotAvailable
28
from weboob.capabilities.profile import Person
29
from weboob.browser.pages import HTMLPage, LoggedPage, JsonPage
30 31
from weboob.browser.elements import ListElement, TableElement, ItemElement, method, DataError
from weboob.browser.filters.standard import (
32 33 34
    CleanText, CleanDecimal, Eval, Filter, Field, MultiFilter, Date,
    Lower, Async, AsyncLoad, Format, Env,
    Regexp,
35
)
36
from weboob.browser.filters.json import Dict
37
from weboob.browser.filters.html import Attr, Link, TableCell
Romain Bignon's avatar
Romain Bignon committed
38
from weboob.tools.capabilities.bank.transactions import FrenchTransaction
39 40


41
class Transaction(FrenchTransaction):
42 43 44
    PATTERNS = [(re.compile(u'^retrait dab (?P<dd>\d{2})/(?P<mm>\d{2})/(?P<yy>\d{4}) (?P<text>.*)'), FrenchTransaction.TYPE_WITHDRAWAL),
                # Withdrawal in foreign currencies will look like "retrait 123 currency"
                (re.compile(u'^retrait (?P<text>.*)'), FrenchTransaction.TYPE_WITHDRAWAL),
45 46
                (re.compile(u'^carte (?P<dd>\d{2})/(?P<mm>\d{2})/(?P<yy>\d{4}) (?P<text>.*)'), FrenchTransaction.TYPE_CARD),
                (re.compile(u'^virement (sepa )?(emis vers|recu|emis)? (?P<text>.*)'), FrenchTransaction.TYPE_TRANSFER),
47
                (re.compile(u'^remise cheque(?P<text>.*)'), FrenchTransaction.TYPE_DEPOSIT),
48 49
                (re.compile(u'^cheque (?P<text>.*)'), FrenchTransaction.TYPE_CHECK),
                (re.compile(u'^prelevement (?P<text>.*)'), FrenchTransaction.TYPE_ORDER),
Florent Fourcot's avatar
Florent Fourcot committed
50
                (re.compile(u'^prlv sepa (?P<text>.*)'), FrenchTransaction.TYPE_ORDER),
51
                (re.compile(u'^prélèvement sepa en faveur de (?P<text>.*)'), FrenchTransaction.TYPE_ORDER),
52
                (re.compile(u'^commission sur (?P<text>.*)'), FrenchTransaction.TYPE_BANK),
53 54
                ]

55

56 57
class AddPref(MultiFilter):
    prefixes = {u'Courant': u'CC-', u'Livret A': 'LA-', u'Orange': 'LEO-',
Florent Fourcot's avatar
Florent Fourcot committed
58
                u'Durable': u'LDD-', u"Titres": 'TITRE-', u'PEA': u'PEA-'}
59 60 61 62 63 64 65 66 67 68

    def filter(self, values):
        el, label = values
        for key, pref in self.prefixes.items():
            if key in label:
                return pref + el
        return el


class AddType(Filter):
69 70 71 72 73 74
    types = {u'Courant': Account.TYPE_CHECKING,
             u'Livret A': Account.TYPE_SAVINGS,
             u'Orange': Account.TYPE_SAVINGS,
             u'Durable': Account.TYPE_SAVINGS,
             u'Titres': Account.TYPE_MARKET,
             u'PEA': Account.TYPE_PEA,
75
             u'Direct Vie': Account.TYPE_LIFE_INSURANCE,
Vincent Paredes's avatar
Vincent Paredes committed
76 77
             u'Assurance Vie': Account.TYPE_LIFE_INSURANCE,
             u'Crédit Immobilier': Account.TYPE_LOAN,
78 79
             u'Prêt Personnel': Account.TYPE_LOAN,
             }
80 81 82 83 84 85 86 87

    def filter(self, label):
        for key, acc_type in self.types.items():
            if key in label:
                return acc_type
        return Account.TYPE_UNKNOWN


88
class PreHashmd5(MultiFilter):
89 90 91
    def filter(self, values):
        concat = ''
        for value in values:
92 93 94 95
            if type(value) is datetime.date:
                concat += value.strftime('%d/%m/%Y')
            else:
                concat += u'%s' % value
96
        return concat.encode('utf-8')
97

Florent Fourcot's avatar
Florent Fourcot committed
98

Florent Fourcot's avatar
Florent Fourcot committed
99 100 101
class INGDate(Date):
    monthvalue = {u'janv.': '01', u'févr.': '02', u'mars': '03', u'avr.': '04',
                  u'mai': '05', u'juin': '06', u'juil.': '07', u'août': '08',
Florent Fourcot's avatar
Florent Fourcot committed
102
                  u'sept.': '09', u'oct.': '10', u'nov.': '11', u'déc.': '12'}
Florent Fourcot's avatar
Florent Fourcot committed
103 104 105

    def filter(self, txt):
        if txt == 'hier':
106
            return date.today() - timedelta(days=1)
Florent Fourcot's avatar
Florent Fourcot committed
107 108
        elif txt == "aujourd'hui":
            return date.today()
109
        elif txt == 'demain':
110 111 112 113 114 115
            return date.today() + timedelta(days=1)
        frenchmonth = txt.split(' ')[1]
        month = self.monthvalue[frenchmonth]
        txt = txt.replace(' ', '')
        txt = txt.replace(frenchmonth, '/%s/' % month)
        return super(INGDate, self).filter(txt)
Florent Fourcot's avatar
Florent Fourcot committed
116 117 118 119 120 121 122 123 124 125 126 127 128 129 130


class INGCategory(Filter):
    catvalue = {u'virt': u"Virement", u'autre': u"Autre",
                u'plvt': u'Prélèvement', u'cb_ret': u"Carte retrait",
                u'cb_ach': u'Carte achat', u'chq': u'Chèque',
                u'frais': u'Frais bancaire', u'sepaplvt': u'Prélèvement'}

    def filter(self, txt):
        txt = txt.split('-')[0].lower()
        try:
            return self.catvalue[txt]
        except:
            return txt

Florent Fourcot's avatar
Florent Fourcot committed
131

132
class AccountsList(LoggedPage, HTMLPage):
Romain Bignon's avatar
Romain Bignon committed
133 134 135
    i = 0

    def has_error(self):
136
        return len(self.doc.xpath('//div[has-class("alert-warning")]')) > 0
Florent Fourcot's avatar
Florent Fourcot committed
137

138 139 140
    def has_link(self):
        return len(self.doc.xpath('//a[contains(@href, "goTo")]'))

141 142 143 144 145 146
    def get_card_list(self):
        card_list = []
        card_elements = self.doc.xpath('//div[has-class("ccinc_cards")]/div[has-class("accordion")]')
        for card in card_elements:
            card_properties = {}

147
            # Regexp parse the text to extract the card number that may be in different formats
148
            card_properties['number'] = Regexp(CleanText('.'), '(\d+[\s|*]+\d+)', default=NotAvailable)(card)
149 150
            debit_info = (CleanText('.//div[@class="debit-info"]', default='')(card))

151 152
            is_deferred = u'Débit différé' in debit_info
            is_immediate = u'Débit immédiat' in debit_info
153 154 155 156 157 158

            if is_immediate:
                card_properties['kind'] = self.browser.IMMEDIATE_CB
            elif is_deferred:
                card_properties['kind'] = self.browser.DEFERRED_CB
            else:
159
                raise DataError("Cannot tell if the card {} is deferred or immediate".format(card_properties['number']))
160 161 162 163 164

            card_list.append(card_properties)

        return card_list

165 166
    @method
    class get_list(ListElement):
167
        item_xpath = '//div[@id="bloc-menu-comptes"]//a[@class="mainclic"]'
168 169 170 171 172

        class item(ItemElement):
            klass = Account

            obj_currency = u'EUR'
173
            obj_label = CleanText('./span[@class="title"]')
174 175 176 177 178
            obj_id = AddPref(Field('_id'), Field('label'))
            obj_type = AddType(Field('label'))
            obj_coming = NotAvailable
            obj__jid = Attr('//input[@name="javax.faces.ViewState"]', 'value')

179
            def obj_balance(self):
180
                balance = CleanDecimal('./span[@class="solde"]/label', replace_dots=True)(self)
181 182
                return -abs(balance) if Field('type')(self) == Account.TYPE_LOAN else balance

183
            def obj__id(self):
184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203
                return CleanText('./span[@class="account-number"]')(self) or CleanText('./span[@class="life-insurance-application"]')(self)

    @method
    class get_detailed_loans(ListElement):
        item_xpath = '//div[@class="mainclic"]'

        class item(ItemElement):
            klass = Loan

            obj_currency = u'EUR'
            obj_label = CleanText('./span[@class="title"]')
            obj_id = AddPref(Field('_id'), Field('label'))
            obj_type = AddType(Field('label'))
            obj_coming = NotAvailable
            obj__jid = Attr('//input[@name="javax.faces.ViewState"]', 'value')
            obj__id = CleanText('./span[@class="account-number"]')

            def obj_balance(self):
                balance = CleanDecimal('./span[@class="solde"]/label', replace_dots=True)(self)
                return -abs(balance)
204

205
    class generic_transactions(ListElement):
206 207 208
        class item(ItemElement):
            klass = Transaction

Romain Bignon's avatar
Romain Bignon committed
209
            obj_id = None  # will be overwrited by the browser
210
            # we use lower for compatibility with the old website
211
            obj_amount = CleanDecimal('.//td[starts-with(@class, "amount")]', replace_dots=True)
Florent Fourcot's avatar
Florent Fourcot committed
212
            obj_date = INGDate(CleanText('.//td[@class="date"]'), dayfirst=True)
213
            obj_rdate = Field('date')
214
            obj__hash = PreHashmd5(Field('date'), Field('raw'), Field('amount'))
Florent Fourcot's avatar
Florent Fourcot committed
215
            obj_category = INGCategory(Attr('.//td[@class="picto"]/span', 'class'))
216

217 218 219
            def obj_raw(self):
                return Transaction.Raw(Lower('.//td[@class="lbl"]'))(self) or Format('%s %s', Field('date'), Field('amount'))(self)

220
            def condition(self):
221 222
                date_field = self.el.find('.//td[@class="date"]')
                if date_field is None or 'À venir' in CleanText().filter(date_field):
223
                    return False
224
                if 'index' in self.env and self.env['index'] > 0 and self.page.i < self.env['index']:
225 226
                    self.page.i += 1
                    return False
227 228
                return True

229 230
    @method
    class iter_asv_investments(ListElement):
231
        item_xpath = '//div[@id="index:accountdetail"]//div[has-class("asv_fond")]'
232 233 234 235 236

        class item(ItemElement):
            klass = Investment

            obj_portfolio_share = Eval(lambda x: x / 100, CleanDecimal('.//dl[@class="ligne-repartition"]/dd', replace_dots=True))
237
            obj_label = CleanText('.//span[@class="asv_cat_lbl"]')
238

239 240 241 242 243
    @method
    class get_coming(generic_transactions):
        item_xpath = '//div[@class="transactions cc future"]//table'

    @method
244
    class get_transactions_cc(generic_transactions):
245 246
        item_xpath = '//div[@class="temporaryTransactionList"]//table'

247 248
    @method
    class get_transactions_others(generic_transactions):
249
        item_xpath = '//table'
250

251
    def get_history_jid(self):
252
        span = Attr('//*[starts-with(@id, "index:j_id")]', 'id')(self.doc)
253
        jid = span.split(':')[1]
254 255
        return jid

256 257 258
    def get_asv_jid(self):
        return self.doc.xpath('//input[@id="javax.faces.ViewState"]/@value')[0]

259
    def islast(self):
260
        havemore = self.doc.getroot().cssselect('.show-more-transactions')
261 262 263
        if len(havemore) == 0:
            return True

264
        nomore = self.doc.getroot().cssselect('.no-more-transactions')
265
        return len(nomore) > 0
Romain Bignon's avatar
Romain Bignon committed
266 267 268 269 270 271

    @property
    def is_asv(self):
        span = self.doc.xpath('//span[@id="index:panelASV"]')
        return len(span) > 0

272
    @property
Baptiste Delpey's avatar
Baptiste Delpey committed
273
    def asv_has_detail(self):
274 275 276 277 278 279 280
        ap = self.doc.xpath('//a[@id="index:asvInclude:goToAsvPartner"] | //p[contains(text(), "Gestion Libre")]')
        return len(ap) > 0

    @property
    def asv_is_other(self):
        a = self.doc.xpath('//a[@id="index:asvInclude:goToAsvPartner"]')
        return len(a) > 0
281

282
    def submit(self):
283 284
        form = self.get_form(name="follow_link")
        form['follow_link:j_idcl'] = "follow_link:goToAsvPartner"
285
        form.submit()
286

Célande Adrien's avatar
Célande Adrien committed
287 288 289 290 291 292 293 294 295 296
    def get_multispace(self):
        multispace = []

        for a in self.doc.xpath('//a[contains(@id, "mainMenu")]'):
            space = {}
            name = CleanText('.')(a)
            if 'Vos comptes' not in name:
                space['name'] = name
            else:
                space['name'] = CleanText('//div[@class="print-content"]/h1')(a)
297

Célande Adrien's avatar
Célande Adrien committed
298 299 300
            space['id'] = Regexp(Attr('.', 'id'), r'mainMenu:(.*)')(a)
            space['form'] = Attr('.', 'onclick')(a)
            space['is_active'] = 'active' in CleanText('./@class')(a)
301

Célande Adrien's avatar
Célande Adrien committed
302
            multispace.append(space)
303

Célande Adrien's avatar
Célande Adrien committed
304
        return multispace
305 306 307 308 309 310 311 312 313

    def fillup_form(self, form, regexp, string):
        # fill form depending on JS
        link = re.search(regexp, string).group(1)
        parts = link.split(',')
        for p in parts:
            f = p.split("':'")
            form[f[0].replace("'", '')] = f[1].replace("'", '')

Célande Adrien's avatar
Célande Adrien committed
314
    def change_space(self, space):
315
        form = self.get_form(id='mainMenu')
Célande Adrien's avatar
Célande Adrien committed
316
        self.fillup_form(form, r"':\{(.*)\}\s\}", space['form'])
317 318 319
        form['AJAXREQUEST'] = '_viewRoot'
        form.submit()

Célande Adrien's avatar
Célande Adrien committed
320 321 322
    def load_space_page(self):
        # The accounts page exists in two forms: with the spaces list and without
        # When having the spaceless page, a form must be submit to access the space page
323 324 325 326 327
        form = self.get_form(id='user-menu')
        on_click = self.doc.xpath('//a[contains(@class, "comptes")]/@onclick')[1]
        self.fillup_form(form, r"\),\{(.*)\},'", on_click)
        form.submit()

328 329 330 331 332 333 334 335
class IbanPage(LoggedPage, HTMLPage):
    def get_iban(self):
        iban = CleanText('//tr[td[1]//text()="IBAN"]/td[2]')(self.doc).strip().replace(' ', '')
        if not iban or 'null' in iban:
            return NotAvailable
        return iban


336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353
class LoanTokenPage(LoggedPage, HTMLPage):
    def on_load(self):
        form = self.get_form()
        form.submit()


class LoanDetailPage(LoggedPage, JsonPage):
    def getdetails(self, loan):
        loan.total_amount = CleanDecimal(Dict('amount'))(self.doc)
        loan.maturity_date = Date(Dict('loanEndDate'))(self.doc)
        loan.duration = Dict('loanDuration')(self.doc)
        loan.rate = CleanDecimal(Dict('variableInterestRate'))(self.doc) / 100
        loan.nb_payments_left = Dict('remainingMonth')(self.doc)
        loan.last_payment_date = Date(Dict('lastRefundDate'))(self.doc)
        loan.next_payment_date = Date(Dict('nextRefundDate'))(self.doc)
        loan.next_payment_amount = CleanDecimal(Dict('monthlyRefund'))(self.doc)


354 355 356 357
class TitreDetails(LoggedPage, HTMLPage):
    def submit(self):
        form = self.get_form()
        form.submit()
Romain Bignon's avatar
Romain Bignon committed
358 359


Baptiste Delpey's avatar
Baptiste Delpey committed
360
class ASVInvest(LoggedPage, HTMLPage):
361
    @method
362
    class iter_investments(TableElement):
363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378
        # Ignore the first line:
        # <tr>
        #     <td colspan="5" class="enteteTableau metaEnteteTableau enteteTableauFirstCol metaEnteteTableauFirstCol">R&eacute;partition de
        #         l'investissement
        #     </td>
        #     <td colspan="3" class="enteteTableau metaEnteteTableau enteteTableauFirstCol metaEnteteTableauFirstCol">
        #         Plus/moins-values&nbsp;(**)
        #     </td>
        # </tr>
        # Then, there is the line of column heads.
        # Ignore also information lines like that:
        # <tr>
        #       <td colspan="8" class="liTableau" align="left">Gestion Pilotée
        #       <td>
        # </tr>
        item_xpath = '//table[@class="Tableau"]//tr[position()>2 and count(./td) >= 8]'
379 380 381 382 383 384 385
        head_xpath = '//table[@class="Tableau"]//tr[position()=2]/td'

        col_label = u'Support(s)'
        col_vdate = re.compile('Date')
        col_unitvalue = u'Valeur de part'
        col_quantity = u'Nombre de parts'
        col_valuation = u'Contre-valeur'
386
        col_unitprice = [u'Prix revient', u'PAM']
387
        col_diff = u'Montant'
388
        col_diff_percent = u'%'
389

390 391
        class item(ItemElement):
            klass = Investment
392 393 394 395 396 397

            # Euro funds links are like that:
            # <td class="lpTableau lpTableauFirstCol"><a href="javascript:alert('Les performances de ce fond ne sont pas consultables.')" onclick="">Eurossima
            # </a></td>
            # So ignore them.
            load_details = Link('.//td[1]//a') & Regexp(pattern='^((?!javascript:).*)', default=NotAvailable) & AsyncLoad
398 399

            def obj_code(self):
400
                val = Async('details', CleanText('//td[@class="libelle-normal" and contains(.,"CodeISIN")]', default=NotAvailable))(self)
401 402 403 404
                if val:
                    return val.split('CodeISIN : ')[1] if val else val
                else:
                    return NotAvailable
405

406
            def obj_diff_percent(self):
407 408 409 410
                diff = CleanDecimal(TableCell('diff_percent'), replace_dots=True, default=NotAvailable)(self)
                if not diff:
                    return diff
                return diff / 100
411 412 413

            obj_label = CleanText(TableCell('label'))
            obj_vdate = Date(CleanText(TableCell('vdate')), dayfirst=True)
414
            obj_unitvalue = CleanDecimal(TableCell('unitvalue'), replace_dots=True, default=NotAvailable)
415
            obj_quantity = CleanDecimal(TableCell('quantity'), default=NotAvailable)
416
            obj_valuation = CleanDecimal(TableCell('valuation'), replace_dots=True)
417
            obj_unitprice = CleanDecimal(TableCell('unitprice', default=None), replace_dots=True, default=NotAvailable)
418
            obj_diff = CleanDecimal(TableCell('diff'), replace_dots=True, default=NotAvailable)
Romain Bignon's avatar
Romain Bignon committed
419 420


421
class DetailFondsPage(LoggedPage, HTMLPage):
422 423
    def get_isin_code(self):
        return CleanText('//td[contains(text(), "CodeISIN")]/b', default=NotAvailable)(self.doc)
424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480


def MyInput(*args, **kwargs):
    args = (u'//input[contains(@name, "%s")]' % args[0], 'value',)
    kwargs.update(default=NotAvailable)
    return Attr(*args, **kwargs)


def MySelect(*args, **kwargs):
    args = (u'//select[contains(@name, "%s")]/option[@selected]' % args[0],)
    kwargs.update(default=NotAvailable)
    return CleanText(*args, **kwargs)


class ProfilePage(LoggedPage, HTMLPage):
    @method
    class get_profile(ItemElement):
        klass = Person

        obj_name = CleanText('//a[has-class("hme-pm")]/span[@title]')
        obj_address = CleanText('//ul[@class="newPostAdress"]//dd[@class="withMessage"]')
        obj_country = CleanText('//dt[label[contains(text(), "Pays")]]/following-sibling::dd')
        obj_email = CleanText('//dt[contains(text(), "Email")]/following-sibling::dd/text()')
        obj_phone = Env('phone')
        obj_mobile = Env('mobile')

        def parse(self, el):
            pattern = '//dt[contains(text(), "%s")]/following-sibling::dd/label'
            phone = CleanText(pattern % "professionnel")(self)
            mobile = CleanText(pattern % "portable")(self)
            self.env['phone'] = phone or mobile
            self.env['mobile'] = mobile

    @method
    class update_profile(ItemElement):
        obj_job = MyInput('category_pro')
        obj_job_contract_type = MySelect('contractType')
        obj_company_name = MyInput('category_empl')
        obj_socioprofessional_category = MySelect('personal_form:csp')

        def obj_job_activity_area(self):
            return MySelect('business_sector')(self) or NotAvailable

        def obj_main_bank(self):
            return MySelect('present_bank')(self) or NotAvailable

        def obj_housing_status(self):
            return MySelect('housingType')(self) or NotAvailable

        def obj_job_start_date(self):
            month = MySelect('seniority_Month')(self)
            year = MySelect('seniority_Year')(self)
            return Date(default=NotAvailable).filter('01/%s/%s' % (month, year)) if month and year else NotAvailable

        def obj_birth_date(self):
            birth_date = self.page.browser.birthday
            return Date().filter("%s/%s/%s" % (birth_date[2:4], birth_date[:2], birth_date[-4:]))