Newer
Older
# This file is part of a weboob module.
# This weboob module is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This weboob module is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
# You should have received a copy of the GNU Lesser General Public License
# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
from __future__ import unicode_literals, division
import re
import requests
import base64
import math
import random
from decimal import Decimal
from io import BytesIO
Sylvie Ye
committed
from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
from weboob.capabilities.base import empty, find_object, NotAvailable
Sylvie Ye
committed
from weboob.capabilities.bank import (
Account, Recipient, TransferError, TransferBankError, Transfer,
AccountOwnership, AddRecipientBankError,
Sylvie Ye
committed
)
from weboob.capabilities.wealth import Investment, MarketOrder, MarketOrderDirection, MarketOrderType
from weboob.capabilities.bill import Document, Subscription, DocumentTypes
from weboob.capabilities.profile import Person, ProfileMissing
from weboob.browser.elements import method, ListElement, TableElement, ItemElement, DictElement
from weboob.browser.exceptions import ServerError
from weboob.browser.pages import LoggedPage, HTMLPage, JsonPage, FormNotFound, pagination, PartialHTMLPage
from weboob.browser.filters.html import Attr, Link, TableCell, AttributeNotFound, AbsoluteLink
Sylvie Ye
committed
from weboob.browser.filters.standard import (
CleanText, Field, Regexp, Format, Date, CleanDecimal, Map, AsyncLoad, Async, Env, Slugify,
BrowserURL, Eval, Currency, Base, Coalesce, MapIn, Lower,
Sylvie Ye
committed
)
Quentin Defenouillere
committed
from weboob.browser.filters.json import Dict
from weboob.exceptions import BrowserUnavailable, BrowserIncorrectPassword, ActionNeeded, ParseError
from weboob.tools.capabilities.bank.transactions import FrenchTransaction, parse_with_patterns
from weboob.tools.captcha.virtkeyboard import MappedVirtKeyboard, VirtKeyboardError
from weboob.tools.compat import unicode, urlparse, parse_qs, urljoin
from weboob.tools.date import parse_french_date
from weboob.tools.capabilities.bank.investments import is_isin_valid, IsinCode
def MyDecimal(*args, **kwargs):
kwargs.update(replace_dots=True, default=Decimal(0))
return CleanDecimal(*args, **kwargs)
def myXOR(value, seed):
s += chr(seed ^ ord(value[i]))
class LCLBasePage(HTMLPage):
def get_from_js(self, pattern, end, is_list=False):
"""
find a pattern in any javascript text
"""
value = None
for script in self.doc.xpath('//script'):
txt = script.text
if txt is None:
continue
start = txt.find(pattern)
if start < 0:
continue
while True:
if value is None:
value = ''
else:
value += ','
value += txt[start + len(pattern):start + txt[start + len(pattern):].find(end) + len(pattern)]
txt = txt[start + len(pattern) + txt[start + len(pattern):].find(end):]
start = txt.find(pattern)
if start < 0:
break
return value
class LCLVirtKeyboard(MappedVirtKeyboard):
symbols = {
'0': '9da2724133f2221482013151735f033c',
'1': '873ab0087447610841ae1332221be37b',
'2': '93ce6c330393ff5980949d7b6c800f77',
'3': 'b2d70c69693784e1bf1f0973d81223c0',
'4': '498c8f5d885611938f94f1c746c32978',
'5': '359bcd60a9b8565917a7bf34522052c3',
'6': 'aba912172f21f78cd6da437cfc4cdbd0',
'7': 'f710190d6b947869879ec02d8e851dfa',
'8': 'b42cc25e1539a15f767aa7a641f3bfec',
}
url = "/outil/UAUT/Clavier/creationClavier?random="
color = (255, 255, 255, 255)
def __init__(self, basepage):
img = basepage.doc.find("//img[@id='idImageClavier']")
self.url += "%s" % str(int(math.floor(int(random.random() * 1000000000000000000000))))
super(LCLVirtKeyboard, self).__init__(
BytesIO(basepage.browser.open(self.url).content),
basepage.doc, img, self.color, "id")
self.check_symbols(self.symbols, basepage.browser.responses_dirname)
def get_symbol_code(self, md5sum):
code = MappedVirtKeyboard.get_symbol_code(self, md5sum)
def get_string_code(self, string):
code += self.get_symbol_code(self.symbols[c])
class LoginPage(HTMLPage):
def on_load(self):
form = self.get_form(xpath='//form[@id="setInfosCGS" or @name="form"]')
except FormNotFound:
return
def login(self, login, passwd):
except VirtKeyboardError as err:
password = vk.get_string_code(passwd)
seed = -1
s = "var aleatoire = "
for script in self.doc.findall("//script"):
if script.text is None or len(script.text) == 0:
offset = script.text.find(s)
if offset != -1:
seed = int(script.text[offset + len(s) + 1:offset + len(s) + 2])
raise ParseError("Variable 'aleatoire' not found")
form = self.get_form('//form[@id="formAuthenticate"]')
form['identifiant'] = login
form['postClavierXor'] = base64.b64encode(
myXOR(password, seed).encode("utf-8")
)
form['identifiantRouting'] = self.browser.IDENTIFIANT_ROUTING
except AttributeError:
pass
form.submit(allow_redirects=False)
def check_error(self):
errors = self.doc.xpath(u'//*[@class="erreur" or @class="messError"]')
if not errors or self.doc.xpath('//a[@href="/outil/UWHO/Accueil/"]'):
return
for error in errors:
error_text = CleanText(error.xpath('./div/text()'))(self.doc)
if 'Suite à la saisie de plusieurs identifiant / code erronés' in error_text:
raise ActionNeeded(error_text)
if 'Votre identifiant ou votre code personnel est incorrect' in error_text:
raise BrowserIncorrectPassword(error_text)
raise BrowserIncorrectPassword()
class RedirectPage(LoginPage, PartialHTMLPage):
def is_here(self):
# During login a form submit with an allow_redirects=False is done
# The submit request can be done on contract urls following by a redirection
# So if we get a 302 this new class avoids misleading on_load
return self.response.status_code == 302
class ContractsPage(LoginPage, PartialHTMLPage):
# after login we are redirect in ContractsPage even if there is an error at login
# I let the error check code here to simplify
# a better solution will be to put error check on browser.py and error parsing in pages.py
self.check_error()
# To avoid skipping contract page the first time we see it,
# and to be able to get the contracts list from it
if self.browser.parsed_contracts:
self.select_contract()
def get_contracts_list(self):
return self.doc.xpath('//input[@name="contratId"]/@value')
def select_contract(self, id_contract=None):
link = self.doc.xpath('//a[contains(text(), "Votre situation globale")]')
self.browser.location(link[0].attrib['href'])
else:
form = self.get_form(nr=0)
if 'contratId' in form:
if id_contract:
form['contratId'] = id_contract
self.browser.current_contract = form['contratId']
class ContractsChoicePage(ContractsPage):
def on_load(self):
self.check_error()
if not self.logged and not self.browser.current_contract:
self.select_contract()
class OwnedItemElement(ItemElement):
def get_ownership(self, owner):
if re.search(r'(m|mr|me|mme|mlle|mle|ml)\.? (.*)\bou (m|mr|me|mme|mlle|mle|ml)\b(.*)', owner, re.IGNORECASE):
return AccountOwnership.CO_OWNER
elif all(n in owner for n in self.env['name'].split()):
return AccountOwnership.OWNER
return AccountOwnership.ATTORNEY
class AccountsPage(LoggedPage, HTMLPage):
def on_load(self):
warn = self.doc.xpath('//div[@id="attTxt"]')
raise BrowserIncorrectPassword(warn[0].text)
return CleanText('//li[@id="nomClient"]/p')(self.doc)
# XXX Ugly Hack to replace account by second occurrence.
# LCL pro website sometimes display the same account twice and only second link is valid to fetch transactions.
def store(self, obj):
assert obj.id
if obj.id in self.objects:
self.logger.warning('There are two objects with the same ID! %s' % obj.id)
self.objects[obj.id] = obj
return obj
item_xpath = '//tr[contains(@onclick, "redirect")]'
flush_at_end = True
klass = Account
def condition(self):
return '/outil/UWLM/ListeMouvement' in self.el.attrib['onclick']
def load_details(self):
link_id = Field('_link_id')(self)
if link_id:
account_url = urljoin(self.page.browser.BASEURL, link_id)
return self.page.browser.async_open(url=account_url)
return NotAvailable
NATURE2TYPE = {
'001': Account.TYPE_SAVINGS,
'004': Account.TYPE_CHECKING,
'005': Account.TYPE_CHECKING,
'006': Account.TYPE_CHECKING,
'007': Account.TYPE_SAVINGS,
'012': Account.TYPE_SAVINGS,
'023': Account.TYPE_CHECKING,
'036': Account.TYPE_SAVINGS,
'046': Account.TYPE_SAVINGS,
'047': Account.TYPE_SAVINGS,
'049': Account.TYPE_SAVINGS,
'058': Account.TYPE_CHECKING,
'068': Account.TYPE_PEA,
'069': Account.TYPE_SAVINGS,
}
obj__link_id = Format('%s&mode=190', Regexp(CleanText('./@onclick'), "'(.*)'"))
obj__agence = Regexp(Field('_link_id'), r'.*agence=(\w+)')
obj__compte = Regexp(Field('_link_id'), r'compte=(\w+)')
obj_id = Format('%s%s', Field('_agence'), Field('_compte'))
obj__transfer_id = Format('%s0000%s', Field('_agence'), Field('_compte'))
obj_label = CleanText('.//div[@class="libelleCompte"]')
obj_currency = FrenchTransaction.Currency('.//td[has-class("right")]')
obj_type = Map(Regexp(Field('_link_id'), r'.*nature=(\w+)'), NATURE2TYPE, default=Account.TYPE_UNKNOWN)
def obj_balance(self):
Damien Mat Jedrzejewski
committed
balance = None
if 'professionnels' in self.page.browser.url and Field('type')(self) == Account.TYPE_CHECKING:
Damien Mat Jedrzejewski
committed
# for pro accounts with comings, balance without comings must be fetched on details page
async_page = Async('details').loaded_page(self)
balance = async_page.get_balance_without_comings_main()
# maybe the next get_balance can be removed
# sometimes it returns the sum of transactions for last x days (47 ?)
if empty(balance):
self.logger.info('GET_BALANCE_MAIN EMPTY')
balance = async_page.get_balance_without_comings()
Damien Mat Jedrzejewski
committed
if not empty(balance):
return balance
return CleanDecimal.French('.//td[has-class("right")]')(self)
def obj_ownership(self):
async_page = Async('details').loaded_page(self)
owner = CleanText('//h5[contains(text(), "Titulaire")]')(async_page.doc)
return self.get_ownership(owner)
def get_deferred_cards(self):
trs = self.doc.xpath('//tr[contains(@onclick, "EncoursCB")]')
links = []
for tr in trs:
parent_id = Regexp(CleanText('./@onclick'), r'.*AGENCE=(\w+).*COMPTE=(\w+).*CLE=(\w+)', r'\1\2\3')(tr)
link = Regexp(CleanText('./@onclick'), "'(.*)'")(tr)
links.append((parent_id, link))
@method
class get_advisor(ItemElement):
klass = Advisor
obj_name = CleanText('//div[@id="contacterMaBqMenu"]//p[@id="itemNomContactMaBq"]/span')
obj_email = obj_mobile = obj_fax = NotAvailable
obj_phone = Regexp(
CleanText('//div[@id="contacterMaBqMenu"]//p[contains(text(), "Tel")]', replace=[(' ', '')]),
r'([\s\d]+)',
default=NotAvailable
)
obj_agency = CleanText('//div[@id="sousContentAgence"]//p[@class="itemSousTitreMenuMaBq"][1]')
def obj_address(self):
address = CleanText(
'//div[@id="sousContentAgence"]//p[@class="itemSousTitreMenuMaBq"][2]',
default=None
)(self)
city = CleanText(
'//div[@id="sousContentAgence"]//p[@class="itemSousTitreMenuMaBq"][3]',
default=None
)(self)
if not (address and city):
return NotAvailable
return "%s %s" % (address, city)
class LoansPage(LoggedPage, HTMLPage):
class get_list(TableElement):
item_xpath = '//table[.//th[contains(text(), "Emprunteur")]]/tbody/tr[td[3]]'
head_xpath = '//table[.//th[contains(text(), "Emprunteur")]]/thead/tr/th'
col_id = re.compile('Emprunteur')
col_balance = [u'Capital restant dû', re.compile('Sommes totales restant dues'), re.compile('Montant disponible')]
class account(ItemElement):
klass = Account
obj_balance = CleanDecimal(TableCell('balance'), replace_dots=True, sign='-')
obj_currency = FrenchTransaction.Currency(TableCell('balance'))
obj_number = Regexp(CleanText(TableCell('id'), replace=[(' ', ''), ('-', '')]), r'(\d{11}[A-Z])')
def obj_label(self):
has_type = CleanText('./ancestor::table[.//th[contains(text(), "Type")]]', default=None)(self)
if has_type:
return CleanText('./td[2]')(self)
else:
return CleanText('./ancestor::table/preceding-sibling::div[1]')(self).split(' - ')[0]
pattern = re.compile(
r'(m|mr|me|mme|mlle|mle|ml)\.? (.*)\b(ou)? (m|mr|me|mme|mlle|mle|ml)\b(.*)',
re.IGNORECASE
)
if pattern.search(CleanText(TableCell('id'))(self)):
return AccountOwnership.CO_OWNER
return AccountOwnership.OWNER
def parse(self, el):
label = Field('label')(self)
trs = self.xpath(
'//td[contains(text(), $label)]/ancestor::tr[1] | ./ancestor::table[1]/tbody/tr',
label=label
)
self.env['id'] = "%s%s%s" % (
Regexp(CleanText(TableCell('id')), r'(\w+)\s-\s(\w+)', r'\1\2')(self),
label.replace(' ', ''),
i,
)
class LoansProPage(LoggedPage, HTMLPage):
@method
class get_list(TableElement):
item_xpath = '//table[.//th[contains(text(), "Emprunteur")]]/tbody/tr[td[3]]'
head_xpath = '//table[.//th[contains(text(), "Emprunteur")]]/thead/tr/th'
flush_at_end = True
col_id = re.compile('Emprunteur')
col_balance = [u'Capital restant dû', re.compile('Sommes totales restant dues')]
class account(ItemElement):
klass = Account
obj_balance = CleanDecimal(TableCell('balance'), replace_dots=True, sign='-')
obj_currency = FrenchTransaction.Currency(TableCell('balance'))
obj_type = Account.TYPE_LOAN
obj_id = Env('id')
obj__transfer_id = None
obj_number = Regexp(CleanText(TableCell('id'), replace=[(' ', ''), ('-', '')]), r'(\d{11}[A-Z])')
def obj_label(self):
has_type = CleanText('./ancestor::table[.//th[contains(text(), "Nature libell")]]', default=None)(self)
if has_type:
return CleanText('./td[3]')(self)
else:
return CleanText('./ancestor::table/preceding-sibling::div[1]')(self).split(' - ')[0]
def parse(self, el):
label = Field('label')(self)
trs = self.xpath(
'//td[contains(text(), $label)]/ancestor::tr[1] | ./ancestor::table[1]/tbody/tr',
label=label
)
i = [i for i in range(len(trs)) if el == trs[i]]
self.env['id'] = "%s%s%s" % (
Regexp(CleanText(TableCell('id')), r'(\w+)\s-\s(\w+)', r'\1\2')(self),
label.replace(' ', ''),
i,
)
class Transaction(FrenchTransaction):
(
re.compile(r'^(?P<category>CB) (?P<text>RETRAIT) DU (?P<dd>\d+)/(?P<mm>\d+)'),
FrenchTransaction.TYPE_WITHDRAWAL,
),
(re.compile(r'^(?P<category>(PRLV|PE)( SEPA)?) (?P<text>.*)'), FrenchTransaction.TYPE_ORDER),
(re.compile(r'^(?P<category>CHQ\.) (?P<text>.*)'), FrenchTransaction.TYPE_CHECK),
(re.compile(r'^(?P<category>RELEVE CB) AU (\d+)/(\d+)/(\d+)'), FrenchTransaction.TYPE_CARD),
(
re.compile(r'^(?P<category>CB) (?P<text>.*) (?P<dd>\d+)/(?P<mm>\d+)/(?P<yy>\d+)'),
FrenchTransaction.TYPE_CARD,
),
(re.compile(r'^(?P<category>(PRELEVEMENT|TELEREGLEMENT|TIP)) (?P<text>.*)'), FrenchTransaction.TYPE_ORDER),
(re.compile(r'^(?P<category>(ECHEANCE\s*)?PRET)(?P<text>.*)'), FrenchTransaction.TYPE_LOAN_PAYMENT),
(
re.compile(r'^(TP-\d+-)?(?P<category>(EVI|VIR(EM(EN)?)?T?)(.PERMANENT)? ((RECU|FAVEUR) TIERS|SEPA RECU)?)( /FRM)?(?P<text>.*)'),
FrenchTransaction.TYPE_TRANSFER,
),
(re.compile(r'^(?P<category>REMBOURST)(?P<text>.*)'), FrenchTransaction.TYPE_PAYBACK),
(re.compile(r'^(?P<category>COM(MISSIONS?)?)(?P<text>.*)'), FrenchTransaction.TYPE_BANK),
(re.compile(r'^(?P<text>(?P<category>REMUNERATION).*)'), FrenchTransaction.TYPE_BANK),
(re.compile(r'^(?P<text>(?P<category>ABON.*?)\s*.*)'), FrenchTransaction.TYPE_BANK),
(re.compile(r'^(?P<text>(?P<category>RESULTAT .*?)\s*.*)'), FrenchTransaction.TYPE_BANK),
(re.compile(r'^(?P<text>(?P<category>TRAIT\..*?)\s*.*)'), FrenchTransaction.TYPE_BANK),
(re.compile(r'(?P<text>(?P<category>COTISATION).*)'), FrenchTransaction.TYPE_BANK),
(re.compile(r'(?P<text>(?P<category>INTERETS).*)'), FrenchTransaction.TYPE_BANK),
(re.compile(r'^(?P<category>REM CHQ) (?P<text>.*)'), FrenchTransaction.TYPE_DEPOSIT),
(re.compile(r'^VIREMENT.*'), FrenchTransaction.TYPE_TRANSFER),
(re.compile(r'.*(PRELEVEMENTS|PRELVT|TIP).*'), FrenchTransaction.TYPE_ORDER),
(re.compile(r'.*CHEQUE.*'), FrenchTransaction.TYPE_CHECK),
(re.compile(r'.*ESPECES.*'), FrenchTransaction.TYPE_DEPOSIT),
(re.compile(r'.*(CARTE|CB).*'), FrenchTransaction.TYPE_CARD),
(re.compile(r'.*(AGIOS|ANNULATIONS|IMPAYES|CREDIT).*'), FrenchTransaction.TYPE_BANK),
(re.compile(r'.*(FRAIS DE TENUE DE COMPTE).*'), FrenchTransaction.TYPE_BANK),
(re.compile(r'.*\b(RETRAIT)\b.*'), FrenchTransaction.TYPE_WITHDRAWAL),
class Pagination(object):
def next_page(self):
links = self.page.doc.xpath('//div[@class="pagination"] /a')
if len(links) == 0:
return
for link in links:
if link.xpath('./span')[0].text == 'Page suivante':
return link.attrib.get('href')
return
class AccountHistoryPage(LoggedPage, HTMLPage):
class _get_operations(Pagination, Transaction.TransactionsElement):
item_xpath = '//table[has-class("tagTab") and (not(@style) or @style="")]/tr'
head_xpath = '//table[has-class("tagTab") and (not(@style) or @style="")]/tr/th'
col_raw = [u'Vos opérations', u'Libellé']
class item(Transaction.TransactionElement):
def fill_env(self, page, parent=None):
# This *Element's parent has only the dateguesser in its env and we want to
# use the same object, not copy it.
self.env = parent.env
def obj_rdate(self):
rdate = self.obj.rdate
date = Field('date')(self)
if rdate > date:
date_guesser = Env('date_guesser')(self)
return date_guesser.guess_date(rdate.day, rdate.month)
return rdate
def obj__el(self):
return self.el
return (
self.parent.get_colnum('date') is not None
and len(self.el.findall('td')) >= 3
and self.el.get('class')
and 'tableTr' not in self.el.get('class')
)
def open_transaction_page(self, tr):
# Those are summary for deferred card transactions,
# they do not have details.
if CleanText('./td[contains(text(), "RELEVE CB")]')(tr._el):
return None
row = Attr('.', 'id', default=None)(tr._el)
assert row, 'HTML format of transactions details changed'
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
if not re.match(r'\d+', row):
return self.browser.open(
Attr('.', 'href')(tr._el),
method='POST',
)
return self.browser.open(
'/outil/UWLM/ListeMouvementsParticulier/accesDetailsMouvement?element=%s' % row,
method='POST',
)
def fix_transaction_stuff(self, obj, tr_page):
if obj.category == 'RELEVE CB':
obj.type = Transaction.TYPE_CARD_SUMMARY
raw = obj.raw
if tr_page:
# TODO move this xpath to the relevant page class
raw = CleanText(
'//td[contains(text(), "Libellé")]/following-sibling::*[1]|//td[contains(text(), "Nom du donneur")]/following-sibling::*[1]',
)(tr_page.doc)
if raw:
if (
obj.raw in raw
or raw in obj.raw
or ' ' not in obj.raw
):
obj.raw = raw
obj.label = raw
else:
obj.label = '%s %s' % (obj.raw, raw)
obj.raw = '%s %s' % (obj.raw, raw)
m = re.search(r'\d+,\d+COM (\d+,\d+)', raw)
if m:
obj.commission = -CleanDecimal(replace_dots=True).filter(m.group(1))
elif not obj.raw:
# Empty transaction label
# TODO move this xpath to the relevant page class
if tr_page:
obj.raw = obj.label = CleanText(
"""//td[contains(text(), "Nature de l'opération")]/following-sibling::*[1]"""
)(tr_page.doc)
if not obj.date:
if tr_page:
obj.date = Date(
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
"""//td[contains(text(), "Date de l'opération")]/following-sibling::*[1]""",
default=''
),
dayfirst=True,
default=NotAvailable
)(tr_page.doc)
obj.rdate = obj.date
if tr_page:
# TODO move this xpath to the relevant page class
obj.vdate = Date(
CleanText(
'//td[contains(text(), "Date de valeur")]/following-sibling::*[1]',
default=''
),
dayfirst=True,
default=NotAvailable
)(tr_page.doc)
# TODO move this xpath to the relevant page class
obj.amount = CleanDecimal(
'//td[contains(text(), "Montant")]/following-sibling::*[1]',
replace_dots=True,
default=NotAvailable
)(tr_page.doc)
# ugly hack to fix broken html
# sometimes transactions have really an amount of 0...
if not obj.amount and CleanDecimal(TableCell('credit'), default=None)(self) is None:
if tr_page:
# TODO move this xpath to the relevant page class
obj.amount = CleanDecimal(
u'//td[contains(text(), "Montant")]/following-sibling::*[1]',
replace_dots=True,
default=NotAvailable
)(tr_page.doc)
obj.type = Transaction.TYPE_UNKNOWN
if tr_page:
typestring = CleanText(
"""//td[contains(text(), "Nature de l'opération")]/following-sibling::*[1]"""
)(tr_page.doc)
if typestring:
for pattern, trtype in Transaction.PATTERNS:
match = pattern.match(typestring)
if match:
obj.type = trtype
break
# Some transactions have no details, but we can find the type of the transaction,
# the label and the category from the raw label.
if obj.type == Transaction.TYPE_UNKNOWN:
parse_with_patterns(obj.raw, obj, Transaction.PATTERNS)
def get_operations(self, date_guesser):
return self._get_operations(self)(date_guesser=date_guesser)
def get_balance_without_comings_main(self):
return CleanDecimal.French(
'//span[@class="mtSolde"]',
default=NotAvailable
)(self.doc)
Damien Mat Jedrzejewski
committed
def get_balance_without_comings(self):
return CleanDecimal.French(
'//span[contains(text(), "Opérations effectuées")]//ancestor::div[1]/following-sibling::div',
default=NotAvailable
)(self.doc)
def deferred_date(self):
deferred_date = Regexp(
CleanText('//div[@class="date"][contains(text(), "Carte")]'),
r'le ([^:]+)',
default=None
)(self.doc)
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
assert deferred_date, 'Cannot find deferred_date'
return parse_french_date(deferred_date).date()
def get_card_summary(self):
amount = CleanDecimal.French('//div[@class="montantEncours"]')(self.doc)
if amount:
t = Transaction()
t.date = t.rdate = self.deferred_date()
t.type = Transaction.TYPE_CARD_SUMMARY
t.label = t.raw = CleanText('//div[@class="date"][contains(text(), "Carte")]')(self.doc)
t.amount = abs(amount)
return t
def format_url(self, url):
cb_type = re.match(r'.*(UWCBEncours.*)/.*', url).group(1)
return '/outil/UWCB/%s/listeOperations' % cb_type
@method
class iter_multi_cards(TableElement):
head_xpath = '//table[@class="tagTab"]/tr/th'
item_xpath = '//table[@class="tagTab"]//tr[position()>1]'
col_label = re.compile('Type')
col_number = re.compile('Numéro')
col_owner = re.compile('Titulaire')
col_coming = re.compile('Montant')
class Item(ItemElement):
klass = Account
obj_type = Account.TYPE_CARD
obj_balance = Decimal(0)
obj_parent = Env('parent_account')
obj_coming = CleanDecimal.French(TableCell('coming'))
obj_currency = Currency(TableCell('coming'))
obj__transfer_id = None
obj__cards_list = CleanText(Env('cards_list'))
def obj__transactions_link(self):
link = Attr('.', 'onclick')(self)
url = re.match('.*\'(.*)\'\\.*', link).group(1)
return self.page.format_url(url)
def obj_number(self):
card_number = re.match('((XXXX ){3}X ([0-9]{3}))', CleanText(TableCell('number'))(self))
return card_number.group(1)[0:16] + card_number.group(1)[-3:]
def obj_label(self):
return '%s %s %s' % (
CleanText(TableCell('label'))(self),
CleanText(TableCell('owner'))(self),
Field('number')(self),
)
def obj_id(self):
card_number = re.match('((XXXX ){3}X([0-9]{3}))', CleanText(Field('number'))(self))
return '%s-%s' % (Env('parent_account')(self).id, card_number.group(3))
def get_single_card(self, parent_account):
account = Account()
card_info = CleanText('//select[@id="selectCard"]/option/text()')(self.doc)
# ex: VISA INFINITE DD M FIRSTNAME LASTNAME N°XXXX XXXX XXXX X103
regex = '(.*)N°((XXXX ){3}X([0-9]{3})).*'
card_infos = re.match(regex, card_info)
coming = CleanDecimal.French('//div[@class="montantEncours"]/text()')(self.doc)
account.id = '%s-%s' % (parent_account.id, card_infos.group(4))
account.type = Account.TYPE_CARD
account.parent = parent_account
account.balance = Decimal('0')
account.coming = coming
account.number = card_infos.group(2)
account.label = card_info
account.currency = parent_account.currency
account._transactions_link = self.format_url(self.url)
account._transfer_id = None
# We need to store this url. It will be useful later to get the transactions.
account._cards_list = self.url
return account
def get_child_cards(self, parent_account):
# There is a selector with only one entry when there is only one card
# But not when there are multiple card.
if self.doc.xpath('//select[@id="selectCard"]'):
return [self.get_single_card(parent_account)]
return list(self.iter_multi_cards(parent_account=parent_account, cards_list=self.url))
@method
class iter_transactions(TableElement):
item_xpath = '//tr[contains(@class, "ligne")]'
head_xpath = '//th'
col_date = re.compile('Date')
col_label = re.compile('Libellé')
col_amount = re.compile('Montant')
class item(ItemElement):
klass = Transaction
obj_rdate = obj_bdate = Date(CleanText(TableCell('date')), dayfirst=True)
obj_type = Transaction.TYPE_DEFERRED_CARD
obj_raw = obj_label = CleanText(TableCell('label'))
obj_amount = CleanDecimal.French(TableCell('amount'))
def obj_date(self):
return self.page.deferred_date()
def condition(self):
if Field('date')(self) < Field('rdate')(self):
self.logger.error(
'skipping transaction with rdate(%s) > date(%s) for label(%s)',
Field('rdate')(self), Field('date')(self), Field('label')(self)
)
return False
return True
TYPES = {
'plan épargne en actions': Account.TYPE_PEA,
"plan d'épargne en actions": Account.TYPE_PEA,
'plan épargne en actions bourse': Account.TYPE_PEA,
"plan d'épargne en actions bourse": Account.TYPE_PEA,
'pea pme bourse': Account.TYPE_PEA,
'pea pme': Account.TYPE_PEA,
def on_load(self):
"""
Sometimes we are directed towards a prior html page before accessing Bourse Page.
Submit the form to access the page that contains the Bourse Page's session cookie.
"""
try:
form = self.get_form(id='form')
except FormNotFound: # already on the targetted page
pass
else:
form.submit()
super(BoursePage, self).on_load()
def open_iframe(self):
# should be done always (in on_load)?
for iframe in self.doc.xpath('//iframe[@id="mainIframe"]'):
self.browser.location(iframe.attrib['src'])
break
return CleanText(
'//b[contains(text(), "Afin de sécuriser vos transactions, nous vous invitons à créer un mot de passe trading")]'
)(self.doc)
if 'onload' in self.doc.xpath('.//body')[0].attrib:
return re.search('"(.*?)"', self.doc.xpath('.//body')[0].attrib['onload']).group(1)
def get_fullhistory(self):
form = self.get_form(id="historyFilter")
form['cashFilter'] = "ALL"
# We can't go above 2 years
form['beginDayfilter'] = (datetime.strptime(form['endDayfilter'], '%d/%m/%Y') - timedelta(days=730)).strftime('%d/%m/%Y')
form.submit()
@method
class get_list(TableElement):
item_xpath = '//table[has-class("tableau_comptes_details")]//tr[td and not(parent::tfoot)]'
head_xpath = '//table[has-class("tableau_comptes_details")]/thead/tr/th'
col_label = 'Comptes'
col_titres = re.compile('Valorisation')
col_especes = re.compile('Solde espèces')
klass = Account
load_details = Field('_market_link') & AsyncLoad
obj__especes = CleanDecimal(TableCell('especes'), replace_dots=True, default=0)
obj__titres = CleanDecimal(TableCell('titres'), replace_dots=True, default=0)
obj_valuation_diff = Async('details') & CleanDecimal(
'//td[contains(text(), "value latente")]/following-sibling::td[1]',
replace_dots=True,
)
obj__market_id = Regexp(Attr(TableCell('label'), 'onclick'), r'nump=(\d+:\d+)')
obj__market_link = Regexp(Attr(TableCell('label'), 'onclick'), r"goTo\('(.*?)'")
obj__link_id = Async('details') & Link(u'//a[text()="Historique"]')
obj_balance = Field('_titres')
obj_currency = Currency(CleanText(TableCell('titres')))
number = CleanText((TableCell('label')(self)[0]).xpath('./div[not(b)]'))(self).replace(' - ', '')
m = re.search(r'(\d{11,})[A-Z]', number)
if m:
number = m.group(0)
return number
return "%s Bourse" % CleanText((TableCell('label')(self)[0]).xpath('./div[b]'))(self)
_label = ' '.join(Field('label')(self).split()[:-1]).lower()
for key in self.page.TYPES:
if key in _label:
return self.page.TYPES.get(key)
return Account.TYPE_MARKET
def obj_ownership(self):
owner = CleanText(TableCell('owner'))(self)
return self.get_ownership(owner)
def get_logout_link(self):
return Link('//a[contains(text(), "Retour aux comptes")]')(self.doc)
item_xpath = '//table[@id="tableValeurs"]/tbody/tr[@id and count(descendant::td) > 1]'
class item(ItemElement):
klass = Investment
obj_label = CleanText('.//td[2]/div/a')
obj_code = (
CleanText('.//td[2]/div/br/following-sibling::text()')
& Regexp(pattern='^([^ ]+).*', default=NotAvailable)
)
obj_quantity = MyDecimal('.//td[3]/span')
obj_diff = MyDecimal('.//td[7]/span')
obj_valuation = MyDecimal('.//td[5]')
def obj_code_type(self):
code = Field('code')(self)
if code and is_isin_valid(code):
return Investment.CODE_TYPE_ISIN
return NotAvailable
def obj_unitvalue(self):
if "%" in CleanText('.//td[4]')(self) and "%" in CleanText('.//td[6]')(self):
return MyDecimal('.//td[4]/text()')(self)
def obj_unitprice(self):
if "%" in CleanText('.//td[4]')(self) and "%" in CleanText('.//td[6]')(self):
return NotAvailable
return MyDecimal('.//td[6]')(self)
@pagination
@method
class iter_history(TableElement):
item_xpath = '//table[@id="historyTable" and thead]/tbody/tr'
head_xpath = '//table[@id="historyTable" and thead]/thead/tr/th'
col_date = 'Date'
col_label = u'Opération'
col_quantity = u'Qté'
col_code = u'Libellé'
col_amount = 'Montant'
def next_page(self):
form = self.page.get_form(id="historyFilter")
form['PAGE'] = int(form['PAGE']) + 1
if self.page.doc.xpath('//*[@data-page = $page]', page=form['PAGE']):
return requests.Request("POST", form.url, data=dict(form))
class item(ItemElement):
klass = Transaction
obj_date = Date(CleanText(TableCell('date')), dayfirst=True)
obj_type = Transaction.TYPE_BANK
obj_amount = CleanDecimal(TableCell('amount'), replace_dots=True)
obj_investments = Env('investments')
def obj_label(self):
return TableCell('label')(self)[0].xpath('./text()')[0].strip()
def parse(self, el):
i = None
if CleanText(TableCell('code'))(self):
i = Investment()
i.label = Field('label')(self)
i.code = unicode(TableCell('code')(self)[0].xpath('./text()[last()]')[0]).strip()
i.quantity = MyDecimal(TableCell('quantity'))(self)
i.valuation = Field('amount')(self)
i.vdate = Field('date')(self)