# -*- coding: utf-8 -*- # Copyright(C) 2016 Edouard Lambert # # This file is part of weboob. # # weboob is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # weboob is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . from __future__ import unicode_literals import re from weboob.browser.pages import HTMLPage, LoggedPage from weboob.browser.elements import ItemElement, ListElement, TableElement, method from weboob.browser.filters.standard import CleanText, Date, Regexp, CleanDecimal, \ Field, Async, AsyncLoad, Eval, Currency from weboob.browser.filters.html import Attr, Link, TableCell from weboob.capabilities.bank import Account, Investment, Transaction from weboob.capabilities.base import NotAvailable, empty from weboob.exceptions import BrowserUnavailable from weboob.tools.compat import urljoin def MyDecimal(*args, **kwargs): kwargs.update(replace_dots=True, default=NotAvailable) return CleanDecimal(*args, **kwargs) class MaintenancePage(HTMLPage): def on_load(self): raise BrowserUnavailable(CleanText().filter(self.doc.xpath('//p'))) class LoginPage(HTMLPage): def login(self, login, password): form = self.get_form('//form[@id="loginForm"]') form['loginForm:name'] = login form['loginForm:password'] = password form['loginForm:login'] = "loginForm:login" form.submit() def get_error(self): return CleanText('//li[@class="erreurBox"]')(self.doc) class AccountsPage(LoggedPage, HTMLPage): TYPES = { 'Assurance Vie': Account.TYPE_LIFE_INSURANCE, 'Capitalisation': Account.TYPE_MARKET, 'Unknown': Account.TYPE_UNKNOWN, } @method class iter_accounts(TableElement): item_xpath = '//table[@role]/tbody/tr' head_xpath = '//table[@role]/thead/tr/th' col_label = u'Produit' col_id = u'Numéro de contrat' col_balance = re.compile(u'Montant') col_currency = u'Currency' class item(ItemElement): klass = Account load_details = Field('url') & AsyncLoad obj_id = CleanText(TableCell('id'), replace=[(' ', '')]) obj_label = CleanText(TableCell('label')) obj_balance = MyDecimal(TableCell('balance')) obj_valuation_diff = Async('details') & MyDecimal('//tr[1]/td[contains(text(), \ "value du contrat")]/following-sibling::td') obj_currency = Currency('//td[contains(@class,"donneeMontant")]') def obj_url(self): return urljoin(self.page.url, Link('.//a')(self)) def obj_type(self): return self.page.TYPES[Async('details', CleanText('//td[contains(text(), \ "Option fiscale")]/following-sibling::td', default="Unknown"))(self)] class TableInvestment(TableElement): col_label = re.compile(r'Supports?') col_vdate = u'Date de valeur' col_unitvalue = u'Valeur de part' col_quantity = u'Nombre de parts' col_portfolio_share = u'%' col__gestion_type = re.compile('Nom du profil') class ItemInvestment(ItemElement): klass = Investment obj_label = CleanText(TableCell('label')) obj_quantity = MyDecimal(TableCell('quantity', default=None)) obj_unitvalue = MyDecimal(TableCell('unitvalue', default=None)) obj_vdate = Date(CleanText(TableCell('vdate', default="")), dayfirst=True, default=NotAvailable) obj_code = Regexp(CleanText('.//td[contains(text(), "Isin")]'), ':[\s]+([\w]+)', default=NotAvailable) obj__invest_type = Regexp(CleanText('.//td[contains(text(), "Type")]'), ':[\s]+([\w ]+)', default=NotAvailable) def obj_valuation(self): valuation = MyDecimal(TableCell('valuation', default=None))(self) h2 = CleanText('./ancestor::div[contains(@id, "Histo")][1]/preceding-sibling::h2[1]')(self) return -valuation if valuation and any(word in h2.lower() for word in self.page.DEBIT_WORDS) else valuation def obj_portfolio_share(self): ps = MyDecimal(TableCell('portfolio_share', default=None))(self) return Eval(lambda x: x / 100, ps)(self) if not empty(ps) else NotAvailable def obj__gestion_type(self): if self.xpath('ancestor::tbody[ends-with(@id, "contratProfilTable_data")]'): # investments are nested in profiles, get profile type profile_table_el = self.xpath('ancestor::tr/ancestor::table[position() = 1]')[0] profile_table = ProfileTableInvestment(self.page, self, profile_table_el) gestion_type = profile_table.get_colnum('_gestion_type') assert gestion_type path = 'ancestor::tr/preceding-sibling::tr[@data-ri][position() = 1][1]/td[%d]' % (gestion_type + 1) return CleanText(path)(self) return NotAvailable class ProfileTableInvestment(TableInvestment): # used only when portfolio is divided in multiple "profiles" head_xpath = '//thead[ends-with(@id, ":contratProfilTable_head")]/tr/th' class DetailsPage(LoggedPage, HTMLPage): def build_doc(self, content): # The full transactions page is a broken XML and requires # content building to scrap all transactions properly: markers = [b'partial-response', b'ongletHistoOperations:ongletHistoriqueOperations'] if all(marker in content for marker in markers): parts = re.findall(br'\!\[CDATA\[(.*?)\!\[CDATA\[', content, re.DOTALL) return super(DetailsPage, self).build_doc(parts[0]) return super(DetailsPage, self).build_doc(content) DEBIT_WORDS = ['rachat', 'frais', u'désinvestir'] def count_transactions(self): return Regexp(CleanText('//span[@class="ui-paginator-current"][1]'), '(?<=sur )(\d+)', default=None)(self.doc) def goto_unitprice(self): form = self.get_form(id='ongletSituation:syntheseContrat') form['javax.faces.source'] = 'ongletSituation:ongletContratTab' form['javax.faces.partial.execute'] = 'ongletSituation:ongletContratTab' form['javax.faces.partial.render'] = 'ongletSituation:ongletContratTab' form['javax.faces.behavior.event'] = 'tabChange' form['javax.faces.partial.event'] = 'tabChange' form['ongletSituation:ongletContratTab_newTab'] = 'ongletSituation:ongletContratTab:PRIX_REVIENT_MOYEN' form['ongletSituation:ongletContratTab_tabindex'] = '1' form.submit() @method class iter_investment(TableInvestment): item_xpath = '//div[contains(@id,"INVESTISSEMENT")]//div[ends-with(@id, ":tableDetailSituationCompte")]//table/tbody/tr[@data-ri]' head_xpath = '//div[contains(@id,"INVESTISSEMENT")]//div[ends-with(@id, ":tableDetailSituationCompte")]//table/thead/tr/th' col_valuation = re.compile('Contre') class item(ItemInvestment): def obj_portfolio_share(self): inv_share = ItemInvestment.obj_portfolio_share(self) if self.xpath('ancestor::tbody[ends-with(@id, "contratProfilTable_data")]'): # investments are nested in profiles, row share is relative to profile share profile_table_el = self.xpath('ancestor::tr/ancestor::table[position() = 1]')[0] profile_table = ProfileTableInvestment(self.page, self, profile_table_el) share_idx = profile_table.get_colnum('portfolio_share') assert share_idx path = 'ancestor::tr/preceding-sibling::tr[@data-ri][position() = 1][1]/td[%d]' % (share_idx + 1) profile_share = MyDecimal(path)(self) assert profile_share #raise Exception('dtc') profile_share = Eval(lambda x: x / 100, profile_share)(self) return inv_share * profile_share else: return inv_share @method class iter_pm_investment(TableInvestment): item_xpath = '//div[contains(@id,"PRIX_REVIENT_MOYEN")]//div[ends-with(@id, ":tableDetailSituationCompte")]//table/tbody/tr[@data-ri]' head_xpath = '//div[contains(@id,"PRIX_REVIENT_MOYEN")]//div[ends-with(@id, ":tableDetailSituationCompte")]//table/thead/tr/th' col_diff = re.compile(u'.*PRM en €') col_diff_percent = re.compile('.*PRM en %') col_unitprice = re.compile('.*Prix de Revient Moyen') class item(ItemInvestment): obj_diff = MyDecimal(TableCell('diff'), default=NotAvailable) obj_diff_percent = Eval(lambda x: x/100, MyDecimal(TableCell('diff_percent'))) obj_unitprice = MyDecimal(TableCell('unitprice')) def obj_diff_percent(self): diff_percent = MyDecimal(TableCell('diff_percent'))(self) if diff_percent: return diff_percent / 100 else: return NotAvailable def go_historytab(self): form = self.get_form(id='ongletSituation:syntheseContrat') form['javax.faces.source'] = 'tabsPrincipauxConsultationContrat' form['javax.faces.partial.execute'] = 'tabsPrincipauxConsultationContrat' form['javax.faces.partial.render'] = 'messageBox+tabsPrincipauxConsultationContrat' form['javax.faces.behavior.event'] = 'tabChange' form['javax.faces.partial.event'] = 'tabChange' form['tabsPrincipauxConsultationContrat_contentLoad'] = 'true' form['tabsPrincipauxConsultationContrat_newTab'] = 'HISTORIQUE_OPERATIONS' form['ongletSituation:ongletContratTab_tabindex'] = '1' form.submit() def go_historyall(self, page_number): form = self.get_form(xpath='//form[contains(@id, "ongletHistoOperations:ongletHistoriqueOperations")]') # The form value varies (for example j_idt913 or j_idt62081) so we need to scrape it dynamically: form_value = Attr('//div[@id="ongletHistoOperations:ongletHistoriqueOperations:newoperations"]/div[1]', 'id')(self.doc) form['javax.faces.partial.ajax'] = 'true' form['javax.faces.partial.execute'] = form_value form['javax.faces.partial.render'] = form_value form['javax.faces.source'] = form_value form[form_value] = form_value form[form_value + '_encodeFeature'] = 'true' form[form_value + '_pagination'] = 'false' form[form_value + '_rows'] = '100' form[form_value + '_first'] = page_number * 100 form.submit() def go_investments_form(self, index): form = self.get_form(xpath='//form[contains(@id, "ongletHistoOperations:ongletHistoriqueOperations")]') form['javax.faces.behavior.event'] = 'rowToggle' form['javax.faces.partial.event'] = 'rowToggle' id_ = Attr('//div[contains(@id, "ongletHistoOperations:ongletHistoriqueOperations")][has-class("listeAvecDetail")]', 'id')(self.doc) form['javax.faces.source'] = id_ form['javax.faces.partial.execute'] = id_ form['javax.faces.partial.render'] = id_ + ':detail ' + id_ form[id_ + '_rowExpansion'] = 'true' form[id_ + '_encodeFeature'] = 'true' form[id_ + '_expandedRowIndex'] = index form.submit() @method class iter_history(ListElement): item_xpath = '//tr[@role="row"]' class item(ItemElement): klass = Transaction obj_label = CleanText('./td[2]') obj_type = Transaction.TYPE_BANK obj__index = Attr('.', 'data-ri') obj_vdate = Date(CleanText('./td[8]'), dayfirst=True) def obj_date(self): return Date(CleanText('./td[6]'), dayfirst=True, default=Field('vdate')(self))(self) def obj_amount(self): # We display the raw amount only if the net amount is not available. raw_amount = MyDecimal('./td[4]')(self) amount = MyDecimal('./td[5]', default=raw_amount)(self) return -amount if amount and any(word in Field('label')(self).lower() for word in self.page.DEBIT_WORDS) else amount def condition(self): # We do not scrape "Arrêté annuel" transactions since it is just a yearly synthesis of the contract, # nor "Fusion-absorption" transactions because they have no amount. return ( "Validé" in CleanText('./td[3]')(self) and "Arrêté annuel" not in Field('label')(self) and "Fusion-absorption" not in Field('label')(self) ) @method class iter_transactions_investments(TableInvestment): item_xpath = '//table[thead[.//span[text()="ISIN"]]]/tbody/tr' head_xpath = '//thead[.//span[text()="ISIN"]]//th' col_isin = 'ISIN' col_valuation = 'Montant net' col_portfolio_share = '%' class item(ItemElement): klass = Investment # Columns do not always appear depending on transactions so we need # to precise "default=NotAvailable" for all TableCell filters. obj_label = CleanText(TableCell('label', default=NotAvailable), default=NotAvailable) obj_vdate = Date(TableCell('vdate', default=NotAvailable), dayfirst=True, default=NotAvailable) obj_unitvalue = MyDecimal(TableCell('unitvalue', default=NotAvailable), default=NotAvailable) obj_quantity = MyDecimal(TableCell('quantity', default=NotAvailable), default=NotAvailable) obj_valuation = MyDecimal(TableCell('valuation', default=NotAvailable), default=NotAvailable) obj_portfolio_share = MyDecimal(TableCell('portfolio_share', default=NotAvailable), default=NotAvailable) def obj_code(self): code = CleanText(TableCell('isin', default=NotAvailable), default=NotAvailable)(self) return code if code != '-' else NotAvailable def obj_code_type(self): return Investment.CODE_TYPE_ISIN if Field('code')(self) else NotAvailable