diff --git a/modules/spirica/browser.py b/modules/spirica/browser.py index fcc0cb9ea49b90aa3435f32bd3de9aa8aae7ec0d..17b24664d44b51d76b8b7f126068f05fa0d83447 100644 --- a/modules/spirica/browser.py +++ b/modules/spirica/browser.py @@ -26,52 +26,57 @@ class SpiricaBrowser(LoginBrowser): TIMEOUT = 60 + login = URL('/securite/login.xhtml', LoginPage) accounts = URL('/sylvea/client/synthese.xhtml', AccountsPage) details = URL('/sylvea/contrat/consultationContratEpargne.xhtml', DetailsPage) maintenance = URL('/maintenance.html', MaintenancePage) - def __init__(self, website, username, password, *args, **kwargs): - super(LoginBrowser, self).__init__(*args, **kwargs) + def __init__(self, website, *args, **kwargs): + super(SpiricaBrowser, self).__init__(*args, **kwargs) self.BASEURL = website - self.username = username - self.password = password + self.cache = {} + self.cache['invs'] = {} + self.cache['trs'] = {} def do_login(self): self.login.go().login(self.username, self.password) if self.login.is_here(): - raise BrowserIncorrectPassword + error = self.page.get_error() + raise BrowserIncorrectPassword(error) def get_subscription_list(self): return iter([]) @need_login def iter_accounts(self): - return self.accounts.stay_or_go().iter_accounts() + if 'accs' not in self.cache.keys(): + self.cache['accs'] = [a for a in self.accounts.stay_or_go().iter_accounts()] + return self.cache['accs'] @need_login def iter_investment(self, account): - # Get form to show PRM - form = self.location(account._link).page.get_investment_form() - return self.location(form.url, data=dict(form)).page.iter_investment() + if account.id not in self.cache['invs']: + # Get form to show PRM + form = self.location(account._link).page.get_investment_form() + invs = [i for i in self.location(form.url, data=dict(form)).page.iter_investment()] + self.cache['invs'][account.id] = invs + return self.cache['invs'][account.id] @need_login def iter_history(self, account): - # Get form to go to History's tab - form = self.location(account._link).page.get_historytab_form() - # Get form to show all transactions - form = self.location(form.url, data=dict(form)).page.get_historyallpages_form() - if form: - self.location(form.url, data=dict(form)) - # Get forms to expand details of all transactions - for form in self.page.get_historyexpandall_form(): - self.location(form.url, data=dict(form)) - # Get all transactions - self.skipped = [] - transactions = [] - for t in self.page.iter_history(): - transactions.append(t) - for t in self.page.iter_history_skipped(): - transactions.append(t) - return iter(sorted(transactions, key=lambda t: t.date, reverse=True)) + if account.id not in self.cache['trs']: + # Get form to go to History's tab + form = self.location(account._link).page.get_historytab_form() + # Get form to show all transactions + form = self.location(form.url, data=dict(form)).page.get_historyallpages_form() + if form: + self.location(form.url, data=dict(form)) + # Get forms to expand details of all transactions + for form in self.page.get_historyexpandall_form(): + # Can't async because of ReadTimeout + self.location(form.url, data=dict(form)) + trs = [t for t in self.page.iter_history()] + self.cache['trs'][account.id] = trs + return self.cache['trs'][account.id] diff --git a/modules/spirica/pages.py b/modules/spirica/pages.py index a5021b2bba40c71c287813cb593b2ff1d8b09d01..3bb1bba3976cb43c24caf5fc41eaa14fb72fcd1a 100644 --- a/modules/spirica/pages.py +++ b/modules/spirica/pages.py @@ -21,9 +21,9 @@ import re from weboob.browser.pages import HTMLPage, LoggedPage -from weboob.browser.elements import ItemElement, TableElement, SkipItem, method +from weboob.browser.elements import ItemElement, TableElement, method from weboob.browser.filters.standard import CleanText, Date, Regexp, CleanDecimal, \ - Env, TableCell, Field, Async, AsyncLoad, Eval + TableCell, Field, Async, AsyncLoad, Eval from weboob.browser.filters.html import Attr, Link from weboob.capabilities.bank import Account, Investment, Transaction from weboob.capabilities.base import NotAvailable @@ -48,6 +48,9 @@ def login(self, login, password): form['loginForm:login'] = "loginForm:login" form.submit() + def get_error(self): + return CleanText('//li[@class="erreurBox"]')(self.doc) + class AccountsPage(LoggedPage, HTMLPage): TYPES = {'Assurance Vie': Account.TYPE_LIFE_INSURANCE, 'Unknown': Account.TYPE_UNKNOWN} @@ -78,6 +81,43 @@ def obj_type(self): "Option fiscale")]/following-sibling::td', default="Unknown"))(self)] +class TableInvestment(TableElement): + col_label = u'Support' + col_vdate = u'Date de valeur' + col_unitvalue = u'Valeur de part' + col_quantity = u'Nombre de parts' + col_portfolio_share = u'%' + + +class ItemInvestment(ItemElement): + klass = Investment + + obj_label = CleanText(TableCell('label')) + obj_quantity = MyDecimal(TableCell('quantity', default=None)) + obj_unitvalue = MyDecimal(TableCell('unitvalue', default=None)) + obj_vdate = Date(CleanText(TableCell('vdate', default="")), dayfirst=True, default=NotAvailable) + + def obj_valuation(self): + valuation = MyDecimal(TableCell('valuation', default=None))(self) + h2 = CleanText('./ancestor::div[contains(@id, "Histo")][1]/preceding-sibling::h2[1]')(self) + return -valuation if valuation and any(word in h2.lower() for word in self.page.DEBIT_WORDS) else valuation + + def obj_portfolio_share(self): + ps = MyDecimal(TableCell('portfolio_share', default=None))(self) + return Eval(lambda x: x / 100, ps)(self) if ps else NotAvailable + + +class TableTransactionsInvestment(TableInvestment): + item_xpath = './tbody/tr' + head_xpath = './thead/tr/th' + + col_code = u'ISIN' + col_valuation = [u'Montant brut', u'Montant net'] + + class item(ItemInvestment): + obj_code = Regexp(CleanText(TableCell('code')), pattern='([A-Z]{2}\d{10})', default=NotAvailable) + + class DetailsPage(LoggedPage, HTMLPage): DEBIT_WORDS = [u'arrêté', 'rachat', 'frais', u'désinvestir'] @@ -90,27 +130,14 @@ def get_investment_form(self): return form @method - class iter_investment(TableElement): + class iter_investment(TableInvestment): item_xpath = '//div[contains(@id, "INVESTISSEMENT")]//table/tbody/tr[@data-ri]' head_xpath = '//div[contains(@id, "INVESTISSEMENT")]//table/thead/tr/th' - col_label = u'Support' - col_vdate = u'Date de valeur' - col_unitvalue = u'Valeur de part' - col_quantity = u'Nombre de parts' col_valuation = re.compile('Contre') - col_portfolio_share = u'%' - - class item(ItemElement): - klass = Investment - obj_label = CleanText(TableCell('label')) + class item(ItemInvestment): obj_code = Regexp(CleanText('.//td[contains(text(), "Isin")]'), ':[\s]+([\w]+)', default=NotAvailable) - obj_quantity = MyDecimal(TableCell('quantity')) - obj_unitvalue = MyDecimal(TableCell('unitvalue')) - obj_valuation = MyDecimal(TableCell('valuation')) - obj_vdate = Date(CleanText(TableCell('vdate')), dayfirst=True, default=NotAvailable) - obj_portfolio_share = Eval(lambda x: x / 100, MyDecimal(TableCell('portfolio_share'))) def obj_unitprice(self): return MyDecimal('//div[contains(@id, "PRIX_REVIENT")]//a[contains(text(), \ @@ -149,36 +176,6 @@ def get_historyexpandall_form(self): form['ongletHistoOperations:newoperations_expandedRowIndex'] = data yield form - def get_investments(self, el, xpath='.'): - # Get all positions of th - positions = {} - keys = {'isin': 'code', 'support': 'label', 'supports': 'label', 'nombre de parts': 'quantity', 'valeur de part': \ - 'unitvalue', 'montant brut': 'valuation', 'date de valeur': 'vdate', '%': 'portfolio_share'} - for position, th in enumerate(el.xpath("%s//thead//th" % xpath)): - key = CleanText().filter(th.xpath('.')).lower() - if key in keys: - positions[keys[key]] = position + 1 - - investments = [] - for tr in el.xpath("%s//tbody/tr[@data-ri]" % xpath): - i = Investment() - i.label = CleanText().filter(tr.xpath('./td[%s]' % positions['label'])) \ - if "label" in positions else NotAvailable - i.code = Regexp(CleanText('./td[%s]' % positions['code']), pattern='([A-Z]{2}\d{10})', default=NotAvailable)(tr) - i.quantity = MyDecimal().filter(tr.xpath('./td[%s]' % positions['quantity'])) \ - if "quantity" in positions else NotAvailable - i.unitvalue = MyDecimal().filter(tr.xpath('./td[%s]' % positions['unitvalue'])) \ - if "unitvalue" in positions else NotAvailable - i.valuation = MyDecimal().filter(tr.xpath('./td[%s]' % positions['valuation'])) \ - if "valuation" in positions else NotAvailable - i.vdate = Date(CleanText('./td[%s]' % positions['vdate']), dayfirst=True, default=NotAvailable)(tr) \ - if "vdate" in positions else NotAvailable - i.portfolio_share = Eval(lambda x: x / 100).filter([MyDecimal().filter(tr.xpath('./td[%s]' % positions['portfolio_share']))]) \ - if "portfolio_share" in positions else NotAvailable - investments.append(i) - - return investments - @method class iter_history(TableElement): item_xpath = '//table/tbody[@id and not(contains(@id, "j_idt"))]/tr[@data-ri]' @@ -197,7 +194,6 @@ class item(ItemElement): obj_label = CleanText(TableCell('label')) obj_vdate = Date(CleanText(TableCell('vdate')), dayfirst=True) obj_type = Transaction.TYPE_BANK - obj_investments = Env('investments') def obj_amount(self): amount = MyDecimal(TableCell('net') if not CleanText(TableCell('brut'))(self) else TableCell('brut'))(self) @@ -209,30 +205,7 @@ def obj_date(self): def condition(self): return u"Validé" in CleanText(TableCell('status'))(self) - def parse(self, el): - if u"Désinvestir" in CleanText('./following-sibling::tr[1]')(self): - self.page.browser.skipped.append([el, el.xpath('./following-sibling::tr[1]')[0]]) - raise SkipItem() - - self.env['investments'] = self.page.get_investments(el, \ - './following-sibling::tr[1]//span[contains(text(), "ISIN")]/ancestor::table[1]') - - def iter_history_skipped(self): - for tr1, tr2 in self.browser.skipped: - for table, h2 in zip(tr2.xpath('.//table[@role]'), tr2.xpath(u'.//h2')): - t = Transaction() - - t.vdate = Date(CleanText('./td[8]'), dayfirst=True)(tr1) - t.date = Date(CleanText('./td[6]'), dayfirst=True, default=t.vdate)(tr1) - t.type = Transaction.TYPE_BANK - t.label = u"%s - %s" % (CleanText().filter(tr1.xpath('./td[2]')), \ - CleanText().filter(h2.xpath('.'))) - t.amount = CleanDecimal(replace_dots=True, default=MyDecimal().filter( \ - tr1.xpath('./td[5]'))).filter(tr1.xpath('./td[4]')) - - if t.amount and any(word in t.label.lower() for word in self.DEBIT_WORDS): - t.amount = -t.amount - - t.investments = self.get_investments(table) - - yield t + def obj_investments(self): + investments = [] + for table in self.el.xpath('./following-sibling::tr[1]//span[contains(text(), "ISIN")]/ancestor::table[1]'): + investments = sum([investments, list(TableTransactionsInvestment(self.page, el=table)())], [])