diff --git a/modules/amazon/browser.py b/modules/amazon/browser.py index fc87804b97e6472e7e73cb2412cb5928d7f12825..a48bfa368aa19cacacb3d01c255385c77bf97dae 100644 --- a/modules/amazon/browser.py +++ b/modules/amazon/browser.py @@ -21,7 +21,9 @@ from datetime import date from weboob.browser import LoginBrowser, URL, need_login, StatesMixin -from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable, ImageCaptchaQuestion, BrowserQuestion +from weboob.exceptions import ( + BrowserIncorrectPassword, BrowserUnavailable, ImageCaptchaQuestion, BrowserQuestion, ActionNeeded +) from weboob.tools.value import Value from weboob.browser.browsers import ClientError @@ -85,6 +87,11 @@ def push_security_otp(self, pin_code): self.location('/ap/signin', data=res_form, headers=self.otp_headers) def handle_security(self): + otp_type = self.page.get_otp_type() + if otp_type == '/ap/signin': + # this otp will be always present until user deactivate it + raise ActionNeeded('You have enabled otp in your options, please deactivate it before synchronize') + if self.page.doc.xpath('//span[@class="a-button-text"]'): self.page.send_code() diff --git a/modules/amazon/pages.py b/modules/amazon/pages.py index 211b6287b2f177eb70b24b931f598a9ff639f6ea..2034e4572cb46f0f213e13c7ea7b56e03bc0dc75 100644 --- a/modules/amazon/pages.py +++ b/modules/amazon/pages.py @@ -45,9 +45,20 @@ def get_sub_link(self): class SecurityPage(HTMLPage): + def get_otp_type(self): + # amazon send us otp in two cases: + # - if it's the first time we connect to this account for an ip => manage it normally + # - if user has activated otp in his options => raise ActionNeeded, an ask user to deactivate it + form = self.get_form(xpath='//form[.//h1]') + url = form.url.replace(self.browser.BASEURL, '') + + # verify: this otp is sent by amazon when we connect to the account for the first time from a new ip or computer + # /ap/signin: this otp is a user activated otp which is always present + assert url in ('verify', '/ap/signin'), url + return url + def get_otp_message(self): - message = self.doc.xpath('//div[@class="a-box-inner"]/p') - return message[0] if message else None + return CleanText('//div[@class="a-box-inner"]/p')(self.doc) def send_code(self): form = self.get_form() diff --git a/modules/billetreduc/browser.py b/modules/billetreduc/browser.py index 1f0e8a63fc061192bd71d50d5fb83c1e674bde3b..4cdd8e433cf4a70d31d0ee243724965b8a1bf90e 100644 --- a/modules/billetreduc/browser.py +++ b/modules/billetreduc/browser.py @@ -27,7 +27,7 @@ class BilletreducBrowser(PagesBrowser): - BASEURL = 'http://www.billetreduc.com' + BASEURL = 'https://www.billetreduc.com' search = URL(r'/recherche.htm', SearchPage) results = URL(r'/search.htm', ResultsPage) diff --git a/modules/billetreduc/pages.py b/modules/billetreduc/pages.py index ac4ad02f5ad4ba56bce9930d42675dabd409c576..48467855963fa45c6b96ea021ea5488efd6181ed 100644 --- a/modules/billetreduc/pages.py +++ b/modules/billetreduc/pages.py @@ -166,7 +166,7 @@ def obj__date_hours(self): p.do_parse() return p.res - m = re.match('le \w+ \d+ \w+ \d+ à (\d+)h(\d*)$', txt) + m = re.match('le \w+ \d+ \w+ \d+ à (\d+)h(\d*)$', txt, re.UNICODE) return [(int(m.group(1)), int(m.group(2) or 0))] obj_start_date = Env('date') diff --git a/modules/bolden/pages.py b/modules/bolden/pages.py index 7588f13e96bf759c99947d8d3784f1de41752bc8..9d5ce4433a7f3e7727fab81f93b13d2375d87f27 100644 --- a/modules/bolden/pages.py +++ b/modules/bolden/pages.py @@ -61,8 +61,8 @@ class item(ItemElement): obj_label = 'Compte Bolden' obj_type = Account.TYPE_MARKET obj_currency = 'EUR' - obj_balance = CleanDecimal('//div[p[has-class("investor-state") and contains(text(),"Total compte Bolden :")]]/p[has-class("investor-status")]', replace_dots=True) - obj_valuation_diff = CleanDecimal('//p[has-class("rent-amount strong dashboard-text")]', replace_dots=True) + obj_balance = CleanDecimal.French('//div[p[has-class("investor-state") and contains(text(),"Total compte Bolden :")]]/p[has-class("investor-status")]') + obj_valuation_diff = CleanDecimal.French('//div[has-class("rent-total")]') @method class iter_investments(TableElement): @@ -93,7 +93,7 @@ def obj__docurl(self): return urljoin(self.page.url, Link('.//a')(TableCell('doc')(self)[0])) def get_liquidity(self): - return CleanDecimal('//div[p[contains(text(), "Fonds disponibles")]]/p[@class="investor-status strong"]', replace_dots=True)(self.doc) + return CleanDecimal.French('//div[p[contains(text(), "Fonds disponibles")]]/p[has-class("investor-status")]')(self.doc) class OperationsPage(LoggedPage, HTMLPage): diff --git a/modules/bouygues/browser.py b/modules/bouygues/browser.py index ed1965183d1695269267126c88ff29eae2973476..83cdd8fb67a82a8e9e421eab145c8eb0606f1ff9 100644 --- a/modules/bouygues/browser.py +++ b/modules/bouygues/browser.py @@ -41,15 +41,16 @@ class BouyguesBrowser(LoginBrowser): home = URL(r'https://www.bouyguestelecom.fr/mon-compte', HomePage) subscriber = URL(r'/personnes/(?P\d+)$', SubscriberPage) subscriptions = URL(r'/personnes/(?P\d+)/comptes-facturation', SubscriptionPage) + subscriptions_details = URL(r'/comptes-facturation/(?P\d+)/contrats-payes', SubscriptionDetailPage) document_file = URL(r'/comptes-facturation/(?P\d+)/factures/\d+/documents', DocumentFilePage) documents = URL(r'/comptes-facturation/(?P\d+)/factures', DocumentsPage) - sms_page = URL(r'http://www.mobile.service.bbox.bouyguestelecom.fr/services/SMSIHD/sendSMS.phtml', - r'http://www.mobile.service.bbox.bouyguestelecom.fr/services/SMSIHD/confirmSendSMS.phtml', + sms_page = URL(r'https://www.secure.bbox.bouyguestelecom.fr/services/SMSIHD/sendSMS.phtml', + r'https://www.secure.bbox.bouyguestelecom.fr/services/SMSIHD/confirmSendSMS.phtml', SendSMSPage) - confirm = URL(r'http://www.mobile.service.bbox.bouyguestelecom.fr/services/SMSIHD/resultSendSMS.phtml', UselessPage) - sms_error_page = URL(r'http://www.mobile.service.bbox.bouyguestelecom.fr/services/SMSIHD/SMS_erreur.phtml', + confirm = URL(r'https://www.secure.bbox.bouyguestelecom.fr/services/SMSIHD/resultSendSMS.phtml', UselessPage) + sms_error_page = URL(r'https://www.secure.bbox.bouyguestelecom.fr/services/SMSIHD/SMS_erreur.phtml', SendSMSErrorPage) profile = URL(r'/personnes/(?P\d+)/coordonnees', ProfilePage) diff --git a/modules/bp/browser.py b/modules/bp/browser.py index 0c6dd3ad7eb4dfeabfb621ac51e08b3ef88637e0..3790abf2c71fff42a6d90cd0e7c35b79bfa9b313 100644 --- a/modules/bp/browser.py +++ b/modules/bp/browser.py @@ -74,7 +74,10 @@ class BPBrowser(LoginBrowser, StatesMixin): '/voscomptes/canalXHTML/pret/encours/detaillerOffrePretConsoListe-encoursPrets.ea', '/voscomptes/canalXHTML/pret/creditRenouvelable/init-consulterCreditRenouvelable.ea', '/voscomptes/canalXHTML/pret/encours/rechercherPret-encoursPrets.ea', + '/voscomptes/canalXHTML/sso/commun/init-integration.ea\?partenaire', + '/voscomptes/canalXHTML/sso/lbpf/souscriptionCristalFormAutoPost.jsp', AccountList) + par_accounts_revolving = URL('https://espaceclientcreditconso.labanquepostale.fr/sav/accueil.do', AccountList) accounts_rib = URL(r'.*voscomptes/canalXHTML/comptesCommun/imprimerRIB/init-imprimer_rib.ea.*', '/voscomptes/canalXHTML/comptesCommun/imprimerRIB/init-selection_rib.ea', AccountRIB) @@ -103,10 +106,11 @@ class BPBrowser(LoginBrowser, StatesMixin): par_account_checking_history = URL('/voscomptes/canalXHTML/CCP/releves_ccp/init-releve_ccp.ea\?typeRecherche=10&compte.numero=(?P.*)', '/voscomptes/canalXHTML/CCP/releves_ccp/afficher-releve_ccp.ea', AccountHistory) - deferred_card_history = URL(r'/voscomptes/canalXHTML/CB/releveCB/init-mouvementsCarteDD.ea\?compte.numero=(?P\w+)&indexCarte=(?P\d+)&typeListe=(?P\d+)', AccountHistory) - deferred_card_history_multi = URL(r'/voscomptes/canalXHTML/CB/releveCB/preparerRecherche-mouvementsCarteDD.ea\?compte.numero=(?P\w+)&indexCarte=(?P\d+)&typeListe=(?P\d+)', AccountHistory) # &typeRecherche=10 + deferred_card_history = URL(r'/voscomptes/canalXHTML/CB/releveCB/init-mouvementsCarteDD.ea\?compte.numero=(?P\w+)&indexCompte=(?P\d+)&typeListe=(?P\d+)', AccountHistory) + deferred_card_history_multi = URL(r'/voscomptes/canalXHTML/CB/releveCB/preparerRecherche-mouvementsCarteDD.ea\?indexCompte=(?P\w+)&indexCarte=(?P\d+)&typeListe=(?P\d+)', AccountHistory) # &typeRecherche=10 par_account_checking_coming = URL('/voscomptes/canalXHTML/CCP/releves_ccp_encours/preparerRecherche-releve_ccp_encours.ea\?compte.numero=(?P.*)&typeRecherche=1', - '/voscomptes/canalXHTML/CB/releveCB/init-mouvementsCarteDD.ea\?compte.numero=(?P.*)&typeListe=1&typeRecherche=10', AccountHistory) + '/voscomptes/canalXHTML/CB/releveCB/init-mouvementsCarteDD.ea\?compte.numero=(?P.*)&typeListe=1&typeRecherche=10', + '/voscomptes/canalXHTML/CCP/releves_ccp_encours/preparerRecherche-releve_ccp_encours.ea\?indexCompte', AccountHistory) par_account_savings_and_invests_history = URL('/voscomptes/canalXHTML/CNE/releveCNE/init-releve_cne.ea\?typeRecherche=10&compte.numero=(?P.*)', '/voscomptes/canalXHTML/CNE/releveCNE/releveCNE-releve_cne.ea', AccountHistory) @@ -218,6 +222,10 @@ def do_login(self): @need_login def get_accounts_list(self): + if self.session.cookies.get('indicateur'): + # Malformed cookie to delete to reach other spaces + del self.session.cookies['indicateur'] + if self.accounts is None: accounts = [] to_check = [] @@ -237,7 +245,7 @@ def get_accounts_list(self): for account in self.page.iter_accounts(): if account.type == Account.TYPE_LOAN: self.location(account.url) - if 'CreditRenouvelable' not in account.url: + if 'initSSO' not in account.url: for loan in self.page.iter_loans(): loan.currency = account.currency accounts.append(loan) @@ -248,9 +256,11 @@ def get_accounts_list(self): student_loan.currency = account.currency accounts.append(student_loan) else: - for loan in self.page.iter_revolving_loans(): - loan.currency = account.currency - accounts.append(loan) + # The main revolving page is not accessible, we can reach it by this new way + self.location(self.absurl('/voscomptes/canalXHTML/sso/lbpf/souscriptionCristalFormAutoPost.jsp')) + self.page.go_revolving() + revolving_loan = self.page.get_revolving_attributes(account) + accounts.append(revolving_loan) page.go() elif account.type == Account.TYPE_PERP: @@ -308,7 +318,7 @@ def get_history(self, account): self.go_linebourse(account) return self.linebourse.iter_history(account.id) - if account.type == Account.TYPE_LOAN: + if account.type in (Account.TYPE_LOAN, Account.TYPE_REVOLVING_CREDIT): return [] if account.type == Account.TYPE_CARD: diff --git a/modules/bp/pages/accountlist.py b/modules/bp/pages/accountlist.py index 4160eb0e1c9ca3c4c8f409022f32168eec99f7be..aa63d305e13d2fc09b9e4589d8c41271f6292987 100644 --- a/modules/bp/pages/accountlist.py +++ b/modules/bp/pages/accountlist.py @@ -30,7 +30,7 @@ from weboob.browser.elements import ListElement, ItemElement, method, TableElement from weboob.browser.pages import LoggedPage, RawPage, PartialHTMLPage, HTMLPage from weboob.browser.filters.html import Link, TableCell -from weboob.browser.filters.standard import CleanText, CleanDecimal, Regexp, Env, Field, BrowserURL, Currency, Async, Date, Format +from weboob.browser.filters.standard import CleanText, CleanDecimal, Regexp, Env, Field, Currency, Async, Date, Format from weboob.exceptions import BrowserUnavailable from weboob.tools.compat import urljoin, unicode @@ -58,6 +58,8 @@ def condition(self): def obj_url(self): url = Link(u'./a', default=NotAvailable)(self) if url: + if 'CreditRenouvelable' in url: + url = Link(u'.//a[contains(text(), "espace de gestion crédit renouvelable")]')(self.el) return urljoin(self.page.url, url) return url @@ -74,9 +76,9 @@ def obj_coming(self): has_coming = False coming = 0 - self.page.browser.open(Field('url')(self)) - coming_operations = self.page.browser.open( - BrowserURL('par_account_checking_coming', accountId=Field('id'))(self)) + details_page = self.page.browser.open(Field('url')(self)) + coming_op_link = Regexp(Link(u'//a[contains(text(), "Opérations à venir")]'), r'../(.*)')(details_page.page.doc) + coming_operations = self.page.browser.open(self.page.browser.BASEURL + '/voscomptes/canalXHTML/CCP/' + coming_op_link) if CleanText('//span[@id="amount_total"]')(coming_operations.page.doc): has_coming = True @@ -91,10 +93,11 @@ def obj_coming(self): return NotAvailable def obj_iban(self): - response = self.page.browser.open( - '/voscomptes/canalXHTML/comptesCommun/imprimerRIB/init-imprimer_rib.ea?numeroCompte=%s' % Field('id')( - self)) - return response.page.get_iban() + rib_link = Link('//a[abbr[contains(text(), "RIB")]]', default=NotAvailable)(self.el) + if rib_link: + response = self.page.browser.open(rib_link) + return response.page.get_iban() + return NotAvailable def obj_type(self): types = {'comptes? bancaires?': Account.TYPE_CHECKING, @@ -137,6 +140,10 @@ def on_load(self): raise BrowserUnavailable() + def go_revolving(self): + form = self.get_form() + form.submit() + @property def no_accounts(self): return len(self.doc.xpath('//iframe[contains(@src, "/comptes_contrats/sans_")] |\ @@ -156,6 +163,22 @@ class item_account(item_account_generic): def condition(self): return item_account_generic.condition(self) + + def get_revolving_attributes(self, account): + loan = Loan() + loan.id = account.id + loan.label = '%s - %s' %(account.label, account.id) + loan.currency = account.currency + loan.url = account.url + + loan.available_amount = CleanDecimal('//tr[td[contains(text(), "Montant Maximum Autorisé") or contains(text(), "Montant autorisé")]]/td[2]')(self.doc) + loan.used_amount = loan.used_amount = CleanDecimal('//tr[td[contains(text(), "Montant Utilisé") or contains(text(), "Montant utilisé")]]/td[2]')(self.doc) + loan.available_amount = CleanDecimal(Regexp(CleanText('//tr[td[contains(text(), "Montant Disponible") or contains(text(), "Montant disponible")]]/td[2]'), r'(.*) au'))(self.doc) + loan._has_cards = False + loan.type = Account.TYPE_REVOLVING_CREDIT + return loan + + @method class iter_revolving_loans(ListElement): item_xpath = '//div[@class="bloc Tmargin"]//dl' diff --git a/modules/caissedepargne/browser.py b/modules/caissedepargne/browser.py index c45cd5c306401c93ad3987981414d80b26ffe97f..5dacfa9d25bd3ce0266f9a8c82ad513d3d8cd0fc 100644 --- a/modules/caissedepargne/browser.py +++ b/modules/caissedepargne/browser.py @@ -333,7 +333,7 @@ def loans_conso(self): days = ('Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun') month = ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec') now = datetime.datetime.today() - d = '%s %s %s %s:%s:%s GMT 0100 (CET)' % (days[now.weekday()], month[now.month - 1], now.year, now.hour, format(now.minute, "02"), now.second) + d = '%s %s %s %s %s:%s:%s GMT+0100 (heure normale d’Europe centrale)' % (days[now.weekday()], now.day, month[now.month - 1], now.year, now.hour, format(now.minute, "02"), now.second) if self.home.is_here(): msg = self.page.loan_unavailable_msg() if msg: @@ -597,7 +597,7 @@ def get_coming(self, account): @need_login def get_investment(self, account): self.deleteCTX() - if account.type not in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_MARKET, Account.TYPE_PEA) or 'measure_id' in account._info: + if account.type not in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_CAPITALISATION, Account.TYPE_MARKET, Account.TYPE_PEA) or 'measure_id' in account._info: raise NotImplementedError() if account.type == Account.TYPE_PEA and account.label == 'PEA NUMERAIRE': @@ -622,7 +622,7 @@ def get_investment(self, account): yield investment return - elif account.type == Account.TYPE_LIFE_INSURANCE: + elif account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_CAPITALISATION): if "MILLEVIE" in account.label: self.page.go_life_insurance(account) label = account.label.split()[-1] diff --git a/modules/caissedepargne/pages.py b/modules/caissedepargne/pages.py index eb87520dc5d1cf22a943be702a22427e347091a3..a9032b18857123517db18e4786d2d62a69698243 100644 --- a/modules/caissedepargne/pages.py +++ b/modules/caissedepargne/pages.py @@ -296,6 +296,8 @@ def _add_account(self, accounts, link, label, account_type, balance): account.type = self.ACCOUNT_TYPES.get(label, info['acc_type'] if 'acc_type' in info else account_type) if 'PERP' in account.label: account.type = Account.TYPE_PERP + if 'NUANCES CAPITALISATI' in account.label: + account.type = Account.TYPE_CAPITALISATION balance = balance or self.get_balance(account) account.balance = Decimal(FrenchTransaction.clean_amount(balance)) if balance and balance is not NotAvailable else NotAvailable @@ -315,10 +317,10 @@ def _add_account(self, accounts, link, label, account_type, balance): accounts[account.id] = account def get_balance(self, account): - if account.type not in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_PERP): + if account.type not in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_PERP, Account.TYPE_CAPITALISATION): return NotAvailable page = self.go_history(account._info).page - balance = page.doc.xpath('.//tr[td[ends-with(@id,"NumContrat")]/a[contains(text(),$id)]]/td[@class="somme"]', id=account.id) + balance = page.doc.xpath('.//tr[td[contains(@id,"NumContrat")]]/td[@class="somme"]/a[contains(@href, $id)]', id=account.id) if len(balance) > 0: balance = CleanText('.')(balance[0]) balance = balance if balance != u'' else NotAvailable @@ -408,7 +410,7 @@ def go_loans_conso(self, tr): if m: account = m.group(1) - form = self.get_form(name="main") + form = self.get_form(id="main") form['__EVENTTARGET'] = 'MM$SYNTHESE_CREDITS' form['__EVENTARGUMENT'] = 'ACTIVDESACT_CREDITCONSO&%s' % account form['m_ScriptManager'] = 'MM$m_UpdatePanel|MM$SYNTHESE_CREDITS' @@ -459,6 +461,9 @@ def get_loan_list(self): account._card_links = [] if "renouvelables" in CleanText('.')(title): + if 'JSESSIONID' in self.browser.session.cookies: + # Need to delete this to access the consumer loans space (a new one will be created) + del self.browser.session.cookies['JSESSIONID'] self.go_loans_conso(tr) d = self.browser.loans_conso() if d: @@ -502,11 +507,10 @@ class item(ItemElement): obj_currency = Currency(MyTableCell("balance")) obj_last_payment_date = Date(CleanText(MyTableCell("last_payment_date"))) obj_next_payment_amount = MyDecimal(MyTableCell("next_payment_amount")) - obj_next_payment_date = Date(CleanText(MyTableCell("next_payment_date"))) - + obj_next_payment_date = Date(CleanText(MyTableCell("next_payment_date", default=''), default=NotAvailable), default=NotAvailable) def go_list(self): - form = self.get_form(name='main') + form = self.get_form(id='main') form['__EVENTARGUMENT'] = "CPTSYNT0" @@ -525,7 +529,7 @@ def go_list(self): # On some pages, navigate to indexPage does not lead to the list of measures, so we need this form ... def go_measure_list(self): - form = self.get_form(name='main') + form = self.get_form(id='main') form['__EVENTARGUMENT'] = "MESLIST0" form['__EVENTTARGET'] = 'Menu_AJAX' @@ -537,7 +541,7 @@ def go_measure_list(self): # This function goes to the accounts page of one measure giving its id def go_measure_accounts_list(self, measure_id): - form = self.get_form(name='main') + form = self.get_form(id='main') form['__EVENTARGUMENT'] = "CPTSYNT0" @@ -556,7 +560,7 @@ def go_measure_accounts_list(self, measure_id): form.submit() def go_loan_list(self): - form = self.get_form(name='main') + form = self.get_form(id='main') form['__EVENTARGUMENT'] = "CRESYNT0" @@ -573,7 +577,7 @@ def go_loan_list(self): form.submit() def go_history(self, info, is_cbtab=False): - form = self.get_form(name='main') + form = self.get_form(id='main') form['__EVENTTARGET'] = 'MM$%s' % (info['type'] if is_cbtab else 'SYNTHESE') form['__EVENTARGUMENT'] = info['link'] @@ -587,7 +591,7 @@ def go_history(self, info, is_cbtab=False): def get_form_to_detail(self, transaction): m = re.match('.*\("(.*)", "(DETAIL_OP&[\d]+).*\)\)', transaction._link) # go to detailcard page - form = self.get_form(name='main') + form = self.get_form(id='main') form['__EVENTTARGET'] = m.group(1) form['__EVENTARGUMENT'] = m.group(2) fix_form(form) @@ -650,7 +654,7 @@ def go_next(self): # link = self.doc.xpath('//a[contains(@id, "lnkSuivante")]') - if len(link) == 0 or 'disabled' in link[0].attrib: + if len(link) == 0 or 'disabled' in link[0].attrib or link[0].attrib.get('class') == 'aspNetDisabled': return False account_type = 'COMPTE' @@ -658,7 +662,7 @@ def go_next(self): if m: account_type = m.group(1) - form = self.get_form(name='main') + form = self.get_form(id='main') form['__EVENTTARGET'] = "MM$HISTORIQUE_%s$lnkSuivante" % account_type form['__EVENTARGUMENT'] = '' @@ -679,7 +683,7 @@ def go_life_insurance(self, account): link = self.doc.xpath('//tr[td[contains(., ' + account.id + ') ]]//a')[0] m = re.search("PostBackOptions?\([\"']([^\"']+)[\"'],\s*['\"]((REDIR_ASS_VIE)?[\d\w&]+)?['\"]", link.attrib.get('href', '')) if m is not None: - form = self.get_form(name='main') + form = self.get_form(id='main') form['__EVENTTARGET'] = m.group(1) form['__EVENTARGUMENT'] = m.group(2) @@ -712,7 +716,7 @@ def go_transfer(self, account): else: link = link[0] m = re.search("PostBackOptions?\([\"']([^\"']+)[\"'],\s*['\"]([^\"']+)?['\"]", link.attrib.get('href', '')) - form = self.get_form(name='main') + form = self.get_form(id='main') if 'MM$HISTORIQUE_COMPTE$btnCumul' in form: del form['MM$HISTORIQUE_COMPTE$btnCumul'] form['__EVENTTARGET'] = m.group(1) @@ -728,7 +732,7 @@ def loan_unavailable_msg(self): return msg def go_subscription(self): - form = self.get_form(name='main') + form = self.get_form(id='main') form['m_ScriptManager'] = 'MM$m_UpdatePanel|MM$Menu_Ajax' form['__EVENTTARGET'] = 'MM$Menu_Ajax' link = Link('//a[contains(@title, "e-Documents") or contains(@title, "Relevés en ligne")]')(self.doc) @@ -1005,7 +1009,7 @@ def get_recipient_value(self, recipient): return recipient_value[0] def init_transfer(self, account, recipient, transfer): - form = self.get_form(name='main') + form = self.get_form(id='main') form['MM$VIREMENT$SAISIE_VIREMENT$ddlCompteDebiter'] = self.get_origin_account_value(account) form['MM$VIREMENT$SAISIE_VIREMENT$ddlCompteCrediter'] = self.get_recipient_value(recipient) form['MM$VIREMENT$SAISIE_VIREMENT$txtLibelleVirement'] = transfer.label @@ -1021,7 +1025,7 @@ class iter_recipients(MyRecipients): pass def continue_transfer(self, origin_label, recipient, label): - form = self.get_form(name='main') + form = self.get_form(id='main') type_ = 'intra' if recipient.category == u'Interne' else 'sepa' fill = lambda s, t: s % (t.upper(), t.capitalize()) form['__EVENTTARGET'] = 'MM$VIREMENT$m_WizardBar$m_lnkNext$m_lnkButton' @@ -1032,7 +1036,7 @@ def continue_transfer(self, origin_label, recipient, label): form.submit() def go_add_recipient(self): - form = self.get_form(name='main') + form = self.get_form(id='main') link = self.doc.xpath(u'//a[span[contains(text(), "Ajouter un compte bénéficiaire")]]')[0] m = re.search("PostBackOptions?\([\"']([^\"']+)[\"'],\s*['\"]([^\"']+)?['\"]", link.attrib.get('href', '')) form['__EVENTTARGET'] = m.group(1) @@ -1051,7 +1055,7 @@ def is_here(self): return bool(CleanText(u'//h2[contains(text(), "Confirmer mon virement")]')(self.doc)) def confirm(self): - form = self.get_form(name='main') + form = self.get_form(id='main') form['__EVENTTARGET'] = 'MM$VIREMENT$m_WizardBar$m_lnkNext$m_lnkButton' form.submit() @@ -1152,7 +1156,7 @@ class iter_recipients(MyRecipients): pass def init_transfer(self, account, recipient, transfer): - form = self.get_form(name='main') + form = self.get_form(id='main') form['MM$VIREMENT$SAISIE_VIREMENT$ddlCompteDebiter'] = self.get_origin_account_value(account) form['MM$VIREMENT$SAISIE_VIREMENT$ddlCompteCrediterPro'] = self.get_recipient_value(recipient) form['MM$VIREMENT$SAISIE_VIREMENT$Libelle'] = transfer.label @@ -1166,7 +1170,7 @@ def init_transfer(self, account, recipient, transfer): form.submit() def go_add_recipient(self): - form = self.get_form(name='main') + form = self.get_form(id='main') form['__EVENTTARGET'] = 'MM$VIREMENT$SAISIE_VIREMENT$ddlCompteCrediterPro' form['MM$VIREMENT$SAISIE_VIREMENT$ddlCompteCrediterPro'] = 'AC' form.submit() @@ -1228,7 +1232,7 @@ def is_here(self): return bool(CleanText(u'//h2[contains(text(), "Authentification réussie")]')(self.doc)) def go_on(self): - form = self.get_form(name='main') + form = self.get_form(id='main') form['__EVENTTARGET'] = 'MM$RETOUR_OK_SOL$m_ChoiceBar$lnkRight' form.submit() @@ -1247,7 +1251,7 @@ def is_here(self): //h2[contains(text(), "Confirmer l\'ajout d\'un compte bénéficiaire")]')(self.doc)) def post_recipient(self, recipient): - form = self.get_form(name='main') + form = self.get_form(id='main') form['__EVENTTARGET'] = '%s$m_WizardBar$m_lnkNext$m_lnkButton' % self.EVENTTARGET form['%s$m_RibIban$txtTitulaireCompte' % self.FORM_FIELD_ADD] = recipient.label for i in range(len(recipient.iban) // 4 + 1): @@ -1255,7 +1259,7 @@ def post_recipient(self, recipient): form.submit() def confirm_recipient(self): - form = self.get_form(name='main') + form = self.get_form(id='main') form['__EVENTTARGET'] = 'MM$WIZARD_AJOUT_COMPTE_EXTERNE$m_WizardBar$m_lnkNext$m_lnkButton' form.submit() @@ -1270,7 +1274,7 @@ def is_here(self): return self.need_auth() and self.doc.xpath('//span[@id="MM_ANR_WS_AUTHENT_ANR_WS_AUTHENT_SAISIE_lblProcedure1"]') def set_browser_form(self): - form = self.get_form(name='main') + form = self.get_form(id='main') form['__EVENTTARGET'] = 'MM$ANR_WS_AUTHENT$m_WizardBar$m_lnkNext$m_lnkButton' self.browser.recipient_form = dict((k, v) for k, v in form.items()) self.browser.recipient_form['url'] = form.url @@ -1306,8 +1310,9 @@ class get_detail(TableElement): def next_page(self): # only for new website, don't have any accounts with enough deferred card transactions on old webiste - if self.page.doc.xpath('//a[contains(@id, "lnkSuivante") and not(contains(@disabled,"disabled"))]'): - form = self.page.get_form(name='main') + if self.page.doc.xpath('//a[contains(@id, "lnkSuivante") and not(contains(@disabled,"disabled")) \ + and not(contains(@class, "aspNetDisabled"))]'): + form = self.page.get_form(id='main') form['__EVENTTARGET'] = "MM$ECRITURE_GLOBALE$lnkSuivante" form['__EVENTARGUMENT'] = '' fix_form(form) @@ -1329,12 +1334,12 @@ def go_form_to_summary(self): # return to first page to_history = Link(self.doc.xpath(u'//a[contains(text(), "Retour à l\'historique")]'))(self.doc) n = re.match('.*\([\'\"](MM\$.*?)[\'\"],.*\)$', to_history) - form = self.get_form(name='main') + form = self.get_form(id='main') form['__EVENTTARGET'] = n.group(1) form.submit() def go_newsite_back_to_summary(self): - form = self.get_form(name='main') + form = self.get_form(id='main') form['__EVENTTARGET'] = "MM$ECRITURE_GLOBALE$lnkRetourHisto" form.submit() @@ -1363,7 +1368,7 @@ def condition(self): def go_document_list(self, sub_id): target = Attr('//select[contains(@id, "ClientsBancaires")]', 'id')(self.doc) - form = self.get_form(name='main') + form = self.get_form(id='main') form['m_ScriptManager'] = target if 'palatine' in self.browser.BASEURL: form['MM$CONSULTATION_NUMERISATION_PALATINE$cboClientsBancaires'] = sub_id @@ -1383,16 +1388,25 @@ class iter_documents(ListElement): class item(ItemElement): klass = Document - obj_label = Format('%s %s', CleanText('./preceding::h3[1]'), CleanText('./span')) - obj_date = Date(CleanText('./span'), dayfirst=True) obj_type = DocumentTypes.OTHER obj_format = 'pdf' obj_url = Regexp(Link('.'), r'WebForm_PostBackOptions\("(\S*)"') - obj_id = Format('%s_%s_%s', Env('sub_id'), CleanText('./span', symbols='/'), Regexp(Field('url'), r'ctl(.*)')) + obj_id = Format('%s_%s_%s', Env('sub_id'), CleanText('./span', symbols='/', replace=[(' ', '_')]), Regexp(Field('url'), r'ctl(.*)')) obj__event_id = Regexp(Attr('.', 'onclick'), r"val\('(.*)'\);", default=None) + def obj_label(self): + if 'Récapitulatif de frais bancaires' in CleanText('./span')(self.el): + return CleanText('./span')(self.el) + return Format('%s %s', CleanText('./preceding::h3[1]'), CleanText('./span'))(self.el) + + def obj_date(self): + if 'Récapitulatif de frais bancaires' in CleanText('./span')(self.el): + year = Regexp(CleanText('./span'), r'(\d{4})')(self.el) + return Date(dayfirst=True).filter('31/12/%s' %year) + return Date(CleanText('./span'), dayfirst=True)(self.el) + def download_document(self, document): - form = self.get_form(name='main') + form = self.get_form(id='main') form['m_ScriptManager'] = document.url form['__EVENTTARGET'] = document.url form['MM$COMPTE_EDOCUMENTS$ctrlEDocumentsConsultationDocument$eventId'] = document._event_id diff --git a/modules/cmso/module.py b/modules/cmso/module.py index 61603dfb3450f1a87e2f2996c91dbc250d4cd8f5..84a01ba62c9472f2c045c4148022bf6fc5e5fede 100644 --- a/modules/cmso/module.py +++ b/modules/cmso/module.py @@ -19,9 +19,9 @@ from __future__ import unicode_literals -from weboob.capabilities.bank import CapBankWealth, AccountNotFound +from weboob.capabilities.bank import CapBankTransfer, CapBankWealth, Account, AccountNotFound, RecipientNotFound from weboob.capabilities.contact import CapContact -from weboob.capabilities.base import find_object +from weboob.capabilities.base import find_object, strict_find_object from weboob.capabilities.profile import CapProfile from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import Value, ValueBackendPassword @@ -33,7 +33,7 @@ __all__ = ['CmsoModule'] -class CmsoModule(Module, CapBankWealth, CapContact, CapProfile): +class CmsoModule(Module, CapBankTransfer, CapBankWealth, CapContact, CapProfile): NAME = 'cmso' MAINTAINER = 'Romain Bignon' EMAIL = 'romain@weboob.org' @@ -69,6 +69,39 @@ def iter_coming(self, account): def iter_investment(self, account): return self.browser.iter_investment(account) + def iter_transfer_recipients(self, origin_account): + if self.config['website'].get() != "par": + raise NotImplementedError() + + if not isinstance(origin_account, Account): + origin_account = self.get_account(origin_account) + return self.browser.iter_recipients(origin_account) + + def init_transfer(self, transfer, **params): + if self.config['website'].get() != "par": + raise NotImplementedError() + + self.logger.info('Going to do a new transfer') + + account = strict_find_object( + self.iter_accounts(), + error=AccountNotFound, + iban=transfer.account_iban, + id=transfer.account_id + ) + + recipient = strict_find_object( + self.iter_transfer_recipients(account.id), + error=RecipientNotFound, + iban=transfer.recipient_iban, + id=transfer.recipient_id + ) + + return self.browser.init_transfer(account, recipient, transfer.amount, transfer.label, transfer.exec_date) + + def execute_transfer(self, transfer, **params): + return self.browser.execute_transfer(transfer, **params) + def iter_contacts(self): if self.config['website'].get() != "par": raise NotImplementedError() diff --git a/modules/cmso/par/browser.py b/modules/cmso/par/browser.py index 846374cdd31e048e49763e84b54183ce851c79f2..7b6a921bf00e03ebd8db99424baba584a85ff44e 100644 --- a/modules/cmso/par/browser.py +++ b/modules/cmso/par/browser.py @@ -22,6 +22,7 @@ import re import json +from datetime import date from functools import wraps from weboob.browser import LoginBrowser, URL, need_login, StatesMixin @@ -33,8 +34,9 @@ from .pages import ( LogoutPage, InfosPage, AccountsPage, HistoryPage, LifeinsurancePage, MarketPage, - AdvisorPage, LoginPage, RecipientsPage, ProfilePage, + AdvisorPage, LoginPage, ProfilePage, ) +from .transfer_pages import TransferInfoPage, RecipientsListPage, TransferPage def retry(exc_check, tries=4): @@ -92,7 +94,11 @@ class CmsoParBrowser(LoginBrowser, StatesMixin): 'https://www.*/domiweb/prive/particulier', MarketPage) advisor = URL('/edrapi/v(?P\w+)/oauth/(?P\w+)', AdvisorPage) - recipients = URL(r'/domiapi/oauth/json/transfer/transferinfos', RecipientsPage) + # transfer + transfer_info = URL(r'/domiapi/oauth/json/transfer/transferinfos', TransferInfoPage) + recipients_list = URL(r'/domiapi/oauth/json/transfer/beneficiariesListTransfer', RecipientsListPage) + init_transfer_page = URL(r'/domiapi/oauth/json/transfer/controlTransferOperation', TransferPage) + execute_transfer_page = URL(r'/domiapi/oauth/json/transfer/transferregister', TransferPage) profile = URL(r'/domiapi/oauth/json/edr/infosPerson', ProfilePage) @@ -158,7 +164,7 @@ def iter_accounts(self): seen = {} - self.recipients.go(data='{"beneficiaryType":"INTERNATIONAL"}', headers=self.json_headers) + self.transfer_info.go(json={"beneficiaryType":"INTERNATIONAL"}) numbers = self.page.get_numbers() # First get all checking accounts... @@ -305,6 +311,74 @@ def iter_investment(self, account): return [] raise NotImplementedError() + @retry((ClientError, ServerError)) + @need_login + def iter_recipients(self, account): + self.transfer_info.go(json={"beneficiaryType":"INTERNATIONAL"}) + + if account.type in (Account.TYPE_LOAN, ): + return + if not account._eligible_debit: + return + + # internal recipient + for rcpt in self.page.iter_titu_accounts(): + if rcpt.id != account.id: + yield rcpt + for rcpt in self.page.iter_manda_accounts(): + if rcpt.id != account.id: + yield rcpt + for rcpt in self.page.iter_legal_rep_accounts(): + if rcpt.id != account.id: + yield rcpt + # external recipient + for rcpt in self.page.iter_external_recipients(): + yield rcpt + + @need_login + def init_transfer(self, account, recipient, amount, reason, exec_date): + self.recipients_list.go(json={"beneficiaryType":"INTERNATIONAL"}) + + transfer_data = { + 'beneficiaryIndex': self.page.get_rcpt_index(recipient), + 'debitAccountIndex': account._index, + 'devise': account.currency, + 'deviseReglement': account.currency, + 'montant': amount, + 'nature': 'externesepa', + 'transferToBeneficiary': True, + } + + if exec_date and exec_date > date.today(): + transfer_data['date'] = int(exec_date.strftime('%s')) * 1000 + else: + transfer_data['immediate'] = True + + # check if recipient is internal or external + if recipient.id != recipient.iban: + transfer_data['nature'] = 'interne' + transfer_data['transferToBeneficiary'] = False + + self.init_transfer_page.go(json=transfer_data) + transfer = self.page.handle_transfer(account, recipient, amount, reason, exec_date) + # transfer_data is used in execute_transfer + transfer._transfer_data = transfer_data + return transfer + + @need_login + def execute_transfer(self, transfer, **params): + assert transfer._transfer_data + + transfer._transfer_data.update({ + 'enregistrerNouveauBeneficiaire': False, + 'creditLabel': 'de %s' % transfer.account_label if not transfer.label else transfer.label, + 'debitLabel': 'vers %s' % transfer.recipient_label, + 'typeFrais': 'SHA' + }) + self.execute_transfer_page.go(json=transfer._transfer_data) + transfer.id = self.page.get_transfer_confirm_id() + return transfer + @retry((ClientError, ServerError)) @need_login def get_advisor(self): diff --git a/modules/cmso/par/pages.py b/modules/cmso/par/pages.py index c6c99b1a45b9e29af317fc3c7e16577d370291eb..6e8e1fe759a5b75dc5de714786a53e66795a89d4 100644 --- a/modules/cmso/par/pages.py +++ b/modules/cmso/par/pages.py @@ -135,6 +135,8 @@ class item(ItemElement): # Iban is available without last 5 numbers, or by sms obj_iban = NotAvailable obj__index = Dict('index') + # to know if we can do transfer on account + obj__eligible_debit = Dict('eligibiliteDebit', default=False) def obj_balance(self): balance = CleanDecimal(Dict('soldeEuro', default="0"))(self) @@ -194,6 +196,8 @@ class item(ItemElement): obj_coming = CleanDecimal(Dict('AVenir', default=None), default=NotAvailable) obj__index = Dict('index') obj__owner = Dict('nomTitulaire') + # to know if we can do transfer on account + obj__eligible_debit = Dict('eligibiliteDebit', default=False) def obj_id(self): type = Field('type')(self) @@ -549,35 +553,6 @@ class update_agency(ItemElement): obj_address = Format('%s %s', Dict('adresse1'), Dict('adresse3')) -class RecipientsPage(LoggedPage, JsonPage): - def get_numbers(self): - # If account information is not available when asking for the - # recipients (server error for ex.), return an empty dictionary - # that will be filled later after being returned the json of the - # account page (containing the accounts IDs too). - if 'listCompteTitulaireCotitulaire' not in self.doc and 'exception' in self.doc: - return {} - - ret = {} - - ret.update({ - d['index']: d['numeroContratSouscrit'] - for d in self.doc['listCompteTitulaireCotitulaire'] - }) - ret.update({ - d['index']: d['numeroContratSouscrit'] - for p in self.doc['listCompteMandataire'].values() - for d in p - }) - ret.update({ - d['index']: d['numeroContratSouscrit'] - for p in self.doc['listCompteLegalRep'].values() - for d in p - }) - - return ret - - class ProfilePage(LoggedPage, JsonPage): # be careful, this page is used in CmsoProBrowser too! diff --git a/modules/cmso/par/transfer_pages.py b/modules/cmso/par/transfer_pages.py new file mode 100644 index 0000000000000000000000000000000000000000..96eed8e9d4abcee64073dcc86346977ae46e78c7 --- /dev/null +++ b/modules/cmso/par/transfer_pages.py @@ -0,0 +1,154 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Sylvie Ye +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import datetime as dt + +from weboob.browser.pages import JsonPage, LoggedPage +from weboob.browser.elements import DictElement, ItemElement, method +from weboob.browser.filters.standard import CleanText, CleanDecimal, Currency +from weboob.browser.filters.json import Dict +from weboob.capabilities.bank import Recipient, Transfer, TransferBankError +from weboob.capabilities.base import NotAvailable + + +class MyRecipientItemElement(ItemElement): + def condition(self): + return Dict('eligibiliteCredit', default=False) + + klass = Recipient + + obj_id = Dict('numeroContratSouscrit') + obj_label = Dict('lib') + obj_iban = NotAvailable + obj_enabled_at = dt.date.today() + obj_category = 'Interne' + obj__index = Dict('index') + + +class RecipientsListPage(LoggedPage, JsonPage): + def get_rcpt_index(self, recipient): + if recipient.category == 'Externe': + for el in self.doc['listBeneficiaries']: + for rcpt in el: + # in this list, recipient iban is like FR111111111111111XXXXX + if rcpt['iban'][:-5] == recipient.iban[:-5] and rcpt['nom'] == recipient.label: + return rcpt['index'] + return recipient._index + + +class TransferInfoPage(LoggedPage, JsonPage): + def get_numbers(self): + # If account information is not available when asking for the + # recipients (server error for ex.), return an empty dictionary + # that will be filled later after being returned the json of the + # account page (containing the accounts IDs too). + if 'listCompteTitulaireCotitulaire' not in self.doc and 'exception' in self.doc: + return {} + + ret = {} + + ret.update({ + d['index']: d['numeroContratSouscrit'] + for d in self.doc['listCompteTitulaireCotitulaire'] + }) + ret.update({ + d['index']: d['numeroContratSouscrit'] + for p in self.doc['listCompteMandataire'].values() + for d in p + }) + ret.update({ + d['index']: d['numeroContratSouscrit'] + for p in self.doc['listCompteLegalRep'].values() + for d in p + }) + + return ret + + @method + class iter_titu_accounts(DictElement): + item_xpath = 'listCompteTitulaireCotitulaire' + + class item(MyRecipientItemElement): + pass + + @method + class iter_manda_accounts(DictElement): + item_xpath = 'listCompteMandataire/*' + + class item(MyRecipientItemElement): + pass + + @method + class iter_legal_rep_accounts(DictElement): + item_xpath = 'listCompteLegalRep/*' + + class item(MyRecipientItemElement): + pass + + @method + class iter_external_recipients(DictElement): + item_xpath = 'listBeneficiaries' + + class item(ItemElement): + klass = Recipient + + obj_id = obj_iban = Dict('iban') + obj_label = Dict('nom') + obj_category = 'Externe' + obj_enabled_at = dt.date.today() + obj__index = Dict('index') + + def condition(self): + return Dict('actif', default=False)(self) + + +class TransferPage(LoggedPage, JsonPage): + def on_load(self): + if self.doc.get('exception') and not self.doc.get('debitAccountOwner'): + if Dict('exception/type')(self.doc) == 1: + # technical error + assert False, 'Error with code %s occured during init_transfer: %s' % \ + (Dict('exception/code')(self.doc), Dict('exception/message')(self.doc)) + elif Dict('exception/type')(self.doc) == 2: + # user error + TransferBankError(message=Dict('exception/message')(self.doc)) + + def handle_transfer(self, account, recipient, amount, reason, exec_date): + transfer = Transfer() + transfer.amount = CleanDecimal(Dict('amount'))(self.doc) + transfer.currency = Currency(Dict('codeDevise'))(self.doc) + transfer.label = reason + + if exec_date: + transfer.exec_date = dt.date.fromtimestamp(int(Dict('date')(self.doc))//1000) + + transfer.account_id = account.id + transfer.account_label = CleanText(Dict('debitAccountLabel'))(self.doc) + transfer.account_balance = CleanDecimal(Dict('debitAccountBalance'))(self.doc) + + transfer.recipient_id = recipient.id + transfer.recipient_iban = recipient.iban + transfer.recipient_label = CleanText(Dict('creditAccountOwner'))(self.doc) + + return transfer + + def get_transfer_confirm_id(self): + return self.doc.get('numeroOperation') diff --git a/modules/cragr/api/browser.py b/modules/cragr/api/browser.py index aa8dcb3c57d9213a7ee2371bea480bf08121abe1..77b8c90c0325b30017d4410356570ecc12562ca7 100644 --- a/modules/cragr/api/browser.py +++ b/modules/cragr/api/browser.py @@ -20,22 +20,28 @@ from __future__ import unicode_literals +from decimal import Decimal import re -from weboob.capabilities.bank import ( - Account, -) -from weboob.capabilities.base import find_object, empty, NotAvailable +from weboob.capabilities.bank import Account, Transaction +from weboob.capabilities.base import empty, NotAvailable from weboob.browser import LoginBrowser, URL, need_login -from weboob.exceptions import BrowserUnavailable, BrowserIncorrectPassword -from weboob.browser.exceptions import ServerError +from weboob.exceptions import BrowserUnavailable, BrowserIncorrectPassword, ActionNeeded +from weboob.browser.exceptions import ServerError, BrowserHTTPNotFound +from weboob.capabilities.bank import Loan from weboob.tools.capabilities.bank.iban import is_iban_valid +from weboob.tools.capabilities.bank.transactions import sorted_transactions from .pages import ( - LoginPage, LoggedOutPage, KeypadPage, SecurityPage, ContractsPage, AccountsPage, AccountDetailsPage, - IbanPage, HistoryPage, ProfilePage, + LoginPage, LoggedOutPage, KeypadPage, SecurityPage, ContractsPage, FirstConnectionPage, AccountsPage, AccountDetailsPage, + TokenPage, IbanPage, HistoryPage, CardsPage, CardHistoryPage, NetfincaRedirectionPage, PredicaRedirectionPage, + PredicaInvestmentsPage, ProfilePage, ProfileDetailsPage, ProProfileDetailsPage, ) +from weboob.tools.capabilities.bank.investments import create_french_liquidity + +from .netfinca_browser import NetfincaBrowser + __all__ = ['CragrAPI'] @@ -44,32 +50,84 @@ class CragrAPI(LoginBrowser): login_page = URL(r'particulier/acceder-a-mes-comptes.html$', LoginPage) keypad = URL(r'particulier/acceder-a-mes-comptes.authenticationKeypad.json', KeypadPage) security_check = URL(r'particulier/acceder-a-mes-comptes.html/j_security_check', SecurityPage) + first_connection = URL(r'.*/operations/interstitielles/premiere-connexion.html', FirstConnectionPage) logged_out = URL(r'.*', LoggedOutPage) + token_page = URL(r'libs/granite/csrf/token.json', TokenPage) + contracts_page = URL(r'particulier/operations/.rechargement.contexte.html\?idBamIndex=(?P)', r'association/operations/.rechargement.contexte.html\?idBamIndex=(?P)', - r'professionnel/operations/.rechargement.contexte.html\?idBamIndex=(?P)', ContractsPage) + r'professionnel/operations/.rechargement.contexte.html\?idBamIndex=(?P)', + r'agriculteur/operations/.rechargement.contexte.html\?idBamIndex=(?P)', + r'entreprise/operations/.rechargement.contexte.html\?idBamIndex=(?P)', ContractsPage) accounts_page = URL(r'particulier/operations/synthese.html', r'association/operations/synthese.html', - r'professionnel/operations/synthese.html', AccountsPage) + r'professionnel/operations/synthese.html', + r'agriculteur/operations/synthese.html', + r'entreprise/operations/synthese.html', AccountsPage) account_details = URL(r'particulier/operations/synthese/jcr:content.produits-valorisation.json/(?P)', r'association/operations/synthese/jcr:content.produits-valorisation.json/(?P)', - r'professionnel/operations/synthese/jcr:content.produits-valorisation.json/(?P)', AccountDetailsPage) + r'professionnel/operations/synthese/jcr:content.produits-valorisation.json/(?P)', + r'agriculteur/operations/synthese/jcr:content.produits-valorisation.json/(?P)', + r'entreprise/operations/synthese/jcr:content.produits-valorisation.json/(?P)', AccountDetailsPage) account_iban = URL(r'particulier/operations/operations-courantes/editer-rib/jcr:content.ibaninformation.json', r'association/operations/operations-courantes/editer-rib/jcr:content.ibaninformation.json', - r'professionnel/operations/operations-courantes/editer-rib/jcr:content.ibaninformation.json', IbanPage) - - history_page = URL(r'particulier/operations/synthese/detail-comptes/jcr:content.n3.compte.infos.json', - r'association/operations/synthese/detail-comptes/jcr:content.n3.compte.infos.json', - r'professionnel/operations/synthese/detail-comptes/jcr:content.n3.compte.infos.json', HistoryPage) + r'professionnel/operations/operations-courantes/editer-rib/jcr:content.ibaninformation.json', + r'agriculteur/operations/operations-courantes/editer-rib/jcr:content.ibaninformation.json', + r'entreprise/operations/operations-courantes/editer-rib/jcr:content.ibaninformation.json', IbanPage) + + cards = URL(r'particulier/operations/moyens-paiement/mes-cartes/jcr:content.listeCartesParCompte.json', + r'association/operations/moyens-paiement/mes-cartes/jcr:content.listeCartesParCompte.json', + r'professionnel/operations/moyens-paiement/mes-cartes/jcr:content.listeCartesParCompte.json', + r'agriculteur/operations/moyens-paiement/mes-cartes/jcr:content.listeCartesParCompte.json', + r'entreprise/operations/moyens-paiement/mes-cartes/jcr:content.listeCartesParCompte.json', CardsPage) + + history = URL(r'particulier/operations/synthese/detail-comptes/jcr:content.n3.operations.json', + r'association/operations/synthese/detail-comptes/jcr:content.n3.operations.json', + r'professionnel/operations/synthese/detail-comptes/jcr:content.n3.operations.json', + r'agriculteur/operations/synthese/detail-comptes/jcr:content.n3.operations.json', + r'entreprise/operations/synthese/detail-comptes/jcr:content.n3.operations.json', HistoryPage) + + card_history = URL(r'particulier/operations/synthese/detail-comptes/jcr:content.n3.operations.encours.carte.debit.differe.json', + r'association/operations/synthese/detail-comptes/jcr:content.n3.operations.encours.carte.debit.differe.json', + r'professionnel/operations/synthese/detail-comptes/jcr:content.n3.operations.encours.carte.debit.differe.json', + r'agriculteur/operations/synthese/detail-comptes/jcr:content.n3.operations.encours.carte.debit.differe.json', + r'entreprise/operations/synthese/detail-comptes/jcr:content.n3.operations.encours.carte.debit.differe.json', CardHistoryPage) + + netfinca_redirection = URL(r'particulier/operations/moco/catitres/jcr:content.init.html', + r'association/operations/moco/catitres/jcr:content.init.html', + r'professionnel/operations/moco/catitres/jcr:content.init.html', + r'agriculteur/operations/moco/catitres/jcr:content.init.html', + r'entreprise/operations/moco/catitres/jcr:content.init.html', + r'particulier/operations/moco/catitres/_jcr_content.init.html', + r'association/operations/moco/catitres/_jcr_content.init.html', + r'professionnel/operations/moco/catitres/_jcr_content.init.html', + r'agriculteur/operations/moco/catitres/_jcr_content.init.html', + r'entreprise/operations/moco/catitres/_jcr_content.init.html', NetfincaRedirectionPage) + + predica_redirection = URL(r'particulier/operations/moco/predica/jcr:content.init.html', + r'association/operations/moco/predica/jcr:content.init.html', + r'professionnel/operations/moco/predica/jcr:content.init.html', + r'agriculteur/operations/moco/predica/jcr:content.init.html', + r'entreprise/operations/moco/predica/jcr:content.init.html', PredicaRedirectionPage) + + predica_investments = URL(r'https://npcprediweb.predica.credit-agricole.fr/rest/detailEpargne/contrat/', PredicaInvestmentsPage) profile_page = URL(r'particulier/operations/synthese/jcr:content.npc.store.client.json', r'association/operations/synthese/jcr:content.npc.store.client.json', - r'professionnel/operations/synthese/jcr:content.npc.store.client.json', ProfilePage) + r'professionnel/operations/synthese/jcr:content.npc.store.client.json', + r'agriculteur/operations/synthese/jcr:content.npc.store.client.json', + r'entreprise/operations/synthese/jcr:content.npc.store.client.json', ProfilePage) + + profile_details = URL(r'particulier/operations/profil/infos-personnelles/gerer-coordonnees.html', ProfileDetailsPage) + pro_profile_details = URL(r'association/operations/profil/infos-personnelles/controler-coordonnees.html', + r'professionnel/operations/profil/infos-personnelles/controler-coordonnees.html', + r'agriculteur/operations/profil/infos-personnelles/controler-coordonnees.html', + r'entreprise/operations/profil/infos-personnelles/controler-coordonnees.html', ProProfileDetailsPage) def __init__(self, website, *args, **kwargs): super(CragrAPI, self).__init__(*args, **kwargs) @@ -78,16 +136,17 @@ def __init__(self, website, *args, **kwargs): self.BASEURL = 'https://%s/' % self.region self.accounts_url = None - def do_login(self): - self.keypad.go() - keypad_password = self.page.build_password(self.password[:6]) - keypad_id = self.page.get_keypad_id() - assert keypad_password, 'Could not obtain keypad password' - assert keypad_id, 'Could not obtain keypad id' + # Netfinca browser: + self.weboob = kwargs.pop('weboob') + dirname = self.responses_dirname + self.netfinca = NetfincaBrowser('', '', logger=self.logger, weboob=self.weboob, responses_dirname=dirname, proxy=self.PROXIES) - self.login_page.go() - # Get the form data to POST the security check: - form = self.page.get_login_form(self.username, keypad_password, keypad_id) + def deinit(self): + super(CragrAPI, self).deinit() + self.netfinca.deinit() + + def do_login(self): + form = self.get_security_form() try: self.security_check.go(data=form) except ServerError as exc: @@ -95,32 +154,65 @@ def do_login(self): error = exc.response.json().get('error') if error: message = error.get('message', '') - if 'Votre identification est incorrecte' in message: + wrongpass_messages = ("Votre identification est incorrecte", "Vous n'avez plus droit") + if any(value in message for value in wrongpass_messages): raise BrowserIncorrectPassword() - assert False, 'Unhandled Server Error encountered: %s' % error.get('message', '') - - # accounts_url may contain '/particulier', '/professionnel' or '/association' + if 'obtenir un nouveau code' in message: + raise ActionNeeded(message) + technical_errors = ('Un incident technique', 'identifiant et votre code personnel') + if any(value in message for value in technical_errors): + # If it is a technical error, we try login again + form = self.get_security_form() + try: + self.security_check.go(data=form) + except ServerError as exc: + error = exc.response.json().get('error') + if error: + message = error.get('message', '') + if 'Un incident technique' in message: + raise BrowserUnavailable(message) + + # accounts_url may contain '/particulier', '/professionnel', '/entreprise', '/agriculteur' or '/association' self.accounts_url = self.page.get_accounts_url() assert self.accounts_url, 'Could not get accounts url from security check' self.location(self.accounts_url) - assert self.accounts_page.is_here(), 'We failed to login after the security check!' + assert self.accounts_page.is_here(), 'We failed to login after the security check: response URL is %s' % self.url # Once the security check is passed, we are logged in. + def get_security_form(self): + self.keypad.go() + keypad_password = self.page.build_password(self.password[:6]) + keypad_id = self.page.get_keypad_id() + assert keypad_password, 'Could not obtain keypad password' + assert keypad_id, 'Could not obtain keypad id' + self.login_page.go() + # Get the form data to POST the security check: + form = self.page.get_login_form(self.username, keypad_password, keypad_id) + return form + @need_login def get_accounts_list(self): # Determine how many spaces are present on the connection: self.location(self.accounts_url) + if not self.accounts_page.is_here(): + # We have been logged out. + self.do_login() total_spaces = self.page.count_spaces() self.logger.info('The total number of spaces on this connection is %s.' % total_spaces) + # Complete accounts list is required to match card parent accounts + # and to avoid accounts that are present on several spaces + all_accounts = {} + deferred_cards = {} + for contract in range(total_spaces): # This request often returns a 500 error so we retry several times. try: - self.contracts_page.go(id_contract=contract) + self.go_to_account_space(contract) except ServerError: self.logger.warning('Server returned error 500 when trying to access space %s, we try again' % contract) try: - self.contracts_page.go(id_contract=contract) + self.go_to_account_space(contract) except ServerError: self.logger.warning('Server returned error 500 twice when trying to access space %s, this space will be skipped' % contract) continue @@ -135,9 +227,19 @@ def get_accounts_list(self): account._contract = contract account.owner_type = self.page.get_owner_type() - # Some accounts have no balance in the main JSON, so we must - # get all the (id, balance) pairs in the account_details JSON: - categories = {int(account._category) for account in accounts_list if account._category != None} + ''' Other accounts have no balance in the main JSON, so we must get all + the (_id_element_contrat, balance) pairs in the account_details JSON. + + Account categories always correspond to the same account types: + # Category 1: Checking accounts, + # Category 2: To be determined, + # Category 3: Savings, + # Category 4: Loans & Credits, + # Category 5: Insurances (skipped), + # Category 6: To be determined, + # Category 7: Market accounts. ''' + + categories = {int(account._category) for account in accounts_list if account._category not in (None, '5')} account_balances = {} loan_ids = {} for category in categories: @@ -145,7 +247,6 @@ def get_accounts_list(self): account_balances.update(self.page.get_account_balances()) loan_ids.update(self.page.get_loan_ids()) - # Getting IBANs for checking accounts if main_account.type == Account.TYPE_CHECKING: params = { 'compteIdx': int(main_account._index), @@ -155,76 +256,265 @@ def get_accounts_list(self): iban = self.page.get_iban() if is_iban_valid(iban): main_account.iban = iban - yield main_account - - for card in main_account._cards: - card.parent = main_account - card.currency = main_account.currency - card.owner_type = main_account.owner_type - card._contract = contract - yield card + if main_account.id not in all_accounts: + all_accounts[main_account.id] = main_account + yield main_account for account in accounts_list: if empty(account.balance): - account.balance = account_balances.get(account.id, NotAvailable) + account.balance = account_balances.get(account._id_element_contrat, NotAvailable) if account.type == Account.TYPE_CHECKING: - try: - params = { - 'compteIdx': int(account._index), - 'grandeFamilleCode': 1, - } - self.account_iban.go(params=params) - iban = self.page.get_iban() - if is_iban_valid(iban): - account.iban = iban - except ServerError: - self.logger.warning('Could not fetch IBAN for checking account "%s %s"', account.label, account.id) - pass - - # TO-DO: Create Loan() object with its related attributes + params = { + 'compteIdx': int(account._index), + 'grandeFamilleCode': int(account._category), + } + self.account_iban.go(params=params) + iban = self.page.get_iban() + if is_iban_valid(iban): + account.iban = iban + # Loans have a specific ID that we need to fetch # so the backend can match loans properly. - # If no there is no loan ID, we keep the account ID. if account.type == Account.TYPE_LOAN: - account.id = loan_ids.get(account.id, account.id) - account.balance = -account.balance + account.id = account.number = loan_ids.get(account._id_element_contrat, account.id) + account = self.switch_account_to_loan(account) elif account.type == Account.TYPE_REVOLVING_CREDIT: - account.id = loan_ids.get(account.id, account.id) - account.balance = 0 - yield account + account.id = account.number = loan_ids.get(account._id_element_contrat, account.id) + account = self.switch_account_to_revolving(account) + if account.id not in all_accounts: + all_accounts[account.id] = account + yield account + + # Fetch all deferred credit cards for this space + self.cards.go() + for card in self.page.iter_card_parents(): + card.number = card.id + card.parent = all_accounts.get(card._parent_id, NotAvailable) + card.currency = card.parent.currency + card.owner_type = card.parent.owner_type + card._category = card.parent._category + card._contract = contract + if card.id not in deferred_cards: + deferred_cards[card.id] = card + + # We must check if cards are unique on their parent account; + # if not, we cannot retrieve their summaries in iter_history. + parent_accounts = [] + for card in deferred_cards.values(): + parent_accounts.append(card.parent.id) + for card in deferred_cards.values(): + if parent_accounts.count(card.parent.id) == 1: + card._unique = True + else: + card._unique = False + yield card + + def switch_account_to_loan(self, account): + loan = Loan() + copy_attrs = ('id', 'number', 'label', 'type', 'currency', '_index', '_category', '_contract', '_id_element_contrat', 'owner_type') + for attr in copy_attrs: + setattr(loan, attr, getattr(account, attr)) + loan.balance = -account.balance + return loan + + def switch_account_to_revolving(self, account): + loan = Loan() + copy_attrs = ('id', 'number', 'label', 'type', 'currency', '_index', '_category', '_contract', '_id_element_contrat', 'owner_type') + for attr in copy_attrs: + setattr(loan, attr, getattr(account, attr)) + loan.balance = Decimal(0) + loan.available_amount = account.balance + return loan @need_login def go_to_account_space(self, contract): - # TO-DO: Figure out a way to determine whether - # we already are on the right account space self.contracts_page.go(id_contract=contract) - assert self.accounts_page.is_here() + if not self.accounts_page.is_here(): + # We have been logged out. + self.do_login() + self.contracts_page.go(id_contract=contract) + assert self.accounts_page.is_here() @need_login - def get_card(self, id): - return find_object(self.get_cards(), id=id) - - @need_login - def get_cards(self, accounts_list=None): - # accounts_list is only used by get_list - raise BrowserUnavailable() - - @need_login - def get_history(self, account): - raise BrowserUnavailable() + def get_history(self, account, coming=False): + if account.type == Account.TYPE_CARD: + card_transactions = [] + self.go_to_account_space(account._contract) + # Deferred cards transactions have a specific JSON. + # Only three months of history available for cards. + value = 0 if coming else 1 + params = { + 'grandeFamilleCode': int(account._category), + 'compteIdx': int(account.parent._index), + 'carteIdx': int(account._index), + 'rechercheEncoursDebite': value + } + self.card_history.go(params=params) + for tr in self.page.iter_card_history(): + card_transactions.append(tr) + + # If the card if not unique on the parent id, it is impossible + # to know which summary corresponds to which card. + if not coming and card_transactions and account._unique: + # Get card summaries from parent account + # until we reach the oldest card transaction + last_transaction = card_transactions[-1] + before_last_transaction = False + params = { + 'compteIdx': int(account.parent._index), + 'grandeFamilleCode': int(account.parent._category), + 'idDevise': str(account.parent.currency), + 'idElementContrat': str(account.parent._id_element_contrat), + } + self.history.go(params=params) + for tr in self.page.iter_history(): + if tr.date < last_transaction.date: + before_last_transaction = True + break + if tr.type == Transaction.TYPE_CARD_SUMMARY: + tr.amount = -tr.amount + card_transactions.append(tr) + + while self.page.has_next_page() and not before_last_transaction: + next_index = self.page.get_next_index() + params = { + 'grandeFamilleCode': int(account.parent._category), + 'compteIdx': int(account.parent._index), + 'idDevise': str(account.parent.currency), + 'startIndex': next_index, + 'count': 100, + } + self.history.go(params=params) + for tr in self.page.iter_history(): + if tr.date < last_transaction.date: + before_last_transaction = True + break + if tr.type == Transaction.TYPE_CARD_SUMMARY: + tr.amount = -tr.amount + card_transactions.append(tr) + + for tr in sorted_transactions(card_transactions): + yield tr + return + + # These three parameters are required to get the transactions for non_card accounts + if empty(account._index) or empty(account._category) or empty(account._id_element_contrat): + return + + self.go_to_account_space(account._contract) + params = { + 'compteIdx': int(account._index), + 'grandeFamilleCode': int(account._category), + 'idDevise': str(account.currency), + 'idElementContrat': str(account._id_element_contrat), + } + self.history.go(params=params) + for tr in self.page.iter_history(): + yield tr + + # Get other transactions 100 by 100: + while self.page.has_next_page(): + next_index = self.page.get_next_index() + params = { + 'grandeFamilleCode': int(account._category), + 'compteIdx': int(account._index), + 'idDevise': str(account.currency), + 'startIndex': next_index, + 'count': 100, + } + self.history.go(params=params) + for tr in self.page.iter_history(): + yield tr @need_login def iter_investment(self, account): - raise BrowserUnavailable() + if account.type in (Account.TYPE_PERP, Account.TYPE_PERCO, Account.TYPE_LIFE_INSURANCE, Account.TYPE_CAPITALISATION): + if account.label == "Vers l'avenir": + # Website crashes when clicking on these Life Insurances... + return + self.go_to_account_space(account._contract) + token = self.token_page.go().get_token() + data = { + 'situation_travail': 'CONTRAT', + 'idelco': account.id, + ':cq_csrf_token': token, + } + self.predica_redirection.go(data=data) + self.predica_investments.go() + for inv in self.page.iter_investments(): + yield inv + + elif account.type == Account.TYPE_PEA and account.label == 'Compte espèce PEA': + yield create_french_liquidity(account.balance) + return + + elif account.type in (Account.TYPE_PEA, Account.TYPE_MARKET): + # Do not try to get to Netfinca if there is no money + # on the account or the server will return an error 500 + if account.balance == 0: + return + self.go_to_account_space(account._contract) + token = self.token_page.go().get_token() + data = { + 'situation_travail': 'BANCAIRE', + 'num_compte': account.id, + 'code_fam_produit': account._fam_product_code, + 'code_fam_contrat_compte': account._fam_contract_code, + ':cq_csrf_token': token, + } + + # For some market accounts, investments are not even accessible, + # and the only way to know if there are investments is to try + # to go to the Netfinca space with the accounts parameters. + try: + self.netfinca_redirection.go(data=data) + except BrowserHTTPNotFound: + self.logger.info('Investments are not available for this account.') + self.go_to_account_space(account._contract) + return + url = self.page.get_url() + if 'netfinca' in url: + self.location(url) + self.netfinca.session.cookies.update(self.session.cookies) + self.netfinca.accounts.go() + for inv in self.netfinca.iter_investments(account): + if inv.code == 'XX-liquidity' and account.type == Account.TYPE_PEA: + # Liquidities are already fetched on the "PEA espèces" + continue + yield inv @need_login def iter_advisor(self): - raise BrowserUnavailable() + self.go_to_account_space(0) + owner_type = self.page.get_owner_type() + self.profile_page.go() + if owner_type == 'PRIV': + advisor = self.page.get_advisor() + self.profile_details.go() + self.page.fill_advisor(obj=advisor) + return advisor + elif owner_type == 'ORGA': + advisor = self.page.get_advisor() + self.pro_profile_details.go() + self.page.fill_advisor(obj=advisor) + return advisor @need_login def get_profile(self): - #self.profile.go() - raise BrowserUnavailable() + # There is one profile per space, so we only fetch the first one + self.go_to_account_space(0) + owner_type = self.page.get_owner_type() + self.profile_page.go() + if owner_type == 'PRIV': + profile = self.page.get_user_profile() + self.profile_details.go() + self.page.fill_profile(obj=profile) + return profile + elif owner_type == 'ORGA': + profile = self.page.get_company_profile() + self.pro_profile_details.go() + self.page.fill_profile(obj=profile) + return profile @need_login def iter_transfer_recipients(self, account): diff --git a/modules/cragr/api/netfinca_browser.py b/modules/cragr/api/netfinca_browser.py new file mode 100644 index 0000000000000000000000000000000000000000..1cffad4f961e838b08725b0b1a4291845ea6fc03 --- /dev/null +++ b/modules/cragr/api/netfinca_browser.py @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012-2019 Budget Insight + + +from weboob.browser import AbstractBrowser + + +class NetfincaBrowser(AbstractBrowser): + PARENT = 'netfinca' + BASEURL = 'https://www.cabourse.credit-agricole.fr' diff --git a/modules/cragr/api/pages.py b/modules/cragr/api/pages.py index 18046a1fa103774cc3db67f5f6205d0c1c9e4dc6..bdffcefbcd562989650d010173d0d9db26ffac87 100644 --- a/modules/cragr/api/pages.py +++ b/modules/cragr/api/pages.py @@ -22,19 +22,23 @@ from decimal import Decimal import re import json +import dateutil from weboob.browser.pages import HTMLPage, JsonPage, LoggedPage -from weboob.exceptions import BrowserUnavailable +from weboob.exceptions import ActionNeeded from weboob.capabilities import NotAvailable from .compat.weboob_capabilities_bank import ( - Account, AccountOwnerType, + Account, AccountOwnerType, Transaction, Investment, ) - +from weboob.capabilities.profile import Person, Company +from weboob.capabilities.contact import Advisor from weboob.browser.elements import DictElement, ItemElement, method from weboob.browser.filters.standard import ( - CleanText, CleanDecimal, Currency as CleanCurrency, Format, Field, Map, Eval + CleanText, CleanDecimal, Currency as CleanCurrency, Format, Field, Map, Eval, Env, Regexp, Date, ) +from weboob.browser.filters.html import Attr from weboob.browser.filters.json import Dict +from weboob.tools.capabilities.bank.investments import is_isin_valid def float_to_decimal(f): @@ -54,7 +58,7 @@ def get_keypad_id(self): class LoginPage(HTMLPage): def get_login_form(self, username, keypad_password, keypad_id): form = self.get_form(id="loginForm") - form['j_username'] = username + form['j_username'] = username[:11] form['j_password'] = keypad_password form['keypadId'] = keypad_id return form @@ -64,9 +68,12 @@ class LoggedOutPage(HTMLPage): def is_here(self): return self.doc.xpath('//b[text()="FIN DE CONNEXION"]') + +class FirstConnectionPage(LoggedPage, HTMLPage): def on_load(self): - self.logger.warning('We have been logged out!') - raise BrowserUnavailable() + message = CleanText('//p[contains(text(), "votre première visite")]')(self.doc) + if message: + raise ActionNeeded(message) class SecurityPage(JsonPage): @@ -74,6 +81,11 @@ def get_accounts_url(self): return Dict('url')(self.doc) +class TokenPage(LoggedPage, JsonPage): + def get_token(self): + return Dict('token')(self.doc) + + class ContractsPage(LoggedPage, HTMLPage): pass @@ -102,12 +114,15 @@ class ContractsPage(LoggedPage, HTMLPage): 'DAV TIGERE': Account.TYPE_SAVINGS, 'CPTEXCPRO': Account.TYPE_SAVINGS, 'CPTEXCENT': Account.TYPE_SAVINGS, + 'DAT': Account.TYPE_SAVINGS, 'PRET PERSO': Account.TYPE_LOAN, 'P. ENTREPR': Account.TYPE_LOAN, 'P. HABITAT': Account.TYPE_LOAN, 'PRET 0%': Account.TYPE_LOAN, 'INV PRO': Account.TYPE_LOAN, 'TRES. PRO': Account.TYPE_LOAN, + 'CT ATT HAB': Account.TYPE_LOAN, + 'PRET CEL': Account.TYPE_LOAN, 'PEA': Account.TYPE_PEA, 'PEAP': Account.TYPE_PEA, 'DAV PEA': Account.TYPE_PEA, @@ -165,8 +180,10 @@ class get_main_account(ItemElement): obj_balance = Eval(float_to_decimal, Dict('comptePrincipal/solde')) obj_currency = CleanCurrency(Dict('comptePrincipal/idDevise')) obj__index = Dict('comptePrincipal/index') - obj__category = None # Main accounts have no category + obj__category = Dict('comptePrincipal/grandeFamilleProduitCode', default=None) obj__id_element_contrat = CleanText(Dict('comptePrincipal/idElementContrat')) + obj__fam_product_code = CleanText(Dict('comptePrincipal/codeFamilleProduitBam')) + obj__fam_contract_code = CleanText(Dict('comptePrincipal/codeFamilleContratBam')) def obj_type(self): _type = Map(CleanText(Dict('comptePrincipal/libelleUsuelProduit')), ACCOUNT_TYPES, Account.TYPE_UNKNOWN)(self) @@ -174,22 +191,6 @@ def obj_type(self): self.logger.warning('We got an untyped account: please add "%s" to ACCOUNT_TYPES.' % CleanText(Dict('comptePrincipal/libelleUsuelProduit'))(self)) return _type - class obj__cards(DictElement): - item_xpath = 'comptePrincipal/cartesDD' - - class item(ItemElement): - klass = Account - - def obj_id(self): - return CleanText(Dict('idCarte'))(self).replace(' ', '') - - obj_label = Format('Carte %s %s', Field('id'), CleanText(Dict('titulaire'))) - obj_type = Account.TYPE_CARD - obj_coming = Eval(float_to_decimal, Dict('encoursCarteM')) - obj_balance = CleanDecimal(0) - obj__index = Dict('index') - obj__category = None - @method class iter_accounts(DictElement): item_xpath = 'grandesFamilles/*/elementsContrats' @@ -199,18 +200,28 @@ class item(ItemElement): klass = Account - obj_id = CleanText(Dict('numeroCompteBam')) - obj_number = CleanText(Dict('numeroCompteBam')) + def obj_id(self): + # Loan ids may be duplicated so we use the contract number for now: + if Field('type')(self) == Account.TYPE_LOAN: + return CleanText(Dict('idElementContrat'))(self) + return CleanText(Dict('numeroCompte'))(self) + + obj_number = CleanText(Dict('numeroCompte')) obj_label = CleanText(Dict('libelleProduit')) obj_currency = CleanCurrency(Dict('idDevise')) obj__index = Dict('index') obj__category = Dict('grandeFamilleProduitCode', default=None) obj__id_element_contrat = CleanText(Dict('idElementContrat')) + obj__fam_product_code = CleanText(Dict('codeFamilleProduitBam')) + obj__fam_contract_code = CleanText(Dict('codeFamilleContratBam')) def obj_type(self): + if CleanText(Dict('libelleUsuelProduit'))(self) in ('HABITATION',): + # No need to log warning for "assurance" accounts + return NotAvailable _type = Map(CleanText(Dict('libelleUsuelProduit')), ACCOUNT_TYPES, Account.TYPE_UNKNOWN)(self) if _type == Account.TYPE_UNKNOWN: - self.logger.warning('We got an untyped account: please add "%s" to ACCOUNT_TYPES.' % CleanText(Dict('libelleUsuelProduit'))(self)) + self.logger.warning('There is an untyped account: please add "%s" to ACCOUNT_TYPES.' % CleanText(Dict('libelleUsuelProduit'))(self)) return _type def obj_balance(self): @@ -227,22 +238,30 @@ def condition(self): class AccountDetailsPage(LoggedPage, JsonPage): def get_account_balances(self): + # We use the 'idElementContrat' key because it is unique + # whereas the account id may not be unique for Loans account_balances = {} for el in self.doc: + # Insurances have no balance, we skip them + if el.get('typeProduit') == 'assurance': + continue value = el.get('solde', el.get('encoursActuel', el.get('valorisationContrat', el.get('montantRestantDu', el.get('capitalDisponible'))))) - assert value is not None, 'Could not find the account balance' - account_balances[Dict('numeroCompte')(el)] = float_to_decimal(value) + if value is None: + continue + account_balances[Dict('idElementContrat')(el)] = float_to_decimal(value) return account_balances def get_loan_ids(self): + # We use the 'idElementContrat' key because it is unique + # whereas the account id may not be unique for Loans loan_ids = {} for el in self.doc: if el.get('numeroCredit'): # Loans - loan_ids[Dict('numeroCompte')(el)] = Dict('numeroCredit')(el) + loan_ids[Dict('idElementContrat')(el)] = Dict('numeroCredit')(el) elif el.get('numeroContrat'): # Revolving credits - loan_ids[Dict('numeroCompte')(el)] = Dict('numeroContrat')(el) + loan_ids[Dict('idElementContrat')(el)] = Dict('numeroContrat')(el) return loan_ids @@ -252,12 +271,191 @@ def get_iban(self): class HistoryPage(LoggedPage, JsonPage): - pass + def has_next_page(self): + return Dict('hasNext')(self.doc) + def get_next_index(self): + return Dict('nextSetStartIndex')(self.doc) -class InvestmentPage(LoggedPage, JsonPage): - pass + @method + class iter_history(DictElement): + item_xpath = 'listeOperations' + + class item(ItemElement): + + TRANSACTION_TYPES = { + 'PAIEMENT PAR CARTE': Transaction.TYPE_CARD, + 'REMISE CARTE': Transaction.TYPE_CARD, + 'PRELEVEMENT CARTE': Transaction.TYPE_CARD_SUMMARY, + 'RETRAIT AU DISTRIBUTEUR': Transaction.TYPE_WITHDRAWAL, + "RETRAIT MUR D'ARGENT": Transaction.TYPE_WITHDRAWAL, + 'FRAIS': Transaction.TYPE_BANK, + 'COTISATION': Transaction.TYPE_BANK, + 'VIREMENT': Transaction.TYPE_TRANSFER, + 'VIREMENT EN VOTRE FAVEUR': Transaction.TYPE_TRANSFER, + 'VIREMENT EMIS': Transaction.TYPE_TRANSFER, + 'CHEQUE EMIS': Transaction.TYPE_CHECK, + 'REMISE DE CHEQUE': Transaction.TYPE_DEPOSIT, + 'PRELEVEMENT': Transaction.TYPE_ORDER, + 'PRELEVT': Transaction.TYPE_ORDER, + 'PRELEVMNT': Transaction.TYPE_ORDER, + 'REMBOURSEMENT DE PRET': Transaction.TYPE_LOAN_PAYMENT, + } + + klass = Transaction + + obj_raw = Format('%s %s %s', CleanText(Dict('libelleTypeOperation')), CleanText(Dict('libelleOperation')), CleanText(Dict('libelleComplementaire'))) + obj_label = Format('%s %s', CleanText(Dict('libelleTypeOperation')), CleanText(Dict('libelleOperation'))) + obj_amount = Eval(float_to_decimal, Dict('montant')) + obj_type = Map(CleanText(Dict('libelleTypeOperation')), TRANSACTION_TYPES, Transaction.TYPE_UNKNOWN) + + def obj_date(self): + return dateutil.parser.parse(Dict('dateValeur')(self)) + + def obj_rdate(self): + return dateutil.parser.parse(Dict('dateOperation')(self)) + + +class CardsPage(LoggedPage, JsonPage): + @method + class iter_card_parents(DictElement): + item_xpath = 'comptes' + + class iter_cards(DictElement): + item_xpath = 'listeCartes' + + def parse(self, el): + self.env['parent_id'] = Dict('idCompte')(el) + + class item(ItemElement): + klass = Account + + def obj_id(self): + return CleanText(Dict('idCarte'))(self).replace(' ', '') + + def condition(self): + assert CleanText(Dict('codeTypeDebitPaiementCarte'))(self) in ('D', 'I') + return CleanText(Dict('codeTypeDebitPaiementCarte'))(self)=='D' + + obj_label = Format('Carte %s %s', Field('id'), CleanText(Dict('titulaire'))) + obj_type = Account.TYPE_CARD + obj_coming = Eval(lambda x: -float_to_decimal(x), Dict('encoursCarteM')) + obj_balance = CleanDecimal(0) + obj__parent_id = Env('parent_id') + obj__index = Dict('index') + obj__id_element_contrat = None + + +class CardHistoryPage(LoggedPage, JsonPage): + @method + class iter_card_history(DictElement): + item_xpath = None + + class item(ItemElement): + klass = Transaction + + obj_raw = CleanText(Dict('libelleOperation')) + obj_label = CleanText(Dict('libelleOperation')) + obj_amount = Eval(float_to_decimal, Dict('montant')) + obj_type = Transaction.TYPE_DEFERRED_CARD + + def obj_date(self): + return dateutil.parser.parse(Dict('datePrelevement')(self)) + + def obj_rdate(self): + return dateutil.parser.parse(Dict('dateOperation')(self)) + + +class NetfincaRedirectionPage(LoggedPage, HTMLPage): + def get_url(self): + return Regexp(Attr('//body', 'onload'), r'document.location="([^"]+)"')(self.doc) + + +class PredicaRedirectionPage(LoggedPage, HTMLPage): + def on_load(self): + form = self.get_form() + form.submit() + + +class PredicaInvestmentsPage(LoggedPage, JsonPage): + @method + class iter_investments(DictElement): + item_xpath = 'listeSupports/support' + + class item(ItemElement): + klass = Investment + + obj_label = CleanText(Dict('lcspt')) + obj_valuation = Eval(float_to_decimal, Dict('mtvalspt')) + + def obj_portfolio_share(self): + portfolio_share = Dict('txrpaspt', default=None)(self) + if portfolio_share: + return Eval(lambda x: float_to_decimal(x / 100), portfolio_share)(self) + return NotAvailable + + def obj_unitvalue(self): + unit_value = Dict('mtliqpaaspt', default=None)(self) + if unit_value: + return Eval(float_to_decimal, unit_value)(self) + return NotAvailable + + def obj_quantity(self): + quantity = Dict('qtpaaspt', default=None)(self) + if quantity: + return Eval(float_to_decimal, quantity)(self) + return NotAvailable + + def obj_code(self): + code = Dict('cdsptisn')(self) + if is_isin_valid(code): + return code + return NotAvailable + + def obj_code_type(self): + if is_isin_valid(Field('code')(self)): + return Investment.CODE_TYPE_ISIN + return NotAvailable class ProfilePage(LoggedPage, JsonPage): - pass \ No newline at end of file + @method + class get_user_profile(ItemElement): + klass = Person + + obj_name = CleanText(Dict('displayName', default=NotAvailable)) + obj_phone = CleanText(Dict('branchPhone', default=NotAvailable)) + obj_birth_date = Date(Dict('birthdate', default=NotAvailable)) + + @method + class get_company_profile(ItemElement): + klass = Company + + obj_name = CleanText(Dict('displayName', default=NotAvailable)) + obj_phone = CleanText(Dict('branchPhone', default=NotAvailable)) + obj_registration_date = Date(Dict('birthdate', default=NotAvailable)) + + @method + class get_advisor(ItemElement): + klass = Advisor + + def obj_name(self): + # If no advisor is displayed, we return the agency advisor. + if Dict('advisorGivenName')(self) and Dict('advisorFamilyName')(self): + return Format('%s %s', CleanText(Dict('advisorGivenName')), CleanText(Dict('advisorFamilyName')))(self) + return Format('%s %s', CleanText(Dict('branchManagerGivenName')), CleanText(Dict('branchManagerFamilyName')))(self) + + +class ProfileDetailsPage(LoggedPage, HTMLPage): + @method + class fill_profile(ItemElement): + obj_email = CleanText('//p[contains(@class, "Data mail")]', default=NotAvailable) + obj_address = CleanText('//p[strong[contains(text(), "Adresse")]]/text()[2]', default=NotAvailable) + + @method + class fill_advisor(ItemElement): + obj_phone = CleanText('//div[@id="blockConseiller"]//a[contains(@class, "advisorNumber")]', default=NotAvailable) + + +class ProProfileDetailsPage(ProfileDetailsPage): + pass diff --git a/modules/cragr/module.py b/modules/cragr/module.py index 3fd1fc802ebe4eafa947df5d18132eb97c92a76d..4830a91360c8fabcc2afdf02092308957b7e6235 100644 --- a/modules/cragr/module.py +++ b/modules/cragr/module.py @@ -84,7 +84,7 @@ class CragrModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapContact }.items())]) CONFIG = BackendConfig(Value('website', label=u'Région', choices=website_choices), - ValueBackendPassword('login', label=u'N° de compte', masked=False), + ValueBackendPassword('login', label=u'N° de compte', masked=False, regexp=r'\d+'), ValueBackendPassword('password', label=u'Code personnel', regexp=r'\d{6}')) BROWSER = ProxyBrowser @@ -98,7 +98,8 @@ def create_default_browser(self): site_conf = self.COMPAT_DOMAINS.get(site_conf, site_conf) return self.create_browser(site_conf, self.config['login'].get(), - self.config['password'].get()) + self.config['password'].get(), + weboob=self.weboob) def iter_accounts(self): return self.browser.get_accounts_list() @@ -114,15 +115,17 @@ def to_date(obj): return obj.date() return obj - for tr in self.browser.get_history(account): + for tr in self.browser.get_history(account, coming): tr_coming = to_date(tr.date) > today if coming == tr_coming: yield tr + elif coming: + break def iter_history(self, account): if account.type == Account.TYPE_CARD: return self._history_filter(account, False) - return self.browser.get_history(account) + return self.browser.get_history(account, False) def iter_coming(self, account): if account.type == Account.TYPE_CARD: diff --git a/modules/cragr/web/browser.py b/modules/cragr/web/browser.py index 838af343238a5b4cb9313145c21c1e119fc862c2..6d45816985449c371f2a5aa137c885cfa8347761 100644 --- a/modules/cragr/web/browser.py +++ b/modules/cragr/web/browser.py @@ -412,7 +412,7 @@ def market_accounts_matching(self, accounts_list, market_accounts_list): account.balance = self.page.get_pea_balance() @need_login - def get_history(self, account): + def get_history(self, account, coming=False): if account.type in (Account.TYPE_MARKET, Account.TYPE_PEA, Account.TYPE_LIFE_INSURANCE, Account.TYPE_PERP): self.logger.warning('This account is not supported') raise NotImplementedError() diff --git a/modules/creditdunord/module.py b/modules/creditdunord/module.py index a6ea0595cdcd3a8172f1ed064d2c084d155562e8..b6ae084dea914f405a9e614c344e901f2d6b6ed7 100644 --- a/modules/creditdunord/module.py +++ b/modules/creditdunord/module.py @@ -83,7 +83,7 @@ def iter_history(self, account): yield tr def iter_coming(self, account): - account = self.get_account(account.id) + account = self.get_account_for_history(account.id) for tr in self.browser.get_history(account, coming=True): if tr._is_coming: yield tr diff --git a/modules/creditmutuel/browser.py b/modules/creditmutuel/browser.py index 8b7ba663f1bee2a0f6354182848a1136ac49b975..dbf2f4f5c0e2422dd1ae1b5488167d2f9176de6d 100644 --- a/modules/creditmutuel/browser.py +++ b/modules/creditmutuel/browser.py @@ -312,8 +312,11 @@ def list_operations(self, page, account): form.pop(k, None) form.submit() # IndexError when form xpath returns [], StopIteration if next called on empty iterable - except (IndexError, StopIteration, FormNotFound): + except (StopIteration, FormNotFound): self.logger.warning('Could not get history on new website') + except IndexError: + # 6 months history is not available + pass while self.page: try: diff --git a/modules/creditmutuel/pages.py b/modules/creditmutuel/pages.py index 9a161a6b5023a728e44e51fa122f51240a561155..7435149665af8b27444c1a3069f2693a698c7d5f 100644 --- a/modules/creditmutuel/pages.py +++ b/modules/creditmutuel/pages.py @@ -907,7 +907,11 @@ def obj_type(self): def obj_original_amount(self): m = re.search(r'(([\s-]\d+)+,\d+)', CleanText(TableCell('commerce'))(self)) if m and not 'FRAIS' in CleanText(TableCell('commerce'))(self): - return Decimal(m.group(1).replace(',', '.').replace(' ', '')).quantize(Decimal('0.01')) + matched_text = m.group(1) + submatch = re.search(r'\d+-(.*)', matched_text) + if submatch: + matched_text = submatch.group(1) + return Decimal(matched_text.replace(',', '.').replace(' ', '')).quantize(Decimal('0.01')) return NotAvailable def obj_original_currency(self): @@ -979,7 +983,11 @@ def obj_type(self): def obj_original_amount(self): m = re.search(r'(([\s-]\d+)+,\d+)', CleanText(TableCell('commerce'))(self)) if m and not 'FRAIS' in CleanText(TableCell('commerce'))(self): - return Decimal(m.group(1).replace(',', '.').replace(' ', '')).quantize(Decimal('0.01')) + matched_text = m.group(1) + submatch = re.search(r'\d+-(.*)', matched_text) + if submatch: + matched_text = submatch.group(1) + return Decimal(matched_text.replace(',', '.').replace(' ', '')).quantize(Decimal('0.01')) return NotAvailable def obj_original_currency(self): @@ -1105,11 +1113,11 @@ def obj_commission(self): @method class iter_investment(TableElement): - item_xpath = '//table[has-class("liste")]/tbody/tr[count(td)>=7]' - head_xpath = '//table[has-class("liste")]/thead/tr/th' + item_xpath = '//table[has-class("liste") and not (@summary="Avances")]/tbody/tr[count(td)>=7]' + head_xpath = '//table[has-class("liste") and not (@summary="Avances")]/thead/tr/th' col_label = 'Support' - col_unitprice = re.compile(r"^Prix d'achat moyen") + col_unitprice = re.compile(r'Prix') col_vdate = re.compile(r'Date de cotation') col_unitvalue = 'Valeur de la part' col_quantity = 'Nombre de parts' diff --git a/modules/entreparticuliers/module.py b/modules/entreparticuliers/module.py index ea2b5e139f9359043f0cc4bad694706070317c25..648c4cdc84b717eeec7457cbd02d43cf3d655163 100644 --- a/modules/entreparticuliers/module.py +++ b/modules/entreparticuliers/module.py @@ -20,7 +20,7 @@ from weboob.tools.backend import Module from weboob.capabilities.housing import (CapHousing, HousingPhoto, - ADVERT_TYPES) + ADVERT_TYPES, Housing) from .browser import EntreparticuliersBrowser @@ -61,4 +61,10 @@ def fill_photo(self, photo, fields): photo.data = self.browser.open(photo.url).content return photo - OBJECTS = {HousingPhoto: fill_photo} + def fill_housing(self, housing, fields): + if len(fields) > 0: + self.browser.get_housing(housing.id, housing) + + return housing + + OBJECTS = {HousingPhoto: fill_photo, Housing: fill_housing} diff --git a/modules/explorimmo/browser.py b/modules/explorimmo/browser.py index c029a787a161a256be2fbff8aba91cd2099f3612..fbbc8aeece95c362e21c3442ed008b7a1d220c49 100644 --- a/modules/explorimmo/browser.py +++ b/modules/explorimmo/browser.py @@ -29,7 +29,7 @@ class ExplorimmoBrowser(PagesBrowser): cities = URL('/rest/locations\?q=(?P.*)', CitiesPage) search = URL('/annonces/resultat/annonces.html\?(?P.*)', SearchPage) - housing_html = URL('/annonce-(?P<_id>.*).html', HousingPage) + housing_html = URL('/annonces/annonce-(?P<_id>.*).html', HousingPage) phone = URL('/rest/classifieds/(?P<_id>.*)/phone', PhonePage) housing = URL('/rest/classifieds/(?P<_id>.*)', '/rest/classifieds/\?(?P.*)', HousingPage2) diff --git a/modules/explorimmo/pages.py b/modules/explorimmo/pages.py index dac434acbe1bab9b2808b5ac366d9d40068feb63..46ac23c6826ff858c77b21ee3a47f2a0b0380668 100644 --- a/modules/explorimmo/pages.py +++ b/modules/explorimmo/pages.py @@ -192,7 +192,7 @@ def obj_utilities(self): ) obj_url = Format( - "http://www.explorimmo.com/annonce-%s.html", + "https://immobilier.lefigaro.fr/annonces/annonce-%s.html", CleanText('./@data-classified-id') ) diff --git a/modules/fortuneo/browser.py b/modules/fortuneo/browser.py index 7d587b48c4a9f6b3e5c1aaf88b89d4065fa62f5b..cb57a68094a58b3a54531903eac647be5e85ce7a 100644 --- a/modules/fortuneo/browser.py +++ b/modules/fortuneo/browser.py @@ -32,7 +32,7 @@ from .pages.login import LoginPage, UnavailablePage from .pages.accounts_list import ( - AccountsList, AccountHistoryPage, CardHistoryPage, InvestmentHistoryPage, PeaHistoryPage, LoanPage, ProfilePage, ProfilePageCSV, SecurityPage, + AccountsList, AccountHistoryPage, CardHistoryPage, InvestmentHistoryPage, PeaHistoryPage, LoanPage, ProfilePage, ProfilePageCSV, SecurityPage, FakeActionPage, ) from .pages.transfer import ( RegisterTransferPage, ValidateTransferPage, ConfirmTransferPage, RecipientsPage, RecipientSMSPage @@ -84,7 +84,7 @@ class Fortuneo(LoginBrowser, StatesMixin): r'fr/prive/mes-comptes/compte-courant/.*/init-confirmer-saisie-virement.jsp', r'/fr/prive/mes-comptes/compte-courant/.*/confirmer-saisie-virement.jsp', ConfirmTransferPage) - + fake_action_page = URL(r'fr/prive/mes-comptes/synthese-globale/synthese-mes-comptes.jsp', FakeActionPage) profile = URL(r'/fr/prive/informations-client.jsp', ProfilePage) profile_csv = URL(r'/PdfStruts\?*', ProfilePageCSV) @@ -160,7 +160,17 @@ def get_accounts_list(self): self.process_action_needed() assert self.accounts_page.is_here() - return self.page.get_list() + accounts_list = self.page.get_list() + if self.fake_action_page.is_here(): + # A false action needed is present, it's a choice to make Fortuno your main bank. + # To avoid it, we need to first detect it on the account_page + # Then make a post request to mimic the click on choice 'later' + # And to finish we must to reload the page with a POST to get the accounts + # before going on the accounts_page, which will have the data. + self.location(self.absurl('ReloadContext?action=1&', base=True), method='POST') + self.accounts_page.go() + accounts_list = self.page.get_list() + return accounts_list def process_action_needed(self): # we have to go in an iframe to know if there are CGUs diff --git a/modules/fortuneo/pages/accounts_list.py b/modules/fortuneo/pages/accounts_list.py index d90c808dad9172a52e1577fac7ce76f3c4726ebd..e9add0e6bead7c0abbe0fe7b0aee1a2cffbbc2e5 100644 --- a/modules/fortuneo/pages/accounts_list.py +++ b/modules/fortuneo/pages/accounts_list.py @@ -452,6 +452,9 @@ def get_list(self): '| //div[@id="as_renouvellementMotDePasse.do_"]//p[contains(text(), "votre mot de passe")]' '| //div[@id="as_afficherSecuriteForteOTPIdentification.do_"]//span[contains(text(), "Pour valider ")]') if global_error_message: + if "Et si vous faisiez de Fortuneo votre banque principale" in CleanText(global_error_message)(self): + self.browser.location('/ReloadContext', data={'action': 4}) + return raise ActionNeeded(CleanText('.')(global_error_message[0])) local_error_message = page.doc.xpath('//div[@id="error"]/p[@class="erreur_texte1"]') if local_error_message: @@ -501,6 +504,9 @@ def get_list(self): return accounts +class FakeActionPage(LoggedPage, HTMLPage): + pass + class LoanPage(LoggedPage, HTMLPage): def get_balance(self): return CleanText(u'//p[@id="c_montantRestant"]//strong')(self.doc) diff --git a/modules/hsbc/pages/account_pages.py b/modules/hsbc/pages/account_pages.py index fd822841a4ec09a054f3076deefbc8ce8af28ceb..03c04a5d4f38422f90f48f5dc6cfa6676c146949 100644 --- a/modules/hsbc/pages/account_pages.py +++ b/modules/hsbc/pages/account_pages.py @@ -160,7 +160,7 @@ class item(ItemElement): klass = Account def condition(self): - return len(self.el.xpath('./td')) > 2 + return len(self.el.xpath('./td')) > 2 and "en opposition" not in CleanText('./td[1]')(self) # Some accounts have no in the first def obj_label(self): diff --git a/modules/ing/browser.py b/modules/ing/browser.py index 56b83fb5cde484d23115e8dd4c5569d6cd478730..a885ce5b1730385b277fc60caced5a91116f903d 100644 --- a/modules/ing/browser.py +++ b/modules/ing/browser.py @@ -16,6 +16,7 @@ # # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals import hashlib @@ -45,10 +46,10 @@ def start_with_main_site(f): def wrapper(*args, **kwargs): browser = args[0] - if browser.url and browser.url.startswith('https://bourse.ingdirect.fr/'): + if browser.url and browser.url.startswith('https://bourse.ing.fr/'): for i in range(3): try: - browser.location('https://bourse.ingdirect.fr/priv/redirectIng.php?pageIng=COMPTE') + browser.location('https://bourse.ing.fr/priv/redirectIng.php?pageIng=CC') except ServerError: pass else: @@ -74,35 +75,35 @@ class IngBrowser(LoginBrowser): lifeback = URL(r'https://ingdirectvie.ing.fr/b2b2c/entreesite/EntAccExit', ReturnPage) # Login and error - loginpage = URL('/public/displayLogin.jsf.*', LoginPage) - errorpage = URL('.*displayCoordonneesCommand.*', StopPage) - actioneeded = URL('/general\?command=displayTRAlertMessage', - '/protected/pages/common/eco1/moveMoneyForbidden.jsf', ActionNeededPage) + loginpage = URL(r'/public/displayLogin.jsf.*', LoginPage) + errorpage = URL(r'.*displayCoordonneesCommand.*', StopPage) + actioneeded = URL(r'/general\?command=displayTRAlertMessage', + r'/protected/pages/common/eco1/moveMoneyForbidden.jsf', ActionNeededPage) # CapBank - accountspage = URL('/protected/pages/index.jsf', - '/protected/pages/asv/contract/(?P.*).jsf', AccountsList) - titredetails = URL('/general\?command=display.*', TitreDetails) - ibanpage = URL('/protected/pages/common/rib/initialRib.jsf', IbanPage) - loantokenpage = URL('general\?command=goToConsumerLoanCommand&redirectUrl=account-details', LoanTokenPage) - loandetailpage = URL('https://subscribe.ing.fr/consumerloan/consumerloan-v1/consumer/details', LoanDetailPage) + accountspage = URL(r'/protected/pages/index.jsf', + r'/protected/pages/asv/contract/(?P.*).jsf', AccountsList) + titredetails = URL(r'/general\?command=display.*', TitreDetails) + ibanpage = URL(r'/protected/pages/common/rib/initialRib.jsf', IbanPage) + loantokenpage = URL(r'general\?command=goToConsumerLoanCommand&redirectUrl=account-details', LoanTokenPage) + loandetailpage = URL(r'https://subscribe.ing.fr/consumerloan/consumerloan-v1/consumer/details', LoanDetailPage) # CapBank-Market - netissima = URL('/data/asv/fiches-fonds/fonds-netissima.html', NetissimaPage) - starttitre = URL('/general\?command=goToAccount&zone=COMPTE', TitrePage) - titrepage = URL('https://bourse.ingdirect.fr/priv/portefeuille-TR.php', TitrePage) - titrehistory = URL('https://bourse.ingdirect.fr/priv/compte.php\?ong=3', TitreHistory) - titrerealtime = URL('https://bourse.ingdirect.fr/streaming/compteTempsReelCK.php', TitrePage) - titrevalue = URL('https://bourse.ingdirect.fr/priv/fiche-valeur.php\?val=(?P.*)&pl=(?P.*)&popup=1', TitreValuePage) - asv_history = URL('https://ingdirectvie.ing.fr/b2b2c/epargne/CoeLisMvt', - 'https://ingdirectvie.ing.fr/b2b2c/epargne/CoeDetMvt', ASVHistory) - asv_invest = URL('https://ingdirectvie.ing.fr/b2b2c/epargne/CoeDetCon', ASVInvest) - detailfonds = URL('https://ingdirectvie.ing.fr/b2b2c/fonds/PerDesFac\?codeFonds=(.*)', DetailFondsPage) + netissima = URL(r'/data/asv/fiches-fonds/fonds-netissima.html', NetissimaPage) + starttitre = URL(r'/general\?command=goToAccount&zone=COMPTE', TitrePage) + titrepage = URL(r'https://bourse.ing.fr/priv/portefeuille-TR.php', TitrePage) + titrehistory = URL(r'https://bourse.ing.fr/priv/compte.php\?ong=3', TitreHistory) + titrerealtime = URL(r'https://bourse.ing.fr/streaming/compteTempsReelCK.php', TitrePage) + titrevalue = URL(r'https://bourse.ing.fr/priv/fiche-valeur.php\?val=(?P.*)&pl=(?P.*)&popup=1', TitreValuePage) + asv_history = URL(r'https://ingdirectvie.ing.fr/b2b2c/epargne/CoeLisMvt', + r'https://ingdirectvie.ing.fr/b2b2c/epargne/CoeDetMvt', ASVHistory) + asv_invest = URL(r'https://ingdirectvie.ing.fr/b2b2c/epargne/CoeDetCon', ASVInvest) + detailfonds = URL(r'https://ingdirectvie.ing.fr/b2b2c/fonds/PerDesFac\?codeFonds=(.*)', DetailFondsPage) # CapDocument - billpage = URL('/protected/pages/common/estatement/eStatement.jsf', BillsPage) + billpage = URL(r'/protected/pages/common/estatement/eStatement.jsf', BillsPage) # CapProfile - profile = URL('/protected/pages/common/profil/(?P\w+).jsf', ProfilePage) + profile = URL(r'/protected/pages/common/profil/(?P\w+).jsf', ProfilePage) - transfer = URL('/protected/pages/common/virement/index.jsf', TransferPage) + transfer = URL(r'/protected/pages/common/virement/index.jsf', TransferPage) __states__ = ['where'] @@ -120,6 +121,10 @@ def __init__(self, *args, **kwargs): self.multispace = None self.current_space = None + # ing website is stateful, so we need to store the current subscription when download document to be sure + # we download file for the right subscription + self.current_subscription = None + def do_login(self): assert self.password.isdigit() assert self.birthday.isdigit() @@ -137,8 +142,7 @@ def do_login(self): @need_login def set_multispace(self): - self.accountspage.go() - self.where = "start" + self.where = 'start' self.page.load_space_page() self.multispace = self.page.get_multispace() @@ -153,7 +157,7 @@ def set_multispace(self): def change_space(self, space): if self.multispace and not self.is_same_space(space, self.current_space): self.accountspage.go() - self.where = "start" + self.where = 'start' self.page.load_space_page() self.page.change_space(space) @@ -168,29 +172,29 @@ def is_same_space(self, a, b): @start_with_main_site def get_market_balance(self, account): - if self.where != "start": + if self.where != 'start': self.accountspage.go() - self.where = "start" + self.where = 'start' self.change_space(account._space) data = self.get_investments_data(account) for i in range(5): if i > 0: - self.logger.debug('Can\'t get market balance, retrying in %s seconds...', (2**i)) + self.logger.debug("Can't get market balance, retrying in %s seconds...", (2**i)) time.sleep(2**i) if self.accountspage.go(data=data).has_link(): break self.starttitre.go() - self.where = u"titre" + self.where = 'titre' self.titrepage.go() self.titrerealtime.go() account.balance = self.page.get_balance() or account.balance self.cache["investments_data"][account.id] = self.page.doc or None @need_login - def get_iban(self, account): + def fill_account(self, account): if account.type in [Account.TYPE_CHECKING, Account.TYPE_SAVINGS]: self.go_account_page(account) account.iban = self.ibanpage.go().get_iban() @@ -199,15 +203,18 @@ def get_iban(self, account): self.get_market_balance(account) @need_login - def get_accounts_on_space(self, space, get_iban=True): + def get_accounts_on_space(self, space, fill_account=True): accounts_list = [] self.change_space(space) for acc in self.page.get_list(): acc._space = space - if get_iban: - self.get_iban(acc) + if fill_account: + try: + self.fill_account(acc) + except ServerError: + pass assert not find_object(accounts_list, id=acc.id), 'There is a duplicate account.' accounts_list.append(acc) @@ -222,25 +229,28 @@ def get_accounts_on_space(self, space, get_iban=True): @need_login @start_with_main_site - def get_accounts_list(self, space=None, get_iban=True): + def get_accounts_list(self, space=None, fill_account=True): self.accountspage.go() - self.where = "start" + self.where = 'start' self.set_multispace() if space: - for acc in self.get_accounts_on_space(space, get_iban=get_iban): + for acc in self.get_accounts_on_space(space, fill_account=fill_account): yield acc elif self.multispace: for space in self.multispace: - for acc in self.get_accounts_on_space(space, get_iban=get_iban): + for acc in self.get_accounts_on_space(space, fill_account=fill_account): yield acc else: for acc in self.page.get_list(): acc._space = None - if get_iban: - self.get_iban(acc) + if fill_account: + try: + self.fill_account(acc) + except ServerError: + pass yield acc for loan in self.iter_detailed_loans(): @@ -251,7 +261,7 @@ def get_accounts_list(self, space=None, get_iban=True): @start_with_main_site def iter_detailed_loans(self): self.accountspage.go() - self.where = "start" + self.where = 'start' for loan in self.page.get_detailed_loans(): data = {'AJAXREQUEST': '_viewRoot', @@ -283,8 +293,7 @@ def return_from_loan_site(self): self.location('https://secure.ing.fr/', data={'token': self.response.text}) def get_account(self, _id, space=None): - return find_object(self.get_accounts_list(get_iban=False, space=space), id=_id, error=AccountNotFound) - + return find_object(self.get_accounts_list(fill_account=False, space=space), id=_id, error=AccountNotFound) def go_account_page(self, account): data = {"AJAX:EVENTS_COUNT": 1, @@ -302,7 +311,7 @@ def go_account_page(self, account): self.only_deferred_cards[account._id] = all( [card['kind'] == self.DEFERRED_CB for card in card_list] ) - self.where = "history" + self.where = 'history' @need_login @start_with_main_site @@ -406,10 +415,10 @@ def go_on_asv_detail(self, account, link): try: if self.page.asv_is_other: jid = self.page.get_asv_jid() - data = {'index': "index", 'javax.faces.ViewState': jid, 'index:j_idcl': "index:asvInclude:goToAsvPartner"} + data = {'index': "index", 'javax.faces.ViewState': jid, 'index:j_idcl': 'index:asvInclude:goToAsvPartner'} self.accountspage.go(data=data) else: - self.accountspage.go(asvpage="manageASVContract") + self.accountspage.go(asvpage='manageASVContract') self.page.submit() self.page.submit() self.location(link) @@ -453,13 +462,13 @@ def go_investments(self, account): break else: - self.logger.warning("Unable to get investments list...") + self.logger.warning('Unable to get investments list...') if self.page.is_asv: return self.starttitre.go() - self.where = u"titre" + self.where = 'titre' self.titrepage.go() @need_login @@ -470,8 +479,8 @@ def get_investments(self, account): self.go_investments(account) - if self.where == u'titre': - if self.cache["investments_data"].get(account.id) is None: + if self.where == 'titre': + if self.cache['investments_data'].get(account.id) is None: self.titrerealtime.go() for inv in self.page.iter_investments(account): yield inv @@ -481,7 +490,7 @@ def get_investments(self, account): for asv_investments in self.page.iter_asv_investments(): shares[asv_investments.label] = asv_investments.portfolio_share if self.go_on_asv_detail(account, '/b2b2c/epargne/CoeDetCon') is not False: - self.where = u"asv" + self.where = 'asv' for inv in self.page.iter_investments(): inv.portfolio_share = shares[inv.label] yield inv @@ -489,7 +498,7 @@ def get_investments(self, account): def get_history_titre(self, account): self.go_investments(account) - if self.where == u'titre': + if self.where == 'titre': self.titrehistory.go() elif self.page.asv_has_detail or account._jid: if self.go_on_asv_detail(account, '/b2b2c/epargne/CoeLisMvt') is False: @@ -535,25 +544,44 @@ def get_subscriptions(self): self.billpage.go() if self.loginpage.is_here(): self.do_login() - return self.billpage.go().iter_account() + subscriptions = list(self.billpage.go().iter_subscriptions()) else: - return self.page.iter_account() + subscriptions = list(self.page.iter_subscriptions()) + + self.cache['subscriptions'] = {} + for sub in subscriptions: + self.cache['subscriptions'][sub.id] = sub + + return subscriptions + + def _go_to_subscription(self, subscription): + # ing website is not stateless, make sure we are on the correct documents page before doing anything else + if self.current_subscription and self.current_subscription.id == subscription.id: + return - @need_login - def get_documents(self, subscription): self.billpage.go() - data = {"AJAXREQUEST": "_viewRoot", - "accountsel_form": "accountsel_form", - subscription._formid: subscription._formid, - "autoScroll": "", - "javax.faces.ViewState": subscription._javax, - "transfer_issuer_radio": subscription.id - } + data = { + "AJAXREQUEST": "_viewRoot", + "accountsel_form": "accountsel_form", + subscription._formid: subscription._formid, + "autoScroll": "", + "javax.faces.ViewState": subscription._javax, + "transfer_issuer_radio": subscription.id + } self.billpage.go(data=data) + self.current_subscription = subscription + + @need_login + def get_documents(self, subscription): + self._go_to_subscription(subscription) return self.page.iter_documents(subid=subscription.id) - def predownload(self, bill): - self.page.postpredown(bill._localid) + def download_document(self, bill): + subid = bill.id.split('-')[0] + # make sure we are on the right page to not download a document from another subscription + self._go_to_subscription(self.cache['subscriptions'][subid]) + self.page.go_to_year(bill._year) + return self.page.download_document(bill) ############# CapProfile ############# @start_with_main_site diff --git a/modules/ing/module.py b/modules/ing/module.py index ba2f2f22c5b407f9e012149e7f2212fd83c6f39e..0e9c12e3d639bfbbd9d90eee9776f06de653a0a3 100644 --- a/modules/ing/module.py +++ b/modules/ing/module.py @@ -16,6 +16,8 @@ # # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals + import re from datetime import timedelta @@ -27,10 +29,9 @@ SubscriptionNotFound, DocumentNotFound, DocumentTypes, ) from weboob.capabilities.profile import CapProfile -from weboob.capabilities.base import find_object, NotAvailable +from weboob.capabilities.base import find_object from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword, ValueDate -from weboob.browser.exceptions import ServerError from .browser import IngBrowser @@ -39,13 +40,13 @@ class INGModule(Module, CapBankWealth, CapBankTransfer, CapDocument, CapProfile): NAME = 'ing' - MAINTAINER = u'Florent Fourcot' + MAINTAINER = 'Florent Fourcot' EMAIL = 'weboob@flo.fourcot.fr' VERSION = '1.4' LICENSE = 'AGPLv3+' DESCRIPTION = 'ING Direct' CONFIG = BackendConfig(ValueBackendPassword('login', - label=u'Numéro client', + label='Numéro client', masked=False), ValueBackendPassword('password', label='Code secret', @@ -89,7 +90,7 @@ def iter_transfer_recipients(self, account): def init_transfer(self, transfer, **params): self.logger.info('Going to do a new transfer') - transfer.label = ' '.join(w for w in re.sub('[^0-9a-zA-Z/\-\?:\(\)\.,\'\+ ]+', '', transfer.label).split()).upper() + transfer.label = ' '.join(w for w in re.sub(r'[^0-9a-zA-Z/\-\?:\(\)\.,\'\+ ]+', '', transfer.label).split()).upper() if transfer.account_iban: account = find_object(self.iter_accounts(), iban=transfer.account_iban, error=AccountNotFound) else: @@ -138,13 +139,8 @@ def iter_documents(self, subscription): def download_document(self, bill): if not isinstance(bill, Bill): bill = self.get_document(bill) - self.get_document(bill.id) - try: - self.browser.predownload(bill) - except ServerError: - return NotAvailable - assert(self.browser.response.headers['content-type'] in ["application/pdf", "application/download"]) - return self.browser.response.content + + return self.browser.download_document(bill).content def get_profile(self): return self.browser.get_profile() diff --git a/modules/ing/pages/bills.py b/modules/ing/pages/bills.py index 0051f442fa54826d69107c362214d3b74979d4cc..bdaa9492c1ac0281b46c62df195b8d6e7bc6ad50 100644 --- a/modules/ing/pages/bills.py +++ b/modules/ing/pages/bills.py @@ -16,9 +16,11 @@ # # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals + from weboob.capabilities.bill import DocumentTypes, Bill, Subscription -from weboob.browser.pages import HTMLPage, LoggedPage, pagination +from weboob.browser.pages import HTMLPage, LoggedPage, pagination, Form from weboob.browser.filters.standard import Filter, CleanText, Format, Field, Env, Date from weboob.browser.filters.html import Attr from weboob.browser.elements import ListElement, ItemElement, method @@ -32,29 +34,60 @@ def filter(self, txt): return formid +class MyForm(Form): + def submit(self, **kwargs): + """ + Submit the form but keep current browser.page + """ + kwargs.setdefault('data_encoding', self.page.encoding) + return self.page.browser.open(self.request, **kwargs) + + class BillsPage(LoggedPage, HTMLPage): @method - class iter_account(ListElement): + class iter_subscriptions(ListElement): item_xpath = '//ul[@class="unstyled striped"]/li' class item(ItemElement): klass = Subscription obj__javax = Attr("//form[@id='accountsel_form']/input[@name='javax.faces.ViewState']", 'value') - obj_id = Attr('input', "value") + obj_id = Attr('input', 'value') obj_label = CleanText('label') obj__formid = FormId(Attr('input', 'onclick')) - def postpredown(self, _id): - _id = _id.split("'")[3] - form = self.get_form(name="downpdf_form") + def get_selected_year(self): + return int(CleanText('//form[@id="years_form"]//ul/li[@class="rich-list-item selected"]')(self.doc)) + + def go_to_year(self, year): + if year == self.get_selected_year(): + return + + ref = Attr('//form[@id="years_form"]//ul//a[text()="%s"]' % year, 'id')(self.doc) + + self.FORM_CLASS = Form + form = self.get_form(name='years_form') + form.pop('years_form:j_idcl') + form.pop('years_form:_link_hidden_') + form['AJAXREQUEST'] = 'years_form:year_region' + form[ref] = ref + + return form.submit() + + def download_document(self, bill): + # MyForm do open, and not location to keep html page as self.page, to reduce number of request on this html page + self.FORM_CLASS = MyForm + _id = bill._localid.split("'")[3] + + form = self.get_form(name='downpdf_form') form['statements_form'] = 'statements_form' form['statements_form:j_idcl'] = _id - form.submit() + return form.submit() @pagination @method class iter_documents(ListElement): + flush_at_end = True item_xpath = '//ul[@id="statements_form:statementsel"]/li' def next_page(self): @@ -62,7 +95,7 @@ def next_page(self): selected = False ref = None for li in lis: - if "rich-list-item selected" in li.attrib['class']: + if 'rich-list-item selected' in li.attrib['class']: selected = True else: if selected: @@ -70,22 +103,29 @@ def next_page(self): break if ref is None: return - form = self.page.get_form(name="years_form") + form = self.page.get_form(name='years_form') form.pop('years_form:j_idcl') form.pop('years_form:_link_hidden_') - form['AJAXREQUEST'] = "years_form:year_region" + form['AJAXREQUEST'] = 'years_form:year_region' form[ref] = ref return form.request + def flush(self): + for obj in reversed(self.objects.values()): + yield obj + class item(ItemElement): klass = Bill - condition = lambda self: not (u"tous les relev" in CleanText('a[1]')(self.el)) and not (u'annuel' in CleanText('a[1]')(self.el)) + condition = lambda self: not ('tous les relev' in CleanText('a[1]')(self.el)) and not ('annuel' in CleanText('a[1]')(self.el)) obj_label = CleanText('a[1]', replace=[(' ', '-')]) - obj_id = Format(u"%s-%s", Env('subid'), Field('label')) + obj_id = Format('%s-%s', Env('subid'), Field('label')) # Force first day of month as label is in form "janvier 2016" - obj_date = Format("1 %s", Field('label')) & Date(parse_func=parse_french_date) - obj_format = u"pdf" + obj_date = Format('1 %s', Field('label')) & Date(parse_func=parse_french_date) + obj_format = 'pdf' obj_type = DocumentTypes.STATEMENT obj__localid = Attr('a[2]', 'onclick') + + def obj__year(self): + return int(CleanText('a[1]')(self).split(' ')[1]) diff --git a/modules/ing/pages/transfer.py b/modules/ing/pages/transfer.py index 034bd085997c3930c70caf5f3e804201b8546a50..b05ec9e098446d0b41e4f9f537fed10c93086829 100644 --- a/modules/ing/pages/transfer.py +++ b/modules/ing/pages/transfer.py @@ -73,7 +73,7 @@ class item(MyRecipient): def parse(self, el): _id = Attr('.', 'data-acct-number')(self) - accounts = [acc for acc in self.page.browser.get_accounts_list(get_iban=False, space=self.env['origin']._space) if _id in acc.id] + accounts = [acc for acc in self.page.browser.get_accounts_list(fill_account=False, space=self.env['origin']._space) if _id in acc.id] assert len(accounts) == 1 account = accounts[0] self.env['id'] = account.id diff --git a/modules/lcl/browser.py b/modules/lcl/browser.py index 10a4132cbcae546c2a44d46ed1f84bead3bd1d8b..bbe000b77d8e39c90a32addb485ce63e9dcb6bdf 100644 --- a/modules/lcl/browser.py +++ b/modules/lcl/browser.py @@ -24,10 +24,10 @@ from functools import wraps from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable -from weboob.browser import LoginBrowser, URL, need_login, StatesMixin +from weboob.browser.browsers import LoginBrowser, URL, need_login, StatesMixin from weboob.browser.exceptions import ServerError from weboob.capabilities.base import NotAvailable -from weboob.capabilities.bank import Account, AddRecipientBankError, AddRecipientStep, Recipient +from .compat.weboob_capabilities_bank import Account, AddRecipientBankError, AddRecipientStep, Recipient, AccountOwnerType from weboob.tools.capabilities.bank.investments import create_french_liquidity from weboob.tools.compat import basestring, urlsplit, parse_qsl, unicode from weboob.tools.value import Value @@ -126,6 +126,7 @@ def __init__(self, *args, **kwargs): self.accounts_list = None self.current_contract = None self.contracts = None + self.owner_type = AccountOwnerType.PRIVATE def load_state(self, state): super(LCLBrowser, self).load_state(state) @@ -284,6 +285,9 @@ def get_accounts_list(self): self.get_accounts() else: self.get_accounts() + + for account in self.accounts_list: + account.owner_type = self.owner_type return iter(self.accounts_list) def get_bourse_accounts_ids(self): @@ -540,6 +544,7 @@ class LCLProBrowser(LCLBrowser): def __init__(self, *args, **kwargs): super(LCLProBrowser, self).__init__(*args, **kwargs) self.session.cookies.set("lclgen", "professionnels", domain=urlsplit(self.BASEURL).hostname) + self.owner_type = AccountOwnerType.ORGANIZATION class ELCLBrowser(LCLBrowser): diff --git a/modules/lcl/compat/__init__.py b/modules/lcl/compat/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/modules/lcl/compat/weboob_capabilities_bank.py b/modules/lcl/compat/weboob_capabilities_bank.py new file mode 100644 index 0000000000000000000000000000000000000000..181890bcecf68a64f746379f6190b3277b58404d --- /dev/null +++ b/modules/lcl/compat/weboob_capabilities_bank.py @@ -0,0 +1,18 @@ +import weboob.capabilities.bank as OLD + +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) + + +__all__ = OLD.__all__ + + +class AccountOwnerType(object): + """ + Specifies the usage of the account + """ + PRIVATE = u'PRIV' + """private personal account""" + ORGANIZATION = u'ORGA' + """professional account""" diff --git a/modules/lcl/enterprise/browser.py b/modules/lcl/enterprise/browser.py index 3f0f168c5828957d9af41777f7d1c24049fc3309..5237a5ab4ca4308af2a97250668c9b110b3e1c64 100644 --- a/modules/lcl/enterprise/browser.py +++ b/modules/lcl/enterprise/browser.py @@ -18,8 +18,9 @@ # along with weboob. If not, see . -from weboob.browser import LoginBrowser, URL, need_login +from weboob.browser.browsers import LoginBrowser, URL, need_login from weboob.exceptions import BrowserIncorrectPassword +from .compat.weboob_capabilities_bank import AccountOwnerType from .pages import LoginPage, MovementsPage, ProfilePage, PassExpiredPage @@ -37,6 +38,8 @@ class LCLEnterpriseBrowser(LoginBrowser): def __init__(self, *args, **kwargs): super(LCLEnterpriseBrowser, self).__init__(*args, **kwargs) self.accounts = None + self.owner_type = AccountOwnerType.ORGANIZATION + def deinit(self): if self.page and self.page.logged: @@ -57,7 +60,10 @@ def do_login(self): def get_accounts_list(self): if not self.accounts: self.accounts = list(self.movements.go().iter_accounts()) - return self.accounts + + for account in self.accounts: + account.owner_type = self.owner_type + yield account @need_login def get_history(self, account): diff --git a/modules/lcl/enterprise/compat/__init__.py b/modules/lcl/enterprise/compat/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/modules/lcl/enterprise/compat/weboob_capabilities_bank.py b/modules/lcl/enterprise/compat/weboob_capabilities_bank.py new file mode 100644 index 0000000000000000000000000000000000000000..181890bcecf68a64f746379f6190b3277b58404d --- /dev/null +++ b/modules/lcl/enterprise/compat/weboob_capabilities_bank.py @@ -0,0 +1,18 @@ +import weboob.capabilities.bank as OLD + +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) + + +__all__ = OLD.__all__ + + +class AccountOwnerType(object): + """ + Specifies the usage of the account + """ + PRIVATE = u'PRIV' + """private personal account""" + ORGANIZATION = u'ORGA' + """professional account""" diff --git a/modules/logicimmo/pages.py b/modules/logicimmo/pages.py index 3c4f2b63e42805b602733c906e1c2ddfa5585f0a..c9d3eea439c6269984c5aeac945c27b37083de11 100644 --- a/modules/logicimmo/pages.py +++ b/modules/logicimmo/pages.py @@ -281,13 +281,12 @@ def obj_house_type(self): offer_details_wrapper + '/div/div/p[@class="offer-type"]/a', 'title' ) - obj_url = Format( - "http://www.logic-immo.com/%s.htm", - CleanText( - './@id', - replace=[('header-offer-', 'detail-location-')] - ) - ) + + obj_url = Format(u'%s%s', + CleanText('./div/div/div/div/div/p/a[@class="offer-link"]/@href'), + CleanText('./div/div/div/div/div/p/a[@class="offer-link"]/\ +@data-orpi', default="")) + obj_area = CleanDecimal( ( offer_details_wrapper + diff --git a/modules/myedenred/pages.py b/modules/myedenred/pages.py index 1caa77671540fe698cc6e65cd764b6e9ae7a5891..eb1216026498e05d8e3ecad2ecb73278918d02db 100644 --- a/modules/myedenred/pages.py +++ b/modules/myedenred/pages.py @@ -71,6 +71,7 @@ class item(ItemElement): obj_date = DateGuesser(CleanText('.//span[contains(., "/")]'), LinearDateGuesser(date_max_bump=timedelta(45))) obj_label = CleanText('.//h3/strong') + obj_raw = Field('label') obj_amount = MyDecimal('./td[@class="al-r"]/div/span[has-class("badge")]') def obj_type(self): amount = Field('amount')(self) diff --git a/modules/n26/browser.py b/modules/n26/browser.py index 3d91db7d6a8af49a2869fed948277ff2966bdbdc..41af807dc96c45ef318efda8bc886ad20591683f 100644 --- a/modules/n26/browser.py +++ b/modules/n26/browser.py @@ -139,6 +139,13 @@ def get_coming(self, categories): @need_login def _internal_get_transactions(self, categories, filter_func): + TYPES = { + 'PT': Transaction.TYPE_CARD, + 'AA': Transaction.TYPE_CARD, + 'CT': Transaction.TYPE_TRANSFER, + 'WEE': Transaction.TYPE_BANK, + } + transactions = self.request('/api/smrt/transactions?limit=1000') for t in transactions: @@ -167,12 +174,7 @@ def _internal_get_transactions(self, categories, filter_func): if "originalAmount" in t: new.original_amount = Decimal(str(t["originalAmount"])) - if t["type"] == 'PT': - new.type = Transaction.TYPE_CARD - elif t["type"] == 'CT': - new.type = Transaction.TYPE_TRANSFER - elif t["type"] == 'WEE': - new.type = Transaction.TYPE_BANK + new.type = TYPES.get(t["type"], Transaction.TYPE_UNKNOWN) if t["category"] in categories: new.category = categories[t["category"]] diff --git a/modules/nef/__init__.py b/modules/nef/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..99f80964f92e6b85200c1c67ed0ba12aae1c8fbf --- /dev/null +++ b/modules/nef/__init__.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Damien Cassou +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + + +from .module import NefModule + + +__all__ = ['NefModule'] diff --git a/modules/nef/browser.py b/modules/nef/browser.py new file mode 100644 index 0000000000000000000000000000000000000000..0ce29206abe254217d2c9a8aa1aeea2ef73f32df --- /dev/null +++ b/modules/nef/browser.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Damien Cassou +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import datetime + +from weboob.browser import LoginBrowser, URL, need_login +from weboob.exceptions import BrowserIncorrectPassword + +from .pages import LoginPage, HomePage, AccountsPage, RecipientsPage, TransactionsPage + +def next_week_string(): + return (datetime.date.today() + datetime.timedelta(weeks=1)).strftime("%Y-%m-%d") + +class NefBrowser(LoginBrowser): + BASEURL = 'https://espace-client.lanef.com' + + home = URL('/templates/home.cfm', HomePage) + main = URL('/templates/main.cfm', HomePage) + download = URL(r'/templates/account/accountActivityListDownload.cfm\?viewMode=CSV&orderBy=TRANSACTION_DATE_DESCENDING&page=1&startDate=2016-01-01&endDate=%s&showBalance=true&AccNum=(?P.*)' % next_week_string(), TransactionsPage) + login = URL('/templates/logon/logon.cfm', LoginPage) + + def do_login(self): + self.login.stay_or_go() + + self.page.login(self.username, self.password) + + if not self.home.is_here(): + raise BrowserIncorrectPassword('Error logging in') + + @need_login + def iter_accounts_list(self): + response = self.main.open(data={ + 'templateName': 'account/accountList.cfm' + }) + + page = AccountsPage(self, response) + return page.get_items() + + @need_login + def iter_transactions_list(self, account): + return self.download.go(account_id=account.id).iter_history() + + # CapBankTransfer + @need_login + def iter_recipients_list(self): + response = self.main.open(data={ + 'templateName': 'beneficiary/beneficiaryList.cfm', + 'LISTTYPE': 'HISTORY' + }) + + page = RecipientsPage(self, response) + return page.get_items() diff --git a/modules/nef/favicon.png b/modules/nef/favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..bdf2478070ee7ccdb3fe328e40401cfaf4a5fb4a Binary files /dev/null and b/modules/nef/favicon.png differ diff --git a/modules/nef/favicon.svg b/modules/nef/favicon.svg new file mode 100644 index 0000000000000000000000000000000000000000..8de72acb735d59195fbf22d6e4aaf2ea2ddc0812 --- /dev/null +++ b/modules/nef/favicon.svg @@ -0,0 +1,86 @@ + + + + + + + + + + image/svg+xml + + + + + + + la Nef + + diff --git a/modules/nef/module.py b/modules/nef/module.py new file mode 100644 index 0000000000000000000000000000000000000000..e8b40d5725086e2500ab31fa38e0bb578b8c4792 --- /dev/null +++ b/modules/nef/module.py @@ -0,0 +1,129 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Damien Cassou +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + + +from weboob.tools.backend import Module, BackendConfig +from weboob.tools.value import ValueBackendPassword +from weboob.capabilities.bank import CapBankTransfer, Account + +from .browser import NefBrowser + + +__all__ = ['NefModule'] + + +class NefModule(Module, CapBankTransfer): + NAME = 'nef' + DESCRIPTION = 'La Nef' + MAINTAINER = 'Damien Cassou' + EMAIL = 'damien@cassou.me' + LICENSE = 'AGPLv3+' + VERSION = '1.4' + + BROWSER = NefBrowser + + CONFIG = BackendConfig(ValueBackendPassword('login', label='username', regexp='.+'), + ValueBackendPassword('password', label='Password')) + + def create_default_browser(self): + return self.create_browser(self.config['login'].get(), + self.config['password'].get()) + + # CapBank + def iter_accounts(self): + """ + Iter accounts. + + :rtype: iter[:class:`Account`] + """ + return self.browser.iter_accounts_list() + + def iter_coming(self, account): + """ + Iter coming transactions on a specific account. + + :param account: account to get coming transactions + :type account: :class:`Account` + :rtype: iter[:class:`Transaction`] + :raises: :class:`AccountNotFound` + """ + return [] + + def iter_history(self, account): + """ + Iter history of transactions on a specific account. + + :param account: account to get history + :type account: :class:`Account` + :rtype: iter[:class:`Transaction`] + :raises: :class:`AccountNotFound` + """ + return self.browser.iter_transactions_list(account) + + def iter_resources(self, objs, split_path): + """ + Iter resources. + + Default implementation of this method is to return on top-level + all accounts (by calling :func:`iter_accounts`). + + :param objs: type of objects to get + :type objs: tuple[:class:`BaseObject`] + :param split_path: path to discover + :type split_path: :class:`list` + :rtype: iter[:class:`BaseObject`] + """ + if Account in objs: + self._restrict_level(split_path) + + return self.iter_accounts() + + # CapBankTransfer + def iter_transfer_recipients(self, account): + """ + Iter recipients availables for a transfer from a specific account. + + :param account: account which initiate the transfer + :type account: :class:`Account` + :rtype: iter[:class:`Recipient`] + :raises: :class:`AccountNotFound` + """ + return self.browser.iter_recipients_list() + + def init_transfer(self, transfer, **params): + """ + Initiate a transfer. + + :param :class:`Transfer` + :rtype: :class:`Transfer` + :raises: :class:`TransferError` + """ + raise NotImplementedError() + + def execute_transfer(self, transfer, **params): + """ + Execute a transfer. + + :param :class:`Transfer` + :rtype: :class:`Transfer` + :raises: :class:`TransferError` + """ + raise NotImplementedError() diff --git a/modules/nef/pages.py b/modules/nef/pages.py new file mode 100644 index 0000000000000000000000000000000000000000..9cef0452fc5052c5f532aa7d590b03a8600bccb5 --- /dev/null +++ b/modules/nef/pages.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Damien Cassou +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.browser.elements import ListElement, DictElement, ItemElement, method, TableElement +from weboob.browser.filters.standard import CleanText, CleanDecimal, Regexp, Field, Date +from weboob.browser.pages import HTMLPage, PartialHTMLPage, CsvPage, LoggedPage +from weboob.browser.filters.json import Dict +from weboob.browser.filters.html import Attr, TableCell + +from weboob.capabilities.bank import Account, Recipient + +from weboob.tools.date import parse_french_date + +from .transaction import Transaction + +class LoginPage(HTMLPage): + def login(self, username, password): + form = self.get_form(name='formSignon') + form['userId'] = username + form['logonId'] = username + form['static'] = password + form.submit() + +class HomePage(LoggedPage, HTMLPage): + pass + +class AccountsPage(LoggedPage, PartialHTMLPage): + ACCOUNT_TYPES = {re.compile('livret'): Account.TYPE_SAVINGS, + re.compile('parts sociales'): Account.TYPE_SAVINGS, + } + + @method + class get_items(ListElement): + item_xpath = '//div[@data-type="account"]' + + class item(ItemElement): + klass = Account + + obj_id = CleanText('.//div/div/div[(position()=3) and (has-class("pc-content-text"))]/span') & Regexp(pattern=r'(\d+) ') + obj_label = CleanText('.//div/div/div[(position()=2) and (has-class("pc-content-text-wrap"))]') + obj_balance = CleanDecimal('./div[position()=3]/span', replace_dots=True) + obj_currency = u'EUR' + + def obj_type(self): + label = Field('label')(self).lower() + + for regex, account_type in self.page.ACCOUNT_TYPES.items(): + if (regex.match(label)): + return account_type + + return Account.TYPE_UNKNOWN + +class RecipientsPage(LoggedPage, PartialHTMLPage): + @method + class get_items(TableElement): + head_xpath = '//table[@id="tblBeneficiaryList"]/thead//td' + item_xpath = '//table[@id="tblBeneficiaryList"]//tr[has-class("beneficiary-data-rows")]' + + col_label = re.compile('Nom.*') + col_iban = re.compile('IBAN.*') + + class item(ItemElement): + klass = Recipient + + obj_id = Attr('.', 'beneficiaryid') + obj_label = CleanText(TableCell('label')) + obj_iban = CleanText(TableCell('iban')) + +class TransactionsPage(LoggedPage, CsvPage): + ENCODING = 'latin-1' + DIALECT = 'excel' + + # lines 1 to 5 are meta-data + # line 6 is empty + # line 7 describes the columns + HEADER = 7 + + @method + class iter_history(DictElement): + class item(ItemElement): + klass = Transaction + + # The CSV contains these columns: + # + # "Date opération","Date Valeur","Référence","Montant","Solde","Libellé" + obj_raw = Transaction.Raw(Dict(u'Libellé')) + obj_amount = CleanDecimal(Dict('Montant'), replace_dots=True) + obj_date = Date(Dict('Date opération'), parse_func=parse_french_date, dayfirst=True) + obj_vdate = Date(Dict('Date Valeur'), parse_func=parse_french_date, dayfirst=True) diff --git a/modules/nef/test.py b/modules/nef/test.py new file mode 100644 index 0000000000000000000000000000000000000000..91c7f44b9ca673ff78ac6ee9a79e1b8f85baeb81 --- /dev/null +++ b/modules/nef/test.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Damien Cassou +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +from weboob.tools.test import BackendTest +from weboob.tools.capabilities.bank.test import BankStandardTest + + +class NefTest(BankStandardTest, BackendTest): + MODULE = 'nef' diff --git a/modules/nef/transaction.py b/modules/nef/transaction.py new file mode 100644 index 0000000000000000000000000000000000000000..324bcd37fd8c0bf8ea0cec55acb4d4177995b315 --- /dev/null +++ b/modules/nef/transaction.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- + +from weboob.tools.capabilities.bank.transactions import FrenchTransaction + +import re + +class Transaction(FrenchTransaction): + PATTERNS = [ + # Money arrives on the account: + (re.compile('^VIR\. O/ .* MOTIF: ?(?P.*)$'), FrenchTransaction.TYPE_TRANSFER), + # Money leaves the account: + (re.compile('^.* VIREMENT SEPA FAVEUR (?P.*)$'), FrenchTransaction.TYPE_TRANSFER), + # Taxes + (re.compile('^TAXE SUR .*$'), FrenchTransaction.TYPE_BANK), + (re.compile(u'^Prélèvements Sociaux.*$'), FrenchTransaction.TYPE_BANK), + # Interest income + (re.compile(u'^Intérêts Créditeurs.*$'), FrenchTransaction.TYPE_BANK), + (re.compile(u'^REMISE DE CHEQUES.*$'), FrenchTransaction.TYPE_DEPOSIT), + (re.compile(u'^VIREMENT D\'ORDRE DE LA NEF.*$'), FrenchTransaction.TYPE_DEPOSIT), + (re.compile(u'^MISE A JOUR STOCK.*$'), FrenchTransaction.TYPE_ORDER) + ] diff --git a/modules/netfinca/__init__.py b/modules/netfinca/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8ee070d95dff6ff9440da677689db5893bfea006 --- /dev/null +++ b/modules/netfinca/__init__.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012-2019 Budget-Insight +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + + +from .module import NetfincaModule + + +__all__ = ['NetfincaModule'] diff --git a/modules/netfinca/browser.py b/modules/netfinca/browser.py new file mode 100644 index 0000000000000000000000000000000000000000..2429c7ee9eed6b04660b7285e7a40600f464b223 --- /dev/null +++ b/modules/netfinca/browser.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012-2019 Budget-Insight +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + + +from weboob.browser import LoginBrowser, URL +from weboob.exceptions import BrowserUnavailable + +from .pages import InvestmentsPage, AccountsPage + + +class NetfincaBrowser(LoginBrowser): + accounts = URL(r'/netfinca-titres/servlet/com.netfinca.frontcr.synthesis.HomeSynthesis', AccountsPage) + investments = URL(r'/netfinca-titres/servlet/com.netfinca.frontcr.account.WalletVal\?nump=(?P.*)', InvestmentsPage) + + def do_login(self): + raise BrowserUnavailable() + + def iter_accounts(self): + self.accounts.stay_or_go() + return self.page.get_accounts() + + def iter_investments(self, account): + self.accounts.stay_or_go() + + nump_id = self.page.get_nump_id(account) + self.investments.go(nump_id=nump_id) + + for invest in self.page.get_investments(account_currency=account.currency): + yield invest + + liquidity = self.page.get_liquidity() + if liquidity: + yield liquidity diff --git a/modules/netfinca/module.py b/modules/netfinca/module.py new file mode 100644 index 0000000000000000000000000000000000000000..615f043b64f91e0176cc11ea2ecd4ae72f588f3e --- /dev/null +++ b/modules/netfinca/module.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012-2019 Budget-Insight +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + + +from weboob.tools.backend import Module + +from .browser import NetfincaBrowser + +__all__ = ['NetfincaModule'] + + +class NetfincaModule(Module): + NAME = 'netfinca' + DESCRIPTION = 'netfinca website' + MAINTAINER = 'Martin Sicot' + EMAIL = 'martin.sicot@budget-insight.com' + LICENSE = 'LGPLv3+' + VERSION = '1.4' + + BROWSER = NetfincaBrowser diff --git a/modules/netfinca/pages.py b/modules/netfinca/pages.py new file mode 100644 index 0000000000000000000000000000000000000000..327588dd81a8aeefd8c71f2bfab551be6680edfb --- /dev/null +++ b/modules/netfinca/pages.py @@ -0,0 +1,205 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2012-2019 Budget-Insight +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re +from decimal import Decimal +import datetime + +from weboob.browser.pages import HTMLPage, LoggedPage +from weboob.browser.elements import method, ItemElement, TableElement +from weboob.browser.filters.standard import CleanText, CleanDecimal, Currency +from weboob.browser.filters.html import TableCell, Attr +from weboob.capabilities.bank import Investment, Account +from weboob.capabilities.base import NotAvailable +from weboob.tools.capabilities.bank.investments import is_isin_valid, create_french_liquidity + + +class AccountsPage(LoggedPage, HTMLPage): + + # UTF8 tag in the meta div, but that's wrong + ENCODING = 'iso-8859-1' + + @method + class get_accounts(TableElement): + + head_xpath = '//table[contains(@class,"tableau_comptes_details")]//th' + + # There is not 'tbody' balise in the table, we have to get all tr and get out thead and tfoot ones + item_xpath = '//table[contains(@class,"tableau_comptes_details")]//tr[not(ancestor::thead) and not(ancestor::tfoot)]' + + col_id = col_label = 'Comptes' + col_owner = 'Titulaire du compte' + col_balance = re.compile(r'.*Valorisation totale.*') + + class item(ItemElement): + klass = Account + + obj_owner = CleanText(TableCell('owner')) + obj_type = Account.TYPE_MARKET + + def obj_id(self): + tablecell = TableCell('id')(self)[0] + id = tablecell.xpath('./div[position()=2]') + return CleanText().filter(id) + + def obj_label(self): + tablecell = TableCell('label')(self)[0] + label = tablecell.xpath('./div[position()=1]') + return CleanText(label)(self) + + def obj_balance(self): + tablecell = TableCell('balance')(self)[0] + b = tablecell.xpath('./span[@class="intraday"]') + balance = CleanDecimal(replace_dots=True).filter(b) + return Decimal(balance) + + def obj_currency(self): + tablecell = TableCell('balance')(self)[0] + text = tablecell.xpath('./span/text()')[0] + regex = '[0-9,.]* (.*)' + currency = Currency().filter(re.search(regex, text).group(1)) + + return currency + + def get_nump_id(self, account): + # Return an element needed in the request in order to access investments details + attr = Attr('//td[contains(@id, "wallet-%s")]' % account.id, 'onclick')(self.doc) + return re.search('([0-9]+:[0-9]+)', attr).group(1) + + +class InvestmentsPage(LoggedPage, HTMLPage): + + # UTF8 tag in the meta div, but that's wrong + ENCODING = 'iso-8859-1' + + @method + class get_investments(TableElement): + + item_xpath = '//table[@id="tableValeurs"]/tbody/tr[starts-with(@id, "ContentDetPosInLine")]' + head_xpath = '//table[@id="tableValeurs"]/thead//th' + + col_label = col_code = 'Valeur / Isin' + col_quantity = ['Quantité', 'Qté'] + col_unitvalue = col_vdate = 'Cours' + col_valuation = ['Valorisation totale', 'Val. totale'] + col_unitprice = 'Prix de revient' + col_diff = '+/- Value latente' + + # Due to a bug in TableCell, column's number match with tdcell-1 + # Had to use each time in xpath to get the right cell + # @todo : Correct the TableCell class and this module + + class item(ItemElement): + klass = Investment + + obj_diff = CleanDecimal(TableCell('diff', colspan=True), replace_dots=True) + obj_unitprice = CleanDecimal(TableCell('unitprice', colspan=True), replace_dots=True) + obj_valuation = CleanDecimal(TableCell('valuation', colspan=True), replace_dots=True) + + def obj_quantity(self): + tablecell = TableCell('quantity', colspan=True)(self)[0] + return CleanDecimal(tablecell.xpath('./span'), replace_dots=True)(self) + + def obj_label(self): + tablecell = TableCell('label', colspan=True)(self)[0] + label = CleanText(tablecell.xpath('./following-sibling::td[@class=""]/div/a')[0])(self) + return label + + def obj_code(self): + # We try to get the code from div. If we didn't find code in url, + # we try to find it in the cell text + + tablecell = TableCell('label', colspan=True)(self)[0] + # url find try + url = tablecell.xpath('./following-sibling::td[position()=1]/div/a')[0].attrib['href'] + code_match = re.search(r'sico=([A-Z0-9]*)', url) + + if code_match: + if is_isin_valid(code_match.group(1)): + return code_match.group(1) + + # cell text find try + text = CleanText(tablecell.xpath('./following-sibling::td[position()=1]/div')[0])(self) + + for code in text.split(' '): + if is_isin_valid(code): + return code + return NotAvailable + + def obj_code_type(self): + if is_isin_valid(self.obj_code()): + return Investment.CODE_TYPE_ISIN + return NotAvailable + + + def obj_unitvalue(self): + currency, unitvalue = self.original_unitvalue() + + if currency == self.env['account_currency']: + return unitvalue + return NotAvailable + + def obj_original_currency(self): + currency, unitvalue = self.original_unitvalue() + + if currency != self.env['account_currency']: + return currency + + def obj_original_unitvalue(self): + currency, unitvalue = self.original_unitvalue() + + if currency != self.env['account_currency']: + return unitvalue + + def obj_vdate(self): + tablecell = TableCell('vdate', colspan=True)(self)[0] + vdate_scrapped = tablecell.xpath('./preceding-sibling::td[position()=1]//span/text()')[0] + + # Scrapped date could be a schedule time (00:00) or a date (01/01/1970) + vdate = NotAvailable + + if ':' in vdate_scrapped: + today = datetime.date.today() + h, m = [int(x) for x in vdate_scrapped.split(':')] + hour = datetime.time(hour=h, minute=m) + vdate = datetime.datetime.combine(today, hour) + + elif '/' in vdate_scrapped: + vdate = datetime.datetime.strptime(vdate_scrapped, '%d/%m/%y') + + return vdate + + # extract unitvalue and currency + def original_unitvalue(self): + tablecell = TableCell('unitvalue', colspan=True)(self)[0] + text = tablecell.xpath('./text()')[0] + + regex = '[0-9,]* (.*)' + currency = Currency().filter(re.search(regex, text).group(1)) + + return currency, CleanDecimal(replace_dots=True).filter(text) + + def get_liquidity(self): + liquidity_element = self.doc.xpath('//td[contains(text(), "Solde espèces en euros")]//following-sibling::td[position()=1]') + assert len(liquidity_element) <= 1 + if liquidity_element: + valuation = CleanDecimal(replace_dots=True).filter(liquidity_element[0]) + return create_french_liquidity(valuation) diff --git a/modules/pap/pages.py b/modules/pap/pages.py index daee4c9829d8a64ea91c9b191390929ff18fddd4..5568f3954a5f2f5357699270fdd90b68195ee2fd 100644 --- a/modules/pap/pages.py +++ b/modules/pap/pages.py @@ -62,13 +62,13 @@ def condition(self): if self.env['query_type'] == POSTS_TYPES.RENT: isNotFurnishedOk = 'meublé' not in title.lower() return ( - Regexp(Link('./div/a[@class="item-title"]'), '/annonces/(.*)', default=None)(self) and + Regexp(Link('./div/a[has-class("item-title")]'), '/annonces/(.*)', default=None)(self) and isNotFurnishedOk ) def parse(self, el): rooms_bedrooms_area = el.xpath( - './/div[@class="clearfix"]/ul[has-class("item-tags")]/li' + './div/a[has-class("item-title")]/ul[has-class("item-tags")]/li' ) self.env['rooms'] = NotLoaded self.env['bedrooms'] = NotLoaded @@ -78,10 +78,10 @@ def parse(self, el): name = CleanText('.')(item) if 'chambre' in name.lower(): name = 'bedrooms' - value = CleanDecimal('./strong')(item) + value = CleanDecimal('.')(item) elif 'pièce' in name.lower(): name = 'rooms' - value = CleanDecimal('./strong')(item) + value = CleanDecimal('.')(item) else: name = 'area' value = CleanDecimal( @@ -94,7 +94,7 @@ def parse(self, el): )(item) self.env[name] = value - obj_id = Regexp(Link('./div/a[@class="item-title"]'), '/annonces/(.*)') + obj_id = Regexp(Link('./div/a[has-class("item-title")]'), '/annonces/(.*)') obj_type = Env('query_type') obj_advert_type = ADVERT_TYPES.PERSONAL @@ -112,24 +112,15 @@ def obj_house_type(self): else: return HOUSE_TYPES.OTHER - obj_title = CleanText('./div/a[@class="item-title"]') + obj_title = CleanText('./div/a[has-class("item-title")]') obj_area = Env('area') - obj_cost = CleanDecimal(CleanText('./div/a[@class="item-title"]/span[@class="item-price"]'), + obj_cost = CleanDecimal(CleanText('./div/a[has-class("item-title")]/span[@class="item-price"]'), replace_dots=True, default=Decimal(0)) obj_currency = Currency( './div/a[@class="item-title"]/span[@class="item-price"]' ) obj_utilities = UTILITIES.UNKNOWN - def obj_date(self): - date = CleanText( - './div/p[@class="item-date"]' - )(self).split(" / ") - if len(date) > 1: - return parse_french_date(date[1].strip()) - else: - return NotLoaded - obj_station = CleanText('./div/p[@class="item-transports"]', default=NotLoaded) def obj_location(self): diff --git a/modules/pap/test.py b/modules/pap/test.py index 9dec085b99e66e5b9e20005d89f535e94c671b0a..3c0e954ccc9496fa5913cef961b12caaef77da22 100644 --- a/modules/pap/test.py +++ b/modules/pap/test.py @@ -27,13 +27,11 @@ class PapTest(BackendTest, HousingTest): FIELDS_ALL_HOUSINGS_LIST = [ "id", "type", "advert_type", "house_type", "url", "title", "area", - "cost", "currency", "utilities", "date", "location", "text" + "cost", "currency", "utilities", "location", "text" ] FIELDS_ANY_HOUSINGS_LIST = [ "photos", "station", - "rooms", - "bedrooms" ] FIELDS_ALL_SINGLE_HOUSING = [ "id", "url", "type", "advert_type", "house_type", "title", "area", @@ -44,8 +42,7 @@ class PapTest(BackendTest, HousingTest): "photos", "rooms", "bedrooms", - "station", - "DPE" + "station" ] def test_pap_rent(self): @@ -96,8 +93,7 @@ def test_pap_viager(self): self.FIELDS_ANY_SINGLE_HOUSING = [ "photos", "bedrooms", - "station", - "DPE" + "station" ] self.check_against_query(query) diff --git a/modules/societegenerale/browser.py b/modules/societegenerale/browser.py index 3bee54b647c3e3f995568076637b031eda5208ae..fa0b906dcebd809e79e0081031fa794100b2a2bb 100644 --- a/modules/societegenerale/browser.py +++ b/modules/societegenerale/browser.py @@ -19,27 +19,26 @@ from __future__ import unicode_literals -from datetime import datetime, date +from datetime import datetime from decimal import Decimal from dateutil.relativedelta import relativedelta -import re from weboob.browser import LoginBrowser, URL, need_login, StatesMixin -from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable, ActionNeeded -from weboob.capabilities.bank import Account, TransferBankError +from weboob.exceptions import BrowserIncorrectPassword, ActionNeeded, BrowserUnavailable +from weboob.capabilities.bank import Account, TransferBankError, AddRecipientStep from weboob.capabilities.base import find_object, NotAvailable -from weboob.browser.exceptions import BrowserHTTPNotFound +from weboob.browser.exceptions import BrowserHTTPNotFound, ClientError from weboob.capabilities.profile import ProfileMissing +from weboob.tools.value import Value, ValueBool from .pages.accounts_list import ( - AccountsMainPage, AccountDetailsPage, AccountsPage, LoansPage, ComingPage, HistoryPage, - CardListPage, CardHistoryPage, - AdvisorPage, HTMLProfilePage, XMLProfilePage, - MarketPage, - LifeInsurance, LifeInsuranceHistory, LifeInsuranceInvest, LifeInsuranceInvest2, + AccountsMainPage, AccountDetailsPage, AccountsPage, LoansPage, HistoryPage, + CardHistoryPage, PeaLiquidityPage, AccountsSynthesesPage, + AdvisorPage, HTMLProfilePage, CreditPage, CreditHistoryPage, + MarketPage, LifeInsurance, LifeInsuranceHistory, LifeInsuranceInvest, LifeInsuranceInvest2, UnavailableServicePage, ) -from .pages.transfer import AddRecipientPage, RecipientJson, TransferJson, SignTransferPage +from .pages.transfer import AddRecipientPage, SignRecipientPage, TransferJson, SignTransferPage from .pages.login import MainPage, LoginPage, BadLoginPage, ReinitPasswordPage, ActionNeededPage, ErrorPage from .pages.subscription import BankStatementPage @@ -52,16 +51,16 @@ class SocieteGenerale(LoginBrowser, StatesMixin): STATE_DURATION = 5 # Bank - accounts_main_page = URL('/restitution/cns_listeprestation.html', - '/com/icd-web/cbo/index.html', AccountsMainPage) - account_details_page = URL('/restitution/cns_detailPrestation.html', AccountDetailsPage) - accounts = URL('/icd/cbo/data/liste-prestations-navigation-authsec.json', AccountsPage) - coming = URL('/restitution/cns_listeEncours.xml', ComingPage) - history = URL('/icd/cbo/data/liste-operations-authsec.json', HistoryPage) - cards_list = URL('/restitution/cns_listeCartesDd.html', - '/restitution/cns_detailCARTE_\w{3}.html', CardListPage) - card_history = URL('/restitution/cns_listeReleveCarteDd.xml', CardHistoryPage) + accounts_main_page = URL(r'/restitution/cns_listeprestation.html', + r'/com/icd-web/cbo/index.html', AccountsMainPage) + account_details_page = URL(r'/restitution/cns_detailPrestation.html', AccountDetailsPage) + accounts = URL(r'/icd/cbo/data/liste-prestations-navigation-authsec.json', AccountsPage) + accounts_syntheses = URL(r'/icd/cbo/data/liste-prestations-authsec.json\?n10_avecMontant=1', AccountsSynthesesPage) + history = URL(r'/icd/cbo/data/liste-operations-authsec.json', HistoryPage) + card_history = URL(r'/restitution/cns_listeReleveCarteDd.xml', CardHistoryPage) loans = URL(r'/abm/restit/listeRestitutionPretsNET.json\?a100_isPretConso=(?P\w+)', LoansPage) + credit = URL(r'/restitution/cns_detailAVPAT.html', CreditPage) + credit_history = URL(r'/restitution/cns_listeEcrCav.xml', CreditHistoryPage) # Recipient add_recipient = URL(r'/personnalisation/per_cptBen_ajouterFrBic.html', @@ -69,7 +68,7 @@ class SocieteGenerale(LoginBrowser, StatesMixin): json_recipient = URL(r'/sec/getsigninfo.json', r'/sec/csa/send.json', r'/sec/oob_sendoob.json', - r'/sec/oob_polling.json', RecipientJson) + r'/sec/oob_polling.json', SignRecipientPage) # Transfer json_transfer = URL(r'/icd/vupri/data/vupri-liste-comptes.json\?an200_isBack=false', r'/icd/vupri/data/vupri-check.json', @@ -78,32 +77,36 @@ class SocieteGenerale(LoginBrowser, StatesMixin): confirm_transfer = URL(r'/icd/vupri/data/vupri-save.json', TransferJson) # Wealth - market = URL('/brs/cct/comti20.html', MarketPage) - life_insurance = URL('/asv/asvcns10.html', '/asv/AVI/asvcns10a.html', '/brs/fisc/fisca10a.html', LifeInsurance) - life_insurance_invest = URL('/asv/AVI/asvcns20a.html', LifeInsuranceInvest) - life_insurance_invest_2 = URL('/asv/PRV/asvcns10priv.html', LifeInsuranceInvest2) - life_insurance_history = URL('/asv/AVI/asvcns2(?P[0-9])c.html', LifeInsuranceHistory) + market = URL(r'/brs/cct/comti20.html', MarketPage) + pea_liquidity = URL(r'/restitution/cns_detailPea.html', PeaLiquidityPage) + life_insurance = URL(r'/asv/asvcns10.html', + r'/asv/AVI/asvcns10a.html', + r'/brs/fisc/fisca10a.html', LifeInsurance) + life_insurance_invest = URL(r'/asv/AVI/asvcns20a.html', LifeInsuranceInvest) + life_insurance_invest_2 = URL(r'/asv/PRV/asvcns10priv.html', LifeInsuranceInvest2) + life_insurance_history = URL(r'/asv/AVI/asvcns2(?P[0-9])c.html', LifeInsuranceHistory) # Profile - advisor = URL('/icd/pon/data/get-contacts.xml', AdvisorPage) + advisor = URL(r'/icd/pon/data/get-contacts.xml', AdvisorPage) html_profile_page = URL(r'/com/dcr-web/dcr/dcr-coordonnees.html', HTMLProfilePage) - xml_profile_page = URL(r'/gms/gmsRestituerAdresseNotificationServlet.xml', XMLProfilePage) # Document bank_statement = URL(r'/restitution/rce_derniers_releves.html', BankStatementPage) bank_statement_search = URL(r'/restitution/rce_recherche.html\?noRedirect=1', r'/restitution/rce_recherche_resultat.html', BankStatementPage) - bad_login = URL('\/acces/authlgn.html', '/error403.html', BadLoginPage) - reinit = URL('/acces/changecodeobligatoire.html', ReinitPasswordPage) - action_needed = URL('/com/icd-web/forms/cct-index.html', - '/com/icd-web/gdpr/gdpr-recueil-consentements.html', - '/com/icd-web/forms/kyc-index.html', + bad_login = URL(r'/acces/authlgn.html', r'/error403.html', BadLoginPage) + reinit = URL(r'/acces/changecodeobligatoire.html', + r'/swm/swm-changemdpobligatoire.html', ReinitPasswordPage) + action_needed = URL(r'/com/icd-web/forms/cct-index.html', + r'/com/icd-web/gdpr/gdpr-recueil-consentements.html', + r'/com/icd-web/forms/kyc-index.html', ActionNeededPage) - unavailable_service_page = URL(r'/com/service-indisponible.html', UnavailableServicePage) - error = URL('https://static.societegenerale.fr/pri/erreur.html', ErrorPage) - login = URL('/sec/vk', LoginPage) - main_page = URL('https://particuliers.societegenerale.fr', MainPage) + unavailable_service_page = URL(r'/com/service-indisponible.html', + r'.*/Technical-pages/503-error-page/unavailable.html', UnavailableServicePage) + error = URL(r'https://static.societegenerale.fr/pri/erreur.html', ErrorPage) + login = URL(r'/sec/vk', LoginPage) + main_page = URL(r'https://particuliers.societegenerale.fr', MainPage) context = None dup = None @@ -111,6 +114,9 @@ class SocieteGenerale(LoginBrowser, StatesMixin): __states__ = ('context', 'dup', 'id_transaction') + def locate_browser(self, state): + self.location('/com/icd-web/cbo/index.html') + def load_state(self, state): if state.get('dup') is not None and state.get('context') is not None: super(SocieteGenerale, self).load_state(state) @@ -132,6 +138,12 @@ def do_login(self): reason, action = self.page.get_error() if reason == 'echec_authent': raise BrowserIncorrectPassword() + elif reason in ('acces_bloq', 'acces_susp', 'pas_acces_bad', ): + raise ActionNeeded() + elif reason == 'err_tech': + # there is message "Service momentanément indisponible. Veuillez réessayer." + # in SG website in that case ... + raise BrowserUnavailable() def iter_cards(self, account): for el in account._cards: @@ -149,17 +161,24 @@ def iter_cards(self, account): @need_login def get_accounts_list(self): self.accounts_main_page.go() + self.page.is_accounts() if self.page.is_old_website(): # go on new_website self.location(self.absurl('/com/icd-web/cbo/index.html')) # get account iban on transfer page - self.json_transfer.go() - account_ibans = self.page.get_account_ibans_dict() + account_ibans = {} + try: + self.json_transfer.go() + except (TransferBankError, ClientError, BrowserUnavailable): + # some user can't access this page + pass + else: + account_ibans = self.page.get_account_ibans_dict() - # get account coming on coming page, coming amount is not available yet on account page - self.coming.go() + # get accounts coming + self.accounts_syntheses.go() account_comings = self.page.get_account_comings() self.accounts.go() @@ -174,96 +193,105 @@ def get_accounts_list(self): if account._prestation_id in account_comings: account.coming = account_comings[account._prestation_id] - if account.type == account.TYPE_LOAN: - self.loans.go(conso=(account._loan_type == 'PR_CONSO')) + if account.type in (account.TYPE_LOAN, account.TYPE_CONSUMER_CREDIT, ): + self.loans.stay_or_go(conso=(account._loan_type == 'PR_CONSO')) account = self.page.get_loan_account(account) - yield account + if account.type == account.TYPE_REVOLVING_CREDIT: + self.loans.stay_or_go(conso=(account._loan_type == 'PR_CONSO')) + account = self.page.get_revolving_account(account) - @need_login - def iter_card_transaction(self, account): - # TODO - raise BrowserUnavailable() - self.account_details_page.go(params={'idprest': account._prestation_id}) - self.location(self.absurl(self.page.get_card_history_link(account))) - - if self.page.get_card_transactions_link(): - self.location(self.absurl(self.page.get_card_transactions_link())) - - year = date.today().year - rdate = None - for tr in self.page.iter_card_history(): - - # search for the first card summary - if tr.date is NotAvailable: - tr.type = tr.TYPE_CARD_SUMMARY - d = re.search(r'(\d{2})\/(\d{2})', tr.label) - if d: - dd, mm = int(d.group(1)), int(d.group(2)) - tr.date = rdate = date(year, mm, dd) - year = tr.date.year - - # if card summary is found, yield transaction with the right rdate - if rdate: - tr.rdate = rdate - yield tr + yield account @need_login def iter_history(self, account): - # TODO: check matching - raise BrowserUnavailable() + if account.type in (account.TYPE_LOAN, account.TYPE_MARKET, account.TYPE_CONSUMER_CREDIT, ): + return - if account.type in (account.TYPE_LOAN, account.TYPE_MARKET, ): + if account.type == Account.TYPE_PEA and not ('Espèces' in account.label or 'ESPECE' in account.label): return - if account.type == account.TYPE_CARD: - for tr in self.iter_card_transaction(account): + if account.type in (account.TYPE_LIFE_INSURANCE, account.TYPE_PERP, ): + # request to get json is not available yet, old request to get html response + self.account_details_page.go(params={'idprest': account._prestation_id}) + link = self.page.get_history_link() + if link: + self.location(self.absurl(link)) + for tr in self.page.iter_li_history(): + yield tr + return + + if account.type == account.TYPE_REVOLVING_CREDIT and account._loan_type != 'PR_CONSO': + # request to get json is not available yet, old request to get html response + self.account_details_page.go(params={'idprest': account._prestation_id}) + self.page.go_history_page() + for tr in self.page.iter_credit_history(): yield tr return - self.history.go(params={'b64e200_prestationIdTechnique': account._internal_id}) + if account.type == account.TYPE_REVOLVING_CREDIT and not account._is_json_histo: + # Waiting for account with transactions + return + + if account.type == account.TYPE_CARD: + self.history.go(params={'b64e200_prestationIdTechnique': account.parent._internal_id}) + for summary_card_tr in self.page.iter_card_transactions(card_number=account.number): + yield summary_card_tr - iter_transactions = self.page.iter_history - if account.type == Account.TYPE_PEA: - iter_transactions = self.page.iter_pea_history + for card_tr in summary_card_tr._card_transactions: + card_tr.date = summary_card_tr.date + yield card_tr + return - for transaction in iter_transactions(): + self.history.go(params={'b64e200_prestationIdTechnique': account._internal_id}) + for transaction in self.page.iter_history(): yield transaction @need_login def iter_coming(self, account): - # TODO: check matching - raise BrowserUnavailable() - if account.type in (account.TYPE_LOAN, account.TYPE_MARKET ): + if account.type in (account.TYPE_LOAN, account.TYPE_MARKET, account.TYPE_PEA, + account.TYPE_LIFE_INSURANCE, account.TYPE_REVOLVING_CREDIT, + account.TYPE_CONSUMER_CREDIT, Account.TYPE_PERP, ): return + internal_id = account._internal_id if account.type == account.TYPE_CARD: - # TODO - return + internal_id = account.parent._internal_id + self.history.go(params={'b64e200_prestationIdTechnique': internal_id}) - self.history.go(params={'b64e200_prestationIdTechnique': account._internal_id}) - for transaction in self.page.iter_coming(): - yield transaction + if account.type == account.TYPE_CARD: + for transaction in self.page.iter_future_transactions(acc_prestation_id=account._prestation_id): + # coming transactions on this page are not in coming balance + # except for defered card coming transaction, use only for it for the moment + if transaction._card_coming: + for card_coming in transaction._card_coming: + card_coming.date = transaction.date + yield card_coming + return + + for intraday_tr in self.page.iter_intraday_comings(): + yield intraday_tr @need_login def iter_investment(self, account): - # TODO - raise BrowserUnavailable() - if account.type not in (Account.TYPE_MARKET, Account.TYPE_LIFE_INSURANCE, Account.TYPE_PEA): + if account.type not in (Account.TYPE_MARKET, Account.TYPE_LIFE_INSURANCE, + Account.TYPE_PEA, Account.TYPE_PERP, ): self.logger.debug('This account is not supported') - return [] + return - raise BrowserUnavailable() + # request to get json is not available yet, old request to get html response + self.account_details_page.go(params={'idprest': account._prestation_id}) if account.type in (Account.TYPE_PEA, Account.TYPE_MARKET): - self.account_details_page.go(params={'idprest': account._prestation_id}) - for invest in self.page.iter_investment(): - # TODO - pass + for invest in self.page.iter_investments(account=account): + yield invest - if account.type == Account.TYPE_LIFE_INSURANCE: - # TODO - pass + if account.type in (Account.TYPE_LIFE_INSURANCE, Account.TYPE_PERP, ): + if self.page.has_link(): + self.life_insurance_invest.go() + + for invest in self.page.iter_investment(): + yield invest @need_login def iter_recipients(self, account): @@ -304,17 +332,26 @@ def execute_transfer(self, transfer): return transfer def end_sms_recipient(self, recipient, **params): - data = [('context', self.context), ('context', self.context), ('dup', self.dup), ('code', params['code']), ('csa_op', 'sign')] - self.add_recipient.go(data=data, headers={'Referer': 'https://particuliers.secure.societegenerale.fr/lgn/url.html'}) + """End adding recipient with OTP SMS authentication""" + data = [ + ('context', [self.context, self.context]), + ('dup', self.dup), + ('code', params['code']), + ('csa_op', 'sign') + ] + # needed to confirm recipient validation + add_recipient_url = self.absurl('/lgn/url.html', base=True) + self.location(add_recipient_url, data=data, headers={'Referer': add_recipient_url}) return self.page.get_recipient_object(recipient) def end_oob_recipient(self, recipient, **params): - r = self.open('https://particuliers.secure.societegenerale.fr/sec/oob_polling.json', data={'n10_id_transaction': self.id_transaction}) - assert r.page.doc['donnees']['transaction_status'] in ('available', 'in_progress'), \ - 'transaction_status is %s' % r.page.doc['donnees']['transaction_status'] - - if r.page.doc['donnees']['transaction_status'] == 'in_progress': - raise ActionNeeded('Veuillez valider le bénéficiaire sur votre application bancaire.') + """End adding recipient with 'pass sécurité' authentication""" + r = self.open( + self.absurl('/sec/oob_polling.json'), + data={'n10_id_transaction': self.id_transaction} + ) + assert self.id_transaction, "Transaction id is missing, can't sign new recipient." + r.page.check_recipient_status() data = [ ('context', self.context), @@ -323,24 +360,51 @@ def end_oob_recipient(self, recipient, **params): ('n10_id_transaction', self.id_transaction), ('oob_op', 'sign') ] + # needed to confirm recipient validation add_recipient_url = self.absurl('/lgn/url.html', base=True) - self.location( - add_recipient_url, - data=data, - headers={'Referer': add_recipient_url} - ) + self.location(add_recipient_url, data=data, headers={'Referer': add_recipient_url}) return self.page.get_recipient_object(recipient) + def send_sms_to_user(self, recipient): + """Add recipient with OTP SMS authentication""" + data = {} + data['csa_op'] = 'sign' + data['context'] = self.context + self.open(self.absurl('/sec/csa/send.json'), data=data) + raise AddRecipientStep(recipient, Value('code', label='Cette opération doit être validée par un Code Sécurité.')) + + def send_notif_to_user(self, recipient): + """Add recipient with 'pass sécurité' authentication""" + data = {} + data['b64_jeton_transaction'] = self.context + r = self.open(self.absurl('/sec/oob_sendoob.json'), data=data) + self.id_transaction = r.page.get_transaction_id() + raise AddRecipientStep(recipient, ValueBool('pass', label='Valider cette opération sur votre applicaton société générale')) + @need_login def new_recipient(self, recipient, **params): if 'code' in params: return self.end_sms_recipient(recipient, **params) if 'pass' in params: return self.end_oob_recipient(recipient, **params) + self.add_recipient.go() self.page.post_iban(recipient) self.page.post_label(recipient) - self.page.double_auth(recipient) + + recipient = self.page.get_recipient_object(recipient, get_info=True) + self.page.update_browser_recipient_state() + data = self.page.get_signinfo_data() + + r = self.open(self.absurl('/sec/getsigninfo.json'), data=data) + sign_method = r.page.get_sign_method() + + # WARNING: this send validation request to user + if sign_method == 'CSA': + return self.send_sms_to_user(recipient) + elif sign_method == 'OOB': + return self.send_notif_to_user(recipient) + assert False, 'Sign process unknown: %s' % sign_method @need_login def get_advisor(self): @@ -349,13 +413,11 @@ def get_advisor(self): @need_login def get_profile(self): self.html_profile_page.go() - profile = self.page.get_profile() - self.xml_profile_page.go() - profile.email = self.page.get_email() - return profile + return self.page.get_profile() @need_login def iter_subscription(self): + self.accounts_main_page.go() try: profile = self.get_profile() subscriber = profile.name diff --git a/modules/societegenerale/module.py b/modules/societegenerale/module.py index 992e3a3e129d6697747295e92d427a6e821cefb7..1619fe185682e9fef190f88e8156f84f869ffb92 100644 --- a/modules/societegenerale/module.py +++ b/modules/societegenerale/module.py @@ -73,18 +73,11 @@ def get_account(self, _id): def iter_coming(self, account): if hasattr(self.browser, 'get_cb_operations'): transactions = list(self.browser.get_cb_operations(account)) - else: - transactions = [tr for tr in self.browser.iter_history(account) if tr._coming] - transactions = sorted_transactions(transactions) - return transactions + return sorted_transactions(transactions) + return self.browser.iter_coming(account) def iter_history(self, account): - if hasattr(self.browser, 'get_cb_operations'): - transactions = list(self.browser.iter_history(account)) - else: - transactions = [tr for tr in self.browser.iter_history(account) if not tr._coming] - transactions = sorted_transactions(transactions) - return transactions + return self.browser.iter_history(account) def iter_investment(self, account): return self.browser.iter_investment(account) @@ -107,7 +100,7 @@ def iter_transfer_recipients(self, origin_account): return self.browser.iter_recipients(origin_account) def new_recipient(self, recipient, **params): - if self.config['website'].get() not in ('pro', ): + if self.config['website'].get() not in ('par', 'pro'): raise NotImplementedError() recipient.label = ' '.join(w for w in re.sub('[^0-9a-zA-Z:\/\-\?\(\)\.,\'\+ ]+', '', recipient.label).split()) return self.browser.new_recipient(recipient, **params) @@ -122,9 +115,9 @@ def init_transfer(self, transfer, **params): if not account: account = strict_find_object(self.iter_accounts(), id=transfer.account_id, error=AccountNotFound) - recipient = strict_find_object(self.iter_transfer_recipients(account.id), iban=transfer.recipient_iban) + recipient = strict_find_object(self.iter_transfer_recipients(account.id), id=transfer.recipient_id) if not recipient: - recipient = strict_find_object(self.iter_transfer_recipients(account.id), id=transfer.recipient_id, error=RecipientNotFound) + recipient = strict_find_object(self.iter_transfer_recipients(account.id), iban=transfer.recipient_iban, error=RecipientNotFound) transfer.amount = transfer.amount.quantize(Decimal('.01')) return self.browser.init_transfer(account, recipient, transfer) diff --git a/modules/societegenerale/pages/accounts_list.py b/modules/societegenerale/pages/accounts_list.py index f1010977e2c8f22666708ff41d7659cc09635c0a..3eef5ff816110693403a5a7b17fd8fd9925b8e63 100644 --- a/modules/societegenerale/pages/accounts_list.py +++ b/modules/societegenerale/pages/accounts_list.py @@ -19,6 +19,7 @@ from __future__ import unicode_literals +import requests import datetime import re @@ -27,18 +28,15 @@ from weboob.capabilities.contact import Advisor from weboob.capabilities.profile import Person, ProfileMissing from weboob.tools.capabilities.bank.transactions import FrenchTransaction -from weboob.tools.capabilities.bank.investments import is_isin_valid -from weboob.tools.compat import urlparse, parse_qsl, urlunparse, urlencode, unicode +from weboob.tools.capabilities.bank.investments import is_isin_valid, create_french_liquidity from weboob.browser.elements import DictElement, ItemElement, TableElement, method, ListElement from weboob.browser.filters.json import Dict from weboob.browser.filters.standard import ( - CleanText, CleanDecimal, Regexp, RegexpError, Currency, Eval, Field, Format, Date, + CleanText, CleanDecimal, Regexp, Currency, Eval, Field, Format, Date, Env, ) from weboob.browser.filters.html import Link, TableCell from weboob.browser.pages import HTMLPage, XMLPage, JsonPage, LoggedPage, pagination -from weboob.exceptions import BrowserUnavailable, ActionNeeded - -from .base import BasePage +from weboob.exceptions import BrowserUnavailable, ActionNeeded, NoAccountsException def MyDecimal(*args, **kwargs): @@ -46,16 +44,45 @@ def MyDecimal(*args, **kwargs): return CleanDecimal(*args, **kwargs) +def eval_decimal_amount(value, decimal_position): + return Eval(lambda x,y: x / 10**y, + CleanDecimal(Dict(value)), + CleanDecimal(Dict(decimal_position))) + + +class JsonBasePage(LoggedPage, JsonPage): + def on_load(self): + if Dict('commun/statut')(self.doc).upper() == 'NOK': + reason = Dict('commun/raison')(self.doc) + action = Dict('commun/action')(self.doc) + + if action and 'BLOCAGE' in action: + raise ActionNeeded() + + if 'le service est momentanement indisponible' in reason: + # TODO: iter account on old website + # can't access new website + raise BrowserUnavailable() + + assert 'pas encore géré' in reason, 'Error %s is not handled yet' % reason + self.browser.logger.warning('This page is not handled yet by SG') + + class AccountsMainPage(LoggedPage, HTMLPage): def is_old_website(self): return Link('//a[contains(text(), "Afficher la nouvelle consultation")]', default=None)(self.doc) + def is_accounts(self): + error_msg = CleanText('//span[@class="error_msg"]')(self.doc) + if 'Vous ne disposez pas de compte consultable' in error_msg: + raise NoAccountsException(error_msg) + class AccountDetailsPage(LoggedPage, HTMLPage): pass -class AccountsPage(LoggedPage, JsonPage): +class AccountsPage(JsonBasePage): @method class iter_accounts(DictElement): item_xpath = 'donnees' @@ -67,21 +94,36 @@ class item(ItemElement): TYPES = { 'COMPTE_COURANT': Account.TYPE_CHECKING, 'PEL': Account.TYPE_SAVINGS, + 'CEL': Account.TYPE_SAVINGS, 'LDD': Account.TYPE_SAVINGS, 'LIVRETA': Account.TYPE_SAVINGS, 'LIVRET_JEUNE': Account.TYPE_SAVINGS, + 'LIVRET_EUROKID': Account.TYPE_SAVINGS, 'COMPTE_SUR_LIVRET': Account.TYPE_SAVINGS, + 'LIVRET_EPARGNE_PLUS': Account.TYPE_SAVINGS, + 'PLAN_EPARGNE_BANCAIRE': Account.TYPE_SAVINGS, + 'LIVRET_EPARGNE_POPULAIRE': Account.TYPE_SAVINGS, 'BANQUE_FRANCAISE_MUTUALISEE': Account.TYPE_SAVINGS, 'PRET_GENERAL': Account.TYPE_LOAN, 'PRET_PERSONNEL_MUTUALISE': Account.TYPE_LOAN, + 'DECLIC_TEMPO': Account.TYPE_MARKET, 'COMPTE_TITRE_GENERAL': Account.TYPE_MARKET, 'PEA_ESPECES': Account.TYPE_PEA, 'PEA_PME_ESPECES': Account.TYPE_PEA, 'COMPTE_TITRE_PEA': Account.TYPE_PEA, 'COMPTE_TITRE_PEA_PME': Account.TYPE_PEA, + 'PROJECTIS': Account.TYPE_LIFE_INSURANCE, 'VIE_FEDER': Account.TYPE_LIFE_INSURANCE, + 'PALISSANDRE': Account.TYPE_LIFE_INSURANCE, 'ASSURANCE_VIE_GENERALE': Account.TYPE_LIFE_INSURANCE, + 'ASSURANCE_VIE_SOGECAP_GENERAL': Account.TYPE_LIFE_INSURANCE, + 'RESERVEA': Account.TYPE_REVOLVING_CREDIT, + 'COMPTE_ALTERNA': Account.TYPE_REVOLVING_CREDIT, + 'CREDIT_CONFIANCE': Account.TYPE_REVOLVING_CREDIT, 'AVANCE_PATRIMOINE': Account.TYPE_REVOLVING_CREDIT, + 'PRET_EXPRESSO': Account.TYPE_CONSUMER_CREDIT, + 'PRET_EVOLUTIF': Account.TYPE_CONSUMER_CREDIT, + 'PERP_EPICEA': Account.TYPE_PERP, } obj_id = obj_number = CleanText(Dict('numeroCompteFormate'), replace=[(' ', '')]) @@ -99,17 +141,28 @@ def obj_type(self): obj__prestation_id = Dict('id') def obj__loan_type(self): - if Field('type')(self) == Account.TYPE_LOAN: + if Field('type')(self) in (Account.TYPE_LOAN, Account.TYPE_CONSUMER_CREDIT, + Account.TYPE_REVOLVING_CREDIT, ): return Dict('codeFamille')(self) return None + def obj__is_json_histo(self): + # For TYPE_REVOLVING_CREDIT, to get transaction + if Field('type')(self) == Account.TYPE_REVOLVING_CREDIT and \ + not Dict('produit')(self) in ('COMPTE_ALTERNA', 'AVANCE_PATRIMOINE'): + return True -class LoansPage(LoggedPage, JsonPage): - def on_load(self): - if 'action' in self.doc['commun'] and self.doc['commun']['action'] == 'BLOCAGE': - raise ActionNeeded() - assert self.doc['commun']['statut'] != 'nok' +class AccountsSynthesesPage(JsonBasePage): + def get_account_comings(self): + account_comings = {} + + for product in Dict('donnees/syntheseParGroupeProduit')(self.doc): + for prestation in Dict('prestations')(product): + account_comings[Dict('id')(prestation)] = CleanDecimal(Dict('soldes/soldeEnCours'))(prestation) + return account_comings + +class LoansPage(JsonBasePage): def get_loan_account(self, account): assert account._prestation_id in Dict('donnees/tabIdAllPrestations')(self.doc), \ 'Loan with prestation id %s should be on this page ...' % account._prestation_id @@ -133,8 +186,63 @@ def get_loan_account(self, account): loan._internal_id = account._internal_id loan._prestation_id = account._prestation_id + loan._loan_type = account._loan_type return loan + def get_revolving_account(self, account): + loan = Loan() + loan.id = loan.number = account.id + loan.label = account.label + loan.type = account.type + + loan.currency = account.currency + loan.balance = account.balance + loan.coming = account.coming + + loan._internal_id = account._internal_id + loan._prestation_id = account._prestation_id + loan._loan_type = account._loan_type + loan._is_json_histo = account._is_json_histo + + if Dict('donnees/tabIdAllPrestations')(self.doc): + for acc in Dict('donnees/tabPrestations')(self.doc): + if CleanText(Dict('idPrestation'))(acc) == account._prestation_id: + + # coming + if Dict('encoursFinMois', default=NotAvailable)(acc): + loan.coming = eval_decimal_amount('encoursFinMois/valeur', 'encoursFinMois/posDecimale')(acc) + + # total amount + if Dict('reserveAutorisee', default=NotAvailable)(acc): + loan.total_amount = eval_decimal_amount('reserveAutorisee/valeur', 'reserveAutorisee/posDecimale')(acc) + elif Dict('montantAutorise', default=NotAvailable)(acc): + loan.total_amount = eval_decimal_amount('montantAutorise/valeur', 'montantAutorise/posDecimale')(acc) + else: + loan.total_amount = eval_decimal_amount('reserveMaximum/valeur', 'reserveMaximum/posDecimale')(acc) + + # available amount + if Dict('montantDisponible', default=NotAvailable)(acc): + loan.available_amount = eval_decimal_amount('montantDisponible/valeur', 'montantDisponible/posDecimale')(acc) + else: + loan.available_amount = eval_decimal_amount('reserveDispo/valeur', 'reserveDispo/posDecimale')(acc) + + # used amount + if Dict('reserveUtilisee', default=NotAvailable)(acc): + loan.used_amount = eval_decimal_amount('reserveUtilisee/valeur', 'reserveUtilisee/posDecimale')(acc) + elif Dict('montantUtilise', default=NotAvailable)(acc): + loan.available_amount = eval_decimal_amount('montantUtilise/valeur', 'montantUtilise/posDecimale')(acc) + + # next payment amount + if Dict('prochaineEcheance', default=NotAvailable)(acc): + loan.next_payment_amount = eval_decimal_amount('prochaineEcheance/valeur', 'prochaineEcheance/posDecimale')(acc) + elif Dict('montantMensualite', default=NotAvailable)(acc): + loan.next_payment_amount = eval_decimal_amount('montantMensualite/valeur', 'montantMensualite/posDecimale')(acc) + loan.last_payment_amount = loan.next_payment_amount + + loan.duration = Dict('dureeNbMois')(acc) + return loan + return loan + class Transaction(FrenchTransaction): PATTERNS = [(re.compile(r'^CARTE \w+ RETRAIT DAB.*? (?P
\d{2})\/(?P\d{2})( (?P\d+)H(?P\d+))? (?P.*)'), @@ -167,97 +275,147 @@ class Transaction(FrenchTransaction): FrenchTransaction.TYPE_CARD_SUMMARY), (re.compile(r'^CREDIT MENSUEL CARTE (?P.*)'), FrenchTransaction.TYPE_CARD_SUMMARY), + (re.compile(r'^CARTE \w+ (?P
\d{2})\/(?P\d{2}) (?P.*)'), + FrenchTransaction.TYPE_CARD), ] -class HistoryPage(LoggedPage, JsonPage): +class TransactionItemElement(ItemElement): + klass = Transaction + + def obj_id(self): + # real transaction id is like: + # /DDMMYYYY/ + if not Dict('idOpe')(self) or Regexp(CleanText(Dict('idOpe')), r'^(\d+)$', default=NotAvailable)(self): + return '' + id_op = Regexp(CleanText(Dict('idOpe')), r'(\d+)/')(self) + if id_op != '0': + # card summary has transaction id '0' + return id_op + + def obj_vdate(self): + if Dict('dateChargement')(self): + return Eval(lambda t: datetime.date.fromtimestamp(int(t)/1000),Dict('dateChargement'))(self) + + obj_date = Eval(lambda t: datetime.date.fromtimestamp(int(t)/1000), Dict('dateOpe')) + obj_amount = CleanDecimal(Dict('mnt')) + obj_raw = Transaction.Raw(Dict('libOpe')) + + +class HistoryPage(JsonBasePage): + def hist_pagination(self, condition): + all_conditions = { + 'history': ( + not Dict('donnees/listeOperations')(self.doc), + not Dict('donnees/recapitulatifCompte/chargerPlusOperations')(self.doc) + ), + 'future': ( + not Dict('donnees/listeOperationsFutures')(self.doc), + not Dict('donnees/recapitulatifCompte/chargerPlusOperations')(self.doc) + ), + 'intraday': ( + not Dict('donnees/listeOperations')(self.doc), + Dict('donnees/listeOperations')(self.doc) and \ + not Dict('donnees/listeOperations/0/statutOperation')(self.doc) == 'INTRADAY', + not Dict('donnees/recapitulatifCompte/chargerPlusOperations')(self.doc), + not Dict('donnees/recapitulatifCompte/encours')(self.doc), + ), + } + + if any(all_conditions[condition]): + return + + if '&an200_operationsSupplementaires=true' in self.browser.url: + return self.browser.url + return self.browser.url + '&an200_operationsSupplementaires=true' + @pagination @method class iter_history(DictElement): def next_page(self): - conditions = ( - not Dict('donnees/listeOperations')(self), - not Dict('donnees/recapitulatifCompte/chargerPlusOperations')(self) - ) - - if any(conditions): - return - - if '&an200_operationsSupplementaires=true' in self.page.url: - return self.page.url - return self.page.url + '&an200_operationsSupplementaires=true' + return self.page.hist_pagination('history') item_xpath = 'donnees/listeOperations' - class item(ItemElement): + class item(TransactionItemElement): def condition(self): return Dict('statutOperation')(self) == 'COMPTABILISE' - klass = Transaction + @pagination + @method + class iter_card_transactions(DictElement): + def next_page(self): + return self.page.hist_pagination('history') - # not 'idOpe' means that it's a comming transaction - def obj_id(self): - if Dict('idOpe')(self): - return Regexp(CleanText(Dict('idOpe')), r'(\d+)/')(self) + item_xpath = 'donnees/listeOperations' - def obj_rdate(self): - if Dict('dateChargement')(self): - return Eval(lambda t: datetime.date.fromtimestamp(int(t)/1000),Dict('dateChargement'))(self) + class item(TransactionItemElement): + def condition(self): + # card summary transaction id is like: + # 0/DDMMYYYY/ + conditions = ( + Dict('idOpe')(self) and \ + Regexp(CleanText(Dict('idOpe')), r'(\d+)/', default=NotAvailable)(self) == '0', + Env('card_number')(self) in Dict('libOpe')(self), + Dict('statutOperation')(self) == 'COMPTABILISE', + ) + return all(conditions) + + obj_type = Transaction.TYPE_CARD_SUMMARY - obj_date = Eval(lambda t: datetime.date.fromtimestamp(int(t)/1000), Dict('dateOpe')) - obj_amount = CleanDecimal(Dict('mnt')) - obj_raw = Dict('libOpe') + def obj_amount(self): + return abs(CleanDecimal(Dict('mnt'))(self)) + class obj__card_transactions(DictElement): + item_xpath = 'listeOpeFilles' - @method - class iter_pea_history(DictElement): - item_xpath = 'donnees/listeOperations' + class tr_item(TransactionItemElement): + def condition(self): + return Dict('statutOperation')(self) == 'COMPTABILISE' - class item(ItemElement): - klass = Transaction + @pagination + @method + class iter_intraday_comings(DictElement): + def next_page(self): + return self.page.hist_pagination('intraday') - obj_date = Eval(lambda t: datetime.date.fromtimestamp(int(t)/1000), Dict('dateOpe')) - obj_amount = CleanDecimal(Dict('mnt')) - obj_raw = Dict('libOpe') + item_xpath = 'donnees/listeOperations' + class item(TransactionItemElement): + def condition(self): + return Dict('statutOperation')(self) == 'INTRADAY' + @pagination @method - class iter_coming(DictElement): - item_xpath = 'donnees/listeOperations' + class iter_future_transactions(DictElement): + def next_page(self): + return self.page.hist_pagination('future') + + item_xpath = 'donnees/listeOperationsFutures' class item(ItemElement): def condition(self): - return Dict('statutOperation')(self) != 'COMPTABILISE' and not Dict('idOpe')(self) + conditions = ( + Dict('operationCategorisable')(self) in ('FUTURE', 'OPERATION_MERE'), + Dict('prestationIdAssocie')(self) == Env('acc_prestation_id')(self) + ) + return all(conditions) klass = Transaction - obj_rdate = Eval(lambda t: datetime.date.fromtimestamp(int(t)/1000), Dict('dateOpe')) - # there is no 'dateChargement' for coming transaction - obj_date = Eval(lambda t: datetime.date.fromtimestamp(int(t)/1000), Dict('dateOpe')) - obj_amount = CleanDecimal(Dict('mnt')) - obj_raw = Dict('libOpe') + obj_date = Date(Dict('dateEcheance')) + obj_amount = CleanDecimal(Dict('montant/value')) + obj_raw = obj_label = Dict('libelleAAfficher') + class obj__card_coming(DictElement): + item_xpath = 'operationsFilles' -class ComingPage(LoggedPage, XMLPage): - def get_account_comings(self): - account_comings = {} - for el in self.doc.xpath('//EnCours'): - prestation_id = CleanText('./@id')(el).replace('montantEncours', '') - coming_amount = MyDecimal(Regexp(CleanText('.'), r'(.*) '))(el) - account_comings[prestation_id] = coming_amount - return account_comings + class tr_item(ItemElement): + klass = Transaction - -class CardListPage(LoggedPage, HTMLPage): - def get_card_history_link(self, account): - for el in self.doc.xpath('//a[contains(@href, "detailCARTE")]'): - if CleanText('.', replace=[(' ', '')])(el) == account.number: - return Link('.')(el) - - def get_card_transactions_link(self): - if 'Le détail de cette carte ne vous est pas accessible' in CleanText('//div')(self.doc): - return NotAvailable - return CleanText('//div[@id="operationsListView"]//select/option[@selected="selected"]/@value')(self.doc) + obj_amount = CleanDecimal(Dict('montant/value')) + obj_date = obj_vdate = Date(Dict('dateEcheance')) + obj_raw = Transaction.Raw(Dict('libelleOrigine')) class CardHistoryPage(LoggedPage, HTMLPage): @@ -288,320 +446,250 @@ def obj_raw(self): return NotAvailable -class MarketPage(LoggedPage, HTMLPage): - # TODO - pass +class CreditPage(LoggedPage, HTMLPage): + def go_history_page(self): + redirection_script = CleanText('//script[contains(text(), "setPrestationURL")]')(self.doc) + history_link = re.search(r'setPrestationURL\("(.*)"\)', redirection_script) + if history_link: + self.browser.location(self.browser.absurl(history_link.group(1))) -class AdvisorPage(LoggedPage, XMLPage): - ENCODING = 'ISO-8859-15' +class CreditHistoryPage(LoggedPage, HTMLPage): + def build_doc(self, content): + # for some reason, lxml discards the first tag inside the CDATA + # (of course, there shouldn't be XML inside the CDATA in the first place) + content = content.replace(b'') + return super(CreditHistoryPage, self).build_doc(content) - def get_advisor(self): - advisor = Advisor() - advisor.name = Format('%s %s', CleanText('//NomConseiller'), CleanText('//PrenomConseiller'))(self.doc) - advisor.phone = CleanText('//NumeroTelephone')(self.doc) - advisor.agency = CleanText('//liloes')(self.doc) - advisor.address = Format('%s %s %s', - CleanText('//ruadre'), - CleanText('//cdpost'), - CleanText('//loadre') - )(self.doc) - advisor.email = CleanText('//Email')(self.doc) - advisor.role = "wealth" if "patrimoine" in CleanText('//LibelleNatureConseiller')(self.doc).lower() else "bank" - yield advisor + @method + class iter_credit_history(ListElement): + item_xpath = '//tr' + class item(ItemElement): + klass = Transaction -class HTMLProfilePage(LoggedPage, HTMLPage): - def on_load(self): - msg = CleanText('//div[@id="connecteur_partenaire"]', default='')(self.doc) - service_unavailable_msg = CleanText('//span[@class="error_msg" and contains(text(), "indisponible")]')(self.doc) + obj_label = CleanText('./@title') + obj_date = Date(CleanText('./td[@headers="Date"]'), dayfirst=True) - if 'Erreur' in msg: - raise BrowserUnavailable(msg) - if service_unavailable_msg: - raise ProfileMissing(service_unavailable_msg) + def obj_amount(self): + credit = MyDecimal(CleanText('./td[contains(@headers, "Credit")]', replace=[(' ', '')]))(self) + if credit: + return credit + return MyDecimal(CleanText('./td[contains(@headers, "Debit")]', replace=[(' ', '')]))(self) - def get_profile(self): - profile = Person() - profile.name = Regexp(CleanText('//div[@id="dcr-conteneur"]//div[contains(text(), "PROFIL DE")]'), r'PROFIL DE (.*)')(self.doc) - profile.address = CleanText('//div[@id="dcr-conteneur"]//div[contains(text(), "ADRESSE")]/following::table//tr[3]/td[2]')(self.doc) - profile.address += ' ' + CleanText('//div[@id="dcr-conteneur"]//div[contains(text(), "ADRESSE")]/following::table//tr[5]/td[2]')(self.doc) - profile.address += ' ' + CleanText('//div[@id="dcr-conteneur"]//div[contains(text(), "ADRESSE")]/following::table//tr[6]/td[2]')(self.doc) - profile.country = CleanText('//div[@id="dcr-conteneur"]//div[contains(text(), "ADRESSE")]/following::table//tr[7]/td[2]')(self.doc) - return profile +class LifeInsurance(LoggedPage, HTMLPage): + def on_load(self): + errors_msg = ( + CleanText('//span[@class="error_msg"]')(self.doc), + CleanText("//div[@class='net2g_asv_error_full_page']")(self.doc) + ) + for error_msg in errors_msg: + if error_msg and 'Le service est momentanément indisponible' in error_msg: + raise BrowserUnavailable(error_msg) + if error_msg and 'Aucune opération' in error_msg: + break + else: + assert not any(errors_msg), 'Some errors are not handle yet' + + def has_link(self): + return Link('//a[@href="asvcns20a.html"]', default=NotAvailable)(self.doc) + + def get_history_link(self): + return Link('//a[img[@alt="Suivi des opérations"]]', default=NotAvailable)(self.doc) + def get_pages(self): + pages = CleanText('//div[@class="net2g_asv_tableau_pager"]')(self.doc) + if pages: + # "pages" value is for example "1/5" + return re.search(r'(\d)/(\d)', pages).group(1, 2) + + def li_pagination(self): + pages = self.get_pages() + if pages: + current_page, total_pages = int(pages[0]), int(pages[1]) + if current_page < total_pages: + data = { + 'a100_asv_action': 'actionSuivPage', + 'a100_asv_numPage': current_page, + 'a100_asv_nbPages': total_pages, + } + return requests.Request('POST', self.browser.url, data=data) + + +class LifeInsuranceInvest(LifeInsurance): + @pagination + @method + class iter_investment(TableElement): + def next_page(self): + return self.page.li_pagination() -class XMLProfilePage(LoggedPage, XMLPage): - def get_email(self): - return CleanText('//AdresseEmailExterne')(self.doc) + item_xpath = '//table/tbody/tr[starts-with(@class, "net2g_asv_tableau_ligne_")]' + head_xpath = '//table/thead/tr/td' + col_label = re.compile('Support') + col_quantity = re.compile("Nombre") + col_unitvalue = re.compile("Valeur") + col_valuation = re.compile("Capital") -# TODO: check if it work -class NotTransferBasePage(BasePage): - def is_transfer_here(self): - # check that we aren't on transfer or add recipient page - return bool(CleanText('//h1[contains(text(), "Effectuer un virement")]')(self.doc)) or \ - bool(CleanText(u'//h3[contains(text(), "Ajouter un compte bénéficiaire de virement")]')(self.doc)) or \ - bool(CleanText(u'//h1[contains(text(), "Ajouter un compte bénéficiaire de virement")]')(self.doc)) or \ - bool(CleanText(u'//h3[contains(text(), "Veuillez vérifier les informations du compte à ajouter")]')(self.doc)) or \ - bool(Link('//a[contains(@href, "per_cptBen_ajouterFrBic")]', default=NotAvailable)(self.doc)) + class item(ItemElement): + klass = Investment + obj_code = Regexp(CleanText(TableCell('label')), r'Code ISIN : (\w+) ', default=NotAvailable) + obj_quantity = MyDecimal(TableCell('quantity'), default=NotAvailable) + obj_unitvalue = MyDecimal(TableCell('unitvalue'), default=NotAvailable) -class Invest(object): - def create_investment(self, cells): - inv = Investment() - inv.quantity = MyDecimal('.')(cells[self.COL_QUANTITY]) - inv.unitvalue = MyDecimal('.')(cells[self.COL_UNITVALUE]) - inv.unitprice = NotAvailable - inv.valuation = MyDecimal('.')(cells[self.COL_VALUATION]) - inv.diff = NotAvailable + # Some PERP invests don't have valuation + obj_valuation = MyDecimal(TableCell('valuation', default=NotAvailable), default=NotAvailable) - link = cells[self.COL_LABEL].xpath('a[contains(@href, "CDCVAL=")]')[0] - m = re.search('CDCVAL=([^&]+)', link.attrib['href']) - if m: - inv.code = m.group(1) - else: - inv.code = NotAvailable - return inv - - -class Market(LoggedPage, BasePage, Invest): - COL_LABEL = 0 - COL_QUANTITY = 1 - COL_UNITPRICE = 2 - COL_VALUATION = 3 - COL_DIFF = 4 - - def get_balance(self, account_type): - return CleanDecimal('//form[@id="listeCTForm"]/table//tr[td[5]]/td[@class="TabCelRight"][1]', replace_dots=True, default=None)(self.doc) - - def get_not_rounded_valuations(self): - def prepare_url(url, fields): - components = urlparse(url) - query_pairs = [(f, v) for (f, v) in parse_qsl(components.query) if f not in fields] - - for (field, value) in fields.items(): - query_pairs.append((field, value)) - - new_query_str = urlencode(query_pairs) - - new_components = ( - components.scheme, - components.netloc, - components.path, - components.params, - new_query_str, - components.fragment - ) - - return urlunparse(new_components) - - not_rounded_valuations = {} - pages = [] - - try: - for i in range(1, CleanDecimal(Regexp(CleanText(u'(//table[form[contains(@name, "detailCompteTitresForm")]]//tr[1])[1]/td[3]/text()'), r'\/(.*)'))(self.doc) + 1): - pages.append(self.browser.open(prepare_url(self.browser.url, {'action': '11', 'idCptSelect': '1', 'numPage': i})).page) - except RegexpError: # no multiple page - pages.append(self) - - for page in pages: - for inv in page.doc.xpath(u'//table[contains(., "Détail du compte")]//tr[2]//table/tr[position() > 1]'): - if len(inv.xpath('.//td')) > 2: - amt = CleanText('.//td[7]/text()')(inv) - if amt == 'Indisponible': - continue - not_rounded_valuations[CleanText('.//td[1]/a/text()')(inv)] = CleanDecimal('.//td[7]/text()', replace_dots=True)(inv) - - return not_rounded_valuations - - def iter_investment(self): - not_rounded_valuations = self.get_not_rounded_valuations() - - doc = self.browser.open('/brs/fisc/fisca10a.html').page.doc - num_page = None - - try: - num_page = int(CleanText('.')(doc.xpath(u'.//tr[contains(td[1], "Relevé des plus ou moins values latentes")]/td[2]')[0]).split('/')[1]) - except IndexError: - pass - - docs = [doc] - - if num_page: - for n in range(2, num_page + 1): - docs.append(self.browser.open('%s%s' % ('/brs/fisc/fisca10a.html?action=12&numPage=', str(n))).page.doc) - - for doc in docs: - # There are two different tables possible depending on the market account type. - is_detailed = bool(doc.xpath(u'//span[contains(text(), "Années d\'acquisition")]')) - tr_xpath = '//tr[@height and td[@colspan="6"]]' if is_detailed else '//tr[count(td)>5]' - for tr in doc.xpath(tr_xpath): - cells = tr.findall('td') - - inv = Investment() - - title_split = cells[self.COL_LABEL].xpath('.//span')[0].attrib['title'].split(' - ') - inv.label = unicode(title_split[0]) - - for code in title_split[1:]: - if is_isin_valid(code): - inv.code = unicode(code) - inv.code_type = Investment.CODE_TYPE_ISIN - break - else: - inv.code = NotAvailable - inv.code_type = NotAvailable + def obj_label(self): + if 'FONDS EN EUROS' in CleanText(TableCell('label'))(self): + return 'FONDS EN EUROS' + return Regexp(CleanText(TableCell('label')), r'Libellé support : (.*) Code ISIN')(self) - if is_detailed: - inv.quantity = MyDecimal('.')(tr.xpath('./following-sibling::tr/td[2]')[0]) - inv.unitprice = MyDecimal('.', replace_dots=True)(tr.xpath('./following-sibling::tr/td[3]')[1]) - inv.unitvalue = MyDecimal('.', replace_dots=True)(tr.xpath('./following-sibling::tr/td[3]')[0]) + def obj_code_type(self): + if Field('label')(self) == 'FONDS EN EUROS': + return NotAvailable + return Investment.CODE_TYPE_ISIN - try: # try to get not rounded value - inv.valuation = not_rounded_valuations[inv.label] - except KeyError: # ok.. take it from the page - inv.valuation = MyDecimal('.')(tr.xpath('./following-sibling::tr/td[4]')[0]) - inv.diff = MyDecimal('.')(tr.xpath('./following-sibling::tr/td[5]')[0]) or \ - MyDecimal('.')(tr.xpath('./following-sibling::tr/td[6]')[0]) - else: - inv.quantity = MyDecimal('.')(cells[self.COL_QUANTITY]) - inv.diff = MyDecimal('.')(cells[self.COL_DIFF]) - inv.unitprice = MyDecimal('.')(cells[self.COL_UNITPRICE].xpath('.//tr[1]/td[2]')[0]) - inv.unitvalue = MyDecimal('.')(cells[self.COL_VALUATION].xpath('.//tr[1]/td[2]')[0]) - inv.valuation = MyDecimal('.')(cells[self.COL_VALUATION].xpath('.//tr[2]/td[2]')[0]) +class LifeInsuranceInvest2(LifeInsuranceInvest): + @method + class iter_investment(TableElement): + item_xpath = '//table/tbody/tr[starts-with(@class, "net2g_asv_tableau_ligne_")]' + head_xpath = '//table/thead/tr/td' - yield inv + col_label = u'Support' + col_valuation = u'Montant' + class item(ItemElement): + klass = Investment + obj_label = CleanText(TableCell('label')) + obj_valuation = MyDecimal(TableCell('valuation')) -class LifeInsurance(LoggedPage, BasePage): - def get_error(self): - try: - return self.doc.xpath("//div[@class='net2g_asv_error_full_page']")[0].text.strip() - except IndexError: - return super(LifeInsurance, self).get_error() - def has_link(self): - return Link('//a[@href="asvcns20a.html"]', default=NotAvailable)(self.doc) +class LifeInsuranceHistory(LifeInsurance): + @pagination + @method + class iter_li_history(TableElement): + def next_page(self): + return self.page.li_pagination() - def get_error_msg(self): - # to check page errors - return CleanText('//span[@class="error_msg"]')(self.doc) + item_xpath = '//table/tbody/tr[starts-with(@class, "net2g_asv_tableau_ligne_")]' + head_xpath = '//table/thead/tr/td' + col_label = 'Opération' + col_date = 'Date' + col_amount = 'Montant' + col__status = 'Statut' -class LifeInsuranceInvest(LifeInsurance, Invest): - COL_LABEL = 0 - COL_QUANTITY = 1 - COL_UNITVALUE = 2 - COL_VALUATION = 3 + class item(ItemElement): + def condition(self): + return (CleanText(TableCell('_status'))(self) == 'Réalisé' and + MyDecimal(TableCell('amount'), default=NotAvailable)(self)) - def iter_investment(self): - for tr in self.doc.xpath("//table/tbody/tr[starts-with(@class, 'net2g_asv_tableau_ligne_')]"): - cells = tr.findall('td') - inv = self.create_investment(cells) - inv.label = unicode(cells[self.COL_LABEL].xpath('a/span')[0].text.strip()) - inv.description = unicode(cells[self.COL_LABEL].xpath('a//div/b[last()]')[0].tail) + klass = Transaction - yield inv + obj_label = CleanText(TableCell('label')) + obj_amount = MyDecimal(TableCell('amount')) - def get_pages(self): - # "pages" value is for example "1/5" - pages = CleanText('//div[@class="net2g_asv_tableau_pager"]')(self.doc) - return re.search(r'/(.*)', pages).group(1) if pages else None + def obj_date(self): + tr_date = CleanText(TableCell('date'))(self) + if len(tr_date) == 4: + # date of transaction with label 'Intérêts crédités au cours de l'année' + # is only year valuation + # set transaction date to the last day year + return datetime.date(int(tr_date), 12, 31) + return Date(dayfirst=True).filter(tr_date) -class LifeInsuranceInvest2(LifeInsuranceInvest): +class MarketPage(LoggedPage, HTMLPage): @method - class iter_investment(TableElement): - item_xpath = '//table/tbody/tr[starts-with(@class, "net2g_asv_tableau_ligne_")]' - head_xpath = '//table/thead/tr/td' + class iter_investments(TableElement): + table_xpath = '//tr[td[contains(@class,"TabTit1l")]]/following-sibling::tr//table' + head_xpath = table_xpath + '//tr[1]/td' + item_xpath = table_xpath + '//tr[position()>1]' - col_label = u'Support' - col_valuation = u'Montant' + col_quantity = 'Quantité' + col_valuation = 'Evaluation' + col_vdate = 'Date' + col_unitvalue = 'Cours' + + def condition(self): + return not 'PAS DE VALEURS DETENUES ACTUELLEMENT SUR CE COMPTE' in \ + CleanText('//td[@class="MessErreur"]')(self.el) class item(ItemElement): + def condition(self): + return self.xpath('./td[contains(@class, "TabCelLeft")]') + klass = Investment - obj_label = CleanText(TableCell('label')) + + obj_code = Regexp(CleanText('./td[1]//@title'), '- (\w+) -') + obj_label = CleanText('./td[1]//text()') + obj_quantity = MyDecimal(TableCell('quantity')) obj_valuation = MyDecimal(TableCell('valuation')) + obj_vdate = Date(Regexp(CleanText(TableCell('vdate')), r'(\d{2}/\d{2}/\d{4})')) + obj_unitvalue = MyDecimal(TableCell('unitvalue')) + def obj_code_type(self): + if is_isin_valid(Field('code')(self)): + return Investment.CODE_TYPE_ISIN + return NotAvailable -class LifeInsuranceHistory(LifeInsurance): - COL_DATE = 0 - COL_LABEL = 1 - COL_AMOUNT = 2 - COL_STATUS = 3 - - def iter_transactions(self): - for tr in self.doc.xpath("//table/tbody/tr[starts-with(@class, 'net2g_asv_tableau_ligne_')]"): - cells = tr.findall('td') - - link = cells[self.COL_LABEL].xpath('a')[0] - # javascript:detailOperation('operationForm', '2'); - m = re.search(", '([0-9]+)'", link.attrib['href']) - if m: - id_trans = m.group(1) - else: - id_trans = '' - - trans = Transaction() - trans._temp_id = id_trans - trans.parse(raw=link.attrib['title'], date=cells[self.COL_DATE].text) - trans.set_amount(cells[self.COL_AMOUNT].text) - # search for 'Réalisé' - trans._coming = 'alis' not in cells[self.COL_STATUS].text.strip() - - if not self.set_date(trans): - continue - - if u'Annulé' in cells[self.COL_STATUS].text.strip(): - continue - - yield trans - - def set_date(self, trans): - """fetch date and vdate from another page""" - # go to the page containing the dates - form = self.get_form(id='operationForm') - form['a100_asv_action'] = 'detail' - form['a100_asv_indexOp'] = trans._temp_id - form.url = '/asv/AVI/asvcns21c.html' - - # but the page sometimes fail - for i in range(3, -1, -1): - page = form.submit().page - doc = page.doc - if not page.get_error(): - break - self.logger.warning('Life insurance history error (%s), retrying %d more times', page.get_error(), i) - else: - self.logger.warning('Life insurance history error (%s), failed', page.get_error()) - return False - - # process the data - date_xpath = '//td[@class="net2g_asv_suiviOperation_element1"]/following-sibling::td' - vdate_xpath = '//td[@class="net2g_asv_tableau_cell_date"]' - - date = CleanText(date_xpath)(doc) - if u"Rejet d'intégration" in date: - return False - - trans.date = self.parse_date(doc, trans, date_xpath, 1) - trans.rdate = trans.date - trans.vdate = self.parse_date(doc, trans, vdate_xpath, 0) - return True - - @staticmethod - def parse_date(doc, trans, xpath, index): - elem = doc.xpath(xpath)[index] - if elem.text: - return trans.parse_date(elem.text.strip()) - else: - return NotAvailable + +class PeaLiquidityPage(LoggedPage, HTMLPage): + def iter_investments(self, account): + yield (create_french_liquidity(account.balance)) + + +class AdvisorPage(LoggedPage, XMLPage): + ENCODING = 'ISO-8859-15' + + def get_advisor(self): + advisor = Advisor() + advisor.name = Format('%s %s', CleanText('//NomConseiller'), CleanText('//PrenomConseiller'))(self.doc) + advisor.phone = CleanText('//NumeroTelephone')(self.doc) + advisor.agency = CleanText('//liloes')(self.doc) + advisor.address = Format('%s %s %s', + CleanText('//ruadre'), + CleanText('//cdpost'), + CleanText('//loadre') + )(self.doc) + advisor.email = CleanText('//Email')(self.doc) + advisor.role = "wealth" if "patrimoine" in CleanText('//LibelleNatureConseiller')(self.doc).lower() else "bank" + yield advisor + + +class HTMLProfilePage(LoggedPage, HTMLPage): + def on_load(self): + msg = CleanText('//div[@id="connecteur_partenaire"]', default='')(self.doc) or \ + CleanText('//body', default='')(self.doc) + service_unavailable_msg = CleanText('//div[@class="message-error" and contains(text(), "indisponible")]')(self.doc) + + if 'Erreur' in msg: + raise BrowserUnavailable(msg) + if service_unavailable_msg: + raise ProfileMissing(service_unavailable_msg) + + def get_profile(self): + profile = Person() + profile.name = Regexp(CleanText('//div[@id="dcr-conteneur"]//div[contains(text(), "PROFIL DE")]'), r'PROFIL DE (.*)')(self.doc) + profile.address = CleanText('//div[@id="dcr-conteneur"]//div[contains(text(), "ADRESSE")]/following::table//tr[3]/td[2]')(self.doc) + profile.address += ' ' + CleanText('//div[@id="dcr-conteneur"]//div[contains(text(), "ADRESSE")]/following::table//tr[5]/td[2]')(self.doc) + profile.address += ' ' + CleanText('//div[@id="dcr-conteneur"]//div[contains(text(), "ADRESSE")]/following::table//tr[6]/td[2]')(self.doc) + profile.country = CleanText('//div[@id="dcr-conteneur"]//div[contains(text(), "ADRESSE")]/following::table//tr[7]/td[2]')(self.doc) + profile.email = CleanText('//span[@id="currentEmail"]')(self.doc) + + return profile class UnavailableServicePage(LoggedPage, HTMLPage): def on_load(self): - if self.doc.xpath('//div[contains(@class, "erreur_404_content")]'): + conditions = ( + self.doc.xpath('//div[contains(@class, "erreur_404_content")]'), + 'Site momentanément indisponible' in CleanText('//h2[contains(@class, "error-page")]')(self.doc), + ) + + if any(conditions): raise BrowserUnavailable() diff --git a/modules/societegenerale/pages/subscription.py b/modules/societegenerale/pages/subscription.py index 6690a2feba8c0a67b4fd72e60b18cdd3d20e8c94..093435fffd4d21ed6c3ce62bcc342a3956db8054 100644 --- a/modules/societegenerale/pages/subscription.py +++ b/modules/societegenerale/pages/subscription.py @@ -37,8 +37,12 @@ class iter_subscription(TableElement): head_xpath = '//table//th' col_id = 'Numéro de Compte' + col_label = 'Derniers relevés' class item(ItemElement): + def condition(self): + return 'Récapitulatif annuel' not in CleanText(TableCell('label'))(self) + klass = Subscription obj_id = CleanText(TableCell('id'), replace=[(' ', '')]) @@ -110,4 +114,6 @@ def iter_documents(self, subscription): yield d def has_error_msg(self): - return CleanText('//div[@class="MessageErreur"]')(self.doc) or CleanText('//span[@class="error_msg"]')(self.doc) + return any((CleanText('//div[@class="MessageErreur"]')(self.doc), + CleanText('//span[@class="error_msg"]')(self.doc), + self.doc.xpath('//div[contains(@class, "error_page")]'), )) diff --git a/modules/societegenerale/pages/transfer.py b/modules/societegenerale/pages/transfer.py index ecc547a8debf014b23b9f7cf40624a0f78db72a6..a202a9cb1fa6dd85573f99347a95ee16e5a3b73a 100644 --- a/modules/societegenerale/pages/transfer.py +++ b/modules/societegenerale/pages/transfer.py @@ -23,7 +23,7 @@ from weboob.browser.pages import LoggedPage, JsonPage, FormNotFound from weboob.browser.elements import method, ItemElement, DictElement from weboob.capabilities.bank import ( - Recipient, Transfer, TransferBankError, AddRecipientBankError, AddRecipientStep, + Recipient, Transfer, TransferBankError, AddRecipientBankError, AddRecipientTimeout, ) from weboob.tools.capabilities.bank.iban import is_iban_valid from weboob.capabilities.base import NotAvailable @@ -32,9 +32,8 @@ ) from weboob.browser.filters.html import Link from weboob.browser.filters.json import Dict -from weboob.tools.value import Value, ValueBool from weboob.tools.json import json -from weboob.exceptions import BrowserUnavailable +from weboob.exceptions import BrowserUnavailable, ActionNeeded from .base import BasePage from .login import MainPage @@ -45,7 +44,7 @@ def on_load(self): if Dict('commun/statut')(self.doc).upper() == 'NOK': if self.doc['commun'].get('action'): raise TransferBankError(message=Dict('commun/action')(self.doc)) - elif self.doc['commun'].get('raison') == 'err_tech': + elif self.doc['commun'].get('raison') in ('err_tech', 'err_is'): # on SG website, there is unavalaible message 'Le service est momentanément indisponible.' raise BrowserUnavailable() else: @@ -167,8 +166,29 @@ def get_confirm_transfer_data(self, password): } -class RecipientJson(LoggedPage, JsonPage): - pass +class SignRecipientPage(LoggedPage, JsonPage): + def on_load(self): + assert Dict('commun/statut')(self.doc).upper() == 'OK', \ + 'Something went wrong on sign recipient page: %s' % Dict('commun/raison')(self.doc) + + def get_sign_method(self): + return Dict('donnees/sign_proc')(self.doc).upper() + + def check_recipient_status(self): + transaction_status = Dict('donnees/transaction_status')(self.doc) + + # check add new recipient status + assert transaction_status in ('available', 'in_progress', 'aborted', 'rejected'), \ + 'transaction_status is %s' % transaction_status + if transaction_status == 'aborted': + raise AddRecipientTimeout() + elif transaction_status == 'rejected': + raise ActionNeeded("La demande d'ajout de bénéficiaire a été annulée.") + elif transaction_status == 'in_progress': + raise ActionNeeded('Veuillez valider le bénéficiaire sur votre application bancaire.') + + def get_transaction_id(self): + return Dict('donnees/id-transaction')(self.doc) class AddRecipientPage(LoggedPage, BasePage): @@ -201,40 +221,26 @@ def get_action_level(self): if 'actionLevel' in CleanText('.')(script): return re.search("'actionLevel': (\d{3}),", script.text).group(1) - def double_auth(self, recipient): + def get_signinfo_data_form(self): try: form = self.get_form(id='formCache') except FormNotFound: - assert False, 'Double auth form not found' + assert False, 'Transfer auth form not found' + return form + def update_browser_recipient_state(self): + form = self.get_signinfo_data_form() + # set browser variable used to continue new recipient self.browser.context = form['context'] self.browser.dup = form['dup'] self.browser.logged = 1 - getsigninfo_data = {} - getsigninfo_data['b64_jeton_transaction'] = form['context'] - getsigninfo_data['action_level'] = self.get_action_level() - r = self.browser.open('https://particuliers.secure.societegenerale.fr/sec/getsigninfo.json', data=getsigninfo_data) - assert r.page.doc['commun']['statut'] == 'ok' - - recipient = self.get_recipient_object(recipient, get_info=True) - self.browser.page = None - if r.page.doc['donnees']['sign_proc'] == 'csa': - send_data = {} - send_data['csa_op'] = 'sign' - send_data['context'] = form['context'] - r = self.browser.open('https://particuliers.secure.societegenerale.fr/sec/csa/send.json', data=send_data) - assert r.page.doc['commun']['statut'] == 'ok' - raise AddRecipientStep(recipient, Value('code', label=u'Cette opération doit être validée par un Code Sécurité.')) - elif r.page.doc['donnees']['sign_proc'] == 'OOB': - oob_data = {} - oob_data['b64_jeton_transaction'] = form['context'] - r = self.browser.open('https://particuliers.secure.societegenerale.fr/sec/oob_sendoob.json', data=oob_data) - assert r.page.doc['commun']['statut'] == 'ok' - self.browser.id_transaction = r.page.doc['donnees']['id-transaction'] - raise AddRecipientStep(recipient, ValueBool('pass', label=u'Valider cette opération sur votre applicaton société générale')) - else: - assert False, 'Sign process unknown: %s' % r.page.doc['donnees']['sign_proc'] + def get_signinfo_data(self): + form = self.get_signinfo_data_form() + signinfo_data = {} + signinfo_data['b64_jeton_transaction'] = form['context'] + signinfo_data['action_level'] = self.get_action_level() + return signinfo_data def get_recipient_object(self, recipient, get_info=False): r = Recipient() diff --git a/modules/societegenerale/sgpe/browser.py b/modules/societegenerale/sgpe/browser.py index def1102ceaa6b88affae7dcc48c7774712a98519..65a4416ab7f6a298f26c82a6d00a242685200166 100644 --- a/modules/societegenerale/sgpe/browser.py +++ b/modules/societegenerale/sgpe/browser.py @@ -467,6 +467,9 @@ def init_transfer(self, account, recipient, transfer): ('an_guichetGestionnaire', account._manage_counter), ('an_codeProduit', account._product_code), ('an_codeSousProduit', account._underproduct_code), + ('n_soldeComptableVeilleMontant', int(account.balance * (10 ** account._decimal_code))), + ('n_soldeComptableVeilleCodeDecimalisation', account._decimal_code), + ('an_soldeComptableVeilleDevise', account._currency_code), ('n_ordreMontantValeur', int(transfer.amount * (10 ** account._decimal_code))), ('n_ordreMontantCodeDecimalisation', account._decimal_code), ('an_ordreMontantCodeDevise', account._currency_code), @@ -479,6 +482,8 @@ def init_transfer(self, account, recipient, transfer): ('cl_beneficiaireCompteIbanFormate', recipient._formatted_iban), ('an_beneficiaireCompteIban', recipient.iban), ('cl_beneficiaireCompteBic', recipient._bic), + ('cl_beneficiaireDateCreation', recipient._created_date), + ('cl_beneficiaireCodeOrigine', recipient._code_origin), ('cl_beneficiaireAdressePays', recipient.iban[:2]), ('an_indicateurIntraAbonnement', 'false'), ('cl_reference', ' '), diff --git a/modules/societegenerale/sgpe/transfer_pages.py b/modules/societegenerale/sgpe/transfer_pages.py index 799c627ba4ef6af114a34c3a1521acf1d668bfff..9ba405811327fb45faf23567a1f86060443f861a 100644 --- a/modules/societegenerale/sgpe/transfer_pages.py +++ b/modules/societegenerale/sgpe/transfer_pages.py @@ -69,6 +69,8 @@ def condition(self): obj__formatted_iban = Dict('coordonnee/0/numeroCompteFormate') obj__bic = Dict('coordonnee/0/BIC') obj__ref = Dict('coordonnee/0/refSICoordonnee') + obj__code_origin = Dict('coordonnee/0/codeOrigine') + obj__created_date = Dict('dateCreationDest') class TransferDatesPage(LoggedPage, ErrorCheckedJsonPage): @@ -121,6 +123,8 @@ def iter_internal_recipients(self): rcpt._account_title = json_data['intituleCompte'] rcpt._bic = json_data['bicCompte'] rcpt._ref = '' + rcpt._code_origin = '' + rcpt._created_date = '' yield rcpt diff --git a/modules/vlille/browser.py b/modules/vlille/browser.py index 9297a53cf33d295581b0915b1d7d754f11579112..e80c448786bf9febd496c2c52b6336d45444ca16 100644 --- a/modules/vlille/browser.py +++ b/modules/vlille/browser.py @@ -28,7 +28,7 @@ class VlilleBrowser(PagesBrowser): - BASEURL = 'https://www.transpole.fr' + BASEURL = 'https://www.ilevia.fr' list_page = URL('/cms/vlille/les-stations-cartographies/', ListStationsPage) def get_station_list(self): diff --git a/modules/vlille/pages.py b/modules/vlille/pages.py index 69400d8004e4d9061b6f87864baa599de8fc3130..0430a937193490739edf96a63110b3f8b866e08f 100644 --- a/modules/vlille/pages.py +++ b/modules/vlille/pages.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from decimal import Decimal from weboob.browser.pages import HTMLPage from weboob.browser.elements import ItemElement, TableElement, method @@ -52,7 +53,7 @@ def _create_bikes_sensor(self, value, gauge_id, last_update, adresse): levelbikes.name = u'Bikes' levelbikes.address = u'%s' % adresse lastvalue = GaugeMeasure() - lastvalue.level = float(value) + lastvalue.level = Decimal(value) lastvalue.date = last_update if lastvalue.level < 1: lastvalue.alarm = u'Empty station' @@ -68,7 +69,7 @@ def _create_attach_sensor(self, value, gauge_id, last_update, adresse): lastvalue = GaugeMeasure() if lastvalue.level < 1: lastvalue.alarm = u'Full station' - lastvalue.level = float(value) + lastvalue.level = Decimal(value) lastvalue.date = last_update levelattach.lastvalue = lastvalue levelattach.history = NotLoaded