diff --git a/modules/afer/browser.py b/modules/afer/browser.py index 088f405746b066ed1067de108fc8bc1af3708344..c3d7dd436de932425d8eb6742ea9d4740cb056b6 100644 --- a/modules/afer/browser.py +++ b/modules/afer/browser.py @@ -29,7 +29,6 @@ class AferBrowser(LoginBrowser): BASEURL = 'https://adherent.gie-afer.fr' - VERIFY = 'afer.pem' login = URL('/web/ega.nsf/listeAdhesions\?OpenForm', LoginPage) bad_login = URL('/names.nsf\?Login', BadLogin) @@ -53,7 +52,9 @@ def do_login(self): raise BrowserIncorrectPassword() if self.bad_login.is_here(): - raise BrowserIncorrectPassword() + error = self.page.get_error() + assert "La saisie de l’identifiant ou du code confidentiel est incorrecte" in error, error + raise BrowserIncorrectPassword(error) @need_login def iter_accounts(self): diff --git a/modules/afer/compat/weboob_capabilities_bank.py b/modules/afer/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/afer/compat/weboob_capabilities_bank.py +++ b/modules/afer/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/afer/pages.py b/modules/afer/pages.py index b4667f860803b9c7b78ee732c6ed4f6ecc152066..bdfefc3e24c9e9e482b47cb18e0642b18746c437 100644 --- a/modules/afer/pages.py +++ b/modules/afer/pages.py @@ -42,6 +42,11 @@ def is_here(self): return bool(self.doc.xpath('//form[@name="_DominoForm"]')) +class BadLogin(HTMLPage): + def get_error(self): + return CleanText('//div[@id="idDivErrorLogin"]')(self.doc) + + class IndexPage(LoggedPage, HTMLPage): def on_load(self): HTMLPage.on_load(self) @@ -156,7 +161,3 @@ def obj_amount(self): return am return (Async('details') & CleanDecimal('//div//tr[2]/td[2]', replace_dots=True, default=NotAvailable))( self) - - -class BadLogin(HTMLPage): - pass diff --git a/modules/amazon/browser.py b/modules/amazon/browser.py index aa7f20ace5a8d0195db45c7685eb032fbf297475..849939644521cc5058395d30a4ef50f508315d7a 100644 --- a/modules/amazon/browser.py +++ b/modules/amazon/browser.py @@ -25,7 +25,10 @@ from weboob.tools.value import Value from weboob.browser.browsers import ClientError -from .pages import LoginPage, SubscriptionsPage, DocumentsPage, HomePage, PanelPage, SecurityPage, LanguagePage, HistoryPage +from .pages import ( + LoginPage, SubscriptionsPage, DocumentsPage, DownloadDocumentPage, HomePage, PanelPage, SecurityPage, + LanguagePage, HistoryPage +) class AmazonBrowser(LoginBrowser, StatesMixin): @@ -44,6 +47,7 @@ class AmazonBrowser(LoginBrowser, StatesMixin): documents = URL(r'/gp/your-account/order-history\?opt=ab&digitalOrders=1(.*)&orderFilter=year-(?P.*)', r'https://www.amazon.fr/gp/your-account/order-history', DocumentsPage) + download_doc = URL(r'/gp/shared-cs/ajax/invoice/invoice.html', DownloadDocumentPage) security = URL('/ap/dcq', '/ap/cvf/', '/ap/mfa', @@ -176,6 +180,8 @@ def iter_documents(self, subscription): documents = [] for y in range(date.today().year - 2, date.today().year + 1): - for doc in self.documents.go(year=y).iter_documents(subid=subscription.id, currency=self.CURRENCY): + self.documents.go(year=y) + request_id = self.page.response.headers['x-amz-rid'] + for doc in self.page.iter_documents(subid=subscription.id, currency=self.CURRENCY, request_id=request_id): documents.append(doc) return documents diff --git a/modules/amazon/pages.py b/modules/amazon/pages.py index da230306947a874d8b7e62ecd057f6ee33c4de4c..7455d59d27ebae3f69c64abe4291bbff1a424a85 100644 --- a/modules/amazon/pages.py +++ b/modules/amazon/pages.py @@ -19,12 +19,12 @@ from __future__ import unicode_literals - -from weboob.browser.pages import HTMLPage, LoggedPage, FormNotFound +from weboob.browser.pages import HTMLPage, LoggedPage, FormNotFound, PartialHTMLPage from weboob.browser.elements import ItemElement, ListElement, method +from weboob.browser.filters.html import Link from weboob.browser.filters.standard import ( CleanText, CleanDecimal, Env, Regexp, Format, - Field, Currency, RegexpError, Date + Field, Currency, RegexpError, Date, Async, AsyncLoad ) from weboob.capabilities.bill import Bill, Subscription from weboob.capabilities.base import NotAvailable @@ -119,11 +119,14 @@ class iter_documents(ListElement): class item(ItemElement): klass = Bill + load_details = Field('_pre_url') & AsyncLoad obj__simple_id = CleanText('.//div[has-class("actions")]//span[has-class("value")]') obj_id = Format('%s_%s', Env('subid'), Field('_simple_id')) - obj_url = Format('/gp/css/summary/print.html/ref=oh_aui_ajax_pi?ie=UTF8&orderID=%s', Field('_simple_id')) - obj_format = 'html' + obj__pre_url = Format('/gp/shared-cs/ajax/invoice/invoice.html?orderId=%s&relatedRequestId=%s&isADriveSubscription=&isHFC=', + Field('_simple_id'), Env('request_id')) + obj_url = Async('details') & Link('//a[contains(@href, "download")]') + obj_format = 'pdf' obj_label = Format('Facture %s', Field('_simple_id')) obj_type = 'bill' @@ -142,3 +145,7 @@ def obj_price(self): def obj_currency(self): currency = Env('currency')(self) return Currency('.//div[has-class("a-col-left")]//span[has-class("value") and contains(., "%s")]' % currency)(self) + + +class DownloadDocumentPage(LoggedPage, PartialHTMLPage): + pass diff --git a/modules/ameli/browser.py b/modules/ameli/browser.py index 1be6c45d7eaa5f3e5c27580d45f476bdd661373d..295037b49d565e7e0e0777eff9673c1354d6ec70 100644 --- a/modules/ameli/browser.py +++ b/modules/ameli/browser.py @@ -17,6 +17,8 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals + from weboob.browser import LoginBrowser, URL, need_login from weboob.exceptions import BrowserIncorrectPassword, ActionNeeded from weboob.tools.compat import basestring @@ -28,13 +30,13 @@ class AmeliBrowser(LoginBrowser): BASEURL = 'https://assure.ameli.fr' - loginp = URL('/PortailAS/appmanager/PortailAS/assure\?.*_pageLabel=as_login_page', LoginPage) - homep = URL('/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&_pageLabel=as_accueil_page', HomePage) - cgup = URL('/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&_pageLabel=as_conditions_generales_page', CguPage) - accountp = URL('/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&_pageLabel=as_info_perso_page', AccountPage) - paymentsp = URL('/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&_pageLabel=as_paiements_page', PaymentsPage) - paymentdetailsp = URL('/PortailAS/paiements.do\?actionEvt=chargerDetailPaiements.*', PaymentDetailsPage) - lastpaymentsp = URL('/PortailAS/paiements.do\?actionEvt=afficherPaiements.*', LastPaymentsPage) + loginp = URL(r'/PortailAS/appmanager/PortailAS/assure\?.*_pageLabel=as_login_page', LoginPage) + homep = URL(r'/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&_pageLabel=as_accueil_page', HomePage) + cgup = URL(r'/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&_pageLabel=as_conditions_generales_page', CguPage) + accountp = URL(r'/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&_pageLabel=as_info_perso_page', AccountPage) + paymentsp = URL(r'/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&_pageLabel=as_paiements_page', PaymentsPage) + paymentdetailsp = URL(r'/PortailAS/paiements.do\?actionEvt=chargerDetailPaiements.*', PaymentDetailsPage) + lastpaymentsp = URL(r'/PortailAS/paiements.do\?actionEvt=afficherPaiements.*', LastPaymentsPage) pdf_page = URL(r'PortailAS/PDFServletReleveMensuel.dopdf\?PDF.moisRecherche=.*', Raw) def do_login(self): diff --git a/modules/ameli/module.py b/modules/ameli/module.py index 2904b217dd4b04955f11742ff03a80ba34ea2deb..f94b28bcf934f3b828bd1bb7c00ad1ac107dc43e 100644 --- a/modules/ameli/module.py +++ b/modules/ameli/module.py @@ -17,6 +17,8 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals + from weboob.capabilities.bill import CapDocument, SubscriptionNotFound, DocumentNotFound, Subscription, Bill from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword @@ -27,8 +29,8 @@ class AmeliModule(Module, CapDocument): NAME = 'ameli' - DESCRIPTION = u'Ameli website: French Health Insurance' - MAINTAINER = u'Christophe Lampin' + DESCRIPTION = 'Ameli website: French Health Insurance' + MAINTAINER = 'Christophe Lampin' EMAIL = 'weboob@lampin.net' VERSION = '1.3' LICENSE = 'AGPLv3+' @@ -75,6 +77,7 @@ def get_document(self, id): def download_document(self, bill): if not isinstance(bill, Bill): bill = self.get_document(bill) - request = self.browser.open(bill.url, stream=True) - assert(request.headers['content-type'] == "application/pdf") - return request.content + response = self.browser.open(bill.url, stream=True) + if not response or response.headers['content-type'] != "application/pdf": + return None + return response.content diff --git a/modules/ameli/pages.py b/modules/ameli/pages.py index 659cbfae4820a52f15e3e65bbd72482bac1cff4b..1118dbda27aae4992323782dc3772de3b0dcf27e 100644 --- a/modules/ameli/pages.py +++ b/modules/ameli/pages.py @@ -27,17 +27,16 @@ from weboob.browser.pages import HTMLPage, RawPage, LoggedPage from weboob.capabilities.bill import Subscription, Detail, Bill from weboob.browser.filters.standard import CleanText, Regexp -from weboob.tools.compat import urljoin # Ugly array to avoid the use of french locale -FRENCH_MONTHS = [u'janvier', u'février', u'mars', u'avril', u'mai', u'juin', u'juillet', u'août', u'septembre', u'octobre', u'novembre', u'décembre'] +FRENCH_MONTHS = ['janvier', 'février', 'mars', 'avril', 'mai', 'juin', 'juillet', 'août', 'septembre', 'octobre', 'novembre', 'décembre'] class AmeliBasePage(HTMLPage): @property def logged(self): - if self.doc.xpath(u'//a[contains(text(), "Déconnexion")]'): + if self.doc.xpath('//a[contains(text(), "Déconnexion")]'): logged = True else: logged = False @@ -45,7 +44,7 @@ def logged(self): return logged def is_error(self): - errors = self.doc.xpath(u'//*[@id="r_errors"]') + errors = self.doc.xpath('//*[@id="r_errors"]') if errors: return errors[0].text_content() @@ -86,7 +85,7 @@ class AccountPage(AmeliBasePage): def iter_subscription_list(self): names_list = self.doc.xpath('//span[@class="NomEtPrenomLabel"]') fullname = CleanText(newlines=True).filter(names_list[0]) - number = re.sub('[^\d]+', '', CleanText('//span[@class="blocNumSecu"]', replace=[(' ','')])(self.doc)) + number = re.sub(r'[^\d]+', '', CleanText('//span[@class="blocNumSecu"]', replace=[(' ','')])(self.doc)) sub = Subscription(number) sub._id = number sub.label = fullname @@ -126,10 +125,10 @@ def iter_documents(self, sub): bil = Bill() bil.id = sub._id + "." + date.strftime("%Y%m") bil.date = date - bil.format = u'pdf' - bil.type = u'bill' - bil.label = u'' + date.strftime("%Y%m%d") - bil.url = urljoin(self.url, '/PortailAS/PDFServletReleveMensuel.dopdf?PDF.moisRecherche=%s' % date.strftime("%m%Y")) + bil.format = 'pdf' + bil.type = 'bill' + bil.label = date.strftime("%Y%m%d") + bil.url = '/PortailAS/PDFServletReleveMensuel.dopdf?PDF.moisRecherche=' + date.strftime("%m%Y") yield bil def get_document(self, bill): @@ -139,7 +138,7 @@ def get_document(self, bill): class PaymentDetailsPage(AmeliBasePage): def iter_payment_details(self, sub): id_str = self.doc.xpath('//div[@class="entete container"]/h2')[0].text.strip() - m = re.match('.*le (.*) pour un montant de.*', id_str) + m = re.match(r'.*le (.*) pour un montant de.*', id_str) if m: blocs_benes = self.doc.xpath('//span[contains(@id,"nomBeneficiaire")]') blocs_prestas = self.doc.xpath('//table[@id="tableauPrestation"]') @@ -179,18 +178,18 @@ def iter_payment_details(self, sub): price = '0' if date_str is None or date_str == '': - det.infos = u'' + det.infos = '' det.datetime = last_date else: - det.infos = '%s (%sj) * %s€' % (date_str, re.sub(r'[^\d,-]+', '', jours), re.sub(r'[^\d,-]+', '', montant)) + det.infos = date_str + ' (' + re.sub(r'[^\d,-]+', '', jours) + 'j) * ' + re.sub(r'[^\d,-]+', '', montant) + '€' det.datetime = datetime.strptime(date_str.split(' ')[3], '%d/%m/%Y').date() last_date = det.datetime - det.price = Decimal(re.sub('[^\d,-]+', '', price).replace(',', '.')) + det.price = Decimal(re.sub(r'[^\d,-]+', '', price).replace(',', '.')) if len(tds) == 5: date_str = Regexp(pattern=r'\w*(\d{2})/(\d{2})/(\d{4}).*', template='\\1/\\2/\\3', default="").filter("".join(tds[0].itertext())) det.id = id + "." + str(line) - det.label = '%s - %s' % (bene, tds[0].xpath('.//span')[0].text.strip()) + det.label = bene + ' - ' + tds[0].xpath('.//span')[0].text.strip() paye = tds[1].text if paye is None: @@ -213,13 +212,13 @@ def iter_payment_details(self, sub): price = tdprice.strip() if date_str is None or date_str == '': - det.infos = u'' + det.infos = '' det.datetime = last_date else: - det.infos = u' Payé %s€ / Base %s€ / Taux %s%%' % (re.sub(r'[^\d,-]+', '', paye), re.sub(r'[^\d,-]+', '', base), re.sub('[^\d,-]+', '', taux)) + det.infos = ' Payé ' + re.sub(r'[^\d,-]+', '', paye) + '€ / Base ' + re.sub(r'[^\d,-]+', '', base) + '€ / Taux ' + re.sub(r'[^\d,-]+', '', taux) + '%' det.datetime = datetime.strptime(date_str, '%d/%m/%Y').date() last_date = det.datetime - det.price = Decimal(re.sub('[^\d,-]+', '', price).replace(',', '.')) + det.price = Decimal(re.sub(r'[^\d,-]+', '', price).replace(',', '.')) line = line + 1 yield det diff --git a/modules/americanexpress/pages/base.py b/modules/americanexpress/pages/base.py deleted file mode 100644 index 9a45b1d41b2f9b9f529446f44b4e475e8591aee1..0000000000000000000000000000000000000000 --- a/modules/americanexpress/pages/base.py +++ /dev/null @@ -1,275 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2013 Romain Bignon -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - -from __future__ import unicode_literals - -import datetime -import re -from dateutil.relativedelta import relativedelta - -from weboob.browser.elements import ItemElement, ListElement, method -from weboob.browser.pages import HTMLPage, LoggedPage, PartialHTMLPage -from weboob.browser.filters.standard import CleanText, CleanDecimal, Currency as CleanCurrency -from weboob.browser.filters.html import AbsoluteLink -from weboob.capabilities.bank import Account -from weboob.capabilities import NotAvailable -from weboob.exceptions import ActionNeeded -from weboob.tools.capabilities.bank.transactions import FrenchTransaction as Transaction -from weboob.tools.compat import urljoin -from weboob.tools.date import ChaoticDateGuesser, parse_french_date - - -def parse_decimal(s): - # we might get 1,399,680 in rupie indonésienne - if s.count(',') > 1 and not s.count('.'): - return CleanDecimal(replace_dots=(',', '.')).filter(s) - # we don't know which decimal format this account will use - comma = s.rfind(',') > s.rfind('.') - return CleanDecimal(replace_dots=comma).filter(s) - - -class WrongLoginPage(HTMLPage): - pass - - -class AccountSuspendedPage(HTMLPage): - pass - - -class NoCardPage(HTMLPage): - def on_load(self): - raise ActionNeeded() - - -class LoginPage(HTMLPage): - def login(self, username, password): - form = self.get_form(name='ssoform') - form['UserID'] = username - form['USERID'] = username - form['Password'] = password - form['PWD'] = password - form.submit() - - -class AccountsPage(LoggedPage, PartialHTMLPage): - def get_account(self): - for div in self.doc.xpath('.//div[@id="card-details"]'): - a = Account() - a.id = CleanText().filter(div.xpath('.//span[@class="acc-num"]')) - a.label = CleanText().filter(div.xpath('.//span[@class="card-desc"]')) - a.type = Account.TYPE_CARD - balance = CleanText().filter(div.xpath('.//span[@class="balance-data"]')) - if balance in (u'Indisponible', u'Indisponible Facturation en cours', ''): - a.balance = NotAvailable - else: - a.currency = a.get_currency(balance) - a.balance = - abs(parse_decimal(balance)) - - # Cancel card don't have a link to watch history - link = self.doc.xpath('.//div[@class="wide-bar"]/h3/a') - if len(link) == 1: - a.url = urljoin(self.url, link[0].attrib['href']) - else: - a.url = None - - return a - - def get_idx_list(self): - fetched = False - for div in self.doc.xpath('//div[@id="card-list"]//div[has-class("card-details")]'): - _id = div.attrib['id'] - idx = re.match(r'card-(\d+)-detail', _id).group(1) - - message = CleanText('.//div[has-class("messages")]')(div).lower() - cancelled = ('annul' in message or 'cancel' in message) - - yield idx, cancelled - fetched = True - - if fetched: - return - - for div in self.doc.xpath('//div[@id="card-detail"]'): - idx = div.xpath('//span[@id="cardSortedIndex"]/@data')[0] - message = CleanText('.//div[has-class("messages")]')(div).lower() - cancelled = ('annul' in message or 'cancel' in message) - - yield idx, cancelled - return - - def get_session(self): - return self.doc.xpath('//form[@id="j-session-form"]//input[@name="Hidden"]/@value') - - -class AccountsPage2(LoggedPage, PartialHTMLPage): - @method - class iter_accounts(ListElement): - item_xpath = '//table[@id="summaryTable"]' - - class item(ItemElement): - klass = Account - - def condition(self): - if 'Votre carte est annulée' in CleanText('.//span[@id="cardSORStatus"]')(self): - self.logger.warning('skipping cancelled card %r', self.obj_id(self)) - return False - return True - - obj_id = CleanText('.//td[@class="cardArtColWidth"]/div[@class="summaryTitles"]') - obj_label = CleanText('.//span[@class="cardTitle"]') - obj_type = Account.TYPE_CARD - - obj_currency = CleanCurrency('.//td[@id="colOSBalance"]/div[@class="summaryValues makeBold"]') - - def obj_balance(self): - return -abs(parse_decimal(CleanText('.//td[@id="colOSBalance"]/div[@class="summaryValues makeBold"]')(self))) - - obj_url = AbsoluteLink('.//a[text()="View Latest Transactions"]', default=AbsoluteLink('.//a[span[text()="Online Statement"] or text()="Détail de vos opérations"]')) - - -class TransactionsPage(LoggedPage, HTMLPage): - def is_last(self): - current = False - for option in self.doc.xpath('//select[@id="viewPeriod"]/option'): - if 'selected' in option.attrib: - current = True - elif current: - return False - - return True - - def get_end_debit_date(self): - for option in self.doc.xpath('//select[@id="viewPeriod"]/option'): - if 'selected' in option.attrib: - m = re.search('(\d+) ([\w\.]+) (\d{4})$', option.text.strip(), re.UNICODE) - if m: - return datetime.date(int(m.group(3)), - self.parse_month(m.group(2)), - int(m.group(1))) - - def get_beginning_debit_date(self): - for option in self.doc.xpath('//select[@id="viewPeriod"]/option'): - if 'selected' in option.attrib: - m = re.search('^(\d+) ([\w\.]+) (\d{4})', option.text.strip(), re.UNICODE) - if m: - return datetime.date(int(m.group(3)), - self.parse_month(m.group(2)), - int(m.group(1))) - return datetime.date.today() - - COL_DATE = 0 - COL_TEXT = 1 - COL_CREDIT = -2 - COL_DEBIT = -1 - - FR_MONTHS = ['janv', u'févr', u'mars', u'avr', u'mai', u'juin', u'juil', u'août', u'sept', u'oct', u'nov', u'déc'] - US_MONTHS = ['Jan', u'Feb', u'Mar', u'Apr', u'May', u'Jun', u'Jul', u'Aug', u'Sep', u'Oct', u'Nov', u'Dec'] - - @classmethod - def parse_month(cls, s): - # there can be fr or us labels even if currency is EUR - s = s.rstrip('.') - try: - return cls.FR_MONTHS.index(s) + 1 - except ValueError: - return cls.US_MONTHS.index(s) + 1 - - def get_next_url(self): - options = self.doc.xpath('//select[@id="viewPeriod"]//option') - url = 'https://global.americanexpress.com/myca/intl/estatement/emea/statement.do?sorted_index=0&BPIndex={}&Face=fr_FR' - return [url.format(option.attrib['value']) for option in options] - - def get_history(self, account): - # checking if the card is still valid - if self.doc.xpath('//div[@id="errorbox"]'): - return - - # adding a time delta because amex have hard time to put the date in a good interval - beginning_date = self.get_beginning_debit_date() - datetime.timedelta(days=360) - end_date = self.get_end_debit_date() - - guesser = ChaoticDateGuesser(beginning_date, end_date) - - # Since the site doesn't provide the debit_date, - # we just use the date of beginning of the previous period. - # If this date + 1 month is greater than today's date, - # then the transaction is coming - end_of_period = None - previous_date = CleanText('//td[@id="colStatementBalance"]/div[3]', default=None)(self.doc) - if previous_date: - end_of_period = (parse_french_date(' '.join(previous_date.split()[1:4])) + relativedelta(months=1)).date() - else: - previous_date = CleanText('//select[@id="viewPeriod"]/option[@selected]', default=None)(self.doc) - if previous_date: - end_of_period = parse_french_date(' '.join(previous_date.split()[:3])) + relativedelta(days=-1) + relativedelta(months=1) - end_of_period = end_of_period.date() - - _id = str(int(account._idforold)) - for tr in reversed(self.doc.xpath('//div[@id="txnsSection"]//tbody[@id="tableBody-txnsCard%s"]/tr[@class="tableStandardText"]' % _id)): - cols = tr.findall('td') - - t = Transaction() - - day, month = CleanText().filter(cols[self.COL_DATE]).split(' ', 1) - day = int(day) - month = self.parse_month(month) - date = guesser.guess_date(day, month) - - vdate = None - try: - detail = cols[self.COL_TEXT].xpath('./div[has-class("hiddenROC")]')[0] - except IndexError: - pass - else: - m = re.search(r' (\d{2} \D{3,4})', (' '.join([txt.strip() for txt in detail.itertext()])).strip()) - if m: - vday, vmonth = m.group(1).strip().split(' ') - vday = int(vday) - vmonth = self.parse_month(vmonth) - vdate = guesser.guess_date(vday, vmonth) - detail.drop_tree() - - raw = (' '.join([txt.strip() for txt in cols[self.COL_TEXT].itertext()])).strip() - credit = CleanText().filter(cols[self.COL_CREDIT]) - debit = CleanText().filter(cols[self.COL_DEBIT]) - if end_of_period is not None and datetime.date.today() < end_of_period: - t._is_coming = True - else: - t._is_coming = False - - t.date = t.rdate = date - t.vdate = vdate - t.raw = re.sub(r'[ ]+', ' ', raw) - t.label = re.sub('(.*?)( \d+)? .*', r'\1', raw).strip() - t.amount = parse_decimal(credit or debit) * (1 if credit else -1) - if t.raw in self.browser.SUMMARY_CARD_LABEL: - t.type = t.TYPE_CARD_SUMMARY - elif t.amount > 0: - t.type = t.TYPE_ORDER - else: - t.date = end_of_period - t.type = t.TYPE_DEFERRED_CARD - - yield t - - -class ActionNeededPage(LoggedPage, HTMLPage): - def on_load(self): - if self.doc.xpath('//meta[contains(@content,"action/home?request_type=un_Activation")]'): - raise ActionNeeded() diff --git a/modules/americanexpress/pages/json.py b/modules/americanexpress/pages/json.py deleted file mode 100644 index 55ff15780a3242236a7db27b2344a2f134a08d64..0000000000000000000000000000000000000000 --- a/modules/americanexpress/pages/json.py +++ /dev/null @@ -1,167 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright(C) 2017 Vincent Ardisson -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . - -from __future__ import unicode_literals - -from ast import literal_eval -from decimal import Decimal -import re - -from weboob.browser.pages import LoggedPage, JsonPage, HTMLPage -from weboob.browser.elements import ItemElement, DictElement, method -from weboob.browser.filters.standard import Date, Eval, CleanText, Field, CleanDecimal -from weboob.browser.filters.json import Dict -from weboob.capabilities.bank import Account, Transaction -from weboob.capabilities.base import NotAvailable -from weboob.tools.json import json -from weboob.tools.compat import basestring -from .base import parse_decimal - - -def float_to_decimal(f): - return Decimal(str(f)) - - -class DashboardPage(LoggedPage, HTMLPage): - def get_user_key(self): - script = CleanText('//script[@id="initial-state"]', replace=[('\\', '')])(self.doc) - m = re.search(r'account_key\",(\"(.*?)\")', script) - if m: - return m.group(2) - return None - - -class AccountsPage3(LoggedPage, HTMLPage): - def iter_accounts(self): - for line in self.doc.xpath('//script[@id="initial-state"]')[0].text.split('\n'): - m = re.search('window.__INITIAL_STATE__ = (.*);', line) - if m: - data = json.loads(literal_eval(m.group(1))) - break - else: - assert False, "data was not found" - - assert data[13] == 'core' - assert len(data[14]) == 3 - - # search for products to get products list - for index, el in enumerate(data[14][2]): - if 'products' in el: - accounts_data = data[14][2][index+1] - - assert len(accounts_data) == 2 - assert accounts_data[1][4] == 'productsList' - - accounts_data = accounts_data[1][5] - token = [] - - for account_data in accounts_data: - if isinstance(account_data, basestring): - token2 = account_data - - elif isinstance(account_data, list) and not account_data[4][2][0]=="Canceled": - acc = Account() - if len(account_data) > 15: - token.append(account_data[-11]) - acc._idforJSON = account_data[10][-1] - else: - acc._idforJSON = account_data[-5][-1] - acc.number = '-%s' % account_data[2][2] - acc._idforold = account_data[2][6] - acc.label = '%s %s' % (account_data[6][4], account_data[10][-1]) - acc._token2 = acc.id = token2 - acc._token = token[-1] - yield acc - - -class JsonBalances(LoggedPage, JsonPage): - def set_balances(self, accounts): - by_token = {a._token2: a for a in accounts} - for d in self.doc: - # coming is what should be refunded at a futur deadline - by_token[d['account_token']].coming = -float_to_decimal(d['total_debits_balance_amount']) - # balance is what is currently due - by_token[d['account_token']].balance = -float_to_decimal(d['remaining_statement_balance_amount']) - - -class JsonBalances2(LoggedPage, JsonPage): - def set_balances(self, accounts): - by_token = {a._token2: a for a in accounts} - for d in self.doc: - by_token[d['account_token']].balance = -float_to_decimal(d['total']['payments_credits_total_amount']) - by_token[d['account_token']].coming = -float_to_decimal(d['total']['debits_total_amount']) - # warning: payments_credits_total_amount is not the coming value here - - -class CurrencyPage(LoggedPage, JsonPage): - def get_currency(self): - return self.doc['currency'] - - -class JsonPeriods(LoggedPage, JsonPage): - def get_periods(self): - return [p['statement_end_date'] for p in self.doc] - - -class JsonHistory(LoggedPage, JsonPage): - def get_count(self): - return self.doc['total_count'] - - @method - class iter_history(DictElement): - item_xpath = 'transactions' - - class item(ItemElement): - klass = Transaction - - def obj_type(self): - if Field('raw')(self) in self.page.browser.SUMMARY_CARD_LABEL: - return Transaction.TYPE_CARD_SUMMARY - elif Field('amount')(self) > 0: - return Transaction.TYPE_ORDER - else: - return Transaction.TYPE_DEFERRED_CARD - - obj_raw = CleanText(Dict('description', default='')) - obj_date = Date(Dict('statement_end_date', default=None), default=None) - obj_rdate = Date(Dict('charge_date')) - obj_vdate = Date(Dict('post_date', default=None), default=NotAvailable) - obj_amount = Eval(lambda x: -float_to_decimal(x), Dict('amount')) - obj_original_currency = Dict('foreign_details/iso_alpha_currency_code', default=NotAvailable) - obj_commission = CleanDecimal(Dict('foreign_details/commission_amount', default=NotAvailable), sign=lambda x: -1, default=NotAvailable) - obj__owner = CleanText(Dict('embossed_name')) - - def obj_original_amount(self): - # amount in the account's currency - amount = Field("amount")(self) - # amount in the transaction's currency - original_amount = Dict('foreign_details/amount', default=NotAvailable)(self) - if Field("original_currency")(self) == "XAF": - original_amount = abs(CleanDecimal(replace_dots=('.')).filter(original_amount)) - elif not original_amount: - return NotAvailable - else: - original_amount = abs(parse_decimal(original_amount)) - if amount < 0: - return -original_amount - else: - return original_amount - - #obj__ref = Dict('reference_id') - obj__ref = Dict('identifier') diff --git a/modules/amundi/compat/weboob_capabilities_bank.py b/modules/amundi/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/amundi/compat/weboob_capabilities_bank.py +++ b/modules/amundi/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/apivie/compat/weboob_capabilities_bank.py b/modules/apivie/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/apivie/compat/weboob_capabilities_bank.py +++ b/modules/apivie/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/aum/module.py b/modules/aum/module.py index fea8549288f8d946456be97328269e8213a5b79a..9ebb0cf2060844195e9c23b5d519beba879534ff 100644 --- a/modules/aum/module.py +++ b/modules/aum/module.py @@ -20,6 +20,7 @@ import time import datetime +from base64 import b64decode from html2text import unescape from dateutil import tz from dateutil.parser import parse as _parse_dt @@ -66,7 +67,7 @@ class AuMModule(Module, CapMessages, CapMessagesPost, CapDating, CapChat, CapCon Value('search_query', label='Search query', default='')) STORAGE = {'profiles_walker': {'viewed': []}, 'queries_queue': {'queue': []}, - 'sluts': {}, + 'contacts': {}, 'notes': {}, } BROWSER = AuMBrowser @@ -175,14 +176,14 @@ def get_thread(self, id, contacts=None, get_profiles=False): child = None msg = None - slut = self._get_slut(id) + contact = self._get_contact(id) if contacts is None: contacts = {} if not thread.title: thread.title = u'Discussion with %s' % mails['who']['pseudo'] - self.storage.set('sluts', int(thread.id), 'status', mails['status']) + self.storage.set('contacts', int(thread.id), 'status', mails['status']) self.storage.save() for mail in mails['results']: @@ -192,7 +193,7 @@ def get_thread(self, id, contacts=None, get_profiles=False): self.report_spam(thread.id) break - if parse_dt(mail['date']) > slut['lastmsg']: + if parse_dt(mail['date']) > contact['lastmsg']: flags |= Message.IS_UNREAD if get_profiles: @@ -261,8 +262,8 @@ def iter_unread_messages(self): self.logger.info('Skipped a spam-unread-thread from %s' % thread['who']['pseudo']) self.report_spam(thread['member']['id']) continue - slut = self._get_slut(thread['who']['id']) - if parse_dt(thread['date']) > slut['lastmsg'] or thread['status'] != slut['status']: + contact = self._get_contact(thread['who']['id']) + if parse_dt(thread['date']) > contact['lastmsg'] or thread['status'] != contact['status']: try: t = self.get_thread(thread['who']['id'], contacts, get_profiles=True) except BrowserUnavailable: @@ -275,17 +276,17 @@ def iter_unread_messages(self): return # Send mail when someone added me in her basket. - # XXX possibly race condition if a slut adds me in her basket + # XXX possibly race condition if a contact adds me in her basket # between the aum.nb_new_baskets() and aum.get_baskets(). with self.browser: - slut = self._get_slut(-self.MAGIC_ID_BASKET) + contact = self._get_contact(-self.MAGIC_ID_BASKET) new_baskets = self.browser.nb_new_baskets() if new_baskets > 0: baskets = self.browser.get_baskets() my_name = self.browser.get_my_name() for basket in baskets: - if parse_dt(basket['date']) <= slut['lastmsg']: + if parse_dt(basket['date']) <= contact['lastmsg']: continue contact = self.get_contact(basket['who']['id']) if self.antispam and not self.antispam.check_contact(contact): @@ -313,31 +314,33 @@ def iter_unread_messages(self): def set_message_read(self, message): if int(message.id) == self.MAGIC_ID_BASKET: # Save the last baskets checks. - slut = self._get_slut(-self.MAGIC_ID_BASKET) - if slut['lastmsg'] < message.date: - slut['lastmsg'] = message.date - self.storage.set('sluts', -self.MAGIC_ID_BASKET, slut) + contact = self._get_contact(-self.MAGIC_ID_BASKET) + if contact['lastmsg'] < message.date: + contact['lastmsg'] = message.date + self.storage.set('contacts', -self.MAGIC_ID_BASKET, contact) self.storage.save() return - slut = self._get_slut(message.thread.id) - if slut['lastmsg'] < message.date: - slut['lastmsg'] = message.date - self.storage.set('sluts', int(message.thread.id), slut) + contact = self._get_contact(message.thread.id) + if contact['lastmsg'] < message.date: + contact['lastmsg'] = message.date + self.storage.set('contacts', int(message.thread.id), contact) self.storage.save() - def _get_slut(self, id): + def _get_contact(self, id): id = int(id) - sluts = self.storage.get('sluts') - if not sluts or id not in sluts: - slut = {'lastmsg': datetime.datetime(1970,1,1), - 'status': None} + contacts = self.storage.get('contacts') + if not contacts or id not in contacts: + contacts = self.storage.get(b64decode('c2x1dHM=')) + if not contacts or id not in contacts: + contact = {'lastmsg': datetime.datetime(1970,1,1), + 'status': None} else: - slut = self.storage.get('sluts', id) + contact = contacts[id] - slut['lastmsg'] = slut.get('lastmsg', datetime.datetime(1970,1,1)).replace(tzinfo=tz.tzutc()) - slut['status'] = slut.get('status', None) - return slut + contact['lastmsg'] = contact.get('lastmsg', datetime.datetime(1970,1,1)).replace(tzinfo=tz.tzutc()) + contact['status'] = contact.get('status', None) + return contact # ---- CapMessagesPost methods --------------------- diff --git a/modules/axabanque/browser.py b/modules/axabanque/browser.py index dba151dd27eac638f8f0af255281d19cc0c070f9..6b3a44b42704990e4f0ef81b79b63036e24c2ccc 100644 --- a/modules/axabanque/browser.py +++ b/modules/axabanque/browser.py @@ -420,7 +420,7 @@ def iter_recipients(self, origin_account_id): for recipient in self.page.get_recipients(): if recipient.iban in seen: - recipient.category = 'EXTERNE' + recipient.category = 'Externe' yield recipient def copy_recipient_obj(self, recipient): @@ -428,7 +428,7 @@ def copy_recipient_obj(self, recipient): rcpt.id = recipient.iban rcpt.iban = recipient.iban rcpt.label = recipient.label - rcpt.category = 'EXTERNE' + rcpt.category = 'Externe' rcpt.enabled_at = date.today() rcpt.currency = 'EUR' return rcpt diff --git a/modules/axabanque/compat/weboob_capabilities_bank.py b/modules/axabanque/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/axabanque/compat/weboob_capabilities_bank.py +++ b/modules/axabanque/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/axabanque/pages/transfer.py b/modules/axabanque/pages/transfer.py index 3757e4a0638e2c09d63d44c00d1c7c77f860b672..a8f7e756582b67940a8190d1dd4aa15787bfc639 100644 --- a/modules/axabanque/pages/transfer.py +++ b/modules/axabanque/pages/transfer.py @@ -32,7 +32,7 @@ CleanText, Date, Regexp, CleanDecimal, Currency, Format, Field, ) from weboob.capabilities.bank import ( - Recipient, Transfer, TransferError, AddRecipientError, RecipientNotFound, + Recipient, Transfer, TransferBankError, AddRecipientError, RecipientNotFound, ) from .compat.weboob_tools_captcha_virtkeyboard import SimpleVirtualKeyboard from weboob.capabilities.base import find_object, NotAvailable @@ -54,7 +54,7 @@ class TransferVirtualKeyboard(SimpleVirtualKeyboard): '1': '12d398f7f389711c5f8298ee68a8af28', '2': 'f43ca3a5dd649d30bf02060ab65c4eff', '3': 'b6dd7864cfd941badb0784be37f7eeb3', - '4': '7138d0a663eef56c699d85dc6c3ac639', + '4': ('7138d0a663eef56c699d85dc6c3ac639', '0faced58777f371097a7a70bb9570dd7', ), '5': 'b71bd38e71ce0b611642a01b6900218f', '6': 'f71f7249413c189165da7b588c2f0493', '7': '81fc65230d7df341e80d02e414f183d4', @@ -95,7 +95,7 @@ class item(ItemElement): obj_id = CleanText(TableCell('id'), replace=[(' ', '')]) obj_iban = Field('id') obj_label = Format('%s - %s', CleanText(TableCell('_acc_name')), CleanText(TableCell('_rcpt_name'))) - obj_category = 'EXTERNE' + obj_category = 'Externe' obj_enabled_at = date.today() obj_currency = 'EUR' obj_bank_name = NotAvailable @@ -218,7 +218,7 @@ def on_load(self): error_xpath = '//span[@class="erreur_phrase"]' if self.doc.xpath(error_xpath): error_msg = CleanText(error_xpath)(self.doc) - raise TransferError(message=error_msg) + raise TransferBankError(message=error_msg) def is_transfer_account(self, acc_id): valide_accounts_xpath = '//select[@id="compteEmetteurSelectionne"]//option[not(contains(@value,"vide0"))]' @@ -255,7 +255,7 @@ def get_recipients(self): rcpt.iban = CleanText('./@value')(recipient) rcpt.id = rcpt.iban rcpt.enabled_at = date.today() - rcpt.category = 'INTERNE' + rcpt.category = 'Interne' yield rcpt @@ -357,7 +357,7 @@ class ConfirmTransferPage(LoggedPage, HTMLPage): def on_load(self): error_msg = '//p[@id="messErreur"]/span' if self.doc.xpath(error_msg): - raise TransferError(message=CleanText(error_msg)(self.doc)) + raise TransferBankError(message=CleanText(error_msg)(self.doc)) confirm_transfer_xpath = '//h2[contains(text(), "Virement enregistr")]' assert self.doc.xpath(confirm_transfer_xpath) diff --git a/modules/axabanque/pages/wealth.py b/modules/axabanque/pages/wealth.py index 40ce75d93b610b7390cdb31c428a45c9e15c4f5a..83f6c4176272a955075ba5754291d6f807322578 100644 --- a/modules/axabanque/pages/wealth.py +++ b/modules/axabanque/pages/wealth.py @@ -41,11 +41,12 @@ class AccountsPage(LoggedPage, HTMLPage): TYPES = {u'assurance vie': Account.TYPE_LIFE_INSURANCE, u'perp': Account.TYPE_PERP, u'novial avenir': Account.TYPE_MADELIN, + u'epargne retraite novial': Account.TYPE_LIFE_INSURANCE, } @method class iter_accounts(ListElement): - item_xpath = '//div[contains(@data-route, "/assurance-vie/")]' + item_xpath = '//div[contains(@data-route, "/savings/")]' class item(ItemElement): klass = Account @@ -56,7 +57,14 @@ class item(ItemElement): obj_label = CleanText('.//h3[has-class("card-title")]') obj_balance = MyDecimal('.//p[has-class("amount-card")]') obj_valuation_diff = MyDecimal('.//p[@class="performance"]') - obj_url = Attr('.', 'data-route') + + def obj_url(self): + url = Attr('.', 'data-route')(self) + # The Assurance Vie xpath recently changed so we must verify that all + # the accounts now have "/savings/" instead of "/assurances-vie/". + assert "/savings/" in url + return url + obj_currency = Currency('.//p[has-class("amount-card")]') obj__acctype = "investment" diff --git a/modules/banquepopulaire/browser.py b/modules/banquepopulaire/browser.py index a3e860f447f96267d41e9969a59fab05ad776a49..ab5374f94d6703024b22a369425745d1660b0bad 100644 --- a/modules/banquepopulaire/browser.py +++ b/modules/banquepopulaire/browser.py @@ -96,7 +96,6 @@ def wrapper(browser, *args, **kwargs): class BanquePopulaire(LoginBrowser): - VERIFY = False login_page = URL(r'https://[^/]+/auth/UI/Login.*', LoginPage) index_page = URL(r'https://[^/]+/cyber/internet/Login.do', IndexPage) accounts_page = URL(r'https://[^/]+/cyber/internet/StartTask.do\?taskInfoOID=mesComptes.*', @@ -128,9 +127,14 @@ class BanquePopulaire(LoginBrowser): transaction_detail_page = URL(r'https://[^/]+/cyber/internet/ContinueTask.do\?.*dialogActionPerformed=DETAIL_ECRITURE.*', TransactionDetailPage) - error_page = URL(r'https://[^/]+/cyber/internet/ContinueTask.do', ErrorPage) + error_page = URL(r'https://[^/]+/cyber/internet/ContinueTask.do', + r'https://[^/]+/_layouts/error.aspx', + ErrorPage) + unavailable_page = URL(r'https://[^/]+/s3f-web/.*', - r'https://[^/]+/static/errors/nondispo.html', UnavailablePage) + r'https://[^/]+/static/errors/nondispo.html', + r'/i-RIA/swc/1.0.0/desktop/index.html', + UnavailablePage) redirect_page = URL(r'https://[^/]+/portailinternet/_layouts/Ibp.Cyi.Layouts/RedirectSegment.aspx.*', RedirectPage) home_page = URL(r'https://[^/]+/portailinternet/Catalogue/Segments/.*.aspx(\?vary=(?P.*))?', diff --git a/modules/banquepopulaire/compat/weboob_capabilities_bank.py b/modules/banquepopulaire/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/banquepopulaire/compat/weboob_capabilities_bank.py +++ b/modules/banquepopulaire/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/banquepopulaire/linebourse_browser.py b/modules/banquepopulaire/linebourse_browser.py index 01da37ef6424ed6e2867f9e22b8ae7dd54145a19..1387384694e14b3ef15427b43da11b5bd6298a92 100644 --- a/modules/banquepopulaire/linebourse_browser.py +++ b/modules/banquepopulaire/linebourse_browser.py @@ -23,4 +23,3 @@ class LinebourseBrowser(AbstractBrowser): PARENT = 'linebourse' - VERIFY = False diff --git a/modules/banquepopulaire/pages.py b/modules/banquepopulaire/pages.py index 1628b9b48a1396a9ca0ec9424bc48543d8d6ecda..f6b75fa29c623ec406aeadb2e60055271b995f5c 100644 --- a/modules/banquepopulaire/pages.py +++ b/modules/banquepopulaire/pages.py @@ -38,6 +38,7 @@ from weboob.capabilities.contact import Advisor from weboob.capabilities import NotAvailable from weboob.tools.capabilities.bank.transactions import FrenchTransaction +from weboob.tools.decorators import retry from weboob.tools.compat import urlsplit, parse_qsl from weboob.tools.json import json from weboob.tools.misc import to_unicode @@ -274,6 +275,9 @@ def on_load(self): h1 = CleanText('//h1[1]')(self.doc) if "est indisponible" in h1: raise BrowserUnavailable(h1) + body = CleanText(".")(self.doc) + if "An unexpected error has occurred." in body: + raise BrowserUnavailable(body) a = Link('//a[@class="btn"][1]', default=None)(self.doc) if not a: @@ -387,6 +391,9 @@ def build_doc(self, data, *args, **kwargs): return None return super(MyHTMLPage, self).build_doc(data, *args, **kwargs) + @retry(KeyError) + # sometime the server redirects to a bad url, not containing token. + # therefore "return args['token']" crashes with a KeyError def get_token(self): vary = None if self.params.get('vary', None) is not None: @@ -437,6 +444,7 @@ class AccountsPage(LoggedPage, MyHTMLPage): u'Liste complète de mon épargne': Account.TYPE_SAVINGS, u'Mes comptes': Account.TYPE_CHECKING, u'Comptes en euros': Account.TYPE_CHECKING, + 'Mes comptes en devises': Account.TYPE_CHECKING, u'Liste complète de mes comptes': Account.TYPE_CHECKING, u'Mes emprunts': Account.TYPE_LOAN, u'Liste complète de mes emprunts': Account.TYPE_LOAN, @@ -527,9 +535,14 @@ def iter_accounts(self, next_pages): else: account.type = account_type - balance = FrenchTransaction.clean_amount(u''.join([txt.strip() for txt in tds[3].itertext()])) + balance_text = u''.join([txt.strip() for txt in tds[3].itertext()]) + balance = FrenchTransaction.clean_amount(balance_text) account.balance = Decimal(balance or '0.0') - account.currency = currency + + if currency is None: + account.currency = Account.get_currency(balance_text) + else: + account.currency = currency if account.type == account.TYPE_LOAN: account.balance = - abs(account.balance) diff --git a/modules/barclays/browser.py b/modules/barclays/browser.py index ee3e9f03d13e8afec275a277051718ead79bf1ff..bf66302b99c0882cfcf8206c2268813c9c094ecd 100644 --- a/modules/barclays/browser.py +++ b/modules/barclays/browser.py @@ -34,7 +34,6 @@ class Barclays(LoginBrowser): - VERIFY = 'certificate.pem' BASEURL = 'https://client.milleis.fr' logout = URL('https://www.milleis.fr/deconnexion') diff --git a/modules/barclays/compat/weboob_capabilities_bank.py b/modules/barclays/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/barclays/compat/weboob_capabilities_bank.py +++ b/modules/barclays/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/becm/browser.py b/modules/becm/browser.py index 0163affba8de338993491f7f8110314242f7ebed..477e65941bce1f76473aa306a5cf3b3c733c2342 100644 --- a/modules/becm/browser.py +++ b/modules/becm/browser.py @@ -17,11 +17,13 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals from weboob.browser.browsers import AbstractBrowser from weboob.browser.profiles import Wget from weboob.browser.url import URL from weboob.browser.browsers import need_login +from .compat.weboob_exceptions import ActionNeeded, AuthMethodNotImplemented from .pages import AdvisorPage, LoginPage @@ -38,6 +40,23 @@ class BECMBrowser(AbstractBrowser): login = URL('/fr/authentification.html', LoginPage) advisor = URL('/fr/banques/Details.aspx\?banque=.*', AdvisorPage) + def do_login(self): + # Clear cookies. + self.do_logout() + self.login.go() + if not self.page.logged: + self.page.login(self.username, self.password) + # Many "Credit Mutuel" customers tried to add their connection to BECM, but the BECM + # website does not return any error when you try to login with correct Crédit Mutuel + # credentials, therefore we must suggest them to try regular Crédit Mutuel if login fails. + if self.login.is_here(): + raise ActionNeeded("La connexion au site de BECM n'a pas fonctionné avec les identifiants fournis.\ + Si vous êtes client du Crédit Mutuel, veuillez réessayer en sélectionnant le module Crédit Mutuel.") + + if self.verify_pass.is_here(): + raise AuthMethodNotImplemented("L'identification renforcée avec la carte n'est pas supportée.") + + @need_login def get_advisor(self): advisor = None diff --git a/modules/americanexpress/pages/__init__.py b/modules/becm/compat/__init__.py similarity index 100% rename from modules/americanexpress/pages/__init__.py rename to modules/becm/compat/__init__.py diff --git a/modules/becm/compat/weboob_exceptions.py b/modules/becm/compat/weboob_exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..e1912dfbf13e2acd7d6a676b5765d962a734925f --- /dev/null +++ b/modules/becm/compat/weboob_exceptions.py @@ -0,0 +1,46 @@ + +from weboob.exceptions import * + + +class AuthMethodNotImplemented(Exception): + pass + + +class CaptchaQuestion(Exception): + """Site requires solving a CAPTCHA (base class)""" + # could be improved to pass the name of the backendconfig key + + def __init__(self, type=None, **kwargs): + super(CaptchaQuestion, self).__init__("The site requires solving a captcha") + self.type = type + for key, value in kwargs.items(): + setattr(self, key, value) + + +class ImageCaptchaQuestion(CaptchaQuestion): + type = 'image_captcha' + + image_data = None + + def __init__(self, image_data): + super(ImageCaptchaQuestion, self).__init__(self.type, image_data=image_data) + + +class NocaptchaQuestion(CaptchaQuestion): + type = 'g_recaptcha' + + website_key = None + website_url = None + + def __init__(self, website_key, website_url): + super(NocaptchaQuestion, self).__init__(self.type, website_key=website_key, website_url=website_url) + + +class RecaptchaQuestion(CaptchaQuestion): + type = 'g_recaptcha' + + website_key = None + website_url = None + + def __init__(self, website_key, website_url): + super(RecaptchaQuestion, self).__init__(self.type, website_key=website_key, website_url=website_url) diff --git a/modules/becm/pages.py b/modules/becm/pages.py index 1c21db9fb1e3c0f8460289b9f8e14b69e6b5ffbd..74eeb05db8402623da6699155e0d7a12d0151e77 100644 --- a/modules/becm/pages.py +++ b/modules/becm/pages.py @@ -22,6 +22,7 @@ from weboob.browser.elements import method, ItemElement from weboob.browser.filters.standard import CleanText, Format from weboob.capabilities import NotAvailable +from weboob.exceptions import BrowserIncorrectPassword class LoginPage(HTMLPage): @@ -37,6 +38,11 @@ def login(self, login, passwd): def logged(self): return self.doc.xpath('//div[@id="e_identification_ok"]') + def on_load(self): + error_msg_xpath = '//div[has-class("err")]//p[contains(text(), "votre mot de passe est faux")]' + if self.doc.xpath(error_msg_xpath): + raise BrowserIncorrectPassword(CleanText(error_msg_xpath)(self.doc)) + class AdvisorPage(LoggedPage, HTMLPage): @method diff --git a/modules/bforbank/compat/weboob_capabilities_bank.py b/modules/bforbank/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/bforbank/compat/weboob_capabilities_bank.py +++ b/modules/bforbank/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/bforbank/pages.py b/modules/bforbank/pages.py index 2247d7095afa5bddfc0acd618702cd17841a2fa1..c95f3723fed24aef2727b821dab4eb489079d8d8 100644 --- a/modules/bforbank/pages.py +++ b/modules/bforbank/pages.py @@ -303,7 +303,8 @@ def get_cards(self, account_id): assert len(divs) msgs = re.compile(u'Vous avez fait opposition sur cette carte bancaire.' + '|Votre carte bancaire a été envoyée.' + - '|BforBank a fait opposition sur votre carte') + '|BforBank a fait opposition sur votre carte' + + '|Pour des raisons de sécurité, la demande de réception du code confidentiel de votre carte par SMS est indisponible') divs = [d for d in divs if not msgs.search(CleanText('.//div[has-class("alert")]', default='')(d))] divs = [d.xpath('.//div[@class="m-card-infos"]')[0] for d in divs] divs = [d for d in divs if not d.xpath('.//div[@class="m-card-infos-body-text"][text()="Débit immédiat"]')] diff --git a/modules/binck/compat/weboob_capabilities_bank.py b/modules/binck/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/binck/compat/weboob_capabilities_bank.py +++ b/modules/binck/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/bnporc/compat/weboob_capabilities_bank.py b/modules/bnporc/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/bnporc/compat/weboob_capabilities_bank.py +++ b/modules/bnporc/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/bnporc/pp/browser.py b/modules/bnporc/pp/browser.py index d432c156f63e9367102f1b4fa40d4fa5a4079457..cbf29d2f613ded7d263e1f078a77c6f1970b11bf 100644 --- a/modules/bnporc/pp/browser.py +++ b/modules/bnporc/pp/browser.py @@ -82,6 +82,7 @@ class BNPParibasBrowser(JsonBrowserMixin, LoginBrowser): '/fr/espace-pro/changer-son-mot-de-passe', '/fr/espace-client/100-connexions', '/fr/espace-prive/mot-de-passe-expire', + '/fr/client/100-connexion', '/fr/systeme/page-indisponible', ConnectionThresholdPage) accounts = URL('udc-wspl/rest/getlstcpt', AccountsPage) ibans = URL('rib-wspl/rpc/comptes', AccountsIBANPage) diff --git a/modules/bnporc/pp/pages.py b/modules/bnporc/pp/pages.py index 6053fd0eeedf5bd6b11370d3ce25144ed73e1f0f..439be56989618022452253335eff8d58412b988d 100644 --- a/modules/bnporc/pp/pages.py +++ b/modules/bnporc/pp/pages.py @@ -610,20 +610,20 @@ def get_params(self): 'init': 'true', 'multiInit': 'false', } - params['a0'] = self.doc['data']['nationVieProInfos']['a0'] - # The number of "p" keys may vary (p0, p1, p2 ... up to p13 or more) - for key, value in self.doc['data']['nationVieProInfos']['listeP'].items(): - params[key] = value - # We must decode the values before constructing the URL: - for k, v in params.items(): - params[k] = unquote_plus(v) - return params + params['a0'] = self.doc['data']['nationVieProInfos']['a0'] + # The number of "p" keys may vary (p0, p1, p2 ... up to p13 or more) + for key, value in self.doc['data']['nationVieProInfos']['listeP'].items(): + params[key] = value + # We must decode the values before constructing the URL: + for k, v in params.items(): + params[k] = unquote_plus(v) + return params class CapitalisationPage(LoggedPage, HTMLPage): def has_contracts(self): - # This message will appear if the page "Assurance Vie" contains no contract. - return not CleanText('//td[@class="message"]/text()[starts-with(., "Pour toute information")]')(self.doc) + # This message will appear if the page "Assurance Vie" contains no contract. + return not CleanText('//td[@class="message"]/text()[starts-with(., "Pour toute information")]')(self.doc) # To be completed with other account labels and types seen on the "Assurance Vie" space: ACCOUNT_TYPES = { @@ -635,32 +635,32 @@ def has_contracts(self): @method class iter_capitalisation(TableElement): - # Other types of tables may appear on the page (such as Alternative Emprunteur/Capital Assuré) - # But they do not contain bank accounts so we must avoid them. + # Other types of tables may appear on the page (such as Alternative Emprunteur/Capital Assuré) + # But they do not contain bank accounts so we must avoid them. item_xpath = '//table/tr[preceding-sibling::tr[th[text()="Libellé du contrat"]]][td[@class="ligneTableau"]]' - head_xpath = '//table/tr/th[@class="headerTableau"]' + head_xpath = '//table/tr/th[@class="headerTableau"]' - col_label = 'Libellé du contrat' - col_id = 'Numéro de contrat' - col_balance = 'Montant' - col_currency = "Monnaie d'affichage" + col_label = 'Libellé du contrat' + col_id = 'Numéro de contrat' + col_balance = 'Montant' + col_currency = "Monnaie d'affichage" class item(ItemElement): klass = Account obj_label = CleanText(TableCell('label')) - obj_id = CleanText(TableCell('id')) + obj_id = CleanText(TableCell('id')) obj_number = CleanText(TableCell('id')) obj_balance = CleanDecimal(TableCell('balance'), replace_dots=True) - obj_coming = None - obj_iban = None + obj_coming = None + obj_iban = None def obj_type(self): - for k, v in self.page.ACCOUNT_TYPES.items(): - if Field('label')(self).startswith(k): - return v - return Account.TYPE_UNKNOWN + for k, v in self.page.ACCOUNT_TYPES.items(): + if Field('label')(self).startswith(k): + return v + return Account.TYPE_UNKNOWN def obj_currency(self): currency = CleanText(TableCell('currency')(self))(self) @@ -670,55 +670,55 @@ def obj_currency(self): def obj__details(self): raw_details = CleanText((TableCell('balance')(self)[0]).xpath('./a/@href'))(self) m = re.search(r"Window\('(.*?)',window", raw_details) - if m: + if m: return m.group(1) def get_params(self, account): - form = self.get_form(xpath='//form[@name="formListeContrats"]') - form['postValue'] = account._details - return form + form = self.get_form(xpath='//form[@name="formListeContrats"]') + form['postValue'] = account._details + return form # The investments vdate is out of the investments table and is the same for all investments: def get_vdate(self): - return parse_french_date(CleanText('//table[tr[th[text()[contains(., "Date de valorisation")]]]]/tr[2]/td[2]')(self.doc)) + return parse_french_date(CleanText('//table[tr[th[text()[contains(., "Date de valorisation")]]]]/tr[2]/td[2]')(self.doc)) @method class iter_investments(TableElement): - # Investment lines contain at least 5 tags + # Investment lines contain at least 5 tags item_xpath = '//table[tr[th[text()[contains(., "Libellé")]]]]/tr[count(td)>=5]' - head_xpath = '//table[tr[th[text()[contains(., "Libellé")]]]]/tr/th[@class="headerTableau"]' + head_xpath = '//table[tr[th[text()[contains(., "Libellé")]]]]/tr/th[@class="headerTableau"]' - col_label = 'Libellé' - col_code = 'Code ISIN' - col_quantity = 'Nombre de parts' - col_valuation = 'Montant' - col_portfolio_share = 'Montant en %' + col_label = 'Libellé' + col_code = 'Code ISIN' + col_quantity = 'Nombre de parts' + col_valuation = 'Montant' + col_portfolio_share = 'Montant en %' class item(ItemElement): klass = Investment - obj_label = CleanText(TableCell('label')) - obj_valuation = CleanDecimal(TableCell('valuation'), replace_dots=True) - obj_portfolio_share = Eval(lambda x: x / 100, CleanDecimal(TableCell('portfolio_share'), replace_dots=True)) - # There is no "unitvalue" information available on the "Assurances Vie" space. - - def obj_quantity(self): - quantity = TableCell('quantity')(self) - if CleanText(quantity)(self) == '-': - return NotAvailable - return CleanDecimal(quantity, replace_dots=True)(self) - - def obj_code(self): - isin = CleanText(TableCell('code')(self))(self) - return isin or NotAvailable - - def obj_code_type(self): - if is_isin_valid(Field('code')(self)): - return Investment.CODE_TYPE_ISIN - return NotAvailable - - def obj_vdate(self): - return self.page.get_vdate() + obj_label = CleanText(TableCell('label')) + obj_valuation = CleanDecimal(TableCell('valuation'), replace_dots=True) + obj_portfolio_share = Eval(lambda x: x / 100, CleanDecimal(TableCell('portfolio_share'), replace_dots=True)) + # There is no "unitvalue" information available on the "Assurances Vie" space. + + def obj_quantity(self): + quantity = TableCell('quantity')(self) + if CleanText(quantity)(self) == '-': + return NotAvailable + return CleanDecimal(quantity, replace_dots=True)(self) + + def obj_code(self): + isin = CleanText(TableCell('code')(self))(self) + return isin or NotAvailable + + def obj_code_type(self): + if is_isin_valid(Field('code')(self)): + return Investment.CODE_TYPE_ISIN + return NotAvailable + + def obj_vdate(self): + return self.page.get_vdate() class MarketListPage(BNPPage): diff --git a/modules/bolden/__init__.py b/modules/bolden/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6e42ccdbf4371f87d47a2c34381d55d9185e1b0c --- /dev/null +++ b/modules/bolden/__init__.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + + +from .module import BoldenModule + + +__all__ = ['BoldenModule'] diff --git a/modules/bolden/browser.py b/modules/bolden/browser.py new file mode 100644 index 0000000000000000000000000000000000000000..b76a339afa798413c190a3913c0bf68a508373ff --- /dev/null +++ b/modules/bolden/browser.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +from datetime import timedelta, datetime + +from weboob.browser import LoginBrowser, need_login, URL +from weboob.capabilities.bill import Document + +from .pages import ( + LoginPage, HomeLendPage, PortfolioPage, OperationsPage, MAIN_ID, ProfilePage, +) + + +class BoldenBrowser(LoginBrowser): + BASEURL = 'https://bolden.fr/' + + login = URL(r'/connexion', LoginPage) + home_lend = URL(r'/tableau-de-bord-investisseur', HomeLendPage) + profile = URL(r'/mon-profil', ProfilePage) + portfolio = URL(r'/InvestorDashboard/GetPortfolio', PortfolioPage) + operations = URL(r'/InvestorDashboard/GetOperations\?startDate=(?P[\d-]+)&endDate=(?P[\d-]+)', OperationsPage) + + def do_login(self): + self.login.go() + self.page.do_login(self.username, self.password) + + if self.login.is_here(): + self.page.check_error() + assert False, 'should not be on login page' + + @need_login + def iter_accounts(self): + self.portfolio.go() + return self.page.iter_accounts() + + @need_login + def iter_history(self, account): + if account.id != MAIN_ID: + return [] + return self._iter_all_history() + + def _iter_all_history(self): + end = datetime.now() + while True: + start = end - timedelta(days=365) + + self.operations.go(start=start.strftime('%Y-%m-%d'), end=end.strftime('%Y-%m-%d')) + transactions = list(self.page.iter_history()) + if not transactions: + break + + last_with_date = None + for tr in transactions: + if tr.date is None: + tr.date = last_with_date.date + tr.label = '%s %s' % (last_with_date.label, tr.label) + else: + last_with_date = tr + + yield tr + + end = start + + @need_login + def get_profile(self): + self.profile.go() + return self.page.get_profile() + + @need_login + def iter_documents(self): + for acc in self.iter_accounts(): + if acc.id == MAIN_ID: + continue + + doc = Document() + doc.id = acc.id + doc.url = acc._docurl + doc.label = 'Contrat %s' % acc.label + doc.type = 'other' + doc.format = 'pdf' + yield doc diff --git a/modules/bolden/module.py b/modules/bolden/module.py new file mode 100644 index 0000000000000000000000000000000000000000..7e51b2081fb9239480f259db075da1dde00714db --- /dev/null +++ b/modules/bolden/module.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + + +from weboob.tools.backend import Module, BackendConfig +from weboob.tools.value import ValueBackendPassword +from weboob.capabilities.bank import CapBank, Account +from weboob.capabilities.base import find_object +from weboob.capabilities.bill import ( + CapDocument, Subscription, SubscriptionNotFound, DocumentNotFound, Document, +) +from weboob.capabilities.profile import CapProfile + +from .browser import BoldenBrowser + + +__all__ = ['BoldenModule'] + + +class BoldenModule(Module, CapBank, CapDocument, CapProfile): + NAME = 'bolden' + DESCRIPTION = 'Bolden' + MAINTAINER = 'Vincent A' + EMAIL = 'dev@indigo.re' + LICENSE = 'AGPLv3+' + VERSION = '1.3' + + BROWSER = BoldenBrowser + + CONFIG = BackendConfig( + ValueBackendPassword('login', label='Email', masked=False), + ValueBackendPassword('password', label='Mot de passe'), + ) + + def create_default_browser(self): + return self.create_browser(self.config['login'].get(), self.config['password'].get()) + + def iter_accounts(self): + return self.browser.iter_accounts() + + def iter_history(self, account): + return self.browser.iter_history(account) + + def get_profile(self): + return self.browser.get_profile() + + def iter_subscription(self): + sub = Subscription() + sub.id = '_bolden_' + sub.subscriber = self.get_profile().name + sub.label = 'Bolden %s' % sub.subscriber + return [sub] + + def get_subscription(self, _id): + if _id == '_bolden_': + return self.iter_subscription()[0] + raise SubscriptionNotFound() + + def iter_documents(self, sub): + if not isinstance(sub, Subscription): + sub = self.get_subscription(sub) + return self.browser.iter_documents() + + def get_document(self, id): + return find_object(self.browser.iter_documents(), id=id, error=DocumentNotFound) + + def download_document(self, doc): + if not isinstance(doc, Document): + doc = self.get_document(doc) + return self.browser.open(doc.url).content + + def iter_resources(self, objs, split_path): + if Account in objs: + self._restrict_level(split_path) + return self.iter_accounts() + if Subscription in objs: + self._restrict_level(split_path) + return self.iter_subscription() diff --git a/modules/bolden/pages.py b/modules/bolden/pages.py new file mode 100644 index 0000000000000000000000000000000000000000..9f9ee35f20d787ac6e5f5ad67ff5e860fa4eaca8 --- /dev/null +++ b/modules/bolden/pages.py @@ -0,0 +1,137 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +from decimal import Decimal + +from weboob.browser.elements import ListElement, ItemElement, method, TableElement +from weboob.browser.filters.html import TableCell, Link, Attr +from weboob.browser.filters.standard import ( + CleanText, CleanDecimal, Slugify, Date, Field, Format, +) +from weboob.browser.pages import HTMLPage, LoggedPage +from weboob.capabilities.bank import Account, Transaction +from weboob.capabilities.profile import Profile +from weboob.exceptions import BrowserIncorrectPassword +from weboob.tools.compat import urljoin + + +MAIN_ID = '_bolden_' + +class LoginPage(HTMLPage): + def do_login(self, username, password): + form = self.get_form(id='loginform') + form['Email'] = username + form['Password'] = password + form.submit() + + def check_error(self): + msg = CleanText('//div[has-class("validation-summary-errors")]')(self.doc) + if 'Tentative de connexion invalide' in msg: + raise BrowserIncorrectPassword(msg) + + +class HomeLendPage(LoggedPage, HTMLPage): + pass + + +class PortfolioPage(LoggedPage, HTMLPage): + @method + class iter_accounts(ListElement): + class get_main(ItemElement): + klass = Account + + obj_id = MAIN_ID + obj_label = 'Compte Bolden' + obj_type = Account.TYPE_CHECKING + obj_currency = 'EUR' + obj_balance = CleanDecimal('//div[p[has-class("investor-state") and contains(text(),"Fonds disponibles :")]]/p[has-class("investor-status")]', replace_dots=True) + #obj_coming = CleanDecimal('//div[p[has-class("investor-state") and contains(text(),"Capital restant dû :")]]/p[has-class("investor-status")]', replace_dots=True) + + class iter_lends(TableElement): + head_xpath = '//div[@class="tab-wallet"]/table/thead//td' + + col_label = 'Emprunteur' + col_coming = 'Capital restant dû' + col_doc = 'Contrat' + + item_xpath = '//div[@class="tab-wallet"]/table/tbody/tr' + + class item(ItemElement): + klass = Account + + obj_label = CleanText(TableCell('label')) + obj_id = Slugify(Field('label')) + obj_type = Account.TYPE_SAVINGS + obj_currency = 'EUR' + obj_coming = CleanDecimal(TableCell('coming'), replace_dots=True) + obj_balance = Decimal('0') + + def obj__docurl(self): + return urljoin(self.page.url, Link('.//a')(TableCell('doc')(self)[0])) + + +class OperationsPage(LoggedPage, HTMLPage): + @method + class iter_history(TableElement): + head_xpath = '//div[@class="tab-wallet"]/table/thead//td' + + col_date = 'Date' + col_label = 'Opération' + col_amount = 'Montant' + + item_xpath = '//div[@class="tab-wallet"]/table/tbody/tr' + + class item(ItemElement): + klass = Transaction + + def condition(self): + return not Field('label')(self).startswith('dont ') + + obj_label = CleanText(TableCell('label')) + + def obj_amount(self): + v = CleanDecimal(TableCell('amount'), replace_dots=True)(self) + if Field('label')(self).startswith('Investissement'): + v = -v + return v + + obj_date = Date(CleanText(TableCell('date')), dayfirst=True, default=None) + + +class ProfilePage(LoggedPage, HTMLPage): + @method + class get_profile(ItemElement): + klass = Profile + + obj_name = Format( + '%s %s', + Attr('//input[@id="SubModel_FirstName"]', 'value'), + Attr('//input[@id="SubModel_LastName"]', 'value'), + ) + obj_phone = Attr('//input[@id="SubModel_Phone"]', 'value') + obj_address = Format( + '%s %s %s %s %s', + Attr('//input[@id="SubModel_Address_Street"]', 'value'), + Attr('//input[@id="SubModel_Address_Suplement"]', 'value'), + Attr('//input[@id="SubModel_Address_PostalCode"]', 'value'), + Attr('//input[@id="SubModel_Address_City"]', 'value'), + CleanText('//select[@id="SubModel_Address_Country"]/option[@selected]'), + ) diff --git a/modules/boursorama/browser.py b/modules/boursorama/browser.py index ec0f1e8418623d10b659725db9476b6eab6e34db..c04a1829ee757124de3a43a9c5e4cfb0766005de 100644 --- a/modules/boursorama/browser.py +++ b/modules/boursorama/browser.py @@ -27,11 +27,11 @@ from weboob.browser.browsers import need_login, StatesMixin from weboob.browser.url import URL from weboob.exceptions import BrowserIncorrectPassword, BrowserHTTPNotFound -from .compat.weboob_browser_exceptions import LoggedOut -from weboob.capabilities.bank import ( +from .compat.weboob_browser_exceptions import LoggedOut, ClientError +from .compat.weboob_capabilities_bank import ( Account, AccountNotFound, TransferError, TransferInvalidAmount, TransferInvalidEmitter, TransferInvalidLabel, TransferInvalidRecipient, - AddRecipientStep, Recipient, + AddRecipientStep, Recipient, Rate ) from weboob.capabilities.contact import Advisor from weboob.tools.captcha.virtkeyboard import VirtKeyboardError @@ -44,7 +44,7 @@ MarketPage, LoanPage, SavingMarketPage, ErrorPage, IncidentPage, IbanPage, ProfilePage, ExpertPage, CardsNumberPage, CalendarPage, HomePage, PEPPage, TransferAccounts, TransferRecipients, TransferCharac, TransferConfirm, TransferSent, - AddRecipientPage, StatusPage, CardHistoryPage, CardCalendarPage, + AddRecipientPage, StatusPage, CardHistoryPage, CardCalendarPage, CurrencyListPage, CurrencyConvertPage, ) @@ -116,6 +116,9 @@ class BoursoramaBrowser(RetryLoginBrowser, StatesMixin): cards = URL('/compte/cav/cb', CardsNumberPage) + currencylist = URL('https://www.boursorama.com/bourse/devises/parite/_detail-parite', CurrencyListPage) + currencyconvert = URL('https://www.boursorama.com/bourse/devises/convertisseur-devises/convertir', CurrencyConvertPage) + __states__ = ('auth_token',) def __init__(self, config=None, *args, **kwargs): @@ -476,3 +479,24 @@ def rcpt_after_sms(self): assert self.page.is_created() return ret + + def iter_currencies(self): + return self.currencylist.go().get_currency_list() + + def get_rate(self, curr_from, curr_to): + r = Rate() + params = { + 'from': curr_from, + 'to': curr_to, + 'amount': '1' + } + r.currency_from = curr_from + r.currency_to = curr_to + r.datetime = datetime.now() + try: + self.currencyconvert.go(params=params) + r.value = self.page.get_rate() + # if a rate is no available the site return a 401 error... + except ClientError: + return + return r diff --git a/modules/boursorama/compat/weboob_capabilities_bank.py b/modules/boursorama/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/boursorama/compat/weboob_capabilities_bank.py +++ b/modules/boursorama/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/boursorama/module.py b/modules/boursorama/module.py index cb164759418e42e4a9121e57d5e62badd799dc0f..22a743cc36aa2cb2f76ad7fa37378655cf98a826 100644 --- a/modules/boursorama/module.py +++ b/modules/boursorama/module.py @@ -24,7 +24,7 @@ import re -from .compat.weboob_capabilities_bank import CapBankWealth, CapBankTransferAddRecipient, Account, AccountNotFound +from .compat.weboob_capabilities_bank import CapBankWealth, CapBankTransferAddRecipient, Account, AccountNotFound, CapCurrencyRate from weboob.capabilities.profile import CapProfile from weboob.capabilities.contact import CapContact from weboob.tools.backend import Module, BackendConfig @@ -36,7 +36,7 @@ __all__ = ['BoursoramaModule'] -class BoursoramaModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapProfile, CapContact): +class BoursoramaModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapProfile, CapContact, CapCurrencyRate): NAME = 'boursorama' MAINTAINER = u'Gabriel Kerneis' EMAIL = 'gabriel@kerneis.info' @@ -100,3 +100,9 @@ def execute_transfer(self, transfer, **kwargs): def transfer_check_label(self, old, new): old = re.sub(r'[€#&$£%><~"\{\}\[\]=@^\\_`|°µ§!;]', '', old).strip() return super(BoursoramaModule, self).transfer_check_label(old, new) + + def iter_currencies(self): + return self.browser.iter_currencies() + + def get_rate(self, currency_from, currency_to): + return self.browser.get_rate(currency_from, currency_to) diff --git a/modules/boursorama/pages.py b/modules/boursorama/pages.py index becab0c0c10da89324d956837ddaa9b2301684ce..79f15916deecf6c75768edbd8d918693abb2567c 100644 --- a/modules/boursorama/pages.py +++ b/modules/boursorama/pages.py @@ -26,7 +26,7 @@ from io import BytesIO from datetime import date -from .compat.weboob_browser_pages import HTMLPage, LoggedPage, pagination, NextPage, FormNotFound, PartialHTMLPage, LoginPage, CsvPage, RawPage +from .compat.weboob_browser_pages import HTMLPage, LoggedPage, pagination, NextPage, FormNotFound, PartialHTMLPage, LoginPage, CsvPage, RawPage, JsonPage from weboob.browser.elements import ListElement, ItemElement, method, TableElement, SkipItem, DictElement from weboob.browser.filters.standard import ( CleanText, CleanDecimal, Field, Format, @@ -35,10 +35,14 @@ ) from weboob.browser.filters.json import Dict from weboob.browser.filters.html import Attr, Link, TableCell, AbsoluteLink -from weboob.capabilities.bank import Account, Investment, Recipient, Transfer, AccountNotFound, AddRecipientError, TransferInvalidAmount -from weboob.capabilities.base import NotAvailable, empty +from weboob.capabilities.bank import ( + Account, Investment, Recipient, Transfer, AccountNotFound, + AddRecipientError, TransferInvalidAmount, +) +from weboob.capabilities.base import NotAvailable, empty, Currency from weboob.capabilities.profile import Person from weboob.tools.capabilities.bank.transactions import FrenchTransaction +from weboob.tools.capabilities.bank.iban import is_iban_valid from weboob.tools.value import Value from weboob.tools.date import parse_french_date from weboob.tools.captcha.virtkeyboard import VirtKeyboard, VirtKeyboardError @@ -249,10 +253,12 @@ def condition(self): def obj_balance(self): if Field('type')(self) != Account.TYPE_CARD: balance = Field('_amount')(self) - if Field('type')(self) in [Account.TYPE_PEA, Account.TYPE_LIFE_INSURANCE]: + if Field('type')(self) in [Account.TYPE_PEA, Account.TYPE_LIFE_INSURANCE, Account.TYPE_MARKET]: page = Async('details').loaded_page(self) if isinstance(page, MarketPage): - return page.get_balance(Field('type')(self)) or balance + updated_balance = page.get_balance(Field('type')(self)) + if updated_balance is not None: + return updated_balance return balance return Decimal('0') @@ -474,8 +480,10 @@ def validate(self, obj): class Myiter_investment(TableElement): - item_xpath = '//table[contains(@class, "operations")]/tbody/tr' - head_xpath = '//table[contains(@class, "operations")]/thead/tr/th' + # We do not scrape the investments contained in the "Engagements en liquidation" table + # so we must check that the

before the
does not contain this title. + item_xpath = '//div[preceding-sibling::h3[text()!="Engagements en liquidation"]]//table[contains(@class, "operations")]/tbody/tr' + head_xpath = '//div[preceding-sibling::h3[text()!="Engagements en liquidation"]]//table[contains(@class, "operations")]/thead/tr/th' col_value = u'Valeur' col_quantity = u'Quantité' @@ -522,7 +530,10 @@ def inner(page, *args, **kwargs): class MarketPage(LoggedPage, HTMLPage): def get_balance(self, account_type): txt = u"Solde au" if account_type is Account.TYPE_LIFE_INSURANCE else u"Total Portefeuille" - return CleanDecimal('//li[h4[contains(text(), "%s")]]/h3' % txt, replace_dots=True, default=None)(self.doc) + # HTML tags are usually h4-h3 but may also be span-span + h_balance = CleanDecimal('//li[h4[contains(text(), "%s")]]/h3' % txt, replace_dots=True, default=None)(self.doc) + span_balance = CleanDecimal('//li/span[contains(text(), "%s")]/following-sibling::span' % txt, replace_dots=True, default=None)(self.doc) + return h_balance or span_balance or None @my_pagination @method @@ -890,6 +901,13 @@ def obj_enabled_at(self): obj__tempid = Attr('.', 'data-value') + def condition(self): + iban = Field('iban')(self) + if iban: + return is_iban_valid(iban) + # some internal accounts don't show iban + return True + def submit_recipient(self, tempid): form = self.get_form(name='CreditAccount') form['CreditAccount[creditAccountKey]'] = tempid @@ -1029,3 +1047,28 @@ def is_created(self): class PEPPage(LoggedPage, HTMLPage): pass + + +class CurrencyListPage(HTMLPage): + @method + class iter_currencies(ListElement): + item_xpath = '//select[@class="c-select currency-change"]/option' + + class item(ItemElement): + klass = Currency + + obj_id = Attr('./.', 'value') + + def get_currency_list(self): + CurIDList = [] + for currency in self.iter_currencies(): + currency.id = currency.id[0:3] + if currency.id not in CurIDList: + CurIDList.append(currency.id) + yield currency + + +class CurrencyConvertPage(JsonPage): + def get_rate(self): + if not 'error' in self.doc: + return round(self.doc['rate'], 4) diff --git a/modules/bouygues/browser.py b/modules/bouygues/browser.py index 8f76af6ccc73d12b6d5bbb3ff16eaa8b20965d47..ed1965183d1695269267126c88ff29eae2973476 100644 --- a/modules/bouygues/browser.py +++ b/modules/bouygues/browser.py @@ -21,7 +21,7 @@ from weboob.browser import LoginBrowser, URL, need_login from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable -from weboob.browser.exceptions import ClientError +from weboob.browser.exceptions import ClientError, HTTPNotFound from weboob.tools.compat import urlparse, parse_qs from .pages import ( DocumentsPage, HomePage, LoginPage, SubscriberPage, SubscriptionPage, SubscriptionDetailPage, @@ -37,21 +37,21 @@ class BouyguesBrowser(LoginBrowser): BASEURL = 'https://api.bouyguestelecom.fr' TIMEOUT = 20 - login = URL('https://www.mon-compte.bouyguestelecom.fr/cas/login', LoginPage) - home = URL('https://www.bouyguestelecom.fr/mon-compte', HomePage) - subscriber = URL('/personnes/(?P\d+)$', SubscriberPage) - subscriptions = URL('/personnes/(?P\d+)/comptes-facturation', SubscriptionPage) - subscriptions_details = URL('/comptes-facturation/(?P\d+)/contrats-payes', SubscriptionDetailPage) - document_file = URL('/comptes-facturation/(?P\d+)/factures/\d+/documents', DocumentFilePage) - documents = URL('/comptes-facturation/(?P\d+)/factures', DocumentsPage) + login = URL(r'https://www.mon-compte.bouyguestelecom.fr/cas/login', LoginPage) + home = URL(r'https://www.bouyguestelecom.fr/mon-compte', HomePage) + subscriber = URL(r'/personnes/(?P\d+)$', SubscriberPage) + subscriptions = URL(r'/personnes/(?P\d+)/comptes-facturation', SubscriptionPage) + subscriptions_details = URL(r'/comptes-facturation/(?P\d+)/contrats-payes', SubscriptionDetailPage) + document_file = URL(r'/comptes-facturation/(?P\d+)/factures/\d+/documents', DocumentFilePage) + documents = URL(r'/comptes-facturation/(?P\d+)/factures', DocumentsPage) - sms_page = URL('http://www.mobile.service.bbox.bouyguestelecom.fr/services/SMSIHD/sendSMS.phtml', - 'http://www.mobile.service.bbox.bouyguestelecom.fr/services/SMSIHD/confirmSendSMS.phtml', + sms_page = URL(r'http://www.mobile.service.bbox.bouyguestelecom.fr/services/SMSIHD/sendSMS.phtml', + r'http://www.mobile.service.bbox.bouyguestelecom.fr/services/SMSIHD/confirmSendSMS.phtml', SendSMSPage) - confirm = URL('http://www.mobile.service.bbox.bouyguestelecom.fr/services/SMSIHD/resultSendSMS.phtml', UselessPage) - sms_error_page = URL('http://www.mobile.service.bbox.bouyguestelecom.fr/services/SMSIHD/SMS_erreur.phtml', + confirm = URL(r'http://www.mobile.service.bbox.bouyguestelecom.fr/services/SMSIHD/resultSendSMS.phtml', UselessPage) + sms_error_page = URL(r'http://www.mobile.service.bbox.bouyguestelecom.fr/services/SMSIHD/SMS_erreur.phtml', SendSMSErrorPage) - profile = URL('/personnes/(?P\d+)/coordonnees', ProfilePage) + profile = URL(r'/personnes/(?P\d+)/coordonnees', ProfilePage) def __init__(self, username, password, lastname, *args, **kwargs): super(BouyguesBrowser, self).__init__(username, password, *args, **kwargs) @@ -67,8 +67,11 @@ def do_login(self): self.page.login(self.username, self.password, self.lastname) - if not self.home.is_here(): - raise BrowserIncorrectPassword() + if self.login.is_here(): + error = self.page.get_error() + if error and 'mot de passe' in error: + raise BrowserIncorrectPassword(error) + raise AssertionError("Unhandled error at login: {}".format(error)) # after login we need to get some tokens to use bouygues api data = { @@ -97,7 +100,7 @@ def post_message(self, message): if self.sms_error_page.is_here(): raise CantSendMessage(self.page.get_error_message()) - receivers = ";".join(list(message.receivers)) if message.receivers else self.username + receivers = ";".join(message.receivers) if message.receivers else self.username self.page.send_sms(message, receivers) if self.sms_error_page.is_here(): @@ -125,8 +128,13 @@ def iter_subscriptions(self): @need_login def iter_documents(self, subscription): - self.location(subscription.url, headers=self.headers) - return self.page.iter_documents(subid=subscription.id) + try: + self.location(subscription.url, headers=self.headers) + return self.page.iter_documents(subid=subscription.id) + except HTTPNotFound as error: + if error.response.json()['error'] == 'facture_introuvable': + return [] + raise @need_login def download_document(self, document): diff --git a/modules/bouygues/module.py b/modules/bouygues/module.py index d28fbc36eb43446d8df1e3f0d65b0d6c0ef4e06a..c28c702c6499582dcbbd4515c46b019c7235b7ac 100644 --- a/modules/bouygues/module.py +++ b/modules/bouygues/module.py @@ -34,7 +34,7 @@ class BouyguesModule(Module, CapMessages, CapMessagesPost, CapDocument, CapProfile): NAME = 'bouygues' - MAINTAINER = u'Bezleputh' + MAINTAINER = 'Bezleputh' EMAIL = 'carton_ben@yahoo.fr' VERSION = '1.3' DESCRIPTION = u'Bouygues Télécom French mobile phone provider' @@ -49,7 +49,7 @@ def create_default_browser(self): def post_message(self, message): if not message.content.strip(): - raise CantSendMessage(u'Message content is empty.') + raise CantSendMessage('Message content is empty.') self.browser.post_message(message) def iter_subscription(self): diff --git a/modules/bouygues/pages.py b/modules/bouygues/pages.py index 02cc1be5396d1c215f154f49c69d6ba0cfb26ae1..965ea584b3419b3ba7f872c5aa004b2bca32656c 100644 --- a/modules/bouygues/pages.py +++ b/modules/bouygues/pages.py @@ -16,10 +16,13 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals + import re from datetime import datetime, timedelta from weboob.capabilities.messages import CantSendMessage +from weboob.exceptions import BrowserIncorrectPassword from weboob.capabilities.base import NotLoaded from weboob.capabilities.bill import Bill, Subscription @@ -32,16 +35,21 @@ class LoginPage(HTMLPage): def login(self, login, password, lastname): - form = self.get_form('//form[@id="log_data"]') + form = self.get_form(id='log_data') form['username'] = login form['password'] = password if 'lastname' in form: + if not lastname: + raise BrowserIncorrectPassword('Le nom de famille est obligatoire.') form['lastname'] = lastname form.submit() + def get_error(self): + return CleanText('//div[@id="alert_msg"]//p')(self.doc) + class HomePage(LoggedPage, HTMLPage): pass @@ -58,7 +66,7 @@ def get_subscriber(self): def get_phone_list(self): num_tel_list = [] for phone in self.doc.get('comptesAcces', []): - num_tel_list.append(' '.join([phone[i:i + 2] for i in range(0, len(phone), 2)])) + num_tel_list.append(' '.join(phone[i:i + 2] for i in range(0, len(phone), 2))) return ' - '.join(num_tel_list) @@ -88,7 +96,7 @@ def get_label(self): class SendSMSPage(HTMLPage): def send_sms(self, message, receivers): - sms_number = CleanDecimal(Regexp(CleanText('//span[@class="txt12-o"][1]/strong'), '(\d*) SMS.*'))(self.doc) + sms_number = CleanDecimal(Regexp(CleanText('//span[@class="txt12-o"][1]/strong'), r'(\d*) SMS.*'))(self.doc) if sms_number == 0: msg = CleanText('//span[@class="txt12-o"][1]')(self.doc) @@ -133,11 +141,10 @@ class item(ItemElement): obj_url = Format('https://api.bouyguestelecom.fr%s', Dict('_links/facturePDF/href')) obj_date = Env('date') obj_duedate = Env('duedate') - obj_format = u"pdf" + obj_format = 'pdf' obj_label = Env('label') - obj_type = u"bill" obj_price = CleanDecimal(Dict('mntTotFacture')) - obj_currency = u'EUR' + obj_currency = 'EUR' def parse(self, el): bill_date = datetime.strptime(Dict('dateFacturation')(self), "%Y-%m-%dT%H:%M:%SZ").date() diff --git a/modules/bp/browser.py b/modules/bp/browser.py index 7cff2720d3393159ccaa609a6f141322890e7a4c..58d292370486d726b402b2a3e1d2cb370e1d9240 100644 --- a/modules/bp/browser.py +++ b/modules/bp/browser.py @@ -31,7 +31,7 @@ LoginPage, Initident, CheckPassword, repositionnerCheminCourant, BadLoginPage, AccountDesactivate, AccountList, AccountHistory, CardsList, UnavailablePage, AccountRIB, Advisor, TransferChooseAccounts, CompleteTransfer, TransferConfirm, TransferSummary, CreateRecipient, ValidateRecipient, - ValidateCountry, ConfirmPage, RcptSummary + ValidateCountry, ConfirmPage, RcptSummary, SubscriptionPage, DownloadPage, ProSubscriptionPage, ) from .pages.accounthistory import ( LifeInsuranceInvest, LifeInsuranceHistory, LifeInsuranceHistoryInv, RetirementHistory, @@ -50,7 +50,6 @@ class BPBrowser(LoginBrowser, StatesMixin): BASEURL = 'https://voscomptesenligne.labanquepostale.fr' - STATE_DURATION = 5 # FIXME beware that '.*' in start of URL() won't match all domains but only under BASEURL @@ -169,6 +168,10 @@ class BPBrowser(LoginBrowser, StatesMixin): profile = URL('/voscomptes/canalXHTML/donneesPersonnelles/consultationDonneesPersonnellesSB490A/init-consulterDonneesPersonnelles.ea', ProfilePage) + subscription = URL('/voscomptes/canalXHTML/relevePdf/relevePdf_historique/reinitialiser-historiqueRelevesPDF.ea', SubscriptionPage) + subscription_search = URL('/voscomptes/canalXHTML/relevePdf/relevePdf_historique/form-historiqueRelevesPDF\.ea', SubscriptionPage) + download_page = URL(r'/voscomptes/canalXHTML/relevePdf/relevePdf_historique/telechargerPDF-historiqueRelevesPDF.ea\?ts=.*&listeRecherche=.*', DownloadPage) + accounts = None def __init__(self, *args, **kwargs): @@ -353,6 +356,8 @@ def get_coming(self, account): @need_login def iter_card_transactions(self, account): def iter_transactions(link, urlobj): + # we go back to main menue otherwise we get an error 500. + self.cards_list.go(account_id=account.parent.id) self.location(link) assert urlobj.is_here() ncard = self.page.params['cardIndex'] @@ -474,6 +479,43 @@ def get_advisor(self): def get_profile(self): return self.profile.go().get_profile() + @need_login + def iter_subscriptions(self): + subscriber = self.get_profile().name + self.subscription.go() + return self.page.iter_subscriptions(subscriber=subscriber) + + @need_login + def iter_documents(self, subscription): + self.subscription.go() + params = self.page.get_params(sub_label=subscription.label) + + for year in self.page.get_years(): + params['formulaire.anneeRecherche'] = year + + if 'PEA' in subscription.label: + for statement_type in self.page.STATEMENT_TYPES: + params['formulaire.typeReleve'] = statement_type + self.subscription_search.go(params=params) + + if self.page.has_error(): + # you may have an error message + # instead of telling you that there are no statement for a year + continue + + for doc in self.page.iter_documents(sub_id=subscription.id): + yield doc + else: + self.subscription_search.go(params=params) + for doc in self.page.iter_documents(sub_id=subscription.id): + yield doc + + @need_login + def download_document(self, document): + download_page = self.open(document.url).page + # may have an iframe + return download_page.get_content() + class BProBrowser(BPBrowser): login_url = "https://banqueenligne.entreprises.labanquepostale.fr/wsost/OstBrokerWeb/loginform?TAM_OP=login&ERROR_CODE=0x00000000&URL=%2Fws_q47%2Fvoscomptes%2Fidentification%2Fidentification.ea%3Forigin%3Dprofessionnels" @@ -486,6 +528,10 @@ class BProBrowser(BPBrowser): useless2 = URL(r'.*/voscomptes/bourseenligne/lancementBourseEnLigne-bourseenligne.ea\?numCompte=(?P\d+)', UselessPage) market_login = URL(r'.*/voscomptes/bourseenligne/oicformautopost.jsp', MarketLoginPage) + subscription = URL(r'(?P.*)/voscomptes/relevespdf/histo-consultationReleveCompte.ea', + r'.*/voscomptes/relevespdf/rechercheHistoRelevesCompte-consultationReleveCompte.ea', ProSubscriptionPage) + download_page = URL(r'.*/voscomptes/relevespdf/telechargerReleveCompteSelectionne-consultationReleveCompte.ea\?idReleveSelectionne=.*', DownloadPage) + BASEURL = 'https://banqueenligne.entreprises.labanquepostale.fr' def set_variables(self): @@ -568,3 +614,38 @@ def get_profile(self): self.location('%s/voscomptes/rib/preparerRIB-rib.ea?%s' % (self.base_url, value)) if self.rib.is_here(): return self.page.get_profile() + + @need_login + def iter_subscriptions(self): + subscriber = self.get_profile().name + self.subscription.go(base_url=self.base_url) + return self.page.iter_subscriptions(subscriber=subscriber) + + @need_login + def iter_documents(self, subscription): + self.subscription.go(base_url=self.base_url) + + for year in self.page.get_years(): + self.page.submit_form(sub_number=subscription._number, year=year) + + if self.page.no_statement(): + self.subscription.go(base_url=self.base_url) + continue + + for doc in self.page.iter_documents(sub_id=subscription.id): + yield doc + + self.subscription.go(base_url=self.base_url) + + @need_login + def download_document(self, document): + # must be sure to be on the right page before downloading + if self.subscription.is_here() and self.page.has_document(document.date): + return self.open(document.url).content + + self.subscription.go(base_url=self.base_url) + sub_number = self.page.get_sub_number(document.id) + year = str(document.date.year) + self.page.submit_form(sub_number=sub_number, year=year) + + return self.open(document.url).content diff --git a/modules/bp/compat/weboob_capabilities_bank.py b/modules/bp/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/bp/compat/weboob_capabilities_bank.py +++ b/modules/bp/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/bp/linebourse_browser.py b/modules/bp/linebourse_browser.py index 6c27cb09047aff17dca09d30dd0971a2396ba726..c878ebbadb6580115f213f259bd70ad5b6ea0067 100644 --- a/modules/bp/linebourse_browser.py +++ b/modules/bp/linebourse_browser.py @@ -22,5 +22,3 @@ class LinebourseBrowser(AbstractBrowser): PARENT = 'linebourse' - - VERIFY = 'linebourse.pem' diff --git a/modules/bp/module.py b/modules/bp/module.py index 9a65da0daca6fcde8bbf93101a9422a33bfc8718..673765a5a191befca579016ee20028788dec83b5 100644 --- a/modules/bp/module.py +++ b/modules/bp/module.py @@ -21,8 +21,12 @@ from decimal import Decimal from .compat.weboob_capabilities_bank import CapBankWealth, CapBankTransferAddRecipient, Account, AccountNotFound, RecipientNotFound, TransferError from weboob.capabilities.contact import CapContact -from weboob.capabilities.base import find_object +from weboob.capabilities.base import find_object, NotAvailable from weboob.capabilities.profile import CapProfile +from weboob.capabilities.bill import ( + CapDocument, Subscription, SubscriptionNotFound, + Document, DocumentNotFound, +) from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword, Value @@ -32,7 +36,11 @@ __all__ = ['BPModule'] -class BPModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapContact, CapProfile): +class BPModule( + Module, CapBankWealth, CapBankTransferAddRecipient, + CapContact, CapProfile, CapDocument, +): + NAME = 'bp' MAINTAINER = u'Nicolas Duhamel' EMAIL = 'nicolas@jombi.fr' @@ -94,8 +102,15 @@ def init_transfer(self, transfer, **params): except (AssertionError, ValueError): raise TransferError('something went wrong') + # format label like label sent by firefox or chromium browser + transfer.label = transfer.label.encode('latin-1', errors="xmlcharrefreplace").decode('latin-1') + return self.browser.init_transfer(account, recipient, amount, transfer) + def transfer_check_label(self, old, new): + old = old.encode('latin-1', errors="xmlcharrefreplace").decode('latin-1') + return super(BPModule, self).transfer_check_label(old, new) + def execute_transfer(self, transfer, **params): return self.browser.execute_transfer(transfer) @@ -110,3 +125,36 @@ def iter_contacts(self): def get_profile(self): return self.browser.get_profile() + + def get_document(self, _id): + subscription_id = _id.split('_')[0] + subscription = self.get_subscription(subscription_id) + return find_object(self.iter_documents(subscription), id=_id, error=DocumentNotFound) + + def get_subscription(self, _id): + return find_object(self.iter_subscription(), id=_id, error=SubscriptionNotFound) + + def iter_documents(self, subscription): + if not isinstance(subscription, Subscription): + subscription = self.get_subscription(subscription) + + return self.browser.iter_documents(subscription) + + def iter_subscription(self): + return self.browser.iter_subscriptions() + + def download_document(self, document): + if not isinstance(document, Document): + document = self.get_document(document) + if document.url is NotAvailable: + return + + return self.browser.download_document(document) + + def iter_resources(self, objs, split_path): + if Account in objs: + self._restrict_level(split_path) + return self.iter_accounts() + if Subscription in objs: + self._restrict_level(split_path) + return self.iter_subscription() diff --git a/modules/bp/pages/__init__.py b/modules/bp/pages/__init__.py index cb865f527948b149760fa7090562adddc7b16738..6f41c8c49d4cdeae8d2a64a8db85edcc0622a659 100644 --- a/modules/bp/pages/__init__.py +++ b/modules/bp/pages/__init__.py @@ -23,8 +23,10 @@ from .accounthistory import AccountHistory, CardsList from .transfer import TransferChooseAccounts, CompleteTransfer, TransferConfirm, TransferSummary, CreateRecipient, ValidateRecipient,\ ValidateCountry, ConfirmPage, RcptSummary +from .subscription import SubscriptionPage, DownloadPage, ProSubscriptionPage __all__ = ['LoginPage', 'Initident', 'CheckPassword', 'repositionnerCheminCourant', "AccountList", 'AccountHistory', 'BadLoginPage', 'AccountDesactivate', 'TransferChooseAccounts', 'CompleteTransfer', 'TransferConfirm', 'TransferSummary', 'UnavailablePage', - 'CardsList', 'AccountRIB', 'Advisor', 'CreateRecipient', 'ValidateRecipient', 'ValidateCountry', 'ConfirmPage', 'RcptSummary'] + 'CardsList', 'AccountRIB', 'Advisor', 'CreateRecipient', 'ValidateRecipient', 'ValidateCountry', 'ConfirmPage', 'RcptSummary', + 'SubscriptionPage', 'DownloadPage', 'ProSubscriptionPage'] diff --git a/modules/bp/pages/accountlist.py b/modules/bp/pages/accountlist.py index e683b9a330a0a04bbdc3c9be57b44c8a7cc6130a..c4d00961d970f6f58f92ee48f05d0aa4105a9974 100644 --- a/modules/bp/pages/accountlist.py +++ b/modules/bp/pages/accountlist.py @@ -108,6 +108,7 @@ def obj_type(self): u'crédits?': Account.TYPE_LOAN, 'plan d\'epargne en actions': Account.TYPE_PEA, 'comptes? attente': Account.TYPE_CHECKING, + 'perp': Account.TYPE_PERP, } # first trying to match with label diff --git a/modules/bp/pages/subscription.py b/modules/bp/pages/subscription.py new file mode 100644 index 0000000000000000000000000000000000000000..49715354ddad236eb366e117c1ad743ec2f87fd4 --- /dev/null +++ b/modules/bp/pages/subscription.py @@ -0,0 +1,169 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2010-2018 Célande Adrien +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.capabilities.bill import Subscription, Document +from weboob.browser.pages import LoggedPage, HTMLPage +from weboob.browser.filters.standard import CleanText, Regexp, Env, Date, Format, Field +from weboob.browser.filters.html import Link, Attr, TableCell +from weboob.browser.elements import ListElement, ItemElement, method, TableElement + + +class SubscriptionPage(LoggedPage, HTMLPage): + # encoding is wrong on the page + ENCODING='ISO-8859-1' + + # because of freaking JS from hell + STATEMENT_TYPES = ('RCE', 'RPT', 'RCO') + + @method + class iter_subscriptions(ListElement): + item_xpath = '//select[@id="compte"]/option' + + class item(ItemElement): + klass = Subscription + + obj_id = Regexp(Attr('.', 'value'), r'\w-(\w+)') + obj_label = CleanText('.') + obj_subscriber = Env('subscriber') + + @method + class iter_documents(ListElement): + def condition(self): + return not ( + CleanText('//p[contains(text(), "est actuellement indisponible")]')(self) + or CleanText('//p[contains(text(), "Aucun e-Relevé n\'est disponible")]')(self) + ) + + item_xpath = '//ul[contains(@class, "liste-cpte")]/li' + # you can have twice the same statement: same month, same subscription + ignore_duplicate = True + + class item(ItemElement): + klass = Document + + obj_id = Format('%s_%s%s', Env('sub_id'), Regexp(CleanText('.//a/@title'), r' (\d{2}) '), CleanText('.//span[contains(@class, "date")]' ,symbols='/')) + obj_label = Format('%s - %s', CleanText('.//span[contains(@class, "lib")]'), CleanText('.//span[contains(@class, "date")]')) + obj_url = Format('/voscomptes/canalXHTML/relevePdf/relevePdf_historique/%s', Link('./a')) + obj_format = 'pdf' + obj_type = 'other' + + def obj_date(self): + date = CleanText('.//span[contains(@class, "date")]')(self) + m = re.search(r'(\d{2}/\d{2}/\d{4})', date) + if m: + return Date(CleanText('.//span[contains(@class, "date")]'), dayfirst=True)(self) + else: + return Date( + Format( + '%s/%s', + Regexp(CleanText('.//a/@title'), r' (\d{2}) '), + CleanText('.//span[contains(@class, "date")]') + ), + dayfirst=True + )(self) + + def get_params(self, sub_label): + # the id is in the label + sub_value = Attr('//select[@id="compte"]/option[contains(text(), "%s")]' % sub_label, 'value')(self.doc) + + form = self.get_form(name='formulaireHistorique') + form['formulaire.numeroCompteRecherche'] = sub_value + return form + + def get_years(self): + return self.doc.xpath('//select[@id="annee"]/option/@value') + + def has_error(self): + return ( + CleanText('//p[contains(text(), "est actuellement indisponible")]')(self.doc) + or CleanText('//p[contains(text(), "Aucun e-Relevé n\'est disponible")]')(self.doc) + ) + + +class DownloadPage(LoggedPage, HTMLPage): + def get_content(self): + if self.doc.xpath('//iframe'): + # the url has the form + # ../relevePdf_telechargement/affichagePDF-telechargementPDF.ea?date=XXX + part_link = Attr('//iframe', 'src')(self.doc).replace('..', '') + return self.browser.open('/voscomptes/canalXHTML/relevePdf%s' % part_link).content + return self.content + +class ProSubscriptionPage(LoggedPage, HTMLPage): + @method + class iter_subscriptions(ListElement): + item_xpath = '//select[@id="numeroCompteRechercher"]/option' + + class item(ItemElement): + klass = Subscription + + obj_label = CleanText('.') + obj_id = Regexp(Field('label'), r'\w - (\w+)') + obj_subscriber = Env('subscriber') + obj__number = Attr('.', 'value') + + @method + class iter_documents(TableElement): + item_xpath = '//table[@id="relevesPDF"]//tr[td]' + head_xpath = '//table[@id="relevesPDF"]//th' + # may have twice the same statement for a given month + ignore_duplicate = True + + col_date = re.compile('Date du relevé') + col_label = re.compile('Type de document') + + class item(ItemElement): + klass = Document + + obj_date = Date(CleanText(TableCell('date')), dayfirst=True) + obj_label = Format('%s %s', CleanText(TableCell('label')), CleanText(TableCell('date'))) + obj_id = Format('%s_%s', Env('sub_id'), CleanText(TableCell('date'), symbols='/')) + # the url uses an id depending on the page where the document is + # by example, if the id is 0, + # it means that it is the first document that you can find + # on the page of the year XXX for the subscription YYYY + obj_url = Link('.//a') + obj_format = 'pdf' + obj_type = 'other' + + def submit_form(self, sub_number, year): + form = self.get_form(name='formRechHisto') + + form['historiqueReleveParametre.numeroCompteRecherche'] = sub_number + form['typeRecherche'] = 'annee' + form['anneeRechercheDefaut'] = year + + form.submit() + + def get_years(self): + return self.doc.xpath('//select[@name="anneeRechercheDefaut"]/option/@value') + + def no_statement(self): + return self.doc.xpath('//p[has-class("noresult")]') + + def has_document(self, date): + return self.doc.xpath('//td[@headers="dateReleve" and contains(text(), "%s")]' % date.strftime('%d/%m/%Y')) + + def get_sub_number(self, doc_id): + sub_id = doc_id.split('_')[0] + return Attr('//select[@id="numeroCompteRechercher"]/option[contains(text(), "%s")]' % sub_id, 'value')(self.doc) diff --git a/modules/bp/pages/transfer.py b/modules/bp/pages/transfer.py index e2be475009d64c6a360069a16faafcf491c6f1de..a1bbc1d38491daf09800143934ee0269cbc33862 100644 --- a/modules/bp/pages/transfer.py +++ b/modules/bp/pages/transfer.py @@ -180,18 +180,13 @@ def handle_response(self, transfer): if u'veuillez saisir votre code de validation' in CleanText('//div[@class="bloc Tmargin"]')(self.doc): raise NotImplementedError() - transfer_date = Date(Regexp( + # there are several regexp for transfer date: + # Date ([\d\/]+)|le ([\d\/]+)|suivant \(([\d\/]+)\) + # be more passive to avoid impulsive reaction from user + transfer.exec_date = Date(Regexp( CleanText('//div[@class="bloc Tmargin"]'), - r'suivant \(([\d\/]+)\)', - default=NotAvailable - ), dayfirst=True, default=NotAvailable)(self.doc) - if transfer_date: - transfer.exec_date = transfer_date - else: - transfer.exec_date = Date(Regexp( - CleanText('//div[@class="bloc Tmargin"]'), - r'Date ([\d\/]+)' - ), dayfirst=True)(self.doc) + r' (\d{2}/\d{2}/\d{4})' + ), dayfirst=True)(self.doc) return transfer diff --git a/modules/bred/bred/browser.py b/modules/bred/bred/browser.py index d4fc75c8291bb93c67ade8038a7231baf73e73ac..07f8599b4c37196c2780c5ab87f354d1c8310b90 100644 --- a/modules/bred/bred/browser.py +++ b/modules/bred/bred/browser.py @@ -201,7 +201,9 @@ def get_history(self, account, coming=False): next_page = bool(transactions) offset += 50 - assert offset < 30000, 'the site may be doing an infinite loop' + # This assert supposedly prevents infinite loops, + # but some customers actually have a lot of transactions. + assert offset < 100000, 'the site may be doing an infinite loop' @need_login def get_investment(self, account): diff --git a/modules/bred/bred/pages.py b/modules/bred/bred/pages.py index a3ad23f289dc781ec777244569d6e10aeb828660..509af9261134f2a7164264d68ad97db0cb03d0df 100644 --- a/modules/bred/bred/pages.py +++ b/modules/bred/bred/pages.py @@ -108,6 +108,7 @@ class AccountsPage(MyJsonPage): '011': Account.TYPE_CARD, # Carte bancaire '020': Account.TYPE_SAVINGS, # Compte sur livret '021': Account.TYPE_SAVINGS, + '022': Account.TYPE_SAVINGS, # Livret d'épargne populaire '023': Account.TYPE_SAVINGS, # LDD Solidaire '025': Account.TYPE_SAVINGS, # Livret Fidélis '027': Account.TYPE_SAVINGS, # Livret A @@ -214,7 +215,7 @@ def iter_history(self, account, operation_list, seen, today, coming): transactions = [] for op in reversed(operation_list): t = Transaction() - t.id = op['id'] + t.id = str(op['id']) if op['id'] in seen: raise ParseError('There are several transactions with the same ID, probably an infinite loop') @@ -261,7 +262,8 @@ def get_profile(self): class EmailsPage(MyJsonPage): def set_email(self, profile): content = self.get_content() - profile.email = content['emailPart'] + if 'emailPart' in content: + profile.email = content['emailPart'] class ErrorPage(LoggedPage, HTMLPage): diff --git a/modules/bred/compat/weboob_capabilities_bank.py b/modules/bred/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/bred/compat/weboob_capabilities_bank.py +++ b/modules/bred/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/bred/dispobank/browser.py b/modules/bred/dispobank/browser.py index 57b2797d4643e9c1c9d138201432fed9dfcba65c..d92b8d7c77d63008fa53c743139d8974071a061c 100644 --- a/modules/bred/dispobank/browser.py +++ b/modules/bred/dispobank/browser.py @@ -46,8 +46,6 @@ class DispoBankBrowser(LoginBrowser): } } - VERIFY = 'cert.pem' - def __init__(self, accnum, *args, **kwargs): super(DispoBankBrowser, self).__init__(*args, **kwargs) self.accnum = accnum diff --git a/modules/caels/compat/weboob_capabilities_bank.py b/modules/caels/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/caels/compat/weboob_capabilities_bank.py +++ b/modules/caels/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/caissedepargne/browser.py b/modules/caissedepargne/browser.py index 10df633a9f7ec7e89c5276d253c6d2e96e5affb9..8298fd3cd36787e802b4a44d81d5b5aeb12d15ab 100644 --- a/modules/caissedepargne/browser.py +++ b/modules/caissedepargne/browser.py @@ -45,7 +45,8 @@ TransferPage, ProTransferPage, TransferConfirmPage, TransferSummaryPage, ProTransferConfirmPage, ProTransferSummaryPage, ProAddRecipientOtpPage, ProAddRecipientPage, SmsPage, SmsPageOption, SmsRequest, AuthentPage, RecipientPage, CanceledAuth, CaissedepargneKeyboard, - TransactionsDetailsPage, LoadingPage, ConsLoanPage, MeasurePage, NatixisLIHis, NatixisLIInv, NatixisRedirectPage + TransactionsDetailsPage, LoadingPage, ConsLoanPage, MeasurePage, NatixisLIHis, NatixisLIInv, NatixisRedirectPage, + SubscriptionPage, ) from .linebourse_browser import LinebourseBrowser @@ -76,6 +77,7 @@ class CaisseEpargne(LoginBrowser, StatesMixin): pro_add_recipient = URL('https://.*/Portail.aspx.*', ProAddRecipientPage) measure_page = URL('https://.*/Portail.aspx.*', MeasurePage) authent = URL('https://.*/Portail.aspx.*', AuthentPage) + subscription = URL('https://.*/Portail.aspx\?tache=(?P).*', SubscriptionPage) home = URL('https://.*/Portail.aspx.*', IndexPage) home_tache = URL('https://.*/Portail.aspx\?tache=(?P).*', IndexPage) error = URL('https://.*/login.aspx', @@ -302,7 +304,12 @@ def get_accounts_list(self): continue self.page.get_valuation_diff(account) - return iter(self.accounts) + # Some accounts have no available balance or label and cause issues + # in the backend so we must exclude them from the accounts list: + self.accounts = [account for account in self.accounts if account.label and account.balance != NotAvailable] + for account in self.accounts: + yield account + @need_login def get_loans_list(self): @@ -660,3 +667,42 @@ def new_recipient(self, recipient, **params): self.page.check_canceled_auth() self.page.set_browser_form() raise AddRecipientStep(self.get_recipient_obj(recipient), Value('sms_password', label=self.page.get_prompt_text())) + + @need_login + def iter_subscription(self): + self.home.go() + # CapDocument is not implemented for professional accounts yet + if any(x in self.url for x in ["netpp", "netpro"]): + raise NotImplementedError() + self.home_tache.go(tache='CPTSYNT1') + self.page.go_subscription() + assert self.subscription.is_here() + + if self.page.has_subscriptions(): + return self.page.iter_subscription() + return [] + + @need_login + def iter_documents(self, subscription): + self.home.go() + self.home_tache.go(tache='CPTSYNT1') + self.page.go_subscription() + assert self.subscription.is_here() + + sub_id = subscription.id + self.page.go_document_list(sub_id=sub_id) + + for doc in self.page.iter_documents(sub_id=sub_id): + yield doc + + @need_login + def download_document(self, document): + self.home.go() + self.home_tache.go(tache='CPTSYNT1') + self.page.go_subscription() + assert self.subscription.is_here() + + sub_id = document.id.split('_')[0] + self.page.go_document_list(sub_id=sub_id) + + return self.page.download_document(document).content diff --git a/modules/caissedepargne/cenet/browser.py b/modules/caissedepargne/cenet/browser.py index b721b2852ea4092036f55ea6ee86db8de0e1e632..ee2ecff68ef67369a2d6b08f6b0301621cf83e7b 100644 --- a/modules/caissedepargne/cenet/browser.py +++ b/modules/caissedepargne/cenet/browser.py @@ -30,7 +30,7 @@ ErrorPage, LoginPage, CenetLoginPage, CenetHomePage, CenetAccountsPage, CenetAccountHistoryPage, CenetCardsPage, - CenetCardSummaryPage, + CenetCardSummaryPage, SubscriptionPage, DownloadDocumentPage, ) from ..pages import CaissedepargneKeyboard @@ -59,6 +59,10 @@ class CenetBrowser(LoginBrowser, StatesMixin): cenet_login = URL(r'https://.*/$', r'https://.*/default.aspx', CenetLoginPage) + subscription = URL('/Web/Api/ApiReleves.asmx/ChargerListeEtablissements', SubscriptionPage) + documents = URL('/Web/Api/ApiReleves.asmx/ChargerListeReleves', SubscriptionPage) + download = URL(r'/Default.aspx\?dashboard=ComptesReleves&lien=SuiviReleves', DownloadDocumentPage) + __states__ = ('BASEURL',) def __init__(self, nuser, *args, **kwargs): @@ -234,3 +238,43 @@ def init_transfer(self, account, recipient, transfer): def new_recipient(self, recipient, **params): raise NotImplementedError() + + @need_login + def iter_subscription(self): + subscriber = self.get_profile().name + json_data = { + 'contexte': '', + 'dateEntree': None, + 'donneesEntree': 'null', + 'filtreEntree': None + } + self.subscription.go(json=json_data) + return self.page.iter_subscription(subscriber=subscriber) + + @need_login + def iter_documents(self, subscription): + sub_id = subscription.id + input_filter = { + 'Page':0, + 'NombreParPage':0, + 'Tris':[], + 'Criteres':[ + {'Champ': 'Etablissement','TypeCritere': 'Equals','Value': sub_id}, + {'Champ': 'DateDebut','TypeCritere': 'Equals','Value': None}, + {'Champ': 'DateFin','TypeCritere': 'Equals','Value': None}, + {'Champ': 'MaxRelevesAffichesParNumero','TypeCritere': 'Equals','Value': '100'} + ] + } + json_data = { + 'contexte': '', + 'dateEntree': None, + 'donneesEntree': 'null', + 'filtreEntree': json.dumps(input_filter) + } + self.documents.go(json=json_data) + return self.page.iter_documents(sub_id=sub_id, sub_label=subscription.label, username=self.username) + + @need_login + def download_document(self, document): + self.download.go() + return self.page.download_form(document).content diff --git a/modules/caissedepargne/cenet/pages.py b/modules/caissedepargne/cenet/pages.py index 5a62d4dd3f99de4f2fdfea16cbae6799f9cfb56b..62bc5a7590864c12563637f45263a94da13708b0 100644 --- a/modules/caissedepargne/cenet/pages.py +++ b/modules/caissedepargne/cenet/pages.py @@ -19,15 +19,17 @@ import re import json +from datetime import datetime from weboob.browser.pages import LoggedPage, HTMLPage, JsonPage from weboob.browser.elements import DictElement, ItemElement, method -from weboob.browser.filters.standard import Date, CleanDecimal, CleanText, Format, Field +from weboob.browser.filters.standard import Date, CleanDecimal, CleanText, Format, Field, Env, Regexp from weboob.browser.filters.json import Dict from weboob.capabilities import NotAvailable from weboob.capabilities.bank import Account, Transaction from weboob.capabilities.contact import Advisor from weboob.capabilities.profile import Profile +from weboob.capabilities.bill import Subscription, Document from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.exceptions import BrowserUnavailable @@ -253,3 +255,56 @@ class Transaction(FrenchTransaction): (re.compile('^FAC CB (?P.*?) (?P
\d{2})/(?P\d{2})', re.IGNORECASE), FrenchTransaction.TYPE_CARD), ] + + +class SubscriptionPage(LoggedPage, CenetJsonPage): + @method + class iter_subscription(DictElement): + item_xpath = 'DonneesSortie' + + class item(ItemElement): + klass = Subscription + + obj_id = CleanText(Dict('Numero')) + obj_label = CleanText(Dict('Intitule')) + obj_subscriber = Env('subscriber') + + @method + class iter_documents(DictElement): + item_xpath = 'DonneesSortie' + + class item(ItemElement): + klass = Document + + obj_id = Format('%s_%s_%s', Env('sub_id'), Dict('Numero'), CleanText(Env('french_date'), symbols='/')) + obj_format = 'pdf' + obj_type = 'other' + obj__numero = CleanText(Dict('Numero')) + obj__sub_id = Env('sub_id') + obj__sub_label = Env('sub_label') + obj__download_id = CleanText(Dict('IdDocument')) + + def obj_date(self): + date = Regexp(Dict('DateArrete'), r'Date\((\d+)\)')(self) + date = int(date) // 1000 + return datetime.fromtimestamp(date).date() + + def obj_label(self): + return '%s %s' % (CleanText(Dict('Libelle'))(self), Env('french_date')(self)) + + def parse(self, el): + self.env['french_date'] = Field('date')(self).strftime('%d/%m/%Y') + + +class DownloadDocumentPage(LoggedPage, HTMLPage): + def download_form(self, document): + data = { + 'Numero': document._numero, + 'Libelle': document._sub_label.replace(' ', '+'), + 'DateArrete': '', + 'IdDocument': document._download_id + } + form = self.get_form(id='aspnetForm') + form['__EVENTTARGET'] = 'btn_telecharger' + form['__EVENTARGUMENT'] = json.dumps(data) + return form.submit() diff --git a/modules/caissedepargne/compat/weboob_capabilities_bank.py b/modules/caissedepargne/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/caissedepargne/compat/weboob_capabilities_bank.py +++ b/modules/caissedepargne/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/caissedepargne/linebourse_browser.py b/modules/caissedepargne/linebourse_browser.py index 6c27cb09047aff17dca09d30dd0971a2396ba726..c878ebbadb6580115f213f259bd70ad5b6ea0067 100644 --- a/modules/caissedepargne/linebourse_browser.py +++ b/modules/caissedepargne/linebourse_browser.py @@ -22,5 +22,3 @@ class LinebourseBrowser(AbstractBrowser): PARENT = 'linebourse' - - VERIFY = 'linebourse.pem' diff --git a/modules/caissedepargne/module.py b/modules/caissedepargne/module.py index 18484bd4d066505572c042b0dd7939ce318eaafe..5f4e95ccdf5488e160dbb6ccdc17f1fc80a590d3 100644 --- a/modules/caissedepargne/module.py +++ b/modules/caissedepargne/module.py @@ -22,6 +22,11 @@ from decimal import Decimal from .compat.weboob_capabilities_bank import CapBankWealth, CapBankTransferAddRecipient, AccountNotFound, Account, RecipientNotFound +from weboob.capabilities.bill import ( + CapDocument, Subscription, SubscriptionNotFound, + Document, DocumentNotFound, +) +from weboob.capabilities.base import NotAvailable from weboob.capabilities.contact import CapContact from weboob.capabilities.profile import CapProfile from weboob.capabilities.base import find_object @@ -33,7 +38,7 @@ __all__ = ['CaisseEpargneModule'] -class CaisseEpargneModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapContact, CapProfile): +class CaisseEpargneModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapDocument, CapContact, CapProfile): NAME = 'caissedepargne' MAINTAINER = u'Romain Bignon' EMAIL = 'romain@weboob.org' @@ -110,3 +115,37 @@ def execute_transfer(self, transfer, **params): def new_recipient(self, recipient, **params): #recipient.label = ' '.join(w for w in re.sub('[^0-9a-zA-Z:\/\-\?\(\)\.,\'\+ ]+', '', recipient.label).split()) return self.browser.new_recipient(recipient, **params) + + def iter_resources(self, objs, split_path): + if Account in objs: + self._restrict_level(split_path) + return self.iter_accounts() + if Subscription in objs: + self._restrict_level(split_path) + return self.iter_subscription() + + def get_subscription(self, _id): + return find_object(self.iter_subscription(), id=_id, error=SubscriptionNotFound) + + def get_document(self, _id): + subscription_id = _id.split('_')[0] + subscription = self.get_subscription(subscription_id) + return find_object(self.iter_documents(subscription), id=_id, error=DocumentNotFound) + + def iter_subscription(self): + return self.browser.iter_subscription() + + def iter_documents(self, subscription): + if not isinstance(subscription, Subscription): + subscription = self.get_subscription(subscription) + + return self.browser.iter_documents(subscription) + + def download_document(self, document): + if not isinstance(document, Document): + document = self.get_document(document) + + if document.url is NotAvailable: + return + + return self.browser.download_document(document) diff --git a/modules/caissedepargne/pages.py b/modules/caissedepargne/pages.py index 714e59e018b1532b5e2eee3a0e7ef5b3c6379ca3..517b4490748b0f0c562f9d95d886f2d87b60545a 100644 --- a/modules/caissedepargne/pages.py +++ b/modules/caissedepargne/pages.py @@ -30,11 +30,12 @@ from weboob.browser.pages import LoggedPage, HTMLPage, JsonPage, pagination, FormNotFound from weboob.browser.elements import ItemElement, method, ListElement, TableElement, SkipItem, DictElement -from weboob.browser.filters.standard import Date, CleanDecimal, Regexp, CleanText, Env, Upper, Field, Eval +from weboob.browser.filters.standard import Date, CleanDecimal, Regexp, CleanText, Env, Upper, Field, Eval, Format from weboob.browser.filters.html import Link, Attr, TableCell from weboob.capabilities import NotAvailable from weboob.capabilities.bank import Account, Investment, Recipient, TransferError, TransferBankError, Transfer,\ AddRecipientError, Loan +from weboob.capabilities.bill import Subscription, Document from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.tools.capabilities.bank.iban import is_rib_valid, rib2iban, is_iban_valid from weboob.tools.captcha.virtkeyboard import GridVirtKeyboard @@ -650,6 +651,13 @@ def loan_unavailable_msg(self): if msg: return msg + def go_subscription(self): + form = self.get_form(name='main') + form['m_ScriptManager'] = 'MM$m_UpdatePanel|MM$Menu_Ajax' + form['__EVENTTARGET'] = 'MM$Menu_Ajax' + form['__EVENTARGUMENT'] = 'CPTEDOC&codeMenu=WCE0' + form.submit() + class ConsLoanPage(JsonPage): def get_conso(self): @@ -1251,3 +1259,59 @@ def go_newsite_back_to_summary(self): form = self.get_form(name='main') form['__EVENTTARGET'] = "MM$ECRITURE_GLOBALE$lnkRetourHisto" form.submit() + + +class SubscriptionPage(LoggedPage, HTMLPage): + def is_here(self): + return self.doc.xpath('//h2[text()="e-Documents"]') + + def has_subscriptions(self): + # This message appears if the customer has not activated the e-Documents yet + return not bool(self.doc.xpath('//a[contains(text(), "Je souscris au service e-Documents")]')) + + @method + class iter_subscription(ListElement): + item_xpath = '//select[contains(@id, "ClientsBancaires")]/option' + + class item(ItemElement): + klass = Subscription + + obj_id = Attr('.', 'value') + obj_label = CleanText('.') + obj_subscriber = CleanText('.') + + def condition(self): + return 'Clos' not in Field('label')(self) + + def go_document_list(self, sub_id): + target = Attr('//select[contains(@id, "ClientsBancaires")]', 'id')(self.doc) + form = self.get_form(name='main') + form['m_ScriptManager'] = target + form['MM$COMPTE_EDOCUMENTS$ctrlEDocumentsConsultationDocument$cboClientsBancaires'] = sub_id + form['__EVENTTARGET'] = target + form.submit() + + def get_years(self): + return self.doc.xpath('//select[contains(@id, "Annee")]/option') + + @method + class iter_documents(ListElement): + item_xpath = '//ul[@class="telecharger"]/li/a' + + class item(ItemElement): + klass = Document + + obj_label = Format('%s %s', CleanText('./preceding::h3[1]'), CleanText('./span')) + obj_date = Date(CleanText('./span'), dayfirst=True) + obj_type = 'other' + obj_format = 'pdf' + obj_url = Regexp(Link('.'), r'WebForm_PostBackOptions\("(\S*)"') + obj_id = Format('%s_%s_%s', Env('sub_id'), CleanText('./span', symbols='/'), Regexp(Field('url'), r'ctl(.*)')) + obj__event_id = Regexp(Attr('.', 'onclick'), r"val\('(.*)'\);") + + def download_document(self, document): + form = self.get_form(name='main') + form['m_ScriptManager'] = document.url + form['__EVENTTARGET'] = document.url + form['MM$COMPTE_EDOCUMENTS$ctrlEDocumentsConsultationDocument$eventId'] = document._event_id + return form.submit() diff --git a/modules/carrefourbanque/browser.py b/modules/carrefourbanque/browser.py index c91770e8d36608523852ece87a3e25bec4b3d218..21d45aa5d5e5f074d81c8e9aab098ed658a9b20c 100644 --- a/modules/carrefourbanque/browser.py +++ b/modules/carrefourbanque/browser.py @@ -19,12 +19,12 @@ from time import sleep from weboob.browser.browsers import LoginBrowser, URL, need_login, StatesMixin -from .compat.weboob_exceptions import BrowserIncorrectPassword, NocaptchaQuestion +from .compat.weboob_exceptions import BrowserIncorrectPassword, NocaptchaQuestion, BrowserUnavailable from weboob.capabilities.bank import Account from weboob.tools.compat import basestring from .pages import ( - LoginPage, HomePage, IncapsulaResourcePage, LoanHistoryPage, CardHistoryPage, SavingHistoryPage, + LoginPage, MaintenancePage, HomePage, IncapsulaResourcePage, LoanHistoryPage, CardHistoryPage, SavingHistoryPage, LifeInvestmentsPage, LifeHistoryPage ) @@ -36,6 +36,7 @@ class CarrefourBanqueBrowser(LoginBrowser, StatesMixin): BASEURL = 'https://www.carrefour-banque.fr' login = URL('/espace-client/connexion', LoginPage) + maintenance = URL('/maintenance', MaintenancePage) incapsula_ressource = URL('/_Incapsula_Resource', IncapsulaResourcePage) home = URL('/espace-client$', HomePage) @@ -88,6 +89,9 @@ def do_login(self): # we got javascript page again, this shouldn't happen assert False, "obfuscated javascript not managed" + if self.maintenance.is_here(): + raise BrowserUnavailable(self.page.get_message()) + self.page.enter_login(self.username) self.page.enter_password(self.password) diff --git a/modules/carrefourbanque/compat/weboob_capabilities_bank.py b/modules/carrefourbanque/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/carrefourbanque/compat/weboob_capabilities_bank.py +++ b/modules/carrefourbanque/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/carrefourbanque/pages.py b/modules/carrefourbanque/pages.py index 1a52a5fca99c22d50f91100867de3bb2f60e028c..35e5251139733d8f72be08e49bd49387778065ca 100644 --- a/modules/carrefourbanque/pages.py +++ b/modules/carrefourbanque/pages.py @@ -140,6 +140,11 @@ def enter_password(self, password): form.submit() +class MaintenancePage(HTMLPage): + def get_message(self): + return CleanText('//div[@class="bloc-title"]/h1//div[has-class("field-item")]')(self.doc) + + class IncapsulaResourcePage(HTMLPage): def __init__(self, *args, **kwargs): # this page can be a html page, or just javascript @@ -231,7 +236,7 @@ def obj_balance(self): @method class iter_life_accounts(ListElement): # Assurances vie - item_xpath = '//div/div[contains(./h2, "Carrefour Horizons") and contains(./p, "Numéro de compte")]/..' + item_xpath = '//div/div[(contains(./h2, "Carrefour Horizons") or contains(./h2, "Carrefour Avenir")) and contains(./p, "Numéro de compte")]/..' class item(item_account_generic): obj_type = Account.TYPE_LIFE_INSURANCE diff --git a/modules/cices/compat/weboob_capabilities_bank.py b/modules/cices/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/cices/compat/weboob_capabilities_bank.py +++ b/modules/cices/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/cmes/compat/weboob_capabilities_bank.py b/modules/cmes/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/cmes/compat/weboob_capabilities_bank.py +++ b/modules/cmes/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/cmso/compat/weboob_capabilities_bank.py b/modules/cmso/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/cmso/compat/weboob_capabilities_bank.py +++ b/modules/cmso/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/cmso/module.py b/modules/cmso/module.py index a12e90eb5c9fe59db49fc0d532f9361b1d1b25cd..0ad63335f4899b441225e43cdfd23fa8a0e46438 100644 --- a/modules/cmso/module.py +++ b/modules/cmso/module.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals from .compat.weboob_capabilities_bank import CapBankWealth, AccountNotFound from weboob.capabilities.contact import CapContact @@ -34,10 +35,10 @@ class CmsoModule(Module, CapBankWealth, CapContact, CapProfile): NAME = 'cmso' - MAINTAINER = u'Romain Bignon' + MAINTAINER = 'Romain Bignon' EMAIL = 'romain@weboob.org' VERSION = '1.3' - DESCRIPTION = u'Crédit Mutuel Sud-Ouest' + DESCRIPTION = 'Crédit Mutuel Sud-Ouest' LICENSE = 'AGPLv3+' CONFIG = BackendConfig(ValueBackendPassword('login', label='Identifiant', masked=False), ValueBackendPassword('password', label='Mot de passe'), diff --git a/modules/cmso/par/browser.py b/modules/cmso/par/browser.py index 559c2e26740c1f1c53bb7419e93ff128a6c4526f..d3d5987d245afaf1d5461fdb6c7da6c40686d6e1 100644 --- a/modules/cmso/par/browser.py +++ b/modules/cmso/par/browser.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals import re import json @@ -52,7 +53,7 @@ def decorator(func): def wrapper(browser, *args, **kwargs): cb = lambda: func(browser, *args, **kwargs) - for i in xrange(tries, 0, -1): + for i in range(tries, 0, -1): try: ret = cb() except exc_check as exc: @@ -220,7 +221,7 @@ def iter_history(self, account): if account.type == Account.TYPE_LIFE_INSURANCE: url = json.loads(self.lifeinsurance.go(accid=account._index).content)['url'] - url = self.location(url).page.get_link(u"opérations") + url = self.location(url).page.get_link("opérations") return self.location(url).page.iter_history() elif account.type in (Account.TYPE_PEA, Account.TYPE_MARKET): diff --git a/modules/cmso/par/pages.py b/modules/cmso/par/pages.py index ff722c2c6d70b8ddd5c4993a7d292a76f2eaa456..a7ee5881796e87414c629771478aca3a5ae72c54 100644 --- a/modules/cmso/par/pages.py +++ b/modules/cmso/par/pages.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals import re import requests @@ -37,6 +38,8 @@ from weboob.capabilities.profile import Profile from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.exceptions import ParseError +from weboob.tools.capabilities.bank.investments import is_isin_valid +from weboob.tools.compat import unicode def MyDecimal(*args, **kwargs): @@ -55,7 +58,7 @@ class LogoutPage(RawPage): class InfosPage(LoggedPage, HTMLPage): def get_typelist(self): url = Attr(None, 'src').filter(self.doc.xpath('//script[contains(@src, "comptes/scripts")]')) - m = re.search('synthesecomptes[^\w]+([^:]+)[^\w]+([^"]+)', self.browser.open(url).content) + m = re.search(r'synthesecomptes[^\w]+([^:]+)[^\w]+([^"]+)', self.browser.open(url).text) return {m.group(1): m.group(2)} @@ -94,8 +97,8 @@ def get_numbers(self): numbers = {} for key in keys: if isinstance(self.doc[key], dict): - keys = [k for k in self.doc[key] if isinstance(k, unicode)] - contracts = [v for v in self.doc[key][k] for k in keys] + keys_ = [k for k in self.doc[key] if isinstance(k, unicode)] + contracts = [v for k in keys_ for v in self.doc[key][k]] else: contracts = [v for v in self.doc[key]] numbers.update({c['index']: c['numeroContratSouscrit'] for c in contracts}) @@ -108,14 +111,14 @@ def parse(self, el): def find_elements(self): selector = self.item_xpath.split('/') - for el in selector: - if isinstance(self.el, dict) and el == '*' and self.el.values(): - self.el = self.el.values()[0] - if el == '*': + for sub_element in selector: + if isinstance(self.el, dict) and self.el and sub_element == '*': + self.el = next(iter(self.el.values())) # replace self.el with its first value + if sub_element == '*': continue - self.el = self.el[el] - for el in self.el: - yield el + self.el = self.el[sub_element] + for sub_element in self.el: + yield sub_element class item(ItemElement): klass = Account @@ -184,7 +187,7 @@ class item(ItemElement): obj_label = Upper(Dict('libelleContrat')) obj_balance = CleanDecimal(Dict('solde', default="0")) - obj_currency = u'EUR' + obj_currency = 'EUR' obj_coming = CleanDecimal(Dict('AVenir', default=None), default=NotAvailable) obj__index = Dict('index') obj__owner = Dict('nomTitulaire') @@ -241,7 +244,7 @@ def obj_id(self): return Dict('numeroContratSouscrit', default=None)(self) or Dict('identifiantTechnique')(self) obj_label = Dict('libelle') - obj_currency = u'EUR' + obj_currency = 'EUR' obj_type = Account.TYPE_LOAN def obj_total_amount(self): @@ -276,14 +279,14 @@ def obj_balance(self): class Transaction(FrenchTransaction): - PATTERNS = [(re.compile(u'^CARTE (?P
\d{2})/(?P\d{2}) (?P.*)'), FrenchTransaction.TYPE_CARD), - (re.compile(u'^(?P(PRLV|PRELEVEMENTS).*)'), FrenchTransaction.TYPE_ORDER), - (re.compile(u'^(?PRET DAB.*)'), FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile(u'^(?PECH.*)'), FrenchTransaction.TYPE_LOAN_PAYMENT), - (re.compile(u'^(?PVIR.*)'), FrenchTransaction.TYPE_TRANSFER), - (re.compile(u'^(?PANN.*)'), FrenchTransaction.TYPE_PAYBACK), - (re.compile(u'^(?P(VRST|VERSEMENT).*)'), FrenchTransaction.TYPE_DEPOSIT), - (re.compile(u'^(?P.*)'), FrenchTransaction.TYPE_BANK) + PATTERNS = [(re.compile(r'^CARTE (?P
\d{2})/(?P\d{2}) (?P.*)'), FrenchTransaction.TYPE_CARD), + (re.compile(r'^(?P(PRLV|PRELEVEMENTS).*)'), FrenchTransaction.TYPE_ORDER), + (re.compile(r'^(?PRET DAB.*)'), FrenchTransaction.TYPE_WITHDRAWAL), + (re.compile(r'^(?PECH.*)'), FrenchTransaction.TYPE_LOAN_PAYMENT), + (re.compile(r'^(?PVIR.*)'), FrenchTransaction.TYPE_TRANSFER), + (re.compile(r'^(?PANN.*)'), FrenchTransaction.TYPE_PAYBACK), + (re.compile(r'^(?P(VRST|VERSEMENT).*)'), FrenchTransaction.TYPE_DEPOSIT), + (re.compile(r'^(?P.*)'), FrenchTransaction.TYPE_BANK) ] @@ -356,12 +359,12 @@ def parse(self, el): class LifeinsurancePage(LoggedPage, HTMLPage): def get_account_id(self): - account_id = Regexp(CleanText('//h1[@class="portlet-title"]'), ur'n° ([\d\s]+)', default=NotAvailable)(self.doc) + account_id = Regexp(CleanText('//h1[@class="portlet-title"]'), r'n° ([\d\s]+)', default=NotAvailable)(self.doc) if account_id: return re.sub(r'\s', '', account_id) def get_link(self, page): - return Link(default=NotAvailable).filter(self.doc.xpath(u'//a[contains(text(), "%s")]' % page)) + return Link(default=NotAvailable).filter(self.doc.xpath('//a[contains(text(), "%s")]' % page)) @pagination @method @@ -370,7 +373,7 @@ class iter_history(TableElement): head_xpath = '//table/thead/tr/th' col_date = re.compile('Date') - col_label = re.compile(u'Libellé') + col_label = re.compile('Libellé') col_amount = re.compile('Montant') next_page = Link('//a[contains(text(), "Suivant") and not(contains(@href, "javascript"))]', default=None) @@ -387,7 +390,7 @@ class iter_investment(TableElement): item_xpath = '//table/tbody/tr[contains(@class, "results")]' head_xpath = '//table/thead/tr/th' - col_label = re.compile(u'Libellé') + col_label = re.compile('Libellé') col_quantity = re.compile('Nb parts') col_vdate = re.compile('Date VL') col_unitvalue = re.compile('VL') @@ -398,13 +401,16 @@ class item(ItemElement): klass = Investment obj_label = CleanText(TableCell('label')) - obj_code = Regexp(Link('./td/a'), 'Isin%253D([^%]+)') + obj_code = Regexp(Link('./td/a'), r'Isin%253D([^%]+)') obj_quantity = MyDecimal(TableCell('quantity')) obj_unitprice = MyDecimal(TableCell('unitprice')) obj_unitvalue = MyDecimal(TableCell('unitvalue')) obj_valuation = MyDecimal(TableCell('valuation')) obj_vdate = Date(CleanText(TableCell('vdate')), dayfirst=True, default=NotAvailable) + def obj_code_type(self): + return Investment.CODE_TYPE_ISIN if is_isin_valid(Field('code')(self)) else NotAvailable + class MarketPage(LoggedPage, HTMLPage): def find_account(self, acclabel, accowner): @@ -468,9 +474,9 @@ class iter_history(TableElement): head_xpath = '//table[has-class("domifrontTb")]/tr[1]/td' col_date = re.compile('Date') - col_label = u'Opération' - col_code = u'Code' - col_quantity = u'Quantité' + col_label = 'Opération' + col_code = 'Code' + col_quantity = 'Quantité' col_amount = re.compile('Montant') class item(ItemElement): @@ -496,13 +502,13 @@ class iter_investment(TableElement): item_xpath = '//table[has-class("domifrontTb")]/tr[not(has-class("LnTit") or has-class("LnTot"))]' head_xpath = '//table[has-class("domifrontTb")]/tr[1]/td' - col_label = u'Valeur' - col_code = u'Code' - col_quantity = u'Qté' - col_vdate = u'Date cours' - col_unitvalue = u'Cours' + col_label = 'Valeur' + col_code = 'Code' + col_quantity = 'Qté' + col_vdate = 'Date cours' + col_unitvalue = 'Cours' col_unitprice = re.compile('P.R.U') - col_valuation = u'Valorisation' + col_valuation = 'Valorisation' class item(ItemElement): klass = Investment @@ -510,13 +516,21 @@ class item(ItemElement): condition = lambda self: not CleanText('//div[has-class("errorConteneur")]', default=None)(self.el) obj_label = Upper(TableCell('label')) - obj_code = CleanText(TableCell('code')) obj_quantity = MyDecimal(TableCell('quantity')) obj_unitprice = MyDecimal(TableCell('unitprice')) obj_unitvalue = MyDecimal(TableCell('unitvalue')) obj_valuation = CleanDecimal(TableCell('valuation'), replace_dots=True) obj_vdate = Date(CleanText(TableCell('vdate')), dayfirst=True, default=NotAvailable) + def obj_code(self): + if Field('label')(self) == "LIQUIDITES": + return 'XX-liquidity' + code = CleanText(TableCell('code'))(self) + return code if is_isin_valid(code) else NotAvailable + + def obj_code_type(self): + return Investment.CODE_TYPE_ISIN if is_isin_valid(Field('code')(self)) else NotAvailable + class AdvisorPage(LoggedPage, JsonPage): @method diff --git a/modules/cmso/pro/browser.py b/modules/cmso/pro/browser.py index ca38fbeb8be87a25148bed45991d6ec139d7a617..8e1aaddbab3c7d27564074828ee7e306496b361f 100644 --- a/modules/cmso/pro/browser.py +++ b/modules/cmso/pro/browser.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals import datetime from dateutil.relativedelta import relativedelta @@ -267,9 +268,7 @@ def iter_investment(self, account): return [] assert self.invest_account.is_here() - invests = list(self.page.iter_investments()) - assert len(invests) < 2, 'implementation should be checked with more than 1 investment' # FIXME - return invests + return self.page.iter_investments() @need_login def get_profile(self): diff --git a/modules/cmso/pro/pages.py b/modules/cmso/pro/pages.py index 6c6c14ad9467848076daa0c7c8e5e6599f1be76a..b1ac8629e2a3ebaad554f77dd88629462825c44a 100644 --- a/modules/cmso/pro/pages.py +++ b/modules/cmso/pro/pages.py @@ -17,18 +17,20 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals import re from weboob.exceptions import BrowserIncorrectPassword from weboob.browser.pages import HTMLPage, JsonPage, pagination from weboob.browser.elements import ListElement, ItemElement, TableElement, method -from weboob.browser.filters.standard import CleanText, CleanDecimal, DateGuesser, Env, Field, Filter, Regexp, Currency +from weboob.browser.filters.standard import CleanText, CleanDecimal, DateGuesser, Env, Field, Filter, Regexp, Currency, Date from weboob.browser.filters.html import Link, Attr, TableCell from weboob.capabilities.bank import Account, Investment from weboob.capabilities.base import NotAvailable from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.tools.compat import urljoin +from weboob.tools.capabilities.bank.investments import is_isin_valid __all__ = ['LoginPage'] @@ -54,7 +56,7 @@ def on_load(self): class SubscriptionPage(HTMLPage): def on_load(self): - if u"Vous ne disposez d'aucun contrat sur cet accès." in CleanText(u'.')(self.doc): + if "Vous ne disposez d'aucun contrat sur cet accès." in CleanText('.')(self.doc): raise BrowserIncorrectPassword() def get_csrf(self): @@ -90,10 +92,10 @@ def logged(self): class AccountsPage(CMSOPage): - TYPES = {u'COMPTE CHEQUES': Account.TYPE_CHECKING, - u'COMPTE TITRES': Account.TYPE_MARKET, - u"ACTIV'EPARGNE": Account.TYPE_SAVINGS, - u"TRESO'VIV": Account.TYPE_SAVINGS, + TYPES = {'COMPTE CHEQUES': Account.TYPE_CHECKING, + 'COMPTE TITRES': Account.TYPE_MARKET, + "ACTIV'EPARGNE": Account.TYPE_SAVINGS, + "TRESO'VIV": Account.TYPE_SAVINGS, } @method @@ -105,7 +107,7 @@ class item(ItemElement): class Type(Filter): def filter(self, label): - for pattern, actype in AccountsPage.TYPES.iteritems(): + for pattern, actype in AccountsPage.TYPES.items(): if label.startswith(pattern): return actype return Account.TYPE_UNKNOWN @@ -181,19 +183,29 @@ class InvestmentAccountPage(CMSOPage): @method class iter_investments(CmsoTableElement): col_label = 'Valeur' - col_isin = 'Code' - col_quantity = u'Qté' + col_code = 'Code' + col_quantity = 'Qté' col_unitvalue = 'Cours' col_valuation = 'Valorisation' + col_vdate = 'Date cours' class item(ItemElement): klass = Investment obj_label = CleanText(TableCell('label')) - obj_code = CleanText(TableCell('isin')) obj_quantity = CleanDecimal(TableCell('quantity'), replace_dots=True) obj_unitvalue = CleanDecimal(TableCell('unitvalue'), replace_dots=True) obj_valuation = CleanDecimal(TableCell('valuation'), replace_dots=True) + obj_vdate = Date(CleanText(TableCell('vdate')), dayfirst=True, default=NotAvailable) + + def obj_code(self): + if Field('label')(self) == "LIQUIDITES": + return 'XX-liquidity' + code = CleanText(TableCell('code'))(self) + return code if is_isin_valid(code) else NotAvailable + + def obj_code_type(self): + return Investment.CODE_TYPE_ISIN if is_isin_valid(Field('code')(self)) else NotAvailable class Transaction(FrenchTransaction): diff --git a/modules/cragr/compat/weboob_capabilities_bank.py b/modules/cragr/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/cragr/compat/weboob_capabilities_bank.py +++ b/modules/cragr/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/cragr/web/browser.py b/modules/cragr/web/browser.py index 98fe2d3608cba6c248274abddea84d4db664f4a6..173e2fddc4019c5062e3190217fc18c5f3e08270 100644 --- a/modules/cragr/web/browser.py +++ b/modules/cragr/web/browser.py @@ -280,7 +280,7 @@ def get_cards(self, accounts_list=None): # accounts_list is only used by get_list self.location(self.accounts_url.format(self.sag)) - for idelco in self.page.iter_idelcos(): + for idelco, parent_id in self.page.iter_idelcos(): if not self.accounts.is_here(): self.location(self.accounts_url.format(self.sag)) @@ -299,6 +299,7 @@ def get_cards(self, accounts_list=None): else: for account in self.page.iter_card(): if accounts_list: + account._parent_id = parent_id account.parent = find_object(accounts_list, id=account._parent_id) yield account @@ -369,14 +370,14 @@ def get_list(self): @need_login def market_accounts_matching(self, accounts_list, market_accounts_list): - for account in accounts_list: - for market_account in market_accounts_list: - if account.id == market_account.id: - account.label = market_account.label or account.label - # Update accounts balance only for TYPE_MARKET because PEA accounts are fused - # with their related DAV PEA here, therefore the balance includes liquidities: - if account.type == Account.TYPE_MARKET: - account.balance = market_account.balance or account.balance + for market_account in market_accounts_list: + account = find_object(accounts_list, id=market_account.id) + if account: + account.label = market_account.label or account.label + # Update accounts balance only for TYPE_MARKET because PEA accounts are fused + # with their related DAV PEA here, therefore the balance includes liquidities: + if account.type == Account.TYPE_MARKET: + account.balance = market_account.balance or account.balance @need_login def get_history(self, account): diff --git a/modules/cragr/web/pages.py b/modules/cragr/web/pages.py index 8ec5c3eaa74ea65003adbe15ecd9dda28706d032..76000ed7072efa9660e7f91ed6a1b4cac9c24e04 100644 --- a/modules/cragr/web/pages.py +++ b/modules/cragr/web/pages.py @@ -38,7 +38,7 @@ from weboob.tools.compat import urlparse, urljoin, unicode from weboob.browser.elements import ListElement, TableElement, ItemElement, method from weboob.browser.filters.standard import Date, CleanText, CleanDecimal, Currency as CleanCurrency, \ - Regexp, Format, Field, Async, AsyncLoad + Regexp, Format, Field from .compat.weboob_browser_filters_html import Link, TableCell, ColumnNotFound @@ -306,30 +306,33 @@ def get_code_caisse(self): def _iter_idelcos_ids(self): for line in self.doc.xpath('//table[@class="ca-table"]//tr[@class="ligne-connexe"]'): # ignore line if preceding line is also a link to deferred card - if line.xpath('./preceding-sibling::tr')[-1].attrib.get('class') == 'ligne-connexe': + li = line.xpath('./preceding-sibling::tr') + if li[-1].attrib.get('class') == 'ligne-connexe': continue try: link = line.xpath('.//a/@href')[0] except IndexError: continue - yield link + + parent_id = re.findall(r'> (?P\d+)', CleanText(li)(self))[-1] + yield link, parent_id def iter_idelcos(self): # Use a set because it is possible to see several times the same link. idelcos = set() - for link in self._iter_idelcos_ids(): + for link, parent_id in self._iter_idelcos_ids(): if link.startswith('javascript:'): m = re.search(r"javascript:fwkPUAvancerForm\('Cartes','(\w+)'\)", link) if m: - idelcos.add(m.group(1)) + idelcos.add((m.group(1), parent_id)) else: m = re.search('IDELCO=(\d+)&', link) if m: - idelcos.add(m.group(1)) + idelcos.add((m.group(1), parent_id)) return idelcos def get_idelco(self, account_idelco): - for link in self._iter_idelcos_ids(): + for link, parent_id in self._iter_idelcos_ids(): if link.startswith('javascript:'): # no need to fetch a "full" link return self.get_form(name=account_idelco) @@ -446,9 +449,6 @@ def obj__spaced_number(self): def obj_url(self): return self.page.url - load_details = Link('.//tr[contains(@class, "ligne-bleu")]/th[@class="cel-texte" and contains(text(), "Solde")]/a') & AsyncLoad - obj__parent_id = Async('details') & Regexp(CleanText('//table[@class="ca-table"][1]//tr[@class="ligne-impaire"]/th'), r'(\d+)') - @method class iter_cards(ListElement): item_xpath = '//table[caption[@class="caption tdb-cartes-caption" or @class="ca-table caption"]]' @@ -1405,6 +1405,7 @@ def find_recipient(self, iban): if iban_text.upper() == 'IBAN:%s' % iban: res = Recipient() res.iban = iban + res.id = iban res.label = CleanText('./td[%s]' % (iban_col-1))(tr) return res diff --git a/modules/creditdunord/compat/weboob_capabilities_bank.py b/modules/creditdunord/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/creditdunord/compat/weboob_capabilities_bank.py +++ b/modules/creditdunord/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/creditdunord/pages.py b/modules/creditdunord/pages.py index f10a8effc13bdbfe7749b54f28c5c1ba316d10c7..722fb48f26612a19cd6b8ab1c0213ba33ae14a3f 100755 --- a/modules/creditdunord/pages.py +++ b/modules/creditdunord/pages.py @@ -28,9 +28,9 @@ from lxml import html from weboob.browser.pages import HTMLPage, LoggedPage -from weboob.browser.elements import method, ItemElement -from weboob.browser.filters.standard import CleanText, Date, CleanDecimal, Regexp -from weboob.browser.filters.html import Attr +from weboob.browser.elements import method, ItemElement, TableElement +from weboob.browser.filters.standard import CleanText, Date, CleanDecimal, Regexp, Field +from weboob.browser.filters.html import Attr, TableCell from weboob.exceptions import ActionNeeded, BrowserIncorrectPassword, BrowserUnavailable, BrowserPasswordExpired from weboob.capabilities.bank import Account, Investment from weboob.capabilities.profile import Profile @@ -619,29 +619,39 @@ def get_market_investment(self): yield inv - def get_deposit_investment(self): - COL_LABEL = 0 - COL_QUANTITY = 3 - COL_UNITVALUE = 4 - COL_VALUATION = 5 + @method + class get_deposit_investment(TableElement): + item_xpath = '//table[@class="datas"]/tr[not(@class="entete")]' + head_xpath = '//table[@class="datas"]/tr[(@class="entete")]/td/b' - for tr in self.doc.xpath('//table[@class="datas"]/tr[not(@class="entete")]'): - cols = tr.findall('td') + col_label = re.compile('Libellé') + col_quantity = 'Quantité' + col_valuation = 'Montant (EUR)' + col_unitvalue = re.compile('Valeur liquidative') + + class item(ItemElement): + klass = Investment + + def obj_label(self): + return CleanText('./a')(TableCell("label")(self)[0]) or CleanText(TableCell("label"))(self) + + def obj_code(self): + link_label = CleanText('./a')(TableCell("label")(self)[0]) + if link_label: + return CleanText('./text()', default=NotAvailable)(TableCell("label")(self)[0]) + + obj_quantity = MyDecimal(TableCell("quantity")) + + obj_unitvalue = MyDecimal(TableCell("unitvalue")) + + def obj_vdate(self): + if Field('unitvalue')(self): + return Date().filter(TableCell("unitvalue")(self)[0].xpath("./text()")[1]) + return Date(dayfirst=True, default=NotAvailable).filter(Regexp(CleanText('(//tr[td[span|b[contains(text(), "Estimation du contrat")]]]/td[2]/span)[2]'), \ + '(\d{2})/(\d{2})/(\d{4})', '\\3-\\2-\\1', default=NotAvailable)(self.page.doc)) + + obj_valuation = MyDecimal(TableCell("valuation")) - inv = Investment() - inv.label = CleanText('.')(cols[COL_LABEL].xpath('.//a')[0]) - inv.code = CleanText('./text()')(cols[COL_LABEL]) - inv.quantity = MyDecimal('.')(cols[COL_QUANTITY]) - inv.unitvalue = MyDecimal().filter(CleanText('.')(cols[COL_UNITVALUE]).split()[0]) - if inv.unitvalue is not NotAvailable: - inv.vdate = Date(dayfirst=True, default=NotAvailable)\ - .filter(Regexp(CleanText('.'), '(\d{2})/(\d{2})/(\d{4})', '\\3-\\2-\\1', default=NotAvailable)(cols[COL_UNITVALUE])) or \ - Date(dayfirst=True, default=NotAvailable)\ - .filter(Regexp(CleanText('//tr[td[span[b[contains(text(), "Estimation du contrat")]]]]/td[2]'), - '(\d{2})/(\d{2})/(\d{4})', '\\3-\\2-\\1', default=NotAvailable)(cols[COL_UNITVALUE])) - inv.valuation = MyDecimal('.')(cols[COL_VALUATION]) - - yield inv def fill_diff_currency(self, account): valuation_diff = CleanText(u'//td[span[contains(text(), "dont +/- value : ")]]//b', default=None)(self.doc) diff --git a/modules/creditdunordpee/compat/weboob_capabilities_bank.py b/modules/creditdunordpee/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/creditdunordpee/compat/weboob_capabilities_bank.py +++ b/modules/creditdunordpee/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/creditmutuel/browser.py b/modules/creditmutuel/browser.py index 5bfdb74fb309a3e94425538dbd8ca5bd93f3b436..d639dcb7b59565c22c9ca11d06e390fd35071cf9 100644 --- a/modules/creditmutuel/browser.py +++ b/modules/creditmutuel/browser.py @@ -36,14 +36,16 @@ from weboob.capabilities import NotAvailable from weboob.tools.compat import urlparse -from .pages import LoginPage, LoginErrorPage, AccountsPage, UserSpacePage, \ - OperationsPage, CardPage, ComingPage, RecipientsListPage, \ - ChangePasswordPage, VerifCodePage, EmptyPage, PorPage, \ - IbanPage, NewHomePage, AdvisorPage, RedirectPage, \ - LIAccountsPage, CardsActivityPage, CardsListPage, \ - CardsOpePage, NewAccountsPage, InternalTransferPage, \ - ExternalTransferPage, RevolvingLoanDetails, RevolvingLoansList, \ - ErrorPage +from .pages import ( + LoginPage, LoginErrorPage, AccountsPage, UserSpacePage, + OperationsPage, CardPage, ComingPage, RecipientsListPage, + ChangePasswordPage, VerifCodePage, EmptyPage, PorPage, + IbanPage, NewHomePage, AdvisorPage, RedirectPage, + LIAccountsPage, CardsActivityPage, CardsListPage, + CardsOpePage, NewAccountsPage, InternalTransferPage, + ExternalTransferPage, RevolvingLoanDetails, RevolvingLoansList, + ErrorPage, SubscriptionPage, +) __all__ = ['CreditMutuelBrowser'] @@ -122,7 +124,9 @@ class CreditMutuelBrowser(LoginBrowser, StatesMixin): internal_transfer = URL('/(?P.*)fr/banque/virements/vplw_vi.html', InternalTransferPage) external_transfer = URL('/(?P.*)fr/banque/virements/vplw_vee.html', ExternalTransferPage) recipients_list = URL('/(?P.*)fr/banque/virements/vplw_bl.html', RecipientsListPage) - error = URL('/validation/infos.cgi', ErrorPage) + error = URL('/(?P.*)validation/infos.cgi', ErrorPage) + + subscription = URL('/(?P.*)fr/banque/MMU2_LstDoc.aspx', SubscriptionPage) currentSubBank = None is_new_website = None @@ -169,6 +173,11 @@ def get_accounts_list(self): if self.currentSubBank is None: self.getCurrentSubBank() self.accounts_list = [] + self.revolving_accounts = [] + + for acc in self.revolving_loan_list.stay_or_go(subbank=self.currentSubBank).iter_accounts(): + self.accounts_list.append(acc) + self.revolving_accounts.append(acc.label.lower()) # Handle cards on tiers page self.cards_activity.go(subbank=self.currentSubBank) @@ -192,8 +201,6 @@ def get_accounts_list(self): for acc in self.li.go(subbank=self.currentSubBank).iter_li_accounts(): self.accounts_list.append(acc) - for acc in self.revolving_loan_list.stay_or_go(subbank=self.currentSubBank).iter_accounts(): - self.accounts_list.append(acc) excluded_label = ['etalis', 'valorisation totale'] self.accounts_list = [acc for acc in self.accounts_list if not any(w in acc.label.lower() for w in excluded_label)] @@ -488,3 +495,27 @@ def new_recipient(self, recipient, **params): raise AddRecipientStep(self.get_recipient_object(recipient), Value(u'Clé', label=self.page.get_question())) else: return self.continue_new_recipient(recipient, **params) + + @need_login + def iter_subscriptions(self): + if self.currentSubBank is None: + self.getCurrentSubBank() + self.subscription.go(subbank=self.currentSubBank) + return self.page.iter_subscriptions() + + @need_login + def iter_documents(self, subscription): + if self.currentSubBank is None: + self.getCurrentSubBank() + self.subscription.go(subbank=self.currentSubBank, params={'typ':'doc'}) + + security_limit = 10 + + for i in range(security_limit): + for doc in self.page.iter_documents(sub_id=subscription.id): + yield doc + + if self.page.is_last_page(): + break + + self.page.next_page() diff --git a/modules/creditmutuel/compat/weboob_capabilities_bank.py b/modules/creditmutuel/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/creditmutuel/compat/weboob_capabilities_bank.py +++ b/modules/creditmutuel/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/creditmutuel/module.py b/modules/creditmutuel/module.py index 86515c0a887e3b7eb127f09f4fb7c102798aa2a7..7afa78f8ae90ae3dd1bfab6f5388c080300fa6ae 100644 --- a/modules/creditmutuel/module.py +++ b/modules/creditmutuel/module.py @@ -21,11 +21,15 @@ from decimal import Decimal -from weboob.capabilities.base import find_object +from weboob.capabilities.base import find_object, NotAvailable from .compat.weboob_capabilities_bank import CapBankWealth, CapBankTransferAddRecipient, AccountNotFound, RecipientNotFound, \ Account, TransferError from weboob.capabilities.contact import CapContact from weboob.capabilities.profile import CapProfile +from weboob.capabilities.bill import ( + CapDocument, Subscription, SubscriptionNotFound, + Document, DocumentNotFound, +) from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword @@ -35,7 +39,11 @@ __all__ = ['CreditMutuelModule'] -class CreditMutuelModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapContact, CapProfile): +class CreditMutuelModule( + Module, CapBankWealth, CapBankTransferAddRecipient, CapDocument, + CapContact, CapProfile, +): + NAME = 'creditmutuel' MAINTAINER = u'Julien Veyssier' EMAIL = 'julien.veyssier@aiur.fr' @@ -107,6 +115,9 @@ def init_transfer(self, transfer, **params): except (AssertionError, ValueError): raise TransferError('something went wrong') + # drop characters that can crash website + transfer.label = transfer.label.encode('cp1252', errors="ignore").decode('cp1252') + return self.browser.init_transfer(account, recipient, amount, transfer.label) def execute_transfer(self, transfer, **params): @@ -119,3 +130,36 @@ def get_profile(self): if not hasattr(self.browser, 'get_profile'): raise NotImplementedError() return self.browser.get_profile() + + def get_document(self, _id): + subscription_id = _id.split('_')[0] + subscription = self.get_subscription(subscription_id) + return find_object(self.iter_documents(subscription), id=_id, error=DocumentNotFound) + + def get_subscription(self, _id): + return find_object(self.iter_subscription(), id=_id, error=SubscriptionNotFound) + + def iter_documents(self, subscription): + if not isinstance(subscription, Subscription): + subscription = self.get_subscription(subscription) + + return self.browser.iter_documents(subscription) + + def iter_subscription(self): + return self.browser.iter_subscriptions() + + def download_document(self, document): + if not isinstance(document, Document): + document = self.get_document(document) + if document.url is NotAvailable: + return + + return self.browser.open(document.url).content + + def iter_resources(self, objs, split_path): + if Account in objs: + self._restrict_level(split_path) + return self.iter_accounts() + if Subscription in objs: + self._restrict_level(split_path) + return self.iter_subscription() diff --git a/modules/creditmutuel/pages.py b/modules/creditmutuel/pages.py index 8ea71b76a2c0f5ea9aab14e0f303a70ba5d88d90..8e0d506f38e8bf3362ae5432fa7dd11388e778e6 100644 --- a/modules/creditmutuel/pages.py +++ b/modules/creditmutuel/pages.py @@ -33,15 +33,19 @@ from weboob.browser.filters.standard import Filter, Env, CleanText, CleanDecimal, Field, \ Regexp, Async, AsyncLoad, Date, Format, Type, Currency from .compat.weboob_browser_filters_html import Link, Attr, TableCell, ColumnNotFound -from weboob.exceptions import BrowserIncorrectPassword, ParseError, NoAccountsException, ActionNeeded, BrowserUnavailable +from .compat.weboob_exceptions import ( + BrowserIncorrectPassword, ParseError, NoAccountsException, ActionNeeded, BrowserUnavailable, + AuthMethodNotImplemented, +) from weboob.capabilities import NotAvailable -from weboob.capabilities.base import empty +from weboob.capabilities.base import empty, find_object from weboob.capabilities.bank import Account, Investment, Recipient, TransferError, TransferBankError, \ Transfer, AddRecipientError, AddRecipientStep, Loan from weboob.capabilities.contact import Advisor from weboob.capabilities.profile import Profile from weboob.tools.capabilities.bank.iban import is_iban_valid from weboob.tools.capabilities.bank.transactions import FrenchTransaction +from weboob.capabilities.bill import Subscription, Document from weboob.tools.compat import urlparse, parse_qs, urljoin, range, unicode from weboob.tools.date import parse_french_date from weboob.tools.value import Value @@ -77,21 +81,11 @@ def on_load(self): if self.doc.xpath(error_msg_xpath): raise BrowserIncorrectPassword(CleanText(error_msg_xpath)(self.doc)) - def convert_uncodable_char_to_xml_entity(self, word): - final_word = '' - for char in word: - try: - char.encode('cp1252') - except UnicodeEncodeError: - char = '&#{};'.format(ord(char)) - final_word += char - return final_word - def login(self, login, passwd): form = self.get_form(xpath='//form[contains(@name, "ident")]') - form['_cm_user'] = login - # format password like password sent by firefox or chromium browser - form['_cm_pwd'] = self.convert_uncodable_char_to_xml_entity(passwd) + # format login/password like login/password sent by firefox or chromium browser + form['_cm_user'] = login.encode('cp1252', errors='xmlcharrefreplace').decode('cp1252') + form['_cm_pwd'] = passwd.encode('cp1252', errors='xmlcharrefreplace').decode('cp1252') form.submit() @property @@ -112,7 +106,7 @@ def on_load(self): action_needed = CleanText('//p[contains(text(), "Votre Carte de Clés Personnelles") and contains(text(), "est révoquée")]')(self.doc) if action_needed: raise ActionNeeded(action_needed) - maintenance = CleanText('//td[@class="ALERTE"]/p/span[contains(text(), "Dans le cadre de l\'amélioration de nos services, nous vous informons que le service est interrompu"]')(self.doc) + maintenance = CleanText('//td[@class="ALERTE"]/p/span[contains(text(), "Dans le cadre de l\'amélioration de nos services, nous vous informons que le service est interrompu")]')(self.doc) if maintenance: raise BrowserUnavailable(maintenance) @@ -175,6 +169,7 @@ class item_account_generic(ItemElement): 'Allure Libre', 'Preference', 'Plan 4', + 'Credit En Reserve', ] def condition(self): @@ -208,9 +203,10 @@ def filter(self, label): obj__card_links = [] def obj__link_id(self): - page = self.page.browser.open(Link('./td[1]//a')(self)).page - if page and page.doc.xpath('//div[@class="fg"]/a[contains(@href, "%s")]' % Field('id')(self)): - return urljoin(page.url, Link('//div[@class="fg"]/a')(page.doc)) + if self.is_revolving(Field('label')(self)): + page = self.page.browser.open(Link('./td[1]//a')(self)).page + if page and page.doc.xpath('//div[@class="fg"]/a[contains(@href, "%s")]' % Field('id')(self)): + return urljoin(page.url, Link('//div[@class="fg"]/a')(page.doc)) return Link('./td[1]//a')(self) def obj_type(self): @@ -225,6 +221,9 @@ def obj_type(self): obj__is_webid = Env('_is_webid') def parse(self, el): + accounting = None + coming = None + page = None link = el.xpath('./td[1]//a')[0].get('href', '') if 'POR_SyntheseLst' in link: raise SkipItem() @@ -270,14 +269,20 @@ def parse(self, el): id = p['webid'][0] self.env['_is_webid'] = True - page = self.page.browser.open(link).page - if isinstance(page, RevolvingLoansList): - # some revolving loans are listed on an other page. On the accountList, there is just a link for this page - # that's why we don't handle it here - raise SkipItem() + if self.is_revolving(Field('label')(self)): + page = self.page.browser.open(link).page + if isinstance(page, RevolvingLoansList): + # some revolving loans are listed on an other page. On the accountList, there is just a link for this page + # that's why we don't handle it here + raise SkipItem() # Handle cards if id in self.parent.objects: + if not page: + page = self.page.browser.open(link).page + # Handle real balances + coming = page.find_amount(u"Opérations à venir") if page else None + accounting = page.find_amount(u"Solde comptable") if page else None # on old website we want card's history in account's history if not page.browser.is_new_website: account = self.parent.objects[id] @@ -330,10 +335,6 @@ def parse(self, el): self.env['id'] = id - # Handle real balances - coming = page.find_amount(u"Opérations à venir") if page else None - accounting = page.find_amount(u"Solde comptable") if page else None - if accounting is not None and accounting + (coming or Decimal('0')) != balance: self.page.logger.warning('%s + %s != %s' % (accounting, coming, balance)) @@ -345,7 +346,7 @@ def parse(self, el): def is_revolving(self, label): return any(revolving_loan_label in label - for revolving_loan_label in item_account_generic.REVOLVING_LOAN_LABELS) + for revolving_loan_label in item_account_generic.REVOLVING_LOAN_LABELS) or label.lower() in self.page.browser.revolving_accounts class AccountsPage(LoggedPage, HTMLPage): @@ -393,22 +394,18 @@ def condition(self): type = Field('type')(self) label = Field('label')(self) details_link = Link('.//a', default=None)(self) - closed_loan = False # mobile accounts are leading to a 404 error when parsing history # furthermore this is not exactly a loan account if re.search(r'Le\sMobile\s+([0-9]{2}\s?){5}', label): return False - if details_link: + if details_link and item_account_generic.condition and type == Account.TYPE_LOAN and not self.is_revolving(label): details = self.page.browser.open(details_link) - if details.page: - closed_loan = 'cloturé' in CleanText( - '//form[@id="P:F"]//div[@class="blocmsg info"]//p', default='')(details.page.doc) - return (item_account_generic.condition(self) - and type == Account.TYPE_LOAN - and not self.is_revolving(label) - and not closed_loan) + if details.page and not 'cloturé' in CleanText('//form[@id="P:F"]//div[@class="blocmsg info"]//p')(details.page.doc): + return True + return False + class item_revolving_loan(item_account_generic): klass = Loan @@ -1117,7 +1114,8 @@ def check_errors(self): 'Le solde de votre compte est insuffisant', 'Nom prénom du bénéficiaire différent du titulaire. Utilisez un compte courant', "Pour effectuer cette opération, vous devez passer par l’intermédiaire d’un compte courant", - 'Montant maximum autorisé au crédit pour ce compte'] + 'Montant maximum autorisé au crédit pour ce compte', + 'Débit interdit sur ce compte'] for message in messages: if message in content: @@ -1366,6 +1364,10 @@ def on_load(self): self.browser.need_clear_storage = True raise AddRecipientError(message=error) + app_validation = self.doc.xpath('//strong[contains(text(), "Démarrez votre application mobile Crédit Mutuel")]') + if app_validation: + raise AuthMethodNotImplemented("La confirmation par validation sur votre application mobile n'est pas supportée") + def has_list(self): return bool(CleanText('//th[contains(text(), "Listes pour virements ordinaires")]')(self.doc)) @@ -1440,7 +1442,7 @@ class item_account(ItemElement): obj_label = CleanText('.//td[2]') obj_total_amount = MyDecimal('.//td[3]') obj_currency = FrenchTransaction.Currency(CleanText('.//td[3]')) - obj_type = Account.TYPE_LOAN + obj_type = Account.TYPE_REVOLVING_CREDIT obj__is_inv = False obj__link_id = None @@ -1460,3 +1462,85 @@ def on_load(self): class RevolvingLoanDetails(LoggedPage, HTMLPage): pass + + +class SubscriptionPage(LoggedPage, HTMLPage): + def submit_form(self, subscriber): + form = self.get_form(id='frmDoc') + form['SelTiers'] = subscriber + form.submit() + + def get_subscriptions(self, subscription_list, subscriber=None): + for account in self.doc.xpath('//table[@class="liste"]//tr//td[contains(text(), "Compte")]'): + sub = Subscription() + sub.id = Regexp(CleanText('.', replace=[('.', ''), (' ', '')]), r'(\d+)')(account) + + if find_object(subscription_list, id=sub.id): + continue + + sub.label = CleanText('.')(account) + + if subscriber != None: + sub.subscriber = CleanText('.')(subscriber) + else: + sub.subscriber = CleanText('//span[@id="NomTiers"]')(self.doc) + + subscription_list.append(sub) + yield sub + + def iter_subscriptions(self): + subscription_list = [] + + options = self.doc.xpath('//select[@id="SelTiers"]/option') + if options: + for opt in options: + subscriber = self.doc.xpath('//select[@id="SelTiers"]/option[contains(text(), "%s")]' % CleanText('.')(opt))[0] + self.submit_form(Attr('.', 'value')(subscriber)) + for sub in self.get_subscriptions(subscription_list, subscriber): + yield sub + else: + for sub in self.get_subscriptions(subscription_list): + yield sub + + @method + class iter_documents(TableElement): + item_xpath = '//table[caption[contains(text(), "Extraits de comptes")]]//tr[td]' + head_xpath = '//table[@class="liste"]//th' + + col_date = 'Date' + col_label = 'Information complémentaire' + col_url = 'Nature du document' + + class item(ItemElement): + def condition(self): + return Env('sub_id')(self) == Regexp(CleanText(TableCell('label'), replace=[('.', ''), (' ', '')]), r'(\d+)')(self) + + klass = Document + + # Some documents may have the same date, name and label; only parts of the PDF href may change, + # so we must pick a unique ID including the href to avoid document duplicates: + obj_id = Format('%s_%s_%s', Env('sub_id'), CleanText(TableCell('date'), replace=[('/', '')]), Regexp(Field('url'), r'NOM=(.*)&RFL=')) + obj_label = Format('%s %s', CleanText(TableCell('url')), CleanText(TableCell('date'))) + obj_date = Date(CleanText(TableCell('date')), dayfirst=True) + obj_format = 'pdf' + obj_type = 'other' + + def obj_url(self): + return urljoin(self.page.url, '/fr/banque/%s' % Link('./a')(TableCell('url')(self)[0])) + + def next_page(self): + form = self.get_form(id='frmDoc') + form['__EVENTTARGET'] = '' + form['__EVENTARGUMENT'] = '' + form['__LASTFOCUS'] = '' + form['SelIndex1'] = '' + form['NEXT.x'] = '7' + form['NEXT.y'] = '8' + form.submit() + + def is_last_page(self): + if self.doc.xpath('//td[has-class("ERREUR")]'): + return True + if re.search(r'(\d\/\d)', CleanText('//div[has-class("blocpaginb")]', symbols=' ')(self.doc)): + return True + return False diff --git a/modules/edf/par/browser.py b/modules/edf/par/browser.py index e24a2a9591c6f5358a297d9caa55ec78ccaf8ca2..bd685cef80d53c3df4b201ff4f52d34303d6cad3 100644 --- a/modules/edf/par/browser.py +++ b/modules/edf/par/browser.py @@ -21,10 +21,11 @@ from time import time from weboob.browser.browsers import LoginBrowser, URL, need_login +from weboob.browser.exceptions import ClientError from .compat.weboob_exceptions import BrowserIncorrectPassword, NocaptchaQuestion from weboob.tools.json import json from .pages import ( - HomePage, LoginPage, ProfilPage, + HomePage, AuthenticatePage, AuthorizePage, CheckAuthenticatePage, ProfilPage, DocumentsPage, WelcomePage, UnLoggedPage, ProfilePage, ) @@ -32,8 +33,10 @@ class EdfBrowser(LoginBrowser): BASEURL = 'https://particulier.edf.fr' - home = URL('/fr/accueil.html', HomePage) - login = URL('/bin/edf_rc/servlets/authentication', LoginPage) + home = URL('/fr/accueil/contrat-et-conso/mon-compte-edf.html', HomePage) + authenticate = URL(r'https://espace-client.edf.fr/sso/json/authenticate', AuthenticatePage) + authorize = URL(r'https://espace-client.edf.fr/sso/oauth2/INTERNET/authorize', AuthorizePage) + check_authenticate = URL('/services/rest/openid/checkAuthenticate', CheckAuthenticatePage) not_connected = URL('/fr/accueil/connexion/mon-espace-client.html', UnLoggedPage) connected = URL('/fr/accueil/espace-client/tableau-de-bord.html', WelcomePage) profil = URL('/services/rest/authenticate/getListContracts', ProfilPage) @@ -46,36 +49,52 @@ class EdfBrowser(LoginBrowser): def __init__(self, config, *args, **kwargs): self.config = config + self.authId = None kwargs['username'] = self.config['login'].get() kwargs['password'] = self.config['password'].get() super(EdfBrowser, self).__init__(*args, **kwargs) def do_login(self): - self.connected.go() - if self.not_connected.is_here(): - if self.config['captcha_response'].get(): - self.login.go(data={'login': self.username, - 'password': self.password, - 'rememberMe': "false", - 'goto': None, - 'gRecaptchaAuthentResponse': self.config['captcha_response'].get()}) - self.connected.go() - - if self.not_connected.is_here(): - raise BrowserIncorrectPassword() - else: - return - + auth_params = {'realm': '/INTERNET'} + if self.config['captcha_response'].get() and self.authId: + self.authenticate.go(method='POST', params=auth_params) + data = self.page.get_data() + data['authId'] = self.authId + data['callbacks'][0]['input'][0]['value'] = self.username + data['callbacks'][1]['input'][0]['value'] = self.password + data['callbacks'][2]['input'][0]['value'] = self.config['captcha_response'].get() + data['callbacks'][3]['input'][0]['value'] = '0' + + try: + self.authenticate.go(json=data, params=auth_params) + except ClientError as error: + resp = error.response + if resp.status_code == 401: + raise BrowserIncorrectPassword(resp.json()['message']) + raise + + self.session.cookies['ivoiream'] = self.page.get_data()['tokenId'] + + # go to this url will auto submit a form which will finalize login + self.connected.go() + + """ + call check_authenticate url before get subscription in profil, or we'll get an error 'invalid session' + we do nothing with this response (which contains false btw) + but edf website expect we call it before or will reject us + """ + self.check_authenticate.go() + else: self.home.go() - if self.page.has_captcha_request(): - website_key = self.page.get_recaptcha_key() # google recaptcha plubic key - website_url = "https://particulier.edf.fr/fr/accueil.html" + # google recaptcha site key is returned here, but it's not a good one, take it from another url + self.authenticate.go(method='POST', params=auth_params) + data = self.page.get_data() + website_key = data['callbacks'][4]['output'][0]['value'] + website_url = "https://espace-client.edf.fr/sso/XUI/#login/&realm=%2FINTERNET" + self.authId = data['authId'] + raise NocaptchaQuestion(website_key=website_key, website_url=website_url) - else: - raise BrowserIncorrectPassword() - else: - return def get_csrf_token(self): return self.csrf_token.go(timestamp=int(time())).get_token() diff --git a/modules/edf/par/pages.py b/modules/edf/par/pages.py index 70a4658ee11a37f4a479422f1e795a2680302210..8fd94a972f5e36ff7cb3749122ade51190ce520d 100644 --- a/modules/edf/par/pages.py +++ b/modules/edf/par/pages.py @@ -21,8 +21,9 @@ from datetime import datetime from decimal import Decimal -from weboob.browser.pages import LoggedPage, JsonPage, HTMLPage -from weboob.browser.filters.standard import Env, Format, Date, Eval, CleanText +from weboob.browser.filters.html import Attr +from weboob.browser.pages import LoggedPage, JsonPage, HTMLPage, RawPage +from weboob.browser.filters.standard import Env, Format, Date, Eval from weboob.browser.elements import ItemElement, DictElement, method from weboob.browser.filters.json import Dict from weboob.capabilities.bill import Bill, Subscription @@ -34,19 +35,26 @@ class HomePage(HTMLPage): def has_captcha_request(self): return len(self.doc.xpath('//div[@class="captcha"]')) > 0 - def get_recaptcha_key(self): - return CleanText(self.doc.xpath('(//input[@name="captchaPublicKeyAuth"]/@value)[1]'))(self.doc) +class AuthenticatePage(JsonPage): + def get_data(self): + return self.doc -class LoginPage(JsonPage): - def is_logged(self): - return "200" in Dict('errorCode')(self.doc) + +class AuthorizePage(HTMLPage): + def on_load(self): + if Attr('//body', 'onload', default=NotAvailable)(self.doc): + self.get_form().submit() class WelcomePage(LoggedPage, HTMLPage): pass +class CheckAuthenticatePage(LoggedPage, RawPage): + pass + + class UnLoggedPage(HTMLPage): pass @@ -107,6 +115,7 @@ def get_bills_informations(self): 'parNumber': Dict('parNumber')(self.doc) } + class ProfilePage(LoggedPage, JsonPage): def get_profile(self): data = self.doc['bp'] diff --git a/modules/entreparticuliers/compat/weboob_capabilities_housing.py b/modules/entreparticuliers/compat/weboob_capabilities_housing.py index a7456e29ce1b0da440e193efaed4689cedb6d533..f763fcbaf7f9b168db3d2069e15ff965d2faac3d 100644 --- a/modules/entreparticuliers/compat/weboob_capabilities_housing.py +++ b/modules/entreparticuliers/compat/weboob_capabilities_housing.py @@ -1,192 +1,31 @@ -# -*- coding: utf-8 -*- -# Copyright(C) 2012 Romain Bignon -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . +import weboob.capabilities.housing as OLD +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) -from weboob.capabilities.base import Capability, BaseObject, Field, IntField, DecimalField, \ - StringField, BytesField, StrEnum, EnumField, UserError -from weboob.capabilities.date import DateField -__all__ = [ - 'CapHousing', 'Housing', 'Query', 'City', 'UTILITIES', 'ENERGY_CLASS', - 'POSTS_TYPES', 'ADVERT_TYPES', 'HOUSE_TYPES', 'TypeNotSupported', - 'HousingPhoto', -] +__all__ = OLD.__all__ -from weboob.capabilities.housing import TypeNotSupported as _TypeNotSupported -class TypeNotSupported(_TypeNotSupported): - """ - Raised when query type is not supported - """ +ENERGY_CLASS = enum(A=u'A', B=u'B', C=u'C', D=u'D', E=u'E', F=u'F', G=u'G') - def __init__(self, - msg='This type of house is not supported by this module'): - super(TypeNotSupported, self).__init__(msg) +POSTS_TYPES = enum(RENT=u'RENT', + SALE=u'SALE', + SHARING=u'SHARING', + FURNISHED_RENT=u'FURNISHED_RENT', + VIAGER=u'VIAGER') -from weboob.capabilities.housing import HousingPhoto as _HousingPhoto -class HousingPhoto(_HousingPhoto): - """ - Photo of a housing. - """ - data = BytesField('Data of photo') - def __init__(self, url): - super(HousingPhoto, self).__init__(url.split('/')[-1], url) +ADVERT_TYPES = enum(PROFESSIONAL=u'Professional', PERSONAL=u'Personal') - def __iscomplete__(self): - return self.data - def __unicode__(self): - return self.url - - def __repr__(self): - return '' % (self.id, len(self.data) if self.data else 0, self.url) - - -class UTILITIES(StrEnum): - INCLUDED = u'C.C.' - EXCLUDED = u'H.C.' - UNKNOWN = u'' - - -class ENERGY_CLASS(StrEnum): - A = u'A' - B = u'B' - C = u'C' - D = u'D' - E = u'E' - F = u'F' - G = u'G' - - -class POSTS_TYPES(StrEnum): - RENT=u'RENT' - SALE = u'SALE' - SHARING = u'SHARING' - FURNISHED_RENT = u'FURNISHED_RENT' - VIAGER = u'VIAGER' - - -class ADVERT_TYPES(StrEnum): - PROFESSIONAL = u'Professional' - PERSONAL = u'Personal' - - -class HOUSE_TYPES(StrEnum): - APART = u'Apartment' - HOUSE = u'House' - PARKING = u'Parking' - LAND = u'Land' - OTHER = u'Other' - UNKNOWN = u'Unknown' - - -from weboob.capabilities.housing import Housing as _Housing -class Housing(_Housing): - """ - Content of a housing. - """ - type = EnumField('Type of housing (rent, sale, sharing)', - POSTS_TYPES) - advert_type = EnumField('Type of advert (professional or personal)', - ADVERT_TYPES) - house_type = EnumField(u'Type of house (apartment, house, parking, …)', - HOUSE_TYPES) - title = StringField('Title of housing') - area = DecimalField('Area of housing, in m2') - cost = DecimalField('Cost of housing') - price_per_meter = DecimalField('Price per meter ratio') - currency = StringField('Currency of cost') - utilities = EnumField('Utilities included or not', UTILITIES) - date = DateField('Date when the housing has been published') - location = StringField('Location of housing') - station = StringField('What metro/bus station next to housing') - text = StringField('Text of the housing') - phone = StringField('Phone number to contact') - photos = Field('List of photos', list) - rooms = DecimalField('Number of rooms') - bedrooms = DecimalField('Number of bedrooms') - details = Field('Key/values of details', dict) - DPE = EnumField('DPE (Energy Performance Certificate)', ENERGY_CLASS) - GES = EnumField('GES (Greenhouse Gas Emissions)', ENERGY_CLASS) - - -from weboob.capabilities.housing import Query as _Query -class Query(_Query): - """ - Query to find housings. - """ - type = EnumField('Type of housing to find (POSTS_TYPES constants)', - POSTS_TYPES) - cities = Field('List of cities to search in', list, tuple) - area_min = IntField('Minimal area (in m2)') - area_max = IntField('Maximal area (in m2)') - cost_min = IntField('Minimal cost') - cost_max = IntField('Maximal cost') - nb_rooms = IntField('Number of rooms') - house_types = Field('List of house types', list, tuple, default=list(HOUSE_TYPES)) - advert_types = Field('List of advert types to filter on', list, tuple, - default=list(ADVERT_TYPES)) - - -from weboob.capabilities.housing import City as _City -class City(_City): - """ - City. - """ - name = StringField('Name of city') - - -from weboob.capabilities.housing import CapHousing as _CapHousing -class CapHousing(_CapHousing): - """ - Capability of websites to search housings. - """ - - def search_housings(self, query): - """ - Search housings. - - :param query: search query - :type query: :class:`Query` - :rtype: iter[:class:`Housing`] - """ - raise NotImplementedError() - - def get_housing(self, housing): - """ - Get an housing from an ID. - - :param housing: ID of the housing - :type housing: str - :rtype: :class:`Housing` or None if not found. - """ - raise NotImplementedError() - - def search_city(self, pattern): - """ - Search a city from a pattern. - - :param pattern: pattern to search - :type pattern: str - :rtype: iter[:class:`City`] - """ - raise NotImplementedError() +HOUSE_TYPES = enum(APART=u'Apartment', + HOUSE=u'House', + PARKING=u'Parking', + LAND=u'Land', + OTHER=u'Other', + UNKNOWN=u'Unknown') diff --git a/modules/explorimmo/browser.py b/modules/explorimmo/browser.py index 69376e8faa099cb3d71fd5290cf6512be9c90007..a2c6ced64f95bc445e5eb7ed3cd14cc4ee2c79f1 100644 --- a/modules/explorimmo/browser.py +++ b/modules/explorimmo/browser.py @@ -25,14 +25,14 @@ class ExplorimmoBrowser(PagesBrowser): - BASEURL = 'https://www.explorimmo.com/' - - cities = URL('rest/locations\?q=(?P.*)', CitiesPage) - search = URL('resultat/annonces.html\?(?P.*)', SearchPage) - housing_html = URL('annonce-(?P<_id>.*).html', HousingPage) - phone = URL('rest/classifieds/(?P<_id>.*)/phone', PhonePage) - housing = URL('rest/classifieds/(?P<_id>.*)', - 'rest/classifieds/\?(?P.*)', HousingPage2) + BASEURL = 'https://immobilier.lefigaro.fr' + + cities = URL('/rest/locations\?q=(?P.*)', CitiesPage) + search = URL('/annonces/resultat/annonces.html\?(?P.*)', SearchPage) + housing_html = URL('/annonce-(?P<_id>.*).html', HousingPage) + phone = URL('/rest/classifieds/(?P<_id>.*)/phone', PhonePage) + housing = URL('/rest/classifieds/(?P<_id>.*)', + '/rest/classifieds/\?(?P.*)', HousingPage2) TYPES = {POSTS_TYPES.RENT: 'location', POSTS_TYPES.SALE: 'vente', diff --git a/modules/explorimmo/compat/weboob_capabilities_housing.py b/modules/explorimmo/compat/weboob_capabilities_housing.py index a7456e29ce1b0da440e193efaed4689cedb6d533..f763fcbaf7f9b168db3d2069e15ff965d2faac3d 100644 --- a/modules/explorimmo/compat/weboob_capabilities_housing.py +++ b/modules/explorimmo/compat/weboob_capabilities_housing.py @@ -1,192 +1,31 @@ -# -*- coding: utf-8 -*- -# Copyright(C) 2012 Romain Bignon -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . +import weboob.capabilities.housing as OLD +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) -from weboob.capabilities.base import Capability, BaseObject, Field, IntField, DecimalField, \ - StringField, BytesField, StrEnum, EnumField, UserError -from weboob.capabilities.date import DateField -__all__ = [ - 'CapHousing', 'Housing', 'Query', 'City', 'UTILITIES', 'ENERGY_CLASS', - 'POSTS_TYPES', 'ADVERT_TYPES', 'HOUSE_TYPES', 'TypeNotSupported', - 'HousingPhoto', -] +__all__ = OLD.__all__ -from weboob.capabilities.housing import TypeNotSupported as _TypeNotSupported -class TypeNotSupported(_TypeNotSupported): - """ - Raised when query type is not supported - """ +ENERGY_CLASS = enum(A=u'A', B=u'B', C=u'C', D=u'D', E=u'E', F=u'F', G=u'G') - def __init__(self, - msg='This type of house is not supported by this module'): - super(TypeNotSupported, self).__init__(msg) +POSTS_TYPES = enum(RENT=u'RENT', + SALE=u'SALE', + SHARING=u'SHARING', + FURNISHED_RENT=u'FURNISHED_RENT', + VIAGER=u'VIAGER') -from weboob.capabilities.housing import HousingPhoto as _HousingPhoto -class HousingPhoto(_HousingPhoto): - """ - Photo of a housing. - """ - data = BytesField('Data of photo') - def __init__(self, url): - super(HousingPhoto, self).__init__(url.split('/')[-1], url) +ADVERT_TYPES = enum(PROFESSIONAL=u'Professional', PERSONAL=u'Personal') - def __iscomplete__(self): - return self.data - def __unicode__(self): - return self.url - - def __repr__(self): - return '' % (self.id, len(self.data) if self.data else 0, self.url) - - -class UTILITIES(StrEnum): - INCLUDED = u'C.C.' - EXCLUDED = u'H.C.' - UNKNOWN = u'' - - -class ENERGY_CLASS(StrEnum): - A = u'A' - B = u'B' - C = u'C' - D = u'D' - E = u'E' - F = u'F' - G = u'G' - - -class POSTS_TYPES(StrEnum): - RENT=u'RENT' - SALE = u'SALE' - SHARING = u'SHARING' - FURNISHED_RENT = u'FURNISHED_RENT' - VIAGER = u'VIAGER' - - -class ADVERT_TYPES(StrEnum): - PROFESSIONAL = u'Professional' - PERSONAL = u'Personal' - - -class HOUSE_TYPES(StrEnum): - APART = u'Apartment' - HOUSE = u'House' - PARKING = u'Parking' - LAND = u'Land' - OTHER = u'Other' - UNKNOWN = u'Unknown' - - -from weboob.capabilities.housing import Housing as _Housing -class Housing(_Housing): - """ - Content of a housing. - """ - type = EnumField('Type of housing (rent, sale, sharing)', - POSTS_TYPES) - advert_type = EnumField('Type of advert (professional or personal)', - ADVERT_TYPES) - house_type = EnumField(u'Type of house (apartment, house, parking, …)', - HOUSE_TYPES) - title = StringField('Title of housing') - area = DecimalField('Area of housing, in m2') - cost = DecimalField('Cost of housing') - price_per_meter = DecimalField('Price per meter ratio') - currency = StringField('Currency of cost') - utilities = EnumField('Utilities included or not', UTILITIES) - date = DateField('Date when the housing has been published') - location = StringField('Location of housing') - station = StringField('What metro/bus station next to housing') - text = StringField('Text of the housing') - phone = StringField('Phone number to contact') - photos = Field('List of photos', list) - rooms = DecimalField('Number of rooms') - bedrooms = DecimalField('Number of bedrooms') - details = Field('Key/values of details', dict) - DPE = EnumField('DPE (Energy Performance Certificate)', ENERGY_CLASS) - GES = EnumField('GES (Greenhouse Gas Emissions)', ENERGY_CLASS) - - -from weboob.capabilities.housing import Query as _Query -class Query(_Query): - """ - Query to find housings. - """ - type = EnumField('Type of housing to find (POSTS_TYPES constants)', - POSTS_TYPES) - cities = Field('List of cities to search in', list, tuple) - area_min = IntField('Minimal area (in m2)') - area_max = IntField('Maximal area (in m2)') - cost_min = IntField('Minimal cost') - cost_max = IntField('Maximal cost') - nb_rooms = IntField('Number of rooms') - house_types = Field('List of house types', list, tuple, default=list(HOUSE_TYPES)) - advert_types = Field('List of advert types to filter on', list, tuple, - default=list(ADVERT_TYPES)) - - -from weboob.capabilities.housing import City as _City -class City(_City): - """ - City. - """ - name = StringField('Name of city') - - -from weboob.capabilities.housing import CapHousing as _CapHousing -class CapHousing(_CapHousing): - """ - Capability of websites to search housings. - """ - - def search_housings(self, query): - """ - Search housings. - - :param query: search query - :type query: :class:`Query` - :rtype: iter[:class:`Housing`] - """ - raise NotImplementedError() - - def get_housing(self, housing): - """ - Get an housing from an ID. - - :param housing: ID of the housing - :type housing: str - :rtype: :class:`Housing` or None if not found. - """ - raise NotImplementedError() - - def search_city(self, pattern): - """ - Search a city from a pattern. - - :param pattern: pattern to search - :type pattern: str - :rtype: iter[:class:`City`] - """ - raise NotImplementedError() +HOUSE_TYPES = enum(APART=u'Apartment', + HOUSE=u'House', + PARKING=u'Parking', + LAND=u'Land', + OTHER=u'Other', + UNKNOWN=u'Unknown') diff --git a/modules/explorimmo/pages.py b/modules/explorimmo/pages.py index 957e7376934b68fd1a83d8c95eee3b9cc6563826..b038d2e364908603fefb39595bf2dacf3d16160c 100644 --- a/modules/explorimmo/pages.py +++ b/modules/explorimmo/pages.py @@ -113,6 +113,7 @@ def condition(self): obj_id = Attr('.', 'data-classified-id') obj_type = Env('query_type') + obj_title = CleanText('./div/h2[@class="item-type"]') def obj_advert_type(self): if self.is_agency(): @@ -133,8 +134,6 @@ def obj_house_type(self): else: return HOUSE_TYPES.OTHER - obj_title = CleanText('.//*[has-class("js-item-title")]') - def obj_location(self): script = CleanText('./script')(self) try: @@ -163,7 +162,7 @@ def obj_cost(self): r'de (.*) à .*', default=0))(self) if cost == 0: - return CleanDecimal(self.price_selector, default=NotLoaded)(self) + return CleanDecimal(self.price_selector, default=NotAvailable)(self) else: return cost @@ -180,7 +179,7 @@ def obj_utilities(self): else: return UTILITIES.UNKNOWN - obj_text = CleanText('./div/div/div[@itemprop="description"]') + obj_text = CleanText('./div/p[@itemprop="description"]') obj_area = CleanDecimal( Regexp( obj_title, @@ -220,11 +219,7 @@ def obj_details(self): return NotLoaded def obj_photos(self): - url = Attr( - '.', - 'data-img', - default=None - )(self) + url = CleanText('./div[has-class("default-img")]/img/@data-src')(self) if url: url = unquote(url) if "http://" in url[3:]: diff --git a/modules/explorimmo/test.py b/modules/explorimmo/test.py index 7c0bd73322a7de221a2a6465fabcdae89e092da7..812e57a11906129f15649b423455becd9c502fdd 100644 --- a/modules/explorimmo/test.py +++ b/modules/explorimmo/test.py @@ -27,10 +27,10 @@ class ExplorimmoTest(BackendTest, HousingTest): FIELDS_ALL_HOUSINGS_LIST = [ "id", "type", "advert_type", "house_type", "title", "location", - "cost", "currency", "utilities", "text", "area", "url" + "utilities", "text", "area", "url" ] FIELDS_ANY_HOUSINGS_LIST = [ - "photos" + "photos", "cost", "currency" ] FIELDS_ALL_SINGLE_HOUSING = [ "id", "url", "type", "advert_type", "house_type", "title", "area", diff --git a/modules/foncia/compat/weboob_capabilities_housing.py b/modules/foncia/compat/weboob_capabilities_housing.py index a7456e29ce1b0da440e193efaed4689cedb6d533..f763fcbaf7f9b168db3d2069e15ff965d2faac3d 100644 --- a/modules/foncia/compat/weboob_capabilities_housing.py +++ b/modules/foncia/compat/weboob_capabilities_housing.py @@ -1,192 +1,31 @@ -# -*- coding: utf-8 -*- -# Copyright(C) 2012 Romain Bignon -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . +import weboob.capabilities.housing as OLD +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) -from weboob.capabilities.base import Capability, BaseObject, Field, IntField, DecimalField, \ - StringField, BytesField, StrEnum, EnumField, UserError -from weboob.capabilities.date import DateField -__all__ = [ - 'CapHousing', 'Housing', 'Query', 'City', 'UTILITIES', 'ENERGY_CLASS', - 'POSTS_TYPES', 'ADVERT_TYPES', 'HOUSE_TYPES', 'TypeNotSupported', - 'HousingPhoto', -] +__all__ = OLD.__all__ -from weboob.capabilities.housing import TypeNotSupported as _TypeNotSupported -class TypeNotSupported(_TypeNotSupported): - """ - Raised when query type is not supported - """ +ENERGY_CLASS = enum(A=u'A', B=u'B', C=u'C', D=u'D', E=u'E', F=u'F', G=u'G') - def __init__(self, - msg='This type of house is not supported by this module'): - super(TypeNotSupported, self).__init__(msg) +POSTS_TYPES = enum(RENT=u'RENT', + SALE=u'SALE', + SHARING=u'SHARING', + FURNISHED_RENT=u'FURNISHED_RENT', + VIAGER=u'VIAGER') -from weboob.capabilities.housing import HousingPhoto as _HousingPhoto -class HousingPhoto(_HousingPhoto): - """ - Photo of a housing. - """ - data = BytesField('Data of photo') - def __init__(self, url): - super(HousingPhoto, self).__init__(url.split('/')[-1], url) +ADVERT_TYPES = enum(PROFESSIONAL=u'Professional', PERSONAL=u'Personal') - def __iscomplete__(self): - return self.data - def __unicode__(self): - return self.url - - def __repr__(self): - return '' % (self.id, len(self.data) if self.data else 0, self.url) - - -class UTILITIES(StrEnum): - INCLUDED = u'C.C.' - EXCLUDED = u'H.C.' - UNKNOWN = u'' - - -class ENERGY_CLASS(StrEnum): - A = u'A' - B = u'B' - C = u'C' - D = u'D' - E = u'E' - F = u'F' - G = u'G' - - -class POSTS_TYPES(StrEnum): - RENT=u'RENT' - SALE = u'SALE' - SHARING = u'SHARING' - FURNISHED_RENT = u'FURNISHED_RENT' - VIAGER = u'VIAGER' - - -class ADVERT_TYPES(StrEnum): - PROFESSIONAL = u'Professional' - PERSONAL = u'Personal' - - -class HOUSE_TYPES(StrEnum): - APART = u'Apartment' - HOUSE = u'House' - PARKING = u'Parking' - LAND = u'Land' - OTHER = u'Other' - UNKNOWN = u'Unknown' - - -from weboob.capabilities.housing import Housing as _Housing -class Housing(_Housing): - """ - Content of a housing. - """ - type = EnumField('Type of housing (rent, sale, sharing)', - POSTS_TYPES) - advert_type = EnumField('Type of advert (professional or personal)', - ADVERT_TYPES) - house_type = EnumField(u'Type of house (apartment, house, parking, …)', - HOUSE_TYPES) - title = StringField('Title of housing') - area = DecimalField('Area of housing, in m2') - cost = DecimalField('Cost of housing') - price_per_meter = DecimalField('Price per meter ratio') - currency = StringField('Currency of cost') - utilities = EnumField('Utilities included or not', UTILITIES) - date = DateField('Date when the housing has been published') - location = StringField('Location of housing') - station = StringField('What metro/bus station next to housing') - text = StringField('Text of the housing') - phone = StringField('Phone number to contact') - photos = Field('List of photos', list) - rooms = DecimalField('Number of rooms') - bedrooms = DecimalField('Number of bedrooms') - details = Field('Key/values of details', dict) - DPE = EnumField('DPE (Energy Performance Certificate)', ENERGY_CLASS) - GES = EnumField('GES (Greenhouse Gas Emissions)', ENERGY_CLASS) - - -from weboob.capabilities.housing import Query as _Query -class Query(_Query): - """ - Query to find housings. - """ - type = EnumField('Type of housing to find (POSTS_TYPES constants)', - POSTS_TYPES) - cities = Field('List of cities to search in', list, tuple) - area_min = IntField('Minimal area (in m2)') - area_max = IntField('Maximal area (in m2)') - cost_min = IntField('Minimal cost') - cost_max = IntField('Maximal cost') - nb_rooms = IntField('Number of rooms') - house_types = Field('List of house types', list, tuple, default=list(HOUSE_TYPES)) - advert_types = Field('List of advert types to filter on', list, tuple, - default=list(ADVERT_TYPES)) - - -from weboob.capabilities.housing import City as _City -class City(_City): - """ - City. - """ - name = StringField('Name of city') - - -from weboob.capabilities.housing import CapHousing as _CapHousing -class CapHousing(_CapHousing): - """ - Capability of websites to search housings. - """ - - def search_housings(self, query): - """ - Search housings. - - :param query: search query - :type query: :class:`Query` - :rtype: iter[:class:`Housing`] - """ - raise NotImplementedError() - - def get_housing(self, housing): - """ - Get an housing from an ID. - - :param housing: ID of the housing - :type housing: str - :rtype: :class:`Housing` or None if not found. - """ - raise NotImplementedError() - - def search_city(self, pattern): - """ - Search a city from a pattern. - - :param pattern: pattern to search - :type pattern: str - :rtype: iter[:class:`City`] - """ - raise NotImplementedError() +HOUSE_TYPES = enum(APART=u'Apartment', + HOUSE=u'House', + PARKING=u'Parking', + LAND=u'Land', + OTHER=u'Other', + UNKNOWN=u'Unknown') diff --git a/modules/fortuneo/browser.py b/modules/fortuneo/browser.py index 66b052d45eaa1c3a7820e2bcb09ec14edef76a4d..d39ffbbe04879a68a32f8529b29e32add08b876f 100644 --- a/modules/fortuneo/browser.py +++ b/modules/fortuneo/browser.py @@ -18,23 +18,32 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals + import time +import json +from datetime import datetime, timedelta -from weboob.browser.browsers import LoginBrowser, URL, need_login +from weboob.browser.browsers import LoginBrowser, URL, need_login, StatesMixin from .compat.weboob_exceptions import AuthMethodNotImplemented, BrowserIncorrectPassword -from weboob.capabilities.bank import Account +from weboob.capabilities.bank import Account, AddRecipientStep, Recipient from weboob.tools.capabilities.bank.transactions import sorted_transactions +from weboob.tools.value import Value from .pages.login import LoginPage, UnavailablePage from .pages.accounts_list import ( AccountsList, AccountHistoryPage, CardHistoryPage, InvestmentHistoryPage, PeaHistoryPage, LoanPage, ) +from .pages.transfer import ( + RegisterTransferPage, ValidateTransferPage, ConfirmTransferPage, RecipientsPage, RecipientSMSPage +) __all__ = ['Fortuneo'] -class Fortuneo(LoginBrowser): +class Fortuneo(LoginBrowser, StatesMixin): BASEURL = 'https://mabanque.fortuneo.fr' + STATE_DURATION = 5 login_page = URL(r'.*identification\.jsp.*', LoginPage) @@ -54,10 +63,36 @@ class Fortuneo(LoginBrowser): loan_contract = URL(r'/fr/prive/mes-comptes/credit-immo/contrat-credit-immo/contrat-pret-immobilier.jsp.*', LoanPage) unavailable = URL(r'/customError/indispo.html', UnavailablePage) + # transfer + recipients = URL( + r'/fr/prive/mes-comptes/compte-courant/realiser-operations/gerer-comptes-externes/consulter-comptes-externes.jsp', + r'/fr/prive/verifier-compte-externe.jsp', + r'fr/prive/mes-comptes/compte-courant/.*/gestion-comptes-externes.jsp', + RecipientsPage) + recipient_sms = URL( + r'/fr/prive/appel-securite-forte-otp-bankone.jsp', + r'/fr/prive/mes-comptes/compte-courant/.*/confirmer-ajout-compte-externe.jsp', + RecipientSMSPage) + register_transfer = URL( + r'/fr/prive/mes-comptes/compte-courant/realiser-operations/saisie-virement.jsp\?ca=(?P)', + RegisterTransferPage) + validate_transfer = URL( + r'/fr/prive/mes-comptes/compte-courant/.*/verifier-saisie-virement.jsp', + ValidateTransferPage) + confirm_transfer = URL( + r'fr/prive/mes-comptes/compte-courant/.*/init-confirmer-saisie-virement.jsp', + r'/fr/prive/mes-comptes/compte-courant/.*/confirmer-saisie-virement.jsp', + ConfirmTransferPage) + + need_reload_state = None + + __states__ = ['need_reload_state', 'add_recipient_form'] + def __init__(self, *args, **kwargs): LoginBrowser.__init__(self, *args, **kwargs) self.investments = {} self.action_needed_processed = False + self.add_recipient_form = None def do_login(self): if not self.login_page.is_here(): @@ -72,6 +107,13 @@ def do_login(self): if self.accounts_page.is_here() and self.page.need_sms(): raise AuthMethodNotImplemented('Authentification with sms is not supported') + def load_state(self, state): + # reload state only for new recipient feature + if state.get('need_reload_state'): + # don't use locate browser for add recipient step + state.pop('url', None) + super(Fortuneo, self).load_state(state) + @need_login def get_investments(self, account): if hasattr(account, '_investment_link'): @@ -127,3 +169,89 @@ def process_action_needed(self): self.accounts_page.go() # go back to the accounts page whenever there was an iframe or not self.action_needed_processed = True + + @need_login + def iter_recipients(self, origin_account): + self.register_transfer.go(ca=origin_account._ca) + if self.page.is_account_transferable(origin_account): + for internal_recipient in self.page.iter_internal_recipients(origin_account_id=origin_account.id): + yield internal_recipient + + self.recipients.go() + for external_recipients in self.page.iter_external_recipients(): + yield external_recipients + + def copy_recipient(self, recipient): + rcpt = Recipient() + rcpt.iban = recipient.iban + rcpt.id = recipient.iban + rcpt.label = recipient.label + rcpt.category = recipient.category + rcpt.enabled_at = datetime.now().replace(microsecond=0) + timedelta(days=1) + rcpt.currency = u'EUR' + return rcpt + + def new_recipient(self, recipient, **params): + if 'code' in params: + self.need_reload_state = None + # to drop and use self.add_recipient_form instead in send_code() + recipient_form = json.loads(self.add_recipient_form) + self.send_code(recipient_form ,params['code']) + assert self.page.rcpt_after_sms() + return self.copy_recipient(recipient) + return self.new_recipient_before_otp(recipient, **params) + + @need_login + def new_recipient_before_otp(self, recipient, **params): + self.recipients.go() + self.page.check_external_iban_form(recipient) + self.page.check_recipient_iban() + + # fill form + self.page.fill_recipient_form(recipient) + rcpt = self.page.get_new_recipient(recipient) + + # get first part of confirm form + send_code_form = self.page.get_send_code_form() + + data = { + 'appelAjax': 'true', + 'domicileUpdated': 'false', + 'numeroSelectionne.value': '', + 'portableUpdated': 'false', + 'proUpdated': 'false', + 'typeOperationSensible': 'AJOUT_BENEFICIAIRE' + } + # this send sms to user + self.location(self.absurl('/fr/prive/appel-securite-forte-otp-bankone.jsp', base=True) , data=data) + # get second part of confirm form + send_code_form.update(self.page.get_send_code_form_input()) + + # save form value and url for statesmixin + self.add_recipient_form = dict(send_code_form) + self.add_recipient_form.update({'url': send_code_form.url}) + + # storage can't handle dict with '.' in key + # to drop when dict with '.' in key is handled + self.add_recipient_form = json.dumps(self.add_recipient_form) + + self.need_reload_state = True + raise AddRecipientStep(rcpt, Value('code', label='Veuillez saisir le code reçu.')) + + def send_code(self, form_data, code): + form_url = form_data['url'] + form_data['otp'] = code + form_data.pop('url') + self.location(self.absurl(form_url, base=True), data=form_data) + + @need_login + def init_transfer(self, account, recipient, amount, label, exec_date): + self.register_transfer.go(ca=account._ca) + self.page.fill_transfer_form(account, recipient, amount, label, exec_date) + return self.page.handle_response(account, recipient, amount, label, exec_date) + + @need_login + def execute_transfer(self, transfer): + self.page.validate_transfer() + self.page.confirm_transfer() + return self.page.transfer_confirmation(transfer) diff --git a/modules/fortuneo/compat/weboob_capabilities_bank.py b/modules/fortuneo/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/fortuneo/compat/weboob_capabilities_bank.py +++ b/modules/fortuneo/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/fortuneo/module.py b/modules/fortuneo/module.py index 494b014ae03540e68ac39c4b4d9d74b70ce1e231..9aab657e641b5ad778f6e6f947f8900df58953db 100644 --- a/modules/fortuneo/module.py +++ b/modules/fortuneo/module.py @@ -19,7 +19,10 @@ from weboob.capabilities.base import find_object -from .compat.weboob_capabilities_bank import CapBankWealth, AccountNotFound +from .compat.weboob_capabilities_bank import ( + CapBankWealth, CapBankTransferAddRecipient, AccountNotFound, RecipientNotFound, + TransferInvalidLabel, Account, +) from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword @@ -29,7 +32,7 @@ __all__ = ['FortuneoModule'] -class FortuneoModule(Module, CapBankWealth): +class FortuneoModule(Module, CapBankWealth, CapBankTransferAddRecipient): NAME = 'fortuneo' MAINTAINER = u'Gilles-Alexandre Quenot' EMAIL = 'gilles.quenot@gmail.com' @@ -64,4 +67,33 @@ def iter_coming(self, account): def iter_investment(self, account): return self.browser.get_investments(account) + def iter_transfer_recipients(self, origin_account): + if isinstance(origin_account, Account): + origin_account = origin_account.id + return self.browser.iter_recipients(self.get_account(origin_account)) + + def new_recipient(self, recipient, **params): + recipient.label = recipient.label[:35].upper() + return self.browser.new_recipient(recipient, **params) + + def init_transfer(self, transfer, **params): + if not transfer.label: + raise TransferInvalidLabel() + + self.logger.info('Going to do a new transfer') + if transfer.account_iban: + account = find_object(self.iter_accounts(), iban=transfer.account_iban, error=AccountNotFound) + else: + account = find_object(self.iter_accounts(), id=transfer.account_id, error=AccountNotFound) + + if transfer.recipient_iban: + recipient = find_object(self.iter_transfer_recipients(account.id), iban=transfer.recipient_iban, error=RecipientNotFound) + else: + recipient = find_object(self.iter_transfer_recipients(account.id), id=transfer.recipient_id, error=RecipientNotFound) + + return self.browser.init_transfer(account, recipient, transfer.amount, transfer.label, transfer.exec_date) + + def execute_transfer(self, transfer, **params): + return self.browser.execute_transfer(transfer) + # vim:ts=4:sw=4 diff --git a/modules/fortuneo/pages/accounts_list.py b/modules/fortuneo/pages/accounts_list.py index 0e4fc12e7f47b1223d6758ea010624510dc13810..9c0bed7d54ba2d9e3addd37427d2e378cc2dbef8 100644 --- a/modules/fortuneo/pages/accounts_list.py +++ b/modules/fortuneo/pages/accounts_list.py @@ -457,6 +457,7 @@ def get_list(self): number = RawText('./a[contains(@class, "numero_compte")]')(cpt).replace(u'N° ', '') account.id = CleanText(None).filter(number).replace(u'N°', '') + account._ca = CleanText('./a[contains(@class, "numero_compte")]/@rel')(cpt) account._card_links = [] card_link = Link('./ul/li/a[contains(text(), "Carte bancaire")]', default='')(cpt) @@ -484,7 +485,7 @@ def get_list(self): if account.type in {Account.TYPE_PEA, Account.TYPE_MARKET}: account.currency = investment_page.get_currency() - else: + elif balance: account.currency = account.get_currency(balance) account.balance = CleanDecimal(None, replace_dots=True).filter(balance) diff --git a/modules/fortuneo/pages/transfer.py b/modules/fortuneo/pages/transfer.py new file mode 100644 index 0000000000000000000000000000000000000000..9e8c5193cedb586d3374e2783ee62918e2a752b1 --- /dev/null +++ b/modules/fortuneo/pages/transfer.py @@ -0,0 +1,244 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Sylvie Ye +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re +from datetime import date, timedelta + +from weboob.browser.pages import HTMLPage, PartialHTMLPage, LoggedPage +from weboob.browser.elements import method, ListElement, ItemElement +from weboob.browser.filters.standard import ( + CleanText, Date, Regexp, CleanDecimal, Currency, Field, Env, +) +from weboob.capabilities.bank import Recipient, Transfer, TransferBankError, AddRecipientError +from weboob.capabilities.base import NotAvailable + + +class RecipientsPage(LoggedPage, HTMLPage): + @method + class iter_external_recipients(ListElement): + # use list element because there are 4th for 7td in one tr + item_xpath = '//div[@id="listeCompteExternes"]/table/tbody//tr[@class="ct"]' + + def condition(self): + return 'Aucun compte externe enregistré' not in CleanText('.')(self) + + class item(ItemElement): + klass = Recipient + + def obj_id(self): + iban = CleanText('./td[6]', replace=[(' ', '')])(self) + iban = re.search(r'(?<=IBAN:)(\w+)BIC', iban).group(1) + return iban + + def obj_label(self): + if Field('_custom_label')(self): + return '{} - {}'.format(Field('_recipient_name')(self), Field('_custom_label')(self)) + return Field('_recipient_name')(self) + + obj__recipient_name = CleanText('./td[2]') + obj__custom_label = CleanText('./td[4]') + obj_iban = NotAvailable + obj_category = 'Externe' + obj_enabled_at = date.today() + obj_currency = 'EUR' + obj_bank_name = CleanText('./td[1]') + + def check_external_iban_form(self, recipient): + form = self.get_form(id='CompteExterneActionForm') + form['codePaysBanque'] = recipient.iban[:2] + form['codeIban'] = recipient.iban + form.url = self.browser.BASEURL + '/fr/prive/verifier-compte-externe.jsp' + form.submit() + + def check_recipient_iban(self): + if not CleanText('//input[@name="codeBic"]/@value')(self.doc): + raise AddRecipientError('Recipient already exist or invalid iban') + + def fill_recipient_form(self, recipient) : + form = self.get_form(id='CompteExterneActionForm') + form['codePaysBanque'] = recipient.iban[:2] + form['codeIban'] = recipient.iban + form['libelleCompte'] = recipient.label + form['nomTitulaireCompte'] = recipient.label + form['methode'] = 'verifierAjout' + form.submit() + + def get_new_recipient(self, recipient): + recipient_xpath = '//form[@id="CompteExterneActionForm"]//ul' + + rcpt = Recipient() + rcpt.label = Regexp(CleanText( + recipient_xpath + '/li[contains(text(), "Nom du titulaire")]', replace=[(' ', '')] + ), r'(?<=Nomdutitulaire:)(\w+)')(self.doc) + rcpt.iban = Regexp(CleanText( + recipient_xpath + '/li[contains(text(), "IBAN")]', replace=[(' ', '')] + ), r'(?<=IBAN:)([A-Z]{2}\d+)')(self.doc) + rcpt.id = rcpt.iban + rcpt.category = 'Externe' + rcpt.enabled_at = date.today() + timedelta(1) + rcpt.currency = 'EUR' + return rcpt + + def get_send_code_form(self): + return self.get_form(id='CompteExterneActionForm') + + +class RecipientSMSPage(LoggedPage, PartialHTMLPage): + def on_load(self): + if not self.doc.xpath('//input[@id="otp"]') and not self.doc.xpath('//div[@class="confirmationAjoutCompteExterne"]'): + raise AddRecipientError(CleanText('//div[@id="aidesecuforte"]/p[contains("Nous vous invitons")]')(self.doc)) + + def build_doc(self, content): + content = '
' + content.decode('latin-1') + '' + return super(RecipientSMSPage, self).build_doc(content.encode('latin-1')) + + def get_send_code_form_input(self): + form = self.get_form() + return form + + def rcpt_after_sms(self): + return self.doc.xpath('//div[@class="confirmationAjoutCompteExterne"]\ + /h2[contains(text(), "ajout de compte externe a bien été prise en compte")]') + + +class RegisterTransferPage(LoggedPage, HTMLPage): + @method + class iter_internal_recipients(ListElement): + item_xpath = '//select[@name="compteACrediter"]/option[not(@selected)]' + + class item(ItemElement): + klass = Recipient + + obj_id = CleanText('./@value') + obj_iban = NotAvailable + obj_label = CleanText('.') + obj__recipient_name = CleanText('.') + obj_category = 'Interne' + obj_enabled_at = date.today() + obj_currency = 'EUR' + obj_bank_name = 'FORTUNEO' + + def condition(self): + # external recipient id contains 43 characters + return len(Field('id')(self)) < 40 and Env('origin_account_id')(self) != Field('id')(self) + + def is_account_transferable(self, origin_account): + for account in self.doc.xpath('//select[@name="compteADebiter"]/option[not(@selected)]'): + if origin_account.id in CleanText('.')(account): + return True + return False + + def get_recipient_transfer_id(self, recipient): + for account in self.doc.xpath('//select[@name="compteACrediter"]/option[not(@selected)]'): + recipient_transfer_id = CleanText('./@value')(account) + + if (recipient.id == recipient_transfer_id + or recipient.id in CleanText('.', replace=[(' ', '')])(account) + ): + return recipient_transfer_id + + def fill_transfer_form(self, account, recipient, amount, label, exec_date): + recipient_transfer_id = self.get_recipient_transfer_id(recipient) + assert recipient_transfer_id + + form = self.get_form(id='SaisieVirementForm') + form['compteADebiter'] = account.id + form['libelleCompteADebiter'] = CleanText( + '//select[@name="compteADebiter"]/option[@value="%s"]' % account.id + )(self.doc) + form['compteACrediter'] = recipient_transfer_id + form['libelleCompteACrediter'] = CleanText( + '//select[@name="compteACrediter"]/option[@value="%s"]' % recipient_transfer_id + )(self.doc) + form['nomBeneficiaire'] = recipient._recipient_name + form['libellePopupDoublon'] = recipient._recipient_name + form['destinationEconomiqueFonds'] = '' + form['periodicite'] = 1 + form['typeDeVirement'] = 'VI' + form['dateDeVirement'] = exec_date.strftime('%d/%m/%Y') + form['montantVirement'] = amount + form['libelleVirementSaisie'] = label + form.submit() + + +class ValidateTransferPage(LoggedPage, HTMLPage): + def on_load(self): + if self.doc.xpath('//form[@id="SaisieVirementForm"]/p[has-class("error")]'): + raise TransferBankError(CleanText( + '//form[@id="SaisieVirementForm"]/p[has-class("error")]/label' + )(self.doc)) + + def check_transfer_data(self, transfer_data): + for t_data in transfer_data: + assert t_data in transfer_data[t_data], '%s not found in transfer summary %s' % (t_data, transfer_data[t_data]) + + def handle_response(self, account, recipient, amount, label, exec_date): + summary_xpath = '//div[@id="as_verifVirement.do_"]//ul' + + transfer = Transfer() + + transfer_data = { + account.id: CleanText( + summary_xpath + '/li[contains(text(), "Compte à débiter")]' + )(self.doc), + recipient.id: CleanText( + summary_xpath + '/li[contains(text(), "Compte à créditer")]', replace=[(' ', '')] + )(self.doc), + recipient._recipient_name: CleanText( + summary_xpath + '/li[contains(text(), "Nom du bénéficiaire")]' + )(self.doc), + label: CleanText(summary_xpath + '/li[contains(text(), "Motif")]')(self.doc), + } + self.check_transfer_data(transfer_data) + + transfer.account_id = account.id + transfer.account_label = account.label + transfer.account_iban = account.iban + + transfer.recipient_id = recipient.id + transfer.recipient_label = recipient.label + transfer.recipient_iban = recipient.iban + + transfer.label = label + transfer.currency = Currency(summary_xpath + '/li[contains(text(), "Montant")]')(self.doc) + transfer.amount = CleanDecimal( + Regexp(CleanText(summary_xpath + '/li[contains(text(), "Montant")]'), r'((\d+)\.?(\d+)?)') + )(self.doc) + transfer.exec_date = Date(Regexp(CleanText( + summary_xpath + '/li[contains(text(), "Date de virement")]' + ), r'(\d+/\d+/\d+)'), dayfirst=True)(self.doc) + + return transfer + + def validate_transfer(self): + form = self.get_form(id='SaisieVirementForm') + form['methode'] = 'valider' + form.submit() + + +class ConfirmTransferPage(LoggedPage, HTMLPage): + def confirm_transfer(self): + confirm_transfer_url = '/fr/prive/mes-comptes/compte-courant/realiser-operations/effectuer-virement/confirmer-saisie-virement.jsp' + self.browser.location(self.browser.BASEURL + confirm_transfer_url, data={'operationTempsReel': 'true'}) + + def transfer_confirmation(self, transfer): + if self.doc.xpath('//div[@class="confirmation_virement"]/h2[contains(text(), "virement a bien été enregistrée")]'): + return transfer diff --git a/modules/gmf/compat/weboob_capabilities_bank.py b/modules/gmf/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/gmf/compat/weboob_capabilities_bank.py +++ b/modules/gmf/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/groupama/browser.py b/modules/groupama/browser.py index 8aeccdcdc7202c0a261c65ac4a910670c9fe83e1..d227a66f231835baff5d7b62c3134d439c96afe2 100644 --- a/modules/groupama/browser.py +++ b/modules/groupama/browser.py @@ -72,7 +72,7 @@ def get_accounts_list(self, balance=True, need_iban=False): if account.balance or not balance: if account.type != Account.TYPE_LIFE_INSURANCE and need_iban: self.location(account._link) - if self.transactions.is_here(): + if self.transactions.is_here() and self.page.has_iban(): self.page.go_iban() account.iban = self.page.get_iban() accounts.append(account) diff --git a/modules/groupama/compat/weboob_capabilities_bank.py b/modules/groupama/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/groupama/compat/weboob_capabilities_bank.py +++ b/modules/groupama/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/groupama/pages.py b/modules/groupama/pages.py index 01b5408462092680b8c9a8e1166850af99d3fddd..fbf99ad762bf25138517d8e8f88f597a4bf93df0 100644 --- a/modules/groupama/pages.py +++ b/modules/groupama/pages.py @@ -161,6 +161,9 @@ def get_coming_link(self): return None return re.sub('[ \t\r\n]+', '', a.attrib['href']) + def has_iban(self): + return self.doc.xpath('//a[@class="rib"]') + def go_iban(self): js_event = Attr("//a[@class='rib']", 'onclick')(self.doc) m = re.search("envoyer(.*);", js_event) diff --git a/modules/groupamaes/compat/weboob_capabilities_bank.py b/modules/groupamaes/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/groupamaes/compat/weboob_capabilities_bank.py +++ b/modules/groupamaes/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/hsbc/compat/weboob_capabilities_bank.py b/modules/hsbc/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/hsbc/compat/weboob_capabilities_bank.py +++ b/modules/hsbc/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/hsbc/pages/account_pages.py b/modules/hsbc/pages/account_pages.py index 4eeb00d4caf165e2ceb93cf34b5c98882863c82e..03e1b1c17e573f97a3f0c924e4c7066abfb4108d 100644 --- a/modules/hsbc/pages/account_pages.py +++ b/modules/hsbc/pages/account_pages.py @@ -125,6 +125,7 @@ class Type(Filter): ('plan assur. innovat.', Account.TYPE_LIFE_INSURANCE), ('hsbc evol pat transf', Account.TYPE_LIFE_INSURANCE), ('bourse libre', Account.TYPE_MARKET), + ('plurival', Account.TYPE_LIFE_INSURANCE), ] def filter(self, label): diff --git a/modules/ing/browser.py b/modules/ing/browser.py index 3f366a6f0005e69232b08d1d03f60ea12a14831a..70dde32b1b62d7f5b0c083f586b18b1a9f984f7e 100644 --- a/modules/ing/browser.py +++ b/modules/ing/browser.py @@ -27,7 +27,7 @@ from weboob.exceptions import BrowserIncorrectPassword from weboob.browser.exceptions import ServerError from weboob.capabilities.bank import Account, AccountNotFound -from weboob.capabilities.base import find_object +from weboob.capabilities.base import find_object, NotAvailable from weboob.tools.capabilities.bank.transactions import FrenchTransaction from .pages import ( @@ -488,12 +488,33 @@ def get_history_titre(self, account): return iter([]) transactions = list() + # In order to reduce the amount of requests just to get ISIN codes, we fill + # a dictionary with already visited investment pages and store their ISIN codes: + isin_codes = {} for tr in self.page.iter_history(): transactions.append(tr) if self.asv_history.is_here(): for tr in transactions: page = tr._detail.result().page if tr._detail else None - tr.investments = list(page.get_investments()) if page and 'numMvt' in page.url else [] + if page and 'numMvt' in page.url: + investment_list = list() + for inv in page.get_investments(): + if inv._code_url in isin_codes: + inv.code = isin_codes.get(inv._code_url) + else: + # Fonds en euros (Eurossima) have no _code_url so we must set their code to None + if inv._code_url: + self.location(inv._code_url) + if self.detailfonds.is_here(): + inv.code = self.page.get_isin_code() + isin_codes[inv._code_url] = inv.code + else: + # In case the page is not available or blocked: + inv.code = NotAvailable + else: + inv.code = None + investment_list.append(inv) + tr.investments = investment_list self.lifeback.go() return iter(transactions) diff --git a/modules/ing/compat/weboob_capabilities_bank.py b/modules/ing/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/ing/compat/weboob_capabilities_bank.py +++ b/modules/ing/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/ing/pages/accounts_list.py b/modules/ing/pages/accounts_list.py index 8a6126ad19aad12306ce14bd080dcff8b0256e1f..9967545b16d902617aa417cdb635617d174e81e8 100644 --- a/modules/ing/pages/accounts_list.py +++ b/modules/ing/pages/accounts_list.py @@ -39,7 +39,7 @@ class Transaction(FrenchTransaction): - PATTERNS = [(re.compile(u'^retrait dab (?P
\d{2})/(?P\d{2})/(?P\d{4}) (?P.*)'), FrenchTransaction.TYPE_WITHDRAWAL), + PATTERNS = [(re.compile(u'^retrait (?P.*)'), FrenchTransaction.TYPE_WITHDRAWAL), (re.compile(u'^carte (?P
\d{2})/(?P\d{2})/(?P\d{4}) (?P.*)'), FrenchTransaction.TYPE_CARD), (re.compile(u'^virement (sepa )?(emis vers|recu|emis)? (?P.*)'), FrenchTransaction.TYPE_TRANSFER), (re.compile(u'^remise cheque(?P.*)'), FrenchTransaction.TYPE_DEPOSIT), @@ -47,6 +47,7 @@ class Transaction(FrenchTransaction): (re.compile(u'^prelevement (?P.*)'), FrenchTransaction.TYPE_ORDER), (re.compile(u'^prlv sepa (?P.*)'), FrenchTransaction.TYPE_ORDER), (re.compile(u'^prélèvement sepa en faveur de (?P.*)'), FrenchTransaction.TYPE_ORDER), + (re.compile(u'^commission sur (?P.*)'), FrenchTransaction.TYPE_BANK), ] @@ -416,7 +417,8 @@ def obj_diff_percent(self): class DetailFondsPage(LoggedPage, HTMLPage): - pass + def get_isin_code(self): + return CleanText('//td[contains(text(), "CodeISIN")]/b', default=NotAvailable)(self.doc) def MyInput(*args, **kwargs): diff --git a/modules/ing/pages/titre.py b/modules/ing/pages/titre.py index a09b416578a6a91a7599ecbea228317d6027be50..8ed4189bf100b57cf59b9c1253a4f1884e56595c 100644 --- a/modules/ing/pages/titre.py +++ b/modules/ing/pages/titre.py @@ -26,7 +26,7 @@ from weboob.capabilities.bank import Investment from weboob.browser.pages import RawPage, HTMLPage, LoggedPage, pagination from weboob.browser.elements import ListElement, TableElement, ItemElement, method -from weboob.browser.filters.standard import CleanDecimal, CleanText, Date, Regexp, Env, Async, AsyncLoad +from weboob.browser.filters.standard import CleanDecimal, CleanText, Date, Regexp, Env from weboob.browser.filters.html import Link, Attr, TableCell from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.tools.compat import unicode @@ -40,7 +40,8 @@ class Transaction(FrenchTransaction): class TitreValuePage(LoggedPage, HTMLPage): def get_isin(self): - return unicode(self.doc.xpath('//div[@id="headFiche"]//span[@id="test3"]/text()')[0].split(' - ')[0].strip()) + isin = self.doc.xpath('//div[@id="headFiche"]//span[@id="test3"]/text()') + return unicode(isin[0].split(' - ')[0].strip()) if isin else NotAvailable class TitrePage(LoggedPage, RawPage): @@ -151,15 +152,15 @@ class get_investments(TableElement): class item(ItemElement): klass = Investment - load_details = Regexp(Attr('./td/a', 'onclick', default=""), 'PageExterne\(\'([^\']+)', default=None) & AsyncLoad - obj_label = CleanText(TableCell('label')) - obj_code = Async('details') & CleanText('//td[contains(text(), "CodeISIN")]/b', default=NotAvailable) obj_quantity = CleanDecimal(TableCell('quantity'), replace_dots=True, default=NotAvailable) obj_unitvalue = CleanDecimal(TableCell('unitvalue'), replace_dots=True, default=NotAvailable) obj_valuation = CleanDecimal(TableCell('valuation'), replace_dots=True) obj_vdate = Date(CleanText(TableCell('vdate')), dayfirst=True) + obj__code_url = Regexp(Attr('./td/a', 'onclick', default=""), r'PageExterne\(\'([^\']+)', default=None) + + @pagination @method class iter_history(TableElement): diff --git a/modules/lcl/compat/weboob_capabilities_bank.py b/modules/lcl/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/lcl/compat/weboob_capabilities_bank.py +++ b/modules/lcl/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/lcl/module.py b/modules/lcl/module.py index dd5c502edc21ec399280c42f99578279c2be251b..684b9ee0cd238ae301e036e3e287e9e2d8e88914 100644 --- a/modules/lcl/module.py +++ b/modules/lcl/module.py @@ -143,7 +143,8 @@ def execute_transfer(self, transfer, **params): return self.browser.execute_transfer(transfer) def transfer_check_label(self, old, new): - old = re.sub(r'/', '', old).strip() + old = re.sub(r"[/<\?='!]", '', old).strip() + old = old.encode('latin-1', errors='replace').decode('latin-1') return super(LCLModule, self).transfer_check_label(old, new) @only_for_websites('par') diff --git a/modules/lcl/pages.py b/modules/lcl/pages.py index 65d0bbf12f1ca9b8db59a90a9b87168464a22794..f919aac9d2c0748f4c7d003ccb23ac8e4a031e0c 100644 --- a/modules/lcl/pages.py +++ b/modules/lcl/pages.py @@ -246,6 +246,7 @@ def condition(self): return '/outil/UWLM/ListeMouvement' in self.el.attrib['onclick'] NATURE2TYPE = {'001': Account.TYPE_SAVINGS, + '004': Account.TYPE_CHECKING, '005': Account.TYPE_CHECKING, '006': Account.TYPE_CHECKING, '007': Account.TYPE_SAVINGS, diff --git a/modules/ldlc/browser.py b/modules/ldlc/browser.py index a3e9dfae919827ebfaadcea218100e8716ff50f5..4df59bd5b5a8739a1163a631bf645b7ca1dd41d1 100644 --- a/modules/ldlc/browser.py +++ b/modules/ldlc/browser.py @@ -18,47 +18,92 @@ # along with weboob. If not, see . - from weboob.browser import LoginBrowser, URL, need_login from weboob.exceptions import BrowserIncorrectPassword -from .pages import HomePage, BillsPage, LoginPage - -class LdlcBrowser(LoginBrowser): - login = URL('/Account/LoginPage.aspx', LoginPage) - bills = URL('/Account/CommandListingPage.aspx', BillsPage) - home = URL('/$', HomePage) +from .pages import LoginPage, HomePage, ParBillsPage, ProBillsPage - def __init__(self, website, *args, **kwargs): - self.website = website - if website == 'pro': - self.BASEURL = 'https://secure.ldlc-pro.com/' - else: - self.BASEURL = 'https://secure.ldlc.com/' - super(LdlcBrowser, self).__init__(*args, **kwargs) +class LdlcBrowser(LoginBrowser): + login = URL(r'/Account/LoginPage.aspx', LoginPage) + home = URL(r'/$', HomePage) def do_login(self): - self.login.stay_or_go().login(self.username, self.password) + self.login.stay_or_go() + website = 'part' if type(self) == LdlcParBrowser else 'pro' + self.page.login(self.username, self.password, website) + if self.login.is_here(): - raise BrowserIncorrectPassword + raise BrowserIncorrectPassword(self.page.get_error()) @need_login def get_subscription_list(self): return self.home.stay_or_go().get_list() + +class LdlcParBrowser(LdlcBrowser): + BASEURL = 'https://secure.ldlc.com' + + bills = URL(r'/Account/CommandListingPage.aspx', ParBillsPage) + @need_login def iter_documents(self, subscription): self.bills.stay_or_go() - bills = list() for value in self.page.get_range(): - if self.website == 'pro': - event = 'ctl00$cphMainContent$ddlDate' - else: - event = 'ctl00$ctl00$cphMainContent$cphMainContent$ddlDate' + self.bills.go(data={'ctl00$ctl00$cphMainContent$cphMainContent$ddlDate': value, '__EVENTTARGET': 'ctl00$cphMainContent$ddlDate'}) + + for bill in self.page.iter_documents(subid=subscription.id): + yield bill + - self.bills.go(data={event: value, '__EVENTTARGET': 'ctl00$cphMainContent$ddlDate'}) +class LdlcProBrowser(LdlcBrowser): + BASEURL = 'https://secure.ldlc-pro.com' + + bills = URL(r'/Account/CommandListingPage.aspx', ProBillsPage) + + @need_login + def iter_documents(self, subscription): + self.bills.stay_or_go() + + for value in self.page.get_range(): + self.bills.go(data={'ctl00$cphMainContent$ddlDate': value, '__EVENTTARGET': 'ctl00$cphMainContent$ddlDate'}) + view_state = self.page.get_view_state() + # we need position to download file + position = 1 + hidden_field = self.page.get_ctl00_actScriptManager_HiddenField() + for bill in self.page.iter_documents(subid=subscription.id): + bill._position = position + bill._view_state = view_state + bill._hidden_field = hidden_field + position += 1 + yield bill + + @need_login + def download_document(self, bill): + data = { + '__EVENTARGUMENT': '', + '__EVENTTARGET': '', + '__LASTFOCUS': '', + '__SCROLLPOSITIONX': 0, + '__SCROLLPOSITIONY': 0, + '__VIEWSTATE': bill._view_state, + 'ctl00$actScriptManager': '', + 'ctl00$cphMainContent$DetailCommand$hfCommand': '', + 'ctl00$cphMainContent$DetailCommand$txtAltEmail': '', + 'ctl00$cphMainContent$ddlDate': bill.date.year, + 'ctl00$cphMainContent$hfCancelCommandId': '', + 'ctl00$cphMainContent$hfCommandId': '', + 'ctl00$cphMainContent$hfCommandSearch': '', + 'ctl00$cphMainContent$hfOrderTri': 1, + 'ctl00$cphMainContent$hfTypeTri': 1, + 'ctl00$cphMainContent$rptCommand$ctl%s$hlFacture.x' % str(bill._position).zfill(2): '7', + 'ctl00$cphMainContent$rptCommand$ctl%s$hlFacture.y' % str(bill._position).zfill(2): '11', + 'ctl00$cphMainContent$txtCommandSearch': '', + 'ctl00$hfCountries': '', + 'ctl00$ucHeaderControl$ctrlSuggestedProductPopUp$HiddenCommandeSupplementaire': '', + 'ctl00$ucHeaderControl$ctrlSuggestedProductPopUp$hiddenPopUp': '', + 'ctl00$ucHeaderControl$txtSearch': 'Rechercher+...', + 'ctl00_actScriptManager_HiddenField': bill._hidden_field + } - for i in self.page.get_documents(subid=subscription.id): - bills.append(i) - return bills + return self.open(bill.url, data=data).content diff --git a/modules/ldlc/module.py b/modules/ldlc/module.py index 0191579c7563dcd10d725fca84893b97e4103114..3c7cffc3687243758fe84cf8d8b94e060ef4cb96 100644 --- a/modules/ldlc/module.py +++ b/modules/ldlc/module.py @@ -23,7 +23,7 @@ from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword, Value -from .browser import LdlcBrowser +from .browser import LdlcParBrowser, LdlcProBrowser __all__ = ['LdlcModule'] @@ -37,18 +37,17 @@ class LdlcModule(Module, CapDocument): LICENSE = 'AGPLv3+' VERSION = '1.3' CONFIG = BackendConfig(Value('login', label='Email'), - ValueBackendPassword('password', label='Password'), - Value('website', label='Site web', default='part', - choices={'pro': 'Professionnels', - 'part': 'Particuliers'})) - - - BROWSER = LdlcBrowser + ValueBackendPassword('password', label='Password'), + Value('website', label='Site web', default='part', + choices={'pro': 'Professionnels', 'part': 'Particuliers'})) def create_default_browser(self): - return self.create_browser(self.config['website'].get(), - self.config['login'].get(), - self.config['password'].get()) + if self.config['website'].get() == 'part': + self.BROWSER = LdlcParBrowser + else: + self.BROWSER = LdlcProBrowser + + return self.create_browser(self.config['login'].get(), self.config['password'].get()) def iter_subscription(self): return self.browser.get_subscription_list() @@ -70,4 +69,7 @@ def iter_documents(self, subscription): def download_document(self, bill): if not isinstance(bill, Bill): bill = self.get_document(bill) - return self.browser.open(bill.url).content + if self.config['website'].get() == 'part': + return self.browser.open(bill.url).content + else: + return self.browser.download_document(bill) diff --git a/modules/ldlc/pages.py b/modules/ldlc/pages.py index c452e280513b532440293a2a1b950297fbdaba76..76725b2a812626d08046e42e4d64b5f2b488d90d 100644 --- a/modules/ldlc/pages.py +++ b/modules/ldlc/pages.py @@ -17,13 +17,22 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals + from weboob.browser.pages import HTMLPage, LoggedPage -from weboob.browser.filters.standard import CleanDecimal, CleanText, Env, Format, QueryValue -from weboob.browser.elements import ListElement, ItemElement, method +from weboob.browser.filters.standard import CleanDecimal, CleanText, Env, Format, QueryValue, TableCell, Currency +from weboob.browser.elements import ListElement, ItemElement, method, TableElement from weboob.browser.filters.html import Attr from weboob.capabilities.bill import Bill, Subscription from weboob.tools.date import parse_french_date + +class HiddenFieldPage(HTMLPage): + def get_ctl00_actScriptManager_HiddenField(self): + param = QueryValue(Attr('//script[contains(@src, "js/CombineScriptsHandler.ashx?")]', 'src'), "_TSM_CombinedScripts_")(self.doc) + return param + + class HomePage(LoggedPage, HTMLPage): @method class get_list(ListElement): @@ -36,41 +45,80 @@ class item(ItemElement): obj_label = CleanText('.//div[@id="divlblTitleFirstNameLastName"]//span') -class LoginPage(HTMLPage): - def login(self, username, password): - form = self.get_form(xpath='//form[@id="aspnetForm"]') - form["ctl00$ctl00$cphMainContent$cphMainContent$txbMail"] = username - form["ctl00$ctl00$cphMainContent$cphMainContent$txbPassword"] = password - form["__EVENTTARGET"] = "ctl00$ctl00$cphMainContent$cphMainContent$butConnexion" - form["ctl00_ctl00_actScriptManager_HiddenField"] = self.get_ctl00_actScriptManager_HiddenField() +class LoginPage(HiddenFieldPage): + def login(self, username, password, website): + form = self.get_form(id='aspnetForm') + if website == 'part': + form["ctl00$ctl00$cphMainContent$cphMainContent$txbMail"] = username + form["ctl00$ctl00$cphMainContent$cphMainContent$txbPassword"] = password + form["__EVENTTARGET"] = "ctl00$ctl00$cphMainContent$cphMainContent$butConnexion" + form["ctl00_ctl00_actScriptManager_HiddenField"] = self.get_ctl00_actScriptManager_HiddenField() + else: + form["ctl00$cphMainContent$txbMail"] = username + form["ctl00$cphMainContent$txbPassword"] = password + form["__EVENTTARGET"] = "ctl00$cphMainContent$butConnexion" + form["ctl00_ctl00_actScriptManager_HiddenField"] = self.get_ctl00_actScriptManager_HiddenField() form.submit() - def get_ctl00_actScriptManager_HiddenField(self): - param = QueryValue(Attr('//script[contains(@src, "js/CombineScriptsHandler.ashx?")]', 'src'), "_TSM_CombinedScripts_")(self.doc) - return param + def get_error(self): + return CleanText('//span[contains(text(), "Identifiants incorrects")]')(self.doc) -class BillsPage(LoggedPage, HTMLPage): +class BillsPage(LoggedPage, HiddenFieldPage): def get_range(self): for value in self.doc.xpath('//div[@class="commandListing content clearfix"]//select/option/@value'): yield value + +class ParBillsPage(BillsPage): @method - class get_documents(ListElement): - item_xpath = '//table[@id="TopListing"]//tr' + class iter_documents(TableElement): + ignore_duplicate = True + item_xpath = '//table[@id="TopListing"]/tr[position()>1]' + head_xpath = '//table[@id="TopListing"]/tr[@class="TopListingHeader"]/td' + + col_id = 'N° de commande' + col_date = 'Date' + col_price = 'Montant TTC' class item(ItemElement): klass = Bill - obj_id = Format('%s_%s', Env('subid'), CleanText('./td[3]')) + obj_id = Format('%s_%s', Env('subid'), CleanText(TableCell('id'))) obj_url = Attr('./td[@class="center" or @class="center pdf"]/a', 'href') - obj_date = Env('date') - obj_format = u"pdf" - obj_type = u"bill" - obj_price = CleanDecimal('./td[@class="center montant"]/span', replace_dots=True) + obj_format = 'pdf' + obj_price = CleanDecimal(TableCell('price'), replace_dots=True) + obj_currency = Currency(TableCell('price')) - def parse(self, el): - self.env['date'] = parse_french_date(el.xpath('./td[2]')[0].text).date() + def obj_date(self): + return parse_french_date(CleanText(TableCell('date'))(self)).date() def condition(self): return CleanText().filter(self.el.xpath('.//td')[-1]) != "" and len(self.el.xpath('./td[@class="center" or @class="center pdf"]/a/@href')) == 1 + + +class ProBillsPage(BillsPage): + def get_view_state(self): + return Attr('//input[@id="__VIEWSTATE"]', 'value')(self.doc) + + @method + class iter_documents(TableElement): + ignore_duplicate = True + item_xpath = '//table[@id="TopListing"]/tr[contains(@class, "rowTable")]' + head_xpath = '//table[@id="TopListing"]/tr[@class="headTable"]/td' + + col_id = 'N° de commande' + col_date = 'Date' + col_price = 'Montant HT' + + class item(ItemElement): + klass = Bill + + obj_id = Format('%s_%s', Env('subid'), CleanText(TableCell('id'))) + obj_url = '/Account/CommandListingPage.aspx' + obj_format = 'pdf' + obj_price = CleanDecimal(TableCell('price'), replace_dots=True) + obj_currency = Currency(TableCell('price')) + + def obj_date(self): + return parse_french_date(CleanText(TableCell('date'))(self)).date() diff --git a/modules/leboncoin/compat/weboob_capabilities_housing.py b/modules/leboncoin/compat/weboob_capabilities_housing.py index a7456e29ce1b0da440e193efaed4689cedb6d533..f763fcbaf7f9b168db3d2069e15ff965d2faac3d 100644 --- a/modules/leboncoin/compat/weboob_capabilities_housing.py +++ b/modules/leboncoin/compat/weboob_capabilities_housing.py @@ -1,192 +1,31 @@ -# -*- coding: utf-8 -*- -# Copyright(C) 2012 Romain Bignon -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . +import weboob.capabilities.housing as OLD +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) -from weboob.capabilities.base import Capability, BaseObject, Field, IntField, DecimalField, \ - StringField, BytesField, StrEnum, EnumField, UserError -from weboob.capabilities.date import DateField -__all__ = [ - 'CapHousing', 'Housing', 'Query', 'City', 'UTILITIES', 'ENERGY_CLASS', - 'POSTS_TYPES', 'ADVERT_TYPES', 'HOUSE_TYPES', 'TypeNotSupported', - 'HousingPhoto', -] +__all__ = OLD.__all__ -from weboob.capabilities.housing import TypeNotSupported as _TypeNotSupported -class TypeNotSupported(_TypeNotSupported): - """ - Raised when query type is not supported - """ +ENERGY_CLASS = enum(A=u'A', B=u'B', C=u'C', D=u'D', E=u'E', F=u'F', G=u'G') - def __init__(self, - msg='This type of house is not supported by this module'): - super(TypeNotSupported, self).__init__(msg) +POSTS_TYPES = enum(RENT=u'RENT', + SALE=u'SALE', + SHARING=u'SHARING', + FURNISHED_RENT=u'FURNISHED_RENT', + VIAGER=u'VIAGER') -from weboob.capabilities.housing import HousingPhoto as _HousingPhoto -class HousingPhoto(_HousingPhoto): - """ - Photo of a housing. - """ - data = BytesField('Data of photo') - def __init__(self, url): - super(HousingPhoto, self).__init__(url.split('/')[-1], url) +ADVERT_TYPES = enum(PROFESSIONAL=u'Professional', PERSONAL=u'Personal') - def __iscomplete__(self): - return self.data - def __unicode__(self): - return self.url - - def __repr__(self): - return '' % (self.id, len(self.data) if self.data else 0, self.url) - - -class UTILITIES(StrEnum): - INCLUDED = u'C.C.' - EXCLUDED = u'H.C.' - UNKNOWN = u'' - - -class ENERGY_CLASS(StrEnum): - A = u'A' - B = u'B' - C = u'C' - D = u'D' - E = u'E' - F = u'F' - G = u'G' - - -class POSTS_TYPES(StrEnum): - RENT=u'RENT' - SALE = u'SALE' - SHARING = u'SHARING' - FURNISHED_RENT = u'FURNISHED_RENT' - VIAGER = u'VIAGER' - - -class ADVERT_TYPES(StrEnum): - PROFESSIONAL = u'Professional' - PERSONAL = u'Personal' - - -class HOUSE_TYPES(StrEnum): - APART = u'Apartment' - HOUSE = u'House' - PARKING = u'Parking' - LAND = u'Land' - OTHER = u'Other' - UNKNOWN = u'Unknown' - - -from weboob.capabilities.housing import Housing as _Housing -class Housing(_Housing): - """ - Content of a housing. - """ - type = EnumField('Type of housing (rent, sale, sharing)', - POSTS_TYPES) - advert_type = EnumField('Type of advert (professional or personal)', - ADVERT_TYPES) - house_type = EnumField(u'Type of house (apartment, house, parking, …)', - HOUSE_TYPES) - title = StringField('Title of housing') - area = DecimalField('Area of housing, in m2') - cost = DecimalField('Cost of housing') - price_per_meter = DecimalField('Price per meter ratio') - currency = StringField('Currency of cost') - utilities = EnumField('Utilities included or not', UTILITIES) - date = DateField('Date when the housing has been published') - location = StringField('Location of housing') - station = StringField('What metro/bus station next to housing') - text = StringField('Text of the housing') - phone = StringField('Phone number to contact') - photos = Field('List of photos', list) - rooms = DecimalField('Number of rooms') - bedrooms = DecimalField('Number of bedrooms') - details = Field('Key/values of details', dict) - DPE = EnumField('DPE (Energy Performance Certificate)', ENERGY_CLASS) - GES = EnumField('GES (Greenhouse Gas Emissions)', ENERGY_CLASS) - - -from weboob.capabilities.housing import Query as _Query -class Query(_Query): - """ - Query to find housings. - """ - type = EnumField('Type of housing to find (POSTS_TYPES constants)', - POSTS_TYPES) - cities = Field('List of cities to search in', list, tuple) - area_min = IntField('Minimal area (in m2)') - area_max = IntField('Maximal area (in m2)') - cost_min = IntField('Minimal cost') - cost_max = IntField('Maximal cost') - nb_rooms = IntField('Number of rooms') - house_types = Field('List of house types', list, tuple, default=list(HOUSE_TYPES)) - advert_types = Field('List of advert types to filter on', list, tuple, - default=list(ADVERT_TYPES)) - - -from weboob.capabilities.housing import City as _City -class City(_City): - """ - City. - """ - name = StringField('Name of city') - - -from weboob.capabilities.housing import CapHousing as _CapHousing -class CapHousing(_CapHousing): - """ - Capability of websites to search housings. - """ - - def search_housings(self, query): - """ - Search housings. - - :param query: search query - :type query: :class:`Query` - :rtype: iter[:class:`Housing`] - """ - raise NotImplementedError() - - def get_housing(self, housing): - """ - Get an housing from an ID. - - :param housing: ID of the housing - :type housing: str - :rtype: :class:`Housing` or None if not found. - """ - raise NotImplementedError() - - def search_city(self, pattern): - """ - Search a city from a pattern. - - :param pattern: pattern to search - :type pattern: str - :rtype: iter[:class:`City`] - """ - raise NotImplementedError() +HOUSE_TYPES = enum(APART=u'Apartment', + HOUSE=u'House', + PARKING=u'Parking', + LAND=u'Land', + OTHER=u'Other', + UNKNOWN=u'Unknown') diff --git a/modules/limetorrents/browser.py b/modules/limetorrents/browser.py index 88914322aa6fc910f3e3b1f9b7af203bf2f8c23e..0178a54bb45fa6ab46cba383f68fba5a27abc319 100644 --- a/modules/limetorrents/browser.py +++ b/modules/limetorrents/browser.py @@ -33,7 +33,7 @@ class LimetorrentsBrowser(PagesBrowser): PROFILE = Wget() TIMEOUT = 30 - BASEURL = 'https://www.limetorrents.io/' + BASEURL = 'https://www.limetorrents.info/' search = URL(r'/search/all/(?P.*)/seeds/(?P[0-9]+)/', SearchPage) torrent = URL(r'/(?P.*)-torrent-(?P[0-9]+)\.html', TorrentPage) diff --git a/modules/logicimmo/browser.py b/modules/logicimmo/browser.py index c62f3d7ff4f48ae4bc0c0a0d5136f75a593c5bfb..8a7acdcf055a01569cea5df715546b0d32b65d92 100644 --- a/modules/logicimmo/browser.py +++ b/modules/logicimmo/browser.py @@ -28,7 +28,6 @@ class LogicimmoBrowser(PagesBrowser): BASEURL = 'http://www.logic-immo.com/' PROFILE = Firefox() - city = URL('asset/t9/getLocalityT9.php\?site=fr&lang=fr&json=%22(?P.*)%22', CitiesPage) search = URL('(?Plocation-immobilier|vente-immobilier|recherche-colocation)-(?P.*)/options/(?P.*)', SearchPage) @@ -47,6 +46,10 @@ class LogicimmoBrowser(PagesBrowser): HOUSE_TYPES.PARKING: '10', HOUSE_TYPES.OTHER: '14'} + def __init__(self, *args, **kwargs): + super(LogicimmoBrowser, self).__init__(*args, **kwargs) + self.session.headers['X-Requested-With'] = 'XMLHttpRequest' + def get_cities(self, pattern): if pattern: return self.city.go(pattern=pattern).get_cities() diff --git a/modules/logicimmo/compat/weboob_capabilities_housing.py b/modules/logicimmo/compat/weboob_capabilities_housing.py index a7456e29ce1b0da440e193efaed4689cedb6d533..f763fcbaf7f9b168db3d2069e15ff965d2faac3d 100644 --- a/modules/logicimmo/compat/weboob_capabilities_housing.py +++ b/modules/logicimmo/compat/weboob_capabilities_housing.py @@ -1,192 +1,31 @@ -# -*- coding: utf-8 -*- -# Copyright(C) 2012 Romain Bignon -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . +import weboob.capabilities.housing as OLD +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) -from weboob.capabilities.base import Capability, BaseObject, Field, IntField, DecimalField, \ - StringField, BytesField, StrEnum, EnumField, UserError -from weboob.capabilities.date import DateField -__all__ = [ - 'CapHousing', 'Housing', 'Query', 'City', 'UTILITIES', 'ENERGY_CLASS', - 'POSTS_TYPES', 'ADVERT_TYPES', 'HOUSE_TYPES', 'TypeNotSupported', - 'HousingPhoto', -] +__all__ = OLD.__all__ -from weboob.capabilities.housing import TypeNotSupported as _TypeNotSupported -class TypeNotSupported(_TypeNotSupported): - """ - Raised when query type is not supported - """ +ENERGY_CLASS = enum(A=u'A', B=u'B', C=u'C', D=u'D', E=u'E', F=u'F', G=u'G') - def __init__(self, - msg='This type of house is not supported by this module'): - super(TypeNotSupported, self).__init__(msg) +POSTS_TYPES = enum(RENT=u'RENT', + SALE=u'SALE', + SHARING=u'SHARING', + FURNISHED_RENT=u'FURNISHED_RENT', + VIAGER=u'VIAGER') -from weboob.capabilities.housing import HousingPhoto as _HousingPhoto -class HousingPhoto(_HousingPhoto): - """ - Photo of a housing. - """ - data = BytesField('Data of photo') - def __init__(self, url): - super(HousingPhoto, self).__init__(url.split('/')[-1], url) +ADVERT_TYPES = enum(PROFESSIONAL=u'Professional', PERSONAL=u'Personal') - def __iscomplete__(self): - return self.data - def __unicode__(self): - return self.url - - def __repr__(self): - return '' % (self.id, len(self.data) if self.data else 0, self.url) - - -class UTILITIES(StrEnum): - INCLUDED = u'C.C.' - EXCLUDED = u'H.C.' - UNKNOWN = u'' - - -class ENERGY_CLASS(StrEnum): - A = u'A' - B = u'B' - C = u'C' - D = u'D' - E = u'E' - F = u'F' - G = u'G' - - -class POSTS_TYPES(StrEnum): - RENT=u'RENT' - SALE = u'SALE' - SHARING = u'SHARING' - FURNISHED_RENT = u'FURNISHED_RENT' - VIAGER = u'VIAGER' - - -class ADVERT_TYPES(StrEnum): - PROFESSIONAL = u'Professional' - PERSONAL = u'Personal' - - -class HOUSE_TYPES(StrEnum): - APART = u'Apartment' - HOUSE = u'House' - PARKING = u'Parking' - LAND = u'Land' - OTHER = u'Other' - UNKNOWN = u'Unknown' - - -from weboob.capabilities.housing import Housing as _Housing -class Housing(_Housing): - """ - Content of a housing. - """ - type = EnumField('Type of housing (rent, sale, sharing)', - POSTS_TYPES) - advert_type = EnumField('Type of advert (professional or personal)', - ADVERT_TYPES) - house_type = EnumField(u'Type of house (apartment, house, parking, …)', - HOUSE_TYPES) - title = StringField('Title of housing') - area = DecimalField('Area of housing, in m2') - cost = DecimalField('Cost of housing') - price_per_meter = DecimalField('Price per meter ratio') - currency = StringField('Currency of cost') - utilities = EnumField('Utilities included or not', UTILITIES) - date = DateField('Date when the housing has been published') - location = StringField('Location of housing') - station = StringField('What metro/bus station next to housing') - text = StringField('Text of the housing') - phone = StringField('Phone number to contact') - photos = Field('List of photos', list) - rooms = DecimalField('Number of rooms') - bedrooms = DecimalField('Number of bedrooms') - details = Field('Key/values of details', dict) - DPE = EnumField('DPE (Energy Performance Certificate)', ENERGY_CLASS) - GES = EnumField('GES (Greenhouse Gas Emissions)', ENERGY_CLASS) - - -from weboob.capabilities.housing import Query as _Query -class Query(_Query): - """ - Query to find housings. - """ - type = EnumField('Type of housing to find (POSTS_TYPES constants)', - POSTS_TYPES) - cities = Field('List of cities to search in', list, tuple) - area_min = IntField('Minimal area (in m2)') - area_max = IntField('Maximal area (in m2)') - cost_min = IntField('Minimal cost') - cost_max = IntField('Maximal cost') - nb_rooms = IntField('Number of rooms') - house_types = Field('List of house types', list, tuple, default=list(HOUSE_TYPES)) - advert_types = Field('List of advert types to filter on', list, tuple, - default=list(ADVERT_TYPES)) - - -from weboob.capabilities.housing import City as _City -class City(_City): - """ - City. - """ - name = StringField('Name of city') - - -from weboob.capabilities.housing import CapHousing as _CapHousing -class CapHousing(_CapHousing): - """ - Capability of websites to search housings. - """ - - def search_housings(self, query): - """ - Search housings. - - :param query: search query - :type query: :class:`Query` - :rtype: iter[:class:`Housing`] - """ - raise NotImplementedError() - - def get_housing(self, housing): - """ - Get an housing from an ID. - - :param housing: ID of the housing - :type housing: str - :rtype: :class:`Housing` or None if not found. - """ - raise NotImplementedError() - - def search_city(self, pattern): - """ - Search a city from a pattern. - - :param pattern: pattern to search - :type pattern: str - :rtype: iter[:class:`City`] - """ - raise NotImplementedError() +HOUSE_TYPES = enum(APART=u'Apartment', + HOUSE=u'House', + PARKING=u'Parking', + LAND=u'Land', + OTHER=u'Other', + UNKNOWN=u'Unknown') diff --git a/modules/myfoncia/compat/weboob_capabilities_housing.py b/modules/myfoncia/compat/weboob_capabilities_housing.py index a7456e29ce1b0da440e193efaed4689cedb6d533..f763fcbaf7f9b168db3d2069e15ff965d2faac3d 100644 --- a/modules/myfoncia/compat/weboob_capabilities_housing.py +++ b/modules/myfoncia/compat/weboob_capabilities_housing.py @@ -1,192 +1,31 @@ -# -*- coding: utf-8 -*- -# Copyright(C) 2012 Romain Bignon -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . +import weboob.capabilities.housing as OLD +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) -from weboob.capabilities.base import Capability, BaseObject, Field, IntField, DecimalField, \ - StringField, BytesField, StrEnum, EnumField, UserError -from weboob.capabilities.date import DateField -__all__ = [ - 'CapHousing', 'Housing', 'Query', 'City', 'UTILITIES', 'ENERGY_CLASS', - 'POSTS_TYPES', 'ADVERT_TYPES', 'HOUSE_TYPES', 'TypeNotSupported', - 'HousingPhoto', -] +__all__ = OLD.__all__ -from weboob.capabilities.housing import TypeNotSupported as _TypeNotSupported -class TypeNotSupported(_TypeNotSupported): - """ - Raised when query type is not supported - """ +ENERGY_CLASS = enum(A=u'A', B=u'B', C=u'C', D=u'D', E=u'E', F=u'F', G=u'G') - def __init__(self, - msg='This type of house is not supported by this module'): - super(TypeNotSupported, self).__init__(msg) +POSTS_TYPES = enum(RENT=u'RENT', + SALE=u'SALE', + SHARING=u'SHARING', + FURNISHED_RENT=u'FURNISHED_RENT', + VIAGER=u'VIAGER') -from weboob.capabilities.housing import HousingPhoto as _HousingPhoto -class HousingPhoto(_HousingPhoto): - """ - Photo of a housing. - """ - data = BytesField('Data of photo') - def __init__(self, url): - super(HousingPhoto, self).__init__(url.split('/')[-1], url) +ADVERT_TYPES = enum(PROFESSIONAL=u'Professional', PERSONAL=u'Personal') - def __iscomplete__(self): - return self.data - def __unicode__(self): - return self.url - - def __repr__(self): - return '' % (self.id, len(self.data) if self.data else 0, self.url) - - -class UTILITIES(StrEnum): - INCLUDED = u'C.C.' - EXCLUDED = u'H.C.' - UNKNOWN = u'' - - -class ENERGY_CLASS(StrEnum): - A = u'A' - B = u'B' - C = u'C' - D = u'D' - E = u'E' - F = u'F' - G = u'G' - - -class POSTS_TYPES(StrEnum): - RENT=u'RENT' - SALE = u'SALE' - SHARING = u'SHARING' - FURNISHED_RENT = u'FURNISHED_RENT' - VIAGER = u'VIAGER' - - -class ADVERT_TYPES(StrEnum): - PROFESSIONAL = u'Professional' - PERSONAL = u'Personal' - - -class HOUSE_TYPES(StrEnum): - APART = u'Apartment' - HOUSE = u'House' - PARKING = u'Parking' - LAND = u'Land' - OTHER = u'Other' - UNKNOWN = u'Unknown' - - -from weboob.capabilities.housing import Housing as _Housing -class Housing(_Housing): - """ - Content of a housing. - """ - type = EnumField('Type of housing (rent, sale, sharing)', - POSTS_TYPES) - advert_type = EnumField('Type of advert (professional or personal)', - ADVERT_TYPES) - house_type = EnumField(u'Type of house (apartment, house, parking, …)', - HOUSE_TYPES) - title = StringField('Title of housing') - area = DecimalField('Area of housing, in m2') - cost = DecimalField('Cost of housing') - price_per_meter = DecimalField('Price per meter ratio') - currency = StringField('Currency of cost') - utilities = EnumField('Utilities included or not', UTILITIES) - date = DateField('Date when the housing has been published') - location = StringField('Location of housing') - station = StringField('What metro/bus station next to housing') - text = StringField('Text of the housing') - phone = StringField('Phone number to contact') - photos = Field('List of photos', list) - rooms = DecimalField('Number of rooms') - bedrooms = DecimalField('Number of bedrooms') - details = Field('Key/values of details', dict) - DPE = EnumField('DPE (Energy Performance Certificate)', ENERGY_CLASS) - GES = EnumField('GES (Greenhouse Gas Emissions)', ENERGY_CLASS) - - -from weboob.capabilities.housing import Query as _Query -class Query(_Query): - """ - Query to find housings. - """ - type = EnumField('Type of housing to find (POSTS_TYPES constants)', - POSTS_TYPES) - cities = Field('List of cities to search in', list, tuple) - area_min = IntField('Minimal area (in m2)') - area_max = IntField('Maximal area (in m2)') - cost_min = IntField('Minimal cost') - cost_max = IntField('Maximal cost') - nb_rooms = IntField('Number of rooms') - house_types = Field('List of house types', list, tuple, default=list(HOUSE_TYPES)) - advert_types = Field('List of advert types to filter on', list, tuple, - default=list(ADVERT_TYPES)) - - -from weboob.capabilities.housing import City as _City -class City(_City): - """ - City. - """ - name = StringField('Name of city') - - -from weboob.capabilities.housing import CapHousing as _CapHousing -class CapHousing(_CapHousing): - """ - Capability of websites to search housings. - """ - - def search_housings(self, query): - """ - Search housings. - - :param query: search query - :type query: :class:`Query` - :rtype: iter[:class:`Housing`] - """ - raise NotImplementedError() - - def get_housing(self, housing): - """ - Get an housing from an ID. - - :param housing: ID of the housing - :type housing: str - :rtype: :class:`Housing` or None if not found. - """ - raise NotImplementedError() - - def search_city(self, pattern): - """ - Search a city from a pattern. - - :param pattern: pattern to search - :type pattern: str - :rtype: iter[:class:`City`] - """ - raise NotImplementedError() +HOUSE_TYPES = enum(APART=u'Apartment', + HOUSE=u'House', + PARKING=u'Parking', + LAND=u'Land', + OTHER=u'Other', + UNKNOWN=u'Unknown') diff --git a/modules/n26/browser.py b/modules/n26/browser.py index f352146f7614a1064cb4f36acb778681c37ea1dd..d160e45296d786bf1e8b18afc52262b28e7ef19a 100644 --- a/modules/n26/browser.py +++ b/modules/n26/browser.py @@ -91,6 +91,7 @@ def do_login(self): @need_login def get_accounts(self): account = self.request('/api/accounts') + spaces = self.request('/api/spaces') a = Account() @@ -100,7 +101,7 @@ def get_accounts(self): a.id = account["id"] a.number = NotAvailable - a.balance = Decimal(str(account["availableBalance"])) + a.balance = Decimal(str(spaces["totalBalance"])) a.iban = account["iban"] a.currency = u'EUR' diff --git a/modules/nalo/__init__.py b/modules/nalo/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4b0d1c8d2bb8f914f4d1337c7d40d33ab5d97ad8 --- /dev/null +++ b/modules/nalo/__init__.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + + +from .module import NaloModule + + +__all__ = ['NaloModule'] diff --git a/modules/nalo/browser.py b/modules/nalo/browser.py new file mode 100644 index 0000000000000000000000000000000000000000..1b1aa5ec84cab611f403bc63fbcb72fa74e277b0 --- /dev/null +++ b/modules/nalo/browser.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +from weboob.browser import LoginBrowser, need_login, URL +from weboob.capabilities.bank import Investment + +from .pages import LoginPage, AccountsPage, AccountPage, InvestPage + + +class NaloBrowser(LoginBrowser): + BASEURL = 'https://nalo.fr' + + login = URL(r'/api/v1/login', LoginPage) + accounts = URL(r'/api/v1/projects/mine/without_details', AccountsPage) + history = URL(r'/api/v1/projects/(?P\d+)/history') + account = URL(r'/api/v1/projects/(?P\d+)', AccountPage) + invests = URL(r'https://app.nalo.fr/scripts/data/data.json', InvestPage) + + token = None + + def do_login(self): + self.login.go(json={ + 'email': self.username, + 'password': self.password, + 'userToken': False, + }) + self.token = self.page.get_token() + + def build_request(self, *args, **kwargs): + if 'json' in kwargs: + kwargs.setdefault('headers', {})['Accept'] = 'application/json' + if self.token: + kwargs.setdefault('headers', {})['Authorization'] = 'Token %s' % self.token + return super(NaloBrowser, self).build_request(*args, **kwargs) + + @need_login + def iter_accounts(self): + self.accounts.go() + return self.page.iter_accounts() + + @need_login + def iter_history(self, account): + self.history.go(id=account.id) + return self.page.iter_history() + + @need_login + def iter_investment(self, account): + self.account.go(id=account.id) + key = self.page.get_invest_key() + + self.invests.go() + data = self.page.get_invest(*key) + for item in data: + inv = Investment() + inv.code = item['isin'] + inv.label = item['name'] + inv.portfolio_share = item['share'] + inv.valuation = account.balance * inv.portfolio_share + yield inv diff --git a/modules/nalo/compat/__init__.py b/modules/nalo/compat/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/modules/nalo/compat/weboob_capabilities_bank.py b/modules/nalo/compat/weboob_capabilities_bank.py new file mode 100644 index 0000000000000000000000000000000000000000..404e8d30ed28683c8a27f183d1e670c10509ce56 --- /dev/null +++ b/modules/nalo/compat/weboob_capabilities_bank.py @@ -0,0 +1,40 @@ + +import weboob.capabilities.bank as OLD + +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) + + +__all__ = OLD.__all__ + + +class CapBankWealth(CapBank): + pass + + +class CapBankPockets(CapBank): + pass + + +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + +class CapBankTransfer(OLD.CapBankTransfer): + def transfer_check_label(self, old, new): + from unidecode import unidecode + + return unidecode(old) == unidecode(new) + + +class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipient): + pass + + +Account.TYPE_MORTGAGE = 17 +Account.TYPE_CONSUMER_CREDIT = 18 +Account.TYPE_REVOLVING_CREDIT = 19 diff --git a/modules/nalo/module.py b/modules/nalo/module.py new file mode 100644 index 0000000000000000000000000000000000000000..79f4095be43f67c3d2bf194feb6994c69f9c55c9 --- /dev/null +++ b/modules/nalo/module.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +from weboob.tools.backend import Module, BackendConfig +from weboob.tools.value import ValueBackendPassword +from .compat.weboob_capabilities_bank import CapBankWealth + +from .browser import NaloBrowser + + +__all__ = ['NaloModule'] + + +class NaloModule(Module, CapBankWealth): + NAME = 'nalo' + DESCRIPTION = 'Nalo' + MAINTAINER = 'Vincent A' + EMAIL = 'dev@indigo.re' + LICENSE = 'AGPLv3+' + VERSION = '1.3' + + BROWSER = NaloBrowser + + CONFIG = BackendConfig( + ValueBackendPassword('login', label='E-mail', masked=False, regexp='.+@.+'), + ValueBackendPassword('password', label='Mot de passe'), + ) + + def create_default_browser(self): + return self.create_browser(self.config['login'].get(), self.config['password'].get()) + + def iter_accounts(self): + return self.browser.iter_accounts() + + def iter_investment(self, account): + return self.browser.iter_investment(account) diff --git a/modules/nalo/pages.py b/modules/nalo/pages.py new file mode 100644 index 0000000000000000000000000000000000000000..5f8b64fe076fe06ed2c96d207178abda16adcb55 --- /dev/null +++ b/modules/nalo/pages.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Vincent A +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +from decimal import Decimal + +from weboob.browser.pages import LoggedPage, JsonPage +from weboob.browser.elements import method, DictElement, ItemElement +from weboob.browser.filters.json import Dict +from weboob.browser.filters.standard import Eval +from weboob.capabilities.bank import Account + + +def float_to_decimal(v): + return Decimal(str(v)) + + +class LoginPage(JsonPage): + def get_token(self): + return self.doc['detail']['token'] + + +class AccountsPage(LoggedPage, JsonPage): + ENCODING = 'utf-8' # chardet is shit + + @method + class iter_accounts(DictElement): + item_xpath = 'detail' + + class item(ItemElement): + klass = Account + + obj_id = Eval(str, Dict('id')) + obj_label = Dict('name') + obj_balance = Eval(float_to_decimal, Dict('current_value')) + obj_valuation_diff = Eval(float_to_decimal, Dict('absolute_performance')) + obj_currency = 'EUR' + obj_type = Account.TYPE_LIFE_INSURANCE + + +class AccountPage(LoggedPage, JsonPage): + def get_invest_key(self): + return self.doc['detail']['project_kind'], self.doc['detail']['risk_level'] + + def get_kind(self): + return self.doc['detail']['project_kind'] + + def get_risk(self): + return self.doc['detail']['risk_level'] + + +class HistoryPage(LoggedPage, JsonPage): + pass + + +class InvestPage(LoggedPage, JsonPage): + ENCODING = 'utf-8' + + def get_invest(self, kind, risk): + for pk in self.doc['portfolios']: + if pk['kind'] == kind: + break + else: + assert False + + for p in pk['target_portfolios']: + if p['risk_id'] == risk: + break + else: + assert False + + for line in p['lines']: + yield { + 'isin': line['isin'], + 'name': line['name'], + 'share': float_to_decimal(line['weight']) / 100, + } diff --git a/modules/orange/browser.py b/modules/orange/browser.py index b36e22a399dd4c4ae96451109dd25f015fbf6c5d..bd514e4efadc42a73f9e0d27b03388df2f22ca25 100644 --- a/modules/orange/browser.py +++ b/modules/orange/browser.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright(C) 2010-2011 Nicolas Duhamel +# Copyright(C) 2012-2014 Vincent Paredes # # This file is part of weboob. # @@ -17,52 +17,101 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . +from __future__ import unicode_literals -#~ from .pages.compose import ClosePage, ComposePage, ConfirmPage, SentPage -#~ from .pages.login import LoginPage +from weboob.browser import LoginBrowser, URL, need_login +from weboob.exceptions import BrowserIncorrectPassword +from .pages import LoginPage, BillsPage +from .pages.bills import SubscriptionsPage, BillsApiPage, ContractsPage +from .pages.profile import ProfilePage +from weboob.browser.exceptions import ClientError, ServerError +from weboob.tools.compat import basestring -from .pages import LoginPage, ComposePage, ConfirmPage -from weboob.deprecated.browser import Browser, BrowserIncorrectPassword +__all__ = ['OrangeBillBrowser'] -__all__ = ['OrangeBrowser'] +class OrangeBillBrowser(LoginBrowser): + BASEURL = 'https://espaceclientv3.orange.fr/' + loginpage = URL('https://login.orange.fr/\?service=sosh&return_url=https://www.sosh.fr/', + 'https://login.orange.fr/front/login', LoginPage) -class OrangeBrowser(Browser): - DOMAIN = 'orange.fr' - PAGES = { - 'http://id.orange.fr/auth_user/bin/auth_user.cgi.*': LoginPage, - 'http://id.orange.fr/auth_user/bin/auth0user.cgi.*': LoginPage, - 'https://id.orange.fr/auth_user/bin/auth_user.cgi.*': LoginPage, - 'https://id.orange.fr/auth_user/bin/auth0user.cgi.*': LoginPage, - 'https://authweb.orange.fr/auth_user/bin/auth_user.cgi.*': LoginPage, - 'https://authweb.orange.fr/auth_user/bin/auth0user.cgi.*': LoginPage, - 'http://smsmms1.orange.fr/./Sms/sms_write.php.*' : ComposePage, - 'http://smsmms1.orange.fr/./Sms/sms_write.php?command=send' : ConfirmPage, - 'https://smsmms1.orange.fr/./Sms/sms_write.php.*' : ComposePage, - 'https://smsmms1.orange.fr/./Sms/sms_write.php?command=send' : ConfirmPage, - } + contracts = URL('https://espaceclientpro.orange.fr/api/contracts\?page=1&nbcontractsbypage=15', ContractsPage) - def get_nb_remaining_free_sms(self): - self.location("http://smsmms1.orange.fr/M/Sms/sms_write.php") - return self.page.get_nb_remaining_free_sms() + subscriptions = URL(r'https://espaceclientv3.orange.fr/js/necfe.php\?zonetype=bandeau&idPage=gt-home-page', SubscriptionsPage) + + billspage = URL('https://m.espaceclientv3.orange.fr/\?page=factures-archives', + 'https://.*.espaceclientv3.orange.fr/\?page=factures-archives', + 'https://espaceclientv3.orange.fr/\?page=factures-archives', + 'https://espaceclientv3.orange.fr/\?page=facture-telecharger', + 'https://espaceclientv3.orange.fr/maf.php', + 'https://espaceclientv3.orange.fr/\?idContrat=(?P.*)&page=factures-historique', + 'https://espaceclientv3.orange.fr/\?page=factures-historique&idContrat=(?P.*)', + BillsPage) + + bills_api = URL('https://espaceclientpro.orange.fr/api/contract/(?P\d+)/bills\?count=(?P)', + BillsApiPage) - def home(self): - self.location("http://smsmms1.orange.fr/M/Sms/sms_write.php") + doc_api = URL('https://espaceclientpro.orange.fr/api/contract/(?P\d+)/bill/(?P.*)/(?P.*)/\?(?P)') + profile = URL('/\?page=profil-infosPerso', ProfilePage) - def is_logged(self): - self.location("http://smsmms1.orange.fr/M/Sms/sms_write.php", no_login=True) - return not self.is_on_page(LoginPage) + def do_login(self): + assert isinstance(self.username, basestring) + assert isinstance(self.password, basestring) - def login(self): - if not self.is_on_page(LoginPage): - self.location('https://authweb.orange.fr/auth_user/bin/auth_user.cgi?url=http://www.orange.fr', no_login=True) - self.page.login(self.username, self.password) - if not self.is_logged(): - raise BrowserIncorrectPassword() + try: + self.loginpage.stay_or_go().login(self.username, self.password) + except ClientError as error: + if error.response.status_code == 401: + raise BrowserIncorrectPassword() + raise + + def get_nb_remaining_free_sms(self): + raise NotImplementedError() def post_message(self, message, sender): - if not self.is_on_page(ComposePage): - self.home() - self.page.post_message(message, sender) + raise NotImplementedError() + + @need_login + def get_subscription_list(self): + profile = self.profile.go().get_profile() + # this only works when there are pro subs. + nb_sub = 0 + try: + for sub in self.contracts.go().iter_subscriptions(): + sub.subscriber = profile.name + yield sub + nb_sub = self.page.doc['totalContracts'] + # assert pagination is not needed + assert nb_sub < 15 + except ServerError: + pass + + if nb_sub > 0: + return + # if nb_sub is 0, we continue, because we can get them in next url + + self.location('https://espaceclientv3.orange.fr/?page=gt-home-page&sosh') + self.subscriptions.go() + for sub in self.page.iter_subscription(): + sub.subscriber = profile.name + yield sub + + @need_login + def iter_documents(self, subscription): + documents = [] + if subscription._is_pro: + for d in self.bills_api.go(subid=subscription.id, count=72).get_bills(subid=subscription.id): + documents.append(d) + # check pagination for this subscription + assert len(documents) != 72 + else: + self.billspage.go(subid=subscription.id) + for b in self.page.get_bills(subid=subscription.id): + documents.append(b) + return iter(documents) + + @need_login + def get_profile(self): + return self.profile.go().get_profile() diff --git a/modules/orange/module.py b/modules/orange/module.py index 8eff4d217fecc01ba65efe2173a609d9edb65f47..d2a6899b8394baa957c57a4296d3d2f3146988f0 100644 --- a/modules/orange/module.py +++ b/modules/orange/module.py @@ -19,52 +19,28 @@ from weboob.capabilities.bill import CapDocument, Subscription, Document, SubscriptionNotFound, DocumentNotFound -from weboob.capabilities.messages import CantSendMessage, CapMessages, CapMessagesPost from weboob.capabilities.base import find_object, NotAvailable -from weboob.capabilities.account import CapAccount, StatusField +from weboob.capabilities.account import CapAccount from weboob.capabilities.profile import CapProfile from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword, Value -from .browser import OrangeBrowser -from .bill.browser import OrangeBillBrowser +from .browser import OrangeBillBrowser __all__ = ['OrangeModule'] -# We need to have a switcher, CapMessages use a browser1 and -# CapDocument use a browser2 -# This will be remove when CapMessages use a browser2 -def browser_switcher(b): - def set_browser(func): - def func_wrapper(*args, **kwargs): - self = args[0] - if self._browser is None or type(self._browser) != b: - self.BROWSER = b - try: - self._browser = self._browsers[b] - except KeyError: - self._browsers[b] = self.create_default_browser() - self._browser = self._browsers[b] - return func(*args, **kwargs) - return func_wrapper - return set_browser - - -class OrangeModule(Module, CapAccount, CapMessages, CapMessagesPost, CapDocument, CapProfile): + +class OrangeModule(Module, CapAccount, CapDocument, CapProfile): NAME = 'orange' - MAINTAINER = u'Lucas Nussbaum' - EMAIL = 'lucas@lucas-nussbaum.net' + MAINTAINER = 'Florian Duguet' + EMAIL = 'florian.duguet@budget-insight.com' VERSION = '1.3' DESCRIPTION = 'Orange French mobile phone provider' LICENSE = 'AGPLv3+' CONFIG = BackendConfig(Value('login', label='Login'), - ValueBackendPassword('password', label='Password'), - Value('phonenumber', label='Phone number', default='') - ) - ACCOUNT_REGISTER_PROPERTIES = None - BROWSER = OrangeBrowser - + ValueBackendPassword('password', label='Password')) + BROWSER = OrangeBillBrowser def __init__(self, *args, **kwargs): self._browsers = dict() @@ -73,38 +49,22 @@ def __init__(self, *args, **kwargs): def create_default_browser(self): return self.create_browser(self.config['login'].get(), self.config['password'].get()) - @browser_switcher(OrangeBrowser) - def get_account_status(self): - return (StatusField('nb_remaining_free_sms', 'Number of remaining free SMS', - self.browser.get_nb_remaining_free_sms()),) - - @browser_switcher(OrangeBrowser) - def post_message(self, message): - if not message.content.strip(): - raise CantSendMessage(u'Message content is empty.') - self.browser.post_message(message, self.config['phonenumber'].get()) - - @browser_switcher(OrangeBillBrowser) def iter_subscription(self): return self.browser.get_subscription_list() - @browser_switcher(OrangeBillBrowser) def get_subscription(self, _id): return find_object(self.iter_subscription(), id=_id, error=SubscriptionNotFound) - @browser_switcher(OrangeBillBrowser) def get_document(self, _id): subid = _id.rsplit('_', 1)[0] subscription = self.get_subscription(subid) return find_object(self.iter_documents(subscription), id=_id, error=DocumentNotFound) - @browser_switcher(OrangeBillBrowser) def iter_documents(self, subscription): if not isinstance(subscription, Subscription): subscription = self.get_subscription(subscription) return self.browser.iter_documents(subscription) - @browser_switcher(OrangeBillBrowser) def download_document(self, document): if not isinstance(document, Document): document = self.get_document(document) @@ -112,6 +72,5 @@ def download_document(self, document): return return self.browser.open(document.url).content - @browser_switcher(OrangeBillBrowser) def get_profile(self): return self.browser.get_profile() diff --git a/modules/orange/pages/__init__.py b/modules/orange/pages/__init__.py index 63ee962d66505b2a9f37bf3e7caa05701f147616..c87ea35fb6914dfd8a2dced64c2bc1f38d254d13 100644 --- a/modules/orange/pages/__init__.py +++ b/modules/orange/pages/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright(C) 2010-2011 Nicolas Duhamel +# Copyright(C) 2010-2011 Vincent Paredes # # This file is part of weboob. # @@ -18,6 +18,6 @@ # along with weboob. If not, see . from .login import LoginPage -from .compose import ComposePage, ConfirmPage +from .bills import BillsPage -__all__ = ['LoginPage', 'ComposePage', 'ConfirmPage'] +__all__ = ['LoginPage', 'BillsPage'] diff --git a/modules/orange/pages/bills.py b/modules/orange/pages/bills.py new file mode 100644 index 0000000000000000000000000000000000000000..c3791ac7d7f9cd65603f5e4d60b5b84bc8c5dc9e --- /dev/null +++ b/modules/orange/pages/bills.py @@ -0,0 +1,181 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2010-2011 Vincent Paredes +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +import re +try: + from html.parser import HTMLParser +except ImportError: + import HTMLParser + +from weboob.browser.pages import HTMLPage, LoggedPage, JsonPage +from weboob.capabilities.bill import Subscription +from weboob.browser.elements import DictElement, ListElement, ItemElement, method, TableElement +from weboob.browser.filters.standard import CleanDecimal, CleanText, Env, Field, Regexp, Date, Currency, BrowserURL, Format +from weboob.browser.filters.html import Link, TableCell +from weboob.browser.filters.javascript import JSValue +from weboob.browser.filters.json import Dict +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.bill import Bill +from weboob.tools.date import parse_french_date +from weboob.tools.compat import urlencode + + +class BillsApiPage(LoggedPage, JsonPage): + @method + class get_bills(DictElement): + item_xpath = 'bills' + # orange's API will sometimes return the temporary bill for the current month along with other bills + # in the json. The url will lead to the exact same document, this is probably not intended behaviour and + # causes weboob to raise a DataError as they'll have identical ids. + ignore_duplicate = True + + class item(ItemElement): + klass = Bill + + obj_date = Date(Dict('dueDate'), parse_func=parse_french_date, default=NotAvailable) + obj_price = CleanDecimal(Dict('amountIncludingTax')) + obj_format = 'pdf' + + def obj_label(self): + return 'Facture du %s' % Field('date')(self) + + def obj_id(self): + return '%s_%s' % (Env('subid')(self), Field('date')(self).strftime('%d%m%Y')) + + def get_params(self): + params = {'billid': Dict('id')(self), 'billDate': Dict('dueDate')(self)} + return urlencode(params) + + obj_url = BrowserURL('doc_api', subid=Env('subid'), dir=Dict('documents/0/mainDir'), fact_type=Dict('documents/0/subDir'), billparams=get_params) + + +class BillsPage(LoggedPage, HTMLPage): + @method + class get_bills(TableElement): + item_xpath = '//table[has-class("table-hover")]/div/div/tr | //table[has-class("table-hover")]/div/tr' + head_xpath = '//table[has-class("table-hover")]/thead/tr/th' + + col_date = 'Date' + col_amount = ['Montant TTC', 'Montant'] + col_ht = 'Montant HT' + col_url = 'Télécharger' + col_infos = 'Infos paiement' + + class item(ItemElement): + klass = Bill + + obj_type = u"bill" + obj_format = u"pdf" + + # TableCell('date') can have other info like: 'duplicata' + obj_date = Date(CleanText('./td[@headers="ec-dateCol"]/text()[not(preceding-sibling::br)]'), parse_func=parse_french_date, dayfirst=True) + + def obj__cell(self): + # sometimes the link to the bill is not in the right column (Thanks Orange!!) + if CleanText(TableCell('url')(self))(self): + return 'url' + return 'infos' + + def obj_price(self): + if CleanText(TableCell('amount')(self))(self): + return CleanDecimal(Regexp(CleanText(TableCell('amount')), '.*?([\d,]+).*', default=NotAvailable), replace_dots=True, default=NotAvailable)(self) + else: + return Field('_ht')(self) + + def obj_currency(self): + if CleanText(TableCell('amount')(self))(self): + return Currency(TableCell('amount')(self))(self) + else: + return Currency(TableCell('ht')(self))(self) + + # Only when a list of documents is present + obj__url_base = Regexp(CleanText('.//ul[@class="liste"]/script', default=None), '.*?contentList[\d]+ \+= \'
  • (.*?). -from weboob.deprecated.browser import Page -from weboob.tools.compat import urlencode -class LoginPage(Page): - def on_loaded(self): - pass +from weboob.browser.pages import HTMLPage - def login(self, user, pwd): - post_data = {"credential" : str(user), - "password" : str(pwd), - "save_user": "false", - "save_pwd" : "false", - "save_TC" : "true", - "action" : "valider", - "usertype" : "", - "service" : "", - "url" : "http://www.orange.fr", - "case" : "", - "origin" : "", } - post_data = urlencode(post_data) - self.browser.addheaders = [('Referer', 'http://id.orange.fr/auth_user/template/auth0user/htm/vide.html'), - ("Content-Type" , 'application/x-www-form-urlencoded') ] +class LoginPage(HTMLPage): + def login(self, username, password): + json_data = { + 'forcePwd': False, + 'login': username, + 'mem': True, + } + self.browser.location('https://login.orange.fr/front/login', json=json_data) - self.browser.open(self.browser.geturl(), data=post_data) - - #~ print "LOGIN!!!" - #~ self.browser.select_form(predicate=lambda form: "id" in form.attrs and form.attrs["id"] == "authentication_form" ) - #~ user_control = self.browser.find_control(id="user_credential") - #~ user_control.value = user - #~ pwd_control = self.browser.find_control(id="user_password") - #~ pwd_control.value = pwd - #~ self.browser.submit() + json_data = { + 'login': username, + 'password': password, + } + self.browser.location('https://login.orange.fr/front/password', json=json_data) diff --git a/modules/orange/pages/profile.py b/modules/orange/pages/profile.py new file mode 100644 index 0000000000000000000000000000000000000000..919d6c32d4bb54393c9d3046fbbb9910857786a8 --- /dev/null +++ b/modules/orange/pages/profile.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2010-2011 Vincent Paredes +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +from weboob.browser.pages import HTMLPage, LoggedPage +from weboob.capabilities.profile import Profile +from weboob.browser.filters.standard import CleanText, Format + + +class ProfilePage(LoggedPage, HTMLPage): + def get_profile(self): + pr = Profile() + pr.email = CleanText('//span[contains(@class, "panelAccount-label") and strong[contains(text(), "Adresse email")]]/following::span[1]/strong')(self.doc) + + if 'Informations indisponibles' not in CleanText('//div[contains(@id, "Address")]')(self.doc): + pr.address = ( + CleanText('//div[contains(@id, "Address")]//div[@class="ec-blocAddress text-primary"]')(self.doc) + or CleanText('//div[contains(@class, "addressLine")][1]//span[@class]')(self.doc) + or CleanText('//div[contains(@class, "row ec-blocAddressList")]')(self.doc) + ) + + phone = CleanText('//span[contains(@class, "panelAccount-label") and strong[contains(text(), "Numéro de mobile")]]/following::span[1]')(self.doc) + if 'non renseigné' not in phone: + pr.phone = phone + + # Civilé + # Nom + # Prénom + if CleanText('//p[contains(@class, "panelAccount-label")]/span[strong[contains(text(), "Civilité")]]')(self.doc): + pr.name = Format('%s %s %s', + CleanText('//p[contains(@class, "panelAccount-label")]/span[strong[contains(text(), "Civilité")]]/following::span[1]'), + CleanText('//p[contains(@class, "panelAccount-label")]/span[strong[contains(text(), "Nom :")]]/following::span[1]'), + CleanText('//p[contains(@class, "panelAccount-label")]/span[strong[contains(text(), "Prénom :")]]/following::span[1]') + )(self.doc) + # Prénom / Nom + elif CleanText('//p[contains(@class, "panelAccount-label")]/span[strong[contains(text(), "Prénom / Nom")]]')(self.doc): + pr.name = CleanText('//p[contains(@class, "panelAccount-label")]/span[strong[contains(text(), "Prénom / Nom")]]/following::span[1]')(self.doc) + # Nom + else: + pr.name = CleanText('//p[contains(@class, "panelAccount-label")]/span[strong[text()="Nom :"]]/following::span[1]')(self.doc) + + return pr diff --git a/modules/pap/compat/weboob_capabilities_housing.py b/modules/pap/compat/weboob_capabilities_housing.py index a7456e29ce1b0da440e193efaed4689cedb6d533..f763fcbaf7f9b168db3d2069e15ff965d2faac3d 100644 --- a/modules/pap/compat/weboob_capabilities_housing.py +++ b/modules/pap/compat/weboob_capabilities_housing.py @@ -1,192 +1,31 @@ -# -*- coding: utf-8 -*- -# Copyright(C) 2012 Romain Bignon -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . +import weboob.capabilities.housing as OLD +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) -from weboob.capabilities.base import Capability, BaseObject, Field, IntField, DecimalField, \ - StringField, BytesField, StrEnum, EnumField, UserError -from weboob.capabilities.date import DateField -__all__ = [ - 'CapHousing', 'Housing', 'Query', 'City', 'UTILITIES', 'ENERGY_CLASS', - 'POSTS_TYPES', 'ADVERT_TYPES', 'HOUSE_TYPES', 'TypeNotSupported', - 'HousingPhoto', -] +__all__ = OLD.__all__ -from weboob.capabilities.housing import TypeNotSupported as _TypeNotSupported -class TypeNotSupported(_TypeNotSupported): - """ - Raised when query type is not supported - """ +ENERGY_CLASS = enum(A=u'A', B=u'B', C=u'C', D=u'D', E=u'E', F=u'F', G=u'G') - def __init__(self, - msg='This type of house is not supported by this module'): - super(TypeNotSupported, self).__init__(msg) +POSTS_TYPES = enum(RENT=u'RENT', + SALE=u'SALE', + SHARING=u'SHARING', + FURNISHED_RENT=u'FURNISHED_RENT', + VIAGER=u'VIAGER') -from weboob.capabilities.housing import HousingPhoto as _HousingPhoto -class HousingPhoto(_HousingPhoto): - """ - Photo of a housing. - """ - data = BytesField('Data of photo') - def __init__(self, url): - super(HousingPhoto, self).__init__(url.split('/')[-1], url) +ADVERT_TYPES = enum(PROFESSIONAL=u'Professional', PERSONAL=u'Personal') - def __iscomplete__(self): - return self.data - def __unicode__(self): - return self.url - - def __repr__(self): - return '' % (self.id, len(self.data) if self.data else 0, self.url) - - -class UTILITIES(StrEnum): - INCLUDED = u'C.C.' - EXCLUDED = u'H.C.' - UNKNOWN = u'' - - -class ENERGY_CLASS(StrEnum): - A = u'A' - B = u'B' - C = u'C' - D = u'D' - E = u'E' - F = u'F' - G = u'G' - - -class POSTS_TYPES(StrEnum): - RENT=u'RENT' - SALE = u'SALE' - SHARING = u'SHARING' - FURNISHED_RENT = u'FURNISHED_RENT' - VIAGER = u'VIAGER' - - -class ADVERT_TYPES(StrEnum): - PROFESSIONAL = u'Professional' - PERSONAL = u'Personal' - - -class HOUSE_TYPES(StrEnum): - APART = u'Apartment' - HOUSE = u'House' - PARKING = u'Parking' - LAND = u'Land' - OTHER = u'Other' - UNKNOWN = u'Unknown' - - -from weboob.capabilities.housing import Housing as _Housing -class Housing(_Housing): - """ - Content of a housing. - """ - type = EnumField('Type of housing (rent, sale, sharing)', - POSTS_TYPES) - advert_type = EnumField('Type of advert (professional or personal)', - ADVERT_TYPES) - house_type = EnumField(u'Type of house (apartment, house, parking, …)', - HOUSE_TYPES) - title = StringField('Title of housing') - area = DecimalField('Area of housing, in m2') - cost = DecimalField('Cost of housing') - price_per_meter = DecimalField('Price per meter ratio') - currency = StringField('Currency of cost') - utilities = EnumField('Utilities included or not', UTILITIES) - date = DateField('Date when the housing has been published') - location = StringField('Location of housing') - station = StringField('What metro/bus station next to housing') - text = StringField('Text of the housing') - phone = StringField('Phone number to contact') - photos = Field('List of photos', list) - rooms = DecimalField('Number of rooms') - bedrooms = DecimalField('Number of bedrooms') - details = Field('Key/values of details', dict) - DPE = EnumField('DPE (Energy Performance Certificate)', ENERGY_CLASS) - GES = EnumField('GES (Greenhouse Gas Emissions)', ENERGY_CLASS) - - -from weboob.capabilities.housing import Query as _Query -class Query(_Query): - """ - Query to find housings. - """ - type = EnumField('Type of housing to find (POSTS_TYPES constants)', - POSTS_TYPES) - cities = Field('List of cities to search in', list, tuple) - area_min = IntField('Minimal area (in m2)') - area_max = IntField('Maximal area (in m2)') - cost_min = IntField('Minimal cost') - cost_max = IntField('Maximal cost') - nb_rooms = IntField('Number of rooms') - house_types = Field('List of house types', list, tuple, default=list(HOUSE_TYPES)) - advert_types = Field('List of advert types to filter on', list, tuple, - default=list(ADVERT_TYPES)) - - -from weboob.capabilities.housing import City as _City -class City(_City): - """ - City. - """ - name = StringField('Name of city') - - -from weboob.capabilities.housing import CapHousing as _CapHousing -class CapHousing(_CapHousing): - """ - Capability of websites to search housings. - """ - - def search_housings(self, query): - """ - Search housings. - - :param query: search query - :type query: :class:`Query` - :rtype: iter[:class:`Housing`] - """ - raise NotImplementedError() - - def get_housing(self, housing): - """ - Get an housing from an ID. - - :param housing: ID of the housing - :type housing: str - :rtype: :class:`Housing` or None if not found. - """ - raise NotImplementedError() - - def search_city(self, pattern): - """ - Search a city from a pattern. - - :param pattern: pattern to search - :type pattern: str - :rtype: iter[:class:`City`] - """ - raise NotImplementedError() +HOUSE_TYPES = enum(APART=u'Apartment', + HOUSE=u'House', + PARKING=u'Parking', + LAND=u'Land', + OTHER=u'Other', + UNKNOWN=u'Unknown') diff --git a/modules/pradoepargne/compat/weboob_capabilities_bank.py b/modules/pradoepargne/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/pradoepargne/compat/weboob_capabilities_bank.py +++ b/modules/pradoepargne/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/s2e/compat/weboob_capabilities_bank.py b/modules/s2e/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/s2e/compat/weboob_capabilities_bank.py +++ b/modules/s2e/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/s2e/pages.py b/modules/s2e/pages.py index ec43d03e0acb774b43c7fb4978353eb98c76c537..4726e343ac702547745237c777f2569796f7510d 100644 --- a/modules/s2e/pages.py +++ b/modules/s2e/pages.py @@ -197,7 +197,7 @@ class AMFSGPage(HTMLPage): CODE_TYPE = Investment.CODE_TYPE_AMF def get_code(self): - return Regexp(CleanText('//div[@id="header_code"]', default=NotAvailable), r'(\d+)')(self.doc) + return Regexp(CleanText('//div[@id="header_code"]'), r'(\d+)', default=NotAvailable)(self.doc) def build_doc(self, data): if not data.strip(): diff --git a/modules/seloger/compat/weboob_capabilities_housing.py b/modules/seloger/compat/weboob_capabilities_housing.py index a7456e29ce1b0da440e193efaed4689cedb6d533..f763fcbaf7f9b168db3d2069e15ff965d2faac3d 100644 --- a/modules/seloger/compat/weboob_capabilities_housing.py +++ b/modules/seloger/compat/weboob_capabilities_housing.py @@ -1,192 +1,31 @@ -# -*- coding: utf-8 -*- -# Copyright(C) 2012 Romain Bignon -# -# This file is part of weboob. -# -# weboob is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# weboob is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with weboob. If not, see . +import weboob.capabilities.housing as OLD +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) -from weboob.capabilities.base import Capability, BaseObject, Field, IntField, DecimalField, \ - StringField, BytesField, StrEnum, EnumField, UserError -from weboob.capabilities.date import DateField -__all__ = [ - 'CapHousing', 'Housing', 'Query', 'City', 'UTILITIES', 'ENERGY_CLASS', - 'POSTS_TYPES', 'ADVERT_TYPES', 'HOUSE_TYPES', 'TypeNotSupported', - 'HousingPhoto', -] +__all__ = OLD.__all__ -from weboob.capabilities.housing import TypeNotSupported as _TypeNotSupported -class TypeNotSupported(_TypeNotSupported): - """ - Raised when query type is not supported - """ +ENERGY_CLASS = enum(A=u'A', B=u'B', C=u'C', D=u'D', E=u'E', F=u'F', G=u'G') - def __init__(self, - msg='This type of house is not supported by this module'): - super(TypeNotSupported, self).__init__(msg) +POSTS_TYPES = enum(RENT=u'RENT', + SALE=u'SALE', + SHARING=u'SHARING', + FURNISHED_RENT=u'FURNISHED_RENT', + VIAGER=u'VIAGER') -from weboob.capabilities.housing import HousingPhoto as _HousingPhoto -class HousingPhoto(_HousingPhoto): - """ - Photo of a housing. - """ - data = BytesField('Data of photo') - def __init__(self, url): - super(HousingPhoto, self).__init__(url.split('/')[-1], url) +ADVERT_TYPES = enum(PROFESSIONAL=u'Professional', PERSONAL=u'Personal') - def __iscomplete__(self): - return self.data - def __unicode__(self): - return self.url - - def __repr__(self): - return '' % (self.id, len(self.data) if self.data else 0, self.url) - - -class UTILITIES(StrEnum): - INCLUDED = u'C.C.' - EXCLUDED = u'H.C.' - UNKNOWN = u'' - - -class ENERGY_CLASS(StrEnum): - A = u'A' - B = u'B' - C = u'C' - D = u'D' - E = u'E' - F = u'F' - G = u'G' - - -class POSTS_TYPES(StrEnum): - RENT=u'RENT' - SALE = u'SALE' - SHARING = u'SHARING' - FURNISHED_RENT = u'FURNISHED_RENT' - VIAGER = u'VIAGER' - - -class ADVERT_TYPES(StrEnum): - PROFESSIONAL = u'Professional' - PERSONAL = u'Personal' - - -class HOUSE_TYPES(StrEnum): - APART = u'Apartment' - HOUSE = u'House' - PARKING = u'Parking' - LAND = u'Land' - OTHER = u'Other' - UNKNOWN = u'Unknown' - - -from weboob.capabilities.housing import Housing as _Housing -class Housing(_Housing): - """ - Content of a housing. - """ - type = EnumField('Type of housing (rent, sale, sharing)', - POSTS_TYPES) - advert_type = EnumField('Type of advert (professional or personal)', - ADVERT_TYPES) - house_type = EnumField(u'Type of house (apartment, house, parking, …)', - HOUSE_TYPES) - title = StringField('Title of housing') - area = DecimalField('Area of housing, in m2') - cost = DecimalField('Cost of housing') - price_per_meter = DecimalField('Price per meter ratio') - currency = StringField('Currency of cost') - utilities = EnumField('Utilities included or not', UTILITIES) - date = DateField('Date when the housing has been published') - location = StringField('Location of housing') - station = StringField('What metro/bus station next to housing') - text = StringField('Text of the housing') - phone = StringField('Phone number to contact') - photos = Field('List of photos', list) - rooms = DecimalField('Number of rooms') - bedrooms = DecimalField('Number of bedrooms') - details = Field('Key/values of details', dict) - DPE = EnumField('DPE (Energy Performance Certificate)', ENERGY_CLASS) - GES = EnumField('GES (Greenhouse Gas Emissions)', ENERGY_CLASS) - - -from weboob.capabilities.housing import Query as _Query -class Query(_Query): - """ - Query to find housings. - """ - type = EnumField('Type of housing to find (POSTS_TYPES constants)', - POSTS_TYPES) - cities = Field('List of cities to search in', list, tuple) - area_min = IntField('Minimal area (in m2)') - area_max = IntField('Maximal area (in m2)') - cost_min = IntField('Minimal cost') - cost_max = IntField('Maximal cost') - nb_rooms = IntField('Number of rooms') - house_types = Field('List of house types', list, tuple, default=list(HOUSE_TYPES)) - advert_types = Field('List of advert types to filter on', list, tuple, - default=list(ADVERT_TYPES)) - - -from weboob.capabilities.housing import City as _City -class City(_City): - """ - City. - """ - name = StringField('Name of city') - - -from weboob.capabilities.housing import CapHousing as _CapHousing -class CapHousing(_CapHousing): - """ - Capability of websites to search housings. - """ - - def search_housings(self, query): - """ - Search housings. - - :param query: search query - :type query: :class:`Query` - :rtype: iter[:class:`Housing`] - """ - raise NotImplementedError() - - def get_housing(self, housing): - """ - Get an housing from an ID. - - :param housing: ID of the housing - :type housing: str - :rtype: :class:`Housing` or None if not found. - """ - raise NotImplementedError() - - def search_city(self, pattern): - """ - Search a city from a pattern. - - :param pattern: pattern to search - :type pattern: str - :rtype: iter[:class:`City`] - """ - raise NotImplementedError() +HOUSE_TYPES = enum(APART=u'Apartment', + HOUSE=u'House', + PARKING=u'Parking', + LAND=u'Land', + OTHER=u'Other', + UNKNOWN=u'Unknown') diff --git a/modules/societegenerale/browser.py b/modules/societegenerale/browser.py index 9a1cd534e27c7addbac65bca750c0396cb0576e5..e275b69c618af7cccad06b7878fd67139df6a6e6 100644 --- a/modules/societegenerale/browser.py +++ b/modules/societegenerale/browser.py @@ -33,7 +33,7 @@ ListRibPage, AdvisorPage, HTMLProfilePage, XMLProfilePage, LoansPage, IbanPage, ) from .pages.transfer import RecipientsPage, TransferPage, AddRecipientPage, RecipientJson -from .pages.login import LoginPage, BadLoginPage, ReinitPasswordPage, ActionNeededPage +from .pages.login import LoginPage, BadLoginPage, ReinitPasswordPage, ActionNeededPage, ErrorPage from .pages.subscription import BankStatementPage @@ -76,6 +76,8 @@ class SocieteGenerale(LoginBrowser, StatesMixin): bank_statement_search = URL(r'/restitution/rce_recherche.html\?noRedirect=1', r'/restitution/rce_recherche_resultat.html', BankStatementPage) + error = URL('https://static.societegenerale.fr/pri/erreur.html', ErrorPage) + accounts_list = None context = None dup = None diff --git a/modules/societegenerale/compat/weboob_capabilities_bank.py b/modules/societegenerale/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/societegenerale/compat/weboob_capabilities_bank.py +++ b/modules/societegenerale/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/societegenerale/module.py b/modules/societegenerale/module.py index 472ff1d85d1bcace89e4df2ecc6b19afe536542d..82d63c4f9b7dbcb3653984fe40a96989c2689f45 100644 --- a/modules/societegenerale/module.py +++ b/modules/societegenerale/module.py @@ -107,6 +107,8 @@ def iter_transfer_recipients(self, origin_account): return self.browser.iter_recipients(origin_account) def new_recipient(self, recipient, **params): + if self.config['website'].get() not in ('par', 'pro'): + raise NotImplementedError() recipient.label = ' '.join(w for w in re.sub('[^0-9a-zA-Z:\/\-\?\(\)\.,\'\+ ]+', '', recipient.label).split()) return self.browser.new_recipient(recipient, **params) diff --git a/modules/societegenerale/pages/login.py b/modules/societegenerale/pages/login.py index 361ba8490983e36312adac7ce378779c55472d4f..525565b4ab1861551ec197004af835f989d895c6 100644 --- a/modules/societegenerale/pages/login.py +++ b/modules/societegenerale/pages/login.py @@ -25,6 +25,8 @@ from weboob.tools.json import json from weboob.exceptions import BrowserUnavailable, BrowserPasswordExpired, ActionNeeded +from weboob.browser.pages import HTMLPage +from weboob.browser.filters.standard import CleanText from .base import BasePage from ..captcha import Captcha, TileError @@ -119,3 +121,9 @@ class ActionNeededPage(BasePage): # Mise à jour des conditions particulières de vos sonventions de comptes titres def on_load(self): raise ActionNeeded() + + +class ErrorPage(HTMLPage): + def on_load(self): + message = CleanText('//span[contains(text(), "Une erreur est survenue lors du chargement de la page")]')(self.doc) + raise BrowserUnavailable(message) diff --git a/modules/societegenerale/pages/transfer.py b/modules/societegenerale/pages/transfer.py index 0af107a9eca0dcfa28fb3481272dd375a87632a0..0121c7b85edeaf5b8ad573ad9579a8badadaf2a0 100644 --- a/modules/societegenerale/pages/transfer.py +++ b/modules/societegenerale/pages/transfer.py @@ -27,7 +27,7 @@ from weboob.browser.pages import LoggedPage, JsonPage, FormNotFound from weboob.browser.elements import method, ListElement, ItemElement, SkipItem from weboob.capabilities.bank import ( - Recipient, TransferError, TransferBankError, TransferInvalidCurrency, Transfer, + Recipient, TransferBankError, TransferInvalidCurrency, Transfer, AddRecipientError, AddRecipientStep, ) from weboob.capabilities.base import find_object, NotAvailable, empty @@ -199,7 +199,7 @@ def get_params(self, _id, _type): elif _type == 'Destinataires' and ((params['cdgude'] + params['nucpde']) == _id or params['codeIBANBenef'] == _id): return params - raise TransferError(u'Paramètres pour le compte %s numéro %s introuvable.' % (_type, _id)) + assert False, u'Paramètres pour le compte %s numéro %s introuvable.' % (_type, _id) def get_account_value(self, _id): for option in self.doc.xpath('//select[@id="SelectEmet"]//option'): @@ -219,8 +219,8 @@ def init_transfer(self, account, recipient, transfer): recipient_params = self.get_params(recipient.id, 'Destinataires') data = OrderedDict() value = self.get_account_value(account.id) or self.get_account_value_by_label(account.label) - if value is None: - raise TransferError("Couldn't retrieve origin account in list") + assert value is not None, "Couldn't retrieve origin account %s in list" % account.id + data['dup'] = re.search('dup=(.*?)(&|$)', value).group(1) data['src'] = re.search('src=(.*?)(&|$)', value).group(1) data['sign'] = re.search('sign=(.*?)(&|$)', value).group(1) @@ -258,12 +258,12 @@ def check_data_consistency(self, transfer): amount = CleanDecimal('.//td[@headers="virement montant"]', replace_dots=True)(self.doc) label = CleanText('.//td[@headers="virement motif"]')(self.doc) exec_date = Date(CleanText('.//td[@headers="virement date"]'), dayfirst=True)(self.doc) - if transfer.amount != amount: - raise TransferError('data consistency failed, %s changed from %s to %s' % ('amount', transfer.amount, amount)) - if transfer.label not in label: - raise TransferError('data consistency failed, %s changed from %s to %s' % ('label', transfer.label, label)) - if not (transfer.exec_date <= exec_date <= transfer.exec_date + timedelta(days=4)): - raise TransferError('data consistency failed, %s changed from %s to %s' % ('exec_date', transfer.exec_date, exec_date)) + assert transfer.amount == amount, \ + 'data consistency failed, %s changed from %s to %s' % ('amount', transfer.amount, amount) + assert transfer.label in label, \ + 'data consistency failed, %s changed from %s to %s' % ('label', transfer.label, label) + assert (transfer.exec_date <= exec_date <= transfer.exec_date + timedelta(days=4)), \ + 'data consistency failed, %s changed from %s to %s' % ('exec_date', transfer.exec_date, exec_date) def create_transfer(self, account, recipient, transfer): transfer = Transfer() diff --git a/modules/societegenerale/sgpe/browser.py b/modules/societegenerale/sgpe/browser.py index 414c862b2fdcbd46af9a53015b8b315699bab74e..2cae330a1c982c410377d12e16e7da79b6a385df 100644 --- a/modules/societegenerale/sgpe/browser.py +++ b/modules/societegenerale/sgpe/browser.py @@ -20,15 +20,18 @@ from __future__ import unicode_literals import re +from datetime import date -from weboob.browser.browsers import LoginBrowser, need_login +from weboob.browser.browsers import LoginBrowser, need_login, StatesMixin from weboob.browser.url import URL from weboob.browser.exceptions import ClientError from weboob.exceptions import BrowserIncorrectPassword from weboob.capabilities.base import find_object from weboob.capabilities.bank import ( - AccountNotFound, RecipientNotFound, + AccountNotFound, RecipientNotFound, AddRecipientStep, AddRecipientError, + Recipient, TransferBankError, ) +from weboob.tools.value import Value from .pages import ( LoginPage, CardsPage, CardHistoryPage, IncorrectLoginPage, @@ -36,7 +39,8 @@ ) from .json_pages import AccountsJsonPage, BalancesJsonPage, HistoryJsonPage, BankStatementPage from .transfer_pages import ( - EasyTransferPage, RecipientsJsonPage, TransferPage, SignTransferPage, + EasyTransferPage, RecipientsJsonPage, TransferPage, SignTransferPage, TransferDatesPage, + AddRecipientPage, AddRecipientStepPage, ConfirmRecipientPage, ) @@ -68,6 +72,9 @@ def do_login(self): raise BrowserIncorrectPassword('Password must be 6 digits long.') self.login.stay_or_go() + if self.page.logged: + return + self.session.cookies.set('PILOTE_OOBA', 'true') try: self.page.login(self.username, self.password) @@ -167,15 +174,17 @@ def iter_documents(self, subscription): self.subscription_form.go(data=data) return self.page.iter_documents(sub_id=subscription.id) -class SGProfessionalBrowser(SGEnterpriseBrowser): +class SGProfessionalBrowser(SGEnterpriseBrowser, StatesMixin): BASEURL = 'https://professionnels.secure.societegenerale.fr' LOGIN_FORM = 'auth_reco' MENUID = 'SBOREL' CERTHASH = '9f5232c9b2283814976608bfd5bba9d8030247f44c8493d8d205e574ea75148e' + STATE_DURATION = 5 incorrect_login = URL('/authent.html', IncorrectLoginPage) profile = URL('/gao/modifier-donnees-perso-saisie.html', ProfileProPage) + transfer_dates = URL('/ord-web/ord//get-dates-execution.json', TransferDatesPage) easy_transfer = URL('/ord-web/ord//ord-virement-simplifie-emetteur.html', EasyTransferPage) internal_recipients = URL('/ord-web/ord//ord-virement-simplifie-beneficiaire.html', EasyTransferPage) external_recipients = URL('/ord-web/ord//ord-liste-compte-beneficiaire-externes.json', RecipientsJsonPage) @@ -184,12 +193,33 @@ class SGProfessionalBrowser(SGEnterpriseBrowser): sign_transfer_page = URL('/ord-web/ord//ord-verifier-habilitation-signature-ordre.json', SignTransferPage) confirm_transfer = URL('/ord-web/ord//ord-valider-signature-ordre.json', TransferPage) + recipients = URL('/ord-web/ord//ord-gestion-tiers-liste.json', RecipientsJsonPage) + add_recipient = URL('/ord-web/ord//ord-fragment-form-tiers.html\?cl_action=ajout&cl_idTiers=', + AddRecipientPage) + add_recipient_step = URL('/ord-web/ord//ord-tiers-calcul-bic.json', + '/ord-web/ord//ord-preparer-signature-destinataire.json', + AddRecipientStepPage) + confirm_new_recipient = URL('/ord-web/ord//ord-creer-destinataire.json', ConfirmRecipientPage) + bank_statement_menu = URL('/icd/syd-front/data/syd-rce-accederDepuisMenu.json', BankStatementPage) bank_statement_search = URL('/icd/syd-front/data/syd-rce-lancerRecherche.json', BankStatementPage) date_max = None date_min = None + new_rcpt_token = None + new_rcpt_validate_form = None + need_reload_state = None + + __states__ = ['need_reload_state', 'new_rcpt_token', 'new_rcpt_validate_form'] + + def load_state(self, state): + # reload state only for new recipient feature + if state.get('need_reload_state'): + state.pop('url', None) + self.need_reload_state = None + super(SGProfessionalBrowser, self).load_state(state) + @need_login def iter_subscription(self): profile = self.get_profile() @@ -225,11 +255,130 @@ def advance_month(self, end_month, end_year, month_range=3): return new_end_month, end_year, begin_month, begin_year + def copy_recipient_obj(self, recipient): + rcpt = Recipient() + rcpt.id = recipient.iban + rcpt.iban = recipient.iban + rcpt.label = recipient.label + rcpt.category = 'Externe' + rcpt.enabled_at = date.today() + return rcpt + + @need_login + def new_recipient(self, recipient, **params): + if 'code' in params: + self.validate_rcpt_with_sms(params['code']) + return self.page.rcpt_after_sms(recipient) + + self.recipients.go() + step_urls = { + 'first_recipient_check': self.absurl('/ord-web/ord//ord-valider-destinataire-avant-maj.json', base=True), + 'get_bic': self.absurl('/ord-web/ord//ord-tiers-calcul-bic.json', base=True), + 'get_token': self.absurl('/ord-web/ord//ord-preparer-signature-destinataire.json', base=True), + 'get_sign_info': self.absurl('/sec/getsigninfo.json', base=True), + 'send_otp_to_user': self.absurl('/sec/csa/send.json', base=True), + } + + self.add_recipient.go(method='POST', headers={'Content-Type': 'application/json;charset=UTF-8'}) + countries = self.page.get_countries() + + # first recipient check + data = { + 'an_codeAction': 'ajout_tiers', + 'an_refSICoordonnee': '', + 'an_refSITiers': '', + 'cl_iban': recipient.iban, + 'cl_raisonSociale': recipient.label, + } + self.location(step_urls['first_recipient_check'], data=data) + + # get bic + data = { + 'an_activateCMU': 'true', + 'an_codePaysBanque': '', + 'an_nature': 'C', + 'an_numeroCompte': recipient.iban, + 'an_topIBAN': 'true', + 'cl_adresse': '', + 'cl_adresseBanque': '', + 'cl_codePays': recipient.iban[:2], + 'cl_libellePaysBanque': '', + 'cl_libellePaysDestinataire': countries[recipient.iban[:2]], + 'cl_nomBanque': '', + 'cl_nomRaisonSociale': recipient.label, + 'cl_ville': '', + 'cl_villeBanque': '', + } + self.location(step_urls['get_bic'], data=data) + bic = self.page.get_response_data() + + # get token + data = { + 'an_coordonnee_codePaysBanque': '', + 'an_coordonnee_nature': 'C', + 'an_coordonnee_numeroCompte': recipient.iban, + 'an_coordonnee_topConfidentiel': 'false', + 'an_coordonnee_topIBAN': 'true', + 'an_refSICoordonnee': '', + 'an_refSIDestinataire': '', + 'cl_adresse': '', + 'cl_codePays': recipient.iban[:2], + 'cl_coordonnee_adresseBanque': '', + 'cl_coordonnee_bic': bic, + 'cl_coordonnee_categories_libelle': '', + 'cl_coordonnee_categories_refSi': '', + 'cl_coordonnee_libellePaysBanque': '', + 'cl_coordonnee_nomBanque': '', + 'cl_coordonnee_villeBanque': '', + 'cl_libellePaysDestinataire': countries[recipient.iban[:2]], + 'cl_nomRaisonSociale': recipient.label, + 'cl_ville': '', + } + self.location(step_urls['get_token'], data=data) + self.new_rcpt_validate_form = data + payload = self.page.get_response_data() + + # get sign info + data = { + 'b64_jeton_transaction': payload['jeton'], + 'action_level': payload['sensibilite'], + } + self.location(step_urls['get_sign_info'], data=data) + + # send otp to user + data = { + 'context': payload['jeton'], + 'csa_op': 'sign' + } + self.location(step_urls['send_otp_to_user'], data=data) + self.new_rcpt_validate_form.update(data) + + rcpt = self.copy_recipient_obj(recipient) + self.need_reload_state = True + raise AddRecipientStep(rcpt, Value('code', label='Veuillez entrer le code reçu par SMS.')) + + @need_login + def validate_rcpt_with_sms(self, code): + if not self.new_rcpt_validate_form: + raise AddRecipientError() + + self.new_rcpt_validate_form.update({'code': code}) + try: + self.confirm_new_recipient.go(data=self.new_rcpt_validate_form) + except ClientError as e: + assert e.response.status_code == 403, \ + 'Something went wrong in add recipient, response status code is %s' % e.response.status_code + raise AddRecipientError(message='Le code entré est incorrect.') + @need_login def iter_recipients(self, origin_account): self.easy_transfer.go() self.page.update_origin_account(origin_account) + if not hasattr(origin_account, '_product_code'): + # check that origin account is updated, if not, this account can't do transfer + return + params = { 'cl_ibanEmetteur': origin_account.iban, 'cl_codeProduit': origin_account._product_code, @@ -251,12 +400,18 @@ def iter_recipients(self, origin_account): 'n_nbOccurences': '10000', } self.external_recipients.go(data=data) - assert self.page.is_all_external_recipient(), "Some recipients are missing" - for external_rcpt in self.page.iter_external_recipients(): - yield external_rcpt + + if self.page.is_external_recipients(): + assert self.page.is_all_external_recipient(), "Some recipients are missing" + for external_rcpt in self.page.iter_external_recipients(): + yield external_rcpt @need_login def init_transfer(self, account, recipient, transfer): + self.transfer_dates.go() + if not self.page.is_date_valid(transfer.exec_date): + raise TransferBankError(message="La date d'exécution du virement est invalide. Elle doit correspondre aux horaires et aux dates d'ouvertures d'agence.") + # update account and recipient info recipient = find_object(self.iter_recipients(account), iban=recipient.iban, error=RecipientNotFound) diff --git a/modules/societegenerale/sgpe/json_pages.py b/modules/societegenerale/sgpe/json_pages.py index 2aa7989c8cee79e74fb926c08124beb0b2652b73..20df079fa393352f1a84e5ea8e174857e22a4741 100644 --- a/modules/societegenerale/sgpe/json_pages.py +++ b/modules/societegenerale/sgpe/json_pages.py @@ -36,6 +36,8 @@ from .pages import Transaction class AccountsJsonPage(LoggedPage, JsonPage): + ENCODING = 'utf-8' + TYPES = {u'COMPTE COURANT': Account.TYPE_CHECKING, u'COMPTE PERSONNEL': Account.TYPE_CHECKING, u'CPTE PRO': Account.TYPE_CHECKING, diff --git a/modules/societegenerale/sgpe/pages.py b/modules/societegenerale/sgpe/pages.py index 5d45e79d42cf4125009541151bac5c143d80f8a5..b63442692e0b138e84858715844bf1295a55792a 100644 --- a/modules/societegenerale/sgpe/pages.py +++ b/modules/societegenerale/sgpe/pages.py @@ -95,6 +95,10 @@ def on_load(self): class LoginPage(SGPEPage): + @property + def logged(self): + return self.doc.xpath('//a[text()="Déconnexion" and @href="/logout"]') + def get_authentication_data(self): infos_data = self.browser.open('/sec/vk/gen_crypto?estSession=0').text infos_data = re.match('^_vkCallback\((.*)\);$', infos_data).group(1) diff --git a/modules/societegenerale/sgpe/transfer_pages.py b/modules/societegenerale/sgpe/transfer_pages.py index 078a0015d93535e8680f6593724d896ba91eb6b1..8794a7c6c4075d273f42ce249812c7f23d68d022 100644 --- a/modules/societegenerale/sgpe/transfer_pages.py +++ b/modules/societegenerale/sgpe/transfer_pages.py @@ -25,15 +25,25 @@ from weboob.browser.pages import LoggedPage, HTMLPage, JsonPage from weboob.browser.elements import method, DictElement, ItemElement +from weboob.browser.filters.standard import CleanText, CleanDecimal from weboob.browser.filters.html import Attr from weboob.browser.filters.json import Dict from weboob.browser.filters.standard import Date, Eval -from weboob.capabilities.bank import Recipient, Transfer +from weboob.capabilities.bank import Recipient, Transfer, Account from .pages import LoginPage -class RecipientsJsonPage(LoggedPage, JsonPage): +class ErrorCheckedJsonPage(JsonPage): + def on_load(self): + assert Dict('commun/statut')(self.doc) == 'ok', \ + 'Something went wrong: %s' % Dict('commun/raison')(self.doc) + + +class RecipientsJsonPage(LoggedPage, ErrorCheckedJsonPage): + def is_external_recipients(self): + return Dict('donnees/items')(self.doc) + def is_all_external_recipient(self): return ( Dict('donnees/nbTotalDestinataires')(self.doc) == len(self.doc['donnees']['items']) @@ -61,6 +71,13 @@ def condition(self): obj__ref = Dict('coordonnee/0/refSICoordonnee') +class TransferDatesPage(LoggedPage, ErrorCheckedJsonPage): + def is_date_valid(self, exec_date): + transfer_dates_list = Dict('donnees/listeDatesExecution')(self.doc) + assert transfer_dates_list + return exec_date.strftime('%d/%m/%Y') in transfer_dates_list + + class EasyTransferPage(LoggedPage, HTMLPage): def update_origin_account(self, origin_account): for account in self.doc.xpath('//ul[@id="idCptFrom"]//li'): @@ -69,7 +86,7 @@ def update_origin_account(self, origin_account): json_data = json.loads(data.replace('"', '"')) if ( - origin_account.label == json_data['libelleCompte'] + origin_account.label == CleanText().filter(json_data['libelleCompte']) and origin_account.iban == json_data['ibanCompte'] ): origin_account._currency_code = json_data['codeDevise'] @@ -85,7 +102,14 @@ def update_origin_account(self, origin_account): origin_account._underproduct_code = json_data['codeSousProduit'] break else: - assert False, 'Account % not found on transfer page' % (origin_account.label) + assumptions = ( + not origin_account.balance, + not origin_account.iban, + origin_account.currency != 'EUR', + origin_account.type == Account.TYPE_PEA, + ) + if not any(assumptions): + assert False, 'Account %s not found on transfer page' % (origin_account.label) def iter_internal_recipients(self): if self.doc.xpath('//ul[@id="idCmptToInterne"]'): @@ -107,10 +131,7 @@ def iter_internal_recipients(self): yield rcpt -class TransferPage(LoggedPage, JsonPage): - def on_load(self): - assert Dict('commun/statut')(self.doc) == 'ok', 'Something went wrong: %s' % Dict('commun/raison')(self.doc) - +class TransferPage(LoggedPage, ErrorCheckedJsonPage): def handle_response(self, origin, recipient, amount, reason, exec_date): account_data = Dict('donnees/detailOrdre/compteEmetteur')(self.doc) recipient_data = Dict('donnees/listOperations/0/compteBeneficiaire')(self.doc) @@ -129,11 +150,11 @@ def handle_response(self, origin, recipient, amount, reason, exec_date): transfer.recipient_iban = Dict('ibanCompte')(recipient_data) transfer.currency = Dict('montantTotalOrdre/codeDevise')(transfer_data) - transfer.amount = Eval( + transfer.amount = CleanDecimal(Eval( lambda x, y: x * (10 ** -y), Dict('montantTotalOrdre/valeurMontant'), Dict('montantTotalOrdre/codeDecimalisation') - )(transfer_data) + ))(transfer_data) transfer.exec_date = Date(Dict('dateExecution'), dayfirst=True)(transfer_data) transfer.label = Dict('libelleClientOrdre')(transfer_data) @@ -159,3 +180,34 @@ def get_confirm_transfer_data(self, password): 'vk_op': 'sign', 'context': token, } + + +class AddRecipientPage(LoggedPage, HTMLPage): + def get_countries(self): + countries = {} + for country in self.doc.xpath('//div[@id="div-pays-tiers"]//li[not(@data-codepays="")]'): + countries.update({ + Attr('.', 'data-codepays')(country): Attr('.', 'data-libellepays')(country) + }) + return countries + + +class AddRecipientStepPage(LoggedPage, ErrorCheckedJsonPage): + def get_response_data(self): + return self.doc['donnees'] + + +class ConfirmRecipientPage(LoggedPage, ErrorCheckedJsonPage): + def rcpt_after_sms(self, recipient): + rcpt_data = self.doc['donnees'] + + assert recipient.label == Dict('nomRaisonSociale')(rcpt_data) + assert recipient.iban == Dict('coordonnee/0/numeroCompte')(rcpt_data) + + rcpt = Recipient() + rcpt.id = Dict('coordonnee/0/refSICoordonnee')(rcpt_data) + rcpt.iban = Dict('coordonnee/0/numeroCompte')(rcpt_data) + rcpt.label = Dict('nomRaisonSociale')(rcpt_data) + rcpt.category = u'Externe' + rcpt.enabled_at = date.today() + return rcpt diff --git a/modules/spirica/compat/weboob_capabilities_bank.py b/modules/spirica/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/spirica/compat/weboob_capabilities_bank.py +++ b/modules/spirica/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/suravenir/compat/weboob_capabilities_bank.py b/modules/suravenir/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/suravenir/compat/weboob_capabilities_bank.py +++ b/modules/suravenir/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 - diff --git a/modules/yggtorrent/browser.py b/modules/yggtorrent/browser.py index 09094345a639c2bed4fefe901490eabb0ab3e861..74d7dde7997e6bb7b4f22fd25b7f1dacd5d12de6 100644 --- a/modules/yggtorrent/browser.py +++ b/modules/yggtorrent/browser.py @@ -35,7 +35,7 @@ class YggtorrentBrowser(LoginBrowser): PROFILE = Wget() TIMEOUT = 30 - BASEURL = 'https://ww1.yggtorrent.is/' + BASEURL = 'https://yggtorrent.to/' home = URL('$', HomePage) login = URL('/user/login$', LoginPage) search = URL(r'/engine/search\?name=(?P.*)&order=desc&sort=seed&do=search', SearchPage) diff --git a/modules/yomoni/compat/weboob_capabilities_bank.py b/modules/yomoni/compat/weboob_capabilities_bank.py index 8f80e948192a8fc53db8e45db7ec386ee5734cf8..404e8d30ed28683c8a27f183d1e670c10509ce56 100644 --- a/modules/yomoni/compat/weboob_capabilities_bank.py +++ b/modules/yomoni/compat/weboob_capabilities_bank.py @@ -17,6 +17,13 @@ class CapBankPockets(CapBank): pass +class Rate(BaseObject, Currency): + pass + +class CapCurrencyRate(CapBank): + pass + + class CapBankTransfer(OLD.CapBankTransfer): def transfer_check_label(self, old, new): from unidecode import unidecode @@ -31,4 +38,3 @@ class CapBankTransferAddRecipient(CapBankTransfer, OLD.CapBankTransferAddRecipie Account.TYPE_MORTGAGE = 17 Account.TYPE_CONSUMER_CREDIT = 18 Account.TYPE_REVOLVING_CREDIT = 19 -