diff --git a/modules/afer/pages.py b/modules/afer/pages.py index 86e5f287142f09cc6ccba26be6efbc381511c5e9..94b3fb40ed1408c26e865962623040907bb7b972 100644 --- a/modules/afer/pages.py +++ b/modules/afer/pages.py @@ -55,6 +55,10 @@ class IndexPage(LoggedPage, HTMLPage): if "prendre connaissance des nouvelles conditions" in msg: raise ActionNeeded(msg) + msg = CleanText('//span[@id="txtErrorAccesBase"]')(self.doc) + if 'Merci de nous envoyer' in msg: + raise ActionNeeded(msg) + # website sometime crash if self.doc.xpath(u'//div[@id="divError"]/span[contains(text(),"Une erreur est survenue")]'): raise BrowserUnavailable() diff --git a/modules/amazon/browser.py b/modules/amazon/browser.py index a239ac910d8e913dbeae4929f673043d3b165bfe..fc87804b97e6472e7e73cb2412cb5928d7f12825 100644 --- a/modules/amazon/browser.py +++ b/modules/amazon/browser.py @@ -71,7 +71,7 @@ class AmazonBrowser(LoginBrowser, StatesMixin): super(AmazonBrowser, self).__init__(*args, **kwargs) def locate_browser(self, state): - pass + self.location(state['url']) def push_security_otp(self, pin_code): res_form = self.otp_form @@ -181,11 +181,12 @@ class AmazonBrowser(LoginBrowser, StatesMixin): @need_login def iter_documents(self, subscription): - documents = [] - - for y in range(date.today().year - 2, date.today().year + 1): - self.documents.go(year=y) + year = date.today().year + old_year = year - 2 + while year >= old_year: + self.documents.go(year=year) request_id = self.page.response.headers['x-amz-rid'] for doc in self.page.iter_documents(subid=subscription.id, currency=self.CURRENCY, request_id=request_id): - documents.append(doc) - return documents + yield doc + + year -= 1 diff --git a/modules/amazon/module.py b/modules/amazon/module.py index c7aded82a94025e0d9e5437a0a8a6bc8796843a7..c6f3065cfa0724f15c8269d105a1874df205fab9 100644 --- a/modules/amazon/module.py +++ b/modules/amazon/module.py @@ -20,11 +20,10 @@ from __future__ import unicode_literals from collections import OrderedDict -from weboob.capabilities.bill import CapDocument, Subscription, Document, SubscriptionNotFound, DocumentNotFound +from weboob.capabilities.bill import DocumentTypes, CapDocument, Subscription, Document, SubscriptionNotFound, DocumentNotFound from weboob.capabilities.base import find_object, NotAvailable from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword, Value -from weboob.tools.pdf import html_to_pdf from .browser import AmazonBrowser from .en.browser import AmazonEnBrowser @@ -64,6 +63,8 @@ class AmazonModule(Module, CapDocument): Value('pin_code', label='OTP response', required=False, default='') ) + accepted_document_types = (DocumentTypes.BILL,) + def create_default_browser(self): self.BROWSER = self.BROWSERS[self.config['website'].get()] return self.create_browser(self.config) @@ -92,11 +93,3 @@ class AmazonModule(Module, CapDocument): return return self.browser.open(document.url).content - - def download_document_pdf(self, document): - if not isinstance(document, Document): - document = self.get_document(document) - if document.url is NotAvailable: - return - - return html_to_pdf(self.browser, url=self.browser.BASEURL + document.url) diff --git a/modules/amazon/pages.py b/modules/amazon/pages.py index 47129b54d88eed4b5a716d26d61d92dea36759ce..211b6287b2f177eb70b24b931f598a9ff639f6ea 100644 --- a/modules/amazon/pages.py +++ b/modules/amazon/pages.py @@ -26,7 +26,7 @@ from weboob.browser.filters.standard import ( CleanText, CleanDecimal, Env, Regexp, Format, Field, Currency, RegexpError, Date, Async, AsyncLoad ) -from weboob.capabilities.bill import Bill, Subscription +from weboob.capabilities.bill import DocumentTypes, Bill, Subscription from weboob.capabilities.base import NotAvailable from weboob.tools.date import parse_french_date @@ -131,7 +131,7 @@ class DocumentsPage(LoggedPage, HTMLPage): obj_url = Async('details') & Link('//a[contains(@href, "download")]|//a[contains(@href, "generated_invoices")]') obj_format = 'pdf' obj_label = Format('Facture %s', Field('_simple_id')) - obj_type = 'bill' + obj_type = DocumentTypes.BILL def obj_date(self): date = Date(CleanText('.//div[has-class("a-span4") and not(has-class("recipient"))]/div[2]'), diff --git a/modules/ameli/module.py b/modules/ameli/module.py index 2747cb4b837878317b7c0e058f3d1b222354d661..8002d9be780595029244b836e0a0c19a768c709a 100644 --- a/modules/ameli/module.py +++ b/modules/ameli/module.py @@ -19,7 +19,7 @@ from __future__ import unicode_literals -from weboob.capabilities.bill import CapDocument, SubscriptionNotFound, DocumentNotFound, Subscription, Bill +from weboob.capabilities.bill import DocumentTypes, CapDocument, SubscriptionNotFound, DocumentNotFound, Subscription, Bill from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword from .browser import AmeliBrowser @@ -38,6 +38,8 @@ class AmeliModule(Module, CapDocument): CONFIG = BackendConfig(ValueBackendPassword('login', label='Numero de SS', regexp=r'^\d{13}$', masked=False), ValueBackendPassword('password', label='Password', masked=True)) + accepted_document_types = (DocumentTypes.BILL,) + def create_default_browser(self): return self.create_browser(self.config['login'].get(), self.config['password'].get()) diff --git a/modules/ameli/pages.py b/modules/ameli/pages.py index b9eb114228a5387a221c0e73c71837876c59185e..245d697ccd25f1a635612492d6ff7cbaabae1830 100644 --- a/modules/ameli/pages.py +++ b/modules/ameli/pages.py @@ -25,7 +25,7 @@ from decimal import Decimal from weboob.browser.filters.html import Attr, XPathNotFound from weboob.browser.pages import HTMLPage, RawPage, LoggedPage -from weboob.capabilities.bill import Subscription, Detail, Bill +from weboob.capabilities.bill import DocumentTypes, Subscription, Detail, Bill from weboob.browser.filters.standard import CleanText, Regexp from weboob.exceptions import BrowserUnavailable @@ -132,7 +132,7 @@ class LastPaymentsPage(LoggedPage, AmeliBasePage): bil.id = sub._id + "." + date.strftime("%Y%m") bil.date = date bil.format = 'pdf' - bil.type = 'bill' + bil.type = DocumentTypes.BILL bil.label = date.strftime("%Y%m%d") bil.url = '/PortailAS/PDFServletReleveMensuel.dopdf?PDF.moisRecherche=' + date.strftime("%m%Y") yield bil diff --git a/modules/amelipro/module.py b/modules/amelipro/module.py index 550b033fc80a6a8f73388e9fcb7c626d493d1cca..d3d6325236915fa76072b7b98cfba0a5b1ed574b 100644 --- a/modules/amelipro/module.py +++ b/modules/amelipro/module.py @@ -17,7 +17,7 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . -from weboob.capabilities.bill import CapDocument, SubscriptionNotFound, DocumentNotFound, Subscription, Bill +from weboob.capabilities.bill import DocumentTypes, CapDocument, SubscriptionNotFound, DocumentNotFound, Subscription, Bill from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword from .browser import AmeliProBrowser @@ -40,6 +40,8 @@ class AmeliProModule(Module, CapDocument): masked=True) ) + accepted_document_types = (DocumentTypes.BILL,) + def create_default_browser(self): self.logger.settings['save_responses'] = False # Set to True to help debugging return self.create_browser(self.config['login'].get(), diff --git a/modules/amelipro/pages.py b/modules/amelipro/pages.py index cc73dfc5855c7fb81d726de6b29e33014b8cf21b..d1d2f4e89fa51ae4341477bfa59fff1bc642f574 100644 --- a/modules/amelipro/pages.py +++ b/modules/amelipro/pages.py @@ -22,7 +22,7 @@ from datetime import datetime import re from decimal import Decimal from weboob.browser.pages import HTMLPage -from weboob.capabilities.bill import Subscription, Detail, Bill +from weboob.capabilities.bill import DocumentTypes, Subscription, Detail, Bill # Ugly array to avoid the use of french locale @@ -111,7 +111,7 @@ class BillsPage(HTMLPage): bil.price = 0 bil.label = u''+date.strftime("%Y%m%d") bil.format = u''+format - bil.type = u'bill' + bil.type = DocumentTypes.BILL filedate = date.strftime("%m%Y") bil.url = u'/PortailPS/fichier.do' bil._data = {'FICHIER.type': format.lower()+'.releveCompteMensuel', diff --git a/modules/americanexpress/pages.py b/modules/americanexpress/pages.py index adb17266ff73861737be95b94a5c961a535058ff..be558e4ded6677689bcdf9c13bdf81cf089d85ea 100644 --- a/modules/americanexpress/pages.py +++ b/modules/americanexpress/pages.py @@ -111,6 +111,7 @@ class AccountsPage(LoggedPage, HTMLPage): acc._idforJSON = account_data[10][-1] else: acc._idforJSON = account_data[-5][-1] + acc._idforJSON = re.sub('\s+', ' ', acc._idforJSON) acc.number = '-%s' % account_data[2][2] acc.label = '%s %s' % (account_data[6][4], account_data[10][-1]) acc._balances_token = acc.id = balances_token diff --git a/modules/barclays/browser.py b/modules/barclays/browser.py index bf66302b99c0882cfcf8206c2268813c9c094ecd..71725779c760980c53874f8be45651039210a4ef 100644 --- a/modules/barclays/browser.py +++ b/modules/barclays/browser.py @@ -29,7 +29,8 @@ from weboob.capabilities.base import NotAvailable from .pages import ( LoginPage, AccountsPage, AccountPage, MarketAccountPage, - LifeInsuranceAccountPage, CardPage, IbanPDFPage, + LifeInsuranceAccountPage, CardPage, IbanPDFPage, ActionNeededPage, + RevolvingAccountPage, LoanAccountPage, ) @@ -41,10 +42,13 @@ class Barclays(LoginBrowser): login = URL('/BconnectDesk/servletcontroller', LoginPage) accounts = URL('/BconnectDesk/servletcontroller', AccountsPage) + loan_account = URL('/BconnectDesk/servletcontroller', LoanAccountPage) account = URL('/BconnectDesk/servletcontroller', AccountPage) card_account = URL('/BconnectDesk/servletcontroller', CardPage) market_account = URL('/BconnectDesk/servletcontroller', MarketAccountPage) life_insurance_account = URL('/BconnectDesk/servletcontroller', LifeInsuranceAccountPage) + revolving_account = URL('/BconnectDesk/servletcontroller', RevolvingAccountPage) + actionNeededPage = URL('/BconnectDesk/servletcontroller', ActionNeededPage) iban = URL('/BconnectDesk/editique', IbanPDFPage) def __init__(self, secret, *args, **kwargs): @@ -127,8 +131,9 @@ class Barclays(LoginBrowser): traccounts = [] for account in accounts: - if account.type != Account.TYPE_LOAN: - self._go_to_account(account) + self._go_to_account(account) + if account.type == Account.TYPE_LOAN: + account = self.page.get_loan_attributes(account) if account.type == Account.TYPE_CARD: if self.page.is_immediate_card(): @@ -138,6 +143,8 @@ class Barclays(LoginBrowser): continue account._attached_account = self.page.do_account_attachment([a for a in accounts if a.type == Account.TYPE_CHECKING]) + if account.type == Account.TYPE_REVOLVING_CREDIT: + account = self.page.get_revolving_attributes(account) account.iban = self.iban.open().get_iban() if self.page.has_iban() else NotAvailable @@ -151,7 +158,7 @@ class Barclays(LoginBrowser): def iter_history(self, account): if account._multiple_type and not self._multiple_account_choice(account): return [] - elif account.type == Account.TYPE_LOAN: + elif account.type in (Account.TYPE_LOAN, Account.TYPE_REVOLVING_CREDIT): return [] self._go_to_account(account) diff --git a/modules/barclays/pages.py b/modules/barclays/pages.py index 61ee0273b7c8cb423d41cd2de467b20a4207b3cf..f3e66ce091f7f436bcad50f6b3f28cfaa8c7d982 100644 --- a/modules/barclays/pages.py +++ b/modules/barclays/pages.py @@ -21,16 +21,14 @@ from __future__ import unicode_literals import re -from six.moves.html_parser import HTMLParser - from weboob.browser.pages import HTMLPage, PDFPage, LoggedPage from weboob.browser.elements import TableElement, ListElement, ItemElement, method from weboob.browser.filters.standard import CleanText, CleanDecimal, Regexp, Field, Date, Eval -from weboob.browser.filters.html import Attr, TableCell -from weboob.capabilities.bank import Account, Investment, NotAvailable +from weboob.browser.filters.html import Attr, TableCell, ReplaceEntities +from weboob.capabilities.bank import Account, Investment, Loan, NotAvailable from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.tools.capabilities.bank.iban import is_iban_valid - +from weboob.exceptions import ActionNeeded def MyDecimal(*args, **kwargs): kwargs.update(replace_dots=True, default=NotAvailable) @@ -109,6 +107,8 @@ class AccountsPage(StatefulPage): ACCOUNT_EXTRA_TYPES = {'BMOOVIE': Account.TYPE_LIFE_INSURANCE, 'B. GESTION VIE': Account.TYPE_LIFE_INSURANCE, 'E VIE MILLEIS': Account.TYPE_LIFE_INSURANCE, + 'BANQUE PRIVILEGE': Account.TYPE_REVOLVING_CREDIT, + 'PRET PERSONNEL': Account.TYPE_LOAN, } ACCOUNT_TYPE_TO_STR = {Account.TYPE_MARKET: 'TTR', Account.TYPE_CARD: 'CRT' @@ -205,16 +205,6 @@ class Transaction(FrenchTransaction): ] -class Entities(CleanText): - """ - Filter to replace HTML entities like "é" or "B" with their unicode counterpart. - """ - def filter(self, data): - h = HTMLParser() - txt = super(Entities, self).filter(data) - return h.unescape(txt) - - class AbstractAccountPage(StatefulPage): def has_iban(self): return len(self.doc.xpath('//a[contains(., "Edition RIB")]/ancestor::node()[2][not(contains(@style, "display: none;"))]')) > 1 @@ -255,7 +245,7 @@ class AbstractAccountPage(StatefulPage): obj_date = Date(CleanText(TableCell('date')), dayfirst=True) obj_vdate = Date(CleanText(TableCell('vdate')), dayfirst=True) obj_amount = MyDecimal(TableCell('credit'), default=TableCell('debit')) - obj_raw = Transaction.Raw(Entities(Regexp(CleanText('.//script[1]'), r"toggleDetails\([^,]+,[^,]+, '(.*?)', '(.*?)', '(.*?)',", r'\1 \2 \3'))) + obj_raw = Transaction.Raw(ReplaceEntities(Regexp(CleanText('.//script[1]'), r"toggleDetails\([^,]+,[^,]+, '(.*?)', '(.*?)', '(.*?)',", r'\1 \2 \3'))) class AccountPage(AbstractAccountPage): @@ -449,6 +439,64 @@ class CardPage(AbstractAccountPage): return (a[1], 'C4__WORKING[1].LISTCONTRATS', form['C4__WORKING[1].LISTCONTRATS'], a[2]) +class RevolvingAccountPage(AbstractAccountPage): + def is_here(self): + return bool(CleanText('//span[contains(., "Crédit renouvelable")]')(self.doc)) + + def has_iban(self): + return False + + def get_revolving_attributes(self, account): + loan = Loan() + + loan.available_amount = CleanDecimal('//div/span[contains(text(), "Montant disponible")]/following-sibling::*[1]', replace_dots=True)(self.doc) + loan.used_amount = CleanDecimal('//div/span[contains(text(), "Montant Utilisé")]/following-sibling::*[1]', replace_dots=True)(self.doc) + loan.total_amount = CleanDecimal('//div/span[contains(text(), "Réserve accordée")]/following-sibling::*[1]', replace_dots=True)(self.doc) + loan.last_payment_amount = CleanDecimal('//div/span[contains(text(), "Echéance Précédente")]/following-sibling::*[1]', replace_dots=True)(self.doc) + loan.last_payment_date = Date(Regexp(CleanText('//div/span[contains(text(), "Echéance Précédente")]/following-sibling::*[2]'), r'(\d{2}\/\d{2}\/\d{4})'), dayfirst=True)(self.doc) + owner_name = CleanText('//a[@class="lien-entete login"]/span')(self.doc) + loan.name = ' '.join(owner_name.split()[1:]) + + loan.id = account.id + loan.currency = account.currency + loan.label = account.label + loan.balance = account.balance + loan.coming = account.coming + loan.type = account.type + loan._uncleaned_id = account._uncleaned_id + loan._multiple_type = account._multiple_type + return loan + + +class LoanAccountPage(AbstractAccountPage): + def is_here(self): + return bool(CleanText('//span[contains(., "Détail compte")]')(self.doc)) + + def has_iban(self): + return False + + def get_loan_attributes(self, account): + loan = Loan() + loan.total_amount = CleanDecimal('//div/span[contains(text(), "Capital initial")]/following-sibling::*[1]', replace_dots=True)(self.doc) + owner_name = CleanText('//a[@class="lien-entete login"]/span')(self.doc) + loan.name = ' '.join(owner_name.split()[1:]) + loan.subscription_date = Date(Regexp(CleanText('//h4[span[contains(text(), "Date de départ du prêt")]]'), r'(\d{2}\/\d{2}\/\d{4})'), dayfirst=True)(self.doc) + loan.maturity_date = Date(Regexp(CleanText('//h4[span[contains(text(), "Date de fin du prêt")]]'), r'(\d{2}\/\d{2}\/\d{4})'), dayfirst=True)(self.doc) + loan.rate = Eval(lambda x: x / 100, CleanDecimal('//div/span[contains(text(), "Taux fixe")]/following-sibling::*[1]', replace_dots=True))(self.doc) + loan.last_payment_amount = CleanDecimal('//div[@class="txt-detail " and not (@style)]//span[contains(text(), "Echéance du")]/following-sibling::span[1]')(self.doc) + loan.last_payment_date = Date(Regexp(CleanText('//div[@class="txt-detail " and not (@style)]//span[contains(text(), "Echéance du")]'), r'(\d{2}\/\d{2}\/\d{4})'), dayfirst=True)(self.doc) + + loan.id = account.id + loan.currency = account.currency + loan.label = account.label + loan.balance = account.balance + loan.coming = account.coming + loan.type = account.type + loan._uncleaned_id = account._uncleaned_id + loan._multiple_type = account._multiple_type + return loan + + class IbanPDFPage(LoggedPage, PDFPage): def get_iban(self): match = re.search(r'Tm \[\((FR[0-9]{2} [A-Z0-9 ]+)\)\]', self.doc.decode('ISO8859-1')) @@ -461,3 +509,15 @@ class IbanPDFPage(LoggedPage, PDFPage): assert is_iban_valid(iban) return iban + + +class ActionNeededPage(LoggedPage, HTMLPage): + def on_load(self): + # We only need 2 sentences because the thirds is too specific + message = CleanText("//h2[contains(@class, 'ecDIB')]")(self.doc).split('.') + raise ActionNeeded('%s.' % '.'.join(message[:-2])) + + def is_here(self): + message = CleanText("//h2[contains(@class, 'ecDIB')]")(self.doc) + text = "Afin de respecter nos obligations réglementaires, nous devons disposer d’une connaissance récente des données de nos clients." + return message and text in message diff --git a/modules/bnporc/enterprise/pages.py b/modules/bnporc/enterprise/pages.py index a02aca40f0b688935d747f9054c1b1b9e66ded18..1269aed097818865601656319635040446a335c2 100644 --- a/modules/bnporc/enterprise/pages.py +++ b/modules/bnporc/enterprise/pages.py @@ -343,7 +343,8 @@ class MarketPage(LoggedPage, HTMLPage): @method class iter_market_accounts(TableElement): def condition(self): - return not self.el.xpath('//table[@id="table-portefeuille"]//tr/td[contains(text(), "Aucun portefeuille à afficher")]') + return not self.el.xpath('//table[@id="table-portefeuille"]//tr/td[contains(text(), "Aucun portefeuille à afficher") \ + or contains(text(), "No portfolio to display")]') item_xpath = '//table[@id="table-portefeuille"]/tbody[@class="main-content"]/tr' head_xpath = '//table[@id="table-portefeuille"]/thead/tr/th/label' diff --git a/modules/bolden/browser.py b/modules/bolden/browser.py index c89cf51344b450e30c4267fd5c3e65e8e9027efd..05d6dc38021364cae362c0a21456fe411770bb2f 100644 --- a/modules/bolden/browser.py +++ b/modules/bolden/browser.py @@ -22,7 +22,7 @@ from __future__ import unicode_literals from datetime import timedelta, datetime from weboob.browser import LoginBrowser, need_login, URL -from weboob.capabilities.bill import Document +from weboob.capabilities.bill import DocumentTypes, Document from weboob.tools.capabilities.bank.investments import create_french_liquidity from .pages import ( @@ -97,6 +97,6 @@ class BoldenBrowser(LoginBrowser): doc.id = inv.id doc.url = inv._docurl doc.label = 'Contrat %s' % inv.label - doc.type = 'other' + doc.type = DocumentTypes.OTHER doc.format = 'pdf' yield doc diff --git a/modules/bolden/module.py b/modules/bolden/module.py index 30d236d72f9a84c76a06137cd1a488867fc39b92..65cc7743342654aaa23f9e1b5cb76d1180e4858d 100644 --- a/modules/bolden/module.py +++ b/modules/bolden/module.py @@ -26,6 +26,7 @@ from weboob.capabilities.bank import CapBankWealth, Account from weboob.capabilities.base import find_object from weboob.capabilities.bill import ( CapDocument, Subscription, SubscriptionNotFound, DocumentNotFound, Document, + DocumentTypes, ) from weboob.capabilities.profile import CapProfile @@ -50,6 +51,8 @@ class BoldenModule(Module, CapBankWealth, CapDocument, CapProfile): ValueBackendPassword('password', label='Mot de passe'), ) + accepted_document_types = (DocumentTypes.OTHER,) + def create_default_browser(self): return self.create_browser(self.config['login'].get(), self.config['password'].get()) diff --git a/modules/boursorama/browser.py b/modules/boursorama/browser.py index 73ebe78e8ab56aecf9f735f23b9912f1b7737531..f365d1e1e265f128feb60d49b4387a0c1cfecece 100644 --- a/modules/boursorama/browser.py +++ b/modules/boursorama/browser.py @@ -36,7 +36,7 @@ from weboob.capabilities.bank import ( from weboob.capabilities.contact import Advisor from weboob.tools.captcha.virtkeyboard import VirtKeyboardError from weboob.tools.value import Value -from weboob.tools.compat import basestring, urlsplit, urlunsplit +from weboob.tools.compat import basestring, urlsplit from weboob.tools.capabilities.bank.transactions import sorted_transactions from .pages import ( @@ -45,7 +45,7 @@ from .pages import ( CardsNumberPage, CalendarPage, HomePage, PEPPage, TransferAccounts, TransferRecipients, TransferCharac, TransferConfirm, TransferSent, AddRecipientPage, StatusPage, CardHistoryPage, CardCalendarPage, CurrencyListPage, CurrencyConvertPage, - AccountsErrorPage, NoAccountPage, + AccountsErrorPage, NoAccountPage, TransferMainPage, ) @@ -85,10 +85,11 @@ class BoursoramaBrowser(RetryLoginBrowser, StatesMixin): saving_pep = URL('/compte/epargne/pep', PEPPage) incident = URL('/compte/cav/(?P.*)/mes-incidents.*', IncidentPage) - transfer_accounts = URL(r'/compte/(?P[^/]+)/(?P\w+)/virements/nouveau/(?P\w+)/1', - r'/compte/(?P[^/]+)/(?P\w+)/virements/nouveau$', - TransferAccounts) - recipients_page = URL(r'/compte/(?P[^/]+)/(?P\w+)/virements/$', + # transfer + transfer_main_page = URL(r'/compte/(?P[^/]+)/(?P\w+)/virements$', TransferMainPage) + transfer_accounts = URL(r'/compte/(?P[^/]+)/(?P\w+)/virements/nouveau$', + r'/compte/(?P[^/]+)/(?P\w+)/virements/nouveau/(?P\w+)/1', TransferAccounts) + recipients_page = URL(r'/compte/(?P[^/]+)/(?P\w+)/virements$', r'/compte/(?P[^/]+)/(?P\w+)/virements/nouveau/(?P\w+)/2', TransferRecipients) transfer_charac = URL(r'/compte/(?P[^/]+)/(?P\w+)/virements/nouveau/(?P\w+)/3', @@ -376,24 +377,25 @@ class BoursoramaBrowser(RetryLoginBrowser, StatesMixin): def iter_transfer_recipients(self, account): if account.type in (Account.TYPE_LOAN, Account.TYPE_LIFE_INSURANCE): return [] - assert account.url + # url transfer preparation url = urlsplit(account.url) parts = [part for part in url.path.split('/') if part] - if account.type == Account.TYPE_SAVINGS: - self.logger.debug('Deleting account name %s to get recipients', parts[-2]) - del parts[-2] - parts.append('virements') - url = url._replace(path='/'.join(parts)) - target = urlunsplit(url) + assert len(parts) > 2, 'Account url missing some important part to iter recipient' + account_type = parts[1] # cav, ord, epargne ... + account_webid = parts[-1] try: - self.location(target) + self.transfer_main_page.go(acc_type=account_type, webid=account_webid) except BrowserHTTPNotFound: return [] + # can check all account available transfer option + if self.transfer_main_page.is_here(): + self.transfer_accounts.go(acc_type=account_type, webid=account_webid) + if self.transfer_accounts.is_here(): try: self.page.submit_account(account.id) diff --git a/modules/boursorama/pages.py b/modules/boursorama/pages.py index 6f32f5b275bf88950c25364182cd60c5c82cb18c..caefbbaa0a270ff3a29d93c5fd74e737cb562b6a 100644 --- a/modules/boursorama/pages.py +++ b/modules/boursorama/pages.py @@ -804,6 +804,10 @@ class NoTransferPage(LoggedPage, HTMLPage): pass +class TransferMainPage(LoggedPage, HTMLPage): + pass + + class TransferAccounts(LoggedPage, HTMLPage): @method class iter_accounts(ListElement): diff --git a/modules/bp/browser.py b/modules/bp/browser.py index 73361b9fdf94ea1887d66db990d95b025ecf0dc2..0c6dd3ad7eb4dfeabfb621ac51e08b3ef88637e0 100644 --- a/modules/bp/browser.py +++ b/modules/bp/browser.py @@ -612,7 +612,7 @@ class BProBrowser(BPBrowser): self.location('%s/voscomptes/rib/init-rib.ea' % self.base_url) value = self.page.get_rib_value(acc.id) if value: - self.location('%s/voscomptes/rib/preparerRIB-rib.ea?%s' % (self.base_url, value)) + self.location('%s/voscomptes/rib/preparerRIB-rib.ea?idxSelection=%s' % (self.base_url, value)) if self.rib.is_here(): acc.iban = self.page.get_iban() @@ -626,7 +626,7 @@ class BProBrowser(BPBrowser): self.location('%s/voscomptes/rib/init-rib.ea' % self.base_url) value = self.page.get_rib_value(acc.id) if value: - self.location('%s/voscomptes/rib/preparerRIB-rib.ea?%s' % (self.base_url, value)) + self.location('%s/voscomptes/rib/preparerRIB-rib.ea?idxSelection=%s' % (self.base_url, value)) if self.rib.is_here(): return self.page.get_profile() diff --git a/modules/bp/pages/subscription.py b/modules/bp/pages/subscription.py index 69ab822ca505bd8c133ede867892c0f33e19699e..eaf9d4e09f491758b888f7fee0d0e908c60c6430 100644 --- a/modules/bp/pages/subscription.py +++ b/modules/bp/pages/subscription.py @@ -21,7 +21,7 @@ from __future__ import unicode_literals import re -from weboob.capabilities.bill import Subscription, Document +from weboob.capabilities.bill import DocumentTypes, Subscription, Document from weboob.browser.pages import LoggedPage, HTMLPage from weboob.browser.filters.standard import CleanText, Regexp, Env, Date, Format, Field from weboob.browser.filters.html import Link, Attr, TableCell @@ -62,7 +62,7 @@ class SubscriptionPage(LoggedPage, HTMLPage): obj_label = Format('%s - %s', CleanText('.//span[contains(@class, "lib")]'), CleanText('.//span[contains(@class, "date")]')) obj_url = Format('/voscomptes/canalXHTML/relevePdf/relevePdf_historique/%s', Link('./a')) obj_format = 'pdf' - obj_type = 'other' + obj_type = DocumentTypes.OTHER def obj_date(self): date = CleanText('.//span[contains(@class, "date")]')(self) @@ -141,7 +141,7 @@ class ProSubscriptionPage(LoggedPage, HTMLPage): # on the page of the year XXX for the subscription YYYY obj_url = Link('.//a') obj_format = 'pdf' - obj_type = 'other' + obj_type = DocumentTypes.OTHER def submit_form(self, sub_number, year): form = self.get_form(name='formRechHisto') diff --git a/modules/caissedepargne/browser.py b/modules/caissedepargne/browser.py index eae03e34b19e80f337343c3dd4ab0ae2ff93299b..c45cd5c306401c93ad3987981414d80b26ffe97f 100644 --- a/modules/caissedepargne/browser.py +++ b/modules/caissedepargne/browser.py @@ -411,6 +411,10 @@ class CaisseEpargne(LoginBrowser, StatesMixin): self.page.submit() if 'offrebourse.com' in self.url: + # Some users may not have access to this. + if self.page.is_error(): + continue + self.update_linebourse_token() page = self.linebourse.go_portfolio(account.id) assert self.linebourse.portfolio.is_here() diff --git a/modules/caissedepargne/cenet/pages.py b/modules/caissedepargne/cenet/pages.py index 038b5c37240cf81992cdf02aa191cc082a58eb1f..e46dd7324900f511cf5717c064028938d3735e61 100644 --- a/modules/caissedepargne/cenet/pages.py +++ b/modules/caissedepargne/cenet/pages.py @@ -29,7 +29,7 @@ from weboob.capabilities import NotAvailable from weboob.capabilities.bank import Account, Transaction from weboob.capabilities.contact import Advisor from weboob.capabilities.profile import Profile -from weboob.capabilities.bill import Subscription, Document +from weboob.capabilities.bill import DocumentTypes, Subscription, Document from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.exceptions import BrowserUnavailable @@ -278,7 +278,7 @@ class SubscriptionPage(LoggedPage, CenetJsonPage): obj_id = Format('%s_%s_%s', Env('sub_id'), Dict('Numero'), CleanText(Env('french_date'), symbols='/')) obj_format = 'pdf' - obj_type = 'other' + obj_type = DocumentTypes.OTHER obj__numero = CleanText(Dict('Numero')) obj__sub_id = Env('sub_id') obj__sub_label = Env('sub_label') diff --git a/modules/caissedepargne/module.py b/modules/caissedepargne/module.py index 8c874a405f1cb9837c2489d2d81a1ed924b5a81e..802551642ca76b8175c6c1b2e7a45debe2c1c5ff 100644 --- a/modules/caissedepargne/module.py +++ b/modules/caissedepargne/module.py @@ -24,7 +24,7 @@ from decimal import Decimal from weboob.capabilities.bank import CapBankWealth, CapBankTransferAddRecipient, AccountNotFound, Account, RecipientNotFound from weboob.capabilities.bill import ( CapDocument, Subscription, SubscriptionNotFound, - Document, DocumentNotFound, + Document, DocumentNotFound, DocumentTypes, ) from weboob.capabilities.base import NotAvailable from weboob.capabilities.contact import CapContact @@ -56,6 +56,8 @@ class CaisseEpargneModule(Module, CapBankWealth, CapBankTransferAddRecipient, Ca Value('nuser', label='User ID (optional)', default='', regexp='\d{0,8}'), Value('pincode', label='pincode', required=False)) + accepted_document_types = (DocumentTypes.OTHER,) + def create_default_browser(self): return self.create_browser(nuser=self.config['nuser'].get(), username=self.config['login'].get(), diff --git a/modules/caissedepargne/pages.py b/modules/caissedepargne/pages.py index b864c231a585dcffb9fff95a783e5684d700ea4a..eb87520dc5d1cf22a943be702a22427e347091a3 100644 --- a/modules/caissedepargne/pages.py +++ b/modules/caissedepargne/pages.py @@ -37,7 +37,7 @@ from weboob.capabilities.bank import ( Account, Investment, Recipient, TransferError, TransferBankError, Transfer, AddRecipientBankError, Loan, ) -from weboob.capabilities.bill import Subscription, Document +from weboob.capabilities.bill import DocumentTypes, Subscription, Document from weboob.tools.capabilities.bank.investments import is_isin_valid from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.tools.capabilities.bank.iban import is_rib_valid, rib2iban, is_iban_valid @@ -762,11 +762,8 @@ class NatixisRedirectPage(LoggedPage, HTMLPage): class MarketPage(LoggedPage, HTMLPage): - def on_load(self): - error = CleanText('//caption[contains(text(),"Erreur")]')(self.doc) - if error: - message = CleanText('//td[contains(@class,"donneeLongIdent")]')(self.doc) - raise BrowserUnavailable(message) + def is_error(self): + return CleanText('//caption[contains(text(),"Erreur")]')(self.doc) def parse_decimal(self, td, percentage=False): value = CleanText('.')(td) @@ -1388,7 +1385,7 @@ class SubscriptionPage(LoggedPage, HTMLPage): obj_label = Format('%s %s', CleanText('./preceding::h3[1]'), CleanText('./span')) obj_date = Date(CleanText('./span'), dayfirst=True) - obj_type = 'other' + obj_type = DocumentTypes.OTHER obj_format = 'pdf' obj_url = Regexp(Link('.'), r'WebForm_PostBackOptions\("(\S*)"') obj_id = Format('%s_%s_%s', Env('sub_id'), CleanText('./span', symbols='/'), Regexp(Field('url'), r'ctl(.*)')) diff --git a/modules/cityscoot/module.py b/modules/cityscoot/module.py index 390cb33bf2763b392ca2c21a8e7988859eeb9a48..1c124175a96ea8b3ed999fa8c502d1b4320c5e24 100644 --- a/modules/cityscoot/module.py +++ b/modules/cityscoot/module.py @@ -20,7 +20,7 @@ from __future__ import unicode_literals -from weboob.capabilities.bill import CapDocument, Subscription, Document, SubscriptionNotFound, DocumentNotFound +from weboob.capabilities.bill import DocumentTypes, CapDocument, Subscription, Document, SubscriptionNotFound, DocumentNotFound from weboob.capabilities.base import find_object, NotAvailable from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword @@ -43,6 +43,8 @@ class CityscootModule(Module, CapDocument): BROWSER = CityscootBrowser + accepted_document_types = (DocumentTypes.BILL,) + def create_default_browser(self): return self.create_browser(self.config['login'].get(), self.config['password'].get()) diff --git a/modules/cityscoot/pages.py b/modules/cityscoot/pages.py index 6be8a99e362b1c2225ed1030a2495e79ac53f798..783dfcfd0de8d4ff662f32c90b0ac9000f06e201 100644 --- a/modules/cityscoot/pages.py +++ b/modules/cityscoot/pages.py @@ -24,7 +24,7 @@ from weboob.browser.pages import HTMLPage, LoggedPage from weboob.browser.elements import ItemElement, TableElement, method from weboob.browser.filters.standard import CleanText, CleanDecimal, Env, Regexp, Format, Date, Async, AsyncLoad from weboob.browser.filters.html import Link -from weboob.capabilities.bill import Bill, Subscription +from weboob.capabilities.bill import DocumentTypes, Bill, Subscription from weboob.capabilities.base import NotAvailable class LoginPage(HTMLPage): @@ -68,7 +68,7 @@ class DocumentsPage(LoggedPage, HTMLPage): obj_date = Async('details') & Date(Regexp(CleanText('.//h3'), r'(\d{2}\/\d{2}\/\d{4})'), dayfirst=True) obj_format = 'html' obj_label = Async('details') & CleanText('.//h3') - obj_type = 'bill' + obj_type = DocumentTypes.BILL obj_price = Async('details') & CleanDecimal('.//td[.="Total"]/following-sibling::td') obj_vat = Async('details') & CleanDecimal('.//td[contains(text(), "TVA")]/following-sibling::td') obj_currency = u'EUR' diff --git a/modules/cmes/browser.py b/modules/cmes/browser.py index 009a636e6ad191c35702c9b1770357e0fdec6be5..25726b1235459da25823d148acfa629ab3e6f8ba 100644 --- a/modules/cmes/browser.py +++ b/modules/cmes/browser.py @@ -23,24 +23,25 @@ from weboob.browser import LoginBrowser, URL, need_login from .pages import ( LoginPage, AccountsPage, FCPEInvestmentPage, - CCBInvestmentPage, HistoryPage, AlertPage, + CCBInvestmentPage, HistoryPage, CustomPage, ) class CmesBrowser(LoginBrowser): BASEURL = 'https://www.cic-epargnesalariale.fr' - login = URL('(?P.*)fr/identification/default.cgi', LoginPage) + login = URL('/espace-client/fr/identification/authentification.html', LoginPage) accounts = URL('(?P.*)fr/espace/devbavoirs.aspx\?mode=net&menu=cpte$', AccountsPage) fcpe_investment = URL(r'/fr/.*GoPositionsParFond.*', r'/fr/espace/devbavoirs.aspx\?.*SituationParFonds.*GoOpenDetailFond.*', r'(?P.*)fr/espace/devbavoirs.aspx\?_tabi=C&a_mode=net&a_menu=cpte&_pid=SituationGlobale&_fid=GoPositionsParFond', r'(?P.*)fr/espace/devbavoirs.aspx\?_tabi=C&a_mode=net&a_menu=cpte&_pid=SituationParFonds.*', FCPEInvestmentPage) - ccb_investment = URL('(?P.*)fr/.*LstSuppCCB.*', CCBInvestmentPage) + ccb_investment = URL('(?P.*)fr/espace/devbavoirs.aspx\?_tabi=C&a_mode=net&a_menu=cpte&_pid=SituationGlobale&_fid=GoCCB', CCBInvestmentPage) history = URL('(?P.*)fr/espace/devbavoirs.aspx\?mode=net&menu=cpte&page=operations', '(?P.*)fr/.*GoOperationsTraitees', '(?P.*)fr/.*GoOperationDetails', HistoryPage) - supp_alert = URL(r'(?P.*)fr/espace/LstSuppAlerte.asp', AlertPage) + custom_page = URL('/fr/espace/personnel/index.html', CustomPage) + def __init__(self, website, username, password, subsite="", *args, **kwargs): super(LoginBrowser, self).__init__(*args, **kwargs) @@ -50,19 +51,26 @@ class CmesBrowser(LoginBrowser): self.subsite = subsite def do_login(self): - self.login.go(subsite=self.subsite).login(self.username, self.password) + self.login.go() + self.page.login(self.username, self.password) if self.login.is_here(): raise BrowserIncorrectPassword @need_login def iter_accounts(self): - return self.accounts.stay_or_go(subsite=self.subsite).iter_accounts() + self.accounts.go(subsite=self.subsite) + + if self.custom_page.is_here(): + # it can be redirected by accounts page, return on accounts page should be enough + self.accounts.go(subsite=self.subsite) + + return self.page.iter_accounts() @need_login def iter_investment(self, account): - fcpe_link = self.accounts.stay_or_go(subsite=self.subsite).get_investment_link() - ccb_link = self.supp_alert.go(subsite=self.subsite).get_pocket_link() + fcpe_link = self.accounts.go(subsite=self.subsite).get_investment_link() + ccb_link = self.accounts.go(subsite=self.subsite).get_pocket_link() if fcpe_link or ccb_link: return self._iter_investment(fcpe_link, ccb_link) @@ -80,18 +88,17 @@ class CmesBrowser(LoginBrowser): @need_login def iter_pocket(self, account): - self.accounts.stay_or_go(subsite=self.subsite) + self.accounts.go(subsite=self.subsite) for inv in self.iter_investment(account): + if inv._pocket_url: + # Only FCPE investments have pocket link: + self.location(inv._pocket_url) + for pocket in self.page.iter_pocket(inv=inv): + yield pocket - self.location(inv._pocket_url) - for pocket in self.page.iter_pocket(inv=inv): - yield pocket - - # don't know if it is still releavent - # need ccb case - ccb_link = self.supp_alert.stay_or_go(subsite=self.subsite).get_pocket_link() + ccb_link = self.accounts.go(subsite=self.subsite).get_pocket_link() if ccb_link: - for inv in self.location(ccb_link).page.iter_pocket(): + for inv in self.location(ccb_link).page.iter_investment(): for poc in self.page.iter_pocket(inv=inv): yield poc diff --git a/modules/cmes/pages.py b/modules/cmes/pages.py index 75233e78d5283b030bda026a61fb9a0006ae58bc..25833ec83b5ba1ee974f35b54d24c779deec418b 100644 --- a/modules/cmes/pages.py +++ b/modules/cmes/pages.py @@ -55,6 +55,9 @@ class AccountsPage(LoggedPage, HTMLPage): def get_investment_link(self): return Link('//a[contains(text(), "Par fonds") or contains(@href,"GoPositionsParFond")]', default=None)(self.doc) + def get_pocket_link(self): + return Link('//a[contains(@href, "CCB")]', default=None)(self.doc) + @method class iter_accounts(ListElement): class item(ItemElement): @@ -80,11 +83,6 @@ class AccountsPage(LoggedPage, HTMLPage): return Currency().filter(CleanText('//table[@class="fiche"]//td/small')(self)) -class AlertPage(LoggedPage, HTMLPage): - def get_pocket_link(self): - return Link('//a[contains(@href, "CCB")]', default=None)(self.doc) - - class FCPEInvestmentPage(LoggedPage, HTMLPage): @method class iter_investment(TableElement): @@ -147,14 +145,13 @@ class CCBInvestmentPage(LoggedPage, HTMLPage): continue inv = Investment() - inv.label = CleanText(el.xpath('./td[has-class("g")]'))(self.doc) + inv.label = CleanText(el.xpath('./td[has-class("i g")]'))(self.doc) inv.valuation = MyDecimal(el.xpath('./td[last()]'))(self.doc) - i = 1 - while i < rowspan: + inv._pocket_url = None + for i in range(1, rowspan): # valuation is not directly written on website, but it's separated by pocket, so we compute it here, # and is also written in footer so it's sum of all valuation, not just one inv.valuation += MyDecimal(el_list[index+i].xpath('./td[last()]'))(self.doc) - i += 1 yield inv @@ -265,3 +262,7 @@ class HistoryPage(LoggedPage, HTMLPage): self.env['label'] = label self.env['amount'] = amount self.env['investments'] = list(page.get_investments()) + + +class CustomPage(LoggedPage, HTMLPage): + pass diff --git a/modules/cragr/module.py b/modules/cragr/module.py index 9e34a58ca1060ca14d8cdd05cac9ee07830355d2..959309647fb4e2aec65a499c91cc32ba80d63e27 100644 --- a/modules/cragr/module.py +++ b/modules/cragr/module.py @@ -28,7 +28,7 @@ from weboob.capabilities.profile import CapProfile from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword, Value -from .web.browser import Cragr +from .proxy_browser import ProxyBrowser __all__ = ['CragrModule'] @@ -82,10 +82,12 @@ class CragrModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapContact 'm.ca-valdefrance.fr': u'Val de France', 'm.lefil.com': u'Pyrénées Gascogne', }.items())]) + CONFIG = BackendConfig(Value('website', label=u'Région', choices=website_choices), ValueBackendPassword('login', label=u'N° de compte', masked=False), ValueBackendPassword('password', label=u'Code personnel', regexp=r'\d{6}')) - BROWSER = Cragr + + BROWSER = ProxyBrowser COMPAT_DOMAINS = { 'm.lefil.com': 'm.ca-pyrenees-gascogne.fr', diff --git a/modules/cragr/web/browser.py b/modules/cragr/web/browser.py index a73de43f5c04b0886361794b3de37a97deed6933..838af343238a5b4cb9313145c21c1e119fc862c2 100644 --- a/modules/cragr/web/browser.py +++ b/modules/cragr/web/browser.py @@ -28,9 +28,10 @@ from weboob.capabilities.bank import ( Account, AddRecipientStep, AddRecipientBankError, RecipientInvalidLabel, Recipient, AccountNotFound, ) -from weboob.capabilities.base import NotLoaded, find_object, empty +from weboob.capabilities.base import find_object, empty from weboob.capabilities.profile import ProfileMissing from weboob.browser import LoginBrowser, URL, need_login, StatesMixin +from weboob.browser.switch import SiteSwitch from weboob.browser.pages import FormNotFound from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable from weboob.tools.date import ChaoticDateGuesser, LinearDateGuesser @@ -42,7 +43,7 @@ from weboob.tools.capabilities.bank.iban import is_iban_valid from weboob.tools.capabilities.bank.investments import create_french_liquidity from .pages import ( - HomePage, LoginPage, LoginErrorPage, AccountsPage, + HomePage, NewWebsitePage, LoginPage, LoginErrorPage, AccountsPage, SavingsPage, TransactionsPage, AdvisorPage, UselessPage, CardsPage, LifeInsurancePage, MarketPage, LoansPage, PerimeterPage, ChgPerimeterPage, MarketHomePage, FirstVisitPage, BGPIPage, @@ -61,6 +62,7 @@ class WebsiteNotSupported(Exception): class Cragr(LoginBrowser, StatesMixin): home_page = URL('/$', '/particuliers.html', 'https://www.*.fr/Vitrine/jsp/CMDS/b.js', HomePage) + new_website = URL(r'https://www.credit-agricole.fr/.*', NewWebsitePage) login_page = URL(r'/stb/entreeBam$', r'/stb/entreeBam\?.*typeAuthentification=CLIC_ALLER.*', LoginPage) @@ -170,6 +172,10 @@ class Cragr(LoginBrowser, StatesMixin): if not self.home_page.is_here(): self.home_page.go() + if self.new_website.is_here(): + self.logger.warning('This connection uses the new API website') + raise SiteSwitch('api') + if self.new_login: self.page.go_to_auth() parsed = urlparse(self.url) @@ -269,7 +275,7 @@ class Cragr(LoginBrowser, StatesMixin): if (self.page and not self.page.get_current()) or self.current_perimeter != perimeter: self.go_perimeter(perimeter) for account in self.get_list(): - if not account in l: + if account not in l: l.append(account) else: l = self.get_list() @@ -380,7 +386,7 @@ class Cragr(LoginBrowser, StatesMixin): # update life insurances with unavailable balance for account in accounts_list: - if account.type == Account.TYPE_LIFE_INSURANCE and not account.balance: + if account.type == Account.TYPE_LIFE_INSURANCE and not account.balance and self.bgpi.is_here(): if account._perimeter != self.current_perimeter: self.go_perimeter(account._perimeter) new_location = self.moveto_insurance_website(account) @@ -388,7 +394,7 @@ class Cragr(LoginBrowser, StatesMixin): account.label, account.balance = self.page.get_li_details(account.id) # be sure that we send accounts with balance - return [acc for acc in accounts_list if acc.balance is not NotLoaded] + return [acc for acc in accounts_list if not empty(acc.balance)] @need_login def market_accounts_matching(self, accounts_list, market_accounts_list): @@ -555,7 +561,7 @@ class Cragr(LoginBrowser, StatesMixin): if m: url = m.group(1) else: - self.logger.warn('Unable to go to market website') + self.logger.warning('Unable to go to market website') raise WebsiteNotSupported() self.open(url) diff --git a/modules/cragr/web/pages.py b/modules/cragr/web/pages.py index 0a8d8b63922583fb4207d9a94dc4d76e88400376..14824a4719481f1ca971f959b32e618e84e7d91a 100644 --- a/modules/cragr/web/pages.py +++ b/modules/cragr/web/pages.py @@ -145,6 +145,10 @@ class HomePage(BasePage): return Regexp(CleanText('.'), r'public_key.+?(\w+)')(self.doc) +class NewWebsitePage(BasePage): + pass + + class LoginPage(BasePage): def on_load(self): if self.doc.xpath('//font[@class="taille2"]'): diff --git a/modules/creditdunord/pages.py b/modules/creditdunord/pages.py index 129e6d5cbad3e1ecfd080ce170081506e1ef7a8d..f3f562edbc0afdc95b9ad3b7da34dc577d1ea302 100755 --- a/modules/creditdunord/pages.py +++ b/modules/creditdunord/pages.py @@ -174,7 +174,7 @@ class LabelsPage(LoggedPage, JsonPage): def get_labels(self): synthesis_labels = ["Synthèse"] loan_labels = ["Crédits en cours", "Crédits perso et immo", "Crédits"] - for element in Dict('donnees/0/submenu')(self.doc): + for element in Dict('donnees/1/submenu')(self.doc): if CleanText(Dict('label'))(element) in synthesis_labels: synthesis_label = CleanText(Dict('link'))(element).split("/")[-1] if CleanText(Dict('label'))(element) in loan_labels: diff --git a/modules/creditmutuel/module.py b/modules/creditmutuel/module.py index faf5c63717724558ca79efb9274becb35cca225f..9595aad89466e4b8d9dbf52be96c94511d1cafa3 100644 --- a/modules/creditmutuel/module.py +++ b/modules/creditmutuel/module.py @@ -30,7 +30,7 @@ from weboob.capabilities.contact import CapContact from weboob.capabilities.profile import CapProfile from weboob.capabilities.bill import ( CapDocument, Subscription, SubscriptionNotFound, - Document, DocumentNotFound, + Document, DocumentNotFound, DocumentTypes, ) from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword @@ -56,6 +56,8 @@ class CreditMutuelModule( ValueBackendPassword('password', label='Mot de passe')) BROWSER = CreditMutuelBrowser + accepted_document_types = (DocumentTypes.OTHER,) + def create_default_browser(self): return self.create_browser(self.config['login'].get(), self.config['password'].get()) diff --git a/modules/creditmutuel/pages.py b/modules/creditmutuel/pages.py index 153fea68ec58455c03169d59dd744fc9382d277d..9a161a6b5023a728e44e51fa122f51240a561155 100644 --- a/modules/creditmutuel/pages.py +++ b/modules/creditmutuel/pages.py @@ -48,7 +48,7 @@ from weboob.capabilities.contact import Advisor from weboob.capabilities.profile import Profile from weboob.tools.capabilities.bank.iban import is_iban_valid from weboob.tools.capabilities.bank.transactions import FrenchTransaction -from weboob.capabilities.bill import Subscription, Document +from weboob.capabilities.bill import DocumentTypes, Subscription, Document from weboob.tools.compat import urlparse, parse_qs, urljoin, range, unicode from weboob.tools.date import parse_french_date from weboob.tools.value import Value @@ -302,10 +302,8 @@ class item_account_generic(ItemElement): card_xpath = multiple_cards_xpath + ' | ' + single_card_xpath for elem in page.doc.xpath(card_xpath): card_id = Regexp(CleanText('.', symbols=' '), r'([\dx]{16})')(elem) - if card_id in self.page.browser.unavailablecards or card_id in [d.id for d in self.page.browser.cards_list]: - raise SkipItem() - - if any(card_id in a.id for a in page.browser.accounts_list): + is_in_accounts = any(card_id in a.id for a in page.browser.accounts_list) + if card_id in self.page.browser.unavailablecards or is_in_accounts: continue card = Account() @@ -772,7 +770,7 @@ class ComingPage(OperationsPage, LoggedPage): class CardPage(OperationsPage, LoggedPage): def select_card(self, card_number): for option in self.doc.xpath('//select[@name="Data_SelectedCardItemKey"]/option'): - card_id = Regexp(CleanText('.', symbols=' '), r'([\dx]+)')(option) + card_id = Regexp(CleanText('.', symbols=' '), r'(\d+x+\d+)')(option) if card_id != card_number: continue if Attr('.', 'selected', default=None)(option): @@ -1826,7 +1824,7 @@ class SubscriptionPage(LoggedPage, HTMLPage): obj_label = Format('%s %s', CleanText(TableCell('url')), CleanText(TableCell('date'))) obj_date = Date(CleanText(TableCell('date')), dayfirst=True) obj_format = 'pdf' - obj_type = 'other' + obj_type = DocumentTypes.OTHER def obj_url(self): return urljoin(self.page.url, '/fr/banque/%s' % Link('./a')(TableCell('url')(self)[0])) diff --git a/modules/edf/module.py b/modules/edf/module.py index 4043c275176c6255974f443b386aa7eb55b39426..c46305eacb8dcf5f564d2da4e8f3f5c29965d969 100644 --- a/modules/edf/module.py +++ b/modules/edf/module.py @@ -18,7 +18,7 @@ # along with weboob. If not, see . -from weboob.capabilities.bill import CapDocument, Subscription, Document, SubscriptionNotFound, DocumentNotFound +from weboob.capabilities.bill import DocumentTypes, CapDocument, Subscription, Document, SubscriptionNotFound, DocumentNotFound from weboob.capabilities.base import find_object from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword, Value @@ -44,6 +44,8 @@ class EdfModule(Module, CapDocument, CapProfile): choices={'par': 'Particulier', 'pro': 'Entreprise'}), Value('captcha_response', label='Reponse Captcha', required=False, default='')) + accepted_document_types = (DocumentTypes.BILL,) + def create_default_browser(self): browsers = {'pro': EdfproBrowser, 'par': EdfBrowser} self.BROWSER = browsers[self.config['website'].get()] diff --git a/modules/edf/par/browser.py b/modules/edf/par/browser.py index 7a5dde931d7eaad9b62acc4d7555f00e5b557bd7..d0c4d343d8e93aa3b2edf1f01d084850755119c5 100644 --- a/modules/edf/par/browser.py +++ b/modules/edf/par/browser.py @@ -23,6 +23,7 @@ from time import time from weboob.browser import LoginBrowser, URL, need_login from weboob.browser.exceptions import ClientError from weboob.exceptions import BrowserIncorrectPassword, NocaptchaQuestion +from weboob.tools.decorators import retry from weboob.tools.json import json from .pages import ( HomePage, AuthenticatePage, AuthorizePage, CheckAuthenticatePage, ProfilPage, @@ -30,6 +31,10 @@ from .pages import ( ) +class BrokenPageError(Exception): + pass + + class EdfBrowser(LoginBrowser): BASEURL = 'https://particulier.edf.fr' @@ -111,6 +116,7 @@ class EdfBrowser(LoginBrowser): return self.bills.go().iter_bills(subid=subscription.id) + @retry(BrokenPageError, tries=2, delay=4) @need_login def download_document(self, document): token = self.get_csrf_token() @@ -127,9 +133,15 @@ class EdfBrowser(LoginBrowser): 'parNumber': document._par_number })).get_bills_informations() - return self.bill_download.go(csrf_token=token, dn='FACTURE', pn=document._par_number, - di=document._doc_number, bn=bills_informations.get('bpNumber'), - an=bills_informations.get('numAcc')).content + self.bill_download.go(csrf_token=token, dn='FACTURE', pn=document._par_number, + di=document._doc_number, bn=bills_informations.get('bpNumber'), + an=bills_informations.get('numAcc')) + + # sometimes we land to another page that tell us, this document doesn't exist, but just sometimes... + # make sure this page is the right one to avoid return a html page as document + if not self.bill_download.is_here(): + raise BrokenPageError() + return self.page.content @need_login def get_profile(self): diff --git a/modules/edf/par/pages.py b/modules/edf/par/pages.py index 3e7a7a56f5d8afba2ca7f1aca919c555047af109..14ad2f229fbe02176d9b91c6437ae5e17d257342 100644 --- a/modules/edf/par/pages.py +++ b/modules/edf/par/pages.py @@ -27,7 +27,7 @@ from weboob.browser.pages import LoggedPage, JsonPage, HTMLPage, RawPage from weboob.browser.filters.standard import Env, Format, Date, Eval from weboob.browser.elements import ItemElement, DictElement, method from weboob.browser.filters.json import Dict -from weboob.capabilities.bill import Bill, Subscription +from weboob.capabilities.bill import DocumentTypes, Bill, Subscription from weboob.capabilities.base import NotAvailable from weboob.capabilities.profile import Profile @@ -96,7 +96,7 @@ class DocumentsPage(LoggedPage, JsonPage): .strftime('%Y-%m-%d'), Dict('creationDate'))) obj_format = 'pdf' obj_label = Format('Facture %s', Dict('documentNumber')) - obj_type = 'bill' + obj_type = DocumentTypes.BILL obj_price = Env('price') obj_currency = 'EUR' obj_vat = NotAvailable diff --git a/modules/edf/pro/pages.py b/modules/edf/pro/pages.py index 029c78e35de75bad8f3214b94e86ceabdae0fe3c..e0be896da63be3cb61d7873860d9020ce0fd008e 100644 --- a/modules/edf/pro/pages.py +++ b/modules/edf/pro/pages.py @@ -26,7 +26,7 @@ from weboob.browser.elements import DictElement, ItemElement, method from weboob.browser.filters.standard import CleanDecimal, CleanText from weboob.browser.filters.html import CleanHTML from weboob.browser.filters.json import Dict -from weboob.capabilities.bill import Subscription, Bill +from weboob.capabilities.bill import DocumentTypes, Subscription, Bill from weboob.exceptions import ActionNeeded from weboob.capabilities.profile import Profile @@ -98,7 +98,7 @@ class DocumentsPage(LoggedPage, JsonPage): doc.date = date.fromtimestamp(int(document['dateEmission'] / 1000)) doc.format = 'PDF' doc.label = 'Facture %s' % document['numFactureLabel'] - doc.type = 'bill' + doc.type = DocumentTypes.BILL doc.price = CleanDecimal().filter(document['montantTTC']) doc.currency = '€' doc._account_billing = document['compteFacturation'] diff --git a/modules/ekwateur/module.py b/modules/ekwateur/module.py index 9b20389ec83099e5165586c02a9771d5b6a6d20a..88c393098d7c3831f14d6da088c654c0fa575a9f 100644 --- a/modules/ekwateur/module.py +++ b/modules/ekwateur/module.py @@ -24,7 +24,7 @@ from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import Value, ValueBackendPassword from weboob.capabilities.base import find_object from weboob.capabilities.bill import ( - CapDocument, Document, DocumentNotFound, Subscription + CapDocument, Document, DocumentNotFound, Subscription, DocumentTypes, ) from .browser import EkwateurBrowser @@ -48,6 +48,8 @@ class EkwateurModule(Module, CapDocument): ValueBackendPassword('password', help='Password'), ) + accepted_document_types = (DocumentTypes.BILL,) + def create_default_browser(self): return self.create_browser(self.config['login'].get(), self.config['password'].get()) diff --git a/modules/ekwateur/pages.py b/modules/ekwateur/pages.py index 426ea3b13a49fb080089d23e75d4a488daea692f..5d79b4a221716d826860be235200e6632f22f76c 100644 --- a/modules/ekwateur/pages.py +++ b/modules/ekwateur/pages.py @@ -30,7 +30,7 @@ from weboob.browser.filters.standard import ( ) from weboob.browser.filters.html import AbsoluteLink, Attr, Link, XPath from weboob.capabilities.base import NotAvailable -from weboob.capabilities.bill import Subscription, Bill, Document +from weboob.capabilities.bill import DocumentTypes, Subscription, Bill, Document class LoginPage(HTMLPage): @@ -95,7 +95,7 @@ class BillsPage(EkwateurPage): CleanText(TableCell('amount')), CleanText(TableCell('date')) ) - obj_type = 'bill' + obj_type = DocumentTypes.BILL obj_price = CleanDecimal(TableCell('amount'), replace_dots=True) obj_currency = Currency(TableCell('amount')) obj_duedate = Date( diff --git a/modules/freemobile/module.py b/modules/freemobile/module.py index 8a563a16d16040a0572a25dc631af90702dd162d..b7dcc573a811990e7e91e9e13e6617a7ab3a938d 100644 --- a/modules/freemobile/module.py +++ b/modules/freemobile/module.py @@ -17,7 +17,7 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . -from weboob.capabilities.bill import CapDocument, Subscription, Bill, SubscriptionNotFound, DocumentNotFound +from weboob.capabilities.bill import DocumentTypes, CapDocument, Subscription, Bill, SubscriptionNotFound, DocumentNotFound from weboob.capabilities.profile import CapProfile from weboob.capabilities.messages import CantSendMessage, CapMessages, CapMessagesPost from weboob.capabilities.base import find_object @@ -46,6 +46,8 @@ class FreeMobileModule(Module, CapDocument, CapMessages, CapMessagesPost, CapPro ) BROWSER = Freemobile + accepted_document_types = (DocumentTypes.BILL,) + def create_default_browser(self): return self.create_browser(self.config['login'].get(), self.config['password'].get()) diff --git a/modules/freemobile/pages/history.py b/modules/freemobile/pages/history.py index b94d6ddc8e0c8c418434b8554153797c3ce40e7a..c37bd267f518f699af4af76543e49e0e8a732e56 100644 --- a/modules/freemobile/pages/history.py +++ b/modules/freemobile/pages/history.py @@ -27,7 +27,7 @@ from weboob.browser.elements import ItemElement, ListElement, method from weboob.browser.filters.standard import Date, CleanText, Filter,\ CleanDecimal, Currency, Regexp, Field, DateTime, Format, Env from weboob.browser.filters.html import AbsoluteLink, Attr -from weboob.capabilities.bill import Detail, Bill +from weboob.capabilities.bill import DocumentTypes, Detail, Bill from weboob.capabilities.base import NotAvailable from weboob.exceptions import ParseError from weboob.tools.compat import unicode @@ -128,7 +128,7 @@ class DetailsPage(LoggedPage, BadUTF8Page): obj_id = Format('%s.%s', Env('subid'), Field('_localid')) obj_date = FormatDate(Field('label')) obj_format = u"pdf" - obj_type = u"bill" + obj_type = DocumentTypes.BILL obj_price = CleanDecimal('div[@class="montant"]', default=Decimal(0), replace_dots=False) obj_currency = Currency('div[@class="montant"]') diff --git a/modules/infomaniak/module.py b/modules/infomaniak/module.py index 4a1c504f2b662dcdacba887ba67fdd419b8d59c5..860fe1a52f8e5b5f1c6764c5b6749e092c67c4cb 100644 --- a/modules/infomaniak/module.py +++ b/modules/infomaniak/module.py @@ -20,7 +20,7 @@ from __future__ import unicode_literals -from weboob.capabilities.bill import CapDocument, Subscription, Document, SubscriptionNotFound, DocumentNotFound +from weboob.capabilities.bill import DocumentTypes, CapDocument, Subscription, Document, SubscriptionNotFound, DocumentNotFound from weboob.capabilities.base import find_object from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword @@ -43,6 +43,8 @@ class InfomaniakModule(Module, CapDocument): BROWSER = InfomaniakBrowser + accepted_document_types = (DocumentTypes.BILL,) + def create_default_browser(self): return self.create_browser(self.config['login'].get(), self.config['password'].get()) diff --git a/modules/infomaniak/pages.py b/modules/infomaniak/pages.py index 27562c78758007d558c20185ccd4d623650d3590..c1295b5ee700095d10f574b448c86a01cade8ae1 100644 --- a/modules/infomaniak/pages.py +++ b/modules/infomaniak/pages.py @@ -27,7 +27,7 @@ from weboob.browser.filters.standard import ( CleanDecimal, Env, Regexp, Format, Currency, Field, Eval, ) from weboob.browser.filters.json import Dict -from weboob.capabilities.bill import Bill, Subscription +from weboob.capabilities.bill import DocumentTypes, Bill, Subscription from weboob.tools.compat import urljoin @@ -65,5 +65,5 @@ class DocumentsPage(LoggedPage, JsonPage): obj_label = Format('Facture %s', _num) obj_price = CleanDecimal(Dict('fMontant')) obj_currency = Currency(Dict('sMontant')) - obj_type = 'bill' + obj_type = DocumentTypes.BILL obj_format = 'pdf' diff --git a/modules/ing/module.py b/modules/ing/module.py index 6ccab17366f9dc5c5e983f18ce6d4521eced9ad4..71a80cc3d7cb0198335b6893d60982720efa6df2 100644 --- a/modules/ing/module.py +++ b/modules/ing/module.py @@ -24,7 +24,7 @@ from decimal import Decimal from weboob.capabilities.bank import CapBankWealth, CapBankTransfer, Account, AccountNotFound, RecipientNotFound from weboob.capabilities.bill import ( CapDocument, Bill, Subscription, - SubscriptionNotFound, DocumentNotFound + SubscriptionNotFound, DocumentNotFound, DocumentTypes, ) from weboob.capabilities.profile import CapProfile from weboob.capabilities.base import find_object, NotAvailable @@ -56,6 +56,8 @@ class INGModule(Module, CapBankWealth, CapBankTransfer, CapDocument, CapProfile) ) BROWSER = IngBrowser + accepted_document_types = (DocumentTypes.STATEMENT,) + def create_default_browser(self): return self.create_browser(self.config['login'].get(), self.config['password'].get(), diff --git a/modules/ing/pages/bills.py b/modules/ing/pages/bills.py index 853d795fd9df735b25a1815f3f9152e0942bc68f..0051f442fa54826d69107c362214d3b74979d4cc 100644 --- a/modules/ing/pages/bills.py +++ b/modules/ing/pages/bills.py @@ -17,7 +17,7 @@ # You should have received a copy of the GNU Affero General Public License # along with weboob. If not, see . -from weboob.capabilities.bill import Bill, Subscription +from weboob.capabilities.bill import DocumentTypes, Bill, Subscription from weboob.browser.pages import HTMLPage, LoggedPage, pagination from weboob.browser.filters.standard import Filter, CleanText, Format, Field, Env, Date from weboob.browser.filters.html import Attr @@ -87,5 +87,5 @@ class BillsPage(LoggedPage, HTMLPage): # Force first day of month as label is in form "janvier 2016" obj_date = Format("1 %s", Field('label')) & Date(parse_func=parse_french_date) obj_format = u"pdf" - obj_type = u"bill" + obj_type = DocumentTypes.STATEMENT obj__localid = Attr('a[2]', 'onclick') diff --git a/modules/ing/pages/login.py b/modules/ing/pages/login.py index 90694459a2626fdcc9346e4073f0344806ec5311..47de0d525b70b1a77b45677185b294f8a3c69d8e 100644 --- a/modules/ing/pages/login.py +++ b/modules/ing/pages/login.py @@ -84,7 +84,6 @@ class INGVirtKeyboard(VirtKeyboard): for i, font in enumerate(elems): if Attr('.', 'class')(font) == "vide": temppasswd += password[i] - self.page.browser.logger.debug('We are looking for : ' + temppasswd) coordinates = self.get_string_code(temppasswd) self.page.browser.logger.debug("Coordonates: " + coordinates) return coordinates diff --git a/modules/lcl/module.py b/modules/lcl/module.py index 897a13967b0a5c31c4603db98abe886575cb9572..2cbd0996927565c3f44285c5ebe2cd7da4cf7228 100644 --- a/modules/lcl/module.py +++ b/modules/lcl/module.py @@ -147,6 +147,9 @@ class LCLModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapContact, def transfer_check_label(self, old, new): old = re.sub(r"[/<\?='!\+:]", '', old).strip() old = old.encode('latin-1', errors='replace').decode('latin-1') + # if no reason given, the site changes the label + if not old and ("INTERNET-FAVEUR" in new): + return True return super(LCLModule, self).transfer_check_label(old, new) @only_for_websites('par', 'elcl', 'pro') diff --git a/modules/materielnet/module.py b/modules/materielnet/module.py index 7f96ceceb172646861d396d858780d793b06f312..3a8f980a92949cdbe5ccfff7c8b809ad055b2b64 100644 --- a/modules/materielnet/module.py +++ b/modules/materielnet/module.py @@ -18,7 +18,7 @@ # along with weboob. If not, see . -from weboob.capabilities.bill import CapDocument, Subscription, Document, SubscriptionNotFound, DocumentNotFound +from weboob.capabilities.bill import DocumentTypes, CapDocument, Subscription, Document, SubscriptionNotFound, DocumentNotFound from weboob.capabilities.base import find_object, NotAvailable from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword @@ -41,6 +41,8 @@ class MaterielnetModule(Module, CapDocument): BROWSER = MaterielnetBrowser + accepted_document_types = (DocumentTypes.BILL,) + def create_default_browser(self): return self.create_browser(self.config['login'].get(), self.config['password'].get()) diff --git a/modules/materielnet/pages.py b/modules/materielnet/pages.py index ace5a75527ad1405914cbcb51b162e6d872b04d3..4a061e93d55fd6ab31236c134a75a71c4454a92b 100644 --- a/modules/materielnet/pages.py +++ b/modules/materielnet/pages.py @@ -25,7 +25,7 @@ from weboob.browser.pages import HTMLPage, LoggedPage, PartialHTMLPage from weboob.browser.filters.standard import CleanText, CleanDecimal, Env, Format, Date, Async, Filter, Regexp, Field from weboob.browser.elements import ListElement, ItemElement, method from weboob.browser.filters.html import Attr, Link -from weboob.capabilities.bill import Bill, Subscription +from weboob.capabilities.bill import DocumentTypes, Bill, Subscription from weboob.capabilities.base import NotAvailable from weboob.exceptions import BrowserIncorrectPassword @@ -94,7 +94,7 @@ class DocumentsPage(LoggedPage, PartialHTMLPage): obj_date = Date(CleanText('./div[contains(@class, "date")]'), dayfirst=True) obj_format = 'pdf' obj_label = Regexp(CleanText('./div[contains(@class, "ref")]'), r' (.*)') - obj_type = 'bill' + obj_type = DocumentTypes.BILL obj_price = CleanDecimal(CleanText('./div[contains(@class, "price")]'), replace_dots=(' ', '€')) obj_currency = 'EUR' diff --git a/modules/oney/pages.py b/modules/oney/pages.py index 700fda7a7c11e8d9c35ed9253980e34d97c7eeb4..7ab2d691bcc6fb60f90262af46074ae87ec25243 100644 --- a/modules/oney/pages.py +++ b/modules/oney/pages.py @@ -250,19 +250,15 @@ class CreditAccountPage(LoggedPage, HTMLPage): class get_account(ItemElement): klass = Account - obj_type = Account.TYPE_REVOLVING_CREDIT - + obj_type = Account.TYPE_CHECKING obj__site = 'other' - - def obj_label(self): - return self.page.browser.card_name - + obj_balance = 0 obj_id = CleanText('//tr[td[text()="Mon numéro de compte"]]/td[@class="droite"]', replace=[(' ', '')]) - obj_balance = CleanDecimal('//div[@id="mod-paiementcomptant"]//tr[td[starts-with(normalize-space(text()),"Montant disponible")]]/td[@class="droite"]') obj_coming = CleanDecimal('//div[@id="mod-paiementcomptant"]//tr[td[contains(text(),"débité le")]]/td[@class="droite"]', sign=lambda _: -1, default=0) obj_currency = Currency('//div[@id="mod-paiementcomptant"]//tr[td[starts-with(normalize-space(text()),"Montant disponible")]]/td[@class="droite"]') - # what's the balance anyway? - # there's "Paiements au comptant" and sometimes "Retraits d'argent au comptant" + + def obj_label(self): + return self.page.browser.card_name class CreditHistory(LoggedPage, XLSPage): diff --git a/modules/onlinenet/module.py b/modules/onlinenet/module.py index c2b0d4c0e167cb6fb996a5a7bdaa8307c2d4b7b2..403ba10ba0be4cebbf3cd4087ed3aa1c586c8511 100644 --- a/modules/onlinenet/module.py +++ b/modules/onlinenet/module.py @@ -18,7 +18,7 @@ # along with weboob. If not, see . -from weboob.capabilities.bill import CapDocument, Subscription, Document, SubscriptionNotFound, DocumentNotFound +from weboob.capabilities.bill import DocumentTypes, CapDocument, Subscription, Document, SubscriptionNotFound, DocumentNotFound from weboob.capabilities.base import find_object, NotAvailable from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword, Value @@ -41,6 +41,8 @@ class OnlinenetModule(Module, CapDocument): BROWSER = OnlinenetBrowser + accepted_document_types = (DocumentTypes.BILL, DocumentTypes.OTHER,) + def create_default_browser(self): return self.create_browser(self.config['login'].get(), self.config['password'].get()) diff --git a/modules/onlinenet/pages.py b/modules/onlinenet/pages.py index 89b87ca9d995182dcb54b6aa21dcad512b5bd6b5..f9f66817576f5d7aeb01e2b78201ba9e4d068a80 100644 --- a/modules/onlinenet/pages.py +++ b/modules/onlinenet/pages.py @@ -24,7 +24,7 @@ from weboob.browser.pages import HTMLPage, LoggedPage from weboob.browser.filters.standard import CleanText, CleanDecimal, Env, Format, Date from weboob.browser.filters.html import Attr, TableCell from weboob.browser.elements import ListElement, ItemElement, TableElement, method -from weboob.capabilities.bill import Bill, Document, Subscription +from weboob.capabilities.bill import DocumentTypes, Bill, Document, Subscription from weboob.capabilities.base import NotAvailable @@ -68,7 +68,7 @@ class DocumentsPage(LoggedPage, HTMLPage): obj_date = Date(CleanText(TableCell('date'))) obj_format = u"pdf" obj_label = Format('Facture %s', CleanDecimal(TableCell('id'))) - obj_type = u"bill" + obj_type = DocumentTypes.BILL obj_price = CleanDecimal(TableCell('price')) obj_currency = u'EUR' @@ -90,7 +90,7 @@ class DocumentsPage(LoggedPage, HTMLPage): obj__url = Attr('.', 'href') obj_format = u"pdf" obj_label = CleanText('.') - obj_type = u"other" + obj_type = DocumentTypes.OTHER def parse(self, el): self.env['username'] = self.page.browser.username diff --git a/modules/orange/module.py b/modules/orange/module.py index be0647335726e144d8e068eef5f2e94914c96e32..863578f801ed5f09bdef692b057ad0b492b9c37a 100644 --- a/modules/orange/module.py +++ b/modules/orange/module.py @@ -18,7 +18,7 @@ # along with weboob. If not, see . -from weboob.capabilities.bill import CapDocument, Subscription, Document, SubscriptionNotFound, DocumentNotFound +from weboob.capabilities.bill import DocumentTypes, CapDocument, Subscription, Document, SubscriptionNotFound, DocumentNotFound from weboob.capabilities.base import find_object, NotAvailable from weboob.capabilities.account import CapAccount from weboob.capabilities.profile import CapProfile @@ -46,6 +46,8 @@ class OrangeModule(Module, CapAccount, CapDocument, CapProfile): self._browsers = dict() super(OrangeModule, self).__init__(*args, **kwargs) + accepted_document_types = (DocumentTypes.BILL,) + def create_default_browser(self): return self.create_browser(self.config['login'].get(), self.config['password'].get()) diff --git a/modules/orange/pages/bills.py b/modules/orange/pages/bills.py index c3791ac7d7f9cd65603f5e4d60b5b84bc8c5dc9e..5d875baede8a181b28759f5c65a285bf05ac556f 100644 --- a/modules/orange/pages/bills.py +++ b/modules/orange/pages/bills.py @@ -33,7 +33,7 @@ from weboob.browser.filters.html import Link, TableCell from weboob.browser.filters.javascript import JSValue from weboob.browser.filters.json import Dict from weboob.capabilities.base import NotAvailable -from weboob.capabilities.bill import Bill +from weboob.capabilities.bill import DocumentTypes, Bill from weboob.tools.date import parse_french_date from weboob.tools.compat import urlencode @@ -82,7 +82,7 @@ class BillsPage(LoggedPage, HTMLPage): class item(ItemElement): klass = Bill - obj_type = u"bill" + obj_type = DocumentTypes.BILL obj_format = u"pdf" # TableCell('date') can have other info like: 'duplicata' diff --git a/modules/ovh/module.py b/modules/ovh/module.py index e395489723bf6a89f0e659e71f0190e9ac246251..3ccdeba32d3ee5df8b7dfdcddf9dfaa5138c1c61 100644 --- a/modules/ovh/module.py +++ b/modules/ovh/module.py @@ -18,7 +18,7 @@ # along with weboob. If not, see . -from weboob.capabilities.bill import CapDocument, Subscription, Bill, SubscriptionNotFound, DocumentNotFound +from weboob.capabilities.bill import DocumentTypes, CapDocument, Subscription, Bill, SubscriptionNotFound, DocumentNotFound from weboob.capabilities.base import find_object from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword, Value @@ -42,6 +42,8 @@ class OvhModule(Module, CapDocument): BROWSER = OvhBrowser + accepted_document_types = (DocumentTypes.BILL,) + def create_default_browser(self): return self.create_browser(self.config) diff --git a/modules/ovh/pages.py b/modules/ovh/pages.py index 5730ac5332819c2de88a3fa684b5a80fef78da8f..be690c0197cbcd5e8ecd4c20734efe747511cf46 100644 --- a/modules/ovh/pages.py +++ b/modules/ovh/pages.py @@ -18,7 +18,7 @@ # along with weboob. If not, see . -from weboob.capabilities.bill import Bill, Subscription +from weboob.capabilities.bill import DocumentTypes, Bill, Subscription from weboob.browser.pages import HTMLPage, LoggedPage, JsonPage from weboob.browser.filters.standard import CleanDecimal, CleanText, Env, Format, Date from weboob.browser.filters.html import Attr @@ -87,7 +87,7 @@ class BillsPage(LoggedPage, JsonPage): obj_id = Format('%s.%s', Env('subid'), Dict('orderId')) obj_date = Date(Dict('billingDate')) obj_format = u"pdf" - obj_type = u"bill" + obj_type = DocumentTypes.BILL obj_price = CleanDecimal(Dict('priceWithTax/value')) obj_url = Dict('pdfUrl') obj_label = Format('Facture %s', Dict('orderId')) diff --git a/modules/pradoepargne/browser.py b/modules/pradoepargne/browser.py index 5cdfef935f472ac3d6ca99a1231c18ed4534820d..185191297ed05e4f3d11990940c056b77524bb4f 100644 --- a/modules/pradoepargne/browser.py +++ b/modules/pradoepargne/browser.py @@ -18,8 +18,15 @@ # along with weboob. If not, see . -from weboob.browser import AbstractBrowser +from weboob.browser import AbstractBrowser, URL +from .pages import LoginPage -class CmesBrowser(AbstractBrowser): +class PradoBrowser(AbstractBrowser): PARENT = 'cmes' + + login = URL('pradoepargne/fr/identification/default.cgi', LoginPage) + + def __init__(self, baseurl, subsite, login, password, *args, **kwargs): + self.weboob = kwargs['weboob'] + super(PradoBrowser, self).__init__(baseurl, login, password, subsite, *args, **kwargs) diff --git a/modules/pradoepargne/module.py b/modules/pradoepargne/module.py index f65ccfd9cae5f08bdd2f75917d8c03b691fb6426..a46ea4ea5b943dda31477cd4b2990ca990409187 100644 --- a/modules/pradoepargne/module.py +++ b/modules/pradoepargne/module.py @@ -23,7 +23,7 @@ from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword from weboob.capabilities.base import find_object -from .browser import CmesBrowser +from .browser import PradoBrowser __all__ = ['PradoepargneModule'] @@ -40,13 +40,13 @@ class PradoepargneModule(Module, CapBankPockets): ValueBackendPassword('login', label='Identifiant', masked=False), ValueBackendPassword('password', label='Mot de passe')) - BROWSER = CmesBrowser + BROWSER = PradoBrowser def create_default_browser(self): return self.create_browser("https://www.gestion-epargne-salariale.fr", + "pradoepargne/", self.config['login'].get(), self.config['password'].get(), - "pradoepargne/", weboob=self.weboob) def get_account(self, _id): diff --git a/modules/societegenerale/browser.py b/modules/societegenerale/browser.py index 5206127c7f1fddf6ef7f76937b87e77aaef3abaf..3bee54b647c3e3f995568076637b031eda5208ae 100644 --- a/modules/societegenerale/browser.py +++ b/modules/societegenerale/browser.py @@ -19,8 +19,10 @@ from __future__ import unicode_literals -from datetime import datetime +from datetime import datetime, date +from decimal import Decimal from dateutil.relativedelta import relativedelta +import re from weboob.browser import LoginBrowser, URL, need_login, StatesMixin from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable, ActionNeeded @@ -28,16 +30,17 @@ from weboob.capabilities.bank import Account, TransferBankError from weboob.capabilities.base import find_object, NotAvailable from weboob.browser.exceptions import BrowserHTTPNotFound from weboob.capabilities.profile import ProfileMissing -from weboob.tools.capabilities.bank.investments import create_french_liquidity from .pages.accounts_list import ( - AccountsList, AccountHistory, CardsList, LifeInsurance, - LifeInsuranceHistory, LifeInsuranceInvest, LifeInsuranceInvest2, Market, UnavailableServicePage, - ListRibPage, AdvisorPage, HTMLProfilePage, XMLProfilePage, LoansPage, IbanPage, ComingPage, - NewLandingPage, + AccountsMainPage, AccountDetailsPage, AccountsPage, LoansPage, ComingPage, HistoryPage, + CardListPage, CardHistoryPage, + AdvisorPage, HTMLProfilePage, XMLProfilePage, + MarketPage, + LifeInsurance, LifeInsuranceHistory, LifeInsuranceInvest, LifeInsuranceInvest2, + UnavailableServicePage, ) from .pages.transfer import AddRecipientPage, RecipientJson, TransferJson, SignTransferPage -from .pages.login import LoginPage, BadLoginPage, ReinitPasswordPage, ActionNeededPage, ErrorPage +from .pages.login import MainPage, LoginPage, BadLoginPage, ReinitPasswordPage, ActionNeededPage, ErrorPage from .pages.subscription import BankStatementPage @@ -45,57 +48,63 @@ __all__ = ['SocieteGenerale'] class SocieteGenerale(LoginBrowser, StatesMixin): - BASEURL = 'https://particuliers.secure.societegenerale.fr' + BASEURL = 'https://particuliers.societegenerale.fr' STATE_DURATION = 5 - login = URL('https://particuliers.societegenerale.fr/index.html', LoginPage) - action_needed = URL('/com/icd-web/forms/cct-index.html', - '/com/icd-web/gdpr/gdpr-recueil-consentements.html', - '/com/icd-web/forms/kyc-index.html', - ActionNeededPage) - bad_login = URL('\/acces/authlgn.html', '/error403.html', BadLoginPage) - reinit = URL('/acces/changecodeobligatoire.html', ReinitPasswordPage) - iban_page = URL(r'/lgn/url\.html\?dup', IbanPage) - accounts = URL('/restitution/cns_listeprestation.html', AccountsList) - coming_page = URL('/restitution/cns_listeEncours.xml', ComingPage) - cards_list = URL('/restitution/cns_listeCartes.*.html', CardsList) - account_history = URL('/restitution/cns_detail.*\.html', '/lgn/url.html', AccountHistory) - market = URL('/brs/cct/comti20.html', Market) - life_insurance = URL('/asv/asvcns10.html', '/asv/AVI/asvcns10a.html', '/brs/fisc/fisca10a.html', LifeInsurance) - life_insurance_invest = URL('/asv/AVI/asvcns20a.html', LifeInsuranceInvest) - life_insurance_invest_2 = URL('/asv/PRV/asvcns10priv.html', LifeInsuranceInvest2) - life_insurance_history = URL('/asv/AVI/asvcns2(?P[0-9])c.html', LifeInsuranceHistory) - list_rib = URL('/restitution/imp_listeRib.html', ListRibPage) - advisor = URL('/com/contacts.html', AdvisorPage) + # Bank + accounts_main_page = URL('/restitution/cns_listeprestation.html', + '/com/icd-web/cbo/index.html', AccountsMainPage) + account_details_page = URL('/restitution/cns_detailPrestation.html', AccountDetailsPage) + accounts = URL('/icd/cbo/data/liste-prestations-navigation-authsec.json', AccountsPage) + coming = URL('/restitution/cns_listeEncours.xml', ComingPage) + history = URL('/icd/cbo/data/liste-operations-authsec.json', HistoryPage) + cards_list = URL('/restitution/cns_listeCartesDd.html', + '/restitution/cns_detailCARTE_\w{3}.html', CardListPage) + card_history = URL('/restitution/cns_listeReleveCarteDd.xml', CardHistoryPage) + loans = URL(r'/abm/restit/listeRestitutionPretsNET.json\?a100_isPretConso=(?P\w+)', LoansPage) - # recipient + # Recipient add_recipient = URL(r'/personnalisation/per_cptBen_ajouterFrBic.html', r'/lgn/url.html', AddRecipientPage) json_recipient = URL(r'/sec/getsigninfo.json', r'/sec/csa/send.json', r'/sec/oob_sendoob.json', r'/sec/oob_polling.json', RecipientJson) - # transfer + # Transfer json_transfer = URL(r'/icd/vupri/data/vupri-liste-comptes.json\?an200_isBack=false', r'/icd/vupri/data/vupri-check.json', r'/lgn/url.html', TransferJson) sign_transfer = URL(r'/icd/vupri/data/vupri-generate-token.json', SignTransferPage) confirm_transfer = URL(r'/icd/vupri/data/vupri-save.json', TransferJson) - loans = URL(r'/abm/restit/listeRestitutionPretsNET.json\?a100_isPretConso=(?P\w+)', LoansPage) + # Wealth + market = URL('/brs/cct/comti20.html', MarketPage) + life_insurance = URL('/asv/asvcns10.html', '/asv/AVI/asvcns10a.html', '/brs/fisc/fisca10a.html', LifeInsurance) + life_insurance_invest = URL('/asv/AVI/asvcns20a.html', LifeInsuranceInvest) + life_insurance_invest_2 = URL('/asv/PRV/asvcns10priv.html', LifeInsuranceInvest2) + life_insurance_history = URL('/asv/AVI/asvcns2(?P[0-9])c.html', LifeInsuranceHistory) + + # Profile + advisor = URL('/icd/pon/data/get-contacts.xml', AdvisorPage) html_profile_page = URL(r'/com/dcr-web/dcr/dcr-coordonnees.html', HTMLProfilePage) xml_profile_page = URL(r'/gms/gmsRestituerAdresseNotificationServlet.xml', XMLProfilePage) - unavailable_service_page = URL(r'/com/service-indisponible.html', UnavailableServicePage) + # Document bank_statement = URL(r'/restitution/rce_derniers_releves.html', BankStatementPage) bank_statement_search = URL(r'/restitution/rce_recherche.html\?noRedirect=1', r'/restitution/rce_recherche_resultat.html', BankStatementPage) - new_landing = URL(r'/com/icd-web/cbo/index.html', NewLandingPage) - + bad_login = URL('\/acces/authlgn.html', '/error403.html', BadLoginPage) + reinit = URL('/acces/changecodeobligatoire.html', ReinitPasswordPage) + action_needed = URL('/com/icd-web/forms/cct-index.html', + '/com/icd-web/gdpr/gdpr-recueil-consentements.html', + '/com/icd-web/forms/kyc-index.html', + ActionNeededPage) + unavailable_service_page = URL(r'/com/service-indisponible.html', UnavailableServicePage) error = URL('https://static.societegenerale.fr/pri/erreur.html', ErrorPage) + login = URL('/sec/vk', LoginPage) + main_page = URL('https://particuliers.societegenerale.fr', MainPage) - accounts_list = None context = None dup = None id_transaction = None @@ -113,162 +122,148 @@ class SocieteGenerale(LoginBrowser, StatesMixin): raise BrowserIncorrectPassword() self.username = self.username[:8] - self.login.stay_or_go() - + self.main_page.go() try: self.page.login(self.username, self.password) except BrowserHTTPNotFound: raise BrowserIncorrectPassword() - if self.login.is_here(): + assert self.login.is_here() + reason, action = self.page.get_error() + if reason == 'echec_authent': raise BrowserIncorrectPassword() - if self.bad_login.is_here(): - error = self.page.get_error() - if error is None: - raise BrowserIncorrectPassword() - elif error.startswith('Votre session a'): - raise BrowserUnavailable('Session has expired') - elif error.startswith('Le service est momentan'): - raise BrowserUnavailable(error) - elif 'niv_auth_insuff' in error: - raise BrowserIncorrectPassword("Niveau d'authentification insuffisant") - elif 'Veuillez contacter' in error: - raise ActionNeeded(error) - else: - raise BrowserIncorrectPassword(error) + def iter_cards(self, account): + for el in account._cards: + if el['carteDebitDiffere']: + card = Account() + card.id = card.number = el['numeroCompteFormate'].replace(' ', '') + card.label = el['labelToDisplay'] + card.coming = Decimal(str(el['montantProchaineEcheance'])) + card.type = Account.TYPE_CARD + card.currency = account.currency + card._internal_id = el['idTechnique'] + card._prestation_id = el['id'] + yield card @need_login def get_accounts_list(self): - if self.accounts_list is None: - self.accounts.stay_or_go() - # the link is not on the new landing page, navigating manually - if self.new_landing.is_here(): - self.logger.info('Falling back on old accounts consulting page.') - self.location('/restitution/cns_listeprestation.html?NoRedirect=true') - self.accounts_list = self.page.get_list() - # Coming amount is on another page, whose url must be retrieved on the main page - self.location(self.page.get_coming_url()) - self.page.set_coming(self.accounts_list) - self.list_rib.go() - if self.list_rib.is_here(): - # Caching rib url, so we don't have to go back and forth for each account - for account in self.accounts_list: - account._rib_url = self.page.get_rib_url(account) - for account in self.accounts_list: - if account.type is Account.TYPE_MARKET: - self.location(account._link_id) - if isinstance(self.page, Market): - account.balance = self.page.get_balance(account.type) or account.balance - if account._rib_url: - self.location(account._rib_url) - if self.iban_page.is_here(): - account.iban = self.page.get_iban() - - for type_ in ['true', 'false']: - self.loans.go(conso=type_) - # some loans page are unavailable - if self.page.doc['commun']['statut'] == 'nok': - continue - self.accounts_list.extend(self.page.iter_accounts()) - - return iter(self.accounts_list) + self.accounts_main_page.go() + + if self.page.is_old_website(): + # go on new_website + self.location(self.absurl('/com/icd-web/cbo/index.html')) + + # get account iban on transfer page + self.json_transfer.go() + account_ibans = self.page.get_account_ibans_dict() + + # get account coming on coming page, coming amount is not available yet on account page + self.coming.go() + account_comings = self.page.get_account_comings() + + self.accounts.go() + for account in self.page.iter_accounts(): + for card in self.iter_cards(account): + card.parent = account + yield card + + if account._prestation_id in account_ibans: + account.iban = account_ibans[account._prestation_id] + + if account._prestation_id in account_comings: + account.coming = account_comings[account._prestation_id] + + if account.type == account.TYPE_LOAN: + self.loans.go(conso=(account._loan_type == 'PR_CONSO')) + account = self.page.get_loan_account(account) + + yield account + + @need_login + def iter_card_transaction(self, account): + # TODO + raise BrowserUnavailable() + self.account_details_page.go(params={'idprest': account._prestation_id}) + self.location(self.absurl(self.page.get_card_history_link(account))) + + if self.page.get_card_transactions_link(): + self.location(self.absurl(self.page.get_card_transactions_link())) + + year = date.today().year + rdate = None + for tr in self.page.iter_card_history(): + + # search for the first card summary + if tr.date is NotAvailable: + tr.type = tr.TYPE_CARD_SUMMARY + d = re.search(r'(\d{2})\/(\d{2})', tr.label) + if d: + dd, mm = int(d.group(1)), int(d.group(2)) + tr.date = rdate = date(year, mm, dd) + year = tr.date.year + + # if card summary is found, yield transaction with the right rdate + if rdate: + tr.rdate = rdate + yield tr @need_login def iter_history(self, account): - if not account._link_id: + # TODO: check matching + raise BrowserUnavailable() + + if account.type in (account.TYPE_LOAN, account.TYPE_MARKET, ): return - self.location(account._link_id) - - if self.cards_list.is_here(): - for card_link in self.page.iter_cards(): - self.location(card_link) - for trans in self.page.iter_transactions(): - yield trans - elif self.account_history.is_here(): - for trans in self.page.iter_transactions(): - yield trans - - elif self.life_insurance.is_here(): - for n in ('0', '1'): - for i in range(3, -1, -1): - self.life_insurance_history.go(n=n) - if not self.page.get_error(): - break - self.logger.warning('Life insurance error (%s), retrying %d more times', self.page.get_error(), i) - else: - self.logger.warning('Life insurance error (%s), failed', self.page.get_error()) - return - - for trans in self.page.iter_transactions(): - yield trans - - # go to next page - while self.page.doc.xpath('//div[@class="net2g_asv_tableau_pager"]/a[contains(@href, "actionSuivPage")]'): - form = self.page.get_form('//form[@id="operationForm"]') - form['a100_asv_action'] = 'actionSuivPage' - form.submit() - for trans in self.page.iter_transactions(): - yield trans - - else: - self.logger.warning('This account is not supported') + + if account.type == account.TYPE_CARD: + for tr in self.iter_card_transaction(account): + yield tr + return + + self.history.go(params={'b64e200_prestationIdTechnique': account._internal_id}) + + iter_transactions = self.page.iter_history + if account.type == Account.TYPE_PEA: + iter_transactions = self.page.iter_pea_history + + for transaction in iter_transactions(): + yield transaction + + @need_login + def iter_coming(self, account): + # TODO: check matching + raise BrowserUnavailable() + if account.type in (account.TYPE_LOAN, account.TYPE_MARKET ): + return + + if account.type == account.TYPE_CARD: + # TODO + return + + self.history.go(params={'b64e200_prestationIdTechnique': account._internal_id}) + for transaction in self.page.iter_coming(): + yield transaction @need_login def iter_investment(self, account): + # TODO + raise BrowserUnavailable() if account.type not in (Account.TYPE_MARKET, Account.TYPE_LIFE_INSURANCE, Account.TYPE_PEA): self.logger.debug('This account is not supported') - return + return [] - if account.type == Account.TYPE_MARKET: - self.location(account._link_id) - for invest in self.page.iter_investment(): - yield invest - - elif account.type == Account.TYPE_LIFE_INSURANCE: - # Life Insurance type whose investments require scraping at '/asv/PRV/asvcns10priv.html': - self.location(account._link_id) - if self.page.has_link(): - # Other Life Insurance pages: - self.life_insurance_invest.go() - - if self.life_insurance.is_here(): - # check that investements are here - error_msg = self.page.get_error_msg() - if error_msg and 'Le service est momentanément indisponible' in error_msg: - raise BrowserUnavailable(error_msg) - - # Yield investments from the first page: + raise BrowserUnavailable() + + if account.type in (Account.TYPE_PEA, Account.TYPE_MARKET): + self.account_details_page.go(params={'idprest': account._prestation_id}) for invest in self.page.iter_investment(): - yield invest - - # Handle investments pagination: - total_pages = self.page.get_pages() - if total_pages: - total_pages = int(total_pages) - for page in range(2, total_pages + 1): - params = { - 'a100_asv_action': 'actionSuivPage', - 'a100_asv_numPage': page - 1, - 'a100_asv_nbPages': total_pages, - } - # Using "self.url" avoids dealing with the multiple possible URLs: - # asvcns10.html, asvcns10priv.html, asvcns20a.html and so on. - self.location(self.url, data=params) - for invest in self.page.iter_investment(): - yield invest - - elif account.type == Account.TYPE_PEA: - # Scraping liquidities for "PEA Espèces" accounts - self.location(account._link_id) - valuation = self.page.get_liquidities() - if valuation != NotAvailable: - yield create_french_liquidity(valuation) - return + # TODO + pass - @need_login - def get_advisor(self): - return self.advisor.stay_or_go().get_advisor() + if account.type == Account.TYPE_LIFE_INSURANCE: + # TODO + pass @need_login def iter_recipients(self, account): @@ -283,6 +278,11 @@ class SocieteGenerale(LoginBrowser, StatesMixin): @need_login def init_transfer(self, account, recipient, transfer): self.json_transfer.go() + + first_transfer_date = self.page.get_first_available_transfer_date() + if transfer.exec_date and transfer.exec_date < first_transfer_date: + transfer.exec_date = first_transfer_date + self.page.init_transfer(account, recipient, transfer) return self.page.handle_response(recipient) @@ -342,6 +342,10 @@ class SocieteGenerale(LoginBrowser, StatesMixin): self.page.post_label(recipient) self.page.double_auth(recipient) + @need_login + def get_advisor(self): + return self.advisor.go().get_advisor() + @need_login def get_profile(self): self.html_profile_page.go() diff --git a/modules/societegenerale/module.py b/modules/societegenerale/module.py index 149855480af7fcb1372956c481c0a4e4c9c006dc..ade9fd8e96034c8a4e8414af14543b820ca33fe3 100644 --- a/modules/societegenerale/module.py +++ b/modules/societegenerale/module.py @@ -107,7 +107,7 @@ class SocieteGeneraleModule(Module, CapBankWealth, CapBankTransferAddRecipient, return self.browser.iter_recipients(origin_account) def new_recipient(self, recipient, **params): - if self.config['website'].get() not in ('par', 'pro'): + if self.config['website'].get() not in ('pro', ): raise NotImplementedError() recipient.label = ' '.join(w for w in re.sub('[^0-9a-zA-Z:\/\-\?\(\)\.,\'\+ ]+', '', recipient.label).split()) return self.browser.new_recipient(recipient, **params) diff --git a/modules/societegenerale/pages/accounts_list.py b/modules/societegenerale/pages/accounts_list.py index 9256d2907ba45dfa8bcd7fec2b667cfcf1870849..f1010977e2c8f22666708ff41d7659cc09635c0a 100644 --- a/modules/societegenerale/pages/accounts_list.py +++ b/modules/societegenerale/pages/accounts_list.py @@ -20,193 +20,132 @@ from __future__ import unicode_literals import datetime -from lxml.etree import XML -from lxml.html import fromstring -from decimal import Decimal, InvalidOperation import re -from weboob.capabilities.base import empty, NotAvailable, find_object -from weboob.capabilities.bank import Account, Investment +from weboob.capabilities.base import NotAvailable +from weboob.capabilities.bank import Account, Investment, Loan from weboob.capabilities.contact import Advisor from weboob.capabilities.profile import Person, ProfileMissing from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.tools.capabilities.bank.investments import is_isin_valid -from weboob.tools.compat import parse_qs, urlparse, parse_qsl, urlunparse, urlencode, unicode -from weboob.tools.date import parse_date as parse_d -from weboob.browser.elements import DictElement, ItemElement, TableElement, method +from weboob.tools.compat import urlparse, parse_qsl, urlunparse, urlencode, unicode +from weboob.browser.elements import DictElement, ItemElement, TableElement, method, ListElement from weboob.browser.filters.json import Dict -from weboob.browser.filters.standard import CleanText, CleanDecimal, Regexp, RegexpError -from weboob.browser.filters.html import Link, TableCell, Attr -from weboob.browser.pages import HTMLPage, XMLPage, JsonPage, LoggedPage -from weboob.exceptions import NoAccountsException, BrowserUnavailable, ActionNeeded +from weboob.browser.filters.standard import ( + CleanText, CleanDecimal, Regexp, RegexpError, Currency, Eval, Field, Format, Date, +) +from weboob.browser.filters.html import Link, TableCell +from weboob.browser.pages import HTMLPage, XMLPage, JsonPage, LoggedPage, pagination +from weboob.exceptions import BrowserUnavailable, ActionNeeded from .base import BasePage + def MyDecimal(*args, **kwargs): kwargs.update(replace_dots=True, default=NotAvailable) return CleanDecimal(*args, **kwargs) -class NotTransferBasePage(BasePage): - def is_transfer_here(self): - # check that we aren't on transfer or add recipient page - return bool(CleanText('//h1[contains(text(), "Effectuer un virement")]')(self.doc)) or \ - bool(CleanText(u'//h3[contains(text(), "Ajouter un compte bénéficiaire de virement")]')(self.doc)) or \ - bool(CleanText(u'//h1[contains(text(), "Ajouter un compte bénéficiaire de virement")]')(self.doc)) or \ - bool(CleanText(u'//h3[contains(text(), "Veuillez vérifier les informations du compte à ajouter")]')(self.doc)) or \ - bool(Link('//a[contains(@href, "per_cptBen_ajouterFrBic")]', default=NotAvailable)(self.doc)) +class AccountsMainPage(LoggedPage, HTMLPage): + def is_old_website(self): + return Link('//a[contains(text(), "Afficher la nouvelle consultation")]', default=None)(self.doc) -class AccountsList(LoggedPage, BasePage): - LINKID_REGEXP = re.compile(".*ch4=(\w+).*") - - TYPES = {u'Compte Bancaire': Account.TYPE_CHECKING, - u'Compte Epargne': Account.TYPE_SAVINGS, - u'Compte Sur Livret': Account.TYPE_SAVINGS, - u'Compte Titres': Account.TYPE_MARKET, - 'Déclic Tempo': Account.TYPE_MARKET, - u'Compte Alterna': Account.TYPE_LOAN, - u'Crédit': Account.TYPE_LOAN, - u'Ldd': Account.TYPE_SAVINGS, - u'Livret': Account.TYPE_SAVINGS, - u'PEA': Account.TYPE_PEA, - u'PEL': Account.TYPE_SAVINGS, - u'Plan Epargne': Account.TYPE_SAVINGS, - u'Prêt': Account.TYPE_LOAN, - 'Avance Patrimoniale': Account.TYPE_LOAN, - } - - def get_coming_url(self): - for script in self.doc.xpath('//script'): - s_content = CleanText('.')(script) - if "var url_encours" in s_content: - break - return re.search(r'url_encours=\"(.+)\"; ', s_content).group(1) - - def get_list(self): - err = CleanText('//span[@class="error_msg"]', default='')(self.doc) - if err == 'Vous ne disposez pas de compte consultable.': - raise NoAccountsException() - - def check_valid_url(url): - pattern = ['/restitution/cns_detailAVPAT.html', - '/restitution/cns_detailAlterna.html', - ] - - for p in pattern: - if url.startswith(p): - return False - return True - - accounts_list = [] +class AccountDetailsPage(LoggedPage, HTMLPage): + pass - for tr in self.doc.getiterator('tr'): - if 'LGNTableRow' not in tr.attrib.get('class', '').split(): - continue - account = Account() - for td in tr.getiterator('td'): - if td.attrib.get('headers', '') == 'TypeCompte': - a = td.find('a') - if a is None: - break - account.label = CleanText('.')(a) - account._link_id = a.get('href', '') - for pattern, actype in self.TYPES.items(): - if account.label.startswith(pattern): - account.type = actype - break - else: - if account._link_id.startswith('/asv/asvcns10.html'): - account.type = Account.TYPE_LIFE_INSURANCE - # Website crashes when going on theses URLs - if not check_valid_url(account._link_id): - account._link_id = None - - elif td.attrib.get('headers', '') == 'NumeroCompte': - account.id = CleanText(u'.', replace=[(' ', '')])(td) - - elif td.attrib.get('headers', '') == 'Libelle': - text = CleanText('.')(td) - if text != '': - account.label = text - - elif td.attrib.get('headers', '') == 'Solde': - div = td.xpath('./div[@class="Solde"]') - if len(div) > 0: - balance = CleanText('.')(div[0]) - if len(balance) > 0 and balance not in ('ANNULEE', 'OPPOSITION'): - try: - account.balance = Decimal(FrenchTransaction.clean_amount(balance)) - except InvalidOperation: - self.logger.error('Unable to parse balance %r' % balance) - continue - account.currency = account.get_currency(balance) - else: - account.balance = NotAvailable - if not account.label or empty(account.balance): - continue +class AccountsPage(LoggedPage, JsonPage): + @method + class iter_accounts(DictElement): + item_xpath = 'donnees' - if account._link_id and 'CARTE_' in account._link_id: - account.type = account.TYPE_CARD - page = self.browser.open(account._link_id).page + class item(ItemElement): + klass = Account - # Layout with several cards - line = CleanText('//table//div[contains(text(), "Liste des cartes")]', replace=[(' ', '')])(page.doc) - m = re.search(r'(\d+)', line) - if m: - parent_id = m.group() - else: - parent_id = CleanText('//div[contains(text(), "Numéro de compte débité")]/following::div[1]', replace=[(' ', '')])(page.doc) - account.parent = find_object(accounts_list, id=parent_id) + # There are more account type to find + TYPES = { + 'COMPTE_COURANT': Account.TYPE_CHECKING, + 'PEL': Account.TYPE_SAVINGS, + 'LDD': Account.TYPE_SAVINGS, + 'LIVRETA': Account.TYPE_SAVINGS, + 'LIVRET_JEUNE': Account.TYPE_SAVINGS, + 'COMPTE_SUR_LIVRET': Account.TYPE_SAVINGS, + 'BANQUE_FRANCAISE_MUTUALISEE': Account.TYPE_SAVINGS, + 'PRET_GENERAL': Account.TYPE_LOAN, + 'PRET_PERSONNEL_MUTUALISE': Account.TYPE_LOAN, + 'COMPTE_TITRE_GENERAL': Account.TYPE_MARKET, + 'PEA_ESPECES': Account.TYPE_PEA, + 'PEA_PME_ESPECES': Account.TYPE_PEA, + 'COMPTE_TITRE_PEA': Account.TYPE_PEA, + 'COMPTE_TITRE_PEA_PME': Account.TYPE_PEA, + 'VIE_FEDER': Account.TYPE_LIFE_INSURANCE, + 'ASSURANCE_VIE_GENERALE': Account.TYPE_LIFE_INSURANCE, + 'AVANCE_PATRIMOINE': Account.TYPE_REVOLVING_CREDIT, + } - if account.type == Account.TYPE_UNKNOWN: - self.logger.debug('Unknown account type: %s', account.label) + obj_id = obj_number = CleanText(Dict('numeroCompteFormate'), replace=[(' ', '')]) + obj_label = Dict('labelToDisplay') + obj_balance = CleanDecimal(Dict('soldes/soldeTotal')) + obj_coming = CleanDecimal(Dict('soldes/soldeEnCours')) + obj_currency = Currency(Dict('soldes/devise')) + obj__cards = Dict('cartes', default=[]) - accounts_list.append(account) + def obj_type(self): + return self.TYPES.get(Dict('produit')(self), Account.TYPE_UNKNOWN) - return accounts_list + # Useful for navigation + obj__internal_id = Dict('idTechnique') + obj__prestation_id = Dict('id') + def obj__loan_type(self): + if Field('type')(self) == Account.TYPE_LOAN: + return Dict('codeFamille')(self) + return None -class ComingPage(LoggedPage, XMLPage): - def set_coming(self, accounts_list): - for a in accounts_list: - a.coming = CleanDecimal('//EnCours[contains(@id, "%s")]' % a.id, replace_dots=True, default=NotAvailable)(self.doc) +class LoansPage(LoggedPage, JsonPage): + def on_load(self): + if 'action' in self.doc['commun'] and self.doc['commun']['action'] == 'BLOCAGE': + raise ActionNeeded() + assert self.doc['commun']['statut'] != 'nok' -class IbanPage(LoggedPage, NotTransferBasePage): - def is_here(self): - if self.is_transfer_here(): - return False - return 'Imprimer ce RIB' in Attr('.//img', 'alt')(self.doc) or \ - CleanText('//span[@class="error_msg"]')(self.doc) + def get_loan_account(self, account): + assert account._prestation_id in Dict('donnees/tabIdAllPrestations')(self.doc), \ + 'Loan with prestation id %s should be on this page ...' % account._prestation_id + for acc in Dict('donnees/tabPrestations')(self.doc): + if CleanText(Dict('idPrestation'))(acc) == account._prestation_id: + loan = Loan() + loan.id = loan.number = account.id + loan.label = account.label + loan.type = account.type - def get_iban(self): - if not CleanText('//span[@class="error_msg"]')(self.doc): - return CleanText().filter(self.doc.xpath("//font[contains(text(),'IBAN')]/b[1]")[0]).replace(' ', '') + loan.currency = Currency(Dict('capitalRestantDu/devise'))(acc) + loan.balance = Eval(lambda x: x / 100, CleanDecimal(Dict('capitalRestantDu/valeur')))(acc) + loan.coming = account.coming + loan.total_amount = Eval(lambda x: x / 100, CleanDecimal(Dict('montantPret/valeur')))(acc) + loan.next_payment_amount = Eval(lambda x: x / 100, CleanDecimal(Dict('montantEcheance/valeur')))(acc) -class CardsList(LoggedPage, BasePage): - def iter_cards(self): - for tr in self.doc.getiterator('tr'): - tds = tr.findall('td') - if len(tds) < 4 or tds[0].attrib.get('class', '') != 'tableauIFrameEcriture1': - continue + loan.duration = Dict('dureeNbMois')(acc) + loan.maturity_date = datetime.datetime.strptime(Dict('dateFin')(acc), '%Y%m%d') - yield tr.xpath('.//a')[0].attrib['href'] + loan._internal_id = account._internal_id + loan._prestation_id = account._prestation_id + return loan class Transaction(FrenchTransaction): - PATTERNS = [(re.compile(r'^CARTE \w+ RETRAIT DAB.*? (?P
\d{2})/(?P\d{2})( (?P\d+)H(?P\d+))? (?P.*)'), + PATTERNS = [(re.compile(r'^CARTE \w+ RETRAIT DAB.*? (?P
\d{2})\/(?P\d{2})( (?P\d+)H(?P\d+))? (?P.*)'), FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile(r'^CARTE \w+ (?P
\d{2})/(?P\d{2})( A (?P\d+)H(?P\d+))? RETRAIT DAB (?P.*)'), + (re.compile(r'^CARTE \w+ (?P
\d{2})\/(?P\d{2})( A (?P\d+)H(?P\d+))? RETRAIT DAB (?P.*)'), FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile(r'^CARTE \w+ REMBT (?P
\d{2})/(?P\d{2})( A (?P\d+)H(?P\d+))? (?P.*)'), + (re.compile(r'^CARTE \w+ REMBT (?P
\d{2})\/(?P\d{2})( A (?P\d+)H(?P\d+))? (?P.*)'), FrenchTransaction.TYPE_PAYBACK), - (re.compile(r'^(?PCARTE) \w+ (?P
\d{2})/(?P\d{2}) (?P.*)'), + (re.compile(r'^(?PCARTE) \w+ (?P
\d{2})\/(?P\d{2}) (?P.*)'), FrenchTransaction.TYPE_CARD), - (re.compile(r'^(?P
\d{2})(?P\d{2})/(?P.*?)/?(-[\d,]+)?$'), + (re.compile(r'^(?P
\d{2})(?P\d{2})\/(?P.*?)\/?(-[\d,]+)?$'), FrenchTransaction.TYPE_CARD), (re.compile(r'^(?P(COTISATION|PRELEVEMENT|TELEREGLEMENT|TIP)) (?P.*)'), FrenchTransaction.TYPE_ORDER), @@ -231,137 +170,182 @@ class Transaction(FrenchTransaction): ] -class AccountHistory(LoggedPage, NotTransferBasePage): - def is_here(self): - return not self.is_transfer_here() +class HistoryPage(LoggedPage, JsonPage): + @pagination + @method + class iter_history(DictElement): + def next_page(self): + conditions = ( + not Dict('donnees/listeOperations')(self), + not Dict('donnees/recapitulatifCompte/chargerPlusOperations')(self) + ) - def on_load(self): - super(AccountHistory, self).on_load() + if any(conditions): + return - msg = CleanText('//span[@class="error_msg"]', default='')(self.doc) - if 'Le service est momentanément indisponible' in msg: - raise BrowserUnavailable(msg) + if '&an200_operationsSupplementaires=true' in self.page.url: + return self.page.url + return self.page.url + '&an200_operationsSupplementaires=true' - debit_date = None + item_xpath = 'donnees/listeOperations' - def get_part_url(self): - for script in self.doc.getiterator('script'): - if script.text is None: - continue + class item(ItemElement): + def condition(self): + return Dict('statutOperation')(self) == 'COMPTABILISE' - m = re.search('var listeEcrCavXmlUrl="(.*)";', script.text) - if m: - return m.group(1) + klass = Transaction - return None + # not 'idOpe' means that it's a comming transaction + def obj_id(self): + if Dict('idOpe')(self): + return Regexp(CleanText(Dict('idOpe')), r'(\d+)/')(self) - def iter_transactions(self): - url = self.get_part_url() - if url is None: - # There are no transactions in this kind of account - return - - is_deferred_card = bool(self.doc.xpath(u'//div[contains(text(), "Différé")]')) - has_summary = False - - if is_deferred_card: - coming_debit_date = None - # get coming debit date for deferred_card - date_string = Regexp(CleanText(u'//option[contains(text(), "détail des factures à débiter le")]'), - r'(\d{2}/\d{2}/\d{4})', - default=NotAvailable)(self.doc) - if date_string: - coming_debit_date = parse_d(date_string) - - while True: - d = XML(self.browser.open(url).content) - el = d.xpath('//dataBody') - if not el: - return + def obj_rdate(self): + if Dict('dateChargement')(self): + return Eval(lambda t: datetime.date.fromtimestamp(int(t)/1000),Dict('dateChargement'))(self) - el = el[0] - s = unicode(el.text).encode('iso-8859-1') - doc = fromstring(s) - - for tr in self._iter_transactions(doc): - if tr.type == Transaction.TYPE_CARD_SUMMARY: - has_summary = True - if is_deferred_card and tr.type is Transaction.TYPE_CARD: - tr.type = Transaction.TYPE_DEFERRED_CARD - if not has_summary: - if coming_debit_date: - tr.date = coming_debit_date - tr._coming = True - yield tr - - el = d.xpath('//dataHeader')[0] - if int(el.find('suite').text) != 1: - return + obj_date = Eval(lambda t: datetime.date.fromtimestamp(int(t)/1000), Dict('dateOpe')) + obj_amount = CleanDecimal(Dict('mnt')) + obj_raw = Dict('libOpe') - url = urlparse(url) - p = parse_qs(url.query) - - args = {} - args['n10_nrowcolor'] = 0 - args['operationNumberPG'] = el.find('operationNumber').text - args['operationTypePG'] = el.find('operationType').text - args['pageNumberPG'] = el.find('pageNumber').text - args['idecrit'] = el.find('idecrit').text or '' - args['sign'] = p['sign'][0] - args['src'] = p['src'][0] - - url = '%s?%s' % (url.path, urlencode(args)) - - def _iter_transactions(self, doc): - t = None - for i, tr in enumerate(doc.xpath('//tr')): - try: - raw = tr.attrib['title'].strip() - except KeyError: - raw = CleanText('./td[@headers="Libelle"]//text()')(tr) - - date = CleanText('./td[@headers="Date"]')(tr) - if date == '': - m = re.search(r'(\d+)/(\d+)', raw) - if not m: - continue - - old_debit_date = self.debit_date - self.debit_date = t.date if t else datetime.date.today() - self.debit_date = self.debit_date.replace(day=int(m.group(1)), month=int(m.group(2))) - - # Need to do it when years/date overlap, causing the previous `.replace()` to - # set the date at the end of the next year instead of the current year - if old_debit_date is None and self.debit_date > datetime.date.today(): - old_debit_date = self.debit_date - - if old_debit_date is not None: - while self.debit_date > old_debit_date: - self.debit_date = self.debit_date.replace(year=self.debit_date.year - 1) - self.logger.error('adjusting debit date to %s', self.debit_date) - - if not t: - continue - - t = Transaction() - - if 'EnTraitement' in tr.get('class', ''): - t._coming = True - else: - t._coming = False - t.set_amount(*reversed([el.text for el in tr.xpath('./td[@class="right"]')])) - if date == '': - # Credit from main account. - t.amount = -t.amount - date = self.debit_date - t.rdate = t.parse_date(date) - t.parse(raw=raw, date=(self.debit_date or date), vdate=(date or None)) + @method + class iter_pea_history(DictElement): + item_xpath = 'donnees/listeOperations' + + class item(ItemElement): + klass = Transaction + + obj_date = Eval(lambda t: datetime.date.fromtimestamp(int(t)/1000), Dict('dateOpe')) + obj_amount = CleanDecimal(Dict('mnt')) + obj_raw = Dict('libOpe') + + + @method + class iter_coming(DictElement): + item_xpath = 'donnees/listeOperations' + + class item(ItemElement): + def condition(self): + return Dict('statutOperation')(self) != 'COMPTABILISE' and not Dict('idOpe')(self) + + klass = Transaction + + obj_rdate = Eval(lambda t: datetime.date.fromtimestamp(int(t)/1000), Dict('dateOpe')) + # there is no 'dateChargement' for coming transaction + obj_date = Eval(lambda t: datetime.date.fromtimestamp(int(t)/1000), Dict('dateOpe')) + obj_amount = CleanDecimal(Dict('mnt')) + obj_raw = Dict('libOpe') + + +class ComingPage(LoggedPage, XMLPage): + def get_account_comings(self): + account_comings = {} + for el in self.doc.xpath('//EnCours'): + prestation_id = CleanText('./@id')(el).replace('montantEncours', '') + coming_amount = MyDecimal(Regexp(CleanText('.'), r'(.*) '))(el) + account_comings[prestation_id] = coming_amount + return account_comings + + +class CardListPage(LoggedPage, HTMLPage): + def get_card_history_link(self, account): + for el in self.doc.xpath('//a[contains(@href, "detailCARTE")]'): + if CleanText('.', replace=[(' ', '')])(el) == account.number: + return Link('.')(el) + + def get_card_transactions_link(self): + if 'Le détail de cette carte ne vous est pas accessible' in CleanText('//div')(self.doc): + return NotAvailable + return CleanText('//div[@id="operationsListView"]//select/option[@selected="selected"]/@value')(self.doc) + - yield t +class CardHistoryPage(LoggedPage, HTMLPage): + @method + class iter_card_history(ListElement): + item_xpath = '//tr' + + class item(ItemElement): + klass = Transaction + + obj_label = CleanText('.//td[@headers="Libelle"]/span') + + def obj_date(self): + if not 'TOTAL DES FACTURES' in Field('label')(self): + return Date(Regexp(CleanText('.//td[@headers="Date"]'), r'\d{2}\/\d{2}\/\d{4}'))(self) + else: + return NotAvailable - def get_liquidities(self): - return CleanDecimal('//td[contains(@headers, "solde")]', replace_dots=True)(self.doc) + def obj_amount(self): + if not 'TOTAL DES FACTURES' in Field('label')(self): + return MyDecimal(CleanText('.//td[contains(@headers, "Debit")]'))(self) + else: + return abs(MyDecimal(CleanText('.//td[contains(@headers, "Debit")]'))(self)) + + def obj_raw(self): + if not 'TOTAL DES FACTURES' in Field('label')(self): + return Transaction.Raw(CleanText('.//td[@headers="Libelle"]/span'))(self) + return NotAvailable + + +class MarketPage(LoggedPage, HTMLPage): + # TODO + pass + + +class AdvisorPage(LoggedPage, XMLPage): + ENCODING = 'ISO-8859-15' + + def get_advisor(self): + advisor = Advisor() + advisor.name = Format('%s %s', CleanText('//NomConseiller'), CleanText('//PrenomConseiller'))(self.doc) + advisor.phone = CleanText('//NumeroTelephone')(self.doc) + advisor.agency = CleanText('//liloes')(self.doc) + advisor.address = Format('%s %s %s', + CleanText('//ruadre'), + CleanText('//cdpost'), + CleanText('//loadre') + )(self.doc) + advisor.email = CleanText('//Email')(self.doc) + advisor.role = "wealth" if "patrimoine" in CleanText('//LibelleNatureConseiller')(self.doc).lower() else "bank" + yield advisor + + +class HTMLProfilePage(LoggedPage, HTMLPage): + def on_load(self): + msg = CleanText('//div[@id="connecteur_partenaire"]', default='')(self.doc) + service_unavailable_msg = CleanText('//span[@class="error_msg" and contains(text(), "indisponible")]')(self.doc) + + if 'Erreur' in msg: + raise BrowserUnavailable(msg) + if service_unavailable_msg: + raise ProfileMissing(service_unavailable_msg) + + def get_profile(self): + profile = Person() + profile.name = Regexp(CleanText('//div[@id="dcr-conteneur"]//div[contains(text(), "PROFIL DE")]'), r'PROFIL DE (.*)')(self.doc) + profile.address = CleanText('//div[@id="dcr-conteneur"]//div[contains(text(), "ADRESSE")]/following::table//tr[3]/td[2]')(self.doc) + profile.address += ' ' + CleanText('//div[@id="dcr-conteneur"]//div[contains(text(), "ADRESSE")]/following::table//tr[5]/td[2]')(self.doc) + profile.address += ' ' + CleanText('//div[@id="dcr-conteneur"]//div[contains(text(), "ADRESSE")]/following::table//tr[6]/td[2]')(self.doc) + profile.country = CleanText('//div[@id="dcr-conteneur"]//div[contains(text(), "ADRESSE")]/following::table//tr[7]/td[2]')(self.doc) + + return profile + + +class XMLProfilePage(LoggedPage, XMLPage): + def get_email(self): + return CleanText('//AdresseEmailExterne')(self.doc) + + +# TODO: check if it work +class NotTransferBasePage(BasePage): + def is_transfer_here(self): + # check that we aren't on transfer or add recipient page + return bool(CleanText('//h1[contains(text(), "Effectuer un virement")]')(self.doc)) or \ + bool(CleanText(u'//h3[contains(text(), "Ajouter un compte bénéficiaire de virement")]')(self.doc)) or \ + bool(CleanText(u'//h1[contains(text(), "Ajouter un compte bénéficiaire de virement")]')(self.doc)) or \ + bool(CleanText(u'//h3[contains(text(), "Veuillez vérifier les informations du compte à ajouter")]')(self.doc)) or \ + bool(Link('//a[contains(@href, "per_cptBen_ajouterFrBic")]', default=NotAvailable)(self.doc)) class Invest(object): @@ -506,6 +490,7 @@ class LifeInsurance(LoggedPage, BasePage): # to check page errors return CleanText('//span[@class="error_msg"]')(self.doc) + class LifeInsuranceInvest(LifeInsurance, Invest): COL_LABEL = 0 COL_QUANTITY = 1 @@ -526,6 +511,7 @@ class LifeInsuranceInvest(LifeInsurance, Invest): pages = CleanText('//div[@class="net2g_asv_tableau_pager"]')(self.doc) return re.search(r'/(.*)', pages).group(1) if pages else None + class LifeInsuranceInvest2(LifeInsuranceInvest): @method class iter_investment(TableElement): @@ -615,103 +601,7 @@ class LifeInsuranceHistory(LifeInsurance): return NotAvailable -class ListRibPage(LoggedPage, BasePage): - def get_rib_url(self, account): - for div in self.doc.xpath('//table//td[@class="fond_cellule"]//div[@class="tableauBodyEcriture1"]//table//tr'): - if account.id == CleanText().filter(div.xpath('./td[2]//div/div')).replace(' ', ''): - href = CleanText().filter(div.xpath('./td[4]//a/@href')) - m = re.search("javascript:windowOpenerRib\('(.*?)'(.*)\)", href) - if m: - return m.group(1) - - -class AdvisorPage(LoggedPage, BasePage): - def get_advisor(self): - fax = CleanText('//div[contains(text(), "Fax")]/following-sibling::div[1]', replace=[(' ', '')])(self.doc) - agency = CleanText('//div[contains(@class, "agence")]/div[last()]')(self.doc) - address = CleanText('//div[contains(text(), "Adresse")]/following-sibling::div[1]')(self.doc) - for div in self.doc.xpath('//div[div[text()="Contacter mon conseiller"]]'): - a = Advisor() - a.name = CleanText('./div[2]')(div) - a.phone = Regexp(CleanText(u'./following-sibling::div[div[contains(text(), "Téléphone")]][1]/div[last()]', replace=[(' ', '')]), '([+\d]+)')(div) - a.fax = fax - a.agency = agency - a.address = address - a.mobile = a.email = NotAvailable - a.role = u"wealth" if "patrimoine" in CleanText('./div[1]')(div) else u"bank" - yield a - - -class HTMLProfilePage(LoggedPage, HTMLPage): - def on_load(self): - msg = CleanText('//div[@id="connecteur_partenaire"]', default='')(self.doc) - service_unavailable_msg = CleanText('//span[@class="error_msg" and contains(text(), "indisponible")]')(self.doc) - - if 'Erreur' in msg: - raise BrowserUnavailable(msg) - if service_unavailable_msg: - raise ProfileMissing(service_unavailable_msg) - - def get_profile(self): - profile = Person() - profile.name = Regexp(CleanText('//div[@id="dcr-conteneur"]//div[contains(text(), "PROFIL DE")]'), r'PROFIL DE (.*)')(self.doc) - profile.address = CleanText('//div[@id="dcr-conteneur"]//div[contains(text(), "ADRESSE")]/following::table//tr[3]/td[2]')(self.doc) - profile.address += ' ' + CleanText('//div[@id="dcr-conteneur"]//div[contains(text(), "ADRESSE")]/following::table//tr[5]/td[2]')(self.doc) - profile.address += ' ' + CleanText('//div[@id="dcr-conteneur"]//div[contains(text(), "ADRESSE")]/following::table//tr[6]/td[2]')(self.doc) - profile.country = CleanText('//div[@id="dcr-conteneur"]//div[contains(text(), "ADRESSE")]/following::table//tr[7]/td[2]')(self.doc) - - return profile - - -class XMLProfilePage(LoggedPage, XMLPage): - def get_email(self): - return CleanText('//AdresseEmailExterne')(self.doc) - - -class LoansPage(LoggedPage, JsonPage): - def on_load(self): - if 'action' in self.doc['commun'] and self.doc['commun']['action'] == 'BLOCAGE': - raise ActionNeeded() - assert self.doc['commun']['statut'] != 'nok' - - @method - class iter_accounts(DictElement): - item_xpath = 'donnees/tabPrestations' - - class item(ItemElement): - klass = Account - - obj_id = Dict('idPrestation') - obj_type = Account.TYPE_LOAN - obj_label = Dict('libelle') - obj_currency = Dict('capitalRestantDu/devise', default=NotAvailable) - obj__link_id = None - - def obj_balance(self): - val = Dict('capitalRestantDu/valeur', default=NotAvailable)(self) - if val is NotAvailable: - return val - - val = Decimal(val) - point = Decimal(Dict('capitalRestantDu/posDecimale')(self)) - assert point >= 0 - return val.scaleb(-point) - - def validate(self, obj): - assert obj.id - assert obj.label - if obj.balance is NotAvailable: - # ... but the account may be in the main AccountsList anyway - self.logger.debug('skipping account %r %r due to missing balance', obj.id, obj.label) - return False - return True - - class UnavailableServicePage(LoggedPage, HTMLPage): def on_load(self): if self.doc.xpath('//div[contains(@class, "erreur_404_content")]'): raise BrowserUnavailable() - - -class NewLandingPage(LoggedPage, HTMLPage): - pass diff --git a/modules/societegenerale/pages/login.py b/modules/societegenerale/pages/login.py index 6e18a7b9201a97cadeb174aac754d78a5259c32e..e78f9d7316d81eb3346811c0806e7b75b44ae2ea 100644 --- a/modules/societegenerale/pages/login.py +++ b/modules/societegenerale/pages/login.py @@ -25,8 +25,9 @@ import re from weboob.tools.json import json from weboob.exceptions import BrowserUnavailable, BrowserPasswordExpired, ActionNeeded -from weboob.browser.pages import HTMLPage +from weboob.browser.pages import HTMLPage, JsonPage from weboob.browser.filters.standard import CleanText +from weboob.browser.filters.json import Dict from .base import BasePage from ..captcha import Captcha, TileError @@ -61,19 +62,10 @@ class PasswordPage(object): return new_grid -class LoginPage(BasePage, PasswordPage): - def on_load(self): - for td in self.doc.getroot().cssselect('td.LibelleErreur'): - if td.text is None: - continue - msg = td.text.strip() - if 'indisponible' in msg: - raise BrowserUnavailable(msg) - +class MainPage(BasePage, PasswordPage): def get_authentication_data(self): - headers = {'Referer': 'https://particuliers.societegenerale.fr/index.html'} url = self.browser.BASEURL + '//sec/vkm/gen_crypto?estSession=0' - infos_data = self.browser.open(url, headers=headers).text + infos_data = self.browser.open(url).text infos_data = re.match('^_vkCallback\((.*)\);$', infos_data).group(1) infos = json.loads(infos_data.replace("'", '"')) @@ -97,20 +89,28 @@ class LoginPage(BasePage, PasswordPage): def login(self, login, password): authentication_data = self.get_authentication_data() - form = self.get_form(id='n2g_authentification') - pwd = authentication_data['img'].get_codes(password[:6]) t = pwd.split(',') newpwd = ','.join(t[self.strange_map[j]] for j in range(6)) - form['codcli'] = login.encode('iso-8859-1') - form['user_id'] = login.encode('iso-8859-1') - form['codsec'] = newpwd - form['cryptocvcs'] = authentication_data['infos']['crypto'].encode('iso-8859-1') - form['vkm_op'] = 'auth' - form.url = 'https://particuliers.secure.societegenerale.fr//acces/authlgn.html' - del form['button'] - form.submit() + data = { + 'top_code_etoile': 0, + 'top_ident': 1, + 'jeton': '', + 'cible': 300, + 'user_id': login.encode('iso-8859-1'), + 'codsec': newpwd, + 'cryptocvcs': authentication_data['infos']['crypto'].encode('iso-8859-1'), + 'vkm_op': 'auth', + } + self.browser.location(self.browser.absurl('/sec/vk/authent.json'), data=data) + + +class LoginPage(JsonPage): + def get_error(self): + if (Dict('commun/statut')(self.doc)).lower() != 'ok': + return Dict('commun/raison')(self.doc), Dict('commun/action')(self.doc) + return None, None class BadLoginPage(BasePage): diff --git a/modules/societegenerale/pages/transfer.py b/modules/societegenerale/pages/transfer.py index a7163d5f59f4d651fcae849535adee5f5ca8fc95..ecc547a8debf014b23b9f7cf40624a0f78db72a6 100644 --- a/modules/societegenerale/pages/transfer.py +++ b/modules/societegenerale/pages/transfer.py @@ -25,6 +25,7 @@ from weboob.browser.elements import method, ItemElement, DictElement from weboob.capabilities.bank import ( Recipient, Transfer, TransferBankError, AddRecipientBankError, AddRecipientStep, ) +from weboob.tools.capabilities.bank.iban import is_iban_valid from weboob.capabilities.base import NotAvailable from weboob.browser.filters.standard import ( CleanText, CleanDecimal, Env, Date, Field, Format, @@ -36,7 +37,7 @@ from weboob.tools.json import json from weboob.exceptions import BrowserUnavailable from .base import BasePage -from .login import LoginPage +from .login import MainPage class TransferJson(LoggedPage, JsonPage): @@ -60,6 +61,15 @@ class TransferJson(LoggedPage, JsonPage): def is_able_to_transfer(self, account): return self.get_acc_transfer_id(account) + def get_first_available_transfer_date(self): + return Date(Dict('donnees/listeEmetteursBeneficiaires/premiereDateExecutionPossible'))(self.doc) + + def get_account_ibans_dict(self): + account_ibans = {} + for account in Dict('donnees/listeEmetteursBeneficiaires/listeDetailEmetteurs')(self.doc): + account_ibans[Dict('identifiantPrestation')(account)] = Dict('iban')(account) + return account_ibans + @method class iter_recipients(DictElement): item_xpath = 'donnees/listeEmetteursBeneficiaires/listeDetailBeneficiaires' @@ -90,7 +100,7 @@ class TransferJson(LoggedPage, JsonPage): return Dict('iban')(self) def condition(self): - return Field('id')(self) != Env('account_id')(self) + return Field('id')(self) != Env('account_id')(self) and is_iban_valid(Field('iban')(self)) def init_transfer(self, account, recipient, transfer): assert self.is_able_to_transfer(account), 'Account %s seems to be not able to do transfer' % account.id @@ -135,7 +145,7 @@ class TransferJson(LoggedPage, JsonPage): return Dict('commun/statut')(self.doc).upper() == 'OK' -class SignTransferPage(LoggedPage, LoginPage): +class SignTransferPage(LoggedPage, MainPage): def get_token(self): result_page = json.loads(self.content) assert result_page['commun']['statut'].upper() == 'OK', 'Something went wrong: %s' % result_page['commun']['raison'] diff --git a/modules/trainline/browser.py b/modules/trainline/browser.py index 37690af2357bb7f3fb40dface98e62907c06973b..27956393aff797c240526564966c1969f1b0fe21 100644 --- a/modules/trainline/browser.py +++ b/modules/trainline/browser.py @@ -25,7 +25,7 @@ from weboob.browser.browsers import APIBrowser from weboob.exceptions import BrowserIncorrectPassword from weboob.browser.filters.standard import CleanDecimal, Date from weboob.browser.exceptions import ClientError -from weboob.capabilities.bill import Bill, Subscription +from weboob.capabilities.bill import DocumentTypes, Bill, Subscription class TrainlineBrowser(APIBrowser): @@ -79,7 +79,7 @@ class TrainlineBrowser(APIBrowser): b.date = Date().filter(proof['created_at']) b.format = u"pdf" b.label = u'Trajet du %s' % Date().filter(trip['departure_date']) - b.type = u"bill" + b.type = DocumentTypes.BILL b.vat = CleanDecimal().filter('0') if pnr['cents']: b.price = CleanDecimal().filter(format(pnr['cents']/float(100), '.2f')) diff --git a/modules/trainline/module.py b/modules/trainline/module.py index a6b226e90cbba9d2f8827c5410667427cb141016..e4027b89bf4dc961f0128bb62717a13eadf2f0a4 100644 --- a/modules/trainline/module.py +++ b/modules/trainline/module.py @@ -18,7 +18,7 @@ # along with weboob. If not, see . -from weboob.capabilities.bill import CapDocument, Subscription, Document, SubscriptionNotFound, DocumentNotFound +from weboob.capabilities.bill import DocumentTypes, CapDocument, Subscription, Document, SubscriptionNotFound, DocumentNotFound from weboob.capabilities.base import find_object, NotAvailable from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword, Value @@ -41,6 +41,8 @@ class TrainlineModule(Module, CapDocument): BROWSER = TrainlineBrowser + accepted_document_types = (DocumentTypes.BILL,) + def create_default_browser(self): return self.create_browser(self.config['login'].get(), self.config['password'].get())