diff --git a/modules/amazon/compat/weboob_exceptions.py b/modules/amazon/compat/weboob_exceptions.py index 941b96a08a0f97192ef081f5cd977f687e00a897..d67a00bde86b749f6c7901b2b87ffac8159e2e3a 100644 --- a/modules/amazon/compat/weboob_exceptions.py +++ b/modules/amazon/compat/weboob_exceptions.py @@ -67,9 +67,12 @@ class AppValidation(DecoupledValidation): from weboob.exceptions import BrowserRedirect as _BrowserRedirect class BrowserRedirect(_BrowserRedirect): - def __init__(self, url): + def __init__(self, url, resource=None): self.url = url + # Needed for transfer redirection + self.resource = resource + def __str__(self): return 'Redirecting to %s' % self.url diff --git a/modules/ameli/__init__.py b/modules/ameli/__init__.py index 7e647021f36f266df3b1d2aeba5288df9000d6d5..915e0d0401e6d8b51b3850919f81d19f35c42914 100644 --- a/modules/ameli/__init__.py +++ b/modules/ameli/__init__.py @@ -1,22 +1,24 @@ # -*- coding: utf-8 -*- -# Copyright(C) 2013-2015 Christophe Lampin +# Copyright(C) 2019 Budget Insight # # This file is part of a weboob module. # # This weboob module is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by +# it under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This weboob module is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. +# GNU Lesser General Public License for more details. # -# You should have received a copy of the GNU Affero General Public License +# You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . +from __future__ import unicode_literals + from .module import AmeliModule diff --git a/modules/ameli/browser.py b/modules/ameli/browser.py index a1106562b37ef58b956fc19b339ee7f93f8c8d4a..72b47aaeb1d83b0347e81b690ed9d2b69ae5c710 100644 --- a/modules/ameli/browser.py +++ b/modules/ameli/browser.py @@ -1,126 +1,80 @@ # -*- coding: utf-8 -*- -# Copyright(C) 2013-2015 Christophe Lampin +# Copyright(C) 2019 Budget Insight # # This file is part of a weboob module. # # This weboob module is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by +# it under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This weboob module is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. +# GNU Lesser General Public License for more details. # -# You should have received a copy of the GNU Affero General Public License +# You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . from __future__ import unicode_literals -from weboob.browser import LoginBrowser, URL, need_login -from weboob.exceptions import BrowserIncorrectPassword, ActionNeeded -from .pages import ( - LoginPage, HomePage, CguPage, AccountPage, LastPaymentsPage, PaymentsPage, PaymentDetailsPage, Raw, UnavailablePage, -) -from weboob.tools.compat import basestring - +from datetime import date +from time import time +from dateutil.relativedelta import relativedelta -__all__ = ['AmeliBrowser'] +from weboob.browser import LoginBrowser, URL, need_login +from .pages import ErrorPage, LoginPage, SubscriptionPage, DocumentsPage class AmeliBrowser(LoginBrowser): BASEURL = 'https://assure.ameli.fr' - loginp = URL(r'/PortailAS/appmanager/PortailAS/assure\?.*_pageLabel=as_login_page', LoginPage) - homep = URL(r'/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&_pageLabel=as_accueil_page', HomePage) - cgup = URL(r'/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&_pageLabel=as_conditions_generales_page', CguPage) - accountp = URL(r'/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&_pageLabel=as_info_perso_page', AccountPage) - paymentsp = URL(r'/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&_pageLabel=as_paiements_page', PaymentsPage) - paymentdetailsp = URL(r'/PortailAS/paiements.do\?actionEvt=chargerDetailPaiements.*', PaymentDetailsPage) - lastpaymentsp = URL(r'/PortailAS/paiements.do\?actionEvt=afficherPaiements.*', LastPaymentsPage) - pdf_page = URL(r'PortailAS/PDFServletReleveMensuel.dopdf\?PDF.moisRecherche=.*', Raw) - unavailablep = URL(r'/vu/INDISPO_COMPTE_ASSURES.html', UnavailablePage) + error_page = URL(r'/vu/INDISPO_COMPTE_ASSURES.html', ErrorPage) + login_page = URL(r'/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&connexioncompte_2actionEvt=afficher.*', LoginPage) + subscription_page = URL(r'/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&_pageLabel=as_info_perso_page.*', SubscriptionPage) + documents_page = URL(r'/PortailAS/paiements.do', DocumentsPage) def do_login(self): - self.logger.debug('call Browser.do_login') - - # set this cookie to get login form in response - self.session.cookies['hbc'] = '' - self.loginp.stay_or_go() - if self.homep.is_here(): - return True - + self.login_page.go() self.page.login(self.username, self.password) - error = self.page.is_error() - if error: - raise BrowserIncorrectPassword(error) - - self.page.locate_to_cgu_page() - if self.cgup.is_here(): - raise ActionNeeded(self.page.get_cgu()) - - self.homep.stay_or_go() # Redirection not interpreted by browser. Manually redirect on homep - - if not self.homep.is_here(): - raise BrowserIncorrectPassword() - - @need_login - def iter_subscription_list(self): - self.logger.debug('call Browser.iter_subscription_list') - self.accountp.stay_or_go() - return self.page.iter_subscription_list() - - @need_login - def get_subscription(self, id): - self.logger.debug('call Browser.get_subscription') - assert isinstance(id, basestring) - for sub in self.iter_subscription_list(): - if id == sub._id: - return sub - return None - - @need_login - def iter_history(self, sub): - transactions = [] - self.logger.debug('call Browser.iter_history') - self.paymentsp.stay_or_go() - payments_url = self.page.get_last_payments_url() - self.location(payments_url) - assert self.lastpaymentsp.is_here() - urls = self.page.iter_last_payments() - for url in urls: - self.location(url) - assert self.paymentdetailsp.is_here() - for payment in self.page.iter_payment_details(sub): - transactions.append(payment) - - # go to a page with a "deconnexion" link so that logged property - # stays True and next call to do_login doesn't crash when using the - # blackbox - self.accountp.go() - - return transactions - @need_login - def iter_documents(self, sub): - self.logger.debug('call Browser.iter_documents') - self.paymentsp.stay_or_go() - payments_url = self.page.get_last_payments_url() - self.location(payments_url) - assert self.lastpaymentsp.is_here() - for document in self.page.iter_documents(sub): - yield document + def iter_subscription(self): + self.subscription_page.go() + return self.page.iter_subscriptions() @need_login - def get_document(self, id): - self.logger.debug('call Browser.get_document') - assert isinstance(id, basestring) - subs = self.iter_subscription_list() - for sub in subs: - for b in self.iter_documents(sub): - if id == b.id: - return b - return False + def iter_documents(self, subscription): + end_date = date.today() + + start_date = end_date - relativedelta(years=1) + # FUN FACT, website tell us documents are available for 6 months + # let's suppose today is 28/05/19, website frontend limit DateDebut to 28/11/18 but we can get a little bit more + # by setting a previous date and get documents that are no longer available for simple user + + params = { + 'Beneficiaire': 'tout_selectionner', + 'DateDebut': start_date.strftime('%d/%m/%Y'), + 'DateFin': end_date.strftime('%d/%m/%Y'), + 'actionEvt': 'afficherPaiementsComplementaires', + 'afficherIJ': 'false', + 'afficherInva': 'false', + 'afficherPT': 'false', + 'afficherRS': 'false', + 'afficherReleves': 'false', + 'afficherRentes': 'false', + 'idNoCache': int(time()*1000) + } + + # the second request is stateful + # first value of actionEvt is afficherPaiementsComplementaires to get all payments from last 6 months + # (start_date 6 months in the past is needed but not enough) + self.documents_page.go(params=params) + + # then we set Rechercher to actionEvt to filter for this subscription, within last 6 months + # without first request we would have filter for this subscription but within last 2 months + params['actionEvt'] = 'Rechercher' + params['Beneficiaire'] = subscription._param + self.documents_page.go(params=params) + return self.page.iter_documents(subid=subscription.id) diff --git a/modules/ameli/module.py b/modules/ameli/module.py index c50ae6960be283863e707d958a15f39325ebc436..0955e3815488b5906bc73726fb14944a113d9073 100644 --- a/modules/ameli/module.py +++ b/modules/ameli/module.py @@ -1,80 +1,70 @@ # -*- coding: utf-8 -*- -# Copyright(C) 2013-2015 Christophe Lampin +# Copyright(C) 2019 Budget Insight # # This file is part of a weboob module. # # This weboob module is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by +# it under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This weboob module is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. +# GNU Lesser General Public License for more details. # -# You should have received a copy of the GNU Affero General Public License +# You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . + from __future__ import unicode_literals -from weboob.capabilities.bill import DocumentTypes, CapDocument, SubscriptionNotFound, DocumentNotFound, Subscription, Bill +from weboob.capabilities.base import find_object from weboob.tools.backend import Module, BackendConfig +from weboob.capabilities.bill import CapDocument, Document, DocumentTypes, SubscriptionNotFound, DocumentNotFound from weboob.tools.value import ValueBackendPassword + from .browser import AmeliBrowser + __all__ = ['AmeliModule'] class AmeliModule(Module, CapDocument): NAME = 'ameli' - DESCRIPTION = 'Ameli website: French Health Insurance' - MAINTAINER = 'Christophe Lampin' - EMAIL = 'weboob@lampin.net' + DESCRIPTION = "le site de l'Assurance Maladie en ligne" + MAINTAINER = 'Florian Duguet' + EMAIL = 'florian.duguet@budget-insight.com' + LICENSE = 'LGPLv3+' VERSION = '1.5' - LICENSE = 'AGPLv3+' + BROWSER = AmeliBrowser - CONFIG = BackendConfig(ValueBackendPassword('login', label='Numero de SS', regexp=r'^\d{13}$', masked=False), - ValueBackendPassword('password', label='Password', masked=True)) + + CONFIG = BackendConfig(ValueBackendPassword('login', label='Mon numero de sécurité sociale', regexp=r'^\d{13}$', masked=False), + ValueBackendPassword('password', label='Mon code (4 à 13 chiffres)', regexp=r'^\d{4,13}', masked=True)) accepted_document_types = (DocumentTypes.BILL,) def create_default_browser(self): - return self.create_browser(self.config['login'].get(), - self.config['password'].get()) + return self.create_browser(self.config['login'].get(), self.config['password'].get()) def iter_subscription(self): - return self.browser.iter_subscription_list() + return self.browser.iter_subscription() def get_subscription(self, _id): - subscription = self.browser.get_subscription(_id) - if not subscription: - raise SubscriptionNotFound() - else: - return subscription - - def iter_documents_history(self, subscription): - if not isinstance(subscription, Subscription): - subscription = self.get_subscription(subscription) - return self.browser.iter_history(subscription) + return find_object(self.iter_subscription(), id=_id, error=SubscriptionNotFound) def iter_documents(self, subscription): - if not isinstance(subscription, Subscription): - subscription = self.get_subscription(subscription) return self.browser.iter_documents(subscription) - def get_document(self, id): - bill = self.browser.get_document(id) - if not bill: - raise DocumentNotFound() - else: - return bill - - def download_document(self, bill): - if not isinstance(bill, Bill): - bill = self.get_document(bill) - response = self.browser.open(bill.url, stream=True) - if not response or response.headers['content-type'] != "application/pdf": - return None - return response.content + def get_document(self, _id): + subid = _id.rsplit('_', 1)[0] + subscription = self.get_subscription(subid) + return find_object(self.iter_documents(subscription), id=_id, error=DocumentNotFound) + + def download_document(self, document): + if not isinstance(document, Document): + document = self.get_document(document) + + return self.browser.open(document.url).content diff --git a/modules/ameli/pages.py b/modules/ameli/pages.py index f60c2d2f41f555f2713a3fe0b6f386570a4f9178..d90467b404199abb43590812998700a34b6541b2 100644 --- a/modules/ameli/pages.py +++ b/modules/ameli/pages.py @@ -1,238 +1,92 @@ -# -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- -# Copyright(C) 2013-2015 Christophe Lampin +# Copyright(C) 2019 Budget Insight # # This file is part of a weboob module. # # This weboob module is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by +# it under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This weboob module is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. +# GNU Lesser General Public License for more details. # -# You should have received a copy of the GNU Affero General Public License +# You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . from __future__ import unicode_literals -from datetime import datetime import re -from decimal import Decimal -from weboob.browser.filters.html import Attr, XPathNotFound -from weboob.browser.pages import HTMLPage, RawPage, LoggedPage -from weboob.capabilities.bill import DocumentTypes, Subscription, Detail, Bill -from .compat.weboob_browser_filters_standard import CleanText, Regexp +from weboob.browser.elements import method, ListElement, ItemElement +from weboob.browser.filters.html import Attr, Link +from .compat.weboob_browser_filters_standard import CleanText, Regexp, CleanDecimal, Currency, Field, Format, Env +from weboob.browser.pages import LoggedPage, HTMLPage, PartialHTMLPage +from weboob.capabilities.bill import Subscription, Bill from weboob.exceptions import BrowserUnavailable +from weboob.tools.date import parse_french_date -# Ugly array to avoid the use of french locale +class LoginPage(HTMLPage): + def login(self, username, password): + form = self.get_form(id='connexioncompte_2connexionCompteForm') + form['connexioncompte_2numSecuriteSociale'] = username + form['connexioncompte_2codeConfidentiel'] = password + form.submit() -FRENCH_MONTHS = ['janvier', 'février', 'mars', 'avril', 'mai', 'juin', 'juillet', 'août', 'septembre', 'octobre', 'novembre', 'décembre'] +class ErrorPage(HTMLPage): + def on_load(self): + msg = CleanText('//div[@id="backgroundId"]//p')(self.doc) + raise BrowserUnavailable(msg) -class AmeliBasePage(HTMLPage): - @property - def logged(self): - if self.doc.xpath('//a[contains(text(), "Déconnexion")]'): - logged = True - else: - logged = False - self.logger.debug('logged: %s' % (logged)) - return logged - def is_error(self): - errors = self.doc.xpath('//*[@id="r_errors"]') - if errors: - return errors[0].text_content() +class SubscriptionPage(LoggedPage, HTMLPage): + @method + class iter_subscriptions(ListElement): + item_xpath = '//div[@id="corps-de-la-page"]//div[@class="tableau"]/div' - errors = CleanText('//p[@class="msg_erreur"]', default='')(self.doc) - if errors: - return errors + class item(ItemElement): + klass = Subscription - errors = CleanText('//div[@class="zone-alerte"]/span')(self.doc) - if errors: - return errors + obj__labelid = Attr('.', 'aria-labelledby') - return False + def obj__birthdate(self): + return CleanText('//button[@id="%s"]//td[@class="dateNaissance"]' % Field('_labelid')(self))(self) + def obj_id(self): + # DON'T TAKE social security number for id because it's a very confidential data, take birth date instead + return ''.join(re.findall(r'\d+', Field('_birthdate')(self))) -class LoginPage(AmeliBasePage): - def login(self, login, password): - form = self.get_form('//form[@name="connexionCompteForm"]') - form['connexioncompte_2numSecuriteSociale'] = login.encode('utf8') - form['connexioncompte_2codeConfidentiel'] = password.encode('utf8') - form.submit() + def obj__param(self): + reversed_date = ''.join(reversed(re.findall(r'\d+', Field('_birthdate')(self)))) + name = CleanText('//button[@id="%s"]//td[@class="nom"]' % Field('_labelid')(self))(self) + return '%s!-!%s!-!1' % (reversed_date, name) - def locate_to_cgu_page(self): - try: - # they've put a head tag inside body, yes i know... - url = Regexp(Attr('//div[@id="connexioncompte_2"]//meta', 'content'), r'url=(.*)')(self.doc) - except XPathNotFound: - # no cgu to validate - return - self.browser.location(url) - - -class CguPage(AmeliBasePage): - def get_cgu(self): - return CleanText('//div[@class="page_nouvelles_cgus"]/p[1]')(self.doc) - - -class HomePage(AmeliBasePage): - pass - - -class AccountPage(AmeliBasePage): - def iter_subscription_list(self): - names_list = self.doc.xpath('//span[@class="NomEtPrenomLabel"]') - fullname = CleanText(newlines=True).filter(names_list[0]) - number = re.sub(r'[^\d]+', '', CleanText('//span[@class="blocNumSecu"]', replace=[(' ', '')])(self.doc)) - sub = Subscription(number) - sub._id = number - sub.label = fullname - firstname = CleanText('//span[@class="prenom-titulaire"]')(self.doc) - sub.subscriber = firstname - yield sub - - -class PaymentsPage(AmeliBasePage): - def get_last_payments_url(self): - begin_date = self.doc.xpath('//input[@id="paiements_1dateDebut"]/@data-mindate')[0] - end_date = self.doc.xpath('//input[@id="paiements_1dateFin"]/@data-maxdate')[0] - url = ('/PortailAS/paiements.do?actionEvt=afficherPaiementsComplementaires&DateDebut=' - + begin_date + '&DateFin=' + end_date + - '&Beneficiaire=tout_selectionner&afficherReleves=false&afficherIJ=false&afficherInva=false' - '&afficherRentes=false&afficherRS=false&indexPaiement=&idNotif=') - return url - - -class LastPaymentsPage(LoggedPage, AmeliBasePage): - def iter_last_payments(self): - elts = self.doc.xpath('//li[@class="rowitem remboursement"]') - for elt in elts: - items = Regexp(CleanText('./@onclick'), r".*ajaxCallRemoteChargerDetailPaiement \('(\w+={0,2})', '(\w+)', '(\d+)', '(\d+)'\).*", '\\1,\\2,\\3,\\4')(elt).split(',') - yield "/PortailAS/paiements.do?actionEvt=chargerDetailPaiements&idPaiement=" + items[0] + "&naturePaiement=" + items[1] + "&indexGroupe=" + items[2] + "&indexPaiement=" + items[3] - - def iter_documents(self, sub): - elts = self.doc.xpath('//li[@class="rowdate"]') - for elt in elts: - try: - elt.xpath('.//a[contains(@id,"lienPDFReleve")]')[0] - except IndexError: - continue - date_str = elt.xpath('.//span[contains(@id,"moisEnCours")]')[0].text - month_str = date_str.split()[0] - date = datetime.strptime(re.sub(month_str, str(FRENCH_MONTHS.index(month_str) + 1), date_str), "%m %Y").date() - bil = Bill() - bil.id = sub._id + "." + date.strftime("%Y%m") - bil.date = date - bil.format = 'pdf' - bil.type = DocumentTypes.BILL - bil.label = date.strftime("%Y%m%d") - bil.url = '/PortailAS/PDFServletReleveMensuel.dopdf?PDF.moisRecherche=' + date.strftime("%m%Y") - yield bil - - def get_document(self, bill): - self.location(bill.url, params=bill._args) - - -class PaymentDetailsPage(AmeliBasePage): - def iter_payment_details(self, sub): - id_str = self.doc.xpath('//div[@class="entete container"]/h2')[0].text.strip() - m = re.match(r'.*le (.*) pour un montant de.*', id_str) - if m: - blocs_benes = self.doc.xpath('//span[contains(@id,"nomBeneficiaire")]') - blocs_prestas = self.doc.xpath('//table[@id="tableauPrestation"]') - i = 0 - last_bloc = len(blocs_benes) - for i in range(0, last_bloc): - bene = blocs_benes[i].text - id_str = m.group(1) - id_date = datetime.strptime(id_str, '%d/%m/%Y').date() - id = sub._id + "." + datetime.strftime(id_date, "%Y%m%d") - table = blocs_prestas[i].xpath('.//tr') - line = 1 - last_date = None - for tr in table: - tds = tr.xpath('.//td') - if len(tds) == 0: - continue - - det = Detail() - - # TO TEST : Indemnités journalières : Pas pu tester de cas de figure similaire dans la nouvelle mouture du site - if len(tds) == 4: - date_str = Regexp(pattern=r'.*
(\d+/\d+/\d+)\).*').filter(tds[0].text) - det.id = id + "." + str(line) - det.label = tds[0].xpath('.//span')[0].text.strip() - - jours = tds[1].text - if jours is None: - jours = '0' - - montant = tds[2].text - if montant is None: - montant = '0' - - price = tds[3].text - if price is None: - price = '0' - - if date_str is None or date_str == '': - det.infos = '' - det.datetime = last_date - else: - det.infos = date_str + ' (' + re.sub(r'[^\d,-]+', '', jours) + 'j) * ' + re.sub(r'[^\d,-]+', '', montant) + '€' - det.datetime = datetime.strptime(date_str.split(' ')[3], '%d/%m/%Y').date() - last_date = det.datetime - det.price = Decimal(re.sub(r'[^\d,-]+', '', price).replace(',', '.')) - - if len(tds) == 5: - date_str = Regexp(pattern=r'\w*(\d{2})/(\d{2})/(\d{4}).*', template='\\1/\\2/\\3', default="").filter("".join(tds[0].itertext())) - det.id = id + "." + str(line) - det.label = bene + ' - ' + tds[0].xpath('.//span')[0].text.strip() - - paye = tds[1].text - if paye is None: - paye = '0' - - base = tds[2].text - if base is None: - base = '0' - - tdtaux = tds[3].xpath('.//span')[0].text - if tdtaux is None: - taux = '0' - else: - taux = tdtaux.strip() - - tdprice = tds[4].xpath('.//span')[0].text - if tdprice is None: - price = '0' - else: - price = tdprice.strip() - - if date_str is None or date_str == '': - det.infos = '' - det.datetime = last_date - else: - det.infos = ' Payé ' + re.sub(r'[^\d,-]+', '', paye) + '€ / Base ' + re.sub(r'[^\d,-]+', '', base) + '€ / Taux ' + re.sub(r'[^\d,-]+', '', taux) + '%' - det.datetime = datetime.strptime(date_str, '%d/%m/%Y').date() - last_date = det.datetime - det.price = Decimal(re.sub(r'[^\d,-]+', '', price).replace(',', '.')) - line = line + 1 - yield det - - -class Raw(LoggedPage, RawPage): - pass - - -class UnavailablePage(HTMLPage): - def on_load(self): - raise BrowserUnavailable(CleanText('//span[@class="texte-indispo"]')(self.doc)) + obj_subscriber = CleanText('.//span[@class="NomEtPrenomLabel"]') + obj_label = obj_subscriber + + +class DocumentsPage(LoggedPage, PartialHTMLPage): + @method + class iter_documents(ListElement): + item_xpath = '//ul[@id="unordered_list"]//li[has-class("rowitem")]' + + class item(ItemElement): + klass = Bill + + obj_id = Format('%s_%s', Env('subid'), Regexp(Field('url'), r'idPaiement=(.*)')) + obj_label = CleanText('.//div[has-class("col-label")]') + obj_price = CleanDecimal.French('.//div[has-class("col-montant")]/span') + obj_currency = Currency('.//div[has-class("col-montant")]/span') + obj_url = Link('.//a[@class="downdetail"]') + obj_format = 'pdf' + + def obj_date(self): + year = Regexp(CleanText('./preceding-sibling::li[@class="rowdate"]//span[@class="mois"]'), r'(\d+)')(self) + day_month = CleanText('.//div[has-class("col-date")]/span')(self) + + return parse_french_date(day_month + ' ' + year) diff --git a/modules/banquepopulaire/pages.py b/modules/banquepopulaire/pages.py index c0cac413503a847bb6558f876584b0411e2cfcb1..6669e79db84f2a1f9293be3e2a48274ca9ec283d 100644 --- a/modules/banquepopulaire/pages.py +++ b/modules/banquepopulaire/pages.py @@ -540,18 +540,20 @@ class AccountsPage(LoggedPage, MyHTMLPage): u'Synthèse': None, # ignore this title } - PATTERN = [(re.compile('.*Titres Pea.*'), Account.TYPE_PEA), - (re.compile(".*Plan D'epargne En Actions.*"), Account.TYPE_PEA), - (re.compile(".*Compte Especes Pea.*"), Account.TYPE_PEA), - (re.compile('.*Plan Epargne Retraite.*'), Account.TYPE_PERP), - (re.compile('.*Titres.*'), Account.TYPE_MARKET), - (re.compile('.*Selection Vie.*'),Account.TYPE_LIFE_INSURANCE), - (re.compile('^Fructi Pulse.*'), Account.TYPE_MARKET), - (re.compile('^(Quintessa|Solevia).*'), Account.TYPE_MARKET), - (re.compile('^Plan Epargne Enfant Mul.*'), Account.TYPE_MARKET), - (re.compile('^Alc Premium'), Account.TYPE_MARKET), - (re.compile('^Plan Epargne Enfant Msu.*'), Account.TYPE_LIFE_INSURANCE), - ] + PATTERN = [ + (re.compile(r'.*Titres Pea.*'), Account.TYPE_PEA), + (re.compile(r".*Plan D'epargne En Actions.*"), Account.TYPE_PEA), + (re.compile(r".*Compte Especes Pea.*"), Account.TYPE_PEA), + (re.compile(r'.*Plan Epargne Retraite.*'), Account.TYPE_PERP), + (re.compile(r'.*Titres.*'), Account.TYPE_MARKET), + (re.compile(r'.*Selection Vie.*'), Account.TYPE_LIFE_INSURANCE), + (re.compile(r'^Fructi Pulse.*'), Account.TYPE_MARKET), + (re.compile(r'^(Quintessa|Solevia).*'), Account.TYPE_MARKET), + (re.compile(r'^Plan Epargne Enfant Mul.*'), Account.TYPE_MARKET), + (re.compile(r'^Alc Premium'), Account.TYPE_MARKET), + (re.compile(r'^Plan Epargne Enfant Msu.*'), Account.TYPE_LIFE_INSURANCE), + (re.compile(r'^Parts Sociales.*'), Account.TYPE_MARKET), + ] def pop_up(self): if self.doc.xpath('//span[contains(text(), "du navigateur Internet.")]'): diff --git a/modules/bnpcards/browser.py b/modules/bnpcards/browser.py index 92e680c76f1b9e5e201cd8f94c11da19c3cf2201..2098a681416bec2214fb72fbbe70164fd23f0006 100644 --- a/modules/bnpcards/browser.py +++ b/modules/bnpcards/browser.py @@ -109,26 +109,28 @@ def iter_accounts(self): account.coming = self.page.get_balance() yield account if self.type == '2': - for rib in self.page.get_rib_list(): + for company in self.page.get_companies(): self.accounts.stay_or_go() - self.page.expand(rib=rib) - - accounts = list(self.page.iter_accounts(rib=rib)) - ids = {} - prev_rib = None - for account in accounts: - if account.id in ids: - self.logger.warning('duplicate account %r', account.id) - account.id += '_%s' % ''.join(account.label.split()) - - if prev_rib != account._rib: - self.coming.go() - self.page.expand(rib=account._rib) - account.coming = self.page.get_balance(account) - prev_rib = account._rib - - ids[account.id] = account - yield account + self.page.expand(company=company) + for rib in self.page.get_rib_list(): + self.page.expand(rib=rib, company=company) + + accounts = list(self.page.iter_accounts(rib=rib, company=company)) + ids = {} + prev_rib = None + for account in accounts: + if account.id in ids: + self.logger.warning('duplicate account %r', account.id) + account.id += '_%s' % ''.join(account.label.split()) + + if prev_rib != account._rib: + self.coming.go() + self.page.expand(rib=account._rib, company=account._company) + account.coming = self.page.get_balance(account) + prev_rib = account._rib + + ids[account.id] = account + yield account # Could be the very same as non corporate but this shitty website seems # completely bugged @@ -136,34 +138,34 @@ def get_ti_corporate_transactions(self, account): if account.id not in self.transactions_dict: self.transactions_dict[account.id] = [] self.ti_histo_go() - self.page.expand(self.page.get_periods()[0], account=account) + self.page.expand(self.page.get_periods()[0], account=account, company=account._company) for tr in sorted_transactions(self.page.get_history()): self.transactions_dict[account.id].append(tr) return self.transactions_dict[account.id] def get_ti_transactions(self, account): self.ti_card_go() - self.page.expand(account=account) + self.page.expand(account=account, company=account._company) for tr in sorted_transactions(self.page.get_history()): yield tr self.ti_histo_go() - self.page.expand(self.page.get_periods()[0], account=account) + self.page.expand(self.page.get_periods()[0], account=account, company=account._company) for period in self.page.get_periods(): - self.page.expand(period, account=account) + self.page.expand(period, account=account, company=account._company) for tr in sorted_transactions(self.page.get_history()): yield tr def get_ge_transactions(self, account): transactions = [] self.coming.go() - self.page.expand(rib=account._rib) + self.page.expand(account=account, rib=account._rib, company=account._company) link = self.page.get_link(account) if link: self.location(link) transactions += self.page.get_history() self.history.go() for period in self.page.get_periods(): - self.page.expand(period, rib=account._rib) + self.page.expand(period, rib=account._rib, company=account._company, account=account) link = self.page.get_link(account) if link: self.location(link) diff --git a/modules/bnpcards/pages.py b/modules/bnpcards/pages.py index 0a99da6aa1d364d0a890dc026213a18887756e85..b90c3cbf6ce9b4917d9b77fa04aa21f1adcf8b26 100644 --- a/modules/bnpcards/pages.py +++ b/modules/bnpcards/pages.py @@ -49,12 +49,18 @@ def login(self, type, username, password): class ExpandablePage(LoggedPage, HTMLPage): - def expand(self, account=None, rib=None): + def expand(self, account=None, rib=None, company=None): form = self.get_form() if rib is not None: form['ribSaisi'] = rib if account is not None: - form['numCarteSaisi'] = account._nav_num + form['nomCarteSaisi'] = account.label # some forms use 'nomCarteSaisi' some use 'titulaireSaisie' + form['titulaireSaisie'] = account.label + if company is not None: + if 'entrepriseSaisie' in form.keys(): # some forms use 'entrepriseSaisie' some use 'entrepriseSaisi' + form['entrepriseSaisie'] = company + else: + form['entrepriseSaisi'] = company # needed if coporate titulaire form.url = form.url.replace('Appliquer', 'Afficher') form.submit() @@ -80,13 +86,16 @@ def get_periods(self): periods.append(period) return periods - def expand(self, period, account=None, rib=None): + def expand(self, period, account=None, rib=None, company=None): form = self.get_form(submit='//input[@value="Display"]') if account is not None: - form['numCarteSaisi'] = account._nav_num + form['nomCarteSaisi'] = account.label + form['titulaireSaisi'] = account.label form['periodeSaisie'] = period if rib is not None: form['ribSaisi'] = rib + if company is not None: + form['entrepriseSaisi'] = company # needed if coporate titulaire form.url = form.url.replace('Appliquer', 'Afficher') form.submit() @@ -109,6 +118,7 @@ class item(ItemElement): obj_label = CleanText('./td[1]') obj_type = Account.TYPE_CARD obj__rib = Env('rib') + obj__company = Env('company') obj_currency = u'EUR' obj_number = CleanText('./td[2]', replace=[(' ', '')]) obj_url = AbsoluteLink('./td[2]/a') @@ -118,6 +128,9 @@ class item(ItemElement): def store(self, obj): return obj + def get_companies(self): + return self.doc.xpath('//select[@name="entrepriseSaisie"]/option/@value') + class ComingPage(ExpandablePage): def get_link(self, account): diff --git a/modules/bnporc/pp/pages.py b/modules/bnporc/pp/pages.py index b46c656fae3d60aaa0573ff3ceeaf1ff0cb69c5f..49f18c735d2bb85f59b782186f9e11f85342b114 100644 --- a/modules/bnporc/pp/pages.py +++ b/modules/bnporc/pp/pages.py @@ -334,6 +334,7 @@ class item(ItemElement): 'Crédit immobilier': Account.TYPE_MORTGAGE, 'Réserve Provisio': Account.TYPE_REVOLVING_CREDIT, 'Prêt personnel': Account.TYPE_CONSUMER_CREDIT, + 'Crédit Silo': Account.TYPE_REVOLVING_CREDIT, } klass = Account diff --git a/modules/boursorama/browser.py b/modules/boursorama/browser.py index 17c74758737319e8d817a92d7ff2c9f206382f7c..d5e9b141b1094bd374bc07d714edff7b2b8d80d8 100644 --- a/modules/boursorama/browser.py +++ b/modules/boursorama/browser.py @@ -333,20 +333,18 @@ def get_history(self, account, coming=False): return self.get_regular_transactions(account, coming) def get_regular_transactions(self, account, coming): - # We look for 3 years of history. - params = {} - params['movementSearch[toDate]'] = (date.today() + relativedelta(days=40)).strftime('%d/%m/%Y') - params['movementSearch[fromDate]'] = (date.today() - relativedelta(years=3)).strftime('%d/%m/%Y') - params['movementSearch[selectedAccounts][]'] = account._webid if not coming: + # We look for 3 years of history. + params = {} + params['movementSearch[toDate]'] = (date.today() + relativedelta(days=40)).strftime('%d/%m/%Y') + params['movementSearch[fromDate]'] = (date.today() - relativedelta(years=3)).strftime('%d/%m/%Y') + params['movementSearch[selectedAccounts][]'] = account._webid self.location('%s/mouvements' % account.url.rstrip('/'), params=params) for transaction in self.page.iter_history(): yield transaction - elif account.type == Account.TYPE_CHECKING: - self.location('%s/mouvements-a-venir' % account.url.rstrip('/'), params=params) - for transaction in self.page.iter_history(coming=True): - yield transaction + # Note: Checking accounts have a 'Mes prélèvements à venir' tab, + # but these transactions have no date anymore so we ignore them. def get_card_transactions(self, account, coming): # All card transactions can be found in the CSV (history and coming), diff --git a/modules/boursorama/pages.py b/modules/boursorama/pages.py index 6f47e9bea2e49d38b15eefc1f29b51f5812e7a1d..bf9e1ca136c4bcb613960cf09e534351c6838820 100644 --- a/modules/boursorama/pages.py +++ b/modules/boursorama/pages.py @@ -460,7 +460,6 @@ def next_page(self): class item(ItemElement): klass = Transaction - obj_date = Date(Attr('.//time', 'datetime')) obj_amount = CleanDecimal('.//div[has-class("list__movement__line--amount")]', replace_dots=True) obj_category = CleanText('.//span[has-class("category")]') obj__account_name = CleanText('.//span[contains(@class, "account__name-xs")]', default=None) diff --git a/modules/bp/browser.py b/modules/bp/browser.py index e93892e8d3f906df42d9cf0942c181462b8998c6..4b11b75474572985a3d1e07efe234379aaa3ed46 100644 --- a/modules/bp/browser.py +++ b/modules/bp/browser.py @@ -617,7 +617,7 @@ def get_accounts_list(self): self.location(self.accounts_url) assert self.pro_accounts_list.is_here() - for account in self.page.get_accounts_list(): + for account in self.page.iter_accounts(): ids.add(account.id) accounts.append(account) @@ -625,7 +625,7 @@ def get_accounts_list(self): self.location(self.accounts_and_loans_url) assert self.pro_accounts_list.is_here() - for account in self.page.get_accounts_list(): + for account in self.page.iter_accounts(): if account.id not in ids: ids.add(account.id) accounts.append(account) diff --git a/modules/bp/pages/accounthistory.py b/modules/bp/pages/accounthistory.py index 9b681853844de72d8952e7d8ad802466359d15ca..c03c9e39c9c7595927fde84d3ae5259fd3f86012 100644 --- a/modules/bp/pages/accounthistory.py +++ b/modules/bp/pages/accounthistory.py @@ -39,31 +39,26 @@ class Transaction(FrenchTransaction): - PATTERNS = [(re.compile(u'^(?PCHEQUE)( N)? (?P.*)'), - FrenchTransaction.TYPE_CHECK), - (re.compile(r'^(?PACHAT CB) (?P.*) (?P
\d{2})\.(?P\d{2}).(?P\d{2,4}).*'), - FrenchTransaction.TYPE_CARD), - (re.compile('^(?P(PRELEVEMENT DE|TELEREGLEMENT|TIP)) (?P.*)'), - FrenchTransaction.TYPE_ORDER), - (re.compile('^(?PECHEANCEPRET)(?P.*)'), - FrenchTransaction.TYPE_LOAN_PAYMENT), - (re.compile(r'^CARTE \w+ (?P
\d{2})/(?P\d{2})/(?P\d{2,4}) A \d+H\d+ (?PRETRAIT DAB) (?P.*)'), - FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile(r'^(?PRETRAIT DAB) (?P
\d{2})/(?P\d{2})/(?P\d{2,4}) \d+H\d+ (?P.*)'), - FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile(r'^(?PRETRAIT) (?P.*) (?P
\d{2})\.(?P\d{2})\.(?P\d{2,4})'), - FrenchTransaction.TYPE_WITHDRAWAL), - (re.compile('^(?PVIR(EMEN)?T?) (DE |POUR )?(?P.*)'), - FrenchTransaction.TYPE_TRANSFER), - (re.compile('^(?PREMBOURST)(?P.*)'), FrenchTransaction.TYPE_PAYBACK), - (re.compile('^(?PCOMMISSIONS)(?P.*)'), FrenchTransaction.TYPE_BANK), - (re.compile('^(?PFRAIS POUR)(?P.*)'), FrenchTransaction.TYPE_BANK), - (re.compile('^(?P(?PREMUNERATION).*)'), FrenchTransaction.TYPE_BANK), - (re.compile('^(?PREMISE DE CHEQUES?) (?P.*)'), FrenchTransaction.TYPE_DEPOSIT), - (re.compile(u'^(?PDEBIT CARTE BANCAIRE DIFFERE.*)'), FrenchTransaction.TYPE_CARD_SUMMARY), - (re.compile('^COTISATION TRIMESTRIELLE.*'), FrenchTransaction.TYPE_BANK), - (re.compile('^(?PFRAIS (TRIMESTRIELS) DE TENUE DE COMPTE.*)'), FrenchTransaction.TYPE_BANK) - ] + PATTERNS = [ + (re.compile(r'^(?PCHEQUE)( N)? (?P.*)'), FrenchTransaction.TYPE_CHECK), + (re.compile(r'^(?PACHAT CB) (?P.*) (?P
\d{2})\.(?P\d{2}).(?P\d{2,4}).*'), FrenchTransaction.TYPE_CARD), + (re.compile(r'^(?P(PRELEVEMENT DE|TELEREGLEMENT|TIP)) (?P.*)'), FrenchTransaction.TYPE_ORDER), + (re.compile(r'^(?PECHEANCEPRET)(?P.*)'), FrenchTransaction.TYPE_LOAN_PAYMENT), + (re.compile(r'^CARTE \w+ (?P
\d{2})/(?P\d{2})/(?P\d{2,4}) A \d+H\d+ (?PRETRAIT DAB) (?P.*)'), FrenchTransaction.TYPE_WITHDRAWAL), + (re.compile(r'^(?PRETRAIT DAB) (?P
\d{2})/(?P\d{2})/(?P\d{2,4}) \d+H\d+ (?P.*)'), FrenchTransaction.TYPE_WITHDRAWAL), + (re.compile(r'^(?PRETRAIT) (?P.*) (?P
\d{2})\.(?P\d{2})\.(?P\d{2,4})'), FrenchTransaction.TYPE_WITHDRAWAL), + (re.compile(r'^(?PVIR(EMEN)?T?) (DE |POUR )?(?P.*)'), FrenchTransaction.TYPE_TRANSFER), + (re.compile(r'^(?PREMBOURST)(?P.*)'), FrenchTransaction.TYPE_PAYBACK), + (re.compile(r'^(?PCOMMISSIONS)(?P.*)'), FrenchTransaction.TYPE_BANK), + (re.compile(r'^(?PFRAIS POUR)(?P.*)'), FrenchTransaction.TYPE_BANK), + (re.compile(r'^(?P(?PREMUNERATION).*)'), FrenchTransaction.TYPE_BANK), + (re.compile(r'^(?PREMISE DE CHEQUES?) (?P.*)'), FrenchTransaction.TYPE_DEPOSIT), + (re.compile(r'^(?PDEBIT CARTE BANCAIRE DIFFERE.*)'), FrenchTransaction.TYPE_CARD_SUMMARY), + (re.compile(r'^COTISATION TRIMESTRIELLE.*'), FrenchTransaction.TYPE_BANK), + (re.compile(r'^REMISE COMMERCIALE.*'), FrenchTransaction.TYPE_BANK), + (re.compile(r'^.*UTILISATION DU DECOUVERT$'), FrenchTransaction.TYPE_BANK), + (re.compile(r'^(?PFRAIS (TRIMESTRIELS) DE TENUE DE COMPTE.*)'), FrenchTransaction.TYPE_BANK) + ] class AccountHistory(LoggedPage, MyHTMLPage): diff --git a/modules/bp/pages/pro.py b/modules/bp/pages/pro.py index f98a6b6c76404796b5ccaac95a23b9dff2ce8c3a..363bb4b7486fed8e82220f9001088ab95fe33b76 100644 --- a/modules/bp/pages/pro.py +++ b/modules/bp/pages/pro.py @@ -17,17 +17,15 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . -import re -from decimal import Decimal +from __future__ import unicode_literals from weboob.browser.elements import ListElement, ItemElement, method -from .compat.weboob_browser_filters_standard import CleanText, CleanDecimal, Date -from weboob.browser.filters.html import Link +from .compat.weboob_browser_filters_standard import CleanText, CleanDecimal, Coalesce, Currency, Date, Map, Field, Regexp +from weboob.browser.filters.html import AbsoluteLink, Link from weboob.browser.pages import LoggedPage, pagination from .compat.weboob_capabilities_bank import Account from weboob.capabilities.profile import Company from weboob.capabilities.base import NotAvailable -from weboob.tools.compat import urljoin, unicode from weboob.exceptions import BrowserUnavailable from .accounthistory import Transaction @@ -36,55 +34,55 @@ class RedirectPage(LoggedPage, MyHTMLPage): def check_for_perso(self): - return self.doc.xpath(u'//p[contains(text(), "L\'identifiant utilisé est celui d\'un compte de Particuliers")]') + return self.doc.xpath('''//p[contains(text(), "L'identifiant utilisé est celui d'un compte de Particuliers")]''') + + +ACCOUNT_TYPES = { + 'Comptes titres': Account.TYPE_MARKET, + 'Comptes épargne': Account.TYPE_SAVINGS, + 'Comptes courants': Account.TYPE_CHECKING, +} class ProAccountsList(LoggedPage, MyHTMLPage): + + # TODO Be careful about connections with personnalized account groups + # According to their presentation video (https://www.labanquepostale.fr/pmo/nouvel-espace-client-business.html), + # on the new website people are able to make personnalized groups of account instead of the usual drop-down categories on which to parse to find a match in ACCOUNT_TYPES + # If clients use the functionnality we might need to add entries new in ACCOUNT_TYPES + def on_load(self): if self.doc.xpath('//div[@id="erreur_generale"]'): - raise BrowserUnavailable(CleanText(u'//div[@id="erreur_generale"]//p[contains(text(), "Le service est momentanément indisponible")]')(self.doc)) - - ACCOUNT_TYPES = {u'comptes titres': Account.TYPE_MARKET, - u'comptes épargne': Account.TYPE_SAVINGS, - # wtf? ^ - u'comptes épargne': Account.TYPE_SAVINGS, - u'comptes courants': Account.TYPE_CHECKING, - } - def get_accounts_list(self): - for table in self.doc.xpath('//div[@class="comptestabl"]/table'): - try: - account_type = self.ACCOUNT_TYPES[table.get('summary').lower()] - if not account_type: - account_type = self.ACCOUNT_TYPES[table.xpath('./caption/text()')[0].strip().lower()] - except (IndexError,KeyError): - account_type = Account.TYPE_UNKNOWN - for tr in table.xpath('./tbody/tr'): - cols = tr.findall('td') - - link = cols[0].find('a') - if link is None: - continue - - a = Account() - a.type = account_type - a.id = unicode(re.search('([A-Z\d]{4}[A-Z\d\*]{3}[A-Z\d]{4})', link.attrib['title']).group(1)) - a.label = unicode(link.attrib['title'].replace('%s ' % a.id, '')) - # We use '.text_content()' to avoid HTML comments like '' - tmp_balance = CleanText(None).filter(cols[1].text_content()) - a.currency = a.get_currency(tmp_balance) - if not a.currency: - a.currency = u'EUR' - a.balance = Decimal(Transaction.clean_amount(tmp_balance)) - a._has_cards = False - a.url = urljoin(self.url, link.attrib['href']) - yield a + raise BrowserUnavailable(CleanText('//div[@id="erreur_generale"]//p[contains(text(), "Le service est momentanément indisponible")]')(self.doc)) + + def is_here(self): + return CleanText('//h1[contains(text(), "Synthèse des comptes")]')(self.doc) + + @method + class iter_accounts(ListElement): + item_xpath = '//div[@id="mainContent"]//div[h3/a]' + + class item(ItemElement): + klass = Account + + obj_id = Regexp(CleanText('./h3/a/@title'), r'([A-Z\d]{4}[A-Z\d\*]{3}[A-Z\d]{4})') + obj_balance = CleanDecimal.French('./span/text()[1]') # This website has the good taste of leaving hard coded HTML comments. This is the way to pin point to the righ text item. + obj_currency = Currency('./span') + obj_url = AbsoluteLink('./h3/a') + + # account are grouped in /div based on their type, we must fetch the closest one relative to item_xpath + obj_type = Map(CleanText('./ancestor::div[1]/preceding-sibling::h2[1]/button/div[@class="title-accordion"]'), ACCOUNT_TYPES, Account.TYPE_UNKNOWN) + + def obj_label(self): + """ Need to get rid of the id wherever we find it in account labels like "LIV A 0123456789N MR MOMO" (livret A) as well as "0123456789N MR MOMO" (checking account) """ + return CleanText('./h3/a/@title')(self).replace('%s ' % Field('id')(self), '') class ProAccountHistory(LoggedPage, MyHTMLPage): @pagination @method class iter_history(ListElement): - item_xpath = u'//div[@id="tabReleve"]//tbody/tr' + item_xpath = '//div[@id="tabReleve"]//tbody/tr' def next_page(self): # The next page on the website can return pages already visited without logical mechanism @@ -96,12 +94,12 @@ def next_page(self): next_page_link = Link(next_page_xpath)(self.el) next_page = self.page.browser.location(next_page_link) first_transaction = CleanText(tr_xpath)(next_page.page.doc) - count = 0 # avoid an infinite loop + count = 0 # avoid an infinite loop while first_transaction in self.page.browser.first_transactions and count < 30: next_page = self.page.browser.location(next_page_link) next_page_link = Link(next_page_xpath)(next_page.page.doc) - first_transaction = CleanText(tr_xpath)(next_page.page.doc) + first_transaction = CleanText(tr_xpath)(next_page.page.doc) count += 1 if count < 30: @@ -112,13 +110,15 @@ class item(ItemElement): obj_date = Date(CleanText('.//td[@headers="date"]'), dayfirst=True) obj_raw = Transaction.Raw('.//td[@headers="libelle"]') - obj_amount = CleanDecimal('.//td[@headers="debit" or @headers="credit"]', - replace_dots=True, default=NotAvailable) + obj_amount = Coalesce( + CleanDecimal.French('.//td[@headers="debit"]', default=NotAvailable), + CleanDecimal.French('.//td[@headers="credit"]', default=NotAvailable), + ) class DownloadRib(LoggedPage, MyHTMLPage): def get_rib_value(self, acc_id): - opt = self.doc.xpath('//div[@class="rechform"]//option') + opt = self.doc.xpath('//select[@id="idxSelection"]/optgroup//option') for o in opt: if acc_id in o.text: return o.xpath('./@value')[0] diff --git a/modules/bp/pages/subscription.py b/modules/bp/pages/subscription.py index c51140b51ecfb324eb44862ecaa86fb9b7430ae1..da8d70845c62335eb782b6db5532b82dcbb3e7c0 100644 --- a/modules/bp/pages/subscription.py +++ b/modules/bp/pages/subscription.py @@ -109,7 +109,7 @@ def get_content(self): class ProSubscriptionPage(LoggedPage, HTMLPage): @method class iter_subscriptions(ListElement): - item_xpath = '//select[@id="numeroCompteRechercher"]/option' + item_xpath = '//select[@id="numeroCompteRechercher"]/option[not(@disabled)]' class item(ItemElement): klass = Subscription diff --git a/modules/bred/bred/browser.py b/modules/bred/bred/browser.py index d5877aba3f57ce09aa8a9afb157132a72cc40851..a07a65455e7165c3f8805ac0774cfa0e4bb895c2 100644 --- a/modules/bred/bred/browser.py +++ b/modules/bred/bred/browser.py @@ -135,9 +135,6 @@ def get_loans_list(self): def get_list(self): self.accounts.go() for acc in self.page.iter_accounts(accnum=self.accnum, current_univers=self.current_univers): - if acc.type == Account.TYPE_CHECKING: - self.iban.go(number=acc._number) - self.page.set_iban(account=acc) yield acc @need_login @@ -227,3 +224,9 @@ def get_profile(self): self.page.set_email(profile=profile) return profile + + @need_login + def fill_account(self, account, fields): + if account.type == Account.TYPE_CHECKING and 'iban' in fields: + self.iban.go(number=account._number) + self.page.set_iban(account=account) diff --git a/modules/bred/module.py b/modules/bred/module.py index a8cdd6a52f560bf5bb1f4bccc334de6db332e894..0f7df0441c2b1c96a6a653933d17a940e13b16af 100644 --- a/modules/bred/module.py +++ b/modules/bred/module.py @@ -18,7 +18,7 @@ # along with this weboob module. If not, see . -from .compat.weboob_capabilities_bank import CapBankWealth, AccountNotFound +from .compat.weboob_capabilities_bank import CapBankWealth, AccountNotFound, Account from weboob.capabilities.base import find_object from weboob.capabilities.profile import CapProfile from weboob.tools.backend import Module, BackendConfig @@ -73,3 +73,13 @@ def iter_investment(self, account): def get_profile(self): return self.browser.get_profile() + + def fill_account(self, account, fields): + if self.config['website'].get() != 'bred': + return + + self.browser.fill_account(account, fields) + + OBJECTS = { + Account: fill_account, + } diff --git a/modules/caissedepargne/pages.py b/modules/caissedepargne/pages.py index 68ed822e35cc7079c3c5e420762ec9474d18af92..035972dcbd61292b81e67e5a602ee5674a8f0da6 100644 --- a/modules/caissedepargne/pages.py +++ b/modules/caissedepargne/pages.py @@ -162,35 +162,36 @@ class Transaction(FrenchTransaction): ] class IndexPage(LoggedPage, HTMLPage): - ACCOUNT_TYPES = {u'Epargne liquide': Account.TYPE_SAVINGS, - u'Compte Courant': Account.TYPE_CHECKING, - u'COMPTE A VUE': Account.TYPE_CHECKING, - u'COMPTE CHEQUE': Account.TYPE_CHECKING, - u'Mes comptes': Account.TYPE_CHECKING, - u'CPT DEPOT PART.': Account.TYPE_CHECKING, - u'CPT DEPOT PROF.': Account.TYPE_CHECKING, - u'Mon épargne': Account.TYPE_SAVINGS, - u'Mes autres comptes': Account.TYPE_SAVINGS, - u'Compte Epargne et DAT': Account.TYPE_SAVINGS, - u'Plan et Contrat d\'Epargne': Account.TYPE_SAVINGS, - u'COMPTE SUR LIVRET': Account.TYPE_SAVINGS, - u'LIVRET DEV.DURABLE': Account.TYPE_SAVINGS, - u'LDD Solidaire': Account.TYPE_SAVINGS, - u'LIVRET A': Account.TYPE_SAVINGS, - u'LIVRET JEUNE': Account.TYPE_SAVINGS, - u'LIVRET GRAND PRIX': Account.TYPE_SAVINGS, - u'LEP': Account.TYPE_SAVINGS, - u'LEL': Account.TYPE_SAVINGS, - u'CPT PARTS SOCIALES': Account.TYPE_SAVINGS, - u'PEL 16 2013': Account.TYPE_SAVINGS, - u'Titres': Account.TYPE_MARKET, - u'Compte titres': Account.TYPE_MARKET, - u'Mes crédits immobiliers': Account.TYPE_LOAN, - u'Mes crédits renouvelables': Account.TYPE_LOAN, - u'Mes crédits consommation': Account.TYPE_LOAN, - u'PEA NUMERAIRE': Account.TYPE_PEA, - u'PEA': Account.TYPE_PEA, - } + ACCOUNT_TYPES = { + 'Epargne liquide': Account.TYPE_SAVINGS, + 'Compte Courant': Account.TYPE_CHECKING, + 'COMPTE A VUE': Account.TYPE_CHECKING, + 'COMPTE CHEQUE': Account.TYPE_CHECKING, + 'Mes comptes': Account.TYPE_CHECKING, + 'CPT DEPOT PART.': Account.TYPE_CHECKING, + 'CPT DEPOT PROF.': Account.TYPE_CHECKING, + 'Mon épargne': Account.TYPE_SAVINGS, + 'Mes autres comptes': Account.TYPE_SAVINGS, + 'Compte Epargne et DAT': Account.TYPE_SAVINGS, + 'Plan et Contrat d\'Epargne': Account.TYPE_SAVINGS, + 'COMPTE SUR LIVRET': Account.TYPE_SAVINGS, + 'LIVRET DEV.DURABLE': Account.TYPE_SAVINGS, + 'LDD Solidaire': Account.TYPE_SAVINGS, + 'LIVRET A': Account.TYPE_SAVINGS, + 'LIVRET JEUNE': Account.TYPE_SAVINGS, + 'LIVRET GRAND PRIX': Account.TYPE_SAVINGS, + 'LEP': Account.TYPE_SAVINGS, + 'LEL': Account.TYPE_SAVINGS, + 'CPT PARTS SOCIALES': Account.TYPE_MARKET, + 'PEL 16 2013': Account.TYPE_SAVINGS, + 'Titres': Account.TYPE_MARKET, + 'Compte titres': Account.TYPE_MARKET, + 'Mes crédits immobiliers': Account.TYPE_LOAN, + 'Mes crédits renouvelables': Account.TYPE_LOAN, + 'Mes crédits consommation': Account.TYPE_LOAN, + 'PEA NUMERAIRE': Account.TYPE_PEA, + 'PEA': Account.TYPE_PEA, + } def build_doc(self, content): content = content.strip(b'\x00') diff --git a/modules/cragr/regions/browser.py b/modules/cragr/regions/browser.py index 7361c7e3904d85e21424e38361425dd68b469de9..738353886a906ea0b9d6d7201ece2675753db9ee 100644 --- a/modules/cragr/regions/browser.py +++ b/modules/cragr/regions/browser.py @@ -273,6 +273,14 @@ def iter_accounts(self): - Multiple perimeters: visit all perimeters one by one and return all accounts. ''' accounts_list = [] + + # Sometimes the URL of the page after login has a session_value=None, + # so we must set it correctly otherwise the next requests will crash. + if not self.session_value: + m = re.search(r'sessionSAG=([^&]+)', self.url) + if m: + self.session_value = m.group(1) + if len(self.perimeters) == 1: self.accounts.stay_or_go(session_value=self.session_value) for account in self.iter_perimeter_accounts(iban=True, all_accounts=True): @@ -530,6 +538,9 @@ def iter_history(self, account, coming=False): # we must skip the ongoing one but fetch the other ones # even if they are in the future. ongoing_coming = self.page.get_ongoing_coming() + if not ongoing_coming: + # This card has no available history or coming. + return card_transactions = [] latest_date = None diff --git a/modules/cragr/regions/pages.py b/modules/cragr/regions/pages.py index e5679e96b2d41d35399513a3c5f7aa00ec52592f..405517e65adb38c143a623a9a886faebe2a40c45 100644 --- a/modules/cragr/regions/pages.py +++ b/modules/cragr/regions/pages.py @@ -196,6 +196,7 @@ def get_iban(self): 'CCHQ': Account.TYPE_CHECKING, 'CCOU': Account.TYPE_CHECKING, 'AUTO ENTRP': Account.TYPE_CHECKING, + 'AUTO ENTRS': Account.TYPE_CHECKING, 'DEVISE USD': Account.TYPE_CHECKING, 'EKO': Account.TYPE_CHECKING, 'DEVISE CHF': Account.TYPE_CHECKING, @@ -417,8 +418,10 @@ def get_ongoing_coming(self): # the coming is positive, it will become 'Opérations créditées' raw_date = Regexp( CleanText('//table[@class="ca-table"]//tr[1]//b[contains(text(), "Opérations débitées") or contains(text(), "Opérations créditées")]'), - r'le (.*) :' + r'le (.*) :', default=None )(self.doc) + if not raw_date: + return None return parse_french_date(raw_date).date() def get_card_transactions(self, latest_date, ongoing_coming): diff --git a/modules/fortuneo/pages/accounts_list.py b/modules/fortuneo/pages/accounts_list.py index 95ed38f973ad55593edb257a2b9b81534bb8d634..02a88af723ce3349abb9705e2923fac8ae921db2 100644 --- a/modules/fortuneo/pages/accounts_list.py +++ b/modules/fortuneo/pages/accounts_list.py @@ -291,6 +291,12 @@ def get_operations(self): amount = tables[i].xpath("./td[5]/text() | ./td[6]/text()") operation.parse(date=date_oper, raw=label, vdate=date_val) + + # There is no difference between card transaction and deferred card transaction + # on the history. + if operation.type == FrenchTransaction.TYPE_CARD: + operation.bdate = operation.rdate + # Needed because operation.parse overwrite operation.label # Theses lines must run after operation.parse. if tables[i].xpath("./td[4]/div/text()"): @@ -327,7 +333,7 @@ def get_operations(self): tr = Transaction() tr.parse(date=date, raw=raw) - tr.rdate = tr.parse_date(rdate) + tr.rdate = tr.bdate = tr.parse_date(rdate) tr.type = tr.TYPE_DEFERRED_CARD if credit: tr.amount = CleanDecimal(None, replace_dots=True).filter(credit) diff --git a/modules/hsbc/pages/account_pages.py b/modules/hsbc/pages/account_pages.py index 688caa394338d9f0a0f28e0f5a8f94c6ab871eee..eb85c1e65d85c15fc733b8f4f3fd13a32cbe39bf 100644 --- a/modules/hsbc/pages/account_pages.py +++ b/modules/hsbc/pages/account_pages.py @@ -46,7 +46,8 @@ class Transaction(FrenchTransaction): (re.compile(r'^DAB (?P
\d{2})/(?P\d{2}) ((?P\d{2})H(?P\d{2}) )?(?P.*?)( CB N°.*)?$'), FrenchTransaction.TYPE_WITHDRAWAL), (re.compile(r'^(IMPAYE REMISE )?CHEQUE( \d+)?'), FrenchTransaction.TYPE_CHECK), (re.compile(r'^IMPAYE REMISE CHEQUE'), FrenchTransaction.TYPE_CHECK), - (re.compile(r'^(COTIS\.?|FRAIS) (?P.*)'), FrenchTransaction.TYPE_BANK), + (re.compile(r'^(COM\.?|COTIS\.?|FRAIS) (?P.*)'), FrenchTransaction.TYPE_BANK), + (re.compile(r'^ARRETE DE COMPTE.*'), FrenchTransaction.TYPE_BANK), (re.compile(r'^REMISE (?P.*)'), FrenchTransaction.TYPE_DEPOSIT), (re.compile(r'^FACTURES CB (?P.*)'), FrenchTransaction.TYPE_CARD_SUMMARY), ] @@ -214,6 +215,8 @@ class item(ItemElement): def obj_balance(self): if Field('type')(self) == Account.TYPE_CARD: return Decimal(0) + elif 'Mes crédits' in CleanText('.//ancestor::div[1]/preceding-sibling::*')(self): + return - abs(Field('_amount')(self)) return Field('_amount')(self) def obj_coming(self): diff --git a/modules/ing/api_browser.py b/modules/ing/api_browser.py index 7890362f797e305a019a656711c46dbcfa502a93..b868a215a93a07069c163390209380f21d1f215d 100644 --- a/modules/ing/api_browser.py +++ b/modules/ing/api_browser.py @@ -27,6 +27,7 @@ from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable, ActionNeeded from weboob.browser.exceptions import ClientError from .compat.weboob_capabilities_bank import TransferBankError, TransferInvalidAmount +from weboob.tools.capabilities.bank.transactions import FrenchTransaction from .api import ( LoginPage, AccountsPage, HistoryPage, ComingPage, @@ -259,6 +260,8 @@ def get_api_history(self, account): for tr in self.page.iter_history(): # transaction id is decreasing first_transaction_id = int(tr._web_id) + if tr.type == FrenchTransaction.TYPE_CARD: + tr.bdate = tr.rdate yield tr # like website, add 1 to the last transaction id of the list to get next transactions page @@ -282,7 +285,10 @@ def get_web_coming(self, account): def get_api_coming(self, account): """iter coming on new website""" self.coming.go(account_uid=account._uid) - return self.page.iter_coming() + for tr in self.page.iter_coming(): + if tr.type == FrenchTransaction.TYPE_CARD: + tr.bdate = tr.rdate + yield tr @need_login def iter_coming(self, account): diff --git a/modules/nef/pages.py b/modules/nef/pages.py index 95b4fae4db4ddc1af66d5bb200269f16986ed2cf..5ceb0fa5a11def7a05b12d7d951088ad5a07eb35 100644 --- a/modules/nef/pages.py +++ b/modules/nef/pages.py @@ -45,9 +45,10 @@ class HomePage(LoggedPage, HTMLPage): pass class AccountsPage(LoggedPage, PartialHTMLPage): - ACCOUNT_TYPES = {re.compile('livret'): Account.TYPE_SAVINGS, - re.compile('parts sociales'): Account.TYPE_SAVINGS, - } + ACCOUNT_TYPES = { + re.compile(r'livret'): Account.TYPE_SAVINGS, + re.compile(r'parts sociales'): Account.TYPE_MARKET, + } @method class get_items(ListElement): diff --git a/modules/okc/browser.py b/modules/okc/browser.py index a12efb841b52a667602458733b0da8be02354fb7..bddc12dcf31fe52c7a3994769de4e2c79ebb6257 100644 --- a/modules/okc/browser.py +++ b/modules/okc/browser.py @@ -17,10 +17,14 @@ # You should have received a copy of the GNU Affero General Public License # along with this weboob module. If not, see . -from weboob.browser import LoginBrowser, URL -from weboob.exceptions import BrowserIncorrectPassword -from weboob.tools.json import json +import re +from weboob.browser.browsers import LoginBrowser, URL +from weboob.browser.browsers import DomainBrowser +from weboob.browser.pages import HTMLPage +from .compat.weboob_browser_filters_standard import CleanText +from weboob.exceptions import BrowserIncorrectPassword, ParseError +from weboob.tools.json import json __all__ = ['OkCBrowser'] @@ -33,24 +37,66 @@ def inner(browser, *args, **kwargs): return inner +class FacebookBrowser(DomainBrowser): + BASEURL = 'https://graph.facebook.com' + + access_token = None + + def login(self, username, password): + self.location('https://www.facebook.com/v2.9/dialog/oauth?app_id=484681304938818&auth_type=rerequest&channel_url=https%3A%2F%2Fstaticxx.facebook.com%2Fconnect%2Fxd_arbiter.php%3Fversion%3D44%23cb%3Df33dd8340f36618%26domain%3Dwww.okcupid.com%26origin%3Dhttps%253A%252F%252Fwww.okcupid.com%252Ff5818a5f355be8%26relation%3Dopener&client_id=484681304938818&display=popup&domain=www.okcupid.com&e2e=%7B%7D&fallback_redirect_uri=https%3A%2F%2Fwww.okcupid.com%2Flogin&locale=en_US&origin=1&redirect_uri=https%3A%2F%2Fstaticxx.facebook.com%2Fconnect%2Fxd_arbiter.php%3Fversion%3D44%23cb%3Df2ce4ca90b82cb4%26domain%3Dwww.okcupid.com%26origin%3Dhttps%253A%252F%252Fwww.okcupid.com%252Ff5818a5f355be8%26relation%3Dopener%26frame%3Df3f40f304ac5e9&response_type=token%2Csigned_request&scope=email%2Cuser_birthday%2Cuser_photos&sdk=joey&version=v2.9') + + page = HTMLPage(self, self.response) + form = page.get_form('//form[@id="login_form"]') + form['email'] = username + form['pass'] = password + self.session.headers['cookie-installing-permission'] = 'required' + self.session.cookies['wd'] = '640x1033' + self.session.cookies['act'] = '1563018648141%2F0' + form.submit(allow_redirects=False) + if 'Location' not in self.response.headers: + raise BrowserIncorrectPassword() + + self.location(self.response.headers['Location']) + + page = HTMLPage(self, self.response) + if len(page.doc.xpath('//td/div[has-class("s")]')) > 0: + raise BrowserIncorrectPassword(CleanText('//td/div[has-class("s")]')(page.doc)) + + script = page.doc.xpath('//script')[0].text + + m = re.search('access_token=([^&]+)&', script) + if m: + self.access_token = m.group(1) + else: + raise ParseError('Unable to find access_token') + + class OkCBrowser(LoginBrowser): BASEURL = 'https://www.okcupid.com' login = URL('/login') - threads = URL('/messages') - messages = URL('/apitun/messages/conversations/global_messaging') + threads = URL('/1/apitun/connections/messages/incoming') + messages = URL('/1/apitun/messages/conversations/(?P\d+)') thread_delete = URL(r'/1/apitun/messages/conversations/(?P\d+)/delete') - message_send = URL('/apitun/messages/send') + message_send = URL('/1/apitun/messages/send') quickmatch = URL(r'/quickmatch\?okc_api=1') like = URL(r'/1/apitun/profile/(?P\d+)/like') - profile = URL(r'/apitun/profile/(?P\d+)') - full_profile = URL(r'/profile/(?P.*)\?okc_api=1') + profile = URL(r'/1/apitun/profile/(?P\d+)') access_token = None me = None + def __init__(self, username, password, facebook, *args, **kwargs): + self.facebook = facebook + + super(OkCBrowser, self).__init__(username, password, *args, **kwargs) + def do_login(self): - r = self.login.go(data={'username': self.username, 'password': self.password, 'okc_api': 1}).json() + if self.facebook: + r = self.login.go(data={'facebook_access_token': self.facebook.access_token, 'okc_api': 1}).json() + + else: + r = self.login.go(data={'username': self.username, 'password': self.password, 'okc_api': 1}).json() if not 'oauth_accesstoken' in r: raise BrowserIncorrectPassword(r['status_str']) @@ -64,14 +110,13 @@ def do_login(self): self.session.headers['Authorization'] = 'Bearer %s' % self.access_token @need_login - def get_threads_list(self, folder=1): - return self.threads.go(params={'okc_api': 1, 'folder': folder, 'messages_dropdown_ajax': 1}).json() + def get_threads_list(self): + return self.threads.go().json()['data'] @need_login def get_thread_messages(self, thread_id): - r = self.messages.go(params={'access_token': self.access_token, - '_json': '{"userids":["%s"]}' % thread_id}).json() - return r[thread_id] + r = self.messages.go(thread_id=thread_id, params={'limit': 20}, headers={'endpoint_version': '2'}).json() + return r @need_login def post_message(self, thread_id, content): @@ -97,13 +142,6 @@ def find_match_profile(self): def do_rate(self, user_id): self.like.go(method='POST', user_id=user_id) - @need_login - def get_username(self, user_id): - return self.profile.go(user_id=user_id).json()['username'] - @need_login def get_profile(self, username): - if username.isdigit(): - username = self.get_username(username) - - return self.full_profile.go(username=username).json() + return self.profile.go(user_id=username).json() diff --git a/modules/okc/compat/__init__.py b/modules/okc/compat/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/modules/okc/compat/weboob_browser_filters_standard.py b/modules/okc/compat/weboob_browser_filters_standard.py new file mode 100644 index 0000000000000000000000000000000000000000..f382d162f07989986480f1ac6e229c930e1b4879 --- /dev/null +++ b/modules/okc/compat/weboob_browser_filters_standard.py @@ -0,0 +1,49 @@ + +import weboob.browser.filters.standard as OLD + +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) + + +try: + __all__ = OLD.__all__ +except AttributeError: + pass + + +class Coalesce(MultiFilter): + """ + Returns the first value that is not falsy, + or default if all values are falsy. + """ + @debug() + def filter(self, values): + for value in values: + if value: + return value + return self.default_or_raise(FilterError('All falsy and no default.')) + + +class MapIn(Filter): + """ + Map the pattern of a selected value to another value using a dict. + """ + + def __init__(self, selector, map_dict, default=_NO_DEFAULT): + """ + :param selector: key from `map_dict` to use + """ + super(MapIn, self).__init__(selector, default=default) + self.map_dict = map_dict + + @debug() + def filter(self, txt): + """ + :raises: :class:`ItemNotFound` if key pattern does not exist in dict + """ + for key in self.map_dict: + if key in txt: + return self.map_dict[key] + + return self.default_or_raise(ItemNotFound('Unable to handle %r on %r' % (txt, self.map_dict))) diff --git a/modules/okc/module.py b/modules/okc/module.py index 04d02f0e64568b9f2eef2c38e15e5f0ad2ef4f06..e11ed9a8907eba317e53f27591d2353785f37acc 100644 --- a/modules/okc/module.py +++ b/modules/okc/module.py @@ -26,9 +26,9 @@ from weboob.capabilities.messages import CapMessages, CapMessagesPost, Message, Thread from weboob.tools.backend import Module, BackendConfig from weboob.tools.misc import to_unicode -from weboob.tools.value import Value, ValueBackendPassword +from weboob.tools.value import Value, ValueBackendPassword, ValueBool -from .browser import OkCBrowser +from .browser import OkCBrowser, FacebookBrowser from .optim.profiles_walker import ProfilesWalker @@ -49,41 +49,43 @@ def set_profile(self, *args): section[key] = ProfileNode(key, key.capitalize().replace('_', ' '), value) def __init__(self, profile): - super(OkcContact, self).__init__(profile['userid'], - profile['username'], - self.STATUS_ONLINE if profile['is_online'] == '1' else self.STATUS_OFFLINE) + super(OkcContact, self).__init__(profile['user']['userid'], + profile['user']['userinfo']['displayname'], + self.STATUS_ONLINE if profile['user']['online'] else self.STATUS_OFFLINE) - self.url = 'https://www.okcupid.com/profile/%s' % self.name - self.summary = profile.get('summary', '') - self.status_msg = 'Last connection at %s' % profile['skinny']['last_online'] + self.url = 'https://www.okcupid.com/profile/%s' % self.id + self.summary = u'' + self.status_msg = profile['extras']['lastOnlineString'] - for no, photo in enumerate(profile['photos']): - self.set_photo(u'image_%i' % no, url=photo['image_url'], thumbnail_url=photo['image_url']) + for no, photo in enumerate(profile['user']['photos']): + self.set_photo(u'image_%i' % no, url=photo['full'], thumbnail_url=photo['full_small']) self.profile = OrderedDict() - self.set_profile('info', 'status', profile['status_str']) - self.set_profile('info', 'orientation', profile['orientation_str']) - self.set_profile('info', 'age', '%s yo' % profile['age']) - self.set_profile('info', 'birthday', '%04d-%02d-%02d' % (profile['birthday']['year'], profile['birthday']['month'], profile['birthday']['day'])) - self.set_profile('info', 'sex', profile['gender_str']) - self.set_profile('info', 'location', profile['location']) - self.set_profile('info', 'join_date', profile['skinny']['join_date']) - self.set_profile('stats', 'match_percent', '%s%%' % profile['matchpercentage']) - self.set_profile('stats', 'friend_percent', '%s%%' % profile['friendpercentage']) - self.set_profile('stats', 'enemy_percent', '%s%%' % profile['enemypercentage']) - for key, value in sorted(profile['skinny'].items()): - self.set_profile('details', key, value or '-') - - for essay in profile['essays']: - if len(essay['essay']) == 0: + if isinstance(profile['user']['details'], dict): + for key, label in profile['user']['details']['_labels'].items(): + self.set_profile('info', label, profile['user']['details']['values'][key]) + else: + for section in profile['user']['details']: + self.set_profile('info', section['info']['name'], section['text']['text']) + + self.set_profile('info', 'orientation', profile['user']['userinfo']['orientation']) + self.set_profile('info', 'age', '%s yo' % profile['user']['userinfo']['age']) + self.set_profile('info', 'sex', profile['user']['userinfo']['gender']) + self.set_profile('info', 'location', profile['user']['userinfo']['location']) + self.set_profile('stats', 'match_percent', '%s%%' % profile['user']['percentages']['match']) + self.set_profile('stats', 'enemy_percent', '%s%%' % profile['user']['percentages']['enemy']) + if 'friend' in profile['user']['percentages']: + self.set_profile('stats', 'friend_percent', '%s%%' % profile['user']['percentages']['friend']) + + for essay in profile['user']['essays']: + if not essay['content']: continue self.summary += '%s:\n' % essay['title'] self.summary += '-' * (len(essay['title']) + 1) self.summary += '\n' - for text in essay['essay']: - self.summary += text['rawtext'] + self.summary += essay['rawtext'] self.summary += '\n\n' self.profile['info'].flags |= ProfileNode.HEAD @@ -99,14 +101,22 @@ class OkCModule(Module, CapMessages, CapContact, CapMessagesPost, CapDating): LICENSE = 'AGPLv3+' DESCRIPTION = u'OkCupid' CONFIG = BackendConfig(Value('username', label='Username'), - ValueBackendPassword('password', label='Password')) + ValueBackendPassword('password', label='Password'), + ValueBool('facebook', label='Do you login with Facebook?', default=False)) STORAGE = {'profiles_walker': {'viewed': []}, 'sluts': {}, } BROWSER = OkCBrowser def create_default_browser(self): - return self.create_browser(self.config['username'].get(), self.config['password'].get()) + if int(self.config['facebook'].get()): + facebook = self.create_browser(klass=FacebookBrowser) + facebook.login(self.config['username'].get(), self.config['password'].get()) + else: + facebook = None + return self.create_browser(self.config['username'].get(), + self.config['password'].get(), + facebook) # ---- CapDating methods --------------------- def init_optimizations(self): @@ -120,10 +130,10 @@ def iter_threads(self): threads = self.browser.get_threads_list() for thread in threads: - t = Thread(thread['userid']) + t = Thread(thread['user']['userid']) t.flags = Thread.IS_DISCUSSION - t.title = u'Discussion with %s' % thread['user']['username'] - t.date = datetime.fromtimestamp(thread['timestamp']) + t.title = u'Discussion with %s' % thread['user']['userinfo']['displayname'] + t.date = datetime.fromtimestamp(thread['time']) yield t def get_thread(self, thread): @@ -140,7 +150,7 @@ def get_thread(self, thread): other = OkcContact(self.browser.get_profile(thread.id)) parent = None - for message in messages['messages']['messages']: + for message in messages['messages']: date = datetime.fromtimestamp(message['timestamp']) flags = 0 @@ -153,6 +163,19 @@ def get_thread(self, thread): else: receiver = other sender = me + if message.get('read', False): + flags |= Message.IS_RECEIVED + # Apply that flag on all previous messages as the 'read' + # attribute is only set on the last read message. + pmsg = parent + while pmsg: + if pmsg.flags & Message.IS_NOT_RECEIVED: + pmsg.flags |= Message.IS_RECEIVED + pmsg.flags &= ~Message.IS_NOT_RECEIVED + pmsg = pmsg.parent + else: + flags |= Message.IS_NOT_RECEIVED + msg = Message(thread=thread, id=message['id'], diff --git a/modules/okc/optim/profiles_walker.py b/modules/okc/optim/profiles_walker.py index 761bd135ed422c30fc3153d2663557714a410fe4..ded695e0360b45d2e51f29b3482e5ae3e07a3890 100644 --- a/modules/okc/optim/profiles_walker.py +++ b/modules/okc/optim/profiles_walker.py @@ -17,8 +17,6 @@ # You should have received a copy of the GNU Affero General Public License # along with this weboob module. If not, see . -from datetime import datetime -from dateutil.relativedelta import relativedelta from random import randint from weboob.capabilities.dating import Optimization @@ -74,13 +72,6 @@ def get_config(self): def view_profile(self): try: - # Remove old threads - for thread in self._browser.get_threads_list(folder=2): # folder 2 is the sentbox - last_message = datetime.fromtimestamp(thread['timestamp']) - if not thread['replied'] and last_message < (datetime.now() - relativedelta(months=6)): - self._logger.info('Removing old thread with %s from %s', thread['user']['username'], last_message) - self._browser.delete_thread(thread['userid']) - # Find a new profile user_id = self._browser.find_match_profile() if user_id in self._visited_profiles: @@ -89,13 +80,12 @@ def view_profile(self): self._browser.do_rate(user_id) profile = self._browser.get_profile(user_id) if self._config['first_message'] != '': - self._browser.post_message(user_id, self._config['first_message'] % {'name': profile['username']}) - self._browser.delete_thread(user_id) - self._logger.info(u'Visited profile of %s ', profile['username']) + self._browser.post_message(user_id, self._config['first_message'] % {'name': profile['user']['userinfo']['displayname']}) + self._logger.info(u'Visited profile of %s: https://www.okcupid.com/profile/%s', profile['user']['userinfo']['displayname'], profile['user']['userid']) # do not forget that we visited this profile, to avoid re-visiting it. self._visited_profiles.add(user_id) self.save() finally: if self._view_cron is not None: - self._view_cron = self._sched.schedule(randint(60, 120), self.view_profile) + self._view_cron = self._sched.schedule(randint(30, 60), self.view_profile) diff --git a/modules/s2e/pages.py b/modules/s2e/pages.py index 6ba9acbbb7a3cff3103f3e87ea4b8f56e4434106..c932d06ae80aa9a92a81bc04c37fb46e98ac26e5 100644 --- a/modules/s2e/pages.py +++ b/modules/s2e/pages.py @@ -26,7 +26,7 @@ from weboob.browser.pages import HTMLPage, XMLPage, RawPage, LoggedPage, pagination, FormNotFound, PartialHTMLPage from weboob.browser.elements import ItemElement, TableElement, SkipItem, method -from .compat.weboob_browser_filters_standard import CleanText, Date, Regexp, Eval, CleanDecimal, Env, Field +from .compat.weboob_browser_filters_standard import CleanText, Date, Regexp, Eval, CleanDecimal, Env, Field, MapIn, Upper from weboob.browser.filters.html import Attr, TableCell from .compat.weboob_capabilities_bank import Account, Investment, Pocket, Transaction from weboob.capabilities.base import NotAvailable @@ -346,6 +346,7 @@ def on_load(self): 'SWISS': Account.TYPE_MARKET, 'RSP': Account.TYPE_RSP, 'CCB': Account.TYPE_DEPOSIT, + 'PERF': Account.TYPE_PERP, } CONDITIONS = { @@ -386,9 +387,7 @@ def condition(self): obj_label = Env('label') def obj_type(self): - if Field('label')(self).startswith('ETOILE'): - return self.page.TYPES.get(Field('label')(self).split()[1].upper(), Account.TYPE_UNKNOWN) - return self.page.TYPES.get(Field('label')(self).split()[0].upper(), Account.TYPE_UNKNOWN) + return MapIn(Upper(Field('label')), self.page.TYPES, Account.TYPE_UNKNOWN)(self) def obj_balance(self): return MyDecimal(TableCell('balance')(self)[0].xpath('.//div[has-class("nowrap")]'))(self) diff --git a/modules/societegenerale/browser.py b/modules/societegenerale/browser.py index 2ea270cdb65a95c8c5c71376b6092b381c37524c..12e9b5813df8ab5476509ba7fa68ecfd2fadd7af 100644 --- a/modules/societegenerale/browser.py +++ b/modules/societegenerale/browser.py @@ -24,6 +24,7 @@ from dateutil.relativedelta import relativedelta from weboob.browser.browsers import LoginBrowser, URL, need_login, StatesMixin +from weboob.capabilities.bill import Document, DocumentTypes from weboob.exceptions import BrowserIncorrectPassword, ActionNeeded, BrowserUnavailable from .compat.weboob_capabilities_bank import Account, TransferBankError, AddRecipientStep, TransactionType, AccountOwnerType from weboob.capabilities.base import find_object, NotAvailable @@ -33,14 +34,14 @@ from .pages.accounts_list import ( AccountsMainPage, AccountDetailsPage, AccountsPage, LoansPage, HistoryPage, - CardHistoryPage, PeaLiquidityPage, AccountsSynthesesPage, + CardHistoryPage, PeaLiquidityPage, AdvisorPage, HTMLProfilePage, CreditPage, CreditHistoryPage, OldHistoryPage, MarketPage, LifeInsurance, LifeInsuranceHistory, LifeInsuranceInvest, LifeInsuranceInvest2, UnavailableServicePage, LoanDetailsPage, ) from .pages.transfer import AddRecipientPage, SignRecipientPage, TransferJson, SignTransferPage from .pages.login import MainPage, LoginPage, BadLoginPage, ReinitPasswordPage, ActionNeededPage, ErrorPage -from .pages.subscription import BankStatementPage +from .pages.subscription import BankStatementPage, RibPdfPage __all__ = ['SocieteGenerale'] @@ -54,8 +55,7 @@ class SocieteGenerale(LoginBrowser, StatesMixin): accounts_main_page = URL(r'/restitution/cns_listeprestation.html', r'/com/icd-web/cbo/index.html', AccountsMainPage) account_details_page = URL(r'/restitution/cns_detailPrestation.html', AccountDetailsPage) - accounts = URL(r'/icd/cbo/data/liste-prestations-navigation-authsec.json', AccountsPage) - accounts_syntheses = URL(r'/icd/cbo/data/liste-prestations-authsec.json\?n10_avecMontant=1', AccountsSynthesesPage) + accounts = URL(r'/icd/cbo/data/liste-prestations-authsec.json\?n10_avecMontant=1', AccountsPage) history = URL(r'/icd/cbo/data/liste-operations-authsec.json', HistoryPage) loans = URL(r'/abm/restit/listeRestitutionPretsNET.json\?a100_isPretConso=(?P\w+)', LoansPage) loan_details_page = URL(r'icd/cbo/data/recapitulatif-prestation-authsec.json', LoanDetailsPage) @@ -100,6 +100,7 @@ class SocieteGenerale(LoginBrowser, StatesMixin): bank_statement = URL(r'/restitution/rce_derniers_releves.html', BankStatementPage) bank_statement_search = URL(r'/restitution/rce_recherche.html\?noRedirect=1', r'/restitution/rce_recherche_resultat.html', BankStatementPage) + rib_pdf_page = URL(r'/com/icd-web/cbo/pdf/rib-authsec.pdf', RibPdfPage) bad_login = URL(r'/acces/authlgn.html', r'/error403.html', BadLoginPage) reinit = URL(r'/acces/changecodeobligatoire.html', @@ -187,7 +188,7 @@ def get_accounts_list(self): else: account_ibans = self.page.get_account_ibans_dict() - self.accounts_syntheses.go() + self.accounts.go() if not self.page.is_new_website_available(): # return in old pages to get accounts @@ -196,12 +197,7 @@ def get_accounts_list(self): yield acc return - # get accounts coming - account_comings = self.page.get_account_comings() - accounts = {} - - self.accounts.go() for account in self.page.iter_accounts(): account._parent_id = None for card in self.iter_cards(account): @@ -215,9 +211,6 @@ def get_accounts_list(self): if account._prestation_id in account_ibans: account.iban = account_ibans[account._prestation_id] - if account._prestation_id in account_comings: - account.coming = account_comings[account._prestation_id] - if account.type in (account.TYPE_LOAN, account.TYPE_CONSUMER_CREDIT, ): self.loans.stay_or_go(conso=(account._loan_type == 'PR_CONSO')) account = self.page.get_loan_account(account) @@ -475,37 +468,69 @@ def iter_subscription(self): except (ProfileMissing, BrowserUnavailable): subscriber = NotAvailable - # subscriptions which have statements are present on the last statement page - self.bank_statement.go() - subscriptions_list = list(self.page.iter_subscription()) + self.accounts.go() + subscriptions_list = list(self.page.iter_subscription(subscriber=subscriber)) - # this way the no statement accounts are excluded - # and the one keeped have all the data and parameters needed self.bank_statement_search.go() - for sub in self.page.iter_searchable_subscription(subscriber=subscriber): - found_sub = find_object(subscriptions_list, id=sub.id) + searchable_subscription_list = list(self.page.iter_searchable_subscription()) + for sub in subscriptions_list: + found_sub = find_object(searchable_subscription_list, id=sub.id) if found_sub: - yield sub - - @need_login - def iter_documents(self, subscribtion): - end_date = datetime.today() + # we need it to get bank statement, but not all subscription have it + sub._rad_button_id = found_sub._rad_button_id + else: + # even without bank statement we still can get RIB document, so we yield subscription anyway + sub._rad_button_id = NotAvailable + yield sub + + def _fetch_rib_document(self, subscription): + d = Document() + d.id = subscription.id + '_RIB' + d.url = self.rib_pdf_page.build(params={'b64e200_prestationIdTechnique': subscription._internal_id}) + d.type = DocumentTypes.RIB + d.format = 'pdf' + d.label = 'RIB' + return d + + def _iter_statements(self, subscription): + # we need _rad_button_id for post_form function + # if not present it means this subscription doesn't have any bank statement + if subscription._rad_button_id is NotAvailable: + return # 5 years since it goes with a 2 months step security_limit = 30 + end_date = datetime.today() i = 0 while i < security_limit: self.bank_statement_search.go() - self.page.post_form(subscribtion, end_date) + self.page.post_form(subscription, end_date) # No more documents if self.page.has_error_msg(): break - for d in self.page.iter_documents(subscribtion): + for d in self.page.iter_documents(subscription): yield d # 3 months step because the documents list is inclusive # from the 08 to the 06, the 06 statement is included end_date = end_date - relativedelta(months=+3) i += 1 + + @need_login + def iter_documents(self, subscription): + yield self._fetch_rib_document(subscription) + for doc in self._iter_statements(subscription): + yield doc + + @need_login + def iter_documents_by_types(self, subscription, accepted_types): + if DocumentTypes.RIB in accepted_types: + yield self._fetch_rib_document(subscription) + + if DocumentTypes.STATEMENT not in accepted_types: + return + + for doc in self._iter_statements(subscription): + yield doc diff --git a/modules/societegenerale/module.py b/modules/societegenerale/module.py index 5e2a9618ea1e83291147a425c377eb46ae80b898..8a3e5a99265982c3e5613fe72c686e63e3682a39 100644 --- a/modules/societegenerale/module.py +++ b/modules/societegenerale/module.py @@ -57,7 +57,7 @@ class SocieteGeneraleModule(Module, CapBankWealth, CapBankTransferAddRecipient, Value('website', label='Type de compte', default='par', choices={'par': 'Particuliers', 'pro': 'Professionnels', 'ent': 'Entreprises'})) - accepted_document_types = (DocumentTypes.STATEMENT,) + accepted_document_types = (DocumentTypes.STATEMENT, DocumentTypes.RIB) def create_default_browser(self): b = {'par': SocieteGenerale, 'pro': SGProfessionalBrowser, 'ent': SGEnterpriseBrowser} @@ -157,6 +157,18 @@ def iter_documents(self, subscription): return self.browser.iter_documents(subscription) + def iter_documents_by_types(self, subscription, accepted_types): + if not isinstance(subscription, Subscription): + subscription = self.get_subscription(subscription) + + if self.config['website'].get() not in ('ent', 'pro'): + for doc in self.browser.iter_documents_by_types(subscription, accepted_types): + yield doc + else: + for doc in self.browser.iter_documents(subscription): + if doc.type in accepted_types: + yield doc + def download_document(self, document): if not isinstance(document, Document): document = self.get_document(document) diff --git a/modules/societegenerale/pages/accounts_list.py b/modules/societegenerale/pages/accounts_list.py index 98e80a16db8a37979866a866e27487995002a324..e777fa3d2f25cb1423c7d3e78338ea76bcd3fe76 100644 --- a/modules/societegenerale/pages/accounts_list.py +++ b/modules/societegenerale/pages/accounts_list.py @@ -26,6 +26,7 @@ from dateutil.relativedelta import relativedelta from weboob.capabilities.base import NotAvailable from .compat.weboob_capabilities_bank import Account, Investment, Loan, AccountOwnership +from weboob.capabilities.bill import Subscription from weboob.capabilities.contact import Advisor from weboob.capabilities.profile import Person, ProfileMissing from weboob.tools.capabilities.bank.transactions import FrenchTransaction @@ -67,9 +68,10 @@ def on_load(self): conditions = ( 'pas encore géré' in reason, # this page is not handled by SG api website - 'le service est momentanement indisponible' in reason, # can't access new website + 'le service est momentanement indisponible' in reason, # can't access new website ) assert any(conditions), 'Error %s is not handled yet' % reason + self.logger.warning('Handled Error "%s"', reason) class AccountsMainPage(LoggedPage, HTMLPage): @@ -117,9 +119,17 @@ class AccountDetailsPage(LoggedPage, HTMLPage): class AccountsPage(JsonBasePage): + def is_new_website_available(self): + if not Dict('commun/raison')(self.doc): + return True + elif 'le service est momentanement indisponible' not in Dict('commun/raison')(self.doc): + return True + self.logger.warning("SG new website is not available yet for this user") + return False + @method class iter_accounts(DictElement): - item_xpath = 'donnees' + item_xpath = 'donnees/syntheseParGroupeProduit/*/prestations' class item(ItemElement): def condition(self): @@ -215,22 +225,17 @@ def obj__is_json_histo(self): not Dict('produit')(self) in ('PLAN_EPARGNE_POPULAIRE', ): return True -class AccountsSynthesesPage(JsonBasePage): - def is_new_website_available(self): - if not Dict('commun/raison')(self.doc): - return True - elif not 'le service est momentanement indisponible' in Dict('commun/raison')(self.doc): - return True - self.logger.warning("SG new website is not available yet for this user") - return False + @method + class iter_subscription(DictElement): + item_xpath = 'donnees/syntheseParGroupeProduit/*/prestations' - def get_account_comings(self): - account_comings = {} + class item(ItemElement): + klass = Subscription - for product in Dict('donnees/syntheseParGroupeProduit')(self.doc): - for prestation in Dict('prestations')(product): - account_comings[Dict('id')(prestation)] = CleanDecimal(Dict('soldes/soldeEnCours'))(prestation) - return account_comings + obj_id = CleanText(Dict('numeroCompteFormate'), replace=[(' ', '')]) + obj_subscriber = Env('subscriber') + obj_label = Format('%s %s', Dict('labelToDisplay'), Field('id')) + obj__internal_id = Dict('idTechnique') class LoanDetailsPage(LoggedPage, JsonPage): @@ -461,6 +466,11 @@ def hist_pagination(self, condition): @pagination @method class iter_history(DictElement): + def condition(self): + # If we reach this point and it's "NOK", that's mean it's a known error handled + # in JsonBasePage and we can't have history for now. + return Dict('commun/statut')(self.el).upper() != 'NOK' + def next_page(self): return self.page.hist_pagination('history') @@ -508,6 +518,11 @@ def condition(self): @pagination @method class iter_intraday_comings(DictElement): + def condition(self): + # If we reach this point and it's "NOK", that mean it's a known error handled + # in JsonBasePage and we can't have history for now. + return Dict('commun/statut')(self.el).upper() != 'NOK' + def next_page(self): return self.page.hist_pagination('intraday') diff --git a/modules/societegenerale/pages/subscription.py b/modules/societegenerale/pages/subscription.py index ba482f10eb518ef947dbb36a18692c84053117b9..f11dd5370463e9253e4de61371cf1506d03c8773 100644 --- a/modules/societegenerale/pages/subscription.py +++ b/modules/societegenerale/pages/subscription.py @@ -24,31 +24,14 @@ from weboob.capabilities.bill import Document, Subscription, DocumentTypes from weboob.browser.elements import TableElement, ItemElement, method -from .compat.weboob_browser_filters_standard import CleanText, Regexp, Env, Date, Format, Field +from .compat.weboob_browser_filters_standard import CleanText, Regexp, Date, Format, Field from weboob.browser.filters.html import Link, TableCell, Attr -from weboob.browser.pages import LoggedPage +from weboob.browser.pages import LoggedPage, RawPage from .base import BasePage -class BankStatementPage(LoggedPage, BasePage): - @method - class iter_subscription(TableElement): - item_xpath = '//table[.//th]//tr[td and @class="LGNTableRow"]' - head_xpath = '//table//th' - - col_id = 'Numéro de Compte' - col_label = 'Type de Compte' - col__last_document_label = 'Derniers relevés' - - class item(ItemElement): - def condition(self): - return 'Récapitulatif annuel' not in CleanText(TableCell('_last_document_label'))(self) - - klass = Subscription - - obj_id = CleanText(TableCell('id'), replace=[(' ', '')]) - obj_label = CleanText(TableCell('label')) +class BankStatementPage(LoggedPage, BasePage): @method class iter_searchable_subscription(TableElement): item_xpath = '//table//tr[@class="fond_ligne"]' @@ -63,7 +46,6 @@ class item(ItemElement): klass = Subscription obj_id = CleanText(TableCell('id'), replace=[(' ', '')]) - obj_subscriber = Env('subscriber') def obj_label(self): label = CleanText(TableCell('label'))(self) @@ -119,3 +101,7 @@ def has_error_msg(self): return any((CleanText('//div[@class="MessageErreur"]')(self.doc), CleanText('//span[@class="error_msg"]')(self.doc), self.doc.xpath('//div[contains(@class, "error_page")]'), )) + + +class RibPdfPage(LoggedPage, RawPage): + pass