From 0b9c99f749cc1640ae14b38e5e742f81cb492383 Mon Sep 17 00:00:00 2001 From: Pierre Nardozi Date: Thu, 20 Aug 2020 18:30:39 +0200 Subject: [PATCH] [ameli] add pages for monthly statement document --- modules/ameli/browser.py | 53 ++++++++++++++++++++---------- modules/ameli/pages.py | 71 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 101 insertions(+), 23 deletions(-) diff --git a/modules/ameli/browser.py b/modules/ameli/browser.py index 1ef59e1555..eabffecd65 100644 --- a/modules/ameli/browser.py +++ b/modules/ameli/browser.py @@ -25,10 +25,12 @@ from weboob.browser import LoginBrowser, URL, need_login from weboob.exceptions import ActionNeeded +from weboob.tools.capabilities.bill.documents import merge_iterators from .pages import ( ErrorPage, LoginPage, RedirectPage, CguPage, - SubscriptionPage, DocumentsPage, CtPage, + SubscriptionPage, DocumentsDetailsPage, CtPage, DocumentsFirstSummaryPage, + DocumentsLastSummaryPage, ) @@ -40,7 +42,15 @@ class AmeliBrowser(LoginBrowser): redirect_page = URL(r'/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&.*validationconnexioncompte.*', RedirectPage) cgu_page = URL(r'/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&_pageLabel=as_conditions_generales_page.*', CguPage) subscription_page = URL(r'/PortailAS/appmanager/PortailAS/assure\?_nfpb=true&_pageLabel=as_info_perso_page.*', SubscriptionPage) - documents_page = URL(r'/PortailAS/paiements.do', DocumentsPage) + documents_details_page = URL(r'/PortailAS/paiements.do', DocumentsDetailsPage) + documents_first_summary_page = URL( + r'PortailAS/appmanager/PortailAS/assure\?_nfpb=true&_pageLabel=as_releve_mensuel_paiement_page', + DocumentsFirstSummaryPage + ) + documents_last_summary_page = URL( + r'PortailAS/portlets/relevemensuelpaiement/relevemensuelpaiement.do\?actionEvt=afficherPlusReleves', + DocumentsLastSummaryPage + ) ct_page = URL(r'/PortailAS/JavaScriptServlet', CtPage) def do_login(self): @@ -58,19 +68,16 @@ def iter_subscription(self): yield self.page.get_subscription() @need_login - def iter_documents(self, subscription): + def _iter_details_documents(self, subscription): end_date = date.today() start_date = end_date - relativedelta(years=1) - # FUN FACT, website tell us documents are available for 6 months - # let's suppose today is 28/05/19, website frontend limit DateDebut to 28/11/18 but we can get a little bit more - # by setting a previous date and get documents that are no longer available for simple user params = { 'Beneficiaire': 'tout_selectionner', 'DateDebut': start_date.strftime('%d/%m/%Y'), 'DateFin': end_date.strftime('%d/%m/%Y'), - 'actionEvt': 'afficherPaiementsComplementaires', + 'actionEvt': 'Rechercher', 'afficherIJ': 'false', 'afficherInva': 'false', 'afficherPT': 'false', @@ -80,14 +87,26 @@ def iter_documents(self, subscription): 'idNoCache': int(time()*1000) } - # the second request is stateful - # first value of actionEvt is afficherPaiementsComplementaires to get all payments from last 6 months - # (start_date 6 months in the past is needed but not enough) - self.documents_page.go(params=params) - - # then we set Rechercher to actionEvt to filter for this subscription, within last 6 months - # without first request we would have filter for this subscription but within last 2 months - params['actionEvt'] = 'Rechercher' - params['Beneficiaire'] = 'tout_selectionner' - self.documents_page.go(params=params) + # website tell us details documents are available for 6 months + self.documents_details_page.go(params=params) return self.page.iter_documents(subid=subscription.id) + + @need_login + def _iter_summary_documents(self, subscription): + # The monthly statements for the last 23 months are available in two parts. + # The first part contains the last 6 months on an HTML page. + self.documents_first_summary_page.go() + for doc in self.page.iter_documents(subid=subscription.id): + yield doc + + # The second part is retrieved in JSON via this page which displays the next 6 months at each iteration. + for _ in range(3): + self.documents_last_summary_page.go() + for doc in self.page.iter_documents(subid=subscription.id): + yield doc + + + @need_login + def iter_documents(self, subscription): + for doc in merge_iterators(self._iter_details_documents(subscription), self._iter_summary_documents(subscription)): + yield doc diff --git a/modules/ameli/pages.py b/modules/ameli/pages.py index 20efc27dd5..24226ece2d 100644 --- a/modules/ameli/pages.py +++ b/modules/ameli/pages.py @@ -23,11 +23,12 @@ from hashlib import sha1 -from weboob.browser.elements import method, ListElement, ItemElement +from weboob.browser.elements import method, ListElement, ItemElement, DictElement from weboob.browser.filters.html import Link -from weboob.browser.filters.standard import CleanText, Regexp, CleanDecimal, Currency, Field, Env -from weboob.browser.pages import LoggedPage, HTMLPage, PartialHTMLPage, RawPage -from weboob.capabilities.bill import Subscription, Bill +from weboob.browser.filters.standard import CleanText, Regexp, CleanDecimal, Currency, Field, Env, Format +from weboob.browser.filters.json import Dict +from weboob.browser.pages import LoggedPage, HTMLPage, PartialHTMLPage, RawPage, JsonPage +from weboob.capabilities.bill import Subscription, Bill, Document, DocumentTypes from weboob.exceptions import BrowserUnavailable from weboob.tools.date import parse_french_date from weboob.tools.json import json @@ -74,12 +75,12 @@ def get_subscription(self): return sub -class DocumentsPage(LoggedPage, PartialHTMLPage): +class DocumentsDetailsPage(LoggedPage, PartialHTMLPage): ENCODING = 'utf-8' def build_doc(self, content): res = json.loads(content) - return super(DocumentsPage, self).build_doc(res['tableauPaiement'].encode('utf-8')) + return super(DocumentsDetailsPage, self).build_doc(res['tableauPaiement'].encode('utf-8')) @method class iter_documents(ListElement): @@ -105,3 +106,61 @@ def obj_date(self): day_month = CleanText('.//div[has-class("col-date")]/span')(self) return parse_french_date(day_month + ' ' + year) + + +class DocumentsFirstSummaryPage(LoggedPage, HTMLPage): + + @method + class iter_documents(ListElement): + item_xpath = '//ul[@id="unordered_list"]//li[@class="rowdate" and .//span[@class="blocTelecharger"]]' + + class item(ItemElement): + klass = Document + + obj_type = DocumentTypes.BILL + obj_label = Format('%s %s', CleanText('.//span[@class="libelle"]'), CleanText('.//span[@class="mois"]')) + obj_url = Link('.//div[@class="col-telechargement"]//a') + obj_format = 'pdf' + + def obj_date(self): + year = Regexp(CleanText('.//span[@class="mois"]'), r'(\d+)')(self) + month = Regexp(CleanText('.//span[@class="mois"]'), r'(\D+)')(self) + + return parse_french_date(month + ' ' + year) + + def obj_id(self): + year = Regexp(CleanText('.//span[@class="mois"]'), r'(\d+)')(self) + month = Regexp(CleanText('.//span[@class="mois"]'), r'(\D+)')(self) + + return '%s_%s' % (Env('subid')(self), parse_french_date(month + ' ' + year).strftime('%Y%m')) + + +class DocumentsLastSummaryPage(LoggedPage, JsonPage): + + @method + class iter_documents(DictElement): + + def find_elements(self): + for doc in self.el['listeDecomptes']: + if doc['montant']: + yield doc + + class item(ItemElement): + klass = Document + + obj_type = DocumentTypes.BILL + obj_url = Dict('urlPDF') + obj_format = 'pdf' + obj_label = Format('Relevé mensuel %s', CleanText(Dict('mois'))) + + def obj_date(self): + year = Regexp(CleanText(Dict('mois')), r'(\d+)')(self) + month = Regexp(CleanText(Dict('mois')), r'(\D+)')(self) + + return parse_french_date(month + ' ' + year) + + def obj_id(self): + year = Regexp(CleanText(Dict('mois')), r'(\d+)')(self) + month = Regexp(CleanText(Dict('mois')), r'(\D+)')(self) + + return '%s_%s' % (Env('subid')(self), parse_french_date(month + ' ' + year).strftime('%Y%m')) -- GitLab