diff --git a/modules/750g/pages.py b/modules/750g/pages.py index 86a275e87bb80cf66e2d7e340b3d7f401f8c093e..689ed5d1cb420de4df27d19d3dc4bd29b5eb1ae4 100644 --- a/modules/750g/pages.py +++ b/modules/750g/pages.py @@ -34,6 +34,8 @@ class Time(Dict): def filter(self, el): if el and not isinstance(el, NotFound): el = el.replace('PT', '') + if el == u'P': + return NotAvailable _time = parse_date(el, dayfirst=False, fuzzy=False) _time = _time - datetime.combine(date.today(), time(0)) return _time.seconds // 60 @@ -59,7 +61,7 @@ class ResultsPage(HTMLPage): obj_id = Regexp(CleanText('./div/h2/a/@href'), '/(.*).htm') - obj_id = CleanText('.') + obj_title = CleanText('./div/h2/a') class obj_picture(ItemElement): diff --git a/modules/binck/browser.py b/modules/binck/browser.py index 5b8ef07866c7ff67d59d46216d29b9f44177a94d..2eeb09a542507a8df373a2c2d7bdae4d73ee5a53 100644 --- a/modules/binck/browser.py +++ b/modules/binck/browser.py @@ -19,29 +19,41 @@ from __future__ import unicode_literals +from lxml import etree +from io import StringIO + from weboob.browser import LoginBrowser, URL, need_login from weboob.exceptions import BrowserIncorrectPassword, ActionNeeded -from weboob.browser.exceptions import HTTPNotFound +from weboob.browser.exceptions import HTTPNotFound, ServerError from weboob.tools.capabilities.bank.investments import create_french_liquidity from .pages import ( - LoginPage, AccountsPage, HomePage, InvestmentPage, HistoryPage, - QuestionPage, ChangePassPage, LogonFlowPage, ViewPage, SwitchPage, + LoginPage, HomePage, AccountsPage, OldAccountsPage, HistoryPage, InvestmentPage, InvestDetailPage, + InvestmentListPage, QuestionPage, ChangePassPage, LogonFlowPage, ViewPage, SwitchPage, ) class BinckBrowser(LoginBrowser): BASEURL = 'https://web.binck.fr' + ''' Delete this attribute when old website is obsolete ''' + old_website_connection = False + login = URL(r'/Logon', LoginPage) - view = URL('PersonIntroduction/Index', ViewPage) + view = URL('/PersonIntroduction/Index', ViewPage) logon_flow = URL(r'/AmlQuestionnairesOverview/LogonFlow$', LogonFlowPage) accounts = URL(r'/PersonAccountOverview/Index', AccountsPage) + old_accounts = URL(r'/AccountsOverview/Index', OldAccountsPage) + account_switch = URL('/Header/SwitchAccount', SwitchPage) - home_page = URL('/Home/Index', HomePage) + home_page = URL(r'/$', + r'/Home/Index', HomePage) investment = URL(r'/PortfolioOverview/GetPortfolioOverview', InvestmentPage) + investment_list = URL(r'PortfolioOverview$', InvestmentListPage) + invest_detail = URL(r'/SecurityInformation/Get', InvestDetailPage) + history = URL(r'/TransactionsOverview/GetTransactions', r'/TransactionsOverview/FilteredOverview', HistoryPage) questions = URL(r'/FDL_Complex_FR_Compte', QuestionPage) @@ -49,7 +61,7 @@ class BinckBrowser(LoginBrowser): def deinit(self): if self.page and self.page.logged: - self.location("/Account/Logoff") + self.location('/Account/Logoff') super(BinckBrowser, self).deinit() def do_login(self): @@ -65,56 +77,129 @@ class BinckBrowser(LoginBrowser): )): raise ActionNeeded(error) raise AssertionError('Unhandled behavior at login: error is "{}"'.format(error)) - elif self.view.is_here(): - self.location(self.page.skip_tuto()) + + @need_login + def switch_account(self, account_id): + self.accounts.stay_or_go() + if self.accounts.is_here(): + token = self.page.get_token() + data = {'accountNumber': account_id} + # Important: the "switch" request without the token will return a 500 error + self.account_switch.go(data=data, headers=token) + # We should be automatically redirected to the accounts page: + assert self.accounts.is_here(), 'switch_account did not redirect to AccountsPage properly' @need_login def iter_accounts(self): - self.accounts.go() - for a in self.page.iter_accounts(): - self.accounts.stay_or_go() - if self.view.is_here(): - self.location(self.page.skip_tuto()) - if self.accounts.is_here(): - token = self.page.get_token() - data = {'accountNumber': a.id} - # Important: the "switch" request without the token will lead to a 500 error - self.account_switch.go(data=data, headers=token) - - # We must get the new token almost everytime we get a new page: - if self.view.is_here(): - self.location(self.page.skip_tuto()) - if self.accounts.is_here(): - token = self.page.get_token() - try: - data = {'grouping': 'SecurityCategory'} - a._invpage = self.investment.go(data=data, headers=token) - except HTTPNotFound: - # if it is not an invest account, the portfolio link may be present but hidden and return a 404 + self.accounts.stay_or_go() + if self.page.has_accounts_table(): + for a in self.page.iter_accounts(): + ''' Delete these attributes when old website is obsolete ''' a._invpage = None - - if a._invpage: - a.valuation_diff = a._invpage.get_valuation_diff() - - # Get history page - data = [('currencyCode', a.currency), ('startDate', ""), ('endDate', "")] - a._histpages = [self.history.go(data=data, headers=token)] - while self.page.doc['EndOfData'] is False: - a._histpages.append(self.history.go(data=self.page.get_nextpage_data(data[:]), headers=token)) - - yield a + a._histpages = None + + self.switch_account(a.id) + # We must get the new token almost everytime we get a new page: + if self.accounts.is_here(): + token = self.page.get_token() + # Get valuation_diff from the investment page + try: + data = {'grouping': 'SecurityCategory'} + a.valuation_diff = self.investment.go(data=data, headers=token).get_valuation_diff() + except HTTPNotFound: + # if it is not an invest account, the portfolio link may be present but hidden and return a 404 + a.valuation_diff = None + yield a + + # Some Binck connections don't have any accounts on the new AccountsPage, + # so we need to fetch them on the OldAccountsPage for now: + else: + ''' Delete this part when old website is obsolete ''' + self.old_website_connection = True + self.old_accounts.go() + for a in self.page.iter_accounts(): + try: + self.old_accounts.stay_or_go().go_to_account(a.id) + except ServerError as exception: + # get html error to parse + parser = etree.HTMLParser() + html_error = etree.parse(StringIO(exception.response.text), parser) + account_error = html_error.xpath('//p[contains(text(), "Votre compte est")]/text()') + if account_error: + raise ActionNeeded(account_error[0]) + else: + raise + + a.iban = self.page.get_iban() + # Get token + token = self.page.get_token() + # Get investment page + data = {'grouping': "SecurityCategory"} + try: + a._invpage = self.investment.go(data=data, headers=token) \ + if self.page.is_investment() else None + except HTTPNotFound: + # if it's not an invest account, the portfolio link may be present but hidden and return a 404 + a._invpage = None + + if a._invpage: + a.valuation_diff = a._invpage.get_valuation_diff() + # Get history page + data = [('currencyCode', a.currency), ('startDate', ""), ('endDate', "")] + a._histpages = [self.history.go(data=data, headers=token)] + while self.page.doc['EndOfData'] is False: + a._histpages.append(self.history.go(data=self.page.get_nextpage_data(data[:]), headers=token)) + + yield a @need_login def iter_investment(self, account): - # Add liquidity investment + # Start with liquidities: if account._liquidity: yield create_french_liquidity(account._liquidity) - if account._invpage: - for inv in account._invpage.iter_investment(currency=account.currency): - yield inv + + ''' Delete this part when old website is obsolete ''' + if self.old_website_connection: + self.old_accounts.stay_or_go().go_to_account(account.id) + if account._invpage: + for inv in account._invpage.iter_investment(currency=account.currency): + if not inv.code: + params = {'securityId': inv._security_id} + self.invest_detail.go(params=params) + if self.invest_detail.is_here(): + inv.code, inv.code_type = self.page.get_isin_code_and_type() + yield inv + return + + self.switch_account(account.id) + token = self.page.get_token() + + try: + data = {'grouping': 'SecurityCategory'} + self.investment.go(data=data, headers=token) + except HTTPNotFound: + return + + for inv in self.page.iter_investment(currency=account.currency): + yield inv @need_login def iter_history(self, account): - for page in account._histpages: + ''' Delete this part when old website is obsolete ''' + if self.old_website_connection: + if account._histpages: + for page in account._histpages: + for tr in page.iter_history(): + yield tr + return + + self.switch_account(account.id) + token = self.page.get_token() + data = [('currencyCode', account.currency), ('startDate', ''), ('endDate', '')] + history_pages = [self.history.go(data=data, headers=token)] + while self.page.doc['EndOfData'] is False: + history_pages.append(self.history.go(data=self.page.get_nextpage_data(data[:]), headers=token)) + + for page in history_pages: for tr in page.iter_history(): yield tr diff --git a/modules/binck/pages.py b/modules/binck/pages.py index 493f60d903155714aa46a2851255f2be0e212b07..659a748a95a846e73404fab64742b0b250eee7a7 100644 --- a/modules/binck/pages.py +++ b/modules/binck/pages.py @@ -22,9 +22,9 @@ from __future__ import unicode_literals import re from weboob.browser.pages import HTMLPage, JsonPage, LoggedPage -from weboob.browser.elements import ItemElement, ListElement, DictElement, method +from weboob.browser.elements import ItemElement, ListElement, DictElement, TableElement, method from weboob.browser.filters.standard import CleanText, Date, Format, CleanDecimal, Eval, Env, Field -from weboob.browser.filters.html import Attr, Link +from weboob.browser.filters.html import Attr, Link, TableCell from weboob.browser.filters.json import Dict from weboob.exceptions import BrowserPasswordExpired, ActionNeeded from weboob.capabilities.bank import Account, Investment @@ -45,8 +45,21 @@ class QuestionPage(HTMLPage): class ViewPage(LoggedPage, HTMLPage): - def skip_tuto(self): - return Link('//a[contains(@href, "Skip") and contains(text(), "Suivant")]')(self.doc) + # We automatically skip the new website tutorial + def on_load(self): + link = Link('//a[contains(@href, "Skip") and contains(text(), "Suivant")]')(self.doc) + assert link, 'ViewPage skipping link was not found' + self.browser.location(link) + + +class HomePage(LoggedPage, HTMLPage): + # We directly go from the home page to the accounts page + def on_load(self): + ''' Remove the old_website_connection part when old website is obsolete ''' + if self.browser.old_website_connection: + self.browser.location('https://web.binck.fr/AccountsOverview/Index') + else: + self.browser.location('https://web.binck.fr/PersonAccountOverview/Index') class ChangePassPage(LoggedPage, HTMLPage): @@ -79,6 +92,10 @@ class AccountsPage(LoggedPage, HTMLPage): 'AV': Account.TYPE_LIFE_INSURANCE, } + ''' Delete this method when the old website is obsolete ''' + def has_accounts_table(self): + return self.doc.xpath('//table[contains(@class, "accountoverview-table")]') + def get_token(self): return [{Attr('.', 'name')(input): Attr('.', 'value')(input)} \ for input in self.doc.xpath('//input[contains(@name, "Token")]')][0] @@ -111,15 +128,58 @@ class AccountsPage(LoggedPage, HTMLPage): return Account.get_currency(CleanText('.//div[contains(text(), "Total des avoirs")]/following::strong[1]')(self)) -class HomePage(AccountsPage): - pass +class OldAccountsPage(LoggedPage, HTMLPage): + ''' + Old website accounts page. We can get rid of this + class when all users have access to the new website. + ''' + TYPES = {'LIVRET': Account.TYPE_SAVINGS, + 'COMPTE-TITRES': Account.TYPE_MARKET, + 'PEA-PME': Account.TYPE_PEA, + 'PEA': Account.TYPE_PEA + } + + def go_to_account(self, number): + form = self.get_form('//form[contains(@action, "Switch")]') + form['accountNumber'] = number + form.submit() + def get_iban(self): + return CleanText('//div[@class="iban"]/text()', replace=[(' ', '')], default=NotAvailable)(self.doc) -class SwitchPage(LoggedPage, HTMLPage): - pass + def get_token(self): + return [{Attr('.', 'name')(input): Attr('.', 'value')(input)} \ + for input in self.doc.xpath('//input[contains(@name, "Token")]')][0] + + def is_investment(self): + # warning: the link can be present even in case of non-investement account + return CleanText('//a[contains(@href, "Portfolio")]', default=False)(self.doc) + + @method + class iter_accounts(TableElement): + item_xpath = '//table[contains(@class, "accountsTable")]/tbody/tr' + head_xpath = '//table[contains(@class, "accountsTable")]/thead/tr/th' + + col_label = 'Intitulé du compte' + col_balance = 'Total Portefeuille' + col_liquidity = 'Espèces' + + class item(ItemElement): + klass = Account + + obj_id = Attr('.', 'data-accountnumber') + obj_label = CleanText(TableCell('label')) + obj_balance = MyDecimal(TableCell('balance')) + obj__liquidity = MyDecimal(TableCell('liquidity')) + + def obj_type(self): + return self.page.TYPES.get(CleanText('./ancestor::section[h1]/h1')(self).upper(), Account.TYPE_UNKNOWN) + + def obj_currency(self): + return Account.get_currency(CleanText(TableCell('balance'))(self)) -class DetailsPage(LoggedPage, HTMLPage): +class SwitchPage(LoggedPage, HTMLPage): pass @@ -147,6 +207,7 @@ class InvestmentPage(LoggedPage, JsonPage): obj_original_unitprice = Env('o_unitprice', default=NotAvailable) obj_original_valuation = Env('o_valuation', default=NotAvailable) obj_original_diff = Env('o_diff', default=NotAvailable) + obj__security_id = Dict('SecurityId') def obj_code(self): if is_isin_valid(Dict('IsinCode')(self)): @@ -173,6 +234,18 @@ class InvestmentPage(LoggedPage, JsonPage): self.env['vdate'] = Date(dayfirst=True).filter(Dict('PortfolioSummary/UpdatedAt')(self.page.doc)) +class InvestmentListPage(LoggedPage, HTMLPage): + pass + + +class InvestDetailPage(LoggedPage, HTMLPage): + def get_isin_code_and_type(self): + code = CleanText('//td[strong[text()="ISIN"]]/following-sibling::td[1]', default=NotAvailable)(self.doc) + if is_isin_valid(code): + return code, Investment.CODE_TYPE_ISIN + return NotAvailable, NotAvailable + + class Transaction(FrenchTransaction): PATTERNS = [(re.compile(r'^(?P(Virement.*|Transfert.*))'), FrenchTransaction.TYPE_TRANSFER), (re.compile(r'^(?PDépôt.*)'), FrenchTransaction.TYPE_DEPOSIT), diff --git a/modules/cmso/par/pages.py b/modules/cmso/par/pages.py index 92c782381740f754ccf6905d26be8963bb0223f0..c6c99b1a45b9e29af317fc3c7e16577d370291eb 100644 --- a/modules/cmso/par/pages.py +++ b/modules/cmso/par/pages.py @@ -419,31 +419,27 @@ class MarketPage(LoggedPage, HTMLPage): def find_account(self, acclabel, accowner): accowner = sorted(accowner.lower().split()) # first name and last name may not be ordered the same way on market site... + def get_ids(ref, acclabel, accowner): + ids = None + for a in self.doc.xpath('//a[contains(@%s, "indiceCompte")]' % ref): + self.logger.debug("get investment from %s" % ref) + label = CleanText('.')(a) + owner = CleanText('./ancestor::tr/preceding-sibling::tr[@class="LnMnTiers"][1]')(a) + owner = re.sub(r' \(.+', '', owner) + owner = sorted(owner.lower().split()) + if label == acclabel and owner == accowner: + ids = list(re.search(r'indiceCompte[^\d]+(\d+).*idRacine[^\d]+(\d+)', Attr('.', ref)(a)).groups()) + ids.append(CleanText('./ancestor::td/preceding-sibling::td')(a)) + self.logger.debug("assign value to ids: {}".format(ids)) + return ids + # Check if history is present if CleanText(default=None).filter(self.doc.xpath('//body/p[contains(text(), "indisponible pour le moment")]')): return False - ids = None - for a in self.doc.xpath('//a[contains(@onclick, "indiceCompte")]'): - self.logger.debug("get investment from onclick") - - label = CleanText('.')(a) - owner = CleanText('./ancestor::tr/preceding-sibling::tr[@class="LnMnTiers"][1]')(a) - owner = sorted(owner.lower().split()) - - if label == acclabel and owner == accowner: - ids = list(re.search(r'indiceCompte[^\d]+(\d+).*idRacine[^\d]+(\d+)', Attr('.', 'onclick')(a)).groups()) - ids.append(CleanText('./ancestor::td/preceding-sibling::td')(a)) - self.logger.debug("assign value to ids: {}".format(ids)) - return ids - - for a in self.doc.xpath('//a[contains(@href, "indiceCompte")]'): - self.logger.debug("get investment from href") - if CleanText('.')(a) == acclabel: - ids = list(re.search(r'indiceCompte[^\d]+(\d+).*idRacine[^\d]+(\d+)', Attr('.', 'href')(a)).groups()) - ids.append(CleanText('./ancestor::td/preceding-sibling::td')(a)) - self.logger.debug("assign value to ids: {}".format(ids)) - return ids + ref = CleanText(self.doc.xpath('//a[contains(@href, "indiceCompte")]'))(self) + return get_ids('onclick', acclabel, accowner) if not ref else get_ids('href', acclabel, accowner) + def get_account_id(self, acclabel, owner): account = self.find_account(acclabel, owner) diff --git a/modules/ensap/browser.py b/modules/ensap/browser.py index 04554b2416fb4232f00425f24da02b31783ed4f0..7811237f562429101ee60bead0c6c598e501d8ca 100644 --- a/modules/ensap/browser.py +++ b/modules/ensap/browser.py @@ -26,7 +26,7 @@ from weboob.exceptions import BrowserIncorrectPassword from weboob.capabilities.base import find_object from weboob.capabilities.bill import DocumentNotFound from .pages import LoginPage, DocumentsPage, HomePage, LoginControlPage,\ - LoginValidityPage + LoginValidityPage, ListYear class EnsapBrowser(LoginBrowser): @@ -37,7 +37,8 @@ class EnsapBrowser(LoginBrowser): loginvalidity = URL('/authentification', LoginValidityPage) authp = URL('/prive/initialiserhabilitation/v1', LoginControlPage) homep = URL('/prive/accueilconnecte/v1', HomePage) - documents = URL('/prive/remuneration/v1', DocumentsPage) + documents = URL('/prive/remuneration/v1/(?P\d+)', DocumentsPage) + listyears = URL('/prive/listeanneeremuneration/v1', ListYear) logged = False token = None @@ -51,16 +52,20 @@ class EnsapBrowser(LoginBrowser): "secret": self.password}) if not self.page.check_logged(): raise BrowserIncorrectPassword() - self.authp.go(data={"": ""}) + self.authp.go(data="{}", headers={'Content-Type': 'application/json'}) self.token = self.page.get_xsrf() self.logged = True @need_login def iter_documents(self, subscription): - self.documents.stay_or_go(headers={"X-XSRF-TOKEN": self.token}) - self.token = self.session.cookies.get("XSRF-TOKEN") -# return self.bills.go().iter_bills(subid=subscription.id) - return self.page.iter_documents() + self.listyears.go() + years = self.page.get_years() + # use reverse order of list to get recent documents first + for year in years[::-1]: + self.documents.stay_or_go(year=year, headers={"X-XSRF-TOKEN": self.token}) + self.token = self.session.cookies.get("XSRF-TOKEN") + for doc in self.page.iter_documents(): + yield doc @need_login def iter_subscription(self): diff --git a/modules/ensap/pages.py b/modules/ensap/pages.py index e3142048916a00f081f26607ddef6157214e56c4..391b8c43e50367f84075a9994a2bc6597f7c646c 100644 --- a/modules/ensap/pages.py +++ b/modules/ensap/pages.py @@ -50,21 +50,26 @@ class HomePage(JsonPage): class DocumentsPage(JsonPage): @method class iter_documents(DictElement): - item_xpath = 'donnee/listeDocument' + item_xpath = 'donnee' ignore_duplicate = True class item(ItemElement): klass = Document - obj_date = Date(Dict('date')) + obj_date = Date(Dict('dateDocument')) obj_format = "pdf" obj_label = Format("%s : %s", Dict('libelle1'), Dict('libelle3')) obj_type = CleanText(Dict('libelleIcone'), replace=[('Icône ', '')]) obj_id = Regexp(Dict('libelle2'), r"(\S+)\.", nth=0) - obj_url = Format("/prive/telechargerdocument/v1?documentUuid=%s", + obj_url = Format("/prive/telechargerdocumentremuneration/v1?documentUuid=%s", Dict('documentUuid')) class LoginControlPage(JsonPage): def get_xsrf(self): return self.get("xcrf") + + +class ListYear(JsonPage): + def get_years(self): + return self.get("donnee") diff --git a/modules/francetelevisions/browser.py b/modules/francetelevisions/browser.py index 7b0a3f099f62ef8690e937a9933b16b7926d42a3..d3cf1fc66b96d2a1e09d0eafb028858d88110bb1 100644 --- a/modules/francetelevisions/browser.py +++ b/modules/francetelevisions/browser.py @@ -54,5 +54,13 @@ class PluzzBrowser(PagesBrowser): for cat in self.home.go(cat=cat).iter_categories(): yield cat + def get_subcategories(self, cat): + for cat in self.home.go(cat=cat).iter_subcategories(cat=cat): + yield cat + + def get_emissions(self, cat): + for cat in self.home.go(cat="%s.html" % "/".join(cat)).iter_emissions(cat=cat): + yield cat + def iter_videos(self, cat=""): return self.home.go(cat=cat).iter_videos() diff --git a/modules/francetelevisions/module.py b/modules/francetelevisions/module.py index 3facf0e7c9544624eefae4b596feac17b6107dcc..245e36f4169f5d81ef0cefa31f94020eb66ba07d 100644 --- a/modules/francetelevisions/module.py +++ b/modules/francetelevisions/module.py @@ -76,33 +76,31 @@ class PluzzModule(Module, CapVideo, CapCollection): if category.path_level == 1: yield category - else: + elif collection.path_level > 0 and split_path[-1] == u'videos': + for v in self.browser.iter_videos("/".join(collection.split_path[:-1])): + yield v + + elif collection.path_level == 1: + yield Collection(collection.split_path + [u'videos'], u'Vidéos') - if split_path[-1] == u'videos': - for v in self.browser.iter_videos("/".join(collection.split_path[:-1])): + for category in self.browser.get_subcategories(collection.split_path[0]): + yield category + + elif collection.path_level == 2: + if split_path[-1] == u'replay-videos': + for v in self.browser.iter_videos("/".join(collection.split_path)): yield v - elif split_path[-1].endswith('-video'): - v = BaseVideo( - "{}/{}".format(self.browser.BASEURL, - "/".join(collection.split_path).replace('-video', '.html'))) - v.title = split_path[-1].replace('-video', '') - yield v else: - iter = 0 - for category in self.browser.get_categories("/".join(collection.split_path)): - if category.path_level == collection.path_level + 1 and \ - category.split_path[0] == collection.split_path[0]: - iter = iter + 1 - yield category - - if iter > 0: - yield Collection(split_path + [u'videos'], u'Vidéos') - else: - for v in self.browser.iter_videos("/".join(collection.split_path).replace('-videos', '.html')): - yield v + for category in self.browser.get_emissions(collection.split_path): + yield category + + elif collection.path_level == 3: + for v in self.browser.iter_videos("/".join([collection.split_path[0], + collection.split_path[-1]])): + yield v def validate_collection(self, objs, collection): - if collection.path_level <= 2: + if collection.path_level <= 3: return raise CollectionNotFound(collection.split_path) diff --git a/modules/francetelevisions/pages.py b/modules/francetelevisions/pages.py index 9abbd8777ea01c7f9ae294057e3cc847e4b3c6a8..38dd88ba8c6bad064388af2f982f35f180939699 100644 --- a/modules/francetelevisions/pages.py +++ b/modules/francetelevisions/pages.py @@ -27,7 +27,7 @@ from weboob.capabilities.collection import Collection from weboob.browser.pages import HTMLPage, JsonPage from weboob.browser.elements import ItemElement, ListElement, method, DictElement -from weboob.browser.filters.standard import CleanText, Regexp, Format, Field, Eval +from weboob.browser.filters.standard import CleanText, Regexp, Format, Field, Env from weboob.browser.filters.html import CleanHTML from weboob.browser.filters.json import Dict @@ -44,12 +44,21 @@ class SearchPage(JsonPage): class item(ItemElement): klass = BaseVideo - obj_id = Format(r"https://www.france.tv/%s/%s-%s.html", Dict('path'), Dict('id'), Dict('url_page')) + obj_id = Format(r"https://www.france.tv/%s/%s-%s.html", + Dict('path'), + Dict('id'), + Dict('url_page')) obj_title = CleanText(Dict('title')) - obj_thumbnail = Eval(Thumbnail, - Format(r'https://www.france.tv%s', - Dict('image/formats/vignette_16x9/urls/w:1024'))) + + def obj_thumbnail(self): + try: + img = Dict('image/formats/vignette_16x9/urls/w:1024', default=None)(self) + + except KeyError: + img = Dict('image/formats/carre/urls/w:400')(self) + + return Thumbnail(r'https://www.france.tv%s' % img) def obj_date(self): return datetime.fromtimestamp(Dict('dates/first_publication_date')(self)) @@ -70,36 +79,77 @@ class HomePage(HTMLPage): class iter_categories(ListElement): ignore_duplicate = True - item_xpath = '//li[has-class("nav-item")]/a' + item_xpath = '//ul[has-class("c-sub-nav-items--channels")]/li/a' class item(ItemElement): klass = Collection def condition(self): - return Regexp(CleanText('./@href'), '//www.france.tv/(.*)', default=False)(self) + return CleanText('./@href')(self)[-1] == '/' def obj_id(self): - id = Regexp(CleanText('./@href', - replace=[('.html', '-video/')]), - '//www.france.tv/(.*)', "\\1", - default=None)(self) - return id[:-1] + id = CleanText('./@href')(self) + return id[1:-1] obj_title = CleanText('.') def obj_split_path(self): return Field('id')(self).split('/') + @method + class iter_subcategories(ListElement): + ignore_duplicate = True + + item_xpath = '//li[@class="c-shortcuts-ctn__replays-links-items"]/a' + + class item(ItemElement): + klass = Collection + + def condition(self): + cat = Env('cat')(self) + return Regexp(CleanText('./@href'), '/%s/.*' % cat, default=False)(self) + + def obj_id(self): + id = CleanText('./@href', replace=[('.html', '/'), + ('https://www.france.tv', '')])(self) + return id[1:-1].split('/')[-1] + + obj_title = CleanText('.') + + def obj_split_path(self): + return [Env('cat')(self)] + [Field('id')(self)] + + @method + class iter_emissions(ListElement): + ignore_duplicate = True + + item_xpath = u'//a[@class="c-card-program__link"]' + + class item(ItemElement): + klass = Collection + + def condition(self): + cat = Env('cat')(self) + return Regexp(CleanText('./@href'), '/%s/.*' % cat[0], default=False)(self) + + def obj_id(self): + id = CleanText('./@href')(self) + return id.split('/')[-1] + + obj_title = CleanText('./@title') + + def obj_split_path(self): + return Env('cat')(self) + [Field('id')(self)] + @method class iter_videos(ListElement): - def parse(self, el): - self.item_xpath = u'//a[@data-video]' + item_xpath = u'//h3[@class="c-card-video__infos"]/a' class item(ItemElement): klass = BaseVideo - obj_id = Format('https:%s', CleanText('./@href')) - obj_title = CleanText(CleanHTML('./div[@class="card-content"]|./div[has-class("card-content")]')) + obj_id = Format('https://www.france.tv%s', CleanText('./@href')) + obj_title = CleanText(CleanHTML('./div[has-class("c-card-video__title")]')) def condition(self): return Field('title')(self) diff --git a/modules/hsbc/module.py b/modules/hsbc/module.py index c152c25649cfa16b9be61a4ec37dfe20c4e5bab0..5293ea5b6970f77832ca10759996ea1a0229d482 100644 --- a/modules/hsbc/module.py +++ b/modules/hsbc/module.py @@ -21,7 +21,7 @@ from weboob.capabilities.bank import CapBankWealth, AccountNotFound from weboob.capabilities.base import find_object from weboob.tools.backend import Module, BackendConfig -from weboob.tools.value import ValueBackendPassword, Value +from weboob.tools.value import ValueBackendPassword from weboob.capabilities.profile import CapProfile from .browser import HSBC @@ -38,7 +38,7 @@ class HSBCModule(Module, CapBankWealth, CapProfile): DESCRIPTION = 'HSBC France' CONFIG = BackendConfig(ValueBackendPassword('login', label='Identifiant', masked=False), ValueBackendPassword('password', label='Mot de passe'), - Value( 'secret', label=u'Réponse secrète')) + ValueBackendPassword('secret', label=u'Réponse secrète')) BROWSER = HSBC def create_default_browser(self): diff --git a/modules/logicimmo/browser.py b/modules/logicimmo/browser.py index ed19270b2a91333629648e2f5041f22bcac4bbb9..e3faf959feba7b801353e3034a127223e042fae2 100644 --- a/modules/logicimmo/browser.py +++ b/modules/logicimmo/browser.py @@ -26,7 +26,7 @@ from .pages import CitiesPage, SearchPage, HousingPage, PhonePage class LogicimmoBrowser(PagesBrowser): - BASEURL = 'http://www.logic-immo.com/' + BASEURL = 'https://www.logic-immo.com/' PROFILE = Firefox() city = URL('asset/t9/getLocalityT9.php\?site=fr&lang=fr&json=%22(?P.*)%22', CitiesPage) diff --git a/modules/weather/pages.py b/modules/weather/pages.py index b097bbafa1390f5ae5417e14555771296bcdd4cd..80de0f53d709db74cbcfa50015d7adbb41847851 100644 --- a/modules/weather/pages.py +++ b/modules/weather/pages.py @@ -46,7 +46,7 @@ class WeatherPage(JsonPage): class get_current(ItemElement): klass = Current - obj_date = DateTime(Dict('vt1currentdatetime/datetime')) + obj_date = DateTime(Dict('vt1currentdatetime/dateTime')) obj_id = Env('city_id') obj_text = Format('%shPa (%s) - humidity %s%% - feels like %s°C - %s', Dict('vt1observation/altimeter'),