From 372520a8a61fc0ac2d0b2e829bc4b58466e3cde8 Mon Sep 17 00:00:00 2001 From: Vincent A Date: Wed, 7 Oct 2020 18:50:45 +0200 Subject: [PATCH] backport master modules fixes --- modules/boursedirect/browser.py | 8 +- modules/boursedirect/pages.py | 39 +++++- modules/boursorama/browser.py | 28 +++- modules/boursorama/pages.py | 30 +++++ modules/bp/pages/transfer.py | 2 +- modules/bred/bred/browser.py | 4 +- modules/caissedepargne/browser.py | 51 ++++--- modules/caissedepargne/module.py | 4 +- modules/caissedepargne/pages.py | 4 +- modules/canalplus/pages.py | 4 +- modules/cragr/browser.py | 12 +- modules/cragr/pages.py | 3 + modules/creditdunord/pages.py | 2 + modules/creditmutuel/browser.py | 13 +- modules/creditmutuel/pages.py | 17 +++ modules/entreparticuliers/pages.py | 2 +- modules/genericnewspaper/pages.py | 2 +- modules/hds/pages.py | 2 +- modules/ing/boursedirect_browser.py | 13 +- modules/ing/boursedirect_pages.py | 12 ++ modules/lcl/module.py | 15 ++- modules/mareeinfo/pages.py | 2 +- modules/redmine/pages/issues.py | 4 +- modules/s2e/browser.py | 28 +++- modules/s2e/pages.py | 2 +- modules/transilien/pages.py | 2 +- modules/unsplash/__init__.py | 26 ++++ modules/unsplash/browser.py | 43 ++++++ modules/unsplash/compat/__init__.py | 0 .../compat/weboob_browser_filters_standard.py | 124 ++++++++++++++++++ .../unsplash/compat/weboob_browser_pages.py | 21 +++ modules/unsplash/module.py | 49 +++++++ modules/unsplash/pages.py | 54 ++++++++ modules/unsplash/test.py | 43 ++++++ 34 files changed, 607 insertions(+), 58 deletions(-) create mode 100644 modules/unsplash/__init__.py create mode 100644 modules/unsplash/browser.py create mode 100644 modules/unsplash/compat/__init__.py create mode 100644 modules/unsplash/compat/weboob_browser_filters_standard.py create mode 100644 modules/unsplash/compat/weboob_browser_pages.py create mode 100644 modules/unsplash/module.py create mode 100644 modules/unsplash/pages.py create mode 100644 modules/unsplash/test.py diff --git a/modules/boursedirect/browser.py b/modules/boursedirect/browser.py index 7279cf356b..d46aaceeeb 100644 --- a/modules/boursedirect/browser.py +++ b/modules/boursedirect/browser.py @@ -30,6 +30,7 @@ LoginPage, PasswordRenewalPage, AccountsPage, HistoryPage, InvestPage, MarketOrdersPage, MarketOrderDetailsPage, LifeInsurancePage, IsinPage, PortfolioPage, JsRedirectPage, + HomePage, ) @@ -38,13 +39,14 @@ class BoursedirectBrowser(LoginBrowser): login = URL(r'/fr/login', LoginPage) password_renewal = URL(r'/fr/changer-mon-mot-de-passe', PasswordRenewalPage) + home = URL(r'/fr/page/inventaire', HomePage) accounts = URL( - r'/priv/compte.php$', - r'/priv/compte.php\?nc=(?P\d+)', + r'/priv/new/compte.php$', + r'/priv/new/compte.php\?nc=(?P\d+)', r'/priv/listeContrats.php\?nc=(?P\d+)', AccountsPage ) - history = URL(r'/priv/compte.php\?ong=3&nc=(?P\d+)', HistoryPage) + history = URL(r'/priv/new/historique-de-compte.php\?ong=3&nc=(?P\d+)', HistoryPage) portfolio = URL(r'/fr/page/portefeuille', PortfolioPage) pre_invests = URL(r'/priv/portefeuille-TR.php\?nc=(?P\d+)') invests = URL(r'/streaming/compteTempsReelCK.php\?stream=0', InvestPage) diff --git a/modules/boursedirect/pages.py b/modules/boursedirect/pages.py index 4fbfad3457..9e15493d4d 100644 --- a/modules/boursedirect/pages.py +++ b/modules/boursedirect/pages.py @@ -86,6 +86,10 @@ def logged(self): ) +class HomePage(BasePage): + pass + + class AccountsPage(BasePage): @method class iter_accounts(ListElement): @@ -333,16 +337,39 @@ class fill_market_order(ItemElement): class HistoryPage(BasePage): @method - class iter_history(ListElement): - item_xpath = '//table[@class="datas retour"]//tr[@class="row1" or @class="row2"]' + class iter_history(TableElement): + item_xpath = '//table[contains(@class,"datas retour")]//tr[@class="row1" or @class="row2"]' + head_xpath = '//table[contains(@class,"datas retour")]//th' + + col_rdate = 'Date opération' + col_date = 'Date affectation' + col_investment_label = 'Libellé' + col_label = 'Opération' + col_investment_quantity = 'Qté' + col_investment_unitvalue = 'Cours' + col_amount = 'Montant net' class item(ItemElement): klass = Transaction - obj_date = Date(CleanText('./td[2]'), dayfirst=True) # Date affectation - obj_rdate = Date(CleanText('./td[1]'), dayfirst=True) # Date opération - obj_label = Format('%s - %s', CleanText('./td[3]/a'), CleanText('./td[4]')) - obj_amount = CleanDecimal.French('./td[7]') + obj_date = Date(CleanText(TableCell('date')), dayfirst=True) # Date affectation + obj_rdate = Date(CleanText(TableCell('rdate')), dayfirst=True) # Date opération + obj_label = Format('%s - %s', CleanText(TableCell('investment_label')), CleanText(TableCell('label'))) + obj_amount = CleanDecimal.French(TableCell('amount')) + + def obj_investments(self): + if CleanDecimal.French(TableCell('unitvalue'), default=None) is None: + return NotAvailable + + investment = Investment() + investment.label = CleanText(TableCell('investment_label'))(self) + investment.valuation = CleanDecimal.French(TableCell('amount'))(self) + investment.unitvalue = CleanDecimal.French( + TableCell('investment_unitvalue'), + default=NotAvailable + )(self) + investment.quantity = CleanDecimal.French(TableCell('investment_quantity'), default=NotAvailable)(self) + return [investment] class IsinPage(HTMLPage): diff --git a/modules/boursorama/browser.py b/modules/boursorama/browser.py index f97878cf8a..9b8c92cdc1 100644 --- a/modules/boursorama/browser.py +++ b/modules/boursorama/browser.py @@ -35,7 +35,7 @@ Account, AccountNotFound, TransferError, TransferInvalidAmount, TransferInvalidEmitter, TransferInvalidLabel, TransferInvalidRecipient, AddRecipientStep, Rate, TransferBankError, AccountOwnership, RecipientNotFound, - AddRecipientTimeout, TransferDateType, Emitter, + AddRecipientTimeout, TransferDateType, Emitter, TransactionType, ) from weboob.capabilities.base import empty, find_object from weboob.capabilities.contact import Advisor @@ -51,7 +51,7 @@ TransferAccounts, TransferRecipients, TransferCharac, TransferConfirm, TransferSent, AddRecipientPage, StatusPage, CardHistoryPage, CardCalendarPage, CurrencyListPage, CurrencyConvertPage, AccountsErrorPage, NoAccountPage, TransferMainPage, PasswordPage, NewTransferRecipients, - NewTransferAccounts, + NewTransferAccounts, CardSumDetailPage, ) from .transfer_pages import TransferListPage, TransferInfoPage @@ -96,6 +96,7 @@ class BoursoramaBrowser(RetryLoginBrowser, TwoFactorBrowser): budget_transactions = URL('/budget/compte/(?P.*)/mouvements.*', HistoryPage) other_transactions = URL('/compte/cav/(?P.*)/mouvements.*', HistoryPage) saving_transactions = URL('/compte/epargne/csl/(?P.*)/mouvements.*', HistoryPage) + card_summary_detail_transactions = URL(r'/contre-valeurs-operation/.*', CardSumDetailPage) saving_pep = URL('/compte/epargne/pep', PEPPage) incident = URL('/compte/cav/(?P.*)/mes-incidents.*', IncidentPage) @@ -456,6 +457,25 @@ def get_regular_transactions(self, account, coming): for transaction in self.page.iter_history(): yield transaction + def get_html_past_card_transactions(self, account): + """ Get card transactions from parent account page """ + + self.otp_location('%s/mouvements' % account.parent.url.rstrip('/')) + for tr in self.page.iter_history(is_card=False): + # get card summaries + if ( + tr.type == TransactionType.CARD_SUMMARY + and account.number in tr.label # in case of several cards per parent account + ): + tr.amount = - tr.amount + yield tr + + # for each summaries, get detailed transactions + self.location(tr._card_sum_detail_link) + for detail_tr in self.page.iter_history(): + detail_tr.date = tr.date + yield detail_tr + # Note: Checking accounts have a 'Mes prélèvements à venir' tab, # but these transactions have no date anymore so we ignore them. @@ -497,8 +517,8 @@ def get_card_transactions(self, account, coming): if self.get_card_transaction(coming, tr): yield tr - for tr in self.page.iter_history(is_card=True): - if self.get_card_transaction(coming, tr): + if not coming: + for tr in self.get_html_past_card_transactions(account): yield tr def get_invest_transactions(self, account, coming): diff --git a/modules/boursorama/pages.py b/modules/boursorama/pages.py index 02557dc7a0..cde06968ca 100644 --- a/modules/boursorama/pages.py +++ b/modules/boursorama/pages.py @@ -700,6 +700,11 @@ def obj_date(self): return date + def obj__card_sum_detail_link(self): + if Field('type')(self) == Transaction.TYPE_CARD_SUMMARY: + return Attr('.//div', 'data-action-url')(self.el) + return NotAvailable + def validate(self, obj): # TYPE_DEFERRED_CARD transactions are already present in the card history # so we only return TYPE_DEFERRED_CARD for the coming: @@ -740,6 +745,25 @@ def get_calendar_link(self): return Link('//a[contains(text(), "calendrier")]')(self.doc) +class CardSumDetailPage(LoggedPage, HTMLPage): + @otp_pagination + @method + class iter_history(ListElement): + item_xpath = '//li[contains(@class, "deffered")]' # this quality website's got all-you-can-eat typos! + + class item(ItemElement): + klass = Transaction + + obj_amount = CleanDecimal.French('.//div[has-class("list-operation-item__amount")]') + obj_raw = Transaction.Raw(CleanText('.//div[has-class("list-operation-item__label-name")]')) + obj_id = Attr('.', 'data-id') + obj__is_coming = False + + def obj_type(self): + # to override CARD typing done by obj.raw + return Transaction.TYPE_DEFERRED_CARD + + class CardHistoryPage(LoggedPage, CsvPage): ENCODING = 'latin-1' FMTPARAMS = {'delimiter': str(';')} @@ -1298,6 +1322,12 @@ def obj_enabled_at(self): class NewTransferAccounts(LoggedPage, HTMLPage): def submit_account(self, account_id): + no_account_msg = CleanText('//div[contains(@class, "alert--warning")]')(self.doc) + if 'Vous ne possédez pas de compte éligible au virement' in no_account_msg: + raise AccountNotFound() + elif no_account_msg: + raise AssertionError('Unhandled error message : "%s"' % no_account_msg) + form = self.get_form() debit_account = CleanText( '//input[./following-sibling::div/span/span[contains(text(), "%s")]]/@value' % account_id diff --git a/modules/bp/pages/transfer.py b/modules/bp/pages/transfer.py index fc087619ec..45c124275b 100644 --- a/modules/bp/pages/transfer.py +++ b/modules/bp/pages/transfer.py @@ -411,7 +411,7 @@ class OtpErrorPage(LoggedPage, PartialHTMLPage): # Need PartialHTMLPage because sometimes we land on this page with # a status_code 302, so the page is empty and the build_doc crash. def get_error(self): - return CleanText('//form//span[@class="warning"]')(self.doc) + return CleanText('//form//span[@class="warning" or @class="app_erreur"]')(self.doc) class RecipientSubmitDevicePage(LoggedPage, MyHTMLPage): diff --git a/modules/bred/bred/browser.py b/modules/bred/bred/browser.py index f322246f4e..da9966562e 100644 --- a/modules/bred/bred/browser.py +++ b/modules/bred/bred/browser.py @@ -280,6 +280,7 @@ def get_history(self, account, coming=False): today = date.today() seen = set() offset = 0 + total_transactions = 0 next_page = True end_date = date.today() last_date = None @@ -309,10 +310,11 @@ def get_history(self, account, coming=False): next_page = len(transactions) > 0 offset += 50 + total_transactions += 50 # This assert supposedly prevents infinite loops, # but some customers actually have a lot of transactions. - assert offset < 100000, 'the site may be doing an infinite loop' + assert total_transactions < 50000, 'the site may be doing an infinite loop' @need_login def iter_investments(self, account): diff --git a/modules/caissedepargne/browser.py b/modules/caissedepargne/browser.py index 38a992d35c..7d11f97dda 100644 --- a/modules/caissedepargne/browser.py +++ b/modules/caissedepargne/browser.py @@ -1512,16 +1512,21 @@ def init_transfer(self, account, recipient, transfer): if self.validation_option.is_here(): self.get_auth_mechanisms_validation_info() - if self.otp_validation['type'] == 'CLOUDCARD': - raise AuthMethodNotImplemented() - - raise TransferStep( - transfer, - Value( - 'otp_sms', - label='Veuillez renseigner le mot de passe unique qui vous a été envoyé par SMS dans le champ réponse.' + if self.otp_validation['type'] == 'SMS': + self.is_send_sms = True + raise TransferStep( + transfer, + Value( + 'otp_sms', + label='Veuillez renseigner le mot de passe unique qui vous a été envoyé par SMS dans le champ réponse.' + ) + ) + elif self.otp_validation['type'] == 'CLOUDCARD': + self.is_app_validation = True + raise AppValidation( + resource=transfer, + message="Veuillez valider le transfert sur votre application mobile.", ) - ) if 'netpro' in self.url: return self.page.create_transfer(account, recipient, transfer) @@ -1530,15 +1535,27 @@ def init_transfer(self, account, recipient, transfer): return self.page.update_transfer(transfer, account, recipient) @need_login - def otp_sms_continue_transfer(self, transfer, **params): - self.is_send_sms = False - assert 'otp_sms' in params, 'OTP SMS is missing' + def otp_validation_continue_transfer(self, transfer, **params): + assert ( + 'resume' in params + or 'otp_sms' in params + ), 'otp_sms or resume is missing' + + if 'resume' in params: + self.is_app_validation = False + + self.do_authentication_validation( + authentication_method='CLOUDCARD', + feature='transfer', + ) + elif 'otp_sms' in params: + self.is_send_sms = False - self.do_authentication_validation( - authentication_method='SMS', - feature='transfer', - otp_sms=params['otp_sms'] - ) + self.do_authentication_validation( + authentication_method='SMS', + feature='transfer', + otp_sms=params['otp_sms'] + ) if self.transfer.is_here(): self.page.continue_transfer(transfer.account_label, transfer.recipient_label, transfer.label) diff --git a/modules/caissedepargne/module.py b/modules/caissedepargne/module.py index d7e4e773c9..a56937dbfb 100644 --- a/modules/caissedepargne/module.py +++ b/modules/caissedepargne/module.py @@ -115,8 +115,8 @@ def iter_transfer_recipients(self, origin_account): return self.browser.iter_recipients(origin_account) def init_transfer(self, transfer, **params): - if 'otp_sms' in params: - return self.browser.otp_sms_continue_transfer(transfer, **params) + if 'otp_sms' in params or 'resume' in params: + return self.browser.otp_validation_continue_transfer(transfer, **params) self.logger.info('Going to do a new transfer') transfer.label = re.sub(r"[^0-9A-Z/?:().,'+ -]+", '', transfer.label.upper()) diff --git a/modules/caissedepargne/pages.py b/modules/caissedepargne/pages.py index 520a466b03..1332516876 100644 --- a/modules/caissedepargne/pages.py +++ b/modules/caissedepargne/pages.py @@ -47,6 +47,7 @@ Transfer, TransferBankError, TransferInvalidOTP, Recipient, AddRecipientBankError, RecipientInvalidOTP, Emitter, EmitterNumberType, AddRecipientError, + TransferError, ) from .compat.weboob_capabilities_wealth import Investment from weboob.capabilities.bill import DocumentTypes, Subscription, Document @@ -188,8 +189,9 @@ def login_errors(self, error): def transfer_errors(self, error): if error == 'FAILED_AUTHENTICATION': - # For the moment, only otp sms is handled raise TransferInvalidOTP(message="Le code SMS que vous avez renseigné n'est pas valide") + elif error == 'AUTHENTICATION_CANCELED': + raise TransferError(message="Le virement a été annulée via l'application mobile.") def recipient_errors(self, error): if error == 'FAILED_AUTHENTICATION': diff --git a/modules/canalplus/pages.py b/modules/canalplus/pages.py index ff3c5f3d64..75fa804dc4 100644 --- a/modules/canalplus/pages.py +++ b/modules/canalplus/pages.py @@ -36,8 +36,8 @@ def get_channels(self): Extract all possible channels (paths) from the page """ channels = list() - for elem in self.doc[2].getchildren(): - for e in elem.getchildren(): + for elem in self.doc[2]: + for e in elem: if e.tag == "NOM": fid, name = self._clean_name(e.text) channels.append(Collection([fid], name)) diff --git a/modules/cragr/browser.py b/modules/cragr/browser.py index 3fcd191e35..f6f4abadda 100644 --- a/modules/cragr/browser.py +++ b/modules/cragr/browser.py @@ -914,16 +914,20 @@ def get_profile(self): # There is one profile per space, so we only fetch the first one self.go_to_account_space(0) owner_type = self.page.get_owner_type() + profile_details = self.page.has_profile_details() self.profile_page.go(space=self.space) + if owner_type == 'PRIV': profile = self.page.get_user_profile() - self.profile_details.go(space=self.space) - self.page.fill_profile(obj=profile) + if profile_details: + self.profile_details.go(space=self.space) + self.page.fill_profile(obj=profile) return profile elif owner_type == 'ORGA': profile = self.page.get_company_profile() - self.pro_profile_details.go(space=self.space) - self.page.fill_profile(obj=profile) + if profile_details: + self.pro_profile_details.go(space=self.space) + self.page.fill_profile(obj=profile) return profile def get_space_info(self): diff --git a/modules/cragr/pages.py b/modules/cragr/pages.py index c81626839b..380560a211 100644 --- a/modules/cragr/pages.py +++ b/modules/cragr/pages.py @@ -314,6 +314,9 @@ def get_connection_id(self): def has_main_account(self): return Dict('comptePrincipal', default=None)(self.doc) + def has_profile_details(self): + return CleanText('//a[text()="Gérer mes coordonnées"]')(self.html_doc) + @method class get_main_account(ItemElement): klass = Account diff --git a/modules/creditdunord/pages.py b/modules/creditdunord/pages.py index 4ba8f2f2f8..3598024053 100755 --- a/modules/creditdunord/pages.py +++ b/modules/creditdunord/pages.py @@ -712,6 +712,7 @@ class get_market_investment(TableElement): col_label = 'Valeur' col_quantity = 'Quantité' col_unitvalue = 'Cours' + col_unitprice = 'Prix de revient' col_valuation = 'Estimation' col_portfolio_share = '%' @@ -722,6 +723,7 @@ class item(ItemElement): obj_valuation = MyDecimal(TableCell('valuation', colspan=True)) obj_quantity = MyDecimal(TableCell('quantity', colspan=True)) obj_unitvalue = MyDecimal(TableCell('unitvalue', colspan=True)) + obj_unitprice = MyDecimal(TableCell('unitprice', colspan=True)) obj_portfolio_share = Eval(lambda x: x / 100, MyDecimal(TableCell('portfolio_share'))) def obj_code(self): diff --git a/modules/creditmutuel/browser.py b/modules/creditmutuel/browser.py index 3d87edead4..2ffb847237 100644 --- a/modules/creditmutuel/browser.py +++ b/modules/creditmutuel/browser.py @@ -59,7 +59,7 @@ ConditionsPage, MobileConfirmationPage, UselessPage, DecoupledStatePage, CancelDecoupled, OtpValidationPage, OtpBlockedErrorPage, TwoFAUnabledPage, LoansOperationsPage, OutagePage, PorInvestmentsPage, PorHistoryPage, PorHistoryDetailsPage, - PorMarketOrdersPage, PorMarketOrderDetailsPage, + PorMarketOrdersPage, PorMarketOrderDetailsPage, SafeTransPage, ) @@ -86,6 +86,7 @@ class CreditMutuelBrowser(TwoFactorBrowser): outage_page = URL(r'/fr/outage.html', OutagePage) twofa_unabled_page = URL(r'/(?P.*)fr/banque/validation.aspx', TwoFAUnabledPage) mobile_confirmation = URL(r'/(?P.*)fr/banque/validation.aspx', MobileConfirmationPage) + safetrans_page = URL(r'/(?P.*)fr/banque/validation.aspx', SafeTransPage) decoupled_state = URL(r'/fr/banque/async/otp/SOSD_OTP_GetTransactionState.htm', DecoupledStatePage) cancel_decoupled = URL(r'/fr/banque/async/otp/SOSD_OTP_CancelTransaction.htm', CancelDecoupled) otp_validation_page = URL(r'/(?P.*)fr/banque/validation.aspx', OtpValidationPage) @@ -98,7 +99,11 @@ class CreditMutuelBrowser(TwoFactorBrowser): AccountsPage) useless_page = URL(r'/(?P.*)fr/banque/paci/defi-solidaire.html', UselessPage) - revolving_loan_list = URL(r'/(?P.*)fr/banque/CR/arrivee.asp\?fam=CR.*', RevolvingLoansList) + revolving_loan_list = URL( + r'/(?P.*)fr/banque/CR/arrivee.asp\?fam=CR.*', + r'/(?P.*)fr/banque/arrivee.asp\?fam=CR.*', + RevolvingLoansList + ) revolving_loan_details = URL(r'/(?P.*)fr/banque/CR/cam9_vis_lstcpt.asp.*', RevolvingLoanDetails) user_space = URL(r'/(?P.*)fr/banque/espace_personnel.aspx', r'/(?P.*)fr/banque/accueil.cgi', @@ -370,6 +375,10 @@ def check_auth_methods(self): assert self.polling_data, "Can't proceed to polling if no polling_data" raise AppValidation(self.page.get_validation_msg()) + if self.safetrans_page.is_here(): + msg = self.page.get_safetrans_message() + raise AuthMethodNotImplemented(msg) + if self.otp_validation_page.is_here(): self.otp_data = self.page.get_otp_data() assert self.otp_data, "Can't proceed to SMS handling if no otp_data" diff --git a/modules/creditmutuel/pages.py b/modules/creditmutuel/pages.py index db01ccb394..56453262f2 100644 --- a/modules/creditmutuel/pages.py +++ b/modules/creditmutuel/pages.py @@ -193,6 +193,23 @@ def check_bypass(self): self.logger.warning('This connexion cannot bypass mobile confirmation') +# PartialHTMLPage: this page shares URL with other pages, +# that might be empty of text while used in a redirection +class SafeTransPage(PartialHTMLPage, AppValidationPage): + # only 'class' and cryptic 'id' tags on this page + # so we scrape based on text, not tags + def is_here(self): + return ( + 'Authentification forte' in CleanText('//p[contains(@id, "title")]')(self.doc) + and CleanText('//*[contains(text(), "confirmer votre connexion avec Safetrans")]')(self.doc) + ) + + def get_safetrans_message(self): + return CleanText( + '//*[contains(text(), "Confirmation Mobile") or contains(text(), "confirmer votre connexion avec Safetrans")]' + )(self.doc) + + class TwoFAUnabledPage(PartialHTMLPage): def is_here(self): return self.doc.xpath('//*[contains(text(), "aucun moyen pour confirmer")]') diff --git a/modules/entreparticuliers/pages.py b/modules/entreparticuliers/pages.py index 667e67eb58..f7582f4862 100644 --- a/modules/entreparticuliers/pages.py +++ b/modules/entreparticuliers/pages.py @@ -97,7 +97,7 @@ class HousingPage(XMLPage): def build_doc(self, content): doc = super(HousingPage, self).build_doc(content).getroot() - for elem in doc.getiterator(): + for elem in doc.iter(): if not hasattr(elem.tag, 'find'): continue i = elem.tag.find('}') diff --git a/modules/genericnewspaper/pages.py b/modules/genericnewspaper/pages.py index 7565537b9c..7beaf4072f 100644 --- a/modules/genericnewspaper/pages.py +++ b/modules/genericnewspaper/pages.py @@ -103,7 +103,7 @@ def get_article(self, _id): return __article def drop_comments(self, base_element): - for comment in base_element.getiterator(Comment): + for comment in base_element.iter(Comment): comment.drop_tree() def try_remove(self, base_element, selector): diff --git a/modules/hds/pages.py b/modules/hds/pages.py index f7eb32811d..edf444428b 100644 --- a/modules/hds/pages.py +++ b/modules/hds/pages.py @@ -129,7 +129,7 @@ class get_author(ItemElement): def obj_description(self): description = u'' - for para in self.el.xpath('//td[has-class("t0")]')[0].getchildren(): + for para in self.el.xpath('//td[has-class("t0")]')[0]: if para.tag not in ('b', 'br'): continue if para.text is not None: diff --git a/modules/ing/boursedirect_browser.py b/modules/ing/boursedirect_browser.py index 13c327e4b1..300e5ef263 100644 --- a/modules/ing/boursedirect_browser.py +++ b/modules/ing/boursedirect_browser.py @@ -24,13 +24,24 @@ from weboob.browser import AbstractBrowser, URL, need_login -from .boursedirect_pages import MarketOrdersPage, MarketOrderDetailsPage +from .boursedirect_pages import ( + MarketOrdersPage, MarketOrderDetailsPage, AccountsPage, HistoryPage, +) class BourseDirectBrowser(AbstractBrowser): PARENT = 'boursedirect' BASEURL = 'https://bourse.ing.fr' + # These URLs have been updated on Bourse Direct but not on ING. + # If they are updated on ING, remove these definitions and associated abstract pages. + accounts = URL( + r'/priv/compte.php$', + r'/priv/compte.php\?nc=(?P\d+)', + r'/priv/listeContrats.php\?nc=(?P\d+)', + AccountsPage + ) + history = URL(r'/priv/compte.php\?ong=3&nc=(?P\d+)', HistoryPage) market_orders = URL(r'/priv/compte.php\?ong=7', MarketOrdersPage) market_orders_details = URL(r'/priv/detailOrdre.php', MarketOrderDetailsPage) diff --git a/modules/ing/boursedirect_pages.py b/modules/ing/boursedirect_pages.py index bbcbf527cf..85adaa20a7 100644 --- a/modules/ing/boursedirect_pages.py +++ b/modules/ing/boursedirect_pages.py @@ -24,6 +24,18 @@ from .compat.weboob_browser_pages import AbstractPage +class AccountsPage(AbstractPage): + PARENT = 'boursedirect' + PARENT_URL = 'accounts' + BROWSER_ATTR = 'package.browser.BoursedirectBrowser' + + +class HistoryPage(AbstractPage): + PARENT = 'boursedirect' + PARENT_URL = 'history' + BROWSER_ATTR = 'package.browser.BoursedirectBrowser' + + class MarketOrdersPage(AbstractPage): PARENT = 'boursedirect' PARENT_URL = 'market_orders' diff --git a/modules/lcl/module.py b/modules/lcl/module.py index 25b0a141e2..2c53c88097 100644 --- a/modules/lcl/module.py +++ b/modules/lcl/module.py @@ -37,7 +37,9 @@ from weboob.tools.backend import Module, BackendConfig from weboob.tools.capabilities.bank.transactions import sorted_transactions from .compat.weboob_tools_value import ValueBackendPassword, Value -from weboob.capabilities.base import find_object, strict_find_object, NotAvailable +from weboob.capabilities.base import ( + find_object, strict_find_object, NotAvailable, empty, +) from .browser import LCLBrowser, LCLProBrowser from .enterprise.browser import LCLEnterpriseBrowser, LCLEspaceProBrowser @@ -173,6 +175,17 @@ def transfer_check_label(self, old, new): return True return super(LCLModule, self).transfer_check_label(old, new) + def transfer_check_account_iban(self, old, new): + # Some accounts' ibans cannot be found anymore on the website. But since we + # kept the iban stored on our side, the 'old' transfer.account_iban is not + # empty when making a transfer. When we do not find the account based on its iban, + # we search it based on its id. So the account is valid, the iban is just empty. + # This check allows to not have an assertion error when making a transfer from + # an account in this situation. + if empty(new): + return True + return old == new + @only_for_websites('par', 'elcl', 'pro') def iter_contacts(self): return self.browser.get_advisor() diff --git a/modules/mareeinfo/pages.py b/modules/mareeinfo/pages.py index 4d4cec00bc..301ac0a43c 100644 --- a/modules/mareeinfo/pages.py +++ b/modules/mareeinfo/pages.py @@ -142,7 +142,7 @@ def _create_low_tide(self, gauge_id, AM=True): return tide def _is_low_tide_first(self, jour): - return XPath('//tr[@id="MareeJours_%s"]/td[1]' % jour)(self)[0].getchildren()[0].tag != 'b' + return list(XPath('//tr[@id="MareeJours_%s"]/td[1]' % jour)(self)[0])[0].tag != 'b' def _get_low_tide_value(self, AM=True, jour=0): slow_tide_pos = 1 if self._is_low_tide_first(jour) else 2 diff --git a/modules/redmine/pages/issues.py b/modules/redmine/pages/issues.py index f1dd222a9a..4f105ec1b5 100644 --- a/modules/redmine/pages/issues.py +++ b/modules/redmine/pages/issues.py @@ -184,11 +184,11 @@ def iter_issues(self): # No results. return - for tr in issues.getiterator('tr'): + for tr in issues.iter('tr'): if not tr.attrib.get('id', '').startswith('issue-'): continue issue = {'id': tr.attrib['id'].replace('issue-', '')} - for td in tr.getiterator('td'): + for td in tr.iter('td'): field = td.attrib.get('class', '') if field in ('checkbox','todo',''): continue diff --git a/modules/s2e/browser.py b/modules/s2e/browser.py index 1185610b7a..adc6b92e92 100644 --- a/modules/s2e/browser.py +++ b/modules/s2e/browser.py @@ -22,9 +22,11 @@ from __future__ import unicode_literals import re +from requests.exceptions import ConnectionError +from urllib3.exceptions import ReadTimeoutError from weboob.browser.browsers import LoginBrowser, URL, need_login, StatesMixin -from weboob.browser.exceptions import ServerError +from weboob.browser.exceptions import ServerError, HTTPNotFound from weboob.exceptions import BrowserIncorrectPassword, ActionNeeded, NoAccountsException from .compat.weboob_capabilities_wealth import Investment from weboob.tools.capabilities.bank.investments import is_isin_valid @@ -229,17 +231,33 @@ def update_investments(self, investments): self.logger.warning('Server returned a Server Error when trying to fetch investment performances.') continue + if not self.bnp_investments.is_here(): + # BNPInvestmentsPage was not accessible, trying the next request + # would lead to a 401 error. + self.logger.warning('Could not access BNP investments page, no investment details will be fetched.') + continue + # Access the BNP API to get the investment details using its ID (found in its label) m = re.search(r'- (\d+)$', inv.label) if m: inv_id = m.group(1) - self.bnp_investment_details.go(id=inv_id) - self.page.fill_investment(obj=inv) + try: + self.bnp_investment_details.go(id=inv_id) + except (ConnectionError, ReadTimeoutError): + # The BNP API times out quite often so we must handle timeout errors + self.logger.warning('Could not connect to the BNP API, no investment details will be fetched.') + continue + else: + self.page.fill_investment(obj=inv) else: - self.logger.warning('Could not fetch BNP investment ID in its label, no details will be fetched.') + self.logger.warning('Could not fetch BNP investment ID in its label, no investment details will be fetched.') elif self.amfcode_amundi.match(inv._link): - self.location(inv._link) + try: + self.location(inv._link) + except HTTPNotFound: + self.logger.warning('Details on AMF Amundi page are not available for this investment.') + continue details_url = self.page.get_details_url() performance_url = self.page.get_performance_url() if details_url: diff --git a/modules/s2e/pages.py b/modules/s2e/pages.py index 53b7d109f6..7a08bc8581 100644 --- a/modules/s2e/pages.py +++ b/modules/s2e/pages.py @@ -275,7 +275,7 @@ class AMFHSBCPage(LoggedPage, XMLPage, CodePage): def build_doc(self, content): doc = super(AMFHSBCPage, self).build_doc(content).getroot() # Remove namespaces - for el in doc.getiterator(): + for el in doc.iter(): if not hasattr(el.tag, 'find'): continue i = el.tag.find('}') diff --git a/modules/transilien/pages.py b/modules/transilien/pages.py index c0a9b1721f..034a3fb4bc 100644 --- a/modules/transilien/pages.py +++ b/modules/transilien/pages.py @@ -38,7 +38,7 @@ class RoadMapDuration(Duration): class DepartureTypeFilter(Filter): def filter(self, el): result = [] - for img in el[0].getiterator(tag='img'): + for img in el[0].iter(tag='img'): result.append(img.attrib['alt']) return u' '.join(result) diff --git a/modules/unsplash/__init__.py b/modules/unsplash/__init__.py new file mode 100644 index 0000000000..d8efe61a98 --- /dev/null +++ b/modules/unsplash/__init__.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Vincent A +# +# This file is part of a weboob module. +# +# This weboob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This weboob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this weboob module. If not, see . + +from __future__ import unicode_literals + + +from .module import UnsplashModule + + +__all__ = ['UnsplashModule'] diff --git a/modules/unsplash/browser.py b/modules/unsplash/browser.py new file mode 100644 index 0000000000..8b4572872c --- /dev/null +++ b/modules/unsplash/browser.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Vincent A +# +# This file is part of a weboob module. +# +# This weboob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This weboob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this weboob module. If not, see . + +from weboob.browser import PagesBrowser, URL + +from .pages import ImageSearch + + +class UnsplashBrowser(PagesBrowser): + BASEURL = 'https://unsplash.com' + + collection_search = URL(r'/napi/search/collections\?query=(?P[^&]+)&page=(?P\d+)&per_page=20') + image_search = URL(r'/napi/search/photos\?query=(?P[^&]+)&page=(?P\d+)&per_page=20', ImageSearch) + + def __init__(self, *args, **kwargs): + super(UnsplashBrowser, self).__init__(*args, **kwargs) + self.session.headers['Authorization'] = 'Client-ID d69927c7ea5c770fa2ce9a2f1e3589bd896454f7068f689d8e41a25b54fa6042' + + def search_image(self, term): + n = 1 + nb_pages = 1 + while n <= nb_pages: + self.image_search.go(term=term, page=n) + for img in self.page.iter_images(): + yield img + nb_pages = self.page.nb_pages() + n += 1 diff --git a/modules/unsplash/compat/__init__.py b/modules/unsplash/compat/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/modules/unsplash/compat/weboob_browser_filters_standard.py b/modules/unsplash/compat/weboob_browser_filters_standard.py new file mode 100644 index 0000000000..7ff30b8761 --- /dev/null +++ b/modules/unsplash/compat/weboob_browser_filters_standard.py @@ -0,0 +1,124 @@ +import weboob.browser.filters.standard as OLD + +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) + +try: + __all__ = OLD.__all__ +except AttributeError: + pass + + +class CleanDecimal(CleanText): + """ + Get a cleaned Decimal value from an element. + + `replace_dots` is False by default. A dot is interpreted as a decimal separator. + + If `replace_dots` is set to True, we remove all the dots. The ',' is used as decimal + separator (often useful for French values) + + If `replace_dots` is a tuple, the first element will be used as the thousands separator, + and the second as the decimal separator. + + See http://en.wikipedia.org/wiki/Thousands_separator#Examples_of_use + + For example, for the UK style (as in 1,234,567.89): + + >>> CleanDecimal('./td[1]', replace_dots=(',', '.')) # doctest: +SKIP + """ + + def __init__(self, selector=None, replace_dots=False, sign=None, legacy=True, default=_NO_DEFAULT): + """ + :param sign: function accepting the text as param and returning the sign + """ + + super(CleanDecimal, self).__init__(selector, default=default) + self.replace_dots = replace_dots + self.sign = sign + self.legacy = legacy + if not legacy: + thousands_sep, decimal_sep = self.replace_dots + self.matching = re.compile(r'([+-]?)\s*(\d[\d%s%s]*|%s\d+)' % tuple(map(re.escape, (thousands_sep, decimal_sep, decimal_sep)))) + self.thousand_check = re.compile(r'^[+-]?\d{1,3}(%s\d{3})*(%s\d*)?$' % tuple(map(re.escape, (thousands_sep, decimal_sep)))) + + @debug() + def filter(self, text): + if type(text) in (float, int, long): + text = str(text) + + if empty(text): + return self.default_or_raise(FormatError('Unable to parse %r' % text)) + + original_text = text = super(CleanDecimal, self).filter(text) + + text = text.replace(u'\u2212', '-') + + if self.legacy: + if self.replace_dots: + if type(self.replace_dots) is tuple: + thousands_sep, decimal_sep = self.replace_dots + else: + thousands_sep, decimal_sep = '.', ',' + text = text.replace(thousands_sep, '').replace(decimal_sep, '.') + + text = re.sub(r'[^\d\-\.]', '', text) + else: + thousands_sep, decimal_sep = self.replace_dots + + matches = self.matching.findall(text) + if not matches: + return self.default_or_raise(NumberFormatError('There is no number to parse')) + elif len(matches) > 1: + return self.default_or_raise(NumberFormatError('There should be exactly one number to parse')) + + text = '%s%s' % (matches[0][0], matches[0][1].strip()) + + if thousands_sep and thousands_sep in text and not self.thousand_check.match(text): + return self.default_or_raise(NumberFormatError('Thousands separator is misplaced in %r' % text)) + + text = text.replace(thousands_sep, '').replace(decimal_sep, '.') + + try: + v = Decimal(text) + except InvalidOperation as e: + return self.default_or_raise(NumberFormatError(e)) + else: + if self.sign is not None: + if callable(self.sign): + v *= self.sign(original_text) + elif self.sign == '+': + return abs(v) + elif self.sign == '-': + return -abs(v) + else: + raise TypeError("'sign' should be a callable or a sign string") + return v + + @classmethod + def US(cls, *args, **kwargs): + kwargs['legacy'] = False + kwargs['replace_dots'] = (',', '.') + return cls(*args, **kwargs) + + @classmethod + def French(cls, *args, **kwargs): + kwargs['legacy'] = False + kwargs['replace_dots'] = (' ', ',') + return cls(*args, **kwargs) + + @classmethod + def SI(cls, *args, **kwargs): + kwargs['legacy'] = False + kwargs['replace_dots'] = (' ', '.') + return cls(*args, **kwargs) + + @classmethod + def Italian(cls, *args, **kwargs): + kwargs['legacy'] = False + kwargs['replace_dots'] = ('.', ',') + return cls(*args, **kwargs) + + +del OLD diff --git a/modules/unsplash/compat/weboob_browser_pages.py b/modules/unsplash/compat/weboob_browser_pages.py new file mode 100644 index 0000000000..b16c529203 --- /dev/null +++ b/modules/unsplash/compat/weboob_browser_pages.py @@ -0,0 +1,21 @@ +import weboob.browser.pages as OLD + +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) + +try: + __all__ = OLD.__all__ +except AttributeError: + pass + + +class JsonPage(OLD.JsonPage): + def get(self, path, default=None): + try: + return next(self.path(path)) + except StopIteration: + return default + + +del OLD diff --git a/modules/unsplash/module.py b/modules/unsplash/module.py new file mode 100644 index 0000000000..5a1c627ca8 --- /dev/null +++ b/modules/unsplash/module.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Vincent A +# +# This file is part of a weboob module. +# +# This weboob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This weboob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this weboob module. If not, see . + +from weboob.tools.backend import Module +from weboob.capabilities.file import CapFile +from weboob.capabilities.image import CapImage, BaseImage + +from .browser import UnsplashBrowser + + +__all__ = ['UnsplashModule'] + + +class UnsplashModule(Module, CapImage): + NAME = 'unsplash' + DESCRIPTION = u'unsplash website' + MAINTAINER = u'Vincent A' + EMAIL = 'dev@indigo.re' + LICENSE = 'AGPLv3+' + VERSION = '2.0' + + BROWSER = UnsplashBrowser + + def search_image(self, pattern, sortby=CapFile.SEARCH_RELEVANCE, nsfw=False): + return self.browser.search_image(pattern) + + def fill_image(self, img, fields): + if 'data' in fields: + img.data = self.browser.open(img.url).content + if 'thumbnail' in fields: + img.thumbnail.data = self.browser.open(img.thumbnail.url).content + + OBJECTS = {BaseImage: fill_image} diff --git a/modules/unsplash/pages.py b/modules/unsplash/pages.py new file mode 100644 index 0000000000..1d3515388b --- /dev/null +++ b/modules/unsplash/pages.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Vincent A +# +# This file is part of a weboob module. +# +# This weboob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This weboob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this weboob module. If not, see . + +from .compat.weboob_browser_pages import JsonPage +from weboob.browser.elements import DictElement, ItemElement, method +from weboob.browser.filters.json import Dict +from .compat.weboob_browser_filters_standard import DateTime, Field, Format +from weboob.capabilities.image import BaseImage, Thumbnail +from weboob.capabilities.file import LICENSES + + +class CollectionSearch(JsonPage): + def do_stuff(self, _id): + raise NotImplementedError() + + +class ImageSearch(JsonPage): + def nb_pages(self): + return self.doc['total_pages'] + + @method + class iter_images(DictElement): + item_xpath = 'results' + + class item(ItemElement): + klass = BaseImage + + obj_id = Dict('id') + obj_nsfw = False + obj_license = LICENSES.PD + obj_author = Dict('user/name') + obj_url = Dict('urls/full') + obj_date = DateTime(Dict('created_at')) + obj_title = Format('%s (%s)', Field('id'), Field('author')) + obj_ext = 'jpg' + + def obj_thumbnail(self): + return Thumbnail(Dict('urls/thumb')(self)) diff --git a/modules/unsplash/test.py b/modules/unsplash/test.py new file mode 100644 index 0000000000..9d38f09027 --- /dev/null +++ b/modules/unsplash/test.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 Vincent A +# +# This file is part of a weboob module. +# +# This weboob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This weboob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this weboob module. If not, see . + +from weboob.tools.test import BackendTest + + +class UnsplashTest(BackendTest): + MODULE = 'unsplash' + + def test_search_img(self): + it = self.backend.search_image('tree') + images = [img for _, img in zip(range(20), it)] + + self.assertEqual(len(images), 20) + for img in images: + assert img.id + assert img.title + assert img.ext + assert img.author + assert img.date + assert img.url + + self.backend.fillobj(img, 'data') + assert img.data + + self.backend.fillobj(img, 'thumbnail') + assert img.thumbnail.data -- GitLab