From c7330d62696bba1a142235b2ae486a2aff77ac04 Mon Sep 17 00:00:00 2001 From: Vincent A Date: Wed, 11 Sep 2019 19:25:18 +0200 Subject: [PATCH] backport master module fixes --- modules/amundi/browser.py | 3 + modules/axabanque/pages/bank.py | 2 +- modules/banquepopulaire/module.py | 56 ++++---- modules/barclays/browser.py | 8 +- modules/bforbank/pages.py | 11 +- modules/binck/browser.py | 1 + modules/binck/pages.py | 4 +- modules/bnpcards/browser.py | 4 + modules/bnporc/company/browser.py | 8 ++ modules/bnporc/enterprise/browser.py | 10 +- modules/bnporc/enterprise/pages.py | 1 + modules/bnporc/module.py | 44 +++++- modules/bnporc/pp/browser.py | 56 +++++++- modules/bnporc/pp/document_pages.py | 120 ++++++++++++++++ modules/bnporc/pp/pages.py | 25 +++- modules/bp/pages/accounthistory.py | 1 + modules/bp/pages/accountlist.py | 1 + modules/caissedepargne/browser.py | 41 ++++-- modules/caissedepargne/pages.py | 113 ++++++++++++++- modules/cragr/regions/pages.py | 12 +- modules/creditmutuel/browser.py | 6 +- modules/creditmutuel/pages.py | 32 +++++ modules/fortuneo/browser.py | 9 +- modules/fortuneo/pages/accounts_list.py | 10 +- modules/hsbc/pages/investments.py | 6 +- modules/ing/api/__init__.py | 6 +- modules/ing/api/transfer_page.py | 36 ++++- modules/ing/api_browser.py | 130 +++++++++++++++++- modules/ing/module.py | 14 +- modules/ing/web/accounts_list.py | 2 +- modules/ldlc/materielnet_pages.py | 9 +- modules/lendosphere/__init__.py | 26 ++++ modules/lendosphere/browser.py | 98 +++++++++++++ modules/lendosphere/compat/__init__.py | 0 .../compat/weboob_browser_filters_standard.py | 49 +++++++ .../compat/weboob_capabilities_bank.py | 48 +++++++ modules/lendosphere/module.py | 57 ++++++++ modules/lendosphere/pages.py | 97 +++++++++++++ modules/linebourse/api/pages.py | 4 +- modules/materielnet/browser.py | 19 ++- modules/materielnet/module.py | 14 +- modules/materielnet/pages.py | 9 +- modules/myedenred/browser.py | 38 +++-- modules/myedenred/pages.py | 10 +- modules/orange/browser.py | 10 +- modules/orange/pages/login.py | 32 ++++- .../societegenerale/pages/accounts_list.py | 35 ++++- modules/spirica/pages.py | 4 +- modules/tapatalk/module.py | 81 ++++++----- modules/wiseed/__init__.py | 26 ++++ modules/wiseed/browser.py | 71 ++++++++++ modules/wiseed/compat/__init__.py | 0 .../compat/weboob_browser_filters_standard.py | 49 +++++++ .../wiseed/compat/weboob_capabilities_bank.py | 48 +++++++ modules/wiseed/module.py | 57 ++++++++ modules/wiseed/pages.py | 101 ++++++++++++++ 56 files changed, 1605 insertions(+), 159 deletions(-) create mode 100644 modules/bnporc/pp/document_pages.py create mode 100644 modules/lendosphere/__init__.py create mode 100644 modules/lendosphere/browser.py create mode 100644 modules/lendosphere/compat/__init__.py create mode 100644 modules/lendosphere/compat/weboob_browser_filters_standard.py create mode 100644 modules/lendosphere/compat/weboob_capabilities_bank.py create mode 100644 modules/lendosphere/module.py create mode 100644 modules/lendosphere/pages.py create mode 100644 modules/wiseed/__init__.py create mode 100644 modules/wiseed/browser.py create mode 100644 modules/wiseed/compat/__init__.py create mode 100644 modules/wiseed/compat/weboob_browser_filters_standard.py create mode 100644 modules/wiseed/compat/weboob_capabilities_bank.py create mode 100644 modules/wiseed/module.py create mode 100644 modules/wiseed/pages.py diff --git a/modules/amundi/browser.py b/modules/amundi/browser.py index 8b043ec329..8064befe8e 100644 --- a/modules/amundi/browser.py +++ b/modules/amundi/browser.py @@ -56,6 +56,9 @@ def iter_accounts(self): @need_login def iter_investment(self, account): + if account.balance == 0: + self.logger.info('Account %s has a null balance, no investment available.', account.label) + return headers = {'X-noee-authorization': 'noeprd %s' % self.token} self.accounts.go(headers=headers) for inv in self.page.iter_investments(account_id=account.id): diff --git a/modules/axabanque/pages/bank.py b/modules/axabanque/pages/bank.py index 38e8463b2e..de065ded63 100644 --- a/modules/axabanque/pages/bank.py +++ b/modules/axabanque/pages/bank.py @@ -515,7 +515,7 @@ class item(ItemElement): obj_code = Regexp(CleanText(TableCell('code')), r'(.{12})', default=NotAvailable) obj_code_type = lambda self: Investment.CODE_TYPE_ISIN if Field('code')(self) is not NotAvailable else NotAvailable - def obj_diff_percent(self): + def obj_diff_ratio(self): diff_percent = MyDecimal(TableCell('diff')(self)[0])(self) return diff_percent/100 if diff_percent != NotAvailable else diff_percent diff --git a/modules/banquepopulaire/module.py b/modules/banquepopulaire/module.py index fa44c75d42..8365adf213 100644 --- a/modules/banquepopulaire/module.py +++ b/modules/banquepopulaire/module.py @@ -17,6 +17,8 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . +from __future__ import unicode_literals + from collections import OrderedDict from functools import reduce @@ -34,38 +36,38 @@ class BanquePopulaireModule(Module, CapBankWealth, CapContact, CapProfile): NAME = 'banquepopulaire' - MAINTAINER = u'Romain Bignon' + MAINTAINER = 'Romain Bignon' EMAIL = 'romain@weboob.org' VERSION = '1.5' - DESCRIPTION = u'Banque Populaire' + DESCRIPTION = 'Banque Populaire' LICENSE = 'LGPLv3+' - website_choices = OrderedDict([(k, u'%s (%s)' % (v, k)) for k, v in sorted({ - 'www.ibps.alpes.banquepopulaire.fr': u'Alpes', - 'www.ibps.alsace.banquepopulaire.fr': u'Alsace Lorraine Champagne', - 'www.ibps.bpalc.banquepopulaire.fr' : u'Alsace Lorraine Champagne', - 'www.ibps.bpaca.banquepopulaire.fr': u'Aquitaine Centre atlantique', - 'www.ibps.atlantique.banquepopulaire.fr': u'Atlantique', - 'www.ibps.bpgo.banquepopulaire.fr': u'Grand Ouest', - 'www.ibps.loirelyonnais.banquepopulaire.fr': u'Auvergne Rhône Alpes', - 'www.ibps.bpaura.banquepopulaire.fr': u'Auvergne Rhône Alpes', - 'www.ibps.banquedesavoie.banquepopulaire.fr': u'Banque de Savoie', - 'www.ibps.bpbfc.banquepopulaire.fr': u'Bourgogne Franche-Comté', - 'www.ibps.bretagnenormandie.cmm.groupe.banquepopulaire.fr': u'Crédit Maritime Bretagne Normandie', - 'www.ibps.atlantique.creditmaritime.groupe.banquepopulaire.fr': u'Crédit Maritime Atlantique', - 'www.ibps.sudouest.creditmaritime.groupe.banquepopulaire.fr': u'Crédit Maritime du Littoral du Sud-Ouest', - 'www.ibps.lorrainechampagne.banquepopulaire.fr': u'Lorraine Champagne', - 'www.ibps.massifcentral.banquepopulaire.fr': u'Massif central', - 'www.ibps.mediterranee.banquepopulaire.fr': u'Méditerranée', - 'www.ibps.nord.banquepopulaire.fr': u'Nord', - 'www.ibps.occitane.banquepopulaire.fr': u'Occitane', - 'www.ibps.ouest.banquepopulaire.fr': u'Ouest', - 'www.ibps.rivesparis.banquepopulaire.fr': u'Rives de Paris', - 'www.ibps.sud.banquepopulaire.fr': u'Sud', - 'www.ibps.valdefrance.banquepopulaire.fr': u'Val de France', + website_choices = OrderedDict([(k, '%s (%s)' % (v, k)) for k, v in sorted({ + 'www.ibps.alpes.banquepopulaire.fr': 'Alpes', + 'www.ibps.alsace.banquepopulaire.fr': 'Alsace Lorraine Champagne', + 'www.ibps.bpalc.banquepopulaire.fr' : 'Alsace Lorraine Champagne', + 'www.ibps.bpaca.banquepopulaire.fr': 'Aquitaine Centre atlantique', + 'www.ibps.atlantique.banquepopulaire.fr': 'Atlantique', + 'www.ibps.bpgo.banquepopulaire.fr': 'Grand Ouest', + 'www.ibps.loirelyonnais.banquepopulaire.fr': 'Auvergne Rhône Alpes', + 'www.ibps.bpaura.banquepopulaire.fr': 'Auvergne Rhône Alpes', + 'www.ibps.banquedesavoie.banquepopulaire.fr': 'Banque de Savoie', + 'www.ibps.bpbfc.banquepopulaire.fr': 'Bourgogne Franche-Comté', + 'www.ibps.bretagnenormandie.cmm.groupe.banquepopulaire.fr': 'Crédit Maritime Bretagne Normandie', + 'www.ibps.atlantique.creditmaritime.groupe.banquepopulaire.fr': 'Crédit Maritime Atlantique', + 'www.ibps.sudouest.creditmaritime.groupe.banquepopulaire.fr': 'Crédit Maritime du Littoral du Sud-Ouest', + 'www.ibps.lorrainechampagne.banquepopulaire.fr': 'Lorraine Champagne', + 'www.ibps.massifcentral.banquepopulaire.fr': 'Massif central', + 'www.ibps.mediterranee.banquepopulaire.fr': 'Méditerranée', + 'www.ibps.nord.banquepopulaire.fr': 'Nord', + 'www.ibps.occitane.banquepopulaire.fr': 'Occitane', + 'www.ibps.ouest.banquepopulaire.fr': 'Ouest', + 'www.ibps.rivesparis.banquepopulaire.fr': 'Rives de Paris', + 'www.ibps.sud.banquepopulaire.fr': 'Sud', + 'www.ibps.valdefrance.banquepopulaire.fr': 'Val de France', }.items(), key=lambda k_v: (k_v[1], k_v[0]))]) - CONFIG = BackendConfig(Value('website', label=u'Région', choices=website_choices), + CONFIG = BackendConfig(Value('website', label='Région', choices=website_choices), ValueBackendPassword('login', label='Identifiant', masked=False), - ValueBackendPassword('password', label='Mot de passee')) + ValueBackendPassword('password', label='Mot de passe')) BROWSER = BanquePopulaire def create_default_browser(self): diff --git a/modules/barclays/browser.py b/modules/barclays/browser.py index 2450ac70af..b69acdfbbe 100644 --- a/modules/barclays/browser.py +++ b/modules/barclays/browser.py @@ -23,7 +23,7 @@ from requests.exceptions import ConnectionError from weboob.browser.browsers import LoginBrowser, URL, need_login -from weboob.exceptions import BrowserIncorrectPassword +from weboob.exceptions import BrowserIncorrectPassword, ActionNeeded from .compat.weboob_capabilities_bank import Account from weboob.capabilities.base import NotAvailable from weboob.tools.decorators import retry @@ -111,8 +111,10 @@ def do_login(self): error_message = self.page.get_error_message() if error_message: - assert 'Saisie incorrecte' in error_message, error_message - raise BrowserIncorrectPassword(error_message) + if 'Saisie incorrecte' in error_message: + raise BrowserIncorrectPassword(error_message) + elif 'Votre accès est suspendu' in error_message: + raise ActionNeeded(error_message) # can't login if there is ' ' in the 2 characters asked if not self.page.login_secret(self.secret): diff --git a/modules/bforbank/pages.py b/modules/bforbank/pages.py index 402b6b6d36..f076eb65e2 100644 --- a/modules/bforbank/pages.py +++ b/modules/bforbank/pages.py @@ -306,10 +306,13 @@ def has_no_card(self): def get_cards(self, account_id): divs = self.doc.xpath('//div[@class="content-boxed"]') assert len(divs) - msgs = re.compile(u'Vous avez fait opposition sur cette carte bancaire.' + - '|Votre carte bancaire a été envoyée.' + - '|BforBank a fait opposition sur votre carte' + - '|Pour des raisons de sécurité, la demande de réception du code confidentiel de votre carte par SMS est indisponible') + msgs = re.compile( + 'Vous avez fait opposition sur cette carte bancaire.' + + '|Votre carte bancaire a été envoyée.' + + '|Carte bancaire commandée.' + + '|BforBank a fait opposition sur votre carte' + + '|Pour des raisons de sécurité, la demande de réception du code confidentiel de votre carte par SMS est indisponible' + ) divs = [d for d in divs if not msgs.search(CleanText('.//div[has-class("alert")]', default='')(d))] divs = [d.xpath('.//div[@class="m-card-infos"]')[0] for d in divs] divs = [d for d in divs if not d.xpath('.//div[@class="m-card-infos-body-text"][text()="Débit immédiat"]')] diff --git a/modules/binck/browser.py b/modules/binck/browser.py index ebcebe0633..fbf7f75c89 100644 --- a/modules/binck/browser.py +++ b/modules/binck/browser.py @@ -59,6 +59,7 @@ class BinckBrowser(LoginBrowser): history = URL(r'/TransactionsOverview/GetTransactions', r'/TransactionsOverview/FilteredOverview', HistoryPage) questions = URL(r'/FDL_Complex_FR_Compte', + r'/FDL_NonComplex_FR_Compte', r'FsmaMandatoryQuestionnairesOverview', QuestionPage) change_pass = URL(r'/ChangePassword/Index', r'/EditSetting/GetSetting\?code=MutationPassword', ChangePassPage) diff --git a/modules/binck/pages.py b/modules/binck/pages.py index 5dcb74c367..7cc58dbf16 100644 --- a/modules/binck/pages.py +++ b/modules/binck/pages.py @@ -43,7 +43,7 @@ def on_load(self): if self.doc.xpath(u'//h1[contains(text(), "Questionnaires connaissance et expérience")]'): form = self.get_form('//form[@action="/FsmaMandatoryQuestionnairesOverview/PostponeQuestionnaires"]') else: - form = self.get_form('//form[@action="/FDL_Complex_FR_Compte/Introduction/SkipQuestionnaire"]') + form = self.get_form('//form[contains(@action, "Complex_FR_Compte/Introduction/SkipQuestionnaire")]') form.submit() @@ -217,7 +217,7 @@ class item(ItemElement): obj_unitprice = Env('unitprice', default=NotAvailable) obj_valuation = MyDecimal(Dict('ValueInEuro')) obj_diff = MyDecimal(Dict('ResultValueInEuro')) - obj_diff_percent = Eval(lambda x: x / 100, MyDecimal(Dict('ResultPercentageInEuro'))) + obj_diff_ratio = Eval(lambda x: x / 100, MyDecimal(Dict('ResultPercentageInEuro'))) obj_original_currency = Env('o_currency', default=NotAvailable) obj_original_unitvalue = Env('o_unitvalue', default=NotAvailable) obj_original_unitprice = Env('o_unitprice', default=NotAvailable) diff --git a/modules/bnpcards/browser.py b/modules/bnpcards/browser.py index 2098a68141..daea4939de 100644 --- a/modules/bnpcards/browser.py +++ b/modules/bnpcards/browser.py @@ -76,10 +76,14 @@ def do_login(self): if self.error.is_here() or self.page.is_error(): raise BrowserIncorrectPassword() if self.type == '2' and self.page.is_corporate(): + self.logger.info('Manager corporate connection') raise SiteSwitch('corporate') # ti corporate and ge corporate are not detected the same way .. if 'corporate' in self.page.url: + self.logger.info('Carholder corporate connection') self.is_corporate = True + else: + self.logger.info('Cardholder connection') def ti_card_go(self): if self.is_corporate: diff --git a/modules/bnporc/company/browser.py b/modules/bnporc/company/browser.py index e0acb3454a..d03c6f700c 100644 --- a/modules/bnporc/company/browser.py +++ b/modules/bnporc/company/browser.py @@ -79,6 +79,14 @@ def iter_history(self, account): (date.today() - timedelta(days=90)).strftime('%Y%m%d'), date.today().strftime('%Y%m%d')) + @need_login + def iter_documents(self, subscription): + raise NotImplementedError() + + @need_login + def iter_subscription(self): + raise NotImplementedError() + @need_login def iter_coming_operations(self, account): return self.get_transactions(account.id, diff --git a/modules/bnporc/enterprise/browser.py b/modules/bnporc/enterprise/browser.py index f508a279fd..47033002d9 100644 --- a/modules/bnporc/enterprise/browser.py +++ b/modules/bnporc/enterprise/browser.py @@ -33,7 +33,7 @@ from .pages import ( LoginPage, AuthPage, AccountsPage, AccountHistoryViewPage, AccountHistoryPage, - ActionNeededPage, TransactionPage, MarketPage, InvestPage + ActionNeededPage, TransactionPage, MarketPage, InvestPage, ) @@ -124,6 +124,14 @@ def iter_history(self, account): return [] return self._iter_history_base(account) + @need_login + def iter_documents(self, subscription): + raise NotImplementedError() + + @need_login + def iter_subscription(self): + raise NotImplementedError() + def _iter_history_base(self, account): dformat = "%Y%m%d" diff --git a/modules/bnporc/enterprise/pages.py b/modules/bnporc/enterprise/pages.py index c9b4a906a2..95a2bc500f 100644 --- a/modules/bnporc/enterprise/pages.py +++ b/modules/bnporc/enterprise/pages.py @@ -230,6 +230,7 @@ class AccountHistoryPage(LoggedPage, JsonPage): u'0083': Transaction.TYPE_DEFERRED_CARD, u'0813': Transaction.TYPE_LOAN_PAYMENT, u'0568': Transaction.TYPE_TRANSFER, + u'1194': Transaction.TYPE_DEFERRED_CARD, # PAYBACK typed as DEFERRED_CARD } @method diff --git a/modules/bnporc/module.py b/modules/bnporc/module.py index 34ab209ccd..4e9244736a 100644 --- a/modules/bnporc/module.py +++ b/modules/bnporc/module.py @@ -33,6 +33,10 @@ from weboob.capabilities.base import find_object, strict_find_object from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import ValueBackendPassword, Value, ValueBool +from weboob.capabilities.bill import ( + Subscription, CapDocument, SubscriptionNotFound, DocumentNotFound, Document, + DocumentTypes, +) from .enterprise.browser import BNPEnterprise from .company.browser import BNPCompany @@ -42,7 +46,7 @@ __all__ = ['BNPorcModule'] -class BNPorcModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapMessages, CapContact, CapProfile): +class BNPorcModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapMessages, CapContact, CapProfile, CapDocument): NAME = 'bnporc' MAINTAINER = u'Romain Bignon' EMAIL = 'romain@weboob.org' @@ -61,6 +65,13 @@ class BNPorcModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapMessag 'ent2': 'Entreprises et PME (nouveau site)'})) STORAGE = {'seen': []} + accepted_document_types = ( + DocumentTypes.STATEMENT, + DocumentTypes.REPORT, + DocumentTypes.BILL, + DocumentTypes.OTHER, + ) + # Store the messages *list* for this duration CACHE_THREADS = timedelta(seconds=3 * 60 * 60) @@ -74,6 +85,14 @@ def create_default_browser(self): self.BROWSER = b[self.config['website'].get()] return self.create_browser(self.config) + def iter_resources(self, objs, split_path): + if Account in objs: + self._restrict_level(split_path) + return self.iter_accounts() + if Subscription in objs: + self._restrict_level(split_path) + return self.iter_subscription() + def iter_accounts(self): return self.browser.iter_accounts() @@ -205,4 +224,27 @@ def set_message_read(self, message): self.storage.get('seen', default=[]).append(message.thread.id) self.storage.save() + def get_subscription(self, _id): + return find_object(self.iter_subscription(), id=_id, error=SubscriptionNotFound) + + def iter_documents(self, subscription): + if not isinstance(subscription, Subscription): + subscription = self.get_subscription(subscription) + + return self.browser.iter_documents(subscription) + + def iter_subscription(self): + return self.browser.iter_subscription() + + def get_document(self, _id): + subscription_id = _id.split('_')[0] + subscription = self.get_subscription(subscription_id) + return find_object(self.iter_documents(subscription), id=_id, error=DocumentNotFound) + + def download_document(self, document): + if not isinstance(document, Document): + document = self.get_document(document) + + return self.browser.open(document.url).content + OBJECTS = {Thread: fill_thread} diff --git a/modules/bnporc/pp/browser.py b/modules/bnporc/pp/browser.py index 868dda4207..175981056a 100644 --- a/modules/bnporc/pp/browser.py +++ b/modules/bnporc/pp/browser.py @@ -30,13 +30,14 @@ AccountNotFound, Account, AddRecipientStep, AddRecipientTimeout, TransferInvalidRecipient, Loan, ) +from weboob.capabilities.bill import Subscription from weboob.capabilities.profile import ProfileMissing from weboob.tools.decorators import retry from weboob.tools.capabilities.bank.transactions import sorted_transactions from weboob.tools.json import json from weboob.browser.exceptions import ServerError from weboob.browser.elements import DataError -from weboob.exceptions import BrowserIncorrectPassword +from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable from weboob.tools.value import Value, ValueBool from weboob.tools.capabilities.bank.investments import create_french_liquidity @@ -50,6 +51,7 @@ UselessPage, TransferAssertionError, LoanDetailsPage, ) +from .document_pages import DocumentsPage, TitulairePage __all__ = ['BNPPartPro', 'HelloBank'] @@ -92,12 +94,14 @@ class BNPParibasBrowser(JsonBrowserMixin, LoginBrowser): r'/fr/espace-pro/changer-son-mot-de-passe', r'/fr/espace-client/100-connexions', r'/fr/espace-prive/mot-de-passe-expire', + r'/fr/client/mdp-expire', r'/fr/client/100-connexion', r'/fr/systeme/page-indisponible', ConnectionThresholdPage) accounts = URL(r'udc-wspl/rest/getlstcpt', AccountsPage) loan_details = URL(r'caraccomptes-wspl/rpc/(?P.*)', LoanDetailsPage) ibans = URL(r'rib-wspl/rpc/comptes', AccountsIBANPage) history = URL(r'rop2-wspl/rest/releveOp', HistoryPage) + history_old = URL(r'rop-wspl/rest/releveOp', HistoryPage) transfer_init = URL(r'virement-wspl/rest/initialisationVirement', TransferInitPage) lifeinsurances = URL(r'mefav-wspl/rest/infosContrat', LifeInsurancesPage) @@ -121,6 +125,9 @@ class BNPParibasBrowser(JsonBrowserMixin, LoginBrowser): advisor = URL(r'/conseiller-wspl/rest/monConseiller', AdvisorPage) + titulaire = URL(r'/demat-wspl/rest/listerTitulairesDemat', TitulairePage) + document = URL(r'/demat-wspl/rest/rechercheCriteresDemat', DocumentsPage) + profile = URL(r'/kyc-wspl/rest/informationsClient', ProfilePage) list_detail_card = URL(r'/udcarte-wspl/rest/listeDetailCartes', ListDetailCardPage) @@ -275,14 +282,21 @@ def iter_history(self, account, coming=False): if not self.card_to_transaction_type: self.list_detail_card.go() self.card_to_transaction_type = self.page.get_card_to_transaction_type() - - self.history.go(data=JSON({ + data = JSON({ "ibanCrypte": account.id, "pastOrPending": 1, "triAV": 0, "startDate": (datetime.now() - relativedelta(years=1)).strftime('%d%m%Y'), "endDate": datetime.now().strftime('%d%m%Y') - })) + }) + try: + self.history.go(data=data) + except BrowserUnavailable: + # old url is still used for certain connections bu we don't know which one is, + # so the same HistoryPage is attained by the old url in another URL object + data[1]['startDate'] = (datetime.now() - relativedelta(years=3)).strftime('%d%m%Y') + # old url authorizes up to 3 years of history + self.history_old.go(data=data) if coming: return sorted_transactions(self.page.iter_coming()) @@ -510,6 +524,40 @@ def iter_threads(self): def get_thread(self, thread): raise NotImplementedError() + @need_login + def iter_documents(self, subscription): + titulaires = self.titulaire.go().get_titulaires() + # Calling '/demat-wspl/rest/listerDocuments' before the request on 'document' + # is necessary when you specify an ikpi, otherwise no documents are returned + self.location('/demat-wspl/rest/listerDocuments') + # When we only have one titulaire, no need to use the ikpi parameter in the request, + # all document are provided with this simple request + data = { + 'dateDebut': (datetime.now() - relativedelta(years=3)).strftime('%d/%m/%Y'), + 'dateFin': datetime.now().strftime('%d/%m/%Y'), + } + # Ikpi is necessary for multi titulaires accounts to get each document of each titulaires + if len(titulaires) > 1: + data['ikpiPersonne'] = subscription._iduser + self.document.go(json=data) + return self.page.iter_documents(sub_id=subscription.id, sub_number=subscription._number, baseurl=self.BASEURL) + + @need_login + def iter_subscription(self): + acc_list = self.iter_accounts() + + for acc in acc_list: + sub = Subscription() + sub.label = acc.label + sub.subscriber = acc._subscriber + sub.id = acc.id + # number is the hidden number of an account like "****1234" + # and it's used in the parsing of the docs in iter_documents + sub._number = acc.number + # iduser is the ikpi affiliate to the account, + # usefull for multi titulaires connexions + sub._iduser = acc._iduser + yield sub class BNPPartPro(BNPParibasBrowser): BASEURL_TEMPLATE = r'https://%s.bnpparibas/' diff --git a/modules/bnporc/pp/document_pages.py b/modules/bnporc/pp/document_pages.py new file mode 100644 index 0000000000..ab6da7eb28 --- /dev/null +++ b/modules/bnporc/pp/document_pages.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2009-2019 Romain Bignon +# +# This file is part of a weboob module. +# +# This weboob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This weboob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this weboob module. If not, see . + +from __future__ import unicode_literals + +import re + +from weboob.browser.elements import DictElement, ItemElement, method +from weboob.browser.filters.json import Dict +from .compat.weboob_browser_filters_standard import Format, Date, Env +from weboob.browser.pages import JsonPage, LoggedPage +from weboob.capabilities.bill import Document, DocumentTypes +from weboob.tools.compat import urlencode + +patterns = { + r'Relevé': DocumentTypes.STATEMENT, + r'Livret(s) A': DocumentTypes.STATEMENT, + r'développement durable': DocumentTypes.STATEMENT, + r'Synthèse': DocumentTypes.STATEMENT, + r'Echelles/Décomptes': DocumentTypes.STATEMENT, + r'épargne logement': DocumentTypes.STATEMENT, + r'Livret(s) jeune': DocumentTypes.STATEMENT, + r'Compte(s) sur Livret': DocumentTypes.STATEMENT, + r'Récapitulatifs annuels': DocumentTypes.REPORT, + r"Avis d'exécution": DocumentTypes.REPORT, + r'Factures': DocumentTypes.BILL, +} + +def get_document_type(family): + for patt, type in patterns.items(): + if re.search(re.escape(patt), family): + return type + return DocumentTypes.OTHER + + +class TitulairePage(LoggedPage, JsonPage): + def get_titulaires(self): + return set([t['idKpiTitulaire'] for t in self.doc['data']['listeTitulairesDemat']['listeTitulaires']]) + + +class DocumentsPage(LoggedPage, JsonPage): + @method + class iter_documents(DictElement): + item_xpath = 'data/rechercheCriteresDemat/*/*/listeDocument' + ignore_duplicate = True + + class item(ItemElement): + klass = Document + + def condition(self): + # There is two type of json, the one with the ibancrypte in it + # and the one with the idcontrat in it, here we check if + # the document belong to the subscritpion. + if 'ibanCrypte' in self.el: + return Env('sub_id')(self) in Dict('ibanCrypte')(self) + else: + return Env('sub_number')(self) in Dict('idContrat')(self) + + obj_date = Date(Dict('dateDoc'), dayfirst=True) + obj_format = 'pdf' + obj_id = Format('%s_%s', Env('sub_id'), Dict('idDoc')) + + def obj_label(self): + if 'ibanCrypte' in self.el: + return '%s %s N° %s' % (Dict('dateDoc')(self), Dict('libelleSousFamille')(self), Dict('numeroCompteAnonymise')(self)) + else: + return '%s %s N° %s' % (Dict('dateDoc')(self), Dict('libelleSousFamille')(self), Dict('idContrat')(self)) + + def obj_url(self): + keys_to_copy = { + 'idDocument' :'idDoc', + 'dateDocument': 'dateDoc', + 'idLocalisation': 'idLocalisation', + 'viDocDocument': 'viDocDocument', + } + # Here we parse the json with ibancrypte in it, for most cases + if 'ibanCrypte' in self.el: + url = 'demat-wspl/rest/consultationDocumentDemat?' + keys_to_copy.update({ + 'typeCpt': 'typeCompte', + 'familleDoc': 'famDoc', + 'ibanCrypte': 'ibanCrypte', + 'typeDoc': 'typeDoc', + 'consulted': 'consulted', + }) + request_params = {'typeFamille': 'R001', 'ikpiPersonne': ''} + # Here we parse the json with idcontrat in it. For the cases present + # on privee.mabanque where sometimes the doc url is different + else: + url = 'demat-wspl/rest/consultationDocumentSpecialBpfDemat?' + keys_to_copy.update({ + 'heureDocument': 'heureDoc', + 'numClient': 'numClient', + 'typeReport': 'typeReport', + }) + request_params = {'ibanCrypte': ''} + + for k, v in keys_to_copy.items(): + request_params[k] = Dict(v)(self) + + return Env('baseurl')(self) + url + urlencode(request_params) + + def obj_type(self): + return get_document_type(Dict('libelleSousFamille')(self)) diff --git a/modules/bnporc/pp/pages.py b/modules/bnporc/pp/pages.py index c517fb61c0..4d9c2c8a87 100644 --- a/modules/bnporc/pp/pages.py +++ b/modules/bnporc/pp/pages.py @@ -30,7 +30,8 @@ from weboob.browser.elements import DictElement, ListElement, TableElement, ItemElement, method from weboob.browser.filters.json import Dict from .compat.weboob_browser_filters_standard import ( - Format, Eval, Regexp, CleanText, Date, CleanDecimal, Field, Coalesce, Map, Env, Currency, + Format, Eval, Regexp, CleanText, Date, CleanDecimal, Field, Coalesce, Map, Env, + Currency, ) from weboob.browser.filters.html import TableCell from weboob.browser.pages import JsonPage, LoggedPage, HTMLPage @@ -107,7 +108,10 @@ def looks_legit(self, password): return True def on_load(self): - msg = CleanText('//div[@class="confirmation"]//span[span]')(self.doc) + msg = ( + CleanText('//div[@class="confirmation"]//span[span]')(self.doc) or + CleanText('//p[contains(text(), "Vous avez atteint la date de fin de vie de votre code secret")]')(self.doc) + ) self.logger.warning('Password expired.') if not self.browser.rotating_password: @@ -306,7 +310,6 @@ def obj_company_siren(self): class AccountsPage(BNPPage): - @method class iter_accounts(DictElement): item_xpath = 'data/infoUdc/familleCompte' @@ -315,6 +318,12 @@ class iter_accounts_details(DictElement): item_xpath = 'compte' class item(ItemElement): + def validate(self, obj): + # We skip loans with a balance of 0 because the JSON returned gives + # us no info (only `null` values on all fields), so there is nothing + # useful to display + return obj.type != Account.TYPE_LOAN or obj.balance != 0 + FAMILY_TO_TYPE = { 1: Account.TYPE_CHECKING, 2: Account.TYPE_SAVINGS, @@ -354,6 +363,8 @@ class item(ItemElement): obj_balance = Dict('soldeDispo') obj_coming = Dict('soldeAVenir') obj_number = Dict('value') + obj__subscriber = Format('%s %s', Dict('titulaire/nom'), Dict('titulaire/prenom')) + obj__iduser = Dict('titulaire/ikpi') def obj_iban(self): iban = Map(Dict('key'), Env('ibans')(self), default=NotAvailable)(self) @@ -381,11 +392,15 @@ class fill_loan_details(ItemElement): obj_rate = Dict('data/tauxRemboursement') obj_nb_payments_left = Dict('data/nbRemboursementRestant') obj_next_payment_date = Date(Dict('data/dateProchainAmortissement'), dayfirst=True) + obj__subscriber = Format('%s %s', Dict('data/titulaire/nom'), Dict('data/titulaire/prenom')) + obj__iduser = None @method class fill_revolving_details(ItemElement): obj_total_amount = Dict('data/montantDisponible') obj_rate = Dict('data/tauxInterets') + obj__subscriber = Format('%s %s', Dict('data/titulaire/nom'), Dict('data/titulaire/prenom')) + obj__iduser = None class AccountsIBANPage(BNPPage): @@ -441,6 +456,8 @@ class RecipientsPage(BNPPage): @method class iter_recipients(DictElement): item_xpath = 'data/infoBeneficiaire/listeBeneficiaire' + # We ignore duplicate because BNP allows differents recipients with the same iban + ignore_duplicate = True class item(MyRecipient): # For the moment, only yield ready to transfer on recipients. @@ -745,6 +762,8 @@ class item(ItemElement): obj_balance = CleanDecimal(TableCell('balance'), replace_dots=True) obj_coming = None obj_iban = None + obj__subscriber = None + obj__iduser = None def obj_type(self): for k, v in self.page.ACCOUNT_TYPES.items(): diff --git a/modules/bp/pages/accounthistory.py b/modules/bp/pages/accounthistory.py index d46ca17228..171459a34c 100644 --- a/modules/bp/pages/accounthistory.py +++ b/modules/bp/pages/accounthistory.py @@ -54,6 +54,7 @@ class Transaction(FrenchTransaction): (re.compile(r'^(?PFRAIS POUR)(?P.*)'), FrenchTransaction.TYPE_BANK), (re.compile(r'^(?P(?PREMUNERATION).*)'), FrenchTransaction.TYPE_BANK), (re.compile(r'^(?PREMISE DE CHEQUES?) (?P.*)'), FrenchTransaction.TYPE_DEPOSIT), + (re.compile(r'^(?PVERSEMENT DAB) (?P.*)'), FrenchTransaction.TYPE_DEPOSIT), (re.compile(r'^(?PDEBIT CARTE BANCAIRE DIFFERE.*)'), FrenchTransaction.TYPE_CARD_SUMMARY), (re.compile(r'^(?PCOTISATION TRIMESTRIELLE).*'), FrenchTransaction.TYPE_BANK), (re.compile(r'^REMISE COMMERCIALE.*'), FrenchTransaction.TYPE_BANK), diff --git a/modules/bp/pages/accountlist.py b/modules/bp/pages/accountlist.py index b37a1f46eb..38e31b396a 100644 --- a/modules/bp/pages/accountlist.py +++ b/modules/bp/pages/accountlist.py @@ -136,6 +136,7 @@ def obj_type(self): 'livrets?': Account.TYPE_SAVINGS, 'epargnes? logement': Account.TYPE_SAVINGS, "autres produits d'epargne": Account.TYPE_SAVINGS, + 'compte relais': Account.TYPE_SAVINGS, 'comptes? titres? et pea': Account.TYPE_MARKET, 'compte-titres': Account.TYPE_MARKET, 'assurances? vie': Account.TYPE_LIFE_INSURANCE, diff --git a/modules/caissedepargne/browser.py b/modules/caissedepargne/browser.py index 9f49bf4f3a..5666933f87 100644 --- a/modules/caissedepargne/browser.py +++ b/modules/caissedepargne/browser.py @@ -52,6 +52,7 @@ SmsPage, SmsPageOption, SmsRequest, AuthentPage, RecipientPage, CanceledAuth, CaissedepargneKeyboard, TransactionsDetailsPage, LoadingPage, ConsLoanPage, MeasurePage, NatixisLIHis, NatixisLIInv, NatixisRedirectPage, SubscriptionPage, CreditCooperatifMarketPage, UnavailablePage, CardsPage, CardsComingPage, CardsOldWebsitePage, TransactionPopupPage, + OldLeviesPage, NewLeviesPage, ) from .linebourse_browser import LinebourseAPIBrowser @@ -86,6 +87,8 @@ class CaisseEpargne(LoginBrowser, StatesMixin): cards_old = URL('https://.*/Portail.aspx.*', CardsOldWebsitePage) cards = URL('https://.*/Portail.aspx.*', CardsPage) cards_coming = URL('https://.*/Portail.aspx.*', CardsComingPage) + old_checkings_levies = URL(r'https://.*/Portail.aspx.*', OldLeviesPage) + new_checkings_levies = URL(r'https://.*/Portail.aspx.*', NewLeviesPage) authent = URL('https://.*/Portail.aspx.*', AuthentPage) subscription = URL('https://.*/Portail.aspx\?tache=(?P).*', SubscriptionPage) transaction_popup = URL(r'https://.*/Portail.aspx.*', TransactionPopupPage) @@ -458,7 +461,7 @@ def get_accounts_list(self): - In CardsPage, there are cards (with "Business" in the label) without checking account on the website (neither history nor coming), so we skip them. - Some card on the CardsPage that have a checking account parent, but if we follow the link to - reach it with CardsComingPage, we find an other card that not in CardsPage. + reach it with CardsComingPage, we find an other card that is not in CardsPage. """ if self.new_website: for account in self.accounts: @@ -736,33 +739,51 @@ def match_cb(tr): @need_login def get_coming(self, account): - if account.type != account.TYPE_CARD: - return [] + if account.type == account.TYPE_CHECKING: + return self.get_coming_checking(account) + elif account.type == account.TYPE_CARD: + return self.get_coming_card(account) + return [] + def get_coming_checking(self, account): + # The accounts list or account history page does not contain comings for checking accounts + # We need to go to a specific levies page where we can find past and coming levies (such as recurring ones) + trs = [] + self.home.go() + self.page.go_cards() # need to go to cards page to have access to the nav bar where we can choose LeviesPage from + if not self.page.levies_page_enabled(): + return trs + self.page.go_levies() # need to go to a general page where we find levies for all accounts before requesting a specific account + if not self.page.comings_enabled(account.id): + return trs + self.page.go_levies(account.id) + if self.new_checkings_levies.is_here() or self.old_checkings_levies.is_here(): + today = datetime.datetime.today().date() + # Today transactions are in this page but also in history page, we need to ignore it as a coming + for tr in self.page.iter_coming(): + if tr.date > today: + trs.append(tr) + return trs + + def get_coming_card(self, account): trs = [] - if not hasattr(account.parent, '_info'): raise NotImplementedError() - # We are on the old website if hasattr(account, '_coming_eventargument'): - if not self.cards_old.is_here(): self.home.go() self.page.go_list() self.page.go_cards() self.page.go_card_coming(account._coming_eventargument) - return sorted_transactions(self.page.iter_coming()) - # We are on the new website. info = account.parent._card_links - # if info is empty, that mean there are no coming yet + # if info is empty, that means there are no comings yet if info: for tr in self._get_history(info.copy(), account): tr.type = tr.TYPE_DEFERRED_CARD trs.append(tr) - return sorted_transactions(trs) @need_login diff --git a/modules/caissedepargne/pages.py b/modules/caissedepargne/pages.py index ed1b9896b3..81d00cd138 100644 --- a/modules/caissedepargne/pages.py +++ b/modules/caissedepargne/pages.py @@ -524,12 +524,12 @@ class item(ItemElement): obj_label = Env('label') obj_type = Loan.TYPE_LOAN obj_total_amount = MyDecimal(MyTableCell("total_amount")) - obj_rate = Eval(lambda x: x / 100, MyDecimal(MyTableCell("rate", default=NotAvailable), default=NotAvailable)) obj_balance = MyDecimal(MyTableCell("balance"), sign=lambda x: -1) obj_currency = Currency(MyTableCell("balance")) obj_last_payment_date = Date(CleanText(MyTableCell("last_payment_date"))) obj_next_payment_amount = MyDecimal(MyTableCell("next_payment_amount")) obj_next_payment_date = Date(CleanText(MyTableCell("next_payment_date", default=''), default=NotAvailable), default=NotAvailable) + obj_rate = MyDecimal(MyTableCell("rate", default=NotAvailable), default=NotAvailable) def submit_form(self, form, eventargument, eventtarget, scriptmanager): form['__EVENTARGUMENT'] = eventargument @@ -538,6 +538,36 @@ def submit_form(self, form, eventargument, eventtarget, scriptmanager): fix_form(form) form.submit() + def go_levies(self, account_id=None): + form = self.get_form(id='main') + if account_id: + # Go to an account specific levies page + eventargument = "" + if "MM$m_CH$IsMsgInit" in form: + # Old website + form['MM$SYNTHESE_SDD_RECUS$m_ExDropDownList'] = account_id + eventtarget = "MM$SYNTHESE_SDD_RECUS$m_ExDropDownList" + scriptmanager = "MM$m_UpdatePanel|MM$SYNTHESE_SDD_RECUS$m_ExDropDownList" + else: + # New website + form['MM$SYNTHESE_SDD_RECUS$ddlCompte'] = account_id + eventtarget = "MM$SYNTHESE_SDD_RECUS$ddlCompte" + scriptmanager = "MM$m_UpdatePanel|MM$SYNTHESE_SDD_RECUS$ddlCompte" + self.submit_form(form, eventargument, eventtarget, scriptmanager,) + else: + # Go to an general levies page page where all levies are found + if "MM$m_CH$IsMsgInit" in form: + # Old website + eventargument = "SDDRSYN0" + eventtarget = "Menu_AJAX" + scriptmanager = "m_ScriptManager|Menu_AJAX" + else: + # New website + eventargument = "SDDRSYN0&codeMenu=WPS1" + eventtarget = "MM$Menu_Ajax" + scriptmanager = "MM$m_UpdatePanel|MM$Menu_Ajax" + self.submit_form(form, eventargument, eventtarget, scriptmanager,) + def go_list(self): form = self.get_form(id='main') @@ -842,6 +872,13 @@ def go_pro_transfer_availability(self): def is_transfer_allowed(self): return not self.doc.xpath('//ul/li[contains(text(), "Aucun compte tiers n\'est disponible")]') + def levies_page_enabled(self): + """ Levies page does not exist in the nav bar for every connections """ + return ( + CleanText('//a/span[contains(text(), "Suivre mes prélèvements reçus")]')(self.doc) or # new website + CleanText('//a[contains(text(), "Suivre les prélèvements reçus")]')(self.doc) # old website + ) + class TransactionPopupPage(LoggedPage, HTMLPage): def is_here(self): @@ -851,6 +888,74 @@ def complete_label(self): return CleanText('''//div[@class="scrollPane"]/table[//caption[contains(text(), "Détail de l'opération")]]//tr[2]''')(self.doc) +class NewLeviesPage(IndexPage): + """ Scrape new website 'Prélèvements' page for comings for checking accounts """ + + def is_here(self): + return CleanText('//h2[contains(text(), "Suivez vos prélèvements reçus")]')(self.doc) + + def comings_enabled(self, account_id): + """ Check if a specific account can be selected on the general levies page """ + return account_id in CleanText('//span[@id="MM_SYNTHESE_SDD_RECUS"]//select/option/@value')(self.doc) + + @method + class iter_coming(TableElement): + head_xpath = '//div[contains(@id, "ListePrelevement_0")]/table[contains(@summary, "Liste des prélèvements en attente")]//tr/th' + item_xpath = '//div[contains(@id, "ListePrelevement_0")]/table[contains(@summary, "Liste des prélèvements en attente")]//tr[contains(@id, "trRowDetail")]' + + col_label = 'Libellé/Référence' + col_coming = 'Montant' + col_date = 'Date' + + class item(ItemElement): + klass = Transaction + + # Transaction typing will mostly not work since transaction as comings will only display the debiting organism in the label + # Labels will bear recognizable patterns only when they move from future to past, where they will be typed by iter_history + # when transactions change state from coming to history 'Prlv' is append to their label, this will help the backend for the matching + obj_raw = Transaction.Raw(Format('Prlv %s', Field('label'))) + obj_label = CleanText(TableCell('label')) + obj_amount = CleanDecimal.French(TableCell('coming'), sign=lambda x: -1) + obj_date = Date(CleanText(TableCell('date')), dayfirst=True) + + def condition(self): + return not CleanText('''//p[contains(text(), "Vous n'avez pas de prélèvement en attente d'exécution.")]''')(self) + + +class OldLeviesPage(IndexPage): + """ Scrape old website 'Prélèvements' page for comings for checking accounts """ + + def is_here(self): + return CleanText('//span[contains(text(), "Suivez vos prélèvements reçus")]')(self.doc) + + def comings_enabled(self, account_id): + """ Check if a specific account can be selected on the general levies page """ + return account_id in CleanText('//span[@id="MM_SYNTHESE_SDD_RECUS"]//select/option/@value')(self.doc) + + @method + class iter_coming(TableElement): + head_xpath = '''//span[contains(text(), "Prélèvements en attente d'exécution")]/ancestor::table[1]/following-sibling::table[1]//tr[contains(@class, "DataGridHeader")]//td''' + item_xpath = '''//span[contains(text(), "Prélèvements en attente d'exécution")]/ancestor::table[1]/following-sibling::table[1]//tr[contains(@class, "DataGridHeader")]//following-sibling::tr''' + + col_label = 'Libellé/Référence' + col_coming = 'Montant' + col_date = 'Date' + + class item(ItemElement): + klass = Transaction + + # Transaction typing will mostly not work since transaction as comings will only display the debiting organism in the label + # Labels will bear recognizable patterns only when they move from future to past, where they will be typed by iter_history + # when transactions change state from coming to history 'Prlv' is append to their label, this will help the backend for the matching + obj_raw = Transaction.Raw(Format('Prlv %s', Field('label'))) + obj_label = CleanText(TableCell('label')) + obj_amount = CleanDecimal.French(TableCell('coming'), sign=lambda x: -1) + obj_date = Date(CleanText(TableCell('date')), dayfirst=True) + + def condition(self): + return not CleanText('''//table[@id="MM_SYNTHESE_SDD_RECUS_rpt_dgList_0"]//td[contains(text(), "Vous n'avez pas de prélèvements")]''')(self) + + class CardsPage(IndexPage): def is_here(self): return CleanText('//h3[normalize-space(text())="Mes cartes (cartes dont je suis le titulaire)"]')(self.doc) @@ -1119,6 +1224,10 @@ def find_elements(self): class item(ItemElement): klass = Transaction + def condition(self): + # Eliminate transactions without amount + return Dict('montantBrut')(self) + obj_raw = Transaction.Raw(Dict('type/libelleLong')) obj_amount = Eval(float_to_decimal, Dict('montantBrut/valeur')) @@ -1169,7 +1278,7 @@ def obj_diff(self): return Eval(float_to_decimal, Dict('montantPlusValue/valeur'))(self) return NotAvailable - def obj_diff_percent(self): + def obj_diff_ratio(self): if Dict('tauxPlusValue')(self): return Eval(lambda x: float_to_decimal(x) / 100, Dict('tauxPlusValue'))(self) return NotAvailable diff --git a/modules/cragr/regions/pages.py b/modules/cragr/regions/pages.py index da1a331a99..d6dabe46c2 100644 --- a/modules/cragr/regions/pages.py +++ b/modules/cragr/regions/pages.py @@ -613,17 +613,17 @@ def parse(self, obj): self.env['vdate'] = NotAvailable if CleanText('//table[@class="ca-table"][caption[span[b[text()="Historique des opérations"]]]]//tr[count(td) = 4]')(self): # History table with 4 columns - self.env['raw'] = CleanText('./td[2]', children=False)(self) + self.env['raw'] = CleanText('./td[2]')(self) self.env['amount'] = CleanDecimal.French('./td[last()]')(self) elif CleanText('//table[@class="ca-table"][caption[span[b[text()="Historique des opérations"]]]]//tr[count(td) = 5]')(self): # History table with 5 columns - self.env['raw'] = CleanText('./td[3]', children=False)(self) + self.env['raw'] = CleanText('./td[3]')(self) self.env['amount'] = CleanDecimal.French('./td[last()]')(self) elif CleanText('//table[@class="ca-table"][caption[span[b[text()="Historique des opérations"]]]]//tr[count(td) = 6]')(self): # History table with 6 columns (contains vdate) - self.env['raw'] = CleanText('./td[4]', children=False)(self) + self.env['raw'] = CleanText('./td[4]')(self) self.env['vdate'] = DateGuesser(CleanText('./td[2]'), Env('date_guesser'))(self) self.env['amount'] = CleanDecimal.French('./td[last()]')(self) @@ -635,11 +635,11 @@ def parse(self, obj): )(self) if CleanText('//table[@class="ca-table"][caption[span[b[text()="Historique des opérations"]]]]//th[a[contains(text(), "Valeur")]]')(self): # With vdate column ('Valeur') - self.env['raw'] = CleanText('./td[4]', children=False)(self) + self.env['raw'] = CleanText('./td[4]')(self) self.env['vdate'] = DateGuesser(CleanText('./td[2]'), Env('date_guesser'))(self) else: # Without any vdate column - self.env['raw'] = CleanText('./td[3]', children=False)(self) + self.env['raw'] = CleanText('./td[3]')(self) else: assert False, 'This type of history table is not handled yet!' @@ -833,7 +833,7 @@ class item(ItemElement): CleanDecimal.French('.//span[@class="box"][span[span[text()="Répartition"]]]/span[2]/span') ) - def obj_diff_percent(self): + def obj_diff_ratio(self): # Euro funds have '-' instead of a diff_percent value if CleanText('.//span[@class="box"][span[span[text()="+/- value latente (%)"]]]/span[2]/span')(self) == '-': return NotAvailable diff --git a/modules/creditmutuel/browser.py b/modules/creditmutuel/browser.py index 823496c2a7..f06fb5534c 100644 --- a/modules/creditmutuel/browser.py +++ b/modules/creditmutuel/browser.py @@ -47,8 +47,8 @@ LIAccountsPage, CardsActivityPage, CardsListPage, CardsOpePage, NewAccountsPage, InternalTransferPage, ExternalTransferPage, RevolvingLoanDetails, RevolvingLoansList, - ErrorPage, SubscriptionPage, NewCardsListPage, CardPage2, - ConditionsPage, + ErrorPage, SubscriptionPage, NewCardsListPage, CardPage2, FiscalityConfirmationPage, + ConditionsPage, MobileConfirmationPage, ) @@ -67,9 +67,11 @@ class CreditMutuelBrowser(LoginBrowser, StatesMixin): r'/(?P.*)fr/banques/particuliers/index.html', LoginPage) login_error = URL(r'/(?P.*)fr/identification/default.cgi', LoginErrorPage) + fiscality = URL(r'/(?P.*)fr/banque/residencefiscale.aspx', FiscalityConfirmationPage) accounts = URL(r'/(?P.*)fr/banque/situation_financiere.cgi', r'/(?P.*)fr/banque/situation_financiere.html', AccountsPage) + mobile_confirmation = URL(r'/(?P.*)fr/banque/validation.aspx', MobileConfirmationPage) revolving_loan_list = URL(r'/(?P.*)fr/banque/CR/arrivee.asp\?fam=CR.*', RevolvingLoansList) revolving_loan_details = URL(r'/(?P.*)fr/banque/CR/cam9_vis_lstcpt.asp.*', RevolvingLoanDetails) user_space = URL(r'/(?P.*)fr/banque/espace_personnel.aspx', diff --git a/modules/creditmutuel/pages.py b/modules/creditmutuel/pages.py index 71e84ab0d5..d5ec506c3b 100644 --- a/modules/creditmutuel/pages.py +++ b/modules/creditmutuel/pages.py @@ -101,6 +101,22 @@ def on_load(self): raise BrowserIncorrectPassword(CleanText('//div[has-class("blocmsg")]')(self.doc)) +class FiscalityConfirmationPage(LoggedPage, HTMLPage): + pass + + +class MobileConfirmationPage(LoggedPage, HTMLPage): + # We land on this page for some connections, but can still bypass this verification for now + def on_load(self): + link = Attr('//a[contains(text(), "Accéder à mon Espace Client sans Confirmation Mobile")]', 'href', default=None)(self.doc) + if link: + self.logger.warning('This connexion is bypassing mobile confirmation') + self.browser.location(link) + else: + self.logger.warning('This connexion cannot bypass mobile confirmation') + assert False, 'This connexion cannot bypass mobile confirmation' + + class EmptyPage(LoggedPage, HTMLPage): REFRESH_MAX = 10.0 @@ -165,9 +181,12 @@ class item_account_generic(ItemElement): ('Livret', Account.TYPE_SAVINGS), ("Plan D'Epargne", Account.TYPE_SAVINGS), ('Tonic Croissance', Account.TYPE_SAVINGS), + ('Tonic Societaire', Account.TYPE_SAVINGS), ('Capital Expansion', Account.TYPE_SAVINGS), ('Épargne', Account.TYPE_SAVINGS), ('Capital Plus', Account.TYPE_SAVINGS), + ('Pep', Account.TYPE_SAVINGS), + ('Compte Duo', Account.TYPE_SAVINGS), ('Compte Garantie Titres', Account.TYPE_MARKET), ]) @@ -243,6 +262,16 @@ def parse(self, el): for td in el.xpath('./td[2] | ./td[3]'): try: balance = CleanDecimal('.', replace_dots=True)(td) + has_child_def_card = CleanText('.//following-sibling::tr[1]//span[contains(text(), "Dépenses cartes prélevées")]')(el) + if Field('type')(self) == Account.TYPE_CHECKING and not has_child_def_card: + # the present day, real balance (without comings) is displayed in the operations page of the account + # need to limit requests to checking accounts with no def cards + details_page_link = Link('.//a', default=None)(self) + if details_page_link: + coming_page = self.page.browser.open(details_page_link).page + balance_without_comings = coming_page.get_balance() + if not empty(balance_without_comings): + balance = balance_without_comings except InvalidOperation: continue else: @@ -712,6 +741,9 @@ def get_coming_link(self): def has_more_operations(self): return bool(self.doc.xpath('//a/span[contains(text(), "Plus d\'opérations")]')) + def get_balance(self): + return CleanDecimal.French('//span[contains(text(), "Dont opérations enregistrées")]', default=NotAvailable)(self.doc) + class CardsOpePage(OperationsPage): def select_card(self, card_number): diff --git a/modules/fortuneo/browser.py b/modules/fortuneo/browser.py index e579fd292c..66040ffc56 100644 --- a/modules/fortuneo/browser.py +++ b/modules/fortuneo/browser.py @@ -290,4 +290,11 @@ def get_profile(self): csv_link = self.page.get_csv_link() if csv_link: self.location(csv_link) - return self.page.get_profile() + return self.page.get_profile() + # The persons name is in a menu not returned in the ProfilePage, so + # we have to go back to the AccountsPage (which is the main page for the website) + # to get the info + person = self.page.get_profile() + self.accounts_page.go() + self.page.fill_person_name(obj=person) + return person diff --git a/modules/fortuneo/pages/accounts_list.py b/modules/fortuneo/pages/accounts_list.py index 02a88af723..f59d03183b 100644 --- a/modules/fortuneo/pages/accounts_list.py +++ b/modules/fortuneo/pages/accounts_list.py @@ -180,7 +180,7 @@ def get_investments(self, account): inv.valuation = self.parse_decimal(cols[self.COL_VALUATION]) inv.diff = self.parse_decimal(cols[self.COL_PERF]) diff_percent = self.parse_decimal(cols[self.COL_PERF_PERCENT]) - inv.diff_percent = diff_percent / 100 if diff_percent else NotAvailable + inv.diff_ratio = diff_percent / 100 if diff_percent else NotAvailable if is_isin_valid(inv.code): inv.code_type = Investment.CODE_TYPE_ISIN @@ -365,6 +365,14 @@ def has_action_needed(self): if warning: raise ActionNeeded(warning[0].text) + @method + class fill_person_name(ItemElement): + klass = Account + + # Contains the title (M., Mme., etc) + last name. + # The first name isn't available in the person's details. + obj_name = CleanText('//span[has-class("mon_espace_nom")]') + def get_iframe_url(self): iframe = self.doc.xpath('//iframe[@id="iframe_centrale"]') if iframe: diff --git a/modules/hsbc/pages/investments.py b/modules/hsbc/pages/investments.py index 52095c6368..3f84a161ec 100644 --- a/modules/hsbc/pages/investments.py +++ b/modules/hsbc/pages/investments.py @@ -385,7 +385,7 @@ def obj_vdate(self): obj_valuation = CleanDecimal(Dict( 'holdingDetailInformation/0/holdingDetailMultipleCurrencyInformation/0/productHoldingMarketValueAmount' ), default=NotAvailable) - obj_diff_percent = CleanDecimal(Dict( + obj_diff_ratio = CleanDecimal(Dict( 'holdingDetailInformation/0/holdingDetailMultipleCurrencyInformation/0' '/profitLossUnrealizedPercent' ), default=NotAvailable) @@ -443,7 +443,7 @@ def obj__invest_account_id(self): obj_unitprice = CleanDecimal(Dict( 'holdingSummaryMultipleCurrencyInformation/0/productHoldingUnitCostAverageAmount' ),default=NotAvailable) - obj_diff_percent = CleanDecimal(Dict( + obj_diff_ratio = CleanDecimal(Dict( 'holdingSummaryMultipleCurrencyInformation/0/profitLossUnrealizedPercent' ), default=NotAvailable) obj_diff = CleanDecimal(Dict( @@ -546,7 +546,7 @@ class item(ItemElement): obj_unitprice = CleanDecimal(TableCell('unitprice'), replace_dots=True) obj_unitvalue = CleanDecimal(TableCell('unitvalue'), replace_dots=True) - def obj_diff_percent(self): + def obj_diff_ratio(self): diff_percent = CleanDecimal( Regexp(CleanText(TableCell('diff_percent')), r'\d+,\d+'), replace_dots=True diff --git a/modules/ing/api/__init__.py b/modules/ing/api/__init__.py index 6673d4880c..51214bad3a 100644 --- a/modules/ing/api/__init__.py +++ b/modules/ing/api/__init__.py @@ -19,11 +19,15 @@ from .login import LoginPage from .accounts_page import AccountsPage, HistoryPage, ComingPage -from .transfer_page import DebitAccountsPage, CreditAccountsPage, TransferPage +from .transfer_page import ( + DebitAccountsPage, CreditAccountsPage, TransferPage, AddRecipientPage, + OtpChannelsPage, ConfirmOtpPage, +) from .profile_page import ProfilePage __all__ = ['LoginPage', 'AccountsPage', 'HistoryPage', 'ComingPage', 'DebitAccountsPage', 'CreditAccountsPage', 'TransferPage', + 'AddRecipientPage', 'OtpChannelsPage', 'ConfirmOtpPage', 'ProfilePage'] diff --git a/modules/ing/api/transfer_page.py b/modules/ing/api/transfer_page.py index 0ebff65fd8..72c371147e 100644 --- a/modules/ing/api/transfer_page.py +++ b/modules/ing/api/transfer_page.py @@ -30,7 +30,9 @@ from weboob.browser.elements import method, DictElement, ItemElement from weboob.browser.filters.json import Dict from .compat.weboob_browser_filters_standard import Env, Field, Date -from .compat.weboob_capabilities_bank import Recipient +from .compat.weboob_capabilities_bank import ( + Recipient, RecipientInvalidIban, RecipientInvalidOTP, +) class TransferINGVirtKeyboard(SimpleVirtualKeyboard): @@ -159,3 +161,35 @@ def get_password_coord(self, password): @property def transfer_is_validated(self): return Dict('acknowledged')(self.doc) + + +class AddRecipientPage(LoggedPage, JsonPage): + def check_recipient(self, recipient): + rcpt = self.doc + return rcpt['accountHolderName'] == recipient.label and rcpt['iban'] == recipient.iban + + def handle_error(self): + if 'error' in self.doc: + if self.doc['error']['code'] == 'EXTERNAL_ACCOUNT.IBAN_NOT_FRENCH': + # not using the bank message because it is too generic + raise RecipientInvalidIban(message="L'IBAN doit correpondre à celui d'une banque domiciliée en France.") + assert False, 'Recipient error not handled' + + +class OtpChannelsPage(LoggedPage, JsonPage): + def get_sms_info(self): + # receive a list of dict + for element in self.doc: + if element['type'] == 'SMS_MOBILE': + return element + assert False, 'No sms info found' + + +class ConfirmOtpPage(LoggedPage, JsonPage): + def handle_error(self): + if 'error' in self.doc: + error_code = self.doc['error']['code'] + if error_code == 'SCA.WRONG_OTP_ATTEMPT': + raise RecipientInvalidOTP(message=self.doc['error']['message']) + + assert False, 'Recipient OTP error not handled' diff --git a/modules/ing/api_browser.py b/modules/ing/api_browser.py index 78eec407fc..e68f4849ef 100644 --- a/modules/ing/api_browser.py +++ b/modules/ing/api_browser.py @@ -22,17 +22,24 @@ import json from collections import OrderedDict from functools import wraps +import re -from weboob.browser.browsers import LoginBrowser, URL +from weboob.browser.browsers import LoginBrowser, URL, StatesMixin from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable, ActionNeeded from weboob.browser.exceptions import ClientError -from .compat.weboob_capabilities_bank import TransferBankError, TransferInvalidAmount +from .compat.weboob_capabilities_bank import ( + TransferBankError, TransferInvalidAmount, + AddRecipientStep, RecipientInvalidOTP, + AddRecipientTimeout, AddRecipientBankError, +) from weboob.tools.capabilities.bank.transactions import FrenchTransaction +from weboob.tools.value import Value from .api import ( LoginPage, AccountsPage, HistoryPage, ComingPage, DebitAccountsPage, CreditAccountsPage, TransferPage, ProfilePage, + AddRecipientPage, OtpChannelsPage, ConfirmOtpPage, ) from .web import StopPage, ActionNeededPage @@ -84,8 +91,9 @@ def wrapper(self, *args, **kwargs): return decorator -class IngAPIBrowser(LoginBrowser): +class IngAPIBrowser(LoginBrowser, StatesMixin): BASEURL = 'https://m.ing.fr' + STATE_DURATION = 10 # Login context = URL(r'/secure/api-v1/session/context') @@ -109,15 +117,32 @@ class IngAPIBrowser(LoginBrowser): init_transfer_page = URL(r'/secure/api-v1/transfers/v2/new/validate', TransferPage) exec_transfer_page = URL(r'/secure/api-v1/transfers/v2/new/execute/pin', TransferPage) + # recipient + add_recipient = URL(r'secure/api-v1/externalAccounts/add/validateRequest', AddRecipientPage) + otp_channels = URL(r'secure/api-v1/sensitiveoperation/ADD_TRANSFER_BENEFICIARY/otpChannels', OtpChannelsPage) + confirm_otp = URL(r'secure/api-v1/sca/confirmOtp', ConfirmOtpPage) + # profile informations = URL(r'/secure/api-v1/customer/info', ProfilePage) + + __states__ = ('need_reload_state', 'add_recipient_info') + def __init__(self, *args, **kwargs): self.birthday = kwargs.pop('birthday') super(IngAPIBrowser, self).__init__(*args, **kwargs) self.old_browser = IngBrowser(*args, **kwargs) self.transfer_data = None + self.need_reload_state = None + self.add_recipient_info = None + + def load_state(self, state): + # reload state only for new recipient + if state.get('need_reload_state'): + state.pop('url', None) + self.need_reload_state = None + super(IngAPIBrowser, self).load_state(state) def handle_login_error(self, r): error_page = r.response.json() @@ -315,7 +340,7 @@ def get_investments(self, account): self.redirect_to_old_browser() return self.old_browser.get_investments(account) - ############# CapTransfer ############# + ############# CapTransferAddRecipient ############# @need_login @need_to_be_on_website('api') def iter_recipients(self, account): @@ -381,6 +406,103 @@ def execute_transfer(self, transfer): assert self.page.transfer_is_validated, "Transfer is not validated" return transfer + @need_login + def send_sms_to_user(self, recipient, sms_info): + """Add recipient with OTP SMS authentication""" + data = { + 'channelType': sms_info['type'], + 'externalAccountsRequest': self.add_recipient_info, + 'sensitiveOperationAction': 'ADD_TRANSFER_BENEFICIARY', + } + + phone_id = sms_info['phone'] + data['channelValue'] = phone_id + self.add_recipient_info['phoneUid'] = phone_id + + self.location(self.absurl('/secure/api-v1/sca/sendOtp', base=True), json=data) + self.need_reload_state = True + raise AddRecipientStep(recipient, Value('code', label='Veuillez saisir le code temporaire envoyé par SMS')) + + def handle_recipient_error(self, r): + error_page = r.response.json() + if 'error' in error_page: + error = error_page['error'] + + # the error message may seem generic + # but after testing multiple cases + # it is the only time that it appears + if error['code'] == 'SENSITIVE_OPERATION.SENSITIVE_OPERATION_NOT_FOUND': + raise AddRecipientTimeout() + elif error['code'] in ( + 'EXTERNAL_ACCOUNT.EXTERNAL_ACCOUNT_ALREADY_EXISTS', + # not allowed to add a recipient + 'EXTERNAL_ACCOUNT.ACCOUNT_RESTRICTION', + ): + raise AddRecipientBankError(message=error['message']) + + assert False, 'Recipient error not handled' + + @need_login + def end_sms_recipient(self, recipient, code): + # create a variable to empty the add_recipient_info + # so that if there is a problem it will not be caught + # in the StatesMixin + rcpt_info = self.add_recipient_info + self.add_recipient_info = None + + if not re.match(r'^\d{6}$', code): + raise RecipientInvalidOTP() + + data = { + 'externalAccountsRequest': rcpt_info, + 'otp': code, + 'sensitiveOperationAction': 'ADD_TRANSFER_BENEFICIARY', + } + + try: + self.confirm_otp.go(json=data) + except ClientError as e: + self.handle_recipient_error(e) + raise + + self.page.handle_error() + + @need_login + @need_to_be_on_website('api') + def new_recipient(self, recipient, **params): + # sms only, we don't handle the call + if 'code' in params: + # part 2 - finalization + self.end_sms_recipient(recipient, params['code']) + + # WARNING: On the recipient list, the IBAN is masked + # so I cannot match it + # The label is not checked by the website + # so I cannot match it + return recipient + + # part 1 - initialization + # Set sign method + self.otp_channels.go() + sms_info = self.page.get_sms_info() + + try: + self.add_recipient.go(json={ + 'accountHolderName': recipient.label, + 'iban': recipient.iban + }) + except ClientError as e: + self.handle_recipient_error(e) + raise + + self.page.handle_error() + + assert self.page.check_recipient(recipient), "The recipients don't match." + self.add_recipient_info = self.page.doc + + # WARNING: this send validation request to user + self.send_sms_to_user(recipient, sms_info) + ############# CapDocument ############# @need_login @need_to_be_on_website('web') diff --git a/modules/ing/module.py b/modules/ing/module.py index 9c67c01a3f..4737ad095a 100644 --- a/modules/ing/module.py +++ b/modules/ing/module.py @@ -21,8 +21,9 @@ from decimal import Decimal from datetime import timedelta +import re -from .compat.weboob_capabilities_bank import CapBankWealth, CapBankTransfer, Account, AccountNotFound, RecipientNotFound +from .compat.weboob_capabilities_bank import CapBankWealth, CapBankTransferAddRecipient, Account, AccountNotFound, RecipientNotFound from weboob.capabilities.bill import ( CapDocument, Bill, Subscription, SubscriptionNotFound, DocumentNotFound, DocumentTypes, @@ -37,7 +38,7 @@ __all__ = ['INGModule'] -class INGModule(Module, CapBankWealth, CapBankTransfer, CapDocument, CapProfile): +class INGModule(Module, CapBankWealth, CapBankTransferAddRecipient, CapDocument, CapProfile): NAME = 'ing' MAINTAINER = 'Florent Fourcot' EMAIL = 'weboob@flo.fourcot.fr' @@ -95,12 +96,19 @@ def iter_investment(self, account): account = self.get_account(account) return self.browser.get_investments(account) - ############# CapTransfer ############# + ############# CapTransferAddRecipient ############# def iter_transfer_recipients(self, account): if not isinstance(account, Account): account = self.get_account(account) return self.browser.iter_recipients(account) + def new_recipient(self, recipient, **params): + cleaned_label = re.sub("[^0-9a-zA-Z:/\-\?\(\)\.,\+ ']", '', recipient.label) + cleaned_label = re.sub(r'\s{2,}', ' ', cleaned_label) + recipient.label = cleaned_label.strip() + + return self.browser.new_recipient(recipient, **params) + def init_transfer(self, transfer, **params): self.logger.info('Going to do a new transfer') diff --git a/modules/ing/web/accounts_list.py b/modules/ing/web/accounts_list.py index a36dffb023..f5eedfe05a 100644 --- a/modules/ing/web/accounts_list.py +++ b/modules/ing/web/accounts_list.py @@ -400,7 +400,7 @@ def obj_code(self): else: return NotAvailable - def obj_diff_percent(self): + def obj_diff_ratio(self): diff = CleanDecimal(TableCell('diff_percent'), replace_dots=True, default=NotAvailable)(self) if not diff: return diff diff --git a/modules/ldlc/materielnet_pages.py b/modules/ldlc/materielnet_pages.py index 31feef0ca4..714dd895bc 100644 --- a/modules/ldlc/materielnet_pages.py +++ b/modules/ldlc/materielnet_pages.py @@ -31,7 +31,10 @@ class LoginPage(PartialHTMLPage): - def login(self, login, password): + def get_recaptcha_sitekey(self): + return Attr('//div[@class="g-recaptcha"]', 'data-sitekey', default=NotAvailable)(self.doc) + + def login(self, login, password, captcha_response=None): maxlength = Attr('//input[@id="Email"]', 'data-val-maxlength-max')(self.doc) regex = Attr('//input[@id="Email"]', 'data-val-regex-pattern')(self.doc) # their regex is: ^([\w\-+\.]+)@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.)|(([\w-]+\.)+))([a-zA-Z]{2,15}|[0-9]{1,3})(\]?)$ @@ -47,6 +50,10 @@ def login(self, login, password): form = self.get_form(xpath='//form[contains(@action, "/Login/Login")]') form['Email'] = login form['Password'] = password + + if captcha_response: + form['g-recaptcha-response'] = captcha_response + form.submit() def get_error(self): diff --git a/modules/lendosphere/__init__.py b/modules/lendosphere/__init__.py new file mode 100644 index 0000000000..8298ff613a --- /dev/null +++ b/modules/lendosphere/__init__.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Vincent A +# +# This file is part of a weboob module. +# +# This weboob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This weboob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this weboob module. If not, see . + +from __future__ import unicode_literals + + +from .module import LendosphereModule + + +__all__ = ['LendosphereModule'] diff --git a/modules/lendosphere/browser.py b/modules/lendosphere/browser.py new file mode 100644 index 0000000000..88cc39398b --- /dev/null +++ b/modules/lendosphere/browser.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Vincent A +# +# This file is part of a weboob module. +# +# This weboob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This weboob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this weboob module. If not, see . + +from __future__ import unicode_literals + +import datetime + +from weboob.browser.browsers import LoginBrowser, URL, need_login +from weboob.tools.capabilities.bank.investments import create_french_liquidity +from .compat.weboob_capabilities_bank import Investment + +from .pages import ( + LoginPage, SummaryPage, GSummaryPage, ProfilePage, ComingPage, +) + + +class AttrURL(URL): + def build(self, *args, **kwargs): + import re + + for pattern in self.urls: + regex = re.compile(pattern) + + for k in regex.groupindex: + if hasattr(self.browser, k) and k not in kwargs: + kwargs[k] = getattr(self.browser, k) + + return super(AttrURL, self).build(*args, **kwargs) + + +class LendosphereBrowser(LoginBrowser): + BASEURL = 'https://www.lendosphere.com' + + login = URL(r'/membres/se-connecter', LoginPage) + dashboard = AttrURL(r'/membres/(?P[a-z0-9-]+)/tableau-de-bord', SummaryPage) + global_summary = AttrURL(r'/membres/(?P[a-z0-9-]+)/dashboard_global_info', GSummaryPage) + coming = AttrURL(r'/membres/(?P[a-z0-9-]+)/mes-echeanciers.csv', ComingPage) + profile = AttrURL(r'/membres/(?P[a-z0-9-]+)', ProfilePage) + + def do_login(self): + self.login.go() + self.page.do_login(self.username, self.password) + + if self.login.is_here(): + self.page.raise_error() + + self.user_id = self.page.params['user_id'] + + @need_login + def iter_accounts(self): + self.global_summary.go() + return [self.page.get_account()] + + @need_login + def iter_investment(self, account): + today = datetime.date.today() + + self.coming.go() + + # unfortunately there doesn't seem to be a page indicating what's + # left to be repaid on each project, so let's sum... + valuations = {} + commissions = {} + for tr in self.page.iter_transactions(): + if tr.date <= today: + continue + + if tr.raw not in valuations: + valuations[tr.raw] = tr.amount + commissions[tr.raw] = tr.commission + else: + valuations[tr.raw] += tr.amount + commissions[tr.raw] += tr.commission + + for label, value in valuations.items(): + inv = Investment() + inv.label = label + inv.valuation = value + inv.diff = commissions[label] + yield inv + + yield create_french_liquidity(account._liquidities) diff --git a/modules/lendosphere/compat/__init__.py b/modules/lendosphere/compat/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/modules/lendosphere/compat/weboob_browser_filters_standard.py b/modules/lendosphere/compat/weboob_browser_filters_standard.py new file mode 100644 index 0000000000..f382d162f0 --- /dev/null +++ b/modules/lendosphere/compat/weboob_browser_filters_standard.py @@ -0,0 +1,49 @@ + +import weboob.browser.filters.standard as OLD + +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) + + +try: + __all__ = OLD.__all__ +except AttributeError: + pass + + +class Coalesce(MultiFilter): + """ + Returns the first value that is not falsy, + or default if all values are falsy. + """ + @debug() + def filter(self, values): + for value in values: + if value: + return value + return self.default_or_raise(FilterError('All falsy and no default.')) + + +class MapIn(Filter): + """ + Map the pattern of a selected value to another value using a dict. + """ + + def __init__(self, selector, map_dict, default=_NO_DEFAULT): + """ + :param selector: key from `map_dict` to use + """ + super(MapIn, self).__init__(selector, default=default) + self.map_dict = map_dict + + @debug() + def filter(self, txt): + """ + :raises: :class:`ItemNotFound` if key pattern does not exist in dict + """ + for key in self.map_dict: + if key in txt: + return self.map_dict[key] + + return self.default_or_raise(ItemNotFound('Unable to handle %r on %r' % (txt, self.map_dict))) diff --git a/modules/lendosphere/compat/weboob_capabilities_bank.py b/modules/lendosphere/compat/weboob_capabilities_bank.py new file mode 100644 index 0000000000..4d5ed8d992 --- /dev/null +++ b/modules/lendosphere/compat/weboob_capabilities_bank.py @@ -0,0 +1,48 @@ + +import weboob.capabilities.bank as OLD +from weboob.capabilities.base import StringField +from weboob.capabilities.date import DateField + +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) + + +try: + __all__ = OLD.__all__ +except AttributeError: + pass + + +# can't create a subclass because of CapBank.iter_resources reimplementations: +# modules will import our subclass, but boobank will call iter_resources with the OLD class +Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') + + +Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') + + +class RecipientInvalidOTP(AddRecipientError): + code = 'invalidOTP' + + +class AccountOwnership(object): + """ + Relationship between the credentials owner (PSU) and the account + """ + OWNER = u'owner' + """The PSU is the account owner""" + CO_OWNER = u'co-owner' + """The PSU is the account co-owner""" + ATTORNEY = u'attorney' + """The PSU is the account attorney""" + + +AccountOwnerType.ASSOCIATION = u'ASSO' + + +try: + __all__ += ['AccountOwnership', 'RecipientInvalidOTP'] +except NameError: + pass diff --git a/modules/lendosphere/module.py b/modules/lendosphere/module.py new file mode 100644 index 0000000000..ca018220df --- /dev/null +++ b/modules/lendosphere/module.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Vincent A +# +# This file is part of a weboob module. +# +# This weboob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This weboob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this weboob module. If not, see . + +from __future__ import unicode_literals + +from weboob.tools.backend import Module, BackendConfig +from weboob.tools.value import ValueBackendPassword +from .compat.weboob_capabilities_bank import CapBankWealth + +from .browser import LendosphereBrowser + + +__all__ = ['LendosphereModule'] + + +class LendosphereModule(Module, CapBankWealth): + NAME = 'lendosphere' + DESCRIPTION = 'Lendosphere' + MAINTAINER = 'Vincent A' + EMAIL = 'dev@indigo.re' + LICENSE = 'LGPLv3+' + VERSION = '1.5' + + BROWSER = LendosphereBrowser + + CONFIG = BackendConfig( + ValueBackendPassword('login', label='Email', masked=False), + ValueBackendPassword('password', label='Mot de passe'), + ) + + def create_default_browser(self): + return self.create_browser( + self.config['login'].get(), + self.config['password'].get(), + ) + + def iter_accounts(self): + return self.browser.iter_accounts() + + def iter_investment(self, account): + return self.browser.iter_investment(account) diff --git a/modules/lendosphere/pages.py b/modules/lendosphere/pages.py new file mode 100644 index 0000000000..0ea4c454c4 --- /dev/null +++ b/modules/lendosphere/pages.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Vincent A +# +# This file is part of a weboob module. +# +# This weboob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This weboob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this weboob module. If not, see . + +from __future__ import unicode_literals + +from weboob.browser.elements import ItemElement, method, DictElement +from weboob.browser.filters.json import Dict +from .compat.weboob_browser_filters_standard import ( + CleanText, CleanDecimal, Date, Field, +) +from weboob.browser.pages import HTMLPage, CsvPage, LoggedPage +from weboob.capabilities.base import NotAvailable +from .compat.weboob_capabilities_bank import Account, Transaction +from weboob.exceptions import BrowserIncorrectPassword + + +MAIN_ID = '_lendosphere_' + + +class LoginPage(HTMLPage): + def do_login(self, username, password): + form = self.get_form(id='new_user') + form['user[email]'] = username + form['user[password]'] = password + form.submit() + + def raise_error(self): + msg = CleanText('//div[has-class("alert-danger")]')(self.doc) + if 'Votre email ou mot de passe est incorrect' in msg: + raise BrowserIncorrectPassword(msg) + assert False, 'unhandled error %r' % msg + + +class SummaryPage(LoggedPage, HTMLPage): + pass + + +class ProfilePage(LoggedPage, HTMLPage): + pass + + +class ComingProjectPage(LoggedPage, HTMLPage): + def iter_projects(self): + return [value for value in self.doc.xpath('//select[@id="offer"]/option/@value') if value != '*'] + + +class ComingPage(LoggedPage, CsvPage): + HEADER = 1 + + @method + class iter_transactions(DictElement): + class item(ItemElement): + klass = Transaction + + obj_type = Transaction.TYPE_BANK + obj_raw = Dict('Projet') + obj_date = Date(Dict('Date'), dayfirst=True) + + obj_gross_amount = CleanDecimal.SI(Dict('Capital rembourse')) + obj_amount = CleanDecimal.SI(Dict('Montant brut')) + obj_commission = CleanDecimal.SI(Dict('Interets')) + + obj__amount_left = CleanDecimal.SI(Dict('Capital restant du')) + + +class GSummaryPage(LoggedPage, HTMLPage): + @method + class get_account(ItemElement): + klass = Account + + obj_id = MAIN_ID + obj_currency = 'EUR' + obj_number = NotAvailable + obj_type = Account.TYPE_MARKET + obj_label = 'Lendosphere' + + obj__liquidities = CleanDecimal.French('//div[div[@class="subtitle"][contains(text(),"Somme disponible")]]/div[@class="amount"]') + obj__invested = CleanDecimal.French('//tr[td[contains(text(),"Echéances brutes restantes*")]]/td[last()]') + + def obj_balance(self): + return Field('_liquidities')(self) + Field('_invested')(self) diff --git a/modules/linebourse/api/pages.py b/modules/linebourse/api/pages.py index 387620a888..cd88f1358b 100644 --- a/modules/linebourse/api/pages.py +++ b/modules/linebourse/api/pages.py @@ -84,10 +84,10 @@ def obj_unitprice(self): return CleanDecimal(Dict('pam'))(self) return NotAvailable - def obj_diff_percent(self): + def obj_diff_ratio(self): if not Env('sign')(self): return NotAvailable - # obj_diff_percent key can have several names: + # obj_diff_ratio key can have several names: if Dict('plvPourcentage', default=None)(self): return CleanDecimal(Dict('plvPourcentage'), sign=lambda x: Env('sign')(self))(self) elif Dict('pourcentagePlv', default=None)(self): diff --git a/modules/materielnet/browser.py b/modules/materielnet/browser.py index e86cf501b1..890d446f2c 100644 --- a/modules/materielnet/browser.py +++ b/modules/materielnet/browser.py @@ -17,9 +17,11 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . +from __future__ import unicode_literals + from weboob.browser import LoginBrowser, URL, need_login -from weboob.exceptions import BrowserIncorrectPassword +from weboob.exceptions import BrowserIncorrectPassword, NocaptchaQuestion from .pages import LoginPage, CaptchaPage, ProfilePage, DocumentsPage, DocumentsDetailsPage @@ -34,7 +36,7 @@ class MaterielnetBrowser(LoginBrowser): BASEURL = 'https://secure.materiel.net' login = MyURL(r'/(?P.*)Login/Login', LoginPage) - captcha = URL('/pm/client/captcha.html', CaptchaPage) + captcha = URL(r'/pm/client/captcha.html', CaptchaPage) profile = MyURL(r'/(?P.*)Account/InformationsSection', r'/pro/Account/InformationsSection', ProfilePage) documents = MyURL(r'/(?P.*)Orders/PartialCompletedOrdersHeader', @@ -42,8 +44,9 @@ class MaterielnetBrowser(LoginBrowser): document_details = MyURL(r'/(?P.*)Orders/PartialCompletedOrderContent', r'/pro/Orders/PartialCompletedOrderContent', DocumentsDetailsPage) - def __init__(self, *args, **kwargs): + def __init__(self, config, *args, **kwargs): super(MaterielnetBrowser, self).__init__(*args, **kwargs) + self.config = config self.is_pro = None self.lang = '' @@ -53,11 +56,17 @@ def par_or_pro_location(self, url, *args, **kwargs): elif self.lang: url = '/' + self.lang[:-1] + url - return super(MaterielnetBrowser, self).location(url, *args, **kwargs) + return self.location(url, *args, **kwargs) def do_login(self): self.login.go() - self.page.login(self.username, self.password) + sitekey = self.page.get_recaptcha_sitekey() + # captcha is not always present + if sitekey: + if not self.config['captcha_response'].get(): + raise NocaptchaQuestion(website_key=sitekey, website_url=self.login.build(lang=self.lang)) + + self.page.login(self.username, self.password, self.config['captcha_response'].get()) if self.captcha.is_here(): BrowserIncorrectPassword() diff --git a/modules/materielnet/module.py b/modules/materielnet/module.py index 1d0c1f0ea4..1b0aa6617a 100644 --- a/modules/materielnet/module.py +++ b/modules/materielnet/module.py @@ -17,11 +17,12 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . +from __future__ import unicode_literals from weboob.capabilities.bill import DocumentTypes, CapDocument, Subscription, Document, SubscriptionNotFound, DocumentNotFound from weboob.capabilities.base import find_object, NotAvailable from weboob.tools.backend import Module, BackendConfig -from weboob.tools.value import ValueBackendPassword +from weboob.tools.value import ValueBackendPassword, Value from .browser import MaterielnetBrowser @@ -31,20 +32,21 @@ class MaterielnetModule(Module, CapDocument): NAME = 'materielnet' - DESCRIPTION = u'Materiel.net' - MAINTAINER = u'Edouard Lambert' + DESCRIPTION = 'Materiel.net' + MAINTAINER = 'Edouard Lambert' EMAIL = 'elambert@budget-insight.com' LICENSE = 'LGPLv3+' VERSION = '1.5' - CONFIG = BackendConfig(ValueBackendPassword('login', label='Email', regexp=r'.+@.+'), - ValueBackendPassword('password', label='Mot de passe')) + CONFIG = BackendConfig(ValueBackendPassword('login', label='Email'), + ValueBackendPassword('password', label='Mot de passe'), + Value('captcha_response', label='Réponse captcha', default='', required=False)) BROWSER = MaterielnetBrowser accepted_document_types = (DocumentTypes.BILL,) def create_default_browser(self): - return self.create_browser(self.config['login'].get(), self.config['password'].get()) + return self.create_browser(self.config, self.config['login'].get(), self.config['password'].get()) def iter_subscription(self): return self.browser.get_subscription_list() diff --git a/modules/materielnet/pages.py b/modules/materielnet/pages.py index 31feef0ca4..714dd895bc 100644 --- a/modules/materielnet/pages.py +++ b/modules/materielnet/pages.py @@ -31,7 +31,10 @@ class LoginPage(PartialHTMLPage): - def login(self, login, password): + def get_recaptcha_sitekey(self): + return Attr('//div[@class="g-recaptcha"]', 'data-sitekey', default=NotAvailable)(self.doc) + + def login(self, login, password, captcha_response=None): maxlength = Attr('//input[@id="Email"]', 'data-val-maxlength-max')(self.doc) regex = Attr('//input[@id="Email"]', 'data-val-regex-pattern')(self.doc) # their regex is: ^([\w\-+\.]+)@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.)|(([\w-]+\.)+))([a-zA-Z]{2,15}|[0-9]{1,3})(\]?)$ @@ -47,6 +50,10 @@ def login(self, login, password): form = self.get_form(xpath='//form[contains(@action, "/Login/Login")]') form['Email'] = login form['Password'] = password + + if captcha_response: + form['g-recaptcha-response'] = captcha_response + form.submit() def get_error(self): diff --git a/modules/myedenred/browser.py b/modules/myedenred/browser.py index 3267f375cf..363ff03ab1 100644 --- a/modules/myedenred/browser.py +++ b/modules/myedenred/browser.py @@ -19,11 +19,13 @@ from __future__ import unicode_literals +from datetime import timedelta + from weboob.browser import LoginBrowser, URL, need_login from weboob.exceptions import BrowserIncorrectPassword -from weboob.tools.capabilities.bank.transactions import merge_iterators -from .pages import LoginPage, AccountsPage, AccountDetailsPage, TransactionsPage +from weboob.tools.date import LinearDateGuesser +from .pages import LoginPage, AccountsPage, AccountDetailsPage, TransactionsPage class MyedenredBrowser(LoginBrowser): BASEURL = 'https://www.myedenred.fr' @@ -55,21 +57,17 @@ def iter_accounts(self): @need_login def iter_history(self, account): - def iter_transactions_by_type(type): - history = self.transactions.go(data={'command': 'Charger les 10 transactions suivantes', - 'ErfBenId': account._product_token, - 'ProductCode': account._product_type, - 'SortBy': 'DateOperation', - 'StartDate': '', - 'EndDate': '', - 'PageNum': 10, - 'OperationType': type, - 'failed': 'false', - 'X-Requested-With': 'XMLHttpRequest' - }) - return history.iter_transactions(subid=account.id) - - if account.id not in self.docs: - iterator = merge_iterators(iter_transactions_by_type(type='Debit'), iter_transactions_by_type(type='Credit')) - self.docs[account.id] = list(iterator) - return self.docs[account.id] + self.transactions.go(data={ + 'command': 'Charger les 10 transactions suivantes', + 'ErfBenId': account._product_token, + 'ProductCode': account._product_type, + 'SortBy': 'DateOperation', + 'StartDate': '', + 'EndDate': '', + 'PageNum': 10, + 'OperationType': 'Default', + 'failed': 'false', + 'X-Requested-With': 'XMLHttpRequest' + }) + for tr in self.page.iter_transactions(subid=account.id, date_guesser=LinearDateGuesser(date_max_bump=timedelta(45))): + yield tr diff --git a/modules/myedenred/pages.py b/modules/myedenred/pages.py index 176a87db7b..c171f1b029 100644 --- a/modules/myedenred/pages.py +++ b/modules/myedenred/pages.py @@ -19,17 +19,14 @@ from __future__ import unicode_literals - from weboob.browser.pages import HTMLPage, PartialHTMLPage, LoggedPage from weboob.browser.elements import ItemElement, method, ListElement from .compat.weboob_browser_filters_standard import ( CleanText, CleanDecimal, - Regexp, DateGuesser, Field + Regexp, DateGuesser, Field, Env ) from .compat.weboob_capabilities_bank import Account, Transaction from weboob.capabilities.base import NotAvailable -from weboob.tools.date import LinearDateGuesser -from datetime import timedelta def MyDecimal(*args, **kwargs): @@ -74,7 +71,10 @@ class iter_transactions(ListElement): class item(ItemElement): klass = Transaction - obj_date = DateGuesser(CleanText('.//span[contains(., "/")]'), LinearDateGuesser(date_max_bump=timedelta(45))) + def condition(self): + return CleanText('./td[@class="al-c"]/span')(self) not in ('transaction refusée', 'transaction en cours de traitement') + + obj_date = DateGuesser(CleanText('.//span[contains(., "/")]'), Env('date_guesser')) obj_label = CleanText('.//h3/strong') obj_raw = Field('label') obj_amount = MyDecimal('./td[@class="al-r"]/div/span[has-class("badge")]') diff --git a/modules/orange/browser.py b/modules/orange/browser.py index 56d0b72c8b..34c058be1c 100644 --- a/modules/orange/browser.py +++ b/modules/orange/browser.py @@ -22,9 +22,9 @@ from requests.exceptions import ConnectTimeout from weboob.browser import LoginBrowser, URL, need_login -from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable, ActionNeeded +from weboob.exceptions import BrowserIncorrectPassword, BrowserUnavailable, ActionNeeded, BrowserPasswordExpired from .pages import LoginPage, BillsPage -from .pages.login import ManageCGI, HomePage +from .pages.login import ManageCGI, HomePage, PasswordPage from .pages.bills import SubscriptionsPage, BillsApiProPage, BillsApiParPage, ContractsPage from .pages.profile import ProfilePage from weboob.browser.exceptions import ClientError, ServerError @@ -41,6 +41,7 @@ class OrangeBillBrowser(LoginBrowser): home_page = URL('https://businesslounge.orange.fr/$', HomePage) loginpage = URL('https://login.orange.fr/\?service=sosh&return_url=https://www.sosh.fr/', 'https://login.orange.fr/front/login', LoginPage) + password_page = URL(r'https://login.orange.fr/front/password', PasswordPage) contracts = URL('https://espaceclientpro.orange.fr/api/contracts\?page=1&nbcontractsbypage=15', ContractsPage) @@ -80,6 +81,11 @@ def do_login(self): raise BrowserIncorrectPassword(error.response.json()) raise + if self.password_page.is_here(): + error_message = self.page.get_change_password_message() + if error_message: + raise BrowserPasswordExpired(error_message) + def get_nb_remaining_free_sms(self): raise NotImplementedError() diff --git a/modules/orange/pages/login.py b/modules/orange/pages/login.py index bc3365d703..28bc3669cf 100644 --- a/modules/orange/pages/login.py +++ b/modules/orange/pages/login.py @@ -17,8 +17,13 @@ # You should have received a copy of the GNU Lesser General Public License # along with this weboob module. If not, see . +from __future__ import unicode_literals -from weboob.browser.pages import HTMLPage, LoggedPage +import lxml.html as html + +from StringIO import StringIO + +from weboob.browser.pages import HTMLPage, LoggedPage, JsonPage from weboob.tools.json import json from .compat.weboob_browser_filters_standard import CleanText, Format @@ -39,6 +44,31 @@ def login(self, username, password): self.browser.location('https://login.orange.fr/front/password', json=json_data) +class PasswordPage(JsonPage): + def get_change_password_message(self): + if self.doc.get('stage') != 'changePassword': + # when stage is not present everything is okay, and if it's not changePassword we prefer do nothing here + return + + if 'mandatory' not in self.doc['options']: + # maybe there are some cases where it's optional + return + + encoding = self.encoding + if encoding == 'latin-1': + encoding = 'latin1' + if encoding: + encoding = encoding.replace('ISO8859_', 'ISO8859-') + + parser = html.HTMLParser(encoding=encoding) + html_doc = html.parse(StringIO(self.doc['view']), parser) + + # message should be: + # Votre mot de passe actuel n’est pas suffisamment sécurisé et doit être renforcé. + # Veuillez le modifier pour accéder à vos services Orange. + return CleanText('//p[@id="cnMsg"]')(html_doc) + + class ManageCGI(HTMLPage): pass diff --git a/modules/societegenerale/pages/accounts_list.py b/modules/societegenerale/pages/accounts_list.py index e777fa3d2f..1d54e5cf7d 100644 --- a/modules/societegenerale/pages/accounts_list.py +++ b/modules/societegenerale/pages/accounts_list.py @@ -31,13 +31,14 @@ from weboob.capabilities.profile import Person, ProfileMissing from weboob.tools.capabilities.bank.transactions import FrenchTransaction from weboob.tools.capabilities.bank.investments import is_isin_valid, create_french_liquidity +from weboob.tools.compat import urlsplit, urlunsplit, urlencode from weboob.browser.elements import DictElement, ItemElement, TableElement, method, ListElement from weboob.browser.filters.json import Dict from .compat.weboob_browser_filters_standard import ( CleanText, CleanDecimal, Regexp, Currency, Eval, Field, Format, Date, Env, Map, Coalesce, empty, ) -from weboob.browser.filters.html import Link, TableCell +from weboob.browser.filters.html import Link, TableCell, Attr from weboob.browser.pages import HTMLPage, XMLPage, JsonPage, LoggedPage, pagination from weboob.exceptions import BrowserUnavailable, ActionNeeded, NoAccountsException @@ -805,8 +806,40 @@ def obj_date(self): class MarketPage(LoggedPage, HTMLPage): + def get_dropdown_menu(self, account_id): + # Get the 'idCptSelect' in a drop-down menu that corresponds the current account + for cpt in self.doc.xpath('//select[@id="idCptSelect"]//option[@value]'): + if account_id in CleanText('.', replace=[(' ', '')])(cpt): + return Attr('.', 'value')(cpt) + + def get_pages(self): + several_pages = CleanText('//tr[td[contains(@class,"TabTit1l")]][count(td)=3]')(self.doc) + if several_pages: + # "several_pages" value is "1/5" for example + return re.search(r'(\d+)/(\d+)', several_pages).group(1, 2) + + def market_pagination(self, account_id): + # Next page is handled by js. Need to build the right url by changing params in current url + several_pages = self.get_pages() + if several_pages: + current_page, total_pages = map(int, several_pages) + if current_page < total_pages: + params = { + 'action': 11, + 'idCptSelect': self.get_dropdown_menu(account_id), + 'numPage': current_page + 1, + } + url_to_keep = urlsplit(self.browser.url)[:3] + url_to_complete = (urlencode(params), '') # '' is the urlsplit().fragment needed for urlunsplit + next_page_url = urlunsplit(url_to_keep + url_to_complete) + return next_page_url + + @pagination @method class iter_investments(TableElement): + def next_page(self): + return self.page.market_pagination(Env('account')(self).id) + table_xpath = '//tr[td[contains(@class,"TabTit1l")]]/following-sibling::tr//table' head_xpath = table_xpath + '//tr[1]/td' item_xpath = table_xpath + '//tr[position()>1]' diff --git a/modules/spirica/pages.py b/modules/spirica/pages.py index 2e5009aeba..7df59217a3 100644 --- a/modules/spirica/pages.py +++ b/modules/spirica/pages.py @@ -216,10 +216,10 @@ class iter_pm_investment(TableInvestment): class item(ItemInvestment): obj_diff = MyDecimal(TableCell('diff'), default=NotAvailable) - obj_diff_percent = Eval(lambda x: x/100, MyDecimal(TableCell('diff_percent'))) + obj_diff_ratio = Eval(lambda x: x/100, MyDecimal(TableCell('diff_percent'))) obj_unitprice = MyDecimal(TableCell('unitprice')) - def obj_diff_percent(self): + def obj_diff_ratio(self): diff_percent = MyDecimal(TableCell('diff_percent'))(self) if diff_percent: return diff_percent / 100 diff --git a/modules/tapatalk/module.py b/modules/tapatalk/module.py index 3bc73b11ba..b161483f85 100644 --- a/modules/tapatalk/module.py +++ b/modules/tapatalk/module.py @@ -21,7 +21,9 @@ import datetime import requests import re -import xmlrpclib + +from six import text_type +from six.moves import urllib, xmlrpc_client from weboob.tools.backend import Module, BackendConfig from weboob.tools.value import Value, ValueBackendPassword @@ -41,29 +43,41 @@ def __init__(self, uri): def request(self, host, handler, request, verbose): response = self._session.post(self._uri, data = request, headers={"Content-Type": "text/xml; charset=UTF-8"}) - p, u = xmlrpclib.getparser() + p, u = xmlrpc_client.getparser() p.feed(response.content) p.close() response.close() return u.close() -class TapatalkServerProxy(xmlrpclib.ServerProxy): +class TapatalkServerProxy(xmlrpc_client.ServerProxy): def __init__(self, uri): transport = RequestsTransport(uri) - xmlrpclib.ServerProxy.__init__(self, uri, transport) + xmlrpc_client.ServerProxy.__init__(self, uri, transport) def __getattr__(self, name): - method = xmlrpclib.ServerProxy.__getattr__(self, name) + method = xmlrpc_client.ServerProxy.__getattr__(self, name) return self._wrap(method) def _wrap(self, method): def call(*args, **kwargs): res = method(*args, **kwargs) if 'result' in res and not res['result']: - raise TapatalkError(str(res.get('result_text'))) + raise TapatalkError(xmlrpc_str(res['result_text'])) return res return call +def xmlrpc_str(data): + """ + Depending on how the XML-RPC server on the other end has been + implemented, strings can either be str (or unicode in python2) + or xmlrpc_client.Binary. Convert the later case in the former, and + ensure that the result is always a str (even if the input is a number) + """ + if isinstance(data, xmlrpc_client.Binary): + return text_type(data.data, 'utf-8') + else: + return text_type(data) + class TapatalkModule(Module, CapMessages): NAME = 'tapatalk' DESCRIPTION = u'Tapatalk-compatible sites' @@ -72,9 +86,10 @@ class TapatalkModule(Module, CapMessages): LICENSE = 'AGPLv3+' VERSION = '1.5' - CONFIG = BackendConfig(Value('username', label='Username', default=''), - ValueBackendPassword('password', label='Password', default=''), - Value('url', label='Site URL', default="https://support.tapatalk.com/mobiquo/mobiquo.php")) + CONFIG = BackendConfig(Value('username', label='Username', default=''), + ValueBackendPassword('password', label='Password', default=''), + Value('url', label='Site URL', default="https://support.tapatalk.com/mobiquo/mobiquo.php"), + Value('message_url_format', label='Message URL format', default='/index.php?/topic/{thread_id}-{thread_title}#entry{message_id}')) def __init__(self, *args, **kwargs): super(TapatalkModule, self).__init__(*args, **kwargs) @@ -88,45 +103,47 @@ def _conn(self): password = self.config['password'].get() self._xmlrpc_client = TapatalkServerProxy(url) try: - self._xmlrpc_client.login(xmlrpclib.Binary(username), xmlrpclib.Binary(password)) + self._xmlrpc_client.login(username, password) except TapatalkError as e: raise BrowserIncorrectPassword(e.message) return self._xmlrpc_client def _get_time(self, post): if 'post_time' in post: - return dateutil.parser.parse(str(post['post_time'])) + return dateutil.parser.parse(xmlrpc_str(post['post_time'])) else: return datetime.datetime.now() def _format_content(self, post): - msg = unicode(str(post['post_content']), 'utf-8') + msg = xmlrpc_str(post['post_content']) msg = re.sub(r'\[url=(.+?)\](.*?)\[/url\]', r'\2', msg) msg = re.sub(r'\[quote\s?.*\](.*?)\[/quote\]', r'

\1

', msg) msg = re.sub(r'\[img\](.*?)\[/img\]', r'', msg) if post.get('icon_url'): - return u' %s' % (post['icon_url'], msg) + return u' %s' % (xmlrpc_str(post['icon_url']), msg) else: return msg def _process_post(self, thread, post, is_root): + message_id = is_root and u'0' or xmlrpc_str(post['post_id']) + message_title = is_root and thread.title or u'Re: %s' % thread.title + # Tapatalk app seems to have hardcoded this construction... I don't think we can do better :( - url = u'%s/index.php?/topic/%s-%s#entry%s' % ( - self.config["url"].get().rstrip('/'), - thread.id, - re.sub(r'[^a-zA-Z0-9-]', '', re.sub(r'\s+', '-', thread.title)), - post['post_id'] - ) + rel_url = self.config['message_url_format'].get().format( + thread_id = urllib.parse.quote(thread.id.encode('utf-8')), + thread_title = urllib.parse.quote(thread.title.encode('utf-8')), + message_id = urllib.parse.quote(message_id.encode('utf-8')), + message_title = urllib.parse.quote(message_title.encode('utf-8'))) message = Message( - id = is_root and "0" or str(post['post_id']), + id = message_id, thread = thread, - sender = unicode(str(post.get('post_author_name', 'Anonymous')), 'utf-8'), - title = is_root and thread.title or u"Re: %s"%thread.title, - url = url, + sender = xmlrpc_str(post.get('post_author_name', u'Anonymous')), + title = message_title, + url = urllib.parse.urljoin(self.config['url'].get(), rel_url), receivers = None, date = self._get_time(post), - content = self._format_content(post),#bbcode(), + content = self._format_content(post), signature = None, parent = thread.root or None, children = [], @@ -140,7 +157,7 @@ def _process_post(self, thread, post, is_root): # First message in the thread is not the root message, # because we asked only for unread messages. Create a non-loaded root # message to allow monboob to fill correctly the References: header - thread.root = Message(id="0", parent=None, children=[message], thread=thread) + thread.root = Message(id=u'0', parent=None, children=[message], thread=thread) message.parent = thread.root return message @@ -161,7 +178,7 @@ def fill_root(thread, start, count, first_unread): count = 50 topic = self._conn.get_thread_by_unread(thread.id, count) if 'title' in fields: - thread.title = unicode(str(topic['topic_title']), 'utf-8') + thread.title = xmlrpc_str(topic['topic_title']) if 'date' in fields: thread.date = self._get_time(topic) if 'root' in fields: @@ -188,14 +205,14 @@ def browse_forum_mode(forum, prefix, mode): count = 50 while True: if mode: - topics = self._conn.get_topic(forum['forum_id'], start, start+count-1, mode) + topics = self._conn.get_topic(xmlrpc_str(forum['forum_id']), start, start+count-1, mode) else: - topics = self._conn.get_topic(forum['forum_id'], start, start+count-1) + topics = self._conn.get_topic(xmlrpc_str(forum['forum_id']), start, start+count-1) all_ignored = True for topic in topics['topics']: - t = Thread(topic['topic_id']) - t.title = unicode(str(topic['topic_title']), 'utf-8') + t = Thread(xmlrpc_str(topic['topic_id'])) + t.title = xmlrpc_str(topic['topic_title']) t.date = self._get_time(topic) if not unread or topic.get('new_post'): all_ignored = False @@ -211,11 +228,11 @@ def process_forum(forum, prefix): yield thread for child in forum.get('child', []): - for thread in process_forum(child, "%s.%s" % (prefix, child['forum_name'])): + for thread in process_forum(child, u"%s.%s" % (prefix, xmlrpc_str(child['forum_name']))): yield thread for forum in self._conn.get_forum(): - for thread in process_forum(forum, "%s" % forum['forum_name']): + for thread in process_forum(forum, xmlrpc_str(forum['forum_name'])): yield thread def iter_unread_messages(self): diff --git a/modules/wiseed/__init__.py b/modules/wiseed/__init__.py new file mode 100644 index 0000000000..685c111381 --- /dev/null +++ b/modules/wiseed/__init__.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Vincent A +# +# This file is part of a weboob module. +# +# This weboob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This weboob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this weboob module. If not, see . + +from __future__ import unicode_literals + + +from .module import WiseedModule + + +__all__ = ['WiseedModule'] diff --git a/modules/wiseed/browser.py b/modules/wiseed/browser.py new file mode 100644 index 0000000000..70a3d04277 --- /dev/null +++ b/modules/wiseed/browser.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Vincent A +# +# This file is part of a weboob module. +# +# This weboob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This weboob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this weboob module. If not, see . + +from __future__ import unicode_literals + +from weboob.browser.browsers import LoginBrowser, need_login, URL, StatesMixin +from .compat.weboob_capabilities_bank import Account + +from .pages import LoginPage, LandPage, InvestPage + + +# TODO implement documents and profile + +class WiseedBrowser(LoginBrowser, StatesMixin): + BASEURL = 'https://www.wiseed.com' + + login = URL('/fr/connexion', LoginPage) + landing = URL('/fr/projets-en-financement', LandPage) + invests = URL('/fr/compte/portefeuille', InvestPage) + + def do_login(self): + self.login.go() + self.page.do_login(self.username, self.password) + + if self.login.is_here(): + self.page.raise_error() + + assert self.landing.is_here() + + @need_login + def iter_accounts(self): + self.invests.stay_or_go() + + acc = Account() + acc.id = '_wiseed_' + acc.type = Account.TYPE_MARKET + acc.number = self.page.get_user_id() + acc.label = 'WiSEED' + acc.currency = 'EUR' + # unfortunately there's little data + acc.balance = sum(inv.valuation for inv in self.iter_investment()) + + return [acc] + + @need_login + def iter_investment(self): + self.invests.stay_or_go() + + yield self.page.get_liquidities() + + for inv in self.page.iter_funded(): + yield inv + + for inv in self.page.iter_funding(): + yield inv diff --git a/modules/wiseed/compat/__init__.py b/modules/wiseed/compat/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/modules/wiseed/compat/weboob_browser_filters_standard.py b/modules/wiseed/compat/weboob_browser_filters_standard.py new file mode 100644 index 0000000000..f382d162f0 --- /dev/null +++ b/modules/wiseed/compat/weboob_browser_filters_standard.py @@ -0,0 +1,49 @@ + +import weboob.browser.filters.standard as OLD + +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) + + +try: + __all__ = OLD.__all__ +except AttributeError: + pass + + +class Coalesce(MultiFilter): + """ + Returns the first value that is not falsy, + or default if all values are falsy. + """ + @debug() + def filter(self, values): + for value in values: + if value: + return value + return self.default_or_raise(FilterError('All falsy and no default.')) + + +class MapIn(Filter): + """ + Map the pattern of a selected value to another value using a dict. + """ + + def __init__(self, selector, map_dict, default=_NO_DEFAULT): + """ + :param selector: key from `map_dict` to use + """ + super(MapIn, self).__init__(selector, default=default) + self.map_dict = map_dict + + @debug() + def filter(self, txt): + """ + :raises: :class:`ItemNotFound` if key pattern does not exist in dict + """ + for key in self.map_dict: + if key in txt: + return self.map_dict[key] + + return self.default_or_raise(ItemNotFound('Unable to handle %r on %r' % (txt, self.map_dict))) diff --git a/modules/wiseed/compat/weboob_capabilities_bank.py b/modules/wiseed/compat/weboob_capabilities_bank.py new file mode 100644 index 0000000000..4d5ed8d992 --- /dev/null +++ b/modules/wiseed/compat/weboob_capabilities_bank.py @@ -0,0 +1,48 @@ + +import weboob.capabilities.bank as OLD +from weboob.capabilities.base import StringField +from weboob.capabilities.date import DateField + +# can't import *, __all__ is incomplete... +for attr in dir(OLD): + globals()[attr] = getattr(OLD, attr) + + +try: + __all__ = OLD.__all__ +except AttributeError: + pass + + +# can't create a subclass because of CapBank.iter_resources reimplementations: +# modules will import our subclass, but boobank will call iter_resources with the OLD class +Account._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') +Loan._fields['ownership'] = StringField('Relationship between the credentials owner (PSU) and the account') + + +Transaction._fields['bdate'] = DateField('Bank date, when the transaction appear on website (usually extracted from column date)') + + +class RecipientInvalidOTP(AddRecipientError): + code = 'invalidOTP' + + +class AccountOwnership(object): + """ + Relationship between the credentials owner (PSU) and the account + """ + OWNER = u'owner' + """The PSU is the account owner""" + CO_OWNER = u'co-owner' + """The PSU is the account co-owner""" + ATTORNEY = u'attorney' + """The PSU is the account attorney""" + + +AccountOwnerType.ASSOCIATION = u'ASSO' + + +try: + __all__ += ['AccountOwnership', 'RecipientInvalidOTP'] +except NameError: + pass diff --git a/modules/wiseed/module.py b/modules/wiseed/module.py new file mode 100644 index 0000000000..f399e9ecae --- /dev/null +++ b/modules/wiseed/module.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Vincent A +# +# This file is part of a weboob module. +# +# This weboob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This weboob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this weboob module. If not, see . + +from __future__ import unicode_literals + +from weboob.tools.backend import Module, BackendConfig +from weboob.tools.value import ValueBackendPassword +from .compat.weboob_capabilities_bank import CapBankWealth + +from .browser import WiseedBrowser + + +__all__ = ['WiseedModule'] + + +class WiseedModule(Module, CapBankWealth): + NAME = 'wiseed' + DESCRIPTION = 'WiSEED' + MAINTAINER = 'Vincent A' + EMAIL = 'dev@indigo.re' + LICENSE = 'LGPLv3+' + VERSION = '1.5' + + BROWSER = WiseedBrowser + + CONFIG = BackendConfig( + ValueBackendPassword('login', label='E-mail', regexp='.*@.*', masked=False), + ValueBackendPassword('password', label='Mot de passe'), + ) + + def create_default_browser(self): + return self.create_browser( + self.config['login'].get(), + self.config['password'].get() + ) + + def iter_accounts(self): + return self.browser.iter_accounts() + + def iter_investment(self, account): + return self.browser.iter_investment() diff --git a/modules/wiseed/pages.py b/modules/wiseed/pages.py new file mode 100644 index 0000000000..7d83008dfa --- /dev/null +++ b/modules/wiseed/pages.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Vincent A +# +# This file is part of a weboob module. +# +# This weboob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This weboob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this weboob module. If not, see . + +from __future__ import unicode_literals + +from weboob.browser.pages import LoggedPage, HTMLPage +from weboob.browser.filters.html import TableCell +from .compat.weboob_browser_filters_standard import CleanText, CleanDecimal, Regexp +from weboob.browser.elements import method, ItemElement, TableElement +from weboob.exceptions import BrowserIncorrectPassword +from .compat.weboob_capabilities_bank import Investment +from weboob.tools.capabilities.bank.investments import create_french_liquidity + + +class LoginPage(HTMLPage): + def do_login(self, login, password): + form = self.get_form(id='recaptcha') # wtf + form['emailConnexion'] = login + form['motDePasseConnexion'] = password + form.submit() + + def raise_error(self): + msg = CleanText('//div[has-class("alert-danger")]')(self.doc) + if 'Email ou mot de passe invalide' in msg: + raise BrowserIncorrectPassword(msg) + assert False, 'unhandled message %r' % msg + + +class LandPage(LoggedPage, HTMLPage): + pass + + +class InvestPage(LoggedPage, HTMLPage): + def get_user_id(self): + return Regexp( + CleanText('//span[contains(text(), "ID Client")]'), + r'ID Client : (\d+)' + )(self.doc) + + def get_liquidities(self): + value = CleanDecimal.French(CleanText('//a[starts-with(text(),"Compte de paiement")]'))(self.doc) + return create_french_liquidity(value) + + @method + class iter_funded(TableElement): + item_xpath = '//table[@id="portefeuilleAction"]/tbody/tr' + + head_xpath = '//table[@id="portefeuilleAction"]/thead//th' + col_bought = 'Vous avez investi' + col_label = 'Investissement dans' + col_valuation = 'Valeur estimée à date' + col_diff_ratio = 'Coef. de performance intermediaire' + + class item(ItemElement): + klass = Investment + + obj_label = CleanText(TableCell('label')) + + # text is "0000000000000100 100,00 €", wtf + obj_valuation = CleanDecimal.SI( + Regexp(CleanText(TableCell('valuation')), r'^000(\d+)\b') + ) + + obj_diff_ratio = CleanDecimal.SI( + Regexp(CleanText(TableCell('diff_ratio')), r'^000(\d+)\b') + ) + + # unitprice and unitvalue are on a dedicated page, let's forget it + + @method + class iter_funding(TableElement): + item_xpath = '//table[has-class("portefeuille-liste") and not(@id)]/tbody/tr' + + head_xpath = '//table[has-class("portefeuille-liste") and not(@id)]/thead//th' + col_label = 'Opération / Cible' + col_details = 'Détails' + + class item(ItemElement): + klass = Investment + + obj_label = CleanText(TableCell('label')) + obj_valuation = CleanDecimal.French(Regexp( + CleanText(TableCell('details')), + r'^(.*?) €', # can be 100,00 € + Frais de 0,90 € + )) -- GitLab