diff --git a/modules/americanexpress/compat/__init__.py b/modules/americanexpress/compat/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/modules/bienici/__init__.py b/modules/bienici/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2d61180cdff4f797b09c1a66a62b8d45ebc3ca3d --- /dev/null +++ b/modules/bienici/__init__.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Antoine BOSSY +# +# This file is part of woob. +# +# woob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# woob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with woob. If not, see . + +from __future__ import unicode_literals + + +from .module import BieniciModule + + +__all__ = ['BieniciModule'] diff --git a/modules/bienici/browser.py b/modules/bienici/browser.py new file mode 100644 index 0000000000000000000000000000000000000000..9f018f774cef37d6f7dbc100d57e11f26aeb3a66 --- /dev/null +++ b/modules/bienici/browser.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Antoine BOSSY +# +# This file is part of woob. +# +# woob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# woob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with woob. If not, see . + +from __future__ import unicode_literals + + +from woob.browser import PagesBrowser, URL +from woob.tools.json import json + +from woob.capabilities.housing import POSTS_TYPES, HOUSE_TYPES +from .pages import Cities, ResultsPage, HousingPage + + +TRANSACTION_TYPE = { + POSTS_TYPES.SALE: 'buy', + POSTS_TYPES.RENT: 'rent', + POSTS_TYPES.SALE: 'buy', + POSTS_TYPES.FURNISHED_RENT: 'rent', + POSTS_TYPES.SHARING: 'rent' +} + + +HOUSE_TYPES = { + HOUSE_TYPES.APART: ['flat'], + HOUSE_TYPES.HOUSE: ['house'], + HOUSE_TYPES.PARKING: ['parking'], + HOUSE_TYPES.LAND: ['terrain'], + HOUSE_TYPES.OTHER: ['others', 'loft', 'shop', 'building', 'castle', 'premises', 'office', 'townhouse'], + HOUSE_TYPES.UNKNOWN: [] +} + + +class BieniciBrowser(PagesBrowser): + BASEURL = 'https://www.bienici.com' + + cities = URL(r'https://res.bienici.com/suggest.json\?q=(?P.+)', Cities) + results = URL(r'/realEstateAds.json\?filters=(?P.+)', ResultsPage) + housing = URL(r'/realEstateAds-one.json\?filters=(?P.*)&onlyRealEstateAd=(?P.*)', HousingPage) + + def get_cities(self, pattern): + return self.cities.go(zipcode=pattern).get_city() + + def search_housing(self, query): + filters = { + 'size': 100, + 'page': 1, + 'resultsPerPage': 24, + 'maxAuthorizedResults': 2400, + 'sortBy': "relevance", + 'sortOrder': "desc", + 'onTheMarket': [True], + 'showAllModels': False, + "zoneIdsByTypes": { + 'zoneIds': [] + }, + 'propertyType': [] + } + + dict_query = query.to_dict() + if dict_query['area_min']: + filters['minArea'] = dict_query['area_min'] + + if dict_query['area_max']: + filters['maxArea'] = dict_query['area_max'] + + if dict_query['cost_min']: + filters['minPrice'] = dict_query['cost_min'] + + if dict_query['cost_max']: + filters['maxPrice'] = dict_query['cost_max'] + + filters['filterType'] = TRANSACTION_TYPE[dict_query['type']] + + for housing_type in dict_query['house_types']: + filters['propertyType'] += HOUSE_TYPES[housing_type] + + for city in dict_query['cities']: + filters['zoneIdsByTypes']['zoneIds'].append(city.id) + + return self.results.go(filters=json.dumps(filters)).get_housings() + + def get_housing(self, housing_id): + # This is to serialize correctly the JSON, and match the URL easier. + filters = { + 'onTheMarket': [True] + } + return self.housing.go(housing_id=housing_id, filters=json.dumps(filters)).get_housing() diff --git a/modules/bienici/module.py b/modules/bienici/module.py new file mode 100644 index 0000000000000000000000000000000000000000..bf66b59494ad5a9ad21d6cf33ccb2e973e5895e1 --- /dev/null +++ b/modules/bienici/module.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Antoine BOSSY +# +# This file is part of woob. +# +# woob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# woob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with woob. If not, see . + +from __future__ import unicode_literals + + +from woob.tools.backend import Module +from woob.capabilities.housing import CapHousing, Housing, HousingPhoto + +from .browser import BieniciBrowser + + +__all__ = ['BieniciModule'] + + +class BieniciModule(Module, CapHousing): + NAME = 'bienici' + DESCRIPTION = 'bienici website' + MAINTAINER = 'Antoine BOSSY' + EMAIL = 'mail+github@abossy.fr' + LICENSE = 'AGPLv3+' + VERSION = '3.0' + + BROWSER = BieniciBrowser + + def get_housing(self, id): + """ + Get an housing from an ID. + + :param housing: ID of the housing + :type housing: str + :rtype: :class:`Housing` or None if not found. + """ + return self.browser.get_housing(id) + + def search_city(self, pattern): + """ + Search a city from a pattern. + + :param pattern: pattern to search + :type pattern: str + :rtype: iter[:class:`City`] + """ + return self.browser.get_cities(pattern) + + def search_housings(self, query): + """ + Search housings. + + :param query: search query + :type query: :class:`Query` + :rtype: iter[:class:`Housing`] + """ + return self.browser.search_housing(query) + + def fill_photo(self, photo, fields): + """ + Fills the photo. + """ + if 'data' in fields and photo.url and not photo.data: + photo.data = self.browser.open(photo.url).content + return photo + + def fill_housing(self, housing, fields): + """ + Fills the housing. + """ + return self.get_housing(housing.id) + + OBJECTS = {HousingPhoto: fill_photo, Housing: fill_housing} diff --git a/modules/bienici/pages.py b/modules/bienici/pages.py new file mode 100644 index 0000000000000000000000000000000000000000..9ca2e0ba34977b242146768f143d8860d17643ef --- /dev/null +++ b/modules/bienici/pages.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2018 Antoine BOSSY +# +# This file is part of woob. +# +# woob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# woob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with woob. If not, see . + +from __future__ import unicode_literals + + +from woob.browser.pages import JsonPage +from woob.browser.elements import ItemElement, DictElement, method +from woob.browser.filters.json import Dict, ItemNotFound +from woob.capabilities.base import NotAvailable +from woob.browser.filters.standard import CleanText, Date, CleanDecimal +from woob.capabilities.housing import City, Housing, HousingPhoto, ENERGY_CLASS + + +class Cities(JsonPage): + @method + class get_city(DictElement): + + class item(ItemElement): + klass = City + + obj_id = Dict('zoneIds/0') + obj_name = CleanText(Dict('name')) + + +class MyItemElement(ItemElement): + klass = Housing + + def condition(self): + return not Dict('userRelativeData/isAdModifier')(self) + + obj_id = Dict('id') + obj_title = Dict('title') + obj_area = Dict('surfaceArea') + obj_cost = Dict('price') + + def obj_price_per_meter(self): + try: + return Dict('pricePerSquareMeter')(self) + except ItemNotFound: + return NotAvailable + + obj_currency = 'EUR' + obj_date = Date(Dict('publicationDate')) + obj_location = CleanDecimal(Dict('postalCode')) + obj_text = Dict('description', '') + + def obj_photos(self): + return [HousingPhoto(photo['url']) for photo in Dict('photos')(self)] + + obj_rooms = Dict('roomsQuantity', 0) + obj_bedrooms = Dict('bedroomsQuantity', 0) + + def obj_DPE(self): + try: + return ENERGY_CLASS[Dict('energyClassification')(self)] + except (KeyError, ItemNotFound): + return NotAvailable + + def obj_GES(self): + try: + return ENERGY_CLASS[Dict('greenhouseGazClassification')(self)] + except (KeyError, ItemNotFound): + return NotAvailable + + +class ResultsPage(JsonPage): + @method + class get_housings(DictElement): + item_xpath = 'realEstateAds' + + class item(MyItemElement): + pass + + +class HousingPage(JsonPage): + @method + class get_housing(MyItemElement): + pass diff --git a/modules/cmso/par/compat/__init__.py b/modules/cmso/par/compat/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/modules/creditmutuel/browser.py b/modules/creditmutuel/browser.py index 7f583a3a3856c735882933b8455a4db1c75bb3f1..1b0cb6d765fbf00d9d7c392e9f86b8ed4d92520f 100644 --- a/modules/creditmutuel/browser.py +++ b/modules/creditmutuel/browser.py @@ -41,7 +41,7 @@ from woob.capabilities.bank import ( Account, AddRecipientStep, Recipient, AccountOwnership, AddRecipientTimeout, TransferStep, TransferBankError, - AddRecipientBankError, + AddRecipientBankError, TransferTimeout, ) from woob.tools.capabilities.bank.investments import create_french_liquidity from woob.capabilities import NotAvailable @@ -226,7 +226,7 @@ def __init__(self, config, *args, **kwargs): 'currentSubBank', 'logged', 'is_new_website', 'need_clear_storage', 'recipient_form', 'twofa_auth_state', 'polling_data', 'otp_data', - 'key_form', + 'key_form', 'transfer_code_form', ) self.twofa_auth_state = {} self.polling_data = {} @@ -234,6 +234,7 @@ def __init__(self, config, *args, **kwargs): self.keep_session = None self.recipient_form = None self.key_form = None + self.transfer_code_form = None self.AUTHENTICATION_METHODS = { 'resume': self.handle_polling, @@ -264,6 +265,7 @@ def load_state(self, state): or state.get('recipient_form') or state.get('otp_data') or state.get('key_form') + or state.get('transfer_code_form') ): # can't start on an url in the middle of a validation process # or server will cancel it and launch another one @@ -343,6 +345,8 @@ def check_otp_blocked(self): raise BrowserUnavailable(error_msg) def handle_sms(self): + if not self.otp_data or 'final_url_params' not in self.otp_data: + raise BrowserIncorrectPassword("Le code de confirmation envoyé par SMS n'est plus utilisable") self.otp_data['final_url_params']['otp_password'] = self.code self.finalize_twofa(self.otp_data) @@ -955,6 +959,8 @@ def iter_recipients(self, origin_account): def continue_transfer(self, transfer, **params): if 'Clé' in params: + if not self.key_form: + raise TransferTimeout(message="La validation du transfert par carte de clés personnelles a expiré") url = self.key_form.pop('url') self.format_personal_key_card_form(params['Clé']) self.location(url, data=self.key_form) @@ -970,9 +976,29 @@ def continue_transfer(self, transfer, **params): if self.login.is_here(): # User took too much time to input the personal key. - raise TransferBankError(message='La validation du transfert par carte de clés personnelles a expiré') + raise TransferBankError(message="La validation du transfert par carte de clés personnelles a expiré") + + transfer_id = self.page.get_transfer_webid() + if transfer_id and (empty(transfer.id) or transfer.id != transfer_id): + transfer.id = self.page.get_transfer_webid() + + elif 'code' in params: + code_form = self.transfer_code_form + if not code_form: + raise TransferTimeout(message="Le code de confirmation envoyé par SMS n'est plus utilisable") + # Specific field of the confirmation page + code_form['Bool:data_input_confirmationDoublon'] = 'true' + self.send_sms(code_form, params['code']) + self.transfer_code_form = None + + # OTP is expired after 15', we end up on login page + if self.login.is_here(): + raise TransferBankError(message="Le code de confirmation envoyé par SMS n'est plus utilisable") + + transfer_id = self.page.get_transfer_webid() + if transfer_id and (empty(transfer.id) or transfer.id != transfer_id): + transfer.id = self.page.get_transfer_webid() - transfer.id = self.page.get_transfer_webid() elif 'resume' in params: self.poll_decoupled(self.polling_data['polling_id']) @@ -980,11 +1006,41 @@ def continue_transfer(self, transfer, **params): self.polling_data['final_url'], data=self.polling_data['final_url_params'], ) - # Dont set `self.polling_data = None` yet because we need to know in - # execute_transfer if we just did an app validation. + self.polling_data = None - # At this point the app validation has already been sent (after validating the - # personal key card code). + transfer = self.check_and_initiate_transfer_otp(transfer) + + return transfer + + def check_and_initiate_transfer_otp(self, transfer, account=None, recipient=None): + if self.page.needs_personal_key_card_validation(): + self.location(self.page.get_card_key_validation_link()) + error = self.page.get_personal_keys_error() + if error: + raise TransferBankError(message=error) + + self.key_form = self.page.get_personal_key_card_code_form() + raise TransferStep( + transfer, + Value('Clé', label=self.page.get_question()) + ) + + if account and transfer: + transfer = self.page.handle_response_create_transfer( + account, recipient, transfer.amount, transfer.label, transfer.exec_date + ) + else: + transfer = self.page.handle_response_reuse_transfer(transfer) + + if self.page.needs_otp_validation(): + self.transfer_code_form = self.page.get_transfer_code_form() + raise TransferStep( + transfer, + Value('code', label='Veuillez saisir le code reçu par sms pour confirmer votre opération') + ) + + # The app validation, if needed, could have already been started + # (for example, after validating the personal key card code). msg = self.page.get_validation_msg() if msg: self.polling_data = self.page.get_polling_data(form_xpath='//form[contains(@action, "virements")]') @@ -1013,35 +1069,17 @@ def init_transfer(self, transfer, account, recipient): self.page.prepare_transfer(account, recipient, transfer.amount, transfer.label, transfer.exec_date) - if self.page.needs_personal_key_card_validation(): - self.location(self.page.get_card_key_validation_link()) - error = self.page.get_personal_keys_error() - if error: - raise TransferBankError(message=error) - - self.key_form = self.page.get_personal_key_card_code_form() - raise TransferStep(transfer, Value('Clé', label=self.page.get_question())) - elif self.page.needs_otp_validation(): - raise AuthMethodNotImplemented("La validation des transferts avec un code sms n'est pas encore disponible.") - - msg = self.page.get_validation_msg() - if msg: - self.polling_data = self.page.get_polling_data(form_xpath='//form[contains(@action, "virements")]') - assert self.polling_data, "Can't proceed without polling data" - raise AppValidation( - resource=transfer, - message=msg, - ) - - return self.page.handle_response(account, recipient, transfer.amount, transfer.label, transfer.exec_date) + new_transfer = self.check_and_initiate_transfer_otp(transfer, account, recipient) + return new_transfer @need_login def execute_transfer(self, transfer, **params): - if self.polling_data: - # If we just did a transfer to a new recipient the transfer has already - # been confirmed with the app validation. - self.polling_data = None - else: + # If we just did a transfer to a new recipient the transfer has already + # been confirmed because of the app validation or the sms otp + # Otherwise, do the confirmation when still needed + if self.page.doc.xpath( + '//form[@id="P:F"]//input[@type="submit" and contains(@value, "Confirmer")]' + ): form = self.page.get_form(id='P:F', submit='//input[@type="submit" and contains(@value, "Confirmer")]') # For the moment, don't ask the user if he confirms the duplicate. form['Bool:data_input_confirmationDoublon'] = 'true' @@ -1104,6 +1142,8 @@ def format_personal_key_card_form(self, key): def continue_new_recipient(self, recipient, **params): if 'Clé' in params: + if not self.key_form: + raise AddRecipientTimeout(message="La validation par carte de clés personnelles a expiré") url = self.key_form.pop('url') self.format_personal_key_card_form(params['Clé']) self.location(url, data=self.key_form) @@ -1119,37 +1159,43 @@ def continue_new_recipient(self, recipient, **params): if self.login.is_here(): # User took too much time to input the personal key. - raise AddRecipientTimeout() + raise AddRecipientBankError(message="La validation par carte de clés personnelles a expiré") self.page.add_recipient(recipient) if self.page.bic_needed(): self.page.ask_bic(self.get_recipient_object(recipient)) self.page.ask_auth_validation(self.get_recipient_object(recipient)) - def send_sms(self, sms): - url = self.recipient_form.pop('url') - self.recipient_form['otp_password'] = sms - self.recipient_form['_FID_DoConfirm.x'] = '1' - self.recipient_form['_FID_DoConfirm.y'] = '1' - self.recipient_form['global_backup_hidden_key'] = '' - self.location(url, data=self.recipient_form) + def send_sms(self, form, sms): + url = form.pop('url') + form['otp_password'] = sms + form['_FID_DoConfirm.x'] = '1' + form['_FID_DoConfirm.y'] = '1' + form['global_backup_hidden_key'] = '' + self.location(url, data=form) - def send_decoupled(self): - url = self.recipient_form.pop('url') - transactionId = self.recipient_form.pop('transactionId') + def send_decoupled(self, form): + url = form.pop('url') + transactionId = form.pop('transactionId') self.poll_decoupled(transactionId) - self.recipient_form['_FID_DoConfirm.x'] = '1' - self.recipient_form['_FID_DoConfirm.y'] = '1' - self.recipient_form['global_backup_hidden_key'] = '' - self.location(url, data=self.recipient_form) + form['_FID_DoConfirm.x'] = '1' + form['_FID_DoConfirm.y'] = '1' + form['global_backup_hidden_key'] = '' + self.location(url, data=form) def end_new_recipient_with_auth_validation(self, recipient, **params): if 'code' in params: - self.send_sms(params['code']) + if not self.recipient_form: + raise AddRecipientTimeout(message="Le code de confirmation envoyé par SMS n'est plus utilisable") + self.send_sms(self.recipient_form, params['code']) + elif 'resume' in params: - self.send_decoupled() + if not self.recipient_form: + raise AddRecipientTimeout(message="Le demande de confirmation a expiré") + self.send_decoupled(self.recipient_form) + self.recipient_form = None self.page = None return self.get_recipient_object(recipient) @@ -1216,7 +1262,19 @@ def iter_subscriptions(self): def iter_documents(self, subscription): if self.currentSubBank is None: self.getCurrentSubBank() + + self.iban.go(subbank=self.currentSubBank) + iban_document = self.page.get_iban_document(subscription) + if iban_document: + yield iban_document + self.subscription.go(subbank=self.currentSubBank, params={'typ': 'doc'}) + + access_not_allowed_msg = "Vous ne disposez pas des droits nécessaires pour accéder à cette partie de l'application." + if access_not_allowed_msg in self.page.error_msg(): + self.logger.warning("Bank user account has insufficient right to access the documents page") + return + link_to_bank_statements = self.page.get_link_to_bank_statements() self.location(link_to_bank_statements) @@ -1241,12 +1299,6 @@ def iter_documents(self, subscription): if self.page.is_last_page(): break - self.iban.go(subbank=self.currentSubBank) - iban_document = self.page.get_iban_document(subscription) - if iban_document: - yield iban_document - - @need_login def iter_emitters(self): diff --git a/modules/creditmutuel/module.py b/modules/creditmutuel/module.py index 44588e3ef79e5e6c9a905771c6bc47221ef41723..9e790a4958a1d48244f3bd694650d7bf58e0157c 100644 --- a/modules/creditmutuel/module.py +++ b/modules/creditmutuel/module.py @@ -117,7 +117,7 @@ def new_recipient(self, recipient, **params): return self.browser.new_recipient(recipient, **params) def init_transfer(self, transfer, **params): - if {'Clé', 'resume'} & set(params.keys()): + if {'Clé', 'resume', 'code'} & set(params.keys()): return self.browser.continue_transfer(transfer, **params) # There is a check on the website, transfer can't be done with too long reason. diff --git a/modules/creditmutuel/pages.py b/modules/creditmutuel/pages.py index b1a08e9002d4d7eca5453f1514259ce49e0bc368..9e32085dfdf5b5e86824a60240043b5c70e265a1 100644 --- a/modules/creditmutuel/pages.py +++ b/modules/creditmutuel/pages.py @@ -859,8 +859,14 @@ class Transaction(FrenchTransaction): (re.compile(r'^RETRAIT DAB (?P
\d{2})(?P\d{2}) (?P.*) CARTE [\*\d]+'), FrenchTransaction.TYPE_WITHDRAWAL), (re.compile(r'^(?P
\d{2})/(?P\d{2})/(?P\d{4}) RETRAIT DAB (?P.*)'), FrenchTransaction.TYPE_WITHDRAWAL), (re.compile(r'^CHEQUE( (?P.*))?$'), FrenchTransaction.TYPE_CHECK), + (re.compile(r'^FACTURE SGT.*'), FrenchTransaction.TYPE_BANK), (re.compile(r'^(F )?COTIS\.? (?P.*)'), FrenchTransaction.TYPE_BANK), + (re.compile(r'^(F )?RETRO\.? (?P.*)'), FrenchTransaction.TYPE_BANK), + (re.compile(r'^EXT.AGIOS'), FrenchTransaction.TYPE_BANK), + (re.compile(r'^(?P(?PINTERETS).*)'), FrenchTransaction.TYPE_BANK), + (re.compile(r'(?PPREL\.(SOC|OBL).*)'), FrenchTransaction.TYPE_BANK), (re.compile(r'^(REMISE|REM CHQ) (?P.*)'), FrenchTransaction.TYPE_DEPOSIT), + (re.compile(r'^VERSEMT PERIOD'), FrenchTransaction.TYPE_DEPOSIT), (re.compile(r'^(?P(ÉCHÉANCE|Echéance)).*'), FrenchTransaction.TYPE_LOAN_PAYMENT), ] @@ -873,7 +879,7 @@ def go_on_history_tab(self): # Maybe obsolete form = self.get_form(id='I1:fm') except FormNotFound: - form = self.get_form(id='I1:P:F') + form = self.get_form(id='I1:P1:F') form['_FID_DoShowListView'] = '' form.submit() @@ -1780,7 +1786,7 @@ def fill_iban(self, accounts): def get_iban_document(self, subscription): for raw in self.doc.xpath('//table[has-class("liste")]//tbody//tr[not(@class)]'): - if raw.xpath('.//td[1]')[0].text_content().startswith(subscription.label.upper()): + if raw.xpath('.//td[1]')[0].text_content().upper().startswith(subscription.label.upper()): iban_document = Document() iban_document.label = 'IBAN {}'.format(subscription.label) iban_document.url = Link(raw.xpath('.//a'))(self.doc) @@ -2061,10 +2067,7 @@ def obj_id(self): return '%s%s' % (bank_info, partial_number) -class InternalTransferPage(LoggedPage, HTMLPage, AppValidationPage): - RECIPIENT_STRING = 'data_input_indiceCompteACrediter' - READY_FOR_TRANSFER_MSG = 'Confirmer un virement entre vos comptes' - SUMMARY_RECIPIENT_TITLE = 'Compte à créditer' +class TransferPageCommon(LoggedPage, HTMLPage, AppValidationPage): IS_PRO_PAGE = False def needs_personal_key_card_validation(self): @@ -2073,6 +2076,12 @@ def needs_personal_key_card_validation(self): def needs_otp_validation(self): return bool(self.doc.xpath('//input[@name="otp_password"]')) + def get_transfer_code_form(self): + form = self.get_form(id='P:F') + code_form = dict(form.items()) + code_form['url'] = form.url + return code_form + def can_transfer_pro(self, origin_account): for li in self.doc.xpath('//ul[@id="idDetailsListCptDebiterVertical:ul"]//ul/li'): if CleanText(li.xpath('.//span[@class="_c1 doux _c1"]'), replace=[(' ', '')])(self) in origin_account: @@ -2087,27 +2096,6 @@ def can_transfer(self, origin_account): if CleanText(li.xpath('.//span[@class="_c1 doux _c1"]'), replace=[(' ', '')])(self) in origin_account: return True - @method - class iter_recipients(ListElement): - def parse(self, el): - if self.page.IS_PRO_PAGE: - self.item_xpath = '//ul[@id="idDetailsListCptCrediterVertical:ul"]//ul/li' - else: - self.item_xpath = '//ul[@id="idDetailsListCptCrediterHorizontal:ul"]//li[@role="radio"]' - - class item(MyRecipient): - condition = lambda self: Field('id')(self) not in self.env['origin_account'].id - - obj_bank_name = 'Crédit Mutuel' - obj_label = CleanText('.//div[@role="presentation"]/em | .//div[not(@id) and not(@role)]') - obj_id = CleanText('.//span[@class="_c1 doux _c1"]', replace=[(' ', '')]) - obj_category = 'Interne' - - def obj_iban(self): - l = [a for a in self.page.browser.get_accounts_list() - if Field('id')(self) in a.id and empty(a.valuation_diff)] - assert len(l) == 1 - return l[0].iban def get_account_index(self, direction, account): for div in self.doc.xpath('//*[has-class("dw_dli_contents")]'): @@ -2127,8 +2115,13 @@ def get_to_account_index(self, account): return self.get_account_index(self.RECIPIENT_STRING, account) def get_transfer_form(self): - # internal and external transfer form are differents - return self.get_form(id='P:F', submit='//input[@type="submit" and contains(@value, "Valider")]') + # internal and external transfer forms are differents ("P:F" vs "P2:F") + # but also the form id is sometimes changed from + # "P1:F" to "P2:F" and from "P2:F" to "P3:F" + # search for other info to get transfer form + transfer_form_xpath = '//form[contains(@action, "fr/banque/virements/vplw") and @method="post"]' + transfer_form_submit_xpath = '//input[@type="submit" and contains(@value, "Valider")]' + return self.get_form(xpath=transfer_form_xpath, submit=transfer_form_submit_xpath) def prepare_transfer(self, account, to, amount, reason, exec_date): form = self.get_transfer_form() @@ -2201,7 +2194,19 @@ def get_transfer_webid(self): parsed = urlparse(self.url) return parse_qs(parsed.query)['_saguid'][0] - def handle_response(self, account, recipient, amount, reason, exec_date): + def handle_response_reuse_transfer(self, transfer): + self.check_errors() + + exec_date, r_amount, currency = self.check_data_consistency( + transfer.account_id, transfer.recipient_id, transfer.amount, transfer.label) + + transfer.exec_date = exec_date + transfer.amount = r_amount + transfer.currency = currency + + return transfer + + def handle_response_create_transfer(self, account, recipient, amount, reason, exec_date): self.check_errors() self.check_success() @@ -2258,7 +2263,35 @@ class iter_emitters(ListEmitters): pass -class ExternalTransferPage(InternalTransferPage): +class InternalTransferPage(TransferPageCommon): + RECIPIENT_STRING = 'data_input_indiceCompteACrediter' + READY_FOR_TRANSFER_MSG = 'Confirmer un virement entre vos comptes' + SUMMARY_RECIPIENT_TITLE = 'Compte à créditer' + + @method + class iter_recipients(ListElement): + def parse(self, el): + if self.page.IS_PRO_PAGE: + self.item_xpath = '//ul[@id="idDetailsListCptCrediterVertical:ul"]//ul/li' + else: + self.item_xpath = '//ul[@id="idDetailsListCptCrediterHorizontal:ul"]//li[@role="radio"]' + + class item(MyRecipient): + condition = lambda self: Field('id')(self) not in self.env['origin_account'].id + + obj_bank_name = 'Crédit Mutuel' + obj_label = CleanText('.//div[@role="presentation"]/em | .//div[not(@id) and not(@role)]') + obj_id = CleanText('.//span[@class="_c1 doux _c1"]', replace=[(' ', '')]) + obj_category = 'Interne' + + def obj_iban(self): + l = [a for a in self.page.browser.get_accounts_list() + if Field('id')(self) in a.id and empty(a.valuation_diff)] + assert len(l) == 1 + return l[0].iban + + +class ExternalTransferPage(TransferPageCommon): RECIPIENT_STRING = 'data_input_indiceBeneficiaire' READY_FOR_TRANSFER_MSG = 'Confirmer un virement vers un bénéficiaire enregistré' SUMMARY_RECIPIENT_TITLE = 'Bénéficiaire à créditer' @@ -2316,17 +2349,6 @@ def obj_iban(self): def parse(self, el): self.env['origin_account']._external_recipients.add(Field('id')(self)) - def get_transfer_form(self): - # transfer form id change from "P1:F" to "P2:F" and from "P2:F" to "P3:F" - # search for other info to get transfer form - transfer_form_xpath = '//form[contains(@action, "fr/banque/virements/vplw") and @method="post"]' - transfer_form_submit_xpath = '//input[@type="submit" and contains(@value, "Valider")]' - return self.get_form(xpath=transfer_form_xpath, submit=transfer_form_submit_xpath) - - @method - class iter_emitters(ListEmitters): - pass - class VerifCodePage(LoggedPage, HTMLPage): HASHES = { @@ -2496,7 +2518,12 @@ def go_to_add(self): form.submit() def get_add_recipient_form(self, recipient): - form = self.get_form(id='P:F') + # form id change from "P:F" to "P2:F" and from "P2:F" to "P3:F" + # search for other info to get transfer form + rcpt_form_xpath = '//form[contains(@action, "fr/banque/virements/vplw") and @method="post"]' + rcpt_form_submit_xpath = '//input[@type="submit" and contains(@value, "Valider")]' + form = self.get_form(xpath=rcpt_form_xpath, submit=rcpt_form_submit_xpath) + del form['_FID_GoI%5fRechercheBIC'] form['[t:dbt%3astring;x(70)]data_input_nom'] = recipient.label form['[t:dbt%3astring;x(34)]data_input_IBANBBAN'] = recipient.iban @@ -2622,6 +2649,9 @@ class RevolvingLoanDetails(LoggedPage, HTMLPage): class SubscriptionPage(LoggedPage, HTMLPage): + def error_msg(self): + return CleanText('//div[@id="errmsg"]/p')(self.doc) + def get_link_to_bank_statements(self): return Link('//a[@id="C:R1:N"]')(self.doc) diff --git a/modules/lesterrains/__init__.py b/modules/lesterrains/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..70483b1d64da074c33eaf39c99d546e043393f74 --- /dev/null +++ b/modules/lesterrains/__init__.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Guntra +# +# This file is part of a woob module. +# +# This woob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This woob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this woob module. If not, see . + +from __future__ import unicode_literals +from .module import LesterrainsModule + + +__all__ = ['LesterrainsModule'] diff --git a/modules/lesterrains/browser.py b/modules/lesterrains/browser.py new file mode 100644 index 0000000000000000000000000000000000000000..daf1db8efd8ba5cf613624eaeb16572efa5d33d8 --- /dev/null +++ b/modules/lesterrains/browser.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Guntra +# +# This file is part of a woob module. +# +# This woob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This woob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this woob module. If not, see . + +from __future__ import unicode_literals + +from woob.browser import PagesBrowser, URL +from woob.capabilities.housing import POSTS_TYPES, HOUSE_TYPES +from woob.tools.compat import urlencode + +from .pages import CitiesPage, SearchPage, HousingPage + + +class LesterrainsBrowser(PagesBrowser): + + BASEURL = 'http://www.les-terrains.com' + TYPES = { + POSTS_TYPES.SALE: 'vente' + } + RET = { + HOUSE_TYPES.LAND: 'Terrain seul' + } + cities = URL(r'/api/get-search.php\?q=(?P.*)', CitiesPage) + search = URL(r'/index.php\?mode_aff=liste&ongletAccueil=Terrains&(?P.*)&distance=0', SearchPage) + housing = URL( + r'/index.php\?page=terrains&mode_aff=un_terrain&idter=(?P<_id>\d+).*', + r'/index.php\?page=terrains&mode_aff=maisonterrain&idter=(?P<_id>\d+).*', + HousingPage + ) + + def get_cities(self, pattern): + return self.cities.open(city=pattern).get_cities() + + def search_housings(self, cities, area_min, area_max, cost_min, cost_max): + + def _get_departement(city): + return city.split(';')[0][:2] + + def _get_ville(city): + return city.split(';')[1] + + for city in cities: + query = urlencode({ + "departement": _get_departement(city), + "ville": _get_ville(city), + "prixMin": cost_min or '', + "prixMax": cost_max or '', + "surfMin": area_min or '', + "surfMax": area_max or '', + }) + for house in self.search.go(query=query).iter_housings(): + yield house + + def get_housing(self, _id, housing=None): + return self.housing.go(_id=_id).get_housing(obj=housing) diff --git a/modules/lesterrains/module.py b/modules/lesterrains/module.py new file mode 100644 index 0000000000000000000000000000000000000000..93e11806837ea7198d37175618f8fde08e9c50e7 --- /dev/null +++ b/modules/lesterrains/module.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Guntra +# +# This file is part of a woob module. +# +# This woob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This woob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this woob module. If not, see . + +from __future__ import unicode_literals +from woob.tools.backend import Module +from woob.capabilities.housing import CapHousing +from .browser import LesterrainsBrowser + + +# Some remarks: +# - post type is hardcoded as POSTS_TYPES.SALE because it makes sense here to have it fixed +# - advert is hardcoded as ADVERT_TYPES.PROFESSIONAL (same) +# - house type is hardcoded as HOUSE_TYPES.LAND (same) +# - Only the first city in the query is taken into account for now (work in progress) +# - If a post has multiple lands, we choose the lowest cost and the highest area to have the best match. +# You'll have to review manually the lands of course and see if there is a good combo cost/area. +# So don't be too happy if you see a cheap big land ;) + +__all__ = ['LesterrainsModule'] + + +class LesterrainsModule(Module, CapHousing): + NAME = 'lesterrains' + DESCRIPTION = 'Les-Terrains.com' + MAINTAINER = 'Guntra' + EMAIL = 'guntra@example.com' + LICENSE = 'LGPLv3+' + VERSION = '3.0' + BROWSER = LesterrainsBrowser + + def search_city(self, pattern): + return self.browser.get_cities(pattern) + + def search_housings(self, query): + cities = ['%s' % c.id for c in query.cities if c.backend == self.name] + if len(cities) == 0: + return list() + return self.browser.search_housings( + cities, + query.area_min, + query.area_max, + query.cost_min, + query.cost_max + ) + + def get_housing(self, housing): + return self.browser.get_housing(housing) diff --git a/modules/lesterrains/pages.py b/modules/lesterrains/pages.py new file mode 100644 index 0000000000000000000000000000000000000000..74efc11724b50fb02d77c6c5fd37573a6737b670 --- /dev/null +++ b/modules/lesterrains/pages.py @@ -0,0 +1,211 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Guntra +# +# This file is part of a woob module. +# +# This woob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This woob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this woob module. If not, see . + +from __future__ import unicode_literals + +from woob.browser.filters.standard import ( + CleanDecimal, CleanText, + Date, Lower, Regexp, QueryValue, +) +from woob.browser.filters.json import Dict +from woob.browser.filters.html import Attr, AbsoluteLink +from woob.browser.elements import ItemElement, ListElement, DictElement, method +from woob.browser.pages import JsonPage, HTMLPage, pagination +from woob.capabilities.base import Currency, NotAvailable +from woob.capabilities.housing import ( + Housing, HousingPhoto, City, + POSTS_TYPES, HOUSE_TYPES, ADVERT_TYPES, UTILITIES +) + + +class CitiesPage(JsonPage): + ENCODING = 'UTF-8' + + def build_doc(self, content): + content = super(CitiesPage, self).build_doc(content) + if content: + return content + else: + return [{"locations": []}] + + @method + class get_cities(DictElement): + item_xpath = 'cities' + + class item(ItemElement): + klass = City + obj_id = Dict('id') & CleanText() & Lower() + obj_name = Dict('value') & CleanText() + + +class SearchPage(HTMLPage): + @pagination + @method + class iter_housings(ListElement): + item_xpath = '//article[has-class("itemListe")]' + next_page = AbsoluteLink('./div[@class="pagination-foot-bloc"]/a[@class="pageActive"][2]') + + class item(ItemElement): + klass = Housing + obj_id = QueryValue( + Attr( + './/div[has-class("presentationItem")]/h2/a', + 'href' + ), + 'idter' + ) + obj_url = AbsoluteLink( + './/h2/a', + default=NotAvailable + ) + obj_type = POSTS_TYPES.SALE + obj_advert_type = ADVERT_TYPES.PROFESSIONAL + obj_house_type = HOUSE_TYPES.LAND + obj_title = CleanText( + './/div[@class="presentationItem"]/h2/a', + default=NotAvailable + ) + + def obj_area(self): + min_area = CleanDecimal( + Regexp( + CleanText('.//div[@class="presentationItem"]/h3'), + r'surface de (\d+) m²', + default=0 + ) + )(self) + max_area = CleanDecimal( + Regexp( + CleanText('.//div[@class="presentationItem"]/h3'), + r'à (\d+) m²', + default=0 + ) + )(self) + return max(min_area, max_area) + + obj_cost = CleanDecimal( + CleanText( + './/div[@class="presentationItem"]/h3/span[1]', + replace=[(".", "")], + default=NotAvailable + ) + ) + obj_currency = Currency.get_currency('€') + obj_date = Date( + CleanText( + './/div[@class="presentationItem"]//span[@class="majItem"]', + replace=[("Mise à jour : ", "")] + ), + default=NotAvailable + ) + obj_location = CleanText( + './/div[@class="presentationItem"]/h2/a/span', + default=NotAvailable + ) + obj_text = CleanText( + './/div[@class="presentationItem"]/p', + default=NotAvailable + ) + obj_phone = CleanText( + './/div[@class="divBoutonContact"]/div[@class="phone-numbers-bloc"]/p[1]/strong', + default=NotAvailable + ) + + def _photos_generator(self): + for photo in self.xpath('.//div[has-class("photoItemListe")]/img/@data-src'): + yield HousingPhoto(self.page.absurl(photo)) + + def obj_photos(self): + return list(self._photos_generator()) + + obj_utilities = UTILITIES.UNKNOWN + + +class HousingPage(HTMLPage): + + @method + class get_housing(ItemElement): + klass = Housing + obj_id = Attr( + '//article//a[has-class("add-to-selection")]', + 'data-id' + ) + + def obj_url(self): + return self.page.url + + obj_type = POSTS_TYPES.SALE + obj_advert_type = ADVERT_TYPES.PROFESSIONAL + obj_house_type = HOUSE_TYPES.LAND + obj_title = CleanText( + '//article[@id="annonceTerrain"]/header/h1', + default=NotAvailable + ) + + def obj_area(self): + areas = [] + for land in self.xpath('//table[@id="price-list"]/tbody/tr'): + areas.append( + CleanDecimal( + './td[2]' + )(land) + ) + return max(areas) + + def obj_cost(self): + prices = [] + for land in self.xpath('//table[@id="price-list"]/tbody/tr'): + prices.append( + CleanDecimal( + CleanText( + './td[3]', + replace=[(".", "")] + ) + )(land) + ) + return min(prices) + + obj_currency = Currency.get_currency('€') + obj_date = Date( + CleanText( + '//section[@id="photos-details"]/div[@class="right-bloc"]/div/div[3]/div[2]/strong', + default=NotAvailable + ), + default=NotAvailable + ) + obj_location = CleanText( + '//article[@id="annonceTerrain"]/header/h1/strong', + default=NotAvailable + ) + obj_text = CleanText( + '//div[@id="informationsTerrain"]/p[2]', + default=NotAvailable + ) + obj_phone = CleanText( + '//div[@id="infos-annonceur"]/div/div/div[@class="phone-numbers-bloc"]/p/strong', + default=NotAvailable + ) + + def obj_photos(self): + photos = [] + for photo in self.xpath('.//div[@id="miniatures-carousel"]/div'): + photos.append(HousingPhoto(self.page.absurl(Attr('./img', 'data-big-photo')(photo)))) + return photos + + obj_utilities = UTILITIES.UNKNOWN diff --git a/modules/lesterrains/test.py b/modules/lesterrains/test.py new file mode 100644 index 0000000000000000000000000000000000000000..3f4e630c8eb41eb7669d537490fb13577fb140d6 --- /dev/null +++ b/modules/lesterrains/test.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Guntra +# +# This file is part of a woob module. +# +# This woob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This woob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this woob module. If not, see . + +from __future__ import unicode_literals +from woob.capabilities.housing import Query, POSTS_TYPES +from woob.tools.capabilities.housing.housing_test import HousingTest +from woob.tools.test import BackendTest + + +class LesterrainsTest(BackendTest, HousingTest): + MODULE = 'lesterrains' + # Fields to be checked for values across all items in housings list + FIELDS_ALL_HOUSINGS_LIST = [ + "id", "url", "type", "advert_type", "house_type" + ] + # Fields to be checked for at least one item in housings list + FIELDS_ANY_HOUSINGS_LIST = [ + "photos" + ] + # Fields to be checked for values across all items when querying + # individually + FIELDS_ALL_SINGLE_HOUSING = [ + "id", "url", "type", "advert_type", "house_type", "title", "area", + "cost", "currency", "date", "location", "text", "phone" + ] + # Fields to be checked for values at least once for all items when querying + # individually + FIELDS_ANY_SINGLE_HOUSING = [ + "photos" + ] + + def test_lesterrains_sale(self): + query = Query() + query.area_min = 500 + query.type = POSTS_TYPES.SALE + query.cities = [] + for city in self.backend.search_city('montastruc la conseillere'): + city.backend = self.backend.name + query.cities.append(city) + self.check_against_query(query) diff --git a/modules/tumblr/browser.py b/modules/tumblr/browser.py index b9fe1727e575d7e12dd1de9cff38f9edbc914754..2bc832884f0d38953ed1271802a1df17b119930a 100644 --- a/modules/tumblr/browser.py +++ b/modules/tumblr/browser.py @@ -101,6 +101,7 @@ def _images_from_post(self, post, index, gallery): if not match: return img = BaseImage( + id=post["id"], index=index, gallery=gallery, url=match.group(1),