diff --git a/modules/avendrealouer/__init__.py b/modules/avendrealouer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..11726874d4e6679f46f1c36e23d0f20e102e1702 --- /dev/null +++ b/modules/avendrealouer/__init__.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 ZeHiro +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + + +from .module import AvendrealouerModule + + +__all__ = ['AvendrealouerModule'] diff --git a/modules/avendrealouer/browser.py b/modules/avendrealouer/browser.py new file mode 100644 index 0000000000000000000000000000000000000000..47c2cfe90dfab018b376177066036c1db7810fee --- /dev/null +++ b/modules/avendrealouer/browser.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 ZeHiro +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + + +from weboob.browser import PagesBrowser, URL +from weboob.capabilities.housing import HOUSE_TYPES + +from .pages import CitiesPage, SearchPage, HousingPage +from .constants import QUERY_TYPES, QUERY_HOUSE_TYPES + + +class AvendrealouerBrowser(PagesBrowser): + BASEURL = 'https://www.avendrealouer.fr' + + cities = URL(r'/common/api/localities\?term=(?P)', CitiesPage) + search = URL(r'/recherche.html\?pageIndex=1&sortPropertyName=Price&sortDirection=Ascending&searchTypeID=(?P.*)&typeGroupCategoryID=1&transactionId=1&localityIds=(?P.*)&typeGroupIds=(?P.*)(?P.*)(?P.*)(?P.*)(?P.*)(?P.*)', SearchPage) + search_one = URL(r'/recherche.html\?localityIds=4-36388&reference=(?P.*)&hasMoreCriterias=true&searchTypeID=1', SearchPage) + housing = URL(r'/[vente|location].*', HousingPage) + + def get_cities(self, pattern): + return self.cities.open(term=pattern).iter_cities() + + def search_housings(self, query): + type_id = QUERY_TYPES[query.type] + + house_types = [] + for house_type in query.house_types: + if house_type == HOUSE_TYPES.UNKNOWN: + house_types = QUERY_HOUSE_TYPES[house_type] + break + house_types.append(QUERY_HOUSE_TYPES[house_type]) + + type_group_ids = ','.join(house_types) + + location_ids = ','.join([city.id for city in query.cities]) + + def build_optional_param(query_field, query_string): + replace = '' + if getattr(query, query_field): + replace = '&%s=%s' % (query_string, getattr(query, query_field)) + return replace + + rooms = '' + if query.nb_rooms: + rooms = str(query.nb_rooms) + for i in range(query.nb_rooms + 1, 6): + rooms += ',%s' % str(i) + rooms = '&roomComfortIds=%s' % rooms + + reg_exp = { + 'type_id': type_id, + 'type_group_ids': type_group_ids, + 'location_ids': location_ids, + 'rooms': rooms, + 'min_price': build_optional_param('cost_min', 'minimumPrice'), + 'max_price': build_optional_param('cost_max', 'maximumPrice'), + 'min_surface': build_optional_param('area_min', 'minimumSurface'), + 'max_surface': build_optional_param('area_max', 'maximumSurface') + } + return self.search.open(**reg_exp).iter_housings() + + def get_housing(self, housing_id, obj=None): + url = self.search_one.open(reference=housing_id).get_housing_url() + return self.open(url).page.get_housing(obj=obj) diff --git a/modules/avendrealouer/constants.py b/modules/avendrealouer/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..5aa777d07c9a3c62ee91e9f17b1223dbf19a9706 --- /dev/null +++ b/modules/avendrealouer/constants.py @@ -0,0 +1,18 @@ +from weboob.capabilities.housing import HOUSE_TYPES, POSTS_TYPES + +QUERY_TYPES = { + POSTS_TYPES.RENT: 2, + POSTS_TYPES.SALE: 1, + POSTS_TYPES.SHARING: 2, # There is no special search for shared appartments. + POSTS_TYPES.FURNISHED_RENT: 2, + POSTS_TYPES.VIAGER: 1 +} + +QUERY_HOUSE_TYPES = { + HOUSE_TYPES.APART: ['1'], + HOUSE_TYPES.HOUSE: ['2'], + HOUSE_TYPES.PARKING: ['7'], + HOUSE_TYPES.LAND: ['3'], + HOUSE_TYPES.OTHER: ['4', '5', '6', '8', '9', '10', '11', '12', '13', '14'], + HOUSE_TYPES.UNKNOWN: ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14'] +} diff --git a/modules/avendrealouer/module.py b/modules/avendrealouer/module.py new file mode 100644 index 0000000000000000000000000000000000000000..c14ec79c6c0d940f0568cfc51df858fc21dddb93 --- /dev/null +++ b/modules/avendrealouer/module.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 ZeHiro +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + + +from weboob.tools.backend import Module +from weboob.capabilities.housing import CapHousing, Housing + +from .browser import AvendrealouerBrowser + + +__all__ = ['AvendrealouerModule'] + + +class AvendrealouerModule(Module, CapHousing): + NAME = u'avendrealouer' + DESCRIPTION = 'avendrealouer website' + MAINTAINER = 'ZeHiro' + EMAIL = 'public@abossy.fr' + LICENSE = 'AGPLv3+' + VERSION = '1.4' + + BROWSER = AvendrealouerBrowser + + def get_housing(self, housing): + """ + Get an housing from an ID. + + :param housing: ID of the housing + :type housing: str + :rtype: :class:`Housing` or None if not found. + """ + return self.browser.get_housing(housing) + + def search_city(self, pattern): + """ + Search a city from a pattern. + + :param pattern: pattern to search + :type pattern: str + :rtype: iter[:class:`City`] + """ + return self.browser.get_cities(pattern) + + def search_housings(self, query): + """ + Search housings. + + :param query: search query + :type query: :class:`Query` + :rtype: iter[:class:`Housing`] + """ + return self.browser.search_housings(query) + + def fill_housing(self, housing, fields): + if 'photos' in fields and housing.photos: + for photo in housing.photos: + photo.data = self.browser.open(photo.url) + return housing + + OBJECTS = {Housing: fill_housing} diff --git a/modules/avendrealouer/pages.py b/modules/avendrealouer/pages.py new file mode 100644 index 0000000000000000000000000000000000000000..5df052b8bedefa9a57b1d22f929d4352c91913c6 --- /dev/null +++ b/modules/avendrealouer/pages.py @@ -0,0 +1,195 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 ZeHiro +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + +from datetime import datetime + +from weboob.browser.pages import HTMLPage, JsonPage, pagination +from weboob.browser.elements import ItemElement, ListElement, method, DictElement +from weboob.browser.filters.html import Attr, AbsoluteLink, Link +from weboob.browser.filters.json import Dict + +from weboob.browser.filters.standard import CleanDecimal, CleanText, Date, Regexp, Async, AsyncLoad + +from weboob.capabilities.housing import City, Housing, UTILITIES, HousingPhoto +from weboob.capabilities.base import NotAvailable, Currency + +from weboob.tools.capabilities.housing.housing import PricePerMeterFilter + + +class CitiesPage(JsonPage): + + @method + class iter_cities(DictElement): + + class item(ItemElement): + klass = City + obj_id = Dict('Value') + obj_name = Dict('Name') + + +class AvendreAlouerItem(ItemElement): + klass = Housing + _url = AbsoluteLink('.//a[has-class("linkCtnr")]') + + load_details = _url & AsyncLoad + + obj_url = _url + obj_id = Async('details') & CleanText(Regexp(CleanText('//p[has-class("property-reference")]'), r'\:(.*)$', default='')) + + obj_title = CleanText('.//a//ul') + obj_area = CleanDecimal( + CleanText('.//a//ul//li[has-class("first")]//following-sibling::li[2]'), + default=NotAvailable + ) + + obj_cost = CleanDecimal( + CleanText('.//span[has-class("price")]') + ) + obj_price_per_meter = PricePerMeterFilter() + obj_currency = CleanText( + Regexp( + CleanText('.//span[has-class("price")]'), + r'[\d\ ]+(.*)' + ) + ) + + obj_location = CleanText('.//span[has-class("loca")]') + obj_text = CleanText('.//p[has-class("propShortDesc")]') + + obj_date = Async('details') & Date( + Regexp( + CleanText('//div[has-class("property-description-main")]'), + r'Mise à jour le ([\d\\]+)', default=datetime.today() + ) + ) + + def obj_details(self): + page_doc = Async('details').loaded_page(self).doc + + return { + 'GES': CleanText('//span[@id="gassymbol"]', '')(page_doc), + 'DPE': CleanText('//span[@id="energysymbol"]', '')(page_doc), + } + + def obj_utilities(self): + price = CleanText('//span[has-class("price-info")]')(self) + if 'CC' in price: + return UTILITIES.INCLUDED + elif 'HC' in price: + return UTILITIES.EXCLUDED + else: + return UTILITIES.UNKNOWN + + obj_station = 'Test' + obj_bedrooms = Async('details') & CleanDecimal( + CleanText('.//td//span[contains(text(), "Chambre")]//following-sibling::span[has-class("r")]'), + default=NotAvailable + ) + + obj_rooms = Async('details') & CleanDecimal( + CleanText('.//td//span[contains(text(), "Pièce")]//following-sibling::span[has-class("r")]'), + default=NotAvailable + ) + + def obj_photos(self): + page_doc = Async('details').loaded_page(self).doc + photos = [] + for photo in page_doc.xpath('//div[@id="bxSliderContainer"]//ul//li//img'): + url = Attr('.', 'src')(photo) + if url[0] != '/': + photos.append(HousingPhoto(url)) + return photos + + def validate(self, obj): + return obj.id != '' + + +class SearchPage(HTMLPage): + @pagination + @method + class iter_housings(ListElement): + item_xpath = './/li[@data-tranid="1"]' + + next_page = AbsoluteLink('./ul[has-class("pagination")]/li/a[has-class("next")]') + + class item(AvendreAlouerItem): + obj_phone = CleanText(Attr('.', 'data-infos')) + + def get_housing_url(self): + return Link('.//a[has-class("picCtnr")]')(self.doc) + + +class HousingPage(HTMLPage): + @method + class get_housing(ItemElement): + klass = Housing + obj_id = Regexp(CleanText('//p[has-class("property-reference")]'), r'\:(.*)$') + + def obj_url(self): + return self.page.url + + obj_area = CleanDecimal( + Regexp( + CleanText('//table[@id="table"]//span[contains(text(), "Surface")]//following-sibling::span[has-class("r")]'), + r'([\d\ ]+)m' + ), + default=NotAvailable + ) + obj_title = CleanText('//span[has-class("mainh1")]') + obj_cost = CleanDecimal('//span[has-class("price-info")]') + obj_currency = Currency.get_currency(u'€') + obj_rooms = CleanDecimal('//table[@id="table"]//span[contains(text(), "Pièce")]//following-sibling::span[has-class("r")]') + obj_bedrooms = CleanDecimal('//table[@id="table"]//span[contains(text(), "Chambre")]//following-sibling::span[has-class("r")]') + obj_location = CleanText(Regexp(CleanText('//span[has-class("mainh1")]'), r',(.+)$')) + obj_text = CleanText('//div[has-class("property-description-main")]') + obj_date = Date( + Regexp( + CleanText('//div[has-class("property-description-main")]'), + r'Mise à jour le ([\d\\]+)', default=datetime.today() + ) + ) + obj_phone = Attr('//button[@id="display-phonenumber-1"]', 'data-phone-number') + + def obj_photos(self): + photos = [] + for photo in self.xpath('//div[@id="bxSliderContainer"]//ul//li//img'): + url = Attr('.', 'src')(photo) + if url[0] != '/': + photos.append(HousingPhoto(url)) + return photos + + def obj_details(self): + return { + 'GES': CleanText('//span[@id="gassymbol"]', '')(self), + 'DPE': CleanText('//span[@id="energysymbol"]', '')(self), + } + + def obj_utilities(self): + price = CleanText('//span[has-class("price-info")]')(self) + if 'CC' in price: + return UTILITIES.INCLUDED + elif 'HC' in price: + return UTILITIES.EXCLUDED + else: + return UTILITIES.UNKNOWN + + obj_station = NotAvailable + obj_price_per_meter = PricePerMeterFilter() diff --git a/modules/avendrealouer/test.py b/modules/avendrealouer/test.py new file mode 100644 index 0000000000000000000000000000000000000000..4e22c4c42c0fed07ebf9d4b7aa68f7c69951f0a9 --- /dev/null +++ b/modules/avendrealouer/test.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2017 ZeHiro +# +# This file is part of weboob. +# +# weboob is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# weboob is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with weboob. If not, see . + +from __future__ import unicode_literals + + +from weboob.tools.test import BackendTest +from weboob.tools.capabilities.housing.housing_test import HousingTest +from weboob.capabilities.housing import Query, POSTS_TYPES + + +class AvendrealouerTest(BackendTest, HousingTest): + MODULE = 'avendrealouer' + + DO_NOT_DISTINGUISH_FURNISHED_RENT = True + + def test_avendre_rent(self): + query = Query() + query.area_min = 20 + query.cost_max = 1500 + query.type = POSTS_TYPES.RENT + query.cities = [] + for city in self.backend.search_city('paris'): + city.backend = self.backend.name + query.cities.append(city) + self.check_against_query(query) + + def test_avendre_sale(self): + query = Query() + query.area_min = 20 + query.type = POSTS_TYPES.SALE + query.cities = [] + for city in self.backend.search_city('paris'): + city.backend = self.backend.name + query.cities.append(city) + self.check_against_query(query) diff --git a/tools/py3-compatible.modules b/tools/py3-compatible.modules index f6f603c0846f6af190b90bb63020d26835fad497..eb94dc43e1260c195f2f8d54284d4cb09dd281a4 100644 --- a/tools/py3-compatible.modules +++ b/tools/py3-compatible.modules @@ -14,6 +14,7 @@ anticaptcha amundi apec arte +avendrealouer axabanque bandcamp banqueaccord