Commit fc71f1cf authored by Antoine BOSSY's avatar Antoine BOSSY Committed by Romain Bignon

Create housing module for avendrealouer.fr

parent 209c4008
Pipeline #1854 failed with stages
in 0 seconds
# -*- coding: utf-8 -*-
# Copyright(C) 2017 ZeHiro
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from __future__ import unicode_literals
from .module import AvendrealouerModule
__all__ = ['AvendrealouerModule']
# -*- coding: utf-8 -*-
# Copyright(C) 2017 ZeHiro
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from __future__ import unicode_literals
from weboob.browser import PagesBrowser, URL
from weboob.capabilities.housing import HOUSE_TYPES
from .pages import CitiesPage, SearchPage, HousingPage
from .constants import QUERY_TYPES, QUERY_HOUSE_TYPES
class AvendrealouerBrowser(PagesBrowser):
BASEURL = 'https://www.avendrealouer.fr'
cities = URL(r'/common/api/localities\?term=(?P<term>)', CitiesPage)
search = URL(r'/recherche.html\?pageIndex=1&sortPropertyName=Price&sortDirection=Ascending&searchTypeID=(?P<type_id>.*)&typeGroupCategoryID=1&transactionId=1&localityIds=(?P<location_ids>.*)&typeGroupIds=(?P<type_group_ids>.*)(?P<rooms>.*)(?P<min_price>.*)(?P<max_price>.*)(?P<min_surface>.*)(?P<max_surface>.*)', SearchPage)
search_one = URL(r'/recherche.html\?localityIds=4-36388&reference=(?P<reference>.*)&hasMoreCriterias=true&searchTypeID=1', SearchPage)
housing = URL(r'/[vente|location].*', HousingPage)
def get_cities(self, pattern):
return self.cities.open(term=pattern).iter_cities()
def search_housings(self, query):
type_id = QUERY_TYPES[query.type]
house_types = []
for house_type in query.house_types:
if house_type == HOUSE_TYPES.UNKNOWN:
house_types = QUERY_HOUSE_TYPES[house_type]
break
house_types.append(QUERY_HOUSE_TYPES[house_type])
type_group_ids = ','.join(house_types)
location_ids = ','.join([city.id for city in query.cities])
def build_optional_param(query_field, query_string):
replace = ''
if getattr(query, query_field):
replace = '&%s=%s' % (query_string, getattr(query, query_field))
return replace
rooms = ''
if query.nb_rooms:
rooms = str(query.nb_rooms)
for i in range(query.nb_rooms + 1, 6):
rooms += ',%s' % str(i)
rooms = '&roomComfortIds=%s' % rooms
reg_exp = {
'type_id': type_id,
'type_group_ids': type_group_ids,
'location_ids': location_ids,
'rooms': rooms,
'min_price': build_optional_param('cost_min', 'minimumPrice'),
'max_price': build_optional_param('cost_max', 'maximumPrice'),
'min_surface': build_optional_param('area_min', 'minimumSurface'),
'max_surface': build_optional_param('area_max', 'maximumSurface')
}
return self.search.open(**reg_exp).iter_housings()
def get_housing(self, housing_id, obj=None):
url = self.search_one.open(reference=housing_id).get_housing_url()
return self.open(url).page.get_housing(obj=obj)
from weboob.capabilities.housing import HOUSE_TYPES, POSTS_TYPES
QUERY_TYPES = {
POSTS_TYPES.RENT: 2,
POSTS_TYPES.SALE: 1,
POSTS_TYPES.SHARING: 2, # There is no special search for shared appartments.
POSTS_TYPES.FURNISHED_RENT: 2,
POSTS_TYPES.VIAGER: 1
}
QUERY_HOUSE_TYPES = {
HOUSE_TYPES.APART: ['1'],
HOUSE_TYPES.HOUSE: ['2'],
HOUSE_TYPES.PARKING: ['7'],
HOUSE_TYPES.LAND: ['3'],
HOUSE_TYPES.OTHER: ['4', '5', '6', '8', '9', '10', '11', '12', '13', '14'],
HOUSE_TYPES.UNKNOWN: ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14']
}
# -*- coding: utf-8 -*-
# Copyright(C) 2017 ZeHiro
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from __future__ import unicode_literals
from weboob.tools.backend import Module
from weboob.capabilities.housing import CapHousing, Housing
from .browser import AvendrealouerBrowser
__all__ = ['AvendrealouerModule']
class AvendrealouerModule(Module, CapHousing):
NAME = u'avendrealouer'
DESCRIPTION = 'avendrealouer website'
MAINTAINER = 'ZeHiro'
EMAIL = 'public@abossy.fr'
LICENSE = 'AGPLv3+'
VERSION = '1.4'
BROWSER = AvendrealouerBrowser
def get_housing(self, housing):
"""
Get an housing from an ID.
:param housing: ID of the housing
:type housing: str
:rtype: :class:`Housing` or None if not found.
"""
return self.browser.get_housing(housing)
def search_city(self, pattern):
"""
Search a city from a pattern.
:param pattern: pattern to search
:type pattern: str
:rtype: iter[:class:`City`]
"""
return self.browser.get_cities(pattern)
def search_housings(self, query):
"""
Search housings.
:param query: search query
:type query: :class:`Query`
:rtype: iter[:class:`Housing`]
"""
return self.browser.search_housings(query)
def fill_housing(self, housing, fields):
if 'photos' in fields and housing.photos:
for photo in housing.photos:
photo.data = self.browser.open(photo.url)
return housing
OBJECTS = {Housing: fill_housing}
# -*- coding: utf-8 -*-
# Copyright(C) 2017 ZeHiro
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from __future__ import unicode_literals
from datetime import datetime
from weboob.browser.pages import HTMLPage, JsonPage, pagination
from weboob.browser.elements import ItemElement, ListElement, method, DictElement
from weboob.browser.filters.html import Attr, AbsoluteLink, Link
from weboob.browser.filters.json import Dict
from weboob.browser.filters.standard import CleanDecimal, CleanText, Date, Regexp, Async, AsyncLoad
from weboob.capabilities.housing import City, Housing, UTILITIES, HousingPhoto
from weboob.capabilities.base import NotAvailable, Currency
from weboob.tools.capabilities.housing.housing import PricePerMeterFilter
class CitiesPage(JsonPage):
@method
class iter_cities(DictElement):
class item(ItemElement):
klass = City
obj_id = Dict('Value')
obj_name = Dict('Name')
class AvendreAlouerItem(ItemElement):
klass = Housing
_url = AbsoluteLink('.//a[has-class("linkCtnr")]')
load_details = _url & AsyncLoad
obj_url = _url
obj_id = Async('details') & CleanText(Regexp(CleanText('//p[has-class("property-reference")]'), r'\:(.*)$', default=''))
obj_title = CleanText('.//a//ul')
obj_area = CleanDecimal(
CleanText('.//a//ul//li[has-class("first")]//following-sibling::li[2]'),
default=NotAvailable
)
obj_cost = CleanDecimal(
CleanText('.//span[has-class("price")]')
)
obj_price_per_meter = PricePerMeterFilter()
obj_currency = CleanText(
Regexp(
CleanText('.//span[has-class("price")]'),
r'[\d\ ]+(.*)'
)
)
obj_location = CleanText('.//span[has-class("loca")]')
obj_text = CleanText('.//p[has-class("propShortDesc")]')
obj_date = Async('details') & Date(
Regexp(
CleanText('//div[has-class("property-description-main")]'),
r'Mise à jour le ([\d\\]+)', default=datetime.today()
)
)
def obj_details(self):
page_doc = Async('details').loaded_page(self).doc
return {
'GES': CleanText('//span[@id="gassymbol"]', '')(page_doc),
'DPE': CleanText('//span[@id="energysymbol"]', '')(page_doc),
}
def obj_utilities(self):
price = CleanText('//span[has-class("price-info")]')(self)
if 'CC' in price:
return UTILITIES.INCLUDED
elif 'HC' in price:
return UTILITIES.EXCLUDED
else:
return UTILITIES.UNKNOWN
obj_station = 'Test'
obj_bedrooms = Async('details') & CleanDecimal(
CleanText('.//td//span[contains(text(), "Chambre")]//following-sibling::span[has-class("r")]'),
default=NotAvailable
)
obj_rooms = Async('details') & CleanDecimal(
CleanText('.//td//span[contains(text(), "Pièce")]//following-sibling::span[has-class("r")]'),
default=NotAvailable
)
def obj_photos(self):
page_doc = Async('details').loaded_page(self).doc
photos = []
for photo in page_doc.xpath('//div[@id="bxSliderContainer"]//ul//li//img'):
url = Attr('.', 'src')(photo)
if url[0] != '/':
photos.append(HousingPhoto(url))
return photos
def validate(self, obj):
return obj.id != ''
class SearchPage(HTMLPage):
@pagination
@method
class iter_housings(ListElement):
item_xpath = './/li[@data-tranid="1"]'
next_page = AbsoluteLink('./ul[has-class("pagination")]/li/a[has-class("next")]')
class item(AvendreAlouerItem):
obj_phone = CleanText(Attr('.', 'data-infos'))
def get_housing_url(self):
return Link('.//a[has-class("picCtnr")]')(self.doc)
class HousingPage(HTMLPage):
@method
class get_housing(ItemElement):
klass = Housing
obj_id = Regexp(CleanText('//p[has-class("property-reference")]'), r'\:(.*)$')
def obj_url(self):
return self.page.url
obj_area = CleanDecimal(
Regexp(
CleanText('//table[@id="table"]//span[contains(text(), "Surface")]//following-sibling::span[has-class("r")]'),
r'([\d\ ]+)m'
),
default=NotAvailable
)
obj_title = CleanText('//span[has-class("mainh1")]')
obj_cost = CleanDecimal('//span[has-class("price-info")]')
obj_currency = Currency.get_currency(u'€')
obj_rooms = CleanDecimal('//table[@id="table"]//span[contains(text(), "Pièce")]//following-sibling::span[has-class("r")]')
obj_bedrooms = CleanDecimal('//table[@id="table"]//span[contains(text(), "Chambre")]//following-sibling::span[has-class("r")]')
obj_location = CleanText(Regexp(CleanText('//span[has-class("mainh1")]'), r',(.+)$'))
obj_text = CleanText('//div[has-class("property-description-main")]')
obj_date = Date(
Regexp(
CleanText('//div[has-class("property-description-main")]'),
r'Mise à jour le ([\d\\]+)', default=datetime.today()
)
)
obj_phone = Attr('//button[@id="display-phonenumber-1"]', 'data-phone-number')
def obj_photos(self):
photos = []
for photo in self.xpath('//div[@id="bxSliderContainer"]//ul//li//img'):
url = Attr('.', 'src')(photo)
if url[0] != '/':
photos.append(HousingPhoto(url))
return photos
def obj_details(self):
return {
'GES': CleanText('//span[@id="gassymbol"]', '')(self),
'DPE': CleanText('//span[@id="energysymbol"]', '')(self),
}
def obj_utilities(self):
price = CleanText('//span[has-class("price-info")]')(self)
if 'CC' in price:
return UTILITIES.INCLUDED
elif 'HC' in price:
return UTILITIES.EXCLUDED
else:
return UTILITIES.UNKNOWN
obj_station = NotAvailable
obj_price_per_meter = PricePerMeterFilter()
# -*- coding: utf-8 -*-
# Copyright(C) 2017 ZeHiro
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from __future__ import unicode_literals
from weboob.tools.test import BackendTest
from weboob.tools.capabilities.housing.housing_test import HousingTest
from weboob.capabilities.housing import Query, POSTS_TYPES
class AvendrealouerTest(BackendTest, HousingTest):
MODULE = 'avendrealouer'
DO_NOT_DISTINGUISH_FURNISHED_RENT = True
def test_avendre_rent(self):
query = Query()
query.area_min = 20
query.cost_max = 1500
query.type = POSTS_TYPES.RENT
query.cities = []
for city in self.backend.search_city('paris'):
city.backend = self.backend.name
query.cities.append(city)
self.check_against_query(query)
def test_avendre_sale(self):
query = Query()
query.area_min = 20
query.type = POSTS_TYPES.SALE
query.cities = []
for city in self.backend.search_city('paris'):
city.backend = self.backend.name
query.cities.append(city)
self.check_against_query(query)
......@@ -14,6 +14,7 @@ anticaptcha
amundi
apec
arte
avendrealouer
axabanque
bandcamp
banqueaccord
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment