The new woob repository is here: https://gitlab.com/woob/woob. This gitlab will be removed soon.

Commit 3a155d65 authored by Guntra's avatar Guntra Committed by Vincent A

new module: lesterrains

parent 3d179964
# -*- coding: utf-8 -*-
# Copyright(C) 2019 Guntra
#
# This file is part of a weboob module.
#
# This weboob module is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This weboob module is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
from __future__ import unicode_literals
from .module import LesterrainsModule
__all__ = ['LesterrainsModule']
# -*- coding: utf-8 -*-
# Copyright(C) 2019 Guntra
#
# This file is part of a weboob module.
#
# This weboob module is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This weboob module is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
from __future__ import unicode_literals
from weboob.browser import PagesBrowser, URL
from weboob.browser.filters.standard import CleanText, Lower, Regexp
from weboob.capabilities.housing import (TypeNotSupported, POSTS_TYPES, HOUSE_TYPES)
from weboob.tools.compat import urlencode
from .pages import CitiesPage, SearchPage, HousingPage
class LesterrainsBrowser(PagesBrowser):
BASEURL = 'http://www.les-terrains.com'
TYPES = {
POSTS_TYPES.SALE: 'vente'
}
RET = {
HOUSE_TYPES.LAND: 'Terrain seul'
}
cities = URL('/api/get-search.php\?q=(?P<city>.*)', CitiesPage)
search = URL('/index.php\?mode_aff=liste&ongletAccueil=Terrains&(?P<query>.*)&distance=0', SearchPage)
housing = URL('/index.php\?page=terrains&mode_aff=un_terrain&idter=(?P<_id>\d+).*', HousingPage)
def get_cities(self, pattern):
return self.cities.open(city=pattern).get_cities()
def search_housings(self, cities, area_min, area_max, cost_min, cost_max):
def _get_departement(city):
return city.split(';')[0][:2]
def _get_ville(city):
return city.split(';')[1]
for city in cities:
query = urlencode({
"departement": _get_departement(city),
"ville": _get_ville(city),
"prixMin": cost_min or '',
"prixMax": cost_max or '',
"surfMin": area_min or '',
"surfMax": area_max or '',
})
for house in self.search.go(query=query).iter_housings():
yield house
def get_housing(self, _id, housing=None):
return self.housing.go(_id = _id).get_housing(obj=housing)
\ No newline at end of file
# -*- coding: utf-8 -*-
# Copyright(C) 2019 Guntra
#
# This file is part of a weboob module.
#
# This weboob module is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This weboob module is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
from __future__ import unicode_literals
from weboob.tools.backend import Module
from weboob.capabilities.housing import CapHousing
from .browser import LesterrainsBrowser
# Some remarks:
# - post type is hardcoded as POSTS_TYPES.SALE because it makes sense here to have it fixed
# - advert is hardcoded as ADVERT_TYPES.PROFESSIONAL (same)
# - house type is hardcoded as HOUSE_TYPES.LAND (same)
# - Only the first city in the query is taken into account for now (work in progress)
# - If a post has multiple lands, we choose the lowest cost and the highest area to have the best match.
# You'll have to review manually the lands of course and see if there is a good combo cost/area.
# So don't be too happy if you see a cheap big land ;)
__all__ = ['LesterrainsModule']
class LesterrainsModule(Module, CapHousing):
NAME = 'lesterrains'
DESCRIPTION = 'Les-Terrains.com'
MAINTAINER = 'Guntra'
EMAIL = 'guntra@example.com'
LICENSE = 'LGPLv3+'
VERSION = '1.6'
BROWSER = LesterrainsBrowser
def search_city(self, pattern):
return self.browser.get_cities(pattern)
def search_housings(self, query):
cities = ['%s' % c.id for c in query.cities if c.backend == self.name]
if len(cities) == 0:
return list()
return self.browser.search_housings(
cities,
query.area_min,
query.area_max,
query.cost_min,
query.cost_max
)
def get_housing(self, housing):
return self.browser.get_housing(housing)
\ No newline at end of file
# -*- coding: utf-8 -*-
# Copyright(C) 2019 Guntra
#
# This file is part of a weboob module.
#
# This weboob module is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This weboob module is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
from __future__ import unicode_literals
from weboob.browser.filters.standard import (
CleanDecimal, CleanText,
Date, Format, Lower, Regexp, QueryValue
)
from weboob.browser.filters.json import Dict
from weboob.browser.filters.html import Attr, AbsoluteLink
from weboob.browser.elements import ItemElement, ListElement, DictElement, method
from weboob.browser.pages import JsonPage, HTMLPage, pagination
from weboob.capabilities.base import Currency
from weboob.capabilities.housing import (
Housing, HousingPhoto, City,
POSTS_TYPES, HOUSE_TYPES, ADVERT_TYPES, UTILITIES
)
class CitiesPage(JsonPage):
ENCODING = 'UTF-8'
def build_doc(self, content):
content = super(CitiesPage, self).build_doc(content)
if content:
return content
else:
return [{"locations": []}]
@method
class get_cities(DictElement):
item_xpath = 'cities'
class item(ItemElement):
klass = City
obj_id = Dict('id') & CleanText() & Lower()
obj_name= Dict('value') & CleanText()
class SearchPage(HTMLPage):
@pagination
@method
class iter_housings(ListElement):
item_xpath = '//article[has-class("itemListe")]'
next_page = AbsoluteLink('./div[@class="pagination-foot-bloc"]/a[@class="pageActive"][2]')
class item(ItemElement):
klass = Housing
obj_id = QueryValue(
Attr(
'.//div[has-class("presentationItem")]/h2/a',
'href'
),
'idter'
)
obj_url = AbsoluteLink('.//h2/a')
obj_type = POSTS_TYPES.SALE
obj_advert_type = ADVERT_TYPES.PROFESSIONAL
obj_house_type = HOUSE_TYPES.LAND
obj_title = CleanText('.//div[@class="presentationItem"]/h2/a')
def obj_area(self):
min_area = CleanDecimal(
Regexp(
CleanText('.//div[@class="presentationItem"]/h3'),
'surface de (\d+) m²'
)
)(self)
max_area = CleanDecimal(
Regexp(
CleanText('.//div[@class="presentationItem"]/h3'),
'à (\d+) m²',
default=0
)
)(self)
if (max_area > min_area):
return max_area
else:
return min_area
obj_cost = CleanDecimal(
CleanText(
'.//div[@class="presentationItem"]/h3/span[1]',
replace=[(".", ""),(" €","")]
)
)
obj_currency = Currency.get_currency(u'€')
obj_date = Date(
CleanText(
'.//div[@class="presentationItem"]//span[@class="majItem"]',
replace=[("Mise à jour : ", "")])
)
obj_location = CleanText('.//div[@class="presentationItem"]/h2/a/span')
obj_text = CleanText('.//div[@class="presentationItem"]/p')
obj_phone = CleanText('.//div[@class="divBoutonContact"]/div[@class="phone-numbers-bloc"]/p[1]/strong')
def _photos_generator(self):
for photo in self.xpath('.//div[has-class("photoItemListe")]/img/@data-src'):
yield HousingPhoto(self.page.absurl(photo))
def obj_photos(self):
return list(self._photos_generator())
obj_utilities = UTILITIES.UNKNOWN
class HousingPage(HTMLPage):
@method
class get_housing(ItemElement):
klass = Housing
obj_id = Attr(
'//article//a[has-class("add-to-selection")]',
'data-id'
)
def obj_url(self):
return self.page.url
obj_type = POSTS_TYPES.SALE
obj_advert_type = ADVERT_TYPES.PROFESSIONAL
obj_house_type = HOUSE_TYPES.LAND
obj_title = CleanText('//article[@id="annonceTerrain"]/header/h1')
def obj_area(self):
max_area = 0
for land in self.xpath('//table[@id="price-list"]/tbody/tr'):
area = CleanDecimal(
CleanText(
'./td[2]',
replace=[("m²","")]
)
)(land)
if area > max_area:
max_area = area
return max_area
def obj_cost(self):
min_cost = 0
for land in self.xpath('//table[@id="price-list"]/tbody/tr'):
cost = CleanDecimal(
CleanText(
'./td[3]',
replace=[(".","")]
)
)(land)
if min_cost == 0:
min_cost = cost
if cost < min_cost:
min_cost = cost
return min_cost
obj_currency = Currency.get_currency(u'€')
obj_date = Date(
CleanText('//section[@id="photos-details"]/div[@class="right-bloc"]/div/div[3]/div[2]/strong')
)
obj_location = CleanText('//article[@id="annonceTerrain"]/header/h1/strong')
obj_text = CleanText('//div[@id="informationsTerrain"]/p[2]')
obj_phone = CleanText('//div[@id="infos-annonceur"]/div/div/div[@class="phone-numbers-bloc"]/p/strong')
def obj_photos(self):
photos = []
for photo in self.xpath('.//div[@id="miniatures-carousel"]/div'):
photos.append(HousingPhoto(self.page.absurl(Attr('./img', 'data-big-photo')(photo))))
return photos
obj_utilities = UTILITIES.UNKNOWN
# -*- coding: utf-8 -*-
# Copyright(C) 2019 Guntra
#
# This file is part of a weboob module.
#
# This weboob module is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This weboob module is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
from __future__ import unicode_literals
from weboob.capabilities.housing import Query, ADVERT_TYPES, POSTS_TYPES
from weboob.tools.capabilities.housing.housing_test import HousingTest
from weboob.tools.test import BackendTest
class LesterrainsTest(BackendTest, HousingTest):
MODULE = 'lesterrains'
# Fields to be checked for values across all items in housings list
FIELDS_ALL_HOUSINGS_LIST = [
"id", "url", "type", "advert_type", "house_type"
]
# Fields to be checked for at least one item in housings list
FIELDS_ANY_HOUSINGS_LIST = [
"photos"
]
# Fields to be checked for values across all items when querying
# individually
FIELDS_ALL_SINGLE_HOUSING = [
"id", "url", "type", "advert_type", "house_type", "title", "area",
"cost", "currency", "date", "location", "text", "phone"
]
# Fields to be checked for values at least once for all items when querying
# individually
FIELDS_ANY_SINGLE_HOUSING = [
"photos"
]
def test_lesterrains_sale(self):
query = Query()
query.area_min = 500
query.type = POSTS_TYPES.SALE
query.cities = []
for city in self.backend.search_city('montastruc la conseillere'):
city.backend = self.backend.name
query.cities.append(city)
self.check_against_query(query)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment