diff --git a/modules/lesterrains/__init__.py b/modules/lesterrains/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4862c1f20a993828ac62c6a5a2d38b3f7cfc019c --- /dev/null +++ b/modules/lesterrains/__init__.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Guntra +# +# This file is part of a weboob module. +# +# This weboob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This weboob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this weboob module. If not, see . + +from __future__ import unicode_literals +from .module import LesterrainsModule + + +__all__ = ['LesterrainsModule'] diff --git a/modules/lesterrains/browser.py b/modules/lesterrains/browser.py new file mode 100644 index 0000000000000000000000000000000000000000..1820a9959d09208807f013bd4241595fbd06c20c --- /dev/null +++ b/modules/lesterrains/browser.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Guntra +# +# This file is part of a weboob module. +# +# This weboob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This weboob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this weboob module. If not, see . + +from __future__ import unicode_literals +from weboob.browser import PagesBrowser, URL +from weboob.browser.filters.standard import CleanText, Lower, Regexp +from weboob.capabilities.housing import (TypeNotSupported, POSTS_TYPES, HOUSE_TYPES) +from weboob.tools.compat import urlencode +from .pages import CitiesPage, SearchPage, HousingPage + + +class LesterrainsBrowser(PagesBrowser): + + BASEURL = 'http://www.les-terrains.com' + + TYPES = { + POSTS_TYPES.SALE: 'vente' + } + + RET = { + HOUSE_TYPES.LAND: 'Terrain seul' + } + + cities = URL('/api/get-search.php\?q=(?P.*)', CitiesPage) + + search = URL('/index.php\?mode_aff=liste&ongletAccueil=Terrains&(?P.*)&distance=0', SearchPage) + + housing = URL('/index.php\?page=terrains&mode_aff=un_terrain&idter=(?P<_id>\d+).*', HousingPage) + + def get_cities(self, pattern): + return self.cities.open(city=pattern).get_cities() + + def search_housings(self, cities, area_min, area_max, cost_min, cost_max): + + def _get_departement(city): + return city.split(';')[0][:2] + + def _get_ville(city): + return city.split(';')[1] + + for city in cities: + query = urlencode({ + "departement": _get_departement(city), + "ville": _get_ville(city), + "prixMin": cost_min or '', + "prixMax": cost_max or '', + "surfMin": area_min or '', + "surfMax": area_max or '', + }) + for house in self.search.go(query=query).iter_housings(): + yield house + + def get_housing(self, _id, housing=None): + return self.housing.go(_id = _id).get_housing(obj=housing) \ No newline at end of file diff --git a/modules/lesterrains/module.py b/modules/lesterrains/module.py new file mode 100644 index 0000000000000000000000000000000000000000..9c4995e28a38770cda8f1cbd2319800ecfdbc03d --- /dev/null +++ b/modules/lesterrains/module.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Guntra +# +# This file is part of a weboob module. +# +# This weboob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This weboob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this weboob module. If not, see . + +from __future__ import unicode_literals +from weboob.tools.backend import Module +from weboob.capabilities.housing import CapHousing +from .browser import LesterrainsBrowser + + +# Some remarks: +# - post type is hardcoded as POSTS_TYPES.SALE because it makes sense here to have it fixed +# - advert is hardcoded as ADVERT_TYPES.PROFESSIONAL (same) +# - house type is hardcoded as HOUSE_TYPES.LAND (same) +# - Only the first city in the query is taken into account for now (work in progress) +# - If a post has multiple lands, we choose the lowest cost and the highest area to have the best match. +# You'll have to review manually the lands of course and see if there is a good combo cost/area. +# So don't be too happy if you see a cheap big land ;) + +__all__ = ['LesterrainsModule'] + +class LesterrainsModule(Module, CapHousing): + + NAME = 'lesterrains' + + DESCRIPTION = 'Les-Terrains.com' + + MAINTAINER = 'Guntra' + + EMAIL = 'guntra@example.com' + + LICENSE = 'LGPLv3+' + + VERSION = '1.6' + + BROWSER = LesterrainsBrowser + + def search_city(self, pattern): + return self.browser.get_cities(pattern) + + def search_housings(self, query): + cities = ['%s' % c.id for c in query.cities if c.backend == self.name] + if len(cities) == 0: + return list() + return self.browser.search_housings( + cities, + query.area_min, + query.area_max, + query.cost_min, + query.cost_max + ) + + def get_housing(self, housing): + return self.browser.get_housing(housing) \ No newline at end of file diff --git a/modules/lesterrains/pages.py b/modules/lesterrains/pages.py new file mode 100644 index 0000000000000000000000000000000000000000..c9904024dc187d8d67f2b17a38b7872d3f6996d7 --- /dev/null +++ b/modules/lesterrains/pages.py @@ -0,0 +1,211 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Guntra +# +# This file is part of a weboob module. +# +# This weboob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This weboob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this weboob module. If not, see . + +from __future__ import unicode_literals +from weboob.browser.filters.standard import ( + CleanDecimal, CleanText, + Date, Format, Lower, Regexp, QueryValue +) +from weboob.browser.filters.json import Dict +from weboob.browser.filters.html import Attr, AbsoluteLink +from weboob.browser.elements import ItemElement, ListElement, DictElement, method +from weboob.browser.pages import JsonPage, HTMLPage, pagination +from weboob.capabilities.base import Currency +from weboob.capabilities.housing import ( + Housing, HousingPhoto, City, + POSTS_TYPES, HOUSE_TYPES, ADVERT_TYPES, UTILITIES +) + + +class CitiesPage(JsonPage): + + ENCODING = 'UTF-8' + + def build_doc(self, content): + content = super(CitiesPage, self).build_doc(content) + if content: + return content + else: + return [{"locations": []}] + + @method + class get_cities(DictElement): + + item_xpath = 'cities' + + class item(ItemElement): + + klass = City + + obj_id = Dict('id') & CleanText() & Lower() + + obj_name= Dict('value') & CleanText() + + +class SearchPage(HTMLPage): + + @pagination + @method + class iter_housings(ListElement): + + item_xpath = '//article[has-class("itemListe")]' + + next_page = AbsoluteLink('./div[@class="pagination-foot-bloc"]/a[@class="pageActive"][2]') + + class item(ItemElement): + + klass = Housing + + obj_id = QueryValue( + Attr( + './/div[has-class("presentationItem")]/h2/a', + 'href' + ), + 'idter' + ) + + obj_url = AbsoluteLink('.//h2/a') + + obj_type = POSTS_TYPES.SALE + + obj_advert_type = ADVERT_TYPES.PROFESSIONAL + + obj_house_type = HOUSE_TYPES.LAND + + obj_title = CleanText('.//div[@class="presentationItem"]/h2/a') + + def obj_area(self): + min_area = CleanDecimal( + Regexp( + CleanText('.//div[@class="presentationItem"]/h3'), + 'surface de (\d+) m²' + ) + )(self) + max_area = CleanDecimal( + Regexp( + CleanText('.//div[@class="presentationItem"]/h3'), + 'à (\d+) m²', + default=0 + ) + )(self) + if (max_area > min_area): + return max_area + else: + return min_area + + obj_cost = CleanDecimal( + CleanText( + './/div[@class="presentationItem"]/h3/span[1]', + replace=[(".", ""),(" €","")] + ) + ) + + obj_currency = Currency.get_currency(u'€') + + obj_date = Date( + CleanText( + './/div[@class="presentationItem"]//span[@class="majItem"]', + replace=[("Mise à jour : ", "")]) + ) + + obj_location = CleanText('.//div[@class="presentationItem"]/h2/a/span') + + obj_text = CleanText('.//div[@class="presentationItem"]/p') + + obj_phone = CleanText('.//div[@class="divBoutonContact"]/div[@class="phone-numbers-bloc"]/p[1]/strong') + + def _photos_generator(self): + for photo in self.xpath('.//div[has-class("photoItemListe")]/img/@data-src'): + yield HousingPhoto(self.page.absurl(photo)) + + def obj_photos(self): + return list(self._photos_generator()) + + obj_utilities = UTILITIES.UNKNOWN + +class HousingPage(HTMLPage): + + @method + class get_housing(ItemElement): + + klass = Housing + + obj_id = Attr( + '//article//a[has-class("add-to-selection")]', + 'data-id' + ) + + def obj_url(self): + return self.page.url + + obj_type = POSTS_TYPES.SALE + + obj_advert_type = ADVERT_TYPES.PROFESSIONAL + + obj_house_type = HOUSE_TYPES.LAND + + obj_title = CleanText('//article[@id="annonceTerrain"]/header/h1') + + def obj_area(self): + max_area = 0 + for land in self.xpath('//table[@id="price-list"]/tbody/tr'): + area = CleanDecimal( + CleanText( + './td[2]', + replace=[("m²","")] + ) + )(land) + if area > max_area: + max_area = area + return max_area + + def obj_cost(self): + min_cost = 0 + for land in self.xpath('//table[@id="price-list"]/tbody/tr'): + cost = CleanDecimal( + CleanText( + './td[3]', + replace=[(".","")] + ) + )(land) + if min_cost == 0: + min_cost = cost + if cost < min_cost: + min_cost = cost + return min_cost + + obj_currency = Currency.get_currency(u'€') + + obj_date = Date( + CleanText('//section[@id="photos-details"]/div[@class="right-bloc"]/div/div[3]/div[2]/strong') + ) + + obj_location = CleanText('//article[@id="annonceTerrain"]/header/h1/strong') + + obj_text = CleanText('//div[@id="informationsTerrain"]/p[2]') + + obj_phone = CleanText('//div[@id="infos-annonceur"]/div/div/div[@class="phone-numbers-bloc"]/p/strong') + + def obj_photos(self): + photos = [] + for photo in self.xpath('.//div[@id="miniatures-carousel"]/div'): + photos.append(HousingPhoto(self.page.absurl(Attr('./img', 'data-big-photo')(photo)))) + return photos + + obj_utilities = UTILITIES.UNKNOWN diff --git a/modules/lesterrains/test.py b/modules/lesterrains/test.py new file mode 100644 index 0000000000000000000000000000000000000000..36edba83be3e9b3cd903386b513d8a4bf615b60e --- /dev/null +++ b/modules/lesterrains/test.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- + +# Copyright(C) 2019 Guntra +# +# This file is part of a weboob module. +# +# This weboob module is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This weboob module is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this weboob module. If not, see . + +from __future__ import unicode_literals +from weboob.capabilities.housing import Query, ADVERT_TYPES, POSTS_TYPES +from weboob.tools.capabilities.housing.housing_test import HousingTest +from weboob.tools.test import BackendTest + + +class LesterrainsTest(BackendTest, HousingTest): + + MODULE = 'lesterrains' + + # Fields to be checked for values across all items in housings list + FIELDS_ALL_HOUSINGS_LIST = [ + "id", "url", "type", "advert_type", "house_type" + ] + + # Fields to be checked for at least one item in housings list + FIELDS_ANY_HOUSINGS_LIST = [ + "photos" + ] + + # Fields to be checked for values across all items when querying + # individually + FIELDS_ALL_SINGLE_HOUSING = [ + "id", "url", "type", "advert_type", "house_type", "title", "area", + "cost", "currency", "date", "location", "text", "phone" + ] + + # Fields to be checked for values at least once for all items when querying + # individually + FIELDS_ANY_SINGLE_HOUSING = [ + "photos" + ] + + def test_lesterrains_sale(self): + query = Query() + query.area_min = 500 + query.type = POSTS_TYPES.SALE + query.cities = [] + for city in self.backend.search_city('montastruc la conseillere'): + city.backend = self.backend.name + query.cities.append(city) + self.check_against_query(query)