Create housing module for avendrealouer.fr

modules/avendrealouer/__init__.py

+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2017      ZeHiro
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import unicode_literals
+
+
+from .module import AvendrealouerModule
+
+
+__all__ = ['AvendrealouerModule']

modules/avendrealouer/browser.py

+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2017      ZeHiro
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import unicode_literals
+
+
+from weboob.browser import PagesBrowser, URL
+from weboob.capabilities.housing import HOUSE_TYPES
+
+from .pages import CitiesPage, SearchPage, HousingPage
+from .constants import QUERY_TYPES, QUERY_HOUSE_TYPES
+
+
+class AvendrealouerBrowser(PagesBrowser):
+    BASEURL = 'https://www.avendrealouer.fr'
+
+    cities = URL(r'/common/api/localities\?term=(?P<term>)', CitiesPage)
+    search = URL(r'/recherche.html\?pageIndex=1&sortPropertyName=Price&sortDirection=Ascending&searchTypeID=(?P<type_id>.*)&typeGroupCategoryID=1&transactionId=1&localityIds=(?P<location_ids>.*)&typeGroupIds=(?P<type_group_ids>.*)(?P<rooms>.*)(?P<min_price>.*)(?P<max_price>.*)(?P<min_surface>.*)(?P<max_surface>.*)', SearchPage)
+    search_one = URL(r'/recherche.html\?localityIds=4-36388&reference=(?P<reference>.*)&hasMoreCriterias=true&searchTypeID=1', SearchPage)
+    housing = URL(r'/[vente|location].*', HousingPage)
+
+    def get_cities(self, pattern):
+        return self.cities.open(term=pattern).iter_cities()
+
+    def search_housings(self, query):
+        type_id = QUERY_TYPES[query.type]
+
+        house_types = []
+        for house_type in query.house_types:
+            if house_type == HOUSE_TYPES.UNKNOWN:
+                house_types = QUERY_HOUSE_TYPES[house_type]
+                break
+            house_types.append(QUERY_HOUSE_TYPES[house_type])
+
+        type_group_ids = ','.join(house_types)
+
+        location_ids = ','.join([city.id for city in query.cities])
+
+        def build_optional_param(query_field, query_string):
+            replace = ''
+            if getattr(query, query_field):
+                replace = '&%s=%s' % (query_string, getattr(query, query_field))
+            return replace
+
+        rooms = ''
+        if query.nb_rooms:
+            rooms = str(query.nb_rooms)
+            for i in range(query.nb_rooms + 1, 6):
+                rooms += ',%s' % str(i)
+            rooms = '&roomComfortIds=%s' % rooms
+
+        reg_exp = {
+            'type_id': type_id,
+            'type_group_ids': type_group_ids,
+            'location_ids': location_ids,
+            'rooms': rooms,
+            'min_price': build_optional_param('cost_min', 'minimumPrice'),
+            'max_price': build_optional_param('cost_max', 'maximumPrice'),
+            'min_surface': build_optional_param('area_min', 'minimumSurface'),
+            'max_surface': build_optional_param('area_max', 'maximumSurface')
+        }
+        return self.search.open(**reg_exp).iter_housings()
+
+    def get_housing(self, housing_id, obj=None):
+        url = self.search_one.open(reference=housing_id).get_housing_url()
+        return self.open(url).page.get_housing(obj=obj)

modules/avendrealouer/constants.py

+from weboob.capabilities.housing import HOUSE_TYPES, POSTS_TYPES
+
+QUERY_TYPES = {
+    POSTS_TYPES.RENT: 2,
+    POSTS_TYPES.SALE: 1,
+    POSTS_TYPES.SHARING: 2,  # There is no special search for shared appartments.
+    POSTS_TYPES.FURNISHED_RENT: 2,
+    POSTS_TYPES.VIAGER: 1
+}
+
+QUERY_HOUSE_TYPES = {
+    HOUSE_TYPES.APART: ['1'],
+    HOUSE_TYPES.HOUSE: ['2'],
+    HOUSE_TYPES.PARKING: ['7'],
+    HOUSE_TYPES.LAND: ['3'],
+    HOUSE_TYPES.OTHER: ['4', '5', '6', '8', '9', '10', '11', '12', '13', '14'],
+    HOUSE_TYPES.UNKNOWN: ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14']
+}

modules/avendrealouer/module.py

+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2017      ZeHiro
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import unicode_literals
+
+
+from weboob.tools.backend import Module
+from weboob.capabilities.housing import CapHousing, Housing
+
+from .browser import AvendrealouerBrowser
+
+
+__all__ = ['AvendrealouerModule']
+
+
+class AvendrealouerModule(Module, CapHousing):
+    NAME = u'avendrealouer'
+    DESCRIPTION = 'avendrealouer website'
+    MAINTAINER = 'ZeHiro'
+    EMAIL = 'public@abossy.fr'
+    LICENSE = 'AGPLv3+'
+    VERSION = '1.4'
+
+    BROWSER = AvendrealouerBrowser
+
+    def get_housing(self, housing):
+        """
+        Get an housing from an ID.
+
+        :param housing: ID of the housing
+        :type housing: str
+        :rtype: :class:`Housing` or None if not found.
+        """
+        return self.browser.get_housing(housing)
+
+    def search_city(self, pattern):
+        """
+        Search a city from a pattern.
+
+        :param pattern: pattern to search
+        :type pattern: str
+        :rtype: iter[:class:`City`]
+        """
+        return self.browser.get_cities(pattern)
+
+    def search_housings(self, query):
+        """
+        Search housings.
+
+        :param query: search query
+        :type query: :class:`Query`
+        :rtype: iter[:class:`Housing`]
+        """
+        return self.browser.search_housings(query)
+
+    def fill_housing(self, housing, fields):
+        if 'photos' in fields and housing.photos:
+            for photo in housing.photos:
+                photo.data = self.browser.open(photo.url)
+        return housing
+
+    OBJECTS = {Housing: fill_housing}

modules/avendrealouer/pages.py

+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2017      ZeHiro
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import unicode_literals
+
+from datetime import datetime
+
+from weboob.browser.pages import HTMLPage, JsonPage, pagination
+from weboob.browser.elements import ItemElement, ListElement, method, DictElement
+from weboob.browser.filters.html import Attr, AbsoluteLink, Link
+from weboob.browser.filters.json import Dict
+
+from weboob.browser.filters.standard import CleanDecimal, CleanText, Date, Regexp, Async, AsyncLoad
+
+from weboob.capabilities.housing import City, Housing, UTILITIES, HousingPhoto
+from weboob.capabilities.base import NotAvailable, Currency
+
+from weboob.tools.capabilities.housing.housing import PricePerMeterFilter
+
+
+class CitiesPage(JsonPage):
+
+    @method
+    class iter_cities(DictElement):
+
+        class item(ItemElement):
+            klass = City
+            obj_id = Dict('Value')
+            obj_name = Dict('Name')
+
+
+class AvendreAlouerItem(ItemElement):
+    klass = Housing
+    _url = AbsoluteLink('.//a[has-class("linkCtnr")]')
+
+    load_details = _url & AsyncLoad
+
+    obj_url = _url
+    obj_id = Async('details') & CleanText(Regexp(CleanText('//p[has-class("property-reference")]'), r'\:(.*)$', default=''))
+
+    obj_title = CleanText('.//a//ul')
+    obj_area = CleanDecimal(
+        CleanText('.//a//ul//li[has-class("first")]//following-sibling::li[2]'),
+        default=NotAvailable
+    )
+
+    obj_cost = CleanDecimal(
+        CleanText('.//span[has-class("price")]')
+    )
+    obj_price_per_meter = PricePerMeterFilter()
+    obj_currency = CleanText(
+        Regexp(
+            CleanText('.//span[has-class("price")]'),
+            r'[\d\ ]+(.*)'
+        )
+    )
+
+    obj_location = CleanText('.//span[has-class("loca")]')
+    obj_text = CleanText('.//p[has-class("propShortDesc")]')
+
+    obj_date = Async('details') & Date(
+        Regexp(
+            CleanText('//div[has-class("property-description-main")]'),
+            r'Mise à jour le ([\d\\]+)', default=datetime.today()
+        )
+    )
+
+    def obj_details(self):
+        page_doc = Async('details').loaded_page(self).doc
+
+        return {
+            'GES': CleanText('//span[@id="gassymbol"]', '')(page_doc),
+            'DPE': CleanText('//span[@id="energysymbol"]', '')(page_doc),
+        }
+
+    def obj_utilities(self):
+        price = CleanText('//span[has-class("price-info")]')(self)
+        if 'CC' in price:
+            return UTILITIES.INCLUDED
+        elif 'HC' in price:
+            return UTILITIES.EXCLUDED
+        else:
+            return UTILITIES.UNKNOWN
+
+    obj_station = 'Test'
+    obj_bedrooms = Async('details') & CleanDecimal(
+        CleanText('.//td//span[contains(text(), "Chambre")]//following-sibling::span[has-class("r")]'),
+        default=NotAvailable
+    )
+
+    obj_rooms = Async('details') & CleanDecimal(
+        CleanText('.//td//span[contains(text(), "Pièce")]//following-sibling::span[has-class("r")]'),
+        default=NotAvailable
+    )
+
+    def obj_photos(self):
+        page_doc = Async('details').loaded_page(self).doc
+        photos = []
+        for photo in page_doc.xpath('//div[@id="bxSliderContainer"]//ul//li//img'):
+            url = Attr('.', 'src')(photo)
+            if url[0] != '/':
+                photos.append(HousingPhoto(url))
+        return photos
+
+    def validate(self, obj):
+        return obj.id != ''
+
+
+class SearchPage(HTMLPage):
+    @pagination
+    @method
+    class iter_housings(ListElement):
+        item_xpath = './/li[@data-tranid="1"]'
+
+        next_page = AbsoluteLink('./ul[has-class("pagination")]/li/a[has-class("next")]')
+
+        class item(AvendreAlouerItem):
+            obj_phone = CleanText(Attr('.', 'data-infos'))
+
+    def get_housing_url(self):
+        return Link('.//a[has-class("picCtnr")]')(self.doc)
+
+
+class HousingPage(HTMLPage):
+    @method
+    class get_housing(ItemElement):
+        klass = Housing
+        obj_id = Regexp(CleanText('//p[has-class("property-reference")]'), r'\:(.*)$')
+
+        def obj_url(self):
+            return self.page.url
+
+        obj_area = CleanDecimal(
+            Regexp(
+                CleanText('//table[@id="table"]//span[contains(text(), "Surface")]//following-sibling::span[has-class("r")]'),
+                r'([\d\ ]+)m'
+            ),
+            default=NotAvailable
+        )
+        obj_title = CleanText('//span[has-class("mainh1")]')
+        obj_cost = CleanDecimal('//span[has-class("price-info")]')
+        obj_currency = Currency.get_currency(u'€')
+        obj_rooms = CleanDecimal('//table[@id="table"]//span[contains(text(), "Pièce")]//following-sibling::span[has-class("r")]')
+        obj_bedrooms = CleanDecimal('//table[@id="table"]//span[contains(text(), "Chambre")]//following-sibling::span[has-class("r")]')
+        obj_location = CleanText(Regexp(CleanText('//span[has-class("mainh1")]'), r',(.+)$'))
+        obj_text = CleanText('//div[has-class("property-description-main")]')
+        obj_date = Date(
+            Regexp(
+                CleanText('//div[has-class("property-description-main")]'),
+                r'Mise à jour le ([\d\\]+)', default=datetime.today()
+            )
+        )
+        obj_phone = Attr('//button[@id="display-phonenumber-1"]', 'data-phone-number')
+
+        def obj_photos(self):
+            photos = []
+            for photo in self.xpath('//div[@id="bxSliderContainer"]//ul//li//img'):
+                url = Attr('.', 'src')(photo)
+                if url[0] != '/':
+                    photos.append(HousingPhoto(url))
+            return photos
+
+        def obj_details(self):
+            return {
+                'GES': CleanText('//span[@id="gassymbol"]', '')(self),
+                'DPE': CleanText('//span[@id="energysymbol"]', '')(self),
+            }
+
+        def obj_utilities(self):
+            price = CleanText('//span[has-class("price-info")]')(self)
+            if 'CC' in price:
+                return UTILITIES.INCLUDED
+            elif 'HC' in price:
+                return UTILITIES.EXCLUDED
+            else:
+                return UTILITIES.UNKNOWN
+
+        obj_station = NotAvailable
+        obj_price_per_meter = PricePerMeterFilter()

modules/avendrealouer/test.py

+# -*- coding: utf-8 -*-
+
+# Copyright(C) 2017      ZeHiro
+#
+# This file is part of weboob.
+#
+# weboob is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# weboob is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with weboob. If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import unicode_literals
+
+
+from weboob.tools.test import BackendTest
+from weboob.tools.capabilities.housing.housing_test import HousingTest
+from weboob.capabilities.housing import Query, POSTS_TYPES
+
+
+class AvendrealouerTest(BackendTest, HousingTest):
+    MODULE = 'avendrealouer'
+
+    DO_NOT_DISTINGUISH_FURNISHED_RENT = True
+
+    def test_avendre_rent(self):
+        query = Query()
+        query.area_min = 20
+        query.cost_max = 1500
+        query.type = POSTS_TYPES.RENT
+        query.cities = []
+        for city in self.backend.search_city('paris'):
+            city.backend = self.backend.name
+            query.cities.append(city)
+        self.check_against_query(query)
+
+    def test_avendre_sale(self):
+        query = Query()
+        query.area_min = 20
+        query.type = POSTS_TYPES.SALE
+        query.cities = []
+        for city in self.backend.search_city('paris'):
+            city.backend = self.backend.name
+            query.cities.append(city)
+        self.check_against_query(query)

tools/py3-compatible.modules

@@ -14,6 +14,7 @@ anticaptcha
 amundi
 apec
 arte
+avendrealouer
 axabanque
 bandcamp
 banqueaccord