Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
weboob
Project overview
Project overview
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
177
Issues
177
List
Boards
Labels
Milestones
Merge Requests
49
Merge Requests
49
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
weboob
weboob
Commits
fc71f1cf
Commit
fc71f1cf
authored
Nov 28, 2017
by
Antoine BOSSY
Committed by
Romain Bignon
Jan 01, 2019
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Create housing module for avendrealouer.fr
parent
209c4008
Pipeline
#1854
failed with stages
in 0 seconds
Changes
7
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Showing
7 changed files
with
452 additions
and
0 deletions
+452
-0
modules/avendrealouer/__init__.py
modules/avendrealouer/__init__.py
+26
-0
modules/avendrealouer/browser.py
modules/avendrealouer/browser.py
+82
-0
modules/avendrealouer/constants.py
modules/avendrealouer/constants.py
+18
-0
modules/avendrealouer/module.py
modules/avendrealouer/module.py
+78
-0
modules/avendrealouer/pages.py
modules/avendrealouer/pages.py
+195
-0
modules/avendrealouer/test.py
modules/avendrealouer/test.py
+52
-0
tools/py3-compatible.modules
tools/py3-compatible.modules
+1
-0
No files found.
modules/avendrealouer/__init__.py
0 → 100644
View file @
fc71f1cf
# -*- coding: utf-8 -*-
# Copyright(C) 2017 ZeHiro
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from
__future__
import
unicode_literals
from
.module
import
AvendrealouerModule
__all__
=
[
'AvendrealouerModule'
]
modules/avendrealouer/browser.py
0 → 100644
View file @
fc71f1cf
# -*- coding: utf-8 -*-
# Copyright(C) 2017 ZeHiro
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from
__future__
import
unicode_literals
from
weboob.browser
import
PagesBrowser
,
URL
from
weboob.capabilities.housing
import
HOUSE_TYPES
from
.pages
import
CitiesPage
,
SearchPage
,
HousingPage
from
.constants
import
QUERY_TYPES
,
QUERY_HOUSE_TYPES
class
AvendrealouerBrowser
(
PagesBrowser
):
BASEURL
=
'https://www.avendrealouer.fr'
cities
=
URL
(
r'/common/api/localities\?term=(?P<term>)'
,
CitiesPage
)
search
=
URL
(
r'/recherche.html\?pageIndex=1&sortPropertyName=Price&sortDirection=Ascending&searchTypeID=(?P<type_id>.*)&typeGroupCategoryID=1&transactionId=1&localityIds=(?P<location_ids>.*)&typeGroupIds=(?P<type_group_ids>.*)(?P<rooms>.*)(?P<min_price>.*)(?P<max_price>.*)(?P<min_surface>.*)(?P<max_surface>.*)'
,
SearchPage
)
search_one
=
URL
(
r'/recherche.html\?localityIds=4-36388&reference=(?P<reference>.*)&hasMoreCriterias=true&searchTypeID=1'
,
SearchPage
)
housing
=
URL
(
r'/[vente|location].*'
,
HousingPage
)
def
get_cities
(
self
,
pattern
):
return
self
.
cities
.
open
(
term
=
pattern
)
.
iter_cities
()
def
search_housings
(
self
,
query
):
type_id
=
QUERY_TYPES
[
query
.
type
]
house_types
=
[]
for
house_type
in
query
.
house_types
:
if
house_type
==
HOUSE_TYPES
.
UNKNOWN
:
house_types
=
QUERY_HOUSE_TYPES
[
house_type
]
break
house_types
.
append
(
QUERY_HOUSE_TYPES
[
house_type
])
type_group_ids
=
','
.
join
(
house_types
)
location_ids
=
','
.
join
([
city
.
id
for
city
in
query
.
cities
])
def
build_optional_param
(
query_field
,
query_string
):
replace
=
''
if
getattr
(
query
,
query_field
):
replace
=
'&
%
s=
%
s'
%
(
query_string
,
getattr
(
query
,
query_field
))
return
replace
rooms
=
''
if
query
.
nb_rooms
:
rooms
=
str
(
query
.
nb_rooms
)
for
i
in
range
(
query
.
nb_rooms
+
1
,
6
):
rooms
+=
',
%
s'
%
str
(
i
)
rooms
=
'&roomComfortIds=
%
s'
%
rooms
reg_exp
=
{
'type_id'
:
type_id
,
'type_group_ids'
:
type_group_ids
,
'location_ids'
:
location_ids
,
'rooms'
:
rooms
,
'min_price'
:
build_optional_param
(
'cost_min'
,
'minimumPrice'
),
'max_price'
:
build_optional_param
(
'cost_max'
,
'maximumPrice'
),
'min_surface'
:
build_optional_param
(
'area_min'
,
'minimumSurface'
),
'max_surface'
:
build_optional_param
(
'area_max'
,
'maximumSurface'
)
}
return
self
.
search
.
open
(
**
reg_exp
)
.
iter_housings
()
def
get_housing
(
self
,
housing_id
,
obj
=
None
):
url
=
self
.
search_one
.
open
(
reference
=
housing_id
)
.
get_housing_url
()
return
self
.
open
(
url
)
.
page
.
get_housing
(
obj
=
obj
)
modules/avendrealouer/constants.py
0 → 100644
View file @
fc71f1cf
from
weboob.capabilities.housing
import
HOUSE_TYPES
,
POSTS_TYPES
QUERY_TYPES
=
{
POSTS_TYPES
.
RENT
:
2
,
POSTS_TYPES
.
SALE
:
1
,
POSTS_TYPES
.
SHARING
:
2
,
# There is no special search for shared appartments.
POSTS_TYPES
.
FURNISHED_RENT
:
2
,
POSTS_TYPES
.
VIAGER
:
1
}
QUERY_HOUSE_TYPES
=
{
HOUSE_TYPES
.
APART
:
[
'1'
],
HOUSE_TYPES
.
HOUSE
:
[
'2'
],
HOUSE_TYPES
.
PARKING
:
[
'7'
],
HOUSE_TYPES
.
LAND
:
[
'3'
],
HOUSE_TYPES
.
OTHER
:
[
'4'
,
'5'
,
'6'
,
'8'
,
'9'
,
'10'
,
'11'
,
'12'
,
'13'
,
'14'
],
HOUSE_TYPES
.
UNKNOWN
:
[
'1'
,
'2'
,
'3'
,
'4'
,
'5'
,
'6'
,
'7'
,
'8'
,
'9'
,
'10'
,
'11'
,
'12'
,
'13'
,
'14'
]
}
modules/avendrealouer/module.py
0 → 100644
View file @
fc71f1cf
# -*- coding: utf-8 -*-
# Copyright(C) 2017 ZeHiro
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from
__future__
import
unicode_literals
from
weboob.tools.backend
import
Module
from
weboob.capabilities.housing
import
CapHousing
,
Housing
from
.browser
import
AvendrealouerBrowser
__all__
=
[
'AvendrealouerModule'
]
class
AvendrealouerModule
(
Module
,
CapHousing
):
NAME
=
u'avendrealouer'
DESCRIPTION
=
'avendrealouer website'
MAINTAINER
=
'ZeHiro'
EMAIL
=
'public@abossy.fr'
LICENSE
=
'AGPLv3+'
VERSION
=
'1.4'
BROWSER
=
AvendrealouerBrowser
def
get_housing
(
self
,
housing
):
"""
Get an housing from an ID.
:param housing: ID of the housing
:type housing: str
:rtype: :class:`Housing` or None if not found.
"""
return
self
.
browser
.
get_housing
(
housing
)
def
search_city
(
self
,
pattern
):
"""
Search a city from a pattern.
:param pattern: pattern to search
:type pattern: str
:rtype: iter[:class:`City`]
"""
return
self
.
browser
.
get_cities
(
pattern
)
def
search_housings
(
self
,
query
):
"""
Search housings.
:param query: search query
:type query: :class:`Query`
:rtype: iter[:class:`Housing`]
"""
return
self
.
browser
.
search_housings
(
query
)
def
fill_housing
(
self
,
housing
,
fields
):
if
'photos'
in
fields
and
housing
.
photos
:
for
photo
in
housing
.
photos
:
photo
.
data
=
self
.
browser
.
open
(
photo
.
url
)
return
housing
OBJECTS
=
{
Housing
:
fill_housing
}
modules/avendrealouer/pages.py
0 → 100644
View file @
fc71f1cf
# -*- coding: utf-8 -*-
# Copyright(C) 2017 ZeHiro
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from
__future__
import
unicode_literals
from
datetime
import
datetime
from
weboob.browser.pages
import
HTMLPage
,
JsonPage
,
pagination
from
weboob.browser.elements
import
ItemElement
,
ListElement
,
method
,
DictElement
from
weboob.browser.filters.html
import
Attr
,
AbsoluteLink
,
Link
from
weboob.browser.filters.json
import
Dict
from
weboob.browser.filters.standard
import
CleanDecimal
,
CleanText
,
Date
,
Regexp
,
Async
,
AsyncLoad
from
weboob.capabilities.housing
import
City
,
Housing
,
UTILITIES
,
HousingPhoto
from
weboob.capabilities.base
import
NotAvailable
,
Currency
from
weboob.tools.capabilities.housing.housing
import
PricePerMeterFilter
class
CitiesPage
(
JsonPage
):
@
method
class
iter_cities
(
DictElement
):
class
item
(
ItemElement
):
klass
=
City
obj_id
=
Dict
(
'Value'
)
obj_name
=
Dict
(
'Name'
)
class
AvendreAlouerItem
(
ItemElement
):
klass
=
Housing
_url
=
AbsoluteLink
(
'.//a[has-class("linkCtnr")]'
)
load_details
=
_url
&
AsyncLoad
obj_url
=
_url
obj_id
=
Async
(
'details'
)
&
CleanText
(
Regexp
(
CleanText
(
'//p[has-class("property-reference")]'
),
r'\:(.*)$'
,
default
=
''
))
obj_title
=
CleanText
(
'.//a//ul'
)
obj_area
=
CleanDecimal
(
CleanText
(
'.//a//ul//li[has-class("first")]//following-sibling::li[2]'
),
default
=
NotAvailable
)
obj_cost
=
CleanDecimal
(
CleanText
(
'.//span[has-class("price")]'
)
)
obj_price_per_meter
=
PricePerMeterFilter
()
obj_currency
=
CleanText
(
Regexp
(
CleanText
(
'.//span[has-class("price")]'
),
r'[\d\ ]+(.*)'
)
)
obj_location
=
CleanText
(
'.//span[has-class("loca")]'
)
obj_text
=
CleanText
(
'.//p[has-class("propShortDesc")]'
)
obj_date
=
Async
(
'details'
)
&
Date
(
Regexp
(
CleanText
(
'//div[has-class("property-description-main")]'
),
r'Mise à jour le ([\d\\]+)'
,
default
=
datetime
.
today
()
)
)
def
obj_details
(
self
):
page_doc
=
Async
(
'details'
)
.
loaded_page
(
self
)
.
doc
return
{
'GES'
:
CleanText
(
'//span[@id="gassymbol"]'
,
''
)(
page_doc
),
'DPE'
:
CleanText
(
'//span[@id="energysymbol"]'
,
''
)(
page_doc
),
}
def
obj_utilities
(
self
):
price
=
CleanText
(
'//span[has-class("price-info")]'
)(
self
)
if
'CC'
in
price
:
return
UTILITIES
.
INCLUDED
elif
'HC'
in
price
:
return
UTILITIES
.
EXCLUDED
else
:
return
UTILITIES
.
UNKNOWN
obj_station
=
'Test'
obj_bedrooms
=
Async
(
'details'
)
&
CleanDecimal
(
CleanText
(
'.//td//span[contains(text(), "Chambre")]//following-sibling::span[has-class("r")]'
),
default
=
NotAvailable
)
obj_rooms
=
Async
(
'details'
)
&
CleanDecimal
(
CleanText
(
'.//td//span[contains(text(), "Pièce")]//following-sibling::span[has-class("r")]'
),
default
=
NotAvailable
)
def
obj_photos
(
self
):
page_doc
=
Async
(
'details'
)
.
loaded_page
(
self
)
.
doc
photos
=
[]
for
photo
in
page_doc
.
xpath
(
'//div[@id="bxSliderContainer"]//ul//li//img'
):
url
=
Attr
(
'.'
,
'src'
)(
photo
)
if
url
[
0
]
!=
'/'
:
photos
.
append
(
HousingPhoto
(
url
))
return
photos
def
validate
(
self
,
obj
):
return
obj
.
id
!=
''
class
SearchPage
(
HTMLPage
):
@
pagination
@
method
class
iter_housings
(
ListElement
):
item_xpath
=
'.//li[@data-tranid="1"]'
next_page
=
AbsoluteLink
(
'./ul[has-class("pagination")]/li/a[has-class("next")]'
)
class
item
(
AvendreAlouerItem
):
obj_phone
=
CleanText
(
Attr
(
'.'
,
'data-infos'
))
def
get_housing_url
(
self
):
return
Link
(
'.//a[has-class("picCtnr")]'
)(
self
.
doc
)
class
HousingPage
(
HTMLPage
):
@
method
class
get_housing
(
ItemElement
):
klass
=
Housing
obj_id
=
Regexp
(
CleanText
(
'//p[has-class("property-reference")]'
),
r'\:(.*)$'
)
def
obj_url
(
self
):
return
self
.
page
.
url
obj_area
=
CleanDecimal
(
Regexp
(
CleanText
(
'//table[@id="table"]//span[contains(text(), "Surface")]//following-sibling::span[has-class("r")]'
),
r'([\d\ ]+)m'
),
default
=
NotAvailable
)
obj_title
=
CleanText
(
'//span[has-class("mainh1")]'
)
obj_cost
=
CleanDecimal
(
'//span[has-class("price-info")]'
)
obj_currency
=
Currency
.
get_currency
(
u'€'
)
obj_rooms
=
CleanDecimal
(
'//table[@id="table"]//span[contains(text(), "Pièce")]//following-sibling::span[has-class("r")]'
)
obj_bedrooms
=
CleanDecimal
(
'//table[@id="table"]//span[contains(text(), "Chambre")]//following-sibling::span[has-class("r")]'
)
obj_location
=
CleanText
(
Regexp
(
CleanText
(
'//span[has-class("mainh1")]'
),
r',(.+)$'
))
obj_text
=
CleanText
(
'//div[has-class("property-description-main")]'
)
obj_date
=
Date
(
Regexp
(
CleanText
(
'//div[has-class("property-description-main")]'
),
r'Mise à jour le ([\d\\]+)'
,
default
=
datetime
.
today
()
)
)
obj_phone
=
Attr
(
'//button[@id="display-phonenumber-1"]'
,
'data-phone-number'
)
def
obj_photos
(
self
):
photos
=
[]
for
photo
in
self
.
xpath
(
'//div[@id="bxSliderContainer"]//ul//li//img'
):
url
=
Attr
(
'.'
,
'src'
)(
photo
)
if
url
[
0
]
!=
'/'
:
photos
.
append
(
HousingPhoto
(
url
))
return
photos
def
obj_details
(
self
):
return
{
'GES'
:
CleanText
(
'//span[@id="gassymbol"]'
,
''
)(
self
),
'DPE'
:
CleanText
(
'//span[@id="energysymbol"]'
,
''
)(
self
),
}
def
obj_utilities
(
self
):
price
=
CleanText
(
'//span[has-class("price-info")]'
)(
self
)
if
'CC'
in
price
:
return
UTILITIES
.
INCLUDED
elif
'HC'
in
price
:
return
UTILITIES
.
EXCLUDED
else
:
return
UTILITIES
.
UNKNOWN
obj_station
=
NotAvailable
obj_price_per_meter
=
PricePerMeterFilter
()
modules/avendrealouer/test.py
0 → 100644
View file @
fc71f1cf
# -*- coding: utf-8 -*-
# Copyright(C) 2017 ZeHiro
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
from
__future__
import
unicode_literals
from
weboob.tools.test
import
BackendTest
from
weboob.tools.capabilities.housing.housing_test
import
HousingTest
from
weboob.capabilities.housing
import
Query
,
POSTS_TYPES
class
AvendrealouerTest
(
BackendTest
,
HousingTest
):
MODULE
=
'avendrealouer'
DO_NOT_DISTINGUISH_FURNISHED_RENT
=
True
def
test_avendre_rent
(
self
):
query
=
Query
()
query
.
area_min
=
20
query
.
cost_max
=
1500
query
.
type
=
POSTS_TYPES
.
RENT
query
.
cities
=
[]
for
city
in
self
.
backend
.
search_city
(
'paris'
):
city
.
backend
=
self
.
backend
.
name
query
.
cities
.
append
(
city
)
self
.
check_against_query
(
query
)
def
test_avendre_sale
(
self
):
query
=
Query
()
query
.
area_min
=
20
query
.
type
=
POSTS_TYPES
.
SALE
query
.
cities
=
[]
for
city
in
self
.
backend
.
search_city
(
'paris'
):
city
.
backend
=
self
.
backend
.
name
query
.
cities
.
append
(
city
)
self
.
check_against_query
(
query
)
tools/py3-compatible.modules
View file @
fc71f1cf
...
...
@@ -14,6 +14,7 @@ anticaptcha
amundi
apec
arte
avendrealouer
axabanque
bandcamp
banqueaccord
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment