[leboncoin] add new housing module laboncoin
This commit is contained in:
Bezleputh
parent
7304f1dee1
commit
6cd0515ffe
5 changed files with 345 additions and 0 deletions
24
modules/leboncoin/__init__.py
Normal file
24
modules/leboncoin/__init__.py
Normal file
|
|
@ -0,0 +1,24 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
# Copyright(C) 2014 Bezleputh
|
||||||
|
#
|
||||||
|
# This file is part of weboob.
|
||||||
|
#
|
||||||
|
# weboob is free software: you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU Affero General Public License as published by
|
||||||
|
# the Free Software Foundation, either version 3 of the License, or
|
||||||
|
# (at your option) any later version.
|
||||||
|
#
|
||||||
|
# weboob is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU Affero General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
|
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
|
||||||
|
from .backend import LeboncoinBackend
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ['LeboncoinBackend']
|
||||||
86
modules/leboncoin/backend.py
Normal file
86
modules/leboncoin/backend.py
Normal file
|
|
@ -0,0 +1,86 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
# Copyright(C) 2014 Bezleputh
|
||||||
|
#
|
||||||
|
# This file is part of weboob.
|
||||||
|
#
|
||||||
|
# weboob is free software: you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU Affero General Public License as published by
|
||||||
|
# the Free Software Foundation, either version 3 of the License, or
|
||||||
|
# (at your option) any later version.
|
||||||
|
#
|
||||||
|
# weboob is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU Affero General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
|
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
|
||||||
|
from weboob.tools.backend import BaseBackend
|
||||||
|
from weboob.capabilities.housing import CapHousing, Query, Housing, HousingPhoto
|
||||||
|
|
||||||
|
from .browser import LeboncoinBrowser
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ['LeboncoinBackend']
|
||||||
|
|
||||||
|
|
||||||
|
class LeboncoinBackend(BaseBackend, CapHousing):
|
||||||
|
NAME = 'leboncoin'
|
||||||
|
DESCRIPTION = u'search house on leboncoin website'
|
||||||
|
MAINTAINER = u'Bezleputh'
|
||||||
|
EMAIL = 'carton_ben@yahoo.fr'
|
||||||
|
LICENSE = 'AGPLv3+'
|
||||||
|
VERSION = '1.0'
|
||||||
|
|
||||||
|
BROWSER = LeboncoinBrowser
|
||||||
|
|
||||||
|
RET = {Query.HOUSE_TYPES.HOUSE: '1',
|
||||||
|
Query.HOUSE_TYPES.APART: '2',
|
||||||
|
Query.HOUSE_TYPES.LAND: '3',
|
||||||
|
Query.HOUSE_TYPES.PARKING: '4',
|
||||||
|
Query.HOUSE_TYPES.OTHER: '5'}
|
||||||
|
|
||||||
|
def get_housing(self, _id):
|
||||||
|
return self.browser.get_housing(_id)
|
||||||
|
|
||||||
|
def fill_housing(self, housing, fields):
|
||||||
|
return self.browser.get_housing(housing.id)
|
||||||
|
|
||||||
|
def fill_photo(self, photo, fields):
|
||||||
|
if 'data' in fields and photo.url and not photo.data:
|
||||||
|
photo.data = self.browser.readurl(photo.url)
|
||||||
|
return photo
|
||||||
|
|
||||||
|
def search_city(self, pattern):
|
||||||
|
return self.browser.get_cities(pattern)
|
||||||
|
|
||||||
|
def search_housings(self, query):
|
||||||
|
cities = []
|
||||||
|
for c in query.cities:
|
||||||
|
cities.append('%s %s' % (c.id, c.name))
|
||||||
|
|
||||||
|
if len(cities) == 0:
|
||||||
|
return list()
|
||||||
|
|
||||||
|
ret = []
|
||||||
|
for g in query.house_types:
|
||||||
|
ret.append(self.RET.get(g))
|
||||||
|
|
||||||
|
if len(ret) == 0:
|
||||||
|
return list()
|
||||||
|
|
||||||
|
_type = query.TYPE_RENT if query.type is None else query.type
|
||||||
|
nb_rooms = '' if not query.nb_rooms else query.nb_rooms
|
||||||
|
area_min = '' if not query.area_min else query.area_min
|
||||||
|
area_max = '' if not query.area_max else query.area_max
|
||||||
|
cost_min = '' if not query.cost_min else query.cost_min
|
||||||
|
cost_max = '' if not query.cost_max else query.cost_max
|
||||||
|
|
||||||
|
return self.browser.search_housings(_type, ','.join(cities), nb_rooms,
|
||||||
|
area_min, area_max,
|
||||||
|
cost_min, cost_max, '&ret='.join(ret))
|
||||||
|
|
||||||
|
OBJECTS = {Housing: fill_housing, HousingPhoto: fill_photo}
|
||||||
55
modules/leboncoin/browser.py
Normal file
55
modules/leboncoin/browser.py
Normal file
|
|
@ -0,0 +1,55 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
# Copyright(C) 2014 Bezleputh
|
||||||
|
#
|
||||||
|
# This file is part of weboob.
|
||||||
|
#
|
||||||
|
# weboob is free software: you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU Affero General Public License as published by
|
||||||
|
# the Free Software Foundation, either version 3 of the License, or
|
||||||
|
# (at your option) any later version.
|
||||||
|
#
|
||||||
|
# weboob is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU Affero General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
|
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
|
||||||
|
from weboob.tools.browser2 import PagesBrowser, URL
|
||||||
|
|
||||||
|
from .pages import CityListPage, HousingListPage, HousingPage
|
||||||
|
|
||||||
|
|
||||||
|
class LeboncoinBrowser(PagesBrowser):
|
||||||
|
BASEURL = 'http://www.leboncoin.fr'
|
||||||
|
city = URL('ajax/location_list.html\?city=(?P<city>.*)&zipcode=(?P<zip>.*)', CityListPage)
|
||||||
|
search = URL('ventes_immobilieres/offres/ile_de_france/occasions/\?ps=(?P<ps>.*)&pe=(?P<pe>.*)&ros=(?P<ros>.*)&location=(?P<location>.*)&sqs=(?P<sqs>.*)&sqe=(?P<sqe>.*)&ret=(?P<ret>.*)',
|
||||||
|
'ventes_immobilieres/offres/ile_de_france/occasions/\?.*',
|
||||||
|
HousingListPage)
|
||||||
|
housing = URL('ventes_immobilieres/(?P<_id>.*).htm', HousingPage)
|
||||||
|
|
||||||
|
def get_cities(self, pattern):
|
||||||
|
city = ''
|
||||||
|
zip_code = ''
|
||||||
|
if pattern.isdigit():
|
||||||
|
zip_code = pattern
|
||||||
|
else:
|
||||||
|
city = pattern
|
||||||
|
|
||||||
|
return self.city.go(city=city, zip=zip_code).get_cities()
|
||||||
|
|
||||||
|
def search_housings(self, type, cities, nb_rooms, area_min, area_max, cost_min, cost_max, ret):
|
||||||
|
# print type achat ou location
|
||||||
|
return self.search.go(location=cities,
|
||||||
|
ros=nb_rooms,
|
||||||
|
sqs=area_min,
|
||||||
|
sqe=area_max,
|
||||||
|
ps=cost_min,
|
||||||
|
pe=cost_max,
|
||||||
|
ret=ret).get_housing_list()
|
||||||
|
|
||||||
|
def get_housing(self, _id):
|
||||||
|
return self.housing.go(_id=_id).get_housing()
|
||||||
141
modules/leboncoin/pages.py
Normal file
141
modules/leboncoin/pages.py
Normal file
|
|
@ -0,0 +1,141 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
# Copyright(C) 2014 Bezleputh
|
||||||
|
#
|
||||||
|
# This file is part of weboob.
|
||||||
|
#
|
||||||
|
# weboob is free software: you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU Affero General Public License as published by
|
||||||
|
# the Free Software Foundation, either version 3 of the License, or
|
||||||
|
# (at your option) any later version.
|
||||||
|
#
|
||||||
|
# weboob is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU Affero General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
|
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
from decimal import Decimal
|
||||||
|
from weboob.tools.browser2.page import HTMLPage, method, pagination
|
||||||
|
from weboob.tools.browser2.elements import ItemElement, ListElement
|
||||||
|
from weboob.tools.browser2.filters import CleanText, Link, Regexp, CleanDecimal, Env, DateTime, Attr
|
||||||
|
from weboob.capabilities.housing import City, Housing, HousingPhoto
|
||||||
|
from datetime import date
|
||||||
|
from weboob.tools.date import DATE_TRANSLATE_FR, LinearDateGuesser
|
||||||
|
|
||||||
|
|
||||||
|
class CityListPage(HTMLPage):
|
||||||
|
@method
|
||||||
|
class get_cities(ListElement):
|
||||||
|
item_xpath = '//li'
|
||||||
|
|
||||||
|
class item(ItemElement):
|
||||||
|
klass = City
|
||||||
|
|
||||||
|
obj_id = CleanText('./span[@class="zipcode"]')
|
||||||
|
obj_name = CleanText('./span[@class="city"]')
|
||||||
|
|
||||||
|
|
||||||
|
class HousingListPage(HTMLPage):
|
||||||
|
@pagination
|
||||||
|
@method
|
||||||
|
class get_housing_list(ListElement):
|
||||||
|
item_xpath = '//div[@class="list-lbc"]/a'
|
||||||
|
|
||||||
|
def next_page(self):
|
||||||
|
return Link('//li[@class="page"]/a')(self)
|
||||||
|
|
||||||
|
class item(ItemElement):
|
||||||
|
klass = Housing
|
||||||
|
|
||||||
|
obj_id = Regexp(Link('.'), 'http://www.leboncoin.fr/ventes_immobilieres/(.*).htm')
|
||||||
|
obj_title = CleanText('./div[@class="lbc"]/div/div[@class="title"]')
|
||||||
|
obj_cost = CleanDecimal('./div[@class="lbc"]/div/div[@class="price"]',
|
||||||
|
replace_dots=(',', '.'),
|
||||||
|
default=Decimal(0))
|
||||||
|
obj_currency = Regexp(CleanText('./div[@class="lbc"]/div/div[@class="price"]'),
|
||||||
|
'.*([%s%s%s])' % (u'€', u'$', u'£'), default=u'€')
|
||||||
|
obj_text = CleanText('./div[@class="lbc"]/div[@class="detail"]')
|
||||||
|
|
||||||
|
def obj_date(self):
|
||||||
|
_date = CleanText('./div[@class="lbc"]/div[@class="date"]',
|
||||||
|
replace=[('Aujourd\'hui', str(date.today().day))])(self)
|
||||||
|
for fr, en in DATE_TRANSLATE_FR:
|
||||||
|
_date = fr.sub(en, _date)
|
||||||
|
|
||||||
|
self.env['tmp'] = _date
|
||||||
|
return DateTime(Env('tmp'), LinearDateGuesser())(self)
|
||||||
|
|
||||||
|
def obj_photos(self):
|
||||||
|
photos = []
|
||||||
|
url = Attr('./div[@class="lbc"]/div[@class="image"]/div/img', 'src', default=None)(self)
|
||||||
|
if url:
|
||||||
|
photos.append(HousingPhoto(url))
|
||||||
|
return photos
|
||||||
|
|
||||||
|
|
||||||
|
class HousingPage(HTMLPage):
|
||||||
|
@method
|
||||||
|
class get_housing(ItemElement):
|
||||||
|
klass = Housing
|
||||||
|
|
||||||
|
def parse(self, el):
|
||||||
|
details = dict()
|
||||||
|
for tr in el.xpath('//div[@class="floatLeft"]/table/tr'):
|
||||||
|
if 'Ville' in CleanText('./th')(tr):
|
||||||
|
self.env['location'] = CleanText('./td')(tr)
|
||||||
|
else:
|
||||||
|
details['%s' % CleanText('./th', replace=[(':', '')])(tr)] = CleanText('./td')(tr)
|
||||||
|
|
||||||
|
for tr in el.xpath('//div[@class="lbcParams criterias"]/table/tr'):
|
||||||
|
if 'Surface' in CleanText('./th')(tr):
|
||||||
|
self.env['area'] = CleanDecimal(Regexp(CleanText('./td'), '(.*)m.*'),
|
||||||
|
replace_dots=(',', '.'))(tr)
|
||||||
|
else:
|
||||||
|
key = '%s' % CleanText('./th', replace=[(':', '')])(tr)
|
||||||
|
if 'GES' in key or 'Classe' in key:
|
||||||
|
details[key] = CleanText('./td/noscript/a')(tr)
|
||||||
|
else:
|
||||||
|
details[key] = CleanText('./td')(tr)
|
||||||
|
|
||||||
|
self.env['details'] = details
|
||||||
|
|
||||||
|
obj_id = Env('_id')
|
||||||
|
obj_title = CleanText('//h2[@id="ad_subject"]')
|
||||||
|
obj_cost = CleanDecimal('//span[@class="price"]', replace_dots=(',', '.'), default=Decimal(0))
|
||||||
|
|
||||||
|
obj_currency = Regexp(CleanText('//span[@class="price"]'),
|
||||||
|
'.*([%s%s%s])' % (u'€', u'$', u'£'))
|
||||||
|
obj_text = CleanText('//div[@class="content"]')
|
||||||
|
obj_location = Env('location')
|
||||||
|
obj_details = Env('details')
|
||||||
|
obj_area = Env('area')
|
||||||
|
|
||||||
|
def obj_date(self):
|
||||||
|
sender = CleanText('//div[@class="upload_by"]/a')(self)
|
||||||
|
_date = CleanText('//div[@class="upload_by"]',
|
||||||
|
replace=[('- Mise en ligne le ', ''),
|
||||||
|
(sender, ''),
|
||||||
|
(u'à', ''),
|
||||||
|
(u'.', '')])(self)
|
||||||
|
|
||||||
|
for fr, en in DATE_TRANSLATE_FR:
|
||||||
|
_date = fr.sub(en, _date)
|
||||||
|
|
||||||
|
self.env['tmp'] = _date
|
||||||
|
return DateTime(Env('tmp'), LinearDateGuesser())(self)
|
||||||
|
|
||||||
|
def obj_photos(self):
|
||||||
|
photos = []
|
||||||
|
for img in self.el.xpath('//div[@id="thumbs_carousel"]/a/span'):
|
||||||
|
url = CleanText(Regexp(Attr('.', 'style',
|
||||||
|
default=''),
|
||||||
|
"background-image: url\('(.*)'\);",
|
||||||
|
default=''),
|
||||||
|
replace=[('thumbs', 'images')],
|
||||||
|
default='')(img)
|
||||||
|
if url:
|
||||||
|
photos.append(HousingPhoto(url))
|
||||||
|
return photos
|
||||||
39
modules/leboncoin/test.py
Normal file
39
modules/leboncoin/test.py
Normal file
|
|
@ -0,0 +1,39 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
# Copyright(C) 2014 Bezleputh
|
||||||
|
#
|
||||||
|
# This file is part of weboob.
|
||||||
|
#
|
||||||
|
# weboob is free software: you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU Affero General Public License as published by
|
||||||
|
# the Free Software Foundation, either version 3 of the License, or
|
||||||
|
# (at your option) any later version.
|
||||||
|
#
|
||||||
|
# weboob is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU Affero General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
|
# along with weboob. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
import itertools
|
||||||
|
from weboob.tools.test import BackendTest
|
||||||
|
from weboob.capabilities.housing import Query
|
||||||
|
|
||||||
|
|
||||||
|
class LeboncoinTest(BackendTest):
|
||||||
|
BACKEND = 'leboncoin'
|
||||||
|
|
||||||
|
def test_leboncoin(self):
|
||||||
|
query = Query()
|
||||||
|
query.cities = []
|
||||||
|
for city in self.backend.search_city('lille'):
|
||||||
|
city.backend = self.backend.name
|
||||||
|
query.cities.append(city)
|
||||||
|
|
||||||
|
results = list(itertools.islice(self.backend.search_housings(query), 0, 20))
|
||||||
|
self.assertTrue(len(results) > 0)
|
||||||
|
|
||||||
|
obj = self.backend.fillobj(results[0])
|
||||||
|
self.assertTrue(obj.area is not None, 'Area for "%s"' % (obj.id))
|
||||||
Loading…
Add table
Add a link
Reference in a new issue