用python的requests，bs4等库爬取了小猪短租网的杭州租房信息。
注意
- 有些房子居然没有写地址，这房还怎么租。
- python的UnboundLocalError： local variable 'xxx' referenced b
参考 博客和博客
其他一些应注意的点写在注释里
这里总共获取了300多条信息，下一步应该考虑把这些信息存储在mongdb里面，并且采用多进程在加速爬取速度。
import requests,time
from bs4 import BeautifulSoup
start_urls = ['http://hz.xiaozhu.com/search-duanzufang-p{}-0/'.format(str(i)) for i in range(1,14)] #摘取13页的信息
count = 0
urls = []
def get_info(url):
try:
r = requests.get(url,timeout=30)
r.raise_for_status()
r.encoding = 'utf-8'
except:
print('error')
soup = BeautifulSoup(r.text,'lxml')
title = soup.select('div.pho_info > h4')
address = soup.select('span.pr5')
price = soup.select('div#pricePart > div.day_l > span')
name = soup.select('a.lorder_name')
sex = 'boy' if soup.findAll('span',class_='member_boy_ico') else 'girl' #简单的判断男女性别
houseImage = soup.select('img#curBigImage')
menberImage = soup.select('div.member_pic > a > img')
info ={
'title':title[0].get_text().strip(),
'address':address[0].get_text().strip() if address else '杭州', #为防止地址不存在，特地加以判断
'price':price[0].get_text(),
'name':name[0].get_text().strip(),
'sex':sex,
'houseImage':houseImage[0].get('src'),
'menberImage':menberImage[0].get('src')
}
global count #全局变量在局部变量使用时需要指定global
print(count)
count = count + 1
print(info)
def get_urls(url,urls):
r = requests.get(url,timeout = 30)
soup = BeautifulSoup(r.text,'lxml')
links = soup.select('a.resule_img_a')
for link in links:
urls.append(link.get('href'))
def main():
for url in start_urls: # 获取所有url
get_urls(url,urls)
for url in urls: #对每个url进行信息获取
get_info(url)
if __name__ == '__main__':
main()
更新：
把数据存入数据库mongdb
import requests,time,pymongo
from bs4 import BeautifulSoup
start_urls = ['http://hz.xiaozhu.com/search-duanzufang-p{}-0/'.format(str(i)) for i in range(1,14)] #摘取13页的信息
count = 0
urls = []
client = pymongo.MongoClient('localhost',27017) #打开数据库客户端
xiaozhu = client.xiaozhu
hz = xiaozhu.hz
def get_info(url):
try:
r = requests.get(url,timeout=30)
r.raise_for_status()
r.encoding = 'utf-8'
except:
print('error')
soup = BeautifulSoup(r.text,'lxml')
title = soup.select('div.pho_info > h4')
address = soup.select('span.pr5')
price = soup.select('div#pricePart > div.day_l > span')
name = soup.select('a.lorder_name')
sex = 'boy' if soup.findAll('span',class_='member_boy_ico') else 'girl' #简单的判断男女性别
houseImage = soup.select('img#curBigImage')
menberImage = soup.select('div.member_pic > a > img')
info ={
'title':title[0].get_text().strip(),
'address':address[0].get_text().strip() if address else '杭州', #为防止地址不存在，特地加以判断
'price':price[0].get_text(),
'name':name[0].get_text().strip(),
'sex':sex,
'houseImage':houseImage[0].get('src'),
'menberImage':menberImage[0].get('src')
}
global count #全局变量在局部变量使用时需要指定global
#print(count)
count = count + 1
#print(info)
hz.insert_one(info) #插入数据库
print('\r当前进度为:{:.2f}%'.format(((count/299)*100)),end="") #在命令行运行可以显示动态窗口,end=""是关键
def get_urls(url,urls):
r = requests.get(url,timeout = 30)
soup = BeautifulSoup(r.text,'lxml')
links = soup.select('a.resule_img_a')
for link in links:
urls.append(link.get('href'))
def main():
for url in start_urls: # 获取所有url
get_urls(url,urls)
for url in urls: #对每个url进行信息获取
get_info(url)
if __name__ == '__main__':
main()
