写了一上午，爬了一下午，心累。
- 采用了多进程爬虫mutliprocessing的Pool
- 采用mongdb数据库存储数据
- 采用requests + BeautifulSoup的路线
思路： - 获得各个分类的链接，获得每个物品的链接，存储
- 从数据库中取链接，获得每个物品的具体信息，存储
- 另外写一个python程序或者从mongdb shell访问数据库，得到进度
- 断点续传的功能，在两个数据库中设置一个关联值（这里采用url），然后分别把数据导入一个set，两集合相减就得到接下来要下载的信息
import pymongo,time,requests
from bs4 import BeautifulSoup
from multiprocessing import Pool
client = pymongo.MongoClient('localhost',27017)
tongcheng = client['tongcheng']
url_list = tongcheng['url_list'] #创建了两个collections
info_list = tongcheng['info_list']
headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:53.0) Gecko/20100101 Firefox/53.0'
} #模拟浏览器访问网站
start_url = 'http://bj.58.com/sale.shtml'
channel_links = []
def get_channel_url(url):
r = requests.get(url,headers=headers)
r.raise_for_status()
soup = BeautifulSoup(r.text,'lxml')
channels = soup.select('ul.ym-mainmnu > li.ym-tab > ul.ym-submnu > li > b > a')
for channel in channels:
channel_links.append('http://bj.58.com'+channel.get('href'))
def get_item_url(channel,page,who_sell=0):
url = '{}/{}/pn{}/'.format(channel,str(who_sell),str(page))
wb_data = requests.get(url,timeout=30,headers=headers)
time.sleep(2)
wb_data.raise_for_status()
soup = BeautifulSoup(wb_data.text,'lxml')
if soup.find('td'):
items = soup.select('a.t')
for item in items:
link = item.get('href').split('?')[0]
if 'zhuanzhuan' in link:
url_list.insert_one({'url':link})
print(link)
def get_pages_url(channel):
for i in range(1,100):
get_item_url(channel,i)
def get_item_info(url):
wb_data = requests.get(url,timeout = 30,headers = headers)
time.sleep(1)
wb_data.raise_for_status()
soup = BeautifulSoup(wb_data.text,'lxml')
title = soup.find('head').find('title').text
price = soup.select('span.price_now > i')
area = soup.select('div.palce_li > span > i')
view = soup.select('span.look_time')
info = {
'title':title.strip(),
'price':price[0].text if price else None,
'area':area[0].text.strip() if area else None,
'view':view[0].text.strip()[:-3] if view else None
}
info_list.insert_one(info)
print(info)
if __name__ == '__main__':
pool = Pool() #创建进程池
get_channel_url(start_url)
pool.map(get_pages_url,channel_links) #采用多进程爬取
pool.map(get_item_info,link['url'] for link in url_list.find())
import pymongo,time
client = pymongo.MongoClient('localhost',27017)
tongcheng = client['tongcheng']
url_list = tongcheng['url_list']
info_list = tongcheng['info_list']
while url_list.find().count() > info_list.find().count():
print(url_list.find().count()+info_list.find().count())
time.sleep(5) #定时得到数据库中的个数
# ================================================= < <链接去重 > > =====================================================
# 设计思路：
# 1.分两个数据库，第一个用于只用于存放抓取下来的 url (ulr_list)；第二个则储存 url 对应的物品详情信息(item_info)
# 2.在抓取过程中在第二个数据库中写入数据的同时，新增一个字段(key) 'index_url' 即该详情对应的链接
# 3.若抓取中断，在第二个存放详情页信息的数据库中的 url 字段应该是第一个数据库中 url 集合的子集
# 4.两个集合的 url 相减得出圣贤应该抓取的 url 还有哪些
db_urls = [item['url'] for item in url_list.find()] # 用列表解析式装入所有要爬取的链接
index_urls = [item['url'] for item in item_info.find()] # 所引出详情信息数据库中所有的现存的 url 字段
x = set(db_urls) # 转换成集合的数据结构
y = set(index_urls)
rest_of_urls = x-y
# 相减
