Python爬虫学习第二周，爬取赶集网二手市场的10万商品数据。
代码：
extract.py
#!/usr/bin/env python
# coding: utf-8
from bs4 import BeautifulSoup
import requests
start_url ='http://bj.ganji.com/wu/'
url_host = 'http://bj.ganji.com'
page_urls = []
headers = {
'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding':'gzip, deflate, sdch',
'Accept-Language':'zh-CN,zh;q=0.8,zh-TW;q=0.6,en;q=0.4',
'Cookie':'ganji_login_act=1468029432459; citydomain=bj; __utmt=1; __utma=32156897.1612008533.1468029433.1468029433.1468029433.1; __utmb=32156897.1.10.1468029433; __utmc=32156897; __utmz=32156897.1468029433.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); ganji_xuuid=63e81818-41ad-4dc4-e480-1821d31a98d3.1468029433479; ganji_uuid=9925341361394064405299; _gl_tracker=%7B%22ca_source%22%3A%22-%22%2C%22ca_name%22%3A%22-%22%2C%22ca_kw%22%3A%22-%22%2C%22ca_id%22%3A%22-%22%2C%22ca_s%22%3A%22self%22%2C%22ca_n%22%3A%22-%22%2C%22ca_i%22%3A%22-%22%2C%22sid%22%3A35833616222%7D',
'Host':'bj.ganji.com',
'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36'
}
def get_channel_urls(url):
wb_data = requests.get(url,headers = headers)
if wb_data.status_code == 200:
Soup = BeautifulSoup(wb_data.text,'lxml')
urls = Soup.select('dl.fenlei dt > a')
for url in urls:
page_urls.append(url_host+url.get('href'))
print(url_host+url.get('href'))
return page_urls
else:
return
#爬取的页码
channel_list = '''
http://bj.ganji.com/jiaju/
http://bj.ganji.com/rirongbaihuo/
http://bj.ganji.com/shouji/
http://bj.ganji.com/shoujihaoma/
http://bj.ganji.com/bangong/
http://bj.ganji.com/nongyongpin/
http://bj.ganji.com/jiadian/
http://bj.ganji.com/ershoubijibendiannao/
http://bj.ganji.com/ruanjiantushu/
http://bj.ganji.com/yingyouyunfu/
http://bj.ganji.com/diannao/
http://bj.ganji.com/xianzhilipin/
http://bj.ganji.com/fushixiaobaxuemao/
http://bj.ganji.com/meironghuazhuang/
http://bj.ganji.com/shuma/
http://bj.ganji.com/laonianyongpin/
http://bj.ganji.com/xuniwupin/
http://bj.ganji.com/qitawupin/
http://bj.ganji.com/ershoufree/
http://bj.ganji.com/wupinjiaohuan/
'''
#get_channel_urls(start_url)
page_parsing.py
#!/usr/bin/env python
# coding: utf-8
from bs4 import BeautifulSoup
import requests
import time
import pymongo
import random
#create the database
client = pymongo.MongoClient('localhost',27017)
ganji = client['ganji'] #table:ganji
url_list = ganji['url_list'] #name:商品url列表
item_info = ganji['item_info'] #sheet:商品详情页
'''
# http://cn-proxy.com/
proxy_list = [
'218.202.111.10:80',
'115.231.162.216:3128',
'119.29.232.113:3128',
]
# 随机获取代理ip
proxy_ip = random.choice(proxy_list)
proxies = {'http': proxy_ip}
'''
# 'o':个人 'a':商家
def get_links_from(channel,pages,who_sells='o'):
headers = {
'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept - Encoding': 'gzip, deflate, sdch',
'Accept - Language': 'zh - CN, zh;q = 0.8, en;q = 0.6',
'Cache - Control': 'max - age = 0',
'Connection': 'keep - alive'
}
list_view = '{}{}{}/'.format(channel,who_sells, str(pages))
wb_data = requests.get(list_view, headers=headers)
i = random.randrange(0,3)
time.sleep(i)
# 检查页面是否不存在，或者被封ip
if wb_data.status_code == 200:
Soup = BeautifulSoup(wb_data.text, 'lxml')
links = Soup.select('.js-item > a')
if Soup.find('ul','pageLink'): #用来判断边界条件
for link in links:
item_link = link.get('href')
headers = requests.head(item_link, allow_redirects=False).headers
if headers['Server'] == 'nginx':
item_link = headers['Location']
url_list.insert_one({'url': item_link})
else:
url_list.insert_one({'url': item_link})
pass
get_item_info_from(item_link)
else:
pass # last page
def get_item_info_from(url):
headers = {
'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept - Encoding': 'gzip, deflate, sdch',
'Accept - Language': 'zh - CN, zh;q = 0.8, en;q = 0.6',
'Cache - Control': 'max - age = 0',
'Connection': 'keep - alive'
}
wb_data = requests.get(url, headers=headers)
if wb_data.status_code != 200:
return
soup = BeautifulSoup(wb_data.text, 'lxml')
no_longer_exist = soup.find('div', 'error')
if no_longer_exist:
pass
else:
prices = soup.select('.f22.fc-orange.f-type') #价格
pub_dates = soup.select('.pr-5') #发布时间
areas = soup.select('ul.det-infor > li:nth-of-type(3) > a') #地点
cates = soup.select('ul.det-infor > li:nth-of-type(1) > span') #类型
data = {
'title': soup.title.text.strip(),
'price': prices[0].text.strip() if len(prices) > 0 else 0,
'pub_date': pub_dates[0].text.strip().split(' ')[0] if len(pub_dates) > 0 else "",
'area': [area.text.strip() for area in areas if area.text.strip() != "-"],
'cates': [cate.text.strip() for cate in cates],
'url':url
}
#print(data)
item_info.insert_one(data)
#get_item_info_from('http://bj.ganji.com/shouji/2215881032x.htm')
main.py
from multiprocessing import Pool
from page_parsing import get_item_info_from, url_list, item_info, get_links_from
from extract import channel_list
def get_all_links_from(channel):
for i in range(1, 100):#100
get_links_from(channel, i)
if __name__ == '__main__':
pool = Pool()
pool.map(get_all_links_from, channel_list.split())
#断点续传 //搜索在url_list中剩下的url
db_urls = [item['url'] for item in url_list.find()]
index_urls = [item['url'] for item in item_info.find()]
x = set(db_urls)
y = set(index_urls)
rest_of_urls = x-y
pool.map(get_item_info_from,rest_of_urls)
count.py
import time
from page_parsing import url_list
while True:
print(url_list.find().count())
time.sleep(5)
总结
大规模数据的爬取之前应该做好爬虫工作流程的设计，设计多个爬虫，分别负责URL链接和每个链接的详情页的爬取。同时设计数据库两个sheet，一个用来存放URL，另一个用来存放商品详情。
爬取测试过程中，有一些网页进行了重定向，用requests的head方法可以获取响应头，通过响应头中的'location'可以获得真实的URL，通过设置请求参数allow_redirects=True可以启用重定向，默认情况下是禁用的。
类似于这样的（未补全网页）http://biz.click.ganji.com/bizClickurl=pZwY0jCfsvFJshI6UhGGshPfUiqJpy提高爬取效率，可以使用多线程和多进程。使用多进程的前提是拥有足够的CPU，因为一个进程会占用一个CPU。对于单核系统，只能使用多线程爬取。
在抓取过程中遇到网络问题而导致程序终止的时候，需要设计断点续传功能保证数据库中抓取的数据不会重复。设计思路是存储商品详情的同时增加一个字段，存储每个商品的URL，如果程序中断，则将所有链接与商品详情表中已抓取链接做差集，抓取剩下的链接。
由于实验室流量限制，没有全爬下来，爬取了部分截图如下,作业做好后参考了下课后视频讲解，使用代理来进行爬取。添加进requests.get()里面。
'''
# http://cn-proxy.com/
proxy_list = [
'218.202.111.10:80',
'115.231.162.216:3128',
'119.29.232.113:3128',
]
# 随机获取代理ip
proxy_ip = random.choice(proxy_list)
proxies = {'http': proxy_ip}
'''
result.png
