练习两层工作流
- 第一步，获取目标url存入数据库（mongoconn.py ）
- 第二步，从数据库中读出url, 并从页面上提取目标信息（homework2_2.py ）
源代码
mongoconn.py
#!usr/bin/env python
#_*_ coding: utf-8 _*_
#
# connect mongodb
import pymongo
def mongoset(db, table):
client = pymongo.MongoClient('localhost', 27017)
data = client[db]
sheet = data[table]
return sheet
def mongoinsert(table, data):
table.insert_many(data)
homework2_2.py
#!usr/bin/env python
#_*_ coding: utf-8 _*_
#
# 爬取手机号
# step1 get all urls, save them to db
# step2 get detail info by accessing those urls
from bs4 import BeautifulSoup
import requests
import time
from mongoconn import mongoset, mongoinsert
def get_soup(url):
source = requests.get(url)
soup = BeautifulSoup(source.text, 'lxml')
return soup
def combineurls(url, page):
pageurls = []
for i in range(1, page+1):
pageurl = '{}{}/'.format(url, i)
pageurls.append(pageurl)
return pageurls
def get_page_urls(url):
curpage = 1
maxpage=0
while curpage > maxpage:
maxpage = curpage
pageurl = url + 'pn' + str(maxpage)
soup = get_soup(pageurl)
pager = soup.select('div.pager > a')
pagenum = pager[len(pager)-3].select('span')[0].get_text() #### -3是临时办法， 需要再想想
curpage = int(pagenum)
urls = combineurls(url+'pn', maxpage)
return urls
def listtodict(urls):
datamany = []
for itemurl in urls:
data = {
'itemurl': itemurl
}
datamany.append(data)
return datamany
def get_item_urls(url):
soup = get_soup(url)
itemlist = soup.select('div.boxlist > ul > li > a.t')
itemurls = []
for item in itemlist:
try:
itemurl = item.get('href')
except:
pass
itemurls.append(itemurl)
time.sleep(1)
return itemurls
def getemtext(element):
return element.get_text().strip().replace('\t', '').replace('\n', '').replace(' ','')
def get_target_info(url):
soup = get_soup(item['url'])
main = soup.select('div.detailPrimary')
if main:
title = main[0].select('div.mainTitle h1')[0]
price = main[0].select('span.price')[0]
data = {
'title': getemtext(title),
'price': getemtext(price),
'url': url
}
return data
if __name__ == '__main__':
### step1, get urls and insert into mongo
table = mongoset('58sale', 'shoujihaourl')
url = 'http://bj.58.com/shoujihao/'
pageurls = get_page_urls(url)
for url in pageurls:
mongoinsert(table, listtodict(get_item_urls(url)))
### step2, get detailed info
table = mongoset('58sale', 'shoujihaourl')
tinfo = mongoset('58sale', 'shoujihaoinfo')
data = table.find()
for item in data:
info = get_target_info(item['url'])
if info:
if not tinfo.count({'url': item['url']}): #to filter out duplication info
print(info)
tinfo.insert_one(info)
time.sleep(1)
- 处理url与mongodb相关的操作分开。用from mongoconn import mongoset, mongoinsert引入mongo相关的函数。
运行结果
-
step1: ‘shoujihaourl’表中存储目标url
Screen Shot 2016-06-30 at 8.44.59 PM.png -
step2: 'shoujihaoinfo'表中存储提取到的详细信息
Screen Shot 2016-06-30 at 8.43.17 PM.png
总结
- 分层处理爬取任务，将已经获取到的信息储存起来，省时省力
