简书7日热门文章数据分析+更新推送（持续更新···）
工作流程
- 爬取简书7日热门文章；
- 存入MongoDB数据库；
- 文章简单的数据分析；
- 再次爬取简书7日热门文章；
- 判断新爬取的文章是否与已爬取的内容重复；
- 如文章不重复，发送邮件提醒；
URL结构
首页URL：http://www.jianshu.com/trending/weekly?utm_medium=index-banner-s&utm_source=desktop
-
其他页面URL说明：当前页面请求URL的seen_snote_ids位于上一页面的
<ul class="note-list" infinite-scroll-url="/trending/weekly">的子标签li的data-note-id属性中，如图。
参考代码
爬虫部分代码
import requests
from lxml import etree
homepage_url = 'http://www.jianshu.com'
base_url = 'http://www.jianshu.com/trending/weekly'
def get_html(url):
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36'
}
resp = requests.get(url, headers=headers)
if resp.status_code == 200:
return resp.text
else:
return None
# 解析首页数据
def parse_first_html(html, db):
root = etree.HTML(html)
articles = root.xpath('//*[contains(concat( " ", @class, " " ), concat( " ", "content", " " ))]')
seen_snote_ids = root.xpath('//ul[@class="note-list"]/li/@data-note-id')
# print(seen_snote_ids)
for article in articles:
author = ''.join(article.xpath('div[@class="author"]/div[@class="name"]/a/text()'))
author_url = homepage_url + ''.join(article.xpath('div[@class="author"]/div[@class="name"]/a/@href'))
title = ''.join(article.xpath('a[@class="title"]/text()'))
article_url = homepage_url + ''.join(article.xpath('a[@class="title"]/@href'))
abstract = ''.join(article.xpath('p[@class="abstract"]/text()')).strip()
read_and_comments = article.xpath('div[@class="meta"]/a/text()')
like_and_pay = article.xpath('div[@class="meta"]/span/text()')
# 数据处理
read_and_comments = ''.join(read_and_comments).strip().split()
like_and_pay = ''.join(like_and_pay).strip().split()
# read_count = meta.xpath('a/text()')
# print(author, author_url, title, article_url, abstract, read_and_comments, like_and_pay)
# print(''.join(like_and_pay).strip().split())
data ={
'author': author,
'author_url': author_url,
'title': title,
'article_url': article_url,
'abstract': abstract,
'read_and_comments': read_and_comments,
'like_and_pay': like_and_pay
}
if not is_data_exist(db, data):
print('保存新数据')
save_data(db, data)
else:
print('数据已存在，更新数据')
update_data(db, data)
return seen_snote_ids
def parse_other_html(html, db):
root = etree.HTML(html)
articles = root.xpath('//li/div[@class="content"]')
seen_snote_ids = root.xpath('//li/@data-note-id')
# print(seen_snote_ids)
for article in articles:
author = ''.join(article.xpath('div[@class="author"]/div[@class="name"]/a/text()'))
author_url = homepage_url + ''.join(article.xpath('div[@class="author"]/div[@class="name"]/a/@href'))
title = ''.join(article.xpath('a[@class="title"]/text()'))
article_url = homepage_url + ''.join(article.xpath('a[@class="title"]/@href'))
abstract = ''.join(article.xpath('p[@class="abstract"]/text()')).strip()
read_and_comments = article.xpath('div[@class="meta"]/a/text()')
like_and_pay = article.xpath('div[@class="meta"]/span/text()')
# 数据处理
read_and_comments = ''.join(read_and_comments).strip().split()
like_and_pay = ''.join(like_and_pay).strip().split()
# read_count = meta.xpath('a/text()')
# print(author, author_url, title, article_url, abstract, read_and_comments, like_and_pay)
data ={
'author': author,
'author_url': author_url,
'title': title,
'article_url': article_url,
'abstract': abstract,
'read_and_comments': read_and_comments,
'like_and_pay': like_and_pay
}
if not is_data_exist(db, data):
print('保存新数据')
save_data(db, data)
else:
print('数据已存在，更新数据')
update_data(db, data)
return seen_snote_ids
def get_other_html(url, page, seen_snote_ids):
url_param = url
page_param = str(page)
seen_snote_ids_param = '?seens_snote_ids%5B%5D=' + '?seens_snote_ids%5B%5D='.join(seen_snote_ids)
url = url + '?page=' + page_param + seen_snote_ids_param
# print(url)
return get_html(url)
# 初始化数据库
def init_sql():
from pymongo import MongoClient
client = MongoClient('localhost', 27017)
# 创建数据库sdifen
db = client.JIANSHU
return db
# 保存数据到数据库中
def save_data(db, data):
db.WEEKLY.insert(data)
# 更新数据：删除原数据，重新保存该数据？？？
def update_data(db, data):
pass
# 判断数据库中是否存在数据
def is_data_exist(db, data):
return db.WEEKLY.find({'article_url': data.get('article_url')}).count()
def main():
db = init_sql()
resp = get_html(base_url)
seen_snote_ids = parse_first_html(resp, db)
for i in range(2, 15):
resp = get_other_html(base_url, i, seen_snote_ids)
seen_snote_ids.extend(parse_other_html(resp, db))
if __name__ == '__main__':
main()
数据分析部分代码
import pandas as pd
import numpy
import matplotlib.pyplot as plt
# 读取MongoDB数据
def read_data():
import pymongo
from pymongo import MongoClient
client = MongoClient()
db = client.JIANSHU
collection = db.WEEKLY
data = pd.DataFrame(list(collection.find()))
return data
# 去重，文章链接一样的数据视为重复数据
data = read_data().drop_duplicates(['article_url'])
userClientCol = ['作者', '文章数']
# 构造DataFrame，注意：需数组转置
userClientDataFrame = pd.DataFrame(numpy.array([list(set(data.get('author'))), [list(data.get('author')).count(level) for level in list(set(data.get('author')))]]).T, columns=userClientCol)
# 根据作者文章数量进行排序，排序方式为降序
userClientDataFrame = userClientDataFrame.sort_values('文章数', ascending=False)
# 选取文章数量前20名的作者
userClientDataFrame = userClientDataFrame.head(15)
plt.figure(figsize=(20,10),dpi=100)
labels = list(userClientDataFrame['作者'])
plt.bar(range(len(labels)),userClientDataFrame['文章数'],tick_label=labels)
plt.title('作者')
plt.show()
发布文章作者前20名
说明：Mac默认的Python或者Anaconda环境下，中文会出现乱码，解决方法如下：
- 下载微软雅黑.ttf文件
- 重命名为DejaVuSans.ttf
- 替换掉/anaconda/lib/python3.6/site-packages/matplotlib/mpl-data/fonts/ttf/DejaVuSans.ttf（Anaconda环境）
