爬取豆瓣top250电影
import requests
from lxml import etree
import pandas as pd
def parse():
"""豆瓣网top250爬虫"""
# 1、获取url地址
# for i in range(0, 226, 25):
# url = 'https://movie.douban.com/top250?start={}&filter='.format(i)
# print(url)
# # 获取 byte的类型的响应
# resp = requests.get(url)
# data = resp.content
headers = {"User-Agent": "ozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36"}
#定义列表存储top250信息[{},{},{}]
movie_info_list=[]
# 1、获取url地址
# 循环每一页
for i in range(0, 226, 25):
url = 'https://movie.douban.com/top250?start={}&filter='.format(i)
# print(url)
# 获取 byte的类型的响应
# resp = requests.get(url)
# data = resp.content
# url = 'https://movie.douban.com/top250?start=0&filter='
# 获取 byte的类型的响应
resp = requests.get(url,headers=headers)
data = resp.content
# 调用etree.HTML获取html对象，然后调用html的xpath语法
html = etree.HTML(data)
movie_list = html.xpath('//div[@id="content"]//ol/li')
# print(len(movie_list))
for movie in movie_list:
# 获取电影序号
serial_number = movie.xpath('./div[@class="item"]/div[@class="pic"]/em/text()')
serial_number = '' if len(serial_number)==0 else serial_number[0]
# print(serial_number)
movie_name = movie.xpath('./div[@class="item"]/div[@class="info"]/div[@class="hd"]/a/span[1]/text()')
movie_name = '' if len(movie_name) == 0 else movie_name[0]
# print(movie_name)
# serial_number = html.xpath('//*[@id="content"]/div/div[1]/ol/li[1]/div/div[1]/em/text()')
# print(serial_number)
# 电影名称
# movie_name =
# 电影介绍
# introduce =
introduce = movie.xpath('./div[@class="item"]/div[@class="info"]/div[@class="bd"]/p/text()')
introduce = '' if len(introduce) == 0 else introduce[0]
#去两端空格操作
introduce=introduce.strip()
# print(introduce)
# 电影星级
# star =
star = movie.xpath('./div[@class="item"]/div[@class="info"]/div[@class="bd"]/div[@class="star"]/span[2]/text()')
star = '' if len(star) == 0 else star[0]
# print(star)
# 电影的评价
# evalute =
evalute = movie.xpath('./div[@class="item"]/div[@class="info"]/div[@class="bd"]/div[@class="star"]/span[4]/text()')
evalute = '人评价' if len(evalute) == 0 else evalute[0]
evalute=evalute.replace('人评价','')
# print(evalute)
# 电影的描述
# describe =
describe = movie.xpath('./div[@class="item"]/div[@class="info"]/div[@class="bd"]/p[@class="quote"]/span[1]/text()')
describe = '' if len(describe) == 0 else describe[0]
# print(describe)
#详情链接的地址
detail_link = movie.xpath('./div[@class="item"]/div[@class="pic"]/a/@href')
detail_link = '' if len(detail_link ) == 0 else detail_link [0]
# print(detail_link )
#图片的地址
img_url=movie.xpath('./div[@class="item"]/div[@class="pic"]/a/img/@src')
img_url = '' if len(img_url ) == 0 else img_url [0]
# print(img_url )
movie_info_list.append({
'serial_number':serial_number,
'movie_name':movie_name,
'introduce':introduce,
'star':star,
'evalute':evalute,
'describe':describe,
'detail_link':detail_link,
'img_url':img_url,
})
# for movie_info in movie_info_list:
# print(movie_info)
# resp=requests.get(movie_info['img_url'])
# if resp.status_code==200:
# #执行图片写入操作
# #参考图片命名方式 0000001.jpg
# img_name='000000{}.jpg'.format(movie_info['serial_number'])
# with open('./imgs/{}'.format(img_name),'wb') as f:
# f.write(resp.content)
for movie_info in movie_info_list:
print(movie_info)
resp = requests.get(movie_info['img_url'])
if resp.status_code == 200:
# 执行图片写入操作
# 参考图片命名方式 001.jpg
img_name = '000000{}.jpg'.format(movie_info['serial_number'])
with open('./imgs/{}'.format(img_name), 'wb')as f:
f.write(resp.content)
# 存储成csv
df = pd.DataFrame(movie_info_list)
df.to_csv('douban_top250_info.csv')
parse()
输出结果：
image.png
爬取猫眼top100
import requests
from lxml import etree
import pandas as pd
def parse():
"""豆瓣网top250爬虫"""
# 1、获取url地址
# for i in range(0, 226, 25):
# url = 'https://movie.douban.com/top250?start={}'.format(i)
# print(url)
# # 获取 byte的类型的响应
# resp = requests.get(url)
# data = resp.content
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36"}
# 定义列表存储top250信息 [{},{},{}]
movie_info_list = []
#
# 循环每一页
for i in range(0, 91, 10):
url = 'https://maoyan.com/board/4?offset={}'.format(i)
# 获取 byte的类型的响应
req = requests.get(url , headers = headers)
req.encoding = 'utf-8'
data = req.text
# 调用etree.HTML获取html对象，然后调用html的xpath语法
html = etree.HTML(data)
movie_list = html.xpath('//div[@class="main"]/dl/dd')
# print(len(movie_list))
for movie in movie_list:
movie_name = movie.xpath('./div/div/div[1]/p[1]/a/text()')
movie_name = '' if len(movie_name) == 0 else movie_name[0]
# print(movie_name)
# 演员介绍
# role =
role = movie.xpath('./div/div/div[1]/p[2]/text()')
role = '' if len(role) == 0 else role[0]
#去两端空格操作
role=role.strip()
# print(role)
# 电影时间
# # time =
time = movie.xpath('./div/div/div[1]/p[3]/text()')
time = '' if len(time) == 0 else time[0]
# print(time)
movie_info_list.append({
'movie_name':movie_name,
'role':role,
'time':time,
})
for movie_info in movie_info_list:
print(movie_info)
parse()
输出结果：
image.png
