python爬虫爬取王者荣耀英雄图片

python爬取数据四步走
1、确定目标
2、分析目标
3、编写代码
4、执行爬虫

1、确定目标

爬取目标:

url = 'https://pvp.qq.com/web201605/herolist.shtml'

注意:有时候有些页面是异步加载，直接请求url地址是获得不了数据的。而王者荣耀英雄页面就是异步加载的，英雄列表数据是动态请求的，F12或者selenium分析找到一个json文件:

# json文件地址
https://pvp.qq.com/web201605/js/herolist.json

2、分析目标

思路：
先在英雄列表得到所有的英雄信息，然后依次循环爬取单个英雄的信息，得到单个英雄的所有皮肤。
选择几个英雄的路径如下:

<a href="herodetail/105.shtml" target="_blank"><img src="//game.gtimg.cn/images/yxzj/img201606/heroimg/105/105.jpg" alt="廉颇" width="91" height="91">廉颇</a>
# 对应的json对象
{
    "0": {
        "ename": 105,
        "cname": "廉颇",
        "title": "正义爆轰",
        "new_type": 0,
        "hero_type": 3,
        "skin_name": "正义爆轰|地狱岩魂"
    }
}
<a href="herodetail/106.shtml" target="_blank"><img src="//game.gtimg.cn/images/yxzj/img201606/heroimg/106/106.jpg" alt="小乔" width="91" height="91">小乔</a>
# 对应的json对象
{
    "1": {
        "ename": 106,
        "cname": "小乔",
        "title": "恋之微风",
        "new_type": 0,
        "hero_type": 2,
        "skin_name": "恋之微风|万圣前夜|天鹅之梦|纯白花嫁|缤纷独角兽"
    }
}
<a href="herodetail/107.shtml" target="_blank"><img src="//game.gtimg.cn/images/yxzj/img201606/heroimg/107/107.jpg" alt="赵云" width="91" height="91">赵云</a>
# 对应的json对象
{
    "2": {
        "ename": 107,
        "cname": "赵云",
        "title": "苍天翔龙",
        "new_type": 0,
        "hero_type": 1,
        "hero_type2": 4,
        "skin_name": "苍天翔龙|忍●炎影|未来纪元|皇家上将|嘻哈天王|白执事|引擎之心"
    }
}
<a href="herodetail/108.shtml" target="_blank"><img src="//game.gtimg.cn/images/yxzj/img201606/heroimg/108/108.jpg" alt="墨子" width="91" height="91">墨子</a>
# 对应的json对象
{
    "3": {
        "ename": 108,
        "cname": "墨子",
        "title": "和平守望",
        "new_type": 0,
        "hero_type": 2,
        "hero_type2": 1,
        "skin_name": "和平守望|金属风暴|龙骑士|进击墨子号"
    }
}
<a href="herodetail/109.shtml" target="_blank"><img src="//game.gtimg.cn/images/yxzj/img201606/heroimg/109/109.jpg" alt="妲己" width="91" height="91">妲己</a>
# 对应的json对象
{
    "4": {
        "ename": 109,
        "cname": "妲己",
        "title": "魅力之狐",
        "pay_type": 11,
        "new_type": 0,
        "hero_type": 2,
        "skin_name": "魅惑之狐|女仆咖啡|魅力维加斯|仙境爱丽丝|少女阿狸|热情桑巴"
    }
}

根据上述，可以推测出英雄图片地址:

# [http://game.gtimg.cn/images/yxzj/img201606/skin/hero-info/(英雄编号)/(英雄编号)-bigskin-(第几个皮肤).jpg](http://game.gtimg.cn/images/yxzj/img201606/skin/hero-info/137/137-bigskin-1.jpg)


https//game.gtimg.cn/images/yxzj/img201606/heroimg + ename + ename + .jpg

3、编写代码

封装请求方法、

# 封装请求方法
  def send_get(self, url):
      try:
          response = requests.get(url, headers=self.headers)
          # 断言测试
          assert response.status_code == 200, '{}请求失败'.format(url)
          return response
      except Exception as e:
          print(e)
          return None

获得首页英雄列表、

# 获得所有的英雄列表
   def hero_list(self):
       # 获取英雄列表
       response = self.send_get(self.hero_list_url)
       if response:
           hero_list_text = response.text  # 获得响应数据
           hero_list_dict = json.loads(hero_list_text)  # 响应数据封装json
           self.process_heroes(**hero_list_dict)  # 调用处理方法
       else:
           print('英雄列表为空,请检查获取URL:{}'.format(self.hero_list_url))

处理英雄列表数据、

# 对英雄列表进行处理
   def process_heroes(self, **hero_list_dict):
       for hero in hero_list_dict['hero']:
           hero_info_url = self.hero_url.format(hero['heroId'])
           resp = self.send_get(hero_info_url)
           if resp:
               hero_info_dict = json.loads(resp.text)
               self.process_hero(**hero_info_dict)
           else:
               print('获取英雄:{}失败,请检查获取URL:{}'.format(hero['name'], self.hero_list_url))

单个英雄数据获取

# 单个英雄处理
   def process_hero(self, **hero_info_dict):
       hero = hero_info_dict['hero']
       skins = hero_info_dict['skins']
       for skin in skins:
           if skin['mainImg']:
               skin_content = self.send_get(skin['mainImg']).content
               hero_image_name = '{}.jpg'.format(skin['name'])
               hero_image_dir = os.path.join(self.base_path, hero['name'] + hero['title'])
               self.save_image(hero_image_dir, hero_image_name, skin_content)
       print('hero:{},skins:{}张,处理完成'.format(hero['name'], len(skins)))
       time.sleep(1)

图片保存、

   def save_image(self,image_dir, image_name, image_content):
       if not os.path.exists(image_dir):
           os.makedirs(image_dir)
       try:
           hero_image_path = os.path.join(image_dir, re.sub(r'[/|?]', '', image_name))
           with open(hero_image_path, 'wb') as image:
               image.write(image_content)
       except Exception as e:
           print('{}保存失败,错误原因:{}'.format(hero_image_path, e))

4、爬虫执行

if __name__ == '__main__':
    LOLHeroSpider().hero_list()

5、完整代码

import requests
import random
import time
import re
import json
import os

# url = https://lol.qq.com/data/info-heros.shtml
# url = 'https://lol.qq.com/data/info-heros.shtml'

# 需要设置USER_AGENT,假装自己是浏览器访问网页
user_agent_list = [
 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0',
 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)',
 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)',
 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)',
 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',
 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',
 'Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',
 'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11',
 'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11'
 ]
USER_AGENT = random.choice(user_agent_list)
header = {
 'User-Agent':USER_AGENT
}

class LOLHeroSpider():
   def __init__(self):
       self.header = header
       self.hero_list_url = 'https://game.gtimg.cn/images/lol/act/img/js/heroList/hero_list.js'
       self.hero_url = 'https://game.gtimg.cn/images/lol/act/img/js/hero/{}.js'
       self.base_path = os.path.join('d:' + os.path.sep, '英雄联盟')
   # 封装请求方法
   def send_get(self, url):
       try:
           response = requests.get(url, headers=self.header)
           # 断言测试
           assert response.status_code == 200, '{}请求失败'.format(url)
           return response
       except Exception as e:
           print(e)
           return None

   # 获得所有的英雄列表
   def hero_list(self):
       # 获取英雄列表
       response = self.send_get(self.hero_list_url)
       if response:
           hero_list_text = response.text  # 获得响应数据
           hero_list_dict = json.loads(hero_list_text)  # 响应数据封装json
           self.process_heroes(**hero_list_dict)  # 调用处理方法
       else:
           print('英雄列表为空,请检查获取URL:{}'.format(self.hero_list_url))

   # 对英雄列表进行处理
   def process_heroes(self, **hero_list_dict):
       for hero in hero_list_dict['hero']:
           hero_info_url = self.hero_url.format(hero['heroId'])
           resp = self.send_get(hero_info_url)
           if resp:
               hero_info_dict = json.loads(resp.text)
               self.process_hero(**hero_info_dict)
           else:
               print('获取英雄:{}失败,请检查获取URL:{}'.format(hero['name'], self.hero_list_url))
   # 单个英雄处理
   def process_hero(self, **hero_info_dict):
       hero = hero_info_dict['hero']
       skins = hero_info_dict['skins']
       for skin in skins:
           if skin['mainImg']:
               skin_content = self.send_get(skin['mainImg']).content
               hero_image_name = '{}.jpg'.format(skin['name'])
               hero_image_dir = os.path.join(self.base_path, hero['name'] + hero['title'])
               self.save_image(hero_image_dir, hero_image_name, skin_content)
       print('hero:{},skins:{}张,处理完成'.format(hero['name'], len(skins)))
       time.sleep(1)
   # 图片保存
   def save_image(self,image_dir, image_name, image_content):
       if not os.path.exists(image_dir):
           os.makedirs(image_dir)
       try:
           hero_image_path = os.path.join(image_dir, re.sub(r'[/|?]', '', image_name))
           with open(hero_image_path, 'wb') as image:
               image.write(image_content)
       except Exception as e:
           print('{}保存失败,错误原因:{}'.format(hero_image_path, e))

if __name__ == '__main__':
    LOLHeroSpider().hero_list()