python爬虫案例（Scrapy版）

以爬取各城市每天的空气质量数据为例

安装好scrapy后（可用conda安装）在cmd中输入以下代码完成爬虫准备工作

cd Desktop
:: 创建projiect
scrapy startproject wheather
cd wheather
:: 需要爬取的网址，以真气网为例
scrapy genspider Wheather www.aqistudy.cn
:: 查看需要爬取的具体网址
scrapy shell https://www.aqistudy.cn/historydata/
view(response)

编辑wheather文件夹中的items.py文件，确定爬取的变量

修改（或新增）WheatherItem的内容，该案例只爬取3个变量，结果如下

class WheatherItem(scrapy.Item):
    city = scrapy.Field(serializer=lambda x: x[8:])  # 城市
    date = scrapy.Field()  # 日期
    quality = scrapy.Field()  # 空气质量
    # location = scrapy.Field()  # 地点

编辑Pipelines.py文件，增加输出格式（可忽略该部分）

该案例添加json输出格式

# 输出json.............................................
import json
class WheatherPipeline(object):
    def __init__(self):
        self.f = open('spdr.json', 'w+', encoding='utf-8')
    def process_item(self, item, spider):
        content = json.dumps(dict(item), ensure_ascii=False) + '\n'
        self.f.write(content)
        return item
    def close_spider(self, spider):
        self.f.close()

编辑settings.py文件

#..............................................
# 改为False
ROBOTSTXT_OBEY = False
# 加代理
USER_AGENT ='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36' 
# log级别设为warning
LOG_LEVEL='WARNING' 
# 导出json
ITEM_PIPELINES = {
   'wheather.pipelines.WheatherPipeline': 300,
}
# 涉及到动态加载，该案例用scrapy-selenium包来嫁接scrapy和selenium。
# 网上有不少教程自己写中间件来引入selenium，可以但是没有必要！！！
DOWNLOADER_MIDDLEWARES  = {
     'scrapy_selenium.SeleniumMiddleware':800
}
SELENIUM_DRIVER_NAME= 'chrome'  # 用chromedriver
SELENIUM_DRIVER_EXECUTABLE_PATH = r"C:/Anaconda3/chromedriver.exe"
SELENIUM_DRIVER_ARGUMENTS = ['--headless',]  # '--start-maximized'
# 原包不提供无图模式，该案例对包的代码进行了修改
# 若不需要无图模式，可忽略余下部分
SELENIUM_DRIVER_EXP_ARGUMENTS=[{"profile.managed_default_content_settings.images": 2}] 
# 修改部分如下
'''
browser_executable_path,driver_exp_arguments):  #.........wdy
or exp_argument in driver_exp_arguments: #..................... wdy
     driver_options.add_experimental_option("prefs", exp_argument) # ...wdy
driver_exp_arguments = crawler.settings.get('SELENIUM_DRIVER_EXP_ARGUMENTS')  #...............wdy
driver_exp_arguments=driver_exp_arguments #............wdy
'''

编辑爬虫文件Wheather.py

# -*- coding: utf-8 -*-
# 爬取中国各个城市的历史天气数据
import scrapy
import re
from scrapy.linkextractors import LinkExtractor
from ..items import WheatherItem
from scrapy_selenium import SeleniumRequest
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC

class WheatherSpider(scrapy.Spider):
    '''
    start_requests，用selenium代替request
    parse，获取城市链接
    parse_month，获取各城市的各日期链接
    parse_final，爬
    '''
    name = 'Wheather'
    allowed_domains = ['www.aqistudy.cn']

    def start_requests(self): # 注意名字不要随意改！！！！！！
        start_urls = 'https://www.aqistudy.cn/historydata/'
        yield SeleniumRequest(url=start_urls, callback=self.parse)
    
    def parse(self, response):
        i=0
        citylink = LinkExtractor(restrict_xpaths='/html/body/div[3]/div/div[1]/div[2]/div[2]')
        citylinks = citylink.extract_links(response)  # 各个城市的天气数据链接   
        for cityurl in citylinks: 
            i+=1
            yield scrapy.Request(url=cityurl.url,meta={'i':i}, callback=self.parse_month)
    
    def parse_month(self, response):
        print(response.meta['i'])  # meta传递数值（对整体没有影响，作者自己看的）
        monthlink = LinkExtractor(restrict_xpaths='/html/body/div[3]/div[1]/div[2]/div[2]/div[2]')
        monthlinks = monthlink.extract_links(response)  # 每个城市各个月的天气数据链接
        for monthurl in monthlinks:
            if int(str(monthurl.url)[-6:]) == 201712:  # 简单起见，只选取了2017年12月的数据
                yield SeleniumRequest(url=monthurl.url,wait_time=4,wait_until=EC.presence_of_element_located((By.XPATH, '/html/body/div[3]/div[1]/div[1]/table/tbody/tr[29]/td[2]')),callback=self.parse_final)
    
    def parse_final(self,response):
        long = len(response.selector.xpath('/html/body/div[3]/div[1]/div[1]/table/tbody/tr'))  # 计算每页一共这么多条
        dit = WheatherItem()
        # response.request.meta['driver'].current_url)[0]
        for line in range(2, long+1):
            dit['city'] = re.findall('(.+?)空气质量',response.selector.xpath('//*[@id="title"]/text()').extract_first())[0]
            dit['date'] = response.selector.xpath('/html/body/div[3]/div[1]/div[1]/table/tbody/tr[%s]/td[1]/text()' % str(line)).extract_first()
            dit['quality'] = response.selector.xpath('/html/body/div[3]/div[1]/div[1]/table/tbody/tr[%s]/td[3]/span/text()' % str(line)).extract_first()
            yield dit
# scrapy crawl somespider -s JOBDIR=crawls/somespider-1 （作者自己看的）

运行和输出

在cmd中输入以下代码可运行代码和输出csv格式的文件

cd wheather
scrapy crawl Wheather -o books.csv