Python爬取环球网国际推荐板块新闻数据

# -*- conding: utf-8 -*-
from lxml import etree
from selenium import webdriver
from pymysql import *
import time
"""
Google 浏览器启动参数
"""
chrome_options = webdriver.ChromeOptions()
# options.add_argument('User-Agent=Mozilla/5.0 (Linux; U; Android 8.1.0; zh-cn; BLA-AL00 Build/HUAWEIBLA-AL00) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.132 MQQBrowser/8.9 Mobile Safari/537.36')
chrome_options.add_argument('--headless')  # 浏览器不提供可视化页面
chrome_options.add_argument('--no-sandbox') #这个参数在linux上执行是必须加上的
browser = webdriver.Chrome(options=chrome_options)
"""
数据库连接
"""
mysqldb = connect(host='IP', port=3306, database='database', user='username', password='password', charset='utf8')
cs = mysqldb.cursor()
"""
information
"""
site_name = "环球网"
url = "https://world.huanqiu.com"
New_urls = []
new_author = []
"""
获取新闻列表
"""
browser.get(url)
for i in range(0, 50):
    js = "var q=document.documentElement.scrollTop={}".format(i * 100)
    browser.execute_script(js)
html = etree.HTML(browser.page_source)
html_urls = html.xpath('//*[@id="recommend"]//a/@href')
for urls in html_urls:
    pingjie_url = url + urls
    New_urls.append(pingjie_url)
"""
从列表提取新闻URL进行访问xpath获取相关数据
"""
data_number = 0
for new_url in New_urls:
    browser.get(new_url)
    news_html = browser.page_source
    html = etree.HTML(news_html)
    title = ''.join(html.xpath('//div[@class="t-container-title"]/h3/text()'))
    author_a = ''.join(html.xpath('//span[@class="source"]/span/a/text()'))
    author_b = ''.join(html.xpath('//span[@class="source"]/a/text()'))
    author_c = ''.join(html.xpath('//span[@class="source"]/span/text()'))
    if author_a == '':
        if author_b == '':
            if author_c == '':
                continue
            else:
                author = author_c
        else:
            author = author_b
    else:
        author = author_a
    new_time_a = ''.join(html.xpath('//p[@class="time"]/text()'))
    new_time_b = ''.join(html.xpath('//*[@id="time"]/text()'))
    if new_time_a != '':
        new_time = new_time_a
    elif new_time_b != '':
        new_time = new_time_b

    article_content_xpath = ''.join(html.xpath('//div[@class="l-con clear"]//section/*/text()'))
    article_content_xpath_one = ''.join(html.xpath('//*[@id="pictureWrap"]/div[1]/div[1]/div/div[1]/div[2]/p/text()'))
    article_content = article_content_xpath + article_content_xpath_one
    """
    insert 插入数据
    """
    query = "insert ignore into toutiao(class,title,source,source_url,behot_time,nowtime,abstract) values (%s, %s, %s, %s, %s, %s, %s)"
    now_time = time.strftime("%Y-%m-%d %H:%M:%S")
    values = (site_name, title, author, new_url, new_time, now_time, article_content)
    cs.execute(query, values)
    print("插入数据成功或数据已存在")
    data_number += 1
    mysqldb.commit()
print("共插入" + data_number + "条数据")
cs.close()
browser.close()