from lxml import etree
import requests
file=open('xpath.html','r',encoding='utf-8')
html=file.read()
file.close()
# print(html)
# url='阳光电影网首页的网址'
# req=requests.get(url)
# req.encoding='gb2312'
# html=req.text
selector=etree.HTML(html)
# print(selector,type(selector))
title=selector.xpath('//title/text()')[0]
# print(title)
h1=selector.xpath('//h1/text()')[0].strip()
# print(h1)
infos=selector.xpath('//a')
for info in infos:
    a_text=info.xpath('text()')[0]
    a_href=info.xpath('@href')[0]
    # print(a_text,a_href)

# 一、div标签文本提取
# 将学习视频中xpath.html文件中div标签下文本值
divs = selector.xpath('//div')
# “第一个div” ，“第二个div” 使用xpath结构化提取并打印输出
for div in divs:
    div_text = div.xpath('text()')[0].strip()
    print(div_text)
# 二、ul标签文本提取
uls = selector.xpath('//ul')
# 将xpath.html文件中ul标签下“流程” ，“xpath学习”，“流程2”文本值使用xpath结构化提取并打印输出
for ul in uls:
    ul_text = ul.xpath('text()')[0].strip()
    print(ul_text)
# 三、过滤标签
# 将xpath.html文件中的第一个div下的前3个a标签的文本及超链接 使用xpath结构化提取，打印输出
print(selector.xpath('//div')[0].xpath('//a/@href'))
# 四、requests模块和lxml&xpath结合提取数据结合上节课requests模块知识，将阳光电影网导航栏的文本及超链接结构化提取
#请求url
url = 'http://www.ygdy8.com/'
#构造headers字典
headers = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    'Accept-Encoding': 'gzip, deflate',
    'Accept-Language': 'zh-CN,zh;q=0.8',
    'Cache-Control': 'max-age=0',
    'Connection': 'keep-alive',
    'Cookie': 'UM_distinctid=15c5ec4f20e377-0798b30518d6b4-5393662-c0000-15c5ec4f20f28b; CNZZDATA5783118=cnzz_eid%3D1150691004-1496237600-%26ntime%3D1496237600; 37cs_user=37cs10138604998; cscpvrich4016_fidx=1; 37cs_show=69',
    'Host': 'www.ygdy8.com',
    'If-Modified-Since': 'Sun, 27 Aug 2017 15:18:27 GMT',
    'If-None-Match': "802356bb471fd31:530",
    'Referer': 'https://www.baidu.com/link?url=cnL9usny1BIZEe-NZUkUbeUE4m9CM23KIysNUsVvzlK&wd=&eqid=c50f090f0001d9880000000259a2e4b0',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36'
}
#定义req为一个requests请求的对象
req = requests.get(url,headers=headers)
#req这个请求对象的status_code方法获取请求的状态码
status_code = req.status_code
# print(status_code)
#指定网页解码方式
req.encoding = 'gb2312'
#获取网页源码 用html变量接收 text content方法灵活运用
html = req.text
# print(html)
selector=etree.HTML(html)
# 查询div的id为menu下的div的class为contain下的a标签不包含onclick事件的所有href
print(selector.xpath("//div[@id='menu']//div[@class='contain']//li//a[not(@onclick)]/@href"))