之前学习了模拟登录新浪微博，没有用到框架，今天晚上尝试用scrapy重新写一遍模拟登录。
存在问题：
1、scrapy 好像不擅长模拟登录，写出来的代码结构还不如requests库写的清楚，当然跟我个人水平有关，scrapy不太会。
2、FormRequest的formdata中不能有int型的字段，不然要报错，网上解决方法要修改scrapy源码，对我来说难度太大，放弃了，直接强制转换为str类型，所幸能成功访问。
3、scrapy 函数之间的相互调用很麻烦，我直接顺序callback一个个函数，有些没有关系的函数也强制顺序访问了，而不是在需要的时候访问，导致程序的结构不清。
4、勉强能用，后续改进空间大。
5、爬到的首页数据没有什么内容，网站用了好多js，暂时不知到如何后续爬区。
6、抛砖引玉，希望哪个大神能看到我的文章，指点我一下如何修改能使爬虫结构更清楚。
from scrapy.http import Request,FormRequest
from scrapy.spiders import CrawlSpider
from urllib.parse import quote_plus
from PIL import Image
import base64
import time
import rsa
import binascii
import requests
import re
class weibo_spider(CrawlSpider):
name = 'weibo'
session = requests.session()
user_agent = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
headers = {
'User-Agent': user_agent
}
def start_requests(self):
start_url = 'http://weibo.com/login.php'
return[Request(start_url,meta={'cookiejar':1},callback=self.get_su)]
def get_su(self,response):
username = input('请输入用户名：')
username_quote = quote_plus(username)
uername_base64 = base64.b64encode(username_quote.encode('utf-8'))
uername_base64 = uername_base64.decode('utf-8')
print(uername_base64)
#print(response.url)
meta = {'cookiejar':response.meta['cookiejar'],
'uername_base64':uername_base64}
pre_url = "http://login.sina.com.cn/sso/prelogin.php?entry=weibo&callback=sinaSSOController.preloginCallBack&su="\
+ uername_base64 + "&rsakt=mod&checkpin=1&client=ssologin.js(v1.4.18)&_=" + str(int(time.time() * 1000))
return[Request(pre_url,meta=meta,callback=self.server_data)]
def server_data(self,response):
#eval的作用是把list,tuple,dict和string相互转化
server_data = eval(response.body.decode('utf-8').replace("sinaSSOController.preloginCallBack",''))
print(server_data)
meta =response.meta
meta['server_data'] = server_data
return[Request('https://baidu.com',meta=meta,callback=self.get_password)]
def get_password(self,response):
password = input('请输入密码:')
servertime = response.meta['server_data']['servertime']
nonce = response.meta['server_data']['nonce']
pubkey = response.meta['server_data']['pubkey']
rsapublickey = int(pubkey,16) #转话为10进制
key = rsa.PublicKey(rsapublickey,65537)
message = str(servertime) + '\t' + str(nonce) + '\n' + str(password)
message = message.encode("utf-8")
passwd = rsa.encrypt(message, key)
passwd = binascii.b2a_hex(passwd)
print(passwd)
meta =response.meta
meta['passwd'] = passwd
return[Request('http://www.sina.com.cn/',meta=meta,callback=self.login)]
def login(self,response):
su = response.meta['uername_base64']
servertime = response.meta['server_data']['servertime']
nonce = response.meta['server_data']['nonce']
rsakv = response.meta['server_data']['rsakv']
password_secret = response.meta['passwd']
postdata = {
'entry':'weibo',
'gateway':'1',
'from':'',
'savestate':'7',
'useticket':'1',
'pagerefer':"http://login.sina.com.cn/sso/logout.php?entry=miniblog&r=http%3A%2F%2Fweibo.com%2Flogout.php%3Fbackurl",
'wsseretry':'servertime_error',
'vsnf':'1',
'su':su,
'service':'miniblog',
'servertime':str(servertime),
'nonce':nonce,
'pwencode':'rsa2',
'rsakv':rsakv,
'sp':password_secret,
'sr':'1536*864',
'encoding':'UTF-8',
'prelt':'105',
'url':'http://weibo.com/ajaxlogin.php?framelogin=1&callback=parent.sinaSSOController.feedBackUrlCallBack',
'returntype':'META'
}
login_url = 'http://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.18)'
print(postdata)
if response.meta['server_data']['showpin'] == 0:
return[FormRequest(login_url,meta={'cookiejar':response.meta['cookiejar']},formdata=postdata,callback=self.get_loop_url,dont_filter = True)]
"""
else: #获得验证吗，手动输入
pcid = response.meta['server_data']['picd']
cha_url = "http://login.sina.com.cn/cgi/pin.php?r="
cha_url = cha_url + str(int(random.random() * 100000000)) + "&s=0&p="
cha_url = cha_url + pcid
cha_page = session.get(cha_url,cookies=response.meta['cookiejar'],headers=headers)
with open("cha.jpg", 'wb') as f:
f.write(cha_page.content)
f.close()
try:
im = Image.open("cha.jpg")
im.show()
im.close()
except:
print(u"请到当前目录下，找到验证码后输入")
"""
def get_loop_url(self,response):
login_loop = response.body.decode('gbk')
pa = r'location\.replace\([\'"](.*?)[\'"]\)'
loop_url = re.findall(pa, login_loop)[0]
print(loop_url)
return[Request(loop_url,meta={'cookiejar':response.meta['cookiejar']},callback=self.after_login)]
def after_login(self,response):
return[Request('http://weibo.com/',meta={'cookiejar':response.meta['cookiejar']},callback=self.parse)]
def parse(self, response):
print(response.body.decode('utf-8'))
