def process_request(self,request,spider): # 获取redis中所有的键(假设redis中只保存了cookies) rediskeys = self.rconn.keys() elem = random.choice(rediskeys) request.cookies = cookie # 在请求中记录当前cookies对应的账号和密码 request.meta["accounttext"] = elem.split(":")[-1]
def process_response(self,request,response,spider): if response.status in [300, 301, 302, 303]: # 获取重定向的url redirect_url = response.headers["location"] if url == "login_url":# 如果是登陆页面,说明当前cookies失效了,需要更新 username,passworod = request.meta['accounttext'].split("--") update_cookie(spider_name,username,password) elif url=="验证页面":# 说明账号被封了 username,passworod = request.meta['accounttext'].split("--") remove_cookie(spider_name,username,password) request = request.copy() request.dont_filter = true return request
如对本文有疑问, 点击进行留言回复!!
网友评论