cloudflare5s后跳转的网页怎么爬取

有很多网站使用了cloudflare 的反爬虫服务&＃xff0c;第一次打开任何页面都需要等待5s才能进入后面的页面。这种页面需要解析js才能获取到跳转参数。完成跳转后才能获取有效COOKIE。

不喜欢bb&＃xff0c;直接上代码了。

# -*- coding: utf-8 -*-
# &＃64;Time : 2019/8/21 20:48
# &＃64;Author : meng_zhihao
# &＃64;Email : 312141830&＃64;qq.com
# &＃64;File : five_seconds_redirect.py&＃39;curl "https://steamdb.info/cdn-cgi/l/chk_jschl?s&＃61;a008fbe38534ed25da1fcfeee8818c71088155e2-1566391545-1800-AS6fBv4Md5hbFH5KuOu3rUO53K8YLifU6bByW039xKgE^%^2BB^%^2Fl3rJNXQjLqvAq^%^2FCNSWqfrbNCiBprNC4fTtXfmasWS20yWx2vBKGjya^%^2BTVhU8PsS8myK8ty1gUsqY7iuvZmw^%^3D^%^3D^&jschl_vc&＃61;bd263529a25342dfc2bf2d06ec6a32f9^&pass&＃61;1566391549.871-VHEZj8fTNM^&jschl_answer&＃61;16.6665600261" -H "authority: steamdb.info" -H "pragma: no-cache" -H "cache-control: no-cache" -H "upgrade-insecure-requests: 1" -H "user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.108 Safari/537.36" -H "accept: text/html,application/xhtml&＃43;xml,application/xml;q&＃61;0.9,image/webp,image/apng,*/*;q&＃61;0.8,application/signed-exchange;v&＃61;b3" -H "referer: https://steamdb.info/" -H "accept-encoding: gzip, deflate, br" -H "accept-language: zh-CN,zh;q&＃61;0.9" -H "COOKIE: __cfduid&＃61;d1117a0185a634e26f5f076daff94a2c01566391545" --compressed&＃39;import requests
from lxml import etree
import re
from js2py import eval_js
import time
import urllib
HEADERS &＃61; {&＃39;User-Agent&＃39;:&＃39;Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36&＃39;}def getXpath(xpath, content): # xptah操作貌似会把中文变成转码&＃xxxx; /text()变unicode编码tree &＃61; etree.HTML(content)out &＃61; []results &＃61; tree.xpath(xpath)for result in results:if &＃39;ElementStringResult&＃39; in str(type(result)) or &＃39;ElementUnicodeResult&＃39; in str(type(result)):out.append(result)else:out.append(etree.tostring(result))return outdef get_js_return(content):""""""# jschl_vc &＃61; "bd263529a25342dfc2bf2d06ec6a32f9"# passwd &＃61; "1566391549.871-VHEZj8fTNM"# jschl_answer &＃61; "16.6665600261"l &＃61; re.findall(r&＃39;name&＃61;"jschl_vc" value&＃61;"(.*?)"&＃39;, content)if l: jschl_vc &＃61; l[0]l &＃61; re.findall(r&＃39;name&＃61;"pass" value&＃61;"(.*?)"&＃39;, content)if l: passwd &＃61; l[0]m &＃61; re.search(r&＃39;setTimeout\(function\(\)\{((?:.|\n)*?)f\.submit\(\)&＃39;, content)if m:s &＃61; m.group(1)l &＃61; s.split("\n")l &＃61; [i for i in l if i.split()]first &＃61; l[0]last &＃61; l[-2]_ret &＃61; re.search(r"(.*?)a\.value\s&＃43;&＃61;\s&＃43;((.*?)121&＃39;)", last)if _ret:last &＃61; _ret.group(1)ret &＃61; _ret.group(2)js &＃61; "function f(){ %s %s %s return %s }" % (first,"""t &＃61; &＃39;https://steamdb.info/&＃39;;r &＃61; t.match(/https?:\/\//)[0];t &＃61; t.substr(r.length); t &＃61; t.substr(0,t.length-1);""", last,ret)jschl_answer &＃61; eval_js(js)()print(jschl_answer)return jschl_vc, passwd, jschl_answerdef get_COOKIE():se &＃61; requests.session()page &＃61; se.get(&＃39;https://steamdb.info/&＃39;,timeout&＃61;10,headers&＃61;HEADERS).content.decode(&＃39;utf8&＃39;)print(page)l &＃61; re.findall(r&＃39;name&＃61;"s" value&＃61;"(.*?)"&＃39;, page)if l:s &＃61; l[0]s &＃61; urllib.parse.quote(s)else:raise Exceptionjschl_vc, passwd, jschl_answer &＃61; get_js_return(page)print(s,jschl_vc, passwd, jschl_answer)time.sleep(4)url &＃61; &＃39;https://steamdb.info/cdn-cgi/l/chk_jschl?s&＃61;%s&jschl_vc&＃61;%s&pass&＃61;%s&jschl_answer&＃61;%s&＃39;%(s,jschl_vc,passwd,jschl_answer)print(url)se.get(url,timeout&＃61;10,headers&＃61;HEADERS)time.sleep(2)new_page &＃61; se.get(&＃39;https://steamdb.info/&＃39;,timeout&＃61;10,headers&＃61;HEADERS,).content.decode(&＃39;utf8&＃39;)print(new_page) # 完整的页面COOKIE &＃61; se.COOKIEs.get_dict() # 完整的COOKIEprint(COOKIE)return COOKIEif __name__ &＃61;&＃61; &＃39;__main__&＃39;:get_COOKIE()