Selenium爬虫登录生成COOKIE
网站登录&＃xff0c;本质上就是登录完成之后服务器给客户端一个凭据&＃xff0c;这个凭据可以存在COOKIE里&＃xff0c;也可以存在别的地方&＃xff0c;客户端下次请求网页的时候带上这个凭据&＃xff0c;服务端就会认为这个客户端是登录过的。所以对于爬虫来说&＃xff0c;生成COOKIEs是代理使用成为了一个必须要做的事情。
示例
安装chrome和对应driver
下载chrome对应版本的chrome deriver
下载对应版本 driver
https://chromedriver.chromium.org/downloads
例如&＃xff1a;
生成COOKIEs
我们使用chrome driver来进行登录和COOKIE的生成
import os
import time
import zipfile
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
class GenCOOKIEs(object):
USER_AGENT &＃61; open(&＃39;useragents.txt&＃39;).readlines()
# 16yun 代理配置
PROXY_HOST &＃61; &＃39;t.16yun.cn&＃39; # proxy or host 亿牛云代理www.16yun.cn
PROXY_PORT &＃61; 31111 # port
PROXY_USER &＃61; &＃39;USERNAME&＃39; # username
PROXY_PASS &＃61; &＃39;PASSWORD&＃39; # password
&＃64;classmethod
def get_chromedriver(cls, use_proxy&＃61;False, user_agent&＃61;None):
manifest_json &＃61; """
{
"version": "1.0.0",
"manifest_version": 2,
"name": "Chrome Proxy",
"permissions": [
"proxy",
"tabs",
"unlimitedStorage",
"storage",
"
"webRequest",
"webRequestBlocking"
],
"background": {
"scripts": ["background.js"]
},
"minimum_chrome_version":"22.0.0"
}
"""
background_js &＃61; """
var config &＃61; {
mode: "fixed_servers",
rules: {
singleProxy: {
scheme: "http",
host: "%s",
port: parseInt(%s)
},
bypassList: ["localhost"]
}
};
chrome.proxy.settings.set({value: config, scope: "regular"}, function() {});
function callbackFn(details) {
return {
authCredentials: {
username: "%s",
password: "%s"
}
};
}
chrome.webRequest.onAuthRequired.addListener(
callbackFn,
{urls: ["
[&＃39;blocking&＃39;]
);
""" % (cls.PROXY_HOST, cls.PROXY_PORT, cls.PROXY_USER, cls.PROXY_PASS)
path &＃61; os.path.dirname(os.path.abspath(__file__))
chrome_options &＃61; webdriver.ChromeOptions()
if use_proxy:
pluginfile &＃61; &＃39;proxy_auth_plugin.zip&＃39;
with zipfile.ZipFile(pluginfile, &＃39;w&＃39;) as zp:
zp.writestr("manifest.json", manifest_json)
zp.writestr("background.js", background_js)
chrome_options.add_extension(pluginfile)
if user_agent:
chrome_options.add_argument(&＃39;--user-agent&＃61;%s&＃39; % user_agent)
driver &＃61; webdriver.Chrome(
os.path.join(path, &＃39;chromedriver&＃39;),
chrome_options&＃61;chrome_options)
return driver
def __init__(self, username, password):
self.url &＃61; &＃39;https://passport.weibo.cn/signin/login?entry&＃61;mweibo&r&＃61;https://m.weibo.cn/&＃39;
self.browser &＃61; self.get_chromedriver(use_proxy&＃61;True, user_agent&＃61;self.USER_AGENT)
self.wait &＃61; WebDriverWait(self.browser, 20)
self.username &＃61; username
self.password &＃61; password
def open(self):
"""
打开网页输入用户名密码并点击
:return: None
"""
self.browser.delete_all_COOKIEs()
self.browser.get(self.url)
username &＃61; self.wait.until(EC.presence_of_element_located((By.ID, &＃39;loginName&＃39;)))
password &＃61; self.wait.until(EC.presence_of_element_located((By.ID, &＃39;loginPassword&＃39;)))
submit &＃61; self.wait.until(EC.element_to_be_clickable((By.ID, &＃39;loginAction&＃39;)))
username.send_keys(self.username)
password.send_keys(self.password)
time.sleep(1)
submit.click()
def password_error(self):
"""
判断是否密码错误
:return:
"""
try:
return WebDriverWait(self.browser, 5).until(
EC.text_to_be_present_in_element((By.ID, &＃39;errorMsg&＃39;), &＃39;用户名或密码错误&＃39;))
except TimeoutException:
return False
def get_COOKIEs(self):
"""
获取COOKIEs
:return:
"""
return self.browser.get_COOKIEs()
def main(self):
"""
入口
:return:
"""
self.open()
if self.password_error():
return {
&＃39;status&＃39;: 2,
&＃39;content&＃39;: &＃39;用户名或密码错误&＃39;
}
# 如果不需要验证码直接登录成功
COOKIEs &＃61; self.get_COOKIEs()
return {
&＃39;status&＃39;: 1,
&＃39;content&＃39;: COOKIEs
}
if __name__ &＃61;&＃61; &＃39;__main__&＃39;:
result &＃61; GenCOOKIEs(
username&＃61;&＃39;180000000&＃39;,
password&＃61;&＃39;16yun&＃39;,
).main()
print(result)
import osimport timeimport zipfilefrom selenium import webdriverfrom selenium.common.exceptions import TimeoutExceptionfrom selenium.webdriver.common.by import Byfrom selenium.webdriver.support import expected_conditions as ECfrom selenium.webdriver.support.ui import WebDriverWaitclass GenCOOKIEs(object): USER_AGENT &＃61; open(&＃39;useragents.txt&＃39;).readlines() # 16yun 代理配置 PROXY_HOST &＃61; &＃39;t.16yun.cn&＃39; # proxy or host 亿牛云代理www.16yun.cn PROXY_PORT &＃61; 31111 # port PROXY_USER &＃61; &＃39;USERNAME&＃39; # username PROXY_PASS &＃61; &＃39;PASSWORD&＃39; # password &＃64;classmethod def get_chromedriver(cls, use_proxy&＃61;False, user_agent&＃61;None): manifest_json &＃61; """ { "version": "1.0.0", "manifest_version": 2, "name": "Chrome Proxy", "permissions": [ "proxy", "tabs", "unlimitedStorage", "storage", "
useragents.txt文件里面是随机的ua
Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.16) Gecko/20110319 Firefox/40Mozilla/5.0 (Windows NT 6.2; rv:39.0) Gecko/20100101 Firefox/39.0Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; QQBrowser/8.3.4769.400)Mozilla/5.0 (Windows NT 6.1; rv:39.0) Gecko/20100101 Firefox/39.0
京公网安备 11010802041100号 | 京ICP备19059560号-4 | PHP1.CN 第一PHP社区 版权所有