day26学习总结与作业

day学习总结与作业

import csvimport requests
from bs4 import BeautifulSoup
import re
headers &＃61; {&＃39;user-agent&＃39;: &＃39;Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36&＃39;
}
url &＃61; &＃39;https://mapi.guazi.com/car-source/carList/pcList?minor&＃61;benz&sourceType&＃61;&ec_buy_car_list_ab&＃61;&location_city&＃61;&district_id&＃61;&tag&＃61;-1&license_date&＃61;&auto_type&＃61;&driving_type&＃61;&gearbox&＃61;&road_haul&＃61;&air_displacement&＃61;&emission&＃61;&car_color&＃61;&guobie&＃61;&bright_spot_config&＃61;&seat&＃61;&fuel_type&＃61;&order&＃61;&priceRange&＃61;0,-1&tag_types&＃61;&diff_city&＃61;&intention_options&＃61;&initialPriceRange&＃61;&monthlyPriceRange&＃61;&transfer_num&＃61;&car_year&＃61;&carid_qigangshu&＃61;&carid_jinqixingshi&＃61;&cheliangjibie&＃61;&page&＃61;1&pageSize&＃61;20&city_filter&＃61;12&city&＃61;12&guazi_city&＃61;12&qpres&＃61;&versionId&＃61;0.0.0.0&osv&＃61;Unknown&platfromSource&＃61;wap&＃39;
response &＃61; requests.get(url, headers&＃61;headers)
result0&＃61;response.json()[&＃39;data&＃39;][&＃39;postList&＃39;]
red&＃61;[]
table &＃61; {&＃39;0xe1d0&＃39;: &＃39;7&＃39;, &＃39;0xe325&＃39;: &＃39;4&＃39;, &＃39;0xe41d&＃39;: &＃39;1&＃39;, &＃39;0xe52e&＃39;: &＃39;9&＃39;, &＃39;0xe630&＃39;: &＃39;2&＃39;, &＃39;0xe76e&＃39;: &＃39;8&＃39;,&＃39;0xe891&＃39;: &＃39;5&＃39;, &＃39;0xe9ce&＃39;: &＃39;0&＃39;, &＃39;0xeaf2&＃39;: &＃39;3&＃39;, &＃39;0xec4c&＃39;: &＃39;6&＃39;, &＃39;0xf88a&＃39;: &＃39;7&＃39;
}
for x in result0:title&＃61;x[&＃39;title&＃39;]license_date&＃61;x[&＃39;license_date&＃39;]road_haul&＃61;x[&＃39;road_haul&＃39;].split(&＃39;;&＃39;)new_road_haul &＃61; &＃39;&＃39;for g in road_haul:if g.startswith(&＃39;&＃&＃39;):new_road_haul &＃43;&＃61; table[hex(int(g[2:]))]elif g.startswith(&＃39;.&＃&＃39;):new_road_haul &＃43;&＃61; &＃39;.&＃39; &＃43; table[hex(int(g[3:]))]else:new_road_haul &＃43;&＃61; gfirst_pay&＃61;x[&＃39;first_pay&＃39;].split(&＃39;;&＃39;)new_first_pay &＃61; &＃39;&＃39;for h in first_pay:if h.startswith(&＃39;&＃&＃39;):new_first_pay &＃43;&＃61; table[hex(int(h[2:]))]elif h.startswith(&＃39;.&＃&＃39;):new_first_pay &＃43;&＃61; &＃39;.&＃39; &＃43; table[hex(int(h[3:]))]else:new_first_pay &＃43;&＃61; hif not new_first_pay:new_first_pay&＃61;&＃39;不支持首付&＃39;buy_out_price&＃61;x[&＃39;price&＃39;].split(&＃39;;&＃39;)new_buy_out_price &＃61; &＃39;&＃39;for h in buy_out_price:if h.startswith(&＃39;&＃&＃39;):new_buy_out_price&＃43;&＃61; table[hex(int(h[2:]))]elif h.startswith(&＃39;.&＃&＃39;):new_buy_out_price &＃43;&＃61; &＃39;.&＃39; &＃43; table[hex(int(h[3:]))]else:new_buy_out_price &＃43;&＃61; hred.append([title,license_date,new_road_haul,new_first_pay,new_buy_out_price])
result1&＃61;csv.writer(open(&＃39;files/ershouche.csv&＃39;,&＃39;w&＃39;,encoding&＃61;&＃39;utf-8&＃39;,newline&＃61;&＃39;&＃39;))
result1.writerow([&＃39;标题&＃39;,&＃39;年限&＃39;,&＃39;里程&＃39;,&＃39;首付&＃39;,&＃39;一次付&＃39;])
result1.writerows(red)


学习总结

import requestsdef get_html(url):headers &＃61; {&＃39;user-agent&＃39;: &＃39;Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36&＃39;}# 1. 使用代理: 给参数proxies赋值# {&＃39;https&＃39;: &＃39;ip:端口&＃39;}# 1&＃xff09;使用固定的代理ipresponse &＃61; requests.get(url, headers&＃61;headers, proxies&＃61;{&＃39;https&＃39;: &＃39;36.25.226.139:4513&＃39;})# 2&＃xff09;通过请求获取代理ip的地址动态获取最新的代理# ip &＃61; requests.get(&＃39;http://d.jghttp.alicloudecs.com/getip?num&＃61;1&type&＃61;1&pro&＃61;&city&＃61;0&yys&＃61;0&port&＃61;11&time&＃61;4&ts&＃61;0&ys&＃61;0&cs&＃61;0&lb&＃61;1&sb&＃61;0&pb&＃61;4&mr&＃61;1®ions&＃61;&＃39;).text.strip()# response &＃61; requests.get(url, headers&＃61;headers, proxies&＃61;{&＃39;https&＃39;: ip})# response &＃61; requests.get(url, headers&＃61;headers)print(response.text)return response.textif __name__ &＃61;&＃61; &＃39;__main__&＃39;:url &＃61; &＃39;https://movie.douban.com/top250&＃39;get_html(url)# while True:# url &＃61; &＃39;https://movie.douban.com/top250&＃39;# get_html(url)
from selenium.webdriver import Chrome, ChromeOptionsoptions &＃61; ChromeOptions()
options.add_experimental_option(&＃39;excludeSwitches&＃39;, [&＃39;enable-automation&＃39;])
options.add_experimental_option("prefs", {"profile.managed_default_content_settings.images": 2})# 1. 使用代理
# --proxy-server&＃61;http://IP:端口号 &＃xff08;ip端口是https的ip&＃xff09;
options.add_argument(&＃39;--proxy-server&＃61;http://122.6.202.214:4510&＃39;)b &＃61; Chrome(options&＃61;options)b.get(&＃39;https://movie.douban.com/top250&＃39;)
import requests# requests完成自动登录的步骤:
"""
第1步&＃xff1a;在谷歌浏览器中打开网页完成登录操作&＃xff0c;然后刷新页面
第2步&＃xff1a;打开当前页面的检查&＃xff0c;在network的All选项下&＃xff0c;找到当前页面的请求&＃xff0c;获取Request Headers中的COOKIE值
第3步&＃xff1a;在用requests发送请求的时候给headers赋值&＃xff0c;在headers中添加 COOKIE对应的键值对
"""
headers &＃61; {&＃39;COOKIE&＃39;: &＃39;_zap&＃61;b7912ec1-8e24-4d72-81cd-8fa2a2b5f78f; d_c0&＃61;"AECfz77blxSPTv9pswdcr3xSHE0eCEMj0Dc&＃61;|1646568427"; _xsrf&＃61;qy8EtuCtpOGgIUmA3g0qSi2edFqOUIBw; __snaker__id&＃61;kvuiOSuliPGkosHY; _9755xjdesxxd_&＃61;32; YD00517437729195%3AWM_TID&＃61;pJYUp8Detk5AUUUUFFc6upRtNshfmnQX; q_c1&＃61;51a6e1898c4d46a594865b3db1dd3e95|1647227093000|1647227093000; NOT_UNREGISTER_WAITING&＃61;1; gdxidpyhxdE&＃61;q6eDxuI%5CS11auZ9%2Be%5C%2BxIx%2F2cg96ULLZchHGSiL8EQvgYs9OmbeiyhhCXoa%5C%2BblJQfD%5CayzzA8oosyB%2FhPPMm7%2Fkd8W5prCdSYuppYUL5qdoyPdjsHcgA8pZgBquceXtX6di5Mu46C7dEKcHakVA7mxxjMZh%2Bre5j%2F4AQta4bulZ39y%5C%3A1647834647437; YD00517437729195%3AWM_NI&＃61;oe9bRvHOQTEjCNug5CHPzre%2BCdBGZr6dru1M9KaaTCyY5hZsiM2d%2FSXjfLKsl91VYoTN6x%2Fvc%2FWYtZWgHtA%2BkNpQdcQgt86C%2F1Vffl9dc8gqk08aF1%2Bp0LL%2BSZzsecFXbUI%3D; YD00517437729195%3AWM_NIKE&＃61;9ca17ae2e6ffcda170e2e6ee99e96f86e9ada5f97cbaac8aa3c84f878a9a85aa7afbeeac8cc97e8c989f8bb32af0fea7c3b92aadbbf783d17e8c919fb5c8628192bb88b8498c90b693e82181bc8ea8d97af6bcbe96b447fb95af82d749bb87fb89d979ab93fe85ae599cb6aaadca79b0e7ffb7dc659892fa99ae7ea1f199baf75295b98895c421a992c0b2e76993b9bcd9b549edef9a87e568fbbcadd9e67ba9f0f882c53be9b000adcf219186b694e8698688aeb6cc37e2a3; captcha_session_v2&＃61;2|1:0|10:1647833755|18:captcha_session_v2|88:Q0FUYjlIemk4MklmZlA5ZlVOWDczbGlhYjRabzMxME1Fc0N6Szl4aGVTQU5JQkdGUklNWFRBVVZtU0hYQUVvLw&＃61;&＃61;|cf267f149c0b9509d0e424579194d94fc6ec1f57567afd34050dc2a47f14bacc; captcha_ticket_v2&＃61;2|1:0|10:1647833769|17:captcha_ticket_v2|704:eyJ2YWxpZGF0ZSI6IkNOMzFfZ2pLT21aR2FUbHdXWFo2SG9wTUY1OWRZTG0tcG1vcGlfNUtrUkpzUThJSmNlV2VJdVVzdEhWNks4TVNBcVhobEZPUkZMcjUuc3N1QXpmUzBHbnIuSUNIdjlnb04tdjdoMlBydUtGSmdmV2FYUUZ2SE10dUl0RU40TzRFQS14dEhtMXJRLWg3UHdRblRMVkt1VG1kX09oekdab0pzUGxUWGFmUVF1NzRfYjc4di1QUmlQVUJMR0s5TEhvTGpDX2JUMF9jNFllNlM1SVFYSGgtMHAwV1ZMR2IwMWVqMS5FdEg1ZzZHRUZNZWMwLU8uMXk3YnY2dzlHdGNmNEE1a3E2Ry5EOWJSVlN4QjRWWmZodU1Gdk54N2lvNkZNUjJCdmw4Q254R1NUeUhtLXI1Yl9xeEZPS00wLnJONjkweW14cTQ5MHNjUzBKeHBwUE9lY05BSDZyNV9Vb3ZxZkFkR191aEQwOGc5ZUkwR3k5X005QVkub3gxOS5QdE1GanQwc1dtNERVbVZwQ3ZJV3R4V21nUS5Nd0tVTk92U3N1ZFlmcXJGMmhIWjUyTXZZUzFLc1RQaGlTNTRQWDRETElCdXQuNmU0ZUpzUFUyUzBWeUJZUXE5WFZSOEUwYzE5MEVYX2x5cEJ2MVdManBzVXJlV0t3d19GcGhTLnguVnlGMyJ9|22db975942e5d8c1799fa7c70120378eed6a34bb8bdd00afa3c5eccf92436296; z_c0&＃61;2|1:0|10:1647833781|4:z_c0|92:Mi4xaW5CWUdRQUFBQUFBUUpfUHZ0dVhGQ1lBQUFCZ0FsVk50VUFsWXdEWFYyXzVOZVRzbGJmbVg4Mk1GNF95bUYtLWd3|9d5f40b82ca91b89a903d5d50df74ed66850c2b3ecf03d86510300f5eb729d96; tst&＃61;r; SESSIONID&＃61;hbQ6zfAEKL3cjwOoewSAbkpnJKR4GtTVNCDhhjrorQ2; KLBRSID&＃61;0a401b23e8a71b70de2f4b37f5b4e379|1647833814|1647833734&＃39;,&＃39;user-agent&＃39;: &＃39;Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36&＃39;
}
response &＃61; requests.get(&＃39;https://www.zhihu.com/&＃39;, headers&＃61;headers)print(response.text)
from selenium.webdriver import Chrome
from selenium.webdriver.common.keys import Keys# 1. 创建浏览器对象打开网页
b &＃61; Chrome()
b.get(&＃39;https://www.taobao.com/&＃39;)# 2. 添加本地保存的COOKIE信息
COOKIE_list &＃61; eval(open(&＃39;files/taobao.txt&＃39;, encoding&＃61;&＃39;utf-8&＃39;).read())
for COOKIE in COOKIE_list:b.add_COOKIE(COOKIE)# 3.重新打开网页
b.get(&＃39;https://www.taobao.com/&＃39;)# 4. 进行后续其他操作
search &＃61; b.find_element_by_id(&＃39;q&＃39;)
search.send_keys(&＃39;雪糕&＃39;)
search.send_keys(Keys.ENTER)
from selenium.webdriver import Chrome# 1. 获取COOKIE保存到本地
# 第一步&＃xff1a;创建浏览器对象&＃xff0c;打开需要自动登录的网站
b &＃61; Chrome()
b.get(&＃39;https://www.taobao.com/&＃39;)# 2. 手动完成登录操作
input(&＃39;是否完成:&＃39;)# 3. 获取COOKIE数据并且保存到本地文件中
COOKIEs &＃61; b.get_COOKIEs()
open(&＃39;files/taobao.txt&＃39;, &＃39;w&＃39;, encoding&＃61;&＃39;utf-8&＃39;).write(str(COOKIEs))