requests获取百度推广信息

2019年的第一篇博客&＃xff0c;恩&＃xff0c;好久没写过博客了&＃xff0c;恩&＃xff0c;忘了&＃xff0c;哈哈&＃xff0c;实在是太懒了

今天写一个爬取百度推广数据的爬虫&＃xff0c;当然我写的肯定不是那么的完美&＃xff0c;但是能用&＃xff0c;大哭

注意&＃xff1a;有的时候&＃xff0c;get或post方法获取数据是会报ssl等错误&＃xff0c;网站要验证啥的&＃xff0c;没搞懂&＃xff0c;网上搜索&＃xff0c;都是设置 verify&＃61;False&＃xff0c;我也懒得去详细分析&＃xff0c;你们要是有兴趣可以去了解一下&＃xff0c;然后这么设置了&＃xff0c;在GET或者post是会有warning提示&＃xff0c;编写代码&＃xff1a;

# 禁用ssl发出的警告
requests.packages.urllib3.disable_warnings()

下面代码&＃xff1a;

def main():
　　#开启一个session对话
main_session &＃61; requests.session()
return main_session

def get_cost_info(main_session, endtime, flag1, flag2):
　　#获取省或市在日期或者月份下的消费信息
if flag1 &＃61;&＃61; &＃39;province&＃39;:
splitDimension &＃61; "provinceName"
else:
splitDimension &＃61; "provinceCityName"
if flag2 &＃61;&＃61; &＃39;month&＃39;:
unitOfTime &＃61; 3
else:
unitOfTime &＃61; 5
headers &＃61; {
&＃39;Accept&＃39;:&＃39;application/json&＃39;,
&＃39;Accept-Encoding&＃39;:&＃39;gzip, deflate&＃39;,
&＃39;Accept-Language&＃39;:&＃39;zh-CN,zh;q&＃61;0.8&＃39;,
&＃39;Connection&＃39;:&＃39;keep-alive&＃39;,
&＃39;Content-Length&＃39;:&＃39;763&＃39;,
&＃39;Content-Type&＃39;:&＃39;application/x-www-form-urlencoded&＃39;,
&＃39;COOKIE&＃39;:setting.COOKIE,
&＃39;DNT&＃39;:&＃39;1&＃39;,
&＃39;Host&＃39;:&＃39;fengchao.baidu.com&＃39;,
&＃39;Origin&＃39;:&＃39;https://fengchao.baidu.com&＃39;,
&＃39;Referer&＃39;:&＃39;https://fengchao.baidu.com/fc/report/dashboard/user/%s/account&＃39;%setting.userid,
&＃39;User-Agent&＃39;:&＃39;Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 BIDUBrowser/8.7 Safari/537.36&＃39;,
&＃39;X-DevTools-Emulate-Network-Conditions-Client-Id&＃39;:&＃39;87F3C66D-3166-46F6-8B46-141057354EBC&＃39;
}
params &＃61; {
"userId":setting.userid,
"ids":[setting.userid],
"idType":2,
"splitDimension":splitDimension,
"limit":[0,1000],
"sortRules":[],
"levelOfDetails":211,
"startDate":"2019-01-01",
"endDate":endtime,
"predicateConditions":[],
"unitOfTime":unitOfTime,
"columns":["userId","date","accountName","impression","click","cost","cpc","ctr","conversion","phoneConversion","bridgeConversion"]
}
formdata &＃61; {
&＃39;reqid&＃39;: setting.reqid,
&＃39;eventId&＃39;: setting.eventId,
&＃39;userid&＃39;: setting.userid,
&＃39;token&＃39;: setting.token,
&＃39;path&＃39;: &＃39;mars/GET/AccountReportDataService/getAccountDataCenterReportData&＃39;,
&＃39;params&＃39;: json.dumps(params)
}
　　#url是去F12---network获取的
url_1 &＃61; &＃39;https://fengchao.baidu.com/hairuo/request.ajax?path&＃61;mars/GET/AccountReportDataService/getAccountDataCenterReportData&reqid&＃61;%s&＃39; % formdata[&＃39;reqid&＃39;]
cont_1 &＃61; main_session.post(url_1, headers&＃61;headers, data&＃61;formdata, verify&＃61;False)
datas &＃61; cont_1.json()
cont_list_1 &＃61; datas[&＃39;data&＃39;][&＃39;ACCOUNT&＃39;][&＃39;rows&＃39;]
for i in cont_list_1:
cont_list_2 &＃61; i[&＃39;subRows&＃39;]
cont_list &＃61; []
for j in cont_list_2:
if flag2 &＃61;&＃61; &＃39;month&＃39;:
time_1 &＃61; j[&＃39;date&＃39;][0:7]
time_list1 &＃61; time_1.split(&＃39;-&＃39;)
date &＃61; &＃39;%s年%s月份&＃39; % (time_list1[0], str(int(time_list1[1])))
else:
date &＃61; j[&＃39;date&＃39;]
zhanghu &＃61; j[&＃39;accountName&＃39;]
province, city &＃61; &＃39;&＃39;, &＃39;&＃39;
if flag1 &＃61;&＃61; &＃39;province&＃39;:
province &＃61; j[&＃39;provinceName&＃39;]
else:
city_list &＃61; j[&＃39;provinceCityName&＃39;].split(&＃39;-&＃39;)
province, city &＃61; city_list[0], city_list[1]
zhanxian &＃61; str(j[&＃39;impression&＃39;])
dianji &＃61; str(j[&＃39;click&＃39;])
xiaofei &＃61; str(j[&＃39;cost&＃39;])
pinjunjiage &＃61; str(&＃39;%.2f&＃39; % (j[&＃39;cpc&＃39;]))
dianjilv &＃61; str(&＃39;{:.2%}&＃39;.format(j[&＃39;ctr&＃39;]))
wangyezhuanhua &＃61; str(j[&＃39;conversion&＃39;])
dianhuazhuanhua &＃61; str(j[&＃39;phoneConversion&＃39;])
shangqiaozhuanhua &＃61; str(j[&＃39;bridgeConversion&＃39;])
cont_list.append([date, zhanghu, province, zhanxian, dianji, xiaofei, pinjunjiage, dianjilv, wangyezhuanhua, dianhuazhuanhua, shangqiaozhuanhua])

 经过测试&＃xff0c;COOKIE一般能保持7天不过期&＃xff0c;七天之后就要重新登录浏览器F12获取