分析Ajax爬取今日头条街拍图片

# -*- coding:UTF-8 -*-
import requests
import time
import os
from hashlib import md5def get_page(offset):params &＃61; {&＃39;aid&＃39;: &＃39;24&＃39;,&＃39;app_name&＃39;: &＃39;web_search&＃39;,&＃39;offset&＃39;: offset,&＃39;format&＃39;: &＃39;json&＃39;,&＃39;keyword&＃39;: &＃39;街拍&＃39;,&＃39;autoload&＃39;: &＃39;true&＃39;,&＃39;count&＃39;: &＃39;20&＃39;,&＃39;en_qc&＃39;: &＃39;1&＃39;,&＃39;cur_tab&＃39;: &＃39;1&＃39;,&＃39;from&＃39;: &＃39;search_tab&＃39;,&＃39;pd&＃39;: &＃39;synthesis&＃39;,&＃39;timestamp&＃39;: int(time.time() * 1000)}url &＃61; &＃39;http://www.toutiao.com/search_content/?&＃39;headers &＃61; {&＃39;User-Agent&＃39;: &＃39;Mozilla/5.0 (Windows NT 10.0; Win64; x64) Ap&＃39;&＃39;pleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36&＃39;,&＃39;referer&＃39;: &＃39;https://www.toutiao.com/search/?keyword&＃61;%E8%A1%97%E6%8B%8D&＃39;,&＃39;x-requested-with&＃39;: &＃39;XMLHttpRequest&＃39;}try:response &＃61; requests.get(url&＃61;url, params&＃61;params, headers&＃61;headers)if response.status_code &＃61;&＃61; 200:return response.json()except requests.ConnectionError:return Nonedef get_images(json):if json.get(&＃39;data&＃39;):for item in json.get(&＃39;data&＃39;):if &＃39;title&＃39; in item and &＃39;image_list&＃39; in item and item[&＃39;image_list&＃39;] !&＃61; []:title &＃61; item[&＃39;title&＃39;]images &＃61; item[&＃39;image_list&＃39;]for image in images:yield {&＃39;image&＃39;: image.get(&＃39;url&＃39;),&＃39;title&＃39;: title}def save_image(offset, item):if not os.path.exists(&＃39;picture&＃39;&＃43;str(offset)):os.mkdir(&＃39;picture&＃39;&＃43;str(offset))try:response &＃61; requests.get("http:" &＃43; item.get(&＃39;image&＃39;))if response.status_code &＃61;&＃61; 200:file_path &＃61; &＃39;{0}/{1}.{2}&＃39;.format(&＃39;picture&＃39;&＃43;str(offset), md5(response.content).hexdigest(), &＃39;jpg&＃39;)if not os.path.exists(file_path):with open(file_path, &＃39;wb&＃39;) as f:f.write(response.content)else:print(&＃39;Already Downloaded&＃39;, file_path)except requests.ConnectionError:print(&＃39;Failed to Save Image&＃39;)if __name__ &＃61;&＃61; &＃39;__main__&＃39;:for offset in range(0, 120, 20):json &＃61; get_page(offset)for item in get_images(json):save_image(offset, item)