简书python爬虫_python爬虫简书评论

某些网站的一些数据是通过js加载的 ,所以爬取下来的数据拿不到,

找到评论的地址 .进行请求获取评论数据

#coding&＃61;utf-8

import json

import requests

def requests_view(response):

import webbrowser

requests_url &＃61; response.url

base_url &＃61; &＃39;

base_url &＃61; base_url.encode(&＃39;utf-8&＃39;)

content &＃61; response.content.replace(b"

tem_html &＃61; open(&＃39;tmp.html&＃39;,&＃39;wb&＃39;)

tem_html.write(content)

tem_html.close()

webbrowser.open_new_tab("tmp.html")

headers &＃61; {

"User-Agent": &＃39;Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36&＃39;}

response &＃61; requests.get("https://www.jianshu.com/notes/26504955/comments?comment_id&＃61;&author_only&＃61;false&since_id&＃61;0&max_id&＃61;1586510606000&order_by&＃61;likes_count&page&＃61;1",headers&＃61;headers)

comments &＃61; json.loads(response.content)

if comments[&＃39;comment_exist&＃39;] &＃61;&＃61; True:

for item in comments[&＃39;comments&＃39;]:

print(item[&＃39;user&＃39;][&＃39;nickname&＃39;],item[&＃39;compiled_content&＃39;])