python爬取知乎发现文件存储[txt，json，csv，mongodb]

使用XPath

import requests
import json
from lxml import etree
from urllib import parse
&＃39;&＃39;&＃39;
遇到不懂的问题&＃xff1f;Python学习交流群&＃xff1a;821460695满足你的需求&＃xff0c;资料都已经上传群文件&＃xff0c;可以自行下载&＃xff01;
&＃39;&＃39;&＃39;
url &＃61; &＃39;https://www.zhihu.com/explore&＃39;
headers &＃61; {&＃39;User-Agent&＃39;: &＃39;Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36&＃39;
}
html &＃61; requests.get(url, headers&＃61;headers).text
# 响应返回的是字符串&＃xff0c;解析为HTML DOM模式 text &＃61; etree.HTML(html)
text &＃61; etree.HTML(html)
# 返回所有内容的结点位置
node_list &＃61; text.xpath(&＃39;//div[&＃64;class&＃61;"explore-feed feed-item"]&＃39;)
items &＃61;{}
for node in node_list:# xpath返回的列表&＃xff0c;这个列表就这一个参数&＃xff0c;用索引方式取出来#问题question &＃61; node.xpath(&＃39;.//h2/a&＃39;)[0].text.replace("\n","")# 作者author &＃61; node.xpath(&＃39;.//*[&＃64;class&＃61;"author-link-line"]/*&＃39;)[0].text#author &＃61; "".join(node.xpath(&＃39;.//*[&＃64;class&＃61;"author-link-line"]//text()&＃39;)).replace("\n","")# 回答answer &＃61; node.xpath(&＃39;.//*[&＃64;class&＃61;"content"]&＃39;)[0].text#answer &＃61; "".join(node.xpath(&＃39;.//*[&＃64;class&＃61;"content"]/text()&＃39;)).strip()#answer &＃61; str(node.xpath(&＃39;.//*[&＃64;class&＃61;"content"]/text()&＃39;))[1:-1]items &＃61; {"question" : question,"author" : author,"answer" : answer,} with open("explore.json", "a") as f:#f.write(json.dumps(items, ensure_ascii &＃61; False).encode("utf-8") &＃43; "\n")f.write(json.dumps(items, ensure_ascii &＃61; False) &＃43; "\n")


####保存为TXT

import requests
from lxml import etree
from urllib import parse
&＃39;&＃39;&＃39;
遇到不懂的问题&＃xff1f;Python学习交流群&＃xff1a;821460695满足你的需求&＃xff0c;资料都已经上传群文件&＃xff0c;可以自行下载&＃xff01;
&＃39;&＃39;&＃39;
url &＃61; &＃39;https://www.zhihu.com/explore&＃39;
headers &＃61; {&＃39;User-Agent&＃39;: &＃39;Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36&＃39;
}
html &＃61; requests.get(url, headers&＃61;headers).text
# 响应返回的是字符串&＃xff0c;解析为HTML DOM模式 text &＃61; etree.HTML(html)
text &＃61; etree.HTML(html)
# 返回所有内容的结点位置
node_list &＃61; text.xpath(&＃39;//div[&＃64;class&＃61;"explore-feed feed-item"]&＃39;)for node in node_list:# xpath返回的列表&＃xff0c;这个列表就这一个参数&＃xff0c;用索引方式取出来#问题question &＃61; node.xpath(&＃39;.//h2/a&＃39;)[0].text.replace("\n","")# 作者author &＃61; node.xpath(&＃39;.//*[&＃64;class&＃61;"author-link-line"]/*&＃39;)[0].text#author &＃61; "".join(node.xpath(&＃39;.//*[&＃64;class&＃61;"author-link-line"]//text()&＃39;)).replace("\n","")# 回答answer &＃61; node.xpath(&＃39;.//*[&＃64;class&＃61;"content"]&＃39;)[0].text#answer &＃61; "".join(node.xpath(&＃39;.//*[&＃64;class&＃61;"content"]/text()&＃39;)).strip()#answer &＃61; str(node.xpath(&＃39;.//*[&＃64;class&＃61;"content"]/text()&＃39;))[1:-1]with open(&＃39;explore.txt&＃39;, &＃39;a&＃39;, encoding&＃61;&＃39;utf-8&＃39;) as file:file.write(&＃39;\n&＃39;.join([question, author, answer]))file.write(&＃39;\n&＃39; &＃43; &＃39;&＃61;&＃39; * 50 &＃43; &＃39;\n&＃39;)


保存为csv

import requests
from lxml import etree
from urllib import parse
import csvurl &＃61; &＃39;https://www.zhihu.com/explore&＃39;
headers &＃61; {&＃39;User-Agent&＃39;: &＃39;Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36&＃39;
}
html &＃61; requests.get(url, headers&＃61;headers).text
# 响应返回的是字符串&＃xff0c;解析为HTML DOM模式 text &＃61; etree.HTML(html)
text &＃61; etree.HTML(html)
# 返回所有内容的结点位置
node_list &＃61; text.xpath(&＃39;//div[&＃64;class&＃61;"explore-feed feed-item"]&＃39;)for node in node_list:# xpath返回的列表&＃xff0c;这个列表就这一个参数&＃xff0c;用索引方式取出来#问题question &＃61; node.xpath(&＃39;.//h2/a&＃39;)[0].text.replace("\n","")# 作者author &＃61; node.xpath(&＃39;.//*[&＃64;class&＃61;"author-link-line"]/*&＃39;)[0].text#author &＃61; "".join(node.xpath(&＃39;.//*[&＃64;class&＃61;"author-link-line"]//text()&＃39;)).replace("\n","")# 回答,为方便展示&＃xff0c;只取部分内容,text[ :10]answer &＃61; node.xpath(&＃39;.//*[&＃64;class&＃61;"content"]&＃39;)[0].text[ :10]#answer &＃61; node.xpath(&＃39;.//*[&＃64;class&＃61;"content"]&＃39;)[0].text#answer &＃61; "".join(node.xpath(&＃39;.//*[&＃64;class&＃61;"content"]/text()&＃39;)).strip()#answer &＃61; str(node.xpath(&＃39;.//*[&＃64;class&＃61;"content"]/text()&＃39;))[1:-1]with open(&＃39;explore.csv&＃39;, &＃39;a&＃39;, encoding&＃61;&＃39;utf-8&＃39;) as csvfile:fieldnames &＃61; [&＃39;question&＃39;, &＃39;author&＃39;, &＃39;answer&＃39;]writer &＃61; csv.DictWriter(csvfile, fieldnames&＃61;fieldnames)writer.writeheader()writer.writerow({&＃39;question&＃39;: question, &＃39;author&＃39;: author, &＃39;answer&＃39;: answer})


####读取csv

1 import csv
2
3 with open(&＃39;explore.csv&＃39;, &＃39;r&＃39;, encoding&＃61;&＃39;utf-8&＃39;) as csvfile:
4 reader &＃61; csv.reader(csvfile)
5 for row in reader:
6 print(row)


####保存到MongoDB

import requests
from lxml import etree
from urllib import parse
from pymongo import MongoClientclient &＃61; MongoClient()
db &＃61; client[&＃39;explore&＃39;]
collection &＃61; db[&＃39;explore&＃39;]url &＃61; &＃39;https://www.zhihu.com/explore&＃39;
headers &＃61; {&＃39;User-Agent&＃39;: &＃39;Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36&＃39;
}
html &＃61; requests.get(url, headers&＃61;headers).text
# 响应返回的是字符串&＃xff0c;解析为HTML DOM模式 text &＃61; etree.HTML(html)
text &＃61; etree.HTML(html)
# 返回所有内容的结点位置
node_list &＃61; text.xpath(&＃39;//div[&＃64;class&＃61;"explore-feed feed-item"]&＃39;)for node in node_list:# xpath返回的列表&＃xff0c;这个列表就这一个参数&＃xff0c;用索引方式取出来#问题question &＃61; node.xpath(&＃39;.//h2/a&＃39;)[0].text.replace("\n","")# 作者author &＃61; node.xpath(&＃39;.//*[&＃64;class&＃61;"author-link-line"]/*&＃39;)[0].text#author &＃61; "".join(node.xpath(&＃39;.//*[&＃64;class&＃61;"author-link-line"]//text()&＃39;)).replace("\n","")# 回答answer &＃61; node.xpath(&＃39;.//*[&＃64;class&＃61;"content"]&＃39;)[0].text#answer &＃61; "".join(node.xpath(&＃39;.//*[&＃64;class&＃61;"content"]/text()&＃39;)).strip()#answer &＃61; str(node.xpath(&＃39;.//*[&＃64;class&＃61;"content"]/text()&＃39;))[1:-1]items &＃61; {"question" : question,"author" : author,"answer" : answer,} if collection.insert(items):print(&＃39;Saved to Mongo&＃39;)