爬虫（七十六）用urllib和re爬取百度贴吧

主题名称&＃xff1a;r&＃39;href&＃61;"/p/\d&＃43;" title&＃61;"(.&＃43;?)"&＃39;

主题作者&＃xff1a;r&＃39;title&＃61;"主题作者: (.&＃43;?)"&＃39;

链接地址&＃xff1a;r&＃39;href&＃61;"/p/(\d&＃43;)"&＃39;

回复数&＃xff1a;r&＃39;title&＃61;"回复">(\d&＃43;)<&＃39;

创建日期&＃xff1a;r&＃39;title&＃61;"创建时间">(.&＃43;?)<&＃39;

核心代码如下&＃xff1a;

import re
# 解析网页源代码&＃xff0c;提取数据
def parse_page(html):# 主题名称titles &＃61; re.findall(r&＃39;href&＃61;"/p/\d&＃43;" title&＃61;"(.&＃43;?)"&＃39;,html)# 主题作者authods &＃61; re.findall(r&＃39;title&＃61;"主题作者: (.&＃43;?)"&＃39;,html)# 链接地址nums &＃61; re.findall(r&＃39;href&＃61;"/p/(\d&＃43;)"&＃39;,html)links &＃61; [&＃39;http://tieba.baidu.com/p/&＃39;&＃43;str(num) for num in nums]# 回复数量focus &＃61; re.findall(r&＃39;title&＃61;"回复">(\d&＃43;)&＃39;,html)# 创建时间ctimes &＃61; re.findall(r&＃39;title&＃61;"创建时间">(.&＃43;?)<&＃39;,html)# 获得结果data &＃61; zip(titles,authods,links,focus,ctimes)# 返回结果return data

&＃xff08;4&＃xff09;保存数据

下面将数据保存为 txt 文件、json 文件和 csv 文件

import json
import csv
# 打开文件
def openfile(fm,fileName):fd &＃61; Noneif fm &＃61;&＃61; &＃39;txt&＃39;:fd &＃61; open(fileName&＃43;&＃39;.txt&＃39;,&＃39;w&＃39;,encoding&＃61;&＃39;utf-8&＃39;)elif fm &＃61;&＃61; &＃39;json&＃39;:fd &＃61; open(fileName&＃43;&＃39;.json&＃39;,&＃39;w&＃39;,encoding&＃61;&＃39;utf-8&＃39;)elif fm &＃61;&＃61; &＃39;csv&＃39;:fd &＃61; open(fileName&＃43;&＃39;.csv&＃39;,&＃39;w&＃39;,encoding&＃61;&＃39;utf-8&＃39;,newline&＃61;&＃39;&＃39;)return fd# 将数据保存到文件
def save2file(fm,fd,data):if fm &＃61;&＃61; &＃39;txt&＃39;:for item in data:fd.write(&＃39;----------------------------------------\n&＃39;)fd.write(&＃39;title&＃xff1a;&＃39; &＃43; str(item[0]) &＃43; &＃39;\n&＃39;)fd.write(&＃39;authod&＃xff1a;&＃39; &＃43; str(item[1]) &＃43; &＃39;\n&＃39;)fd.write(&＃39;link&＃xff1a;&＃39; &＃43; str(item[2]) &＃43; &＃39;\n&＃39;)fd.write(&＃39;focus&＃xff1a;&＃39; &＃43; str(item[3]) &＃43; &＃39;\n&＃39;)fd.write(&＃39;ctime&＃xff1a;&＃39; &＃43; str(item[4]) &＃43; &＃39;\n&＃39;)if fm &＃61;&＃61; &＃39;json&＃39;:temp &＃61; (&＃39;title&＃39;,&＃39;authod&＃39;,&＃39;link&＃39;,&＃39;focus&＃39;,&＃39;ctime&＃39;)for item in data:json.dump(dict(zip(temp,item)),fd,ensure_ascii&＃61;False)if fm &＃61;&＃61; &＃39;csv&＃39;:writer &＃61; csv.writer(fd)for item in data:writer.writerow(item)

2、编码实现

完整代码如下&＃xff0c;也很简单&＃xff0c;还不到 100 行

import urllib.request
import urllib.parse
import re
import json
import csv
import time
import random# 获取网页源代码
def get_page(url):headers &＃61; {&＃39;USER-AGENT&＃39;:&＃39;Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36&＃39;}req &＃61; urllib.request.Request(url&＃61;url,headers&＃61;headers)response &＃61; urllib.request.urlopen(req)html &＃61; response.read().decode(&＃39;utf-8&＃39;)return html# 解析网页源代码&＃xff0c;提取数据
def parse_page(html):titles &＃61; re.findall(r&＃39;href&＃61;"/p/\d&＃43;" title&＃61;"(.&＃43;?)"&＃39;,html)authods &＃61; re.findall(r&＃39;title&＃61;"主题作者: (.&＃43;?)"&＃39;,html)nums &＃61; re.findall(r&＃39;href&＃61;"/p/(\d&＃43;)"&＃39;,html)links &＃61; [&＃39;http://tieba.baidu.com/p/&＃39;&＃43;str(num) for num in nums]focus &＃61; re.findall(r&＃39;title&＃61;"回复">(\d&＃43;)&＃39;,html)ctimes &＃61; re.findall(r&＃39;title&＃61;"创建时间">(.&＃43;?)<&＃39;,html)data &＃61; zip(titles,authods,links,focus,ctimes)return data# 打开文件
def openfile(fm,fileName):if fm &＃61;&＃61; &＃39;txt&＃39;:return open(fileName&＃43;&＃39;.txt&＃39;,&＃39;w&＃39;,encoding&＃61;&＃39;utf-8&＃39;)elif fm &＃61;&＃61; &＃39;json&＃39;:return open(fileName&＃43;&＃39;.json&＃39;,&＃39;w&＃39;,encoding&＃61;&＃39;utf-8&＃39;)elif fm &＃61;&＃61; &＃39;csv&＃39;:return open(fileName&＃43;&＃39;.csv&＃39;,&＃39;w&＃39;,encoding&＃61;&＃39;utf-8&＃39;,newline&＃61;&＃39;&＃39;)else:return None# 将数据保存到文件
def save2file(fm,fd,data):if fm &＃61;&＃61; &＃39;txt&＃39;:for item in data:fd.write(&＃39;----------------------------------------\n&＃39;)fd.write(&＃39;title&＃xff1a;&＃39; &＃43; str(item[0]) &＃43; &＃39;\n&＃39;)fd.write(&＃39;authod&＃xff1a;&＃39; &＃43; str(item[1]) &＃43; &＃39;\n&＃39;)fd.write(&＃39;link&＃xff1a;&＃39; &＃43; str(item[2]) &＃43; &＃39;\n&＃39;)fd.write(&＃39;focus&＃xff1a;&＃39; &＃43; str(item[3]) &＃43; &＃39;\n&＃39;)fd.write(&＃39;ctime&＃xff1a;&＃39; &＃43; str(item[4]) &＃43; &＃39;\n&＃39;)if fm &＃61;&＃61; &＃39;json&＃39;:temp &＃61; (&＃39;title&＃39;,&＃39;authod&＃39;,&＃39;link&＃39;,&＃39;focus&＃39;,&＃39;ctime&＃39;)for item in data:json.dump(dict(zip(temp,item)),fd,ensure_ascii&＃61;False)if fm &＃61;&＃61; &＃39;csv&＃39;:writer &＃61; csv.writer(fd)for item in data:writer.writerow(item)# 开始爬取网页
def crawl():kw &＃61; input(&＃39;请输入主题贴吧名字&＃xff1a;&＃39;)base_url &＃61; &＃39;http://tieba.baidu.com/f?kw&＃61;&＃39; &＃43; urllib.parse.quote(kw) &＃43; &＃39;&ie&＃61;utf-8&pn&＃61;{page}&＃39;fm &＃61; input(&＃39;请输入文件保存格式&＃xff08;txt、json、csv&＃xff09;&＃xff1a;&＃39;)while fm!&＃61;&＃39;txt&＃39; and fm!&＃61;&＃39;json&＃39; and fm!&＃61;&＃39;csv&＃39;:fm &＃61; input(&＃39;输入错误&＃xff0c;请重新输入文件保存格式&＃xff08;txt、json、csv&＃xff09;&＃xff1a;&＃39;)fd &＃61; openfile(fm,kw)page &＃61; 0total_page &＃61; int(re.findall(r&＃39;共有主题数(\d&＃43;)个&＃39;,get_page(base_url.format(page&＃61;str(0))))[0])print(&＃39;开始爬取&＃39;)while page

请继续关注我


记得点赞加关注哦&＃xff0c;记得加鸡腿啊