Python爬虫之Requests库爬取海量图片！数据都是钱啊

Requests 是一个 Python 的 HTTP 客户端库。

Request支持HTTP连接保持和连接池&＃xff0c;支持使用COOKIE保持会话&＃xff0c;支持文件上传&＃xff0c;支持自动响应内容的编码&＃xff0c;支持国际化的URL和POST数据自动编码。

在python内置模块的基础上进行了高度的封装从而使得python进行网络请求时&＃xff0c;变得人性化&＃xff0c;使用Requests可以轻而易举的完成浏览器可有的任何操作。 现代&＃xff0c;国际化&＃xff0c;友好 。

目录

一、Requests基础

二、发送请求与接收响应&＃xff08;基本GET请求&＃xff09;

三、发送请求与接收响应&＃xff08;基本POST请求&＃xff09;

四、response属性

五、代理

六、COOKIE和session

七、案例

一、Requests基础

1.安装Requests库

pip install requests

2.使用Requests库

import requests

二、发送请求与接收响应&＃xff08;基本GET请求&＃xff09;

response &＃61; requests.get(url)

1.传送 parmas参数

• 参数包含在url中

response &＃61; requests.get("http://httpbin.org/get?name&＃61;zhangsan&age&＃61;22")
print(response.text)

• 通过get方法传送参数

data &＃61; {"name": "zhangsan","age": 30}
response &＃61; requests.get("http://httpbin.org/get", params&＃61;data)
print(response.text)

2.模拟发送请求头&＃xff08;传送headers参数&＃xff09;

headers &＃61; {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36"
}
response &＃61; requests.get("http://httpbin.org/get", headers&＃61;headers)
print(response.text)

三、发送请求与接收响应&＃xff08;基本POST请求&＃xff09;

response &＃61; requests.post(url, data &＃61; data, headers&＃61;headers)

四、response属性

五、代理

proxies &＃61; {"http": "https://175.44.148.176:9000","https": "https://183.129.207.86:14002"
}
response &＃61; requests.get("https://www.baidu.com/", proxies&＃61;proxies)

六、COOKIE和session

• 使用的COOKIE和session好处&＃xff1a;很多网站必须登录之后(或者获取某种权限之后)才能能够请求到相关数据。
• 使用的COOKIE和session的弊端&＃xff1a;一套COOKIE和session往往和一个用户对应.请求太快&＃xff0c;请求次数太多&＃xff0c;容易被服务器识别为爬虫&＃xff0c;从而使账号收到损害。

1.不需要COOKIE的时候尽量不去使用COOKIE。

2.为了获取登录之后的页面&＃xff0c;我们必须发送带有COOKIEs的请求&＃xff0c;此时为了确保账号安全应该尽量降低数据

采集速度。

1.COOKIE

&＃xff08;1&＃xff09;获取COOKIE信息

response.COOKIEs

2.session

&＃xff08;1&＃xff09;构造session回话对象

session &＃61; requests.session()

示例&＃xff1a;

def login_renren():login_url &＃61; &＃39;http://www.renren.com/SysHome.do&＃39;headers &＃61; {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36"}session &＃61; requests.session()login_data &＃61; {"email": "账号","password": "密码"}response &＃61; session.post(login_url, data&＃61;login_data, headers&＃61;headers)response &＃61; session.get("http://www.renren.com/971909762/newsfeed/photo")print(response.text)login_renren()

七、案例

案例1&＃xff1a;百度贴吧页面爬取&＃xff08;GET请求&＃xff09;

import requests
import sysclass BaiduTieBa:def __init__(self, name, pn, ):self.name &＃61; nameself.url &＃61; "http://tieba.baidu.com/f?kw&＃61;{}&ie&＃61;utf-8&pn&＃61;{}".format(name, pn)self.headers &＃61; {# "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36"# 使用较老版本的请求头&＃xff0c;该浏览器不支持js"User-Agent": "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)"}self.url_list &＃61; [self.url &＃43; str(pn*50) for pn in range(pn)]print(self.url_list)def get_data(self, url):"""请求数据:param url::return:"""response &＃61; requests.get(url, headers&＃61;self.headers)return response.contentdef save_data(self, data, num):"""保存数据:param data::param num::return:"""file_name &＃61; "./pages/" &＃43; self.name &＃43; "_" &＃43; str(num) &＃43; ".html"with open(file_name, "wb") as f:f.write(data)def run(self):for url in self.url_list:data &＃61; self.get_data(url)num &＃61; self.url_list.index(url)self.save_data(data, num)if __name__ &＃61;&＃61; "__main__":name &＃61; sys.argv[1]pn &＃61; int(sys.argv[2])baidu &＃61; BaiduTieBa(name, pn)baidu.run()

案例2&＃xff1a;金山词霸翻译&＃xff08;POST请求&＃xff09;

import requests
import sys
import jsonclass JinshanCiBa:def __init__(self, words):self.url &＃61; "http://fy.iciba.com/ajax.php?a&＃61;fy"self.headers &＃61; {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0","X-Requested-With": "XMLHttpRequest"}self.post_data &＃61; {"f": "auto","t": "auto","w": words}def get_data(self):"""请求数据:param url::return:"""response &＃61; requests.post(self.url, data&＃61;self.post_data, headers&＃61;self.headers)return response.textdef show_translation(self):"""显示翻译结果:param data::param num::return:"""response &＃61; self.get_data()json_data &＃61; json.loads(response, encoding&＃61;&＃39;utf-8&＃39;)if json_data[&＃39;status&＃39;] &＃61;&＃61; 0:translation &＃61; json_data[&＃39;content&＃39;][&＃39;word_mean&＃39;]elif json_data[&＃39;status&＃39;] &＃61;&＃61; 1:translation &＃61; json_data[&＃39;content&＃39;][&＃39;out&＃39;]else:translation &＃61; Noneprint(translation)def run(self):self.show_translation()if __name__ &＃61;&＃61; "__main__":words &＃61; sys.argv[1]ciba &＃61; JinshanCiBa(words)ciba.run()

案例3&＃xff1a;百度贴吧图片爬取

&＃xff08;1&＃xff09;普通版

从已下载页面中提取url来爬取图片&＃xff08;页面下载方法见案例1&＃xff09;

from lxml import etree
import requestsclass DownloadPhoto:def __init__(self):passdef download_img(self, url):response &＃61; requests.get(url)index &＃61; url.rfind(&＃39;/&＃39;)file_name &＃61; url[index &＃43; 1:]print("下载图片&＃xff1a;" &＃43; file_name)save_name &＃61; "./photo/" &＃43; file_namewith open(save_name, "wb") as f:f.write(response.content)def parse_photo_url(self, page):html &＃61; etree.parse(page, etree.HTMLParser())nodes &＃61; html.xpath("//a[contains(&＃64;class, &＃39;thumbnail&＃39;)]/img/&＃64;bpic")print(nodes)print(len(nodes))for node in nodes:self.download_img(node)if __name__ &＃61;&＃61; "__main__":download &＃61; DownloadPhoto()for i in range(6000):download.parse_photo_url("./pages/校花_{}.html".format(i))

&＃xff08;2&＃xff09;多线程版

main.py

import requests
from lxml import etreefrom file_download import DownLoadExecutioner, file_downloadclass XiaoHua:def __init__(self, init_url):self.init_url &＃61; init_urlself.download_executioner &＃61; DownLoadExecutioner()def start(self):self.download_executioner.start()self.download_img(self.init_url)def download_img(self, url):html_text &＃61; file_download(url, type&＃61;&＃39;text&＃39;)html &＃61; etree.HTML(html_text)img_urls &＃61; html.xpath("//a[contains(&＃64;class,&＃39;thumbnail&＃39;)]/img/&＃64;bpic")self.download_executioner.put_task(img_urls)# 获取下一页的连接next_page &＃61; html.xpath("//div[&＃64;id&＃61;&＃39;frs_list_pager&＃39;]/a[contains(&＃64;class,&＃39;next&＃39;)]/&＃64;href")next_page &＃61; "http:" &＃43; next_page[0]self.download_img(next_page)if __name__ &＃61;&＃61; &＃39;__main__&＃39;:x &＃61; XiaoHua("http://tieba.baidu.com/f?kw&＃61;校花&ie&＃61;utf-8")x.start()

file_download.py

import requests
import threading
from queue import Queuedef file_download(url, type&＃61;&＃39;content&＃39;):headers &＃61; {&＃39;User-Agent&＃39;: &＃39;Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko&＃39;}r &＃61; requests.get(url, headers&＃61;headers)if type &＃61;&＃61; &＃39;text&＃39;:return r.textreturn r.contentclass DownLoadExecutioner(threading.Thread):def __init__(self):super().__init__()self.q &＃61; Queue(maxsize&＃61;50)# 图片保存目录self.save_dir &＃61; &＃39;./img/&＃39;# 图片计数self.index &＃61; 0def put_task(self, urls):if isinstance(urls, list):for url in urls:self.q.put(url)else:self.q.put(urls)def run(self):while True:url &＃61; self.q.get()content &＃61; file_download(url)# 截取图片名称index &＃61; url.rfind(&＃39;/&＃39;)file_name &＃61; url[index&＃43;1:]save_name &＃61; self.save_dir &＃43; file_namewith open(save_name, &＃39;wb&＃43;&＃39;) as f:f.write(content)self.index &＃43;&＃61; 1print(save_name &＃43; "下载成功! 当前已下载图片总数&＃xff1a;" &＃43; str(self.index))

&＃xff08;3&＃xff09;线程池版

main.py

import requests
from lxml import etreefrom file_download_pool import DownLoadExecutionerPool, file_downloadclass XiaoHua:def __init__(self, init_url):self.init_url &＃61; init_urlself.download_executioner &＃61; DownLoadExecutionerPool()def start(self):self.download_img(self.init_url)def download_img(self, url):html_text &＃61; file_download(url, type&＃61;&＃39;text&＃39;)html &＃61; etree.HTML(html_text)img_urls &＃61; html.xpath("//a[contains(&＃64;class,&＃39;thumbnail&＃39;)]/img/&＃64;bpic")self.download_executioner.put_task(img_urls)# 获取下一页的连接next_page &＃61; html.xpath("//div[&＃64;id&＃61;&＃39;frs_list_pager&＃39;]/a[contains(&＃64;class,&＃39;next&＃39;)]/&＃64;href")next_page &＃61; "http:" &＃43; next_page[0]self.download_img(next_page)if __name__ &＃61;&＃61; &＃39;__main__&＃39;:x &＃61; XiaoHua("http://tieba.baidu.com/f?kw&＃61;校花&ie&＃61;utf-8")x.start()

file_download_pool.py

import requests
import concurrent.futures as futuresdef file_download(url, type&＃61;&＃39;content&＃39;):headers &＃61; {&＃39;User-Agent&＃39;: &＃39;Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko&＃39;}r &＃61; requests.get(url, headers&＃61;headers)if type &＃61;&＃61; &＃39;text&＃39;:return r.textreturn r.contentclass DownLoadExecutionerPool():def __init__(self):super().__init__()# 图片保存目录self.save_dir &＃61; &＃39;./img_pool/&＃39;# 图片计数self.index &＃61; 0# 线程池self.ex &＃61; futures.ThreadPoolExecutor(max_workers&＃61;30)def put_task(self, urls):if isinstance(urls, list):for url in urls:self.ex.submit(self.save_img, url)else:self.ex.submit(self.save_img, urls)def save_img(self, url):content &＃61; file_download(url)# 截取图片名称index &＃61; url.rfind(&＃39;/&＃39;)file_name &＃61; url[index&＃43;1:]save_name &＃61; self.save_dir &＃43; file_namewith open(save_name, &＃39;wb&＃43;&＃39;) as f:f.write(content)self.index &＃43;&＃61; 1print(save_name &＃43; "下载成功! 当前已下载图片总数&＃xff1a;" &＃43; str(self.index))

作者&＃xff1a;Recalcitrant

链接&＃xff1a; https://www.jianshu.com/p/140... 是一个 Python 的 HTTP 客户端库。

Request支持HTTP连接保持和连接池&＃xff0c;支持使用COOKIE保持会话&＃xff0c;支持文件上传&＃xff0c;支持自动响应内容的编码&＃xff0c;支持国际化的URL和POST数据自动编码。

在python内置模块的基础上进行了高度的封装&＃xff0c;从而使得python进行网络请求时&＃xff0c;变得人性化&＃xff0c;使用Requests可以轻而易举的完成浏览器可有的任何操作。现代&＃xff0c;国际化&＃xff0c;友好。

requests会自动实现持久连接keep-alive

目录

一、Requests基础

二、发送请求与接收响应&＃xff08;基本GET请求&＃xff09;

三、发送请求与接收响应&＃xff08;基本POST请求&＃xff09;

四、response属性

五、代理

六、COOKIE和session

七、案例

一、Requests基础

1.安装Requests库

pip install requests

2.使用Requests库

import requests

二、发送请求与接收响应&＃xff08;基本GET请求&＃xff09;

response &＃61; requests.get(url)

1.传送 parmas参数

• 参数包含在url中

response &＃61; requests.get("http://httpbin.org/get?name&＃61;zhangsan&age&＃61;22")
print(response.text)

• 通过get方法传送参数

data &＃61; {"name": "zhangsan","age": 30}
response &＃61; requests.get("http://httpbin.org/get", params&＃61;data)
print(response.text)

2.模拟发送请求头&＃xff08;传送headers参数&＃xff09;

headers &＃61; {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36"
}
response &＃61; requests.get("http://httpbin.org/get", headers&＃61;headers)
print(response.text)

三、发送请求与接收响应&＃xff08;基本POST请求&＃xff09;

response &＃61; requests.post(url, data &＃61; data, headers&＃61;headers)

四、response属性

属性描述response.text获取str类型&＃xff08;Unicode编码&＃xff09;的响应response.content获取bytes类型的响应response.status_code获取响应状态码response.headers获取响应头response.request获取响应对应的请求

五、代理

proxies &＃61; {"http": "https://175.44.148.176:9000","https": "https://183.129.207.86:14002"
}
response &＃61; requests.get("https://www.baidu.com/", proxies&＃61;proxies)

六、COOKIE和session

• 使用的COOKIE和session好处&＃xff1a;很多网站必须登录之后(或者获取某种权限之后)才能能够请求到相关数据。
• 使用的COOKIE和session的弊端&＃xff1a;一套COOKIE和session往往和一个用户对应.请求太快&＃xff0c;请求次数太多&＃xff0c;容易被服务器识别为爬虫&＃xff0c;从而使账号收到损害。

1.不需要COOKIE的时候尽量不去使用COOKIE。

2.为了获取登录之后的页面&＃xff0c;我们必须发送带有COOKIEs的请求&＃xff0c;此时为了确保账号安全应该尽量降低数据

采集速度。

1.COOKIE

&＃xff08;1&＃xff09;获取COOKIE信息

response.COOKIEs

2.session

&＃xff08;1&＃xff09;构造session回话对象

session &＃61; requests.session()

示例&＃xff1a;

def login_renren():login_url &＃61; &＃39;http://www.renren.com/SysHome.do&＃39;headers &＃61; {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36"}session &＃61; requests.session()login_data &＃61; {"email": "账号","password": "密码"}response &＃61; session.post(login_url, data&＃61;login_data, headers&＃61;headers)response &＃61; session.get("http://www.renren.com/971909762/newsfeed/photo")print(response.text)login_renren()

七、案例

案例1&＃xff1a;百度贴吧页面爬取&＃xff08;GET请求&＃xff09;

import requests
import sysclass BaiduTieBa:def __init__(self, name, pn, ):self.name &＃61; nameself.url &＃61; "http://tieba.baidu.com/f?kw&＃61;{}&ie&＃61;utf-8&pn&＃61;{}".format(name, pn)self.headers &＃61; {# "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36"# 使用较老版本的请求头&＃xff0c;该浏览器不支持js"User-Agent": "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)"}self.url_list &＃61; [self.url &＃43; str(pn*50) for pn in range(pn)]print(self.url_list)def get_data(self, url):"""请求数据:param url::return:"""response &＃61; requests.get(url, headers&＃61;self.headers)return response.contentdef save_data(self, data, num):"""保存数据:param data::param num::return:"""file_name &＃61; "./pages/" &＃43; self.name &＃43; "_" &＃43; str(num) &＃43; ".html"with open(file_name, "wb") as f:f.write(data)def run(self):for url in self.url_list:data &＃61; self.get_data(url)num &＃61; self.url_list.index(url)self.save_data(data, num)if __name__ &＃61;&＃61; "__main__":name &＃61; sys.argv[1]pn &＃61; int(sys.argv[2])baidu &＃61; BaiduTieBa(name, pn)baidu.run()

案例2&＃xff1a;金山词霸翻译&＃xff08;POST请求&＃xff09;

import requests
import sys
import jsonclass JinshanCiBa:def __init__(self, words):self.url &＃61; "http://fy.iciba.com/ajax.php?a&＃61;fy"self.headers &＃61; {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0","X-Requested-With": "XMLHttpRequest"}self.post_data &＃61; {"f": "auto","t": "auto","w": words}def get_data(self):"""请求数据:param url::return:"""response &＃61; requests.post(self.url, data&＃61;self.post_data, headers&＃61;self.headers)return response.textdef show_translation(self):"""显示翻译结果:param data::param num::return:"""response &＃61; self.get_data()json_data &＃61; json.loads(response, encoding&＃61;&＃39;utf-8&＃39;)if json_data[&＃39;status&＃39;] &＃61;&＃61; 0:translation &＃61; json_data[&＃39;content&＃39;][&＃39;word_mean&＃39;]elif json_data[&＃39;status&＃39;] &＃61;&＃61; 1:translation &＃61; json_data[&＃39;content&＃39;][&＃39;out&＃39;]else:translation &＃61; Noneprint(translation)def run(self):self.show_translation()if __name__ &＃61;&＃61; "__main__":words &＃61; sys.argv[1]ciba &＃61; JinshanCiBa(words)ciba.run()

案例3&＃xff1a;百度贴吧图片爬取

&＃xff08;1&＃xff09;普通版

从已下载页面中提取url来爬取图片&＃xff08;页面下载方法见案例1&＃xff09;

from lxml import etree
import requestsclass DownloadPhoto:def __init__(self):passdef download_img(self, url):response &＃61; requests.get(url)index &＃61; url.rfind(&＃39;/&＃39;)file_name &＃61; url[index &＃43; 1:]print("下载图片&＃xff1a;" &＃43; file_name)save_name &＃61; "./photo/" &＃43; file_namewith open(save_name, "wb") as f:f.write(response.content)def parse_photo_url(self, page):html &＃61; etree.parse(page, etree.HTMLParser())nodes &＃61; html.xpath("//a[contains(&＃64;class, &＃39;thumbnail&＃39;)]/img/&＃64;bpic")print(nodes)print(len(nodes))for node in nodes:self.download_img(node)if __name__ &＃61;&＃61; "__main__":download &＃61; DownloadPhoto()for i in range(6000):download.parse_photo_url("./pages/校花_{}.html".format(i))

&＃xff08;2&＃xff09;多线程版

main.py

import requests
from lxml import etreefrom file_download import DownLoadExecutioner, file_downloadclass XiaoHua:def __init__(self, init_url):self.init_url &＃61; init_urlself.download_executioner &＃61; DownLoadExecutioner()def start(self):self.download_executioner.start()self.download_img(self.init_url)def download_img(self, url):html_text &＃61; file_download(url, type&＃61;&＃39;text&＃39;)html &＃61; etree.HTML(html_text)img_urls &＃61; html.xpath("//a[contains(&＃64;class,&＃39;thumbnail&＃39;)]/img/&＃64;bpic")self.download_executioner.put_task(img_urls)# 获取下一页的连接next_page &＃61; html.xpath("//div[&＃64;id&＃61;&＃39;frs_list_pager&＃39;]/a[contains(&＃64;class,&＃39;next&＃39;)]/&＃64;href")next_page &＃61; "http:" &＃43; next_page[0]self.download_img(next_page)if __name__ &＃61;&＃61; &＃39;__main__&＃39;:x &＃61; XiaoHua("http://tieba.baidu.com/f?kw&＃61;校花&ie&＃61;utf-8")x.start()

file_download.py

import requests
import threading
from queue import Queuedef file_download(url, type&＃61;&＃39;content&＃39;):headers &＃61; {&＃39;User-Agent&＃39;: &＃39;Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko&＃39;}r &＃61; requests.get(url, headers&＃61;headers)if type &＃61;&＃61; &＃39;text&＃39;:return r.textreturn r.contentclass DownLoadExecutioner(threading.Thread):def __init__(self):super().__init__()self.q &＃61; Queue(maxsize&＃61;50)# 图片保存目录self.save_dir &＃61; &＃39;./img/&＃39;# 图片计数self.index &＃61; 0def put_task(self, urls):if isinstance(urls, list):for url in urls:self.q.put(url)else:self.q.put(urls)def run(self):while True:url &＃61; self.q.get()content &＃61; file_download(url)# 截取图片名称index &＃61; url.rfind(&＃39;/&＃39;)file_name &＃61; url[index&＃43;1:]save_name &＃61; self.save_dir &＃43; file_namewith open(save_name, &＃39;wb&＃43;&＃39;) as f:f.write(content)self.index &＃43;&＃61; 1print(save_name &＃43; "下载成功! 当前已下载图片总数&＃xff1a;" &＃43; str(self.index))

&＃xff08;3&＃xff09;线程池版

main.py

import requests
from lxml import etreefrom file_download_pool import DownLoadExecutionerPool, file_downloadclass XiaoHua:def __init__(self, init_url):self.init_url &＃61; init_urlself.download_executioner &＃61; DownLoadExecutionerPool()def start(self):self.download_img(self.init_url)def download_img(self, url):html_text &＃61; file_download(url, type&＃61;&＃39;text&＃39;)html &＃61; etree.HTML(html_text)img_urls &＃61; html.xpath("//a[contains(&＃64;class,&＃39;thumbnail&＃39;)]/img/&＃64;bpic")self.download_executioner.put_task(img_urls)# 获取下一页的连接next_page &＃61; html.xpath("//div[&＃64;id&＃61;&＃39;frs_list_pager&＃39;]/a[contains(&＃64;class,&＃39;next&＃39;)]/&＃64;href")next_page &＃61; "http:" &＃43; next_page[0]self.download_img(next_page)if __name__ &＃61;&＃61; &＃39;__main__&＃39;:x &＃61; XiaoHua("http://tieba.baidu.com/f?kw&＃61;校花&ie&＃61;utf-8")x.start()

file_download_pool.py

import requests
import concurrent.futures as futuresdef file_download(url, type&＃61;&＃39;content&＃39;):headers &＃61; {&＃39;User-Agent&＃39;: &＃39;Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko&＃39;}r &＃61; requests.get(url, headers&＃61;headers)if type &＃61;&＃61; &＃39;text&＃39;:return r.textreturn r.contentclass DownLoadExecutionerPool():def __init__(self):super().__init__()# 图片保存目录self.save_dir &＃61; &＃39;./img_pool/&＃39;# 图片计数self.index &＃61; 0# 线程池self.ex &＃61; futures.ThreadPoolExecutor(max_workers&＃61;30)def put_task(self, urls):if isinstance(urls, list):for url in urls:self.ex.submit(self.save_img, url)else:self.ex.submit(self.save_img, urls)def save_img(self, url):content &＃61; file_download(url)# 截取图片名称index &＃61; url.rfind(&＃39;/&＃39;)file_name &＃61; url[index&＃43;1:]save_name &＃61; self.save_dir &＃43; file_namewith open(save_name, &＃39;wb&＃43;&＃39;) as f:f.write(content)self.index &＃43;&＃61; 1print(save_name &＃43; "下载成功! 当前已下载图片总数&＃xff1a;" &＃43; str(self.index))

完整项目代码获取点这里