使用python获取nature系列期刊封面高清图片

nature作为科学界最顶级的期刊之一&＃xff0c;其期刊封面审美也一直很在线&＃xff0c;兼具科学和艺术的美感

为了方便快速获取nature系列封面&＃xff0c;这里用python requests模块进行自动化请求并使用BeautifulSoup模块进行html解析

import requests
from bs4 import BeautifulSoup
import ospath &＃61; &＃39;C:\\Users\\User\\Desktop\\nature 封面\\nature 正刊&＃39;
# path &＃61; os.getcwd()
if not os.path.exists(path):os.makedirs(path)print("新建文件夹 nature正刊")# 在这里改变要下载哪期的封面
# 注意下载是从后往前下载的&＃xff0c;所以start_volume应大于等于end_volume
start_volume &＃61; 501
end_volume &＃61; 500
# nature_url &＃61; &＃39;https://www.nature.com/ng/volumes/&＃39; # nature genetics
nature_url&＃61;&＃39;https://www.nature.com/nature/volumes/&＃39; # nature 正刊
kv &＃61; {&＃39;User-Agent&＃39;:&＃39;Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36&＃39;
}
while start_volume >&＃61; end_volume:try:volume_url &＃61; nature_url &＃43; str(start_volume)volume_response &＃61; requests.get(url&＃61;volume_url, headers&＃61;kv, timeout&＃61;120)except Exception:print(str(start_volume) &＃43; "请求异常")with open(path &＃43; "\\异常.txt", &＃39;at&＃39;) as txt:txt.write(str(start_volume) &＃43; "请求异常\n")continuevolume_response.encoding &＃61; &＃39;utf-8&＃39;volume_soup &＃61; BeautifulSoup(volume_response.text, &＃39;html.parser&＃39;)ul_tag &＃61; volume_soup.find_all(&＃39;ul&＃39;,class_&＃61;&＃39;ma0 clean-list grid-auto-fill grid-auto-fill-w220 very-small-column medium-row-gap&＃39;)img_list &＃61; ul_tag[0].find_all("img")issue_number &＃61; 0for img_tag in img_list:issue_number &＃43;&＃61; 1filename &＃61; path &＃43; &＃39;\\&＃39; &＃43; str(start_volume) &＃43; &＃39;_&＃39; &＃43; str(issue_number) &＃43; &＃39;.png&＃39;if os.path.exists(filename):print(filename &＃43; "已经存在")continueprint("Loading...........................")img_url &＃61; &＃39;https:&＃39; &＃43; img_tag.get("src").replace("w200", "w1000")try:img_response &＃61; requests.get(img_url, timeout&＃61;240, headers&＃61;kv)except Exception:print(start_volume, issue_number, &＃39;???????????异常????????&＃39;)with open(path &＃43; "\\异常.txt", &＃39;at&＃39;) as txt:txt.write(str(start_volume) &＃43; &＃39;_&＃39; &＃43; str(issue_number) &＃43; "请求异常\n")continuewith open(filename, &＃39;wb&＃39;) as imgfile:imgfile.write(img_response.content)print("成功下载图片&＃xff1a;" &＃43; str(start_volume) &＃43; &＃39;_&＃39; &＃43; str(issue_number))start_volume -&＃61; 1


运行结果&＃xff1a;

以上部分代码可以自动下载nature和nature genetics的封面&＃xff0c;这两个期刊的网站结构跟其他子刊略有不同&＃xff0c;其他子刊可以用以下代码来进行爬虫&＃xff1a;

import requests
from bs4 import BeautifulSoup
import osother_journals &＃61; {&＃39;nature biomedical engineering&＃39;: &＃39;natbiomedeng&＃39;,&＃39;nature methods&＃39;: &＃39;nmeth&＃39;,&＃39;nature astronomy&＃39;: &＃39;natastron&＃39;,&＃39;nature medicine&＃39;: &＃39;nm&＃39;,&＃39;nature protocols&＃39;: &＃39;nprot&＃39;,&＃39;nature microbiology&＃39;: &＃39;nmicrobiol&＃39;,&＃39;nature cell biology&＃39;: &＃39;ncb&＃39;,&＃39;nature nanotechnology&＃39;: &＃39;nnano&＃39;,&＃39;nature immunology&＃39;: &＃39;ni&＃39;,&＃39;nature energy&＃39;: &＃39;nenergy&＃39;,&＃39;nature materials&＃39;: &＃39;nmat&＃39;,&＃39;nature cancer&＃39;: &＃39;natcancer&＃39;,&＃39;nature neuroscience&＃39;: &＃39;neuro&＃39;,&＃39;nature machine intelligence&＃39;: &＃39;natmachintell&＃39;,&＃39;nature metabolism&＃39;: &＃39;natmetab&＃39;,&＃39;nature food&＃39;: &＃39;natfood&＃39;,&＃39;nature ecology & evolution&＃39;: "natecolevol","nature stuctural & molecular biology":"nsmb","nature physics":"nphys","nature human behavior":"nathumbehav","nature chemical biology":"nchembio"
}nature_journal &＃61; {# 要下载的期刊放这里&＃39;nature plants&＃39;: &＃39;nplants&＃39;,&＃39;nature biotechnology&＃39;: &＃39;nbt&＃39;
}
folder_Name &＃61; "nature 封面"
kv &＃61; {&＃39;User-Agent&＃39;:&＃39;Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36&＃39;
}def makefile(path):folder &＃61; os.path.exists(path)if not folder:os.makedirs(path)print("Make file -- " &＃43; path &＃43; " -- successfully!")else:raise AssertionError################################################################
def getCover(url, journal, year, filepath, startyear&＃61;2022, endyear&＃61;2022):# 注意endyear是比startyear小的,因为是从endyear开始由后往前来下载的if not (endyear <&＃61; year <&＃61; startyear):returntry:issue_response &＃61; requests.get("https://www.nature.com" &＃43; url,timeout&＃61;120,headers&＃61;kv)except Exception:print(journal &＃43; " " &＃43; str(year) &＃43; " Error")returnissue_response.encoding &＃61; &＃39;gbk&＃39;if &＃39;Page not found&＃39; in issue_response.text:print(journal &＃43; " Page not found")returnissue_soup &＃61; BeautifulSoup(issue_response.text, &＃39;html.parser&＃39;)cover_image &＃61; issue_soup.find_all("img", class_&＃61;&＃39;image-constraint pt10&＃39;)for image in cover_image:image_url &＃61; image.get("src")print("Start loading img.............................")image_url &＃61; image_url.replace("w200", "w1000")if (image_url[-2] &＃61;&＃61; &＃39;/&＃39;):month &＃61; "0" &＃43; image_url[-1]else:month &＃61; image_url[-2:]image_name &＃61; nature_journal[journal] &＃43; "_" &＃43; str(year) &＃43; "_" &＃43; month &＃43; ".png"if os.path.exists(filepath &＃43; journal &＃43; "\\" &＃43; image_name):print(image_url &＃43; " 已经存在")continueprint(image_url)try:image_response &＃61; requests.get("http:" &＃43; image_url,timeout&＃61;240,headers&＃61;kv)except Exception:print("获取图片异常:" &＃43; image_name)continuewith open(filepath &＃43; journal &＃43; "\\" &＃43; image_name,&＃39;wb&＃39;) as downloaded_img:downloaded_img.write(image_response.content)def main():try:path &＃61; os.getcwd() &＃43; &＃39;\\&＃39;makefile(path &＃43; folder_Name)except Exception:print("文件夹 --nature 封面-- 已经存在")path &＃61; path &＃43; folder_Name &＃43; "\\"for journal in nature_journal:try:makefile(path &＃43; journal)except AssertionError:print("File -- " &＃43; path &＃43; " -- has already exist!")try:volume_response &＃61; requests.get("https://www.nature.com/" &＃43;nature_journal[journal] &＃43;"/volumes",timeout&＃61;120,headers&＃61;kv)except Exception:print(journal &＃43; " 异常")continuevolume_response.encoding &＃61; &＃39;gbk&＃39;volume_soup &＃61; BeautifulSoup(volume_response.text, &＃39;html.parser&＃39;)volume_list &＃61; volume_soup.find_all(&＃39;ul&＃39;,class_&＃61;&＃39;clean-list ma0 clean-list grid-auto-fill medium-row-gap background-white&＃39;)number_of_volume &＃61; 0for volume_child in volume_list[0].children:if volume_child &＃61;&＃61; &＃39;\n&＃39;:continueissue_url &＃61; volume_child.find_all("a")[0].get("href")print(issue_url)print(2020 - number_of_volume)getCover(issue_url,journal,year&＃61;(2020 - number_of_volume),filepath&＃61;path,startyear&＃61;2022, endyear&＃61;2022)number_of_volume &＃43;&＃61; 1if __name__ &＃61;&＃61; "__main__":main()print("Finish Everything!")


运行结果&＃xff1a;