爬虫实战酷狗音乐数据抓取XPath，Pyquery,Beautifulsoup数据提取对比实战

网站：

http://www.kugou.com/yy/html/rank.html

爬取目标：

酷酷狗飙升榜的歌手，歌曲名字，歌曲链接等内容，存到Mysql数据库中

网页解析：

此次爬取采用三种解析方式：

代码如下：

import requests
from lxml import etree
import pymongo
from pyquery import PyQuery as pq
from bs4 import BeautifulSoupdef get_info():url = &＃39;http://www.kugou.com/yy/html/rank.html&＃39;try:respOnse= requests.get(url)if response.status_code == 200:# print(response)return response.textexcept requests.ConnectionError:return Nonedef get_detail_info(response):"""xpath抓取数据"""html = etree.HTML(response)# xpathresult = html.xpath(&＃39;//div[@id="rankWrap"]/div[@class="pc_temp_songlist pc_rank_songlist_short"]/ul/li&＃39;)# print(result)list1 = []for msg in result:#查找歌手名字还有歌曲名字msg_star = msg.xpath(&＃39;./@title&＃39;)[0]#查找歌曲的榜名msg_address = msg.xpath(&＃39;./span[@class="pc_temp_tips_l"]/i/@title&＃39;)[0]#查找歌曲链接msg_lianjie = msg.xpath(&＃39;./a/@href&＃39;)[0]dic = {"msg_star":msg_star,&＃39;msg_address&＃39;:msg_address,&＃39;msg_lianjie&＃39;:msg_lianjie,}# print(dic)list1.append(dic)return list1def get_detail_info_css(response):"""pyquery抓取"""# print(response)doc = pq(response)# print(doc)items = doc(&＃39;#rankWrap&＃39;)lis = items.find(&＃39;ul&＃39;).find(&＃39;li&＃39;)print(type(lis))lis = doc(lis)list1 = []for msg in lis.items():# print(type(msg))msg_star = msg.attr.titlemsg_address = msg.children(&＃39;.pc_temp_tips_l&＃39;).find(&＃39;i&＃39;).attr.titlemsg_lianjie = msg.find(&＃39;a&＃39;).attr.hrefdic = {"msg_star": msg_star,&＃39;msg_address&＃39;: msg_address,&＃39;msg_lianjie&＃39;: msg_lianjie,}list1.append(dic)return list1def get_detail_info_xml(response):"""beautifulsoup抓取"""list1 =[]soup = BeautifulSoup(response,&＃39;lxml&＃39;)info = soup.find(class_=&＃39;pc_temp_songlist pc_rank_songlist_short&＃39;).ulfor msg in info.select(&＃39;li&＃39;):# print(msg)msg_star = msg.attrs[&＃39;title&＃39;]msg_address = msg.find(class_=&＃39;pc_temp_tips_l&＃39;).i.attrs[&＃39;title&＃39;]msg_lianjie = msg.a.attrs[&＃39;href&＃39;]print(msg_lianjie)dic = {"msg_star": msg_star,&＃39;msg_address&＃39;: msg_address,&＃39;msg_lianjie&＃39;: msg_lianjie,}list1.append(dic)return list1def db(list1):# print(list1)client = pymongo.MongoClient(host=&＃39;localhost&＃39;,port=27017)db = client.testcollection = db.musicfor music_info in list1:print(music_info)result = collection.insert(music_info)print(result)def main():#获取响应respOnse= get_info()# xpath提取# get_detail_info(response)# list1 = get_detail_info(response)# pyquery提取# get_detail_info_css(response)# list1 = get_detail_info_css(response)# bs4提取get_detail_info_xml(response)list1 = get_detail_info_xml(response)db(list1)if __name__ == &＃39;__main__&＃39;:main()