基于pydpier爬取1药网(转载)

#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Created on 2019-02-02 08:59:40
# Project: oneDrugfrom pyspider.libs.base_handler import *
from pymongo import MongoClient
import reclass Handler(BaseHandler):crawl_config &＃61; {}def __init__(self):self.client &＃61; MongoClient(&＃39;mongodb://localhost:27017&＃39;)self.drug &＃61; self.client.drugdef insert_goods(self, data):collection &＃61; self.drug[&＃39;goods&＃39;]collection.update({&＃39;goods_id&＃39;: data[&＃39;goods_id&＃39;]}, data, True)def insert_comments(self, data):collection &＃61; self.drug[&＃39;comments&＃39;]collection.insert_one(data)&＃64;every(minutes&＃61;24 * 60)def on_start(self):self.crawl(&＃39;https://www.111.com.cn/categories/&＃39;, callback&＃61;self.categories_page, validate_cert&＃61;False,fetch_type&＃61;&＃39;js&＃39;)&＃64;config(age&＃61;10 * 24 * 60 * 60)def categories_page(self, response):for each in response.doc(&＃39;.allsort em > a&＃39;).items():self.crawl(each.attr.href, callback&＃61;self.cagetory_list_page, validate_cert&＃61;False, fetch_type&＃61;&＃39;js&＃39;)&＃64;config(priority&＃61;1)def cagetory_list_page(self, response):for each in response.doc(&＃39;#itemSearchList a[target&＃61;"_blank"][class&＃61;"product_pic pro_img"]&＃39;).items():self.crawl(each.attr.href, callback&＃61;self.detail_page, validate_cert&＃61;False, fetch_type&＃61;&＃39;js&＃39;)next &＃61; response.doc(&＃39;#search_table > div.turnPageBottom > a.page_next&＃39;).attr.hrefself.crawl(next, callback&＃61;self.cagetory_list_page, validate_cert&＃61;False, fetch_type&＃61;&＃39;js&＃39;)&＃64;config(priority&＃61;2)def detail_page(self, response):goods_id &＃61; response.doc(&＃39;#gallery_view > ul > li.item_number&＃39;).text()cagetory_one &＃61; response.doc(&＃39;body > div.wrap.clearfix > div > span:nth-child(3) > a&＃39;).text()cagetory_two &＃61; response.doc(&＃39;body > div.wrap.clearfix > div > span:nth-child(5) > a&＃39;).text()cagetory_three &＃61; response.doc(&＃39;body > div.wrap.clearfix > div > span:nth-child(7) > a&＃39;).text()merchants &＃61; response.doc(&＃39;div.middle_property > span:nth-child(1)&＃39;).text()goods_name &＃61; response.doc(&＃39;div.middle_property > h1&＃39;).text()goods_desc &＃61; response.doc(&＃39;div.middle_property > span.red.giftRed&＃39;).text()goods_price &＃61; response.doc(&＃39;div.middle_property > div.shangpin_info > dl:nth-child(2) > dd > span.good_price&＃39;).text()total_comments &＃61; response.doc(&＃39;#fristReviewCount > span > a&＃39;).text()brand &＃61; response.doc(&＃39;#tabCon > div:nth-child(1) > div.goods_intro > table > tbody > tr:nth-child(2) > td:nth-child(2)&＃39;).text()spec &＃61; response.doc(&＃39;#tabCon > div:nth-child(1) > div.goods_intro > table > tbody > tr:nth-child(2) > td:nth-child(4)&＃39;).text()weight &＃61; response.doc(&＃39;#tabCon > div:nth-child(1) > div.goods_intro > table > tbody > tr:nth-child(3) > td:nth-child(2)&＃39;).text()manufacturers &＃61; response.doc(&＃39;#tabCon > div:nth-child(1) > div.goods_intro > table > tbody > tr:nth-child(3) > td:nth-child(4)&＃39;).text()approval_number &＃61; response.doc(&＃39;#tabCon > div:nth-child(1) > div.goods_intro > table > tbody > tr:nth-child(4) > td:nth-child(2)&＃39;).text()drug_type &＃61; response.doc(&＃39;#tabCon > div:nth-child(1) > div.goods_intro > table > tbody > tr:nth-child(4) > td:nth-child(4)&＃39;).text()instructions &＃61; {}if response.doc(&＃39;#prodDetailCotentDiv > table > tbody > tr:nth-child(1) > th&＃39;).text():for i in range(3, 22):instructions_key &＃61; \response.doc(&＃39;#prodDetailCotentDiv > table > tbody > tr:nth-child({}) > th&＃39;.format(i)).text().split(" ")[0]instructions_value &＃61; response.doc(&＃39;#prodDetailCotentDiv > table > tbody > tr:nth-child({}) > td&＃39;.format(i)).text()instructions[instructions_key] &＃61; instructions_valuetotal_comments &＃61; response.doc(&＃39;#itemComments > span&＃39;).text()good_comments &＃61; response.doc(&＃39;#productExperience > div > ul > li:nth-child(2) > a > span&＃39;).text()mid_comments &＃61; response.doc(&＃39;#productExperience > div > ul > li:nth-child(3) > a > span&＃39;).text()bad_comments &＃61; response.doc(&＃39;#productExperience > div > ul > li:nth-child(4) > a > span&＃39;).text()url_id &＃61; re.findall(&＃39;\d&＃43;&＃39;, response.url)[1]goods_data &＃61; {&＃39;url_id&＃39;: url_id,&＃39;goods_id&＃39;: goods_id,&＃39;goods_name&＃39;: goods_name,&＃39;goods_desc&＃39;: goods_desc,&＃39;goods_price&＃39;: goods_price,&＃39;merchants&＃39;: merchants,&＃39;cagetory&＃39;: {&＃39;1&＃39;: cagetory_one,&＃39;2&＃39;: cagetory_two,&＃39;3&＃39;: cagetory_three},&＃39;drug_detail&＃39;: {&＃39;brand&＃39;: brand,&＃39;spec&＃39;: spec,&＃39;weight&＃39;: weight,&＃39;manufacturers&＃39;: manufacturers,&＃39;approval_number&＃39;: approval_number,&＃39;drug_type&＃39;: drug_type},&＃39;instructions&＃39;: instructions,&＃39;comments&＃39;: {&＃39;total_comments&＃39;: total_comments,&＃39;good_comments&＃39;: good_comments,&＃39;mid_comments&＃39;: mid_comments,&＃39;bad_comments&＃39;: bad_comments}}self.insert_goods(goods_data)

from pymongo import MongoClient
import requests
from bs4 import BeautifulSoup
import re
import socketclass Drug:def __init__(self):self.clint &＃61; MongoClient(&＃39;mongodb://localhost:27017&＃39;)self.drug &＃61; self.clint.drugself.collection &＃61; self.drug[&＃39;goods&＃39;]self.comm_collection &＃61; self.drug[&＃39;comments&＃39;]def dbmodify(self):for data in self.collection.find({},{"goods_id":1,"goods_price":1}):try:_id &＃61; data[&＃39;_id&＃39;]id &＃61; data[&＃39;goods_id&＃39;].split("&＃xff1a;")[1]price &＃61; data[&＃39;goods_price&＃39;].split("&＃xffe5;")[1]self.collection.update({&＃39;_id&＃39;: _id},{&＃39;$set&＃39;:{&＃39;goods_id&＃39;:id,&＃39;goods_price&＃39;:price}})print(_id, id, price)except IndexError:passdef getBaseArgument(self,goods_id):base_url &＃61; &＃39;https://www.111.com.cn/interfaces/review/list/html.action&＃39;data &＃61; {&＃39;goodsId&＃39;: goods_id,&＃39;pageIndex&＃39;: 1,&＃39;score&＃39;: &＃39;1&_19020301&＃39;}try:self.collection.update_one({&＃39;url_id&＃39;: goods_id}, {&＃39;$set&＃39;: {&＃39;commspider&＃39;: True}})requests.packages.urllib3.disable_warnings()requests.adapters.DEFAULT_RETRIES &＃61; 5# 设置连接活跃状态为Falses &＃61; requests.session()s.keep_alive &＃61; Falser &＃61; s.get(base_url, params&＃61;data, timeout &＃61; 5,verify&＃61;False)r.close()soup &＃61; BeautifulSoup(r.text, &＃39;html.parser&＃39;)if soup.find_all("div", class_&＃61;"view_no_result"):return "No Comments!"else:total_page_text &＃61; soup.find_all(text&＃61;re.compile(r&＃39;共\d&＃43;页&＃39;))[0]pattern &＃61; re.compile(r&＃39;\d&＃43;&＃39;)total_page &＃61; pattern.findall(total_page_text)return total_page[0]except requests.exceptions.RequestException as e:print(e)def getCommlist(self,goods_id, total_page):base_url &＃61; &＃39;https://www.111.com.cn/interfaces/review/list/html.action&＃39;try:for i in range(1, int(total_page)):data &＃61; {&＃39;goodsId&＃39;: goods_id,&＃39;pageIndex&＃39;: i,&＃39;score&＃39;: &＃39;1&_19020301&＃39;}try:requests.packages.urllib3.disable_warnings()requests.adapters.DEFAULT_RETRIES &＃61; 15# 设置连接活跃状态为Falses &＃61; requests.session()s.keep_alive &＃61; Falser &＃61; s.get(base_url, params&＃61;data, timeout &＃61; 5,verify&＃61;False)r.close()soup &＃61; BeautifulSoup(r.text, &＃39;html.parser&＃39;)for tr in soup.find_all("tr"):comments &＃61; {}try:comments[&＃39;goodsId&＃39;] &＃61; goods_idcomments[&＃39;content&＃39;] &＃61; tr.find(&＃39;p&＃39;).text.strip()comments[&＃39;date&＃39;] &＃61; tr.find(&＃39;p&＃39;, attrs&＃61;{&＃39;class&＃39;: &＃39;eval_date&＃39;}).text.strip()self.comm_collection.insert_one(comments)except:print(goods_id &＃43; "Have some problem!\n")print(comments)except requests.exceptions.RequestException as e:print(e)except ValueError:return "No Comments! Try next!"def getComments(self):i &＃61; 0goods_list &＃61; []for data in self.collection.find({&＃39;commspider&＃39;: False}, {"url_id"}):id &＃61; data[&＃39;url_id&＃39;]goods_list.append(id)length &＃61; len(goods_list)print("总共 {} 条商品".format(length))for good in goods_list:total_page &＃61; self.getBaseArgument(good)comments &＃61; self.getCommlist(good,total_page)i &＃61; i &＃43; 1print("总共 {} 条商品\n目前第 {} 条\n商品编号 {} \n".format(length,i, good))print(comments)test &＃61; Drug().getComments()