python爬虫框架Scrapy使用

安装

pip install -i https://pypi.tuna.tsinghua.edu.cn/simple scrapy


创建爬虫项目

scrapy startproject mypachong


项目结构

创建Spider

scrapy genspider quotes


处理文本内容

class QuotesSpider(scrapy.Spider):name &＃61; &＃39;quotes&＃39;allowed_domains &＃61; [&＃39;book.zongheng.com&＃39;]start_urls &＃61; [&＃39;http://book.zongheng.com/showchapter/2313244.html&＃39;]def parse(self, response):quotes &＃61; response.css(&＃39;.chapter-list li&＃39;)for quote in quotes:chapter &＃61; quote.css(&＃39;a::attr(href)&＃39;).extract_first()url &＃61; response.urljoin(chapter)yield scrapy.Request(url&＃61;url, callback&＃61;self.parse_content)def parse_content(self, response):book &＃61;response.css(&＃39;.reader_crumb a::text&＃39;).extract()[2]chapter &＃61;response.css(&＃39;.title_txtbox::text&＃39;).extract_first()quotes &＃61; response.css(&＃39;.content&＃39;)for quote in quotes:text &＃61; quote.css(&＃39;p::text&＃39;).extract()item &＃61; MypachongItem()item[&＃39;text&＃39;] &＃61; textitem[&＃39;book&＃39;] &＃61; bookitem[&＃39;chapter&＃39;] &＃61; chapteryield item

class MypachongItem(scrapy.Item):text &＃61; scrapy.Field()book &＃61; scrapy.Field()chapter &＃61; scrapy.Field()


随机浏览器头&＃xff1a;

class RandomUA(object):def process_request(self, request, spider):ua &＃61; random.choice(USER_AGENTS)request.headers.setdefault(&＃39;User-Agent&＃39;, ua)


USER_AGENTS &＃61; [&＃39;Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50&＃39;,&＃39;Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50&＃39;,&＃39;Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0&＃39;,&＃39;Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)&＃39;,&＃39;Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)&＃39;,&＃39;Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1&＃39;,&＃39;Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36&＃39;,&＃39;Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11&＃39;,&＃39;Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11&＃39;,&＃39;Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)&＃39;,&＃39;Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36&＃39;,&＃39;Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko&＃39;
]


处理抓取下来的数据&＃xff1a;

class MypachongPipeline(object):def __init__(self):self.file &＃61; open(&＃39;items.json&＃39;, &＃39;w&＃39;)def process_item(self, item, spider):line &＃61; json.dumps(dict(item), ensure_ascii&＃61;False) &＃43; "\n"self.file.write(line)return item


启动爬虫

scrapy crawl quotes