从零实现一个高性能网络爬虫（一）网络请求分析及代码实现

 1 package com.cnblogs.wycm;
 2 
 3 import com.alibaba.fastjson.JSON;
 4 import com.alibaba.fastjson.JSONObject;
 5 import org.apache.http.client.methods.CloseableHttpResponse;
 6 import org.apache.http.client.methods.HttpGet;
 7 import org.apache.http.impl.client.CloseableHttpClient;
 8 import org.apache.http.impl.client.HttpClients;
 9 import org.apache.http.util.EntityUtils;
10 
11 import java.io.IOException;
12 
13 /**
14  * 获取wo-yan-chen-mo关注的所有知乎用户信息
15  * 只是把用户资料打印出来，没有具体解析（关于解析出详细数据可以采用正则表达式、json库、jsonpath等方式）
16  */
17 public class Demo {
18     public static void main(String[] args) throws IOException {
19         //创建http客户端
20         CloseableHttpClient httpClient = HttpClients.createDefault();
21 
22         String url = "https://www.zhihu.com/api/v4/members/wo-yan-chen-mo/followees?include=data%5B*%5D.answer_count%2Carticles_count%2Cgender%2Cfollower_count%2Cis_followed%2Cis_following%2Cbadge%5B%3F(type%3Dbest_answerer)%5D.topics&offset=0&limit=20";
23 
24         //创建http request(GET)
25         HttpGet request = new HttpGet(url);
26 
27         //设置http request header
28         request.setHeader("authorization", "oauth c3cef7c66a1843f8b3a9e6a1e3160e20");
29         //执行http请求
30         CloseableHttpResponse respOnse= httpClient.execute(request);
31         //打印response
32         String respOnseStr= EntityUtils.toString(response.getEntity());
33         System.out.println(responseStr);
34 
35         String nextPageUrl = getNextPageUrl(responseStr);
36         boolean isEnd = getIsEnd(responseStr);
37 
38         while (!isEnd && nextPageUrl != null){
39             //创建http request(GET)
40             request = new HttpGet(nextPageUrl);
41 
42             //设置http request header
43             request.setHeader("authorization", "oauth c3cef7c66a1843f8b3a9e6a1e3160e20");
44             respOnse= httpClient.execute(request);
45             //打印response
46             respOnseStr= EntityUtils.toString(response.getEntity());
47             System.out.println(responseStr);
48             nextPageUrl = getNextPageUrl(responseStr);
49             isEnd = getIsEnd(responseStr);
50         }
51     }
52 
53     /**
54      * 获取next url
55      * @param responseStr
56      * @return
57      */
58     private static String getNextPageUrl(String responseStr){
59         JSONObject jsOnObject= (JSONObject) JSON.parse(responseStr);
60         jsOnObject= (JSONObject) jsonObject.get("paging");
61         return jsonObject.get("next").toString();
62     }
63 
64     /**
65      * 获取is_end
66      * @param responseStr
67      * @return
68      */
69     private static boolean getIsEnd(String responseStr){
70         JSONObject jsOnObject= (JSONObject) JSON.parse(responseStr);
71         jsOnObject= (JSONObject) jsonObject.get("paging");
72         return (boolean) jsonObject.get("is_end");
73     }
74 }

 1 　　<dependency>
 2       <groupId>org.apache.httpcomponentsgroupId>
 3       <artifactId>httpclientartifactId>
 4       <version>4.5version>
 5     dependency>
 6 
 7     
 8     <dependency>
 9       <groupId>com.alibabagroupId>
10       <artifactId>fastjsonartifactId>
11       <version>1.2.31version>
12     dependency>