【java爬虫】HttpClient4.5模拟登录知乎

0.关于HttpClient

虽然在 JDK 的 java net包中已经提供了访问 HTTP 协议的基本功能，但是对于大部分应用程序来说，JDK 库本身提供的功能还不够丰富和灵活。HttpClient 是 Apache Jakarta Common 下的子项目，用来提供高效的、最新的、功能丰富的支持 HTTP 协议的客户端编程工具包，并且它支持 HTTP 协议最新的版本和建议。

1.HttpClient4.5模拟登录网站步骤

由于网站模拟登录都会有验证码，关于验证码的处理一般有以下两种方式：

1. 验证码识别接口(大多都是收费的)，这种情况适合大规模登录。
2. 将验证码下载到本地，手动输入验证码（以下例子采用这种方式）。

2.模拟登录的分析过程

package com.wy;

import org.apache.http.NameValuePair;
import org.apache.http.client.config.COOKIESpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.COOKIE.COOKIESpecProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.COOKIE.BestMatchSpecFactory;
import org.apache.http.impl.COOKIE.BrowserCompatSpecFactory;
import org.apache.http.message.BasicNameValuePair;
import org.json.JSONObject;

import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;
import java.util.Scanner;

/**
 * 模拟登录知乎
 */
public class Main {
    //知乎首页
    final private static String INDEX_URL = "https://www.zhihu.com";
    //邮箱登录地址
    final private static String EMAIL_LOGIN_URL = "https://www.zhihu.com/login/email";
    //手机号码登录地址
    final private static String PHONENUM_LOGIN_URL = "https://www.zhihu.com/login/phone_num";
    //登录验证码地址
    final private static String YZM_URL = "https://www.zhihu.com/captcha.gif?type=login";

    private CloseableHttpClient httpClient;
    private HttpClientContext httpClientContext;

    /**
     * 初始化CloseableHttpClient、HttpClientContext并设置COOKIE策略
     */
    public Main(){
        RequestConfig globalCOnfig= RequestConfig.custom()
                .setCOOKIESpec(COOKIESpecs.BROWSER_COMPATIBILITY)
                .build();
        httpClient = HttpClients.custom()
                .setDefaultRequestConfig(globalConfig)
                .build();

        httpClientCOntext= HttpClientContext.create();
        Registry registry = RegistryBuilder
                . create()
                .register(COOKIESpecs.BEST_MATCH, new BestMatchSpecFactory())
                .register(COOKIESpecs.BROWSER_COMPATIBILITY,
                        new BrowserCompatSpecFactory()).build();
        httpClientContext.setCOOKIESpecRegistry(registry);
    }
    /**
     *
     * @param emailOrPhoneNum 邮箱或手机号码
     * @param pwd 密码
     * @return
     */
    public boolean login(String emailOrPhoneNum,
                         String pwd){
        String yzm = null;
        String loginState = null;
        HttpGet getRequest = new HttpGet(INDEX_URL);
        HttpClientUtil.getWebPage(httpClient, httpClientContext, getRequest, "utf-8", false);
        HttpPost request = null;
        List formParams = new ArrayList();
        if(emailOrPhoneNum.contains("@")){
            //通过邮箱登录
            request = new HttpPost(EMAIL_LOGIN_URL);
            formParams.add(new BasicNameValuePair("email", emailOrPhoneNum));
        }
        else {
            //通过手机号码登录
            request = new HttpPost(PHONENUM_LOGIN_URL);
            formParams.add(new BasicNameValuePair("phone_num", emailOrPhoneNum));
        }
        yzm = yzm(httpClient, httpClientContext, YZM_URL);
        formParams.add(new BasicNameValuePair("captcha", yzm));
        formParams.add(new BasicNameValuePair("_xsrf", ""));//这个参数可以不用
        formParams.add(new BasicNameValuePair("password", pwd));
        formParams.add(new BasicNameValuePair("remember_me", "true"));
        UrlEncodedFormEntity entity = null;
        try {
            entity = new UrlEncodedFormEntity(formParams, "utf-8");
        } catch (UnsupportedEncodingException e) {
            e.printStackTrace();
        }
        request.setEntity(entity);
        loginState = HttpClientUtil.getWebPage(httpClient, httpClientContext, request, "utf-8", false);//登录
        JSONObject jo = new JSONObject(loginState);
        if(jo.get("r").toString().equals("0")){
            System.out.println("登录知乎成功");
            getRequest = new HttpGet("https://www.zhihu.com");
            /**
             * 访问首页
             */
            HttpClientUtil.getWebPage(httpClient, httpClientContext,getRequest, "utf-8", false);
            return true;
        }else{
            throw new RuntimeException(HttpClientUtil.decodeUnicode(loginState));
        }
    }
    /**
     * 下载验证码至本地，并手动输入验证码
     * @return
     */
    public String yzm(CloseableHttpClient httpClient,HttpClientContext context, String url){
        String verificatiOnCodePath= System.getProperty("user.home");
        //下载验证码至本地
        HttpClientUtil.downloadFile(httpClient, context, url, verificationCodePath, "/yzm.jpg",true);
        System.out.println("请输入 " + verificationCodePath + "\\yzm.jpg 下的验证码：");
        Scanner sc = new Scanner(System.in);
        String yzm = sc.nextLine();
        return yzm;
    }
    public static void main(String[] args){
        Main modelLogin = new Main();
        modelLogin.login("邮箱或手机号码", "密码");
    }
}