Httpclientgzip乱码问题解决

■出现问题的原因推测

被反爬了,缺少了COOKIE,你请求出来的信息就是运行一段js,

生成COOKIE,看到args1了么,这个是密钥,下面的也不是编码的,就是js混淆的问题

防爬网站需要携带一些基础http头模拟成浏览器登录

https://www.jianshu.com/p/401a25134b89

■前言

以下代码运行的返回值

■代码

package com.sxz.timecontroal;import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URLDecoder;
import java.util.zip.GZIPInputStream;import org.apache.http.Header;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;public class CheckTimeWithNet {static final String LOGINURL &＃61; "https://blog.csdn.net/sxzlc/article/list/3";public static void main(final String[] args) {final DefaultHttpClient httpclient &＃61; new DefaultHttpClient();final HttpGet httpGet &＃61; new HttpGet(LOGINURL);HttpResponse response &＃61; null;try {httpGet.addHeader("Accept-Encoding", "gzip, deflate"); response &＃61; httpclient.execute(httpGet); } catch (final ClientProtocolException cpException) {} catch (final IOException ioException) {}// verify response is HTTP OKfinal int statusCode &＃61; response.getStatusLine().getStatusCode();if (statusCode !&＃61; HttpStatus.SC_OK) {System.out.println("Error authenticating to Force.com: "&＃43;statusCode);return;}System.out.println("---------------------Status code Info Start---------------------");System.out.println(response.getStatusLine());System.out.println("---------------------Status code Info end ---------------------");System.out.println("---------------------Head Info Start---------------------");final Header[] hs &＃61; response.getAllHeaders();for(final Header h:hs){System.out.println(h.getName() &＃43; ":" &＃43; h.getValue());}System.out.println("---------------------Head Info End ---------------------");String getResult &＃61; null;try {// response.setEntity(new GzipDecompressingEntity(response.getEntity())); // getResult &＃61; EntityUtils.toString(response.getEntity(),"UTF-8");getResult &＃61; getStringFromResponseUzip(response);} catch (final Exception ioException) {// Handle system IO exception}System.out.println(getResult);}public static String getStringFromResponseUzip(final HttpResponse response) throws Exception {if (response &＃61;&＃61; null) {return null;}String responseText &＃61; "";//InputStream in &＃61; response.getEntity().getContent();final InputStream in &＃61; response.getEntity().getContent();final Header[] headers &＃61; response.getHeaders("Content-Encoding");for(final Header h : headers){System.out.println(h.getValue());if(h.getValue().indexOf("gzip") > -1){//For GZip responsetry{final GZIPInputStream gzin &＃61; new GZIPInputStream(in);final InputStreamReader isr &＃61; new InputStreamReader(gzin,"UTF-8");responseText &＃61; getStringFromStream(isr);//responseText &＃61; URLDecoder.decode(responseText, "utf-8");}catch (final IOException exception){exception.printStackTrace();}return responseText;}}responseText &＃61; EntityUtils.toString(response.getEntity(),"utf-8");return responseText;}public static String getStringFromStream(final InputStreamReader isr) throws Exception{final BufferedReader br &＃61; new BufferedReader(isr);final StringBuilder sb &＃61; new StringBuilder();String tmp;while((tmp &＃61; br.readLine())!&＃61;null){sb.append(tmp);sb.append("\r\n");}br.close();isr.close();return sb.toString();}
}

■运行结果

---------------------Status code Info Start---------------------
HTTP/1.1 200 OK
---------------------Status code Info end  ---------------------
---------------------Head Info Start---------------------
Server:Tengine
Date:Sat, 07 Dec 2019 12:20:38 GMT
Content-Type:text/html; charset&＃61;utf-8
Transfer-Encoding:chunked
Connection:keep-alive
Set-COOKIE:acw_tc&＃61;2760820215757212385795097e52a909ebbcda96b20e30f4c216c0bfbc89e6;path&＃61;/;HttpOnly;Max-Age&＃61;2678401
Content-Encoding:gzip
cache-control:no-cache, no-store
Pragma:no-cache
Strict-Transport-Security:max-age&＃61;86400
---------------------Head Info End  ---------------------
gzip

■后续

解压后为16进制代码&＃xff0c;有待解决。。。

\x65 z

这是 URLENCODE造成的&＃xff0c;使用URLDECODE解决

感谢&＃xff0c;[gybao]大神的帮助

https://bbs.csdn.net/topics/395274030

但是&＃xff0c;没有使用URLDECODE&＃xff0c;之前的代码&＃xff0c;在运行一下&＃xff0c;竟然直接成功了。

但是&＃xff0c;我之前是怎么跑出这种效果的&＃xff0c;原因不明。。。　推测问题的原因在下面记述

■再次修改后的代码

对于目前最新代码的说明

当能进入到下面79行的分支中时&＃xff0c;不论有没有85行都不会出现乱码问题。

代码

package com.sxz.timecontroal;import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URLDecoder;
import java.util.zip.GZIPInputStream;import org.apache.http.Header;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;public class CheckTimeWithNet {//static final String LOGINURL &＃61; "https://blog.csdn.net/sxzlc?orderby&＃61;ViewCount";static final String LOGINURL &＃61; "https://blog.csdn.net/sxzlc/article/list/2?orderby&＃61;ViewCount";public static void main(final String[] args) {final DefaultHttpClient httpclient &＃61; new DefaultHttpClient();final HttpGet httpGet &＃61; new HttpGet(LOGINURL);HttpResponse response &＃61; null;try {httpGet.addHeader("Accept-Encoding", "gzip, deflate"); response &＃61; httpclient.execute(httpGet); } catch (final ClientProtocolException cpException) {} catch (final IOException ioException) {}// verify response is HTTP OKfinal int statusCode &＃61; response.getStatusLine().getStatusCode();if (statusCode !&＃61; HttpStatus.SC_OK) {System.out.println("Error authenticating to Force.com: "&＃43;statusCode);return;}System.out.println("---------------------Status code Info Start---------------------");System.out.println(response.getStatusLine());System.out.println("---------------------Status code Info end ---------------------");System.out.println("---------------------Head Info Start---------------------");final Header[] hs &＃61; response.getAllHeaders();for(final Header h:hs){System.out.println(h.getName() &＃43; ":" &＃43; h.getValue());}System.out.println("---------------------Head Info End ---------------------");String getResult &＃61; null;try {// response.setEntity(new GzipDecompressingEntity(response.getEntity())); // getResult &＃61; EntityUtils.toString(response.getEntity(),"UTF-8");getResult &＃61; getStringFromResponseUzip(response);} catch (final Exception ioException) {// Handle system IO exception}System.out.println(getResult);}public static String getStringFromResponseUzip(final HttpResponse response) throws Exception {if (response &＃61;&＃61; null) {return null;}String responseText &＃61; "";//InputStream in &＃61; response.getEntity().getContent();final InputStream in &＃61; response.getEntity().getContent();final Header[] headers &＃61; response.getHeaders("Content-Encoding");for(final Header h : headers){System.out.println(h.getValue());if(h.getValue().indexOf("gzip") > -1){//For GZip responsetry{final GZIPInputStream gzin &＃61; new GZIPInputStream(in);final InputStreamReader isr &＃61; new InputStreamReader(gzin,"UTF-8");responseText &＃61; getStringFromStream(isr);responseText &＃61; URLDecoder.decode(responseText, "UTF-8");}catch (final IOException exception){exception.printStackTrace();}System.out.println("---------------------is gzip---------------------");return responseText;}}System.out.println("---------------------is not gzip---------------------");responseText &＃61; EntityUtils.toString(response.getEntity(),"utf-8");return responseText;}public static String getStringFromStream(final InputStreamReader isr) throws Exception{final BufferedReader br &＃61; new BufferedReader(isr);final StringBuilder sb &＃61; new StringBuilder();String tmp;while((tmp &＃61; br.readLine())!&＃61;null){sb.append(tmp);sb.append("\r\n");}br.close();isr.close();return sb.toString();}
}

以上代码运行后的结果

如果Get不设定gzip是&＃xff0c;

---

推测出现问题的原因&＃xff1a;

-------------------------------------------------------

■原因推测

还是网站那边做了什么特殊的处理

上午之所以好用&＃xff0c;是因为网站那边返回的结果没有进行 gzip压缩&＃xff0c;

而下午请求同样的地址&＃xff0c;经过了gzip压缩&＃xff0c;所以在解析处理的时候&＃xff0c;无法正常解析。

■现象1

下午再次同样的运行代码&＃xff0c;又出现了乱码的问题&＃xff0c;

加上DECODE也没有用&＃xff08;以下88&＃xff0c;87行&＃xff09;&＃xff0c;估计解码时出现问题&＃xff0c;直接返回NULL了

现象2

上午再cmd 窗口中&＃xff0c;使用CURL 上面的地址

可以返回页面的HTML&＃xff0c;下午就不行了&＃xff0c;返回效果如下。

■关于URLEncode的确认

上面的乱码抽取了一部分&＃xff0c;确定是URL编码&＃xff0c;但是在解码全部字符串的时候&＃xff0c;返回值为NULL

■补充说明

而且&＃xff0c;感觉乱码是&＃xff0c;返回的信息&＃xff0c;和上午返回所有的页面HTML代码相比较&＃xff0c;少了很多&＃xff01;

-------------------------------------------------------

■后续&＃xff08;结果说明1&＃xff09;

关于一会儿是 gzip&＃xff0c; 一会儿不是&＃xff0c;

原因推测是&＃xff0c;因为负载平衡&＃xff0c;每次访问的服务器不一样。

基于 Nginx 的两个版本&＃xff08;Openresty和Tengine&＃xff09;

・gzip的server信息 Tengine

------------------------------------

------------------------------------

・不是gzip时的server信息

------------------------------------

 TODO

------------------------------------

----

---