HttpUrlConnection底层实现和关于javahost绑定ip即时生效的设置及分析

最近有个需求需要对于获取URL页面进行host绑定并且立即生效&＃xff0c;在java里面实现可以用代理服务器来实现&＃xff1a;因为在测试环境下可能需要通过绑定来访问测试环境的应用
实现代码如下&＃xff1a;

public static String getResponseText(String queryUrl,String host,String ip) { //queryUrl&＃xff0c;完整的url&＃xff0c;host和ip需要绑定的host和ip
InputStream is &＃61; null;
BufferedReader br &＃61; null;
StringBuffer res &＃61; new StringBuffer();
try {
HttpURLConnection httpUrlConn &＃61; null;
URL url &＃61; new URL(queryUrl);
if(ip!&＃61;null){
String str[] &＃61; ip.split("\\.");
byte[] b &＃61;new byte[str.length];
for(int i&＃61;0,len&＃61;str.length;i b[i] &＃61; (byte)(Integer.parseInt(str[i],10));
}
Proxy proxy &＃61; new Proxy(Proxy.Type.HTTP,
new InetSocketAddress(InetAddress.getByAddress(b), 80)); //b是绑定的ip&＃xff0c;生成proxy代理对象&＃xff0c;因为http底层是socket实现&＃xff0c;
httpUrlConn &＃61; (HttpURLConnection) url
.openConnection(proxy);
}else{
httpUrlConn &＃61; (HttpURLConnection) url
.openConnection();
}
httpUrlConn.setRequestMethod("GET");
httpUrlConn.setDoOutput(true);
httpUrlConn.setConnectTimeout(2000);
httpUrlConn.setReadTimeout(2000);
httpUrlConn.setDefaultUseCaches(false);
httpUrlConn.setUseCaches(false);

is &＃61; httpUrlConn.getInputStream();


那么底层对于proxy对象到底是怎么处理&＃xff0c;底层的socket实现到底怎么样&＃xff0c;带着这个疑惑看了下jdk的rt.jar对于这块的处理
httpUrlConn &＃61; (HttpURLConnection) url.openConnection(proxy)

java.net.URL类里面的openConnection方法&＃xff1a;
public URLConnection openConnection(Proxy proxy){
…
return handler.openConnection(this, proxy); Handler是sun.net.www.protocol.http.Handler.java类&＃xff0c;继承java.net. URLStreamHandler.java类&＃xff0c;用来处理http连接请求响应的。
}

Handler的方法&＃xff1a;
protected java.net.URLConnection openConnection(URL u, Proxy p)
throws IOException {
return new HttpURLConnection(u, p, this);
}

只是简单的生成sun.net.www.protocl.http.HttpURLConnection对象&＃xff0c;并进行初始化
protected HttpURLConnection(URL u, Proxy p, Handler handler) {
super(u);
requests &＃61; new MessageHeader(); 请求头信息生成类
responses &＃61; new MessageHeader(); 响应头信息解析类
this.handler &＃61; handler;
instProxy &＃61; p; 代理服务器对象
COOKIEHandler &＃61; (COOKIEHandler)java.security.AccessController.doPrivileged(
new java.security.PrivilegedAction() {
public Object run() {
return COOKIEHandler.getDefault();
}
});
cacheHandler &＃61; (ResponseCache)java.security.AccessController.doPrivileged(
new java.security.PrivilegedAction() {
public Object run() {
return ResponseCache.getDefault();
}
});
}


最终在httpUrlConn.getInputStream();才进行socket连接&＃xff0c;发送http请求&＃xff0c;解析http响应信息。具体过程如下&＃xff1a;

sun.net.www.protocl.http.HttpURLConnection.java的getInputStream方法&＃xff1a;

public synchronized InputStream getInputStream() throws IOException {

...socket连接
connect();
...
ps &＃61; (PrintStream)http.getOutputStream(); 获得输出流&＃xff0c;打开连接之后已经生成。

if (!streaming()) {
writeRequests(); 输出http请求头信息
}
...
http.parseHTTP(responses, pi, this); 解析响应信息
if(logger.isLoggable(Level.FINEST)) {
logger.fine(responses.toString());
}
inputStream &＃61; http.getInputStream(); 获得输入流
}

其中connect()调用方法链&＃xff1a;
plainConnect(){
...
Proxy p &＃61; null;
if (sel !&＃61; null) {
URI uri &＃61; sun.net.www.ParseUtil.toURI(url);
Iterator it &＃61; sel.select(uri).iterator();
while (it.hasNext()) {
p &＃61; it.next();
try {
if (!failedOnce) {
http &＃61; getNewHttpClient(url, p, connectTimeout);
...
}

getNewHttpClient(){
...
return HttpClient.New(url, p, connectTimeout, useCache);
...
}

下面跟进去最终建立socket连接的代码&＃xff1a;
sun.net.www.http.HttpClient.java的openServer()方法建立socket连接&＃xff1a;

protected synchronized void openServer() throws IOException {
...
if ((proxy !&＃61; null) && (proxy.type() &＃61;&＃61; Proxy.Type.HTTP)) {
sun.net.www.URLConnection.setProxiedHost(host);
if (security !&＃61; null) {
security.checkConnect(host, port);
}
privilegedOpenServer((InetSocketAddress) proxy.address());最终socket连接的是设置的代理服务器的地址&＃xff0c;
...
}

private synchronized void privilegedOpenServer(final InetSocketAddress server)
throws IOException
{
try {
java.security.AccessController.doPrivileged(
new java.security.PrivilegedExceptionAction() {
public Object run() throws IOException {
openServer(server.getHostName(), server.getPort()); 注意openserver函数 这里的server的getHostName是设置的代理服务器&＃xff0c;&＃xff08;ip或者hostname&＃xff0c;如果是host绑定设置的代理服务器的ip&＃xff0c;那么这里getHostName出来的就是ip地址&＃xff0c;可以去查看InetSocketAddress类的getHostName方法&＃xff09;
return null;
}
});
} catch (java.security.PrivilegedActionException pae) {
throw (IOException) pae.getException();
}
}

public void openServer(String server, int port) throws IOException {
serverSocket &＃61; doConnect(server, port); 生成的Socket连接对象
try {
serverOutput &＃61; new PrintStream(
new BufferedOutputStream(serverSocket.getOutputStream()),
false, encoding); 生成输出流&＃xff0c;
} catch (UnsupportedEncodingException e) {
throw new InternalError(encoding&＃43;" encoding not found");
}
serverSocket.setTcpNoDelay(true);
}


protected Socket doConnect (String server, int port)
throws IOException, UnknownHostException {
Socket s;
if (proxy !&＃61; null) {
if (proxy.type() &＃61;&＃61; Proxy.Type.SOCKS) {
s &＃61; (Socket) AccessController.doPrivileged(
new PrivilegedAction() {
public Object run() {
return new Socket(proxy);
}});
} else
s &＃61; new Socket(Proxy.NO_PROXY);
} else
s &＃61; new Socket();
// Instance specific timeouts do have priority, that means
// connectTimeout & readTimeout (-1 means not set)
// Then global default timeouts
// Then no timeout.
if (connectTimeout >&＃61; 0) {
s.connect(new InetSocketAddress(server, port), connectTimeout);
} else {
if (defaultConnectTimeout > 0) {
s.connect(new InetSocketAddress(server, port), defaultConnectTimeout);//连接到代理服务器&＃xff0c;看下面Socket类的connect方法代码
} else {
s.connect(new InetSocketAddress(server, port));
}
}
if (readTimeout >&＃61; 0)
s.setSoTimeout(readTimeout);
else if (defaultSoTimeout > 0) {
s.setSoTimeout(defaultSoTimeout);
}
return s;
}

上面的new InetSocketAddress(server, port)这里会涉及到java DNS cache的处理&＃xff0c;

public InetSocketAddress(String hostname, int port) {
if (port <0 || port > 0xFFFF) {
throw new IllegalArgumentException("port out of range:" &＃43; port);
}
if (hostname &＃61;&＃61; null) {
throw new IllegalArgumentException("hostname can&＃39;t be null");
}
try {
addr &＃61; InetAddress.getByName(hostname); //这里会有java DNS缓存的处理&＃xff0c;先从缓存取hostname绑定的ip地址&＃xff0c;如果取不到再通过OS的DNS cache机制去取&＃xff0c;取不到再从DNS服务器上取。
} catch(UnknownHostException e) {
this.hostname &＃61; hostname;
addr &＃61; null;
}
this.port &＃61; port;
}



当然最终的Socket.java的connect方法
java.net.socket

public void connect(SocketAddress endpoint, int timeout) throws IOException {
if (endpoint &＃61;&＃61; null)

if (timeout <0)
throw new IllegalArgumentException("connect: timeout can&＃39;t be negative");

if (isClosed())
throw new SocketException("Socket is closed");

if (!oldImpl && isConnected())
throw new SocketException("already connected");

if (!(endpoint instanceof InetSocketAddress))
throw new IllegalArgumentException("Unsupported address type");

InetSocketAddress epoint &＃61; (InetSocketAddress) endpoint;

SecurityManager security &＃61; System.getSecurityManager();
if (security !&＃61; null) {
if (epoint.isUnresolved())
security.checkConnect(epoint.getHostName(),
epoint.getPort());
else
security.checkConnect(epoint.getAddress().getHostAddress(),
epoint.getPort());
}
if (!created)
createImpl(true);
if (!oldImpl)
impl.connect(epoint, timeout);
else if (timeout &＃61;&＃61; 0) {
if (epoint.isUnresolved()) //如果没有设置SocketAddress的ip地址&＃xff0c;则用域名去访问
impl.connect(epoint.getAddress().getHostName(),
epoint.getPort());
else
impl.connect(epoint.getAddress(), epoint.getPort()); 最终socket连接的是设置的SocketAddress的ip地址&＃xff0c;
} else
throw new UnsupportedOperationException("SocketImpl.connect(addr, timeout)");
connected &＃61; true;
/*
* If the socket was not bound before the connect, it is now because
* the kernel will have picked an ephemeral port & a local address
*/
bound &＃61; true;
}



我们再看下通过socket来发送HTTP请求的处理代码&＃xff0c;也就是sun.net.www.protocl.http.HttpURLConnection.java的getInputStream方法中调用的writeRequests()方法&＃xff1a;
private void writeRequests() throws IOException { 这段代码就是封装http请求的头请求信息&＃xff0c;通过socket发送出去
/* print all message headers in the MessageHeader
* onto the wire - all the ones we&＃39;ve set and any
* others that have been set
*/
// send any pre-emptive authentication
if (http.usingProxy) {
setPreemptiveProxyAuthentication(requests);
}
if (!setRequests) {

/* We&＃39;re very particular about the order in which we
* set the request headers here. The order should not
* matter, but some careless CGI programs have been
* written to expect a very particular order of the
* standard headers. To name names, the order in which
* Navigator3.0 sends them. In particular, we make *sure*
* to send Content-type: <> and Content-length:<> second
* to last and last, respectively, in the case of a POST
* request.
*/
if (!failedOnce)
requests.prepend(method &＃43; " " &＃43; http.getURLFile()&＃43;" " &＃43;
httpVersion, null);
if (!getUseCaches()) {
requests.setIfNotSet ("Cache-Control", "no-cache");
requests.setIfNotSet ("Pragma", "no-cache");
}
requests.setIfNotSet("User-Agent", userAgent);
int port &＃61; url.getPort();
String host &＃61; url.getHost();
if (port !&＃61; -1 && port !&＃61; url.getDefaultPort()) {
host &＃43;&＃61; ":" &＃43; String.valueOf(port);
}
requests.setIfNotSet("Host", host);
requests.setIfNotSet("Accept", acceptString);

/*
* For HTTP/1.1 the default behavior is to keep connections alive.
* However, we may be talking to a 1.0 server so we should set
* keep-alive just in case, except if we have encountered an error
* or if keep alive is disabled via a system property
*/

// Try keep-alive only on first attempt
if (!failedOnce && http.getHttpKeepAliveSet()) {
if (http.usingProxy) {
requests.setIfNotSet("Proxy-Connection", "keep-alive");
} else {
requests.setIfNotSet("Connection", "keep-alive");
}
} else {
/*
* RFC 2616 HTTP/1.1 section 14.10 says:
* HTTP/1.1 applications that do not support persistent
* connections MUST include the "close" connection option
* in every message
*/
requests.setIfNotSet("Connection", "close");
}
// Set modified since if necessary
long modTime &＃61; getIfModifiedSince();
if (modTime !&＃61; 0 ) {
Date date &＃61; new Date(modTime);
//use the preferred date format according to RFC 2068(HTTP1.1),
// RFC 822 and RFC 1123
SimpleDateFormat fo &＃61;
new SimpleDateFormat ("EEE, dd MMM yyyy HH:mm:ss &＃39;GMT&＃39;", Locale.US);
fo.setTimeZone(TimeZone.getTimeZone("GMT"));
requests.setIfNotSet("If-Modified-Since", fo.format(date));
}
// check for preemptive authorization
AuthenticationInfo sauth &＃61; AuthenticationInfo.getServerAuth(url);
if (sauth !&＃61; null && sauth.supportsPreemptiveAuthorization() ) {
// Sets "Authorization"
requests.setIfNotSet(sauth.getHeaderName(), sauth.getHeaderValue(url,method));
currentServerCredentials &＃61; sauth;
}

if (!method.equals("PUT") && (poster !&＃61; null || streaming())) {
requests.setIfNotSet ("Content-type",
"application/x-www-form-urlencoded");
}

if (streaming()) {
if (chunkLength !&＃61; -1) {
requests.set ("Transfer-Encoding", "chunked");
} else {
requests.set ("Content-Length", String.valueOf(fixedContentLength));
}
} else if (poster !&＃61; null) {
/* add Content-Length & POST/PUT data */
synchronized (poster) {
/* close it, so no more data can be added */
poster.close();
requests.set("Content-Length",
String.valueOf(poster.size()));
}
}

// get applicable COOKIEs based on the uri and request headers
// add them to the existing request headers
setCOOKIEHeader();
…
}


再来看看把socket响应信息解析为http的响应信息的代码&＃xff1a;
sun.net.www.http.HttpClient.java的parseHTTP方法&＃xff1a;
private boolean parseHTTPHeader(MessageHeader responses, ProgressSource pi, HttpURLConnection httpuc)
throws IOException {
/* If "HTTP/*" is found in the beginning, return true. Let
* HttpURLConnection parse the mime header itself.
*
* If this isn&＃39;t valid HTTP, then we don&＃39;t try to parse a header
* out of the beginning of the response into the responses,
* and instead just queue up the output stream to it&＃39;s very beginning.
* This seems most reasonable, and is what the NN browser does.
*/

keepAliveConnections &＃61; -1;
keepAliveTimeout &＃61; 0;

boolean ret &＃61; false;
byte[] b &＃61; new byte[8];

try {
int nread &＃61; 0;
serverInput.mark(10);
while (nread <8) {
int r &＃61; serverInput.read(b, nread, 8 - nread);
if (r <0) {
break;
}
nread &＃43;&＃61; r;
}
String keep&＃61;null;
ret &＃61; b[0] &＃61;&＃61; &＃39;H&＃39; && b[1] &＃61;&＃61; &＃39;T&＃39;
&& b[2] &＃61;&＃61; &＃39;T&＃39; && b[3] &＃61;&＃61; &＃39;P&＃39; && b[4] &＃61;&＃61; &＃39;/&＃39; &&
b[5] &＃61;&＃61; &＃39;1&＃39; && b[6] &＃61;&＃61; &＃39;.&＃39;;
serverInput.reset();
if (ret) { // is valid HTTP - response started w/ "HTTP/1."
responses.parseHeader(serverInput);

// we&＃39;ve finished parsing http headers
// check if there are any applicable COOKIEs to set (in cache)
if (COOKIEHandler !&＃61; null) {
URI uri &＃61; ParseUtil.toURI(url);
// NOTE: That cast from Map shouldn&＃39;t be necessary but
// a bug in javac is triggered under certain circumstances
// So we do put the cast in as a workaround until
// it is resolved.
if (uri !&＃61; null)
COOKIEHandler.put(uri, (Map>)responses.getHeaders());
}

/* decide if we&＃39;re keeping alive:
* This is a bit tricky. There&＃39;s a spec, but most current
* servers (10/1/96) that support this differ in dialects.
* If the server/client misunderstand each other, the
* protocol should fall back onto HTTP/1.0, no keep-alive.
*/
if (usingProxy) { // not likely a proxy will return this
keep &＃61; responses.findValue("Proxy-Connection");
}
if (keep &＃61;&＃61; null) {
keep &＃61; responses.findValue("Connection");
}
if (keep !&＃61; null && keep.toLowerCase().equals("keep-alive")) {
/* some servers, notably Apache1.1, send something like:
* "Keep-Alive: timeout&＃61;15, max&＃61;1" which we should respect.
*/
HeaderParser p &＃61; new HeaderParser(
responses.findValue("Keep-Alive"));
if (p !&＃61; null) {
/* default should be larger in case of proxy */
keepAliveConnections &＃61; p.findInt("max", usingProxy?50:5);
keepAliveTimeout &＃61; p.findInt("timeout", usingProxy?60:5);
}
} else if (b[7] !&＃61; &＃39;0&＃39;) {
/*
* We&＃39;re talking 1.1 or later. Keep persistent until
* the server says to close.
*/
if (keep !&＃61; null) {
/*
* The only Connection token we understand is close.
* Paranoia: if there is any Connection header then
* treat as non-persistent.
*/
keepAliveConnections &＃61; 1;
} else {
keepAliveConnections &＃61; 5;
}
}
……
}


对于java.net包的http&＃xff0c;ftp等各种协议的底层实现&＃xff0c;可以参考rt.jar下面的几个包的代码&＃xff1a;
sun.net.www.protocl下的几个包。


在http client中也可以设置代理&＃xff1a;
HostConfiguration conf &＃61; new HostConfiguration();
conf.setHost(host);
conf.setProxy(ip, 80);
statusCode &＃61; httpclient.executeMethod(conf,getMethod);

httpclient自己也是基于socket封装的http处理的库。底层代理的实现是一样的。


另外一种不设置代理&＃xff0c;通过反射修改InetAddress的cache也是ok的。但是这种方法非常不推荐&＃xff0c;不要使用&＃xff0c;因为对于proxy代理服务器概念了解不清楚&＃xff0c;最开始还使用这种方法&＃xff0c;
public static void jdkDnsNoCache(final String host, final String ip)
throws SecurityException, NoSuchFieldException,
IllegalArgumentException, IllegalAccessException {
if (StringUtils.isBlank(host)) {
return;
}
final Class clazz &＃61; java.net.InetAddress.class;
final Field cacheField &＃61; clazz.getDeclaredField("addressCache");
cacheField.setAccessible(true);
final Object o &＃61; cacheField.get(clazz);
Class clazz2 &＃61; o.getClass();
final Field cacheMapField &＃61; clazz2.getDeclaredField("cache");
cacheMapField.setAccessible(true);
final Map cacheMap &＃61; (Map) cacheMapField.get(o);
AccessController.doPrivileged(new PrivilegedAction() {
public Object run() {
try {
synchronized (o) {// 同步是必须的,因为o可能会有多个线程同时访问修改。
// cacheMap.clear();//这步比较关键&＃xff0c;用于清除原来的缓存
// cacheMap.remove(host);
if (!StringUtils.isBlank(ip)) {
InetAddress inet &＃61; InetAddress.getByAddress(host,IPUtil.int2byte(ip));
InetAddress addressstart &＃61; InetAddress.getByName(host);
Object cacheEntry &＃61; cacheMap.get(host);
cacheMap.put(host,newCacheEntry(inet,cacheEntry));
// cacheMap.put(host,newCacheEntry(newInetAddress(host, ip)));
}else{
cacheMap.remove(host);
}
// System.out.println(getStaticProperty(
// "java.net.InetAddress", "addressCacheInit"));
// System.out.println(invokeStaticMethod("java.net.InetAddress","getCachedAddress",new
// Object[]{host}));
}
} catch (Throwable te) {
throw new RuntimeException(te);
}
return null;
}
});
final Map cacheMapafter &＃61; (Map) cacheMapField.get(o);
System.out.println(cacheMapafter);

}

关于java中对于DNS的缓存设置可以参考&＃xff1a;
1.在${java_home}/jre/lib/secuiry/java.secuiry文件&＃xff0c;修改下面为
networkaddress.cache.negative.ttl&＃61;0 DNS解析不成功的缓存时间
networkaddress.cache.ttl&＃61;0 DNS解析成功的缓存的时间
2.jvm启动时增加下面两个启动环境变量
-Dsun.net.inetaddr.ttl&＃61;0
-Dsun.net.inetaddr.negative.ttl&＃61;0


如果在java程序中使用&＃xff0c;可以这么设置设置&＃xff1a;
java.security.Security.setProperty("networkaddress.cache.ttl" , "0");
java.security.Security.setProperty("networkaddress.cache.negative.ttl" , "0");

还有几篇文档链接可以查看&＃xff1a;
http://www.rgagnon.com/javadetails/java-0445.html
http://bugs.sun.com/bugdatabase/view_bug.do?bug_id&＃61;6247501

linux下关于&＃xff2f;&＃xff33;　&＃xff24;&＃xff2e;&＃xff33;设置的几个文件是
/etc/resolve.conf
/etc/nscd.conf
/etc/nsswitch.conf

http://www.linuxfly.org/post/543/
http://linux.die.net/man/5/nscd.conf
http://www.linuxhomenetworking.com/wiki/index.php/Quick_HOWTO_:_Ch18_:_Configuring_DNS
http://linux.die.net/man/5/nscd.conf