博学，切问，近思--詹子知 (https://jameszhan.github.io)

日前笔者在使用HttpClient在处理大数据请求的时候，在连续发请求的时候经常会出现异常 java.io.IOException: chunked stream ended unexpectedly。使用HttpMethod的abort方法也不能完全避免这种异常的出现，但是对于小数据的请求，这种异常就基本上难得一见了。对于同样的页面请求，如何减少网络的数据传输量呢。众所周知，现在大部分的Web Server都是支持数据的压缩传输的。要知道，一般的网页内容经过压缩，大小可以减少到原来的20%以下，而对于纯英文为网站，网页内容更是可以减少到原来内容的5%以下。而要使Web Server对数据进行压缩传输，只需要在请求头上加入Accept-Encoding:gzip, deflate。public HttpMethod createHttpMethod(String url, String type, NameValuePair[] params, String contentType) { HttpMethod method = null; if (type.equalsIgnoreCase("POST")) { method = new PostMethod(url); method.setRequestHeader("Content-Type", contentType); if(params != null){ ((PostMethod) method).setRequestBody(params); } } else { method = new GetMethod(url); if(params != null){ method.setQueryString(params); } } method.setRequestHeader("Accept-Encoding", "gzip, deflate"); return method; }

 

这个时候，如果你请求的Web Server支持Gzip，返回来的响应便是被压缩后的数据，那么把压缩后的数据解析成原来的网页内容便是客户端要做的事情了。对于当前的主流浏览器，都是支持对压缩数据自动解压的，而在我们的应用程序中，我们只要对象网页流稍作处理，便可以得到原来的网页内容。

protected String doSuccess(HttpMethod method) throws IOException { InputStream in = method.getResponseBodyAsStream(); Header contentEncodingHeader = method.getResponseHeader("Content-Encoding"); if (contentEncodingHeader != null) { String contentEncoding = contentEncodingHeader.getValue(); if (contentEncoding.toLowerCase(Locale.US).indexOf("gzip") != -1) { in = new GZIPInputStream(in); } } return decoder.decode(in); }

上一篇文章，我们介绍了如何检查文档输入流的编码，本节我们就可以利用上文的HtmlInputStreamDecoder类来把文档流来解析文档内容。完整的代码如下：

import java.io.IOException; import java.io.InputStream; import java.util.Locale; import java.util.zip.GZIPInputStream; import org.apache.commons.httpclient.Header; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpException; import org.apache.commons.httpclient.HttpMethod; import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; import org.apache.commons.httpclient.NameValuePair; import org.apache.commons.httpclient.URI; import org.apache.commons.httpclient.URIException; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.methods.PostMethod; import org.apache.log4j.Logger; public class HttpRequest { private static final Logger LOGGER = Logger.getLogger(HttpRequest.class); private HttpClient client; private HtmlInputStreamDecoder decoder; public HttpRequest() { this(new HttpClient(new MultiThreadedHttpConnectionManager())); } public HttpRequest(HttpClient client) { this.client = client; this.decoder = new HtmlInputStreamDecoder(); } public String doRequest(HttpMethod method) { String html = null; try { int statusCode = client.executeMethod(method); switch (statusCode) { case HttpStatus.SC_OK: html = doSuccess(method); break; case HttpStatus.SC_MOVED_PERMANENTLY: case HttpStatus.SC_MOVED_TEMPORARILY: case HttpStatus.SC_SEE_OTHER: case HttpStatus.SC_TEMPORARY_REDIRECT: doRedirect(method); break; default: html = doError(method); } } catch (HttpException e) { LOGGER.error("Http error occur while visit the url.", e); } catch (IOException e) { LOGGER.error("IO error occur while visit the url.", e); } finally { method.abort(); method.releaseConnection(); } return html; } protected String doSuccess(HttpMethod method) throws IOException { InputStream in = method.getResponseBodyAsStream(); Header contentEncodingHeader = method.getResponseHeader("Content-Encoding"); if (contentEncodingHeader != null) { String contentEncoding = contentEncodingHeader.getValue(); if (contentEncoding.toLowerCase(Locale.US).indexOf("gzip") != -1) { in = new GZIPInputStream(in); } } return decoder.decode(in); } protected String doError(HttpMethod method) { LOGGER.error("Error Response: " + method.getStatusLine()); return method.getStatusText(); } protected void doRedirect(HttpMethod method) throws URIException { Header locationHeader = method.getResponseHeader("location"); if (locationHeader != null) { String location = locationHeader.getValue(); if (location == null) { location = "/"; } doRequest(new GetMethod(getRedirectUrl(method.getURI(), location))); } } public HttpMethod createHttpMethod(String url, String type, NameValuePair[] params, String contentType) { HttpMethod method = null; if (type.equalsIgnoreCase("POST")) { method = new PostMethod(url); method.setRequestHeader("Content-Type", contentType); if(params != null){ ((PostMethod) method).setRequestBody(params); } } else { method = new GetMethod(url); if(params != null){ method.setQueryString(params); } } method.setRequestHeader("Accept-Encoding", "gzip, deflate"); return method; } protected static String getRedirectUrl(URI origin, String location) throws URIException { String redirect = null; if (location.startsWith("http:")) { redirect = location; } else if (location.startsWith("/")) { origin.setPath(location); redirect = origin.getURI(); } else { redirect = origin.getURI().replaceAll("(?<=/)[^/]+$", location); } return redirect; } }

 

代码示例：

public static void main(String[] args) { HttpRequest request = new HttpRequest(); HttpMethod method = request.createHttpMethod("http://www.csdn.com", "GET", null, "text/html"); String html = request.doRequest(method); System.out.println(html); }