入门案例：

　　1.创建maven工程，引入依赖：

<dependencies>
    <!-- HttpClient -->
    <dependency>
        <groupId>org.apache.httpcomponents</groupId>
        <artifactId>httpclient</artifactId>
        <version>4.5.3</version>
    </dependency>
    <!-- 日志 -->
    <dependency>
        <groupId>org.slf4j</groupId>
        <artifactId>slf4j-log4j12</artifactId>
        <version>1.7.25</version>
    </dependency>
    <!-- junit -->
    <dependency>
        <groupId>junit</groupId>
        <artifactId>junit</artifactId>
        <version>4.12</version>
    </dependency>
</dependencies>

　　2.加入log4j.properties

log4j.rootLogger=DEBUG,A1
log4j.logger.com.fgy = DEBUG

log4j.appender.A1=org.apache.log4j.ConsoleAppender
log4j.appender.A1.layout=org.apache.log4j.PatternLayout
log4j.appender.A1.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss,SSS} [%t] [%c]-[%p] %m%n

　　3.编写测试代码，抓取数据

@Test
public void testFirst() throws IOException {
    CloseableHttpClient client = HttpClients.createDefault();
    HttpGet httpGet = new HttpGet("https://www.cnblogs.com/roadlandscape/");
    CloseableHttpResponse response = client.execute(httpGet);
    if (response.getStatusLine().getStatusCode() == 200) {
        String context = EntityUtils.toString(response.getEntity(), "utf-8");
        System.out.println(context);
    }
}

带参数的GET请求：

@Test
public void testGetParam() {
    CloseableHttpClient client = HttpClients.createDefault();
    // HttpGet httpGet = new HttpGet("https://s.taobao.com/search?q=vivo");
    CloseableHttpResponse response = null;
    try {
        URIBuilder builder = new URIBuilder("https://s.taobao.com/search");
        builder.addParameter("q", "vivo");
        HttpGet httpGet = new HttpGet(builder.build());
        response = client.execute(httpGet);
        if (response.getStatusLine().getStatusCode() == 200) {
            String context = EntityUtils.toString(response.getEntity(), "utf-8");
            System.out.println(context);
        }
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        if (response != null) {
            try {
                response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        try {
            client.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

POST请求：

@Test
public void testPost() {
    CloseableHttpClient client = HttpClients.createDefault();
    HttpPost httpPost = new HttpPost("https://www.cnblogs.com/roadlandscape/");
    CloseableHttpResponse response = null;
    try {
        response = client.execute(httpPost);
        if (response.getStatusLine().getStatusCode() == 200) {
            String context = EntityUtils.toString(response.getEntity(), "utf-8");
            System.out.println(context);
        }
    } catch (IOException e) {
        e.printStackTrace();
    } finally {
        if (response != null) {
            try {
                response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        try {
            client.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

带参数的POST请求：

@Test
public void testPostParam() {
    CloseableHttpClient client = HttpClients.createDefault();
    CloseableHttpResponse response = null;
    try {
        HttpPost httpPost = new HttpPost("https://s.taobao.com/search");
        // 创建存放参数的list
        List<NameValuePair> params = new ArrayList<>();
        params.add(new BasicNameValuePair("q", "vivo"));
        // 创建表单数据 Entity
        UrlEncodedFormEntity entity = new UrlEncodedFormEntity(params, "utf-8");
        // 设置表单 Entity 到 httpPost 中
        httpPost.setEntity(entity);
        response = client.execute(httpPost);
        if (response.getStatusLine().getStatusCode() == 200) {
            String context = EntityUtils.toString(response.getEntity(), "utf-8");
            System.out.println(context);
        }
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        if (response != null) {
            try {
                response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        try {
            client.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

连接池：

　　如果每次请求都要创建HttpClient，会有频繁创建和销毁的问题，可以使用连接池来解决这个问题。

public class ConPool {
    public static void main(String[] args) throws IOException {
        PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
        // 设置最大连接数
        cm.setMaxTotal(200);
        // 设置每个主机的并发数
        cm.setDefaultMaxPerRoute(20);

        doGet(cm);
    }

    private static void doGet(PoolingHttpClientConnectionManager cm) throws IOException {
        CloseableHttpClient client = HttpClients.custom().setConnectionManager(cm).build();
        HttpGet httpGet = new HttpGet("https://www.cnblogs.com/roadlandscape/");

        // 设置请求参数
        RequestConfig requestConfig = RequestConfig.custom()
                .setConnectTimeout(1000) // 设置创建连接的最长时间
                .setConnectionRequestTimeout(500) // 设置获取连接的最长时间
                .setSocketTimeout(10 * 1000) // 设计数据传输的最长时间
                .build();
        httpGet.setConfig(requestConfig);

        CloseableHttpResponse response = client.execute(httpGet);
        if (response.getStatusLine().getStatusCode() == 200) {
            String context = EntityUtils.toString(response.getEntity(), "utf-8");
            System.out.println(context);
        }
    }
}

 

1.1.1. 编写代码