/**
 * 封装HttpClient，交给Spring容器管理
 */
@Component
public class HttpUtils {
    //httpclient连接池！
    private PoolingHttpClientConnectionManager clientConnectionManager;

    /**
     * 在构造方法中new一个
     */
    public HttpUtils() {
        this.clientConnectionManager = new PoolingHttpClientConnectionManager();
        //设置最大连接数
        this.clientConnectionManager.setMaxTotal(100);
        //设置每个主机的最大连接数
        this.clientConnectionManager.setDefaultMaxPerRoute(10);
    }

    /**
     * 使用get请求获得页面
     * @param url
     * @return 页面数据
     */
    public String doGetHtml(String url){
        //获取HttpClient对象
        CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(this.clientConnectionManager).build();
        //创建HttpGet请求对象，设置url地址
        HttpGet httpGet = new HttpGet(url);
        //设置请求信息
        httpGet.setConfig(this.getConfig());
        CloseableHttpResponse httpResponse=null;
        //使用HttpClient发起请求，获得相应
        try {
            httpResponse = httpClient.execute(httpGet);
            if(httpResponse.getStatusLine().getStatusCode()==200){
                //判断Entity是否为空，如果不为空就可以使用EntityUtils
                if(httpResponse.getEntity()!=null){
                    String content = EntityUtils.toString(httpResponse.getEntity(),"utf-8");
                    return content;
                }else{
                    return "ERROR";
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
        }finally {
            if(httpResponse!=null){
                try {
                    httpResponse.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
        //解析响应，返回结果
        return "";
    }

    /**
     * 设置RequestConfig
     * @return
     */
    private RequestConfig getConfig() {
        RequestConfig config=RequestConfig.custom()
                .setConnectTimeout(1000)    //创建链接的最长时间
                .setConnectionRequestTimeout(500)   //获取链接的最长时间
                .setSocketTimeout(10000)    //数据传输的最长时间
                .build();
        return config;
    }

    /**
     * 下载图片
     * @param url
     * @return 图片名称
     */
    public String doGetImage(String url){
//获取HttpClient对象
        CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(this.clientConnectionManager).build();
        //创建HttpGet请求对象，设置url地址
        HttpGet httpGet = new HttpGet(url);
        //设置请求信息
        httpGet.setConfig(this.getConfig());
        CloseableHttpResponse httpResponse=null;
        //使用HttpClient发起请求，获得相应
        try {
            httpResponse = httpClient.execute(httpGet);
            if(httpResponse.getStatusLine().getStatusCode()==200){
                //判断Entity是否为空，如果不为空就可以使用EntityUtils
                if(httpResponse.getEntity()!=null){
                    //下载图片
                    //获取图片后缀
                    String exName=url.substring(url.lastIndexOf("."));
                    //创建图片名，重命名图片
                    String picName= UUID.randomUUID().toString()+exName;
                    //下载图片
                    OutputStream outputStream=new FileOutputStream(new File("G:/IJDailyCode/Crawler/src/main/resources/downloadImg/"+picName));
                    httpResponse.getEntity().writeTo(outputStream);
                    //返回图片名称
                    return picName;
                }else{
                    return "ERROR";
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
        }finally {
            if(httpResponse!=null){
                try {
                    httpResponse.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
        //出现其他问题返回结果
        return "";
    }

@Component
public class GetTask {
    @Autowired
    HttpUtils httpUtils;
    @Autowired
    ProductService productService;

    @Scheduled(fixedDelay = 100*1000)
    public void crawlerMain(){
        String mainUrl="https://zhongchou.modian.com/all/top_time/going/";
        /*页码for循环*/
        for(int page=1;page<=8;++page){
            String tempUrl=mainUrl+page;
            String html = httpUtils.doGetHtml(tempUrl);
            /*解析页面，获取商品数据*/
            this.parse(html);
        }
    }

    private void parse(String html) {
        Document doc = Jsoup.parse(html);
        Elements proElms = doc.select("div.pro_field > ul > li");
        for (Element proElm:proElms) {
            /**
             * 乱七八糟的搜寻匹配项
             */
            String proId = proElm.attr("data-pro-id");
            String proAuthor=proElm.select("div.author > a > p").text();
            // 重复的使用选择器找信息。。。。。。。。。。。。。
            String imgUrl=infoDoc.getElementById("big_logo").attr("src");

            //如果实体的属性是null,它就会忽略它,这里只传一个proId参数就好
            Product proExample=new Product();
            proExample.setProid(proId);
            
            //查询并判断数据是否存在
            List<Product> examples = productService.findAll(proExample);
            System.out.println("list有无数据:"+examples.size());
            if(examples.size()>0){
                System.out.println("===数据已存在===");
                continue;
            }
            
            /*绑定数值*/
            proExample.setProauthor(proAuthor);
            // 重复的数值绑定操作。。。。。。。。。
            proExample.setProstatus(proStatus);
            /*下载图片*/
            String proImgPath=httpUtils.doGetImage(imgUrl);
            proExample.setProimgpath(proImgPath);
            /*提交保存*/
            productService.save(proExample);
        }
    }
}    

@Service
@Transactional
public class ProductServiceImpl implements ProductService {
    @Autowired
    private ProductDao productDao;
    @Override
    public void save(Product product) {
        this.productDao.save(product);
    }

    @Override
    public List<Product> findAll(Product product) {
        Example<Product> example = Example.of(product);
        List<Product> products = productDao.findAll(example);
        return products;
    }
}