HttpURLConnection获取页面数据

public class ZabbixParseDataServiceImpl implements ZabbixParseDataService {

    private static final CookieManager MANAGER;
    private static final String ZABBIX_FILE="";
    private static final String LOGIN_URL="http://jinrongzabbix.cnsuning.com/index.php";
    private static final String AGENT_STR = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36";

    private static final String DATA_URL="http://jinrongzabbix.cnsuning.com/latest.php?groupid=#{GROUPID}&hostid=0";

    private static final String GROUD_ID_STR = "groupId";

    private static final Logger log = LoggerFactory.getLogger(ZabbixParseDataServiceImpl.class);
    @Autowired
    private PaymentRedisUtilService paymentRedisUtilService;

    @Autowired
    private ZabbixParseInfoDao zabbixParseInfoDao;


    static {
        MANAGER = new CookieManager();
        MANAGER.setCookiePolicy(CookiePolicy.ACCEPT_ALL);
        CookieHandler.setDefault(MANAGER);
    }

    /**
     * 解析zabbix的数据
     * @return
     */
    @Override
   public void parseZabbixData(){
        log.info("zabbix获取数据开始。");
        String resultStr= null;
        try {
            Map<String,String> resultMap=getZabbixData();
            log.info("zabbix数据存redis中参数{}:"+JSONObject.toJSONString(resultMap));
            resultStr = paymentRedisUtilService.hmset(CommonConstants.REDIS_ZABBIX_KEY,resultMap);
        } catch (Exception e) {
            log.error("zabbix数据获取并存redis中出现异常。",e);
        }
        log.info("zabbix数据存redis中结果为:"+resultStr);
        log.info("zabbix获取数据结束。");
    }

    /**
     * zabbix获取一键付统计数据
     */
    @Override
    public void parserZabbixData4DisposablePay() {
        log.info("zabbix获取一键付统计数据开始");
        List<Map<String, Object>> resultList = null;
        try {
            resultList = getZabbixData4PayMonitor();
            if (null != resultList && !resultList.isEmpty()) {
                zabbixParseInfoDao.insertDisposablePayInfo(resultList);
            }
        } catch (Exception e) {
            log.error("zabbix一键付统计数据处理出现异常{}",e);
        }
        log.info("zabbix获取一键付统计数据结束,获取结果{}"+resultList);
    }

    /**
     * 获取zabbix一键付监控实时数据
     * @return
     */
    public List<Map<String, Object>> getZabbixData4PayMonitor() {
        List<Map<String, Object>> monitorInfoList = new ArrayList<>();
        //登录和获取查询数据url
        login("", "");
        String url = ScmProperty.get(ZABBIX_FILE, "");
        String ip = "";

        Map<String, Map<String, Object>> ipInfoMap = new HashMap<>();
        String dateStr = DateUtil.dateTime2String(new Date());
        Elements elements = getElements(url,".tableinfo .odd_row");
        //对获取页面进行数据提取
        for (int i = 0; i < elements.size(); i++) {
            Elements tds = elements.get(i).select("td");
            String ongIp = tds.get(1).select(".menu-host").text().trim();
            if (!"".equals(ongIp)) {
                ip = ongIp;
            }

            Map<String, Object> ipInfo = ipInfoMap.get(ip);
            if (null == ipInfo) {
                ipInfo = new HashMap<>();
                ipInfo.put("ip", ip);
                ipInfo.put("date", dateStr);
                ipInfo.put("sysName", "EPPSPGS");
                ipInfoMap.put(ip, ipInfo);
            }

            //跳过标识
            boolean continueFlag = false;
            if ("".equals(ongIp)) {
                String feild = tds.get(2).text().trim().split(" ")[0];
                if ("CPU空闲时间百分比".equals(feild)) {
                    Float rate = Float.parseFloat(tds.get(4).text().replace("%", ""));
                    ipInfo.put("cpuFreeRate", rate);
                    continueFlag = true;
                }

                if ("可用内存百分比".equals(feild)) {
                    Float rate = Float.parseFloat(tds.get(4).text().replace("%", ""));
                    ipInfo.put("memeryUseRate", rate);
                    continueFlag = true;
                }

                if (continueFlag) {
                    continue;
                }

                String[] sps = tds.get(2).text().trim().split(" ");
                if (sps.length == 1) {
                    feild = sps[0];
                } else {
                    feild = sps[0] + " " + sps[1];
                }

                if ("磁盘vda IO使用率".equals(feild)) {
                    Float rate = Float.parseFloat(tds.get(4).text().replace("%", ""));
                    ipInfo.put("ioUseRate", rate);
                }
            }
        }

        for (Map.Entry<String, Map<String, Object>> entry : ipInfoMap.entrySet()) {
            monitorInfoList.add(entry.getValue());
        }

        return monitorInfoList;
    }

    /**
     * 得到zabbix的数据
     * @return
     */
    public  Map<String,String> getZabbixData(){
        //账号登录
        login("", "");
        //删选页面的规则
        String select = ".tableinfo .odd_row";
        //创建一个返回对象
        Map<String,String> resultMap=new HashMap<>();
        //存放各个系统
        Map<String,Map<String,Object>> systemsMap=new HashMap<>();
        JSONObject zabbixJson;
        try {
            //获取scm上面的配置
            zabbixJson=JSONObject.parseObject(ScmProperty.get(ZABBIX_FILE,""));
        } catch (Exception e) {
            log.error("Scm获取zabbix.properties文件的键为ZABBIX失败",e);
            //获取失败new出一个对象
            zabbixJson=new JSONObject();
        }
        //循环scm上面配置  按照配置查询
        for(Map.Entry<String, Object> zabbixParam: zabbixJson.entrySet()){
            JSONObject fieldJson=(JSONObject)zabbixParam.getValue();
            //组合请求的url
            String url=DATA_URL.replace("#{GROUPID}",fieldJson.getString(GROUD_ID_STR));
            //获取应用名
            String applicationName=zabbixParam.getKey().trim();
            if(null==applicationName || applicationName.split("_").length<2){
                continue;
            }
            //分解系统应用名称获取系统名,应用名称
            String[] applicationNameArr=applicationName.split("_");
            //获取英文的系统名变成大写  当redis的key
            String systemNameUpper=applicationNameArr[0].toUpperCase();
            //获取应用的大写字母
            String applicationNameUpper=applicationNameArr[1].toUpperCase();
            if(!systemsMap.containsKey(systemNameUpper)){
                systemsMap.put(systemNameUpper,new HashMap<String,Object>());
            }
            systemsMap.get(systemNameUpper).put(applicationNameUpper,selectDocment(getElements(url,select),fieldJson));
        }
        //组合返回的对象
        for(Map.Entry<String,Map<String,Object>> systemMap:systemsMap.entrySet()){
            resultMap.put(systemMap.getKey(),JSONObject.toJSONString(systemMap.getValue()));
        }
        return resultMap;
    }

    /**
     * 登录zabbix
     * @param userKey
     * @param pwdKey
     */
    private void login(String userKey,String pwdKey){
        String loginUrl="";
        try {
            loginUrl=LOGIN_URL+"?request=&name="+ScmProperty.get(ZABBIX_FILE, userKey)+"&password="+ScmProperty.get(ZABBIX_FILE, pwdKey)+"&enter=Sign+in";
        } catch (Exception e) {
            log.error("Scm获取zabbix.properties文件的账号和密码失败。",e);
        }
        try {
            requestData(loginUrl,false,new Task<String>() {
                @Override
                public String doAction(HttpURLConnection conn) {
                    return conn.getHeaderField("Location");
                }
            });
        } catch (Exception e) {
            log.error("zabbix解析登录失败。",e);
        }
    }

    //获取页面条件符合的Elements
    private Elements getElements(String url, String select){
        try {
            String requestStr=requestData(url,true,new Task<String>() {
                @Override
                public String doAction(HttpURLConnection conn) throws IOException {
                    StringBuilder builder = new StringBuilder();
                    BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream(),"UTF-8"));
                    String line;
                    while ((line = reader.readLine()) != null){
                        builder.append(line);
                    }
                    reader.close();
                    return builder.toString();
                }
            });
            Document doc = Jsoup.parse(requestStr);
            return doc.select(select);
        } catch (IOException e) {
            log.error("zabbix获取数据失败。",e);
        }
        return new Elements();
    }

    //请求返回数据
    private <T> T requestData(String url,boolean idDoOutput,Task<T> task) throws IOException{
            URL serverUrl = new URL(url);
            HttpURLConnection conn = (HttpURLConnection) serverUrl.openConnection();
            // http正文内,因此需要设为true
            if(idDoOutput){
                conn.setDoOutput(true);
            }
            conn.setRequestMethod("GET");
            conn.setUseCaches(false);
            conn.addRequestProperty("Accept-Charset", "UTF-8;");
            conn.addRequestProperty("User-Agent", AGENT_STR);
            T t = task.doAction(conn);
            conn.connect();
            return t;
    }

    private interface Task<T> {
        /**
         * @param conn
         * @return
         */
        T doAction(HttpURLConnection conn) throws IOException ;
    }
    /**
     * @Description: 筛选  组装 成对象
     * @return List<travedParam> 返回类型
     * @author yangjing
     * @date 2017-2-25 下午4:33:29
     */
    private Map<String,Object> selectDocment(Elements elements,JSONObject fieldJson){
        //返回对象
        Map<String,Object> applicationMap= new HashMap<>();
        try {
            Map<String,PaymentMonitorDto> ipMap=new HashMap<>();
            applicationMap.put(GROUD_ID_STR,fieldJson.getString(GROUD_ID_STR));
            applicationMap.put("dataMap",ipMap);
            String ip = null;
            for (int i = 0; i < elements.size(); i++) {
                Elements tds=elements.get(i).select("td");
                //如果为3的话是ip
                if(tds.size()==3){
                    ip = tds.get(1).text().trim();
                    setIpMap(ipMap,ip);
                }
                //如果是7的话 是属性与值
                if(tds.size()==7){
                    String item = tds.get(2).text().trim();
                    String value = tds.get(4).text().trim();
                    setField(fieldJson,item,ipMap,ip,value);
                }
            }
        } catch (Exception e) {
            log.error("zabbix获取中页面筛选数据异常。groupId:"+fieldJson.getString(GROUD_ID_STR),e);
        }
        return applicationMap;
    }

    public void setIpMap(Map<String,PaymentMonitorDto> ipMap, String ip){
        if(!ipMap.containsKey(ip)){
            ipMap.put(ip,new PaymentMonitorDto());
        }
    }

    public void setField(JSONObject fieldJson,String item,Map<String,PaymentMonitorDto> ipMap, String ip,String value){
        if(fieldJson.containsKey(item)){
            setIpMap(ipMap,ip);
            setField(ipMap.get(ip),fieldJson.getString(item),value);
        }
    }

    private void setField(PaymentMonitorDto param,String field,String value){
        switch (field){
            case "server_cpuRate":param.setServer_cpuRate(value);
                break;
            case "server_IO":param.setServer_IO(value);
                break;
            case "server_diskRate":param.setServer_diskRate(value);
                break;
            case "server_memoryRate":param.setServer_memoryRate(value);
                break;
            case "app_connections":param.setApp_connections(value);
                break;
            case "app_cpuRate":param.setApp_cpuRate(value);
                break;
            case "app_memoryRate": param.setApp_memoryRate(value);
                break;
            default: break;
        }
    }
}
上一篇:java.net.HttpURLConnection


下一篇:linux中top查看cpu使用率超过100%