DSL和java调用

1 索引

1.1 创建索引

PUT logs_20210210

1.2 添加mapping

POST logs_20210210
{
  "properties": {
    "msg": {
      "type": "text",
      "fields": {
        "keyword": {
          "type": "keyword"
        }
      }
    },
    "cost": {
      "type": "long"
    },
    "level": {
      "type": "keyword"
    },
    "logger": {
      "type": "keyword"
    },
    "id": {
      "type": "keyword"
    },
    "threadNo": {
      "type": "float"
    },
    "day": {
      "type": "date",
      "format": "8uuuu-MM-dd"
    },
    "timestamp": {
      "type": "date"
    }
  }
}

1.3 插入数据:

POST log_20210128/doc
{
  "level": "INFO",
  "logger": "c.f.s.controller.IndexController",
  "id": "da579fb935524fe387ab6c9483c82bdc",
  "threadNo": 36966.33,
  "cost": 455,
  "msg": "This is INFO message",
  "day": "2020-01-15",
  "timestamp": "2021-01-15T01:23:23.332Z"
}

POST log_20210128/doc
{
  "level": "DEBUG",
  "logger": "c.f.s.controller.IndexService",
  "id": "0bb205917188487383ac5c310a377d00",
  "threadNo": 76966.33,
  "cost": 25,
  "msg": "这是一条日志",
  "day": "2020-01-16",
  "timestamp": "2021-01-16T11:23:23.300Z"
}

2 查询

2.1 基本查询

2.1.1 精确查询

据id查询,不进行分词。

POST logs_20210210/_search
{
    "query": {
        "term": {
            "id": {
                "value": "da579fb935524fe387ab6c9483c82bdc"
            }
        }
    }
}

多内容精确查询:

POST logs_20210210/_search
{
  "query": {
    "terms": {
      "id": [
        "da579fb935524fe387ab6c9483c82bdc",
        "0bb205917188487383ac5c310a377d00"
      ]
    }
  }
}

代码:

public void termQuery() {
	try {
		// 构建查询条件(注意:termQuery 支持多种格式查询,如 boolean、int、double、string 等,这里使用的是 string 的查询)
		SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
		searchSourceBuilder.query(QueryBuilders.termQuery("id", "da579fb935524fe387ab6c9483c82bdc"));
		// 创建查询请求对象,将查询对象配置到其中
		SearchRequest searchRequest = new SearchRequest("logs_20210210");
		searchRequest.source(searchSourceBuilder);
		// 执行查询,然后处理响应结果
		SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
		// 根据状态和数据条数验证是否返回了数据
		if (RestStatus.OK.equals(searchResponse.status()) && searchResponse.getHits().totalHits > 0) {
			SearchHits hits = searchResponse.getHits();
			for (SearchHit hit : hits) {
				// 将 JSON 转换成对象
				Log userInfo = JSON.parseObject(hit.getSourceAsString(), Log.class);
				// // 输出查询信息
				log.info(userInfo.toString());
			}
		}
	} catch (IOException e) {
		System.out.print(e);
	}
}

public void termsQuery() {
	try {
		// 构建查询条件(注意:termsQuery 支持多种格式查询,如 boolean、int、double、string 等,这里使用的是 string
		// 的查询)
		SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
		searchSourceBuilder.query(QueryBuilders.termsQuery("id", "da579fb935524fe387ab6c9483c82bdc",
				"0bb205917188487383ac5c310a377d00"));
		// 创建查询请求对象,将查询对象配置到其中
		SearchRequest searchRequest = new SearchRequest("logs_20210210");
		searchRequest.source(searchSourceBuilder);
		// 执行查询,然后处理响应结果
		SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
		// 根据状态和数据条数验证是否返回了数据
		if (RestStatus.OK.equals(searchResponse.status()) && searchResponse.getHits().totalHits > 0) {
			SearchHits hits = searchResponse.getHits();
			for (SearchHit hit : hits) {
				// 将 JSON 转换成对象
				Log userInfo = JSON.parseObject(hit.getSourceAsString(), Log.class);
				// 输出查询信息
				log.info(userInfo.toString());
			}
		}
	} catch (IOException e) {
		log.error("", e);
	}
}

2.1.2 匹配查询

查询所有符合条件的数据,并分页

POST logs_20210210/_search
{
  "query": {
    "match_all": {}
  },
  "from": 0,
  "size": 10,
  "sort": [
    {
      "cost": {
        "order": "asc"
      }
    }
  ]
}

匹配查询数据:

POST logs_20210210/_search
{
  "query": {
    "match": {
      "msg": "This is DEBUG message"
    }
  }
}

词语匹配查询
词语匹配进行查询,匹配 msg中为 君莫笑 的msg信息:

GET logs_20210210/_search
{
  "query": {
    "match_phrase": {
      "msg": "君莫笑"
    }
  }
}

内容多字段查询
查询在字段 logger,msg中存在 IndexController2 内容的信息:

GET logs_20210210/_search
{
  "query": {
    "multi_match": {
      "query": "IndexController2",
      "fields": ["logger","msg"]
    }
  }
}

代码:

public void matchAllQuery() {
    try {
        // 构建查询条件
        MatchAllQueryBuilder matchAllQueryBuilder = QueryBuilders.matchAllQuery();
        // 创建查询源构造器
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        searchSourceBuilder.query(matchAllQueryBuilder);
        // 设置分页
        searchSourceBuilder.from(0);
        searchSourceBuilder.size(3);
        // 设置排序
        searchSourceBuilder.sort("cost", SortOrder.ASC);
        // 创建查询请求对象,将查询对象配置到其中
        SearchRequest searchRequest = new SearchRequest("logs_20210210");
        searchRequest.source(searchSourceBuilder);
        // 执行查询,然后处理响应结果
        SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
        // 根据状态和数据条数验证是否返回了数据
        if (RestStatus.OK.equals(searchResponse.status()) && searchResponse.getHits().totalHits > 0) {
            SearchHits hits = searchResponse.getHits();
            for (SearchHit hit : hits) {
                // 将 JSON 转换成对象
                Log logEntity = JSON.parseObject(hit.getSourceAsString(), Log.class);
                // 输出查询信息
                log.info(logEntity.toString());
            }
        }
    } catch (IOException e) {
        log.error("", e);
    }
}

public void matchQuery() {
    try {
        // 构建查询条件
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        searchSourceBuilder.query(QueryBuilders.matchQuery("msg", "*连"));
        // 创建查询请求对象,将查询对象配置到其中
        SearchRequest searchRequest = new SearchRequest("logs_20210210");
        searchRequest.source(searchSourceBuilder);
        // 执行查询,然后处理响应结果
        SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
        // 根据状态和数据条数验证是否返回了数据
        if (RestStatus.OK.equals(searchResponse.status()) && searchResponse.getHits().totalHits > 0) {
            SearchHits hits = searchResponse.getHits();
            for (SearchHit hit : hits) {
                // 将 JSON 转换成对象
                Log logEntity = JSON.parseObject(hit.getSourceAsString(), Log.class);
                // 输出查询信息
                log.info(logEntity.toString());
            }
        }
    } catch (IOException e) {
        log.error("", e);
    }
}

public void matchPhraseQuery() {
    try {
        // 构建查询条件
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        searchSourceBuilder.query(QueryBuilders.matchPhraseQuery("msg", "醉里连"));
        // 创建查询请求对象,将查询对象配置到其中
        SearchRequest searchRequest = new SearchRequest("logs_20210210");
        searchRequest.source(searchSourceBuilder);
        // 执行查询,然后处理响应结果
        SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
        // 根据状态和数据条数验证是否返回了数据
        if (RestStatus.OK.equals(searchResponse.status()) && searchResponse.getHits().totalHits > 0) {
            SearchHits hits = searchResponse.getHits();
            for (SearchHit hit : hits) {
                // 将 JSON 转换成对象
                Log logEntity = JSON.parseObject(hit.getSourceAsString(), Log.class);
                // 输出查询信息
                log.info(logEntity.toString());
            }
        }
    } catch (IOException e) {
        log.error("", e);
    }
}

public void matchMultiQuery() {
    try {
        // 构建查询条件
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        searchSourceBuilder.query(QueryBuilders.multiMatchQuery("IndexController2", "logger", "msg"));
        // 创建查询请求对象,将查询对象配置到其中
        SearchRequest searchRequest = new SearchRequest("logs_20210210");
        searchRequest.source(searchSourceBuilder);
        // 执行查询,然后处理响应结果
        SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
        // 根据状态和数据条数验证是否返回了数据
        if (RestStatus.OK.equals(searchResponse.status()) && searchResponse.getHits().totalHits > 0) {
            SearchHits hits = searchResponse.getHits();
            for (SearchHit hit : hits) {
                // 将 JSON 转换成对象
            	Log  logEntity= JSON.parseObject(hit.getSourceAsString(), Log.class);
                // 输出查询信息
                log.info(logEntity.toString());
            }
        }
    } catch (IOException e) {
        log.error("", e);
    }
}

2.1.3 模糊查询

查询词中有部分错误的情况下也能召回正确的结果。匹配规则为莱温斯坦距离(Levenshtein distance)和莱文斯坦距离扩展版(Damerau–Levenshtein distance),对模糊查询的支持有两种方式match query和fuzzy query。莱温斯坦距离是编辑距离的一种,指两个字符串之间,由一个转成另一个所需的最少编辑操作次数。可以通过查询参数进行配置允许的规则。如下查询所有以level结尾的3 54。

GET logs_20210210/_search
{
  "query": {
    "fuzzy": {
      "level": "\\*3 54"
    }
  }
}
# 注意转义字符,*匹配多个,?匹配一个

代码:

public void fuzzyQuery() {
    try {
        // 构建查询条件
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        searchSourceBuilder.query(QueryBuilders.fuzzyQuery("level", "*54").fuzziness(Fuzziness.AUTO));
        // 创建查询请求对象,将查询对象配置到其中
        SearchRequest searchRequest = new SearchRequest("logs_20210210");
        searchRequest.source(searchSourceBuilder);
        // 执行查询,然后处理响应结果
        SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
        // 根据状态和数据条数验证是否返回了数据
        if (RestStatus.OK.equals(searchResponse.status()) && searchResponse.getHits().totalHits > 0) {
            SearchHits hits = searchResponse.getHits();
            for (SearchHit hit : hits) {
                // 将 JSON 转换成对象
                Log l = JSON.parseObject(hit.getSourceAsString(), Log.class);
                // 输出查询信息
                log.info(l.toString());
            }
        }
    } catch (IOException e) {
        log.error("", e);
    }
}

2.1.4 范围查询

1> Restful操作实例
查询cost ≥ 900 的数据:

GET log_20210128/_search
{
  "query": {
    "range": {
      "cost": {
        "gte": 900
      }
    }
  }
}

代码:

public void rangeQuery() {
	try {
		// 构建查询条件
		SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
		searchSourceBuilder.query(QueryBuilders.rangeQuery("cost").gte(900));
		// 创建查询请求对象,将查询对象配置到其中
		SearchRequest searchRequest = new SearchRequest("logs_20210210");
		searchRequest.source(searchSourceBuilder);
		// 执行查询,然后处理响应结果
		SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
		// 根据状态和数据条数验证是否返回了数据
		if (RestStatus.OK.equals(searchResponse.status()) && searchResponse.getHits().totalHits > 0) {
			SearchHits hits = searchResponse.getHits();
			for (SearchHit hit : hits) {
				// 将 JSON 转换成对象
				Log l = JSON.parseObject(hit.getSourceAsString(), Log.class);
				// 输出查询信息
				log.info(l.toString());
			}
		}
	} catch (IOException e) {
		log.error("", e);
	}
}

2.1.5 通配符查询

查询所有以 “message” 结尾的msg数据:

GET log_20210128/_search
{
  "query": {
    "wildcard": {
      "msg.keyword": {
        "value": "*message"
      }
    }
  }
}

代码:

public void wildcardQuery() {
	try {
		// 构建查询条件
		SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
		searchSourceBuilder.query(QueryBuilders.wildcardQuery("msg.keyword", "*message"));
		// 创建查询请求对象,将查询对象配置到其中
		SearchRequest searchRequest = new SearchRequest("logs_20210210");
		searchRequest.source(searchSourceBuilder);
		// 执行查询,然后处理响应结果
		SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
		// 根据状态和数据条数验证是否返回了数据
		if (RestStatus.OK.equals(searchResponse.status()) && searchResponse.getHits().totalHits > 0) {
			SearchHits hits = searchResponse.getHits();
			for (SearchHit hit : hits) {
				// 将 JSON 转换成对象
				Log l = JSON.parseObject(hit.getSourceAsString(), Log.class);
				// 输出查询信息
				log.info(l.toString());
			}
		}
	} catch (IOException e) {
		log.error("", e);
	}
} 

2.1.6 布尔查询

查询在 timestamp在2021-01-10~2021-01-17之间,且logger为c.f.s.controller.IndexController和c.f.s.controller.IndexService的数据:

POST log_20210128/_search
{
  "query": {
    "bool": {
      "filter": {
        "range": {
          "timestamp": {
            "format": "8uuuu-MM-dd",
            "gte": "2021-01-10",
            "lte": "2021-01-17"
          }
        }
      },
      "must": [
        {
          "terms": {
            "logger": [
              "c.f.s.controller.IndexController",
              "c.f.s.controller.IndexService"
            ]
          }
        }
      ]
    }
  }
}

-trim,match,match_phrase,boolean,wildcard,
代码:

public void boolQuery() {
	try {
		// 创建 Bool 查询构建器
		BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
		// 构建查询条件
		boolQueryBuilder
				.must(QueryBuilders.termsQuery("logger", "c.f.s.controller.IndexController",
						"c.f.s.controller.IndexService"))
				.filter().add(QueryBuilders.rangeQuery("timestamp").format("8uuuu-MM-dd").gte("2021-01-10")
						.lte("2021-01-17"));
		// 构建查询源构建器
		SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
		searchSourceBuilder.query(boolQueryBuilder);
		// 创建查询请求对象,将查询对象配置到其中
		SearchRequest searchRequest = new SearchRequest("logs_20210210");
		searchRequest.source(searchSourceBuilder);
		// 执行查询,然后处理响应结果
		SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
		// 根据状态和数据条数验证是否返回了数据
		if (RestStatus.OK.equals(searchResponse.status()) && searchResponse.getHits().totalHits > 0) {
			SearchHits hits = searchResponse.getHits();
			for (SearchHit hit : hits) {
				// 将 JSON 转换成对象
				Log l = JSON.parseObject(hit.getSourceAsString(), Log.class);
				// 输出查询信息
				log.info(l.toString());
			}
		}
	} catch (IOException e) {
		log.error("", e);
	}
}

2.2 聚合查询

2.2.1 metrics

满足某个条件的文档集合。分桶后会对文档进行某些有意义的指标计算。
demo:

# 计算平均耗时
{"size":0,"aggs":{"cost_avg":{"avg":{"field":"cost"}}}}

计算指标:stats(统计各项信息)、count(总个数)、sum、max、min、cardinality (求不重复的值有多少)、percentile(百分比)、extended_stats(拓展的统计各项信息)。
统计耗时总数、耗时最高值、耗时最低值、耗时平均、耗时总和:
{
  "size": 0,
  "aggs": {
    "cost_stats": {
      "stats": {
        "field": "cost"
      }
    }
  }
}
# 响应
{
  "took": 2325,
  "timed_out": false,
  "_shards": {
    "total": 5,
    "successful": 5,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": 2,
    "max_score": 0,
    "hits": []
  },
  "aggregations": {
    "cost_stats": {
      "count": 2,
      "min": 25,
      "max": 455,
      "avg": 240,
      "sum": 480
    }
  }
}

拓展的统计各项信息:

GET logs_20210210/_search
{"size":0,"aggs":{"cost_avg":{"extended_stats":{"field":"cost"}}}}
# 注意,size为0,不会返回文档信息,只返回聚合信息。
{
  "took": 4,
  "timed_out": false,
  "_shards": {
    "total": 5,
    "successful": 5,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": 2,
    "max_score": 0,
    "hits": []
  },
  "aggregations": {
    "cost_stats": {
      "count": 2,
      "min": 25,
      "max": 455,
      "avg": 240,
      "sum": 480,
      "sum_of_squares": 207650,
      "variance": 46225,
      "std_deviation": 215,
      "std_deviation_bounds": {
        "upper": 670,
        "lower": -190
      }
    }
  }
}

统计cost最低值:

POST log_20210128/_search
{
  "size": 0,
  "aggs": {
    "cost_min": {
      "min": {
        "field": "cost"
      }
    }
  }
}

统计cost最高值:

POST log_20210128/_search
{
  "size": 0,
  "aggs": {
    "cost_max": {
      "max": {
        "field": "cost"
      }
    }
  }
}

统计cost平均值:

POST log_20210128/_search
{
  "size": 0,
  "aggs": {
    "cost_avg": {
      "avg": {
        "field": "cost"
      }
    }
  }
}

统计cost总值:

POST log_20210128/_search
{
  "size": 0,
  "aggs": {
    "cost_sum": {
      "sum": {
        "field": "cost"
      }
    }
  }
}

统计cost总数:

POST log_20210128/_search
{
  "size": 0,
  "aggs": {
    "log_count": {
      "value_count": {
        "field": "scostalary"
      }
    }
  }
}

统计cost百分位:

POST log_20210128/_search
{
  "size": 0,
  "aggs": {
    "cost_percentiles": {
      "percentiles": {
        "field": "cost"
      }
    }
  }
}

代码:

public Object aggregationStats() {
    String responseResult = "";
    try {
        // 设置聚合条件
        AggregationBuilder aggr = AggregationBuilders.stats("cost_stats").field("cost");
        // 查询源构建器
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        searchSourceBuilder.aggregation(aggr);
        // 设置查询结果不返回,只返回聚合结果
        searchSourceBuilder.size(0);
        // 创建查询请求对象,将查询条件配置到其中
        SearchRequest request = new SearchRequest("logs_20210210");
        request.source(searchSourceBuilder);
        // 执行请求
        SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
        // 获取响应中的聚合信息
        Aggregations aggregations = response.getAggregations();
        // 输出内容
        if (RestStatus.OK.equals(response.status()) || aggregations != null) {
            // 转换为 Stats 对象
            ParsedStats aggregation = aggregations.get("cost_stats");
            log.info("-------------------------------------------");
            log.info("聚合信息:");
            log.info("count:{}", aggregation.getCount());
            log.info("avg:{}", aggregation.getAvg());
            log.info("max:{}", aggregation.getMax());
            log.info("min:{}", aggregation.getMin());
            log.info("sum:{}", aggregation.getSum());
            log.info("-------------------------------------------");
        }
        // 根据具体业务逻辑返回不同结果,这里为了方便直接将返回响应对象Json串
        responseResult = response.toString();
    } catch (IOException e) {
        log.error("", e);
    }
    return responseResult;
}

public Object aggregationMin() {
    String responseResult = "";
    try {
        // 设置聚合条件
        AggregationBuilder aggr = AggregationBuilders.min("cost_min").field("cost");
        // 查询源构建器
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        searchSourceBuilder.aggregation(aggr);
        searchSourceBuilder.size(0);
        // 创建查询请求对象,将查询条件配置到其中
        SearchRequest request = new SearchRequest("logs_20210210");
        request.source(searchSourceBuilder);
        // 执行请求
        SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
        // 获取响应中的聚合信息
        Aggregations aggregations = response.getAggregations();
        // 输出内容
        if (RestStatus.OK.equals(response.status()) || aggregations != null) {
            // 转换为 Min 对象
            ParsedMin aggregation = aggregations.get("cost_min");
            log.info("-------------------------------------------");
            log.info("聚合信息:");
            log.info("min:{}", aggregation.getValue());
            log.info("-------------------------------------------");
        }
        // 根据具体业务逻辑返回不同结果,这里为了方便直接将返回响应对象Json串
        responseResult = response.toString();
    } catch (IOException e) {
        log.error("", e);
    }
    return responseResult;
}

public Object aggregationMax() {
    String responseResult = "";
    try {
        // 设置聚合条件
        AggregationBuilder aggr = AggregationBuilders.max("cost_max").field("cost");
        // 查询源构建器
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        searchSourceBuilder.aggregation(aggr);
        searchSourceBuilder.size(0);
        // 创建查询请求对象,将查询条件配置到其中
        SearchRequest request = new SearchRequest("logs_20210210");
        request.source(searchSourceBuilder);
        // 执行请求
        SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
        // 获取响应中的聚合信息
        Aggregations aggregations = response.getAggregations();
        // 输出内容
        if (RestStatus.OK.equals(response.status()) || aggregations != null) {
            // 转换为 Max 对象
            ParsedMax aggregation = aggregations.get("cost_max");
            log.info("-------------------------------------------");
            log.info("聚合信息:");
            log.info("max:{}", aggregation.getValue());
            log.info("-------------------------------------------");
        }
        // 根据具体业务逻辑返回不同结果,这里为了方便直接将返回响应对象Json串
        responseResult = response.toString();
    } catch (IOException e) {
        log.error("", e);
    }
    return responseResult;
}

public Object aggregationAvg() {
    String responseResult = "";
    try {
        // 设置聚合条件
        AggregationBuilder aggr = AggregationBuilders.avg("cost_avg").field("cost");
        // 查询源构建器
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        searchSourceBuilder.aggregation(aggr);
        searchSourceBuilder.size(0);
        // 创建查询请求对象,将查询条件配置到其中
        SearchRequest request = new SearchRequest("logs_20210210");
        request.source(searchSourceBuilder);
        // 执行请求
        SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
        // 获取响应中的聚合信息
        Aggregations aggregations = response.getAggregations();
        // 输出内容
        if (RestStatus.OK.equals(response.status()) || aggregations != null) {
            // 转换为 Avg 对象
            ParsedAvg aggregation = aggregations.get("cost_avg");
            log.info("-------------------------------------------");
            log.info("聚合信息:");
            log.info("avg:{}", aggregation.getValue());
            log.info("-------------------------------------------");
        }
        // 根据具体业务逻辑返回不同结果,这里为了方便直接将返回响应对象Json串
        responseResult = response.toString();
    } catch (IOException e) {
        log.error("", e);
    }
    return responseResult;
}

public Object aggregationSum() {
    String responseResult = "";
    try {
        // 设置聚合条件
        SumAggregationBuilder aggr = AggregationBuilders.sum("cost_sum").field("cost");
        // 查询源构建器
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        searchSourceBuilder.aggregation(aggr);
        searchSourceBuilder.size(0);
        // 创建查询请求对象,将查询条件配置到其中
        SearchRequest request = new SearchRequest("logs_20210210");
        request.source(searchSourceBuilder);
        // 执行请求
        SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
        // 获取响应中的聚合信息
        Aggregations aggregations = response.getAggregations();
        // 输出内容
        if (RestStatus.OK.equals(response.status()) || aggregations != null) {
            // 转换为 Sum 对象
            ParsedSum aggregation = aggregations.get("cost_sum");
            log.info("-------------------------------------------");
            log.info("聚合信息:");
            log.info("sum:{}", String.valueOf((aggregation.getValue())));
            log.info("-------------------------------------------");
        }
        // 根据具体业务逻辑返回不同结果,这里为了方便直接将返回响应对象Json串
        responseResult = response.toString();
    } catch (IOException e) {
        log.error("", e);
    }
    return responseResult;
}

public Object aggregationCount() {
    String responseResult = "";
    try {
        // 设置聚合条件
        AggregationBuilder aggr = AggregationBuilders.count("cost_count").field("cost");
        // 查询源构建器
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        searchSourceBuilder.aggregation(aggr);
        searchSourceBuilder.size(0);
        // 创建查询请求对象,将查询条件配置到其中
        SearchRequest request = new SearchRequest("logs_20210210");
        request.source(searchSourceBuilder);
        // 执行请求
        SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
        // 获取响应中的聚合信息
        Aggregations aggregations = response.getAggregations();
        // 输出内容
        if (RestStatus.OK.equals(response.status()) || aggregations != null) {
            // 转换为 ValueCount 对象
            ParsedValueCount aggregation = aggregations.get("cost_count");
            log.info("-------------------------------------------");
            log.info("聚合信息:");
            log.info("count:{}", aggregation.getValue());
            log.info("-------------------------------------------");
        }
        // 根据具体业务逻辑返回不同结果,这里为了方便直接将返回响应对象Json串
        responseResult = response.toString();
    } catch (IOException e) {
        log.error("", e);
    }
    return responseResult;
}

public Object aggregationPercentiles() {
    String responseResult = "";
    try {
        // 设置聚合条件
        AggregationBuilder aggr = AggregationBuilders.percentiles("cost_percentiles").field("cost");
        // 查询源构建器
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        searchSourceBuilder.aggregation(aggr);
        searchSourceBuilder.size(0);
        // 创建查询请求对象,将查询条件配置到其中
        SearchRequest request = new SearchRequest("logs_20210210");
        request.source(searchSourceBuilder);
        // 执行请求
        SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
        // 获取响应中的聚合信息
        Aggregations aggregations = response.getAggregations();
        // 输出内容
        if (RestStatus.OK.equals(response.status()) || aggregations != null) {
            // 转换为 Percentiles 对象
            ParsedPercentiles aggregation = aggregations.get("cost_percentiles");
            log.info("-------------------------------------------");
            log.info("聚合信息:");
            for (Percentile percentile : aggregation) {
                log.info("百分位:{}:{}", percentile.getPercent(), percentile.getValue());
            }
            log.info("-------------------------------------------");
        }
        // 根据具体业务逻辑返回不同结果,这里为了方便直接将返回响应对象Json串
        responseResult = response.toString();
    } catch (IOException e) {
        log.error("", e);
    }
    return responseResult;
}

2.2.2 bucket

为某个桶中的文档计算得到的统计信息。类似mysql中的group by,会据条件对文档进行划分。
level共两类DEBUG和INFO,查询每一类的个数:

POST logs_20210210/_search
{"size":0,"aggs":{"terms_test":{"terms":{"field":"level"}}}}

# 返回结果:
{...
"aggregations": {
"terms_test": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "INFO",
"doc_count": 3774674
}
,
{
"key": "DEBUG",
"doc_count": 3771326
}
]
}
}
}
# doc_count_error_upper_bound:该值表示未进入最终术语列表的术语的最大潜在文档计数
# sum_other_doc_count:所有不属于响应的bucket的文档计数之和

聚合类别:ranges、date_range、histogram和date_histogram等。
按耗时范围进行聚合分桶,统计耗时在 0-500、500-1500 和 1500 以上的数据:

POST logs_20210210/_search
{
  "size": 0,
  "aggs": {
    "cost_range_bucket": {
      "range": {
        "field": "cost",
        "ranges": [
          {
            "key": "低耗时",
            "to": 500
          },
          {
            "key": "中间耗时",
            "from": 500,
            "to": 1500
          },
          {
            "key": "高耗时",
            "from": 1500
          }
        ]
      }
    }
  }
}

按照时间范围进行分桶,统计 2021-01-16~2021-01-18和 2021-01-18~2021-01-20产生的数据:

{
  "size": 0,
  "aggs": {
    "timestamp_range_bucket": {
      "date_range": {
        "field": "timestamp",
        "format": "8uuuu-MM-dd",
        "ranges": [
          {
            "key": "2021-01-18~2021-01-20",
            "from": "2021-01-16",
            "to": "2021-01-18"
          },
          {
            "key": "2021-01-18~2021-01-20",
            "from": "2021-01-18",
            "to": "2021-01-20"
          }
        ]
      }
    }
  }
}

按cost多少进行聚合分桶,设置统计的最小值为 0,最大值为 3000,区段间隔为 1000:

{
  "size": 0,
  "aggs": {
    "cost_histogram": {
      "histogram": {
        "field": "cost",
        "extended_bounds": {
          "min": 0,
          "max": 3000
        },
        "interval": 1000
      }
    }
  }
}

按day进行分桶:
{
  "size": 0,
  "aggs": {
    "day_histogram": {
      "date_histogram": {
        "format": "8uuuu-MM-dd",
        "field": "day",
        "interval": "day"
      }
    }
  }
}

代码:

public void aggrBucketTerms() {
	try {
		AggregationBuilder aggr = AggregationBuilders.terms("level_bucket").field("level");
		// 查询源构建器
		SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
		searchSourceBuilder.size(10);
		searchSourceBuilder.aggregation(aggr);
		// 创建查询请求对象,将查询条件配置到其中
		SearchRequest request = new SearchRequest("logs_20210210");
		request.source(searchSourceBuilder);
		// 执行请求
		SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
		// 获取响应中的聚合信息
		Aggregations aggregations = response.getAggregations();
		// 输出内容
		if (RestStatus.OK.equals(response.status())) {
			// 分桶
			Terms byCompanyAggregation = aggregations.get("level_bucket");
			List<? extends Terms.Bucket> buckets = byCompanyAggregation.getBuckets();
			// 输出各个桶的内容
			log.info("-------------------------------------------");
			log.info("聚合信息:");
			for (Terms.Bucket bucket : buckets) {
				log.info("桶名:{} | 总数:{}", bucket.getKeyAsString(), bucket.getDocCount());
			}
			log.info("-------------------------------------------");
		}
	} catch (IOException e) {
		log.error("", e);
	}
}
public void aggrBucketRange() {
	try {
		AggregationBuilder aggr = AggregationBuilders.range("cost_range_bucket").field("cost")
				.addUnboundedTo("低耗时", 500).addRange("中间耗时", 500, 1500).addUnboundedFrom("高耗时", 1500);
		// 查询源构建器
		SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
		searchSourceBuilder.size(0);
		searchSourceBuilder.aggregation(aggr);
		// 创建查询请求对象,将查询条件配置到其中
		SearchRequest request = new SearchRequest("logs_20210210");
		request.source(searchSourceBuilder);
		// 执行请求
		SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
		// 获取响应中的聚合信息
		Aggregations aggregations = response.getAggregations();
		// 输出内容
		if (RestStatus.OK.equals(response.status())) {
			// 分桶
			Range byCompanyAggregation = aggregations.get("cost_range_bucket");
			List<? extends Range.Bucket> buckets = byCompanyAggregation.getBuckets();
			// 输出各个桶的内容
			log.info("-------------------------------------------");
			log.info("聚合信息:");
			for (Range.Bucket bucket : buckets) {
				log.info("桶名:{} | 总数:{}", bucket.getKeyAsString(), bucket.getDocCount());
			}
			log.info("-------------------------------------------");
		}
	} catch (IOException e) {
		log.error("", e);
	}
}
public void aggrBucketDateRange() {
	try {
		AggregationBuilder aggr = AggregationBuilders.dateRange("timestamp_range_bucket").field("timestamp")
				.format("8uuuu-MM-dd").addRange("2021-01-01~2021-01-31", "2021-01-01", "2021-01-31")
				.addRange("2021-02-01~2021-02-28", "2021-02-01", "2021-02-28");
		// 查询源构建器
		SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
		searchSourceBuilder.size(0);
		searchSourceBuilder.aggregation(aggr);
		// 创建查询请求对象,将查询条件配置到其中
		SearchRequest request = new SearchRequest("logs_20210210");
		request.source(searchSourceBuilder);
		// 执行请求
		SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
		// 获取响应中的聚合信息
		Aggregations aggregations = response.getAggregations();
		// 输出内容
		if (RestStatus.OK.equals(response.status())) {
			// 分桶
			Range byCompanyAggregation = aggregations.get("timestamp_range_bucket");
			List<? extends Range.Bucket> buckets = byCompanyAggregation.getBuckets();
			// 输出各个桶的内容
			log.info("-------------------------------------------");
			log.info("聚合信息:");
			for (Range.Bucket bucket : buckets) {
				log.info("桶名:{} | 总数:{}", bucket.getKeyAsString(), bucket.getDocCount());
			}
			log.info("-------------------------------------------");
		}
	} catch (IOException e) {
		log.error("", e);
	}
}
public void aggrBucketHistogram() {
	try {
		AggregationBuilder aggr = AggregationBuilders.histogram("cost_histogram").field("cost")
				.extendedBounds(0, 3000).interval(1000);
		// 查询源构建器
		SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
		searchSourceBuilder.size(0);
		searchSourceBuilder.aggregation(aggr);
		// 创建查询请求对象,将查询条件配置到其中
		SearchRequest request = new SearchRequest("logs_20210210");
		request.source(searchSourceBuilder);
		// 执行请求
		SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
		// 获取响应中的聚合信息
		Aggregations aggregations = response.getAggregations();
		// 输出内容
		if (RestStatus.OK.equals(response.status())) {
			// 分桶
			Histogram byCompanyAggregation = aggregations.get("cost_histogram");
			List<? extends Histogram.Bucket> buckets = byCompanyAggregation.getBuckets();
			// 输出各个桶的内容
			log.info("-------------------------------------------");
			log.info("聚合信息:");
			for (Histogram.Bucket bucket : buckets) {
				log.info("桶名:{} | 总数:{}", bucket.getKeyAsString(), bucket.getDocCount());
			}
			log.info("-------------------------------------------");
		}
	} catch (IOException e) {
		log.error("", e);
	}
}
public void aggrBucketDateHistogram() {
	try {
		AggregationBuilder aggr = AggregationBuilders.dateHistogram("day_histogram").field("day").interval(1)
				.dateHistogramInterval(DateHistogramInterval.DAY).format("8uuuu-MM-dd");
		// 查询源构建器
		SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
		searchSourceBuilder.size(0);
		searchSourceBuilder.aggregation(aggr);
		// 创建查询请求对象,将查询条件配置到其中
		SearchRequest request = new SearchRequest("logs_20210210");
		request.source(searchSourceBuilder);
		// 执行请求
		SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
		// 获取响应中的聚合信息
		Aggregations aggregations = response.getAggregations();
		// 输出内容
		if (RestStatus.OK.equals(response.status())) {
			// 分桶
			Histogram byCompanyAggregation = aggregations.get("day_histogram");
			List<? extends Histogram.Bucket> buckets = byCompanyAggregation.getBuckets();
			// 输出各个桶的内容
			log.info("-------------------------------------------");
			log.info("聚合信息:");
			for (Histogram.Bucket bucket : buckets) {
				log.info("桶名:{} | 总数:{}", bucket.getKeyAsString(), bucket.getDocCount());
			}
			log.info("-------------------------------------------");
		}
	} catch (IOException e) {
		log.error("", e);
	}
}

2.2.3 metrics和bucket组合

一个聚合就是一些桶和指标的组合。桶中可以有多个嵌套的桶。
按照level分桶、然后统计每个level耗时的最高值:

{
  "size": 0,
  "aggs": {
    "cost_bucket": {
      "terms": {
        "field": "level",
        "size": "10"
      },
      "aggs": {
        "cost_max_log": {
          "top_hits": {
            "size": 1,
            "sort": [
              {
                "cost": {
                  "order": "desc"
                }
              }
            ]
          }
        }
      }
    }
  }
}
# top_hits聚合器可以有效地用于通过桶聚合器按特定字段对结果集进行分组。 一个或多个存储桶聚合器确定结果集被切入的属性。

代码:

public void aggregationTopHits() {
    try {
        AggregationBuilder testTop = AggregationBuilders.topHits("cost_max_user")
                .size(1)
                .sort("cost", SortOrder.DESC);
        AggregationBuilder levelBucket = AggregationBuilders.terms("level_bucket")
                .field("level")
                .size(10);
        levelBucket.subAggregation(testTop);
        // 查询源构建器
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        searchSourceBuilder.size(0);
        searchSourceBuilder.aggregation(levelBucket);
        // 创建查询请求对象,将查询条件配置到其中
        SearchRequest request = new SearchRequest("logs_20210210");
        request.source(searchSourceBuilder);
        // 执行请求
        SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
        // 获取响应中的聚合信息
        Aggregations aggregations = response.getAggregations();
        // 输出内容
        if (RestStatus.OK.equals(response.status())) {
            // 分桶
            Terms byCompanyAggregation = aggregations.get("level_bucket");
            List<? extends Terms.Bucket> buckets = byCompanyAggregation.getBuckets();
            // 输出各个桶的内容
            log.info("-------------------------------------------");
            log.info("聚合信息:");
            for (Terms.Bucket bucket : buckets) {
                log.info("桶名:{}", bucket.getKeyAsString());
                ParsedTopHits topHits = bucket.getAggregations().get("cost_max_user");
                for (SearchHit hit:topHits.getHits()){
                    log.info(hit.getSourceAsString());
                }
            }
            log.info("-------------------------------------------");
        }
    } catch (IOException e) {
        log.error("", e);
    }
}

3 删除

3.1 删除数据和表结构

3.1.1 删除索引

DELETE 索引名称

3.1.2 删除多个索引

DELETE 索引名称1,索引名称2

3.1.3 删除以pre开头的索引

DELETE pre*

3.1.4 删除全部索引,配置文件中禁止后将不能使用。

DELETE /* 
DELETE _all

3.2 删除数据不删除表结构

3.2.1 根据主键删除

DELETE 索引名称/文档名称/主键编号

3.2.2 根据匹配条件,删除数据

POST 索引名称/文档名称/_delete_by_query   
{
  "query":{
    "term":{
      "_id":100000100
    }
  }
}

3.2.3 删除所有数据,但是不删除表结构

POST /testindex/testtype/_delete_by_query?pretty
{
    "query": {
        "match_all": {
        }
    }
}

4 keyword和text区别:


text:会分词,然后进行索引,用于全文搜索。支持模糊、精确查询。不支持聚合。
keyword:不进行分词,直接索引。keyword用于关键词搜索、 支持模糊、精确查询。支持聚合。
动态映射:若不指定类型,则es字符串默认被映射成text和keyword类型。当成keyword查询是需要添加”.keyword“,即”level.keyword“。

{
    "level": {
        "type": "text",
        "fields": {
            "keyword": {
                "type": "keyword",
                "ignore_above": 256
            }
        }
    }
}

参考:https://blog.csdn.net/jkjiang123/article/details/106431222

上一篇:给表达式添加运算符


下一篇:P1093 [NOIP2007 普及组] 奖学金 AC代码及详解