ElasticSearch7.4.2：RestHighLevelClient
这次项目应客户要求使用7.4.2 highLevel-client版本，由于之前做的es搜索还是使用SpringData+Transport来操作,所以这次也是看了好久的官方api以及好多大神的笔记，但是由于是版本太高，使用的人可能或许太少或许大神是没时间写笔记记录，所以做的过程中也遇到了好多的问题和踩了好多的坑，所以现在记录一下自己做的过程，一方面希望能给大家提供一些帮助，一方面也算是对这方面的回顾，也希望能有大神提供一些建议和批评，互相讨论，共同进步。
一：pom依赖
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>7.4.0</version>
</dependency>
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>7.4.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/commons-logging/commons-logging -->
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>1.2</version>
</dependency>
<!-- https://mvnrepository.com/artifact/commons-codec/commons-codec -->
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.13</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpcore -->
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
<version>4.4.12</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient -->
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.10</version>
</dependency>
二：后台创建index，mapping和同步数据
一：controller层
@RestController
public class EsController {
@Autowired
EsSearchService esSearchService;
@PostMapping("checkIndex")
//检查该index是否存在
public boolean checkIndex(String index){
return esSearchService.checkIndex(index);
}
@PostMapping("createIndex")
//创建index，可以自己手动输入创建，也可以写在代码中，我是直接写在代码中里，一方面是懒，也方便管理
public String createIndex(){
return esSearchService.createIndex();
}
@PostMapping("createMapping")
//创建mapping，可以用dts也可以自己写，我是自己用API写的，API创建的需要注意数据类型
public String createMapping(){
return esSearchService.createMapping();
}
@PostMapping("saveById")
//通过id新增
public int synchroEsDatas(@RequestBody List<EsSearchDTO> es){
return esSearchService.synchroEsDatas(es);
}
@PostMapping("saveAll")
//批量新增
public int save(){
return esSearchService.synchroEsDatasAll();
}
二：实现类
public class EsSearchServiceImp implements EsSearchService{
//如果是用的阿里云的Es服务器，可以在这里注入
@Value("${aliyun.es.username}")
private String username;
@Value("${aliyun.es.password}")
private String password;
@Value("${aliyun.es.hostname}")
private String hostname;
@Value("${aliyun.es.port}")
private int port;
@Value("${aliyun.es.protocol}")
private String protocol;
//获取es连接（不需要密码可以直接连接）
//private RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient
// .builder(new HttpHost(hostname, port, protocol)));
//使用密码连接（阿里云给的API）
public RestHighLevelClient getRestClient(){
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY,new UsernamePasswordCredentials(username, password));
RestClientBuilder builder = RestClient.builder(new HttpHost(hostname, port, protocol))
.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
return httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
}
}).setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
@Override
public RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestBuilder) {
requestBuilder.setConnectTimeout(5000);
requestBuilder.setSocketTimeout(40000);
requestBuilder.setConnectionRequestTimeout(1000);
return requestBuilder;
}
});
RestHighLevelClient restHighLevelClient = new RestHighLevelClient(builder);
return restHighLevelClient;
}
@Resource
private EsSearchMapper esSearchMapper;
// @Async("taskExecutor") 加线程池自动同步
//异步同步所有数据
@Override
public Integer synchroEsDatasAll(){
int syncount = 0;
try {
//异步执行延迟5秒
Thread.sleep(5000);
//获取需要同步的记录
List<EsSearchEntity> list = esSearchMapper.selectESObject(null);
if(list != null && list.size() >0 && !list.isEmpty()){
log.info("ES同步开始");
//同步ES
syncount = this.saveDocumentByIndex(list);
log.info("ES同步完成");
} else {
log.info("未获取到ES同步记录集");
}
} catch (Exception e){
e.printStackTrace();
log.error("ES同步数据异常");
}
return syncount;
}
//删除数据
public void deleteUtil(String id) throws IOException {
Map<String,Object> _Map=new HashMap<String,Object>();
_Map.put("id", id);
DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(IndexData.indexName);
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
for (Map.Entry<String,Object> id:_Map.entrySet()){
boolQueryBuilder.must(QueryBuilders.termQuery(id.getKey(),id.getValue()));
}
deleteByQueryRequest.setQuery(boolQueryBuilder);
BulkByScrollResponse bulkByScrollResponse = getRestClient().deleteByQuery(deleteByQueryRequest,RequestOptions.DEFAULT);
bulkByScrollResponse.getDeleted();
}
@Override
//判断index是否存在
public boolean checkIndex(String index) {
try {
GetIndexRequest request = new GetIndexRequest(index);
boolean result = getRestClient().indices().exists(request, RequestOptions.DEFAULT);
return result;
} catch (IOException e) {
e.printStackTrace();
return false;
}
}
@Override
//创建index
public String createIndex() {
String result = "创建成功";
CreateIndexRequest createIndexRequest = new CreateIndexRequest(IndexData.indexName);
try {
CreateIndexResponse createIndexResponse = getRestClient().indices().create(createIndexRequest,RequestOptions.DEFAULT);
if (!createIndexResponse.isAcknowledged()){
result = "创建失败";
}else{
result = "索引已经存在";
}
} catch (IOException e) {
e.printStackTrace();
result = "接口异常";
}
return result;
}
@Override
//创建mapping
//注意数据格式，此版本已经取去除String格式，改为text和keyword格式，其中text格式支持分词和建立索引，支持模糊查询和精确查询，不支持聚合，keyword不支持分词，支持模糊查询和精确查询，支持聚合查询，排序
public String createMapping() {
String result = "mapping创建成功";
PutMappingRequest putMappingRequest = new PutMappingRequest(IndexData.indexName);
XContentBuilder builder = null;
try {
builder = XContentFactory.jsonBuilder()
.startObject()
.startObject("properties")
.startObject("id")
.field("type","keyword")
.field("index",true)
.endObject()
.startObject("pics")
.field("type","text")
.field("index",false)
.endObject()
.startObject("name")
.field("type","text")
.field("index",true)
//分词器采用ik_smart分词器
.field("analyzer","ik_smart")
.endObject()
.startObject("prices")
.field("type","double")
.field("index",true)
.endObject()
//可以按照城市排序，需要在其中再套一层，并且格式为keyword
.startObject("city")
.field("type","text")
.startObject("fields")
.startObject("raw")
.field("type","keyword")
.endObject()
.endObject()
.endObject()
//支持指定时间格式
.startObject("createTime")
.field("type","date")
.field("format","yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis")
.endObject()
.endObject()
.endObject();
} catch (IOException e) {
e.printStackTrace();
}
putMappingRequest.source(builder);
try {
AcknowledgedResponse putMappingResponse = getRestClient().indices().putMapping(putMappingRequest, RequestOptions.DEFAULT);
System.out.println(putMappingResponse);
if (!putMappingResponse.isAcknowledged()){
result = "接口执行失败";
}else {
result = "mapping“”已经存在";
}
} catch (IOException e) {
result = "mapping创建接口异常";
}
return result;
}
@Override
//将查询出的数据通过实体类映射到es中
//注意：字段映射必须和es中的一致，可以直接映射，也可以转换为json映射
public int saveDocumentByIndex(List<EsSearchEntity> esSearchList) {
BulkRequest bulkRequest = new BulkRequest();
Map<String,Object> map = new HashMap<>();
int count = 0;
for (EsSearchEntity esSearch:esSearchList){
String json = JSON.toJSONString(esSearch);
JSONObject jsonObject = JSONObject.parseObject(json);
Date date = new Date();
map.put("id",jsonObject.getString("id"));
map.put("pics",jsonObject.getString("pics"));
map.put("name",jsonObject.getString("name"));
map.put("prices",jsonObject.getBigDecimal("prices"));
map.put("city",jsonObject.getString("city"));
//时间采用当前时间
map.put("createTime",date);
bulkRequest.add(new IndexRequest(IndexData.indexName,IndexData.type,esSearch.getBid()).source(map));
bulkRequest.timeout("10m");
try {
BulkResponse bulkResponse = getRestClient().bulk(bulkRequest, RequestOptions.DEFAULT);
count ++;
} catch (IOException e) {
log.info(esSearch.getBid() + "同步失败");
e.printStackTrace();
}
}
log.info("es同步完成,更新总记录数:" + count);
return count;
}
}
二：前台执行查询、分页、高亮
一：controller层
@RestController
@RequestMapping("es")
@Api(tags = "ES搜索")
public class ESController {
@Autowired
EsSearchService esSearchService;
@PostMapping("search")
//queryData中为查询的字段、排序字段
//pageNum，pageSize是分页
public Map<String,List<Map<String, Object>>> search( QueryData queryData,int pageNum,int pageSize){
return esSearchService.searchProductByQuery(queryData,pageNum,pageSize);
}
}
二：实现类
@Override
public Map<String, List<Map<String, Object>>> searchProductByQuery(QueryData queryData,int pageNum,int pageSize) {
//searchRequest 是用于查询
SearchRequest searchRequest = new SearchRequest();
//searchRequest1 用于统计查询的总数，由于Es中设置了from和size之后就无法查询出总数，所以就加了一次查询，如果大家有好的办法请务必告知于我，非常感谢！！！！
SearchRequest searchRequest1 = new SearchRequest();
//现在的版本已经不支持指定type啦，可以去掉，因为现在的type都是_doc
searchRequest.indices(IndexData.indexName).types(IndexData.type);
//将searchSourceBuilder加入到searchRequest中
//由于好多的查询都用到了查询条件，所以我i直接将sourceBuilder进行了封装，用的时候可以直接调用，避免代码冗杂
searchRequest.source(sourceBuilder(queryData,pageNum,pageSize));
searchRequest1.source(sourceBuilderTotal(queryData));
//现在的Es的查询返回值都是map集合，需要用map来接
List<Map<String, Object>> listTotal = searchResult(searchRequest1);
同步执行
List<Map<String, Object>> list = searchResult(searchRequest);
Map<String, List<Map<String, Object>>> listMap = new HashMap<>();
//查询出来的值
listMap.put("List", List);
//查询出的总数
listMap.put("total",listTotal.size);
return listMap;
}
封装的sourceBuilder
//用于查询
public SearchSourceBuilder sourceBuilder(QueryData queryData,int pageNum,int pageSize) {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
//开始搜索的位置
searchSourceBuilder.from((pageNum-1)*pageSize);
//搜索结果的数量大小
searchSourceBuilder.size(pageSize);
//设置超时时间
searchSourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
//按照名称查询
MatchQueryBuilder matchQueryBuilder = QueryBuilders.matchQuery("name", queryData.getQueryName());
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
//嵌套查询，满足其一即可
boolQueryBuilder.should(matchQueryBuilder);
//按照价格区间排序
if (null != queryData.getMinPrices() && null != queryData.getMaxPrices()) {
RangeQueryBuilder queryBuilder = QueryBuilders
.rangeQuery("prices")
.from(queryData.getMinPrices())
.to(queryData.getMaxPrices());
searchSourceBuilder.query(queryBuilder);
}
if (StringUtils.equals("asc", queryData.getAsc())) {
//按照价格升序
searchSourceBuilder.sort(new FieldSortBuilder("prices")
.order(SortOrder.ASC));
} else if (StringUtils.equals("desc", queryData.getDesc())) {
//按照价格降序
searchSourceBuilder.sort(new FieldSortBuilder("prices")
.order(SortOrder.DESC));
} else if (StringUtils.isNotBlank(queryData.getSite())) {
//按照地域排序
searchSourceBuilder.sort(new FieldSortBuilder("city.raw")
.order(SortOrder.ASC));
}
//此为高亮字段，作为封装类方法在这里被引用
searchSourceBuilder.highlighter(highlight());
//需要查询出来的字段
String[] includeFields = new String[]{"id", "name", "city", "pics", "createTime"};
//不需要的字段
String[] excludeFields = new String[]{""};
searchSourceBuilder.fetchSource(includeFields, excludeFields);
searchSourceBuilder.query(boolQueryBuilder);
return searchSourceBuilder;
}
//获取总页数
public SearchSourceBuilder sourceBuilderTotal(QueryData queryData) {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
//开始搜索的位置
searchSourceBuilder.from(0);
//搜索结果的数量大小
searchSourceBuilder.size(10000);
//按照商品/服务/解决方案名称查询
MatchQueryBuilder matchQueryBuilder = QueryBuilders.matchQuery("pname", queryData.getQueryName());
uilder boolQueryBuilder = QueryBuilders.boolQuery();
//嵌套查询，满足其一即可
boolQueryBuilder.should(matchQueryBuilder);
searchSourceBuilder.query(boolQueryBuilder);
return searchSourceBuilder;
}
封装的高亮字段
public HighlightBuilder highlight() {
//设置高亮
HighlightBuilder highlightBuilder = new HighlightBuilder();
//设置高亮的前缀和后缀
String preTags = "<em class='titleColor' style='color: #f60'>";
String postTags = "</em>";
highlightBuilder.preTags(preTags);
highlightBuilder.postTags(postTags);
HighlightBuilder.Field pname = new HighlightBuilder.Field("name");
highlightBuilder.field(name);
return highlightBuilder;
}
重点来了！！！查询的结果集！！
public List<Map<String, Object>> searchResult(SearchRequest searchRequest) {
List<Map<String, Object>> list = new ArrayList<>();
List<String> fieldList = new ArrayList();
//需要高亮的字段
fieldList.add("pname");
try {
SearchResponse searchResponse = getRestClient().search(searchRequest, RequestOptions.DEFAULT);
RestStatus status = searchResponse.status();
log.info("状态：" + status);
TimeValue took = searchResponse.getTook();
log.info("执行时间：" + took);
Boolean terminatedEarly = searchResponse.isTerminatedEarly();
log.info("请求终止：" + terminatedEarly);
boolean timedOut = searchResponse.isTimedOut();
log.info("超时时间：" + timedOut);
//获得响应的文档
SearchHits hits = searchResponse.getHits();
//迭代取出数据
for (SearchHit hit : hits) {
Map<String, Object> sourceAsMap = hit.getSourceAsMap();
for (String field : fieldList()) {
String keyValue = (String) sourceAsMap.get(field);
HighlightField highlightFieldValue = hit.getHighlightFields().get(field);
if (highlightFieldValue == null) {
sourceAsMap.put(field, keyValue);
} else {
String highLightContent = highlightFieldValue.getFragments()[0].toString();
sourceAsMap.put(field, highLightContent);
}
}
list.add(sourceAsMap);
}
} catch (IOException e) {
e.printStackTrace();
}
return list;
}