es开启数据压缩，es增加游标，searchAfter，范围查询方法

cf-framework/cf-common/src/main/java/com/cf/imes/framework/common/util/json/JsonUtils.java

@@ -10,16 +10,15 @@ import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
-import lombok.Data;
 import lombok.SneakyThrows;
-import lombok.ToString;
 import lombok.extern.slf4j.Slf4j;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.lang.reflect.Field;
 import java.lang.reflect.Type;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.List;
 import java.util.zip.DataFormatException;
 import java.util.zip.Deflater;
 import java.util.zip.Inflater;
@@ -271,181 +270,4 @@ public class JsonUtils {
     }
 
 
-
-
-
-    /*
-
-
-
-    * */
-    public static List<List<Object>> jsonTo2DArray(String jsonString) {
-        try {
-            ObjectMapper objectMapper = new ObjectMapper();
-            JsonNode jsonArray = objectMapper.readTree(jsonString);
-
-            Set<String> headers = new LinkedHashSet<>();
-            List<List<Object>> result = new ArrayList<>();
-
-            // 从数组中的所有对象收集标头
-            if (jsonArray.isArray()) {
-                for (JsonNode jsonObject : jsonArray) {
-                    Iterator<String> fieldNames = jsonObject.fieldNames();
-                    while (fieldNames.hasNext()) {
-                        headers.add(fieldNames.next());
-
-                    }
-                }
-            }
-            result.add(new ArrayList<>(headers));
-
-            // 提取行
-            for (JsonNode jsonObject : jsonArray) {
-                List<Object> row = new ArrayList<>();
-                for (String header : headers) {
-                    getRow(jsonObject, header, row);
-                }
-                result.add(row);
-            }
-
-            return result;
-        } catch (Exception e) {
-            e.printStackTrace();
-            return new ArrayList<>();
-        }
-    }
-
-    /**
-     * 提取行
-     *
-     * @param jsonObject
-     * @param header
-     * @param row
-     * @return
-     */
-    private static void getRow(JsonNode jsonObject, String header, List<Object> row) {
-        JsonNode valueNode = jsonObject.get(header);
-        if (valueNode != null) {
-            if (valueNode.isTextual()) {
-                row.add(valueNode.asText());
-            } else if (valueNode.isInt()) {
-                row.add(valueNode.asInt());
-            } else if (valueNode.isBoolean()) {
-                row.add(valueNode.asBoolean());
-            } else if (valueNode.isDouble()) {
-                row.add(valueNode.asDouble());
-            } else if (valueNode.isArray() || valueNode.isObject()) {
-                row.add(jsonTo2DArray(valueNode.toString()));
-            } else {
-                row.add(valueNode.asText());
-            }
-        } else {
-            row.add(null); // 处理缺少的值
-        }
-    }
-
-
-    // 抽取 json数据，不改变数据结构对应的值，只改变数据结构（参考 HPACK 算法） 实现json数据抽取
-    public static List<List<Object>> getData(Object object) {
-        String jsonString = toJsonString(object);
-
-        ObjectMapper mapper = new ObjectMapper();
-        try {
-            List<Map<String, Object>> list = mapper.readValue(jsonString, new TypeReference<>() {});
-            List<List<Object>> result = convertTo2DArray(list);
-
-            //
-            return result;
-
-        } catch (IOException e) {
-
-            e.printStackTrace();
-        }
-
-        return null;
-
-    }
-
-
-    private static List<List<Object>> convertTo2DArray(List<Map<String, Object>> list) {
-        List<List<Object>> result = new ArrayList<>();
-
-        // Get the headers from the first map
-        if (list.size() > 0) {
-            Map<String, Object> firstMap = list.get(0);
-            List<Object> headers = new ArrayList<>(firstMap.keySet());
-            result.add(headers);
-            //
-            // Add rows
-            for (Map<String, Object> map : list) {
-                //
-                List<Object> row = new ArrayList<>();
-                for (Object header : headers) {
-                    row.add(map.get(header));
-                }
-                result.add(row);
-            }
-        }
-
-        return result;
-    }
-
-
-
-    // 将所有抽取后的 json 数据都加入一个统一的数据类型中，方便前端解析哪些代码是抽取的数据，进行数据还原
-    @Data
-    @ToString(callSuper = true)
-    public static class ExtractedData {
-        List<List<Object>> extractedData;
-
-    }
-
-
-    public static List<List<List<Object>>> fetchData(Object obj) {
-        List<List<List<Object>>> data = new ArrayList<>();
-        try {
-            // 获取所有字段
-            Field[] fields = obj.getClass().getDeclaredFields();
-            for (Field field : fields) {
-                // 跳过静态或常量字段
-                if (java.lang.reflect.Modifier.isStatic(field.getModifiers()) ||
-                        java.lang.reflect.Modifier.isFinal(field.getModifiers())) {
-                    continue;
-                }
-
-                // 确保字段是可访问的
-                field.setAccessible(true);
-
-                // 获取字段值
-                Object fieldValue = field.get(obj);
-
-                // 如果字段值是一个对象，递归调用 fetchData
-                if (fieldValue != null && isCustomObject(fieldValue)) {
-                    data.add(JsonUtils.jsonTo2DArray(toJsonString(fieldValue)));
-                    fetchData(fieldValue);
-                }
-            }
-
-
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-        return data;
-    }
-
-    // 判断一个对象是否是自定义的类对象
-    private static boolean isCustomObject(Object obj) {
-        Class<?> clazz = obj.getClass();
-        // 自定义对象通常不会是 Java 的基本类型或标准库类型
-        // 这里可以根据具体需求进行更加复杂的判断
-        return !clazz.isPrimitive() &&
-                !clazz.getName().startsWith("java.") &&
-                !clazz.getName().startsWith("javax.") &&
-                !clazz.getName().startsWith("sun.") &&
-                !clazz.isAssignableFrom(Collection.class) &&
-                !clazz.isAssignableFrom(Map.class);
-    }
-
-
-
 }

cf-framework/cf-spring-boot-starter-elasticsearch/src/main/java/com/cf/imes/framework/es/config/ChenfengElasticsearchAutoConfiguration.java

@@ -20,6 +20,7 @@ import org.apache.http.conn.ssl.NoopHostnameVerifier;
 import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
 import org.apache.http.impl.nio.reactor.IOReactorConfig;
+import org.apache.http.message.BasicHeader;
 import org.apache.http.ssl.SSLContextBuilder;
 import org.apache.http.ssl.SSLContexts;
 import org.elasticsearch.client.RestClient;
@@ -43,6 +44,7 @@ import java.security.KeyStoreException;
 import java.security.NoSuchAlgorithmException;
 import java.security.cert.CertificateException;
 import java.time.Duration;
+import java.util.List;
 
 
 /**
@@ -140,6 +142,8 @@ public class ChenfengElasticsearchAutoConfiguration {
                             .setConnectTimeout((int) properties.getConnectionTimeout().toMillis())
                             .setSocketTimeout((int) properties.getSocketTimeout().toMillis())
                             .setConnectionRequestTimeout((int) properties.getConnectionRequestTimeout().toMillis())
+                            // 启用数据压缩
+                            .setContentCompressionEnabled(true)
                             .build());
             if (ObjectUtil.isNotNull(credentialsProvider)) {
                 httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
@@ -150,6 +154,7 @@ public class ChenfengElasticsearchAutoConfiguration {
             }
             // 手动开启keepalive
             httpAsyncClientBuilder.setDefaultIOReactorConfig(IOReactorConfig.custom().setSoKeepAlive(true).build());
+            httpAsyncClientBuilder.setDefaultHeaders(List.of(new BasicHeader("Accept-Encoding", "gzip")));
             // 手动设置保活时长
             httpAsyncClientBuilder.setKeepAliveStrategy(((response, context) -> Duration.ofMinutes(5).toMillis()));
             return httpAsyncClientBuilder;

cf-module-prod-executor/cf-module-prod-executor-biz/src/main/java/com/cf/imes/module/executor/enums/EsIndexEnum.java

@@ -0,0 +1,43 @@
+package com.cf.imes.module.executor.enums;
+
+
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+
+
+/**
+ * @author ES 索引类型
+ */
+
+@RequiredArgsConstructor
+@Getter
+public enum EsIndexEnum {
+
+
+    ORDER_PLATE_MODEL("生产单板材造型", "imes_order_plate_model"),
+
+    ORDER_OPTIMIZE_PLATE_MODEL("排单优化数据", "imes_order_optimize_plate_model"),
+
+    ORDER_PARTS_REMARK_MODEL("生产单配件备注", "imes_order_parts_remark_model"),
+
+    PLAN_PROCESS_SCHEME_OPTIMIZE_MODEL("排单的加工方案组对应的优化信息", "imes_plan_process_scheme_optimize_model"),
+
+    PLAN_ACTUAL_GOODS_MODEL("排单混单时选择的实际生产的大板信息", "imes_plan_actual_goods_model"),
+
+    PLAN_PROCESS_SCHEME_CONFIG("排单的加工方案组对应的配置信息", "imes_plan_process_scheme_config_model"),
+
+    ORDER_PLATE_COMPRESS_MODEL("生产单板材造型的压缩数据","imes_order_plate_compress_model");
+
+    /**
+     * 索引名称
+     */
+    private final String name;
+
+    /**
+     * 索引
+     */
+    private final String index;
+
+
+
+}

cf-module-prod-executor/cf-module-prod-executor-biz/src/main/java/com/cf/imes/module/executor/service/optimizeplan/OptimizePlanServiceImpl.java

@@ -80,6 +80,7 @@ import static com.cf.imes.framework.common.exception.util.ServiceExceptionUtil.e
 import static com.cf.imes.framework.common.util.json.JsonUtils.*;
 import static com.cf.imes.framework.security.core.util.SecurityFrameworkUtils.getUserOrganId;
 import static com.cf.imes.module.executor.enums.ErrorCodeConstants.*;
+import static com.cf.imes.module.executor.enums.EsIndexEnum.*;
 import static com.cf.imes.module.system.enums.ErrorCodeConstants.*;
 
 /**
@@ -966,14 +967,7 @@ public class OptimizePlanServiceImpl implements OptimizePlanService {
     @Override
     public List<OrderModelDO> getPlateModel(Long orderId) {
 
-        List<PlateDO> plateDOS = plateMapper.selectPlateNum(Collections.singletonList(orderId), getUserOrganId());
-
-        if(plateDOS.isEmpty()){
-            return new ArrayList<>();
-        }
-
-        return esUtils.getEsDocument(FIELD_ORDER_ID, orderId, plateDOS.size(), OptimizePlanService.ORDER_PLATE_MODEL, OrderModelDO.class);
-
+        return esUtils.getEsDocumentByScroll(FIELD_ORDER_ID,orderId,1000,ORDER_PLATE_MODEL.getIndex(),OrderModelDO.class);
 
     }
 

cf-module-prod-executor/cf-module-prod-executor-biz/src/main/java/com/cf/imes/module/executor/util/elasticsearch/EsUtils.java

@@ -5,17 +5,21 @@ import cn.hutool.core.collection.CollUtil;
 import co.elastic.clients.elasticsearch.ElasticsearchClient;
 import co.elastic.clients.elasticsearch._types.ElasticsearchException;
 import co.elastic.clients.elasticsearch._types.FieldValue;
+import co.elastic.clients.elasticsearch._types.SortOrder;
+import co.elastic.clients.elasticsearch._types.Time;
 import co.elastic.clients.elasticsearch._types.query_dsl.Query;
-import co.elastic.clients.elasticsearch.core.DeleteByQueryRequest;
-import co.elastic.clients.elasticsearch.core.SearchRequest;
-import co.elastic.clients.elasticsearch.core.SearchResponse;
+import co.elastic.clients.elasticsearch.core.*;
 import co.elastic.clients.elasticsearch.core.search.Hit;
 import co.elastic.clients.elasticsearch.indices.RefreshRequest;
+import co.elastic.clients.json.JsonData;
 import co.elastic.clients.json.JsonpMappingException;
 import com.cf.imes.framework.common.exception.ServiceException;
 import com.cf.imes.framework.common.util.json.JsonUtils;
 import com.cf.imes.framework.es.core.dal.ESDocument;
 import com.cf.imes.framework.es.core.service.ESDocumentService;
+import com.cf.imes.framework.security.core.LoginUser;
+import com.cf.imes.framework.security.core.util.SecurityFrameworkUtils;
+import com.cf.imes.module.executor.dal.dataobject.ordermodel.OrderModelDataTest;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Component;
 
@@ -118,7 +122,7 @@ public class EsUtils {
         SearchRequest.Builder builder = new SearchRequest.Builder();
         builder.index(index);
 
-        builder.query(q -> q.terms(b -> b.field(filedName).terms(t->t.value(fieldValues))));
+        builder.query(q -> q.bool(b->b.filter(f->f.terms(t->t.field(filedName).terms(te->te.value(fieldValues))))));
         try {
             SearchResponse<T> search = elasticsearchClient.search(builder.build(), targetClass);
             List<Hit<T>> hits = search.hits().hits();
@@ -143,7 +147,8 @@ public class EsUtils {
         builder.index(index);
         builder.size(size);
 
-        builder.query(q -> q.terms(b -> b.field(filedName).terms(t->t.value(fieldValues))));
+        builder.query(q-> q.bool(b-> b.filter(f-> f.terms(t-> t.field(filedName).terms(v->v.value(fieldValues))))));
+
         try {
             SearchResponse<T> search = elasticsearchClient.search(builder.build(), targetClass);
             List<Hit<T>> hits = search.hits().hits();
@@ -288,13 +293,190 @@ public class EsUtils {
 
 
 
+
+
+    // 范围查询（包括最大值和最小值），仅限于造型数据查询使用
+    public <T> List<T> getEsDocumentByRange(String filedName, Long value, String index, int minSortId, int maxSortId, Class<T> targetClass) {
+
+        SearchRequest.Builder builder = new SearchRequest.Builder();
+        builder.index(index);
+        int size = maxSortId - minSortId;
+        builder.size( size <= 0 ? 100 : size+2 );
+
+        builder.query(q->q.bool(b->b.filter(f->{
+            f.term(t->t.field(filedName).value(value));
+            f.range(r -> r.field("sortId").gte(JsonData.of(minSortId)).lte(JsonData.of(maxSortId)));
+            return f;
+        })));
+
+
+        try {
+            SearchResponse<T> search = elasticsearchClient.search(builder.build(), targetClass);
+            List<Hit<T>> hits = search.hits().hits();
+            if (CollUtil.isNotEmpty(hits)) {
+                return hits.stream().map(Hit::source).toList();
+            }
+            return new ArrayList<>();
+        }catch (JsonpMappingException e){
+            log.error(e.getMessage());
+            throw new ServiceException(ORDER_DATA_ERROR);
+        }catch (IOException | ElasticsearchException e) {
+            log.error(e.getMessage());
+            throw new ServiceException(DATA_DATA_ERROR);
+        }
+    }
+
+
+
+
+
+
+
+
+    // searchAfter 查询造型数据
+    public <T> List<T> getEsDocumentBySearchAfter(String filedName,
+                                                  Long value,
+                                                  String index,
+                                                  int size,
+                                                  int querySortId,
+                                                  Class<T> targetClass){
+
+        List<T> results = new ArrayList<>();
+
+        try {
+
+
+            SearchRequest.Builder searchBuilder = new SearchRequest.Builder()
+                    .index(index)
+                    .size(size)
+                    .sort(s -> s.field(f -> f.field("sortId").order(SortOrder.Asc)))
+                    .query(q -> q.bool(b -> b
+                            .filter(m -> m.term(t -> t.field(filedName).value(value)))
+                    ));
+
+            while (true) {
+
+                searchBuilder.searchAfter(FieldValue.of(querySortId));
+
+                SearchResponse<T> response = elasticsearchClient.search(searchBuilder.build(), targetClass);
+
+                List<Hit<T>> hits = response.hits().hits();
+                if (hits.isEmpty()) {
+                    break;
+                }
+
+                for (Hit<T> hit : hits) {
+                    results.add(hit.source());
+                }
+
+                querySortId += size;
+
+            }
+
+            return results;
+
+        } catch (JsonpMappingException e){
+            log.error(e.getMessage());
+            throw new ServiceException(ORDER_DATA_ERROR);
+        }catch (IOException | ElasticsearchException e) {
+            log.error(e.getMessage());
+            throw new ServiceException(DATA_DATA_ERROR);
+        }
+
+    }
+
+
+
+
+
+
+
+    // scroll 游标查询，目前仅用于查询造型数据
+    public <T> List<T> getEsDocumentByScroll(String filedName, Long value,Integer size, String index, Class<T> targetClass){
+
+        // scroll 保持游标有效时间
+        final String scrollTime = "1m";
+
+        List<T> result = new ArrayList<>();
+
+
+        String scrollId = null;
+
+        try {
+
+            // 执行第一次搜索，开启 scroll
+            SearchResponse<T> searchResponse = elasticsearchClient.search(s -> s
+                            .index(index)
+                            .scroll(Time.of(t -> t.time(scrollTime)))
+                            .size(size)
+                            .query(q->q.bool(b->b.filter(f->f.term(t->t.field(filedName).value(value))))),
+//                            .source(sou->sou.filter(f->f.includes("orderId","plateId","plateModelData","sortId"))),
+                    targetClass
+            );
+
+
+            scrollId = searchResponse.scrollId();
+            List<Hit<T>> hits = searchResponse.hits().hits();
+            hits.forEach(hit -> result.add(hit.source()));
+
+            // 循环 scroll 拉取剩余数据
+            while (!hits.isEmpty()) {
+                String finalScrollId = scrollId;
+                ScrollResponse<T> scrollResponse = elasticsearchClient.scroll(s -> s
+                        .scrollId(finalScrollId)
+                        .scroll(Time.of(t -> t.time(scrollTime))), targetClass);
+
+                scrollId = scrollResponse.scrollId();
+                hits = scrollResponse.hits().hits();
+
+                if (!hits.isEmpty()) {
+                    hits.forEach(hit -> result.add(hit.source()));
+                }
+            }
+
+            return result;
+
+        }catch (IOException | ElasticsearchException e){
+            log.error(e.getMessage());
+            throw new ServiceException(DATA_DATA_ERROR);
+        }finally {
+            // 清理 scroll
+            clearScroll(scrollId);
+        }
+
+    }
+
+
+
+
+
+
+    private void clearScroll(String scrollId) {
+        try {
+            if (scrollId != null && !scrollId.isEmpty()) {
+                elasticsearchClient.clearScroll(c -> c.scrollId(scrollId));
+            }
+        } catch (Exception e) {
+            log.warn("Failed to clear scroll: {}", e.getMessage());
+        }
+    }
+
+
+
+
+
+
+
+
+
+
     // ES 文档删除
     public void deleteEsDocument(String filedName,Long value, String index) {
 
         DeleteByQueryRequest.Builder builder = new DeleteByQueryRequest.Builder();
         builder.index(index);
 
-        builder.query(q -> q.term(b -> b.field(filedName).value(value)));
+        builder.query(q -> q.bool(b->b.filter(f->f.term(t->t.field(filedName).value(value)))));
 
         try {
             elasticsearchClient.deleteByQuery(builder.build());
@@ -351,7 +533,7 @@ public class EsUtils {
         DeleteByQueryRequest.Builder builder = new DeleteByQueryRequest.Builder();
         builder.index(index);
 
-        builder.query(q -> q.terms(b -> b.field(filedName).terms(e -> e.value(fieldValues))));
+        builder.query(q -> q.bool(b->b.filter(f->f.terms(t->t.field(filedName).terms(te->te.value(fieldValues))))));
 
         try {
             elasticsearchClient.deleteByQuery(builder.build());
@@ -384,6 +566,30 @@ public class EsUtils {
 
 
 
+    public void savePlateModelEs(String index,List<OrderModelDataTest > targetClass) {
+        try {
+
+            BulkRequest.Builder br = new BulkRequest.Builder();
+            LoginUser loginUser = SecurityFrameworkUtils.getLoginUser();
+            targetClass.forEach(esDocument -> {
+                esDocument.setOrganId(loginUser.getOrganId());
+                br.operations(op -> op.index(idx -> idx
+                        .index(index)
+                        .document(esDocument)));
+            });
+            elasticsearchClient.bulk(br.build());
+
+        } catch (IOException | ElasticsearchException e) {
+            e.printStackTrace();
+            log.error(e.getMessage());
+            throw new ServiceException(DATA_DATA_ERROR);
+        }
+
+    }
+
+
+
+
 
 
     // ES 文档数据刷新

script/es/imes_plan_process_scheme_config_model.index

@@ -0,0 +1,58 @@
+PUT imes_plan_process_scheme_config_model
+{
+  "settings": {},
+  "mappings": {
+    "properties": {
+      "planId": {
+        "type": "long"
+      },
+      "processId": {
+        "type": "long"
+      },
+      "planConfigId": {
+        "type": "long"
+      },
+      "planName": {
+        "type": "keyword"
+      },
+      "processLineConfig": {
+        "type": "keyword"
+      },
+      "processLineName": {
+        "type": "keyword"
+      },
+      "optimizationConfig": {
+        "type": "keyword"
+      },
+      "optimizationConfigName": {
+        "type": "keyword"
+      },
+      "analyzer": {
+        "type": "keyword"
+      },
+      "analyzerName": {
+        "type": "keyword"
+      },
+      "id": {
+        "type": "keyword"
+      },
+      "organId": {
+        "type": "long"
+      },
+      "updateTime": {
+        "type": "date",
+        "format": "strict_date_optional_time||yyyy-MM-dd HH:mm:ss||epoch_millis"
+      },
+      "updater": {
+        "type": "keyword"
+      },
+      "createTime": {
+        "type": "date",
+        "format": "strict_date_optional_time||yyyy-MM-dd HH:mm:ss||epoch_millis"
+      },
+      "creator": {
+        "type": "keyword"
+      }
+    }
+  }
+}