diff --git a/cf-framework/cf-common/src/main/java/com/cf/imes/framework/common/util/json/JsonUtils.java b/cf-framework/cf-common/src/main/java/com/cf/imes/framework/common/util/json/JsonUtils.java index 56f8418f4..436a6de90 100644 --- a/cf-framework/cf-common/src/main/java/com/cf/imes/framework/common/util/json/JsonUtils.java +++ b/cf-framework/cf-common/src/main/java/com/cf/imes/framework/common/util/json/JsonUtils.java @@ -10,16 +10,15 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import lombok.Data; import lombok.SneakyThrows; -import lombok.ToString; import lombok.extern.slf4j.Slf4j; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.lang.reflect.Field; import java.lang.reflect.Type; -import java.util.*; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; import java.util.zip.DataFormatException; import java.util.zip.Deflater; import java.util.zip.Inflater; @@ -271,181 +270,4 @@ public class JsonUtils { } - - - - /* - - - - * */ - public static List> jsonTo2DArray(String jsonString) { - try { - ObjectMapper objectMapper = new ObjectMapper(); - JsonNode jsonArray = objectMapper.readTree(jsonString); - - Set headers = new LinkedHashSet<>(); - List> result = new ArrayList<>(); - - // 从数组中的所有对象收集标头 - if (jsonArray.isArray()) { - for (JsonNode jsonObject : jsonArray) { - Iterator fieldNames = jsonObject.fieldNames(); - while (fieldNames.hasNext()) { - headers.add(fieldNames.next()); - - } - } - } - result.add(new ArrayList<>(headers)); - - // 提取行 - for (JsonNode jsonObject : jsonArray) { - List row = new ArrayList<>(); - for (String header : headers) { - getRow(jsonObject, header, row); - } - result.add(row); - } - - return result; - } catch (Exception e) { - e.printStackTrace(); - return new ArrayList<>(); - } - } - - /** - * 提取行 - * - * @param jsonObject - * @param header - * @param row - * @return - */ - private static void getRow(JsonNode jsonObject, String header, List row) { - JsonNode valueNode = jsonObject.get(header); - if (valueNode != null) { - if (valueNode.isTextual()) { - row.add(valueNode.asText()); - } else if (valueNode.isInt()) { - row.add(valueNode.asInt()); - } else if (valueNode.isBoolean()) { - row.add(valueNode.asBoolean()); - } else if (valueNode.isDouble()) { - row.add(valueNode.asDouble()); - } else if (valueNode.isArray() || valueNode.isObject()) { - row.add(jsonTo2DArray(valueNode.toString())); - } else { - row.add(valueNode.asText()); - } - } else { - row.add(null); // 处理缺少的值 - } - } - - - // 抽取 json数据，不改变数据结构对应的值，只改变数据结构（参考 HPACK 算法） 实现json数据抽取 - public static List> getData(Object object) { - String jsonString = toJsonString(object); - - ObjectMapper mapper = new ObjectMapper(); - try { - List> list = mapper.readValue(jsonString, new TypeReference<>() {}); - List> result = convertTo2DArray(list); - - // - return result; - - } catch (IOException e) { - - e.printStackTrace(); - } - - return null; - - } - - - private static List> convertTo2DArray(List> list) { - List> result = new ArrayList<>(); - - // Get the headers from the first map - if (list.size() > 0) { - Map firstMap = list.get(0); - List headers = new ArrayList<>(firstMap.keySet()); - result.add(headers); - // - // Add rows - for (Map map : list) { - // - List row = new ArrayList<>(); - for (Object header : headers) { - row.add(map.get(header)); - } - result.add(row); - } - } - - return result; - } - - - - // 将所有抽取后的 json 数据都加入一个统一的数据类型中，方便前端解析哪些代码是抽取的数据，进行数据还原 - @Data - @ToString(callSuper = true) - public static class ExtractedData { - List> extractedData; - - } - - - public static List>> fetchData(Object obj) { - List>> data = new ArrayList<>(); - try { - // 获取所有字段 - Field[] fields = obj.getClass().getDeclaredFields(); - for (Field field : fields) { - // 跳过静态或常量字段 - if (java.lang.reflect.Modifier.isStatic(field.getModifiers()) || - java.lang.reflect.Modifier.isFinal(field.getModifiers())) { - continue; - } - - // 确保字段是可访问的 - field.setAccessible(true); - - // 获取字段值 - Object fieldValue = field.get(obj); - - // 如果字段值是一个对象，递归调用 fetchData - if (fieldValue != null && isCustomObject(fieldValue)) { - data.add(JsonUtils.jsonTo2DArray(toJsonString(fieldValue))); - fetchData(fieldValue); - } - } - - - } catch (Exception e) { - e.printStackTrace(); - } - return data; - } - - // 判断一个对象是否是自定义的类对象 - private static boolean isCustomObject(Object obj) { - Class clazz = obj.getClass(); - // 自定义对象通常不会是 Java 的基本类型或标准库类型 - // 这里可以根据具体需求进行更加复杂的判断 - return !clazz.isPrimitive() && - !clazz.getName().startsWith("java.") && - !clazz.getName().startsWith("javax.") && - !clazz.getName().startsWith("sun.") && - !clazz.isAssignableFrom(Collection.class) && - !clazz.isAssignableFrom(Map.class); - } - - - } diff --git a/cf-framework/cf-spring-boot-starter-elasticsearch/src/main/java/com/cf/imes/framework/es/config/ChenfengElasticsearchAutoConfiguration.java b/cf-framework/cf-spring-boot-starter-elasticsearch/src/main/java/com/cf/imes/framework/es/config/ChenfengElasticsearchAutoConfiguration.java index 204646e87..5936087c1 100644 --- a/cf-framework/cf-spring-boot-starter-elasticsearch/src/main/java/com/cf/imes/framework/es/config/ChenfengElasticsearchAutoConfiguration.java +++ b/cf-framework/cf-spring-boot-starter-elasticsearch/src/main/java/com/cf/imes/framework/es/config/ChenfengElasticsearchAutoConfiguration.java @@ -20,6 +20,7 @@ import org.apache.http.conn.ssl.NoopHostnameVerifier; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; import org.apache.http.impl.nio.reactor.IOReactorConfig; +import org.apache.http.message.BasicHeader; import org.apache.http.ssl.SSLContextBuilder; import org.apache.http.ssl.SSLContexts; import org.elasticsearch.client.RestClient; @@ -43,6 +44,7 @@ import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; import java.security.cert.CertificateException; import java.time.Duration; +import java.util.List; /** @@ -140,6 +142,8 @@ public class ChenfengElasticsearchAutoConfiguration { .setConnectTimeout((int) properties.getConnectionTimeout().toMillis()) .setSocketTimeout((int) properties.getSocketTimeout().toMillis()) .setConnectionRequestTimeout((int) properties.getConnectionRequestTimeout().toMillis()) + // 启用数据压缩 + .setContentCompressionEnabled(true) .build()); if (ObjectUtil.isNotNull(credentialsProvider)) { httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider); @@ -150,6 +154,7 @@ public class ChenfengElasticsearchAutoConfiguration { } // 手动开启keepalive httpAsyncClientBuilder.setDefaultIOReactorConfig(IOReactorConfig.custom().setSoKeepAlive(true).build()); + httpAsyncClientBuilder.setDefaultHeaders(List.of(new BasicHeader("Accept-Encoding", "gzip"))); // 手动设置保活时长 httpAsyncClientBuilder.setKeepAliveStrategy(((response, context) -> Duration.ofMinutes(5).toMillis())); return httpAsyncClientBuilder; diff --git a/cf-module-prod-executor/cf-module-prod-executor-biz/src/main/java/com/cf/imes/module/executor/enums/EsIndexEnum.java b/cf-module-prod-executor/cf-module-prod-executor-biz/src/main/java/com/cf/imes/module/executor/enums/EsIndexEnum.java new file mode 100644 index 000000000..c39f61f8e --- /dev/null +++ b/cf-module-prod-executor/cf-module-prod-executor-biz/src/main/java/com/cf/imes/module/executor/enums/EsIndexEnum.java @@ -0,0 +1,43 @@ +package com.cf.imes.module.executor.enums; + + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + + +/** + * @author ES 索引类型 + */ + +@RequiredArgsConstructor +@Getter +public enum EsIndexEnum { + + + ORDER_PLATE_MODEL("生产单板材造型", "imes_order_plate_model"), + + ORDER_OPTIMIZE_PLATE_MODEL("排单优化数据", "imes_order_optimize_plate_model"), + + ORDER_PARTS_REMARK_MODEL("生产单配件备注", "imes_order_parts_remark_model"), + + PLAN_PROCESS_SCHEME_OPTIMIZE_MODEL("排单的加工方案组对应的优化信息", "imes_plan_process_scheme_optimize_model"), + + PLAN_ACTUAL_GOODS_MODEL("排单混单时选择的实际生产的大板信息", "imes_plan_actual_goods_model"), + + PLAN_PROCESS_SCHEME_CONFIG("排单的加工方案组对应的配置信息", "imes_plan_process_scheme_config_model"), + + ORDER_PLATE_COMPRESS_MODEL("生产单板材造型的压缩数据","imes_order_plate_compress_model"); + + /** + * 索引名称 + */ + private final String name; + + /** + * 索引 + */ + private final String index; + + + +} diff --git a/cf-module-prod-executor/cf-module-prod-executor-biz/src/main/java/com/cf/imes/module/executor/service/optimizeplan/OptimizePlanServiceImpl.java b/cf-module-prod-executor/cf-module-prod-executor-biz/src/main/java/com/cf/imes/module/executor/service/optimizeplan/OptimizePlanServiceImpl.java index 8fb82dedc..953b913aa 100644 --- a/cf-module-prod-executor/cf-module-prod-executor-biz/src/main/java/com/cf/imes/module/executor/service/optimizeplan/OptimizePlanServiceImpl.java +++ b/cf-module-prod-executor/cf-module-prod-executor-biz/src/main/java/com/cf/imes/module/executor/service/optimizeplan/OptimizePlanServiceImpl.java @@ -80,6 +80,7 @@ import static com.cf.imes.framework.common.exception.util.ServiceExceptionUtil.e import static com.cf.imes.framework.common.util.json.JsonUtils.*; import static com.cf.imes.framework.security.core.util.SecurityFrameworkUtils.getUserOrganId; import static com.cf.imes.module.executor.enums.ErrorCodeConstants.*; +import static com.cf.imes.module.executor.enums.EsIndexEnum.*; import static com.cf.imes.module.system.enums.ErrorCodeConstants.*; /** @@ -966,14 +967,7 @@ public class OptimizePlanServiceImpl implements OptimizePlanService { @Override public List getPlateModel(Long orderId) { - List plateDOS = plateMapper.selectPlateNum(Collections.singletonList(orderId), getUserOrganId()); - - if(plateDOS.isEmpty()){ - return new ArrayList<>(); - } - - return esUtils.getEsDocument(FIELD_ORDER_ID, orderId, plateDOS.size(), OptimizePlanService.ORDER_PLATE_MODEL, OrderModelDO.class); - + return esUtils.getEsDocumentByScroll(FIELD_ORDER_ID,orderId,1000,ORDER_PLATE_MODEL.getIndex(),OrderModelDO.class); } diff --git a/cf-module-prod-executor/cf-module-prod-executor-biz/src/main/java/com/cf/imes/module/executor/util/elasticsearch/EsUtils.java b/cf-module-prod-executor/cf-module-prod-executor-biz/src/main/java/com/cf/imes/module/executor/util/elasticsearch/EsUtils.java index f0ab56d5c..9e05c957d 100644 --- a/cf-module-prod-executor/cf-module-prod-executor-biz/src/main/java/com/cf/imes/module/executor/util/elasticsearch/EsUtils.java +++ b/cf-module-prod-executor/cf-module-prod-executor-biz/src/main/java/com/cf/imes/module/executor/util/elasticsearch/EsUtils.java @@ -5,17 +5,21 @@ import cn.hutool.core.collection.CollUtil; import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch._types.ElasticsearchException; import co.elastic.clients.elasticsearch._types.FieldValue; +import co.elastic.clients.elasticsearch._types.SortOrder; +import co.elastic.clients.elasticsearch._types.Time; import co.elastic.clients.elasticsearch._types.query_dsl.Query; -import co.elastic.clients.elasticsearch.core.DeleteByQueryRequest; -import co.elastic.clients.elasticsearch.core.SearchRequest; -import co.elastic.clients.elasticsearch.core.SearchResponse; +import co.elastic.clients.elasticsearch.core.*; import co.elastic.clients.elasticsearch.core.search.Hit; import co.elastic.clients.elasticsearch.indices.RefreshRequest; +import co.elastic.clients.json.JsonData; import co.elastic.clients.json.JsonpMappingException; import com.cf.imes.framework.common.exception.ServiceException; import com.cf.imes.framework.common.util.json.JsonUtils; import com.cf.imes.framework.es.core.dal.ESDocument; import com.cf.imes.framework.es.core.service.ESDocumentService; +import com.cf.imes.framework.security.core.LoginUser; +import com.cf.imes.framework.security.core.util.SecurityFrameworkUtils; +import com.cf.imes.module.executor.dal.dataobject.ordermodel.OrderModelDataTest; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; @@ -118,7 +122,7 @@ public class EsUtils { SearchRequest.Builder builder = new SearchRequest.Builder(); builder.index(index); - builder.query(q -> q.terms(b -> b.field(filedName).terms(t->t.value(fieldValues)))); + builder.query(q -> q.bool(b->b.filter(f->f.terms(t->t.field(filedName).terms(te->te.value(fieldValues)))))); try { SearchResponse search = elasticsearchClient.search(builder.build(), targetClass); List> hits = search.hits().hits(); @@ -143,7 +147,8 @@ public class EsUtils { builder.index(index); builder.size(size); - builder.query(q -> q.terms(b -> b.field(filedName).terms(t->t.value(fieldValues)))); + builder.query(q-> q.bool(b-> b.filter(f-> f.terms(t-> t.field(filedName).terms(v->v.value(fieldValues)))))); + try { SearchResponse search = elasticsearchClient.search(builder.build(), targetClass); List> hits = search.hits().hits(); @@ -288,13 +293,190 @@ public class EsUtils { + + + // 范围查询（包括最大值和最小值），仅限于造型数据查询使用 + public List getEsDocumentByRange(String filedName, Long value, String index, int minSortId, int maxSortId, Class targetClass) { + + SearchRequest.Builder builder = new SearchRequest.Builder(); + builder.index(index); + int size = maxSortId - minSortId; + builder.size( size <= 0 ? 100 : size+2 ); + + builder.query(q->q.bool(b->b.filter(f->{ + f.term(t->t.field(filedName).value(value)); + f.range(r -> r.field("sortId").gte(JsonData.of(minSortId)).lte(JsonData.of(maxSortId))); + return f; + }))); + + + try { + SearchResponse search = elasticsearchClient.search(builder.build(), targetClass); + List> hits = search.hits().hits(); + if (CollUtil.isNotEmpty(hits)) { + return hits.stream().map(Hit::source).toList(); + } + return new ArrayList<>(); + }catch (JsonpMappingException e){ + log.error(e.getMessage()); + throw new ServiceException(ORDER_DATA_ERROR); + }catch (IOException | ElasticsearchException e) { + log.error(e.getMessage()); + throw new ServiceException(DATA_DATA_ERROR); + } + } + + + + + + + + + // searchAfter 查询造型数据 + public List getEsDocumentBySearchAfter(String filedName, + Long value, + String index, + int size, + int querySortId, + Class targetClass){ + + List results = new ArrayList<>(); + + try { + + + SearchRequest.Builder searchBuilder = new SearchRequest.Builder() + .index(index) + .size(size) + .sort(s -> s.field(f -> f.field("sortId").order(SortOrder.Asc))) + .query(q -> q.bool(b -> b + .filter(m -> m.term(t -> t.field(filedName).value(value))) + )); + + while (true) { + + searchBuilder.searchAfter(FieldValue.of(querySortId)); + + SearchResponse response = elasticsearchClient.search(searchBuilder.build(), targetClass); + + List> hits = response.hits().hits(); + if (hits.isEmpty()) { + break; + } + + for (Hit hit : hits) { + results.add(hit.source()); + } + + querySortId += size; + + } + + return results; + + } catch (JsonpMappingException e){ + log.error(e.getMessage()); + throw new ServiceException(ORDER_DATA_ERROR); + }catch (IOException | ElasticsearchException e) { + log.error(e.getMessage()); + throw new ServiceException(DATA_DATA_ERROR); + } + + } + + + + + + + + // scroll 游标查询，目前仅用于查询造型数据 + public List getEsDocumentByScroll(String filedName, Long value,Integer size, String index, Class targetClass){ + + // scroll 保持游标有效时间 + final String scrollTime = "1m"; + + List result = new ArrayList<>(); + + + String scrollId = null; + + try { + + // 执行第一次搜索，开启 scroll + SearchResponse searchResponse = elasticsearchClient.search(s -> s + .index(index) + .scroll(Time.of(t -> t.time(scrollTime))) + .size(size) + .query(q->q.bool(b->b.filter(f->f.term(t->t.field(filedName).value(value))))), +// .source(sou->sou.filter(f->f.includes("orderId","plateId","plateModelData","sortId"))), + targetClass + ); + + + scrollId = searchResponse.scrollId(); + List> hits = searchResponse.hits().hits(); + hits.forEach(hit -> result.add(hit.source())); + + // 循环 scroll 拉取剩余数据 + while (!hits.isEmpty()) { + String finalScrollId = scrollId; + ScrollResponse scrollResponse = elasticsearchClient.scroll(s -> s + .scrollId(finalScrollId) + .scroll(Time.of(t -> t.time(scrollTime))), targetClass); + + scrollId = scrollResponse.scrollId(); + hits = scrollResponse.hits().hits(); + + if (!hits.isEmpty()) { + hits.forEach(hit -> result.add(hit.source())); + } + } + + return result; + + }catch (IOException | ElasticsearchException e){ + log.error(e.getMessage()); + throw new ServiceException(DATA_DATA_ERROR); + }finally { + // 清理 scroll + clearScroll(scrollId); + } + + } + + + + + + + private void clearScroll(String scrollId) { + try { + if (scrollId != null && !scrollId.isEmpty()) { + elasticsearchClient.clearScroll(c -> c.scrollId(scrollId)); + } + } catch (Exception e) { + log.warn("Failed to clear scroll: {}", e.getMessage()); + } + } + + + + + + + + + + // ES 文档删除 public void deleteEsDocument(String filedName,Long value, String index) { DeleteByQueryRequest.Builder builder = new DeleteByQueryRequest.Builder(); builder.index(index); - builder.query(q -> q.term(b -> b.field(filedName).value(value))); + builder.query(q -> q.bool(b->b.filter(f->f.term(t->t.field(filedName).value(value))))); try { elasticsearchClient.deleteByQuery(builder.build()); @@ -351,7 +533,7 @@ public class EsUtils { DeleteByQueryRequest.Builder builder = new DeleteByQueryRequest.Builder(); builder.index(index); - builder.query(q -> q.terms(b -> b.field(filedName).terms(e -> e.value(fieldValues)))); + builder.query(q -> q.bool(b->b.filter(f->f.terms(t->t.field(filedName).terms(te->te.value(fieldValues)))))); try { elasticsearchClient.deleteByQuery(builder.build()); @@ -384,6 +566,30 @@ public class EsUtils { + public void savePlateModelEs(String index,List targetClass) { + try { + + BulkRequest.Builder br = new BulkRequest.Builder(); + LoginUser loginUser = SecurityFrameworkUtils.getLoginUser(); + targetClass.forEach(esDocument -> { + esDocument.setOrganId(loginUser.getOrganId()); + br.operations(op -> op.index(idx -> idx + .index(index) + .document(esDocument))); + }); + elasticsearchClient.bulk(br.build()); + + } catch (IOException | ElasticsearchException e) { + e.printStackTrace(); + log.error(e.getMessage()); + throw new ServiceException(DATA_DATA_ERROR); + } + + } + + + + // ES 文档数据刷新 diff --git a/script/es/imes_plan_process_scheme_config_model.index b/script/es/imes_plan_process_scheme_config_model.index new file mode 100644 index 000000000..30621fe52 --- /dev/null +++ b/script/es/imes_plan_process_scheme_config_model.index @@ -0,0 +1,58 @@ +PUT imes_plan_process_scheme_config_model +{ + "settings": {}, + "mappings": { + "properties": { + "planId": { + "type": "long" + }, + "processId": { + "type": "long" + }, + "planConfigId": { + "type": "long" + }, + "planName": { + "type": "keyword" + }, + "processLineConfig": { + "type": "keyword" + }, + "processLineName": { + "type": "keyword" + }, + "optimizationConfig": { + "type": "keyword" + }, + "optimizationConfigName": { + "type": "keyword" + }, + "analyzer": { + "type": "keyword" + }, + "analyzerName": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "organId": { + "type": "long" + }, + "updateTime": { + "type": "date", + "format": "strict_date_optional_time||yyyy-MM-dd HH:mm:ss||epoch_millis" + }, + "updater": { + "type": "keyword" + }, + "createTime": { + "type": "date", + "format": "strict_date_optional_time||yyyy-MM-dd HH:mm:ss||epoch_millis" + }, + "creator": { + "type": "keyword" + } + } + } +} \ No newline at end of file