mirror of
http://192.168.1.205:9980/cf_devdept2/cf_imes_server.git
synced 2026-08-12 21:02:08 +08:00
es开启数据压缩，es增加游标，searchAfter，范围查询方法
This commit is contained in:
liuzhaotian
+3
-181
@@ -10,16 +10,15 @@ import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.SerializationFeature;
|
||||
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
|
||||
import lombok.Data;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.ToString;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.lang.reflect.Field;
|
||||
import java.lang.reflect.Type;
|
||||
import java.util.*;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Base64;
|
||||
import java.util.List;
|
||||
import java.util.zip.DataFormatException;
|
||||
import java.util.zip.Deflater;
|
||||
import java.util.zip.Inflater;
|
||||
@@ -271,181 +270,4 @@ public class JsonUtils {
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
/*
|
||||
|
||||
|
||||
|
||||
* */
|
||||
public static List<List<Object>> jsonTo2DArray(String jsonString) {
|
||||
try {
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
JsonNode jsonArray = objectMapper.readTree(jsonString);
|
||||
|
||||
Set<String> headers = new LinkedHashSet<>();
|
||||
List<List<Object>> result = new ArrayList<>();
|
||||
|
||||
// 从数组中的所有对象收集标头
|
||||
if (jsonArray.isArray()) {
|
||||
for (JsonNode jsonObject : jsonArray) {
|
||||
Iterator<String> fieldNames = jsonObject.fieldNames();
|
||||
while (fieldNames.hasNext()) {
|
||||
headers.add(fieldNames.next());
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
result.add(new ArrayList<>(headers));
|
||||
|
||||
// 提取行
|
||||
for (JsonNode jsonObject : jsonArray) {
|
||||
List<Object> row = new ArrayList<>();
|
||||
for (String header : headers) {
|
||||
getRow(jsonObject, header, row);
|
||||
}
|
||||
result.add(row);
|
||||
}
|
||||
|
||||
return result;
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
return new ArrayList<>();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 提取行
|
||||
*
|
||||
* @param jsonObject
|
||||
* @param header
|
||||
* @param row
|
||||
* @return
|
||||
*/
|
||||
private static void getRow(JsonNode jsonObject, String header, List<Object> row) {
|
||||
JsonNode valueNode = jsonObject.get(header);
|
||||
if (valueNode != null) {
|
||||
if (valueNode.isTextual()) {
|
||||
row.add(valueNode.asText());
|
||||
} else if (valueNode.isInt()) {
|
||||
row.add(valueNode.asInt());
|
||||
} else if (valueNode.isBoolean()) {
|
||||
row.add(valueNode.asBoolean());
|
||||
} else if (valueNode.isDouble()) {
|
||||
row.add(valueNode.asDouble());
|
||||
} else if (valueNode.isArray() || valueNode.isObject()) {
|
||||
row.add(jsonTo2DArray(valueNode.toString()));
|
||||
} else {
|
||||
row.add(valueNode.asText());
|
||||
}
|
||||
} else {
|
||||
row.add(null); // 处理缺少的值
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// 抽取 json数据，不改变数据结构对应的值，只改变数据结构（参考 HPACK 算法） 实现json数据抽取
|
||||
public static List<List<Object>> getData(Object object) {
|
||||
String jsonString = toJsonString(object);
|
||||
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
try {
|
||||
List<Map<String, Object>> list = mapper.readValue(jsonString, new TypeReference<>() {});
|
||||
List<List<Object>> result = convertTo2DArray(list);
|
||||
|
||||
//
|
||||
return result;
|
||||
|
||||
} catch (IOException e) {
|
||||
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
return null;
|
||||
|
||||
}
|
||||
|
||||
|
||||
private static List<List<Object>> convertTo2DArray(List<Map<String, Object>> list) {
|
||||
List<List<Object>> result = new ArrayList<>();
|
||||
|
||||
// Get the headers from the first map
|
||||
if (list.size() > 0) {
|
||||
Map<String, Object> firstMap = list.get(0);
|
||||
List<Object> headers = new ArrayList<>(firstMap.keySet());
|
||||
result.add(headers);
|
||||
//
|
||||
// Add rows
|
||||
for (Map<String, Object> map : list) {
|
||||
//
|
||||
List<Object> row = new ArrayList<>();
|
||||
for (Object header : headers) {
|
||||
row.add(map.get(header));
|
||||
}
|
||||
result.add(row);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
|
||||
// 将所有抽取后的 json 数据都加入一个统一的数据类型中，方便前端解析哪些代码是抽取的数据，进行数据还原
|
||||
@Data
|
||||
@ToString(callSuper = true)
|
||||
public static class ExtractedData {
|
||||
List<List<Object>> extractedData;
|
||||
|
||||
}
|
||||
|
||||
|
||||
public static List<List<List<Object>>> fetchData(Object obj) {
|
||||
List<List<List<Object>>> data = new ArrayList<>();
|
||||
try {
|
||||
// 获取所有字段
|
||||
Field[] fields = obj.getClass().getDeclaredFields();
|
||||
for (Field field : fields) {
|
||||
// 跳过静态或常量字段
|
||||
if (java.lang.reflect.Modifier.isStatic(field.getModifiers()) ||
|
||||
java.lang.reflect.Modifier.isFinal(field.getModifiers())) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// 确保字段是可访问的
|
||||
field.setAccessible(true);
|
||||
|
||||
// 获取字段值
|
||||
Object fieldValue = field.get(obj);
|
||||
|
||||
// 如果字段值是一个对象，递归调用 fetchData
|
||||
if (fieldValue != null && isCustomObject(fieldValue)) {
|
||||
data.add(JsonUtils.jsonTo2DArray(toJsonString(fieldValue)));
|
||||
fetchData(fieldValue);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
// 判断一个对象是否是自定义的类对象
|
||||
private static boolean isCustomObject(Object obj) {
|
||||
Class<?> clazz = obj.getClass();
|
||||
// 自定义对象通常不会是 Java 的基本类型或标准库类型
|
||||
// 这里可以根据具体需求进行更加复杂的判断
|
||||
return !clazz.isPrimitive() &&
|
||||
!clazz.getName().startsWith("java.") &&
|
||||
!clazz.getName().startsWith("javax.") &&
|
||||
!clazz.getName().startsWith("sun.") &&
|
||||
!clazz.isAssignableFrom(Collection.class) &&
|
||||
!clazz.isAssignableFrom(Map.class);
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user