新增webcad生产单导入：1、大数据列表jdbc替换mp；2、切割大事务；

cf-module-prod-plan/cf-module-prod-plan-biz/src/main/java/com/cf/imes/module/plan/service/orderImport/WebCadOrderImportServiceImpl.java

@@ -26,10 +26,10 @@ import com.cf.imes.module.system.api.systemconfig.SystemConfigApi;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.jdbc.datasource.DataSourceTransactionManager;
 import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
 import org.springframework.stereotype.Service;
-import org.springframework.transaction.annotation.Transactional;
 
 import javax.annotation.Resource;
 
@@ -105,8 +105,10 @@ public class WebCadOrderImportServiceImpl implements WebCadOrderImportService {
     @Value("${chenfeng.plan.impot.cadImportPlateNumThreshold:20000}")
     private int cadImportPlateNumThreshold;
 
+    @Resource
+    private JdbcTemplate jdbcTemplate;
+
     @Override
-    @Transactional(rollbackFor = Exception.class)
     public void webCadOrderImport(WebCadDataReqVO webCadDataReqVO) {
         Long organId = OrganContextHolder.getOrganId();
         String redisUniqueKey = "webcad";
@@ -121,7 +123,7 @@ public class WebCadOrderImportServiceImpl implements WebCadOrderImportService {
                 throw new ServiceException(ORDER_IMPORT_ORGAN_LOCK_ERROR);
             }
             WebCadOrderImportFactory webCadOrderImportFactory = new WebCadOrderImportFactory(organId, orderMapper, snowFlakeGenerator, orderBodyMapper, plateMapper, orderItemMapper, orderGroupMapper,
-                    idWorker, orderInputProcessor, orderPartsMapper, goodsMapper, rawGoodsMapper, plateGoodMapper, customPlateNoGenerateService, transactionManager, systemConfigApi, threadPoolTaskExecutor, cadImportPlateNumThreshold);
+                    idWorker, orderInputProcessor, orderPartsMapper, goodsMapper, rawGoodsMapper, plateGoodMapper, customPlateNoGenerateService, transactionManager, systemConfigApi, threadPoolTaskExecutor, cadImportPlateNumThreshold, jdbcTemplate);
             webCadOrderImportFactory.analyzeTempData(webCadDataReqVO);
         } catch (ServiceException se) {
           throw se;

cf-module-prod-plan/cf-module-prod-plan-biz/src/main/java/com/cf/imes/module/plan/service/orderImport/factory/WebCadOrderImportFactory.java

@@ -65,20 +65,27 @@ import com.cf.imes.module.system.api.systemconfig.dto.SystemConfigRespDTO;
 import com.cf.imes.module.system.api.systemconfig.dto.SystemConfigSealEdgeRespDTO;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang.StringUtils;
+import org.springframework.jdbc.core.BatchPreparedStatementSetter;
+import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.jdbc.datasource.DataSourceTransactionManager;
 import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
 import org.springframework.transaction.TransactionDefinition;
 import org.springframework.transaction.TransactionStatus;
 
 import java.math.BigDecimal;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Timestamp;
 import java.time.LocalDateTime;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.Collectors;
@@ -103,7 +110,7 @@ public class WebCadOrderImportFactory {
     /**
      * 批量操作阈值
      */
-    private static final int BATCH_THRESHOLD_NUMBER = 500;
+    private static final int BATCH_THRESHOLD_NUMBER = 800;
 
     private OrderMapper orderMapper;
 
@@ -137,6 +144,8 @@ public class WebCadOrderImportFactory {
 
     private ThreadPoolTaskExecutor threadPoolTaskExecutor;
 
+    private JdbcTemplate jdbcTemplate;
+
     private Long organId;
 
     // 批次id
@@ -221,7 +230,8 @@ public class WebCadOrderImportFactory {
                                     DataSourceTransactionManager transactionManager,
                                     SystemConfigApi systemConfigApi,
                                     ThreadPoolTaskExecutor threadPoolTaskExecutor,
-                                    int cadImportPlateNumThreshold) {
+                                    int cadImportPlateNumThreshold,
+                                    JdbcTemplate jdbcTemplate) {
         this.organId = organId;
         this.orderMapper = orderMapper;
         this.snowFlakeGenerator = snowFlakeGenerator;
@@ -245,6 +255,7 @@ public class WebCadOrderImportFactory {
         sealEdgeConfigList = new ArrayList<>();
         peek = DynamicDataSourceContextHolder.peek();
         this.cadImportPlateNumThreshold = cadImportPlateNumThreshold;
+        this.jdbcTemplate = jdbcTemplate;
     }
 
     public void analyzeTempData(WebCadDataReqVO webCadDataReqVO) {
@@ -287,19 +298,36 @@ public class WebCadOrderImportFactory {
             analyzeParts(webCadDataReqVO.getParts());
             log.debug("====================【cad拆单解析配件结束】====================");
 
-            orderItemBatchInsertWithinThreshold(orderItemDOS, true);
+            // 清理引用
+            webCadDataReqVO.setBlocks(null);
+            webCadDataReqVO.setParts(null);
+            webCadDataReqVO.setMaterials(null);
+            webCadDataReqVO.setProcessGroup(null);
+            webCadDataReqVO.setDouleRoomTree(null);
+            webCadDataReqVO.setOrders(null);
+
+            log.debug("====================【cad拆单同步保存开始】====================");
+            // 同步保存板件
+            savePlateSync();
+            // 同步保存配件
+            savePartSync();
+            // 同步保存item
+            saveItemSync();
+            log.debug("====================【cad拆单同步保存开始】====================");
 
             log.debug("====================【cad拆单异步保存开始】====================");
             // 异步保存数据
             saveDataAsync();
             log.debug("====================【cad拆单异步保存结束】====================");
 
+            if (plateNumReachThreshold) {
                 log.warn(ServiceExceptionUtil.exception(WEBCAD_ORDER_IMPORT_ORDER_PLATENUM_REACH_THRESHOLD_ERROR, orderId).getMessage());
+            }
 
         } catch (Exception e) {
             if (esInsert) {
                 CompletableFuture.runAsync(() -> {
-                    // 数据库数据回滚，此时删除 ES 里的数据
+                    // 数据库数据回滚，同步删除 ES 里当前批次的数据
                     orderInputProcessor.deleteByOrderIdBatch(orderId, batchId, ORDER_PLATE_MODEL);
                 });
             }
@@ -646,8 +674,6 @@ public class WebCadOrderImportFactory {
         }
         // 所有自定义板编号生成完成后更新配置到生产单
         customPlateNoGenerateService.updateCustomPlateNoGenerateConfig(plateNoGenerateConfigVO, orderDO);
-        // 批量插入板件
-        orderPlateBatchInsertWithinThreshold(plateDOS, true);
         // 批量插入造型数据
         orderModelsBatchInsertWithinThreshold(true);
     }
@@ -832,7 +858,6 @@ public class WebCadOrderImportFactory {
             return;
         }
         plateDOS.add(plateDO);
-        orderPlateBatchInsertWithinThreshold(plateDOS, false);
 
         // 创建es造型数据
         createModel(plateDO, block, blockInfo, plateRemark);
@@ -1318,7 +1343,6 @@ public class WebCadOrderImportFactory {
                         .organId(organId)
                         .build();
                 orderItemDOS.add(orderItemDO);
-                orderItemBatchInsertWithinThreshold(orderItemDOS, false);
             }
         } else {
             // 没有归属的加工组，只创建一个orderItem
@@ -1337,7 +1361,6 @@ public class WebCadOrderImportFactory {
                     .organId(organId)
                     .build();
             orderItemDOS.add(orderItemDO);
-            orderItemBatchInsertWithinThreshold(orderItemDOS, false);
         }
     }
 
@@ -1502,7 +1525,6 @@ public class WebCadOrderImportFactory {
                         .organId(organId)
                         .build();
                 orderItemDOS.add(orderItemDO);
-                orderItemBatchInsertWithinThreshold(orderItemDOS, false);
             }
             OrderItemDO orderItemDO = OrderItemDO
                     .builder()
@@ -1519,7 +1541,6 @@ public class WebCadOrderImportFactory {
                     .organId(organId)
                     .build();
             orderItemDOS.add(orderItemDO);
-            orderItemBatchInsertWithinThreshold(orderItemDOS, false);
         } else {
             OrderItemDO orderItemDO = OrderItemDO
                     .builder()
@@ -1536,7 +1557,6 @@ public class WebCadOrderImportFactory {
                     .organId(organId)
                     .build();
             orderItemDOS.add(orderItemDO);
-            orderItemBatchInsertWithinThreshold(orderItemDOS, false);
         }
     }
 
@@ -1618,7 +1638,6 @@ public class WebCadOrderImportFactory {
             orderPartsDO.setCreateTime(now);
             orderPartsDO.setUpdateTime(now);
             orderPartsDOS.add(orderPartsDO);
-            orderPartBatchInsertWithinThreshold(orderPartsDOS, false);
 
             // 配件id回显到列表
             partValue.forEach(p -> p.setImesPartsId(partId));
@@ -1657,12 +1676,10 @@ public class WebCadOrderImportFactory {
             orderPartsDO.setCreateTime(now);
             orderPartsDO.setUpdateTime(now);
             orderPartsDOS.add(orderPartsDO);
-            orderPartBatchInsertWithinThreshold(orderPartsDOS, false);
 
             // 配件id回显到列表
             module.setImesPartsId(partId);
         }
-        orderPartBatchInsertWithinThreshold(orderPartsDOS, true);
     }
 
 
@@ -1789,57 +1806,6 @@ public class WebCadOrderImportFactory {
         }
     }
 
-    /**
-     * 每达到阈值就批量插入一次orderItem
-     *
-     * @param list
-     * @param isLast 是否最后一次插入，不是就需要清理list
-     */
-    private void orderItemBatchInsertWithinThreshold(List<OrderItemDO> list, boolean isLast) {
-        // 达到批量的阈值就做一次插入
-        if (CollUtil.isNotEmpty(list) && (list.size() >= BATCH_THRESHOLD_NUMBER || isLast)) {
-            orderItemMapper.insertBatchSomeColumn(list);
-            if (!isLast) {
-                // 存储完成清理 list
-                list.clear();
-            }
-        }
-    }
-
-    /**
-     * 每达到阈值就批量插入一次orderPlate
-     *
-     * @param list
-     * @param isLast 是否最后一次插入，不是就需要清理list
-     */
-    private void orderPlateBatchInsertWithinThreshold(List<PlateDO> list, boolean isLast) {
-        // 达到批量的阈值就做一次插入
-        if (CollUtil.isNotEmpty(list) && (list.size() >= BATCH_THRESHOLD_NUMBER || isLast)) {
-            plateMapper.insertBatchSomeColumn(list);
-            if (!isLast) {
-                // 存储完成清理 list
-                list.clear();
-            }
-        }
-    }
-
-    /**
-     * 每达到阈值就批量插入一次orderPart
-     *
-     * @param list
-     * @param isLast 是否最后一次插入，不是就需要清理list
-     */
-    private void orderPartBatchInsertWithinThreshold(List<OrderPartsDO> list, boolean isLast) {
-        // 达到批量的阈值就做一次插入
-        if (CollUtil.isNotEmpty(list) && (list.size() >= BATCH_THRESHOLD_NUMBER || isLast)) {
-            orderPartsMapper.insertBatchSomeColumn(list);
-            if (!isLast) {
-                // 存储完成清理 list
-                list.clear();
-            }
-        }
-    }
-
     /**
      * 每达到阈值就批量插入一次生产单造型数据和压缩数据
      *
@@ -1873,6 +1839,243 @@ public class WebCadOrderImportFactory {
         }
     }
 
+    /**
+     * 同步保存板件
+     */
+    private void savePlateSync() {
+        TransactionStatus transactionStatus = transactionManager.getTransaction(TransactionDefinition.withDefaults());
+
+        try {
+            List<List<PlateDO>> plateList = batchProcess(plateDOS);
+
+            // 同步插入order_plate
+            batchSavePlate(plateList);
+            // 清理order_plate相关的列表
+            plateList.clear();
+
+        } catch (Exception e) {
+            log.error("WEBCAD import savePlate error", e);
+            // 事务回滚
+            transactionManager.rollback(transactionStatus);
+            // 抛出异常回滚
+            throw new ServiceException(WEBCAD_ORDER_IMPORT_SAVEDATA_ERROR);
+        } finally {
+            // 清理order_plate相关的列表
+            plateDOS.clear();
+        }
+
+        // 事务提交
+        transactionManager.commit(transactionStatus);
+    }
+
+    /**
+     * 批次插入板件
+     *
+     * @param plateList 分批次后的板件列表
+     */
+    private void batchSavePlate(List<List<PlateDO>> plateList) {
+        for (List<PlateDO> plates : plateList) {
+            String sql = "INSERT INTO order_plate (id,organ_id,order_id,name,plate_no,type,goods_id,width,height,thickness,split_width,split_height,split_thickness,seal_left,seal_right,seal_up,seal_down,area,texture,hole_face,hole_arrange,unregular_point_count,front_hole_count,back_hole_count,side_hole_count,front_model_count,back_model_count,is_door,open_door_type,offset_x,offset_y,is_arc_across,module_type_id,is_special_shaped,is_sculpt,is_row_hole,is_optimized,is_cutted,filter_type,remark,is_cancel,deleted,custom_plate_no,create_time,update_time,creator,updater) VALUES    (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
+            jdbcTemplate.batchUpdate(sql, new BatchPreparedStatementSetter() {
+                @Override
+                public void setValues(PreparedStatement ps, int index) throws SQLException {
+                    PlateDO item = plates.get(index);
+                    ps.setLong(1, item.getId());
+                    ps.setLong(2, item.getOrganId());
+                    ps.setLong(3, item.getOrderId());
+                    ps.setString(4, item.getName());
+                    ps.setString(5, item.getPlateNo());
+                    ps.setInt(6, item.getType());
+                    ps.setLong(7, item.getGoodsId());
+                    ps.setBigDecimal(8, item.getWidth());
+                    ps.setBigDecimal(9, item.getHeight());
+                    ps.setBigDecimal(10, item.getThickness());
+                    ps.setBigDecimal(11, item.getSplitWidth());
+                    ps.setBigDecimal(12, item.getSplitHeight());
+                    ps.setBigDecimal(13, item.getSplitThickness());
+                    ps.setBigDecimal(14, item.getSealLeft());
+                    ps.setBigDecimal(15, item.getSealRight());
+                    ps.setBigDecimal(16, item.getSealUp());
+                    ps.setBigDecimal(17, item.getSealDown());
+                    ps.setBigDecimal(18, item.getArea());
+                    ps.setInt(19, item.getTexture());
+                    ps.setInt(20, item.getHoleFace());
+                    ps.setInt(21, item.getHoleArrange());
+                    ps.setInt(22, item.getUnregularPointCount());
+                    ps.setInt(23, item.getFrontHoleCount());
+                    ps.setInt(24, item.getBackHoleCount());
+                    ps.setInt(25, item.getSideHoleCount());
+                    ps.setInt(26, item.getFrontModelCount());
+                    ps.setInt(27, item.getBackModelCount());
+                    ps.setBoolean(28, item.getIsDoor());
+                    ps.setInt(29, item.getOpenDoorType());
+                    ps.setBigDecimal(30, item.getOffsetX());
+                    ps.setBigDecimal(31, item.getOffsetY());
+                    ps.setBoolean(32, item.getIsArcAcross());
+                    ps.setLong(33, item.getModuleTypeId());
+                    ps.setBoolean(34, item.getIsSpecialShaped());
+                    ps.setBoolean(35, item.getIsSculpt());
+                    ps.setBoolean(36, item.getIsRowHole());
+                    ps.setBoolean(37, item.getIsOptimized());
+                    ps.setInt(38, item.getIsCutted());
+                    ps.setString(39, item.getFilterType());
+                    ps.setString(40, item.getRemark());
+                    ps.setBoolean(41, item.getIsCancel());
+                    ps.setBoolean(42, item.getDeleted());
+                    ps.setString(43, item.getCustomPlateNo());
+                    ps.setTimestamp(44, Timestamp.valueOf(item.getCreateTime()));
+                    ps.setTimestamp(45, Timestamp.valueOf(item.getUpdateTime()));
+                    ps.setString(46, item.getCreator());
+                    ps.setString(47, item.getUpdater());
+                }
+
+                @Override
+                public int getBatchSize() {
+                    return plates.size();
+                }
+            });
+        }
+    }
+
+    /**
+     * 同步保存配件
+     */
+    private void savePartSync() {
+        TransactionStatus transactionStatus = transactionManager.getTransaction(TransactionDefinition.withDefaults());
+
+        try {
+            List<List<OrderPartsDO>> partList = batchProcess(orderPartsDOS);
+
+            // 同步插入order_plate
+            batchSavePart(partList);
+            // 清理order_plate相关的列表
+            partList.clear();
+
+        } catch (Exception e) {
+            log.error("WEBCAD import savePart error", e);
+            // 事务回滚
+            transactionManager.rollback(transactionStatus);
+            // 抛出异常回滚
+            throw new ServiceException(WEBCAD_ORDER_IMPORT_SAVEDATA_ERROR);
+        } finally {
+            // 清理order_plate相关的列表
+            orderPartsDOS.clear();
+        }
+
+        // 事务提交
+        transactionManager.commit(transactionStatus);
+    }
+
+    /**
+     * 批次插入配件
+     *
+     * @param partList 分批次后的配件列表
+     */
+    private void batchSavePart(List<List<OrderPartsDO>> partList) {
+        for (List<OrderPartsDO> plates : partList) {
+            String sql = "INSERT INTO order_parts (id,organ_id,order_id,goods_id,name,color,material,category,type,width,length,thickness,model,spec,brand,factory,unit,price,is_composite,subparts,remark,deleted,create_time,update_time,creator,updater) VALUES    (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
+            jdbcTemplate.batchUpdate(sql, new BatchPreparedStatementSetter() {
+                @Override
+                public void setValues(PreparedStatement ps, int index) throws SQLException {
+                    OrderPartsDO item = plates.get(index);
+                    ps.setLong(1, item.getId());
+                    ps.setLong(2, item.getOrganId());
+                    ps.setLong(3, item.getOrderId());
+                    ps.setString(4, item.getGoodsId());
+                    ps.setString(5, item.getName());
+                    ps.setString(6, item.getColor());
+                    ps.setString(7, item.getMaterial());
+                    ps.setString(8, item.getCategory());
+                    ps.setString(9, item.getType());
+                    ps.setDouble(10, item.getWidth());
+                    ps.setDouble(11, item.getLength());
+                    ps.setDouble(12, item.getThickness());
+                    ps.setString(13, item.getModel());
+                    ps.setString(14, item.getSpec());
+                    ps.setString(15, item.getBrand());
+                    ps.setString(16, item.getFactory());
+                    ps.setString(17, item.getUnit());
+                    ps.setDouble(18, item.getPrice());
+                    ps.setBoolean(19, item.getIsComposite());
+                    ps.setString(20, item.getSubparts());
+                    ps.setString(21, item.getRemark());
+                    ps.setBoolean(22, item.getDeleted());
+                    ps.setTimestamp(23, Timestamp.valueOf(item.getCreateTime()));
+                    ps.setTimestamp(24, Timestamp.valueOf(item.getUpdateTime()));
+                    ps.setString(25, item.getCreator());
+                    ps.setString(26, item.getUpdater());
+                }
+
+                @Override
+                public int getBatchSize() {
+                    return plates.size();
+                }
+            });
+        }
+    }
+
+    /**
+     * 同步保存item
+     */
+    private void saveItemSync() {
+        TransactionStatus transactionStatus = transactionManager.getTransaction(TransactionDefinition.withDefaults());
+
+        try {
+            List<List<OrderItemDO>> itemList = batchProcess(orderItemDOS);
+
+            // 同步插入order_plate
+            batchSaveItem(itemList);
+            // 清理order_plate相关的列表
+            itemList.clear();
+
+        } catch (Exception e) {
+            log.error("WEBCAD import saveItem error", e);
+            // 事务回滚
+            transactionManager.rollback(transactionStatus);
+            // 抛出异常回滚
+            throw new ServiceException(WEBCAD_ORDER_IMPORT_SAVEDATA_ERROR);
+        } finally {
+            // 清理order_plate相关的列表
+            orderItemDOS.clear();
+        }
+
+        // 事务提交
+        transactionManager.commit(transactionStatus);
+    }
+
+    /**
+     * 批次插入item
+     *
+     * @param itemList 分批次后的item列表
+     */
+    private void batchSaveItem(List<List<OrderItemDO>> itemList) {
+        for (List<OrderItemDO> items : itemList) {
+            String sql = "INSERT INTO order_item (id,order_id,type,room_id,body_id,package_id,group_id,plate_id,parts_id,num,organ_id) VALUES    (?,?,?,?,?,?,?,?,?,?,?)";
+            jdbcTemplate.batchUpdate(sql, new BatchPreparedStatementSetter() {
+                @Override
+                public void setValues(PreparedStatement ps, int index) throws SQLException {
+                    OrderItemDO item = items.get(index);
+                    ps.setLong(1, item.getId());
+                    ps.setLong(2, item.getOrderId());
+                    ps.setInt(3, item.getType());
+                    ps.setLong(4, item.getRoomId());
+                    ps.setLong(5, item.getBodyId());
+                    ps.setLong(6, item.getPackageId());
+                    ps.setLong(7, item.getGroupId());
+                    ps.setLong(8, item.getPlateId());
+                    ps.setLong(9, item.getPartsId());
+                    ps.setDouble(10, item.getNum());
+                    ps.setLong(11, item.getOrganId());
+                }
+
+                @Override
+                public int getBatchSize() {
+                    return items.size();
+                }
+            });
+        }
+    }
+
     /**
      * 异步保存数据
      * orderBody、orderItem、orderGroup
@@ -1900,12 +2103,12 @@ public class WebCadOrderImportFactory {
         }
 
         if (hasError.get()) {
-            // 所有任务事务提交
+            // 事务回滚
             transactionManager.rollback(transactionStatus);
             // 抛出异常回滚主线程的数据
             throw new ServiceException(WEBCAD_ORDER_IMPORT_SAVEDATA_ERROR);
         } else {
-            // 所有任务事务回滚
+            // 事务提交
             transactionManager.commit(transactionStatus);
         }
     }