错误修正

MesETL.App/Helpers/DatabaseHelper.cs

@@ -5,9 +5,19 @@ namespace MesETL.App.Helpers;
 
 public static class DatabaseHelper
 {
+    public static MySqlConnection CreateConnection(string connStr)
+    {
+        var newConnStr = new MySqlConnectionStringBuilder(connStr)
+        {
+            ConnectionTimeout = 30,
+            DefaultCommandTimeout = 0,
+        }.ConnectionString;
+        return new MySqlConnection(newConnStr);
+    }
+    
     public static async Task<DataSet> QueryTableAsync(string connStr, string sql)
     {
-        await using var conn = new MySqlConnection(connStr);
+        await using var conn = CreateConnection(connStr);
         if(conn.State is not ConnectionState.Open)
             await conn.OpenAsync();
         await using var cmd = conn.CreateCommand();
@@ -19,7 +29,7 @@ public static class DatabaseHelper
 
     public static async Task<object?> QueryScalarAsync(string connStr, string sql)
     {
-        await using var conn = new MySqlConnection(connStr);
+        await using var conn = CreateConnection(connStr);
         if(conn.State is not ConnectionState.Open)
             await conn.OpenAsync();
         await using var cmd = conn.CreateCommand();
@@ -29,7 +39,7 @@ public static class DatabaseHelper
 
     public static async Task<int> NonQueryAsync(string connStr, string sql)
     {
-        await using var conn = new MySqlConnection(connStr);
+        await using var conn = CreateConnection(connStr);
         if(conn.State is not ConnectionState.Open)
             await conn.OpenAsync();
         await using var cmd = conn.CreateCommand();
@@ -39,7 +49,7 @@ public static class DatabaseHelper
 
     public static async Task<int> TransactionAsync(string connStr, string sql, params MySqlParameter[] parameters)
     {
-        await using var conn = new MySqlConnection(connStr);
+        await using var conn = CreateConnection(connStr);
         if(conn.State is not ConnectionState.Open)
             await conn.OpenAsync();
         await using var trans = await conn.BeginTransactionAsync();

MesETL.App/HostedServices/MainHostedService.cs

@@ -50,6 +50,7 @@ public class MainHostedService : BackgroundService
 
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
     {
+        _logger.LogInformation("Command argument detected, execute for each database");
         var command = _config["Command"];
         if (!string.IsNullOrEmpty(command))
         {

MesETL.App/Program.cs

@@ -131,7 +131,9 @@ async Task RunProgram()
             TableNames.OrderModuleExtra,
             TableNames.OrderModuleItem,
             TableNames.OrderPackage,
+#if USE_TEST_DB
             TableNames.OrderPatchDetail,
+#endif
             
             TableNames.OrderProcess,
             TableNames.OrderProcessStep,
@@ -314,12 +316,6 @@ async Task RunProgram()
             var cache = context.Cacher;
             switch (record.TableName)
             {
-#if USE_TEST_DB
-                // Order表移除IsBatch列
-                case TableNames.Order:
-                    record.RemoveField("IsBatch");
-                    break;
-#endif
                 //OrderBlockPlan将OrderNo长度<2的置空
                 case TableNames.OrderBlockPlan:
                     if (record["OrderNos"].Length < 2)
@@ -347,10 +343,6 @@ async Task RunProgram()
                 // OrderProcess添加ShardKey列，NextStepID的空值转换为0
                 case TableNames.OrderProcess:
                     record.AddField("ShardKey", CalculateShardKeyByOrderNo(record["OrderNo"]));
-#if USE_TEST_DB
-                    if(record["NextStepID"] == "\\N")
-                        record["NextStepID"] = "0";
-#endif
                     break;
                 // OrderProcessStep，OrderProcessStepItem添加ShardKey列
                 case TableNames.OrderProcessStep:
@@ -363,9 +355,6 @@ async Task RunProgram()
                         TableNames.OrderProcessStepItem, TableNames.OrderProcessStep, "OrderProcessID", "脏数据未处理"));
                     break;
                 case TableNames.SimplePlanOrder:
-#if USE_TEST_DB
-                    record.RemoveField("ProcessState");
-#endif
                     record.AddField("Deleted", "0");
                     break;
             }
@@ -531,7 +520,7 @@ async Task RunProgram()
             { "process_item_exp.ItemJson", ColumnType.Text },
             { "report_template.Template", ColumnType.Text },
             { "report_template.SourceConfig", ColumnType.Text },
-            { "order_block_plan.OrderNos", ColumnType.Json },
+            { "order_block_plan.OrderNos", ColumnType.Text },
             { "order_block_plan.BlockInfo", ColumnType.Text },
         };
 #endif

MesETL.App/Services/ETL/MySqlDestination.cs

@@ -137,9 +137,11 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
 
                     // 在这里处理特殊列
                     #region HandleFields
-                    if (field.Length == 2 && field == @"\N") // MyDumper NULL
+
+                    const string NULL = "NULL";
+                    if (field.Length == 2 && field == @"\N") // MyDumper导出的NULL为'\N'（'\'不是转义字符）
                     {
-                        recordSb.Append("NULL");
+                        recordSb.Append(NULL);
                         goto Escape;
                     }
                     
@@ -148,14 +150,18 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
                         case ColumnType.Text:
                             if(string.IsNullOrEmpty(field))
                                 recordSb.Append("''");
+                            else if (field == NULL)
+                                recordSb.Append(NULL);
                             else recordSb.Append($"_utf8mb4 0x{field}");
                             break;
                         case ColumnType.Blob:
                             if (string.IsNullOrEmpty(field))
                                 recordSb.Append("''");
+                            else if (field == NULL)
+                                recordSb.Append(NULL);
                             else recordSb.Append($"0x{field}");
                             break;
-                        case ColumnType.Json:
+                        case ColumnType.Json:// 生产库没有JSON列，仅用于测试库进行测试
                             if(string.IsNullOrEmpty(field))
                                 recordSb.Append("'[]'"); // JObject or JArray?
                             else if (_options.Value.TreatJsonAsHex)

MesETL.App/appsettings.json

@@ -7,7 +7,7 @@
   "Input":{
     "InputDir": "D:\\Dump\\MockData", // Csv数据输入目录
     "UseMock": false, // 使用模拟数据进行测试
-    "MockCountMultiplier": 0.5 // 模拟数据量级的乘数
+    "MockCountMultiplier": 0 // 模拟数据量级的乘数
   },
   "Transform":{
     "StrictMode": false, // 设为true时如果数据转换发生错误，立刻停止程序
@@ -21,7 +21,7 @@
     "MaxAllowedPacket": 67108864, 
     "FlushCount": 10000, // 每次提交记录条数
     "MaxDatabaseOutputTask" : 4, // 每个数据库最大提交任务数
-    "TreatJsonAsHex": false, // 将json列作为16进制格式输出(0x前缀)
+    "TreatJsonAsHex": false // 将json列作为16进制格式输出(0x前缀)，生产库是没有json列的
   },
   "RedisCache": {
     "Configuration": "192.168.1.246:6380",