添加配置项

MesETL.App/Services/ETL/MySqlDestination.cs

@@ -66,7 +66,7 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
        
         try
         {
-            var excuseList = GetExcuseList(_recordCache, maxAllowPacket).ToList();
+            var excuseList = GetExcuseList(_recordCache, maxAllowPacket);
             foreach (var insertSql in excuseList)
             {
                 cmd.CommandText = insertSql;
@@ -106,6 +106,7 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
     public IEnumerable<string> GetExcuseList(IDictionary<string, IList<DataRecord>> tableRecords,int maxAllowPacket)
     {
         var sb = new StringBuilder("SET AUTOCOMMIT = 1;\n");
+        var appendCount = 0;
         foreach (var (tableName, records) in tableRecords)
         {
             if (records.Count == 0)
@@ -187,6 +188,14 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
                 // 若字符数量即将大于限制，则返回SQL，清空StringBuilder，保留当前记录的索引值，然后转到StartBuild标签重新开始一轮INSERT
                 if (sb.Length + recordSb.Length + 23 > maxAllowPacket)
                 {
+                    if (appendCount == 0) // 如果单条记录超出maxAllowedPacket
+                    {
+                        sb.Append(recordSb);
+                        _logger.LogWarning("{Table}表单条数据的SQL超出了配置的MaxAllowedPacket，字符数{Count}", tableName,
+                            sb.Length + recordSb.Length + 23);
+                    }
+
+                    TryAddForUpdateSuffix(tableName, sb);
                     sb.Append(';').AppendLine();
                     sb.Append("SET AUTOCOMMIT = 1;");
                     yield return sb.ToString();
@@ -198,8 +207,10 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
                     sb.Append(',').AppendLine();
                 noCommas = false;
                 sb.Append(recordSb); // StringBuilder.Append(StringBuilder)不会分配多余的内存
+                appendCount++;
             }
 
+            TryAddForUpdateSuffix(tableName, sb);
             sb.Append(';');
             sb.Append("COMMIT;");
             yield return sb.ToString();
@@ -207,7 +218,24 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
         }
     }
 
-
+    /// <summary>
+    /// 数据必须是同一张表
+    /// </summary>
+    /// <param name="tableName"></param>
+    /// <param name="sb"></param>
+    private void TryAddForUpdateSuffix(string tableName, StringBuilder sb)
+    {
+        var forUpdate = _options.Value.TryGetForUpdate(tableName, out var forUpdateSql);
+        if (forUpdate)
+        {
+            sb.AppendLine($"""
+                           AS new
+                           ON DUPLICATE KEY UPDATE
+                           {forUpdateSql}
+                           """);
+        }
+    }
+    
     public void Dispose()
     {
         _conn.Close();