多项新特性和更改：

- 添加模拟数据生成器；
- GC时添加碎片整理；
- 优化日志输出，添加更多DEBUG监控项目；
- 修复输出时分库配置逻辑的严重错误；
- 优化了少许内存性能，减少Lambda闭包分配；

MesETL.App/HostedServices/FileInputService.cs

@@ -1,4 +1,5 @@
-using MesETL.App.Const;
+using System.Runtime;
+using MesETL.App.Const;
 using MesETL.App.HostedServices.Abstractions;
 using MesETL.App.Options;
 using MesETL.App.Services;
@@ -65,7 +66,7 @@ public class FileInputService : IInputService
                 if (GC.GetTotalMemory(false) > _memoryThreshold)
                 {
                     _logger.LogWarning("内存使用率过高，暂缓输入");
-                    GC.Collect();
+                    GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
                     GC.Collect();
                     await Task.Delay(3000, cancellationToken);
                 }

MesETL.App/HostedServices/OutputService.cs

@@ -1,4 +1,5 @@
 using System.Buffers;
+using System.Text;
 using MesETL.App.Const;
 using MesETL.App.Helpers;
 using MesETL.App.HostedServices.Abstractions;
@@ -46,7 +47,7 @@ public class OutputService : IOutputService
     public async Task ExecuteAsync(CancellationToken ct)
     {
         _logger.LogInformation("***** 输出服务已启动 *****");
-        var dbTaskManager = new TaskManager(5);
+        var dbTaskManager = new TaskManager(_queuePool.Queues.Count);
         var dbTasks = new Dictionary<string, Task>();
         while (!_context.IsTransformCompleted)
         {
@@ -74,7 +75,7 @@ public class OutputService : IOutputService
         _logger.LogInformation("*****开启输出线程，数据库: {db} *****", db);
         var taskManager = new TaskManager(_outputOptions.Value.MaxDatabaseOutputTask);
         var ignoreOutput = new HashSet<string>(_outputOptions.Value.NoOutput);
-        var tmp = new List<DataRecord>();
+        var tmp = new List<DataRecord>(_outputOptions.Value.FlushCount);
         while (!_context.IsTransformCompleted || queue.Count > 0)
         {
             if (ct.IsCancellationRequested)
@@ -111,6 +112,8 @@ public class OutputService : IOutputService
         // 等待所有子任务完成
         await TaskExtensions.WaitUntil(() => taskManager.RunningTaskCount == 0, 10, ct);
         
+        _logger.LogDebug("输出线程结束，清理剩余记录[{Count}]", tmp.Count);
+        
         // 清理剩余记录
         if (tmp.Count > 0)
         {

MesETL.App/HostedServices/TaskMonitorService.cs

@@ -84,18 +84,21 @@ public class TaskMonitorService
                 _context.MaxMemoryUsage = Math.Max(_context.MaxMemoryUsage, memory);
                 logger.LogStatus("系统监控", new Dictionary<string, string>
                 {
-                    {"输入速度",_context.IsInputCompleted ? "OK" : $"{inputSpeed:F2}/s" },
-                    {"转换速度", _context.IsTransformCompleted ? "OK" : $"{transformSpeed:F2}/s" },
-                    {"输出速度", _context.IsOutputCompleted ? "OK" : $"{outputSpeed:F2}/s" },
-                    
-                    {"| 输入队列长度", _producerQueue.Count.ToString() },
-                    {"输出队列长度", _queuePool.Queues.Values.Sum(queue => queue.Count).ToString()}, 
-                    {"内存使用", $"{memory} MiB"},
+                    { "输入速度", _context.IsInputCompleted ? "OK" : $"{inputSpeed:F2}/s" },
+                    { "转换速度", _context.IsTransformCompleted ? "OK" : $"{transformSpeed:F2}/s" },
+                    { "输出速度", _context.IsOutputCompleted ? "OK" : $"{outputSpeed:F2}/s" },
+
+                    { "| 输入队列长度", _producerQueue.Count.ToString() },
+                    { "输出队列长度", _queuePool.Queues.Values.Sum(queue => queue.Count).ToString() },
+                    { "内存使用", $"{memory} MiB" },
                 });
 
                 var dict = _context.TableProgress
-                    .ToDictionary(kv => kv.Key, kv => $"{kv.Value.input}/{kv.Value.output}");
-                logger.LogStatus("系统监控: 表处理进度", dict, ITaskMonitorLogger.LogLevel.Progress);
+                    .ToDictionary(kv => kv.Key, kv => $"{kv.Value.input}:{kv.Value.output}");
+                logger.LogStatus("系统监控: 表处理进度(I:O)", dict, ITaskMonitorLogger.LogLevel.Progress);
+                logger.LogStatus("系统监控：输出队列状态", 
+                    _queuePool.Queues.ToDictionary(q => q.Key, q => q.Value.Count.ToString()), 
+                    ITaskMonitorLogger.LogLevel.Progress);
                 var sb = new StringBuilder("表处理进度：\n");
                 foreach (var kv in dict)
                 {

MesETL.App/HostedServices/TransformService.cs

@@ -116,7 +116,7 @@ public class TransformService : ITransformService
                         }
                     }
                 }
-                            
+
                 await _queuePool[record.Database].EnqueueAsync(record);
                 _context.AddTransform();
             }

MesETL.App/HostedServices/VoidOutputService.cs

@@ -1,4 +1,5 @@
-using MesETL.App.HostedServices.Abstractions;
+using System.Numerics;
+using MesETL.App.HostedServices.Abstractions;
 using MesETL.App.Services;
 using Microsoft.Extensions.Logging;
 
@@ -11,6 +12,8 @@ public class VoidOutputService : IOutputService
     private readonly RecordQueuePool _queuePool;
     private readonly ProcessContext _context;
 
+    private BigInteger _total;
+
     public VoidOutputService(
         ProcessContext context, ILogger<VoidOutputService> logger, RecordQueuePool queuePool)
     {
@@ -31,13 +34,18 @@ public class VoidOutputService : IOutputService
                     _queuePool.RemoveQueue(pair.Key);
                     continue;
                 }
-                if(!pair.Value.TryDequeue(out var record)) continue;
+
+                if (!pair.Value.TryDequeue(out var record))
+                    continue;
+                
+                _total += record.FieldCharCount;
                 _context.AddOutput();
             }
         }
 
         _context.CompleteOutput();
         _logger.LogInformation("***** Void Output Service Stopped *****");
+        _logger.LogInformation("平均列字符数：{Number}", _total / _context.OutputCount);
         return Task.CompletedTask;
     }
 }