修复错误的试运行模式

修复输出线程没有捕获异常的严重错误；

MesETL.App/Cache/MemoryCache.cs

@@ -0,0 +1,51 @@
+using System.Collections.Concurrent;
+
+namespace MesETL.App.Cache;
+
+public class MemoryCache : ICacher
+{
+    private readonly ConcurrentDictionary<string, string> _stringCache = new();
+    private readonly ConcurrentDictionary<string, Dictionary<string, string>> _hashCache = new();
+    
+    public static MemoryCache? Instance { get; private set; }
+
+    public MemoryCache()
+    {
+        Instance = this;
+    }
+    
+    public Task<string?> GetStringAsync(string key)
+    {
+        return _stringCache.TryGetValue(key, out var value) ? Task.FromResult<string?>(value) : Task.FromResult((string?)null);
+    }
+
+    public Task SetStringAsync(string key, string value)
+    {
+        _stringCache[key] = value;
+        return Task.CompletedTask;
+    }
+
+    public Task<bool> ExistsAsync(string key)
+    {
+        return Task.FromResult(_stringCache.ContainsKey(key));
+    }
+
+    public Task SetHashAsync(string key, IReadOnlyDictionary<string, string> hash)
+    {
+        _hashCache[key] = hash.ToDictionary(x => x.Key, x => x.Value);
+        return Task.CompletedTask;
+    }
+
+    public Task<Dictionary<string, string>> GetHashAsync(string key)
+    {
+        return Task.FromResult(_hashCache[key]);
+    }
+
+    public void Delete(Func<string,bool> keySelector)
+    {
+        foreach (var k in _stringCache.Keys.Where(keySelector))
+        {
+            _stringCache.TryRemove(k, out _);
+        }
+    }
+}

MesETL.App/Cache/RedisCache.cs

@@ -57,6 +57,7 @@ public static class RedisCacheExtensions
     {
         var conn = ConnectionMultiplexer.Connect(options.Configuration 
                                                  ?? throw new ApplicationException("未配置Redis连接字符串"));
+        services.AddSingleton(conn);
         services.AddSingleton<ICacher>(new RedisCache(conn, options.Database, options.InstanceName));
         return services;
     }

MesETL.App/Const/TableNames.cs

@@ -8,9 +8,11 @@ public static class TableNames
     public const string OrderBlockPlanItem = "order_block_plan_item";
     public const string OrderBlockPlanResult = "order_block_plan_result";
     public const string OrderBoxBlock = "order_box_block";
+    public const string OrderBoxViewConfig = "order_box_view_config";
     public const string OrderDataBlock = "order_data_block";
     public const string OrderDataGoods = "order_data_goods";
     public const string OrderDataParts = "order_data_parts";
+    public const string OrderExtra = "order_extra";
     public const string OrderItem = "order_item";
     public const string OrderModule = "order_module";
     public const string OrderModuleExtra = "order_module_extra";
@@ -23,6 +25,7 @@ public static class TableNames
     public const string OrderProcessStep = "order_process_step";
     public const string OrderProcessStepItem = "order_process_step_item";
     public const string OrderScrapBoard = "order_scrap_board";
+    public const string OrderExtraList = "order_extra_list";
     public const string ProcessGroup = "process_group";
     public const string ProcessInfo = "process_info";
     public const string ProcessItemExp = "process_item_exp";

MesETL.App/DataRecord.cs

@@ -2,35 +2,12 @@
 
 public class DataRecord : ICloneable
 {
-    public static bool TryGetField(DataRecord record, string columnName, out string value)
-    {
-        value = string.Empty;
-        if (record.Headers is null)
-            throw new InvalidOperationException("Cannot get field when headers of a record have not been set.");
-        var idx = IndexOfIgnoreCase(record.Headers, columnName);
-        if (idx == -1)
-            return false;
-        value = record.Fields[idx];
-        return true;
-    }
-
-    public static string GetField(DataRecord record, string columnName)
-    {
-        if (record.Headers is null)
-            throw new InvalidOperationException("Headers have not been set.");
-        var idx = IndexOfIgnoreCase(record.Headers, columnName);
-        if (idx is -1)
-            throw new IndexOutOfRangeException(
-                $"Column name '{columnName}' not found in this record, table name '{record.TableName}'.");
-        return record.Fields[idx];
-    }
-
     private static int IndexOfIgnoreCase(IList<string> list, string value)
     {
         var idx = -1;
         for (var i = 0; i < list.Count; i++)
         {
-            if (list[i].Equals(value, StringComparison.OrdinalIgnoreCase)) 
+            if (list[i].Equals(value, StringComparison.OrdinalIgnoreCase))
             {
                 idx = i;
                 break;
@@ -40,77 +17,167 @@ public class DataRecord : ICloneable
         return idx;
     }
 
-    public IList<string> Fields { get; }
-    public IList<string> Headers { get; }
+    private readonly List<string> _fields;
+    private readonly List<string> _headers;
+
+    /// <summary>
+    /// 字段列表
+    /// </summary>
+    public IReadOnlyList<string> Fields => _fields;
+
+    /// <summary>
+    /// 表头列表
+    /// </summary>
+    public IReadOnlyList<string> Headers => _headers;
+
+    /// <summary>
+    /// 来源表名
+    /// </summary>
     public string TableName { get; }
+
+    /// <summary>
+    /// 需要输出的数据库
+    /// </summary>
     public string? Database { get; set; }
 
+    /// <summary>
+    /// 所有字段的总字符数量
+    /// </summary>
+    public long FieldCharCount { get; }
 
-    public DataRecord(IEnumerable<string> fields, string tableName, IEnumerable<string> headers, string? database = null)
+    /// <summary>
+    /// 忽略这个记录，不会被输出
+    /// </summary>
+    public bool Ignore { get; set; }
+
+
+    public DataRecord(IEnumerable<string> fields, string tableName, IEnumerable<string> headers,
+        string? database = null)
     {
-        Fields = fields.ToList();
+        _fields = fields.ToList();
         TableName = tableName;
-        Headers = headers.ToList();
+        _headers = headers.ToList();
         Database = database;
-        
-        if (Fields.Count != Headers.Count)
+
+        if (_fields.Count != _headers.Count)
             throw new ArgumentException(
-                $"The number of fields does not match the number of headers. Expected: {Headers.Count} Got: {Fields.Count} Fields: {string.Join(',', Fields)}",
+                $"The number of fields does not match the number of headers. Expected: {_headers.Count} Got: {_fields.Count} Fields: {string.Join(',', _fields)}",
                 nameof(fields));
+
+        FieldCharCount = _fields.Sum(x => (long)x.Length);
     }
 
+    /// <summary>
+    /// 使用索引访问字段
+    /// </summary>
+    /// <param name="index"></param>
     public string this[int index]
     {
-        get => Fields[index];
-        set => Fields[index] = value;
+        get => _fields[index];
+        set => _fields[index] = value;
     }
 
+    /// <summary>
+    /// 使用列名访问字段，不区分大小写
+    /// </summary>
+    /// <param name="columnName"></param>
     public string this[string columnName]
     {
-        get => GetField(this, columnName);
+        get => GetField(columnName);
         set => SetField(columnName, value);
     }
 
-    public int FieldCount => Fields.Count;
-
-    public bool TryGetField(string columnName, out string value) => TryGetField(this, columnName, out value);
-
-    public bool SetField(string columnName, string value) => SetField(this, columnName, value);
-
-    public bool SetField(DataRecord record, string columnName, string value)
+    /// <summary>
+    /// 尝试获取某个字段值
+    /// </summary>
+    /// <param name="columnName"></param>
+    /// <param name="value"></param>
+    /// <returns></returns>
+    /// <exception cref="InvalidOperationException"></exception>
+    public bool TryGetField(string columnName, out string value)
     {
-        if (record.Headers is null)
-            throw new InvalidOperationException("Headers have not been set.");
-        var idx = IndexOfIgnoreCase(record.Headers, columnName);
-        if (idx is -1)
-            throw new IndexOutOfRangeException(
-                $"Column name '{columnName}' not found in this record, table name '{record.TableName}");
-        record.Fields[idx] = value;
+        value = string.Empty;
+        if (_headers is null)
+            throw new InvalidOperationException("Cannot get field when headers of a record have not been set.");
+        var idx = IndexOfIgnoreCase(_headers, columnName);
+        if (idx == -1)
+            return false;
+        value = _fields[idx];
         return true;
     }
-    
-    public void AddField(string columnName, string value)
+
+    /// <summary>
+    /// 获取一条记录的某个字段值
+    /// TODO: 最好能优化至O(1)
+    /// </summary>
+    /// <param name="columnName"></param>
+    /// <returns></returns>
+    /// <exception cref="InvalidOperationException"></exception>
+    /// <exception cref="IndexOutOfRangeException"></exception>
+    public string GetField(string columnName)
     {
-        if (IndexOfIgnoreCase(Headers, columnName) != -1)
-            throw new InvalidOperationException($"{TableName}: 列名 '{columnName}' 已存在");
-        
-        Fields.Add(value);
-        Headers.Add(columnName);
-    }
-    
-    public void RemoveField(string columnName)
-    {
-        var idx = IndexOfIgnoreCase(Headers, columnName);
-        if (idx == -1)
-            throw new InvalidOperationException($"{TableName}: 列名 '{columnName}' 不存在");
-        
-        Fields.RemoveAt(idx);
-        Headers.Remove(columnName);
+        if (_headers is null)
+            throw new InvalidOperationException("记录的表头尚未设置，无法赋值");
+        var idx = IndexOfIgnoreCase(_headers, columnName);
+        if (idx is -1)
+            throw new IndexOutOfRangeException(
+                $"列 '{columnName}' 不存在于该纪录中，表名 '{TableName}");
+        return _fields[idx];
     }
 
-    public bool HeaderExists(string columnName) => IndexOfIgnoreCase(Headers, columnName) != -1;
+    /// <summary>
+    /// 为记录的一个字段赋值，如果该字段名不存在则会抛出异常
+    /// </summary>
+    /// <param name="columnName">列名</param>
+    /// <param name="value">值</param>
+    /// <returns></returns>
+    /// <exception cref="InvalidOperationException">该记录的表头尚未初始化，你可能在错误的阶段调用了该方法</exception>
+    /// <exception cref="IndexOutOfRangeException">输入的字段名不存在于该记录中</exception>
+    public void SetField(string columnName, string value)
+    {
+        // 表头检查
+        if (_headers is null)
+            throw new InvalidOperationException("记录的表头尚未设置，无法赋值");
+        var idx = IndexOfIgnoreCase(_headers, columnName);
+        if (idx is -1)
+            throw new IndexOutOfRangeException(
+                $"列 '{columnName}' 不存在于该纪录中，表名 '{TableName}");
+        _fields[idx] = value;
+    }
+
+    /// <summary>
+    /// 在记录中追加一个字段
+    /// </summary>
+    /// <param name="columnName">字段名</param>
+    /// <param name="value">字段值</param>
+    /// <exception cref="InvalidOperationException">记录的表头尚未初始化，你可能在错误的阶段调用了此方法</exception>
+    /// <exception cref="ArgumentException">提供的字段名已存在于该记录中</exception>
+    public void AppendField(string columnName, string value)
+    {
+        if (_headers is null)
+            throw new InvalidOperationException("记录的表头尚未设置，无法赋值");
+        var idx = IndexOfIgnoreCase(_headers, columnName);
+        if (idx is > 0)
+            throw new ArgumentException($"字段名 '{columnName}' 已存在于该记录中，无法重复添加", nameof(columnName));
+
+        _headers.Add(columnName);
+        _fields.Add(value);
+    }
+
+    public void RemoveField(string columnName)
+    {
+        var idx = IndexOfIgnoreCase(_headers, columnName);
+        if (idx == -1)
+            throw new InvalidOperationException($"{TableName}: 列名 '{columnName}' 不存在");
+
+        _fields.RemoveAt(idx);
+        _headers.Remove(columnName);
+    }
+
+    public bool HeaderExists(string columnName) => IndexOfIgnoreCase(_headers, columnName) != -1;
+
     public object Clone()
     {
-        return new DataRecord(new List<string>(Fields), TableName, new List<string>(Headers), Database);
+        return new DataRecord(new List<string>(_fields), TableName, new List<string>(_headers), Database);
     }
 }

MesETL.App/Helpers/DumpDataHelper.cs

@@ -1,5 +1,7 @@
 using System.Text.Json;
 using System.Text.RegularExpressions;
+using MesETL.App.HostedServices;
+using Serilog;
 using ZstdSharp;
 
 namespace MesETL.App.Helpers;
@@ -27,16 +29,13 @@ public static partial class DumpDataHelper
         string[] ParseHeader(ReadOnlySpan<char> headerStr)
         {
             headerStr = headerStr[1..^1];
-            Span<Range> ranges = stackalloc Range[50];
-            var count = headerStr.Split(ranges, ',');
-            var arr = new string[count];
-
-            for (var i = 0; i < count; i++)
+            var headers = new List<string>();
+            foreach (var range in headerStr.Split(','))
             {
-                arr[i] = headerStr[ranges[i]].Trim("@`").ToString(); // 消除列名的反引号，如果是变量则消除@
+                headers.Add(headerStr[range].Trim("@`").ToString()); // 消除列名的反引号，如果是变量则消除@
             }
-        
-            return arr;
+
+            return headers.ToArray();
         }
     }
     
@@ -45,6 +44,7 @@ public static partial class DumpDataHelper
     /// </summary>
     /// <param name="filePath"></param>
     /// <returns></returns>
+    [Obsolete("用ParseMyDumperFile替代")]
     public static string GetTableNameFromCsvFileName(ReadOnlySpan<char> filePath)
     {
         filePath = filePath[(filePath.LastIndexOf('\\') + 1)..];
@@ -68,6 +68,30 @@ public static partial class DumpDataHelper
 
         return filePath[(firstDotIdx+1)..secondDotIdx].ToString();
     }
+    
+    public enum MyDumperFileType { Dat, Sql }
+
+    public record MyDumperFileMeta(string Path, string Database, string TableName, int Index, MyDumperFileType Type);
+
+    public static MyDumperFileMeta ParseMyDumperFile(ReadOnlySpan<char> path)
+    {
+        try
+        {
+            var fileName = Path.GetFileName(path).ToString();
+            var parts = fileName.Split('.');
+            var type = parts[3] switch
+            {
+                "dat" => MyDumperFileType.Dat,
+                "sql" => MyDumperFileType.Sql,
+                _ => throw new ArgumentException("不支持的MyDumper文件类型", nameof(path))
+            };
+            return new MyDumperFileMeta(path.ToString(), parts[0], parts[1], int.Parse(parts[2]), type);
+        }
+        catch (Exception e)
+        {
+            throw new ArgumentException($"此文件不是MyDumper导出的文件 {path}", nameof(path), e);
+        }
+    }
 
     /// <summary>
     /// 从MyDumper导出的SQL文件内容中读取CSV文件名
@@ -122,17 +146,46 @@ public static partial class DumpDataHelper
         var reader = new StreamReader(ds);
         return await reader.ReadToEndAsync();
     }
-    
-    public static bool IsJson(string str)
+
+    /// <summary>
+    /// 适用于文件输入服务以及MyDumper Zst导出目录的文件元数据构建函数
+    /// </summary>
+    /// <param name="filePath"></param>
+    /// <returns></returns>
+    /// <exception cref="ApplicationException"></exception>
+    public static FileInputInfo? MyDumperFileInputMetaBuilder(string filePath)
     {
+        // 只查找后缀为.dat.zst的文件
+        if (!filePath.EndsWith(".dat.zst")) return null;
+            
+        var fileMeta = ParseMyDumperFile(filePath);
+        var inputDir = Path.GetDirectoryName(filePath);
+        string[]? headers;
         try
         {
-            JsonDocument.Parse(str);
-            return true;
+            // 查找同目录下同表的SQL文件
+            var sqlFile = Directory.GetFiles(inputDir!)
+                .SingleOrDefault(f => f.Equals(filePath.Replace(".dat.zst", ".sql.zst")));
+            if (sqlFile is null)
+            {
+                Log.Debug("{TableName}表的SQL文件不存在", fileMeta.TableName);
+                return null;
+            }
+            headers = GetCsvHeadersFromSqlFile(
+                DecompressZstAsStringAsync(File.OpenRead(sqlFile)).Result);
         }
-        catch (JsonException)
+        catch (InvalidOperationException e)
         {
-            return false;
+            throw new ApplicationException($"目录下不止一个{fileMeta.TableName}表的SQL文件", e);
         }
+
+        return new FileInputInfo
+        {
+            FileName = filePath,
+            TableName = fileMeta.TableName,
+            Headers = headers,
+            Database = fileMeta.Database,
+            Part = fileMeta.Index
+        };
     }
 }

MesETL.App/HostedServices/FileInputService.cs

@@ -1,7 +1,10 @@
-using MesETL.App.HostedServices.Abstractions;
+using System.Runtime;
+using MesETL.App.Const;
+using MesETL.App.HostedServices.Abstractions;
 using MesETL.App.Options;
 using MesETL.App.Services;
 using MesETL.App.Services.ETL;
+using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
@@ -12,16 +15,11 @@ public record FileInputInfo
 {
     public required string FileName { get; init; }
     public required string TableName { get; init; }
+    public required string Database { get; init; }
+    public required int Part { get; init; }
     public required string[] Headers { get; init; }
 }
 
-public enum FileInputType
-{
-    MyDumperCsv,
-    MyDumperZst,
-    ErrorLog,
-}
-
 /// <summary>
 /// 从输入目录中导入文件
 /// </summary>
@@ -32,56 +30,102 @@ public class FileInputService : IInputService
     private readonly IOptions<DataInputOptions> _dataInputOptions;
     private readonly ProcessContext _context;
     private readonly DataReaderFactory _dataReaderFactory;
+    private readonly long _memoryThreshold;
+    private readonly bool _dryRun;
+    private readonly int _dryRunCount;
 
     public FileInputService(ILogger<FileInputService> logger,
             IOptions<DataInputOptions> dataInputOptions,
             ProcessContext context, 
             [FromKeyedServices(Const.ConstVar.Producer)] DataRecordQueue producerQueue, 
-            DataReaderFactory dataReaderFactory)
+            DataReaderFactory dataReaderFactory, 
+            IConfiguration configuration)
     {
         _logger = logger;
         _dataInputOptions = dataInputOptions;
         _context = context;
         _producerQueue = producerQueue;
         _dataReaderFactory = dataReaderFactory;
+        _memoryThreshold = (long)(configuration.GetValue<double>("MemoryThreshold", 8) * 1024 * 1024 * 1024);
+        _dryRun = configuration.GetValue("DryRun", false);
+        _dryRunCount = configuration.GetValue("DryRunCount", 100_000);
     }
 
     public async Task ExecuteAsync(CancellationToken cancellationToken)
     {
         var inputDir = _dataInputOptions.Value.InputDir ?? throw new ApplicationException("未配置文件输入目录");
-        _logger.LogInformation("***** Input service started, working directory: {InputDir} *****", inputDir);
+        _logger.LogInformation("***** 输入服务已启动，工作目录为：{InputDir} *****", inputDir);
+        if (_dryRun)
+            _logger.LogInformation("***** 试运行模式已开启，只读取前 {Count} 行数据 *****", _dryRunCount);
+
+        var orderedInfo = GetOrderedInputInfo(inputDir);
         
-        var trans = _dataInputOptions.Value.FileInputMetaBuilder;
-        if(trans is null) throw new ApplicationException("未配置文件名-表名映射委托");
-        FileInputInfo[] infoArr = Directory.GetFiles(inputDir)
-            .Select(f => trans(f))
+        foreach (var info in orderedInfo)
+        {
+            var file = Path.GetFileName(info.FileName);
+            _logger.LogInformation("正在读取文件：{FileName}, 对应的数据表：{TableName}", file, info.TableName);
+            using var source = _dataReaderFactory.CreateReader(info.FileName, info.TableName, info.Headers);
+            var countBuffer = 0;
+            
+            if (_dryRun && _context.TableProgress.GetValueOrDefault(info.TableName, (input: 0, output: 0)).input >= _dryRunCount)
+                continue;
+            
+            while (await source.ReadAsync())
+            {
+                if (GC.GetTotalMemory(false) > _memoryThreshold)
+                {
+                    _logger.LogWarning("内存使用率过高，暂缓输入");
+                    GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
+                    GC.Collect();
+                    await Task.Delay(3000, cancellationToken);
+                }
+                var record = source.Current;
+                await _producerQueue.EnqueueAsync(record);
+                countBuffer++;
+                _context.AddInput();
+                
+                // 避免影响性能，每1000条更新一次表输入进度
+                if (countBuffer >= 1000)
+                {
+                    _context.AddTableInput(info.TableName, countBuffer);
+                    countBuffer = 0;
+                    // 试运行模式下，超出了指定行数则停止输入
+                    if (_dryRun && _context.TableProgress[info.TableName].input >= _dryRunCount)
+                    {
+                        break;
+                    }
+                }
+            }
+
+            _context.AddTableInput(info.TableName, countBuffer);
+            _logger.LogInformation("文件 {File} 输入完成", file);
+            _dataInputOptions.Value.OnTableInputCompleted?.Invoke(info.TableName);
+        }
+
+        _context.CompleteInput();
+        _logger.LogInformation("***** 输入服务{DryRun}已执行完毕 *****", _dryRun ? " (试运行)" : "");
+    }
+
+    public IEnumerable<FileInputInfo> GetOrderedInputInfo(string dir)
+    {
+        var metaBuilder = _dataInputOptions.Value.FileInputMetaBuilder;
+        if(metaBuilder is null) throw new ApplicationException("未配置文件名->表名的映射委托函数");
+        var files = Directory.GetFiles(dir);
+        FileInputInfo[] infoArr = files
+            .Select(f => metaBuilder(f))
             .Where(info => info is not null).ToArray()!;
         
         var orderedInfo = GetFilesInOrder(infoArr).ToArray();
         
-        _logger.LogInformation("***** {Count} files founded in directory，{OrderedCount} files is matched with configuration *****", infoArr.Length, orderedInfo.Length);
-        foreach (var info in orderedInfo)
+        _logger.LogInformation("***** 输入目录中发现 {Count} 个文件， {InfoCount} 个文件可用，{OrderedCount} 个文件符合当前输入配置 *****", 
+            files.Length, infoArr.Length, orderedInfo.Length);
+        foreach (var info in orderedInfo.GroupBy(i => i.TableName))
         {
-            _logger.LogDebug("Table {TableName}: {FileName}", info.TableName, info.FileName);
+            _logger.LogDebug("表 {TableName} 发现 {FileCount} 个对应文件：\n{FileName}", 
+                info.Key, info.Count(), string.Join('\n', info.Select(f => f.FileName)));
         }
 
-        foreach (var info in orderedInfo)
-        {
-            _logger.LogInformation("Reading file: {FileName}, table: {TableName}", info.FileName, info.TableName);
-            using var source = _dataReaderFactory.CreateReader(info.FileName,info.TableName,info.Headers);
-
-            while (await source.ReadAsync())
-            {
-                var record = source.Current;
-                _producerQueue.Enqueue(record);
-                _context.AddInput();
-            }
-            
-            _logger.LogInformation("Input of table: '{TableName}' finished", info.TableName);
-        }
-
-        _context.CompleteInput();
-        _logger.LogInformation("***** Input service finished *****");
+        return orderedInfo;
     }
 
     /// <summary>
@@ -90,7 +134,8 @@ public class FileInputService : IInputService
     /// <returns></returns>
     private IEnumerable<FileInputInfo> GetFilesInOrder(FileInputInfo[] inputFiles)
     {
-        var tableOrder = _dataInputOptions.Value.TableOrder;
+        var tableOrder = _dataInputOptions.Value.TableOrder ?? typeof(TableNames).GetFields().Select(f => f.GetValue(null) as string).ToArray();
+        var ignoreTable = _dataInputOptions.Value.TableIgnoreList;
         if (tableOrder is null or { Length: 0 })
             return inputFiles;
 
@@ -100,10 +145,14 @@ public class FileInputService : IInputService
         {
             foreach (var tableName in tableOrder)
             {
-                var target = inputFiles.FirstOrDefault(f =>
-                    f.TableName.Equals(tableName, StringComparison.OrdinalIgnoreCase));
-                if (target is not null)
+                var targets = inputFiles.Where(f =>
+                    f.TableName.Equals(tableName, StringComparison.OrdinalIgnoreCase) &&
+                    !ignoreTable.Contains(f.TableName));
+
+                foreach (var target in targets)
+                {
                     yield return target;
+                }
             }
         }
     }

MesETL.App/HostedServices/MainHostedService.cs

@@ -5,6 +5,7 @@ using MesETL.App.HostedServices.Abstractions;
 using MesETL.App.Options;
 using MesETL.App.Services;
 using MesETL.App.Services.ErrorRecorder;
+using MesETL.Shared.Helper;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
@@ -50,17 +51,20 @@ public class MainHostedService : BackgroundService
 
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
     {
-        _logger.LogInformation("Command argument detected, execute for each database");
         var command = _config["Command"];
         if (!string.IsNullOrEmpty(command))
         {
-            _logger.LogInformation("***** Running Sql Command *****");
+            _logger.LogInformation("检测到命令参数传入，将对所有配置的数据库执行输入的命令。。。");
+            _logger.LogInformation("***** 执行SQL命令 *****");
             await ExecuteEachDatabase(command, stoppingToken);
+            _logger.LogInformation("***** 执行完成 *****");
             Environment.Exit(0);
         }
         
         _stopwatch = Stopwatch.StartNew();
-        await SetVariableAsync(); // 开启延迟写入，禁用重做日志 >>> 重做日志处于禁用状态时不要关闭数据库服务！
+        var enableUnsafeVar = _config.GetValue<bool>("UnsafeVariable", false);
+        if (enableUnsafeVar)
+            await SetVariableAsync(); // 开启延迟写入，禁用重做日志 >>> 重做日志处于禁用状态时不要关闭数据库服务！
 
         var monitorTask = Task.Run(async () => await _taskMonitor.Monitor(stoppingToken), stoppingToken);
         var inputTask = ExecuteAndCatch(
@@ -72,15 +76,18 @@ public class MainHostedService : BackgroundService
 
         await Task.WhenAll(inputTask, transformTask, outputTask);
         _stopwatch.Stop();
-        _logger.LogInformation("***** All tasks completed *****");
-        _logger.LogInformation("***** ElapseTime: {Time}", (_stopwatch.ElapsedMilliseconds / 1000f).ToString("F3"));
+        _logger.LogInformation("***** 所有传输任务均已完成 *****");
+        if (_context.HasException)
+            _logger.LogError("***** 传输过程中有错误发生 *****");
+        _logger.LogInformation("***** 耗时：{Time}", (_stopwatch.ElapsedMilliseconds / 1000f).ToString("F3"));
         await Task.Delay(5000, stoppingToken);
 
-        await SetVariableAsync(false); // 关闭延迟写入，开启重做日志
+        if(enableUnsafeVar)
+            await SetVariableAsync(false); // 关闭延迟写入，开启重做日志
         if (!stoppingToken.IsCancellationRequested)
         {
             await ExportResultAsync();
-            _logger.LogInformation("The execution result export to {Path}",
+            _logger.LogInformation("传输结果已保存至 {Path}",
                 Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"Result-{ErrorRecorder.UID}.md"));
 
             Environment.Exit(0);
@@ -109,6 +116,10 @@ public class MainHostedService : BackgroundService
     {
         var connStr = _databaseOptions.Value.ConnectionString
                       ?? throw new ApplicationException("分库配置中没有配置数据库");
+        if (enable)
+            _logger.LogWarning("已开启MySQL延迟写入功能并禁用重做日志，请注意数据安全");
+        else _logger.LogInformation("不安全变量已关闭");
+        
         if (enable)
         {
             await DatabaseHelper.NonQueryAsync(connStr,
@@ -152,33 +163,41 @@ public class MainHostedService : BackgroundService
     private async Task ExportResultAsync()
     {
         var sb = new StringBuilder();
-        if (_context.HasException)
-            sb.AppendLine("# Program Completed With Error");
-        else sb.AppendLine("# Program Completed Successfully");
-        sb.AppendLine("## Process Count");
+
+        var title = (_config.GetValue("DryRun", false), _context.HasException) switch
+        {
+            (true, true) => "# 试运行结束，**请注意处理异常**",
+            (true, false) => "# 试运行结束，没有发生异常",
+            (false, true) => "# 程序执行完毕，**但中途发生了异常**",
+            (false, false) => "# 程序执行完毕，没有发生错误"
+        };
+        sb.AppendLine(title);
+          
+        sb.AppendLine("## 处理计数");
         var processCount = new[]
         {
-            new { State = "Input", Count = _context.InputCount },
-            new { State = "Transform", Count = _context.TransformCount },
-            new { State = "Output", Count = _context.OutputCount }
+            new { 操作 = "输入", 数量 = _context.InputCount },
+            new { 操作 = "转换", 数量 = _context.TransformCount },
+            new { 操作 = "输出", 数量 = _context.OutputCount }
         };
         sb.AppendLine(processCount.ToMarkdownTable());
         sb.AppendLine("\n---\n");
-        sb.AppendLine("## Table Output Progress");
+        sb.AppendLine("## 表输入/输出计数");
         var tableOutputProgress = _context.TableProgress.Select(pair =>
-            new { Table = pair.Key, Count = pair.Value });
+            new { 表名 = pair.Key, 计数 = pair.Value }).OrderBy(s => s.表名);
         sb.AppendLine(tableOutputProgress.ToMarkdownTable());
         sb.AppendLine("\n---\n");
-        sb.AppendLine("## Result");
+        sb.AppendLine("## 总览");
         var elapsedTime = (_stopwatch!.ElapsedMilliseconds / 1000f);
         var result = new[]
         {
-            new { Field = "ElapsedTime", Value = elapsedTime.ToString("F2") },
+            new { 条目 = "耗时", 值 = elapsedTime.ToString("F2") + " 秒" },
             new
             {
-                Field = "Average Output Speed",
-                Value = (_context.OutputCount / elapsedTime).ToString("F2") + "records/s"
-            }
+                条目 = "平均处理速度",
+                值 = (_context.OutputCount / elapsedTime).ToString("F2") + " 条记录/秒"
+            },
+            new { 条目 = "内存占用峰值", 值 = _context.MaxMemoryUsage + " 兆字节" }
         };
         sb.AppendLine(result.ToMarkdownTable());
         await File.WriteAllTextAsync(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"Result-{ErrorRecorder.UID}.md"),

MesETL.App/HostedServices/OutputService.cs

@@ -1,16 +1,22 @@
-using MesETL.App.Helpers;
+using System.Buffers;
+using System.Text;
+using MesETL.App.Const;
+using MesETL.App.Helpers;
 using MesETL.App.HostedServices.Abstractions;
 using MesETL.App.Options;
 using MesETL.App.Services;
 using MesETL.App.Services.ErrorRecorder;
+using MesETL.Shared.Helper;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using MySqlConnector;
 using MySqlDestination = MesETL.App.Services.ETL.MySqlDestination;
-using TaskExtensions = MesETL.App.Helpers.TaskExtensions;
+using TaskExtensions = MesETL.Shared.Helper.TaskExtensions;
 
 namespace MesETL.App.HostedServices;
 
+public record DataOutputContext(IServiceProvider Serivces);
+
 /// <summary>
 /// 数据导出服务，将数据导出至MySql服务
 /// </summary>
@@ -21,24 +27,27 @@ public class OutputService : IOutputService
     private readonly ProcessContext _context;
     private readonly ErrorRecorderFactory _errorRecorderFactory;
     private readonly RecordQueuePool _queuePool;
+    private readonly IServiceProvider _services;
 
     public OutputService(ILogger<OutputService> logger,
         IOptions<DatabaseOutputOptions> outputOptions,
         ProcessContext context,
         RecordQueuePool queuePool, 
-        ErrorRecorderFactory errorRecorderFactory)
+        ErrorRecorderFactory errorRecorderFactory, 
+        IServiceProvider services)
     {
         _logger = logger;
         _outputOptions = outputOptions;
         _context = context;
         _queuePool = queuePool;
         _errorRecorderFactory = errorRecorderFactory;
+        _services = services;
     }
 
     public async Task ExecuteAsync(CancellationToken ct)
     {
-        _logger.LogInformation("***** Output service started *****");
-        var dbTaskManager = new TaskManager(5);
+        _logger.LogInformation("***** 输出服务已启动 *****");
+        var dbTaskManager = new TaskManager(_queuePool.Queues.Count);
         var dbTasks = new Dictionary<string, Task>();
         while (!_context.IsTransformCompleted)
         {
@@ -47,7 +56,18 @@ public class OutputService : IOutputService
                 if (!dbTasks.ContainsKey(db))
                 {
                     dbTasks.Add(db, await dbTaskManager.CreateTaskAsync(
-                        async () => await StartDatabaseWorker(db, queue, ct), ct));
+                        async () =>
+                        {
+                            try
+                            {
+                                await StartDatabaseWorker(db, queue, ct);
+                            }
+                            catch (Exception e)
+                            {
+                                _logger.LogError(e, "输出线程发生错误");
+                                _queuePool.RemoveQueue(db);
+                            }
+                        }, ct));
                 }
             }
 
@@ -57,20 +77,23 @@ public class OutputService : IOutputService
         await TaskExtensions.WaitUntil(() => dbTaskManager.RunningTaskCount == 0, 25, ct);
         
         _context.CompleteOutput();
-        _logger.LogInformation("***** Output service finished *****");
+        _outputOptions.Value.OutputFinished?.Invoke(new DataOutputContext(_services));
+        _logger.LogInformation("***** 输出服务执行完毕 *****");
     }
 
     private async Task StartDatabaseWorker(string db, DataRecordQueue queue, CancellationToken ct = default)
     {
         _logger.LogInformation("*****开启输出线程，数据库: {db} *****", db);
         var taskManager = new TaskManager(_outputOptions.Value.MaxDatabaseOutputTask);
-        var tmp = new List<DataRecord>();
+        var ignoreOutput = new HashSet<string>(_outputOptions.Value.NoOutput);
+        var tmp = new List<DataRecord>(_outputOptions.Value.FlushCount);
         while (!_context.IsTransformCompleted || queue.Count > 0)
         {
             if (ct.IsCancellationRequested)
                 break;
-            
-            if (!queue.TryDequeue(out var record)) continue;
+
+            if (!queue.TryDequeue(out var record) || record.Ignore || ignoreOutput.Contains(record.TableName))
+                continue;
             
             var dbName = record.Database ?? throw new ApplicationException("输出的记录缺少数据库名");
             if(dbName != db)
@@ -100,13 +123,15 @@ public class OutputService : IOutputService
         // 等待所有子任务完成
         await TaskExtensions.WaitUntil(() => taskManager.RunningTaskCount == 0, 10, ct);
         
+        _logger.LogDebug("输出线程结束，清理剩余记录[{Count}]", tmp.Count);
+        
         // 清理剩余记录
         if (tmp.Count > 0)
         {
             await FlushAsync(db, tmp);
         }
 
-        _logger.LogInformation("*****输出线程结束，数据库: {db} *****", db);
+        _logger.LogInformation("***** 输出线程结束，数据库: {db} *****", db);
     }
 
     private async Task FlushAsync(string dbName, IEnumerable<DataRecord> records)
@@ -136,8 +161,9 @@ public class OutputService : IOutputService
         await output.FlushAsync(_outputOptions.Value.MaxAllowedPacket);
         foreach (var (key, value) in tableOutput)
         {
+            _context.AddOutput(value);
             _context.AddTableOutput(key, value);
         }
-        _logger.LogTrace("Flushed {Count} records", tableOutput.Values.Sum(i => i));
+        // _logger.LogTrace("输出任务：刷新了 {Count} 条记录", tableOutput.Values.Sum(i => i));
     }
 }

MesETL.App/HostedServices/TaskMonitorService.cs

@@ -1,4 +1,5 @@
 using System.Diagnostics;
+using System.Text;
 using MesETL.App.Services;
 using MesETL.App.Services.Loggers;
 using Microsoft.Extensions.DependencyInjection;
@@ -14,6 +15,8 @@ public class TaskMonitorService
     private readonly ProcessContext _context;
     private readonly DataRecordQueue _producerQueue;
     private readonly RecordQueuePool _queuePool;
+    
+    private string _outputPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Log/progress.txt");
 
     public TaskMonitorService(ProcessContext context,
         [FromKeyedServices(Const.ConstVar.Producer)]
@@ -77,32 +80,43 @@ public class TaskMonitorService
             //     running, error, completed, canceled, outputSpeed);
             foreach (var logger in _monitorLoggers)
             {
-                logger.LogStatus("Monitor: Progress status", new Dictionary<string, string>
+                var memory = GC.GetTotalMemory(false) / 1024 / 1024;
+                _context.MaxMemoryUsage = Math.Max(_context.MaxMemoryUsage, memory);
+                logger.LogStatus("系统监控", new Dictionary<string, string>
                 {
-                    {"Input",_context.IsInputCompleted ? "completed" : $"running {inputSpeed:F2} records/s" },
-                    {"Transform", _context.IsTransformCompleted ? "completed" : $"running {transformSpeed:F2} records/s" },
-                    {"Output", _context.IsOutputCompleted ? "completed" : $"running {outputSpeed:F2} records/s" }
+                    { "输入速度", _context.IsInputCompleted ? "OK" : $"{inputSpeed:F2}/s" },
+                    { "转换速度", _context.IsTransformCompleted ? "OK" : $"{transformSpeed:F2}/s" },
+                    { "输出速度", _context.IsOutputCompleted ? "OK" : $"{outputSpeed:F2}/s" },
+
+                    { "| 输入队列长度", _producerQueue.Count.ToString() },
+                    { "输出队列长度", _queuePool.Queues.Values.Sum(queue => queue.Count).ToString() },
+                    { "内存使用", $"{memory} MiB" },
                 });
 
-                logger.LogStatus("Monitor: Table output progress",
-                    _context.TableProgress
-                        .ToDictionary(kv => kv.Key, kv => kv.Value.ToString()),
+                var dict = _context.TableProgress
+                    .ToDictionary(kv => kv.Key, kv => $"{kv.Value.input}:{kv.Value.output}");
+                logger.LogStatus("系统监控: 表处理进度(I:O)", dict, ITaskMonitorLogger.LogLevel.Progress);
+                logger.LogStatus("系统监控：输出队列状态", 
+                    _queuePool.Queues.ToDictionary(q => q.Key, q => q.Value.Count.ToString()), 
                     ITaskMonitorLogger.LogLevel.Progress);
-            
-                logger.LogStatus("Monitor: Process count", new Dictionary<string, string>
+                var sb = new StringBuilder("表处理进度：\n");
+                foreach (var kv in dict)
                 {
-                    {"Input", inputCount.ToString()},
-                    {"Transform", transformCount.ToString()},
-                    {"Output", outputCount.ToString()}
-                }, ITaskMonitorLogger.LogLevel.Progress);
+                    sb.Append(kv.Key).AppendLine(kv.Value);
+                }
+
+                sb.AppendLine($"数据记录字段的最大长度：{_producerQueue.LongestFieldCharCount}");
+                await File.WriteAllTextAsync(_outputPath, sb.ToString(), CancellationToken.None);
+                
             
-                logger.LogStatus("Monitor: Queue", new Dictionary<string, string>
-                {
-                    {"Producer queue records", _producerQueue.Count.ToString() },
-                    {"Output queues", _queuePool.Queues.Count.ToString() },
-                    {"Output queue records", _queuePool.Queues.Values.Sum(queue => queue.Count).ToString()}, 
-                });
+                // logger.LogStatus("Monitor: Process count", new Dictionary<string, string>
+                // {
+                //     {"Input", inputCount.ToString()},
+                //     {"Transform", transformCount.ToString()},
+                //     {"Output", outputCount.ToString()}
+                // }, ITaskMonitorLogger.LogLevel.Progress);
             }
+
             
             await Task.Delay(5000, stoppingToken);
 

MesETL.App/HostedServices/TransformService.cs

@@ -1,4 +1,5 @@
 using MesETL.App.Cache;
+using MesETL.App.Const;
 using MesETL.App.HostedServices.Abstractions;
 using MesETL.App.Options;
 using MesETL.App.Services;
@@ -9,7 +10,7 @@ using Microsoft.Extensions.Options;
 
 namespace MesETL.App.HostedServices;
 
-public record DataTransformContext(DataRecord Record, ICacher Cacher, ILogger Logger);
+public record DataTransformContext(DataRecord Record, ICacher Cacher, ILogger Logger, IServiceProvider Services);
 
 /// <summary>
 /// 数据处理服务，对导入后的数据进行处理
@@ -23,6 +24,7 @@ public class TransformService : ITransformService
     private readonly ProcessContext _context;
     private readonly ICacher _cache;
     private readonly ErrorRecorderFactory _errorRecorderFactory;
+    private readonly IServiceProvider _services;
 
 
     public TransformService(ILogger<TransformService> logger, 
@@ -31,7 +33,8 @@ public class TransformService : ITransformService
         RecordQueuePool queuePool,
         ProcessContext context,
         ICacher cache, 
-        ErrorRecorderFactory errorRecorderFactory)
+        ErrorRecorderFactory errorRecorderFactory,
+        IServiceProvider services)
     {
         _logger = logger;
         _options = options;
@@ -40,19 +43,32 @@ public class TransformService : ITransformService
         _context = context;
         _cache = cache;
         _errorRecorderFactory = errorRecorderFactory;
+        _services = services;
     }
 
     public async Task ExecuteAsync(CancellationToken cancellationToken)
     {
-        _logger.LogInformation("***** Data transform service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId);
+        _logger.LogInformation("***** 数据转换服务已启动 *****");
+        
+        await TransformWorker2();
+        
+        _context.CompleteTransform();
+        _logger.LogInformation("***** 数据转换服务执行完毕 *****");
+    }
 
+    public async Task TransformWorker(DataRecordQueue queue)
+    {
         while (!_context.IsInputCompleted || _producerQueue.Count > 0)
         {
-            if (!_producerQueue.TryDequeue(out var record)) continue;
-
+            if (!_producerQueue.TryDequeue(out var record))
+            {
+                await Task.Delay(100);
+                continue;
+            }
+            
             try
             {
-                var context = new DataTransformContext(record, _cache, _logger);
+                var context = new DataTransformContext(record, _cache, _logger, _services);
                 if (_options.Value.EnableFilter)
                 {
                     // 数据过滤
@@ -68,7 +84,7 @@ public class TransformService : ITransformService
                     {
                         record = await replacer(context);
                     }
-                }    
+                }
                 
                 // 字段缓存
                 var cacher = _options.Value.RecordCache;
@@ -79,9 +95,6 @@ public class TransformService : ITransformService
                 var dbFilter = _options.Value.DatabaseFilter 
                                ?? throw new ApplicationException("未配置数据库过滤器");
                 record.Database = dbFilter(record);
-            
-                _queuePool[record.Database].Enqueue(record);
-                _context.AddTransform();
 
                 if (_options.Value.EnableReBuilder)
                 {
@@ -92,12 +105,86 @@ public class TransformService : ITransformService
                         foreach (var rc in addRecords)
                         {
                             if(dbFilter is not null)
-                                rc.Database =dbFilter.Invoke(record);
-                            _queuePool[record.Database].Enqueue(rc);
+                                rc.Database = dbFilter.Invoke(record);
+                            await queue.EnqueueAsync(rc);
                             _context.AddTransform();
                         }
                     }
                 }
+
+                await queue.EnqueueAsync(record);
+                _context.AddTransform();
+            }
+            catch (Exception e)
+            {
+                _context.AddException(e);
+                var errorRecorder = _errorRecorderFactory.CreateTransform();
+                await errorRecorder.LogErrorRecordAsync(record, e);
+                if (!_options.Value.StrictMode)
+                    _logger.LogError(e, "数据转换时发生错误");
+                else throw;
+            }
+        }
+    }
+    
+    public async Task TransformWorker2()
+    {
+        while (!_context.IsInputCompleted || _producerQueue.Count > 0)
+        {
+            if (!_producerQueue.TryDequeue(out var record))
+            {
+                await Task.Delay(100);
+                continue;
+            }
+            
+            try
+            {
+                var context = new DataTransformContext(record, _cache, _logger, _services);
+                if (_options.Value.EnableFilter)
+                {
+                    // 数据过滤
+                    var filter = _options.Value.RecordFilter;
+                    if (filter is not null && await filter(context) == false) continue;
+                }
+
+                if (_options.Value.EnableReplacer)
+                {
+                    // 数据替换
+                    var replacer = _options.Value.RecordModify;
+                    if (replacer is not null)
+                    {
+                        record = await replacer(context);
+                    }
+                }
+                
+                // 字段缓存
+                var cacher = _options.Value.RecordCache;
+                if(cacher is not null)
+                    await cacher.Invoke(context);
+            
+                //计算需要分流的数据库
+                var dbFilter = _options.Value.DatabaseFilter 
+                               ?? throw new ApplicationException("未配置数据库过滤器");
+                record.Database = dbFilter(record);
+
+                if (_options.Value.EnableReBuilder)
+                {
+                    //数据重建
+                    var addRecords = _options.Value.RecordReBuild?.Invoke(context);
+                    if (addRecords is { Count: > 0 })
+                    {
+                        foreach (var rc in addRecords)
+                        {
+                            if(dbFilter is not null)
+                                rc.Database = dbFilter.Invoke(record);
+                            await _queuePool[record.Database].EnqueueAsync(rc);
+                            _context.AddTransform();
+                        }
+                    }
+                }
+
+                await _queuePool[record.Database].EnqueueAsync(record);
+                _context.AddTransform();
             }
             catch (Exception e)
             {
@@ -110,7 +197,5 @@ public class TransformService : ITransformService
             }
         }
         _context.CompleteTransform();
-        
-        _logger.LogInformation("***** Data transformation service finished *****");
     }
 }

MesETL.App/HostedServices/VoidOutputService.cs

@@ -1,4 +1,5 @@
-using MesETL.App.HostedServices.Abstractions;
+using System.Numerics;
+using MesETL.App.HostedServices.Abstractions;
 using MesETL.App.Services;
 using Microsoft.Extensions.Logging;
 
@@ -11,6 +12,8 @@ public class VoidOutputService : IOutputService
     private readonly RecordQueuePool _queuePool;
     private readonly ProcessContext _context;
 
+    private BigInteger _total;
+
     public VoidOutputService(
         ProcessContext context, ILogger<VoidOutputService> logger, RecordQueuePool queuePool)
     {
@@ -24,20 +27,25 @@ public class VoidOutputService : IOutputService
         _logger.LogInformation("***** Void Output Service Started *****");
         while (!_context.IsTransformCompleted || _queuePool.Queues.Count > 0)
         {
-            foreach (var pair in _queuePool.Queues) // 内存优化
+            foreach (var pair in _queuePool.Queues)
             {
                 if (_context.IsTransformCompleted && pair.Value.Count == 0)
                 {
                     _queuePool.RemoveQueue(pair.Key);
                     continue;
                 }
-                if(!pair.Value.TryDequeue(out var record)) continue;
+
+                if (!pair.Value.TryDequeue(out var record))
+                    continue;
+                
+                _total += record.FieldCharCount;
                 _context.AddOutput();
             }
         }
 
         _context.CompleteOutput();
         _logger.LogInformation("***** Void Output Service Stopped *****");
+        _logger.LogInformation("平均列字符数：{Number}", _total / _context.OutputCount);
         return Task.CompletedTask;
     }
 }

MesETL.App/MesETL.App.csproj

@@ -3,7 +3,7 @@
     <PropertyGroup>
         <AssemblyName>MesETL</AssemblyName>
         <OutputType>Exe</OutputType>
-        <TargetFramework>net8.0</TargetFramework>
+        <TargetFramework>net9.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
         <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
@@ -21,15 +21,18 @@
     </ItemGroup>
 
     <ItemGroup>
-      <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
-      <PackageReference Include="MySqlConnector" Version="2.3.3" />
-      <PackageReference Include="Serilog" Version="3.1.2-dev-02097" />
+      <PackageReference Include="Microsoft.Extensions.Hosting" Version="9.0.0" />
+      <PackageReference Include="Microsoft.Extensions.ObjectPool" Version="9.0.0" />
+      <PackageReference Include="Serilog" Version="4.1.0" />
       <PackageReference Include="Serilog.Extensions.Hosting" Version="8.0.0" />
-      <PackageReference Include="Serilog.Sinks.Console" Version="5.0.1" />
-      <PackageReference Include="Serilog.Sinks.File" Version="5.0.1-dev-00972" />
-      <PackageReference Include="ServiceStack.Text" Version="8.0.0" />
-      <PackageReference Include="StackExchange.Redis" Version="2.7.17" />
-      <PackageReference Include="ZstdSharp.Port" Version="0.7.4" />
+      <PackageReference Include="Serilog.Sinks.Console" Version="6.0.0" />
+      <PackageReference Include="Serilog.Sinks.File" Version="6.0.0" />
+      <PackageReference Include="StackExchange.Redis" Version="2.8.16" />
+      <PackageReference Include="ZstdSharp.Port" Version="0.8.2" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\MesETL.Shared\MesETL.Shared.csproj" />
     </ItemGroup>
 
 </Project>

MesETL.App/Options/DataInputOptions.cs

@@ -4,6 +4,9 @@ namespace MesETL.App.Options
 {
     public class DataInputOptions
     {
+        /// <summary>
+        /// 文件输入的目录
+        /// </summary>
         public string? InputDir { get; set; }
 
         #region CSV
@@ -22,26 +25,47 @@ namespace MesETL.App.Options
 
         #region Mock
 
+        /// <summary>
+        /// <para>生成模拟数据进行测试</para>
+        /// <para>启用后在读取数据时会截取ZST文件中的CSV文件的第一条记录，然后复制成指定数量的数据</para>
+        /// </summary>
         public bool UseMock { get; set; }
 
+        /// <summary>
+        /// 当开启模拟数据生成时，模拟数据的倍数
+        /// </summary>
         public double MockCountMultiplier { get; set; } = 1;
         
         /// <summary>
-        /// Table -> Mock Count 暂时为手动配置
+        /// 配置每张表生成模拟数据的规则，此属性暂时在程序中配置
         /// </summary>
         public Dictionary<string, TableMockConfig>? TableMockConfig { get; set; }
         
         #endregion
 
 
-        #region ManualSet
+        #region Reader
         
+        /// <summary>
+        /// <para>配置输入表及其顺序，如果为空则按照程序默认的顺序。</para>
+        /// <para>该值如果存在，程序会按照集合中表的顺序来读取数据，不在集合中的表将被忽略！</para>
+        /// </summary>
         public string[]? TableOrder { get; set; }
+
+        /// <summary>
+        /// 忽略集合中配置的表，不进行读取
+        /// </summary>
+        public string[] TableIgnoreList { get; set; } = [];
         
         /// <summary>
         /// 配置如何从文件名转换为表名和表头
         /// </summary>
-        public Func<string, FileInputInfo?>? FileInputMetaBuilder { get; set; } //TODO: 抽离
+        public Func<string, FileInputInfo?>? FileInputMetaBuilder { get; set; }
+        
+        /// <summary>
+        /// 表输入完成事件
+        /// </summary>
+        public Action<string>? OnTableInputCompleted { get; set; }
 
         #endregion
     }

MesETL.App/Options/DataTransformOptions.cs

@@ -20,26 +20,26 @@ public class DataTransformOptions
     /// <summary>
     /// yyyyMM
     /// </summary>
-    public string CleanDate { get; set; } = "202301";
+    public string CleanDate { get; set; } = "202401";
     
     /// <summary>
     /// Record -> Database name
-    /// 对记录进行数据库过滤
+    /// 决定记录应当被插入到哪一个数据库中
     /// </summary>
     public Func<DataRecord, string>? DatabaseFilter { get; set; }
     /// <summary>
     /// Context -> Should output
-    /// 配置对数据过滤的条件
+    /// 对记录进行过滤，返回false则不输出
     /// </summary>
     public Func<DataTransformContext, Task<bool>>? RecordFilter { get; set; }//数据过滤方法
     /// <summary>
     /// Context -> New record
-    /// 对当前记录进行修改或完整替换
+    /// 对当前记录进行修改或完整替换，你可以在这里修改记录中的字段，或者新增/删除字段
     /// </summary>
     public Func<DataTransformContext, Task<DataRecord>>? RecordModify { get; set; }//数据替换
     /// <summary>
     /// Context -> New rebuild records
-    /// 使用当前记录对某些数据进行重建
+    /// 基于当前记录新增多个记录
     /// </summary>
     public Func<DataTransformContext, IList<DataRecord>?>? RecordReBuild { get; set; }//新增数据
     /// <summary>

MesETL.App/Options/DatabaseOutputOptions.cs

@@ -1,25 +1,73 @@
-namespace MesETL.App.Options;
+using MesETL.App.HostedServices;
+
+namespace MesETL.App.Options;
 
 public class DatabaseOutputOptions
 {
+    /// <summary>
+    /// 输出数据库的连接字符串
+    /// </summary>
     public string? ConnectionString { get; set; }
 
-    public int MaxAllowedPacket { get; set; } = 64 * 1024 * 1024;
-
-    public int FlushCount { get; set; } = 10000;
-
-    public int MaxDatabaseOutputTask { get; set; } = 4;
-
-    public bool TreatJsonAsHex { get; set; } = true;
-    
+    /// <summary>
+    /// MySql max_allowed_packet变量值大小
+    /// </summary>
+    public int MaxAllowedPacket { get; set; } = 32 * 1024 * 1024;
 
     /// <summary>
-    /// 配置导入数据的特殊列
+    /// 每次Insert提交的数据量
+    /// </summary>
+    public int FlushCount { get; set; } = 10000;
+
+    /// <summary>
+    /// 每个数据库最大提交任务数
+    /// </summary>
+    public int MaxDatabaseOutputTask { get; set; } = 4;
+
+    /// <summary>
+    /// 将json列作为16进制格式输出(0x前缀)
+    /// </summary>
+    public bool TreatJsonAsHex { get; set; } = true;
+
+    /// <summary>
+    /// 不对某些表进行输出
+    /// </summary>
+    public string[] NoOutput { get; set; } = [];
+    
+    /// <summary>
+    /// <para>当某张表的键出现重复时，在输出时使用ON DUPLICATE KEY UPDATE更新该条记录</para>
+    /// <para>表名为键，更新的字段为值</para>
+    /// <example>
+    /// <code>
+    /// {
+    ///     // 当order_data_parts表的键出现重复时，使用ON DUPLICATE KEY UPDATE更新已存在记录的CompanyID为新插入记录的值
+    ///     "order_data_parts": "CompanyID = new.CompanyID"
+    /// }
+    /// </code>
+    /// </example>
+    /// </summary>
+    public Dictionary<string, string>? ForUpdate { get; set; }
+
+    /// <summary>
+    /// 配置导入数据的特殊列，请在代码中配置
     /// </summary>
     public Dictionary<string, ColumnType> ColumnTypeConfig { get; set; } = new(); // "table.column" -> type
+    
+    /// <summary>
+    /// 所有数据都输出完毕时的事件，请在代码中配置
+    /// </summary>
+    public Action<DataOutputContext>? OutputFinished { get; set; } 
 
     public ColumnType GetColumnType(string table, string column)
     {
-        return ColumnTypeConfig.GetValueOrDefault($"{table}.{column}", ColumnType.UnDefine);
+        return ColumnTypeConfig.GetValueOrDefault(string.Concat(table, ".", column), ColumnType.UnDefine);
+    }
+
+    public bool TryGetForUpdate(string table, out string? forUpdate)
+    {
+        forUpdate = null;
+        if (ForUpdate is null || !ForUpdate.TryGetValue(table, out forUpdate))
+            return false;
+        return true;
     }
 }

MesETL.App/Options/RedisCacheOptions.cs

@@ -1,8 +1,20 @@
 namespace MesETL.App.Options;
 
+/// <summary>
+/// Redis缓存选项
+/// </summary>
 public class RedisCacheOptions
 {
+    /// <summary>
+    /// Redis连接字符串
+    /// </summary>
     public string? Configuration { get; init; }
+    /// <summary>
+    /// Redis实例名称
+    /// </summary>
     public string InstanceName { get; init; } = "";
+    /// <summary>
+    /// 使用的数据库序号
+    /// </summary>
     public int Database { get; init; } = 0;
 }

MesETL.App/Options/TableMockConfig.cs

@@ -1,5 +1,8 @@
 namespace MesETL.App.Options;
 
+/// <summary>
+/// 表模拟数据生成规则
+/// </summary>
 public struct TableMockConfig
 {
     /// <summary>

MesETL.App/Options/TenantDbOptions.cs

@@ -1,5 +1,8 @@
 namespace MesETL.App.Options;
 
+/// <summary>
+/// 多租户分库配置
+/// </summary>
 public class TenantDbOptions
 {
     public string? TenantKey { get; set; }
@@ -16,8 +19,24 @@ public class TenantDbOptions
         // DbList.ForEach(pair => dictionary.Add(pair.Value, pair.Key));
         // 注意配置顺序
         if(DbGroup is null) throw new ApplicationException("分库配置中没有发现任何数据库");
-        var dbName = DbGroup.Cast<KeyValuePair<string, int>?>()
-            .FirstOrDefault(pair => pair?.Value != null && pair.Value.Value > tenantKeyValue)!.Value.Key;
+
+        #region 性能较低，不使用
+        
+        // var dbName = DbGroup.Cast<KeyValuePair<string, int>?>()
+        //     .FirstOrDefault(pair => pair?.Value != null && pair.Value.Value > tenantKeyValue)!.Value.Key;
+
+        #endregion
+        
+        string? dbName = null;
+        foreach (var (key, value) in DbGroup)
+        {
+            if (value > tenantKeyValue)
+            {
+                dbName = key;
+                break;
+            }
+        }
+        
         return dbName ??
                throw new ArgumentOutOfRangeException(nameof(tenantKeyValue),
                    $"分库配置中没有任何符合'{nameof(tenantKeyValue)}'值的数据库");

MesETL.App/Program.cs

@@ -1,5 +1,6 @@
-// #define USE_TEST_DB // 测试库的结构与生产库不一样，如果使用测试库运行，则加上USE_TEST_DB预处理器指令
+// #define FIX_PLAN_ITEM // 测试环境对OrderBlockPlanItem表进行修复时使用
 
+using System.Text;
 using MesETL.App;
 using MesETL.App.Services;
 using MesETL.App.Services.ETL;
@@ -10,6 +11,8 @@ using MesETL.App.HostedServices.Abstractions;
 using MesETL.App.Options;
 using MesETL.App.Services.ErrorRecorder;
 using MesETL.App.Services.Loggers;
+using MesETL.App.Services.Seq;
+using MesETL.Shared.Compression;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
@@ -68,9 +71,10 @@ async Task RunProgram()
         options.DbGroup = tenantDbOptions.DbGroup;
     });
     host.Services.Configure<RedisCacheOptions>(redisSection);
-    
+
     var oldestTime = DateTime.ParseExact(transformOptions.CleanDate, "yyyyMM", System.Globalization.DateTimeFormatInfo.InvariantInfo);
-    var oldestTimeInt = int.Parse(transformOptions.CleanDate);
+    var oldestTimeInt_yyyyMM = int.Parse(transformOptions.CleanDate);
+    var oldestTimeInt_yyMM = int.Parse(transformOptions.CleanDate[2..]);
     
     // 输入配置
     host.Services.Configure<DataInputOptions>(options =>
@@ -79,137 +83,18 @@ async Task RunProgram()
         options.UseMock = inputOptions.UseMock;
         options.TableMockConfig = inputOptions.TableMockConfig;
         options.MockCountMultiplier = inputOptions.MockCountMultiplier;
+        options.TableIgnoreList = inputOptions.TableIgnoreList;
+        options.TableOrder = inputOptions.TableOrder;
         
-        // 配置文件输入方法
-        options.FileInputMetaBuilder = fileName =>
-        {
-            if (fileName.EndsWith(".dat.zst"))
-            {
-                var tableName = DumpDataHelper.GetTableNameFromCsvFileName(
-                    Path.GetFileNameWithoutExtension(fileName)); // 去除.zst
-                string[]? headers;
-                try
-                {
-                    // 查找同目录下同表的SQL文件
-                    var sqlFile = Directory.GetFiles(options.InputDir)
-                        .SingleOrDefault(f => f.Equals(fileName.Replace(".dat.zst",".sql.zst")));
-                    if (sqlFile is null)
-                        return null;
-                    headers = DumpDataHelper.GetCsvHeadersFromSqlFile(
-                        DumpDataHelper.DecompressZstAsStringAsync(File.OpenRead(sqlFile)).Result);
-                }
-                catch (InvalidOperationException e)
-                {
-                    throw new ApplicationException($"目录下不止一个{tableName}表的SQL文件", e);
-                }
-                
-                return new FileInputInfo
-                {
-                    FileName = fileName,
-                    TableName = tableName,
-                    Headers = headers
-                };
-            }
-            return null;
-        };
+        // 配置文件元数据构建方法
+        options.FileInputMetaBuilder = DumpDataHelper.MyDumperFileInputMetaBuilder;
+        
+        // 配置表输入完成事件
+        options.OnTableInputCompleted = null;
 
-        options.TableOrder =
-        [
-            TableNames.Machine,
-            
-            TableNames.Order,
-            TableNames.OrderBoxBlock,       // 依赖Order.CompanyID
-            
-            TableNames.OrderBlockPlan,      
-            TableNames.OrderBlockPlanResult,// 依赖OrderBlockPlan.CompanyID / 删除
-            
-            TableNames.OrderItem,
-            TableNames.OrderDataBlock,
-            TableNames.OrderDataGoods,
-            TableNames.OrderDataParts,
-            TableNames.OrderModule,
-            TableNames.OrderModuleExtra,
-            TableNames.OrderModuleItem,
-            TableNames.OrderPackage,
-#if USE_TEST_DB
-            TableNames.OrderPatchDetail,
-#endif
-            
-            TableNames.OrderProcess,
-            TableNames.OrderProcessStep,
-            TableNames.OrderProcessStepItem,// 依赖OrderProcess.ShardKey / 删除
-            
-            TableNames.OrderProcessSchedule,
-            TableNames.OrderScrapBoard,
-            TableNames.ProcessGroup,
-            TableNames.ProcessInfo,
-            TableNames.ProcessItemExp,
-            TableNames.ProcessScheduleCapacity,
-            TableNames.ProcessStepEfficiency,
-            TableNames.ReportTemplate,
-            TableNames.SimplePackage,
-            TableNames.SimplePlanOrder,
-            TableNames.SysConfig,
-            TableNames.WorkCalendar,
-            TableNames.WorkShift,
-            TableNames.WorkTime
-        ];
-
-        // options.TableMockConfig = new Dictionary<string, TableMockConfig>
-        // {
-        //     { TableNames.Machine, new TableMockConfig(true, 14655, ["ID"]) },
-        //     { TableNames.Order, new TableMockConfig(true, 5019216, ["OrderNo"]) },
-        //     { TableNames.OrderDataBlock, new TableMockConfig(true, 731800334, ["ID"]) },
-        //     { TableNames.OrderDataGoods, new TableMockConfig(true, 25803671, ["ID"]) },
-        //     { TableNames.OrderDataParts, new TableMockConfig(true, 468517543, ["ID"]) },
-        //     { TableNames.OrderModule, new TableMockConfig(true, 103325385, ["ID"]) },
-        //     { TableNames.OrderModuleExtra, new TableMockConfig(true, 54361321, ["ID"]) },
-        //     { TableNames.OrderModuleItem, new TableMockConfig(true, 69173339, ["ID"]) },
-        //     { TableNames.OrderPackage, new TableMockConfig(true, 16196195, ["ID"]) },
-        //     { TableNames.OrderProcess, new TableMockConfig(true, 3892685, ["ID"]) },
-        //     { TableNames.OrderProcessStep, new TableMockConfig(true, 8050349, ["ID"]) },
-        //     { TableNames.OrderProcessStepItem, new TableMockConfig(true, 14538058, ["ID"]) },
-        //     { TableNames.OrderScrapBoard, new TableMockConfig(true, 123998, ["ID"]) },
-        //     { TableNames.ProcessGroup, new TableMockConfig(true, 1253, ["ID"]) },
-        //     { TableNames.ProcessInfo, new TableMockConfig(true, 7839, ["ID"]) },
-        //     { TableNames.ProcessItemExp, new TableMockConfig(true, 28, ["ID"]) },
-        //     { TableNames.ProcessScheduleCapacity, new TableMockConfig(true, 39736, ["ID"]) },
-        //     { TableNames.ProcessStepEfficiency, new TableMockConfig(true, 8, ["ID"]) },
-        //     { TableNames.ReportTemplate, new TableMockConfig(true, 7337, ["ID"]) },
-        //     { TableNames.SimplePackage, new TableMockConfig(true, 130436, ["ID"]) },
-        //     { TableNames.SysConfig, new TableMockConfig(true, 2296, ["ID"]) },
-        //     { TableNames.WorkCalendar, new TableMockConfig(true, 11, ["ID"]) },
-        //     { TableNames.WorkShift, new TableMockConfig(true, 59, ["ID"]) },
-        //     { TableNames.WorkTime, new TableMockConfig(true, 62, ["ID"]) }
-        // };
+        // 配置表模拟数据
         options.TableMockConfig = new Dictionary<string, TableMockConfig>
-        {
-            { TableNames.Machine, new TableMockConfig(true, 14655, ["ID"]) },
-            { TableNames.Order, new TableMockConfig(true, 50192, ["OrderNo"]) },
-            { TableNames.OrderDataBlock, new TableMockConfig(true, 7318003, ["ID"]) },
-            { TableNames.OrderDataGoods, new TableMockConfig(true, 258036, ["ID"]) },
-            { TableNames.OrderDataParts, new TableMockConfig(true, 4685175, ["ID"]) },
-            { TableNames.OrderItem, new TableMockConfig(true, 13298896, ["ID"])},
-            { TableNames.OrderModule, new TableMockConfig(true, 1033253, ["ID"]) },
-            { TableNames.OrderModuleExtra, new TableMockConfig(true, 543613, ["ID"]) },
-            { TableNames.OrderModuleItem, new TableMockConfig(true, 691733, ["ID"]) },
-            { TableNames.OrderPackage, new TableMockConfig(true, 161961, ["ID"]) },
-            { TableNames.OrderProcess, new TableMockConfig(true, 38926, ["ID"]) },
-            { TableNames.OrderProcessStep, new TableMockConfig(true, 80503, ["ID"]) },
-            { TableNames.OrderProcessStepItem, new TableMockConfig(true, 145380, ["ID"]) },
-            { TableNames.OrderScrapBoard, new TableMockConfig(true, 1239, ["ID"]) },
-            { TableNames.ProcessGroup, new TableMockConfig(true, 125, ["ID"]) },
-            { TableNames.ProcessInfo, new TableMockConfig(true, 783, ["ID"]) },
-            { TableNames.ProcessItemExp, new TableMockConfig(true, 28, ["ID"]) },
-            { TableNames.ProcessScheduleCapacity, new TableMockConfig(true, 39736, ["ID"]) },
-            { TableNames.ProcessStepEfficiency, new TableMockConfig(true, 8, ["ID"]) },
-            { TableNames.ReportTemplate, new TableMockConfig(true, 7337, ["ID"]) },
-            { TableNames.SimplePackage, new TableMockConfig(true, 130436, ["ID"]) },
-            { TableNames.SysConfig, new TableMockConfig(true, 2296, ["Key"]) },
-            { TableNames.WorkCalendar, new TableMockConfig(true, 11, ["ID"]) },
-            { TableNames.WorkShift, new TableMockConfig(true, 59, ["ID"]) },
-            { TableNames.WorkTime, new TableMockConfig(true, 62, ["ID"]) }
-        };
+            { };
     });
 
     host.Services.Configure<DataTransformOptions>(options =>
@@ -225,89 +110,59 @@ async Task RunProgram()
         // order_block_plan_item和order_package_item表不导入，根据order_item数据直接重建
         
         // 数据清理
-        options.RecordFilter = async context =>
+        options.RecordFilter = async context => // TODO: OPT: oldestTime等外部变量会产生闭包
         {
             var record = context.Record;
-            var cache = context.Cacher;
             switch (record.TableName)
             {
-                // OrderBoxBlock删除对应Order.OrderNo不存在的对象
-                case TableNames.OrderBoxBlock:
-                {
-                    if (!await cache.ExistsAsync(CacheKeysFunc.Order_OrderNo_CompanyID(record["OrderNo"])))
-                        return false;
-                    break;
-                }
-                // OrderBlockPlan删除CreateTime < 202301的，Json列合法检查
+                // 清理CreateTime < 202401的
                 case TableNames.OrderBlockPlan:
                 {
-                    var time = DateTime.Parse(record["CreateTime"].Trim('"','\''));
-                    if (time < oldestTime)
+                    var creationTime = DateTime.Parse(record["CreateTime"].AsSpan().Trim(['"', '\'']));
+                    if (creationTime < oldestTime)
+                    {
                         return false;
-
-                    // if (!DumpDataHelper.IsJson(record["OrderNos"])) return false;
+                    }
                     break;
                 }
-                // OrderBlockPlanResult删除对应order_block_plan.ID不存在的对象
-                case TableNames.OrderBlockPlanResult:
+                // 忽略OrderBlockPlanItem
+                case TableNames.OrderBlockPlanItem:
                 {
-                    if (!await cache.ExistsAsync(CacheKeysFunc.OrderBlockPlan_ID_CompanyID(record["ID"])))
+                    return false;
+                }
+                // 清理(Status != 0 || Deleted = 1) && ID前四位 < 2401的
+                case TableNames.OrderScrapBoard:
+                {
+                    var status = record["Status"].AsSpan();
+                    var deleted = record["Deleted"].AsSpan();
+                    var idPref = int.Parse(record["ID"].AsSpan()[..4]);
+                    if ((status is not "0" || deleted is "1") && idPref < oldestTimeInt_yyMM)
+                    {
                         return false;
+                    }
                     break;
                 }
-                // OrderDataGoods Json列合法检查
-                case TableNames.OrderDataGoods:
-                {
-                    if (!DumpDataHelper.IsJson(record["ExtraProp"])) return false;
-                    break;
-                }
-                // OrderModule删除OrderNo < 202301的
-                case TableNames.OrderModule:
-                {
-                    var orderNo = record["OrderNo"];
-                    if(int.Parse(orderNo.AsSpan(0, 6).ToString()) < oldestTimeInt)
-                        return false;
-                    break;
-                }
-                // OrderProcess删除OrderNo < 202301的
-                case TableNames.OrderProcess:
-                {
-                    var orderNo = record["OrderNo"];
-                    if(int.Parse(orderNo.AsSpan(0, 6).ToString()) < oldestTimeInt)
-                        return false;
-                    break;
-                }
-                // OrderProcessStep删除OrderNo < 202301的
-                case TableNames.OrderProcessStep:
-                {
-                    var orderNo = record["OrderNo"];
-                    if(int.Parse(orderNo.AsSpan(0, 6).ToString()) < oldestTimeInt)
-                        return false;
-                    break;
-                }
-                // OrderProcessStepStep删除对应OrderProcess.ID不存在的对象
-                case TableNames.OrderProcessStepItem:
-                {
-                    if (!await cache.ExistsAsync(CacheKeysFunc.OrderProcess_ID_ShardKey(record["OrderProcessID"])))
-                        return false;
-                    break;
-                }
-                // SimplePackage删除OrderNo < 202301的
+                // 清理OrderNo < 202401的
                 case TableNames.SimplePackage:
                 {
-                    var orderNo = record["OrderNo"];
-                    if(int.Parse(orderNo.AsSpan(0, 6).ToString()) < oldestTimeInt)
+                    var orderNo = int.Parse(record["OrderNo"].AsSpan()[..4]);
+                    if (orderNo < oldestTimeInt_yyMM)
+                    {
                         return false;
+                    }
                     break;
                 }
-                // SimplePlanOrder删除CreateTime < 202301的
+                // 清理CreateTime < 202401的
                 case TableNames.SimplePlanOrder:
                 {
-                    var time = DateTime.Parse(record["CreateTime"].Trim('"', '\''));
-                    if (time < oldestTime)
+                    var creationTime = DateTime.Parse(record["CreateTime"].AsSpan().Trim(['"', '\'']));
+                    if (creationTime < oldestTime)
+                    {
                         return false;
+                    }
                     break;
                 }
+                default: break;
             }
 
             return true;
@@ -343,80 +198,52 @@ async Task RunProgram()
             var cache = context.Cacher;
             switch (record.TableName)
             {
-                // Machine处理非空列
-                case TableNames.Machine:
-                    ReplaceIfMyDumperNull(record, "Name", DefaultStr);
-                    ReplaceIfMyDumperNull(record, "CreateTime", DefaultDateTime);
-                    ReplaceIfMyDumperNull(record, "CreatorID", DefaultInt);
-                    ReplaceIfMyDumperNull(record, "EditTime", DefaultDateTime);
-                    ReplaceIfMyDumperNull(record, "EditorID", DefaultInt);
-                    ReplaceIfMyDumperNull(record, "Settings", DefaultText);
-                    break;
-                // Order处理非空列
-                case TableNames.Order:
-                    ReplaceIfMyDumperNull(record, "Deleted", DefaultInt);
-                    break;
-                // OrderBlockPlan处理text->json列
-                case TableNames.OrderBlockPlan:
-                    // 将所有值为'[]'(即字符串长度小等于2(16进制长度小于4))的置空 [] = 0x5b5d
-                    if (record["OrderNos"].Length <= 4)
-                        record["OrderNos"] = "NULL";
-                    break;
-                // OrderBlockPlanResult，添加CompanyID
-                case TableNames.OrderBlockPlanResult:
-                    record.AddField("CompanyID", 
-                        // 获取OrderBlockPlan.ID -> CompanyID
-                        ThrowIfNoCached(await cache.GetStringAsync(CacheKeysFunc.OrderBlockPlan_ID_CompanyID(record["ID"])),
-                            TableNames.OrderBlockPlanResult, TableNames.OrderBlockPlan, "ID", "无法获取对应的CompanyID"));
-                    break;
-                // OrderBoxBlock添加CompanyID列
+                // 重构Data列二进制数据
                 case TableNames.OrderBoxBlock:
-                    record.AddField("CompanyID", 
-                        // 获取Order.OrderNo -> CompanyID
-                        ThrowIfNoCached(await cache.GetStringAsync(CacheKeysFunc.Order_OrderNo_CompanyID(record["OrderNo"])),
-                            TableNames.OrderBoxBlock, TableNames.Order, "OrderNo", "无法获取对应的CompanyID"));
+                {
+                    var data = record["Data"];
+                    if (data is not ConstVar.MyDumperNull and ConstVar.Null)
+                    {
+                        var hex = Encoding.UTF8.GetString(Convert.FromHexString(data));
+                        record["Data"] = hex;
+                    }
+
                     break;
-                // OrderModule添加ShardKey列，移除ViewFileName列
-                case TableNames.OrderModule:
-                    record.AddField("ShardKey", CalculateShardKeyByOrderNo(record["OrderNo"]));
-                    record.RemoveField("ViewFileName");
+                }
+                // 将JsonStr列转换为Data列，添加CompressionType列
+                case TableNames.OrderModuleExtra:
+                {
+                    record.AppendField("CompressionType", "1");
+                    record.AppendField("Data",
+                        Convert.ToHexString(DeflateArchive.Compress(Convert.FromHexString(record["JsonStr"]))));
+                    record.RemoveField("JsonStr");
                     break;
-                // OrderProcess添加ShardKey列，NextStepID的空值转换为0
-                case TableNames.OrderProcess:
-                    record.AddField("ShardKey", CalculateShardKeyByOrderNo(record["OrderNo"]));
+                }
+                // 删除ID列，让数据库自行递增
+                // TODO: 数据表改进，删除ID列或是替换为流水号
+                case TableNames.ProcessStepEfficiency:
+                {
+                    record.RemoveField("ID");
                     break;
-                // OrderProcessStep添加ShardKey
-                case TableNames.OrderProcessStep:
-                    record.AddField("ShardKey", CalculateShardKeyByOrderNo(record["OrderNo"]));
+                }
+                case TableNames.ProcessScheduleCapacity:
+                {
+                    record.RemoveField("ID");
                     break;
-                // OrderProcessStepItem添加ShardKey列，处理非空列
-                case TableNames.OrderProcessStepItem:
-                    ReplaceIfMyDumperNull(record, "DataID", DefaultInt);
-                    record.AddField("ShardKey", 
-                        // 获取OrderProcess.ID -> ShardKey
-                    ThrowIfNoCached(await cache.GetStringAsync(CacheKeysFunc.OrderProcess_ID_ShardKey(record["OrderProcessID"])), 
-                        TableNames.OrderProcessStepItem, TableNames.OrderProcessStep, "OrderProcessID", "无法获取对应的ShardKey"));
+                }
+                case TableNames.SysConfig:
+                {
+                    record.RemoveField("Key");
                     break;
-                // OrderScrapBoard处理非空列
-                case TableNames.OrderScrapBoard:
-                    ReplaceIfMyDumperNull(record, "Color", DefaultStr);
-                    ReplaceIfMyDumperNull(record, "GoodsName", DefaultStr);
-                    ReplaceIfMyDumperNull(record, "Material", DefaultStr);
-                    ReplaceIfMyDumperNull(record, "MaterialName", DefaultStr);
-                    break;
-                // ProcessItemExp处理非空列
-                case TableNames.ProcessItemExp:
-                    ReplaceIfMyDumperNull(record, "MaxPartsID", DefaultInt);
-                    ReplaceIfMyDumperNull(record, "ProcessGroupID", DefaultInt);
-                    break;
-                // SimplePlanOrder处理非空列，添加Deleted
+                }
+                // 移除PlaceData列（如果存在的话，生产库已经删除）
                 case TableNames.SimplePlanOrder:
-                    ReplaceIfMyDumperNull(record, "CreateTime", DefaultDateTime);
-                    ReplaceIfMyDumperNull(record, "UpdateTime", DefaultDateTime);
-                    ReplaceIfMyDumperNull(record, "CompanyID", DefaultInt);
-                    ReplaceIfMyDumperNull(record, "SingleName", DefaultStr);
-                    record.AddField("Deleted", "0");
+                {
+                    if(record.HeaderExists("PlaceData"))
+                        record.RemoveField("PlaceData");
                     break;
+                }
+                default: break;
             }
 
             return record;
@@ -431,34 +258,7 @@ async Task RunProgram()
         };
         
         // 数据缓存
-        options.RecordCache = async context =>
-        {
-            var record = context.Record;
-            var cache = context.Cacher;
-            switch (record.TableName)
-            {
-                // 缓存Order.OrderNo -> CompanyID
-                case TableNames.Order:
-                    await cache.SetStringAsync(
-                        CacheKeysFunc.Order_OrderNo_CompanyID(record["OrderNo"]),
-                        record["CompanyID"]);
-                    break;
-                
-                // 缓存OrderBlockPlan.ID -> CompanyID
-                case TableNames.OrderBlockPlan:
-                    await cache.SetStringAsync(
-                        CacheKeysFunc.OrderBlockPlan_ID_CompanyID(record["ID"]),
-                        record["CompanyID"]);
-                    break;
-                
-                // 缓存OrderProcess.ID -> ShardKey
-                case TableNames.OrderProcess:
-                    await cache.SetStringAsync(
-                        CacheKeysFunc.OrderProcess_ID_ShardKey(record["ID"]),
-                        record["ShardKey"]);
-                    break;
-            }
-        };
+        options.RecordCache = null;
         
         // 数据库过滤
         options.DatabaseFilter = record =>
@@ -471,36 +271,62 @@ async Task RunProgram()
         options.RecordReBuild = context =>
         {
             var record = context.Record;
-            var resultList = new List<DataRecord>();
-            // 分流OrderItem表
+            
+            // 将OrderExtra表迁移至OrderWaveGroup表
+            if (record.TableName == TableNames.OrderExtra)
+            {
+                record.Ignore = true;
+                var resultList = new List<DataRecord>();
+                var seq = context.Services.GetRequiredService<SeqService>();
+                string[] headers = ["OrderNo", "ShardKey", "ConfigType", "ConfigJson", "CompanyID"];
+                var id = seq.AddCachedSeq(SeqConfig.OrderWaveGroupID);
+                var orderWaveGroup = new DataRecord(
+                    [id.ToString(), ..headers.Select(c => record[c])],
+                    TableNames.OrderExtraList,
+                    ["ID", "OrderNo", "ShardKey", "Type", "ConfigJson", "CompanyID"]);
+                resultList.Add(orderWaveGroup);
+                return resultList;
+            }
+            
+            // 通过OrderItem重建OrderBlockPlanItem表
             if (record.TableName == TableNames.OrderItem)
             {
+#if FIX_PLAN_ITEM
+                record.Ignore = true; 
+#endif
+                var resultList = new List<DataRecord>();
                 record.TryGetField("ID", out var itemId);
                 record.TryGetField("ShardKey", out var shardKey);
                 record.TryGetField("PlanID", out var planId);
                 record.TryGetField("PackageID", out var packageId);
                 record.TryGetField("CompanyID", out var companyId);
                 if(!int.TryParse(planId, out var pid))
-                    throw new ApplicationException($"数据发生异常：OrderItem.PlanID，值: {planId}"); 
+                    throw new ApplicationException($"数据发生异常：OrderItem.PlanID，值: {(string.IsNullOrWhiteSpace(planId) ? "NULL" : planId)}"); 
                 if (pid > 0)
                 {
-                    resultList.Add(new DataRecord(new[] { itemId, shardKey, planId, companyId },
+                    resultList.Add(new DataRecord([itemId, shardKey, planId, companyId],
                         TableNames.OrderBlockPlanItem,
                         ["ItemID", "ShardKey", "PlanID", "CompanyID"]
-                        ));
+                    ));
                 }
                 if(!int.TryParse(packageId, out var pkid))
-                    throw new ApplicationException($"数据发生异常：OrderItem.PackageID，值: {packageId}"); 
+                    throw new ApplicationException($"数据发生异常：OrderItem.PackageID，值: {(string.IsNullOrWhiteSpace(packageId) ? "NULL" : packageId)}"); 
                 if(pkid > 0)
                 {
-                    resultList.Add(new DataRecord(new[] { itemId, shardKey, packageId, companyId },
+                    resultList.Add(new DataRecord([itemId, shardKey, packageId, companyId],
                         TableNames.OrderPackageItem,
                         [ "ItemID", "ShardKey", "PackageID", "CompanyID" ]
                     ));
                 }
-            }
-            return resultList;
+                
+                record.RemoveField("PlanID");
+                record.RemoveField("PackageID");
 
+                return resultList;
+            }
+            
+
+            return ArraySegment<DataRecord>.Empty;
         };
     });
     
@@ -508,96 +334,74 @@ async Task RunProgram()
     {
         options.ConnectionString = outputOptions.ConnectionString;
         options.FlushCount = outputOptions.FlushCount;
-        options.MaxAllowedPacket = outputOptions.MaxAllowedPacket;
+        options.MaxAllowedPacket = outputOptions.MaxAllowedPacket / 2;
         options.MaxDatabaseOutputTask = outputOptions.MaxDatabaseOutputTask;
         options.TreatJsonAsHex = outputOptions.TreatJsonAsHex;
-
-#if USE_TEST_DB
-        // Test Server
-         options.ColumnTypeConfig = new Dictionary<string, ColumnType>
-         {
-             { "simple_plan_order.PlaceData", ColumnType.Blob },
-             { "order_block_plan_result.PlaceData", ColumnType.Blob },
-             { "order_box_block.Data", ColumnType.Blob },
-             { "order_data_goods.ExtraProp", ColumnType.Json },
-             { "order_module_extra.JsonStr", ColumnType.Text },
-             { "process_info.Users", ColumnType.Text },
-             { "order_process_schdule.CustomOrderNo", ColumnType.Text },
-             { "order_process_schdule.OrderProcessStepName", ColumnType.Text },
-             { "order_process_schdule.AreaName", ColumnType.Text },
-             { "order_process_schdule.ConsigneeAddress", ColumnType.Text },
-             { "order_process_schdule.ConsigneePhone", ColumnType.Text },
-             { "report_source.Sql", ColumnType.Text },
-             { "report_source.KeyValue", ColumnType.Text },
-             { "report_source.Setting", ColumnType.Text },
-             { "order_data_block.RemarkJson", ColumnType.Text },
-             { "order_patch_detail.BlockDetail", ColumnType.Json },
-             { "order_scrap_board.OutLineJson", ColumnType.Text },
-             { "simple_package.Items", ColumnType.Json },
-             { "order_batch_pack_config.Setting", ColumnType.Text },
-             { "machine.Settings", ColumnType.Text },
-             { "sys_config.Value", ColumnType.Text },
-             { "sys_config.JsonStr", ColumnType.Text },
-             { "process_item_exp.ItemJson", ColumnType.Text },
-             { "report_template.Template", ColumnType.Text },
-             { "report_template.SourceConfig", ColumnType.Text },
-             { "order_block_plan.OrderNos", ColumnType.Json },
-             { "order_block_plan.BlockInfo", ColumnType.Text },
-         };
-#else
-        // 配置列类型
+        options.NoOutput = outputOptions.NoOutput;
+        options.ForUpdate = outputOptions.ForUpdate;
+        
+        // 配置列的类型以便于在输出时区分二进制内容
         // Prod server
         options.ColumnTypeConfig = new Dictionary<string, ColumnType>
         {
-            { "simple_plan_order.PlaceData", ColumnType.Blob },
-            { "order_block_plan_result.PlaceData", ColumnType.Blob },
-            { "order_box_block.Data", ColumnType.Blob },
-            { "order_data_goods.ExtraProp", ColumnType.Text },
-            { "order_module_extra.JsonStr", ColumnType.Text },
-            { "process_info.Users", ColumnType.Text },
-            { "order_process_schdule.CustomOrderNo", ColumnType.Text },
-            { "order_process_schdule.OrderProcessStepName", ColumnType.Text },
-            { "order_process_schdule.AreaName", ColumnType.Text },
-            { "order_process_schdule.ConsigneeAddress", ColumnType.Text },
-            { "order_process_schdule.ConsigneePhone", ColumnType.Text },
-            { "report_source.Sql", ColumnType.Text },
-            { "report_source.KeyValue", ColumnType.Text },
-            { "report_source.Setting", ColumnType.Text },
-            { "order_data_block.RemarkJson", ColumnType.Text },
-            { "order_patch_detail.BlockDetail", ColumnType.Text },
-            { "order_scrap_board.OutLineJson", ColumnType.Text },
-            { "simple_package.Items", ColumnType.Text },
-            { "order_batch_pack_config.Setting", ColumnType.Text },
-            { "machine.Settings", ColumnType.Text },
-            { "sys_config.Value", ColumnType.Text },
-            { "sys_config.JsonStr", ColumnType.Text },
-            { "process_item_exp.ItemJson", ColumnType.Text },
-            { "report_template.Template", ColumnType.Text },
-            { "report_template.SourceConfig", ColumnType.Text },
-            { "order_block_plan.OrderNos", ColumnType.Text },
-            { "order_block_plan.BlockInfo", ColumnType.Text },
+            {"machine.Settings", ColumnType.Text},
+            {"order_block_plan.BlockInfo", ColumnType.Text},
+            {"order_block_plan.OrderNos", ColumnType.Json},
+            {"order_block_plan_result.PlaceData", ColumnType.Blob},
+            {"order_box_block.Data", ColumnType.Blob},
+            {"order_data_block.RemarkJson", ColumnType.Text},
+            {"order_data_goods.ExtraProp", ColumnType.Json},
+            {"order_extra.ConfigJson", ColumnType.Json},
+            {"order_module_extra.Data", ColumnType.Blob},
+            {"order_module_extra.JsonStr", ColumnType.Text},
+            {"order_patch_detail.BlockDetail", ColumnType.Json},
+            {"order_process_schdule.AreaName", ColumnType.Text},
+            {"order_process_schdule.ConsigneeAddress", ColumnType.Text},
+            {"order_process_schdule.ConsigneePhone", ColumnType.Text},
+            {"order_process_schdule.CustomOrderNo", ColumnType.Text},
+            {"order_process_schdule.OrderProcessStepName", ColumnType.Text},
+            {"order_scrap_board.OutLineJson", ColumnType.Text},
+            {"order_wave_group.ConfigJson", ColumnType.Json},
+            {"process_info.Users", ColumnType.Text},
+            {"process_item_exp.ItemJson", ColumnType.Text},
+            {"report_template.SourceConfig", ColumnType.Text},
+            {"report_template.Template", ColumnType.Text},
+            {"simple_package.Items", ColumnType.Json},
+            {"sys_config.JsonStr", ColumnType.Text},
+            {"sys_config.Value", ColumnType.Text}
+        };
+
+        options.OutputFinished += ctx =>
+        {
+            var seq = ctx.Serivces.GetRequiredService<SeqService>();
+            seq.ApplyToDatabaseAsync().GetAwaiter().GetResult();
         };
-#endif
     });
     
     host.Services.AddLogging(builder =>
     {
         builder.ClearProviders();
-        builder.AddSerilog(new LoggerConfiguration()
+        var logger = new LoggerConfiguration()
+            .MinimumLevel.Verbose()
             .WriteTo.Console()
-            .WriteTo.File(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"./Log/Error/{ErrorRecorder.UID}.log"), 
-                restrictedToMinimumLevel:LogEventLevel.Error)
+            .WriteTo.File(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"./Log/Error/{ErrorRecorder.UID}.log"),
+                restrictedToMinimumLevel: LogEventLevel.Error)
             // .WriteTo.File("./Log/Info/{ErrorRecorder.UID}.log", restrictedToMinimumLevel:LogEventLevel.Information) //性能考虑暂不使用
-            .CreateLogger()
-        );
+            .CreateLogger();
+        builder.AddSerilog(logger);
+        Log.Logger = logger;
     });
 
     host.Services.AddDataSourceFactory();
     host.Services.AddErrorRecorderFactory();
     host.Services.AddSingleton<ProcessContext>();
-    host.Services.AddKeyedSingleton<DataRecordQueue>(ConstVar.Producer, new DataRecordQueue(200_000));
-    host.Services.AddRecordQueuePool(tenantDbOptions.DbGroup.Keys.Select(key => (key:key, queue:new DataRecordQueue(60_000))).ToArray());
-    host.Services.AddSingleton<ITaskMonitorLogger, CacheTaskMonitorLogger>();
+    host.Services.AddSingleton<SeqService>();
+    var prodLen = host.Configuration.GetRequiredSection("RecordQueue").GetValue<int>("ProducerQueueLength");
+    var consLen = host.Configuration.GetRequiredSection("RecordQueue").GetValue<int>("ConsumerQueueLength");
+    var maxCharCount = host.Configuration.GetRequiredSection("RecordQueue").GetValue<long>("MaxByteCount") / 2;
+    host.Services.AddKeyedSingleton<DataRecordQueue>(ConstVar.Producer, new DataRecordQueue(prodLen, maxCharCount));
+    host.Services.AddRecordQueuePool(tenantDbOptions.DbGroup.Keys.Select(key => (key:key, queue:new DataRecordQueue(consLen, maxCharCount))).ToArray());
+    // host.Services.AddSingleton<ITaskMonitorLogger, CacheTaskMonitorLogger>();
     host.Services.AddSingleton<ITaskMonitorLogger, LoggerTaskMonitorLogger>();
 
     host.Services.AddHostedService<MainHostedService>();
@@ -605,7 +409,8 @@ async Task RunProgram()
     host.Services.AddSingleton<ITransformService, TransformService>();
     host.Services.AddSingleton<IOutputService, OutputService>();
     host.Services.AddSingleton<TaskMonitorService>();
-    host.Services.AddRedisCache(redisOptions);
+    // host.Services.AddRedisCache(redisOptions);
+    host.Services.AddSingleton<ICacher, MemoryCache>();
     var app = host.Build();
     await app.RunAsync();
 }

MesETL.App/Services/DataRecordQueue.cs

@@ -1,5 +1,6 @@
 using System.Collections.Concurrent;
 using System.Diagnostics.CodeAnalysis;
+using TaskExtensions = MesETL.Shared.Helper.TaskExtensions;
 
 namespace MesETL.App.Services;
 
@@ -10,26 +11,39 @@ public class DataRecordQueue : IDisposable
 {
     private readonly BlockingCollection<DataRecord> _queue;
 
+    private long _currentCharCount;
+    private readonly long _maxCharCount = 2_147_483_648; // 4GiB
+
     public int Count => _queue.Count;
     public bool IsCompleted => _queue.IsCompleted;
     public bool IsAddingCompleted => _queue.IsAddingCompleted;
 
+    public long LongestFieldCharCount { get; private set; }
+
     public event Action? OnRecordWrite;
     public event Action? OnRecordRead;
 
-    public DataRecordQueue() : this(500_000) // 默认容量最大500K
+    public DataRecordQueue() : this(500_000, 2_147_483_648) // 默认容量最大500K
     {
     }
     
-    public DataRecordQueue(int boundedCapacity)
+    public DataRecordQueue(int boundedCapacity, long maxCharCount)
     {
         _queue = new BlockingCollection<DataRecord>(boundedCapacity); 
+        _maxCharCount = maxCharCount;
     }
     
+    public void CompleteAdding() => _queue.CompleteAdding();
+    
     public bool TryDequeue([MaybeNullWhen(false)] out DataRecord record)
     {
         if (_queue.TryTake(out record))
         {
+            // if (record.Database is not null)
+            // {
+            //     Console.WriteLine("out " + record.Database);
+            // }
+            Interlocked.Add(ref _currentCharCount, -record.FieldCharCount);
             OnRecordRead?.Invoke();
             return true;
         }
@@ -37,13 +51,21 @@ public class DataRecordQueue : IDisposable
         return false;
     }
 
-    public DataRecord Dequeue() => _queue.Take();
 
-    public void CompleteAdding() => _queue.CompleteAdding();
-
-    public void Enqueue(DataRecord record)
+    public async Task EnqueueAsync(DataRecord record)
     {
+        var charCount = record.FieldCharCount;
+        LongestFieldCharCount = Math.Max(LongestFieldCharCount, charCount);
+        if (_currentCharCount + charCount > _maxCharCount)
+        {
+            // 不用Task.WaitUntil是为了防止产生Lambda闭包
+            while (!(_currentCharCount + charCount < _maxCharCount))
+            {
+                await Task.Delay(50);
+            }
+        }
         _queue.Add(record);
+        Interlocked.Add(ref _currentCharCount, charCount);
         OnRecordWrite?.Invoke();
     }
 

MesETL.App/Services/ETL/CsvReader.cs

@@ -11,10 +11,11 @@ public class CsvReader : IDataReader
 {
     protected readonly string? FilePath;
     protected readonly Lazy<StreamReader> Reader;
+    private Stream? _stream;
     protected readonly ILogger? Logger;
     protected readonly string TableName;
 
-    public DataRecord Current { get; protected set; } = null!;
+    public DataRecord Current { get; protected set; } = default!;
     public string[] Headers { get; }
     public string Delimiter { get; }
     public char QuoteChar { get; }
@@ -22,15 +23,18 @@ public class CsvReader : IDataReader
     public CsvReader(Stream stream, string tableName, string[] headers, string delimiter = ",", char quoteChar = '"', ILogger? logger = null)
     : this(tableName, headers, delimiter, quoteChar, logger)
     {
-        Reader = new Lazy<StreamReader>(() => new StreamReader(stream));
+        Reader = new Lazy<StreamReader>(() => new StreamReader(stream),false);
     }
 
     public CsvReader(string filePath, string tableName, string[] headers, string delimiter = ",", char quoteChar = '"', ILogger? logger = null)
     : this(tableName, headers, delimiter, quoteChar, logger)
     {
-        var fs = File.OpenRead(filePath);
         FilePath = filePath;
-        Reader = new Lazy<StreamReader>(() => new StreamReader(fs));
+        Reader = new Lazy<StreamReader>(() =>
+        {
+            _stream = File.OpenRead(filePath);
+            return new StreamReader(_stream);
+        });
     }
 
     private CsvReader(string tableName, string[] headers, string delimiter = ",", char quoteChar = '"', ILogger? logger = null)
@@ -49,43 +53,44 @@ public class CsvReader : IDataReader
         if (string.IsNullOrWhiteSpace(str))
             return false;
 
-        var fields = ParseRow(str, QuoteChar, Delimiter);
+        var fields = ParseRowFaster(str, QuoteChar, Delimiter[0]);
         Current = new DataRecord(fields, TableName, Headers);
         return true;
     }
 
-    public string[] ParseRow(ReadOnlySpan<char> source, char quoteChar, string delimiter)
+    public static string[] ParseRow(ReadOnlySpan<char> source, char quoteChar, char delimiter)
     {
         var result = new List<string>();
         var index = -1;
-        var current = new StringBuilder();
+        var current = new StringBuilder(source.Length);
         var hasQuote = false;
         var hasSlash = false;
         while (index < source.Length - 1)
         {
             index++;
-            if (hasSlash == false && source[index] == '\\')
+            var currChar = source[index];
+            if (hasSlash == false && currChar == '\\')
             {
                 hasSlash = true;
                 current.Append('\\');
                 continue;
             }
 
-            if (hasSlash == false && source[index] == quoteChar)
+            if (hasSlash == false && currChar == quoteChar)
             {
                 hasQuote = !hasQuote;
-                current.Append(source[index]);
+                current.Append(currChar);
                 continue;
             }
 
-            if (hasQuote == false && source[index] == delimiter[0])
+            if (hasQuote == false && currChar == delimiter)
             {
                 result.Add(current.ToString());
                 current.Clear();
             }
             else
             {
-                current.Append(source[index]);
+                current.Append(currChar);
             }
 
             hasSlash = false;
@@ -94,10 +99,62 @@ public class CsvReader : IDataReader
         result.Add(current.ToString());
         return result.ToArray();
     }
+    
+    public static List<string> ParseRowFaster(ReadOnlySpan<char> source, char quoteChar, char delimiter, int columnCount = 10)
+    {
+        var result = new List<string>(columnCount);
+        var index = -1;
+        var hasQuote = false;
+        var hasSlash = false;
+        var start = 0;
+        var end = 0;
+        var len = source.Length - 1;
+        while (index < len)
+        {
+            ++index;
+            var currChar = source[index];
+            
+            if (!hasSlash)
+            {
+                if (currChar is '\\')
+                {
+                    hasSlash = true;
+                    ++end;
+                    continue;
+                }
+
+                if (currChar == quoteChar)
+                {
+                    hasQuote = !hasQuote;
+                    ++end;
+                    continue;
+                }
+            }
+            
+            if (!hasQuote && currChar == delimiter)
+            {
+                result.Add(source[start..(end)].ToString()); // 超大型字符串会在LOH中分配内存，没救
+                start = end + 1;
+                ++end;
+            }
+            else
+            {
+                ++end;
+            }
+
+            hasSlash = false;
+        }
+
+        result.Add(source[start..end].ToString());
+        return result;
+    }
 
     public virtual void Dispose()
     {
-        if(Reader.IsValueCreated)
+        if (Reader.IsValueCreated)
+        {
             Reader.Value.Dispose();
+            _stream?.Dispose();
+        }
     }
 }

MesETL.App/Services/ETL/MySqlDestination.cs

@@ -3,6 +3,7 @@ using System.Text.RegularExpressions;
 using MesETL.App.Const;
 using MesETL.App.Helpers;
 using MesETL.App.Options;
+using MesETL.Shared.Helper;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using MySqlConnector;
@@ -14,6 +15,9 @@ namespace MesETL.App.Services.ETL;
 /// </summary>
 public partial class MySqlDestination : IDisposable, IAsyncDisposable
 {
+    /// <summary>
+    /// table => records
+    /// </summary>
     private readonly Dictionary<string, IList<DataRecord>> _recordCache;
     private readonly MySqlConnection _conn;
     private readonly ILogger _logger;
@@ -61,12 +65,12 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
             return;
 
         var cmd = _conn.CreateCommand();
-        cmd.CommandTimeout = 3 * 60;
+        cmd.CommandTimeout = 0;
        
         try
         {
-            var excuseList = GetExcuseList(_recordCache, maxAllowPacket).ToList();
-            foreach (var insertSql in excuseList)
+            var executionList = GetExecutionList(_recordCache, maxAllowPacket);
+            foreach (var insertSql in executionList)
             {
                 cmd.CommandText = insertSql;
                 try
@@ -102,9 +106,10 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
     [GeneratedRegex("INSERT INTO `([^`]+)`")]
     private static partial Regex MatchTableName();
 
-    public IEnumerable<string> GetExcuseList(IDictionary<string, IList<DataRecord>> tableRecords,int maxAllowPacket)
+    public IEnumerable<string> GetExecutionList(IDictionary<string, IList<DataRecord>> tableRecords, int maxAllowPacket)
     {
-        var sb = new StringBuilder("SET AUTOCOMMIT = 1;\n");
+        var sb = new StringBuilder(_options.Value.FlushCount * 128);
+        var appendCount = 0;
         foreach (var (tableName, records) in tableRecords)
         {
             if (records.Count == 0)
@@ -112,15 +117,19 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
             
             var recordIdx = 0;
             StartBuild:
+            sb.AppendLine("SET AUTOCOMMIT = 0;\n");
             var noCommas = true;
             
+            // 标准列顺序，插入时的字段需要按照该顺序排列
+            var headers = records[0].Headers;
+            
             // INSERT INTO ... VALUES >>>
             sb.Append($"INSERT INTO `{tableName}`(");
-            for (var i = 0; i < records[0].Headers.Count; i++)
+            for (var i = 0; i < headers.Count; i++)
             {
                 var header = records[0].Headers[i];
                 sb.Append($"`{header}`");
-                if (i != records[0].Headers.Count - 1)
+                if (i != headers.Count - 1)
                     sb.Append(',');
             }
 
@@ -130,11 +139,20 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
             for (;recordIdx < records.Count; recordIdx++)
             {
                 var record = records[recordIdx];
+                
+                // 数据列校验
+                if (record.Headers.Count != headers.Count)
+                {
+                    throw new InvalidOperationException($"数据异常，数据列数量出现冲突，表名：{tableName}");
+                }
+                
                 var recordSb = new StringBuilder();
                 recordSb.Append('(');
-                for (var fieldIdx = 0; fieldIdx < record.Fields.Count; fieldIdx++)
+                for (var idx = 0; idx < headers.Count; idx++)
                 {
-                    var field = record.Fields[fieldIdx];
+                    var header = headers[idx];
+                    // TODO: 可进行性能优化
+                    var field = record[header];
 
                     // 在这里处理特殊列
                     #region HandleFields
@@ -145,7 +163,7 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
                         goto Escape;
                     }
                     
-                    switch (_options.Value.GetColumnType(record.TableName, record.Headers[fieldIdx]))
+                    switch (_options.Value.GetColumnType(record.TableName, header))
                     {
                         case ColumnType.Text:
                             if(string.IsNullOrEmpty(field))
@@ -161,12 +179,12 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
                                 recordSb.Append(ConstVar.Null);
                             else recordSb.Append($"0x{field}");
                             break;
-                        case ColumnType.Json:// 生产库没有JSON列，仅用于测试库进行测试
-                            if(string.IsNullOrEmpty(field))
-                                recordSb.Append("'[]'"); // JObject or JArray?
+                        case ColumnType.Json: // Mydumper v0.16.7-5导出的Json为字符串，且会将逗号转义，需要适配
+                            if(string.IsNullOrEmpty(field)) 
+                                recordSb.Append(ConstVar.Null);
                             else if (_options.Value.TreatJsonAsHex)
                                 recordSb.Append($"_utf8mb4 0x{field}");
-                            else recordSb.AppendLine(field);
+                            else recordSb.AppendLine(field.Replace("\\,", ","));
                             break;
                         case ColumnType.UnDefine:
                         default:
@@ -177,7 +195,7 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
                     Escape:
 
                     #endregion
-                    if (fieldIdx != record.Fields.Count - 1)
+                    if (idx != headers.Count - 1)
                         recordSb.Append(',');
                 }
 
@@ -186,8 +204,16 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
                 // 若字符数量即将大于限制，则返回SQL，清空StringBuilder，保留当前记录的索引值，然后转到StartBuild标签重新开始一轮INSERT
                 if (sb.Length + recordSb.Length + 23 > maxAllowPacket)
                 {
+                    if (appendCount == 0) // 如果单条记录超出maxAllowedPacket
+                    {
+                        sb.Append(recordSb);
+                        _logger.LogWarning("{Table}表单条数据的SQL超出了配置的MaxAllowedPacket，字符数{Count}", tableName,
+                            sb.Length + recordSb.Length + 23);
+                    }
+
+                    TryAddForUpdateSuffix(tableName, sb);
                     sb.Append(';').AppendLine();
-                    sb.Append("SET AUTOCOMMIT = 1;");
+                    sb.Append("COMMIT;");
                     yield return sb.ToString();
                     sb.Clear();
                     goto StartBuild;
@@ -197,8 +223,10 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
                     sb.Append(',').AppendLine();
                 noCommas = false;
                 sb.Append(recordSb); // StringBuilder.Append(StringBuilder)不会分配多余的内存
+                appendCount++;
             }
 
+            TryAddForUpdateSuffix(tableName, sb);
             sb.Append(';');
             sb.Append("COMMIT;");
             yield return sb.ToString();
@@ -206,16 +234,35 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
         }
     }
 
-
+    /// <summary>
+    /// 数据必须是同一张表
+    /// </summary>
+    /// <param name="tableName"></param>
+    /// <param name="sb"></param>
+    private void TryAddForUpdateSuffix(string tableName, StringBuilder sb)
+    {
+        var forUpdate = _options.Value.TryGetForUpdate(tableName, out var forUpdateSql);
+        if (forUpdate)
+        {
+            sb.AppendLine($"""
+                           AS new
+                           ON DUPLICATE KEY UPDATE
+                           {forUpdateSql}
+                           """);
+        }
+    }
+    
     public void Dispose()
     {
         _conn.Close();
         _conn.Dispose();
+        _recordCache.Clear();
     }
 
     public async ValueTask DisposeAsync()
     {
         await _conn.CloseAsync();
         await _conn.DisposeAsync();
+        _recordCache.Clear();
     }
 }

MesETL.App/Services/ETL/ZstReader.cs

@@ -1,4 +1,6 @@
-using Microsoft.Extensions.Logging;
+using System.Runtime.InteropServices;
+using System.Text;
+using Microsoft.Extensions.Logging;
 using ZstdSharp;
 
 namespace MesETL.App.Services.ETL;
@@ -9,38 +11,91 @@ namespace MesETL.App.Services.ETL;
 public class ZstReader : CsvReader
 {
     protected new readonly Lazy<StreamReader> Reader;
+    private Stream? _stream;
 
-
+    private readonly List<char> _str = new(1024);
+    private readonly char[] _charBuffer = new char[1024];
+    private int _charLen = 0;
+    private int _charPos = 0;
+    
+    
     public ZstReader(string filePath, string tableName, string[] headers, string delimiter = ",", char quoteChar = '\"', ILogger? logger = null) 
         : base(filePath, tableName, headers, delimiter, quoteChar, logger)
     {
-        var ds = new DecompressionStream(File.OpenRead(filePath));
-        Reader = new Lazy<StreamReader>(() => new StreamReader(ds));
+        Reader = new Lazy<StreamReader>(() =>
+        {
+             _stream = new DecompressionStream(File.OpenRead(filePath));
+            return new StreamReader(_stream);
+        }, false);
+        ReadBuffer();
     }
         
     public ZstReader(Stream stream, string tableName, string[] headers, string delimiter = ",", char quoteChar = '\"', ILogger? logger = null) 
         : base(stream, tableName, headers, delimiter, quoteChar, logger)
     {
         var ds = new DecompressionStream(stream);
-        Reader = new Lazy<StreamReader>(() => new StreamReader(ds));
+        Reader = new Lazy<StreamReader>(() => new StreamReader(ds), false);
+        ReadBuffer();
+    }
+
+    private int ReadBuffer()
+    {
+        _charLen = _charPos = 0;
+        _charLen = Reader.Value.ReadBlock(_charBuffer);
+        return _charLen;
     }
 
     public override async ValueTask<bool> ReadAsync()
     {
-        var str = await Reader.Value.ReadLineAsync();
-        if (string.IsNullOrWhiteSpace(str))
+        // 缓冲区已经读取完毕，并且流状态为EOF
+        if (_charPos == _charLen && ReadBuffer() == 0)
             return false;
 
-        var fields = ParseRow(str, QuoteChar, Delimiter);
-        Current = new DataRecord(fields, TableName, Headers);
+        do
+        {
+            // 读取缓冲区
+            var span = _charBuffer.AsSpan(_charPos, _charLen - _charPos);
+            var newLineIdx = span.IndexOfAny('\r', '\n');
+            // 读取到行，结合当前构建字符串转换进行转换
+            if (newLineIdx >= 0)
+            {
+                if (_str.Count == 0) // => 可以提高一点性能...
+                {
+                    var fields = ParseRowFaster(span[..newLineIdx], QuoteChar, Delimiter[0]);
+                    Current = new DataRecord(fields, TableName, Headers);
+                }
+                else
+                {
+                    _str.AddRange(span[..newLineIdx]);
+                    var fields = ParseRowFaster(CollectionsMarshal.AsSpan(_str), QuoteChar, Delimiter[0]);
+                    Current = new DataRecord(fields, TableName, Headers);
+                }
+                _str.Clear();
+
+                var ch = span[newLineIdx];
+                _charPos += newLineIdx + 1;
+                if (ch == '\r' && (_charPos < _charLen || ReadBuffer() > 0) && _charBuffer[_charPos] == '\n') // 跳过CRLF
+                    ++_charPos;
+                return true;
+            }
+            // 未读取到行，将缓冲区插入构建字符串
+            _str.AddRange(span);
+        } while (ReadBuffer() > 0);
+
+        var f = ParseRowFaster(CollectionsMarshal.AsSpan(_str), QuoteChar, Delimiter[0]);
+        Current = new DataRecord(f, TableName, Headers);
+        _str.Clear();
         return true;
     }
 
     public override void Dispose()
     {
         base.Dispose();
-        if(Reader.IsValueCreated) 
+        if (Reader.IsValueCreated)
+        {
             Reader.Value.Dispose();
+            _stream?.Dispose();
+        }
     }
 
 }

MesETL.App/Services/ErrorRecorder/ErrorRecorder.cs

@@ -1,4 +1,5 @@
 using MesETL.App.Helpers;
+using MesETL.Shared.Helper;
 using Microsoft.Extensions.Logging;
 
 namespace MesETL.App.Services.ErrorRecorder;

MesETL.App/Services/ErrorRecorder/OutputErrorRecorder.cs

@@ -9,6 +9,7 @@ public sealed class OutputErrorRecorder : ErrorRecorder
     private readonly string _outputDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"ErrorRecords/{UID}/Output");
     private readonly string _database;
     private readonly Dictionary<string, int> _logIndex = new();
+    private static readonly object Lock = new();
 
     public OutputErrorRecorder(string database, ILogger logger) : base(logger)
     {
@@ -50,7 +51,17 @@ public sealed class OutputErrorRecorder : ErrorRecorder
 
 
                       """;
-        await File.AppendAllTextAsync(filePath, content, Encoding.UTF8);
+        try
+        {
+            lock (Lock)
+            {
+                File.AppendAllText(filePath, content, Encoding.UTF8);
+            }
+        }
+        catch(Exception e)
+        {
+            Logger.LogError(e, "输出错误日志时发生错误");   
+        }
     }
 
     /// <summary>

MesETL.App/Services/Loggers/CacheTaskMonitorLogger.cs

@@ -1,4 +1,5 @@
-using MesETL.App.Cache;
+using System.Text;
+using MesETL.App.Cache;
 
 namespace MesETL.App.Services.Loggers;
 

MesETL.App/Services/Loggers/LoggerTaskMonitorLogger.cs

@@ -17,7 +17,7 @@ public class LoggerTaskMonitorLogger : ITaskMonitorLogger
         var sb = new StringBuilder();
         sb.Append($"{name}: {{");
         sb.AppendJoin(',', properties.Select((pair, i) => $" {pair.Key}: {pair.Value}"));
-        sb.Append('}');
+        sb.Append([' ', '}']);
         // var args = new List<string> { name };
         // properties.Aggregate(args, (args, pair) =>
         // {

MesETL.App/Services/ProcessContext.cs

@@ -11,7 +11,7 @@ public class ProcessContext
     private long _inputCount;
     private long _transformCount;
     private long _outputCount;
-    private readonly ConcurrentDictionary<string, long> _tableProgress = new();
+    private readonly ConcurrentDictionary<string, (long input, long output)> _tableProgress = new();
     public bool HasException => _hasException;
     public bool IsInputCompleted { get; private set; }
     public bool IsTransformCompleted { get; private set; }
@@ -35,15 +35,17 @@ public class ProcessContext
         set => Interlocked.Exchange(ref _outputCount, value);
     }
 
+    public long MaxMemoryUsage { get; set; }
+
 
     // TableName -> Count
-    public IReadOnlyDictionary<string, long> TableProgress => _tableProgress;
+    public IReadOnlyDictionary<string, (long input, long output)> TableProgress => _tableProgress;
 
     public void CompleteInput() => IsInputCompleted = true;
 
     public void CompleteTransform() => IsTransformCompleted = true;
     public void CompleteOutput() => IsOutputCompleted = true;
-    public bool AddException(Exception e) => _hasException = true;
+    public bool AddException(Exception e) => _hasException = true; // 没打算存起来，暂时先加个标记
 
     public void AddInput() => Interlocked.Increment(ref _inputCount);
 
@@ -55,16 +57,21 @@ public class ProcessContext
     public void AddOutput() => Interlocked.Increment(ref _outputCount);
     public void AddOutput(int count) => Interlocked.Add(ref _outputCount, count);
 
-    public void AddTableOutput(string table, int count)
+    public void AddTableInput(string table, int count)
     {
-        _tableProgress.AddOrUpdate(table, count, (k, v) => v + count);
-        AddOutput(count);
+        _tableProgress.AddOrUpdate(table, (input: count, output: 0), (k, tuple) =>
+        {
+            tuple.input += count;
+            return tuple;
+        });
     }
 
-    public long GetTableOutput(string table)
+    public void AddTableOutput(string table, int count)
     {
-        if(!_tableProgress.TryGetValue(table, out var count))
-            throw new ApplicationException($"未找到表{table}输出记录");
-        return count;
+        _tableProgress.AddOrUpdate(table, (input: 0, output: count), (k, tuple) =>
+        {
+            tuple.output += count;
+            return tuple;
+        });
     }
 }

MesETL.App/Services/RecordQueuePool.cs

@@ -9,7 +9,8 @@ public class RecordQueuePool
 
     public IReadOnlyDictionary<string, DataRecordQueue> Queues => _queues;
 
-    public void AddQueue(string key, int boundedCapacity = 200_0000) => AddQueue(key, new DataRecordQueue(boundedCapacity));
+    public void AddQueue(string key, int boundedCapacity = 200_0000, long maxCharCount = 2_147_483_648) 
+        => AddQueue(key, new DataRecordQueue(boundedCapacity, maxCharCount));
     
     public void AddQueue(string key, DataRecordQueue queue)
     {

MesETL.App/Services/Seq/SeqConfig.cs

@@ -0,0 +1,42 @@
+// ReSharper disable InconsistentNaming
+namespace MesETL.App.Services.Seq;
+
+public class SeqConfig(string Name, bool Recycle = true, int Step = 1, long Max = 999_999_999)
+{
+    public string Name { get; init; } = Name;
+    public bool Recycle { get; init; } = Recycle;
+    public int Step { get; init; } = Step;
+    public long Max { get; init; } = Max;
+
+    public static readonly SeqConfig ItemNo = new("seq_ItemNo", true, 1, 999_999_999);
+    public static readonly SeqConfig OrderModuleID = new("seq_order_module_id", false);
+    public static readonly SeqConfig OrderDataID = new("seq_order_data_id", false);
+    public static readonly SeqConfig OrderItemID = new("seq_order_item", false);
+    public static readonly SeqConfig ProcessStepID = new("seq_step_id", false);
+    public static readonly SeqConfig PackageNo = new("seq_pack_no", true, 1, 9_999_999);
+    public static readonly SeqConfig PlanNo = new("seq_plan_order", true, 1, 999_999);
+    public static readonly SeqConfig SimplePlanNo = new("seq_simple_plan_order", true, 1, 999_999);
+
+    // 下面这些类型的流水号在BaseService添加实体时进行生成
+    public static readonly SeqConfig MachineID = new("seq_machine_id", false);
+    public static readonly SeqConfig OrderBlockPlanID = new("seq_order_block_plan_id", false);
+    public static readonly SeqConfig OrderDataGoodsID = new("seq_order_data_goods_id", false);
+    public static readonly SeqConfig OrderPackageID = new("seq_order_pack_id", false);
+    public static readonly SeqConfig OrderProcessID = new("seq_order_process_id", false);
+    public static readonly SeqConfig OrderProcessStepItemID = new("seq_order_process_step_item_id", false);
+    public static readonly SeqConfig ProcessGroupID = new("seq_process_group_id", false);
+    public static readonly SeqConfig ProcessInfoID = new("seq_process_info_id", false);
+    public static readonly SeqConfig ProcessItemExpID = new("seq_process_item_exp_id", false);
+    public static readonly SeqConfig ProcessScheduleCapacityID = new("seq_process_schedule_capacity_id", false);
+    public static readonly SeqConfig ProcessStepEfficiencyID = new("seq_process_step_efficiency_id", false);
+    public static readonly SeqConfig ReportTemplateID = new("seq_report_template_id", false);
+    public static readonly SeqConfig SysConfigKey = new("seq_sys_config_key", false);
+    public static readonly SeqConfig WorkCalendarID = new("seq_work_calendar_id", false);
+    public static readonly SeqConfig WorkShiftID = new("seq_work_shift_id", false);
+    public static readonly SeqConfig WorkTimeID = new("seq_work_time_id", false);
+    public static readonly SeqConfig OrderPatchDetailID = new("seq_order_patch_detail_id", false);
+    public static readonly SeqConfig OrderModuleExtraID = new("seq_order_module_extra_id", false);
+    public static readonly SeqConfig SimplePackageID = new("seq_simple_pack_id", false);
+    public static readonly SeqConfig OrderModuleItemID = new("seq_order_module_item_id", false);
+    public static readonly SeqConfig OrderWaveGroupID = new("seq_order_wave_group_id", false);
+}

MesETL.App/Services/Seq/SeqService.cs

@@ -0,0 +1,136 @@
+using System.Collections.Concurrent;
+using System.Diagnostics;
+using System.Runtime.CompilerServices;
+using System.Text;
+using MesETL.App.Options;
+using MesETL.Shared.Helper;
+using Microsoft.Extensions.Options;
+using MySqlConnector;
+
+namespace MesETL.App.Services.Seq;
+
+public class SeqService
+{
+    private readonly string _connectionString;
+    private readonly ConcurrentDictionary<SeqConfig, long> _cachedSequence;
+
+    public IReadOnlyDictionary<SeqConfig, long> CachedSequence => _cachedSequence;
+
+    public SeqService(IOptions<DatabaseOutputOptions> options)
+    {
+        var connStr = options.Value.ConnectionString ?? throw new ApplicationException("未配置输出数据库连接字符串");
+        var builder = new MySqlConnectionStringBuilder(connStr)
+        {
+            Database = "mes_global"
+        };
+        _connectionString = builder.ConnectionString;
+
+        _cachedSequence = new ConcurrentDictionary<SeqConfig, long>();
+    }
+
+    private async Task<long> UpdateSequenceID(string name,int step,long max,bool recycle, int add)
+    {
+        var sql = new StringBuilder(
+            $"""
+            INSERT INTO seq (SeqName,CurrentVal,Increment,MinVal,MaxVal,UpdateTime)
+            VALUES ({name},{add},{step},1,{max},NOW()) 
+            ON DUPLICATE KEY UPDATE UpdateTime = NOW(),
+            """);
+        if (recycle)
+        {
+            sql.Append($"CurrentVal = (@updatedVal := IF(CurrentVal + {add} >= MaxVal, {add}, CurrentVal + {add}));");
+        }
+        else
+        {
+            sql.Append($"CurrentVal = (@updatedVal := CurrentVal + {add});");
+        }
+        sql.Append("SELECT @updatedVal;");
+        var result = await DatabaseHelper.QueryScalarAsync(_connectionString, sql.ToString());
+        return Convert.ToInt64(result);
+    }
+
+    public async Task<long> PeekKey(SeqConfig config)
+    {
+        var sql = $"SELECT CurrentVal FROM seq WHERE SeqName = '{config.Name}' LIMIT 1;";
+        return Convert.ToInt64(await DatabaseHelper.QueryScalarAsync(_connectionString, sql));
+    }
+    
+    public async Task<long[]> GetKeys(SeqConfig config, int count)
+    {
+        if (count < 1) return [];
+
+        var list = new long[count];
+        var add = config.Step * count;
+        var lastId = await UpdateSequenceID(config.Name, config.Step, config.Max, config.Recycle, add);
+        var step = Convert.ToInt64(config.Step);
+        for (var i = count - 1; i > -1; i--)
+        {
+            list[i] = lastId;
+            lastId -= step;
+        }
+
+        return list;
+    }
+
+    /// <summary>
+    /// 添加并取得一个缓存的流水号
+    /// </summary>
+    /// <param name="config"></param>
+    /// <returns></returns>
+    public long AddCachedSeq(SeqConfig config)
+    {
+        if (!_cachedSequence.TryGetValue(config, out var val))
+        {
+            var seq = PeekKey(config).GetAwaiter().GetResult();
+            val = seq;
+            _cachedSequence[config] = val;
+        }
+
+        var step = config.Step;
+        if (config.Recycle)
+        {
+            val = val + step >= config.Max ? val : val + step;
+        }
+        else val += step;
+        _cachedSequence[config] = val;
+        return val;
+    }
+    
+    /// <summary>
+    /// 移除一个缓存的流水号
+    /// </summary>
+    /// <param name="config"></param>
+    public bool RemoveCachedSeq(SeqConfig config)
+    {
+        return _cachedSequence.Remove(config, out _);
+    }
+    
+    /// <summary>
+    /// 清空所有缓存的流水号
+    /// </summary>
+    public void ClearCache()
+    {
+        _cachedSequence.Clear();
+    }
+
+    /// <summary>
+    /// 将缓存的流水号应用至数据库
+    /// </summary>
+    public async Task ApplyToDatabaseAsync()
+    {
+        if (_cachedSequence.Count == 0) return;
+        var sql = GenerateCachedSeqSql();
+        await DatabaseHelper.NonQueryAsync(_connectionString, sql);
+    }
+
+    private string GenerateCachedSeqSql()
+    {
+        var sb = new StringBuilder();
+        foreach (var kv in _cachedSequence)
+        {
+            sb.AppendLine($"UPDATE seq SET CurrentVal = {kv.Value} WHERE SeqName = '{kv.Key.Name}';");
+        }
+
+        return sb.ToString();
+    }
+}

MesETL.App/Services/TaskManager.cs

@@ -1,5 +1,6 @@
-using ApplicationException = System.ApplicationException;
-using TaskExtensions = MesETL.App.Helpers.TaskExtensions;
+using Serilog;
+using ApplicationException = System.ApplicationException;
+using TaskExtensions = MesETL.Shared.Helper.TaskExtensions;
 
 namespace MesETL.App.Services;
 
@@ -37,6 +38,8 @@ public class TaskManager
     {
         var task = Task.Run(async () =>
         {
+            // Log.Logger.Verbose("[任务管理器] 新的任务已创建");
+            Interlocked.Increment(ref _runningTaskCount);
             try
             {
                 await func();
@@ -45,13 +48,13 @@ public class TaskManager
             catch(Exception ex)
             {
                 OnException?.Invoke(ex);
+                Log.Logger.Error(ex, "[任务管理器] 执行任务时出错");
             }
             finally
             {
                 Interlocked.Decrement(ref _runningTaskCount);
             }
         }, cancellationToken);
-        Interlocked.Increment(ref _runningTaskCount);
         return task;
     }
 
@@ -59,8 +62,10 @@ public class TaskManager
     {
         var task = Task.Factory.StartNew(async obj => // 性能考虑，这个lambda中不要捕获任何外部变量!
         {
+            // Log.Logger.Verbose("[任务管理器] 新的任务已创建");
             if (obj is not Tuple<Func<object?, Task>, object?> tuple)
                 throw new ApplicationException("这个异常不该出现");
+            Interlocked.Increment(ref _runningTaskCount);
             try
             {
                 await tuple.Item1(tuple.Item2);
@@ -69,13 +74,13 @@ public class TaskManager
             catch(Exception ex)
             {
                 OnException?.Invoke(ex);
+                Log.Logger.Error(ex, "[任务管理器] 执行任务时出错");
             }
             finally
             {
                 Interlocked.Decrement(ref _runningTaskCount);
             }
         }, Tuple.Create(func, arg), cancellationToken).Unwrap();
-        Interlocked.Increment(ref _runningTaskCount);
         return task;
     }
 }

MesETL.App/appsettings.json

@@ -1,31 +1,46 @@
 {
+  "MemoryThreshold": 6,
+  "GCIntervalMilliseconds": -1,
+  "UnsafeVariable": true,
+  "DryRun": true, // 试运行，仅输入每张表的前100000条数据
   "Logging": {
     "LogLevel": {
-      "Default": "Debug"
+      "Default": "Trace"
     }
   },
   "Input":{
-    "InputDir": "D:\\Dump\\MyDumper-ZST 2024-02-05", // Csv数据输入目录
+    "InputDir": "D:\\Data\\DatabaseDump\\Prod_Mock_CSV_2024-12-31", // Csv数据输入目录
     "UseMock": false, // 使用模拟数据进行测试
-    "MockCountMultiplier": 1 // 模拟数据量级的乘数
+    "MockCountMultiplier": 1, // 模拟数据量级的乘数
+//    "TableOrder": ["order_item"], // 按顺序输入的表
+    "TableIgnoreList": [] // 忽略输入的表
   },
   "Transform":{
-    "StrictMode": false, // 设为true时如果数据转换发生错误，立刻停止程序
+    "StrictMode": true, // 设为true时如果数据转换发生错误，立刻停止程序
     "EnableFilter": true, // 启用数据过滤
     "EnableReplacer": true, // 启用数据修改
     "EnableReBuilder": true, // 启用数据重建
-    "CleanDate": "202301" // 当数据过滤开启时，删除这个时间之前的数据
+    "CleanDate": "202401" // 当数据过滤开启时，删除这个时间之前的数据
   },
   "Output":{
-    "ConnectionString": "Server=127.0.0.1;Port=3306;UserId=root;Password=cfmes123456;", // 要分库，不用加'Database='了
+    "ConnectionString": "Server=127.0.0.1;Port=3306;UserId=root;Password=123456;", // 要分库，不用加'Database='了
     "MaxAllowedPacket": 67108864, 
     "FlushCount": 10000, // 每次提交记录条数
     "MaxDatabaseOutputTask" : 4, // 每个数据库最大提交任务数
-    "TreatJsonAsHex": false // 将json列作为16进制格式输出(0x前缀)，生产库是没有json列的
+    "TreatJsonAsHex": false, // 使Json列输出时带上"0x"前缀
+    "NoOutput": [], // 不输出的表
+    "ForUpdate":
+    {
+    }
+  },  
+  "RecordQueue":{
+    "ProducerQueueLength": 20000, // 输入队列最大长度
+    "ConsumerQueueLength": 20000, // 每个输出队列最大长度
+    "MaxByteCount": 3221225472 // 队列最大字节数
   },
   "RedisCache": {
     "Configuration": "192.168.1.246:6380",
-    "InstanceName" : "mes-etl:"
+    "InstanceName" : "mes-etl:" 
   },
   "TenantDb": // 分库配置
   {
@@ -39,10 +54,18 @@
       },
       "prod":{
         "mesdb_1": 5000,
-        "mesdb_2": 10000,
-        "mesdb_3": 15000,
-        "mesdb_4": 20000,
-        "mesdb_5": 2147483647
+        "mesdb_2": 7500,
+        "mesdb_3": 10000,
+        "mesdb_4": 15000,
+        "mesdb_5": 20000,
+        "mesdb_6": 2147483647
+      },
+      "mock_void":{
+        "mesdb_1_void": 5000,
+        "mesdb_2_void": 10000,
+        "mesdb_3_void": 15000,
+        "mesdb_4_void": 20000,
+        "mesdb_5_void": 2147483647
       }
     }
   }

MesETL.Clean/MesETL.Clean.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <TargetFramework>net8.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\MesETL.Shared\MesETL.Shared.csproj" />
+    </ItemGroup>
+
+</Project>

MesETL.Clean/Program.cs

@@ -0,0 +1,55 @@
+using MesETL.Shared.Helper;
+
+var connStr = GetArg("-s") ?? throw new ApplicationException("未配置数据库连接字符串");
+var eachLimit = int.Parse(GetArg("-l") ?? "1000");
+var parallelTask = int.Parse(GetArg("-p") ?? "4");
+
+var deletionCount = 0;
+
+Console.WriteLine("Running Deletion...");
+
+_ = Task.Run(async () =>
+{
+    while (true)
+    {
+        await Task.Delay(5000);
+        Console.WriteLine($"[{DateTime.Now}] DELETE COUNT: {deletionCount}");
+    }
+});
+
+await Parallel.ForAsync(0, parallelTask, async (i, token) =>
+{
+    while (true)
+    {
+        var effectRows = await DatabaseHelper.NonQueryAsync(connStr,
+            $"DELETE FROM `order_data_block` WHERE CompanyID = 0 ORDER BY ID LIMIT {eachLimit};", token);
+        if(effectRows == 0)
+            break;
+        Interlocked.Add(ref deletionCount, effectRows);
+    }
+});
+
+Console.WriteLine($"[{DateTime.Now}] DELETE COUNT: {deletionCount}");
+return;
+string? GetArg(string instruct)
+{
+    var idx = Array.IndexOf(args, instruct);
+    if (idx == -1)
+        return null;
+    if (args[idx + 1].StartsWith('-'))
+        throw new ArgumentException("Argument Lost", nameof(instruct));
+    return args[idx + 1];
+}
+
+// var match = await DatabaseHelper.QueryTableAsync(connStr, 
+//     $"SELECT `ID` FROM `order_data_block` WHERE CompanyID = 0 LIMIT {eachLimit};", 
+//     token);
+// var rows = match.Tables[0].Rows;
+// if (rows.Count == 0)
+//     return;
+//     
+// foreach (DataRow row in rows)
+// {
+//     var id = row["ID"].ToString();
+//     await DatabaseHelper.NonQueryAsync(connStr, $"DELETE FROM `order_data_block` WHERE `ID` = {id}", token);
+// }

MesETL.Shared/Compression/DeflateArchive.cs

@@ -0,0 +1,39 @@
+using System.IO.Compression;
+
+namespace MesETL.Shared.Compression;
+
+/// <summary>
+/// Deflate压缩工具类
+/// </summary>
+public static class DeflateArchive
+{
+    /// <summary>
+    /// 解压Deflate
+    /// </summary>
+    /// <param name="input"></param>
+    /// <returns></returns>
+    public static byte[] Decompress(byte[] input)
+    {
+        using var msi = new MemoryStream(input);
+        using var mso = new MemoryStream();
+        using var ds = new DeflateStream(msi, CompressionMode.Decompress);
+        ds.CopyTo(mso);
+        ds.Flush();
+        return mso.ToArray();
+    }
+
+    /// <summary>
+    /// 压缩Deflate
+    /// </summary>
+    /// <param name="input"></param>
+    /// <returns></returns>
+    public static byte[] Compress(byte[] input)
+    {
+        using var msi = new MemoryStream(input);
+        using var mso = new MemoryStream();
+        using var ds = new DeflateStream(mso, CompressionMode.Compress);
+        msi.CopyTo(ds);
+        ds.Flush();
+        return mso.ToArray();
+    }
+}

MesETL.Shared/Helper/DatabaseHelper.cs

@@ -1,10 +1,15 @@
 using System.Data;
 using MySqlConnector;
 
-namespace MesETL.App.Helpers;
+namespace MesETL.Shared.Helper;
 
 public static class DatabaseHelper
 {
+    /// <summary>
+    /// 创建一个MySql连接
+    /// </summary>
+    /// <param name="connStr"></param>
+    /// <returns></returns>
     public static MySqlConnection CreateConnection(string connStr)
     {
         var newConnStr = new MySqlConnectionStringBuilder(connStr)
@@ -15,11 +20,18 @@ public static class DatabaseHelper
         return new MySqlConnection(newConnStr);
     }
     
-    public static async Task<DataSet> QueryTableAsync(string connStr, string sql)
+    /// <summary>
+    /// 使用语句查询数据库
+    /// </summary>
+    /// <param name="connStr"></param>
+    /// <param name="sql"></param>
+    /// <param name="ct"></param>
+    /// <returns></returns>
+    public static async Task<DataSet> QueryTableAsync(string connStr, string sql, CancellationToken ct = default)
     {
         await using var conn = CreateConnection(connStr);
         if(conn.State is not ConnectionState.Open)
-            await conn.OpenAsync();
+            await conn.OpenAsync(ct);
         await using var cmd = conn.CreateCommand();
         cmd.CommandText = sql;
         var ds = new DataSet();
@@ -27,26 +39,47 @@ public static class DatabaseHelper
         return ds;
     }
 
-    public static async Task<object?> QueryScalarAsync(string connStr, string sql)
+    /// <summary>
+    /// 使用语句进行标量查询
+    /// </summary>
+    /// <param name="connStr"></param>
+    /// <param name="sql"></param>
+    /// <param name="ct"></param>
+    /// <returns></returns>
+    public static async Task<object?> QueryScalarAsync(string connStr, string sql, CancellationToken ct = default)
     {
         await using var conn = CreateConnection(connStr);
         if(conn.State is not ConnectionState.Open)
-            await conn.OpenAsync();
+            await conn.OpenAsync(ct);
         await using var cmd = conn.CreateCommand();
         cmd.CommandText = sql;
-        return await cmd.ExecuteScalarAsync();
+        return await cmd.ExecuteScalarAsync(ct);
     }
 
-    public static async Task<int> NonQueryAsync(string connStr, string sql)
+    /// <summary>
+    /// 执行非查询语句
+    /// </summary>
+    /// <param name="connStr"></param>
+    /// <param name="sql"></param>
+    /// <param name="ct"></param>
+    /// <returns></returns>
+    public static async Task<int> NonQueryAsync(string connStr, string sql, CancellationToken ct = default)
     {
         await using var conn = CreateConnection(connStr);
         if(conn.State is not ConnectionState.Open)
-            await conn.OpenAsync();
+            await conn.OpenAsync(ct);
         await using var cmd = conn.CreateCommand();
         cmd.CommandText = sql;
-        return await cmd.ExecuteNonQueryAsync();
+        return await cmd.ExecuteNonQueryAsync(ct);
     }
 
+    /// <summary>
+    /// 在事务中执行语句
+    /// </summary>
+    /// <param name="connStr"></param>
+    /// <param name="sql"></param>
+    /// <param name="parameters"></param>
+    /// <returns></returns>
     public static async Task<int> TransactionAsync(string connStr, string sql, params MySqlParameter[] parameters)
     {
         await using var conn = CreateConnection(connStr);

MesETL.Shared/Helper/Extensions.Dictionary.cs

@@ -1,4 +1,4 @@
-namespace MesETL.App.Helpers;
+namespace MesETL.Shared.Helper;
 
 public static class DictionaryExtensions
 {

MesETL.Shared/Helper/Extensions.IEnumerable.cs

@@ -1,7 +1,7 @@
 using System.Diagnostics.CodeAnalysis;
 using System.Reflection;
 
-namespace MesETL.App.Helpers;
+namespace MesETL.Shared.Helper;
 #nullable disable
 public static class EnumerableExtensions
 {

MesETL.Shared/Helper/Extensions.Lang.cs

@@ -0,0 +1,63 @@
+namespace Azusa.Shared.Extensions;
+
+/// <summary>
+/// 使用Range作为参数的迭代器方法
+/// <br/>
+/// 扩展foreach关键字来实现类似<c>foreach (var i in 1..5)</c>的效果
+/// </summary>
+public static class ForeachExtensions
+{
+    /// <summary>
+    /// 拓展Range结构实现GetEnumerator方法供foreach读取，实现foreach(var i in x..y)
+    /// </summary>
+    /// <returns></returns>
+    public static CustomIntEnumerator GetEnumerator(this Range range)
+    {
+        return new CustomIntEnumerator(range);
+    }
+
+    /// <summary>
+    /// 拓展int类实现GetEnumerator方法供foreach读取，实现foreach(var i in x)
+    /// </summary>
+    /// <param name="end"></param>
+    /// <returns></returns>
+    public static CustomIntEnumerator GetEnumerator(this int end)
+    {
+        return new CustomIntEnumerator(end);
+        return new CustomIntEnumerator(new Range(0, end));//在执行空函数时性能比上一句低10倍，Why
+    }
+}
+
+//使用引用结构体增强性能
+public ref struct CustomIntEnumerator
+{
+    private int _current;
+    private readonly int _end;
+
+    public CustomIntEnumerator(Range range)
+    {
+        //避免某些时候从结尾开始编制
+        // x.. 时会产生Range(x,^0)
+        if (range.End.IsFromEnd)
+        {
+            throw new NotSupportedException("不支持从结尾编制索引");
+        }
+        _current = range.Start.Value - 1;
+        _end = range.End.Value - 1;//迭代器不包含范围的尾部
+    }
+
+    public CustomIntEnumerator(int end)
+    {
+        _current = -1;
+        _end = end;
+    }
+
+    /* 注意，供foeach使用的迭代器不需要实现IEnumerator接口，只需要提供Current属性以及MoveNext方法即可，*/
+    public int Current => _current;
+
+    public bool MoveNext()
+    {
+        _current++;
+        return _current <= _end;
+    }
+}

MesETL.Shared/Helper/Extensions.String.cs

@@ -1,7 +1,7 @@
 using System.Globalization;
 using System.Text;
 
-namespace MesETL.App.Helpers;
+namespace MesETL.Shared.Helper;
 
 public static class StringExtensions
 {

MesETL.Shared/Helper/Extensions.Task.cs

@@ -1,4 +1,4 @@
-namespace MesETL.App.Helpers;
+namespace MesETL.Shared.Helper;
 
 public static class TaskExtensions
 {

MesETL.Shared/MesETL.Shared.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net8.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <PackageReference Include="MySqlConnector" Version="2.4.0" />
+    </ItemGroup>
+
+</Project>

MesETL.Test/DataFix.cs

@@ -1,5 +1,6 @@
 using System.Data;
 using MesETL.App.Helpers;
+using MesETL.Shared.Helper;
 using MySqlConnector;
 using Xunit.Abstractions;
 

MesETL.Test/DatabaseToolBox.cs

@@ -1,6 +1,7 @@
 using System.Data;
 using System.Text;
 using MesETL.App.Helpers;
+using MesETL.Shared.Helper;
 using MySqlConnector;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
@@ -11,7 +12,7 @@ namespace TestProject1;
 public class DatabaseToolBox
 {
     private readonly ITestOutputHelper _output;
-    public const string ConnStr = "Server=127.0.0.1;Port=3306;UserId=root;Password=cfmes123456;";
+    public const string ConnStr = "Server=localhost;Port=3306;UserId=root;Password=123456;";
 
     public DatabaseToolBox(ITestOutputHelper output)
     {
@@ -107,6 +108,24 @@ public class DatabaseToolBox
         }).ToArray();
     }
 
+    [Theory]
+    [InlineData(["mesdb_1"])]
+    [InlineData(["mesdb_2"])]
+    [InlineData(["mesdb_3"])]
+    [InlineData(["mesdb_4"])]
+    [InlineData(["mesdb_5"])]
+    public async Task ShowIndex(string database)
+    {
+        var indexes = await GetAllTableIndexes(database);
+        var sb = new StringBuilder();
+        foreach (var (tableName, indexName, isUnique, columnName, tableIndexType) in indexes!)
+        {
+            sb.AppendLine($"Drop {(isUnique ? "UNIQUE" : string.Empty)} INDEX `{indexName}` ON `{database}`.`{tableName}`;");
+        }
+
+        _output.WriteLine(sb.ToString());
+    }
+
     [Theory]
     [InlineData(["cferp_test_1", "D:/Indexes_cferp_test_1.json"])]
     [InlineData(["cferp_test_2", "D:/Indexes_cferp_test_2.json"])]
@@ -137,9 +156,11 @@ public class DatabaseToolBox
     }
 
     [Theory]
-    [InlineData(["cferp_test_1"])]
-    [InlineData(["cferp_test_2"])]
-    [InlineData(["cferp_test_3"])]
+    [InlineData(["mesdb_1"])]
+    [InlineData(["mesdb_2"])]
+    [InlineData(["mesdb_3"])]
+    [InlineData(["mesdb_4"])]
+    [InlineData(["mesdb_5"])]
     public async Task DropAllIndex(string database)
     {
         var indexes = await GetAllTableIndexes(database);
@@ -151,4 +172,53 @@ public class DatabaseToolBox
         await DatabaseHelper.NonQueryAsync(ConnStr, sb.ToString());
         _output.WriteLine($"Dropped {indexes.Length} indexes from {database}");
     }
+
+    [Theory]
+    [InlineData("mesdb_1")]
+    [InlineData("mesdb_2")]
+    [InlineData("mesdb_3")]
+    [InlineData("mesdb_4")]
+    [InlineData("mesdb_5")]
+    [InlineData("mesdb_6")]
+    public async Task TruncateAllTable(string database)
+    {
+        var tables = await DatabaseHelper.QueryTableAsync(ConnStr,
+            $"""
+             SELECT TABLE_NAME FROM information_schema.`TABLES` WHERE TABLE_SCHEMA = '{database}';
+             """);
+        var sb = new StringBuilder();
+        sb.AppendLine($"USE `{database}`;");
+        foreach (DataRow row in tables.Tables[0].Rows)
+        {
+            var tableName = row["TABLE_NAME"].ToString();
+            var sql = $"""
+                       TRUNCATE TABLE `{tableName}`;
+                       """;
+            sb.AppendLine(sql);
+        }
+        await DatabaseHelper.NonQueryAsync(ConnStr, sb.ToString());
+    }
+
+    [Theory]
+    [InlineData("cferp_test_1")]
+    [InlineData("cferp_test_2")]
+    [InlineData("cferp_test_3")]
+    public async Task AnalyzeAllTable(string database)
+    {
+        var tables = await DatabaseHelper.QueryTableAsync(ConnStr,
+            $"""
+             SELECT TABLE_NAME FROM information_schema.`TABLES` WHERE TABLE_SCHEMA = '{database}';
+             """);
+        var sb = new StringBuilder();
+        sb.AppendLine($"USE `{database}`;");
+        foreach (DataRow row in tables.Tables[0].Rows)
+        {
+            var tableName = row["TABLE_NAME"].ToString();
+            var sql = $"""
+                       ANALYZE TABLE `{tableName}`;
+                       """;
+            sb.AppendLine(sql);
+        }
+        await DatabaseHelper.NonQueryAsync(ConnStr, sb.ToString());
+    }
 }

MesETL.Test/InputServiceTest.cs

@@ -0,0 +1,54 @@
+using MesETL.App.Helpers;
+using MesETL.App.HostedServices;
+using MesETL.App.Options;
+using MesETL.App.Services;
+using MesETL.App.Services.ETL;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Options;
+using TestProject1.XUnit;
+using Xunit.Abstractions;
+
+namespace TestProject1;
+
+public class InputServiceTest : TestBase
+{
+    private readonly ITestOutputHelper _output;
+
+    public InputServiceTest(ITestOutputHelper output) : base(output)
+    {
+        _output = output;
+    }
+
+    /// <summary>
+    /// 测试文件输入服务是否能正确的认到应有的文件
+    /// </summary>
+    /// <param name="inputDir"></param>
+    /// <param name="tableOrder"></param>
+    /// <param name="ignored"></param>
+    /// <param name="assertCount"></param>
+    [Theory]
+    [InlineData(@"D:\Data\DatabaseDump\MyDumper-ZST 2024-12-3", null, new string[0], 152)] // 没有seq和三个库的efmigration
+    [InlineData(@"D:\Data\DatabaseDump\MyDumper-ZST 2024-12-3", new[] { "order", "machine" }, new string[0],
+        11)] // 只有order和machine
+    [InlineData(@"D:\Data\DatabaseDump\MyDumper-ZST 2024-12-3", null, new[] { "order", "machine" },
+        152 - 11)] // 忽略order和machine
+    public void Test_InputInfo_Get_And_Order(string inputDir, string[]? tableOrder, string[] ignored, int assertCount)
+    {
+        var options = new OptionsWrapper<DataInputOptions>(new DataInputOptions()
+        {
+            InputDir = inputDir,
+            FileInputMetaBuilder = DumpDataHelper.MyDumperFileInputMetaBuilder,
+            TableOrder = tableOrder,
+            TableIgnoreList = ignored
+        });
+        var ctx = new ProcessContext();
+        var queue = new DataRecordQueue();
+        var dataReaderFactory = new DataReaderFactory(CreateXUnitLogger<DataReaderFactory>(), options);
+        var sut = new FileInputService(CreateXUnitLogger<FileInputService>(), options, ctx, queue, dataReaderFactory,
+            new ConfigurationManager());
+
+        var result = sut.GetOrderedInputInfo(inputDir).ToArray();
+        WriteJson(result);
+        Assert.True(assertCount == result.Length);
+    }
+}

MesETL.Test/MesDatabaseHelper.cs

@@ -1,4 +1,5 @@
 using MesETL.App.Helpers;
+using MesETL.Shared.Helper;
 
 namespace TestProject1;
 

MesETL.Test/MesETL.Test.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-        <TargetFramework>net8.0</TargetFramework>
+        <TargetFramework>net9.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
 
@@ -11,13 +11,14 @@
     </PropertyGroup>
 
     <ItemGroup>
-        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.0"/>
-        <PackageReference Include="xunit" Version="2.4.2"/>
-        <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.12.0" />
+        <PackageReference Include="NSubstitute" Version="5.3.0" />
+        <PackageReference Include="xunit" Version="2.9.2" />
+        <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
             <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
             <PrivateAssets>all</PrivateAssets>
         </PackageReference>
-        <PackageReference Include="coverlet.collector" Version="6.0.0">
+        <PackageReference Include="coverlet.collector" Version="6.0.2">
             <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
             <PrivateAssets>all</PrivateAssets>
         </PackageReference>

MesETL.Test/Services/SeqServiceTests.cs

@@ -0,0 +1,33 @@
+using System.Reflection;
+using Azusa.Shared.Extensions;
+using MesETL.App.Options;
+using MesETL.App.Services.Seq;
+using Microsoft.Extensions.Options;
+using TestProject1.XUnit;
+using Xunit.Abstractions;
+
+namespace TestProject1.Services;
+
+public class SeqServiceTests : TestBase
+{
+    public SeqServiceTests(ITestOutputHelper output) : base(output)
+    {
+    }
+
+    [Fact]
+    public void Test_Sequence_Sql_Generation()
+    {
+        var sut = new SeqService(new OptionsWrapper<DatabaseOutputOptions>(new DatabaseOutputOptions()
+        {
+            ConnectionString = "Server=127.0.0.1;Port=3306;UserId=root;Password=123456;"
+        }));
+
+        foreach (var i in 10)
+        {
+            Write("Seq: " + sut.AddCachedSeq(SeqConfig.OrderWaveGroupID));
+        }
+
+        var sql = typeof(SeqService).GetMethod("GenerateCachedSeqSql", BindingFlags.Instance | BindingFlags.NonPublic)!.Invoke(sut, []);
+        Write(sql ?? "null");
+    }
+}

MesETL.Test/Test.cs

@@ -0,0 +1,152 @@
+using System.Collections.Concurrent;
+using System.Diagnostics;
+using MesETL.App.Services.ETL;
+using MesETL.Shared.Helper;
+using Xunit.Abstractions;
+using ZstdSharp;
+
+namespace TestProject1;
+
+public class Test
+{
+    private readonly ITestOutputHelper _output;
+
+    public Test(ITestOutputHelper output)
+    {
+        _output = output;
+    }
+
+    [Theory]
+    [InlineData([@"D:\Dump\NewMockData2\cferp.order_box_block.00000.dat.zst"])]
+    public async Task ZstdDecompressTest(string inputFile)
+    {
+        var count = 0;
+        var flag = true;
+        var sw = Stopwatch.StartNew();
+        var reader = new StreamReader(new DecompressionStream(File.OpenRead(inputFile)));
+        var monitor = Task.Run(async () =>
+        {
+            var lastElapse = sw.ElapsedMilliseconds;
+            var lastCount = 0;
+            while (flag)
+            {
+                await Task.Delay(2000);
+
+                _output.WriteLine($"speed: {(count - lastCount) / ((sw.ElapsedMilliseconds - lastElapse) / 1000f)}");
+                lastElapse = sw.ElapsedMilliseconds;
+                lastCount = count;
+            }
+        });
+        while (!reader.EndOfStream)
+        {
+            var str = await reader.ReadLineAsync();
+            char a;
+            // foreach (var c in str)
+            // {
+            //     a = c;
+            // }
+            CsvReader.ParseRowFaster(str, '"', ',');
+            count++;
+        }
+
+        flag = false;
+        monitor.Wait();
+    }
+
+    
+    public static IEnumerable<object[]> ParseRowData()
+    {
+        yield return
+            [@"20220104020855,""2022-01-04 10:06:46"",1455,""0001-01-01 00:00:00"",""1"",0,""2"",""0"",\N,""0"",22010"];
+        yield return 
+            [@"20220104020858,""2022-01-04 15:08:22"",1455,""0001-01-01 00:00:00"",""1"",838,""2"",""0"",""5"",""0"",22010"];
+        yield return
+            [@"5586326,20220104020855,220105981029,""1"",482278,482279,3768774,0,0,""1.000"",1455,22010"];
+        yield return
+            [@"130658,""PD220104002302"",3,4616,""2022-01-04 15:10:40"",1443,""2022-01-04 15:10:40"",""2022-01-04 15:10:51"",0,"""",0,1455,""0001-01-01 00:00:00"",1,5B32303232303130343032303835385D,E590B8E5A1912D2DE590B8E5A1912D2D31382D2D323030302A3630302D2D3130E789872D2D352E3936333B6361696C69616F2D2D79616E73652D2D392D2D323031302A313137342D2D31E789872D2D322E3336,""0"",0"];
+    }
+    [Theory]
+    [MemberData(nameof(ParseRowData))]
+    public void ParseRowFasterTest(string row)
+    {
+        var fields = CsvReader.ParseRowFaster(row, '"', ',');
+        _output.WriteLine(string.Join(',', fields));
+    }
+
+    [Fact]
+    public void DictMemoryTest()
+    {
+        var dict = new ConcurrentDictionary<string, string>();
+        for (int i = 0; i < 3000000; i++)
+        {
+            dict.AddOrUpdate(Guid.NewGuid().ToString(), Random.Shared.NextInt64(1000000000L, 9999999999L).ToString(), (_, __) => Random.Shared.NextInt64(1000000000L, 9999999999L).ToString());
+        }
+
+        while (true)
+        {
+            
+        }
+    }
+
+    [Fact]
+    public void GetResult()
+    {
+        var input =
+            """
+            machine: 19303/19061
+            order: 3416759/3415192
+            order_block_plan: 2934281/1968850
+            order_block_plan_item: 0/235927707
+            order_block_plan_result: 1375479/277667
+            order_box_block: 23457666/23450841
+            order_data_block: 513012248/513012248
+            order_data_goods: 18655270/18655270
+            order_data_parts: 353139066/353139066
+            order_item: 955274320/955274320
+            order_module: 102907480/56935691
+            order_module_extra: 40044077/40044077
+            order_module_item: 49209022/49209022
+            order_package: 12012712/12012712
+            order_package_item: 0/80605124
+            order_process: 4045309/2682043
+            order_process_step: 8343418/5505158
+            order_process_step_item: 14856509/9787696
+            order_scrap_board: 136096/136090
+            process_group: 1577/1543
+            process_info: 9212/9008
+            process_item_exp: 30/30
+            process_schdule_capacity: 42442/42442
+            process_step_efficiency: 8/8
+            report_template: 7358/7338
+            simple_package: 142861/137730
+            simple_plan_order: 1167004/854699
+            simple_plan_order: 0/55677
+            sys_config: 2608/2608
+            work_calendar: 11/11
+            work_shift: 73/73
+            work_time: 77/77
+            order_process_step_item: 14856509/9790701
+            order_process_step: 8343418/5506925
+            order_module: 102907480/56935691
+            order_process: 4045309/2682043
+            report_template: 7358/7358
+            process_info: 9212/9212
+            process_group: 1577/1577
+            order_block_plan_result: 1375479/277667
+            order_box_block: 23457666/23457666
+            order_block_plan: 2934281/1968850
+            order: 3416759/3416759
+            machine: 19303/19303
+            order_scrap_board: 136096/136096
+            """;
+
+        var arr = input.Split('\n').Select(s => 
+        {
+            var x = s.Split(':');
+            var y = x[1].Split('/').Select(i => long.Parse(i)).ToArray();
+            return new {TABLE_NAME = x[0], INPUT = y[0], OUTPUT = y[1], FILTER = y[0] - y[1]};
+        }).OrderBy(s => s.TABLE_NAME);
+
+        _output.WriteLine(arr.ToMarkdownTable());
+    }
+}

MesETL.Test/XUnit/Configuration/XUnitConfiguration.cs

@@ -0,0 +1,16 @@
+using Microsoft.Extensions.Configuration;
+
+namespace TestProject1.XUnit.Configuration;
+
+public static class XUnitConfiguration
+{
+    public static IConfiguration Configuration { get; }
+    
+    static XUnitConfiguration()
+    {
+        Configuration = new ConfigurationBuilder()
+            .SetBasePath(AppContext.BaseDirectory)
+            .AddJsonFile("appsettings.json", false, true)
+            .Build();
+    }
+}

MesETL.Test/XUnit/Logging/XUnitLogger.cs

@@ -0,0 +1,39 @@
+using Microsoft.Extensions.Logging;
+using Xunit.Abstractions;
+
+namespace TestProject1.XUnit.Logging;
+
+/// <summary>
+/// 适用于Xunit的日志记录器，使用ITestOutputHelper输出
+/// </summary>
+public class XunitLogger : ILogger
+{
+    private readonly ITestOutputHelper _testOutputHelper;
+    private readonly string _categoryName;
+
+    public XunitLogger(ITestOutputHelper testOutputHelper, string categoryName)
+    {
+        _testOutputHelper = testOutputHelper;
+        _categoryName = categoryName;
+    }
+
+    public IDisposable? BeginScope<TState>(TState state) where TState : notnull
+        => NoopDisposable.Instance;
+
+    public bool IsEnabled(LogLevel logLevel)
+        => true;
+
+    public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception,
+        Func<TState, Exception?, string> formatter)
+    {
+        _testOutputHelper.WriteLine($"{_categoryName} [{eventId}] {formatter(state, exception)}");
+        if (exception != null)
+            _testOutputHelper.WriteLine(exception.ToString());
+    }
+
+    private class NoopDisposable : IDisposable
+    {
+        public static readonly NoopDisposable Instance = new();
+        public void Dispose() { }
+    }
+}

MesETL.Test/XUnit/Logging/XUnitLoggerExtensions.cs

@@ -0,0 +1,13 @@
+using Microsoft.Extensions.Logging;
+using Xunit.Abstractions;
+
+namespace TestProject1.XUnit.Logging;
+
+internal static class XUnitLoggerExtensions
+{
+    public static ILoggingBuilder AddXUnitLogger(this ILoggingBuilder builder, ITestOutputHelper output)
+    {
+        builder.AddProvider(new XunitLoggerProvider(output));
+        return builder;
+    }
+}

MesETL.Test/XUnit/Logging/XUnitLoggerProvider.cs

@@ -0,0 +1,20 @@
+using Microsoft.Extensions.Logging;
+using Xunit.Abstractions;
+
+namespace TestProject1.XUnit.Logging;
+
+public class XunitLoggerProvider : ILoggerProvider
+{
+    private readonly ITestOutputHelper _testOutputHelper;
+
+    public XunitLoggerProvider(ITestOutputHelper testOutputHelper)
+    {
+        _testOutputHelper = testOutputHelper;
+    }
+
+    public ILogger CreateLogger(string categoryName)
+        => new XunitLogger(_testOutputHelper, categoryName);
+
+    public void Dispose()
+    { }
+}

MesETL.Test/XUnit/TestBase.cs

@@ -0,0 +1,38 @@
+using System.Text.Json;
+using Microsoft.Extensions.Logging;
+using Serilog;
+using TestProject1.XUnit.Logging;
+using Xunit.Abstractions;
+
+namespace TestProject1.XUnit;
+
+public class TestBase
+{
+    private readonly LoggerFactory _loggerFactory;
+    protected readonly ITestOutputHelper Output;
+
+    private readonly JsonSerializerOptions _jsonSerializerOptions =
+        new(JsonSerializerDefaults.Web) { WriteIndented = true };
+
+    public TestBase(ITestOutputHelper output)
+    {
+        Output = output;
+        Console.SetOut(new XUnitConsoleWriter(output));
+        _loggerFactory = new LoggerFactory([new XunitLoggerProvider(Output)]);
+    }
+
+    protected void Write(object obj)
+    {
+        Output.WriteLine(obj.ToString());
+    }
+    
+    protected void WriteJson<T>(T obj)
+    {
+        Console.WriteLine(JsonSerializer.Serialize(obj, _jsonSerializerOptions));
+    }
+
+    protected ILogger<T> CreateXUnitLogger<T>()
+    {
+        return _loggerFactory.CreateLogger<T>();
+    }
+}

MesETL.Test/XUnit/XUnitConsoleWriter.cs

@@ -0,0 +1,18 @@
+using Xunit.Abstractions;
+
+namespace TestProject1.XUnit;
+
+public class XUnitConsoleWriter : StringWriter
+{
+    private ITestOutputHelper output;
+    
+    public XUnitConsoleWriter(ITestOutputHelper output)
+    {
+        this.output = output;
+    }
+
+    public override void WriteLine(string? m)
+    {
+        output.WriteLine(m);
+    }
+}

MesETL.sln

@@ -4,6 +4,14 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MesETL.App", "MesETL.App\Me
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MesETL.Test", "MesETL.Test\MesETL.Test.csproj", "{8679D5B6-5853-446E-9882-7B7A8E270500}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Mesdb.Cli", "Mesdb.Cli\Mesdb.Cli.csproj", "{68307B05-3D66-4322-A42F-C044C1E8BA3B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MesETL.Shared", "MesETL.Shared\MesETL.Shared.csproj", "{FE134001-0E22-458B-BEF2-29712A29087E}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MesETL.Clean", "MesETL.Clean\MesETL.Clean.csproj", "{E1B2BED0-EBA6-4A14-BAD5-8EC4E528D7E0}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Mesdb.DataGenerator", "Mesdb.DataGenerator\Mesdb.DataGenerator.csproj", "{2B7F3837-5ECD-4D24-B674-FDDA1C887A68}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -18,5 +26,21 @@ Global
 		{8679D5B6-5853-446E-9882-7B7A8E270500}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{8679D5B6-5853-446E-9882-7B7A8E270500}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{8679D5B6-5853-446E-9882-7B7A8E270500}.Release|Any CPU.Build.0 = Release|Any CPU
+		{68307B05-3D66-4322-A42F-C044C1E8BA3B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{68307B05-3D66-4322-A42F-C044C1E8BA3B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{68307B05-3D66-4322-A42F-C044C1E8BA3B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{68307B05-3D66-4322-A42F-C044C1E8BA3B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{FE134001-0E22-458B-BEF2-29712A29087E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{FE134001-0E22-458B-BEF2-29712A29087E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{FE134001-0E22-458B-BEF2-29712A29087E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{FE134001-0E22-458B-BEF2-29712A29087E}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E1B2BED0-EBA6-4A14-BAD5-8EC4E528D7E0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E1B2BED0-EBA6-4A14-BAD5-8EC4E528D7E0}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E1B2BED0-EBA6-4A14-BAD5-8EC4E528D7E0}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E1B2BED0-EBA6-4A14-BAD5-8EC4E528D7E0}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2B7F3837-5ECD-4D24-B674-FDDA1C887A68}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2B7F3837-5ECD-4D24-B674-FDDA1C887A68}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2B7F3837-5ECD-4D24-B674-FDDA1C887A68}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2B7F3837-5ECD-4D24-B674-FDDA1C887A68}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

Mesdb.Cli/BatchDbExtensions.cs

@@ -0,0 +1,52 @@
+using System.Collections.Concurrent;
+using System.Data;
+using MesETL.Shared.Helper;
+
+namespace Mesdb.Cli;
+
+public static class BatchDbExtensions
+{
+    public static async Task<IDictionary<string, IDictionary<string,long>>> CountDatabasesAsync(string connStr, IList<string> dbNames, CancellationToken cancellationToken = default)
+    {
+        var result = new ConcurrentDictionary<string, IDictionary<string,long>>();
+        var tables = await DatabaseHelper.QueryTableAsync(connStr,
+            $"""
+             SELECT TABLE_NAME FROM information_schema.`TABLES` WHERE TABLE_SCHEMA = '{dbNames[0]}';
+             """);
+
+        await Parallel.ForEachAsync(dbNames, async (dbName, ct) =>
+        {
+            await Parallel.ForEachAsync(tables.Tables[0].Rows.Cast<DataRow>(), async (row, ct) =>
+            {
+                var tableName = row[0].ToString()!;
+                var count = (long)(await DatabaseHelper.QueryScalarAsync(connStr,
+                    $"SELECT COUNT(1) FROM `{dbName}`.`{tableName}`;", ct))!;
+                result.AddOrUpdate(dbName, new ConcurrentDictionary<string, long>(), (db, dict) =>
+                {
+                    dict.AddOrUpdate(tableName, count, (table, num) => num + count);
+                    return dict;
+                });
+            });
+        });
+        return result;
+    }
+
+    public static async Task AnalyzeAllAsync(string connStr, IList<string> dbNames)
+    {
+        var tables = await DatabaseHelper.QueryTableAsync(connStr,
+            $"""
+             SELECT TABLE_NAME FROM information_schema.`TABLES` WHERE TABLE_SCHEMA = '{dbNames[0]}';
+             """);
+
+        await Parallel.ForEachAsync(dbNames, async (dbName, ct) =>
+        {
+            await Parallel.ForEachAsync(tables.Tables[0].Rows.Cast<DataRow>(), async (row, ct) =>
+            {
+                var tableName = row[0].ToString()!;
+                var result = (await DatabaseHelper.QueryTableAsync(connStr,
+                    $"ANALYZE TABLE `{dbName}`.`{tableName}`;", ct));
+                Console.WriteLine(string.Join('\t', result.Tables[0].Rows[0].ItemArray.Select(x => x.ToString())));
+            });
+        });
+    }
+}

Mesdb.Cli/Mesdb.Cli.csproj

@@ -0,0 +1,22 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <TargetFramework>net8.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <PackageReference Include="Cocona" Version="2.2.0" />
+      <PackageReference Include="Microsoft.Extensions.Hosting" Version="9.0.0" />
+      <PackageReference Include="Serilog" Version="4.1.0" />
+      <PackageReference Include="Serilog.Extensions.Hosting" Version="8.0.0" />
+      <PackageReference Include="Serilog.Sinks.Console" Version="6.0.0" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\MesETL.Shared\MesETL.Shared.csproj" />
+    </ItemGroup>
+
+</Project>

Mesdb.Cli/Program.cs

@@ -0,0 +1,52 @@
+using Cocona;
+using MesETL.Shared.Helper;
+using Mesdb.Cli;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Hosting;
+
+var host = Host.CreateApplicationBuilder(args);
+host.Configuration.AddCommandLine(args, new Dictionary<string, string>
+{
+    { "-s", "ConnectionString" },
+    { "--ConnectionString", "ConnectionString" },
+    { "-B", "Databases" },
+    { "--Databases", "Databases" },
+    { "-a", "All" },
+    { "-c", "Command"},
+    { "--Command", "Command" },
+    { "--Sql", "Command" }
+}); 
+host.Build();
+var connStr = host.Configuration.GetValue<string>("ConnectionString") ?? throw new ApplicationException("没有配置数据库连接字符串");
+var databases = host.Configuration.GetValue<string>("Databases")?.Split(',').ToList() ?? throw new ApplicationException("没有配置数据库");
+var all = host.Configuration.GetValue<bool>("All");
+
+if (args.Length > 1 && args[0] == "count")
+{
+    var result  = await BatchDbExtensions.CountDatabasesAsync(connStr, databases);
+    if (all)
+    {
+        foreach (var (k, v) in result)
+        {
+            Console.WriteLine(k + ":");
+            Console.WriteLine(v.Select(pair => new { TABLE_NAME = pair.Key, COUNT = pair.Value }).ToMarkdownTable());
+        }
+    }
+    else
+    {
+        var allCount = result.Aggregate(new Dictionary<string, long>(), (dict, pair) =>
+        {
+            foreach (var (k, v) in pair.Value)
+            {
+                dict.AddOrUpdate(k, v, (key, num) => num + v);
+            }
+            return dict;
+        });
+        Console.WriteLine(allCount.Select(pair => new { TABLE_NAME = pair.Key, COUNT = pair.Value }).ToMarkdownTable());
+    }
+}
+
+if (args.Length > 1 && args[0] == "analyze")
+{
+    await BatchDbExtensions.AnalyzeAllAsync(connStr, databases);
+}

Mesdb.Cli/Schema/DB.cs

@@ -0,0 +1,44 @@
+using MesETL.Shared.Helper;
+using MySqlConnector;
+using Serilog;
+
+namespace Mesdb.Cli.Schema;
+
+public class DB
+{
+    public required string ConnectionString { get; init; }
+    public required IReadOnlyList<Database> Databases { get; init; }
+
+    public static DB Create(string connStr, IEnumerable<string> dbNames)
+    {
+        var databases = new List<Database>();
+        foreach (var dbName in dbNames)
+        {
+            var dbConnStr = new MySqlConnectionStringBuilder(connStr)
+            {
+                Database = dbName
+            }.ConnectionString;
+
+            try
+            {
+                _ = DatabaseHelper.NonQueryAsync(dbConnStr, "SHOW DATABASES;").Result;
+                databases.Add(new Database(dbName, dbConnStr));
+            }
+            catch (Exception e)
+            {
+                Log.Logger.Fatal(e, "无法连接到数据库: {DbName} ", dbName);
+                throw;
+            }
+        }
+
+        return new DB
+        {
+            ConnectionString = connStr,
+            Databases = databases
+        };
+    }
+
+    private DB()
+    {
+    }
+}

Mesdb.Cli/Schema/Database.cs

@@ -0,0 +1,50 @@
+using System.Data;
+using MesETL.Shared.Helper;
+using MySqlConnector;
+
+namespace Mesdb.Cli.Schema;
+
+public class Database
+{
+    public static async Task<Table[]> FetchTableAsync(string dbName, string connStr)
+    {
+        var tables = await DatabaseHelper.QueryTableAsync(connStr,
+            $"""
+             SELECT TABLE_NAME FROM information_schema.`TABLES` WHERE TABLE_SCHEMA = '{dbName}';
+             """);
+        return tables.Tables[0].Rows.Cast<DataRow>().Select(row => new Table{Name = row[0].ToString()!}).ToArray();
+    }
+    
+    
+    public string Name { get; }
+    public string ConnectionString { get; }
+    public IReadOnlyList<Table> Tables { get; }
+
+    public Database(string name, string connStr)
+    {
+        var trueConnStr = new MySqlConnectionStringBuilder(connStr)
+        {
+            Database = name
+        }.ConnectionString;
+
+        var tables = FetchTableAsync(name, trueConnStr).Result;
+        Name = name;
+        ConnectionString = trueConnStr;
+        Tables = tables;
+    }
+    
+    public Task ExecuteNonQueryAsync(string sql, CancellationToken cancellationToken = default)
+    {
+        return DatabaseHelper.NonQueryAsync(ConnectionString, sql, cancellationToken);
+    }
+
+    public Task<DataSet> ExecuteQueryAsync(string sql, CancellationToken cancellationToken = default)
+    {
+        return DatabaseHelper.QueryTableAsync(ConnectionString, sql, cancellationToken);
+    }
+    
+    public Task<object?> ExecuteScalarAsync(string sql, CancellationToken cancellationToken = default)
+    {
+        return DatabaseHelper.QueryScalarAsync(ConnectionString, sql, cancellationToken);
+    }
+}

Mesdb.Cli/Schema/Table.cs

@@ -0,0 +1,8 @@
+namespace Mesdb.Cli.Schema;
+
+public class Table
+{
+    
+    public required string Name { get; init; }
+    
+}

Mesdb.DataGenerator/Mesdb.DataGenerator.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <TargetFramework>net9.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\MesETL.App\MesETL.App.csproj" />
+    </ItemGroup>
+
+</Project>

Mesdb.DataGenerator/MockHelper.cs

@@ -0,0 +1,11 @@
+namespace Mesdb.DataGenerator;
+
+public static class MockHelper
+{
+    public static string RandomString(int length)
+    {
+        const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
+        return new string(Enumerable.Repeat(chars, length)
+            .Select(s => s[Random.Shared.Next(s.Length)]).ToArray());
+    }
+}

Mesdb.DataGenerator/MockInputOptions.cs

@@ -0,0 +1,26 @@
+using MesETL.App;
+
+namespace Mesdb.DataGenerator;
+
+public class MockInputOptions
+{
+    public IReadOnlyDictionary<string, TableMockOptions> Rules { get; set; } =
+        new Dictionary<string, TableMockOptions>();
+}
+
+public class TableMockOptions
+{
+    public long Amount { get; set; }
+    public Func<TableMockContext, DataRecord> Generate  { get; set; }
+
+    public TableMockOptions(long amount, Func<TableMockContext, DataRecord> generate)
+    {
+        Amount = amount;
+        Generate = generate;
+    }
+}
+
+public struct TableMockContext
+{
+    public long Index { get; set; }
+}

Mesdb.DataGenerator/MockInputService.cs

@@ -0,0 +1,60 @@
+using System.Runtime;
+using MesETL.App.Const;
+using MesETL.App.HostedServices.Abstractions;
+using MesETL.App.Services;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace Mesdb.DataGenerator;
+
+public class MockInputService : IInputService
+{
+    private readonly DataRecordQueue _producerQueue;
+    private readonly ProcessContext _context;
+    private readonly IOptions<MockInputOptions> _options;
+    private readonly ILogger _logger;
+    private readonly long _memoryThreshold;
+
+    public MockInputService([FromKeyedServices(ConstVar.Producer)]DataRecordQueue producerQueue, ProcessContext context, IOptions<MockInputOptions> options,
+        ILogger<MockInputService> logger, IConfiguration configuration)
+    {
+        _producerQueue = producerQueue;
+        _context = context;
+        _options = options;
+        _logger = logger;
+        _memoryThreshold = (long)(configuration.GetValue<double>("MemoryThreshold", 8) * 1024 * 1024 * 1024);
+    }
+
+    public async Task ExecuteAsync(CancellationToken cancellationToken)
+    {
+        _logger.LogInformation("***** 开始模拟输入数据 *****");
+        foreach (var (table, options) in _options.Value.Rules)
+        {
+            _logger.LogInformation("模拟表 '{TableName}' 输入，数量: {Amount}", table, options.Amount);
+
+            for (int i = 0; i < options.Amount; i++)
+            {
+                if (GC.GetTotalMemory(false) > _memoryThreshold)
+                {
+                    _logger.LogWarning("内存使用率过高，暂缓输入");
+                    GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
+                    GC.Collect();
+                    await Task.Delay(3000, cancellationToken);
+                }
+                var ctx = new TableMockContext()
+                {
+                    Index = i,
+                };
+                var record = options.Generate(ctx);
+                await _producerQueue.EnqueueAsync(record);
+                _context.AddInput(1);
+                _context.AddTableInput(table, 1);
+            }
+            _logger.LogInformation("表 '{TableName}' 输入完成", table);
+        }
+        _context.CompleteInput();
+        _logger.LogInformation("***** 模拟数据输入完成 *****");
+    }
+}

Mesdb.DataGenerator/Program.cs

@@ -0,0 +1,304 @@
+// See https://aka.ms/new-console-template for more information
+
+using System.Text;
+using Mesdb.DataGenerator;
+using MesETL.App;
+using MesETL.App.Cache;
+using MesETL.App.Const;
+using MesETL.App.HostedServices;
+using MesETL.App.HostedServices.Abstractions;
+using MesETL.App.Options;
+using MesETL.App.Services;
+using MesETL.App.Services.ErrorRecorder;
+using MesETL.App.Services.ETL;
+using MesETL.App.Services.Loggers;
+using MesETL.App.Services.Seq;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Serilog;
+using Serilog.Events;
+
+await RunProgram();
+return;
+
+async Task RunProgram()
+{
+    ThreadPool.SetMaxThreads(200, 200);
+    var host = Host.CreateApplicationBuilder(args);
+
+    var inputOptions = host.Configuration.GetRequiredSection("Input").Get<DataInputOptions>()
+                       ?? throw new ApplicationException("缺少Input配置");
+
+    var transformOptions = host.Configuration.GetRequiredSection("Transform").Get<DataTransformOptions>()
+                           ?? throw new ApplicationException("缺少Transform配置");
+
+    var outputOptions = host.Configuration.GetRequiredSection("Output").Get<DatabaseOutputOptions>()
+                        ?? throw new ApplicationException("缺少Output配置");
+
+    var tenantDbSection = host.Configuration.GetRequiredSection("TenantDb");
+    var tenantDbOptions = new TenantDbOptions()
+    {
+        TenantKey = tenantDbSection.GetValue<string>(nameof(TenantDbOptions.TenantKey)) ??
+                    throw new ApplicationException("分库配置缺少分库键TenantKey"),
+        UseDbGroup = tenantDbSection.GetValue<string>(nameof(TenantDbOptions.UseDbGroup)) ??
+                     throw new ApplicationException("分库配置缺少使用分库组UseDbGroup")
+    };
+    host.Services.Configure<TenantDbOptions>(options =>
+    {
+        options.TenantKey = tenantDbOptions.TenantKey;
+        options.DbGroup = tenantDbOptions.DbGroup;
+        options.UseDbGroup = tenantDbOptions.UseDbGroup;
+    });
+    tenantDbOptions.DbGroup = tenantDbSection.GetRequiredSection($"DbGroups:{tenantDbOptions.UseDbGroup}")
+                                  .Get<Dictionary<string, int>>()
+                              ?? throw new ApplicationException($"分库配置无法解析分库组{tenantDbOptions.UseDbGroup}，请检查配置");
+
+    host.Services.Configure<MockInputOptions>(options =>
+    {
+        const float Multiplexer = 1F;
+        var SampleSharedKeys = Enumerable.Range(0, 11).Select(i => (23010 + i * 10).ToString()).Concat(
+            Enumerable.Range(0, 11).Select(i => (24010 + i * 10).ToString())).ToArray();
+        options.Rules = new Dictionary<string, TableMockOptions>()
+        {
+            {
+                TableNames.Order, new TableMockOptions((long)(2912406 * Multiplexer), context =>
+                {
+                    string[] headers =
+                    [
+                        "OrderNo", "ShardKey", "CreateTime", "CompanyID", "SchduleDeliveryDate", "OrderType",
+                        "OrderSort",
+                        "CadDataType", "Deleted", "ProcessState"
+                    ];
+                    string[] fields =
+                    [
+                        (20241210000000 + context.Index).ToString(),
+                        SampleSharedKeys[Random.Shared.Next(SampleSharedKeys.Length)],
+                        $"\"{DateTime.Now:yyyy-MM-dd HH:mm:ss}\"",
+                        Random.Shared.Next(1, 28888).ToString(),
+                        $"\"{DateTime.Now.AddDays(Random.Shared.Next(1, 30)):yyyy-MM-dd HH:mm:ss}\"",
+                        Random.Shared.Next(0, 3).ToString(),
+                        Random.Shared.Next(0, 3).ToString(),
+                        Random.Shared.Next(0, 3).ToString(),
+                        Random.Shared.Next(0, 2).ToString(),
+                        Random.Shared.Next(0, 2).ToString()
+                    ];
+                    return new DataRecord(fields, TableNames.Order, headers);
+                })
+            },
+            {
+                TableNames.OrderItem, new TableMockOptions((long)(820241144 * Multiplexer), context =>
+                {
+                    string[] headers =
+                    [
+                        "ID", "ShardKey", "OrderNo", "ItemNo", "ItemType", "RoomID", "BoxID", "DataID", "PlanID",
+                        "PackageID", "Num", "CompanyID"
+                    ];
+                    string[] fields =
+                    [
+                        context.Index.ToString(),
+                        SampleSharedKeys[Random.Shared.Next(SampleSharedKeys.Length)],
+                        (20241210000000 + Random.Shared.Next(0, 2912406)).ToString(),
+                        (2412000000000 + context.Index).ToString(),
+                        Random.Shared.Next(0, 3).ToString(),
+                        Random.Shared.Next(1, 1000000).ToString(),
+                        Random.Shared.Next(1, 1000000).ToString(),
+                        Random.Shared.Next(1, 1000000).ToString(),
+                        Random.Shared.Next(1, 1306670366).ToString(),
+                        Random.Shared.Next(1, 10000000).ToString(),
+                        (Random.Shared.Next(1, 2000000) / 1000).ToString("F3"),
+                        Random.Shared.Next(1, 28888).ToString(),
+                    ];
+                    return new DataRecord(fields, TableNames.OrderItem, headers);
+                })
+            },
+            {
+                TableNames.OrderDataBlock, new TableMockOptions((long)(428568719 * Multiplexer), context =>
+                {
+                    string[] headers =
+                    [
+                        "ID", "ShardKey", "OrderNo", "BoardName", "BoardType", "GoodsID", "Width", "Height",
+                        "Thickness",
+                        "SpliteWidth", "SpliteHeight", "SpliteThickness", "SealedLeft", "SealedRight", "SealedUp",
+                        "SealedDown", "Area", "Wave", "HoleFace", "PaiKong", "RemarkJson", "UnRegularPointCount",
+                        "FrontHoleCount", "BackHoleCount", "SideHoleCount", "FrontModelCount", "BackModelCount",
+                        "IsDoor",
+                        "OpenDoorType", "CompanyID", "BorderLengthHeavy", "BorderLengthLight", "IsHXDJX",
+                        "ModuleTypeID",
+                        "PlanFilterType"
+                    ];
+                    string[] BoardNames = ["左侧板", "右侧板", "左开门板", "右开门板", "薄背板", "顶板", "底板", "地脚线", "后地脚", "层板", "立板"];
+                    string[] fields =
+                    [
+                        context.Index.ToString(),
+                        SampleSharedKeys[Random.Shared.Next(SampleSharedKeys.Length)],
+                        (20241210000000 + Random.Shared.Next(0, 2912406)).ToString(),
+                        $"\"{BoardNames[Random.Shared.Next(0, BoardNames.Length)]}\"",
+                        Random.Shared.Next(0, 3).ToString(),
+                        Random.Shared.Next(1, 1000000).ToString(),
+                        (Random.Shared.Next(1, 2000000) / 1000).ToString("F3"),
+                        (Random.Shared.Next(1, 1200000) / 1000).ToString("F3"),
+                        (Random.Shared.Next(1, 18000) / 1000).ToString("F3"),
+                        (Random.Shared.Next(1, 2000000) / 1000).ToString("F3"),
+                        (Random.Shared.Next(1, 1200000) / 1000).ToString("F3"),
+                        (Random.Shared.Next(1, 18000) / 1000).ToString("F3"),
+                        (Random.Shared.Next(1, 2000) / 1000).ToString("F2"),
+                        (Random.Shared.Next(1, 2000) / 1000).ToString("F2"),
+                        (Random.Shared.Next(1, 2000) / 1000).ToString("F2"),
+                        (Random.Shared.Next(1, 2000) / 1000).ToString("F2"),
+                        (Random.Shared.Next(1, 3000) / 1000).ToString("F3"),
+                        Random.Shared.Next(0, 3).ToString(),
+                        Random.Shared.Next(0, 3).ToString(),
+                        Random.Shared.Next(0, 3).ToString(),
+                        Convert.ToHexString(Encoding.UTF8.GetBytes(MockHelper.RandomString(100))),
+                        Random.Shared.Next(0, 4).ToString(),
+                        Random.Shared.Next(0, 4).ToString(),
+                        Random.Shared.Next(0, 4).ToString(),
+                        Random.Shared.Next(0, 4).ToString(),
+                        Random.Shared.Next(0, 4).ToString(),
+                        Random.Shared.Next(0, 4).ToString(),
+                        Random.Shared.Next(0, 2).ToString(),
+                        Random.Shared.Next(0, 6).ToString(),
+                        Random.Shared.Next(1, 28888).ToString(),
+                        (Random.Shared.Next(1, 2000000) / 1000).ToString("F3"),
+                        (Random.Shared.Next(1, 2000000) / 1000).ToString("F3"),
+                        Random.Shared.Next(0, 2).ToString(),
+                        Random.Shared.Next(0, 100000).ToString(),
+                        Random.Shared.Next(0, 5).ToString(),
+                    ];
+                    return new DataRecord(fields, TableNames.OrderDataBlock, headers);
+                })
+            },
+            {
+                TableNames.OrderBoxBlock, new TableMockOptions((long)(20163038 * Multiplexer), context =>
+                {
+                    string[] headers = ["BoxID", "ShardKey", "OrderNo", "Data", "CompanyID"];
+                    string[] fields =
+                    [
+                        context.Index.ToString(),
+                        SampleSharedKeys[Random.Shared.Next(SampleSharedKeys.Length)],
+                        (20241210000000 + Random.Shared.Next(0, 2912406)).ToString(),
+                        OrderBoxBlockHelper.RandomPick(),
+                        Random.Shared.Next(1, 28888).ToString(),
+                    ];
+                    return new DataRecord(fields, TableNames.OrderBoxBlock, headers);
+                })
+            },
+            {
+                TableNames.OrderModuleExtra, new TableMockOptions((long)(33853580 * Multiplexer), context =>
+                {
+                    string[] headers =
+                    [
+                        "ID", "ShardKey", "OrderNo", "RoomID", "BoxID", "PropType", "JsonStr", "CompanyID"
+                    ];
+                    string[] fields =
+                    [
+                        context.Index.ToString(),
+                        SampleSharedKeys[Random.Shared.Next(SampleSharedKeys.Length)],
+                        (20241210000000 + Random.Shared.Next(0, 2912406)).ToString(),
+                        Random.Shared.Next(1, 1000000).ToString(),
+                        Random.Shared.Next(1, 1000000).ToString(),
+                        Random.Shared.Next(0, 14).ToString(),
+                        OrderModuleExtraHelper.PickJson(),
+                        Random.Shared.Next(1, 28888).ToString(),
+                    ];
+                    return new DataRecord(fields, TableNames.OrderModuleExtra, headers);
+                })
+            }
+        };
+    });
+
+    host.Services.Configure<DataTransformOptions>(options =>
+    {
+        options.StrictMode = transformOptions.StrictMode;
+        options.EnableFilter = transformOptions.EnableFilter;
+        options.EnableReplacer = transformOptions.EnableReplacer;
+        options.EnableReBuilder = transformOptions.EnableReBuilder;
+        
+        options.DatabaseFilter = record =>
+        {
+            var companyId = int.Parse(record[tenantDbOptions.TenantKey]); // 每个实体都应存在CompanyID，否则异常
+            return tenantDbOptions.GetDbNameByTenantKeyValue(companyId);
+        };
+    });
+
+    host.Services.Configure<DatabaseOutputOptions>(options =>
+    {
+        options.ConnectionString = outputOptions.ConnectionString;
+        options.FlushCount = outputOptions.FlushCount;
+        options.MaxAllowedPacket = outputOptions.MaxAllowedPacket / 2;
+        options.MaxDatabaseOutputTask = outputOptions.MaxDatabaseOutputTask;
+        options.TreatJsonAsHex = outputOptions.TreatJsonAsHex;
+        options.NoOutput = outputOptions.NoOutput;
+        options.ForUpdate = outputOptions.ForUpdate;
+
+        // 配置列的类型以便于在输出时区分二进制内容
+        // Prod server
+        options.ColumnTypeConfig = new Dictionary<string, ColumnType>
+        {
+            { "machine.Settings", ColumnType.Text },
+            { "order_block_plan.BlockInfo", ColumnType.Text },
+            { "order_block_plan.OrderNos", ColumnType.Json },
+            { "order_block_plan_result.PlaceData", ColumnType.Blob },
+            { "order_box_block.Data", ColumnType.Blob },
+            { "order_data_block.RemarkJson", ColumnType.Text },
+            { "order_data_goods.ExtraProp", ColumnType.Json },
+            { "order_extra.ConfigJson", ColumnType.Json },
+            { "order_module_extra.Data", ColumnType.Blob },
+            { "order_module_extra.JsonStr", ColumnType.Text },
+            { "order_patch_detail.BlockDetail", ColumnType.Json },
+            { "order_process_schdule.AreaName", ColumnType.Text },
+            { "order_process_schdule.ConsigneeAddress", ColumnType.Text },
+            { "order_process_schdule.ConsigneePhone", ColumnType.Text },
+            { "order_process_schdule.CustomOrderNo", ColumnType.Text },
+            { "order_process_schdule.OrderProcessStepName", ColumnType.Text },
+            { "order_scrap_board.OutLineJson", ColumnType.Text },
+            { "order_wave_group.ConfigJson", ColumnType.Json },
+            { "process_info.Users", ColumnType.Text },
+            { "process_item_exp.ItemJson", ColumnType.Text },
+            { "report_template.SourceConfig", ColumnType.Text },
+            { "report_template.Template", ColumnType.Text },
+            { "simple_package.Items", ColumnType.Json },
+            { "sys_config.JsonStr", ColumnType.Text },
+            { "sys_config.Value", ColumnType.Text }
+        };
+    });
+
+    host.Services.AddLogging(builder =>
+    {
+        builder.ClearProviders();
+        var logger = new LoggerConfiguration()
+            .MinimumLevel.Verbose()
+            .WriteTo.Console()
+            .WriteTo.File(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"./Log/Error/{ErrorRecorder.UID}.log"),
+                restrictedToMinimumLevel: LogEventLevel.Error)
+            // .WriteTo.File("./Log/Info/{ErrorRecorder.UID}.log", restrictedToMinimumLevel:LogEventLevel.Information) //性能考虑暂不使用
+            .CreateLogger();
+        builder.AddSerilog(logger);
+        Log.Logger = logger;
+    });
+
+    host.Services.AddDataSourceFactory();
+    host.Services.AddErrorRecorderFactory();
+    host.Services.AddSingleton<ProcessContext>();
+    host.Services.AddSingleton<SeqService>();
+    var prodLen = host.Configuration.GetRequiredSection("RecordQueue").GetValue<int>("ProducerQueueLength");
+    var consLen = host.Configuration.GetRequiredSection("RecordQueue").GetValue<int>("ConsumerQueueLength");
+    var maxCharCount = host.Configuration.GetRequiredSection("RecordQueue").GetValue<long>("MaxByteCount") / 2;
+    host.Services.AddKeyedSingleton<DataRecordQueue>(ConstVar.Producer, new DataRecordQueue(prodLen, maxCharCount));
+    host.Services.AddRecordQueuePool(tenantDbOptions.DbGroup.Keys
+        .Select(key => (key: key, queue: new DataRecordQueue(consLen, maxCharCount))).ToArray());
+    // host.Services.AddSingleton<ITaskMonitorLogger, CacheTaskMonitorLogger>();
+    host.Services.AddSingleton<ITaskMonitorLogger, LoggerTaskMonitorLogger>();
+
+    host.Services.AddHostedService<MainHostedService>();
+    host.Services.AddSingleton<IInputService, MockInputService>();
+    host.Services.AddSingleton<ITransformService, TransformService>();
+    host.Services.AddSingleton<IOutputService, OutputService>();
+    host.Services.AddSingleton<TaskMonitorService>();
+    // host.Services.AddRedisCache(redisOptions);
+    host.Services.AddSingleton<ICacher, MemoryCache>();
+    var app = host.Build();
+    await app.RunAsync();
+}