Update

ConsoleApp2.sln

@@ -2,6 +2,8 @@
 Microsoft Visual Studio Solution File, Format Version 12.00
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ConsoleApp2", "ConsoleApp2\ConsoleApp2.csproj", "{155E4B04-E88C-4BA4-AED2-B13E0A0432B5}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestProject1", "TestProject1\TestProject1.csproj", "{8679D5B6-5853-446E-9882-7B7A8E270500}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -12,5 +14,9 @@ Global
 		{155E4B04-E88C-4BA4-AED2-B13E0A0432B5}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{155E4B04-E88C-4BA4-AED2-B13E0A0432B5}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{155E4B04-E88C-4BA4-AED2-B13E0A0432B5}.Release|Any CPU.Build.0 = Release|Any CPU
+		{8679D5B6-5853-446E-9882-7B7A8E270500}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{8679D5B6-5853-446E-9882-7B7A8E270500}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{8679D5B6-5853-446E-9882-7B7A8E270500}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{8679D5B6-5853-446E-9882-7B7A8E270500}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

ConsoleApp2/Cache/CacheKeys.cs

@@ -0,0 +1,9 @@
+namespace ConsoleApp2.Cache;
+
+#nullable disable
+public static class CacheKeys
+{
+    public static Func<string, string> Order_OrderNo_CompanyID { get; set; }
+    public static Func<string, string> OrderBlockPlan_ID_CompanyID { get; set; }
+    public static Func<string, string> OrderProcess_ID_ShardKey { get; set; }
+}

ConsoleApp2/Cache/ICacher.cs

@@ -0,0 +1,10 @@
+namespace ConsoleApp2.Cache;
+
+public interface ICacher
+{
+    Task<string?> GetStringAsync(string key);
+    Task SetStringAsync(string key, string value);
+    Task<bool> ExistsAsync(string key);
+    Task SetHashAsync(string key, IReadOnlyDictionary<string, string> hash);
+    Task<Dictionary<string, string>> GetHashAsync(string key);
+}

ConsoleApp2/Cache/RedisCache.cs

@@ -0,0 +1,63 @@
+using ConsoleApp2.Options;
+using Microsoft.Extensions.DependencyInjection;
+using StackExchange.Redis;
+
+namespace ConsoleApp2.Cache;
+
+public class RedisCache : ICacher
+{
+    private readonly IDatabase _db;
+
+    public string KeyPrefix { get; set; }
+
+    public RedisCache(IConnectionMultiplexer conn, int dataBase, string keyPrefix = "")
+    {
+        _db = conn.GetDatabase(dataBase);
+        KeyPrefix = keyPrefix;
+    }
+
+    public async Task<string?> GetStringAsync(string key)
+    {
+        var value = await _db.StringGetAsync($"{KeyPrefix}{key}");
+        return !value.HasValue ? null : value.ToString();
+    }
+
+    public async Task SetStringAsync(string key, string value)
+    {
+        if (!await _db.StringSetAsync($"{KeyPrefix}{key}", value))
+            throw new RedisCommandException("设置Redis缓存失败");
+    }
+
+    public Task<bool> ExistsAsync(string key)
+    {
+        return _db.KeyExistsAsync($"{KeyPrefix}{key}");
+    }
+
+    public Task SetHashAsync(string key, IReadOnlyDictionary<string, string> hash)
+    {
+        return _db.HashSetAsync($"{KeyPrefix}{key}", hash.Select(pair => new HashEntry(pair.Key, pair.Value)).ToArray());
+    }
+
+    public async Task<Dictionary<string, string>> GetHashAsync(string key)
+    {
+        var entries = await _db.HashGetAllAsync($"{KeyPrefix}{key}");
+        var result = new Dictionary<string, string>();
+        foreach (var entry in entries)
+        {
+            result.Add(entry.Name.ToString(), entry.Value.ToString());
+        }
+
+        return result;
+    }
+}
+
+public static class RedisCacheExtensions
+{
+    public static IServiceCollection AddRedisCache(this IServiceCollection services, RedisCacheOptions options)
+    {
+        var conn = ConnectionMultiplexer.Connect(options.Configuration 
+                                                 ?? throw new ApplicationException("未配置Redis连接字符串"));
+        services.AddSingleton<ICacher>(new RedisCache(conn, options.Database, options.InstanceName));
+        return services;
+    }
+}

ConsoleApp2/ConsoleApp2.csproj

@@ -19,7 +19,6 @@
     </ItemGroup>
 
     <ItemGroup>
-      <PackageReference Include="Microsoft.Extensions.Caching.StackExchangeRedis" Version="8.0.0" />
       <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
       <PackageReference Include="MySqlConnector" Version="2.3.3" />
       <PackageReference Include="Serilog" Version="3.1.2-dev-02097" />
@@ -27,6 +26,7 @@
       <PackageReference Include="Serilog.Sinks.Console" Version="5.0.1" />
       <PackageReference Include="Serilog.Sinks.File" Version="5.0.1-dev-00972" />
       <PackageReference Include="ServiceStack.Text" Version="8.0.0" />
+      <PackageReference Include="StackExchange.Redis" Version="2.7.17" />
       <PackageReference Include="ZstdSharp.Port" Version="0.7.4" />
     </ItemGroup>
 

ConsoleApp2/Const/ProcessStep.cs

@@ -2,6 +2,5 @@
 
 public static class ProcessStep
 {
-    public const string Producer = "Producer";
-    public const string Consumer = "Consumer";
+    public const string Produce = "Producer";
 }

ConsoleApp2/Const/TableNames.cs

@@ -0,0 +1,38 @@
+namespace ConsoleApp2.Const;
+
+public static class TableNames
+{
+    public const string Machine = "machine";
+    public const string Order = "order";
+    public const string OrderBlockPlan = "order_block_plan";
+    public const string OrderBlockPlanItem = "order_block_plan_item";
+    public const string OrderBlockPlanResult = "order_block_plan_result";
+    public const string OrderBoxBlock = "order_box_block";
+    public const string OrderDataBlock = "order_data_block";
+    public const string OrderDataGoods = "order_data_goods";
+    public const string OrderDataParts = "order_data_parts";
+    public const string OrderItem = "order_item";
+    public const string OrderModule = "order_module";
+    public const string OrderModuleExtra = "order_module_extra";
+    public const string OrderModuleItem = "order_module_item";
+    public const string OrderPackage = "order_package";
+    public const string OrderPackageItem = "order_package_item";
+    public const string OrderPatchDetail = "order_patch_detail";
+    public const string OrderProcess = "order_process";
+    public const string OrderProcessSchedule = "order_process_schdule";
+    public const string OrderProcessStep = "order_process_step";
+    public const string OrderProcessStepItem = "order_process_step_item";
+    public const string OrderScrapBoard = "order_scrap_board";
+    public const string ProcessGroup = "process_group";
+    public const string ProcessInfo = "process_info";
+    public const string ProcessItemExp = "process_item_exp";
+    public const string ProcessScheduleCapacity = "process_schdule_capacity";
+    public const string ProcessStepEfficiency = "process_step_efficiency";
+    public const string ReportTemplate = "report_template";
+    public const string SimplePackage = "simple_package";
+    public const string SimplePlanOrder = "simple_plan_order";
+    public const string SysConfig = "sys_config";
+    public const string WorkCalendar = "work_calendar";
+    public const string WorkShift = "work_shift";
+    public const string WorkTime = "work_time";
+}

ConsoleApp2/DataRecord.cs

@@ -1,16 +1,13 @@
-using System.ComponentModel.Design;
+namespace ConsoleApp2;
 
-namespace ConsoleApp2;
-
-
-public class DataRecord
+public class DataRecord : ICloneable
 {
     public static bool TryGetField(DataRecord record, string columnName, out string value)
     {
         value = string.Empty;
         if (record.Headers is null)
             throw new InvalidOperationException("Cannot get field when headers of a record have not been set.");
-        var idx = Array.IndexOf(record.Headers, columnName);
+        var idx = IndexOfIgnoreCase(record.Headers, columnName);
         if (idx == -1)
             return false;
         value = record.Fields[idx];
@@ -21,34 +18,46 @@ public class DataRecord
     {
         if (record.Headers is null)
             throw new InvalidOperationException("Headers have not been set.");
-        var idx = Array.IndexOf(record.Headers, columnName);
+        var idx = IndexOfIgnoreCase(record.Headers, columnName);
         if (idx is -1)
-            throw new IndexOutOfRangeException($"Column name {columnName} not found in this record, table name {record.TableName}.");
+            throw new IndexOutOfRangeException(
+                $"Column name '{columnName}' not found in this record, table name '{record.TableName}'.");
         return record.Fields[idx];
     }
 
-
-    public string[] Fields { get; }
-
-    public string[] Headers { get; }
-
-    public string TableName { get; }
-
-    public string Database { get; set; }
-    public int CompanyID { get; set; }
-
-
-    public DataRecord(string[] fields, string tableName, string[] headers, int companyID=0)
+    private static int IndexOfIgnoreCase(IList<string> list, string value)
     {
-        if (fields.Length != headers.Length)
-            throw new ArgumentException(
-                $"The number of fields does not match the number of headers. Expected: {headers.Length} Got: {fields.Length} Fields: {string.Join(',', fields)}",
-                nameof(fields));
+        var idx = -1;
+        for (var i = 0; i < list.Count; i++)
+        {
+            if (list[i].Equals(value, StringComparison.OrdinalIgnoreCase)) 
+            {
+                idx = i;
+                break;
+            }
+        }
 
-        Fields = fields;
+        return idx;
+    }
+
+    public string? RawField { get; set; }
+    public IList<string> Fields { get; }
+    public IList<string> Headers { get; }
+    public string TableName { get; }
+    public string? Database { get; set; }
+
+
+    public DataRecord(IEnumerable<string> fields, string tableName, IEnumerable<string> headers, string? database = null)
+    {
+        Fields = fields.ToList();
         TableName = tableName;
-        Headers = headers;
-        CompanyID = companyID;
+        Headers = headers.ToList();
+        Database = database;
+        
+        if (Fields.Count != Headers.Count)
+            throw new ArgumentException(
+                $"The number of fields does not match the number of headers. Expected: {Headers.Count} Got: {Fields.Count} Fields: {string.Join(',', Fields)}",
+                nameof(fields));
     }
 
     public string this[int index]
@@ -63,32 +72,46 @@ public class DataRecord
         set => SetField(columnName, value);
     }
 
-    public int Count => Fields.Length;
+    public int FieldCount => Fields.Count;
 
     public bool TryGetField(string columnName, out string value) => TryGetField(this, columnName, out value);
 
-    public bool SetField(string columnName,  string value) => SetField(this, columnName,value);
+    public bool SetField(string columnName, string value) => SetField(this, columnName, value);
 
-    public string GetCacheKey(string columnName) => GetCacheKey(this, columnName);
-
-    public  bool SetField( DataRecord record,string columnName,string value)
+    public bool SetField(DataRecord record, string columnName, string value)
     {
         if (record.Headers is null)
             throw new InvalidOperationException("Headers have not been set.");
-        var idx = Array.IndexOf(record.Headers, columnName);
+        var idx = IndexOfIgnoreCase(record.Headers, columnName);
         if (idx is -1)
-            throw new IndexOutOfRangeException("Column name not found in this record.");
+            throw new IndexOutOfRangeException(
+                $"Column name '{columnName}' not found in this record, table name '{record.TableName}");
         record.Fields[idx] = value;
         return true;
     }
-    public string GetCacheKey(DataRecord record, string columnName)
+    
+    public void AddField(string columnName, string value)
     {
-        if (TryGetField(record, columnName, out var  value))
-        {
-            return $"{TableName}_{value}";
-        }else
-            throw new IndexOutOfRangeException($"Column name:{columnName} not found in this record.");
-
+        if (IndexOfIgnoreCase(Headers, columnName) != -1)
+            throw new InvalidOperationException($"{TableName}: 列名 '{columnName}' 已存在");
+        
+        Fields.Add(value);
+        Headers.Add(columnName);
+    }
+    
+    public void RemoveField(string columnName)
+    {
+        var idx = IndexOfIgnoreCase(Headers, columnName);
+        if (idx == -1)
+            throw new InvalidOperationException($"{TableName}: 列名 '{columnName}' 不存在");
+        
+        Fields.RemoveAt(idx);
+        Headers.Remove(columnName);
+    }
 
+    public bool HeaderExists(string columnName) => IndexOfIgnoreCase(Headers, columnName) != -1;
+    public object Clone()
+    {
+        return new DataRecord(new List<string>(Fields), TableName, new List<string>(Headers), Database);
     }
 }

ConsoleApp2/Helpers/Database/DatabaseHelper.cs

@@ -0,0 +1,54 @@
+using System.Data;
+using MySqlConnector;
+
+namespace ConsoleApp2.Helpers.Database;
+
+public static class DatabaseHelper
+{
+    public static async Task<DataSet> QueryTableAsync(string connStr, string sql)
+    {
+        await using var conn = new MySqlConnection(connStr);
+        if(conn.State is not ConnectionState.Open)
+            await conn.OpenAsync();
+        await using var cmd = conn.CreateCommand();
+        cmd.CommandText = sql;
+        var ds = new DataSet();
+        var adapter = new MySqlDataAdapter(cmd).Fill(ds);
+        return ds;
+    }
+
+    public static async Task<object?> QueryScalarAsync(string connStr, string sql)
+    {
+        await using var conn = new MySqlConnection(connStr);
+        if(conn.State is not ConnectionState.Open)
+            await conn.OpenAsync();
+        await using var cmd = conn.CreateCommand();
+        cmd.CommandText = sql;
+        return await cmd.ExecuteScalarAsync();
+    }
+
+    public static async Task<int> NonQueryAsync(string connStr, string sql)
+    {
+        await using var conn = new MySqlConnection(connStr);
+        if(conn.State is not ConnectionState.Open)
+            await conn.OpenAsync();
+        await using var cmd = conn.CreateCommand();
+        cmd.CommandText = sql;
+        return await cmd.ExecuteNonQueryAsync();
+    }
+
+    public static async Task<int> TransactionAsync(string connStr, string sql, params MySqlParameter[] parameters)
+    {
+        await using var conn = new MySqlConnection(connStr);
+        if(conn.State is not ConnectionState.Open)
+            await conn.OpenAsync();
+        await using var trans = await conn.BeginTransactionAsync();
+        await using var cmd = conn.CreateCommand();
+        cmd.CommandText = sql;
+        cmd.Transaction = trans;
+        cmd.Parameters.AddRange(parameters);
+        var rows = await cmd.ExecuteNonQueryAsync();
+        await trans.CommitAsync();
+        return rows;
+    }
+}

ConsoleApp2/Helpers/DumpDataHelper.cs

@@ -1,6 +1,5 @@
-using ConsoleApp2.Options;
-using System.Text;
-using System.Text.RegularExpressions;
+using System.Text.RegularExpressions;
+using ZstdSharp;
 
 namespace ConsoleApp2.Helpers;
 
@@ -11,32 +10,41 @@ public static partial class DumpDataHelper
     [GeneratedRegex(@"\([^)]*\)")]
     private static partial Regex MatchBrackets();
     
-    
-    public static async Task<string[]> GetCsvHeadersFromSqlFileAsync(string txt)
+    /// <summary>
+    /// 从MyDumper导出的SQL文件内容中读取表头
+    /// </summary>
+    /// <param name="content"></param>
+    /// <returns></returns>
+    /// <exception cref="ArgumentException"></exception>
+    public static string[] GetCsvHeadersFromSqlFile(string content)
     {
-        //var txt = await File.ReadAllTextAsync(filePath);
-        var match = MatchBrackets().Match(txt);
-
+        var match = MatchBrackets().Match(content);
+        if (!match.Success)
+            throw new ArgumentException("输入的SQL内容有误，无法提取表头", nameof(content));
         return ParseHeader(match.ValueSpan);
-    }
-
-    private static string[] ParseHeader(ReadOnlySpan<char> headerStr)
-    {
-        headerStr = headerStr[1..^1];
-        Span<Range> ranges = stackalloc Range[50];
-        var count = headerStr.Split(ranges, ',');
-        var arr = new string[count];
-
-        for (var i = 0; i < count; i++)
+        
+        string[] ParseHeader(ReadOnlySpan<char> headerStr)
         {
-            arr[i] = headerStr[ranges[i]].Trim("@`").ToString(); // 消除列名的反引号，如果是变量则消除@
-        }
+            headerStr = headerStr[1..^1];
+            Span<Range> ranges = stackalloc Range[50];
+            var count = headerStr.Split(ranges, ',');
+            var arr = new string[count];
 
-        return arr;
+            for (var i = 0; i < count; i++)
+            {
+                arr[i] = headerStr[ranges[i]].Trim("@`").ToString(); // 消除列名的反引号，如果是变量则消除@
+            }
+        
+            return arr;
+        }
     }
     
-    
-    public static string GetTableName(ReadOnlySpan<char> filePath)
+    /// <summary>
+    /// 从MyDumper导出的Csv文件名解析出表名
+    /// </summary>
+    /// <param name="filePath"></param>
+    /// <returns></returns>
+    public static string GetTableNameFromCsvFileName(ReadOnlySpan<char> filePath)
     {
         filePath = filePath[(filePath.LastIndexOf('\\') + 1)..];
         var firstDotIdx = -1;
@@ -60,13 +68,24 @@ public static partial class DumpDataHelper
         return filePath[(firstDotIdx+1)..secondDotIdx].ToString();
     }
 
-    public static async Task<string[]> GetCsvFileNamesFromSqlFileAsync(string txt,Regex regex)
+    /// <summary>
+    /// 从MyDumper导出的SQL文件内容中读取CSV文件名
+    /// </summary>
+    /// <param name="txt"></param>
+    /// <param name="regex"></param>
+    /// <returns></returns>
+    public static Task<string[]> GetCsvFileNamesFromSqlFileAsync(string txt, Regex regex)
     {
         //var txt = await File.ReadAllTextAsync(filePath);
         var matches = regex.Matches(txt);
-        return matches.Select(match => match.ValueSpan[1..^1].ToString()).ToArray();
+        return Task.FromResult(matches.Select(match => match.ValueSpan[1..^1].ToString()).ToArray());
     }
     
+    /// <summary>
+    /// 检查字符串是否为16进制
+    /// </summary>
+    /// <param name="str"></param>
+    /// <returns></returns>
     public static bool CheckHexField(string? str)
     {
         if (string.IsNullOrWhiteSpace(str))
@@ -90,9 +109,17 @@ public static partial class DumpDataHelper
 
         return true;
     }
+    
+    /// <summary>
+    /// 将输入流以ZSTD标准解压为字符串
+    /// </summary>
+    /// <param name="stream"></param>
+    /// <returns></returns>
+    public static async Task<string> DecompressZstAsStringAsync(Stream stream)
+    {
+        await using var ds = new DecompressionStream(stream);
+        var reader = new StreamReader(ds);
+        return await reader.ReadToEndAsync();
+    }
 
-    // public static string EliminateEscapeChars(ReadOnlySpan<char> str)
-    // {
-    //     char[] escapeChars = ['0','\''];
-    // }
 }

ConsoleApp2/Helpers/Extensions.IEnumerable.cs

@@ -0,0 +1,67 @@
+using System.Diagnostics.CodeAnalysis;
+using System.Reflection;
+
+namespace ConsoleApp2.Helpers;
+
+public static class EnumerableExtensions
+{
+    public static string ToMarkdownTable<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicMethods)]T>(this IEnumerable<T> source)
+    {
+        var properties = typeof(T).GetRuntimeProperties();
+        var fields = typeof(T)
+            .GetRuntimeFields()
+            .Where(f => f.IsPublic);
+
+        var gettables = Enumerable.Union(
+            properties.Select(p => new { p.Name, GetValue = (Func<object, object>)p.GetValue, Type = p.PropertyType }),
+            fields.Select(p => new { p.Name, GetValue = (Func<object, object>)p.GetValue, Type = p.FieldType }));
+
+        var maxColumnValues = source
+            .Select(x => gettables.Select(p => p.GetValue(x)?.ToString()?.Length ?? 0))
+            .Union(new[] { gettables.Select(p => p.Name.Length) }) // Include header in column sizes
+            .Aggregate(
+                new int[gettables.Count()].AsEnumerable(),
+                (accumulate, x) => accumulate.Zip(x, Math.Max))
+            .ToArray();
+
+        var columnNames = gettables.Select(p => p.Name);
+
+        var headerLine = "| " + string.Join(" | ", columnNames.Select((n, i) => n.PadRight(maxColumnValues[i]))) + " |";
+
+        var isNumeric = new Func<Type, bool>(type =>
+            type == typeof(Byte) ||
+            type == typeof(SByte) ||
+            type == typeof(UInt16) ||
+            type == typeof(UInt32) ||
+            type == typeof(UInt64) ||
+            type == typeof(Int16) ||
+            type == typeof(Int32) ||
+            type == typeof(Int64) ||
+            type == typeof(Decimal) ||
+            type == typeof(Double) ||
+            type == typeof(Single));
+
+        var rightAlign = new Func<Type, char>(type => isNumeric(type) ? ':' : ' ');
+
+        var headerDataDividerLine =
+            "| " +
+            string.Join(
+                "| ",
+                gettables.Select((g, i) => new string('-', maxColumnValues[i]) + rightAlign(g.Type))) +
+            "|";
+
+        var lines = new[]
+        {
+            headerLine,
+            headerDataDividerLine,
+        }.Union(
+            source
+                .Select(s =>
+                    "| " + string.Join(" | ",
+                        gettables.Select((n, i) => (n.GetValue(s)?.ToString() ?? "").PadRight(maxColumnValues[i]))) +
+                    " |"));
+
+        return lines
+            .Aggregate((p, c) => p + Environment.NewLine + c);
+    }
+}

ConsoleApp2/Helpers/Extensions.String.cs

@@ -5,6 +5,12 @@ namespace ConsoleApp2.Helpers;
 
 public static class StringExtensions
 {
+    /// <summary>
+    /// 截断字符串
+    /// </summary>
+    /// <param name="this"></param>
+    /// <param name="maxLength">截断长度</param>
+    /// <returns></returns>
     public static string Omit(this ReadOnlySpan<char> @this, int maxLength)
     {
         if (@this.Length > maxLength)
@@ -12,8 +18,20 @@ public static class StringExtensions
         return @this.ToString();
     }
 
+    /// <summary>
+    /// 截断字符串
+    /// </summary>
+    /// <param name="this"></param>
+    /// <param name="maxLength">截断长度</param>
+    /// <returns></returns>
     public static string Omit(this string @this, int maxLength) => Omit(@this.AsSpan(), maxLength);
 
+    /// <summary>
+    /// 将16进制字符串转换为字符串
+    /// </summary>
+    /// <param name="hexString"></param>
+    /// <param name="encoding"></param>
+    /// <returns></returns>
     public static string FromHex(ReadOnlySpan<char> hexString, Encoding? encoding = null)
     {
         encoding ??= Encoding.UTF8;
@@ -38,6 +56,11 @@ public static class StringExtensions
         return encoding.GetString(bytes);
     }
     
+    /// <summary>
+    /// 检查是否为JSON字符串
+    /// </summary>
+    /// <param name="hexStr"></param>
+    /// <returns></returns>
     public static bool CheckJsonHex(ReadOnlySpan<char> hexStr)
     {
         if (hexStr.Length < 2)

ConsoleApp2/Helpers/Extensions.Task.cs

@@ -0,0 +1,19 @@
+namespace ConsoleApp2.Helpers;
+
+public static class TaskExtensions
+{
+    public static async Task WaitUntil(Func<bool> condition, int pollDelay = 25, CancellationToken ct = default)
+    {
+        try
+        {
+            while (!condition())
+            {
+                await Task.Delay(pollDelay, ct);
+            }
+        }
+        catch(TaskCanceledException)
+        {
+            // CancellationToken激活时Task.Delay通过抛异常来结束停止等待，不用管它
+        }
+    }
+}

ConsoleApp2/Helpers/ValidateConsole.cs

@@ -1,31 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace ConsoleApp2.Helpers
-{
-    public static class ValidateConsole
-    {
-      public static void ValidateInput<T>(Func<string,bool> converter,string message)
-        {
-            Console.Write(message);
-           string ? input = Console.ReadLine();
-            while (true)
-            {
-                if (!string.IsNullOrEmpty(input))
-                {
-                    var result = converter(input);
-                    if (result == false)
-                    {
-                        Console.WriteLine($"输入的内容不合法,请重新输入!");
-                        input = Console.ReadLine();
-                    }
-                    else break;
-                }
-                break;
-            }
-        }
-    }
-}

ConsoleApp2/HostedServices/Abstractions/IDataReader.cs

@@ -6,8 +6,9 @@ using System.Threading.Tasks;
 
 namespace ConsoleApp2.HostedServices.Abstractions
 {
-    public interface  IDataSource:IDisposable
+    public interface IDataReader : IDisposable
     {
-        public Task DoEnqueue(Action<DataRecord> action);
+        DataRecord Current { get; }
+        ValueTask<bool> ReadAsync();
     }
 }

ConsoleApp2/HostedServices/Abstractions/IInputService.cs

@@ -5,5 +5,5 @@ namespace ConsoleApp2.HostedServices.Abstractions;
 
 public interface IInputService
 {
-    public Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, ProcessContext context, CancellationToken cancellationToken);
+    public Task ExecuteAsync(CancellationToken cancellationToken);
 }

ConsoleApp2/HostedServices/Abstractions/IOutputService.cs

@@ -5,5 +5,5 @@ namespace ConsoleApp2.HostedServices.Abstractions;
 
 public interface IOutputService
 {
-    public Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken);
+    public Task ExecuteAsync(CancellationToken ct);
 }

ConsoleApp2/HostedServices/Abstractions/ITransformService.cs

@@ -5,5 +5,5 @@ namespace ConsoleApp2.HostedServices.Abstractions;
 
 public interface ITransformService
 {
-    public Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken);
+    public Task ExecuteAsync(CancellationToken cancellationToken);
 }

ConsoleApp2/HostedServices/FileInputService.cs

@@ -0,0 +1,111 @@
+using ConsoleApp2.Const;
+using ConsoleApp2.HostedServices.Abstractions;
+using ConsoleApp2.Options;
+using ConsoleApp2.Services;
+using ConsoleApp2.Services.ETL;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace ConsoleApp2.HostedServices;
+
+public record FileInputInfo
+{
+    public required string FileName { get; init; }
+    public required string TableName { get; init; }
+    public required string[] Headers { get; init; }
+}
+
+public enum FileInputType
+{
+    MyDumperCsv,
+    MyDumperZst,
+    ErrorLog,
+}
+
+/// <summary>
+/// 从输入目录中导入文件
+/// </summary>
+public class FileInputService : IInputService
+{
+    private readonly ILogger _logger;
+    private readonly DataRecordQueue _producerQueue;
+    private readonly IOptions<DataInputOptions> _dataInputOptions;
+    private readonly ProcessContext _context;
+    private readonly DataReaderFactory _dataReaderFactory;
+
+    public FileInputService(ILogger<FileInputService> logger,
+            IOptions<DataInputOptions> dataInputOptions,
+            ProcessContext context, 
+            [FromKeyedServices(ProcessStep.Produce)] DataRecordQueue producerQueue, 
+            DataReaderFactory dataReaderFactory)
+    {
+        _logger = logger;
+        _dataInputOptions = dataInputOptions;
+        _context = context;
+        _producerQueue = producerQueue;
+        _dataReaderFactory = dataReaderFactory;
+    }
+
+    public async Task ExecuteAsync(CancellationToken cancellationToken)
+    {
+        var inputDir = _dataInputOptions.Value.InputDir ?? throw new ApplicationException("未配置文件输入目录");
+        _logger.LogInformation("***** Input service started, working directory: {InputDir} *****", inputDir);
+        
+        var trans = _dataInputOptions.Value.FileInputMetaBuilder;
+        if(trans is null) throw new ApplicationException("未配置文件名-表名映射委托");
+        FileInputInfo[] infoArr = Directory.GetFiles(inputDir)
+            .Select(f => trans(f))
+            .Where(info => info is not null).ToArray()!;
+        
+        var orderedInfo = GetFilesInOrder(infoArr).ToArray();
+        
+        _logger.LogInformation("***** {Count} files founded in directory，{OrderedCount} files is matched with configuration *****", infoArr.Length, orderedInfo.Length);
+        foreach (var info in orderedInfo)
+        {
+            _logger.LogDebug("Table {TableName}: {FileName}", info.TableName, info.FileName);
+        }
+
+        foreach (var info in orderedInfo)
+        {
+            _logger.LogInformation("Reading file: {FileName}, table: {TableName}", info.FileName, info.TableName);
+            var source = _dataReaderFactory.CreateReader(info.FileName,info.TableName,info.Headers);
+
+            while (await source.ReadAsync())
+            {
+                var record = source.Current;
+                _producerQueue.Enqueue(record);
+                _context.AddInput();
+            }
+            
+            _logger.LogInformation("Input of table: '{TableName}' finished", info.TableName);
+        }
+
+        _context.CompleteInput();
+        _logger.LogInformation("***** Input service finished *****");
+    }
+
+    /// <summary>
+    /// 读取配置，按照配置的表顺序来返回
+    /// </summary>
+    /// <returns></returns>
+    private IEnumerable<FileInputInfo> GetFilesInOrder(FileInputInfo[] inputFiles)
+    {
+        var tableOrder = _dataInputOptions.Value.TableOrder;
+        if (tableOrder is null or { Length: 0 })
+            return inputFiles;
+
+        return Yield();
+
+        IEnumerable<FileInputInfo> Yield()
+        {
+            foreach (var tableName in tableOrder)
+            {
+                var target = inputFiles.FirstOrDefault(f =>
+                    f.TableName.Equals(tableName, StringComparison.OrdinalIgnoreCase));
+                if (target is not null)
+                    yield return target;
+            }
+        }
+    }
+}

ConsoleApp2/HostedServices/InputService.cs

@@ -1,59 +0,0 @@
-using ConsoleApp2.Const;
-using ConsoleApp2.Helpers;
-using ConsoleApp2.HostedServices.Abstractions;
-using ConsoleApp2.Options;
-using ConsoleApp2.Services;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Options;
-
-namespace ConsoleApp2.HostedServices;
-
-/// <summary>
-/// 从MyDumper导出的CSV文件中导入表头和数据
-/// </summary>
-public class InputService : IInputService
-{
-    private readonly ILogger _logger;
-    private readonly IOptions<DataInputOptions> _dataInputOptions;
-    private readonly ProcessContext _context;
-
-    public InputService(ILogger<InputService> logger,
-            IOptions<DataInputOptions> dataInputOptions,
-            ProcessContext context)
-    {
-        _logger = logger;
-        _dataInputOptions = dataInputOptions;
-        _context = context;
-    }
-
-    public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, ProcessContext context,CancellationToken cancellationToken)
-    {
-        var inputDir = _dataInputOptions.Value.InputDir;
-        _logger.LogInformation("***** Csv input service start, working dir: {InputDir}, thread id: {ThreadId} *****", inputDir, Environment.CurrentManagedThreadId);
-        var files = Directory.GetFiles(inputDir);
-        if (files.Length == 0)
-        {
-            _logger.LogInformation("No source files found in {InputDir}", inputDir);
-            return;
-        }
-        var count = 0;
-        foreach (var tableName in tasksOptions.TableInfoConfig.Keys)
-        {
-            _logger.LogInformation("Working table: {tableName}", tableName);
-            var source = _dataInputOptions.Value.CreateSource?.Invoke(tableName);
-            await source.DoEnqueue((record) =>
-            {
-                _context.AddInput();
-                producerQueue.Enqueue(record);
-                count++;
-
-            });
-            
-            _logger.LogInformation("table:'{tableName}' input completed", tableName);
-        }
-
-        context.CompleteInput();
-        _logger.LogInformation("***** Csv input service completed *****");
-    }
-}

ConsoleApp2/HostedServices/MainHostedService.cs

@@ -1,110 +1,226 @@
-using ConsoleApp2.HostedServices.Abstractions;
+using System.Diagnostics;
+using System.Text;
+using ConsoleApp2.Helpers;
+using ConsoleApp2.Helpers.Database;
+using ConsoleApp2.HostedServices.Abstractions;
 using ConsoleApp2.Options;
 using ConsoleApp2.Services;
+using ConsoleApp2.Services.ErrorRecorder;
+using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
 namespace ConsoleApp2.HostedServices;
 
 public class MainHostedService : BackgroundService
 {
-    private readonly ILogger _logger;
+    private Stopwatch? _stopwatch;
     private readonly IInputService _input;
     private readonly ITransformService _transform;
     private readonly IOutputService _output;
+    private readonly ILogger _logger;
     private readonly ProcessContext _context;
+    
+    private readonly IOptions<DatabaseOutputOptions> _databaseOptions;
+    private readonly IOptions<TenantDbOptions> _tenantDbOptions;
+    private readonly IConfiguration _config;
 
-    public MainHostedService(ILogger<MainHostedService> logger, IInputService input, ITransformService transform, IOutputService output, ProcessContext context)
+    public MainHostedService(IInputService input, 
+        ITransformService transform,
+        IOutputService output, 
+        ILogger<MainHostedService> logger, 
+        IOptions<TenantDbOptions> tenantDbOptions, 
+        IOptions<DatabaseOutputOptions> databaseOptions, 
+        IConfiguration config, 
+        ProcessContext context)
     {
-        _logger = logger;
         _input = input;
         _transform = transform;
         _output = output;
+        _logger = logger;
+        _tenantDbOptions = tenantDbOptions;
+        _databaseOptions = databaseOptions;
+        _config = config;
         _context = context;
     }
 
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
     {
-        var taskFun = (TasksOptions taskOp, DataRecordQueue producerQueue, DataRecordQueue consumerQueue, ProcessContext context) =>
+        _stopwatch = Stopwatch.StartNew();
+        var inputTask = ExecuteAndCatch(
+            async () => await _input.ExecuteAsync(stoppingToken), "文件输入程序出现异常", stoppingToken);
+        var transformTask = ExecuteAndCatch(
+            async () => await _transform.ExecuteAsync(stoppingToken), "转换程序出现异常", stoppingToken);
+        var outputTask = ExecuteAndCatch(
+            async () => await _output.ExecuteAsync(stoppingToken), "输出程序出现异常", stoppingToken);
+
+        await Task.WhenAll(inputTask, transformTask, outputTask);
+        _stopwatch.Stop();
+        _logger.LogInformation("***** All tasks completed *****");
+        _logger.LogInformation("***** ElapseTime: {Time}", (_stopwatch.ElapsedMilliseconds / 1000f).ToString("F3"));
+        await Task.Delay(5000, stoppingToken);
+
+
+        if (!stoppingToken.IsCancellationRequested)
         {
-            var inputTask = Task.Run(async () =>
-            {
-                try
-                {
-                    await _input.ExecuteAsync(taskOp, producerQueue, context, stoppingToken);
-                }
-                catch (Exception ex)
-                {
-                    _logger.LogCritical("Exception occurred on inputService:{Message},{StackTrace}", ex.Message, ex.StackTrace);
-                    throw;
-                }
+            await ExportResultAsync();
+            _logger.LogInformation("The execution result export to {Path}",
+                Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"Result-{ErrorRecorder.UID}.md"));
+            if (_config["RestoreIndex"] is not null)
+                await RestoreIndexAsync();
 
-            });
-            var transformTask = Task.Run(async () =>
-            {
-                try
-                {
-                    await _transform.ExecuteAsync(taskOp, producerQueue, consumerQueue, context, stoppingToken);
-                }
-                catch (Exception ex)
-                {
-                    _logger.LogCritical("Exception occurred on transformService:{Message},{StackTrace}", ex.Message, ex.StackTrace);
-                    throw;
-                }
-
-            });
-            var outputTask = Task.Run(async () =>
-            {
-                try
-                {
-                    await _output.ExecuteAsync(taskOp, consumerQueue, context,stoppingToken);
-                }
-                catch (Exception ex)
-                {
-                    _logger.LogCritical("Exception occurred on outputService:{Message},{StackTrace}", ex.Message, ex.StackTrace);
-                    throw;
-                }
-
-            });
-        };
-
-        var bigTablesDic = new Dictionary<string, TableInfo>
+            Environment.Exit(0);
+        }
+        else Environment.Exit(1);
+    }
+    
+    private Task ExecuteAndCatch(Func<Task> func, string message, CancellationToken ct)
+    {
+        return Task.Run(async () =>
         {
-            {"order_block_plan",new TableInfo{SimulaRowCount=2725553 }},//CreateTime < 202301的删除 
-            {"order_block_plan_result",new TableInfo{SimulaRowCount=1174096 }},
-            {"order_box_block",new TableInfo{SimulaRowCount=29755672 }},
-            {"order_item",new TableInfo{SimulaRowCount=1345520079 }},
-            {"simple_plan_order",new TableInfo{SimulaRowCount=351470 }},//CreateTime < 202301的删除
-        };
-        taskFun(new TasksOptions { TableInfoConfig = bigTablesDic, OutPutOptions = new OutPutOptions { FlushCount = 10000, OutPutTaskCount = 2 } },
-            new DataRecordQueue(), new DataRecordQueue(),new ProcessContext());
-        var smallTablesDic = new Dictionary<string, TableInfo>
+            try
+            {
+                await func();
+            }
+            catch (Exception e)
+            {
+                _logger.LogCritical(e, "{Msg}\t{ErrMsg}", message, e.Message);
+                _context.AddException(e);
+                Environment.Exit(1);
+            }
+        }, ct);
+    }
+
+    /// <summary>
+    /// 还原所有数据库的索引...
+    /// </summary>
+    /// <returns></returns>
+    /// <exception cref="ApplicationException"></exception>
+    private Task RestoreIndexAsync()
+    {
+        var databases = _tenantDbOptions.Value.DbList?.Keys 
+                        ?? throw new ApplicationException("无法还原索引，因为分库配置中没有配置数据库");
+        var connStr = _databaseOptions.Value.ConnectionString 
+                      ?? throw new ApplicationException("无法还原索引，因为没有配置数据库连接字符串");
+        var list = new List<Task>();
+        foreach(var db in databases)
         {
-            {"machine",new TableInfo{SimulaRowCount=14655 }},
-            {"order",new TableInfo{SimulaRowCount=5019216 }},
-            {"order_data_block",new TableInfo{SimulaRowCount=731800334 }},
-            {"order_data_goods",new TableInfo{SimulaRowCount=25803671 }},
-            {"order_data_parts",new TableInfo{SimulaRowCount=468517543 }},
-            {"order_module",new TableInfo{SimulaRowCount=103325385 }},
-            {"order_module_extra",new TableInfo{SimulaRowCount=54361321 }},
-            {"order_module_item",new TableInfo{SimulaRowCount=69173339 }},
-            {"order_package",new TableInfo{SimulaRowCount=16196195 }},
-            {"order_process",new TableInfo{SimulaRowCount=3892685 }},//orderNo < 202301的  
-            {"order_process_step",new TableInfo{SimulaRowCount=8050349 }},//orderNo < 202301的删除   
-            {"order_process_step_item",new TableInfo{SimulaRowCount=14538058 }},//orderNo < 202301的删除 
-            {"order_scrap_board",new TableInfo{SimulaRowCount=123998 }},
-            {"process_group",new TableInfo{SimulaRowCount=1253 }},
-            {"process_info",new TableInfo{SimulaRowCount=7839 }},
-            {"process_item_exp",new TableInfo{SimulaRowCount=28 }},
-            {"process_schdule_capacity",new TableInfo{SimulaRowCount=39736 }},
-            {"process_step_efficiency",new TableInfo{SimulaRowCount=8 }},
-            {"report_template",new TableInfo{SimulaRowCount=7337 }},
-            {"simple_package",new TableInfo{SimulaRowCount=130436 }},//orderNo < 202301的删除
-            {"sys_config",new TableInfo{SimulaRowCount=2296 }},
-            {"work_calendar",new TableInfo{SimulaRowCount=11 }},
-            {"work_shift",new TableInfo{SimulaRowCount=59 }},
-            {"work_time",new TableInfo{SimulaRowCount=62 }},
+            var task = DatabaseHelper.NonQueryAsync(connStr + $";Database={db};",
+                """
+                CREATE INDEX `idx_CompanyID` ON `machine` (`CompanyID`);
+
+                CREATE INDEX `idx_companyid` ON `order` (`CompanyID`);
+
+                CREATE INDEX `idx_CompanyID` ON `order_block_plan` (`CompanyID`);
+
+                CREATE INDEX `idx_PlanID` ON `order_block_plan_item` (`PlanID`);
+
+                CREATE INDEX `idx_orderno` ON `order_box_block` (`OrderNo`);
+
+                CREATE INDEX `index_OrderNo` ON `order_data_block` (`OrderNo`);
+
+                CREATE INDEX `index_OrderNo` ON `order_data_goods` (`OrderNo`);
+
+                CREATE INDEX `index_OrderNo` ON `order_data_parts` (`OrderNo`);
+
+                CREATE INDEX `index_ItemNo` ON `order_item` (`ItemNo`);
+
+                CREATE INDEX `index_OrderNo` ON `order_item` (`OrderNo`);
+
+                CREATE INDEX `index_PackageID` ON `order_item` (`PackageID`);
+
+                CREATE INDEX `index_PlanID` ON `order_item` (`PlanID`);
+
+                CREATE INDEX `idx_OrderNo` ON `order_module` (`OrderNo`);
+
+                CREATE INDEX `index_OrderNo` ON `order_module_extra` (`OrderNo`);
+
+                CREATE INDEX `index_OrderNo` ON `order_module_item` (`OrderNo`);
+
+                CREATE INDEX `idx_OrderNo` ON `order_package` (`OrderNo`);
+
+                CREATE INDEX `idx_PakageNo` ON `order_package` (`PakageNo`);
+
+                CREATE INDEX `idx_PackageID` ON `order_package_item` (`PackageID`);
+
+                CREATE INDEX `idx_companyid` ON `order_patch_detail` (`CompanyID`);
+
+                CREATE INDEX `idx_OrderNo` ON `order_process` (`OrderNo`);
+
+                CREATE INDEX `index_CompanyID` ON `order_process_schdule` (`CompanyID`);
+
+                CREATE INDEX `IX_order_process_step_OrderProcessID` ON `order_process_step` (`OrderProcessID`);
+
+                CREATE INDEX `idx_OrderProcessID` ON `order_process_step_item` (`OrderProcessID`);
+
+                CREATE INDEX `idx_OrderProcessStepID` ON `order_process_step_item` (`OrderProcessStepID`);
+
+                CREATE INDEX `idx_CompanyID` ON `order_scrap_board` (`CompanyID`);
+
+                CREATE INDEX `idx_CompanyID` ON `process_group` (`CompanyID`);
+
+                CREATE INDEX `idx_CompanyID` ON `process_info` (`CompanyID`);
+
+                CREATE INDEX `index_CompanyID` ON `process_item_exp` (`CompanyID`);
+
+                CREATE INDEX `idx_CompanyID` ON `process_schdule_capacity` (`CompanyID`);
+
+                CREATE INDEX `idx_CompanyID` ON `process_step_efficiency` (`CompanyID`);
+
+                CREATE INDEX `idx_CompanyID` ON `report_template` (`CompanyID`);
+
+                CREATE INDEX `indx_OrderNo` ON `simple_package` (`OrderNo`);
+
+                CREATE INDEX `idx_CompanyID` ON `simple_plan_order` (`CompanyID`);
+
+                CREATE INDEX `idx_CompanyID` ON `sys_config` (`CompanyID`);
+
+                CREATE INDEX `idx` ON `work_calendar` (`CompanyID`);
+
+                CREATE INDEX `idx_CompanyID` ON `work_shift` (`CompanyID`);
+
+                CREATE INDEX `IX_work_time_ShiftID` ON `work_time` (`ShiftID`);
+                """);
+            list.Add(task);
+        }
+
+        return Task.WhenAll(list);
+    }
+
+    private async Task ExportResultAsync()
+    {
+        var sb = new StringBuilder();
+        if (_context.HasException)
+            sb.AppendLine("# Program Completed With Error");
+        else sb.AppendLine("# Program Completed Successfully");
+        sb.AppendLine("## Process Count");
+        var processCount = new[]
+        {
+            new { State = "Input", Count = _context.InputCount },
+            new { State = "Transform", Count = _context.TransformCount },
+            new { State = "Output", Count = _context.OutputCount }
         };
-        taskFun(new TasksOptions { TableInfoConfig = smallTablesDic, OutPutOptions = new OutPutOptions { FlushCount = 20000, OutPutTaskCount = 4 } },
-            new DataRecordQueue(), new DataRecordQueue(), new ProcessContext());
+        sb.AppendLine(processCount.ToMarkdownTable());
+        sb.AppendLine("\n---\n");
+        sb.AppendLine("## Table Output Progress");
+        var tableOutputProgress = _context.TableProgress.Select(pair =>
+            new { Table = pair.Key, Count = pair.Value });
+        sb.AppendLine(tableOutputProgress.ToMarkdownTable());
+        sb.AppendLine("\n---\n");
+        sb.AppendLine("## Result");
+        var elapsedTime = (_stopwatch!.ElapsedMilliseconds / 1000f);
+        var result = new[]
+        {
+            new { Field = "ElapsedTime", Value = elapsedTime.ToString("F2") },
+            new
+            {
+                Field = "Average Output Speed",
+                Value = (_context.OutputCount / elapsedTime).ToString("F2") + "records/s"
+            }
+        };
+        sb.AppendLine(result.ToMarkdownTable());
+        await File.WriteAllTextAsync(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"Result-{ErrorRecorder.UID}.md"),
+            sb.ToString());
     }
 }

ConsoleApp2/HostedServices/OutputService.cs

@@ -2,8 +2,11 @@
 using ConsoleApp2.HostedServices.Abstractions;
 using ConsoleApp2.Options;
 using ConsoleApp2.Services;
+using ConsoleApp2.Services.ErrorRecorder;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using MySqlDestination = ConsoleApp2.Services.ETL.MySqlDestination;
+using TaskExtensions = ConsoleApp2.Helpers.TaskExtensions;
 
 namespace ConsoleApp2.HostedServices;
 
@@ -14,96 +17,116 @@ public class OutputService : IOutputService
 {
     private readonly ILogger _logger;
     private readonly IOptions<DatabaseOutputOptions> _outputOptions;
-    private readonly IOptions<DataTransformOptions> _transformOptions;
     private readonly ProcessContext _context;
-    private readonly TaskManager _taskManager;
-    private readonly ErrorRecorder _errorRecorder;
+    private readonly ErrorRecorderFactory _errorRecorderFactory;
+    private readonly RecordQueuePool _queuePool;
 
     public OutputService(ILogger<OutputService> logger,
         IOptions<DatabaseOutputOptions> outputOptions,
         ProcessContext context,
-        TaskManager taskManager, 
-        IOptions<DataTransformOptions> transformOptions, 
-        ErrorRecorder errorRecorder)
+        RecordQueuePool queuePool, 
+        ErrorRecorderFactory errorRecorderFactory)
     {
         _logger = logger;
         _outputOptions = outputOptions;
         _context = context;
-        _taskManager = taskManager;
-        _transformOptions = transformOptions;
-        _errorRecorder = errorRecorder;
+        _queuePool = queuePool;
+        _errorRecorderFactory = errorRecorderFactory;
     }
 
-    public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken)
+    public async Task ExecuteAsync(CancellationToken ct)
     {
-        _logger.LogInformation("***** Mysql output service started *****");
-        _taskManager.CreateTasks(async () =>
+        _logger.LogInformation("***** Output service started *****");
+        var dbTaskManager = new TaskManager(5);
+        var dbTasks = new Dictionary<string, Task>();
+        while (!_context.IsTransformCompleted)
         {
-            //k: database v: records，按照要导出的数据库名分组
-            var databaseDict = new Dictionary<string, List<DataRecord>>();
-            while (!context.IsTransformCompleted || consumerQueue.Count > 0)
+            foreach (var (db, queue) in _queuePool.Queues)
             {
-                if (!consumerQueue.TryDequeue(out var record)) continue;
-                var dbName = record.Database;
-                var records = databaseDict.AddOrUpdate(dbName, [record], (_, list) =>
+                if (!dbTasks.ContainsKey(db))
                 {
-                    list.Add(record);
-                    return list;
-                });
-                
-                if (records.Count >= tasksOptions.OutPutOptions.FlushCount)
-                {
-                    await FlushAsync(dbName, records);
-                    records.Clear();
+                    dbTasks.Add(db, await dbTaskManager.CreateTaskAsync(
+                        async () => await StartDatabaseWorker(db, queue, ct), ct));
                 }
             }
 
-            foreach (var (db, records) in databaseDict)
+            await Task.Delay(500, ct);
+        }
+
+        await TaskExtensions.WaitUntil(() => dbTaskManager.RunningTaskCount == 0, 25, ct);
+        
+        _context.CompleteOutput();
+        _logger.LogInformation("***** Output service finished *****");
+    }
+
+    private async Task StartDatabaseWorker(string db, DataRecordQueue queue, CancellationToken ct = default)
+    {
+        _logger.LogInformation("*****开启输出线程，数据库: {db} *****", db);
+        var taskManager = new TaskManager(_outputOptions.Value.MaxDatabaseOutputTask);
+        var tmp = new List<DataRecord>();
+        while (!_context.IsTransformCompleted || queue.Count > 0)
+        {
+            if (ct.IsCancellationRequested)
+                break;
+            
+            if (!queue.TryDequeue(out var record)) continue;
+            
+            var dbName = record.Database ?? throw new ApplicationException("输出的记录缺少数据库名");
+            if(dbName != db)
+                throw new ApplicationException($"输出记录的数据与当前输出线程不匹配，记录：{dbName}, 输出线程：{db}");
+            tmp.Add(record);
+
+            if (tmp.Count >= _outputOptions.Value.FlushCount)
             {
-                if (records.Count > 0)
+                var list = tmp;
+                tmp = [];
+                await taskManager.CreateTaskAsync(async arg => // 转换为方法组
                 {
-                    await FlushAsync(db, records);
-                    records.Clear();
-                }
+                    var tuple = arg as Tuple<string, List<DataRecord>>;
+                    try
+                    {
+                        await FlushAsync(tuple!.Item1, tuple.Item2);
+                    }
+                    catch (Exception e)
+                    {
+                        _logger.LogError(e, "输出记录时发生错误");
+                        throw;
+                    }
+                }, Tuple.Create(dbName, list), ct);
             }
+        }
 
-            databaseDict.Clear();
-            _logger.LogInformation("***** Mysql output thread completed *****");
-        }, tasksOptions.OutPutOptions.OutPutTaskCount);
-
-        await _taskManager.WaitAll();
-        //_context.CompleteOutput();
-        _logger.LogInformation("***** Mysql output service completed *****");
+        // 等待所有子任务完成
+        await TaskExtensions.WaitUntil(() => taskManager.RunningTaskCount == 0, 10, ct);
+        
+        // 清理剩余记录
+        if (tmp.Count > 0)
+        {
+            await FlushAsync(db, tmp);
+        }
 
+        _logger.LogInformation("*****输出线程结束，数据库: {db} *****", db);
     }
 
     private async Task FlushAsync(string dbName, IEnumerable<DataRecord> records)
     {
-        var count = 0;
-        var connStr = _outputOptions.Value.ConnectionString ?? throw new InvalidOperationException("ConnectionString is null");
-        await using var output = new MySqlDestination($"{connStr};Database={dbName};", _logger, _context, _transformOptions, _errorRecorder);
-        //if (records == null || records.Count() == 0) return;
-        //var dbName = $"cferp_test_1";
-        //if (records != null && records.Count() > 0)
-        //{
-        //    dbName = $"cferp_test_{records.FirstOrDefault()?.CompanyID}";
-        //}
+        var connStr = _outputOptions.Value.ConnectionString ??
+                      throw new InvalidOperationException("连接字符串为空");
+        await using var output = new MySqlDestination($"{connStr};Database={dbName};", _logger,
+            _outputOptions, _errorRecorderFactory.CreateOutput(dbName), _context);
 
-        //await  using var output = new MySqlDestination(new MySqlConnectionStringBuilder
-        //{
-        //    Server = "127.0.0.1",
-        //    Port = 34309,
-        //    Database = dbName,
-        //    UserID = "root",
-        //    Password = "123456",
-        //    MaximumPoolSize = 50,
-        //}.ConnectionString, _logger,true);
+        var tableOutput = new Dictionary<string, int>();
         foreach (var record in records)
         {
             await output.WriteRecordAsync(record);
-            count++;
+            tableOutput.AddOrUpdate(record.TableName, 1, (_, v) => v + 1);
         }
+
         await output.FlushAsync(_outputOptions.Value.MaxAllowedPacket);
-        _context.AddOutput(count);
+        foreach (var (key, value) in tableOutput)
+        {
+            _context.AddTableOutput(key, value);
+        }
+        _logger.LogTrace("Flushed {Count} records", tableOutput.Values.Sum(i => i));
     }
 }

ConsoleApp2/HostedServices/TaskMonitorService.cs

@@ -1,6 +1,7 @@
 using System.Diagnostics;
 using ConsoleApp2.Const;
 using ConsoleApp2.Services;
+using ConsoleApp2.Services.Loggers;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
@@ -12,33 +13,33 @@ namespace ConsoleApp2.HostedServices;
 /// </summary>
 public class TaskMonitorService : BackgroundService
 {
-    private readonly IHostApplicationLifetime _lifetime;
     private readonly ILogger<TaskMonitorService> _logger;
+    private readonly IEnumerable<ITaskMonitorLogger> _monitorLoggers;
     private readonly ProcessContext _context;
     private readonly DataRecordQueue _producerQueue;
-    private readonly DataRecordQueue _consumerQueue;
+    private readonly RecordQueuePool _queuePool;
 
-    public TaskMonitorService(IHostApplicationLifetime lifetime,
+    public TaskMonitorService(
         ILogger<TaskMonitorService> logger,
         ProcessContext context,
-        [FromKeyedServices(ProcessStep.Producer)]
+        [FromKeyedServices(ProcessStep.Produce)]
         DataRecordQueue producerQueue,
-        [FromKeyedServices(ProcessStep.Consumer)]
-        DataRecordQueue consumerQueue)
+        RecordQueuePool queuePool,
+        IEnumerable<ITaskMonitorLogger> monitorLoggers)
     {
-        _lifetime = lifetime;
         _logger = logger;
         _context = context;
         _producerQueue = producerQueue;
-        _consumerQueue = consumerQueue;
+        _queuePool = queuePool;
+        _monitorLoggers = monitorLoggers;
     }
 
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
     {
-        await Task.Factory.StartNew(Monitor, stoppingToken);
+        await Task.Run(() => Monitor(stoppingToken), stoppingToken);
     }
 
-    private async Task Monitor()
+    private async Task Monitor(CancellationToken stoppingToken)
     {
         var sw = Stopwatch.StartNew();
         var lastTime = sw.ElapsedMilliseconds;
@@ -47,7 +48,7 @@ public class TaskMonitorService : BackgroundService
         var lastOutputCount = _context.OutputCount;
 
         bool endCheck = false;
-        while (true)
+        while (!stoppingToken.IsCancellationRequested)
         {
             EndCheck:
             // var running = 0;
@@ -82,22 +83,40 @@ public class TaskMonitorService : BackgroundService
             var inputSpeed = (inputCount - lastInputCount) / elapseTime;
             var transformSpeed = (transformCount - lastTransformCount) / elapseTime;
             var outputSpeed = (outputCount - lastOutputCount) / elapseTime;
-
+            
             // _logger.LogInformation(
             //     "Task monitor: running: {Running}, error: {Error}, completed: {Completed}, canceled: {Canceled}, outputSpeed: {Speed} records/s",
             //     running, error, completed, canceled, outputSpeed);
-            _logger.LogInformation(
-                "Process monitor: input: {inputStatus}, transform: {transformStatus}, output: {outputStatus}\nInput: {InputCount}, Transform: {TransformCount}, Output: {OutputCount}",
-                _context.IsInputCompleted ? "completed" : $"running {inputSpeed:F2} records/s",
-                _context.IsTransformCompleted ? "completed" : $"running {transformSpeed:F2} records/s",
-                _context.IsOutputCompleted ? "completed" : $"running {outputSpeed:F2} records/s",
-                inputCount,
-                transformCount,
-                outputCount);
-            _logger.LogInformation("Queue monitor: producer queue: {ProducerQueue}, consumer queue: {ConsumerQueue}",
-                _producerQueue.Count, _consumerQueue.Count);
+            foreach (var logger in _monitorLoggers)
+            {
+                logger.LogStatus("Monitor: Progress status", new Dictionary<string, string>
+                {
+                    {"Input",_context.IsInputCompleted ? "completed" : $"running {inputSpeed:F2} records/s" },
+                    {"Transform", _context.IsTransformCompleted ? "completed" : $"running {transformSpeed:F2} records/s" },
+                    {"Output", _context.IsOutputCompleted ? "completed" : $"running {outputSpeed:F2} records/s" }
+                });
 
-            await Task.Delay(5000);
+                logger.LogStatus("Monitor: Table output progress",
+                    _context.TableProgress
+                        .ToDictionary(kv => kv.Key, kv => kv.Value.ToString()),
+                    ITaskMonitorLogger.LogLevel.Progress);
+            
+                logger.LogStatus("Monitor: Process count", new Dictionary<string, string>
+                {
+                    {"Input", inputCount.ToString()},
+                    {"Transform", transformCount.ToString()},
+                    {"Output", outputCount.ToString()}
+                }, ITaskMonitorLogger.LogLevel.Progress);
+            
+                logger.LogStatus("Monitor: Queue", new Dictionary<string, string>
+                {
+                    {"Producer queue records", _producerQueue.Count.ToString() },
+                    {"Output queues", _queuePool.Queues.Count.ToString() },
+                    {"Output queue records", _queuePool.Queues.Values.Sum(queue => queue.Count).ToString()}, 
+                });
+            }
+            
+            await Task.Delay(5000, stoppingToken);
 
             lastTime = time;
             lastInputCount = inputCount;
@@ -114,9 +133,6 @@ public class TaskMonitorService : BackgroundService
                 break;
             }
         }
-        
-        _logger.LogInformation("***** All tasks completed *****");
-        _logger.LogInformation("***** ElapseTime: {Time}", (sw.ElapsedMilliseconds / 1000f).ToString("F3"));
-        // _lifetime.StopApplication();
     }
+
 }

ConsoleApp2/HostedServices/TestInputService.cs

@@ -1,83 +0,0 @@
-using ConsoleApp2.HostedServices.Abstractions;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using ConsoleApp2.Const;
-using ConsoleApp2.Options;
-using ConsoleApp2.Services;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Options;
-using System.Reflection.PortableExecutable;
-using System.Collections.Concurrent;
-using ConsoleApp2.SimulationService;
-
-namespace ConsoleApp2.HostedServices
-{
-    public class TestInputService : IInputService
-    {
-        private readonly ILogger _logger;
-        private readonly IOptions<CsvOptions> _csvOptions;
-        private readonly DataRecordQueue _producerQueue;
-        private readonly ProcessContext _context;
-        public TestInputService(ILogger<TestInputService> logger,
-    IOptions<CsvOptions> csvOptions,
-    [FromKeyedServices(ProcessStep.Producer)] DataRecordQueue producerQueue,
-    ProcessContext context)
-        {
-            _logger = logger;
-            _csvOptions = csvOptions;
-            _producerQueue = producerQueue;
-            _context = context;
-        }
-        public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, ProcessContext context, CancellationToken cancellationToken)
-        {
-            var tableName = "order_item";
-            var headers = new string[] { "ID","OrderNo","ItemNo","ItemType","RoomID","BoxID","DataID","PlanID","PackageID","Num","CompanyID","ShardKey" };
-            var dataCount = 1200000000L;
-            var tempCount = 80000;
-            var tempRecords=new List<DataRecord>();
-            var comanyID = 1;
-            short[] shareKeys = { 23040, 23070, 23100, 24000, 24040, 24070, 24100, 25000, 25040, 25070, 25100 };
-            int[] companyIds = { 1, 2, 3, 4 };
-            var sk = shareKeys.First();
-            var companyID = companyIds.First();
-
-            var shareKeyInterval = 20000;
-            var getShareKeyTimes = 0;
-            var getCompanyIDTimes = 0;
-            var shareKeyIntervalCount = 0;
-            for (long i = 1; i <= dataCount; i++)
-            {
-                shareKeyIntervalCount++;
-                if (shareKeyIntervalCount > shareKeyInterval) {
-                    sk=DataHelper.GetShareKey(getShareKeyTimes);
-                    getShareKeyTimes++;
-                    shareKeyIntervalCount = 0;
-                }
-                var fields = new string[] { i.ToString(), "20220104020855", (220105981029 + i).ToString(), "1", "482278", "482279", "3768774", "0", "0", "1", companyID.ToString(), sk.ToString() };
-                var record = new DataRecord(fields, tableName, headers, comanyID);
-                tempRecords.Add(record);
-                if (tempRecords.Count >= tempCount)
-                {
-                    foreach (var rc in tempRecords)
-                    {
-                        _context.AddInput();
-                        _producerQueue.Enqueue(rc);
-                        if (cancellationToken.IsCancellationRequested)
-                            return;
-                    }
-                    tempRecords.Clear();
-                    companyID = DataHelper. GetCompanyId(getCompanyIDTimes);
-                    getCompanyIDTimes++;
-                }
-            }
-           
-            _context.CompleteInput();
-            _logger.LogInformation("***** Csv input service completed *****");
-        }
-
-    }
-}

ConsoleApp2/HostedServices/TransformService.cs

@@ -1,15 +1,17 @@
-using ConsoleApp2.Const;
+using ConsoleApp2.Cache;
+using ConsoleApp2.Const;
 using ConsoleApp2.HostedServices.Abstractions;
 using ConsoleApp2.Options;
 using ConsoleApp2.Services;
-using Microsoft.Extensions.Caching.Distributed;
+using ConsoleApp2.Services.ErrorRecorder;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
-using StackExchange.Redis;
 
 namespace ConsoleApp2.HostedServices;
 
+public record DataTransformContext(DataRecord Record, ICacher Cacher, ILogger Logger);
+
 /// <summary>
 /// 数据处理服务，对导入后的数据进行处理
 /// </summary>
@@ -17,65 +19,99 @@ public class TransformService : ITransformService
 {
     private readonly ILogger _logger;
     private readonly IOptions<DataTransformOptions> _options;
+    private readonly DataRecordQueue _producerQueue;
+    private readonly RecordQueuePool _queuePool;
     private readonly ProcessContext _context;
-    private readonly IDistributedCache _cache;
-    private readonly TaskManager _taskManager;
+    private readonly ICacher _cache;
+    private readonly ErrorRecorderFactory _errorRecorderFactory;
 
 
     public TransformService(ILogger<TransformService> logger, 
         IOptions<DataTransformOptions> options,
+        [FromKeyedServices(ProcessStep.Produce)] DataRecordQueue producerQueue,
+        RecordQueuePool queuePool,
         ProcessContext context,
-        IDistributedCache cache,
-        TaskManager taskManager)
+        ICacher cache, 
+        ErrorRecorderFactory errorRecorderFactory)
     {
         _logger = logger;
         _options = options;
+        _producerQueue = producerQueue;
+        _queuePool = queuePool;
         _context = context;
         _cache = cache;
-        _taskManager = taskManager;
+        _errorRecorderFactory = errorRecorderFactory;
     }
 
-    public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken)
+    public async Task ExecuteAsync(CancellationToken cancellationToken)
     {
         _logger.LogInformation("***** Data transform service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId);
 
-        _taskManager.CreateTasks(async () =>
+        while (!_context.IsInputCompleted || _producerQueue.Count > 0)
         {
-            while ((!context.IsInputCompleted || producerQueue.Count > 0))
-            {
-                if (!producerQueue.TryDequeue(out var record)) continue;
+            if (!_producerQueue.TryDequeue(out var record)) continue;
 
-                //过滤不要的record
-                if (await _options.Value.RecordFilter?.Invoke(record, _cache) == false) continue;
-                //修改record
-                _options.Value.RecordModify?.Invoke(record);
-                //缓存record
-                await _options.Value.RecordCache?.Invoke(record, _cache);
-                //替换record
-                var replaceRecord = await _options.Value.RecordReplace?.Invoke(record, _cache);
-                if (replaceRecord != null)
+            try
+            {
+                var context = new DataTransformContext(record, _cache, _logger);
+                if (_options.Value.EnableFilter)
                 {
-                    record = replaceRecord;
+                    // 数据过滤
+                    var filter = _options.Value.RecordFilter;
+                    if (filter is not null && await filter(context) == false) continue;
                 }
-                //计算需要分流的数据库
-                record.Database = _options.Value.DatabaseFilter.Invoke(record);
-                consumerQueue.Enqueue(record);
-                _context.AddTransform();
-                //数据增加
-                var addRecords = _options.Value.RecordAdd?.Invoke(record);
-                if (addRecords is { Count: > 0 })
+
+                if (_options.Value.EnableReplacer)
                 {
-                    foreach (var rc in addRecords)
+                    // 数据替换
+                    var replacer = _options.Value.RecordModify;
+                    if (replacer is not null)
                     {
-                        rc.Database = _options.Value.DatabaseFilter.Invoke(record);
-                        consumerQueue.Enqueue(rc);
-                        _context.AddTransform();
+                        record = await replacer(context);
+                    }
+                }    
+                
+                // 字段缓存
+                var cacher = _options.Value.RecordCache;
+                if(cacher is not null)
+                    await cacher.Invoke(context);
+            
+                //计算需要分流的数据库
+                var dbFilter = _options.Value.DatabaseFilter 
+                               ?? throw new ApplicationException("未配置数据库过滤器");
+                record.Database = dbFilter(record);
+            
+                _queuePool[record.Database].Enqueue(record);
+                _context.AddTransform();
+
+                if (_options.Value.EnableReBuilder)
+                {
+                    //数据重建
+                    var addRecords = _options.Value.RecordReBuild?.Invoke(context);
+                    if (addRecords is { Count: > 0 })
+                    {
+                        foreach (var rc in addRecords)
+                        {
+                            if(dbFilter is not null)
+                                rc.Database =dbFilter.Invoke(record);
+                            _queuePool[record.Database].Enqueue(rc);
+                            _context.AddTransform();
+                        }
                     }
                 }
             }
-            context.CompleteTransform();
-        },tasksOptions.TransformTaskCount,cancellationToken);
-        await _taskManager.WaitAll();
-        _logger.LogInformation("***** Data transformation service completed *****");
+            catch (Exception e)
+            {
+                _context.AddException(e);
+                var errorRecorder = _errorRecorderFactory.CreateTransform();
+                await errorRecorder.LogErrorRecordAsync(record, e);
+                if (!_options.Value.StrictMode)
+                    _logger.LogError(e, "数据转换时发生错误");
+                else throw;
+            }
+        }
+        _context.CompleteTransform();
+        
+        _logger.LogInformation("***** Data transformation service finished *****");
     }
 }

ConsoleApp2/HostedServices/VoidOutputService.cs

@@ -1,9 +1,5 @@
-using ConsoleApp2.Const;
-using ConsoleApp2.HostedServices.Abstractions;
-using ConsoleApp2.Options;
+using ConsoleApp2.HostedServices.Abstractions;
 using ConsoleApp2.Services;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 
 namespace ConsoleApp2.HostedServices;
@@ -12,28 +8,36 @@ namespace ConsoleApp2.HostedServices;
 public class VoidOutputService : IOutputService
 {
     private readonly ILogger _logger;
-    private readonly DataRecordQueue _consumerQueue;
+    private readonly RecordQueuePool _queuePool;
     private readonly ProcessContext _context;
 
-    public VoidOutputService([FromKeyedServices(ProcessStep.Consumer)] DataRecordQueue consumerQueue,
-        ProcessContext context, ILogger<VoidOutputService> logger)
+    public VoidOutputService(
+        ProcessContext context, ILogger<VoidOutputService> logger, RecordQueuePool queuePool)
     {
-        _consumerQueue = consumerQueue;
         _context = context;
         _logger = logger;
+        _queuePool = queuePool;
     }
 
-    public Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken)
+    public Task ExecuteAsync(CancellationToken ct)
     {
-        _logger.LogInformation("***** Void output service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId);
-        while (!_context.IsTransformCompleted || _consumerQueue.Count > 0)
+        _logger.LogInformation("***** Void Output Service Started *****");
+        while (!_context.IsTransformCompleted || _queuePool.Queues.Count > 0)
         {
-            if (_consumerQueue.TryDequeue(out var record))
+            foreach (var pair in _queuePool.Queues) // 内存优化
+            {
+                if (_context.IsTransformCompleted && pair.Value.Count == 0)
+                {
+                    _queuePool.RemoveQueue(pair.Key);
+                    continue;
+                }
+                if(!pair.Value.TryDequeue(out var record)) continue;
                 _context.AddOutput();
+            }
         }
 
         _context.CompleteOutput();
-        _logger.LogInformation("***** Void output service completed *****");
+        _logger.LogInformation("***** Void Output Service Stopped *****");
         return Task.CompletedTask;
     }
 }

ConsoleApp2/Options/CommandOptions.cs

@@ -1,21 +0,0 @@
-using System;
-using System.ComponentModel;
-using System.Configuration;
-namespace ConsoleApp2.Options
-{
-    public class CommandOptions
-    {
-
-        public string InputDir { get; set; } = "./MyDumper";
-
-        public int TaskCount { get; set; } = 16;
-
-        public int FlushCount { get; set; } = 20000;
-
-        public bool Isutf8mb4 { get; set; } = true;
-
-        public short OldestShardKey { get; set; } = 23010;
-        public string OldestTime { get; set; } = "202301";
-
-    }
-}

ConsoleApp2/Options/CsvOptions.cs

@@ -1,19 +0,0 @@
-namespace ConsoleApp2.Options;
-
-public class CsvOptions
-{
-    /// <summary>
-    /// MyDumper导出的CSV文件目录
-    /// </summary>
-    //public string InputDir { get; set; } = "./";
-
-    /// <summary>
-    /// 字符串的包围符号，默认为双引号"
-    /// </summary>
-    public char QuoteChar { get; set; } = '"';
-
-    /// <summary>
-    /// 每个字段的分割符，默认逗号,
-    /// </summary>
-    public string Delimiter { get; set; } = ",";
-}

ConsoleApp2/Options/DataInputOptions.cs

@@ -1,18 +1,53 @@
-using ConsoleApp2.Services;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Reflection.PortableExecutable;
-using System.Text;
-using System.Threading.Tasks;
+using ConsoleApp2.HostedServices;
 
 namespace ConsoleApp2.Options
 {
-    public enum InputFileType { CSV, JWT, JSV }
     public class DataInputOptions
     {
-        public string InputDir { get; set; } = "./";
+        public string? InputDir { get; set; }
 
-        public Func<string, CsvSource>? CreateSource {  get; set; }
+        #region CSV
+        
+        /// <summary>
+        /// 字符串的包围符号，默认为双引号"
+        /// </summary>
+        public char QuoteChar { get; set; } = '"';
+
+        /// <summary>
+        /// 每个字段的分割符，默认逗号,
+        /// </summary>
+        public string Delimiter { get; set; } = ",";
+
+        #endregion
+        
+        /// <summary>
+        /// yyyyMM
+        /// </summary>
+        public string CleanDate { get; set; } = "202301";
+
+        #region Mock
+
+        public bool UseMock { get; set; }
+
+        public double MockCountMultiplier { get; set; } = 1;
+        
+        /// <summary>
+        /// Table -> Mock Count 暂时为手动配置
+        /// </summary>
+        public Dictionary<string, TableMockConfig>? TableMockConfig { get; set; }
+        
+        #endregion
+
+
+        #region ManualSet
+        
+        public string[]? TableOrder { get; set; }
+        
+        /// <summary>
+        /// 配置如何从文件名转换为表名和表头
+        /// </summary>
+        public Func<string, FileInputInfo?>? FileInputMetaBuilder { get; set; } //TODO: 抽离
+
+        #endregion
     }
 }

ConsoleApp2/Options/DataTransformOptions.cs

@@ -1,5 +1,6 @@
-using Microsoft.Extensions.Caching.Distributed;
-using StackExchange.Redis;
+using ConsoleApp2.Cache;
+using ConsoleApp2.HostedServices;
+using Microsoft.Extensions.Logging;
 
 namespace ConsoleApp2.Options;
 
@@ -13,23 +14,34 @@ public enum ColumnType
 
 public class DataTransformOptions
 {
-    public Func<DataRecord, string>? DatabaseFilter { get; set; }
-
-    public Func<string, string>? TransformBinary { get; set; }//Binary转字符串方法
-
-    public Func<DataRecord, IDistributedCache, Task<bool>>? RecordFilter { get; set; }//数据过滤方法
-    public Action<DataRecord>? RecordModify { get; set; }//数据修改
-    public Func<DataRecord, IDistributedCache, Task<DataRecord?>>? RecordReplace { get; set; }//数据替换
-    public Func<DataRecord, IList<DataRecord>?>? RecordAdd { get; set; }//数据替换
-    public Func<DataRecord, IDistributedCache, Task>? RecordCache { get; set; }//数据缓存
-
+    public bool StrictMode { get; set; } = true;
+    public bool EnableFilter { get; set; } = true;
+    public bool EnableReplacer { get; set; } = true;
+    public bool EnableReBuilder { get; set; } = true;
+    
     /// <summary>
-    /// 配置导入数据的特殊列
+    /// Record -> Database name
+    /// 对记录进行数据库过滤
     /// </summary>
-    public Dictionary<string, ColumnType> ColumnTypeConfig { get; set; } = new(); // "table.column" -> type
-
-    public ColumnType GetColumnType(string table, string column)
-    {
-        return ColumnTypeConfig.GetValueOrDefault($"{table}.{column}", ColumnType.UnDefine);
-    }
-}
+    public Func<DataRecord, string>? DatabaseFilter { get; set; }
+    /// <summary>
+    /// Context -> Should output
+    /// 配置对数据过滤的条件
+    /// </summary>
+    public Func<DataTransformContext, Task<bool>>? RecordFilter { get; set; }//数据过滤方法
+    /// <summary>
+    /// Context -> New record
+    /// 对当前记录进行修改或完整替换
+    /// </summary>
+    public Func<DataTransformContext, Task<DataRecord>>? RecordModify { get; set; }//数据替换
+    /// <summary>
+    /// Context -> New rebuild records
+    /// 使用当前记录对某些数据进行重建
+    /// </summary>
+    public Func<DataTransformContext, IList<DataRecord>?>? RecordReBuild { get; set; }//新增数据
+    /// <summary>
+    /// Context -> void
+    /// 对数据的某些字段进行缓存
+    /// </summary>
+    public Func<DataTransformContext, Task>? RecordCache { get; set; }//数据缓存
+}

ConsoleApp2/Options/DatabaseOutputOptions.cs

@@ -2,9 +2,24 @@
 
 public class DatabaseOutputOptions
 {
-    /// <summary>
-    /// 数据库连接字符串
-    /// </summary>
     public string? ConnectionString { get; set; }
-    public int MaxAllowedPacket { get; set; } = 64*1024*1024;
+
+    public int MaxAllowedPacket { get; set; } = 64 * 1024 * 1024;
+
+    public int FlushCount { get; set; } = 10000;
+
+    public int MaxDatabaseOutputTask { get; set; } = 4;
+
+    public bool TreatJsonAsHex { get; set; } = true;
+    
+
+    /// <summary>
+    /// 配置导入数据的特殊列
+    /// </summary>
+    public Dictionary<string, ColumnType> ColumnTypeConfig { get; set; } = new(); // "table.column" -> type
+
+    public ColumnType GetColumnType(string table, string column)
+    {
+        return ColumnTypeConfig.GetValueOrDefault($"{table}.{column}", ColumnType.UnDefine);
+    }
 }

ConsoleApp2/Options/RedisCacheOptions.cs

@@ -0,0 +1,8 @@
+namespace ConsoleApp2.Options;
+
+public class RedisCacheOptions
+{
+    public string? Configuration { get; init; }
+    public string InstanceName { get; init; } = "";
+    public int Database { get; init; } = 0;
+}

ConsoleApp2/Options/TableMockConfig.cs

@@ -0,0 +1,32 @@
+namespace ConsoleApp2.Options;
+
+public struct TableMockConfig
+{
+    /// <summary>
+    /// 使用深拷贝
+    /// </summary>
+    public bool UseDeepCopy { get; set; }
+    /// <summary>
+    /// 模拟数据量
+    /// </summary>
+    public long MockCount { get; set; }
+
+    /// <summary>
+    /// 需要开启MockCount
+    /// </summary>
+    public string[]? AutoIncrementColumn { get; set; } = null; // TODO: 换为自定义委托
+    
+    public void Deconstruct(out bool useDeepCopy, out long mockCount, out string[]? autoIncrementColumn)
+    {
+        useDeepCopy = UseDeepCopy;
+        mockCount = MockCount;
+        autoIncrementColumn = AutoIncrementColumn;
+    }
+
+    public TableMockConfig(bool useDeepCopy, long mockCount, string[]? autoIncrementColumn)
+    {
+        UseDeepCopy = useDeepCopy;
+        MockCount = mockCount;
+        AutoIncrementColumn = autoIncrementColumn;
+    }
+}

ConsoleApp2/Options/TasksOptions.cs

@@ -1,24 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace ConsoleApp2.Options
-{
-    public class TableInfo
-    {
-        public long SimulaRowCount { get; set; }//模拟的记录条数
-    }
-    public class TasksOptions
-    {
-        public Dictionary<string, TableInfo> TableInfoConfig { get; set; } = new();
-        public int TransformTaskCount { get; set; } = 1;
-        public OutPutOptions OutPutOptions { get; set; }=new();
-    }
-    public class OutPutOptions
-    {
-        public int FlushCount { get; set; } = 10000;
-        public int OutPutTaskCount { get; set; } = 2;
-    }
-}

ConsoleApp2/Options/TenantDbOptions.cs

@@ -2,22 +2,23 @@ namespace ConsoleApp2.Options;
 
 public class TenantDbOptions
 {
-    public string TenantKey { get; set; }
+    public string? TenantKey { get; set; }
 
     /// <summary>
     /// Key-Value: {DbName}-{TenantKeyLessThan}
     /// </summary>
-    public Dictionary<string, int> DbList { get; set; }
+    public Dictionary<string, int>? DbList { get; set; }
 
     public string GetDbNameByTenantKeyValue(int tenantKeyValue)
     {
         // var dictionary = new SortedDictionary<int, string>();
         // DbList.ForEach(pair => dictionary.Add(pair.Value, pair.Key));
         // 注意配置顺序
+        if(DbList is null) throw new ApplicationException("分库配置中没有发现任何数据库");
         var dbName = DbList.Cast<KeyValuePair<string, int>?>()
             .FirstOrDefault(pair => pair?.Value != null && pair.Value.Value > tenantKeyValue)!.Value.Key;
         return dbName ??
                throw new ArgumentOutOfRangeException(nameof(tenantKeyValue),
-                   $"已配置的数据库中没有任何符合'{nameof(tenantKeyValue)}'值的对象");
+                   $"分库配置中没有任何符合'{nameof(tenantKeyValue)}'值的数据库");
     }
 }

ConsoleApp2/Program.cs

@@ -1,429 +1,509 @@
-using ConsoleApp2;
+#define USE_TEST_DB // 测试库的结构与生产库不一样，如果使用测试库运行，则加上USE_TEST_DB预处理器指令
+
+using ConsoleApp2;
+using ConsoleApp2.Cache;
 using ConsoleApp2.Const;
+using ConsoleApp2.Helpers;
 using ConsoleApp2.HostedServices;
 using ConsoleApp2.HostedServices.Abstractions;
 using ConsoleApp2.Options;
 using ConsoleApp2.Services;
-using Microsoft.Extensions.Caching.StackExchangeRedis;
+using ConsoleApp2.Services.ErrorRecorder;
+using ConsoleApp2.Services.ETL;
+using ConsoleApp2.Services.Loggers;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using MySqlConnector;
 using Serilog;
-using Microsoft.Extensions.Caching.Distributed;
 using Serilog.Events;
 
-
 await RunProgram();
 return;
 
 async Task RunProgram()
 {
-
-
-    //var inputDir= "D:\\MyDumper";
-    //ValidateConsole.ValidateInput<string>((_inputDir) =>
-    //{
-    //    if (Directory.Exists(_inputDir))
-    //     {
-    //        inputDir = _inputDir;
-    //        return true;
-    //    }
-    //    else return false;
-    //}, "请输入读取csv文件的目录(默认为当前目录下MyDumper文件夹):");
-
-    //var maxTask = 16;
-    //ValidateConsole.ValidateInput<string>((_inputDir) =>
-    //{
-
-    //    _ = int.TryParse(_inputDir.ToString(), out var _taskCount);
-    //    if (_taskCount > 0) {
-    //        maxTask = _taskCount;
-    //        return true;
-    //    }
-    //    else return false;
-    //}, "请输入执行输出的线程数量(默认为16):");
-
-    //var flushCount = 2_0000;
-    //ValidateConsole.ValidateInput<string>((_inputDir) =>
-    //{
-    //    _ = int.TryParse(_inputDir.ToString(), out var _flashCount);
-    //    if (_flashCount > 0)
-    //    {
-    //        flushCount = _flashCount;
-    //        return true;
-    //    } else return false; 
-
-    //}, "请输入单次插入的行数(默认为20000):");
-
     ThreadPool.SetMaxThreads(200, 200);
     var host = Host.CreateApplicationBuilder(args);
     host.Configuration.AddJsonFile(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "appsettings.json"), false, false);
-    var commandOptions = host.Configuration.GetSection("CmdOptions").Get<CommandOptions>() ?? 
-                         throw new ApplicationException("未发现配置文件");
-    Console.WriteLine($"InputDir:{commandOptions?.InputDir}");
-
-    var oldestTime = DateTime.ParseExact(commandOptions.OldestTime, "yyyyMM", System.Globalization.DateTimeFormatInfo.InvariantInfo);
-    //host.Services.Configure<InputTableOptions>(option =>
-    //{
-    //    option.TableInfoConfig = new Dictionary<string, TableInfo>
-    //    {
-
-    //        //order_block_plan_item从order_item表查询，然后程序插入                
-    //        //order_package_item从order_item表查询，然后程序插入
-    //        //order_patch_detail生产没有这个表，不处理
-
-
-    //        {"machine",new TableInfo{SimulaRowCount=14655 }},
-    //        {"order",new TableInfo{SimulaRowCount=5019216 }},
-    //        {"order_block_plan",new TableInfo{SimulaRowCount=2725553 }},//CreateTime < 202301的删除
-    //        {"order_block_plan_result",new TableInfo{SimulaRowCount=1174096 }},
-    //        {"order_box_block",new TableInfo{SimulaRowCount=29755672 }},
-    //        {"order_data_block",new TableInfo{SimulaRowCount=731800334 }},
-    //        {"order_data_goods",new TableInfo{SimulaRowCount=25803671 }},
-    //        {"order_data_parts",new TableInfo{SimulaRowCount=468517543 }},
-    //        {"order_item",new TableInfo{SimulaRowCount=1345520079 }},
-    //        {"order_module",new TableInfo{SimulaRowCount=103325385 }},
-    //        {"order_module_extra",new TableInfo{SimulaRowCount=54361321 }},
-    //        {"order_module_item",new TableInfo{SimulaRowCount=69173339 }},
-    //        {"order_package",new TableInfo{SimulaRowCount=16196195 }},
-    //        {"order_process",new TableInfo{SimulaRowCount=3892685 }},//orderNo < 202301的
-    //        {"order_process_step",new TableInfo{SimulaRowCount=8050349 }},//orderNo < 202301的删除
-    //        {"order_process_step_item",new TableInfo{SimulaRowCount=14538058 }},//orderNo < 202301的删除
-    //        {"order_scrap_board",new TableInfo{SimulaRowCount=123998 }},
-    //        {"process_group",new TableInfo{SimulaRowCount=1253 }},
-    //        {"process_info",new TableInfo{SimulaRowCount=7839 }},
-    //        {"process_item_exp",new TableInfo{SimulaRowCount=28 }},
-    //        {"process_schdule_capacity",new TableInfo{SimulaRowCount=39736 }},
-    //        {"process_step_efficiency",new TableInfo{SimulaRowCount=8 }},
-    //        {"report_template",new TableInfo{SimulaRowCount=7337 }},
-    //        {"simple_package",new TableInfo{SimulaRowCount=130436 }},//orderNo < 202301的删除
-    //        {"simple_plan_order",new TableInfo{SimulaRowCount=351470 }},//CreateTime < 202301的删除
-    //        {"sys_config",new TableInfo{SimulaRowCount=2296 }},
-    //        {"work_calendar",new TableInfo{SimulaRowCount=11 }},
-    //        {"work_shift",new TableInfo{SimulaRowCount=59 }},
-    //        {"work_time",new TableInfo{SimulaRowCount=62 }},
-    //    };
-    //});
-    host.Services.Configure<CsvOptions>(option =>
+    host.Configuration.AddCommandLine(args, new Dictionary<string, string>
     {
-        option.Delimiter = ",";
-        option.QuoteChar = '"';
-    });
+        {"-d", "Input:InputDir"},
+        {"--InputDir", "Input:InputDir"},
+        {"-s", "Output:ConnectionString"},
+        {"--ConnectionString", "Output:ConnectionString"},
+        {"-r", "RedisCache:Configuration"},
+        {"--Redis", "RedisCache:Configuration"},
+        {"-i", "RestoreIndex"},
+        {"--RestoreIndex", "RestoreIndex"}
+    }); 
+
+    var inputOptions = host.Configuration.GetRequiredSection("Input").Get<DataInputOptions>() 
+                       ?? throw new ApplicationException("缺少Input配置");
+
+    var transformOptions = host.Configuration.GetRequiredSection("Transform").Get<DataTransformOptions>()
+                        ?? throw new ApplicationException("缺少Transform配置");
+
+    var outputOptions = host.Configuration.GetRequiredSection("Output").Get<DatabaseOutputOptions>() 
+                        ?? throw new ApplicationException("缺少Output配置");
+
+    var redisSection = host.Configuration.GetRequiredSection("RedisCache");
+    var redisOptions = redisSection.Get<RedisCacheOptions>() ?? throw new ApplicationException("缺少RedisCache配置");
+    
+    var tenantDbSection = host.Configuration.GetRequiredSection("TenantDb");
+    var tenantDbOptions = tenantDbSection.Get<TenantDbOptions>() ?? throw new ApplicationException("缺少TenantDb配置");
+    if(tenantDbOptions.TenantKey is null) throw new ApplicationException("分库配置中缺少分库键");
+    if(tenantDbOptions.DbList is null) throw new ApplicationException("分库配置中没有发现任何数据库");
+    
+    host.Services.Configure<TenantDbOptions>(tenantDbSection);
+    host.Services.Configure<RedisCacheOptions>(redisSection);
+    
+    var oldestTime = DateTime.ParseExact(inputOptions.CleanDate, "yyyyMM", System.Globalization.DateTimeFormatInfo.InvariantInfo);
+    var oldestTimeInt = int.Parse(inputOptions.CleanDate);
+    
+    // 输入配置
     host.Services.Configure<DataInputOptions>(options =>
     {
-        options.InputDir = commandOptions.InputDir;
-        var _csvOptions = new CsvOptions { Delimiter = ",", QuoteChar = '"' };
-        options.CreateSource = (string tableName) =>
+        options.InputDir = inputOptions.InputDir ?? throw new ApplicationException("未配置输入目录");
+        options.UseMock = inputOptions.UseMock;
+        options.TableMockConfig = inputOptions.TableMockConfig;
+        options.MockCountMultiplier = inputOptions.MockCountMultiplier;
+        
+        // 配置文件输入方法
+        options.FileInputMetaBuilder = fileName =>
         {
-            var source = new ZstSource(commandOptions.InputDir, tableName, _csvOptions.Delimiter, _csvOptions.QuoteChar);
-            return source;
+            if (fileName.EndsWith(".dat.zst"))
+            {
+                var tableName = DumpDataHelper.GetTableNameFromCsvFileName(
+                    Path.GetFileNameWithoutExtension(fileName)); // 去除.zst
+                string[]? headers;
+                try
+                {
+                    // 查找同目录下同表的SQL文件
+                    var sqlFile = Directory.GetFiles(options.InputDir)
+                        .SingleOrDefault(f => f.Equals(fileName.Replace(".dat.zst",".sql.zst")));
+                    if (sqlFile is null)
+                        return null;
+                    headers = DumpDataHelper.GetCsvHeadersFromSqlFile(
+                        DumpDataHelper.DecompressZstAsStringAsync(File.OpenRead(sqlFile)).Result);
+                }
+                catch (InvalidOperationException e)
+                {
+                    throw new ApplicationException($"目录下不止一个{tableName}表的SQL文件", e);
+                }
+                
+                return new FileInputInfo
+                {
+                    FileName = fileName,
+                    TableName = tableName,
+                    Headers = headers
+                };
+            }
+            return null;
+        };
+
+        options.TableOrder =
+        [
+            TableNames.Machine,
+            
+            TableNames.Order,
+            TableNames.OrderBoxBlock,       // 依赖Order.CompanyID
+            
+            TableNames.OrderBlockPlan,      
+            TableNames.OrderBlockPlanResult,// 依赖OrderBlockPlan.CompanyID / 删除
+            
+            TableNames.OrderItem,
+            TableNames.OrderDataBlock,
+            TableNames.OrderDataGoods,
+            TableNames.OrderDataParts,
+            TableNames.OrderModule,
+            TableNames.OrderModuleExtra,
+            TableNames.OrderModuleItem,
+            TableNames.OrderPackage,
+            TableNames.OrderPatchDetail,
+            
+            TableNames.OrderProcess,
+            TableNames.OrderProcessStep,
+            TableNames.OrderProcessStepItem,// 依赖OrderProcess.ShardKey / 删除
+            
+            TableNames.OrderProcessSchedule,
+            TableNames.OrderScrapBoard,
+            TableNames.ProcessGroup,
+            TableNames.ProcessInfo,
+            TableNames.ProcessItemExp,
+            TableNames.ProcessScheduleCapacity,
+            TableNames.ProcessStepEfficiency,
+            TableNames.ReportTemplate,
+            TableNames.SimplePackage,
+            TableNames.SimplePlanOrder,
+            TableNames.SysConfig,
+            TableNames.WorkCalendar,
+            TableNames.WorkShift,
+            TableNames.WorkTime
+        ];
+
+        // options.TableMockConfig = new Dictionary<string, TableMockConfig>
+        // {
+        //     { TableNames.Machine, new TableMockConfig(true, 14655, ["ID"]) },
+        //     { TableNames.Order, new TableMockConfig(true, 5019216, ["OrderNo"]) },
+        //     { TableNames.OrderDataBlock, new TableMockConfig(true, 731800334, ["ID"]) },
+        //     { TableNames.OrderDataGoods, new TableMockConfig(true, 25803671, ["ID"]) },
+        //     { TableNames.OrderDataParts, new TableMockConfig(true, 468517543, ["ID"]) },
+        //     { TableNames.OrderModule, new TableMockConfig(true, 103325385, ["ID"]) },
+        //     { TableNames.OrderModuleExtra, new TableMockConfig(true, 54361321, ["ID"]) },
+        //     { TableNames.OrderModuleItem, new TableMockConfig(true, 69173339, ["ID"]) },
+        //     { TableNames.OrderPackage, new TableMockConfig(true, 16196195, ["ID"]) },
+        //     { TableNames.OrderProcess, new TableMockConfig(true, 3892685, ["ID"]) },
+        //     { TableNames.OrderProcessStep, new TableMockConfig(true, 8050349, ["ID"]) },
+        //     { TableNames.OrderProcessStepItem, new TableMockConfig(true, 14538058, ["ID"]) },
+        //     { TableNames.OrderScrapBoard, new TableMockConfig(true, 123998, ["ID"]) },
+        //     { TableNames.ProcessGroup, new TableMockConfig(true, 1253, ["ID"]) },
+        //     { TableNames.ProcessInfo, new TableMockConfig(true, 7839, ["ID"]) },
+        //     { TableNames.ProcessItemExp, new TableMockConfig(true, 28, ["ID"]) },
+        //     { TableNames.ProcessScheduleCapacity, new TableMockConfig(true, 39736, ["ID"]) },
+        //     { TableNames.ProcessStepEfficiency, new TableMockConfig(true, 8, ["ID"]) },
+        //     { TableNames.ReportTemplate, new TableMockConfig(true, 7337, ["ID"]) },
+        //     { TableNames.SimplePackage, new TableMockConfig(true, 130436, ["ID"]) },
+        //     { TableNames.SysConfig, new TableMockConfig(true, 2296, ["ID"]) },
+        //     { TableNames.WorkCalendar, new TableMockConfig(true, 11, ["ID"]) },
+        //     { TableNames.WorkShift, new TableMockConfig(true, 59, ["ID"]) },
+        //     { TableNames.WorkTime, new TableMockConfig(true, 62, ["ID"]) }
+        // };
+        options.TableMockConfig = new Dictionary<string, TableMockConfig>
+        {
+            { TableNames.Machine, new TableMockConfig(true, 14655, ["ID"]) },
+            { TableNames.Order, new TableMockConfig(true, 50192, ["OrderNo"]) },
+            { TableNames.OrderDataBlock, new TableMockConfig(true, 7318003, ["ID"]) },
+            { TableNames.OrderDataGoods, new TableMockConfig(true, 258036, ["ID"]) },
+            { TableNames.OrderDataParts, new TableMockConfig(true, 4685175, ["ID"]) },
+            { TableNames.OrderItem, new TableMockConfig(true, 13298896, ["ID"])},
+            { TableNames.OrderModule, new TableMockConfig(true, 1033253, ["ID"]) },
+            { TableNames.OrderModuleExtra, new TableMockConfig(true, 543613, ["ID"]) },
+            { TableNames.OrderModuleItem, new TableMockConfig(true, 691733, ["ID"]) },
+            { TableNames.OrderPackage, new TableMockConfig(true, 161961, ["ID"]) },
+            { TableNames.OrderProcess, new TableMockConfig(true, 38926, ["ID"]) },
+            { TableNames.OrderProcessStep, new TableMockConfig(true, 80503, ["ID"]) },
+            { TableNames.OrderProcessStepItem, new TableMockConfig(true, 145380, ["ID"]) },
+            { TableNames.OrderScrapBoard, new TableMockConfig(true, 1239, ["ID"]) },
+            { TableNames.ProcessGroup, new TableMockConfig(true, 125, ["ID"]) },
+            { TableNames.ProcessInfo, new TableMockConfig(true, 783, ["ID"]) },
+            { TableNames.ProcessItemExp, new TableMockConfig(true, 28, ["ID"]) },
+            { TableNames.ProcessScheduleCapacity, new TableMockConfig(true, 39736, ["ID"]) },
+            { TableNames.ProcessStepEfficiency, new TableMockConfig(true, 8, ["ID"]) },
+            { TableNames.ReportTemplate, new TableMockConfig(true, 7337, ["ID"]) },
+            { TableNames.SimplePackage, new TableMockConfig(true, 130436, ["ID"]) },
+            { TableNames.SysConfig, new TableMockConfig(true, 2296, ["Key"]) },
+            { TableNames.WorkCalendar, new TableMockConfig(true, 11, ["ID"]) },
+            { TableNames.WorkShift, new TableMockConfig(true, 59, ["ID"]) },
+            { TableNames.WorkTime, new TableMockConfig(true, 62, ["ID"]) }
         };
     });
 
     host.Services.Configure<DataTransformOptions>(options =>
     {
-        var tenantDbOptions = host.Configuration.GetRequiredSection("TenantDb").Get<TenantDbOptions>() 
-            ?? throw new ApplicationException("分库配置项不存在");
+        // 数据缓存键格式为[TableName]-[ColumnName@ColumnValue]-[CacheColumnName]
+        static string BuildCacheKey(string tableName, string columnName, string columnValue, string cacheColumnName) 
+            => $"{tableName}-{columnName}@{columnValue}-{cacheColumnName}";
+
+        static string CalculateShardKeyByOrderNo(ReadOnlySpan<char> orderNo)
+            => $"{orderNo[2..6]}0";
+        
+        options.StrictMode = transformOptions.StrictMode;
+        options.EnableFilter = transformOptions.EnableFilter;
+        options.EnableReplacer = transformOptions.EnableReplacer;
+        options.EnableReBuilder = transformOptions.EnableReBuilder;
+        
+        // order_block_plan_item和order_package_item表不导入，根据order_item数据直接重建
+        
+        // 数据清理
+        options.RecordFilter = async context =>
+        {
+            var record = context.Record;
+            var cache = context.Cacher;
+            switch (record.TableName)
+            {
+                // OrderBoxBlock删除对应Order.OrderNo不存在的对象
+                case TableNames.OrderBoxBlock:
+                {
+                    if (!await cache.ExistsAsync(CacheKeys.Order_OrderNo_CompanyID(record["OrderNo"])))
+                        return false;
+                    break;
+                }
+                // OrderBlockPlan删除CreateTime < 202301的
+                case TableNames.OrderBlockPlan:
+                {
+                    var time = DateTime.Parse(record["CreateTime"].Trim('"'));
+                    if (time < oldestTime)
+                        return false;
+                    break;
+                }
+                // OrderBlockPlanResult删除对应order_block_plan.ID不存在的对象
+                case TableNames.OrderBlockPlanResult:
+                {
+                    if (!await cache.ExistsAsync(CacheKeys.OrderBlockPlan_ID_CompanyID(record["ID"])))
+                        return false;
+                    break;
+                }
+                // OrderModule删除OrderNo < 202301的
+                case TableNames.OrderModule:
+                {
+                    var orderNo = record["OrderNo"];
+                    if(int.Parse(orderNo.AsSpan(0, 6).ToString()) < oldestTimeInt)
+                        return false;
+                    break;
+                }
+                // OrderProcess删除OrderNo < 202301的
+                case TableNames.OrderProcess:
+                {
+                    var orderNo = record["OrderNo"];
+                    if(int.Parse(orderNo.AsSpan(0, 6).ToString()) < oldestTimeInt)
+                        return false;
+                    break;
+                }
+                // OrderProcessStep删除OrderNo < 202301的
+                case TableNames.OrderProcessStep:
+                {
+                    var orderNo = record["OrderNo"];
+                    if(int.Parse(orderNo.AsSpan(0, 6).ToString()) < oldestTimeInt)
+                        return false;
+                    break;
+                }
+                // OrderProcessStepStep删除对应OrderProcess.ID不存在的对象
+                case TableNames.OrderProcessStepItem:
+                {
+                    if (!await cache.ExistsAsync(CacheKeys.OrderProcess_ID_ShardKey(record["OrderProcessID"])))
+                        return false;
+                    break;
+                }
+                // SimplePackage删除OrderNo < 202301的
+                case TableNames.SimplePackage:
+                {
+                    var orderNo = record["OrderNo"];
+                    if(int.Parse(orderNo.AsSpan(0, 6).ToString()) < oldestTimeInt)
+                        return false;
+                    break;
+                }
+                // SimplePlanOrder删除CreateTime < 202301的
+                case TableNames.SimplePlanOrder:
+                {
+                    var time = DateTime.Parse(record["CreateTime"].Trim('"'));
+                    if (time < oldestTime)
+                        return false;
+                    break;
+                }
+            }
+
+            return true;
+        };
+        
+        // 数据替换
+        options.RecordModify = async context =>
+        {
+            var record = context.Record;
+            var cache = context.Cacher;
+            switch (record.TableName)
+            {
+#if USE_TEST_DB
+                // Order表移除IsBatch列
+                case TableNames.Order:
+                    record.RemoveField("IsBatch");
+                    break;
+#endif
+                //OrderBlockPlan将OrderNo长度<2的置空
+                case TableNames.OrderBlockPlan:
+                    if (record["OrderNos"].Length < 2)
+                        record["OrderNos"] = "";
+                    break;
+                // OrderBlockPlanResult表添加CompanyID列
+                case TableNames.OrderBlockPlanResult:
+                    record.AddField("CompanyID",
+                        // 获取OrderBlockPlan.ID -> CompanyID
+                        ThrowIfNoCached(await cache.GetStringAsync(CacheKeys.OrderBlockPlan_ID_CompanyID(record["ID"])),
+                            TableNames.OrderBlockPlanResult, TableNames.OrderBlockPlan, "ID", "脏数据未处理"));
+                    break;
+                // OrderBoxBlock添加CompanyID列
+                case TableNames.OrderBoxBlock:
+                    record.AddField("CompanyID", 
+                        // 获取Order.OrderNo -> CompanyID
+                        ThrowIfNoCached(await cache.GetStringAsync(CacheKeys.Order_OrderNo_CompanyID(record["OrderNo"])),
+                            TableNames.OrderBoxBlock, TableNames.Order, "OrderNo", "脏数据未处理"));
+                    break;
+                // OrderModule添加ShardKey列，移除ViewFileName列
+                case TableNames.OrderModule:
+                    record.AddField("ShardKey", CalculateShardKeyByOrderNo(record["OrderNo"]));
+                    record.RemoveField("ViewFileName");
+                    break;
+                // OrderProcess添加ShardKey列，NextStepID的空值转换为0
+                case TableNames.OrderProcess:
+                    record.AddField("ShardKey", CalculateShardKeyByOrderNo(record["OrderNo"]));
+#if USE_TEST_DB
+                    if(record["NextStepID"] == "\\N")
+                        record["NextStepID"] = "0";
+#endif
+                    break;
+                // OrderProcessStep，OrderProcessStepItem添加ShardKey列
+                case TableNames.OrderProcessStep:
+                    record.AddField("ShardKey", CalculateShardKeyByOrderNo(record["OrderNo"]));
+                    break;
+                case TableNames.OrderProcessStepItem:
+                    record.AddField("ShardKey", 
+                        // 获取OrderProcess.ID -> ShardKey
+                    ThrowIfNoCached(await cache.GetStringAsync(CacheKeys.OrderProcess_ID_ShardKey(record["OrderProcessID"])), 
+                        TableNames.OrderProcessStepItem, TableNames.OrderProcessStep, "OrderProcessID", "脏数据未处理"));
+                    break;
+                case TableNames.SimplePlanOrder:
+#if USE_TEST_DB
+                    record.RemoveField("ProcessState");
+#endif
+                    record.AddField("Deleted", "0");
+                    break;
+            }
+
+            return record;
+
+            string ThrowIfNoCached(string? cached, string tableName, string cachedTableName, string cachedColumn, string appendMessage = "")
+            {
+                if (cached is null)
+                    throw new InvalidDataException(
+                        $"{tableName}数据异常，在缓存中未找到对应{cachedTableName}.{cachedColumn}\t{appendMessage}");
+                return cached;
+            }
+        };
+        
+        // 数据缓存
+        options.RecordCache = async context =>
+        {
+            var record = context.Record;
+            var cache = context.Cacher;
+            switch (record.TableName)
+            {
+                // 缓存Order.OrderNo -> CompanyID
+                case TableNames.Order:
+                    CacheKeys.Order_OrderNo_CompanyID = orderNo =>
+                        BuildCacheKey(TableNames.Order, "OrderNo", orderNo, "CompanyID");
+                    await cache.SetStringAsync(
+                        CacheKeys.Order_OrderNo_CompanyID(record["OrderNo"]),
+                        record["CompanyID"]);
+                    break;
+                
+                // 缓存OrderBlockPlan.ID -> CompanyID
+                case TableNames.OrderBlockPlan:
+                    CacheKeys.OrderBlockPlan_ID_CompanyID = id =>
+                        BuildCacheKey(TableNames.OrderBlockPlan, "ID", id, "CompanyID");
+                    await cache.SetStringAsync(
+                        CacheKeys.OrderBlockPlan_ID_CompanyID(record["ID"]),
+                        record["CompanyID"]);
+                    break;
+                
+                // 缓存OrderProcess.ID -> ShardKey
+                case TableNames.OrderProcess:
+                    CacheKeys.OrderProcess_ID_ShardKey = id =>
+                        BuildCacheKey(TableNames.OrderProcess, "ID", id, "ShardKey");
+                    await cache.SetStringAsync(
+                        CacheKeys.OrderProcess_ID_ShardKey(record["ID"]),
+                        record["ShardKey"]);
+                    break;
+            }
+        };
+        
+        // 数据库过滤
         options.DatabaseFilter = record =>
         {
             var companyId = int.Parse(record[tenantDbOptions.TenantKey]); // 每个实体都应存在CompanyID，否则异常
             return tenantDbOptions.GetDbNameByTenantKeyValue(companyId);
         };
-
-        options.TransformBinary = field => commandOptions != null && commandOptions.Isutf8mb4 ? $"_utf8mb4 0x{field}" : $"0x{field}";
-        //数据过滤
-        options.RecordFilter = async (record, cache) =>
-        {
-            //var index = Array.IndexOf(record.Headers, "ShardKey");
-            if (record.TryGetField("ShardKey", out var skStr))
-            {
-                short.TryParse(skStr, out var sk);
-                if (sk < commandOptions.OldestShardKey) return false;
-            }
-
-            if (record.TryGetField("CreateTime", out var createTime))
-            {
-                _ = DateTime.TryParse(createTime.Replace("\"", ""), out var time);
-                if (time < oldestTime) return false;
-            }
-            if (record.TryGetField("OrderNo", out var orderNo))
-            {
-                try
-                {
-                    var yearMonth = orderNo.Substring(0, 6);
-
-                    var dt = DateTime.ParseExact(yearMonth, "yyyyMM", System.Globalization.DateTimeFormatInfo.InvariantInfo);
-                    if (dt < oldestTime) return false;
-
-                }
-                catch (Exception ex)
-                {
-                    return false;//订单号转换失败，跳过
-
-                }
-                
-            }
-
-            if (record.TableName == "order_package")
-            {
-                if (record.TryGetField("PakageNo", out var pkNo))
-                {
-                    if (pkNo.Length <= 2) return false;
-                }
-            }
-            if (record.TableName == "order_block_plan")
-            {
-                if (record.TryGetField("OrderNos", out var nos))
-                {
-                    if (nos.Length <= 2) return false;
-                }
-              
-            }
-            if (record.TableName == "order_process_step" || record.TableName == "order_process_step_item")
-            {
-                //如果缓存中不存在OrderProcessID,则丢弃
-                //if(record.TryGetField("OrderProcessID",out var orderProcessID))
-
- 
-                    var value = await cache.GetStringAsync(record.GetCacheKey("OrderProcessID"));
-                    if (string.IsNullOrEmpty(value))return false;
-                
-            }
-            if (record.TableName == "order_block_plan_result" )
-            {
-                //如果缓存中不存在ID,则丢弃(ID 对应order_block_plan中的ID)
-
-                    var value = await cache.GetStringAsync(record.GetCacheKey("ID"));
-                    if (string.IsNullOrEmpty(value)) return false;
-                
-            }
-            return true;
-
-        };
-        //数据修改
-        options.RecordModify = (record) =>
-        {
-            if (record.TableName == "order_block_plan")
-            {
-                if (record.TryGetField("OrderNos", out var nos))
-                {
-                    if (nos.Length <= 2) record.SetField("OrderNos", "");
-                }
-
-            }
-            if (record.TableName == "order_process")//修改order_process.NextStepID的默认值为0
-            {
-                 
-                if (record.TryGetField("NextStepID", out var idStr))
-                {
-
-                    if (idStr == "\\N")
-                    {
-                        record.SetField("NextStepID", "0");
-                    }
-                }
-            }
-
-        };
-        //数据缓存
-        options.RecordCache = async (record, cache) =>
-        {
-            if (record.TableName == "order")
-            {
-                if (record.TryGetField("OrderNo", out var orderNo))
-                {
-                    if (record.TryGetField("CompanyID", out var companyid))
-                    {
-                        await cache.SetStringAsync(record.GetCacheKey("OrderNo"), companyid);
-                    }
-                    
-                }
-            }
-            if (record.TableName == "order_process")
-            {
-                if (record.TryGetField("OrderNo", out var orderNo))
-                {
-                    var yearMonth = orderNo.Substring(2, 4);
-                    var sk = yearMonth + "0";
-                   
-                    if( record.TryGetField("ID", out var id))
-                    {
-                        await cache.SetStringAsync(record.GetCacheKey("ID"), sk);
-                    }
-          
-                }
-            }
-            if (record.TableName == "order_block_plan")
-            {
-                if (record.TryGetField("CompanyID", out var companyid))
-                {
-                    record.TryGetField("ID", out var id);
-                    await cache.SetStringAsync(record.GetCacheKey("ID"), companyid);
-                }
-            }
-        };
-        //数据替换
-        options.RecordReplace = async (record, cache) =>
-        {
-            //删除数据源里simple_plan_order.ProcessState 字段和值
-
-            if (record.TableName == "simple_plan_order")//修改order_process.NextStepID的默认值为0
-            {
-                var nextStepIdIndex = Array.IndexOf(record.Headers, "ProcessState");
-                if (nextStepIdIndex > -1)
-                {
-                    var headers = record.Headers.Where(t => t != "ProcessState").ToArray();
-                    var fs = record.Fields.ToList();
-                    fs.RemoveAt(nextStepIdIndex);
-                    var fields = fs.ToArray();
-                    return new DataRecord(fields, record.TableName, headers, record.CompanyID);
-                }
-            }
-            if (record.TableName == "order")//修改order_process.NextStepID的默认值为0
-            {
-                var nextStepIdIndex = Array.IndexOf(record.Headers, "IsBatch");
-                if (nextStepIdIndex > -1)
-                {
-                    var headers = record.Headers.Where(t => t != "IsBatch").ToArray();
-                    var fs = record.Fields.ToList();
-                    fs.RemoveAt(nextStepIdIndex);
-                    var fields = fs.ToArray();
-                    return new DataRecord(fields, record.TableName, headers, record.CompanyID);
-                }
-            }
-            if (record.TableName == "order_block_plan_result")//修改order_process.NextStepID的默认值为0
-            {
-                if (record.TryGetField("ID", out var id))
-                {
-                    var headers = new List<string>(record.Headers);
-                    var fields =new List<string>(record.Fields);
-                    headers.Add("CompanyID");
-                    var companyidResult =await cache.GetStringAsync(record.GetCacheKey("ID"));
-
-                    _ = int.TryParse(companyidResult, out var companyid);
-                    fields.Add(companyid.ToString());
-                    return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), companyid);
-                }
-            }
-            if(record.TableName == "order_box_block")
-            {
-                if (!record.TryGetField("CompanyID", out var companyid))
-                {
-                    if (record.TryGetField("OrderNo", out var orderNo))
-                    {
-                        var headers = new List<string>(record.Headers);
-                        var fields = new List<string>(record.Fields);
-                        headers.Add("CompanyID");
-                        var companyidResult = await cache.GetStringAsync(record.GetCacheKey("OrderNo"));
-                        _ = int.TryParse(companyidResult, out var cpid);
-                        fields.Add(cpid.ToString());
-                        return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), cpid);
-                    }
-                }
-            }
-            if (record.TableName == "order_module")
-            {
-                if (record.TryGetField("ViewFileName",out var value))
-                {
-                    var index=Array.IndexOf(record.Headers, "ViewFileName");
-                    var headers = new List<string>(record.Headers);
-                    headers.RemoveAt(index); 
-                    var fields = new List<string>(record.Fields);
-                    fields.RemoveAt(index);
-                    return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), record.CompanyID);
-                }
-                
-                if (!record.TryGetField("ShardKey", out var skStr))
-                {
-                    if (record.TryGetField("OrderNo", out var orderNo))
-                    {
-                        var yearMonth = orderNo.Substring(2, 4);
-                        var sk = yearMonth + "0";
-                        var headers = new List<string>(record.Headers);
-                        var fields = new List<string>(record.Fields);
-                        headers.Add("ShardKey");
-                        fields.Add(sk);
-                        return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), record.CompanyID);
-                    }
-                }
-            }
-            if (record.TableName == "order_process")
-            {
-                if (!record.TryGetField("ShardKey", out var skStr))
-                {
-                    if(record.TryGetField("OrderNo", out var orderNo))
-                    {
-                        var yearMonth = orderNo.Substring(2, 4);
-                        var sk = yearMonth + "0";
-                        var headers = new List<string>(record.Headers);
-                        var fields = new List<string>(record.Fields);
-                        headers.Add("ShardKey");
-                        fields.Add(sk);
-                        return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), record.CompanyID);
-                    }
-                }
-            }
-            if(record.TableName == "order_process_step"|| record.TableName == "order_process_step_item")
-            {
-                if (!record.TryGetField("ShardKey",out var sk))
-                {
-                    if (record.TryGetField("OrderProcessID",out var processID))
-                    {
-                        var shardKey =await cache.GetStringAsync(record.GetCacheKey("OrderProcessID"));
-                        var headers = new List<string>(record.Headers);
-                        var fields = new List<string>(record.Fields);
-                        headers.Add("ShardKey");
-                        fields.Add(shardKey??"0");
-                        return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), record.CompanyID);
-                    }
-                }
-            }
-            
-            // 旧数据库order_data_block.CompanyID数据有误，重新计算
-            if (record.TableName == "order_data_block")  
-            {
-                var companyID = await cache.GetStringAsync($"order_{record["OrderNo"]}");
-                if (string.IsNullOrEmpty(companyID))
-                    // throw new InvalidDataException($"没有获取到OrderNo={record["OrderNo"]}的CompanyID");
-                    return record;
-                record["CompanyID"] = companyID;
-                return record;
-            }
-            return null;
-        };
-        //数据生成
-        options.RecordAdd = (record) =>
+        
+        // 数据重建
+        options.RecordReBuild = context =>
         {
+            var record = context.Record;
             var resultList = new List<DataRecord>();
-            if (record.TableName == "order_item")
+            // 分流OrderItem表
+            if (record.TableName == TableNames.OrderItem)
             {
-                record.TryGetField("ID", out var itemID);
+                record.TryGetField("ID", out var itemId);
                 record.TryGetField("ShardKey", out var shardKey);
-                record.TryGetField("PlanID", out var planID);
-                record.TryGetField("PackageID", out var packageID);
-                record.TryGetField("CompanyID", out var companyID);
-                _=int.TryParse(planID, out var pid);
+                record.TryGetField("PlanID", out var planId);
+                record.TryGetField("PackageID", out var packageId);
+                record.TryGetField("CompanyID", out var companyId);
+                if(!int.TryParse(planId, out var pid))
+                    throw new ApplicationException($"数据发生异常：OrderItem.PlanID，值: {planId}"); 
                 if (pid > 0)
                 {
-                    resultList.Add(new DataRecord(new[] { itemID, shardKey, planID, companyID },
-                        "order_block_plan_item",
-                        new[] { "ItemID", "ShardKey", "PlanID", "CompanyID" }));
+                    resultList.Add(new DataRecord(new[] { itemId, shardKey, planId, companyId },
+                        TableNames.OrderBlockPlanItem,
+                        ["ItemID", "ShardKey", "PlanID", "CompanyID"]
+                        ));
                 }
-                _ = int.TryParse(packageID, out var pkid);
+                if(!int.TryParse(packageId, out var pkid))
+                    throw new ApplicationException($"数据发生异常：OrderItem.PackageID，值: {packageId}"); 
                 if(pkid > 0)
                 {
-                    resultList.Add(new DataRecord(new[] { itemID, shardKey, packageID, companyID },
-                        "order_package_item",
-                        new[] { "ItemID", "ShardKey", "PackageID", "CompanyID" }
-                        ));
+                    resultList.Add(new DataRecord(new[] { itemId, shardKey, packageId, companyId },
+                        TableNames.OrderPackageItem,
+                        [ "ItemID", "ShardKey", "PackageID", "CompanyID" ]
+                    ));
                 }
             }
             return resultList;
 
         };
-        options.ColumnTypeConfig = new()
+    });
+    
+    host.Services.Configure<DatabaseOutputOptions>(options =>
+    {
+        options.ConnectionString = new MySqlConnectionStringBuilder(outputOptions.ConnectionString ?? throw new ApplicationException("未配置数据库连接字符串"))
+        {
+            CharacterSet = "utf8mb4",
+            AllowUserVariables = true,
+            IgnoreCommandTransaction = true,
+            TreatTinyAsBoolean = false,
+            ConnectionTimeout = 60,
+            DefaultCommandTimeout = 0,
+            SslMode = MySqlSslMode.None,
+        }.ConnectionString;
+        options.FlushCount = outputOptions.FlushCount;
+        options.MaxAllowedPacket = outputOptions.MaxAllowedPacket;
+        options.MaxDatabaseOutputTask = outputOptions.MaxDatabaseOutputTask;
+        options.TreatJsonAsHex = outputOptions.TreatJsonAsHex;
+
+#if USE_TEST_DB
+        // Test Server
+         options.ColumnTypeConfig = new Dictionary<string, ColumnType>
+         {
+             { "simple_plan_order.PlaceData", ColumnType.Blob },
+             { "order_block_plan_result.PlaceData", ColumnType.Blob },
+             { "order_box_block.Data", ColumnType.Blob },
+             { "order_data_goods.ExtraProp", ColumnType.Json },
+             { "order_module_extra.JsonStr", ColumnType.Text },
+             { "process_info.Users", ColumnType.Text },
+             { "order_process_schdule.CustomOrderNo", ColumnType.Text },
+             { "order_process_schdule.OrderProcessStepName", ColumnType.Text },
+             { "order_process_schdule.AreaName", ColumnType.Text },
+             { "order_process_schdule.ConsigneeAddress", ColumnType.Text },
+             { "order_process_schdule.ConsigneePhone", ColumnType.Text },
+             { "report_source.Sql", ColumnType.Text },
+             { "report_source.KeyValue", ColumnType.Text },
+             { "report_source.Setting", ColumnType.Text },
+             { "order_data_block.RemarkJson", ColumnType.Text },
+             { "order_patch_detail.BlockDetail", ColumnType.Json },
+             { "order_scrap_board.OutLineJson", ColumnType.Text },
+             { "simple_package.Items", ColumnType.Json },
+             { "order_batch_pack_config.Setting", ColumnType.Text },
+             { "machine.Settings", ColumnType.Text },
+             { "sys_config.Value", ColumnType.Text },
+             { "sys_config.JsonStr", ColumnType.Text },
+             { "process_item_exp.ItemJson", ColumnType.Text },
+             { "report_template.Template", ColumnType.Text },
+             { "report_template.SourceConfig", ColumnType.Text },
+             { "order_block_plan.OrderNos", ColumnType.Json },
+             { "order_block_plan.BlockInfo", ColumnType.Text },
+         };
+#else
+        // 配置列类型
+        // Prod server
+        options.ColumnTypeConfig = new Dictionary<string, ColumnType>
         {
             { "simple_plan_order.PlaceData", ColumnType.Blob },
             { "order_block_plan_result.PlaceData", ColumnType.Blob },
@@ -453,26 +533,7 @@ async Task RunProgram()
             { "order_block_plan.OrderNos", ColumnType.Json },
             { "order_block_plan.BlockInfo", ColumnType.Text },
         };
-    });
-    host.Services.Configure<DatabaseOutputOptions>(options =>
-    {
-        //options.ConnectionString = new MySqlConnectionStringBuilder
-        //{
-        //    Server = "127.0.0.1",
-        //    Port = 33309,
-        //    Database = "cferp_test",
-        //    UserID = "root",
-        //    Password = "123456",
-        //    MaximumPoolSize = 50, // 这个值应当小于 max_connections
-        //}.ConnectionString;
-        options.ConnectionString = new MySqlConnectionStringBuilder(host.Configuration.GetConnectionString("MySqlMaster"))
-        {
-            CharacterSet = "utf8",
-            AllowUserVariables = true,
-            IgnoreCommandTransaction = true,
-            TreatTinyAsBoolean = false,
-            MaximumPoolSize = 50
-        }.ConnectionString;
+#endif
     });
     
     host.Services.AddLogging(builder =>
@@ -480,29 +541,26 @@ async Task RunProgram()
         builder.ClearProviders();
         builder.AddSerilog(new LoggerConfiguration()
             .WriteTo.Console()
-            .WriteTo.File("./log/error.log", restrictedToMinimumLevel:LogEventLevel.Error)
-            // .WriteTo.File("./log/info.log", restrictedToMinimumLevel:LogEventLevel.Information) //性能考虑暂不使用
+            .WriteTo.File($"./Log/Error/{ErrorRecorder.UID}.log", restrictedToMinimumLevel:LogEventLevel.Error)
+            // .WriteTo.File("./Log/Info/{ErrorRecorder.UID}.log", restrictedToMinimumLevel:LogEventLevel.Information) //性能考虑暂不使用
             .CreateLogger()
         );
     });
 
-    host.Services.AddScoped<ProcessContext>();
-    host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Producer);
-    host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Consumer);
-    host.Services.AddTransient<TaskManager>();
-    host.Services.AddSingleton<ErrorRecorder>();
+    host.Services.AddDataSourceFactory();
+    host.Services.AddErrorRecorderFactory();
+    host.Services.AddSingleton<ProcessContext>();
+    host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Produce);
+    host.Services.AddRecordQueuePool(tenantDbOptions.DbList.Keys.Select(key => (key:key, queue:new DataRecordQueue(500_000))).ToArray());
+    host.Services.AddSingleton<ITaskMonitorLogger, CacheTaskMonitorLogger>();
+    host.Services.AddSingleton<ITaskMonitorLogger, LoggerTaskMonitorLogger>();
 
     host.Services.AddHostedService<MainHostedService>();
     host.Services.AddHostedService<TaskMonitorService>();
-    host.Services.AddSingleton<IInputService, InputService>();
+    host.Services.AddSingleton<IInputService, FileInputService>();
     host.Services.AddSingleton<ITransformService, TransformService>();
     host.Services.AddSingleton<IOutputService, OutputService>();
-    var redisOptions = host.Configuration.GetSection("RedisCacheOptions").Get<RedisCacheOptions>() ?? new RedisCacheOptions();
-    host.Services.AddStackExchangeRedisCache(options =>
-    {
-        options.Configuration = redisOptions.Configuration;
-        options.InstanceName = redisOptions.InstanceName;
-    });
+    host.Services.AddRedisCache(redisOptions);
     var app = host.Build();
     await app.RunAsync();
 }

ConsoleApp2/README.md

@@ -1,49 +0,0 @@
-## 说明
-使用该程序来对MyDumper导出的CSV数据进行读取，转换，然后导出到其他数据库中。
-
-1. 用MyDumper从数据库导出CSV数据  
-
-   使用MyDumper Docker镜像
-    ```sh
-    docker run --rm --net=host -v D:/Dump:/home/backup mydumper/mydumper:v0.15.2-6 mydumper `
-   -h 127.0.0.1 -P 33306 -u root -p 123456 `
-   -B cferp_test --no-schemas --csv --hex-blob `
-   -o /home/backup
-    ```
-   将挂载卷，数据库连接和输出目录替换  
-   不导出数据库结构(--no-schemas)，
-   导出完的目录下应当包含.sql文件以及.dat文件
-
-2. 在Program.cs中修改`CsvOptions`配置
-   ```cs
-   host.Services.Configure<CsvOptions>(option =>
-    {
-        option.Delimiter = ",";
-        option.QuoteChar = '"';
-        option.InputDir = "D:/Dump/Test";
-    });
-   ```
-   将`option.InputDir`配置为MyDumper导出的数据目录
-
-3. 在Program.cs中修改`DatabaseOutputOptions`配置
-   ```cs
-   host.Services.Configure<DatabaseOutputOptions>(options =>
-    {
-        options.ConnectionString = new MySqlConnectionStringBuilder
-        {
-            Server = "127.0.0.1",
-            Port = 33306,
-            Database = "cferp_test_1",
-            UserID = "root",
-            Password = "123456",
-            MaximumPoolSize = 50, 
-        }.ConnectionString;
-        options.MaxTask = 16;
-        options.FlushCount = 200;
-    });
-   ```
-   将`MySqlConnectionStringBuilder`的属性修改为程序要导出至的数据库
-   > 后续将这些配置通过命令行传递
-
-4. 运行程序
-   > 注意，测试数据库`cferp_test`中的`order_process_step`表存在外键，如果要导出到和测试库同结构的数据库，记得先把外键删除。

ConsoleApp2/RestoreIndex.sql

@@ -0,0 +1,73 @@
+CREATE INDEX `idx_CompanyID` ON `machine` (`CompanyID`);
+
+CREATE INDEX `idx_companyid` ON `order` (`CompanyID`);
+
+CREATE INDEX `idx_CompanyID` ON `order_block_plan` (`CompanyID`);
+
+CREATE INDEX `idx_PlanID` ON `order_block_plan_item` (`PlanID`);
+
+CREATE INDEX `idx_orderno` ON `order_box_block` (`OrderNo`);
+
+CREATE INDEX `index_OrderNo` ON `order_data_block` (`OrderNo`);
+
+CREATE INDEX `index_OrderNo` ON `order_data_goods` (`OrderNo`);
+
+CREATE INDEX `index_OrderNo` ON `order_data_parts` (`OrderNo`);
+
+CREATE INDEX `index_ItemNo` ON `order_item` (`ItemNo`);
+
+CREATE INDEX `index_OrderNo` ON `order_item` (`OrderNo`);
+
+CREATE INDEX `index_PackageID` ON `order_item` (`PackageID`);
+
+CREATE INDEX `index_PlanID` ON `order_item` (`PlanID`);
+
+CREATE INDEX `idx_OrderNo` ON `order_module` (`OrderNo`);
+
+CREATE INDEX `index_OrderNo` ON `order_module_extra` (`OrderNo`);
+
+CREATE INDEX `index_OrderNo` ON `order_module_item` (`OrderNo`);
+
+CREATE INDEX `idx_OrderNo` ON `order_package` (`OrderNo`);
+
+CREATE INDEX `idx_PakageNo` ON `order_package` (`PakageNo`);
+
+CREATE INDEX `idx_PackageID` ON `order_package_item` (`PackageID`);
+
+CREATE INDEX `idx_companyid` ON `order_patch_detail` (`CompanyID`);
+
+CREATE INDEX `idx_OrderNo` ON `order_process` (`OrderNo`);
+
+CREATE INDEX `index_CompanyID` ON `order_process_schdule` (`CompanyID`);
+
+CREATE INDEX `IX_order_process_step_OrderProcessID` ON `order_process_step` (`OrderProcessID`);
+
+CREATE INDEX `idx_OrderProcessID` ON `order_process_step_item` (`OrderProcessID`);
+
+CREATE INDEX `idx_OrderProcessStepID` ON `order_process_step_item` (`OrderProcessStepID`);
+
+CREATE INDEX `idx_CompanyID` ON `order_scrap_board` (`CompanyID`);
+
+CREATE INDEX `idx_CompanyID` ON `process_group` (`CompanyID`);
+
+CREATE INDEX `idx_CompanyID` ON `process_info` (`CompanyID`);
+
+CREATE INDEX `index_CompanyID` ON `process_item_exp` (`CompanyID`);
+
+CREATE INDEX `idx_CompanyID` ON `process_schdule_capacity` (`CompanyID`);
+
+CREATE INDEX `idx_CompanyID` ON `process_step_efficiency` (`CompanyID`);
+
+CREATE INDEX `idx_CompanyID` ON `report_template` (`CompanyID`);
+
+CREATE INDEX `indx_OrderNo` ON `simple_package` (`OrderNo`);
+
+CREATE INDEX `idx_CompanyID` ON `simple_plan_order` (`CompanyID`);
+
+CREATE INDEX `idx_CompanyID` ON `sys_config` (`CompanyID`);
+
+CREATE INDEX `idx` ON `work_calendar` (`CompanyID`);
+
+CREATE INDEX `idx_CompanyID` ON `work_shift` (`CompanyID`);
+
+CREATE INDEX `IX_work_time_ShiftID` ON `work_time` (`ShiftID`);

ConsoleApp2/Services/CsvSource.cs

@@ -1,184 +0,0 @@
-using System.Reflection.PortableExecutable;
-using System.Text;
-using System.Text.RegularExpressions;
-using ConsoleApp2.Helpers;
-using ConsoleApp2.HostedServices.Abstractions;
-using Microsoft.Extensions.Logging;
-
-namespace ConsoleApp2.Services;
-
-/// <summary>
-/// CSV文件读取
-/// </summary>
-public class CsvSource:IDataSource
-{
-    protected readonly string _inputDir;
-    //protected readonly StreamReader _reader;
-    private readonly ILogger? _logger;
-    protected readonly string _tableName;
-    protected  string? _sqlFilePath;
-    protected readonly string? _sqlFileText;
-    protected string[]? headers;
-    protected string[]? csvFiles;
-    public string? CurrentRaw { get; protected set; }
-    public string Delimiter { get; private set; }
-    public char QuoteChar { get; private set; }
-    public CsvSource(string inputDir,string tableName,string delimiter = ",", char quoteChar = '"',
-        ILogger? logger = null)
-    {
-        _inputDir = inputDir;
-        _tableName = tableName;
-        _logger = logger;
-        Delimiter = delimiter;
-        QuoteChar = quoteChar;
-        string pattern = $"^.*\\.{tableName}\\..*\\.sql$";
-        _sqlFilePath = Directory.GetFiles(_inputDir).FirstOrDefault(s => Regex.Match(s, pattern).Success);
-
-
-    }
-
-
-    public string[] ParseRow(string row, char quoteChar, string delimiter)
-    {
-        var span = row.AsSpan();
-        var result = new List<string>();
-
-        if (span.Length == 0)
-            throw new ArgumentException("The row is empty", nameof(row));
-
-        var isInQuote = span[0] == quoteChar;
-        var start = 0;
-        for (var i = 1; i < span.Length; i++)
-        {
-            if (span[i] == quoteChar)
-            {
-                isInQuote = !isInQuote;
-            }
-            // delimiter需要足够复杂
-            else if (/*!isInQuote && */span.Length > i + delimiter.Length && span[i..(i + delimiter.Length)].Equals(delimiter, StringComparison.CurrentCulture)) // field matched
-            {
-                string field;
-                if (span[start] == quoteChar && span[i - 1] == quoteChar) // enclosed by quoteChar
-                    field = span[(start + 1)..(i - 1)].ToString(); // escape quoteChar
-                else
-                    field = span[start..i].ToString();
-
-                start = i + delimiter.Length;
-
-                if (field == "\\N")
-                    field = "NULL";
-
-                result.Add(field);
-                
-                continue;
-            }
-        }
-
-        result.Add(span[start..].ToString());
-
-
-        for (var i = 0; i < result.Count; i++)
-        {
-            var field = result[i];
-            if (DumpDataHelper.CheckHexField(field) && StringExtensions.CheckJsonHex(field))
-            {
-                result[i] = StringExtensions.FromHex(field);
-            }
-        }
-
-        return result.ToArray();
-    }
-    
-    public string[] ParseRow2(ReadOnlySpan<char> source, char quoteChar, string delimiter)
-    {
-        var result = new List<string>();
-        var index = -1;
-        StringBuilder current = new StringBuilder();
-        bool hasQuote = false;
-        bool hasSlash = false;
-        while (index < source.Length-1)
-        {
-            index++;
-            if (hasSlash == false && source[index] == '\\')
-            {
-                hasSlash = true;
-                current.Append('\\');
-                continue;
-            }
-            if (hasSlash ==false && source[index] == quoteChar)
-            {
-                hasQuote = !hasQuote;
-                current.Append(source[index]);
-                continue;
-            }
-            if (hasQuote==false && source[index] == delimiter[0])
-            {
-                result.Add(current.ToString());
-                current.Clear();
-            }
-            else
-            {
-                current.Append(source[index]);
-            }
-
-            hasSlash = false;
-        }
-
-        result.Add(current.ToString());
-        return result.ToArray();
-    }
-    public virtual async Task GetHeaderAndCsvFiles()
-    {
-        var text = await File.ReadAllTextAsync(_sqlFilePath);
-        headers = await DumpDataHelper.GetCsvHeadersFromSqlFileAsync(text);
-        csvFiles = await DumpDataHelper.GetCsvFileNamesFromSqlFileAsync(text, new Regex(@"'.+\.dat.zst'"));
-
-    }
-    public virtual async Task DoEnqueue(Action<DataRecord> action)
-    {
-        await GetHeaderAndCsvFiles();
-        foreach (var file in csvFiles)
-        {
-            var filePath= Path.Combine(_inputDir, file);
-            using (var fs = File.OpenRead(filePath))
-            {
-                using (StreamReader sr = new StreamReader(fs))
-                {
-                    while (!sr.EndOfStream)
-                    {
-                        var line = await sr.ReadLineAsync();
-                        var fields = ParseRow2(line, QuoteChar, Delimiter);
-                        var record = new DataRecord(fields, _tableName, headers);
-                        action?.Invoke(record);
-                    }
-                }
-            }
-
-          
-        }
-    }
-    public virtual async Task<DataRecord?> GetTestRecord()
-    {
-        await GetHeaderAndCsvFiles();
-        var file = csvFiles.FirstOrDefault();
-        if (file != null)
-        {
-            var filePath = Path.Combine(_inputDir, file);
-            using (var fs = File.OpenRead(filePath))
-            {
-                using (StreamReader sr = new StreamReader(fs))
-                {
-                    var line = await sr.ReadLineAsync();
-                    var fields = ParseRow2(line, QuoteChar, Delimiter);
-                    var record = new DataRecord(fields, _tableName, headers);
-                    return record;
-                }
-            } 
-        }
-        return null;
-    }
-    public void Dispose()
-    {
-       // _reader.Dispose();
-    }
-}

ConsoleApp2/Services/DataRecordQueue.cs

@@ -17,9 +17,13 @@ public class DataRecordQueue : IDisposable
     public event Action? OnRecordWrite;
     public event Action? OnRecordRead;
 
-    public DataRecordQueue()
+    public DataRecordQueue() : this(1000000) // 默认容量最大1M
     {
-        _queue = new BlockingCollection<DataRecord>(2000_000); // 队列最长为20W条记录
+    }
+    
+    public DataRecordQueue(int boundedCapacity)
+    {
+        _queue = new BlockingCollection<DataRecord>(boundedCapacity); 
     }
     
     public bool TryDequeue([MaybeNullWhen(false)] out DataRecord record)

ConsoleApp2/Services/ETL/CsvReader.cs

@@ -0,0 +1,106 @@
+using System.Text;
+using ConsoleApp2.HostedServices.Abstractions;
+using Microsoft.Extensions.Logging;
+
+namespace ConsoleApp2.Services.ETL;
+
+/// <summary>
+/// CSV文件读取
+/// </summary>
+public class CsvReader : IDataReader
+{
+    protected readonly string? FilePath;
+    protected readonly Lazy<StreamReader> Reader;
+    protected readonly ILogger? Logger;
+    protected readonly string TableName;
+
+    public DataRecord Current { get; protected set; } = null!;
+    public string[] Headers { get; }
+    public string? CurrentRaw { get; protected set; }
+    public string Delimiter { get; }
+    public char QuoteChar { get; }
+    
+    public CsvReader(Stream stream, string tableName, string[] headers, string delimiter = ",", char quoteChar = '"', ILogger? logger = null)
+    : this(tableName, headers, delimiter, quoteChar, logger)
+    {
+        Reader = new Lazy<StreamReader>(() => new StreamReader(stream));
+    }
+
+    public CsvReader(string filePath, string tableName, string[] headers, string delimiter = ",", char quoteChar = '"', ILogger? logger = null)
+    : this(tableName, headers, delimiter, quoteChar, logger)
+    {
+        var fs = File.OpenRead(filePath);
+        FilePath = filePath;
+        Reader = new Lazy<StreamReader>(() => new StreamReader(fs));
+    }
+
+    private CsvReader(string tableName, string[] headers, string delimiter = ",", char quoteChar = '"', ILogger? logger = null)
+    {
+        TableName = tableName;
+        Headers = headers;
+        Logger = logger;
+        Delimiter = delimiter;
+        QuoteChar = quoteChar;
+        Reader = null!;
+    }
+
+    public virtual async ValueTask<bool> ReadAsync()
+    {
+        var str = await Reader.Value.ReadLineAsync();
+        if (string.IsNullOrWhiteSpace(str))
+            return false;
+
+        CurrentRaw = str;
+
+        var fields = ParseRow(str, QuoteChar, Delimiter);
+        Current = new DataRecord(fields, TableName, Headers){RawField = str};
+        return true;
+    }
+
+    public string[] ParseRow(ReadOnlySpan<char> source, char quoteChar, string delimiter)
+    {
+        var result = new List<string>();
+        var index = -1;
+        var current = new StringBuilder();
+        var hasQuote = false;
+        var hasSlash = false;
+        while (index < source.Length - 1)
+        {
+            index++;
+            if (hasSlash == false && source[index] == '\\')
+            {
+                hasSlash = true;
+                current.Append('\\');
+                continue;
+            }
+
+            if (hasSlash == false && source[index] == quoteChar)
+            {
+                hasQuote = !hasQuote;
+                current.Append(source[index]);
+                continue;
+            }
+
+            if (hasQuote == false && source[index] == delimiter[0])
+            {
+                result.Add(current.ToString());
+                current.Clear();
+            }
+            else
+            {
+                current.Append(source[index]);
+            }
+
+            hasSlash = false;
+        }
+
+        result.Add(current.ToString());
+        return result.ToArray();
+    }
+
+    public virtual void Dispose()
+    {
+        if(Reader.IsValueCreated)
+            Reader.Value.Dispose();
+    }
+}

ConsoleApp2/Services/ETL/DataReaderFactory.cs

@@ -0,0 +1,46 @@
+using ConsoleApp2.HostedServices.Abstractions;
+using ConsoleApp2.Options;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace ConsoleApp2.Services.ETL;
+
+public class DataReaderFactory
+{
+    private readonly ILogger<DataReaderFactory> _logger;
+    private readonly IOptions<DataInputOptions> _options;
+
+    public DataReaderFactory(ILogger<DataReaderFactory> logger, IOptions<DataInputOptions> options)
+    {
+        _logger = logger;
+        _options = options;
+    }
+
+    public IDataReader CreateReader(string filePath, string tableName, string[] headers)
+    {
+        if (_options.Value.UseMock)
+        {
+            if (_options.Value.TableMockConfig is null)
+                throw new ApplicationException("未配置表模拟数据量级");
+            _logger.LogDebug("***** Using {Type} data source *****", "ZSTD mock");
+            var mockConfig = _options.Value.TableMockConfig.GetValueOrDefault(tableName, 
+                new TableMockConfig { MockCount = 1, UseDeepCopy = false });
+            mockConfig.MockCount = (long)Math.Ceiling(mockConfig.MockCount * _options.Value.MockCountMultiplier); 
+            return new ZstMockReader(mockConfig, filePath,
+                tableName, headers, _options.Value.Delimiter, _options.Value.QuoteChar, _logger);
+        }
+
+        _logger.LogDebug("***** Using {Type} data source *****", "ZSTD");
+        return new ZstReader(filePath, tableName, headers, _options.Value.Delimiter, _options.Value.QuoteChar, _logger);
+    }
+}
+
+public static class DataSourceFactoryExtensions
+{
+    public static IServiceCollection AddDataSourceFactory(this IServiceCollection services)
+    {
+        services.AddSingleton<DataReaderFactory>();
+        return services;
+    }
+}

ConsoleApp2/Services/ETL/MySqlDestination.cs

@@ -1,14 +1,12 @@
-using System.Data.Common;
-using System.Text;
+using System.Text;
 using System.Text.RegularExpressions;
 using ConsoleApp2.Helpers;
 using ConsoleApp2.Options;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using MySqlConnector;
-using ServiceStack;
 
-namespace ConsoleApp2.Services;
+namespace ConsoleApp2.Services.ETL;
 
 /// <summary>
 /// Mysql导出
@@ -18,29 +16,29 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
     private readonly Dictionary<string, IList<DataRecord>> _recordCache;
     private readonly MySqlConnection _conn;
     private readonly ILogger _logger;
+    private readonly IOptions<DatabaseOutputOptions> _options;
+    private readonly ErrorRecorder.OutputErrorRecorder _outputErrorRecorder;
     private readonly ProcessContext _context;
-    private readonly IOptions<DataTransformOptions> _transformOptions;
-    private readonly ErrorRecorder _errorRecorder;
 
     public MySqlDestination(
         string connStr, 
         ILogger logger, 
-        ProcessContext context, 
-        IOptions<DataTransformOptions> transformOptions,
-        ErrorRecorder errorRecorder)
+        IOptions<DatabaseOutputOptions> options,
+        ErrorRecorder.OutputErrorRecorder outputErrorRecorder, 
+        ProcessContext context)
     {
         _conn = new MySqlConnection(connStr);
         _conn.Open();
         _recordCache = new Dictionary<string, IList<DataRecord>>();
         _logger = logger;
+        _options = options;
+        _outputErrorRecorder = outputErrorRecorder;
         _context = context;
-        _transformOptions = transformOptions;
-        _errorRecorder = errorRecorder;
     }
     
     public Task WriteRecordAsync(DataRecord record)
     {
-        _recordCache.AddOrUpdate(record.TableName, [record], (key, value) =>
+        _recordCache.AddOrUpdate(record.TableName, [record], (_, value) =>
         {
             value.Add(record);
             return value;
@@ -76,22 +74,23 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
                 }
                 catch (Exception e)
                 {
-                    _logger.LogError(e, "Error when flushing records, sql: {Sql}", cmd.CommandText.Omit(1000));
-                    
+                    _logger.LogError(e, "插入数据库时发生错误, sql: {Sql}", cmd.CommandText.Omit(1000));
+                    _context.AddException(e);
                     var match = MatchTableName().Match(cmd.CommandText);
                     if (match is { Success: true, Groups.Count: > 1 })
                     {
                         var tableName = match.Groups[1].Value;
-                        await _errorRecorder.LogErrorSqlAsync(cmd.CommandText, tableName, e);
+                        await _outputErrorRecorder.LogErrorSqlAsync(cmd.CommandText, tableName, e);
                     }
-                    else await _errorRecorder.LogErrorSqlAsync(cmd.CommandText, e);
+                    else await _outputErrorRecorder.LogErrorSqlAsync(cmd.CommandText, e);
                 }
             }
             _recordCache.Clear();
         }
         catch (Exception e)
         {
-            _logger.LogError(e, "Error when serialize records, record:");
+            _logger.LogError(e, "序列化记录时发生错误");
+            throw;
         }
         finally
         {
@@ -104,7 +103,7 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
 
     public IEnumerable<string> GetExcuseList(IDictionary<string, IList<DataRecord>> tableRecords,int maxAllowPacket)
     {
-        var sb = new StringBuilder();
+        var sb = new StringBuilder("SET AUTOCOMMIT = 1;\n");
         foreach (var (tableName, records) in tableRecords)
         {
             if (records.Count == 0)
@@ -116,11 +115,11 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
             
             // INSERT INTO ... VALUES >>>
             sb.Append($"INSERT INTO `{tableName}`(");
-            for (var i = 0; i < records[0].Headers.Length; i++)
+            for (var i = 0; i < records[0].Headers.Count; i++)
             {
                 var header = records[0].Headers[i];
                 sb.Append($"`{header}`");
-                if (i != records[0].Headers.Length - 1)
+                if (i != records[0].Headers.Count - 1)
                     sb.Append(',');
             }
 
@@ -132,7 +131,7 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
                 var record = records[recordIdx];
                 var recordSb = new StringBuilder();
                 recordSb.Append('(');
-                for (var fieldIdx = 0; fieldIdx < record.Fields.Length; fieldIdx++)
+                for (var fieldIdx = 0; fieldIdx < record.Fields.Count; fieldIdx++)
                 {
                     var field = record.Fields[fieldIdx];
 
@@ -144,12 +143,12 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
                         goto Escape;
                     }
                     
-                    switch (_transformOptions.Value.GetColumnType(record.TableName, record.Headers[fieldIdx]))
+                    switch (_options.Value.GetColumnType(record.TableName, record.Headers[fieldIdx]))
                     {
                         case ColumnType.Text:
-                            recordSb.Append(string.IsNullOrEmpty(field)
-                                ? "''"
-                                : _transformOptions.Value.TransformBinary?.Invoke(field) ?? field);
+                            if(string.IsNullOrEmpty(field))
+                                recordSb.Append("''");
+                            else recordSb.Append($"_utf8mb4 0x{field}");
                             break;
                         case ColumnType.Blob:
                             if (string.IsNullOrEmpty(field))
@@ -157,9 +156,11 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
                             else recordSb.Append($"0x{field}");
                             break;
                         case ColumnType.Json:
-                            recordSb.Append(string.IsNullOrEmpty(field)
-                                ? "\"[]\""
-                                : _transformOptions.Value.TransformBinary?.Invoke(field) ?? field);
+                            if(string.IsNullOrEmpty(field))
+                                recordSb.Append("'[]'"); // JObject or JArray?
+                            else if (_options.Value.TreatJsonAsHex)
+                                recordSb.Append($"_utf8mb4 0x{field}");
+                            else recordSb.AppendLine(field);
                             break;
                         case ColumnType.UnDefine:
                         default:
@@ -170,16 +171,17 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
                     Escape:
 
                     #endregion
-                    if (fieldIdx != record.Fields.Length - 1)
+                    if (fieldIdx != record.Fields.Count - 1)
                         recordSb.Append(',');
                 }
 
                 recordSb.Append(')');
 
                 // 若字符数量即将大于限制，则返回SQL，清空StringBuilder，保留当前记录的索引值，然后转到StartBuild标签重新开始一轮INSERT
-                if (sb.Length + recordSb.Length + 1 > maxAllowPacket)
+                if (sb.Length + recordSb.Length + 23 > maxAllowPacket)
                 {
-                    sb.Append(';');
+                    sb.Append(';').AppendLine();
+                    sb.Append("SET AUTOCOMMIT = 1;");
                     yield return sb.ToString();
                     sb.Clear();
                     goto StartBuild;
@@ -192,6 +194,7 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
             }
 
             sb.Append(';');
+            sb.Append("COMMIT;");
             yield return sb.ToString();
             sb.Clear();
         }

ConsoleApp2/Services/ETL/ZstMockReader.cs

@@ -0,0 +1,64 @@
+using ConsoleApp2.Options;
+using Microsoft.Extensions.Logging;
+
+namespace ConsoleApp2.Services.ETL;
+
+/// <summary>
+/// 截取提供ZST文件中的第一行，然后复制成指定数量的数据
+/// </summary>
+public class ZstMockReader : ZstReader
+{
+    private long _currentCount;
+    private readonly long _mockCount;
+    private DataRecord? _template;
+    private readonly bool _deepCopy;
+    private readonly string[]? _autoIncrementColumn;
+    
+    static readonly IReadOnlyList<int> Range = [500, 1500, 2500];
+    
+    public ZstMockReader(TableMockConfig mockConfig, string filePath, string tableName, string[] headers, string delimiter = ",", char quoteChar = '\"', ILogger? logger = null) : base(filePath, tableName, headers, delimiter, quoteChar, logger)
+    {
+        _mockCount = mockConfig.MockCount;
+        _deepCopy = mockConfig.UseDeepCopy;
+        _autoIncrementColumn = mockConfig.AutoIncrementColumn;
+    }
+
+    public ZstMockReader(TableMockConfig mockConfig, Stream stream, string tableName, string[] headers, string delimiter = ",", char quoteChar = '\"', ILogger? logger = null) : base(stream, tableName, headers, delimiter, quoteChar, logger)
+    {
+        _mockCount = mockConfig.MockCount;
+        _deepCopy = mockConfig.UseDeepCopy;
+        _autoIncrementColumn = mockConfig.AutoIncrementColumn;
+    }
+
+    public override async ValueTask<bool> ReadAsync()
+    {
+        if (_template is null)
+        {
+            if (!await base.ReadAsync())
+                throw new InvalidOperationException("所提供的ZST源为空，无法生成模板数据");
+            _template = Current.Clone() as DataRecord;
+            if (_template is null)
+                throw new ApplicationException("记录拷贝失败");
+            _currentCount++;
+            return true;
+        }
+
+        if (_deepCopy)
+        {
+            Current = _template.Clone() as DataRecord ?? throw new ApplicationException("记录拷贝失败");
+            if(_autoIncrementColumn is not null)
+            {
+                foreach (var column in _autoIncrementColumn)
+                {
+                    Current[column] = (Convert.ToInt64(Current[column]) + 1).ToString();
+                    _template = Current;
+                }
+            }
+
+            Current["CompanyID"] = Range[Random.Shared.Next(0, Range.Count)].ToString();//随机CompanyID
+        }
+        else Current = _template;
+        _currentCount++;
+        return _currentCount < _mockCount;
+    }
+}

ConsoleApp2/Services/ETL/ZstReader.cs

@@ -0,0 +1,48 @@
+using Microsoft.Extensions.Logging;
+using ZstdSharp;
+
+namespace ConsoleApp2.Services.ETL;
+
+/// <summary>
+/// 解压ZST文件，从中读取CSV数据
+/// </summary>
+public class ZstReader : CsvReader
+{
+    protected new readonly Lazy<StreamReader> Reader;
+
+
+    public ZstReader(string filePath, string tableName, string[] headers, string delimiter = ",", char quoteChar = '\"', ILogger? logger = null) 
+        : base(filePath, tableName, headers, delimiter, quoteChar, logger)
+    {
+        var ds = new DecompressionStream(File.OpenRead(filePath));
+        Reader = new Lazy<StreamReader>(() => new StreamReader(ds));
+    }
+        
+    public ZstReader(Stream stream, string tableName, string[] headers, string delimiter = ",", char quoteChar = '\"', ILogger? logger = null) 
+        : base(stream, tableName, headers, delimiter, quoteChar, logger)
+    {
+        var ds = new DecompressionStream(stream);
+        Reader = new Lazy<StreamReader>(() => new StreamReader(ds));
+    }
+
+    public override async ValueTask<bool> ReadAsync()
+    {
+        var str = await Reader.Value.ReadLineAsync();
+        if (string.IsNullOrWhiteSpace(str))
+            return false;
+
+        CurrentRaw = str;
+
+        var fields = ParseRow(str, QuoteChar, Delimiter);
+        Current = new DataRecord(fields, TableName, Headers) {RawField = str};
+        return true;
+    }
+
+    public override void Dispose()
+    {
+        base.Dispose();
+        if(Reader.IsValueCreated) 
+            Reader.Value.Dispose();
+    }
+
+}

ConsoleApp2/Services/ErrorRecorder/ErrorRecorder.cs

@@ -0,0 +1,79 @@
+using ConsoleApp2.Helpers;
+using Microsoft.Extensions.Logging;
+
+namespace ConsoleApp2.Services.ErrorRecorder;
+
+public class ErrorRecorder
+{
+    protected ILogger Logger;
+
+    /// <summary>
+    /// 当次执行标识
+    /// </summary>
+    public static readonly string UID = DateTime.Now.ToString("yyyy-MM-dd HH-mm-ss");
+
+    public ErrorRecorder(ILogger logger)
+    {
+        Logger = logger;
+    }
+    
+    
+     public static async Task LogErrorRecordAsync(string outputDir, DataRecord record, Exception exception)
+     {
+         if(!Directory.Exists(outputDir))
+             Directory.CreateDirectory(outputDir);
+         var content = $"""
+                        ### {exception.Message}
+                        {record.RawField}
+                        """;
+         var path = Path.Combine(outputDir, $"{record.TableName}.errlog");
+         await File.AppendAllTextAsync(path, content);
+     }
+     
+     public static async Task LogErrorRecordAsync(string outputDir, IEnumerable<DataRecord> records, Exception exception)
+     {
+         if(!Directory.Exists(outputDir))
+             Directory.CreateDirectory(outputDir);
+         var tableMapping = new Dictionary<string, Tuple<List<DataRecord>, StreamWriter>>();
+         foreach (var record in records)
+         {
+             tableMapping.AddOrUpdate(record.TableName,
+                 Tuple.Create((List<DataRecord>) [record], new StreamWriter(File.OpenRead(record.TableName))),
+                 (_, tuple) =>
+                 {
+                     tuple.Item1.Add(record);
+                     return tuple;
+                 });
+         }
+
+         var maxParallelism = 5;
+         for (var i = 0; i < tableMapping.Count; i+=maxParallelism)
+         {
+             await Parallel.ForEachAsync(tableMapping.Take(maxParallelism), async (pair, token) =>
+             {
+                 var (records, writer) = pair.Value;
+                 foreach (var record in records)
+                 {
+                     var content =
+                         $"""
+                          ### {exception.Message}
+                          {record.RawField}
+                          """;
+                     await writer.WriteLineAsync(content);
+                     if (token.IsCancellationRequested)
+                         break;
+                 }
+                 await writer.DisposeAsync();
+             });
+         }
+     }
+     
+     public void ClearErrorRecords(string dir)
+     {
+         Logger.LogInformation("***** Clear error records *****");
+         foreach (var file in Directory.GetFiles(dir, "*.errlog", SearchOption.AllDirectories))
+         {
+             File.Delete(file);
+         }
+     }
+}

ConsoleApp2/Services/ErrorRecorder/ErrorRecorderFactory.cs

@@ -0,0 +1,27 @@
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+
+namespace ConsoleApp2.Services.ErrorRecorder;
+
+public class ErrorRecorderFactory
+{
+    private readonly ILogger<ErrorRecorderFactory> _logger;
+
+    public ErrorRecorderFactory(ILogger<ErrorRecorderFactory> logger)
+    {
+        _logger = logger;
+    }
+    
+    public OutputErrorRecorder CreateOutput(string database) => new(database, _logger);
+    public TransformErrorRecorder CreateTransform() => new(_logger);
+    public InputErrorRecorder CreateInput() => new(_logger);
+}
+
+public static class ErrorRecorderFactoryExtensions
+{
+    public static IServiceCollection AddErrorRecorderFactory(this IServiceCollection services)
+    {
+        services.AddSingleton<ErrorRecorderFactory>();
+        return services;
+    }
+}

ConsoleApp2/Services/ErrorRecorder/InputErrorRecorder.cs

@@ -0,0 +1,19 @@
+using Microsoft.Extensions.Logging;
+
+namespace ConsoleApp2.Services.ErrorRecorder;
+
+public sealed class InputErrorRecorder : ErrorRecorder
+{
+    private readonly string _outputDir =
+        Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"ErrorRecords/{UID}/Input");
+    
+    public InputErrorRecorder(ILogger logger) : base(logger)
+    {
+    }
+    
+    public Task LogErrorRecordAsync(DataRecord record, Exception exception) =>
+        LogErrorRecordAsync(_outputDir, record, exception);
+
+    public Task LogErrorRecordAsync(IEnumerable<DataRecord> records, Exception exception) =>
+        LogErrorRecordAsync(_outputDir, records, exception);
+}

ConsoleApp2/Services/ErrorRecorder/OutputErrorRecorder.cs

@@ -1,27 +1,19 @@
 using System.Text;
 using Microsoft.Extensions.Logging;
 
-namespace ConsoleApp2.Services;
+namespace ConsoleApp2.Services.ErrorRecorder;
 
-public class ErrorRecorder
+
+public sealed class OutputErrorRecorder : ErrorRecorder
 {
-    private readonly string _outputDir = "./ErrorRecords";
-    private readonly ILogger _logger;
+    private readonly string _outputDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"/ErrorRecords/{UID}/Output");
+    private readonly string _database;
     private readonly Dictionary<string, int> _logIndex = new();
 
-    /// <summary>
-    /// 当次执行标识
-    /// </summary>
-    private static readonly string UID = DateTime.Now.ToString("yyyy-MM-dd HH-mm-ss");
-
-    public ErrorRecorder(ILogger<ErrorRecorder> logger)
+    public OutputErrorRecorder(string database, ILogger logger) : base(logger)
     {
-        _logger = logger;
-        var dir = Path.Combine(_outputDir, UID);
-        if (!Directory.Exists(dir))
-        {
-            Directory.CreateDirectory(dir);
-        }
+        _database = database;
+        Logger = logger;
     }
 
     /// <summary>
@@ -32,25 +24,28 @@ public class ErrorRecorder
     /// <param name="exception"></param>
     public async Task LogErrorSqlAsync(string commandText, string tableName, Exception exception)
     {
+        if (!Directory.Exists(_outputDir))
+            Directory.CreateDirectory(_outputDir);
         if (!_logIndex.TryGetValue(tableName, out var idx))
         {
             idx = 0;
             _logIndex.Add(tableName, idx);
         }
-        var filePath = Path.Combine(_outputDir, UID, $"{tableName}-{idx}.errlog");
+        var filePath = Path.Combine(_outputDir, $"{tableName}-{idx}.errlog");
             
         if (File.Exists(filePath) && new FileInfo(filePath).Length > 10 * 1024 * 1024)
         {
             ++idx;
             _logIndex[tableName] = idx;
-            filePath = Path.Combine(_outputDir, UID, $"{tableName}-{idx}.errlog");
+            filePath = Path.Combine(_outputDir, $"{tableName}-{idx}.errlog");
         }
         var content = $"""
                       /* [{DateTime.Now:yyyy-MM-dd HH:mm:ss}]
-                       * Error occurred when export table '{tableName}':
+                       * Error occurred when export table '{_database}.{tableName}':
                        * {exception.Message}
                        */
                       
+                      USE `{_database}`;
                       {commandText}
 
 
@@ -65,7 +60,9 @@ public class ErrorRecorder
     /// <param name="exception"></param>
     public async Task LogErrorSqlAsync(string commandText, Exception exception)
     {
-        var filePath = Path.Combine(_outputDir, UID, "UnknownTables.errlog");
+        if (!Directory.Exists(_outputDir))
+            Directory.CreateDirectory(_outputDir);
+        var filePath = Path.Combine(_outputDir, "UnknownTables.errlog");
         var content = $"""
                        /* [{DateTime.Now:yyyy-MM-dd HH:mm:ss}]
                         * Error occurred when export table with unknown table name:
@@ -78,27 +75,4 @@ public class ErrorRecorder
         await File.AppendAllTextAsync(filePath, content, Encoding.UTF8);
     }
 
-    public async Task LogErrorRecordsAsync(IDictionary<string, DataRecord> records, Exception exception)
-    {
-        var pathDict = new Dictionary<string, string>();
-        foreach (var pair in records)
-        {
-            if(!pathDict.TryGetValue(pair.Key, out var path))
-            {
-                path = Path.Combine(_outputDir, UID, "ErrorRecords", $"{pair.Key}.errlog");
-                pathDict.Add(pair.Key, path);
-            }
-            //
-            await File.AppendAllTextAsync(path, string.Join(',', pair.Value.Fields));
-        }
-    }
-    
-    public void ClearErrorRecords()
-    {
-        _logger.LogInformation("***** Clear error records *****");
-        foreach (var file in Directory.GetFiles(_outputDir, "*.errlog", SearchOption.AllDirectories))
-        {
-            File.Delete(file);
-        }
-    }
 }

ConsoleApp2/Services/ErrorRecorder/TransformErrorRecorder.cs

@@ -0,0 +1,20 @@
+using Microsoft.Extensions.Logging;
+
+namespace ConsoleApp2.Services.ErrorRecorder;
+
+public sealed class TransformErrorRecorder : ErrorRecorder
+{
+    private readonly string _outputDir =
+        Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"ErrorRecords/{UID}/Transform");
+
+
+    public TransformErrorRecorder(ILogger logger) : base(logger)
+    {
+    }
+
+    public Task LogErrorRecordAsync(DataRecord record, Exception exception) =>
+        LogErrorRecordAsync(_outputDir, record, exception);
+
+    public Task LogErrorRecordAsync(IEnumerable<DataRecord> records, Exception exception) =>
+        LogErrorRecordAsync(_outputDir, records, exception);
+}

ConsoleApp2/Services/JsvSource.cs

@@ -1,44 +0,0 @@
-using ConsoleApp2.Helpers;
-using ConsoleApp2.HostedServices.Abstractions;
-using Microsoft.Extensions.Logging;
-using ServiceStack.Text;
-
-namespace ConsoleApp2.Services;
-
-/// <summary>
-/// 读取Jsv格式文件
-/// </summary>
-[Obsolete]
-public class JsvSource:IDataSource
-{
-    private readonly string _inputDir;
-    private readonly JsvStringSerializer _jsv;
-    private readonly StreamReader _reader;
-    // ReSharper disable once PrivateFieldCanBeConvertedToLocalVariable
-    private readonly ILogger? _logger;
-    private readonly string _tableName;
-
-    public DataRecord Current { get; protected set; } = null!;
-    public string[]? Headers { get; }
-    public bool EndOfSource => _reader.EndOfStream;
-
-    public JsvSource(string inputDir,string tableName, ILogger? logger = null)
-    {
-        _inputDir = inputDir;
-        _tableName = tableName;
-        _jsv = new JsvStringSerializer();
-       // _reader = new StreamReader(filePath);
-        //Headers = headers;
-        _logger = logger;
-        // _logger?.LogInformation("Reading file: {FilePath}", filePath);
-        //_tableName = DumpDataHelper.GetTableName(filePath);
-    }
-    public async Task DoEnqueue(Action<DataRecord> action)
-    {
-    }
-
-    public  void Dispose()
-    {
-        _reader.Dispose();
-    }
-}

ConsoleApp2/Services/Loggers/CacheTaskMonitorLogger.cs

@@ -0,0 +1,19 @@
+using ConsoleApp2.Cache;
+
+namespace ConsoleApp2.Services.Loggers;
+
+public class CacheTaskMonitorLogger : ITaskMonitorLogger
+{
+    private readonly ICacher _cacher;
+
+    public CacheTaskMonitorLogger(ICacher cacher)
+    {
+        _cacher = cacher;
+    }
+
+    public void LogStatus(string name, IReadOnlyDictionary<string, string> properties, ITaskMonitorLogger.LogLevel logLevel)
+    {
+        if(logLevel is ITaskMonitorLogger.LogLevel.Progress)
+            _cacher.SetHashAsync(name, properties);
+    }
+}

ConsoleApp2/Services/Loggers/ITaskMonitorLogger.cs

@@ -0,0 +1,12 @@
+namespace ConsoleApp2.Services.Loggers;
+
+public interface ITaskMonitorLogger
+{
+    public enum LogLevel
+    {
+        Info,
+        Debug,
+        Progress,
+    }
+    void LogStatus(string name, IReadOnlyDictionary<string, string> properties, LogLevel logLevel = LogLevel.Info);
+}

ConsoleApp2/Services/Loggers/LoggerTaskMonitorLogger.cs

@@ -0,0 +1,41 @@
+using System.Text;
+using Microsoft.Extensions.Logging;
+
+namespace ConsoleApp2.Services.Loggers;
+
+public class LoggerTaskMonitorLogger : ITaskMonitorLogger
+{
+    private readonly ILogger _logger;
+
+    public LoggerTaskMonitorLogger(ILogger<LoggerTaskMonitorLogger> logger)
+    {
+        _logger = logger;
+    }
+
+    public void LogStatus(string name, IReadOnlyDictionary<string, string> properties, ITaskMonitorLogger.LogLevel logLevel)
+    {
+        var sb = new StringBuilder();
+        sb.Append($"{name}: {{");
+        sb.AppendJoin(',', properties.Select((pair, i) => $" {pair.Key}: {pair.Value}"));
+        sb.Append('}');
+        // var args = new List<string> { name };
+        // properties.Aggregate(args, (args, pair) =>
+        // {
+        //     args.Add(pair.Key);
+        //     args.Add(pair.Value);
+        //     return args;
+        // });
+        switch (logLevel)
+        {
+            case ITaskMonitorLogger.LogLevel.Info:
+                _logger.LogInformation("{message}", sb.ToString());
+                break;
+            case ITaskMonitorLogger.LogLevel.Progress:
+            case ITaskMonitorLogger.LogLevel.Debug:
+                _logger.LogDebug("{message}", sb.ToString());
+                break;
+            default:
+                throw new ArgumentOutOfRangeException(nameof(logLevel), logLevel, null);
+        }
+    }
+}

ConsoleApp2/Services/ProcessContext.cs

@@ -1,39 +1,49 @@
-namespace ConsoleApp2.Services;
+using System.Collections.Concurrent;
+
+namespace ConsoleApp2.Services;
 
 /// <summary>
 /// 处理上下文类，标识处理进度
 /// </summary>
 public class ProcessContext
 {
-    private int _inputCount;
-    private int _transformCount;
-    private int _outputCount;
+    private bool _hasException;
+    private long _inputCount;
+    private long _transformCount;
+    private long _outputCount;
+    private readonly ConcurrentDictionary<string, long> _tableProgress = new();
+    public bool HasException => _hasException;
     public bool IsInputCompleted { get; private set; }
     public bool IsTransformCompleted { get; private set; }
     public bool IsOutputCompleted { get; private set; }
 
-    public int InputCount
+    public long InputCount
     {
         get => _inputCount;
-        private set => _inputCount = value;
+        set => Interlocked.Exchange(ref _inputCount, value);
     }
 
-    public int TransformCount
+    public long TransformCount
     {
         get => _transformCount;
-        private set => _transformCount = value;
+        set => Interlocked.Exchange(ref _transformCount, value);
     }
 
-    public int OutputCount
+    public long OutputCount
     {
         get => _outputCount;
-        private set => _outputCount = value;
+        set => Interlocked.Exchange(ref _outputCount, value);
     }
-    
+
+
+    // TableName -> Count
+    public IReadOnlyDictionary<string, long> TableProgress => _tableProgress;
+
     public void CompleteInput() => IsInputCompleted = true;
 
     public void CompleteTransform() => IsTransformCompleted = true;
     public void CompleteOutput() => IsOutputCompleted = true;
+    public bool AddException(Exception e) => _hasException = true;
 
     public void AddInput() => Interlocked.Increment(ref _inputCount);
 
@@ -44,4 +54,17 @@ public class ProcessContext
 
     public void AddOutput() => Interlocked.Increment(ref _outputCount);
     public void AddOutput(int count) => Interlocked.Add(ref _outputCount, count);
+
+    public void AddTableOutput(string table, int count)
+    {
+        _tableProgress.AddOrUpdate(table, count, (k, v) => v + count);
+        AddOutput(count);
+    }
+
+    public long GetTableOutput(string table)
+    {
+        if(!_tableProgress.TryGetValue(table, out var count))
+            throw new ApplicationException($"未找到表{table}输出记录");
+        return count;
+    }
 }

ConsoleApp2/Services/RecordQueuePool.cs

@@ -0,0 +1,65 @@
+using System.Collections.Concurrent;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace ConsoleApp2.Services;
+
+public class RecordQueuePool
+{
+    private readonly ConcurrentDictionary<string, DataRecordQueue> _queues = new();
+
+    public IReadOnlyDictionary<string, DataRecordQueue> Queues => _queues;
+
+    public void AddQueue(string key, int boundedCapacity = 200_0000) => AddQueue(key, new DataRecordQueue(boundedCapacity));
+    
+    public void AddQueue(string key, DataRecordQueue queue)
+    {
+        if (!_queues.TryAdd(key, queue))
+            throw new InvalidOperationException($"请勿添加重复的队列，队列名: {key}");
+    }
+    
+    public void RemoveQueue(string key, bool dispose = true)
+    {
+        if (!_queues.Remove(key, out var queue))
+            throw new InvalidOperationException($"未找到对应的队列，队列名：{key}");
+        if (dispose) queue.Dispose();
+    }
+    
+    public DataRecordQueue GetQueue(string key)
+    {
+        return _queues[key];
+    }
+
+    public DataRecordQueue this[string key]
+    {
+        get => GetQueue(key);
+        set => AddQueue(key, value);
+    }
+}
+
+public static class MultiRecordQueueExtensions
+{
+    public static IServiceCollection AddRecordQueuePool(this IServiceCollection services, params string[] keys)
+    {
+        var pool = new RecordQueuePool();
+        foreach (var key in keys)
+        {
+            pool.AddQueue(key);
+        }
+
+        services.AddSingleton(pool);
+        return services;
+    }
+
+    public static IServiceCollection AddRecordQueuePool(this IServiceCollection services,
+        params (string key, DataRecordQueue queue)[] queues)
+    {
+        var pool = new RecordQueuePool();
+        foreach (var (key, queue) in queues)
+        {
+            pool.AddQueue(key, queue);
+        }
+
+        services.AddSingleton(pool);
+        return services;
+    }
+}

ConsoleApp2/Services/TaskManager.cs

@@ -1,5 +1,5 @@
-using System.Collections.Concurrent;
-using Microsoft.Extensions.Logging;
+using ApplicationException = System.ApplicationException;
+using TaskExtensions = ConsoleApp2.Helpers.TaskExtensions;
 
 namespace ConsoleApp2.Services;
 
@@ -8,42 +8,74 @@ namespace ConsoleApp2.Services;
 /// </summary>
 public class TaskManager
 {
-    private readonly ConcurrentBag<Task> _tasks;
-    private readonly ILogger _logger;
+    private int _runningTaskCount;
 
-    public int TaskCount => _tasks.Count;
-    public int RunningTaskCount => _tasks.Count(task => !task.IsCompleted);
-    public IReadOnlyCollection<Task> Tasks => _tasks;
+    public int RunningTaskCount => _runningTaskCount;
+    public int MaxTaskCount { get; }
 
-    public TaskManager(ILogger<TaskManager> logger)
+    public event Action<Exception>? OnException;
+    public event Action? OnTaskCompleteSuccessfully;
+
+    public TaskManager(int maxTaskCount)
     {
-        _tasks = new ConcurrentBag<Task>();
-        _logger = logger;
+        MaxTaskCount = maxTaskCount;
     }
 
-    public void CreateTask(Func<Task> func, CancellationToken cancellationToken = default)
+    public async ValueTask<Task> CreateTaskAsync(Func<Task> func, CancellationToken cancellationToken = default)
     {
-        var task = Task.Run(func, cancellationToken);
-        _tasks.Add(task);
-        _logger.LogDebug("New task created");
+        await TaskExtensions.WaitUntil(() => _runningTaskCount < MaxTaskCount, 25, cancellationToken);
+        return RunTask(func, cancellationToken);
+    }
+
+    public async ValueTask<Task> CreateTaskAsync(Func<object?, Task> func, object? arg, CancellationToken ct = default)
+    {
+        await TaskExtensions.WaitUntil(() => _runningTaskCount < MaxTaskCount, 25, ct);
+        return RunTaskNoClosure(func, arg, ct);
     }
     
-    public void CreateTasks(Func<Task> func,int taskCount, CancellationToken cancellationToken = default)
+    private Task RunTask(Func<Task> func, CancellationToken cancellationToken = default)
     {
-        for (int i = 0; i < taskCount; i++)
+        var task = Task.Run(async () =>
         {
-            CreateTask(func, cancellationToken);
-        }
-    }
-    public async Task WaitAll()
-    {
-        await Task.WhenAll(_tasks);
+            try
+            {
+                await func();
+                OnTaskCompleteSuccessfully?.Invoke();
+            }
+            catch(Exception ex)
+            {
+                OnException?.Invoke(ex);
+            }
+            finally
+            {
+                Interlocked.Decrement(ref _runningTaskCount);
+            }
+        }, cancellationToken);
+        Interlocked.Increment(ref _runningTaskCount);
+        return task;
     }
 
-    public void ClearTask()
+    private Task RunTaskNoClosure(Func<object?, Task> func, object? arg, CancellationToken cancellationToken = default)
     {
-        if(RunningTaskCount != 0)
-            throw new InvalidOperationException("Unable to clear task. There are still running tasks");
-        _tasks.Clear();
+        var task = Task.Factory.StartNew(async obj => // 性能考虑，这个lambda中不要捕获任何外部变量!
+        {
+            if (obj is not Tuple<Func<object?, Task>, object?> tuple)
+                throw new ApplicationException("这个异常不该出现");
+            try
+            {
+                await tuple.Item1(tuple.Item2);
+                OnTaskCompleteSuccessfully?.Invoke();
+            }
+            catch(Exception ex)
+            {
+                OnException?.Invoke(ex);
+            }
+            finally
+            {
+                Interlocked.Decrement(ref _runningTaskCount);
+            }
+        }, Tuple.Create(func, arg), cancellationToken).Unwrap();
+        Interlocked.Increment(ref _runningTaskCount);
+        return task;
     }
 }

ConsoleApp2/Services/ZstSource.cs

@@ -1,96 +0,0 @@
-using ConsoleApp2.Helpers;
-using Microsoft.Extensions.Logging;
-using System.IO;
-using System.Text.RegularExpressions;
-using ZstdSharp;
-namespace ConsoleApp2.Services
-{
-    public class ZstSource : CsvSource
-    {
-        public ZstSource(string inputDir, string tableName, string delimiter = ",", char quoteChar = '"',
-        ILogger? logger = null) : base(inputDir, tableName, delimiter = ",", quoteChar = '"', logger = null)
-        {
-            //throw new Exception("aaa");
-            string pattern = $"^.*\\.{tableName}\\..*\\.sql.zst$";
-             _sqlFilePath = Directory.GetFiles(_inputDir).FirstOrDefault(s => Regex.Match(s, pattern).Success);
-
-        }
-        private async Task<string> DecompressFile(string filePath)
-        {
-            using (var input = File.OpenRead(filePath))
-            {
-                using (var decopress = new DecompressionStream(input))
-                {
-
-                    var ms = new MemoryStream();
-                    decopress.CopyTo(ms);
-                    ms.Seek(0, SeekOrigin.Begin);
-                    StreamReader reader = new StreamReader(ms);
-                    var text = await reader.ReadToEndAsync();
-                    return text;
-
-                }
-            }
-        }
-        public override async Task GetHeaderAndCsvFiles()
-        {
-            var text = await DecompressFile(_sqlFilePath);
-            headers=await DumpDataHelper.GetCsvHeadersFromSqlFileAsync(text);
-            csvFiles=await DumpDataHelper.GetCsvFileNamesFromSqlFileAsync(text, new Regex(@"'.+\.dat.zst'"));
-
-        }
-        public override async Task DoEnqueue(Action<DataRecord> action)
-        {
-           await GetHeaderAndCsvFiles();
-            foreach (var file in csvFiles)
-            {
-                var filePath = Path.Combine(_inputDir, file);
-                using (var input = File.OpenRead(filePath))
-                {
-                    using (var decopress = new DecompressionStream(input)) 
-                    {
-                        using( var reader = new StreamReader(decopress))
-                        {
-                            while (!reader.EndOfStream)
-                            {
-                                var line = await reader.ReadLineAsync();
-                                var fields = ParseRow2(line, QuoteChar, Delimiter);
-                                var record = new DataRecord(fields, _tableName, headers);
-                                action?.Invoke(record);
-                            }
-                        }
-                    }
-                }
-
-            }
-        }
-        public override async Task<DataRecord?> GetTestRecord()
-        {
-            await GetHeaderAndCsvFiles();
-            var file = csvFiles.FirstOrDefault();
-            if (file != null)
-            {
-                var filePath = Path.Combine(_inputDir, file);
-                using (var input = File.OpenRead(filePath))
-                {
-                    using (var decopress = new DecompressionStream(input))
-                    {
-                        using (var reader = new StreamReader(decopress))
-                        {
-                            var line = await reader.ReadLineAsync();
-                            var fields = ParseRow2(line, QuoteChar, Delimiter);
-                            var record = new DataRecord(fields, _tableName, headers);
-                            return record;
-                        }
-
-                    }
-                }
-            }
-            return null;
-        }
-        public void Dispose()
-        {
-            //_reader.Dispose();
-        }
-    }
-}

ConsoleApp2/SimulationService/SimulationInputService.cs

@@ -12,6 +12,7 @@ using System.Text.RegularExpressions;
 namespace ConsoleApp2.SimulationService
 {
 
+    #if false
     public class SimulationInputService : IInputService
     {
         private readonly ILogger _logger;
@@ -131,4 +132,5 @@ namespace ConsoleApp2.SimulationService
             _logger.LogInformation("***** Csv input service completed *****");
         }
     }
+#endif
 }

ConsoleApp2/appsettings.json

@@ -1,17 +1,31 @@
 {
-  "CmdOptions": {
-    "InputFileType": "CSV",
-    "InputDir": "D:/MyDumper-ZST",
-    "TaskCount": 6,
-    "FlushCount": 10000,
-    "Isutf8mb4": true,
-    "OldestShardKey": 23000,
-    "OldestTime": "202301"
+  "Logging": {
+    "LogLevel": {
+      "Default": "Debug"
+    }
   },
-  "ConnectionStrings": {
-    "MySqlMaster": "Server=127.0.0.1;Port=33309;UserId=root;Password=123456;" // 要分库，不用加'Database='了
+  "Input":{
+    "InputDir": "D:\\Dump\\MockData",
+    "CleanDate": "202301",
+    "StrictMode": false,
+    "UseMock": true,
+    "MockCountMultiplier": 0.05
   },
-  "RedisCacheOptions": {
+  "Transform":{
+    "StrictMode": true,
+    "EnableFilter": false,
+    "EnableReplacer": false,
+    "EnableReBuilder": false
+  },
+  "Output":{
+    "ConnectionString": "Server=127.0.0.1;Port=3306;UserId=root;Password=cfmes123456;", // 要分库，不用加'Database='了
+    "MaxAllowedPacket": 67108864,
+    "FlushCount": 20000,
+    "MaxDatabaseOutputTask" : 4,
+    "TreatJsonAsHex": false,
+    "RestoreIndex": true
+  },
+  "RedisCache": {
     "Configuration": "192.168.1.246:6380",
     "InstanceName" : "mes-etl:"
   },
@@ -28,6 +42,7 @@
       "cferp_test_1": 1000,
       "cferp_test_2": 2000,
       "cferp_test_3": 2147483647
+//      "cferp_void_1": 2147483647
     }
   }
 }

TestProject1/DataFix.cs

@@ -0,0 +1,142 @@
+using System.Data;
+using ConsoleApp2.Helpers;
+using ConsoleApp2.Helpers.Database;
+using MySqlConnector;
+using Xunit.Abstractions;
+
+namespace TestProject1;
+
+public class DataFix
+{
+    public const string ConnStr = "Server=192.168.1.245;Port=3306;UserId=root;Password=ruixinjie!@#123;";
+    public static string[] DbNames = ["cferp_test_1", "cferp_test_2", "cferp_test_3"];
+    
+    private readonly ITestOutputHelper _output;
+    
+    public DataFix(ITestOutputHelper output)
+    {
+        _output = output;
+    }
+    
+    private string MakeConnStr(string dbName) => ConnStr + $"Database={dbName};";
+
+    /// <summary>
+    /// 查找cferp_test_1中CompanyID = 0的order_box_block，根据OrderNo查找对应订单的CompanyID，然后删除重新插入相应的数据库
+    /// 如果没有找到对应订单，则保留CompanyID为0
+    /// </summary>
+    [Fact]
+    public async Task FixOrderBoxBlockCompanyID()
+    {
+        var ds = await DatabaseHelper.QueryTableAsync(MakeConnStr(DbNames[0]), "SELECT * FROM `order_box_block` WHERE COMPANYID = 0");
+        var dict = new Dictionary<long, int>();//orderNo -> CompanyID
+        foreach (DataRow row in ds.Tables[0].Rows)
+        {
+            // foreach (var column in row.ItemArray)
+            // {
+            //     Console.Write(column.ToString() + '\t');
+            // }
+
+            var orderNo = Convert.ToInt64(row["OrderNo"]);
+            var boxId = Convert.ToInt64(row["BoxID"]);
+
+            int? companyId = null;
+            foreach (var db in DbNames)
+            {
+                if(!dict.TryGetValue(orderNo, out var cid)) // 可以提到外面
+                {
+                    var result = await DatabaseHelper.QueryScalarAsync(ConnStr + $"Database={db};",
+                        $"SELECT CompanyID FROM `order` WHERE OrderNo = {orderNo}");
+                    if (result is null or 0) continue;
+                    
+                    companyId = Convert.ToInt32(result);
+                    dict.Add(orderNo, companyId.Value);
+                    break;
+                }
+                else
+                {
+                    companyId = cid;
+                }
+            }
+
+            if (companyId is null or 0)
+            {
+                _output.WriteLine($"OrderBoxBlock:{boxId} - OrderNo {orderNo} not found");
+                continue;
+            }
+
+            row["CompanyID"] = companyId;
+            await DatabaseHelper.TransactionAsync(ConnStr,
+                $"""
+                 DELETE FROM cferp_test_1.`order_box_block` WHERE BoxID = {boxId};
+                 INSERT INTO {TenantDbHelper.GetDbNameByTenantKeyValue(companyId.Value)}.`order_box_block`
+                 VALUES(@c1, @c2, @c3, @c4, @c5);
+                 """, [new MySqlParameter("c1", row[0]),
+                    new MySqlParameter("c2", row[1]),
+                    new MySqlParameter("c3", row[2]),
+                    new MySqlParameter("c4", row[3]),
+                    new MySqlParameter("c5", row[4])]);
+            _output.WriteLine($"OrderBoxBock:{boxId} CompanyID -> {companyId}");
+        }
+    }
+
+    [Theory]
+    [InlineData(["order_data_block", "ID", 0])]
+    [InlineData(["order_data_block", "ID", 1])]
+    [InlineData(["order_data_block", "ID", 2])]
+    public async Task FixCompanyIdWithOwnOrderNo(string tableName, string keyName, int dbNameIndex)
+    {
+        var ds = await DatabaseHelper.QueryTableAsync(MakeConnStr(DbNames[dbNameIndex]), 
+            $"SELECT * FROM `{tableName}` WHERE COMPANYID = 0");
+        var dict = new Dictionary<long, int>();//orderNo -> CompanyID
+        foreach (DataRow row in ds.Tables[0].Rows)
+        {
+            // foreach (var column in row.ItemArray)
+            // {
+            //     Console.Write(column.ToString() + '\t');
+            // }
+
+            var orderNo = Convert.ToInt64(row["OrderNo"]);
+            var key = Convert.ToInt32(row[keyName]);
+
+            int? companyId = null;
+            if(!dict.TryGetValue(orderNo, out var cid))
+            {
+                companyId = await MesDatabaseHelper.TrySearchCompanyId(ConnStr, DbNames,
+                    $"SELECT CompanyID FROM `order` WHERE OrderNo = {orderNo}");
+            }
+            else
+            {
+                companyId = cid;
+            }
+
+            if (companyId is null or 0)
+            {
+                _output.WriteLine($"{tableName}:{key} - OrderNo {orderNo} not found");
+                continue;
+            }
+
+            row["CompanyID"] = companyId;
+            // ON DUPLICATE KEY UPDATE
+            await DatabaseHelper.TransactionAsync(ConnStr,
+                $"""
+                 DELETE FROM {DbNames[dbNameIndex]}.`{tableName}` WHERE ID = {key};
+                 INSERT INTO {TenantDbHelper.GetDbNameByTenantKeyValue(companyId.Value)}.`{tableName}`
+                 VALUES({string.Join(',', Enumerable.Range(0, row.ItemArray.Length).Select(i => $"@c{i}"))})
+                 ON DUPLICATE KEY UPDATE CompanyID = {companyId};
+                 """, 
+                Enumerable.Range(0, row.ItemArray.Length).Select(i => new MySqlParameter($"c{i}", row[i])).ToArray());
+            _output.WriteLine($"{tableName}:{key} CompanyID -> {companyId}");
+        }
+    }
+
+    [Theory]
+    [InlineData("order", 0)]
+    [InlineData("order", 1)]
+    [InlineData("order", 2)]
+    public async Task FixShardKeyWithOwnOrderNo(string tableName, int dbNameIndex)
+    {
+        var r = await DatabaseHelper.NonQueryAsync(MakeConnStr(DbNames[dbNameIndex]),
+            $"UPDATE `{tableName}` SET ShardKey = CONCAT(SUBSTR(`order`.OrderNo,3,4),'0') WHERE ShardKey = 0");
+        _output.WriteLine($"Affect Rows: {r}");
+    }
+}

TestProject1/DatabaseToolBox.cs

@@ -0,0 +1,154 @@
+using System.Data;
+using System.Text;
+using ConsoleApp2.Helpers.Database;
+using MySqlConnector;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using Xunit.Abstractions;
+
+namespace TestProject1;
+
+public class DatabaseToolBox
+{
+    private readonly ITestOutputHelper _output;
+    public const string ConnStr = "Server=127.0.0.1;Port=33309;UserId=root;Password=123456;";
+
+    public DatabaseToolBox(ITestOutputHelper output)
+    {
+        _output = output;
+    }
+
+    [Theory]
+    [InlineData("cferp_void_1")]
+    public async Task AlterAllTableToBlackHole(string database)
+    {
+        var connStr = ConnStr + $"Database={database}";
+        var tables = await DatabaseHelper.QueryTableAsync(connStr,
+            $"""
+             SELECT TABLE_NAME FROM information_schema.`TABLES` WHERE TABLE_SCHEMA = '{database}';
+             """);
+
+        foreach (DataRow row in tables.Tables[0].Rows)
+        {
+            var tableName = row["TABLE_NAME"].ToString();
+            var sql = $"""
+                       ALTER TABLE `{tableName}` REMOVE PARTITIONING;
+                       """;
+            try
+            {
+                await DatabaseHelper.NonQueryAsync(connStr, sql);
+            }
+            catch (MySqlException e) when (e.ErrorCode == MySqlErrorCode.PartitionManagementOnNoPartitioned)
+            {
+            }
+
+            sql = $"""
+                   ALTER TABLE `{tableName}` ENGINE=BLACKHOLE;
+                   """;
+            await DatabaseHelper.NonQueryAsync(connStr, sql);
+        }
+    }
+
+    public async Task<long> CountAllTable(string connStr, string database)
+    {
+        var count = 0L;
+        var set = await DatabaseHelper.QueryTableAsync(connStr,
+            $"""
+             SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = '{database}';
+             """);
+        foreach (DataRow row in set.Tables[0].Rows)
+        {
+            count += Convert.ToInt64(await DatabaseHelper.QueryScalarAsync(connStr,
+                $"""
+                 SELECT COUNT(1) FROM `{database}`.`{row[0]}`;
+                 """));
+        }
+        _output.WriteLine($"Record count: {count} records");
+        return count;
+    }
+
+    [Theory]
+    [InlineData([new[]{"cferp_test_1", "cferp_test_2", "cferp_test_3"}])]
+    public async Task CountAllDatabase(IEnumerable<string> databases)
+    {
+        var count = 0L;
+        var connStr = "Server=192.168.1.245;Port=3306;UserId=root;Password=ruixinjie!@#123;";
+        foreach (var db in databases)
+        {
+            count += await CountAllTable(connStr, db);
+        }
+
+        _output.WriteLine(count.ToString());
+    }
+
+    public async Task<TableIndex[]> GetAllTableIndexes(string database)
+    {
+        var data = await DatabaseHelper.QueryTableAsync(ConnStr,
+            $"""
+            SELECT TABLE_NAME, INDEX_NAME, NON_UNIQUE, COLUMN_NAME, INDEX_TYPE FROM information_schema.STATISTICS
+                     WHERE TABLE_SCHEMA = '{database}'
+                       AND INDEX_NAME != 'PRIMARY'
+                       AND INDEX_TYPE = 'BTREE'
+            """);
+        return data.Tables[0].Rows.Cast<DataRow>().Select(row =>
+        {
+            return new TableIndex(row["TABLE_NAME"].ToString()!,
+                row["INDEX_NAME"].ToString()!,
+                !Convert.ToBoolean(row["NON_UNIQUE"]),
+                row["COLUMN_NAME"].ToString()!,
+                row["INDEX_TYPE"] switch
+                {
+                    "BTREE" => TableIndex.TableIndexType.BTree,
+                    "PRIMARY" => TableIndex.TableIndexType.Primary,
+                    "HASH" => TableIndex.TableIndexType.Hash,
+                    _ => throw new ArgumentOutOfRangeException(nameof(row))
+                }
+            );
+        }).ToArray();
+    }
+
+    [Theory]
+    [InlineData(["cferp_test_1", "D:/Indexes_cferp_test_1.json"])]
+    [InlineData(["cferp_test_2", "D:/Indexes_cferp_test_2.json"])]
+    [InlineData(["cferp_test_3", "D:/Indexes_cferp_test_3.json"])]
+    public async Task ExportAllIndexes(string database, string outputPath)
+    {
+        var indexes = await GetAllTableIndexes(database);
+        var json = JArray.FromObject(indexes);
+        await File.WriteAllTextAsync(outputPath, json.ToString());
+        _output.WriteLine($"Exported {indexes.Length} indexes to '{outputPath}'");
+    }
+
+    [Theory]
+    [InlineData("cferp_test_1", "D:/Indexes_cferp_test_1.json")]
+    [InlineData("cferp_test_2", "D:/Indexes_cferp_test_2.json")]
+    [InlineData("cferp_test_3", "D:/Indexes_cferp_test_3.json")]
+    public async Task ImportAllIndexes(string database, string importPath)
+    {
+        var json = await File.ReadAllTextAsync(importPath);
+        var indexes = JsonConvert.DeserializeObject<TableIndex[]>(json);
+        var sb = new StringBuilder();
+        foreach (var (tableName, indexName, isUnique, columnName, tableIndexType) in indexes!)
+        {
+            sb.AppendLine($"CREATE {(isUnique ? "UNIQUE" : string.Empty)} INDEX `{indexName}` ON `{database}`.`{tableName}` (`{columnName}`) USING {tableIndexType};");
+        }
+        await DatabaseHelper.NonQueryAsync(ConnStr, sb.ToString());
+        _output.WriteLine($"Import {indexes.Length} indexes to '{database}' from '{importPath}'");
+    }
+
+    [Theory]
+    [InlineData(["cferp_test_1"])]
+    [InlineData(["cferp_test_2"])]
+    [InlineData(["cferp_test_3"])]
+    public async Task DropAllIndex(string database)
+    {
+        var indexes = await GetAllTableIndexes(database);
+        var sb = new StringBuilder();
+        foreach (var (tableName, indexName) in indexes)
+        {
+            sb.AppendLine($"DROP INDEX `{indexName}` ON `{database}`.`{tableName}`;");
+        }
+        await DatabaseHelper.NonQueryAsync(ConnStr, sb.ToString());
+        _output.WriteLine($"Dropped {indexes.Length} indexes from {database}");
+    }
+}

TestProject1/GlobalUsings.cs

@@ -0,0 +1 @@
+global using Xunit;

TestProject1/MesDatabaseHelper.cs

@@ -0,0 +1,20 @@
+using ConsoleApp2.Helpers.Database;
+
+namespace TestProject1;
+
+public static class MesDatabaseHelper
+{
+    public static async Task<int?> TrySearchCompanyId(string connStr, IEnumerable<string> dbNames, string scalarQuery)
+    {
+        foreach (var db in dbNames)
+        {
+            var result = await DatabaseHelper.QueryScalarAsync(connStr + $"Database={db};", scalarQuery);
+            if (result is null or 0) continue;
+                
+            var companyId = Convert.ToInt32(result);
+            return companyId;
+        }
+
+        return null;
+    }
+}

TestProject1/TableIndex.cs

@@ -0,0 +1,24 @@
+namespace TestProject1;
+
+public record TableIndex(string TableName, string IndexName, bool IsUnique, string ColumnName, TableIndex.TableIndexType IndexType)
+{
+    public enum TableIndexType
+    {
+        BTree,
+        Hash,
+        Primary
+    }
+
+    public void Deconstruct(out string tableName, out string indexName)
+    {
+        tableName = TableName;
+        indexName = IndexName;
+    }
+
+    public void Deconstruct(out string tableName, out string indexName, out string columnName)
+    {
+        tableName = TableName;
+        indexName = IndexName;
+        columnName = ColumnName;
+    }
+}

TestProject1/TenantDbHelper.cs

@@ -0,0 +1,26 @@
+namespace TestProject1;
+
+public static class TenantDbHelper
+{
+    /// <summary>
+    /// Key-Value: {DbName}-{TenantKeyLessThan}
+    /// </summary>
+    public static Dictionary<string, int> DbList { get; set; } = new Dictionary<string, int>
+    {
+        { "cferp_test_1", 1000 },
+        { "cferp_test_2", 2000 },
+        { "cferp_test_3", int.MaxValue },
+    };
+
+    public static string GetDbNameByTenantKeyValue(int tenantKeyValue)
+    {
+        // var dictionary = new SortedDictionary<int, string>();
+        // DbList.ForEach(pair => dictionary.Add(pair.Value, pair.Key));
+        // 注意配置顺序
+        var dbName = DbList.Cast<KeyValuePair<string, int>?>()
+            .FirstOrDefault(pair => pair?.Value != null && pair.Value.Value > tenantKeyValue)!.Value.Key;
+        return dbName ??
+               throw new ArgumentOutOfRangeException(nameof(tenantKeyValue),
+                   $"已配置的数据库中没有任何符合'{nameof(tenantKeyValue)}'值的对象");
+    }
+}

TestProject1/TestProject1.csproj

@@ -0,0 +1,29 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net8.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+
+        <IsPackable>false</IsPackable>
+        <IsTestProject>true</IsTestProject>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.0"/>
+        <PackageReference Include="xunit" Version="2.4.2"/>
+        <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
+            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+            <PrivateAssets>all</PrivateAssets>
+        </PackageReference>
+        <PackageReference Include="coverlet.collector" Version="6.0.0">
+            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+            <PrivateAssets>all</PrivateAssets>
+        </PackageReference>
+    </ItemGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\ConsoleApp2\ConsoleApp2.csproj" />
+    </ItemGroup>
+
+</Project>