ConsoleApp2.sln

@@ -2,8 +2,6 @@
 Microsoft Visual Studio Solution File, Format Version 12.00
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ConsoleApp2", "ConsoleApp2\ConsoleApp2.csproj", "{155E4B04-E88C-4BA4-AED2-B13E0A0432B5}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestProject1", "TestProject1\TestProject1.csproj", "{8679D5B6-5853-446E-9882-7B7A8E270500}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -14,9 +12,5 @@ Global
 		{155E4B04-E88C-4BA4-AED2-B13E0A0432B5}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{155E4B04-E88C-4BA4-AED2-B13E0A0432B5}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{155E4B04-E88C-4BA4-AED2-B13E0A0432B5}.Release|Any CPU.Build.0 = Release|Any CPU
-		{8679D5B6-5853-446E-9882-7B7A8E270500}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{8679D5B6-5853-446E-9882-7B7A8E270500}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{8679D5B6-5853-446E-9882-7B7A8E270500}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{8679D5B6-5853-446E-9882-7B7A8E270500}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

ConsoleApp2/Cache/CacheKeys.cs

@@ -1,9 +0,0 @@
-namespace ConsoleApp2.Cache;
-
-#nullable disable
-public static class CacheKeys
-{
-    public static Func<string, string> Order_OrderNo_CompanyID { get; set; }
-    public static Func<string, string> OrderBlockPlan_ID_CompanyID { get; set; }
-    public static Func<string, string> OrderProcess_ID_ShardKey { get; set; }
-}

ConsoleApp2/Cache/ICacher.cs

@@ -1,10 +0,0 @@
-namespace ConsoleApp2.Cache;
-
-public interface ICacher
-{
-    Task<string?> GetStringAsync(string key);
-    Task SetStringAsync(string key, string value);
-    Task<bool> ExistsAsync(string key);
-    Task SetHashAsync(string key, IReadOnlyDictionary<string, string> hash);
-    Task<Dictionary<string, string>> GetHashAsync(string key);
-}

ConsoleApp2/Cache/RedisCache.cs

@@ -1,63 +0,0 @@
-using ConsoleApp2.Options;
-using Microsoft.Extensions.DependencyInjection;
-using StackExchange.Redis;
-
-namespace ConsoleApp2.Cache;
-
-public class RedisCache : ICacher
-{
-    private readonly IDatabase _db;
-
-    public string KeyPrefix { get; set; }
-
-    public RedisCache(IConnectionMultiplexer conn, int dataBase, string keyPrefix = "")
-    {
-        _db = conn.GetDatabase(dataBase);
-        KeyPrefix = keyPrefix;
-    }
-
-    public async Task<string?> GetStringAsync(string key)
-    {
-        var value = await _db.StringGetAsync($"{KeyPrefix}{key}");
-        return !value.HasValue ? null : value.ToString();
-    }
-
-    public async Task SetStringAsync(string key, string value)
-    {
-        if (!await _db.StringSetAsync($"{KeyPrefix}{key}", value))
-            throw new RedisCommandException("设置Redis缓存失败");
-    }
-
-    public Task<bool> ExistsAsync(string key)
-    {
-        return _db.KeyExistsAsync($"{KeyPrefix}{key}");
-    }
-
-    public Task SetHashAsync(string key, IReadOnlyDictionary<string, string> hash)
-    {
-        return _db.HashSetAsync($"{KeyPrefix}{key}", hash.Select(pair => new HashEntry(pair.Key, pair.Value)).ToArray());
-    }
-
-    public async Task<Dictionary<string, string>> GetHashAsync(string key)
-    {
-        var entries = await _db.HashGetAllAsync($"{KeyPrefix}{key}");
-        var result = new Dictionary<string, string>();
-        foreach (var entry in entries)
-        {
-            result.Add(entry.Name.ToString(), entry.Value.ToString());
-        }
-
-        return result;
-    }
-}
-
-public static class RedisCacheExtensions
-{
-    public static IServiceCollection AddRedisCache(this IServiceCollection services, RedisCacheOptions options)
-    {
-        var conn = ConnectionMultiplexer.Connect(options.Configuration 
-                                                 ?? throw new ApplicationException("未配置Redis连接字符串"));
-        services.AddSingleton<ICacher>(new RedisCache(conn, options.Database, options.InstanceName));
-        return services;
-    }
-}

ConsoleApp2/ConsoleApp2.csproj

@@ -19,6 +19,7 @@
     </ItemGroup>
 
     <ItemGroup>
+      <PackageReference Include="Microsoft.Extensions.Caching.StackExchangeRedis" Version="8.0.0" />
       <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
       <PackageReference Include="MySqlConnector" Version="2.3.3" />
       <PackageReference Include="Serilog" Version="3.1.2-dev-02097" />
@@ -26,7 +27,6 @@
       <PackageReference Include="Serilog.Sinks.Console" Version="5.0.1" />
       <PackageReference Include="Serilog.Sinks.File" Version="5.0.1-dev-00972" />
       <PackageReference Include="ServiceStack.Text" Version="8.0.0" />
-      <PackageReference Include="StackExchange.Redis" Version="2.7.17" />
       <PackageReference Include="ZstdSharp.Port" Version="0.7.4" />
     </ItemGroup>
 

ConsoleApp2/Const/ProcessStep.cs

@@ -2,5 +2,6 @@
 
 public static class ProcessStep
 {
-    public const string Produce = "Producer";
+    public const string Producer = "Producer";
+    public const string Consumer = "Consumer";
 }

ConsoleApp2/Const/TableNames.cs

@@ -1,38 +0,0 @@
-namespace ConsoleApp2.Const;
-
-public static class TableNames
-{
-    public const string Machine = "machine";
-    public const string Order = "order";
-    public const string OrderBlockPlan = "order_block_plan";
-    public const string OrderBlockPlanItem = "order_block_plan_item";
-    public const string OrderBlockPlanResult = "order_block_plan_result";
-    public const string OrderBoxBlock = "order_box_block";
-    public const string OrderDataBlock = "order_data_block";
-    public const string OrderDataGoods = "order_data_goods";
-    public const string OrderDataParts = "order_data_parts";
-    public const string OrderItem = "order_item";
-    public const string OrderModule = "order_module";
-    public const string OrderModuleExtra = "order_module_extra";
-    public const string OrderModuleItem = "order_module_item";
-    public const string OrderPackage = "order_package";
-    public const string OrderPackageItem = "order_package_item";
-    public const string OrderPatchDetail = "order_patch_detail";
-    public const string OrderProcess = "order_process";
-    public const string OrderProcessSchedule = "order_process_schdule";
-    public const string OrderProcessStep = "order_process_step";
-    public const string OrderProcessStepItem = "order_process_step_item";
-    public const string OrderScrapBoard = "order_scrap_board";
-    public const string ProcessGroup = "process_group";
-    public const string ProcessInfo = "process_info";
-    public const string ProcessItemExp = "process_item_exp";
-    public const string ProcessScheduleCapacity = "process_schdule_capacity";
-    public const string ProcessStepEfficiency = "process_step_efficiency";
-    public const string ReportTemplate = "report_template";
-    public const string SimplePackage = "simple_package";
-    public const string SimplePlanOrder = "simple_plan_order";
-    public const string SysConfig = "sys_config";
-    public const string WorkCalendar = "work_calendar";
-    public const string WorkShift = "work_shift";
-    public const string WorkTime = "work_time";
-}

ConsoleApp2/DataRecord.cs

@@ -1,13 +1,16 @@
-namespace ConsoleApp2;
+using System.ComponentModel.Design;
 
-public class DataRecord : ICloneable
+namespace ConsoleApp2;
+
+
+public class DataRecord
 {
     public static bool TryGetField(DataRecord record, string columnName, out string value)
     {
         value = string.Empty;
         if (record.Headers is null)
             throw new InvalidOperationException("Cannot get field when headers of a record have not been set.");
-        var idx = IndexOfIgnoreCase(record.Headers, columnName);
+        var idx = Array.IndexOf(record.Headers, columnName);
         if (idx == -1)
             return false;
         value = record.Fields[idx];
@@ -18,46 +21,34 @@ public class DataRecord : ICloneable
     {
         if (record.Headers is null)
             throw new InvalidOperationException("Headers have not been set.");
-        var idx = IndexOfIgnoreCase(record.Headers, columnName);
+        var idx = Array.IndexOf(record.Headers, columnName);
         if (idx is -1)
-            throw new IndexOutOfRangeException(
+            throw new IndexOutOfRangeException($"Column name {columnName} not found in this record, table name {record.TableName}.");
-                $"Column name '{columnName}' not found in this record, table name '{record.TableName}'.");
         return record.Fields[idx];
     }
 
-    private static int IndexOfIgnoreCase(IList<string> list, string value)
-    {
-        var idx = -1;
-        for (var i = 0; i < list.Count; i++)
-        {
-            if (list[i].Equals(value, StringComparison.OrdinalIgnoreCase)) 
-            {
-                idx = i;
-                break;
-            }
-        }
 
-        return idx;
+    public string[] Fields { get; }
-    }
+
+    public string[] Headers { get; }
 
-    public string? RawField { get; set; }
-    public IList<string> Fields { get; }
-    public IList<string> Headers { get; }
     public string TableName { get; }
-    public string? Database { get; set; }
+
+    public string Database { get; set; }
+    public int CompanyID { get; set; }
 
 
-    public DataRecord(IEnumerable<string> fields, string tableName, IEnumerable<string> headers, string? database = null)
+    public DataRecord(string[] fields, string tableName, string[] headers, int companyID=0)
     {
-        Fields = fields.ToList();
+        if (fields.Length != headers.Length)
-        TableName = tableName;
-        Headers = headers.ToList();
-        Database = database;
-        
-        if (Fields.Count != Headers.Count)
             throw new ArgumentException(
-                $"The number of fields does not match the number of headers. Expected: {Headers.Count} Got: {Fields.Count} Fields: {string.Join(',', Fields)}",
+                $"The number of fields does not match the number of headers. Expected: {headers.Length} Got: {fields.Length} Fields: {string.Join(',', fields)}",
                 nameof(fields));
+
+        Fields = fields;
+        TableName = tableName;
+        Headers = headers;
+        CompanyID = companyID;
     }
 
     public string this[int index]
@@ -72,46 +63,32 @@ public class DataRecord : ICloneable
         set => SetField(columnName, value);
     }
 
-    public int FieldCount => Fields.Count;
+    public int Count => Fields.Length;
 
     public bool TryGetField(string columnName, out string value) => TryGetField(this, columnName, out value);
 
-    public bool SetField(string columnName, string value) => SetField(this, columnName, value);
+    public bool SetField(string columnName,  string value) => SetField(this, columnName,value);
 
-    public bool SetField(DataRecord record, string columnName, string value)
+    public string GetCacheKey(string columnName) => GetCacheKey(this, columnName);
+
+    public  bool SetField( DataRecord record,string columnName,string value)
     {
         if (record.Headers is null)
             throw new InvalidOperationException("Headers have not been set.");
-        var idx = IndexOfIgnoreCase(record.Headers, columnName);
+        var idx = Array.IndexOf(record.Headers, columnName);
         if (idx is -1)
-            throw new IndexOutOfRangeException(
+            throw new IndexOutOfRangeException("Column name not found in this record.");
-                $"Column name '{columnName}' not found in this record, table name '{record.TableName}");
         record.Fields[idx] = value;
         return true;
     }
-    
+    public string GetCacheKey(DataRecord record, string columnName)
-    public void AddField(string columnName, string value)
     {
-        if (IndexOfIgnoreCase(Headers, columnName) != -1)
+        if (TryGetField(record, columnName, out var  value))
-            throw new InvalidOperationException($"{TableName}: 列名 '{columnName}' 已存在");
+        {
+            return $"{TableName}_{value}";
+        }else
+            throw new IndexOutOfRangeException($"Column name:{columnName} not found in this record.");
 
-        Fields.Add(value);
-        Headers.Add(columnName);
-    }
 
-    public void RemoveField(string columnName)
-    {
-        var idx = IndexOfIgnoreCase(Headers, columnName);
-        if (idx == -1)
-            throw new InvalidOperationException($"{TableName}: 列名 '{columnName}' 不存在");
-        
-        Fields.RemoveAt(idx);
-        Headers.Remove(columnName);
-    }
-
-    public bool HeaderExists(string columnName) => IndexOfIgnoreCase(Headers, columnName) != -1;
-    public object Clone()
-    {
-        return new DataRecord(new List<string>(Fields), TableName, new List<string>(Headers), Database);
     }
 }

ConsoleApp2/Helpers/Database/DatabaseHelper.cs

@@ -1,54 +0,0 @@
-using System.Data;
-using MySqlConnector;
-
-namespace ConsoleApp2.Helpers.Database;
-
-public static class DatabaseHelper
-{
-    public static async Task<DataSet> QueryTableAsync(string connStr, string sql)
-    {
-        await using var conn = new MySqlConnection(connStr);
-        if(conn.State is not ConnectionState.Open)
-            await conn.OpenAsync();
-        await using var cmd = conn.CreateCommand();
-        cmd.CommandText = sql;
-        var ds = new DataSet();
-        var adapter = new MySqlDataAdapter(cmd).Fill(ds);
-        return ds;
-    }
-
-    public static async Task<object?> QueryScalarAsync(string connStr, string sql)
-    {
-        await using var conn = new MySqlConnection(connStr);
-        if(conn.State is not ConnectionState.Open)
-            await conn.OpenAsync();
-        await using var cmd = conn.CreateCommand();
-        cmd.CommandText = sql;
-        return await cmd.ExecuteScalarAsync();
-    }
-
-    public static async Task<int> NonQueryAsync(string connStr, string sql)
-    {
-        await using var conn = new MySqlConnection(connStr);
-        if(conn.State is not ConnectionState.Open)
-            await conn.OpenAsync();
-        await using var cmd = conn.CreateCommand();
-        cmd.CommandText = sql;
-        return await cmd.ExecuteNonQueryAsync();
-    }
-
-    public static async Task<int> TransactionAsync(string connStr, string sql, params MySqlParameter[] parameters)
-    {
-        await using var conn = new MySqlConnection(connStr);
-        if(conn.State is not ConnectionState.Open)
-            await conn.OpenAsync();
-        await using var trans = await conn.BeginTransactionAsync();
-        await using var cmd = conn.CreateCommand();
-        cmd.CommandText = sql;
-        cmd.Transaction = trans;
-        cmd.Parameters.AddRange(parameters);
-        var rows = await cmd.ExecuteNonQueryAsync();
-        await trans.CommitAsync();
-        return rows;
-    }
-}

ConsoleApp2/Helpers/DumpDataHelper.cs

@@ -1,5 +1,6 @@
-using System.Text.RegularExpressions;
+using ConsoleApp2.Options;
-using ZstdSharp;
+using System.Text;
+using System.Text.RegularExpressions;
 
 namespace ConsoleApp2.Helpers;
 
@@ -10,41 +11,32 @@ public static partial class DumpDataHelper
     [GeneratedRegex(@"\([^)]*\)")]
     private static partial Regex MatchBrackets();
     
-    /// <summary>
+    
-    /// 从MyDumper导出的SQL文件内容中读取表头
+    public static async Task<string[]> GetCsvHeadersFromSqlFileAsync(string txt)
-    /// </summary>
-    /// <param name="content"></param>
-    /// <returns></returns>
-    /// <exception cref="ArgumentException"></exception>
-    public static string[] GetCsvHeadersFromSqlFile(string content)
     {
-        var match = MatchBrackets().Match(content);
+        //var txt = await File.ReadAllTextAsync(filePath);
-        if (!match.Success)
+        var match = MatchBrackets().Match(txt);
-            throw new ArgumentException("输入的SQL内容有误，无法提取表头", nameof(content));
+
         return ParseHeader(match.ValueSpan);
-        
-        string[] ParseHeader(ReadOnlySpan<char> headerStr)
-        {
-            headerStr = headerStr[1..^1];
-            Span<Range> ranges = stackalloc Range[50];
-            var count = headerStr.Split(ranges, ',');
-            var arr = new string[count];
-
-            for (var i = 0; i < count; i++)
-            {
-                arr[i] = headerStr[ranges[i]].Trim("@`").ToString(); // 消除列名的反引号，如果是变量则消除@
-            }
-        
-            return arr;
-        }
     }
 
-    /// <summary>
+    private static string[] ParseHeader(ReadOnlySpan<char> headerStr)
-    /// 从MyDumper导出的Csv文件名解析出表名
+    {
-    /// </summary>
+        headerStr = headerStr[1..^1];
-    /// <param name="filePath"></param>
+        Span<Range> ranges = stackalloc Range[50];
-    /// <returns></returns>
+        var count = headerStr.Split(ranges, ',');
-    public static string GetTableNameFromCsvFileName(ReadOnlySpan<char> filePath)
+        var arr = new string[count];
+
+        for (var i = 0; i < count; i++)
+        {
+            arr[i] = headerStr[ranges[i]].Trim("@`").ToString(); // 消除列名的反引号，如果是变量则消除@
+        }
+
+        return arr;
+    }
+    
+    
+    public static string GetTableName(ReadOnlySpan<char> filePath)
     {
         filePath = filePath[(filePath.LastIndexOf('\\') + 1)..];
         var firstDotIdx = -1;
@@ -68,24 +60,13 @@ public static partial class DumpDataHelper
         return filePath[(firstDotIdx+1)..secondDotIdx].ToString();
     }
 
-    /// <summary>
+    public static async Task<string[]> GetCsvFileNamesFromSqlFileAsync(string txt,Regex regex)
-    /// 从MyDumper导出的SQL文件内容中读取CSV文件名
-    /// </summary>
-    /// <param name="txt"></param>
-    /// <param name="regex"></param>
-    /// <returns></returns>
-    public static Task<string[]> GetCsvFileNamesFromSqlFileAsync(string txt, Regex regex)
     {
         //var txt = await File.ReadAllTextAsync(filePath);
         var matches = regex.Matches(txt);
-        return Task.FromResult(matches.Select(match => match.ValueSpan[1..^1].ToString()).ToArray());
+        return matches.Select(match => match.ValueSpan[1..^1].ToString()).ToArray();
     }
     
-    /// <summary>
-    /// 检查字符串是否为16进制
-    /// </summary>
-    /// <param name="str"></param>
-    /// <returns></returns>
     public static bool CheckHexField(string? str)
     {
         if (string.IsNullOrWhiteSpace(str))
@@ -110,16 +91,8 @@ public static partial class DumpDataHelper
         return true;
     }
 
-    /// <summary>
+    // public static string EliminateEscapeChars(ReadOnlySpan<char> str)
-    /// 将输入流以ZSTD标准解压为字符串
+    // {
-    /// </summary>
+    //     char[] escapeChars = ['0','\''];
-    /// <param name="stream"></param>
+    // }
-    /// <returns></returns>
-    public static async Task<string> DecompressZstAsStringAsync(Stream stream)
-    {
-        await using var ds = new DecompressionStream(stream);
-        var reader = new StreamReader(ds);
-        return await reader.ReadToEndAsync();
-    }
-
 }

ConsoleApp2/Helpers/Extensions.IEnumerable.cs

@@ -1,67 +0,0 @@
-using System.Diagnostics.CodeAnalysis;
-using System.Reflection;
-
-namespace ConsoleApp2.Helpers;
-
-public static class EnumerableExtensions
-{
-    public static string ToMarkdownTable<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicMethods)]T>(this IEnumerable<T> source)
-    {
-        var properties = typeof(T).GetRuntimeProperties();
-        var fields = typeof(T)
-            .GetRuntimeFields()
-            .Where(f => f.IsPublic);
-
-        var gettables = Enumerable.Union(
-            properties.Select(p => new { p.Name, GetValue = (Func<object, object>)p.GetValue, Type = p.PropertyType }),
-            fields.Select(p => new { p.Name, GetValue = (Func<object, object>)p.GetValue, Type = p.FieldType }));
-
-        var maxColumnValues = source
-            .Select(x => gettables.Select(p => p.GetValue(x)?.ToString()?.Length ?? 0))
-            .Union(new[] { gettables.Select(p => p.Name.Length) }) // Include header in column sizes
-            .Aggregate(
-                new int[gettables.Count()].AsEnumerable(),
-                (accumulate, x) => accumulate.Zip(x, Math.Max))
-            .ToArray();
-
-        var columnNames = gettables.Select(p => p.Name);
-
-        var headerLine = "| " + string.Join(" | ", columnNames.Select((n, i) => n.PadRight(maxColumnValues[i]))) + " |";
-
-        var isNumeric = new Func<Type, bool>(type =>
-            type == typeof(Byte) ||
-            type == typeof(SByte) ||
-            type == typeof(UInt16) ||
-            type == typeof(UInt32) ||
-            type == typeof(UInt64) ||
-            type == typeof(Int16) ||
-            type == typeof(Int32) ||
-            type == typeof(Int64) ||
-            type == typeof(Decimal) ||
-            type == typeof(Double) ||
-            type == typeof(Single));
-
-        var rightAlign = new Func<Type, char>(type => isNumeric(type) ? ':' : ' ');
-
-        var headerDataDividerLine =
-            "| " +
-            string.Join(
-                "| ",
-                gettables.Select((g, i) => new string('-', maxColumnValues[i]) + rightAlign(g.Type))) +
-            "|";
-
-        var lines = new[]
-        {
-            headerLine,
-            headerDataDividerLine,
-        }.Union(
-            source
-                .Select(s =>
-                    "| " + string.Join(" | ",
-                        gettables.Select((n, i) => (n.GetValue(s)?.ToString() ?? "").PadRight(maxColumnValues[i]))) +
-                    " |"));
-
-        return lines
-            .Aggregate((p, c) => p + Environment.NewLine + c);
-    }
-}

ConsoleApp2/Helpers/Extensions.String.cs

@@ -5,12 +5,6 @@ namespace ConsoleApp2.Helpers;
 
 public static class StringExtensions
 {
-    /// <summary>
-    /// 截断字符串
-    /// </summary>
-    /// <param name="this"></param>
-    /// <param name="maxLength">截断长度</param>
-    /// <returns></returns>
     public static string Omit(this ReadOnlySpan<char> @this, int maxLength)
     {
         if (@this.Length > maxLength)
@@ -18,20 +12,8 @@ public static class StringExtensions
         return @this.ToString();
     }
 
-    /// <summary>
-    /// 截断字符串
-    /// </summary>
-    /// <param name="this"></param>
-    /// <param name="maxLength">截断长度</param>
-    /// <returns></returns>
     public static string Omit(this string @this, int maxLength) => Omit(@this.AsSpan(), maxLength);
 
-    /// <summary>
-    /// 将16进制字符串转换为字符串
-    /// </summary>
-    /// <param name="hexString"></param>
-    /// <param name="encoding"></param>
-    /// <returns></returns>
     public static string FromHex(ReadOnlySpan<char> hexString, Encoding? encoding = null)
     {
         encoding ??= Encoding.UTF8;
@@ -56,11 +38,6 @@ public static class StringExtensions
         return encoding.GetString(bytes);
     }
     
-    /// <summary>
-    /// 检查是否为JSON字符串
-    /// </summary>
-    /// <param name="hexStr"></param>
-    /// <returns></returns>
     public static bool CheckJsonHex(ReadOnlySpan<char> hexStr)
     {
         if (hexStr.Length < 2)

ConsoleApp2/Helpers/Extensions.Task.cs

@@ -1,19 +0,0 @@
-namespace ConsoleApp2.Helpers;
-
-public static class TaskExtensions
-{
-    public static async Task WaitUntil(Func<bool> condition, int pollDelay = 25, CancellationToken ct = default)
-    {
-        try
-        {
-            while (!condition())
-            {
-                await Task.Delay(pollDelay, ct);
-            }
-        }
-        catch(TaskCanceledException)
-        {
-            // CancellationToken激活时Task.Delay通过抛异常来结束停止等待，不用管它
-        }
-    }
-}

ConsoleApp2/Helpers/ValidateConsole.cs

@@ -0,0 +1,31 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace ConsoleApp2.Helpers
+{
+    public static class ValidateConsole
+    {
+      public static void ValidateInput<T>(Func<string,bool> converter,string message)
+        {
+            Console.Write(message);
+           string ? input = Console.ReadLine();
+            while (true)
+            {
+                if (!string.IsNullOrEmpty(input))
+                {
+                    var result = converter(input);
+                    if (result == false)
+                    {
+                        Console.WriteLine($"输入的内容不合法,请重新输入!");
+                        input = Console.ReadLine();
+                    }
+                    else break;
+                }
+                break;
+            }
+        }
+    }
+}

ConsoleApp2/HostedServices/Abstractions/IDataSource.cs

@@ -6,9 +6,8 @@ using System.Threading.Tasks;
 
 namespace ConsoleApp2.HostedServices.Abstractions
 {
-    public interface IDataReader : IDisposable
+    public interface  IDataSource:IDisposable
     {
-        DataRecord Current { get; }
+        public Task DoEnqueue(Action<DataRecord> action);
-        ValueTask<bool> ReadAsync();
     }
 }

ConsoleApp2/HostedServices/Abstractions/IInputService.cs

@@ -5,5 +5,5 @@ namespace ConsoleApp2.HostedServices.Abstractions;
 
 public interface IInputService
 {
-    public Task ExecuteAsync(CancellationToken cancellationToken);
+    public Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, ProcessContext context, CancellationToken cancellationToken);
 }

ConsoleApp2/HostedServices/Abstractions/IOutputService.cs

@@ -5,5 +5,5 @@ namespace ConsoleApp2.HostedServices.Abstractions;
 
 public interface IOutputService
 {
-    public Task ExecuteAsync(CancellationToken ct);
+    public Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken);
 }

ConsoleApp2/HostedServices/Abstractions/ITransformService.cs

@@ -5,5 +5,5 @@ namespace ConsoleApp2.HostedServices.Abstractions;
 
 public interface ITransformService
 {
-    public Task ExecuteAsync(CancellationToken cancellationToken);
+    public Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken);
 }

ConsoleApp2/HostedServices/FileInputService.cs

@@ -1,111 +0,0 @@
-using ConsoleApp2.Const;
-using ConsoleApp2.HostedServices.Abstractions;
-using ConsoleApp2.Options;
-using ConsoleApp2.Services;
-using ConsoleApp2.Services.ETL;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Options;
-
-namespace ConsoleApp2.HostedServices;
-
-public record FileInputInfo
-{
-    public required string FileName { get; init; }
-    public required string TableName { get; init; }
-    public required string[] Headers { get; init; }
-}
-
-public enum FileInputType
-{
-    MyDumperCsv,
-    MyDumperZst,
-    ErrorLog,
-}
-
-/// <summary>
-/// 从输入目录中导入文件
-/// </summary>
-public class FileInputService : IInputService
-{
-    private readonly ILogger _logger;
-    private readonly DataRecordQueue _producerQueue;
-    private readonly IOptions<DataInputOptions> _dataInputOptions;
-    private readonly ProcessContext _context;
-    private readonly DataReaderFactory _dataReaderFactory;
-
-    public FileInputService(ILogger<FileInputService> logger,
-            IOptions<DataInputOptions> dataInputOptions,
-            ProcessContext context, 
-            [FromKeyedServices(ProcessStep.Produce)] DataRecordQueue producerQueue, 
-            DataReaderFactory dataReaderFactory)
-    {
-        _logger = logger;
-        _dataInputOptions = dataInputOptions;
-        _context = context;
-        _producerQueue = producerQueue;
-        _dataReaderFactory = dataReaderFactory;
-    }
-
-    public async Task ExecuteAsync(CancellationToken cancellationToken)
-    {
-        var inputDir = _dataInputOptions.Value.InputDir ?? throw new ApplicationException("未配置文件输入目录");
-        _logger.LogInformation("***** Input service started, working directory: {InputDir} *****", inputDir);
-        
-        var trans = _dataInputOptions.Value.FileInputMetaBuilder;
-        if(trans is null) throw new ApplicationException("未配置文件名-表名映射委托");
-        FileInputInfo[] infoArr = Directory.GetFiles(inputDir)
-            .Select(f => trans(f))
-            .Where(info => info is not null).ToArray()!;
-        
-        var orderedInfo = GetFilesInOrder(infoArr).ToArray();
-        
-        _logger.LogInformation("***** {Count} files founded in directory，{OrderedCount} files is matched with configuration *****", infoArr.Length, orderedInfo.Length);
-        foreach (var info in orderedInfo)
-        {
-            _logger.LogDebug("Table {TableName}: {FileName}", info.TableName, info.FileName);
-        }
-
-        foreach (var info in orderedInfo)
-        {
-            _logger.LogInformation("Reading file: {FileName}, table: {TableName}", info.FileName, info.TableName);
-            var source = _dataReaderFactory.CreateReader(info.FileName,info.TableName,info.Headers);
-
-            while (await source.ReadAsync())
-            {
-                var record = source.Current;
-                _producerQueue.Enqueue(record);
-                _context.AddInput();
-            }
-            
-            _logger.LogInformation("Input of table: '{TableName}' finished", info.TableName);
-        }
-
-        _context.CompleteInput();
-        _logger.LogInformation("***** Input service finished *****");
-    }
-
-    /// <summary>
-    /// 读取配置，按照配置的表顺序来返回
-    /// </summary>
-    /// <returns></returns>
-    private IEnumerable<FileInputInfo> GetFilesInOrder(FileInputInfo[] inputFiles)
-    {
-        var tableOrder = _dataInputOptions.Value.TableOrder;
-        if (tableOrder is null or { Length: 0 })
-            return inputFiles;
-
-        return Yield();
-
-        IEnumerable<FileInputInfo> Yield()
-        {
-            foreach (var tableName in tableOrder)
-            {
-                var target = inputFiles.FirstOrDefault(f =>
-                    f.TableName.Equals(tableName, StringComparison.OrdinalIgnoreCase));
-                if (target is not null)
-                    yield return target;
-            }
-        }
-    }
-}

ConsoleApp2/HostedServices/InputService.cs

@@ -0,0 +1,59 @@
+using ConsoleApp2.Const;
+using ConsoleApp2.Helpers;
+using ConsoleApp2.HostedServices.Abstractions;
+using ConsoleApp2.Options;
+using ConsoleApp2.Services;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace ConsoleApp2.HostedServices;
+
+/// <summary>
+/// 从MyDumper导出的CSV文件中导入表头和数据
+/// </summary>
+public class InputService : IInputService
+{
+    private readonly ILogger _logger;
+    private readonly IOptions<DataInputOptions> _dataInputOptions;
+    private readonly ProcessContext _context;
+
+    public InputService(ILogger<InputService> logger,
+            IOptions<DataInputOptions> dataInputOptions,
+            ProcessContext context)
+    {
+        _logger = logger;
+        _dataInputOptions = dataInputOptions;
+        _context = context;
+    }
+
+    public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, ProcessContext context,CancellationToken cancellationToken)
+    {
+        var inputDir = _dataInputOptions.Value.InputDir;
+        _logger.LogInformation("***** Csv input service start, working dir: {InputDir}, thread id: {ThreadId} *****", inputDir, Environment.CurrentManagedThreadId);
+        var files = Directory.GetFiles(inputDir);
+        if (files.Length == 0)
+        {
+            _logger.LogInformation("No source files found in {InputDir}", inputDir);
+            return;
+        }
+        var count = 0;
+        foreach (var tableName in tasksOptions.TableInfoConfig.Keys)
+        {
+            _logger.LogInformation("Working table: {tableName}", tableName);
+            var source = _dataInputOptions.Value.CreateSource?.Invoke(tableName);
+            await source.DoEnqueue((record) =>
+            {
+                _context.AddInput();
+                producerQueue.Enqueue(record);
+                count++;
+
+            });
+            
+            _logger.LogInformation("table:'{tableName}' input completed", tableName);
+        }
+
+        context.CompleteInput();
+        _logger.LogInformation("***** Csv input service completed *****");
+    }
+}

ConsoleApp2/HostedServices/MainHostedService.cs

@@ -1,226 +1,110 @@
-using System.Diagnostics;
+using ConsoleApp2.HostedServices.Abstractions;
-using System.Text;
-using ConsoleApp2.Helpers;
-using ConsoleApp2.Helpers.Database;
-using ConsoleApp2.HostedServices.Abstractions;
 using ConsoleApp2.Options;
 using ConsoleApp2.Services;
-using ConsoleApp2.Services.ErrorRecorder;
-using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Options;
-
 namespace ConsoleApp2.HostedServices;
 
 public class MainHostedService : BackgroundService
 {
-    private Stopwatch? _stopwatch;
+    private readonly ILogger _logger;
     private readonly IInputService _input;
     private readonly ITransformService _transform;
     private readonly IOutputService _output;
-    private readonly ILogger _logger;
     private readonly ProcessContext _context;
 
-    private readonly IOptions<DatabaseOutputOptions> _databaseOptions;
+    public MainHostedService(ILogger<MainHostedService> logger, IInputService input, ITransformService transform, IOutputService output, ProcessContext context)
-    private readonly IOptions<TenantDbOptions> _tenantDbOptions;
-    private readonly IConfiguration _config;
-
-    public MainHostedService(IInputService input, 
-        ITransformService transform,
-        IOutputService output, 
-        ILogger<MainHostedService> logger, 
-        IOptions<TenantDbOptions> tenantDbOptions, 
-        IOptions<DatabaseOutputOptions> databaseOptions, 
-        IConfiguration config, 
-        ProcessContext context)
     {
+        _logger = logger;
         _input = input;
         _transform = transform;
         _output = output;
-        _logger = logger;
-        _tenantDbOptions = tenantDbOptions;
-        _databaseOptions = databaseOptions;
-        _config = config;
         _context = context;
     }
 
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
     {
-        _stopwatch = Stopwatch.StartNew();
+        var taskFun = (TasksOptions taskOp, DataRecordQueue producerQueue, DataRecordQueue consumerQueue, ProcessContext context) =>
-        var inputTask = ExecuteAndCatch(
-            async () => await _input.ExecuteAsync(stoppingToken), "文件输入程序出现异常", stoppingToken);
-        var transformTask = ExecuteAndCatch(
-            async () => await _transform.ExecuteAsync(stoppingToken), "转换程序出现异常", stoppingToken);
-        var outputTask = ExecuteAndCatch(
-            async () => await _output.ExecuteAsync(stoppingToken), "输出程序出现异常", stoppingToken);
-
-        await Task.WhenAll(inputTask, transformTask, outputTask);
-        _stopwatch.Stop();
-        _logger.LogInformation("***** All tasks completed *****");
-        _logger.LogInformation("***** ElapseTime: {Time}", (_stopwatch.ElapsedMilliseconds / 1000f).ToString("F3"));
-        await Task.Delay(5000, stoppingToken);
-
-
-        if (!stoppingToken.IsCancellationRequested)
         {
-            await ExportResultAsync();
+            var inputTask = Task.Run(async () =>
-            _logger.LogInformation("The execution result export to {Path}",
-                Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"Result-{ErrorRecorder.UID}.md"));
-            if (_config["RestoreIndex"] is not null)
-                await RestoreIndexAsync();
-
-            Environment.Exit(0);
-        }
-        else Environment.Exit(1);
-    }
-    
-    private Task ExecuteAndCatch(Func<Task> func, string message, CancellationToken ct)
-    {
-        return Task.Run(async () =>
-        {
-            try
             {
-                await func();
+                try
-            }
+                {
-            catch (Exception e)
+                    await _input.ExecuteAsync(taskOp, producerQueue, context, stoppingToken);
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogCritical("Exception occurred on inputService:{Message},{StackTrace}", ex.Message, ex.StackTrace);
+                    throw;
+                }
+
+            });
+            var transformTask = Task.Run(async () =>
             {
-                _logger.LogCritical(e, "{Msg}\t{ErrMsg}", message, e.Message);
+                try
-                _context.AddException(e);
+                {
-                Environment.Exit(1);
+                    await _transform.ExecuteAsync(taskOp, producerQueue, consumerQueue, context, stoppingToken);
-            }
+                }
-        }, ct);
+                catch (Exception ex)
-    }
+                {
+                    _logger.LogCritical("Exception occurred on transformService:{Message},{StackTrace}", ex.Message, ex.StackTrace);
+                    throw;
+                }
 
-    /// <summary>
+            });
-    /// 还原所有数据库的索引...
+            var outputTask = Task.Run(async () =>
-    /// </summary>
+            {
-    /// <returns></returns>
+                try
-    /// <exception cref="ApplicationException"></exception>
+                {
-    private Task RestoreIndexAsync()
+                    await _output.ExecuteAsync(taskOp, consumerQueue, context,stoppingToken);
-    {
+                }
-        var databases = _tenantDbOptions.Value.DbList?.Keys 
+                catch (Exception ex)
-                        ?? throw new ApplicationException("无法还原索引，因为分库配置中没有配置数据库");
+                {
-        var connStr = _databaseOptions.Value.ConnectionString 
+                    _logger.LogCritical("Exception occurred on outputService:{Message},{StackTrace}", ex.Message, ex.StackTrace);
-                      ?? throw new ApplicationException("无法还原索引，因为没有配置数据库连接字符串");
+                    throw;
-        var list = new List<Task>();
+                }
-        foreach(var db in databases)
-        {
-            var task = DatabaseHelper.NonQueryAsync(connStr + $";Database={db};",
-                """
-                CREATE INDEX `idx_CompanyID` ON `machine` (`CompanyID`);
 
-                CREATE INDEX `idx_companyid` ON `order` (`CompanyID`);
+            });
-
-                CREATE INDEX `idx_CompanyID` ON `order_block_plan` (`CompanyID`);
-
-                CREATE INDEX `idx_PlanID` ON `order_block_plan_item` (`PlanID`);
-
-                CREATE INDEX `idx_orderno` ON `order_box_block` (`OrderNo`);
-
-                CREATE INDEX `index_OrderNo` ON `order_data_block` (`OrderNo`);
-
-                CREATE INDEX `index_OrderNo` ON `order_data_goods` (`OrderNo`);
-
-                CREATE INDEX `index_OrderNo` ON `order_data_parts` (`OrderNo`);
-
-                CREATE INDEX `index_ItemNo` ON `order_item` (`ItemNo`);
-
-                CREATE INDEX `index_OrderNo` ON `order_item` (`OrderNo`);
-
-                CREATE INDEX `index_PackageID` ON `order_item` (`PackageID`);
-
-                CREATE INDEX `index_PlanID` ON `order_item` (`PlanID`);
-
-                CREATE INDEX `idx_OrderNo` ON `order_module` (`OrderNo`);
-
-                CREATE INDEX `index_OrderNo` ON `order_module_extra` (`OrderNo`);
-
-                CREATE INDEX `index_OrderNo` ON `order_module_item` (`OrderNo`);
-
-                CREATE INDEX `idx_OrderNo` ON `order_package` (`OrderNo`);
-
-                CREATE INDEX `idx_PakageNo` ON `order_package` (`PakageNo`);
-
-                CREATE INDEX `idx_PackageID` ON `order_package_item` (`PackageID`);
-
-                CREATE INDEX `idx_companyid` ON `order_patch_detail` (`CompanyID`);
-
-                CREATE INDEX `idx_OrderNo` ON `order_process` (`OrderNo`);
-
-                CREATE INDEX `index_CompanyID` ON `order_process_schdule` (`CompanyID`);
-
-                CREATE INDEX `IX_order_process_step_OrderProcessID` ON `order_process_step` (`OrderProcessID`);
-
-                CREATE INDEX `idx_OrderProcessID` ON `order_process_step_item` (`OrderProcessID`);
-
-                CREATE INDEX `idx_OrderProcessStepID` ON `order_process_step_item` (`OrderProcessStepID`);
-
-                CREATE INDEX `idx_CompanyID` ON `order_scrap_board` (`CompanyID`);
-
-                CREATE INDEX `idx_CompanyID` ON `process_group` (`CompanyID`);
-
-                CREATE INDEX `idx_CompanyID` ON `process_info` (`CompanyID`);
-
-                CREATE INDEX `index_CompanyID` ON `process_item_exp` (`CompanyID`);
-
-                CREATE INDEX `idx_CompanyID` ON `process_schdule_capacity` (`CompanyID`);
-
-                CREATE INDEX `idx_CompanyID` ON `process_step_efficiency` (`CompanyID`);
-
-                CREATE INDEX `idx_CompanyID` ON `report_template` (`CompanyID`);
-
-                CREATE INDEX `indx_OrderNo` ON `simple_package` (`OrderNo`);
-
-                CREATE INDEX `idx_CompanyID` ON `simple_plan_order` (`CompanyID`);
-
-                CREATE INDEX `idx_CompanyID` ON `sys_config` (`CompanyID`);
-
-                CREATE INDEX `idx` ON `work_calendar` (`CompanyID`);
-
-                CREATE INDEX `idx_CompanyID` ON `work_shift` (`CompanyID`);
-
-                CREATE INDEX `IX_work_time_ShiftID` ON `work_time` (`ShiftID`);
-                """);
-            list.Add(task);
-        }
-
-        return Task.WhenAll(list);
-    }
-
-    private async Task ExportResultAsync()
-    {
-        var sb = new StringBuilder();
-        if (_context.HasException)
-            sb.AppendLine("# Program Completed With Error");
-        else sb.AppendLine("# Program Completed Successfully");
-        sb.AppendLine("## Process Count");
-        var processCount = new[]
-        {
-            new { State = "Input", Count = _context.InputCount },
-            new { State = "Transform", Count = _context.TransformCount },
-            new { State = "Output", Count = _context.OutputCount }
         };
-        sb.AppendLine(processCount.ToMarkdownTable());
+
-        sb.AppendLine("\n---\n");
+        var bigTablesDic = new Dictionary<string, TableInfo>
-        sb.AppendLine("## Table Output Progress");
-        var tableOutputProgress = _context.TableProgress.Select(pair =>
-            new { Table = pair.Key, Count = pair.Value });
-        sb.AppendLine(tableOutputProgress.ToMarkdownTable());
-        sb.AppendLine("\n---\n");
-        sb.AppendLine("## Result");
-        var elapsedTime = (_stopwatch!.ElapsedMilliseconds / 1000f);
-        var result = new[]
         {
-            new { Field = "ElapsedTime", Value = elapsedTime.ToString("F2") },
+            {"order_block_plan",new TableInfo{SimulaRowCount=2725553 }},//CreateTime < 202301的删除 
-            new
+            {"order_block_plan_result",new TableInfo{SimulaRowCount=1174096 }},
-            {
+            {"order_box_block",new TableInfo{SimulaRowCount=29755672 }},
-                Field = "Average Output Speed",
+            {"order_item",new TableInfo{SimulaRowCount=1345520079 }},
-                Value = (_context.OutputCount / elapsedTime).ToString("F2") + "records/s"
+            {"simple_plan_order",new TableInfo{SimulaRowCount=351470 }},//CreateTime < 202301的删除
-            }
         };
-        sb.AppendLine(result.ToMarkdownTable());
+        taskFun(new TasksOptions { TableInfoConfig = bigTablesDic, OutPutOptions = new OutPutOptions { FlushCount = 10000, OutPutTaskCount = 2 } },
-        await File.WriteAllTextAsync(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"Result-{ErrorRecorder.UID}.md"),
+            new DataRecordQueue(), new DataRecordQueue(),new ProcessContext());
-            sb.ToString());
+        var smallTablesDic = new Dictionary<string, TableInfo>
+        {
+            {"machine",new TableInfo{SimulaRowCount=14655 }},
+            {"order",new TableInfo{SimulaRowCount=5019216 }},
+            {"order_data_block",new TableInfo{SimulaRowCount=731800334 }},
+            {"order_data_goods",new TableInfo{SimulaRowCount=25803671 }},
+            {"order_data_parts",new TableInfo{SimulaRowCount=468517543 }},
+            {"order_module",new TableInfo{SimulaRowCount=103325385 }},
+            {"order_module_extra",new TableInfo{SimulaRowCount=54361321 }},
+            {"order_module_item",new TableInfo{SimulaRowCount=69173339 }},
+            {"order_package",new TableInfo{SimulaRowCount=16196195 }},
+            {"order_process",new TableInfo{SimulaRowCount=3892685 }},//orderNo < 202301的  
+            {"order_process_step",new TableInfo{SimulaRowCount=8050349 }},//orderNo < 202301的删除   
+            {"order_process_step_item",new TableInfo{SimulaRowCount=14538058 }},//orderNo < 202301的删除 
+            {"order_scrap_board",new TableInfo{SimulaRowCount=123998 }},
+            {"process_group",new TableInfo{SimulaRowCount=1253 }},
+            {"process_info",new TableInfo{SimulaRowCount=7839 }},
+            {"process_item_exp",new TableInfo{SimulaRowCount=28 }},
+            {"process_schdule_capacity",new TableInfo{SimulaRowCount=39736 }},
+            {"process_step_efficiency",new TableInfo{SimulaRowCount=8 }},
+            {"report_template",new TableInfo{SimulaRowCount=7337 }},
+            {"simple_package",new TableInfo{SimulaRowCount=130436 }},//orderNo < 202301的删除
+            {"sys_config",new TableInfo{SimulaRowCount=2296 }},
+            {"work_calendar",new TableInfo{SimulaRowCount=11 }},
+            {"work_shift",new TableInfo{SimulaRowCount=59 }},
+            {"work_time",new TableInfo{SimulaRowCount=62 }},
+        };
+        taskFun(new TasksOptions { TableInfoConfig = smallTablesDic, OutPutOptions = new OutPutOptions { FlushCount = 20000, OutPutTaskCount = 4 } },
+            new DataRecordQueue(), new DataRecordQueue(), new ProcessContext());
     }
 }

ConsoleApp2/HostedServices/OutputService.cs

@@ -2,11 +2,8 @@
 using ConsoleApp2.HostedServices.Abstractions;
 using ConsoleApp2.Options;
 using ConsoleApp2.Services;
-using ConsoleApp2.Services.ErrorRecorder;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
-using MySqlDestination = ConsoleApp2.Services.ETL.MySqlDestination;
-using TaskExtensions = ConsoleApp2.Helpers.TaskExtensions;
 
 namespace ConsoleApp2.HostedServices;
 
@@ -17,116 +14,96 @@ public class OutputService : IOutputService
 {
     private readonly ILogger _logger;
     private readonly IOptions<DatabaseOutputOptions> _outputOptions;
+    private readonly IOptions<DataTransformOptions> _transformOptions;
     private readonly ProcessContext _context;
-    private readonly ErrorRecorderFactory _errorRecorderFactory;
+    private readonly TaskManager _taskManager;
-    private readonly RecordQueuePool _queuePool;
+    private readonly ErrorRecorder _errorRecorder;
 
     public OutputService(ILogger<OutputService> logger,
         IOptions<DatabaseOutputOptions> outputOptions,
         ProcessContext context,
-        RecordQueuePool queuePool, 
+        TaskManager taskManager, 
-        ErrorRecorderFactory errorRecorderFactory)
+        IOptions<DataTransformOptions> transformOptions, 
+        ErrorRecorder errorRecorder)
     {
         _logger = logger;
         _outputOptions = outputOptions;
         _context = context;
-        _queuePool = queuePool;
+        _taskManager = taskManager;
-        _errorRecorderFactory = errorRecorderFactory;
+        _transformOptions = transformOptions;
+        _errorRecorder = errorRecorder;
     }
 
-    public async Task ExecuteAsync(CancellationToken ct)
+    public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken)
     {
-        _logger.LogInformation("***** Output service started *****");
+        _logger.LogInformation("***** Mysql output service started *****");
-        var dbTaskManager = new TaskManager(5);
+        _taskManager.CreateTasks(async () =>
-        var dbTasks = new Dictionary<string, Task>();
-        while (!_context.IsTransformCompleted)
         {
-            foreach (var (db, queue) in _queuePool.Queues)
+            //k: database v: records，按照要导出的数据库名分组
+            var databaseDict = new Dictionary<string, List<DataRecord>>();
+            while (!context.IsTransformCompleted || consumerQueue.Count > 0)
             {
-                if (!dbTasks.ContainsKey(db))
+                if (!consumerQueue.TryDequeue(out var record)) continue;
+                var dbName = record.Database;
+                var records = databaseDict.AddOrUpdate(dbName, [record], (_, list) =>
                 {
-                    dbTasks.Add(db, await dbTaskManager.CreateTaskAsync(
+                    list.Add(record);
-                        async () => await StartDatabaseWorker(db, queue, ct), ct));
+                    return list;
+                });
+                
+                if (records.Count >= tasksOptions.OutPutOptions.FlushCount)
+                {
+                    await FlushAsync(dbName, records);
+                    records.Clear();
                 }
             }
 
-            await Task.Delay(500, ct);
+            foreach (var (db, records) in databaseDict)
-        }
-
-        await TaskExtensions.WaitUntil(() => dbTaskManager.RunningTaskCount == 0, 25, ct);
-        
-        _context.CompleteOutput();
-        _logger.LogInformation("***** Output service finished *****");
-    }
-
-    private async Task StartDatabaseWorker(string db, DataRecordQueue queue, CancellationToken ct = default)
-    {
-        _logger.LogInformation("*****开启输出线程，数据库: {db} *****", db);
-        var taskManager = new TaskManager(_outputOptions.Value.MaxDatabaseOutputTask);
-        var tmp = new List<DataRecord>();
-        while (!_context.IsTransformCompleted || queue.Count > 0)
-        {
-            if (ct.IsCancellationRequested)
-                break;
-            
-            if (!queue.TryDequeue(out var record)) continue;
-            
-            var dbName = record.Database ?? throw new ApplicationException("输出的记录缺少数据库名");
-            if(dbName != db)
-                throw new ApplicationException($"输出记录的数据与当前输出线程不匹配，记录：{dbName}, 输出线程：{db}");
-            tmp.Add(record);
-
-            if (tmp.Count >= _outputOptions.Value.FlushCount)
             {
-                var list = tmp;
+                if (records.Count > 0)
-                tmp = [];
-                await taskManager.CreateTaskAsync(async arg => // 转换为方法组
                 {
-                    var tuple = arg as Tuple<string, List<DataRecord>>;
+                    await FlushAsync(db, records);
-                    try
+                    records.Clear();
-                    {
+                }
-                        await FlushAsync(tuple!.Item1, tuple.Item2);
-                    }
-                    catch (Exception e)
-                    {
-                        _logger.LogError(e, "输出记录时发生错误");
-                        throw;
-                    }
-                }, Tuple.Create(dbName, list), ct);
             }
-        }
 
-        // 等待所有子任务完成
+            databaseDict.Clear();
-        await TaskExtensions.WaitUntil(() => taskManager.RunningTaskCount == 0, 10, ct);
+            _logger.LogInformation("***** Mysql output thread completed *****");
+        }, tasksOptions.OutPutOptions.OutPutTaskCount);
 
-        // 清理剩余记录
+        await _taskManager.WaitAll();
-        if (tmp.Count > 0)
+        //_context.CompleteOutput();
-        {
+        _logger.LogInformation("***** Mysql output service completed *****");
-            await FlushAsync(db, tmp);
-        }
 
-        _logger.LogInformation("*****输出线程结束，数据库: {db} *****", db);
     }
 
     private async Task FlushAsync(string dbName, IEnumerable<DataRecord> records)
     {
-        var connStr = _outputOptions.Value.ConnectionString ??
+        var count = 0;
-                      throw new InvalidOperationException("连接字符串为空");
+        var connStr = _outputOptions.Value.ConnectionString ?? throw new InvalidOperationException("ConnectionString is null");
-        await using var output = new MySqlDestination($"{connStr};Database={dbName};", _logger,
+        await using var output = new MySqlDestination($"{connStr};Database={dbName};", _logger, _context, _transformOptions, _errorRecorder);
-            _outputOptions, _errorRecorderFactory.CreateOutput(dbName), _context);
+        //if (records == null || records.Count() == 0) return;
+        //var dbName = $"cferp_test_1";
+        //if (records != null && records.Count() > 0)
+        //{
+        //    dbName = $"cferp_test_{records.FirstOrDefault()?.CompanyID}";
+        //}
 
-        var tableOutput = new Dictionary<string, int>();
+        //await  using var output = new MySqlDestination(new MySqlConnectionStringBuilder
+        //{
+        //    Server = "127.0.0.1",
+        //    Port = 34309,
+        //    Database = dbName,
+        //    UserID = "root",
+        //    Password = "123456",
+        //    MaximumPoolSize = 50,
+        //}.ConnectionString, _logger,true);
         foreach (var record in records)
         {
             await output.WriteRecordAsync(record);
-            tableOutput.AddOrUpdate(record.TableName, 1, (_, v) => v + 1);
+            count++;
         }
-
         await output.FlushAsync(_outputOptions.Value.MaxAllowedPacket);
-        foreach (var (key, value) in tableOutput)
+        _context.AddOutput(count);
-        {
-            _context.AddTableOutput(key, value);
-        }
-        _logger.LogTrace("Flushed {Count} records", tableOutput.Values.Sum(i => i));
     }
 }

ConsoleApp2/HostedServices/TaskMonitorService.cs

@@ -1,7 +1,6 @@
 using System.Diagnostics;
 using ConsoleApp2.Const;
 using ConsoleApp2.Services;
-using ConsoleApp2.Services.Loggers;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
@@ -13,33 +12,33 @@ namespace ConsoleApp2.HostedServices;
 /// </summary>
 public class TaskMonitorService : BackgroundService
 {
+    private readonly IHostApplicationLifetime _lifetime;
     private readonly ILogger<TaskMonitorService> _logger;
-    private readonly IEnumerable<ITaskMonitorLogger> _monitorLoggers;
     private readonly ProcessContext _context;
     private readonly DataRecordQueue _producerQueue;
-    private readonly RecordQueuePool _queuePool;
+    private readonly DataRecordQueue _consumerQueue;
 
-    public TaskMonitorService(
+    public TaskMonitorService(IHostApplicationLifetime lifetime,
         ILogger<TaskMonitorService> logger,
         ProcessContext context,
-        [FromKeyedServices(ProcessStep.Produce)]
+        [FromKeyedServices(ProcessStep.Producer)]
         DataRecordQueue producerQueue,
-        RecordQueuePool queuePool,
+        [FromKeyedServices(ProcessStep.Consumer)]
-        IEnumerable<ITaskMonitorLogger> monitorLoggers)
+        DataRecordQueue consumerQueue)
     {
+        _lifetime = lifetime;
         _logger = logger;
         _context = context;
         _producerQueue = producerQueue;
-        _queuePool = queuePool;
+        _consumerQueue = consumerQueue;
-        _monitorLoggers = monitorLoggers;
     }
 
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
     {
-        await Task.Run(() => Monitor(stoppingToken), stoppingToken);
+        await Task.Factory.StartNew(Monitor, stoppingToken);
     }
 
-    private async Task Monitor(CancellationToken stoppingToken)
+    private async Task Monitor()
     {
         var sw = Stopwatch.StartNew();
         var lastTime = sw.ElapsedMilliseconds;
@@ -48,7 +47,7 @@ public class TaskMonitorService : BackgroundService
         var lastOutputCount = _context.OutputCount;
 
         bool endCheck = false;
-        while (!stoppingToken.IsCancellationRequested)
+        while (true)
         {
             EndCheck:
             // var running = 0;
@@ -87,36 +86,18 @@ public class TaskMonitorService : BackgroundService
             // _logger.LogInformation(
             //     "Task monitor: running: {Running}, error: {Error}, completed: {Completed}, canceled: {Canceled}, outputSpeed: {Speed} records/s",
             //     running, error, completed, canceled, outputSpeed);
-            foreach (var logger in _monitorLoggers)
+            _logger.LogInformation(
-            {
+                "Process monitor: input: {inputStatus}, transform: {transformStatus}, output: {outputStatus}\nInput: {InputCount}, Transform: {TransformCount}, Output: {OutputCount}",
-                logger.LogStatus("Monitor: Progress status", new Dictionary<string, string>
+                _context.IsInputCompleted ? "completed" : $"running {inputSpeed:F2} records/s",
-                {
+                _context.IsTransformCompleted ? "completed" : $"running {transformSpeed:F2} records/s",
-                    {"Input",_context.IsInputCompleted ? "completed" : $"running {inputSpeed:F2} records/s" },
+                _context.IsOutputCompleted ? "completed" : $"running {outputSpeed:F2} records/s",
-                    {"Transform", _context.IsTransformCompleted ? "completed" : $"running {transformSpeed:F2} records/s" },
+                inputCount,
-                    {"Output", _context.IsOutputCompleted ? "completed" : $"running {outputSpeed:F2} records/s" }
+                transformCount,
-                });
+                outputCount);
+            _logger.LogInformation("Queue monitor: producer queue: {ProducerQueue}, consumer queue: {ConsumerQueue}",
+                _producerQueue.Count, _consumerQueue.Count);
 
-                logger.LogStatus("Monitor: Table output progress",
+            await Task.Delay(5000);
-                    _context.TableProgress
-                        .ToDictionary(kv => kv.Key, kv => kv.Value.ToString()),
-                    ITaskMonitorLogger.LogLevel.Progress);
-            
-                logger.LogStatus("Monitor: Process count", new Dictionary<string, string>
-                {
-                    {"Input", inputCount.ToString()},
-                    {"Transform", transformCount.ToString()},
-                    {"Output", outputCount.ToString()}
-                }, ITaskMonitorLogger.LogLevel.Progress);
-            
-                logger.LogStatus("Monitor: Queue", new Dictionary<string, string>
-                {
-                    {"Producer queue records", _producerQueue.Count.ToString() },
-                    {"Output queues", _queuePool.Queues.Count.ToString() },
-                    {"Output queue records", _queuePool.Queues.Values.Sum(queue => queue.Count).ToString()}, 
-                });
-            }
-            
-            await Task.Delay(5000, stoppingToken);
 
             lastTime = time;
             lastInputCount = inputCount;
@@ -133,6 +114,9 @@ public class TaskMonitorService : BackgroundService
                 break;
             }
         }
-    }
         
+        _logger.LogInformation("***** All tasks completed *****");
+        _logger.LogInformation("***** ElapseTime: {Time}", (sw.ElapsedMilliseconds / 1000f).ToString("F3"));
+        // _lifetime.StopApplication();
+    }
 }

ConsoleApp2/HostedServices/TestInputService.cs

@@ -0,0 +1,83 @@
+using ConsoleApp2.HostedServices.Abstractions;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using ConsoleApp2.Const;
+using ConsoleApp2.Options;
+using ConsoleApp2.Services;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using System.Reflection.PortableExecutable;
+using System.Collections.Concurrent;
+using ConsoleApp2.SimulationService;
+
+namespace ConsoleApp2.HostedServices
+{
+    public class TestInputService : IInputService
+    {
+        private readonly ILogger _logger;
+        private readonly IOptions<CsvOptions> _csvOptions;
+        private readonly DataRecordQueue _producerQueue;
+        private readonly ProcessContext _context;
+        public TestInputService(ILogger<TestInputService> logger,
+    IOptions<CsvOptions> csvOptions,
+    [FromKeyedServices(ProcessStep.Producer)] DataRecordQueue producerQueue,
+    ProcessContext context)
+        {
+            _logger = logger;
+            _csvOptions = csvOptions;
+            _producerQueue = producerQueue;
+            _context = context;
+        }
+        public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, ProcessContext context, CancellationToken cancellationToken)
+        {
+            var tableName = "order_item";
+            var headers = new string[] { "ID","OrderNo","ItemNo","ItemType","RoomID","BoxID","DataID","PlanID","PackageID","Num","CompanyID","ShardKey" };
+            var dataCount = 1200000000L;
+            var tempCount = 80000;
+            var tempRecords=new List<DataRecord>();
+            var comanyID = 1;
+            short[] shareKeys = { 23040, 23070, 23100, 24000, 24040, 24070, 24100, 25000, 25040, 25070, 25100 };
+            int[] companyIds = { 1, 2, 3, 4 };
+            var sk = shareKeys.First();
+            var companyID = companyIds.First();
+
+            var shareKeyInterval = 20000;
+            var getShareKeyTimes = 0;
+            var getCompanyIDTimes = 0;
+            var shareKeyIntervalCount = 0;
+            for (long i = 1; i <= dataCount; i++)
+            {
+                shareKeyIntervalCount++;
+                if (shareKeyIntervalCount > shareKeyInterval) {
+                    sk=DataHelper.GetShareKey(getShareKeyTimes);
+                    getShareKeyTimes++;
+                    shareKeyIntervalCount = 0;
+                }
+                var fields = new string[] { i.ToString(), "20220104020855", (220105981029 + i).ToString(), "1", "482278", "482279", "3768774", "0", "0", "1", companyID.ToString(), sk.ToString() };
+                var record = new DataRecord(fields, tableName, headers, comanyID);
+                tempRecords.Add(record);
+                if (tempRecords.Count >= tempCount)
+                {
+                    foreach (var rc in tempRecords)
+                    {
+                        _context.AddInput();
+                        _producerQueue.Enqueue(rc);
+                        if (cancellationToken.IsCancellationRequested)
+                            return;
+                    }
+                    tempRecords.Clear();
+                    companyID = DataHelper. GetCompanyId(getCompanyIDTimes);
+                    getCompanyIDTimes++;
+                }
+            }
+           
+            _context.CompleteInput();
+            _logger.LogInformation("***** Csv input service completed *****");
+        }
+
+    }
+}

ConsoleApp2/HostedServices/TransformService.cs

@@ -1,17 +1,15 @@
-using ConsoleApp2.Cache;
+using ConsoleApp2.Const;
-using ConsoleApp2.Const;
 using ConsoleApp2.HostedServices.Abstractions;
 using ConsoleApp2.Options;
 using ConsoleApp2.Services;
-using ConsoleApp2.Services.ErrorRecorder;
+using Microsoft.Extensions.Caching.Distributed;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using StackExchange.Redis;
 
 namespace ConsoleApp2.HostedServices;
 
-public record DataTransformContext(DataRecord Record, ICacher Cacher, ILogger Logger);
-
 /// <summary>
 /// 数据处理服务，对导入后的数据进行处理
 /// </summary>
@@ -19,99 +17,65 @@ public class TransformService : ITransformService
 {
     private readonly ILogger _logger;
     private readonly IOptions<DataTransformOptions> _options;
-    private readonly DataRecordQueue _producerQueue;
-    private readonly RecordQueuePool _queuePool;
     private readonly ProcessContext _context;
-    private readonly ICacher _cache;
+    private readonly IDistributedCache _cache;
-    private readonly ErrorRecorderFactory _errorRecorderFactory;
+    private readonly TaskManager _taskManager;
 
 
     public TransformService(ILogger<TransformService> logger, 
         IOptions<DataTransformOptions> options,
-        [FromKeyedServices(ProcessStep.Produce)] DataRecordQueue producerQueue,
-        RecordQueuePool queuePool,
         ProcessContext context,
-        ICacher cache, 
+        IDistributedCache cache,
-        ErrorRecorderFactory errorRecorderFactory)
+        TaskManager taskManager)
     {
         _logger = logger;
         _options = options;
-        _producerQueue = producerQueue;
-        _queuePool = queuePool;
         _context = context;
         _cache = cache;
-        _errorRecorderFactory = errorRecorderFactory;
+        _taskManager = taskManager;
     }
 
-    public async Task ExecuteAsync(CancellationToken cancellationToken)
+    public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken)
     {
         _logger.LogInformation("***** Data transform service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId);
 
-        while (!_context.IsInputCompleted || _producerQueue.Count > 0)
+        _taskManager.CreateTasks(async () =>
         {
-            if (!_producerQueue.TryDequeue(out var record)) continue;
+            while ((!context.IsInputCompleted || producerQueue.Count > 0))
-
-            try
             {
-                var context = new DataTransformContext(record, _cache, _logger);
+                if (!producerQueue.TryDequeue(out var record)) continue;
-                if (_options.Value.EnableFilter)
+
+                //过滤不要的record
+                if (await _options.Value.RecordFilter?.Invoke(record, _cache) == false) continue;
+                //修改record
+                _options.Value.RecordModify?.Invoke(record);
+                //缓存record
+                await _options.Value.RecordCache?.Invoke(record, _cache);
+                //替换record
+                var replaceRecord = await _options.Value.RecordReplace?.Invoke(record, _cache);
+                if (replaceRecord != null)
                 {
-                    // 数据过滤
+                    record = replaceRecord;
-                    var filter = _options.Value.RecordFilter;
-                    if (filter is not null && await filter(context) == false) continue;
                 }
-
-                if (_options.Value.EnableReplacer)
-                {
-                    // 数据替换
-                    var replacer = _options.Value.RecordModify;
-                    if (replacer is not null)
-                    {
-                        record = await replacer(context);
-                    }
-                }    
-                
-                // 字段缓存
-                var cacher = _options.Value.RecordCache;
-                if(cacher is not null)
-                    await cacher.Invoke(context);
-            
                 //计算需要分流的数据库
-                var dbFilter = _options.Value.DatabaseFilter 
+                record.Database = _options.Value.DatabaseFilter.Invoke(record);
-                               ?? throw new ApplicationException("未配置数据库过滤器");
+                consumerQueue.Enqueue(record);
-                record.Database = dbFilter(record);
-            
-                _queuePool[record.Database].Enqueue(record);
                 _context.AddTransform();
-
+                //数据增加
-                if (_options.Value.EnableReBuilder)
+                var addRecords = _options.Value.RecordAdd?.Invoke(record);
+                if (addRecords is { Count: > 0 })
                 {
-                    //数据重建
+                    foreach (var rc in addRecords)
-                    var addRecords = _options.Value.RecordReBuild?.Invoke(context);
-                    if (addRecords is { Count: > 0 })
                     {
-                        foreach (var rc in addRecords)
+                        rc.Database = _options.Value.DatabaseFilter.Invoke(record);
-                        {
+                        consumerQueue.Enqueue(rc);
-                            if(dbFilter is not null)
+                        _context.AddTransform();
-                                rc.Database =dbFilter.Invoke(record);
-                            _queuePool[record.Database].Enqueue(rc);
-                            _context.AddTransform();
-                        }
                     }
                 }
             }
-            catch (Exception e)
+            context.CompleteTransform();
-            {
+        },tasksOptions.TransformTaskCount,cancellationToken);
-                _context.AddException(e);
+        await _taskManager.WaitAll();
-                var errorRecorder = _errorRecorderFactory.CreateTransform();
+        _logger.LogInformation("***** Data transformation service completed *****");
-                await errorRecorder.LogErrorRecordAsync(record, e);
-                if (!_options.Value.StrictMode)
-                    _logger.LogError(e, "数据转换时发生错误");
-                else throw;
-            }
-        }
-        _context.CompleteTransform();
-        
-        _logger.LogInformation("***** Data transformation service finished *****");
     }
 }

ConsoleApp2/HostedServices/VoidOutputService.cs

@@ -1,5 +1,9 @@
-using ConsoleApp2.HostedServices.Abstractions;
+using ConsoleApp2.Const;
+using ConsoleApp2.HostedServices.Abstractions;
+using ConsoleApp2.Options;
 using ConsoleApp2.Services;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 
 namespace ConsoleApp2.HostedServices;
@@ -8,36 +12,28 @@ namespace ConsoleApp2.HostedServices;
 public class VoidOutputService : IOutputService
 {
     private readonly ILogger _logger;
-    private readonly RecordQueuePool _queuePool;
+    private readonly DataRecordQueue _consumerQueue;
     private readonly ProcessContext _context;
 
-    public VoidOutputService(
+    public VoidOutputService([FromKeyedServices(ProcessStep.Consumer)] DataRecordQueue consumerQueue,
-        ProcessContext context, ILogger<VoidOutputService> logger, RecordQueuePool queuePool)
+        ProcessContext context, ILogger<VoidOutputService> logger)
     {
+        _consumerQueue = consumerQueue;
         _context = context;
         _logger = logger;
-        _queuePool = queuePool;
     }
 
-    public Task ExecuteAsync(CancellationToken ct)
+    public Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken)
     {
-        _logger.LogInformation("***** Void Output Service Started *****");
+        _logger.LogInformation("***** Void output service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId);
-        while (!_context.IsTransformCompleted || _queuePool.Queues.Count > 0)
+        while (!_context.IsTransformCompleted || _consumerQueue.Count > 0)
         {
-            foreach (var pair in _queuePool.Queues) // 内存优化
+            if (_consumerQueue.TryDequeue(out var record))
-            {
-                if (_context.IsTransformCompleted && pair.Value.Count == 0)
-                {
-                    _queuePool.RemoveQueue(pair.Key);
-                    continue;
-                }
-                if(!pair.Value.TryDequeue(out var record)) continue;
                 _context.AddOutput();
-            }
         }
 
         _context.CompleteOutput();
-        _logger.LogInformation("***** Void Output Service Stopped *****");
+        _logger.LogInformation("***** Void output service completed *****");
         return Task.CompletedTask;
     }
 }

ConsoleApp2/Options/CommandOptions.cs

@@ -0,0 +1,21 @@
+using System;
+using System.ComponentModel;
+using System.Configuration;
+namespace ConsoleApp2.Options
+{
+    public class CommandOptions
+    {
+
+        public string InputDir { get; set; } = "./MyDumper";
+
+        public int TaskCount { get; set; } = 16;
+
+        public int FlushCount { get; set; } = 20000;
+
+        public bool Isutf8mb4 { get; set; } = true;
+
+        public short OldestShardKey { get; set; } = 23010;
+        public string OldestTime { get; set; } = "202301";
+
+    }
+}

ConsoleApp2/Options/CsvOptions.cs

@@ -0,0 +1,19 @@
+namespace ConsoleApp2.Options;
+
+public class CsvOptions
+{
+    /// <summary>
+    /// MyDumper导出的CSV文件目录
+    /// </summary>
+    //public string InputDir { get; set; } = "./";
+
+    /// <summary>
+    /// 字符串的包围符号，默认为双引号"
+    /// </summary>
+    public char QuoteChar { get; set; } = '"';
+
+    /// <summary>
+    /// 每个字段的分割符，默认逗号,
+    /// </summary>
+    public string Delimiter { get; set; } = ",";
+}

ConsoleApp2/Options/DataInputOptions.cs

@@ -1,53 +1,18 @@
-using ConsoleApp2.HostedServices;
+using ConsoleApp2.Services;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection.PortableExecutable;
+using System.Text;
+using System.Threading.Tasks;
 
 namespace ConsoleApp2.Options
 {
+    public enum InputFileType { CSV, JWT, JSV }
     public class DataInputOptions
     {
-        public string? InputDir { get; set; }
+        public string InputDir { get; set; } = "./";
 
-        #region CSV
+        public Func<string, CsvSource>? CreateSource {  get; set; }
-        
-        /// <summary>
-        /// 字符串的包围符号，默认为双引号"
-        /// </summary>
-        public char QuoteChar { get; set; } = '"';
-
-        /// <summary>
-        /// 每个字段的分割符，默认逗号,
-        /// </summary>
-        public string Delimiter { get; set; } = ",";
-
-        #endregion
-        
-        /// <summary>
-        /// yyyyMM
-        /// </summary>
-        public string CleanDate { get; set; } = "202301";
-
-        #region Mock
-
-        public bool UseMock { get; set; }
-
-        public double MockCountMultiplier { get; set; } = 1;
-        
-        /// <summary>
-        /// Table -> Mock Count 暂时为手动配置
-        /// </summary>
-        public Dictionary<string, TableMockConfig>? TableMockConfig { get; set; }
-        
-        #endregion
-
-
-        #region ManualSet
-        
-        public string[]? TableOrder { get; set; }
-        
-        /// <summary>
-        /// 配置如何从文件名转换为表名和表头
-        /// </summary>
-        public Func<string, FileInputInfo?>? FileInputMetaBuilder { get; set; } //TODO: 抽离
-
-        #endregion
     }
 }

ConsoleApp2/Options/DataTransformOptions.cs

@@ -1,6 +1,5 @@
-using ConsoleApp2.Cache;
+using Microsoft.Extensions.Caching.Distributed;
-using ConsoleApp2.HostedServices;
+using StackExchange.Redis;
-using Microsoft.Extensions.Logging;
 
 namespace ConsoleApp2.Options;
 
@@ -14,34 +13,23 @@ public enum ColumnType
 
 public class DataTransformOptions
 {
-    public bool StrictMode { get; set; } = true;
+    public Func<DataRecord, string>? DatabaseFilter { get; set; }
-    public bool EnableFilter { get; set; } = true;
+
-    public bool EnableReplacer { get; set; } = true;
+    public Func<string, string>? TransformBinary { get; set; }//Binary转字符串方法
-    public bool EnableReBuilder { get; set; } = true;
+
+    public Func<DataRecord, IDistributedCache, Task<bool>>? RecordFilter { get; set; }//数据过滤方法
+    public Action<DataRecord>? RecordModify { get; set; }//数据修改
+    public Func<DataRecord, IDistributedCache, Task<DataRecord?>>? RecordReplace { get; set; }//数据替换
+    public Func<DataRecord, IList<DataRecord>?>? RecordAdd { get; set; }//数据替换
+    public Func<DataRecord, IDistributedCache, Task>? RecordCache { get; set; }//数据缓存
 
     /// <summary>
-    /// Record -> Database name
+    /// 配置导入数据的特殊列
-    /// 对记录进行数据库过滤
     /// </summary>
-    public Func<DataRecord, string>? DatabaseFilter { get; set; }
+    public Dictionary<string, ColumnType> ColumnTypeConfig { get; set; } = new(); // "table.column" -> type
-    /// <summary>
+
-    /// Context -> Should output
+    public ColumnType GetColumnType(string table, string column)
-    /// 配置对数据过滤的条件
+    {
-    /// </summary>
+        return ColumnTypeConfig.GetValueOrDefault($"{table}.{column}", ColumnType.UnDefine);
-    public Func<DataTransformContext, Task<bool>>? RecordFilter { get; set; }//数据过滤方法
+    }
-    /// <summary>
-    /// Context -> New record
-    /// 对当前记录进行修改或完整替换
-    /// </summary>
-    public Func<DataTransformContext, Task<DataRecord>>? RecordModify { get; set; }//数据替换
-    /// <summary>
-    /// Context -> New rebuild records
-    /// 使用当前记录对某些数据进行重建
-    /// </summary>
-    public Func<DataTransformContext, IList<DataRecord>?>? RecordReBuild { get; set; }//新增数据
-    /// <summary>
-    /// Context -> void
-    /// 对数据的某些字段进行缓存
-    /// </summary>
-    public Func<DataTransformContext, Task>? RecordCache { get; set; }//数据缓存
 }

ConsoleApp2/Options/DatabaseOutputOptions.cs

@@ -2,24 +2,9 @@
 
 public class DatabaseOutputOptions
 {
-    public string? ConnectionString { get; set; }
-
-    public int MaxAllowedPacket { get; set; } = 64 * 1024 * 1024;
-
-    public int FlushCount { get; set; } = 10000;
-
-    public int MaxDatabaseOutputTask { get; set; } = 4;
-
-    public bool TreatJsonAsHex { get; set; } = true;
-    
-
     /// <summary>
-    /// 配置导入数据的特殊列
+    /// 数据库连接字符串
     /// </summary>
-    public Dictionary<string, ColumnType> ColumnTypeConfig { get; set; } = new(); // "table.column" -> type
+    public string? ConnectionString { get; set; }
-
+    public int MaxAllowedPacket { get; set; } = 64*1024*1024;
-    public ColumnType GetColumnType(string table, string column)
-    {
-        return ColumnTypeConfig.GetValueOrDefault($"{table}.{column}", ColumnType.UnDefine);
-    }
 }

ConsoleApp2/Options/RedisCacheOptions.cs

@@ -1,8 +0,0 @@
-namespace ConsoleApp2.Options;
-
-public class RedisCacheOptions
-{
-    public string? Configuration { get; init; }
-    public string InstanceName { get; init; } = "";
-    public int Database { get; init; } = 0;
-}

ConsoleApp2/Options/TableMockConfig.cs

@@ -1,32 +0,0 @@
-namespace ConsoleApp2.Options;
-
-public struct TableMockConfig
-{
-    /// <summary>
-    /// 使用深拷贝
-    /// </summary>
-    public bool UseDeepCopy { get; set; }
-    /// <summary>
-    /// 模拟数据量
-    /// </summary>
-    public long MockCount { get; set; }
-
-    /// <summary>
-    /// 需要开启MockCount
-    /// </summary>
-    public string[]? AutoIncrementColumn { get; set; } = null; // TODO: 换为自定义委托
-    
-    public void Deconstruct(out bool useDeepCopy, out long mockCount, out string[]? autoIncrementColumn)
-    {
-        useDeepCopy = UseDeepCopy;
-        mockCount = MockCount;
-        autoIncrementColumn = AutoIncrementColumn;
-    }
-
-    public TableMockConfig(bool useDeepCopy, long mockCount, string[]? autoIncrementColumn)
-    {
-        UseDeepCopy = useDeepCopy;
-        MockCount = mockCount;
-        AutoIncrementColumn = autoIncrementColumn;
-    }
-}

ConsoleApp2/Options/TasksOptions.cs

@@ -0,0 +1,24 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace ConsoleApp2.Options
+{
+    public class TableInfo
+    {
+        public long SimulaRowCount { get; set; }//模拟的记录条数
+    }
+    public class TasksOptions
+    {
+        public Dictionary<string, TableInfo> TableInfoConfig { get; set; } = new();
+        public int TransformTaskCount { get; set; } = 1;
+        public OutPutOptions OutPutOptions { get; set; }=new();
+    }
+    public class OutPutOptions
+    {
+        public int FlushCount { get; set; } = 10000;
+        public int OutPutTaskCount { get; set; } = 2;
+    }
+}

ConsoleApp2/Options/TenantDbOptions.cs

@@ -2,23 +2,22 @@ namespace ConsoleApp2.Options;
 
 public class TenantDbOptions
 {
-    public string? TenantKey { get; set; }
+    public string TenantKey { get; set; }
 
     /// <summary>
     /// Key-Value: {DbName}-{TenantKeyLessThan}
     /// </summary>
-    public Dictionary<string, int>? DbList { get; set; }
+    public Dictionary<string, int> DbList { get; set; }
 
     public string GetDbNameByTenantKeyValue(int tenantKeyValue)
     {
         // var dictionary = new SortedDictionary<int, string>();
         // DbList.ForEach(pair => dictionary.Add(pair.Value, pair.Key));
         // 注意配置顺序
-        if(DbList is null) throw new ApplicationException("分库配置中没有发现任何数据库");
         var dbName = DbList.Cast<KeyValuePair<string, int>?>()
             .FirstOrDefault(pair => pair?.Value != null && pair.Value.Value > tenantKeyValue)!.Value.Key;
         return dbName ??
                throw new ArgumentOutOfRangeException(nameof(tenantKeyValue),
-                   $"分库配置中没有任何符合'{nameof(tenantKeyValue)}'值的数据库");
+                   $"已配置的数据库中没有任何符合'{nameof(tenantKeyValue)}'值的对象");
     }
 }

ConsoleApp2/Program.cs

@@ -1,509 +1,429 @@
-#define USE_TEST_DB // 测试库的结构与生产库不一样，如果使用测试库运行，则加上USE_TEST_DB预处理器指令
+using ConsoleApp2;
-
-using ConsoleApp2;
-using ConsoleApp2.Cache;
 using ConsoleApp2.Const;
-using ConsoleApp2.Helpers;
 using ConsoleApp2.HostedServices;
 using ConsoleApp2.HostedServices.Abstractions;
 using ConsoleApp2.Options;
 using ConsoleApp2.Services;
-using ConsoleApp2.Services.ErrorRecorder;
+using Microsoft.Extensions.Caching.StackExchangeRedis;
-using ConsoleApp2.Services.ETL;
-using ConsoleApp2.Services.Loggers;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using MySqlConnector;
 using Serilog;
+using Microsoft.Extensions.Caching.Distributed;
 using Serilog.Events;
 
+
 await RunProgram();
 return;
 
 async Task RunProgram()
 {
+
+
+    //var inputDir= "D:\\MyDumper";
+    //ValidateConsole.ValidateInput<string>((_inputDir) =>
+    //{
+    //    if (Directory.Exists(_inputDir))
+    //     {
+    //        inputDir = _inputDir;
+    //        return true;
+    //    }
+    //    else return false;
+    //}, "请输入读取csv文件的目录(默认为当前目录下MyDumper文件夹):");
+
+    //var maxTask = 16;
+    //ValidateConsole.ValidateInput<string>((_inputDir) =>
+    //{
+
+    //    _ = int.TryParse(_inputDir.ToString(), out var _taskCount);
+    //    if (_taskCount > 0) {
+    //        maxTask = _taskCount;
+    //        return true;
+    //    }
+    //    else return false;
+    //}, "请输入执行输出的线程数量(默认为16):");
+
+    //var flushCount = 2_0000;
+    //ValidateConsole.ValidateInput<string>((_inputDir) =>
+    //{
+    //    _ = int.TryParse(_inputDir.ToString(), out var _flashCount);
+    //    if (_flashCount > 0)
+    //    {
+    //        flushCount = _flashCount;
+    //        return true;
+    //    } else return false; 
+
+    //}, "请输入单次插入的行数(默认为20000):");
+
     ThreadPool.SetMaxThreads(200, 200);
     var host = Host.CreateApplicationBuilder(args);
     host.Configuration.AddJsonFile(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "appsettings.json"), false, false);
-    host.Configuration.AddCommandLine(args, new Dictionary<string, string>
+    var commandOptions = host.Configuration.GetSection("CmdOptions").Get<CommandOptions>() ?? 
+                         throw new ApplicationException("未发现配置文件");
+    Console.WriteLine($"InputDir:{commandOptions?.InputDir}");
+
+    var oldestTime = DateTime.ParseExact(commandOptions.OldestTime, "yyyyMM", System.Globalization.DateTimeFormatInfo.InvariantInfo);
+    //host.Services.Configure<InputTableOptions>(option =>
+    //{
+    //    option.TableInfoConfig = new Dictionary<string, TableInfo>
+    //    {
+
+    //        //order_block_plan_item从order_item表查询，然后程序插入                
+    //        //order_package_item从order_item表查询，然后程序插入
+    //        //order_patch_detail生产没有这个表，不处理
+
+
+    //        {"machine",new TableInfo{SimulaRowCount=14655 }},
+    //        {"order",new TableInfo{SimulaRowCount=5019216 }},
+    //        {"order_block_plan",new TableInfo{SimulaRowCount=2725553 }},//CreateTime < 202301的删除
+    //        {"order_block_plan_result",new TableInfo{SimulaRowCount=1174096 }},
+    //        {"order_box_block",new TableInfo{SimulaRowCount=29755672 }},
+    //        {"order_data_block",new TableInfo{SimulaRowCount=731800334 }},
+    //        {"order_data_goods",new TableInfo{SimulaRowCount=25803671 }},
+    //        {"order_data_parts",new TableInfo{SimulaRowCount=468517543 }},
+    //        {"order_item",new TableInfo{SimulaRowCount=1345520079 }},
+    //        {"order_module",new TableInfo{SimulaRowCount=103325385 }},
+    //        {"order_module_extra",new TableInfo{SimulaRowCount=54361321 }},
+    //        {"order_module_item",new TableInfo{SimulaRowCount=69173339 }},
+    //        {"order_package",new TableInfo{SimulaRowCount=16196195 }},
+    //        {"order_process",new TableInfo{SimulaRowCount=3892685 }},//orderNo < 202301的
+    //        {"order_process_step",new TableInfo{SimulaRowCount=8050349 }},//orderNo < 202301的删除
+    //        {"order_process_step_item",new TableInfo{SimulaRowCount=14538058 }},//orderNo < 202301的删除
+    //        {"order_scrap_board",new TableInfo{SimulaRowCount=123998 }},
+    //        {"process_group",new TableInfo{SimulaRowCount=1253 }},
+    //        {"process_info",new TableInfo{SimulaRowCount=7839 }},
+    //        {"process_item_exp",new TableInfo{SimulaRowCount=28 }},
+    //        {"process_schdule_capacity",new TableInfo{SimulaRowCount=39736 }},
+    //        {"process_step_efficiency",new TableInfo{SimulaRowCount=8 }},
+    //        {"report_template",new TableInfo{SimulaRowCount=7337 }},
+    //        {"simple_package",new TableInfo{SimulaRowCount=130436 }},//orderNo < 202301的删除
+    //        {"simple_plan_order",new TableInfo{SimulaRowCount=351470 }},//CreateTime < 202301的删除
+    //        {"sys_config",new TableInfo{SimulaRowCount=2296 }},
+    //        {"work_calendar",new TableInfo{SimulaRowCount=11 }},
+    //        {"work_shift",new TableInfo{SimulaRowCount=59 }},
+    //        {"work_time",new TableInfo{SimulaRowCount=62 }},
+    //    };
+    //});
+    host.Services.Configure<CsvOptions>(option =>
     {
-        {"-d", "Input:InputDir"},
+        option.Delimiter = ",";
-        {"--InputDir", "Input:InputDir"},
+        option.QuoteChar = '"';
-        {"-s", "Output:ConnectionString"},
-        {"--ConnectionString", "Output:ConnectionString"},
-        {"-r", "RedisCache:Configuration"},
-        {"--Redis", "RedisCache:Configuration"},
-        {"-i", "RestoreIndex"},
-        {"--RestoreIndex", "RestoreIndex"}
     });
-
-    var inputOptions = host.Configuration.GetRequiredSection("Input").Get<DataInputOptions>() 
-                       ?? throw new ApplicationException("缺少Input配置");
-
-    var transformOptions = host.Configuration.GetRequiredSection("Transform").Get<DataTransformOptions>()
-                        ?? throw new ApplicationException("缺少Transform配置");
-
-    var outputOptions = host.Configuration.GetRequiredSection("Output").Get<DatabaseOutputOptions>() 
-                        ?? throw new ApplicationException("缺少Output配置");
-
-    var redisSection = host.Configuration.GetRequiredSection("RedisCache");
-    var redisOptions = redisSection.Get<RedisCacheOptions>() ?? throw new ApplicationException("缺少RedisCache配置");
-    
-    var tenantDbSection = host.Configuration.GetRequiredSection("TenantDb");
-    var tenantDbOptions = tenantDbSection.Get<TenantDbOptions>() ?? throw new ApplicationException("缺少TenantDb配置");
-    if(tenantDbOptions.TenantKey is null) throw new ApplicationException("分库配置中缺少分库键");
-    if(tenantDbOptions.DbList is null) throw new ApplicationException("分库配置中没有发现任何数据库");
-    
-    host.Services.Configure<TenantDbOptions>(tenantDbSection);
-    host.Services.Configure<RedisCacheOptions>(redisSection);
-    
-    var oldestTime = DateTime.ParseExact(inputOptions.CleanDate, "yyyyMM", System.Globalization.DateTimeFormatInfo.InvariantInfo);
-    var oldestTimeInt = int.Parse(inputOptions.CleanDate);
-    
-    // 输入配置
     host.Services.Configure<DataInputOptions>(options =>
     {
-        options.InputDir = inputOptions.InputDir ?? throw new ApplicationException("未配置输入目录");
+        options.InputDir = commandOptions.InputDir;
-        options.UseMock = inputOptions.UseMock;
+        var _csvOptions = new CsvOptions { Delimiter = ",", QuoteChar = '"' };
-        options.TableMockConfig = inputOptions.TableMockConfig;
+        options.CreateSource = (string tableName) =>
-        options.MockCountMultiplier = inputOptions.MockCountMultiplier;
-        
-        // 配置文件输入方法
-        options.FileInputMetaBuilder = fileName =>
         {
-            if (fileName.EndsWith(".dat.zst"))
+            var source = new ZstSource(commandOptions.InputDir, tableName, _csvOptions.Delimiter, _csvOptions.QuoteChar);
-            {
+            return source;
-                var tableName = DumpDataHelper.GetTableNameFromCsvFileName(
-                    Path.GetFileNameWithoutExtension(fileName)); // 去除.zst
-                string[]? headers;
-                try
-                {
-                    // 查找同目录下同表的SQL文件
-                    var sqlFile = Directory.GetFiles(options.InputDir)
-                        .SingleOrDefault(f => f.Equals(fileName.Replace(".dat.zst",".sql.zst")));
-                    if (sqlFile is null)
-                        return null;
-                    headers = DumpDataHelper.GetCsvHeadersFromSqlFile(
-                        DumpDataHelper.DecompressZstAsStringAsync(File.OpenRead(sqlFile)).Result);
-                }
-                catch (InvalidOperationException e)
-                {
-                    throw new ApplicationException($"目录下不止一个{tableName}表的SQL文件", e);
-                }
-                
-                return new FileInputInfo
-                {
-                    FileName = fileName,
-                    TableName = tableName,
-                    Headers = headers
-                };
-            }
-            return null;
-        };
-
-        options.TableOrder =
-        [
-            TableNames.Machine,
-            
-            TableNames.Order,
-            TableNames.OrderBoxBlock,       // 依赖Order.CompanyID
-            
-            TableNames.OrderBlockPlan,      
-            TableNames.OrderBlockPlanResult,// 依赖OrderBlockPlan.CompanyID / 删除
-            
-            TableNames.OrderItem,
-            TableNames.OrderDataBlock,
-            TableNames.OrderDataGoods,
-            TableNames.OrderDataParts,
-            TableNames.OrderModule,
-            TableNames.OrderModuleExtra,
-            TableNames.OrderModuleItem,
-            TableNames.OrderPackage,
-            TableNames.OrderPatchDetail,
-            
-            TableNames.OrderProcess,
-            TableNames.OrderProcessStep,
-            TableNames.OrderProcessStepItem,// 依赖OrderProcess.ShardKey / 删除
-            
-            TableNames.OrderProcessSchedule,
-            TableNames.OrderScrapBoard,
-            TableNames.ProcessGroup,
-            TableNames.ProcessInfo,
-            TableNames.ProcessItemExp,
-            TableNames.ProcessScheduleCapacity,
-            TableNames.ProcessStepEfficiency,
-            TableNames.ReportTemplate,
-            TableNames.SimplePackage,
-            TableNames.SimplePlanOrder,
-            TableNames.SysConfig,
-            TableNames.WorkCalendar,
-            TableNames.WorkShift,
-            TableNames.WorkTime
-        ];
-
-        // options.TableMockConfig = new Dictionary<string, TableMockConfig>
-        // {
-        //     { TableNames.Machine, new TableMockConfig(true, 14655, ["ID"]) },
-        //     { TableNames.Order, new TableMockConfig(true, 5019216, ["OrderNo"]) },
-        //     { TableNames.OrderDataBlock, new TableMockConfig(true, 731800334, ["ID"]) },
-        //     { TableNames.OrderDataGoods, new TableMockConfig(true, 25803671, ["ID"]) },
-        //     { TableNames.OrderDataParts, new TableMockConfig(true, 468517543, ["ID"]) },
-        //     { TableNames.OrderModule, new TableMockConfig(true, 103325385, ["ID"]) },
-        //     { TableNames.OrderModuleExtra, new TableMockConfig(true, 54361321, ["ID"]) },
-        //     { TableNames.OrderModuleItem, new TableMockConfig(true, 69173339, ["ID"]) },
-        //     { TableNames.OrderPackage, new TableMockConfig(true, 16196195, ["ID"]) },
-        //     { TableNames.OrderProcess, new TableMockConfig(true, 3892685, ["ID"]) },
-        //     { TableNames.OrderProcessStep, new TableMockConfig(true, 8050349, ["ID"]) },
-        //     { TableNames.OrderProcessStepItem, new TableMockConfig(true, 14538058, ["ID"]) },
-        //     { TableNames.OrderScrapBoard, new TableMockConfig(true, 123998, ["ID"]) },
-        //     { TableNames.ProcessGroup, new TableMockConfig(true, 1253, ["ID"]) },
-        //     { TableNames.ProcessInfo, new TableMockConfig(true, 7839, ["ID"]) },
-        //     { TableNames.ProcessItemExp, new TableMockConfig(true, 28, ["ID"]) },
-        //     { TableNames.ProcessScheduleCapacity, new TableMockConfig(true, 39736, ["ID"]) },
-        //     { TableNames.ProcessStepEfficiency, new TableMockConfig(true, 8, ["ID"]) },
-        //     { TableNames.ReportTemplate, new TableMockConfig(true, 7337, ["ID"]) },
-        //     { TableNames.SimplePackage, new TableMockConfig(true, 130436, ["ID"]) },
-        //     { TableNames.SysConfig, new TableMockConfig(true, 2296, ["ID"]) },
-        //     { TableNames.WorkCalendar, new TableMockConfig(true, 11, ["ID"]) },
-        //     { TableNames.WorkShift, new TableMockConfig(true, 59, ["ID"]) },
-        //     { TableNames.WorkTime, new TableMockConfig(true, 62, ["ID"]) }
-        // };
-        options.TableMockConfig = new Dictionary<string, TableMockConfig>
-        {
-            { TableNames.Machine, new TableMockConfig(true, 14655, ["ID"]) },
-            { TableNames.Order, new TableMockConfig(true, 50192, ["OrderNo"]) },
-            { TableNames.OrderDataBlock, new TableMockConfig(true, 7318003, ["ID"]) },
-            { TableNames.OrderDataGoods, new TableMockConfig(true, 258036, ["ID"]) },
-            { TableNames.OrderDataParts, new TableMockConfig(true, 4685175, ["ID"]) },
-            { TableNames.OrderItem, new TableMockConfig(true, 13298896, ["ID"])},
-            { TableNames.OrderModule, new TableMockConfig(true, 1033253, ["ID"]) },
-            { TableNames.OrderModuleExtra, new TableMockConfig(true, 543613, ["ID"]) },
-            { TableNames.OrderModuleItem, new TableMockConfig(true, 691733, ["ID"]) },
-            { TableNames.OrderPackage, new TableMockConfig(true, 161961, ["ID"]) },
-            { TableNames.OrderProcess, new TableMockConfig(true, 38926, ["ID"]) },
-            { TableNames.OrderProcessStep, new TableMockConfig(true, 80503, ["ID"]) },
-            { TableNames.OrderProcessStepItem, new TableMockConfig(true, 145380, ["ID"]) },
-            { TableNames.OrderScrapBoard, new TableMockConfig(true, 1239, ["ID"]) },
-            { TableNames.ProcessGroup, new TableMockConfig(true, 125, ["ID"]) },
-            { TableNames.ProcessInfo, new TableMockConfig(true, 783, ["ID"]) },
-            { TableNames.ProcessItemExp, new TableMockConfig(true, 28, ["ID"]) },
-            { TableNames.ProcessScheduleCapacity, new TableMockConfig(true, 39736, ["ID"]) },
-            { TableNames.ProcessStepEfficiency, new TableMockConfig(true, 8, ["ID"]) },
-            { TableNames.ReportTemplate, new TableMockConfig(true, 7337, ["ID"]) },
-            { TableNames.SimplePackage, new TableMockConfig(true, 130436, ["ID"]) },
-            { TableNames.SysConfig, new TableMockConfig(true, 2296, ["Key"]) },
-            { TableNames.WorkCalendar, new TableMockConfig(true, 11, ["ID"]) },
-            { TableNames.WorkShift, new TableMockConfig(true, 59, ["ID"]) },
-            { TableNames.WorkTime, new TableMockConfig(true, 62, ["ID"]) }
         };
     });
 
     host.Services.Configure<DataTransformOptions>(options =>
     {
-        // 数据缓存键格式为[TableName]-[ColumnName@ColumnValue]-[CacheColumnName]
+        var tenantDbOptions = host.Configuration.GetRequiredSection("TenantDb").Get<TenantDbOptions>() 
-        static string BuildCacheKey(string tableName, string columnName, string columnValue, string cacheColumnName) 
+            ?? throw new ApplicationException("分库配置项不存在");
-            => $"{tableName}-{columnName}@{columnValue}-{cacheColumnName}";
-
-        static string CalculateShardKeyByOrderNo(ReadOnlySpan<char> orderNo)
-            => $"{orderNo[2..6]}0";
-        
-        options.StrictMode = transformOptions.StrictMode;
-        options.EnableFilter = transformOptions.EnableFilter;
-        options.EnableReplacer = transformOptions.EnableReplacer;
-        options.EnableReBuilder = transformOptions.EnableReBuilder;
-        
-        // order_block_plan_item和order_package_item表不导入，根据order_item数据直接重建
-        
-        // 数据清理
-        options.RecordFilter = async context =>
-        {
-            var record = context.Record;
-            var cache = context.Cacher;
-            switch (record.TableName)
-            {
-                // OrderBoxBlock删除对应Order.OrderNo不存在的对象
-                case TableNames.OrderBoxBlock:
-                {
-                    if (!await cache.ExistsAsync(CacheKeys.Order_OrderNo_CompanyID(record["OrderNo"])))
-                        return false;
-                    break;
-                }
-                // OrderBlockPlan删除CreateTime < 202301的
-                case TableNames.OrderBlockPlan:
-                {
-                    var time = DateTime.Parse(record["CreateTime"].Trim('"'));
-                    if (time < oldestTime)
-                        return false;
-                    break;
-                }
-                // OrderBlockPlanResult删除对应order_block_plan.ID不存在的对象
-                case TableNames.OrderBlockPlanResult:
-                {
-                    if (!await cache.ExistsAsync(CacheKeys.OrderBlockPlan_ID_CompanyID(record["ID"])))
-                        return false;
-                    break;
-                }
-                // OrderModule删除OrderNo < 202301的
-                case TableNames.OrderModule:
-                {
-                    var orderNo = record["OrderNo"];
-                    if(int.Parse(orderNo.AsSpan(0, 6).ToString()) < oldestTimeInt)
-                        return false;
-                    break;
-                }
-                // OrderProcess删除OrderNo < 202301的
-                case TableNames.OrderProcess:
-                {
-                    var orderNo = record["OrderNo"];
-                    if(int.Parse(orderNo.AsSpan(0, 6).ToString()) < oldestTimeInt)
-                        return false;
-                    break;
-                }
-                // OrderProcessStep删除OrderNo < 202301的
-                case TableNames.OrderProcessStep:
-                {
-                    var orderNo = record["OrderNo"];
-                    if(int.Parse(orderNo.AsSpan(0, 6).ToString()) < oldestTimeInt)
-                        return false;
-                    break;
-                }
-                // OrderProcessStepStep删除对应OrderProcess.ID不存在的对象
-                case TableNames.OrderProcessStepItem:
-                {
-                    if (!await cache.ExistsAsync(CacheKeys.OrderProcess_ID_ShardKey(record["OrderProcessID"])))
-                        return false;
-                    break;
-                }
-                // SimplePackage删除OrderNo < 202301的
-                case TableNames.SimplePackage:
-                {
-                    var orderNo = record["OrderNo"];
-                    if(int.Parse(orderNo.AsSpan(0, 6).ToString()) < oldestTimeInt)
-                        return false;
-                    break;
-                }
-                // SimplePlanOrder删除CreateTime < 202301的
-                case TableNames.SimplePlanOrder:
-                {
-                    var time = DateTime.Parse(record["CreateTime"].Trim('"'));
-                    if (time < oldestTime)
-                        return false;
-                    break;
-                }
-            }
-
-            return true;
-        };
-        
-        // 数据替换
-        options.RecordModify = async context =>
-        {
-            var record = context.Record;
-            var cache = context.Cacher;
-            switch (record.TableName)
-            {
-#if USE_TEST_DB
-                // Order表移除IsBatch列
-                case TableNames.Order:
-                    record.RemoveField("IsBatch");
-                    break;
-#endif
-                //OrderBlockPlan将OrderNo长度<2的置空
-                case TableNames.OrderBlockPlan:
-                    if (record["OrderNos"].Length < 2)
-                        record["OrderNos"] = "";
-                    break;
-                // OrderBlockPlanResult表添加CompanyID列
-                case TableNames.OrderBlockPlanResult:
-                    record.AddField("CompanyID",
-                        // 获取OrderBlockPlan.ID -> CompanyID
-                        ThrowIfNoCached(await cache.GetStringAsync(CacheKeys.OrderBlockPlan_ID_CompanyID(record["ID"])),
-                            TableNames.OrderBlockPlanResult, TableNames.OrderBlockPlan, "ID", "脏数据未处理"));
-                    break;
-                // OrderBoxBlock添加CompanyID列
-                case TableNames.OrderBoxBlock:
-                    record.AddField("CompanyID", 
-                        // 获取Order.OrderNo -> CompanyID
-                        ThrowIfNoCached(await cache.GetStringAsync(CacheKeys.Order_OrderNo_CompanyID(record["OrderNo"])),
-                            TableNames.OrderBoxBlock, TableNames.Order, "OrderNo", "脏数据未处理"));
-                    break;
-                // OrderModule添加ShardKey列，移除ViewFileName列
-                case TableNames.OrderModule:
-                    record.AddField("ShardKey", CalculateShardKeyByOrderNo(record["OrderNo"]));
-                    record.RemoveField("ViewFileName");
-                    break;
-                // OrderProcess添加ShardKey列，NextStepID的空值转换为0
-                case TableNames.OrderProcess:
-                    record.AddField("ShardKey", CalculateShardKeyByOrderNo(record["OrderNo"]));
-#if USE_TEST_DB
-                    if(record["NextStepID"] == "\\N")
-                        record["NextStepID"] = "0";
-#endif
-                    break;
-                // OrderProcessStep，OrderProcessStepItem添加ShardKey列
-                case TableNames.OrderProcessStep:
-                    record.AddField("ShardKey", CalculateShardKeyByOrderNo(record["OrderNo"]));
-                    break;
-                case TableNames.OrderProcessStepItem:
-                    record.AddField("ShardKey", 
-                        // 获取OrderProcess.ID -> ShardKey
-                    ThrowIfNoCached(await cache.GetStringAsync(CacheKeys.OrderProcess_ID_ShardKey(record["OrderProcessID"])), 
-                        TableNames.OrderProcessStepItem, TableNames.OrderProcessStep, "OrderProcessID", "脏数据未处理"));
-                    break;
-                case TableNames.SimplePlanOrder:
-#if USE_TEST_DB
-                    record.RemoveField("ProcessState");
-#endif
-                    record.AddField("Deleted", "0");
-                    break;
-            }
-
-            return record;
-
-            string ThrowIfNoCached(string? cached, string tableName, string cachedTableName, string cachedColumn, string appendMessage = "")
-            {
-                if (cached is null)
-                    throw new InvalidDataException(
-                        $"{tableName}数据异常，在缓存中未找到对应{cachedTableName}.{cachedColumn}\t{appendMessage}");
-                return cached;
-            }
-        };
-        
-        // 数据缓存
-        options.RecordCache = async context =>
-        {
-            var record = context.Record;
-            var cache = context.Cacher;
-            switch (record.TableName)
-            {
-                // 缓存Order.OrderNo -> CompanyID
-                case TableNames.Order:
-                    CacheKeys.Order_OrderNo_CompanyID = orderNo =>
-                        BuildCacheKey(TableNames.Order, "OrderNo", orderNo, "CompanyID");
-                    await cache.SetStringAsync(
-                        CacheKeys.Order_OrderNo_CompanyID(record["OrderNo"]),
-                        record["CompanyID"]);
-                    break;
-                
-                // 缓存OrderBlockPlan.ID -> CompanyID
-                case TableNames.OrderBlockPlan:
-                    CacheKeys.OrderBlockPlan_ID_CompanyID = id =>
-                        BuildCacheKey(TableNames.OrderBlockPlan, "ID", id, "CompanyID");
-                    await cache.SetStringAsync(
-                        CacheKeys.OrderBlockPlan_ID_CompanyID(record["ID"]),
-                        record["CompanyID"]);
-                    break;
-                
-                // 缓存OrderProcess.ID -> ShardKey
-                case TableNames.OrderProcess:
-                    CacheKeys.OrderProcess_ID_ShardKey = id =>
-                        BuildCacheKey(TableNames.OrderProcess, "ID", id, "ShardKey");
-                    await cache.SetStringAsync(
-                        CacheKeys.OrderProcess_ID_ShardKey(record["ID"]),
-                        record["ShardKey"]);
-                    break;
-            }
-        };
-        
-        // 数据库过滤
         options.DatabaseFilter = record =>
         {
             var companyId = int.Parse(record[tenantDbOptions.TenantKey]); // 每个实体都应存在CompanyID，否则异常
             return tenantDbOptions.GetDbNameByTenantKeyValue(companyId);
         };
 
-        // 数据重建
+        options.TransformBinary = field => commandOptions != null && commandOptions.Isutf8mb4 ? $"_utf8mb4 0x{field}" : $"0x{field}";
-        options.RecordReBuild = context =>
+        //数据过滤
+        options.RecordFilter = async (record, cache) =>
         {
-            var record = context.Record;
+            //var index = Array.IndexOf(record.Headers, "ShardKey");
-            var resultList = new List<DataRecord>();
+            if (record.TryGetField("ShardKey", out var skStr))
-            // 分流OrderItem表
-            if (record.TableName == TableNames.OrderItem)
             {
-                record.TryGetField("ID", out var itemId);
+                short.TryParse(skStr, out var sk);
+                if (sk < commandOptions.OldestShardKey) return false;
+            }
+
+            if (record.TryGetField("CreateTime", out var createTime))
+            {
+                _ = DateTime.TryParse(createTime.Replace("\"", ""), out var time);
+                if (time < oldestTime) return false;
+            }
+            if (record.TryGetField("OrderNo", out var orderNo))
+            {
+                try
+                {
+                    var yearMonth = orderNo.Substring(0, 6);
+
+                    var dt = DateTime.ParseExact(yearMonth, "yyyyMM", System.Globalization.DateTimeFormatInfo.InvariantInfo);
+                    if (dt < oldestTime) return false;
+
+                }
+                catch (Exception ex)
+                {
+                    return false;//订单号转换失败，跳过
+
+                }
+                
+            }
+
+            if (record.TableName == "order_package")
+            {
+                if (record.TryGetField("PakageNo", out var pkNo))
+                {
+                    if (pkNo.Length <= 2) return false;
+                }
+            }
+            if (record.TableName == "order_block_plan")
+            {
+                if (record.TryGetField("OrderNos", out var nos))
+                {
+                    if (nos.Length <= 2) return false;
+                }
+              
+            }
+            if (record.TableName == "order_process_step" || record.TableName == "order_process_step_item")
+            {
+                //如果缓存中不存在OrderProcessID,则丢弃
+                //if(record.TryGetField("OrderProcessID",out var orderProcessID))
+
+ 
+                    var value = await cache.GetStringAsync(record.GetCacheKey("OrderProcessID"));
+                    if (string.IsNullOrEmpty(value))return false;
+                
+            }
+            if (record.TableName == "order_block_plan_result" )
+            {
+                //如果缓存中不存在ID,则丢弃(ID 对应order_block_plan中的ID)
+
+                    var value = await cache.GetStringAsync(record.GetCacheKey("ID"));
+                    if (string.IsNullOrEmpty(value)) return false;
+                
+            }
+            return true;
+
+        };
+        //数据修改
+        options.RecordModify = (record) =>
+        {
+            if (record.TableName == "order_block_plan")
+            {
+                if (record.TryGetField("OrderNos", out var nos))
+                {
+                    if (nos.Length <= 2) record.SetField("OrderNos", "");
+                }
+
+            }
+            if (record.TableName == "order_process")//修改order_process.NextStepID的默认值为0
+            {
+                 
+                if (record.TryGetField("NextStepID", out var idStr))
+                {
+
+                    if (idStr == "\\N")
+                    {
+                        record.SetField("NextStepID", "0");
+                    }
+                }
+            }
+
+        };
+        //数据缓存
+        options.RecordCache = async (record, cache) =>
+        {
+            if (record.TableName == "order")
+            {
+                if (record.TryGetField("OrderNo", out var orderNo))
+                {
+                    if (record.TryGetField("CompanyID", out var companyid))
+                    {
+                        await cache.SetStringAsync(record.GetCacheKey("OrderNo"), companyid);
+                    }
+                    
+                }
+            }
+            if (record.TableName == "order_process")
+            {
+                if (record.TryGetField("OrderNo", out var orderNo))
+                {
+                    var yearMonth = orderNo.Substring(2, 4);
+                    var sk = yearMonth + "0";
+                   
+                    if( record.TryGetField("ID", out var id))
+                    {
+                        await cache.SetStringAsync(record.GetCacheKey("ID"), sk);
+                    }
+          
+                }
+            }
+            if (record.TableName == "order_block_plan")
+            {
+                if (record.TryGetField("CompanyID", out var companyid))
+                {
+                    record.TryGetField("ID", out var id);
+                    await cache.SetStringAsync(record.GetCacheKey("ID"), companyid);
+                }
+            }
+        };
+        //数据替换
+        options.RecordReplace = async (record, cache) =>
+        {
+            //删除数据源里simple_plan_order.ProcessState 字段和值
+
+            if (record.TableName == "simple_plan_order")//修改order_process.NextStepID的默认值为0
+            {
+                var nextStepIdIndex = Array.IndexOf(record.Headers, "ProcessState");
+                if (nextStepIdIndex > -1)
+                {
+                    var headers = record.Headers.Where(t => t != "ProcessState").ToArray();
+                    var fs = record.Fields.ToList();
+                    fs.RemoveAt(nextStepIdIndex);
+                    var fields = fs.ToArray();
+                    return new DataRecord(fields, record.TableName, headers, record.CompanyID);
+                }
+            }
+            if (record.TableName == "order")//修改order_process.NextStepID的默认值为0
+            {
+                var nextStepIdIndex = Array.IndexOf(record.Headers, "IsBatch");
+                if (nextStepIdIndex > -1)
+                {
+                    var headers = record.Headers.Where(t => t != "IsBatch").ToArray();
+                    var fs = record.Fields.ToList();
+                    fs.RemoveAt(nextStepIdIndex);
+                    var fields = fs.ToArray();
+                    return new DataRecord(fields, record.TableName, headers, record.CompanyID);
+                }
+            }
+            if (record.TableName == "order_block_plan_result")//修改order_process.NextStepID的默认值为0
+            {
+                if (record.TryGetField("ID", out var id))
+                {
+                    var headers = new List<string>(record.Headers);
+                    var fields =new List<string>(record.Fields);
+                    headers.Add("CompanyID");
+                    var companyidResult =await cache.GetStringAsync(record.GetCacheKey("ID"));
+
+                    _ = int.TryParse(companyidResult, out var companyid);
+                    fields.Add(companyid.ToString());
+                    return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), companyid);
+                }
+            }
+            if(record.TableName == "order_box_block")
+            {
+                if (!record.TryGetField("CompanyID", out var companyid))
+                {
+                    if (record.TryGetField("OrderNo", out var orderNo))
+                    {
+                        var headers = new List<string>(record.Headers);
+                        var fields = new List<string>(record.Fields);
+                        headers.Add("CompanyID");
+                        var companyidResult = await cache.GetStringAsync(record.GetCacheKey("OrderNo"));
+                        _ = int.TryParse(companyidResult, out var cpid);
+                        fields.Add(cpid.ToString());
+                        return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), cpid);
+                    }
+                }
+            }
+            if (record.TableName == "order_module")
+            {
+                if (record.TryGetField("ViewFileName",out var value))
+                {
+                    var index=Array.IndexOf(record.Headers, "ViewFileName");
+                    var headers = new List<string>(record.Headers);
+                    headers.RemoveAt(index); 
+                    var fields = new List<string>(record.Fields);
+                    fields.RemoveAt(index);
+                    return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), record.CompanyID);
+                }
+                
+                if (!record.TryGetField("ShardKey", out var skStr))
+                {
+                    if (record.TryGetField("OrderNo", out var orderNo))
+                    {
+                        var yearMonth = orderNo.Substring(2, 4);
+                        var sk = yearMonth + "0";
+                        var headers = new List<string>(record.Headers);
+                        var fields = new List<string>(record.Fields);
+                        headers.Add("ShardKey");
+                        fields.Add(sk);
+                        return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), record.CompanyID);
+                    }
+                }
+            }
+            if (record.TableName == "order_process")
+            {
+                if (!record.TryGetField("ShardKey", out var skStr))
+                {
+                    if(record.TryGetField("OrderNo", out var orderNo))
+                    {
+                        var yearMonth = orderNo.Substring(2, 4);
+                        var sk = yearMonth + "0";
+                        var headers = new List<string>(record.Headers);
+                        var fields = new List<string>(record.Fields);
+                        headers.Add("ShardKey");
+                        fields.Add(sk);
+                        return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), record.CompanyID);
+                    }
+                }
+            }
+            if(record.TableName == "order_process_step"|| record.TableName == "order_process_step_item")
+            {
+                if (!record.TryGetField("ShardKey",out var sk))
+                {
+                    if (record.TryGetField("OrderProcessID",out var processID))
+                    {
+                        var shardKey =await cache.GetStringAsync(record.GetCacheKey("OrderProcessID"));
+                        var headers = new List<string>(record.Headers);
+                        var fields = new List<string>(record.Fields);
+                        headers.Add("ShardKey");
+                        fields.Add(shardKey??"0");
+                        return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), record.CompanyID);
+                    }
+                }
+            }
+            
+            // 旧数据库order_data_block.CompanyID数据有误，重新计算
+            if (record.TableName == "order_data_block")  
+            {
+                var companyID = await cache.GetStringAsync($"order_{record["OrderNo"]}");
+                if (string.IsNullOrEmpty(companyID))
+                    // throw new InvalidDataException($"没有获取到OrderNo={record["OrderNo"]}的CompanyID");
+                    return record;
+                record["CompanyID"] = companyID;
+                return record;
+            }
+            return null;
+        };
+        //数据生成
+        options.RecordAdd = (record) =>
+        {
+            var resultList = new List<DataRecord>();
+            if (record.TableName == "order_item")
+            {
+                record.TryGetField("ID", out var itemID);
                 record.TryGetField("ShardKey", out var shardKey);
-                record.TryGetField("PlanID", out var planId);
+                record.TryGetField("PlanID", out var planID);
-                record.TryGetField("PackageID", out var packageId);
+                record.TryGetField("PackageID", out var packageID);
-                record.TryGetField("CompanyID", out var companyId);
+                record.TryGetField("CompanyID", out var companyID);
-                if(!int.TryParse(planId, out var pid))
+                _=int.TryParse(planID, out var pid);
-                    throw new ApplicationException($"数据发生异常：OrderItem.PlanID，值: {planId}"); 
                 if (pid > 0)
                 {
-                    resultList.Add(new DataRecord(new[] { itemId, shardKey, planId, companyId },
+                    resultList.Add(new DataRecord(new[] { itemID, shardKey, planID, companyID },
-                        TableNames.OrderBlockPlanItem,
+                        "order_block_plan_item",
-                        ["ItemID", "ShardKey", "PlanID", "CompanyID"]
+                        new[] { "ItemID", "ShardKey", "PlanID", "CompanyID" }));
-                        ));
                 }
-                if(!int.TryParse(packageId, out var pkid))
+                _ = int.TryParse(packageID, out var pkid);
-                    throw new ApplicationException($"数据发生异常：OrderItem.PackageID，值: {packageId}"); 
                 if(pkid > 0)
                 {
-                    resultList.Add(new DataRecord(new[] { itemId, shardKey, packageId, companyId },
+                    resultList.Add(new DataRecord(new[] { itemID, shardKey, packageID, companyID },
-                        TableNames.OrderPackageItem,
+                        "order_package_item",
-                        [ "ItemID", "ShardKey", "PackageID", "CompanyID" ]
+                        new[] { "ItemID", "ShardKey", "PackageID", "CompanyID" }
-                    ));
+                        ));
                 }
             }
             return resultList;
 
         };
-    });
+        options.ColumnTypeConfig = new()
-    
-    host.Services.Configure<DatabaseOutputOptions>(options =>
-    {
-        options.ConnectionString = new MySqlConnectionStringBuilder(outputOptions.ConnectionString ?? throw new ApplicationException("未配置数据库连接字符串"))
-        {
-            CharacterSet = "utf8mb4",
-            AllowUserVariables = true,
-            IgnoreCommandTransaction = true,
-            TreatTinyAsBoolean = false,
-            ConnectionTimeout = 60,
-            DefaultCommandTimeout = 0,
-            SslMode = MySqlSslMode.None,
-        }.ConnectionString;
-        options.FlushCount = outputOptions.FlushCount;
-        options.MaxAllowedPacket = outputOptions.MaxAllowedPacket;
-        options.MaxDatabaseOutputTask = outputOptions.MaxDatabaseOutputTask;
-        options.TreatJsonAsHex = outputOptions.TreatJsonAsHex;
-
-#if USE_TEST_DB
-        // Test Server
-         options.ColumnTypeConfig = new Dictionary<string, ColumnType>
-         {
-             { "simple_plan_order.PlaceData", ColumnType.Blob },
-             { "order_block_plan_result.PlaceData", ColumnType.Blob },
-             { "order_box_block.Data", ColumnType.Blob },
-             { "order_data_goods.ExtraProp", ColumnType.Json },
-             { "order_module_extra.JsonStr", ColumnType.Text },
-             { "process_info.Users", ColumnType.Text },
-             { "order_process_schdule.CustomOrderNo", ColumnType.Text },
-             { "order_process_schdule.OrderProcessStepName", ColumnType.Text },
-             { "order_process_schdule.AreaName", ColumnType.Text },
-             { "order_process_schdule.ConsigneeAddress", ColumnType.Text },
-             { "order_process_schdule.ConsigneePhone", ColumnType.Text },
-             { "report_source.Sql", ColumnType.Text },
-             { "report_source.KeyValue", ColumnType.Text },
-             { "report_source.Setting", ColumnType.Text },
-             { "order_data_block.RemarkJson", ColumnType.Text },
-             { "order_patch_detail.BlockDetail", ColumnType.Json },
-             { "order_scrap_board.OutLineJson", ColumnType.Text },
-             { "simple_package.Items", ColumnType.Json },
-             { "order_batch_pack_config.Setting", ColumnType.Text },
-             { "machine.Settings", ColumnType.Text },
-             { "sys_config.Value", ColumnType.Text },
-             { "sys_config.JsonStr", ColumnType.Text },
-             { "process_item_exp.ItemJson", ColumnType.Text },
-             { "report_template.Template", ColumnType.Text },
-             { "report_template.SourceConfig", ColumnType.Text },
-             { "order_block_plan.OrderNos", ColumnType.Json },
-             { "order_block_plan.BlockInfo", ColumnType.Text },
-         };
-#else
-        // 配置列类型
-        // Prod server
-        options.ColumnTypeConfig = new Dictionary<string, ColumnType>
         {
             { "simple_plan_order.PlaceData", ColumnType.Blob },
             { "order_block_plan_result.PlaceData", ColumnType.Blob },
@@ -533,7 +453,26 @@ async Task RunProgram()
             { "order_block_plan.OrderNos", ColumnType.Json },
             { "order_block_plan.BlockInfo", ColumnType.Text },
         };
-#endif
+    });
+    host.Services.Configure<DatabaseOutputOptions>(options =>
+    {
+        //options.ConnectionString = new MySqlConnectionStringBuilder
+        //{
+        //    Server = "127.0.0.1",
+        //    Port = 33309,
+        //    Database = "cferp_test",
+        //    UserID = "root",
+        //    Password = "123456",
+        //    MaximumPoolSize = 50, // 这个值应当小于 max_connections
+        //}.ConnectionString;
+        options.ConnectionString = new MySqlConnectionStringBuilder(host.Configuration.GetConnectionString("MySqlMaster"))
+        {
+            CharacterSet = "utf8",
+            AllowUserVariables = true,
+            IgnoreCommandTransaction = true,
+            TreatTinyAsBoolean = false,
+            MaximumPoolSize = 50
+        }.ConnectionString;
     });
     
     host.Services.AddLogging(builder =>
@@ -541,26 +480,29 @@ async Task RunProgram()
         builder.ClearProviders();
         builder.AddSerilog(new LoggerConfiguration()
             .WriteTo.Console()
-            .WriteTo.File($"./Log/Error/{ErrorRecorder.UID}.log", restrictedToMinimumLevel:LogEventLevel.Error)
+            .WriteTo.File("./log/error.log", restrictedToMinimumLevel:LogEventLevel.Error)
-            // .WriteTo.File("./Log/Info/{ErrorRecorder.UID}.log", restrictedToMinimumLevel:LogEventLevel.Information) //性能考虑暂不使用
+            // .WriteTo.File("./log/info.log", restrictedToMinimumLevel:LogEventLevel.Information) //性能考虑暂不使用
             .CreateLogger()
         );
     });
 
-    host.Services.AddDataSourceFactory();
+    host.Services.AddScoped<ProcessContext>();
-    host.Services.AddErrorRecorderFactory();
+    host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Producer);
-    host.Services.AddSingleton<ProcessContext>();
+    host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Consumer);
-    host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Produce);
+    host.Services.AddTransient<TaskManager>();
-    host.Services.AddRecordQueuePool(tenantDbOptions.DbList.Keys.Select(key => (key:key, queue:new DataRecordQueue(500_000))).ToArray());
+    host.Services.AddSingleton<ErrorRecorder>();
-    host.Services.AddSingleton<ITaskMonitorLogger, CacheTaskMonitorLogger>();
-    host.Services.AddSingleton<ITaskMonitorLogger, LoggerTaskMonitorLogger>();
 
     host.Services.AddHostedService<MainHostedService>();
     host.Services.AddHostedService<TaskMonitorService>();
-    host.Services.AddSingleton<IInputService, FileInputService>();
+    host.Services.AddSingleton<IInputService, InputService>();
     host.Services.AddSingleton<ITransformService, TransformService>();
     host.Services.AddSingleton<IOutputService, OutputService>();
-    host.Services.AddRedisCache(redisOptions);
+    var redisOptions = host.Configuration.GetSection("RedisCacheOptions").Get<RedisCacheOptions>() ?? new RedisCacheOptions();
+    host.Services.AddStackExchangeRedisCache(options =>
+    {
+        options.Configuration = redisOptions.Configuration;
+        options.InstanceName = redisOptions.InstanceName;
+    });
     var app = host.Build();
     await app.RunAsync();
 }

ConsoleApp2/README.md

@@ -0,0 +1,49 @@
+## 说明
+使用该程序来对MyDumper导出的CSV数据进行读取，转换，然后导出到其他数据库中。
+
+1. 用MyDumper从数据库导出CSV数据  
+
+   使用MyDumper Docker镜像
+    ```sh
+    docker run --rm --net=host -v D:/Dump:/home/backup mydumper/mydumper:v0.15.2-6 mydumper `
+   -h 127.0.0.1 -P 33306 -u root -p 123456 `
+   -B cferp_test --no-schemas --csv --hex-blob `
+   -o /home/backup
+    ```
+   将挂载卷，数据库连接和输出目录替换  
+   不导出数据库结构(--no-schemas)，
+   导出完的目录下应当包含.sql文件以及.dat文件
+
+2. 在Program.cs中修改`CsvOptions`配置
+   ```cs
+   host.Services.Configure<CsvOptions>(option =>
+    {
+        option.Delimiter = ",";
+        option.QuoteChar = '"';
+        option.InputDir = "D:/Dump/Test";
+    });
+   ```
+   将`option.InputDir`配置为MyDumper导出的数据目录
+
+3. 在Program.cs中修改`DatabaseOutputOptions`配置
+   ```cs
+   host.Services.Configure<DatabaseOutputOptions>(options =>
+    {
+        options.ConnectionString = new MySqlConnectionStringBuilder
+        {
+            Server = "127.0.0.1",
+            Port = 33306,
+            Database = "cferp_test_1",
+            UserID = "root",
+            Password = "123456",
+            MaximumPoolSize = 50, 
+        }.ConnectionString;
+        options.MaxTask = 16;
+        options.FlushCount = 200;
+    });
+   ```
+   将`MySqlConnectionStringBuilder`的属性修改为程序要导出至的数据库
+   > 后续将这些配置通过命令行传递
+
+4. 运行程序
+   > 注意，测试数据库`cferp_test`中的`order_process_step`表存在外键，如果要导出到和测试库同结构的数据库，记得先把外键删除。

ConsoleApp2/RestoreIndex.sql

@@ -1,73 +0,0 @@
-CREATE INDEX `idx_CompanyID` ON `machine` (`CompanyID`);
-
-CREATE INDEX `idx_companyid` ON `order` (`CompanyID`);
-
-CREATE INDEX `idx_CompanyID` ON `order_block_plan` (`CompanyID`);
-
-CREATE INDEX `idx_PlanID` ON `order_block_plan_item` (`PlanID`);
-
-CREATE INDEX `idx_orderno` ON `order_box_block` (`OrderNo`);
-
-CREATE INDEX `index_OrderNo` ON `order_data_block` (`OrderNo`);
-
-CREATE INDEX `index_OrderNo` ON `order_data_goods` (`OrderNo`);
-
-CREATE INDEX `index_OrderNo` ON `order_data_parts` (`OrderNo`);
-
-CREATE INDEX `index_ItemNo` ON `order_item` (`ItemNo`);
-
-CREATE INDEX `index_OrderNo` ON `order_item` (`OrderNo`);
-
-CREATE INDEX `index_PackageID` ON `order_item` (`PackageID`);
-
-CREATE INDEX `index_PlanID` ON `order_item` (`PlanID`);
-
-CREATE INDEX `idx_OrderNo` ON `order_module` (`OrderNo`);
-
-CREATE INDEX `index_OrderNo` ON `order_module_extra` (`OrderNo`);
-
-CREATE INDEX `index_OrderNo` ON `order_module_item` (`OrderNo`);
-
-CREATE INDEX `idx_OrderNo` ON `order_package` (`OrderNo`);
-
-CREATE INDEX `idx_PakageNo` ON `order_package` (`PakageNo`);
-
-CREATE INDEX `idx_PackageID` ON `order_package_item` (`PackageID`);
-
-CREATE INDEX `idx_companyid` ON `order_patch_detail` (`CompanyID`);
-
-CREATE INDEX `idx_OrderNo` ON `order_process` (`OrderNo`);
-
-CREATE INDEX `index_CompanyID` ON `order_process_schdule` (`CompanyID`);
-
-CREATE INDEX `IX_order_process_step_OrderProcessID` ON `order_process_step` (`OrderProcessID`);
-
-CREATE INDEX `idx_OrderProcessID` ON `order_process_step_item` (`OrderProcessID`);
-
-CREATE INDEX `idx_OrderProcessStepID` ON `order_process_step_item` (`OrderProcessStepID`);
-
-CREATE INDEX `idx_CompanyID` ON `order_scrap_board` (`CompanyID`);
-
-CREATE INDEX `idx_CompanyID` ON `process_group` (`CompanyID`);
-
-CREATE INDEX `idx_CompanyID` ON `process_info` (`CompanyID`);
-
-CREATE INDEX `index_CompanyID` ON `process_item_exp` (`CompanyID`);
-
-CREATE INDEX `idx_CompanyID` ON `process_schdule_capacity` (`CompanyID`);
-
-CREATE INDEX `idx_CompanyID` ON `process_step_efficiency` (`CompanyID`);
-
-CREATE INDEX `idx_CompanyID` ON `report_template` (`CompanyID`);
-
-CREATE INDEX `indx_OrderNo` ON `simple_package` (`OrderNo`);
-
-CREATE INDEX `idx_CompanyID` ON `simple_plan_order` (`CompanyID`);
-
-CREATE INDEX `idx_CompanyID` ON `sys_config` (`CompanyID`);
-
-CREATE INDEX `idx` ON `work_calendar` (`CompanyID`);
-
-CREATE INDEX `idx_CompanyID` ON `work_shift` (`CompanyID`);
-
-CREATE INDEX `IX_work_time_ShiftID` ON `work_time` (`ShiftID`);

ConsoleApp2/Services/CsvSource.cs

@@ -0,0 +1,184 @@
+using System.Reflection.PortableExecutable;
+using System.Text;
+using System.Text.RegularExpressions;
+using ConsoleApp2.Helpers;
+using ConsoleApp2.HostedServices.Abstractions;
+using Microsoft.Extensions.Logging;
+
+namespace ConsoleApp2.Services;
+
+/// <summary>
+/// CSV文件读取
+/// </summary>
+public class CsvSource:IDataSource
+{
+    protected readonly string _inputDir;
+    //protected readonly StreamReader _reader;
+    private readonly ILogger? _logger;
+    protected readonly string _tableName;
+    protected  string? _sqlFilePath;
+    protected readonly string? _sqlFileText;
+    protected string[]? headers;
+    protected string[]? csvFiles;
+    public string? CurrentRaw { get; protected set; }
+    public string Delimiter { get; private set; }
+    public char QuoteChar { get; private set; }
+    public CsvSource(string inputDir,string tableName,string delimiter = ",", char quoteChar = '"',
+        ILogger? logger = null)
+    {
+        _inputDir = inputDir;
+        _tableName = tableName;
+        _logger = logger;
+        Delimiter = delimiter;
+        QuoteChar = quoteChar;
+        string pattern = $"^.*\\.{tableName}\\..*\\.sql$";
+        _sqlFilePath = Directory.GetFiles(_inputDir).FirstOrDefault(s => Regex.Match(s, pattern).Success);
+
+
+    }
+
+
+    public string[] ParseRow(string row, char quoteChar, string delimiter)
+    {
+        var span = row.AsSpan();
+        var result = new List<string>();
+
+        if (span.Length == 0)
+            throw new ArgumentException("The row is empty", nameof(row));
+
+        var isInQuote = span[0] == quoteChar;
+        var start = 0;
+        for (var i = 1; i < span.Length; i++)
+        {
+            if (span[i] == quoteChar)
+            {
+                isInQuote = !isInQuote;
+            }
+            // delimiter需要足够复杂
+            else if (/*!isInQuote && */span.Length > i + delimiter.Length && span[i..(i + delimiter.Length)].Equals(delimiter, StringComparison.CurrentCulture)) // field matched
+            {
+                string field;
+                if (span[start] == quoteChar && span[i - 1] == quoteChar) // enclosed by quoteChar
+                    field = span[(start + 1)..(i - 1)].ToString(); // escape quoteChar
+                else
+                    field = span[start..i].ToString();
+
+                start = i + delimiter.Length;
+
+                if (field == "\\N")
+                    field = "NULL";
+
+                result.Add(field);
+                
+                continue;
+            }
+        }
+
+        result.Add(span[start..].ToString());
+
+
+        for (var i = 0; i < result.Count; i++)
+        {
+            var field = result[i];
+            if (DumpDataHelper.CheckHexField(field) && StringExtensions.CheckJsonHex(field))
+            {
+                result[i] = StringExtensions.FromHex(field);
+            }
+        }
+
+        return result.ToArray();
+    }
+    
+    public string[] ParseRow2(ReadOnlySpan<char> source, char quoteChar, string delimiter)
+    {
+        var result = new List<string>();
+        var index = -1;
+        StringBuilder current = new StringBuilder();
+        bool hasQuote = false;
+        bool hasSlash = false;
+        while (index < source.Length-1)
+        {
+            index++;
+            if (hasSlash == false && source[index] == '\\')
+            {
+                hasSlash = true;
+                current.Append('\\');
+                continue;
+            }
+            if (hasSlash ==false && source[index] == quoteChar)
+            {
+                hasQuote = !hasQuote;
+                current.Append(source[index]);
+                continue;
+            }
+            if (hasQuote==false && source[index] == delimiter[0])
+            {
+                result.Add(current.ToString());
+                current.Clear();
+            }
+            else
+            {
+                current.Append(source[index]);
+            }
+
+            hasSlash = false;
+        }
+
+        result.Add(current.ToString());
+        return result.ToArray();
+    }
+    public virtual async Task GetHeaderAndCsvFiles()
+    {
+        var text = await File.ReadAllTextAsync(_sqlFilePath);
+        headers = await DumpDataHelper.GetCsvHeadersFromSqlFileAsync(text);
+        csvFiles = await DumpDataHelper.GetCsvFileNamesFromSqlFileAsync(text, new Regex(@"'.+\.dat.zst'"));
+
+    }
+    public virtual async Task DoEnqueue(Action<DataRecord> action)
+    {
+        await GetHeaderAndCsvFiles();
+        foreach (var file in csvFiles)
+        {
+            var filePath= Path.Combine(_inputDir, file);
+            using (var fs = File.OpenRead(filePath))
+            {
+                using (StreamReader sr = new StreamReader(fs))
+                {
+                    while (!sr.EndOfStream)
+                    {
+                        var line = await sr.ReadLineAsync();
+                        var fields = ParseRow2(line, QuoteChar, Delimiter);
+                        var record = new DataRecord(fields, _tableName, headers);
+                        action?.Invoke(record);
+                    }
+                }
+            }
+
+          
+        }
+    }
+    public virtual async Task<DataRecord?> GetTestRecord()
+    {
+        await GetHeaderAndCsvFiles();
+        var file = csvFiles.FirstOrDefault();
+        if (file != null)
+        {
+            var filePath = Path.Combine(_inputDir, file);
+            using (var fs = File.OpenRead(filePath))
+            {
+                using (StreamReader sr = new StreamReader(fs))
+                {
+                    var line = await sr.ReadLineAsync();
+                    var fields = ParseRow2(line, QuoteChar, Delimiter);
+                    var record = new DataRecord(fields, _tableName, headers);
+                    return record;
+                }
+            } 
+        }
+        return null;
+    }
+    public void Dispose()
+    {
+       // _reader.Dispose();
+    }
+}

ConsoleApp2/Services/DataRecordQueue.cs

@@ -17,13 +17,9 @@ public class DataRecordQueue : IDisposable
     public event Action? OnRecordWrite;
     public event Action? OnRecordRead;
 
-    public DataRecordQueue() : this(1000000) // 默认容量最大1M
+    public DataRecordQueue()
     {
-    }
+        _queue = new BlockingCollection<DataRecord>(2000_000); // 队列最长为20W条记录
-    
-    public DataRecordQueue(int boundedCapacity)
-    {
-        _queue = new BlockingCollection<DataRecord>(boundedCapacity); 
     }
     
     public bool TryDequeue([MaybeNullWhen(false)] out DataRecord record)

ConsoleApp2/Services/ETL/CsvReader.cs

@@ -1,106 +0,0 @@
-using System.Text;
-using ConsoleApp2.HostedServices.Abstractions;
-using Microsoft.Extensions.Logging;
-
-namespace ConsoleApp2.Services.ETL;
-
-/// <summary>
-/// CSV文件读取
-/// </summary>
-public class CsvReader : IDataReader
-{
-    protected readonly string? FilePath;
-    protected readonly Lazy<StreamReader> Reader;
-    protected readonly ILogger? Logger;
-    protected readonly string TableName;
-
-    public DataRecord Current { get; protected set; } = null!;
-    public string[] Headers { get; }
-    public string? CurrentRaw { get; protected set; }
-    public string Delimiter { get; }
-    public char QuoteChar { get; }
-    
-    public CsvReader(Stream stream, string tableName, string[] headers, string delimiter = ",", char quoteChar = '"', ILogger? logger = null)
-    : this(tableName, headers, delimiter, quoteChar, logger)
-    {
-        Reader = new Lazy<StreamReader>(() => new StreamReader(stream));
-    }
-
-    public CsvReader(string filePath, string tableName, string[] headers, string delimiter = ",", char quoteChar = '"', ILogger? logger = null)
-    : this(tableName, headers, delimiter, quoteChar, logger)
-    {
-        var fs = File.OpenRead(filePath);
-        FilePath = filePath;
-        Reader = new Lazy<StreamReader>(() => new StreamReader(fs));
-    }
-
-    private CsvReader(string tableName, string[] headers, string delimiter = ",", char quoteChar = '"', ILogger? logger = null)
-    {
-        TableName = tableName;
-        Headers = headers;
-        Logger = logger;
-        Delimiter = delimiter;
-        QuoteChar = quoteChar;
-        Reader = null!;
-    }
-
-    public virtual async ValueTask<bool> ReadAsync()
-    {
-        var str = await Reader.Value.ReadLineAsync();
-        if (string.IsNullOrWhiteSpace(str))
-            return false;
-
-        CurrentRaw = str;
-
-        var fields = ParseRow(str, QuoteChar, Delimiter);
-        Current = new DataRecord(fields, TableName, Headers){RawField = str};
-        return true;
-    }
-
-    public string[] ParseRow(ReadOnlySpan<char> source, char quoteChar, string delimiter)
-    {
-        var result = new List<string>();
-        var index = -1;
-        var current = new StringBuilder();
-        var hasQuote = false;
-        var hasSlash = false;
-        while (index < source.Length - 1)
-        {
-            index++;
-            if (hasSlash == false && source[index] == '\\')
-            {
-                hasSlash = true;
-                current.Append('\\');
-                continue;
-            }
-
-            if (hasSlash == false && source[index] == quoteChar)
-            {
-                hasQuote = !hasQuote;
-                current.Append(source[index]);
-                continue;
-            }
-
-            if (hasQuote == false && source[index] == delimiter[0])
-            {
-                result.Add(current.ToString());
-                current.Clear();
-            }
-            else
-            {
-                current.Append(source[index]);
-            }
-
-            hasSlash = false;
-        }
-
-        result.Add(current.ToString());
-        return result.ToArray();
-    }
-
-    public virtual void Dispose()
-    {
-        if(Reader.IsValueCreated)
-            Reader.Value.Dispose();
-    }
-}

ConsoleApp2/Services/ETL/DataReaderFactory.cs

@@ -1,46 +0,0 @@
-using ConsoleApp2.HostedServices.Abstractions;
-using ConsoleApp2.Options;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Options;
-
-namespace ConsoleApp2.Services.ETL;
-
-public class DataReaderFactory
-{
-    private readonly ILogger<DataReaderFactory> _logger;
-    private readonly IOptions<DataInputOptions> _options;
-
-    public DataReaderFactory(ILogger<DataReaderFactory> logger, IOptions<DataInputOptions> options)
-    {
-        _logger = logger;
-        _options = options;
-    }
-
-    public IDataReader CreateReader(string filePath, string tableName, string[] headers)
-    {
-        if (_options.Value.UseMock)
-        {
-            if (_options.Value.TableMockConfig is null)
-                throw new ApplicationException("未配置表模拟数据量级");
-            _logger.LogDebug("***** Using {Type} data source *****", "ZSTD mock");
-            var mockConfig = _options.Value.TableMockConfig.GetValueOrDefault(tableName, 
-                new TableMockConfig { MockCount = 1, UseDeepCopy = false });
-            mockConfig.MockCount = (long)Math.Ceiling(mockConfig.MockCount * _options.Value.MockCountMultiplier); 
-            return new ZstMockReader(mockConfig, filePath,
-                tableName, headers, _options.Value.Delimiter, _options.Value.QuoteChar, _logger);
-        }
-
-        _logger.LogDebug("***** Using {Type} data source *****", "ZSTD");
-        return new ZstReader(filePath, tableName, headers, _options.Value.Delimiter, _options.Value.QuoteChar, _logger);
-    }
-}
-
-public static class DataSourceFactoryExtensions
-{
-    public static IServiceCollection AddDataSourceFactory(this IServiceCollection services)
-    {
-        services.AddSingleton<DataReaderFactory>();
-        return services;
-    }
-}

ConsoleApp2/Services/ETL/ZstMockReader.cs

@@ -1,64 +0,0 @@
-using ConsoleApp2.Options;
-using Microsoft.Extensions.Logging;
-
-namespace ConsoleApp2.Services.ETL;
-
-/// <summary>
-/// 截取提供ZST文件中的第一行，然后复制成指定数量的数据
-/// </summary>
-public class ZstMockReader : ZstReader
-{
-    private long _currentCount;
-    private readonly long _mockCount;
-    private DataRecord? _template;
-    private readonly bool _deepCopy;
-    private readonly string[]? _autoIncrementColumn;
-    
-    static readonly IReadOnlyList<int> Range = [500, 1500, 2500];
-    
-    public ZstMockReader(TableMockConfig mockConfig, string filePath, string tableName, string[] headers, string delimiter = ",", char quoteChar = '\"', ILogger? logger = null) : base(filePath, tableName, headers, delimiter, quoteChar, logger)
-    {
-        _mockCount = mockConfig.MockCount;
-        _deepCopy = mockConfig.UseDeepCopy;
-        _autoIncrementColumn = mockConfig.AutoIncrementColumn;
-    }
-
-    public ZstMockReader(TableMockConfig mockConfig, Stream stream, string tableName, string[] headers, string delimiter = ",", char quoteChar = '\"', ILogger? logger = null) : base(stream, tableName, headers, delimiter, quoteChar, logger)
-    {
-        _mockCount = mockConfig.MockCount;
-        _deepCopy = mockConfig.UseDeepCopy;
-        _autoIncrementColumn = mockConfig.AutoIncrementColumn;
-    }
-
-    public override async ValueTask<bool> ReadAsync()
-    {
-        if (_template is null)
-        {
-            if (!await base.ReadAsync())
-                throw new InvalidOperationException("所提供的ZST源为空，无法生成模板数据");
-            _template = Current.Clone() as DataRecord;
-            if (_template is null)
-                throw new ApplicationException("记录拷贝失败");
-            _currentCount++;
-            return true;
-        }
-
-        if (_deepCopy)
-        {
-            Current = _template.Clone() as DataRecord ?? throw new ApplicationException("记录拷贝失败");
-            if(_autoIncrementColumn is not null)
-            {
-                foreach (var column in _autoIncrementColumn)
-                {
-                    Current[column] = (Convert.ToInt64(Current[column]) + 1).ToString();
-                    _template = Current;
-                }
-            }
-
-            Current["CompanyID"] = Range[Random.Shared.Next(0, Range.Count)].ToString();//随机CompanyID
-        }
-        else Current = _template;
-        _currentCount++;
-        return _currentCount < _mockCount;
-    }
-}

ConsoleApp2/Services/ETL/ZstReader.cs

@@ -1,48 +0,0 @@
-using Microsoft.Extensions.Logging;
-using ZstdSharp;
-
-namespace ConsoleApp2.Services.ETL;
-
-/// <summary>
-/// 解压ZST文件，从中读取CSV数据
-/// </summary>
-public class ZstReader : CsvReader
-{
-    protected new readonly Lazy<StreamReader> Reader;
-
-
-    public ZstReader(string filePath, string tableName, string[] headers, string delimiter = ",", char quoteChar = '\"', ILogger? logger = null) 
-        : base(filePath, tableName, headers, delimiter, quoteChar, logger)
-    {
-        var ds = new DecompressionStream(File.OpenRead(filePath));
-        Reader = new Lazy<StreamReader>(() => new StreamReader(ds));
-    }
-        
-    public ZstReader(Stream stream, string tableName, string[] headers, string delimiter = ",", char quoteChar = '\"', ILogger? logger = null) 
-        : base(stream, tableName, headers, delimiter, quoteChar, logger)
-    {
-        var ds = new DecompressionStream(stream);
-        Reader = new Lazy<StreamReader>(() => new StreamReader(ds));
-    }
-
-    public override async ValueTask<bool> ReadAsync()
-    {
-        var str = await Reader.Value.ReadLineAsync();
-        if (string.IsNullOrWhiteSpace(str))
-            return false;
-
-        CurrentRaw = str;
-
-        var fields = ParseRow(str, QuoteChar, Delimiter);
-        Current = new DataRecord(fields, TableName, Headers) {RawField = str};
-        return true;
-    }
-
-    public override void Dispose()
-    {
-        base.Dispose();
-        if(Reader.IsValueCreated) 
-            Reader.Value.Dispose();
-    }
-
-}

ConsoleApp2/Services/ErrorRecorder.cs

@@ -1,19 +1,27 @@
 using System.Text;
 using Microsoft.Extensions.Logging;
 
-namespace ConsoleApp2.Services.ErrorRecorder;
+namespace ConsoleApp2.Services;
 
-
+public class ErrorRecorder
-public sealed class OutputErrorRecorder : ErrorRecorder
 {
-    private readonly string _outputDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"/ErrorRecords/{UID}/Output");
+    private readonly string _outputDir = "./ErrorRecords";
-    private readonly string _database;
+    private readonly ILogger _logger;
     private readonly Dictionary<string, int> _logIndex = new();
 
-    public OutputErrorRecorder(string database, ILogger logger) : base(logger)
+    /// <summary>
+    /// 当次执行标识
+    /// </summary>
+    private static readonly string UID = DateTime.Now.ToString("yyyy-MM-dd HH-mm-ss");
+
+    public ErrorRecorder(ILogger<ErrorRecorder> logger)
     {
-        _database = database;
+        _logger = logger;
-        Logger = logger;
+        var dir = Path.Combine(_outputDir, UID);
+        if (!Directory.Exists(dir))
+        {
+            Directory.CreateDirectory(dir);
+        }
     }
 
     /// <summary>
@@ -24,28 +32,25 @@ public sealed class OutputErrorRecorder : ErrorRecorder
     /// <param name="exception"></param>
     public async Task LogErrorSqlAsync(string commandText, string tableName, Exception exception)
     {
-        if (!Directory.Exists(_outputDir))
-            Directory.CreateDirectory(_outputDir);
         if (!_logIndex.TryGetValue(tableName, out var idx))
         {
             idx = 0;
             _logIndex.Add(tableName, idx);
         }
-        var filePath = Path.Combine(_outputDir, $"{tableName}-{idx}.errlog");
+        var filePath = Path.Combine(_outputDir, UID, $"{tableName}-{idx}.errlog");
             
         if (File.Exists(filePath) && new FileInfo(filePath).Length > 10 * 1024 * 1024)
         {
             ++idx;
             _logIndex[tableName] = idx;
-            filePath = Path.Combine(_outputDir, $"{tableName}-{idx}.errlog");
+            filePath = Path.Combine(_outputDir, UID, $"{tableName}-{idx}.errlog");
         }
         var content = $"""
                       /* [{DateTime.Now:yyyy-MM-dd HH:mm:ss}]
-                       * Error occurred when export table '{_database}.{tableName}':
+                       * Error occurred when export table '{tableName}':
                        * {exception.Message}
                        */
                       
-                      USE `{_database}`;
                       {commandText}
 
 
@@ -60,9 +65,7 @@ public sealed class OutputErrorRecorder : ErrorRecorder
     /// <param name="exception"></param>
     public async Task LogErrorSqlAsync(string commandText, Exception exception)
     {
-        if (!Directory.Exists(_outputDir))
+        var filePath = Path.Combine(_outputDir, UID, "UnknownTables.errlog");
-            Directory.CreateDirectory(_outputDir);
-        var filePath = Path.Combine(_outputDir, "UnknownTables.errlog");
         var content = $"""
                        /* [{DateTime.Now:yyyy-MM-dd HH:mm:ss}]
                         * Error occurred when export table with unknown table name:
@@ -75,4 +78,27 @@ public sealed class OutputErrorRecorder : ErrorRecorder
         await File.AppendAllTextAsync(filePath, content, Encoding.UTF8);
     }
 
+    public async Task LogErrorRecordsAsync(IDictionary<string, DataRecord> records, Exception exception)
+    {
+        var pathDict = new Dictionary<string, string>();
+        foreach (var pair in records)
+        {
+            if(!pathDict.TryGetValue(pair.Key, out var path))
+            {
+                path = Path.Combine(_outputDir, UID, "ErrorRecords", $"{pair.Key}.errlog");
+                pathDict.Add(pair.Key, path);
+            }
+            //
+            await File.AppendAllTextAsync(path, string.Join(',', pair.Value.Fields));
+        }
+    }
+    
+    public void ClearErrorRecords()
+    {
+        _logger.LogInformation("***** Clear error records *****");
+        foreach (var file in Directory.GetFiles(_outputDir, "*.errlog", SearchOption.AllDirectories))
+        {
+            File.Delete(file);
+        }
+    }
 }

ConsoleApp2/Services/ErrorRecorder/ErrorRecorder.cs

@@ -1,79 +0,0 @@
-using ConsoleApp2.Helpers;
-using Microsoft.Extensions.Logging;
-
-namespace ConsoleApp2.Services.ErrorRecorder;
-
-public class ErrorRecorder
-{
-    protected ILogger Logger;
-
-    /// <summary>
-    /// 当次执行标识
-    /// </summary>
-    public static readonly string UID = DateTime.Now.ToString("yyyy-MM-dd HH-mm-ss");
-
-    public ErrorRecorder(ILogger logger)
-    {
-        Logger = logger;
-    }
-    
-    
-     public static async Task LogErrorRecordAsync(string outputDir, DataRecord record, Exception exception)
-     {
-         if(!Directory.Exists(outputDir))
-             Directory.CreateDirectory(outputDir);
-         var content = $"""
-                        ### {exception.Message}
-                        {record.RawField}
-                        """;
-         var path = Path.Combine(outputDir, $"{record.TableName}.errlog");
-         await File.AppendAllTextAsync(path, content);
-     }
-     
-     public static async Task LogErrorRecordAsync(string outputDir, IEnumerable<DataRecord> records, Exception exception)
-     {
-         if(!Directory.Exists(outputDir))
-             Directory.CreateDirectory(outputDir);
-         var tableMapping = new Dictionary<string, Tuple<List<DataRecord>, StreamWriter>>();
-         foreach (var record in records)
-         {
-             tableMapping.AddOrUpdate(record.TableName,
-                 Tuple.Create((List<DataRecord>) [record], new StreamWriter(File.OpenRead(record.TableName))),
-                 (_, tuple) =>
-                 {
-                     tuple.Item1.Add(record);
-                     return tuple;
-                 });
-         }
-
-         var maxParallelism = 5;
-         for (var i = 0; i < tableMapping.Count; i+=maxParallelism)
-         {
-             await Parallel.ForEachAsync(tableMapping.Take(maxParallelism), async (pair, token) =>
-             {
-                 var (records, writer) = pair.Value;
-                 foreach (var record in records)
-                 {
-                     var content =
-                         $"""
-                          ### {exception.Message}
-                          {record.RawField}
-                          """;
-                     await writer.WriteLineAsync(content);
-                     if (token.IsCancellationRequested)
-                         break;
-                 }
-                 await writer.DisposeAsync();
-             });
-         }
-     }
-     
-     public void ClearErrorRecords(string dir)
-     {
-         Logger.LogInformation("***** Clear error records *****");
-         foreach (var file in Directory.GetFiles(dir, "*.errlog", SearchOption.AllDirectories))
-         {
-             File.Delete(file);
-         }
-     }
-}

ConsoleApp2/Services/ErrorRecorder/ErrorRecorderFactory.cs

@@ -1,27 +0,0 @@
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Logging;
-
-namespace ConsoleApp2.Services.ErrorRecorder;
-
-public class ErrorRecorderFactory
-{
-    private readonly ILogger<ErrorRecorderFactory> _logger;
-
-    public ErrorRecorderFactory(ILogger<ErrorRecorderFactory> logger)
-    {
-        _logger = logger;
-    }
-    
-    public OutputErrorRecorder CreateOutput(string database) => new(database, _logger);
-    public TransformErrorRecorder CreateTransform() => new(_logger);
-    public InputErrorRecorder CreateInput() => new(_logger);
-}
-
-public static class ErrorRecorderFactoryExtensions
-{
-    public static IServiceCollection AddErrorRecorderFactory(this IServiceCollection services)
-    {
-        services.AddSingleton<ErrorRecorderFactory>();
-        return services;
-    }
-}

ConsoleApp2/Services/ErrorRecorder/InputErrorRecorder.cs

@@ -1,19 +0,0 @@
-using Microsoft.Extensions.Logging;
-
-namespace ConsoleApp2.Services.ErrorRecorder;
-
-public sealed class InputErrorRecorder : ErrorRecorder
-{
-    private readonly string _outputDir =
-        Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"ErrorRecords/{UID}/Input");
-    
-    public InputErrorRecorder(ILogger logger) : base(logger)
-    {
-    }
-    
-    public Task LogErrorRecordAsync(DataRecord record, Exception exception) =>
-        LogErrorRecordAsync(_outputDir, record, exception);
-
-    public Task LogErrorRecordAsync(IEnumerable<DataRecord> records, Exception exception) =>
-        LogErrorRecordAsync(_outputDir, records, exception);
-}

ConsoleApp2/Services/ErrorRecorder/TransformErrorRecorder.cs

@@ -1,20 +0,0 @@
-using Microsoft.Extensions.Logging;
-
-namespace ConsoleApp2.Services.ErrorRecorder;
-
-public sealed class TransformErrorRecorder : ErrorRecorder
-{
-    private readonly string _outputDir =
-        Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"ErrorRecords/{UID}/Transform");
-
-
-    public TransformErrorRecorder(ILogger logger) : base(logger)
-    {
-    }
-
-    public Task LogErrorRecordAsync(DataRecord record, Exception exception) =>
-        LogErrorRecordAsync(_outputDir, record, exception);
-
-    public Task LogErrorRecordAsync(IEnumerable<DataRecord> records, Exception exception) =>
-        LogErrorRecordAsync(_outputDir, records, exception);
-}

ConsoleApp2/Services/JsvSource.cs

@@ -0,0 +1,44 @@
+using ConsoleApp2.Helpers;
+using ConsoleApp2.HostedServices.Abstractions;
+using Microsoft.Extensions.Logging;
+using ServiceStack.Text;
+
+namespace ConsoleApp2.Services;
+
+/// <summary>
+/// 读取Jsv格式文件
+/// </summary>
+[Obsolete]
+public class JsvSource:IDataSource
+{
+    private readonly string _inputDir;
+    private readonly JsvStringSerializer _jsv;
+    private readonly StreamReader _reader;
+    // ReSharper disable once PrivateFieldCanBeConvertedToLocalVariable
+    private readonly ILogger? _logger;
+    private readonly string _tableName;
+
+    public DataRecord Current { get; protected set; } = null!;
+    public string[]? Headers { get; }
+    public bool EndOfSource => _reader.EndOfStream;
+
+    public JsvSource(string inputDir,string tableName, ILogger? logger = null)
+    {
+        _inputDir = inputDir;
+        _tableName = tableName;
+        _jsv = new JsvStringSerializer();
+       // _reader = new StreamReader(filePath);
+        //Headers = headers;
+        _logger = logger;
+        // _logger?.LogInformation("Reading file: {FilePath}", filePath);
+        //_tableName = DumpDataHelper.GetTableName(filePath);
+    }
+    public async Task DoEnqueue(Action<DataRecord> action)
+    {
+    }
+
+    public  void Dispose()
+    {
+        _reader.Dispose();
+    }
+}

ConsoleApp2/Services/Loggers/CacheTaskMonitorLogger.cs

@@ -1,19 +0,0 @@
-using ConsoleApp2.Cache;
-
-namespace ConsoleApp2.Services.Loggers;
-
-public class CacheTaskMonitorLogger : ITaskMonitorLogger
-{
-    private readonly ICacher _cacher;
-
-    public CacheTaskMonitorLogger(ICacher cacher)
-    {
-        _cacher = cacher;
-    }
-
-    public void LogStatus(string name, IReadOnlyDictionary<string, string> properties, ITaskMonitorLogger.LogLevel logLevel)
-    {
-        if(logLevel is ITaskMonitorLogger.LogLevel.Progress)
-            _cacher.SetHashAsync(name, properties);
-    }
-}

ConsoleApp2/Services/Loggers/ITaskMonitorLogger.cs

@@ -1,12 +0,0 @@
-namespace ConsoleApp2.Services.Loggers;
-
-public interface ITaskMonitorLogger
-{
-    public enum LogLevel
-    {
-        Info,
-        Debug,
-        Progress,
-    }
-    void LogStatus(string name, IReadOnlyDictionary<string, string> properties, LogLevel logLevel = LogLevel.Info);
-}

ConsoleApp2/Services/Loggers/LoggerTaskMonitorLogger.cs

@@ -1,41 +0,0 @@
-using System.Text;
-using Microsoft.Extensions.Logging;
-
-namespace ConsoleApp2.Services.Loggers;
-
-public class LoggerTaskMonitorLogger : ITaskMonitorLogger
-{
-    private readonly ILogger _logger;
-
-    public LoggerTaskMonitorLogger(ILogger<LoggerTaskMonitorLogger> logger)
-    {
-        _logger = logger;
-    }
-
-    public void LogStatus(string name, IReadOnlyDictionary<string, string> properties, ITaskMonitorLogger.LogLevel logLevel)
-    {
-        var sb = new StringBuilder();
-        sb.Append($"{name}: {{");
-        sb.AppendJoin(',', properties.Select((pair, i) => $" {pair.Key}: {pair.Value}"));
-        sb.Append('}');
-        // var args = new List<string> { name };
-        // properties.Aggregate(args, (args, pair) =>
-        // {
-        //     args.Add(pair.Key);
-        //     args.Add(pair.Value);
-        //     return args;
-        // });
-        switch (logLevel)
-        {
-            case ITaskMonitorLogger.LogLevel.Info:
-                _logger.LogInformation("{message}", sb.ToString());
-                break;
-            case ITaskMonitorLogger.LogLevel.Progress:
-            case ITaskMonitorLogger.LogLevel.Debug:
-                _logger.LogDebug("{message}", sb.ToString());
-                break;
-            default:
-                throw new ArgumentOutOfRangeException(nameof(logLevel), logLevel, null);
-        }
-    }
-}

ConsoleApp2/Services/MySqlDestination.cs

@@ -1,12 +1,14 @@
-using System.Text;
+using System.Data.Common;
+using System.Text;
 using System.Text.RegularExpressions;
 using ConsoleApp2.Helpers;
 using ConsoleApp2.Options;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using MySqlConnector;
+using ServiceStack;
 
-namespace ConsoleApp2.Services.ETL;
+namespace ConsoleApp2.Services;
 
 /// <summary>
 /// Mysql导出
@@ -16,29 +18,29 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
     private readonly Dictionary<string, IList<DataRecord>> _recordCache;
     private readonly MySqlConnection _conn;
     private readonly ILogger _logger;
-    private readonly IOptions<DatabaseOutputOptions> _options;
-    private readonly ErrorRecorder.OutputErrorRecorder _outputErrorRecorder;
     private readonly ProcessContext _context;
+    private readonly IOptions<DataTransformOptions> _transformOptions;
+    private readonly ErrorRecorder _errorRecorder;
 
     public MySqlDestination(
         string connStr, 
         ILogger logger, 
-        IOptions<DatabaseOutputOptions> options,
+        ProcessContext context, 
-        ErrorRecorder.OutputErrorRecorder outputErrorRecorder, 
+        IOptions<DataTransformOptions> transformOptions,
-        ProcessContext context)
+        ErrorRecorder errorRecorder)
     {
         _conn = new MySqlConnection(connStr);
         _conn.Open();
         _recordCache = new Dictionary<string, IList<DataRecord>>();
         _logger = logger;
-        _options = options;
-        _outputErrorRecorder = outputErrorRecorder;
         _context = context;
+        _transformOptions = transformOptions;
+        _errorRecorder = errorRecorder;
     }
     
     public Task WriteRecordAsync(DataRecord record)
     {
-        _recordCache.AddOrUpdate(record.TableName, [record], (_, value) =>
+        _recordCache.AddOrUpdate(record.TableName, [record], (key, value) =>
         {
             value.Add(record);
             return value;
@@ -74,23 +76,22 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
                 }
                 catch (Exception e)
                 {
-                    _logger.LogError(e, "插入数据库时发生错误, sql: {Sql}", cmd.CommandText.Omit(1000));
+                    _logger.LogError(e, "Error when flushing records, sql: {Sql}", cmd.CommandText.Omit(1000));
-                    _context.AddException(e);
+                    
                     var match = MatchTableName().Match(cmd.CommandText);
                     if (match is { Success: true, Groups.Count: > 1 })
                     {
                         var tableName = match.Groups[1].Value;
-                        await _outputErrorRecorder.LogErrorSqlAsync(cmd.CommandText, tableName, e);
+                        await _errorRecorder.LogErrorSqlAsync(cmd.CommandText, tableName, e);
                     }
-                    else await _outputErrorRecorder.LogErrorSqlAsync(cmd.CommandText, e);
+                    else await _errorRecorder.LogErrorSqlAsync(cmd.CommandText, e);
                 }
             }
             _recordCache.Clear();
         }
         catch (Exception e)
         {
-            _logger.LogError(e, "序列化记录时发生错误");
+            _logger.LogError(e, "Error when serialize records, record:");
-            throw;
         }
         finally
         {
@@ -103,7 +104,7 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
 
     public IEnumerable<string> GetExcuseList(IDictionary<string, IList<DataRecord>> tableRecords,int maxAllowPacket)
     {
-        var sb = new StringBuilder("SET AUTOCOMMIT = 1;\n");
+        var sb = new StringBuilder();
         foreach (var (tableName, records) in tableRecords)
         {
             if (records.Count == 0)
@@ -115,11 +116,11 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
             
             // INSERT INTO ... VALUES >>>
             sb.Append($"INSERT INTO `{tableName}`(");
-            for (var i = 0; i < records[0].Headers.Count; i++)
+            for (var i = 0; i < records[0].Headers.Length; i++)
             {
                 var header = records[0].Headers[i];
                 sb.Append($"`{header}`");
-                if (i != records[0].Headers.Count - 1)
+                if (i != records[0].Headers.Length - 1)
                     sb.Append(',');
             }
 
@@ -131,7 +132,7 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
                 var record = records[recordIdx];
                 var recordSb = new StringBuilder();
                 recordSb.Append('(');
-                for (var fieldIdx = 0; fieldIdx < record.Fields.Count; fieldIdx++)
+                for (var fieldIdx = 0; fieldIdx < record.Fields.Length; fieldIdx++)
                 {
                     var field = record.Fields[fieldIdx];
 
@@ -143,12 +144,12 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
                         goto Escape;
                     }
                     
-                    switch (_options.Value.GetColumnType(record.TableName, record.Headers[fieldIdx]))
+                    switch (_transformOptions.Value.GetColumnType(record.TableName, record.Headers[fieldIdx]))
                     {
                         case ColumnType.Text:
-                            if(string.IsNullOrEmpty(field))
+                            recordSb.Append(string.IsNullOrEmpty(field)
-                                recordSb.Append("''");
+                                ? "''"
-                            else recordSb.Append($"_utf8mb4 0x{field}");
+                                : _transformOptions.Value.TransformBinary?.Invoke(field) ?? field);
                             break;
                         case ColumnType.Blob:
                             if (string.IsNullOrEmpty(field))
@@ -156,11 +157,9 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
                             else recordSb.Append($"0x{field}");
                             break;
                         case ColumnType.Json:
-                            if(string.IsNullOrEmpty(field))
+                            recordSb.Append(string.IsNullOrEmpty(field)
-                                recordSb.Append("'[]'"); // JObject or JArray?
+                                ? "\"[]\""
-                            else if (_options.Value.TreatJsonAsHex)
+                                : _transformOptions.Value.TransformBinary?.Invoke(field) ?? field);
-                                recordSb.Append($"_utf8mb4 0x{field}");
-                            else recordSb.AppendLine(field);
                             break;
                         case ColumnType.UnDefine:
                         default:
@@ -171,17 +170,16 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
                     Escape:
 
                     #endregion
-                    if (fieldIdx != record.Fields.Count - 1)
+                    if (fieldIdx != record.Fields.Length - 1)
                         recordSb.Append(',');
                 }
 
                 recordSb.Append(')');
 
                 // 若字符数量即将大于限制，则返回SQL，清空StringBuilder，保留当前记录的索引值，然后转到StartBuild标签重新开始一轮INSERT
-                if (sb.Length + recordSb.Length + 23 > maxAllowPacket)
+                if (sb.Length + recordSb.Length + 1 > maxAllowPacket)
                 {
-                    sb.Append(';').AppendLine();
+                    sb.Append(';');
-                    sb.Append("SET AUTOCOMMIT = 1;");
                     yield return sb.ToString();
                     sb.Clear();
                     goto StartBuild;
@@ -194,7 +192,6 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
             }
 
             sb.Append(';');
-            sb.Append("COMMIT;");
             yield return sb.ToString();
             sb.Clear();
         }

ConsoleApp2/Services/ProcessContext.cs

@@ -1,49 +1,39 @@
-using System.Collections.Concurrent;
+namespace ConsoleApp2.Services;
-
-namespace ConsoleApp2.Services;
 
 /// <summary>
 /// 处理上下文类，标识处理进度
 /// </summary>
 public class ProcessContext
 {
-    private bool _hasException;
+    private int _inputCount;
-    private long _inputCount;
+    private int _transformCount;
-    private long _transformCount;
+    private int _outputCount;
-    private long _outputCount;
-    private readonly ConcurrentDictionary<string, long> _tableProgress = new();
-    public bool HasException => _hasException;
     public bool IsInputCompleted { get; private set; }
     public bool IsTransformCompleted { get; private set; }
     public bool IsOutputCompleted { get; private set; }
 
-    public long InputCount
+    public int InputCount
     {
         get => _inputCount;
-        set => Interlocked.Exchange(ref _inputCount, value);
+        private set => _inputCount = value;
     }
 
-    public long TransformCount
+    public int TransformCount
     {
         get => _transformCount;
-        set => Interlocked.Exchange(ref _transformCount, value);
+        private set => _transformCount = value;
     }
 
-    public long OutputCount
+    public int OutputCount
     {
         get => _outputCount;
-        set => Interlocked.Exchange(ref _outputCount, value);
+        private set => _outputCount = value;
     }
     
-
-    // TableName -> Count
-    public IReadOnlyDictionary<string, long> TableProgress => _tableProgress;
-
     public void CompleteInput() => IsInputCompleted = true;
 
     public void CompleteTransform() => IsTransformCompleted = true;
     public void CompleteOutput() => IsOutputCompleted = true;
-    public bool AddException(Exception e) => _hasException = true;
 
     public void AddInput() => Interlocked.Increment(ref _inputCount);
 
@@ -54,17 +44,4 @@ public class ProcessContext
 
     public void AddOutput() => Interlocked.Increment(ref _outputCount);
     public void AddOutput(int count) => Interlocked.Add(ref _outputCount, count);
-
-    public void AddTableOutput(string table, int count)
-    {
-        _tableProgress.AddOrUpdate(table, count, (k, v) => v + count);
-        AddOutput(count);
-    }
-
-    public long GetTableOutput(string table)
-    {
-        if(!_tableProgress.TryGetValue(table, out var count))
-            throw new ApplicationException($"未找到表{table}输出记录");
-        return count;
-    }
 }

ConsoleApp2/Services/RecordQueuePool.cs

@@ -1,65 +0,0 @@
-using System.Collections.Concurrent;
-using Microsoft.Extensions.DependencyInjection;
-
-namespace ConsoleApp2.Services;
-
-public class RecordQueuePool
-{
-    private readonly ConcurrentDictionary<string, DataRecordQueue> _queues = new();
-
-    public IReadOnlyDictionary<string, DataRecordQueue> Queues => _queues;
-
-    public void AddQueue(string key, int boundedCapacity = 200_0000) => AddQueue(key, new DataRecordQueue(boundedCapacity));
-    
-    public void AddQueue(string key, DataRecordQueue queue)
-    {
-        if (!_queues.TryAdd(key, queue))
-            throw new InvalidOperationException($"请勿添加重复的队列，队列名: {key}");
-    }
-    
-    public void RemoveQueue(string key, bool dispose = true)
-    {
-        if (!_queues.Remove(key, out var queue))
-            throw new InvalidOperationException($"未找到对应的队列，队列名：{key}");
-        if (dispose) queue.Dispose();
-    }
-    
-    public DataRecordQueue GetQueue(string key)
-    {
-        return _queues[key];
-    }
-
-    public DataRecordQueue this[string key]
-    {
-        get => GetQueue(key);
-        set => AddQueue(key, value);
-    }
-}
-
-public static class MultiRecordQueueExtensions
-{
-    public static IServiceCollection AddRecordQueuePool(this IServiceCollection services, params string[] keys)
-    {
-        var pool = new RecordQueuePool();
-        foreach (var key in keys)
-        {
-            pool.AddQueue(key);
-        }
-
-        services.AddSingleton(pool);
-        return services;
-    }
-
-    public static IServiceCollection AddRecordQueuePool(this IServiceCollection services,
-        params (string key, DataRecordQueue queue)[] queues)
-    {
-        var pool = new RecordQueuePool();
-        foreach (var (key, queue) in queues)
-        {
-            pool.AddQueue(key, queue);
-        }
-
-        services.AddSingleton(pool);
-        return services;
-    }
-}

ConsoleApp2/Services/TaskManager.cs

@@ -1,5 +1,5 @@
-using ApplicationException = System.ApplicationException;
+using System.Collections.Concurrent;
-using TaskExtensions = ConsoleApp2.Helpers.TaskExtensions;
+using Microsoft.Extensions.Logging;
 
 namespace ConsoleApp2.Services;
 
@@ -8,74 +8,42 @@ namespace ConsoleApp2.Services;
 /// </summary>
 public class TaskManager
 {
-    private int _runningTaskCount;
+    private readonly ConcurrentBag<Task> _tasks;
+    private readonly ILogger _logger;
 
-    public int RunningTaskCount => _runningTaskCount;
+    public int TaskCount => _tasks.Count;
-    public int MaxTaskCount { get; }
+    public int RunningTaskCount => _tasks.Count(task => !task.IsCompleted);
+    public IReadOnlyCollection<Task> Tasks => _tasks;
 
-    public event Action<Exception>? OnException;
+    public TaskManager(ILogger<TaskManager> logger)
-    public event Action? OnTaskCompleteSuccessfully;
-
-    public TaskManager(int maxTaskCount)
     {
-        MaxTaskCount = maxTaskCount;
+        _tasks = new ConcurrentBag<Task>();
+        _logger = logger;
     }
 
-    public async ValueTask<Task> CreateTaskAsync(Func<Task> func, CancellationToken cancellationToken = default)
+    public void CreateTask(Func<Task> func, CancellationToken cancellationToken = default)
     {
-        await TaskExtensions.WaitUntil(() => _runningTaskCount < MaxTaskCount, 25, cancellationToken);
+        var task = Task.Run(func, cancellationToken);
-        return RunTask(func, cancellationToken);
+        _tasks.Add(task);
+        _logger.LogDebug("New task created");
     }
     
-    public async ValueTask<Task> CreateTaskAsync(Func<object?, Task> func, object? arg, CancellationToken ct = default)
+    public void CreateTasks(Func<Task> func,int taskCount, CancellationToken cancellationToken = default)
     {
-        await TaskExtensions.WaitUntil(() => _runningTaskCount < MaxTaskCount, 25, ct);
+        for (int i = 0; i < taskCount; i++)
-        return RunTaskNoClosure(func, arg, ct);
-    }
-    
-    private Task RunTask(Func<Task> func, CancellationToken cancellationToken = default)
-    {
-        var task = Task.Run(async () =>
         {
-            try
+            CreateTask(func, cancellationToken);
-            {
+        }
-                await func();
+    }
-                OnTaskCompleteSuccessfully?.Invoke();
+    public async Task WaitAll()
-            }
+    {
-            catch(Exception ex)
+        await Task.WhenAll(_tasks);
-            {
-                OnException?.Invoke(ex);
-            }
-            finally
-            {
-                Interlocked.Decrement(ref _runningTaskCount);
-            }
-        }, cancellationToken);
-        Interlocked.Increment(ref _runningTaskCount);
-        return task;
     }
 
-    private Task RunTaskNoClosure(Func<object?, Task> func, object? arg, CancellationToken cancellationToken = default)
+    public void ClearTask()
     {
-        var task = Task.Factory.StartNew(async obj => // 性能考虑，这个lambda中不要捕获任何外部变量!
+        if(RunningTaskCount != 0)
-        {
+            throw new InvalidOperationException("Unable to clear task. There are still running tasks");
-            if (obj is not Tuple<Func<object?, Task>, object?> tuple)
+        _tasks.Clear();
-                throw new ApplicationException("这个异常不该出现");
-            try
-            {
-                await tuple.Item1(tuple.Item2);
-                OnTaskCompleteSuccessfully?.Invoke();
-            }
-            catch(Exception ex)
-            {
-                OnException?.Invoke(ex);
-            }
-            finally
-            {
-                Interlocked.Decrement(ref _runningTaskCount);
-            }
-        }, Tuple.Create(func, arg), cancellationToken).Unwrap();
-        Interlocked.Increment(ref _runningTaskCount);
-        return task;
     }
 }

ConsoleApp2/Services/ZstSource.cs

@@ -0,0 +1,96 @@
+using ConsoleApp2.Helpers;
+using Microsoft.Extensions.Logging;
+using System.IO;
+using System.Text.RegularExpressions;
+using ZstdSharp;
+namespace ConsoleApp2.Services
+{
+    public class ZstSource : CsvSource
+    {
+        public ZstSource(string inputDir, string tableName, string delimiter = ",", char quoteChar = '"',
+        ILogger? logger = null) : base(inputDir, tableName, delimiter = ",", quoteChar = '"', logger = null)
+        {
+            //throw new Exception("aaa");
+            string pattern = $"^.*\\.{tableName}\\..*\\.sql.zst$";
+             _sqlFilePath = Directory.GetFiles(_inputDir).FirstOrDefault(s => Regex.Match(s, pattern).Success);
+
+        }
+        private async Task<string> DecompressFile(string filePath)
+        {
+            using (var input = File.OpenRead(filePath))
+            {
+                using (var decopress = new DecompressionStream(input))
+                {
+
+                    var ms = new MemoryStream();
+                    decopress.CopyTo(ms);
+                    ms.Seek(0, SeekOrigin.Begin);
+                    StreamReader reader = new StreamReader(ms);
+                    var text = await reader.ReadToEndAsync();
+                    return text;
+
+                }
+            }
+        }
+        public override async Task GetHeaderAndCsvFiles()
+        {
+            var text = await DecompressFile(_sqlFilePath);
+            headers=await DumpDataHelper.GetCsvHeadersFromSqlFileAsync(text);
+            csvFiles=await DumpDataHelper.GetCsvFileNamesFromSqlFileAsync(text, new Regex(@"'.+\.dat.zst'"));
+
+        }
+        public override async Task DoEnqueue(Action<DataRecord> action)
+        {
+           await GetHeaderAndCsvFiles();
+            foreach (var file in csvFiles)
+            {
+                var filePath = Path.Combine(_inputDir, file);
+                using (var input = File.OpenRead(filePath))
+                {
+                    using (var decopress = new DecompressionStream(input)) 
+                    {
+                        using( var reader = new StreamReader(decopress))
+                        {
+                            while (!reader.EndOfStream)
+                            {
+                                var line = await reader.ReadLineAsync();
+                                var fields = ParseRow2(line, QuoteChar, Delimiter);
+                                var record = new DataRecord(fields, _tableName, headers);
+                                action?.Invoke(record);
+                            }
+                        }
+                    }
+                }
+
+            }
+        }
+        public override async Task<DataRecord?> GetTestRecord()
+        {
+            await GetHeaderAndCsvFiles();
+            var file = csvFiles.FirstOrDefault();
+            if (file != null)
+            {
+                var filePath = Path.Combine(_inputDir, file);
+                using (var input = File.OpenRead(filePath))
+                {
+                    using (var decopress = new DecompressionStream(input))
+                    {
+                        using (var reader = new StreamReader(decopress))
+                        {
+                            var line = await reader.ReadLineAsync();
+                            var fields = ParseRow2(line, QuoteChar, Delimiter);
+                            var record = new DataRecord(fields, _tableName, headers);
+                            return record;
+                        }
+
+                    }
+                }
+            }
+            return null;
+        }
+        public void Dispose()
+        {
+            //_reader.Dispose();
+        }
+    }
+}

ConsoleApp2/SimulationService/SimulationInputService.cs

@@ -12,7 +12,6 @@ using System.Text.RegularExpressions;
 namespace ConsoleApp2.SimulationService
 {
 
-    #if false
     public class SimulationInputService : IInputService
     {
         private readonly ILogger _logger;
@@ -132,5 +131,4 @@ namespace ConsoleApp2.SimulationService
             _logger.LogInformation("***** Csv input service completed *****");
         }
     }
-#endif
 }

ConsoleApp2/appsettings.json

@@ -1,31 +1,17 @@
 {
-  "Logging": {
+  "CmdOptions": {
-    "LogLevel": {
+    "InputFileType": "CSV",
-      "Default": "Debug"
+    "InputDir": "D:/MyDumper-ZST",
-    }
+    "TaskCount": 6,
+    "FlushCount": 10000,
+    "Isutf8mb4": true,
+    "OldestShardKey": 23000,
+    "OldestTime": "202301"
   },
-  "Input":{
+  "ConnectionStrings": {
-    "InputDir": "D:\\Dump\\MockData",
+    "MySqlMaster": "Server=127.0.0.1;Port=33309;UserId=root;Password=123456;" // 要分库，不用加'Database='了
-    "CleanDate": "202301",
-    "StrictMode": false,
-    "UseMock": true,
-    "MockCountMultiplier": 0.05
   },
-  "Transform":{
+  "RedisCacheOptions": {
-    "StrictMode": true,
-    "EnableFilter": false,
-    "EnableReplacer": false,
-    "EnableReBuilder": false
-  },
-  "Output":{
-    "ConnectionString": "Server=127.0.0.1;Port=3306;UserId=root;Password=cfmes123456;", // 要分库，不用加'Database='了
-    "MaxAllowedPacket": 67108864,
-    "FlushCount": 20000,
-    "MaxDatabaseOutputTask" : 4,
-    "TreatJsonAsHex": false,
-    "RestoreIndex": true
-  },
-  "RedisCache": {
     "Configuration": "192.168.1.246:6380",
     "InstanceName" : "mes-etl:"
   },
@@ -42,7 +28,6 @@
       "cferp_test_1": 1000,
       "cferp_test_2": 2000,
       "cferp_test_3": 2147483647
-//      "cferp_void_1": 2147483647
     }
   }
 }

TestProject1/DataFix.cs

@@ -1,142 +0,0 @@
-using System.Data;
-using ConsoleApp2.Helpers;
-using ConsoleApp2.Helpers.Database;
-using MySqlConnector;
-using Xunit.Abstractions;
-
-namespace TestProject1;
-
-public class DataFix
-{
-    public const string ConnStr = "Server=192.168.1.245;Port=3306;UserId=root;Password=ruixinjie!@#123;";
-    public static string[] DbNames = ["cferp_test_1", "cferp_test_2", "cferp_test_3"];
-    
-    private readonly ITestOutputHelper _output;
-    
-    public DataFix(ITestOutputHelper output)
-    {
-        _output = output;
-    }
-    
-    private string MakeConnStr(string dbName) => ConnStr + $"Database={dbName};";
-
-    /// <summary>
-    /// 查找cferp_test_1中CompanyID = 0的order_box_block，根据OrderNo查找对应订单的CompanyID，然后删除重新插入相应的数据库
-    /// 如果没有找到对应订单，则保留CompanyID为0
-    /// </summary>
-    [Fact]
-    public async Task FixOrderBoxBlockCompanyID()
-    {
-        var ds = await DatabaseHelper.QueryTableAsync(MakeConnStr(DbNames[0]), "SELECT * FROM `order_box_block` WHERE COMPANYID = 0");
-        var dict = new Dictionary<long, int>();//orderNo -> CompanyID
-        foreach (DataRow row in ds.Tables[0].Rows)
-        {
-            // foreach (var column in row.ItemArray)
-            // {
-            //     Console.Write(column.ToString() + '\t');
-            // }
-
-            var orderNo = Convert.ToInt64(row["OrderNo"]);
-            var boxId = Convert.ToInt64(row["BoxID"]);
-
-            int? companyId = null;
-            foreach (var db in DbNames)
-            {
-                if(!dict.TryGetValue(orderNo, out var cid)) // 可以提到外面
-                {
-                    var result = await DatabaseHelper.QueryScalarAsync(ConnStr + $"Database={db};",
-                        $"SELECT CompanyID FROM `order` WHERE OrderNo = {orderNo}");
-                    if (result is null or 0) continue;
-                    
-                    companyId = Convert.ToInt32(result);
-                    dict.Add(orderNo, companyId.Value);
-                    break;
-                }
-                else
-                {
-                    companyId = cid;
-                }
-            }
-
-            if (companyId is null or 0)
-            {
-                _output.WriteLine($"OrderBoxBlock:{boxId} - OrderNo {orderNo} not found");
-                continue;
-            }
-
-            row["CompanyID"] = companyId;
-            await DatabaseHelper.TransactionAsync(ConnStr,
-                $"""
-                 DELETE FROM cferp_test_1.`order_box_block` WHERE BoxID = {boxId};
-                 INSERT INTO {TenantDbHelper.GetDbNameByTenantKeyValue(companyId.Value)}.`order_box_block`
-                 VALUES(@c1, @c2, @c3, @c4, @c5);
-                 """, [new MySqlParameter("c1", row[0]),
-                    new MySqlParameter("c2", row[1]),
-                    new MySqlParameter("c3", row[2]),
-                    new MySqlParameter("c4", row[3]),
-                    new MySqlParameter("c5", row[4])]);
-            _output.WriteLine($"OrderBoxBock:{boxId} CompanyID -> {companyId}");
-        }
-    }
-
-    [Theory]
-    [InlineData(["order_data_block", "ID", 0])]
-    [InlineData(["order_data_block", "ID", 1])]
-    [InlineData(["order_data_block", "ID", 2])]
-    public async Task FixCompanyIdWithOwnOrderNo(string tableName, string keyName, int dbNameIndex)
-    {
-        var ds = await DatabaseHelper.QueryTableAsync(MakeConnStr(DbNames[dbNameIndex]), 
-            $"SELECT * FROM `{tableName}` WHERE COMPANYID = 0");
-        var dict = new Dictionary<long, int>();//orderNo -> CompanyID
-        foreach (DataRow row in ds.Tables[0].Rows)
-        {
-            // foreach (var column in row.ItemArray)
-            // {
-            //     Console.Write(column.ToString() + '\t');
-            // }
-
-            var orderNo = Convert.ToInt64(row["OrderNo"]);
-            var key = Convert.ToInt32(row[keyName]);
-
-            int? companyId = null;
-            if(!dict.TryGetValue(orderNo, out var cid))
-            {
-                companyId = await MesDatabaseHelper.TrySearchCompanyId(ConnStr, DbNames,
-                    $"SELECT CompanyID FROM `order` WHERE OrderNo = {orderNo}");
-            }
-            else
-            {
-                companyId = cid;
-            }
-
-            if (companyId is null or 0)
-            {
-                _output.WriteLine($"{tableName}:{key} - OrderNo {orderNo} not found");
-                continue;
-            }
-
-            row["CompanyID"] = companyId;
-            // ON DUPLICATE KEY UPDATE
-            await DatabaseHelper.TransactionAsync(ConnStr,
-                $"""
-                 DELETE FROM {DbNames[dbNameIndex]}.`{tableName}` WHERE ID = {key};
-                 INSERT INTO {TenantDbHelper.GetDbNameByTenantKeyValue(companyId.Value)}.`{tableName}`
-                 VALUES({string.Join(',', Enumerable.Range(0, row.ItemArray.Length).Select(i => $"@c{i}"))})
-                 ON DUPLICATE KEY UPDATE CompanyID = {companyId};
-                 """, 
-                Enumerable.Range(0, row.ItemArray.Length).Select(i => new MySqlParameter($"c{i}", row[i])).ToArray());
-            _output.WriteLine($"{tableName}:{key} CompanyID -> {companyId}");
-        }
-    }
-
-    [Theory]
-    [InlineData("order", 0)]
-    [InlineData("order", 1)]
-    [InlineData("order", 2)]
-    public async Task FixShardKeyWithOwnOrderNo(string tableName, int dbNameIndex)
-    {
-        var r = await DatabaseHelper.NonQueryAsync(MakeConnStr(DbNames[dbNameIndex]),
-            $"UPDATE `{tableName}` SET ShardKey = CONCAT(SUBSTR(`order`.OrderNo,3,4),'0') WHERE ShardKey = 0");
-        _output.WriteLine($"Affect Rows: {r}");
-    }
-}

TestProject1/DatabaseToolBox.cs

@@ -1,154 +0,0 @@
-using System.Data;
-using System.Text;
-using ConsoleApp2.Helpers.Database;
-using MySqlConnector;
-using Newtonsoft.Json;
-using Newtonsoft.Json.Linq;
-using Xunit.Abstractions;
-
-namespace TestProject1;
-
-public class DatabaseToolBox
-{
-    private readonly ITestOutputHelper _output;
-    public const string ConnStr = "Server=127.0.0.1;Port=33309;UserId=root;Password=123456;";
-
-    public DatabaseToolBox(ITestOutputHelper output)
-    {
-        _output = output;
-    }
-
-    [Theory]
-    [InlineData("cferp_void_1")]
-    public async Task AlterAllTableToBlackHole(string database)
-    {
-        var connStr = ConnStr + $"Database={database}";
-        var tables = await DatabaseHelper.QueryTableAsync(connStr,
-            $"""
-             SELECT TABLE_NAME FROM information_schema.`TABLES` WHERE TABLE_SCHEMA = '{database}';
-             """);
-
-        foreach (DataRow row in tables.Tables[0].Rows)
-        {
-            var tableName = row["TABLE_NAME"].ToString();
-            var sql = $"""
-                       ALTER TABLE `{tableName}` REMOVE PARTITIONING;
-                       """;
-            try
-            {
-                await DatabaseHelper.NonQueryAsync(connStr, sql);
-            }
-            catch (MySqlException e) when (e.ErrorCode == MySqlErrorCode.PartitionManagementOnNoPartitioned)
-            {
-            }
-
-            sql = $"""
-                   ALTER TABLE `{tableName}` ENGINE=BLACKHOLE;
-                   """;
-            await DatabaseHelper.NonQueryAsync(connStr, sql);
-        }
-    }
-
-    public async Task<long> CountAllTable(string connStr, string database)
-    {
-        var count = 0L;
-        var set = await DatabaseHelper.QueryTableAsync(connStr,
-            $"""
-             SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = '{database}';
-             """);
-        foreach (DataRow row in set.Tables[0].Rows)
-        {
-            count += Convert.ToInt64(await DatabaseHelper.QueryScalarAsync(connStr,
-                $"""
-                 SELECT COUNT(1) FROM `{database}`.`{row[0]}`;
-                 """));
-        }
-        _output.WriteLine($"Record count: {count} records");
-        return count;
-    }
-
-    [Theory]
-    [InlineData([new[]{"cferp_test_1", "cferp_test_2", "cferp_test_3"}])]
-    public async Task CountAllDatabase(IEnumerable<string> databases)
-    {
-        var count = 0L;
-        var connStr = "Server=192.168.1.245;Port=3306;UserId=root;Password=ruixinjie!@#123;";
-        foreach (var db in databases)
-        {
-            count += await CountAllTable(connStr, db);
-        }
-
-        _output.WriteLine(count.ToString());
-    }
-
-    public async Task<TableIndex[]> GetAllTableIndexes(string database)
-    {
-        var data = await DatabaseHelper.QueryTableAsync(ConnStr,
-            $"""
-            SELECT TABLE_NAME, INDEX_NAME, NON_UNIQUE, COLUMN_NAME, INDEX_TYPE FROM information_schema.STATISTICS
-                     WHERE TABLE_SCHEMA = '{database}'
-                       AND INDEX_NAME != 'PRIMARY'
-                       AND INDEX_TYPE = 'BTREE'
-            """);
-        return data.Tables[0].Rows.Cast<DataRow>().Select(row =>
-        {
-            return new TableIndex(row["TABLE_NAME"].ToString()!,
-                row["INDEX_NAME"].ToString()!,
-                !Convert.ToBoolean(row["NON_UNIQUE"]),
-                row["COLUMN_NAME"].ToString()!,
-                row["INDEX_TYPE"] switch
-                {
-                    "BTREE" => TableIndex.TableIndexType.BTree,
-                    "PRIMARY" => TableIndex.TableIndexType.Primary,
-                    "HASH" => TableIndex.TableIndexType.Hash,
-                    _ => throw new ArgumentOutOfRangeException(nameof(row))
-                }
-            );
-        }).ToArray();
-    }
-
-    [Theory]
-    [InlineData(["cferp_test_1", "D:/Indexes_cferp_test_1.json"])]
-    [InlineData(["cferp_test_2", "D:/Indexes_cferp_test_2.json"])]
-    [InlineData(["cferp_test_3", "D:/Indexes_cferp_test_3.json"])]
-    public async Task ExportAllIndexes(string database, string outputPath)
-    {
-        var indexes = await GetAllTableIndexes(database);
-        var json = JArray.FromObject(indexes);
-        await File.WriteAllTextAsync(outputPath, json.ToString());
-        _output.WriteLine($"Exported {indexes.Length} indexes to '{outputPath}'");
-    }
-
-    [Theory]
-    [InlineData("cferp_test_1", "D:/Indexes_cferp_test_1.json")]
-    [InlineData("cferp_test_2", "D:/Indexes_cferp_test_2.json")]
-    [InlineData("cferp_test_3", "D:/Indexes_cferp_test_3.json")]
-    public async Task ImportAllIndexes(string database, string importPath)
-    {
-        var json = await File.ReadAllTextAsync(importPath);
-        var indexes = JsonConvert.DeserializeObject<TableIndex[]>(json);
-        var sb = new StringBuilder();
-        foreach (var (tableName, indexName, isUnique, columnName, tableIndexType) in indexes!)
-        {
-            sb.AppendLine($"CREATE {(isUnique ? "UNIQUE" : string.Empty)} INDEX `{indexName}` ON `{database}`.`{tableName}` (`{columnName}`) USING {tableIndexType};");
-        }
-        await DatabaseHelper.NonQueryAsync(ConnStr, sb.ToString());
-        _output.WriteLine($"Import {indexes.Length} indexes to '{database}' from '{importPath}'");
-    }
-
-    [Theory]
-    [InlineData(["cferp_test_1"])]
-    [InlineData(["cferp_test_2"])]
-    [InlineData(["cferp_test_3"])]
-    public async Task DropAllIndex(string database)
-    {
-        var indexes = await GetAllTableIndexes(database);
-        var sb = new StringBuilder();
-        foreach (var (tableName, indexName) in indexes)
-        {
-            sb.AppendLine($"DROP INDEX `{indexName}` ON `{database}`.`{tableName}`;");
-        }
-        await DatabaseHelper.NonQueryAsync(ConnStr, sb.ToString());
-        _output.WriteLine($"Dropped {indexes.Length} indexes from {database}");
-    }
-}

TestProject1/GlobalUsings.cs

@@ -1 +0,0 @@
-global using Xunit;

TestProject1/MesDatabaseHelper.cs

@@ -1,20 +0,0 @@
-using ConsoleApp2.Helpers.Database;
-
-namespace TestProject1;
-
-public static class MesDatabaseHelper
-{
-    public static async Task<int?> TrySearchCompanyId(string connStr, IEnumerable<string> dbNames, string scalarQuery)
-    {
-        foreach (var db in dbNames)
-        {
-            var result = await DatabaseHelper.QueryScalarAsync(connStr + $"Database={db};", scalarQuery);
-            if (result is null or 0) continue;
-                
-            var companyId = Convert.ToInt32(result);
-            return companyId;
-        }
-
-        return null;
-    }
-}

TestProject1/TableIndex.cs

@@ -1,24 +0,0 @@
-namespace TestProject1;
-
-public record TableIndex(string TableName, string IndexName, bool IsUnique, string ColumnName, TableIndex.TableIndexType IndexType)
-{
-    public enum TableIndexType
-    {
-        BTree,
-        Hash,
-        Primary
-    }
-
-    public void Deconstruct(out string tableName, out string indexName)
-    {
-        tableName = TableName;
-        indexName = IndexName;
-    }
-
-    public void Deconstruct(out string tableName, out string indexName, out string columnName)
-    {
-        tableName = TableName;
-        indexName = IndexName;
-        columnName = ColumnName;
-    }
-}

TestProject1/TenantDbHelper.cs

@@ -1,26 +0,0 @@
-namespace TestProject1;
-
-public static class TenantDbHelper
-{
-    /// <summary>
-    /// Key-Value: {DbName}-{TenantKeyLessThan}
-    /// </summary>
-    public static Dictionary<string, int> DbList { get; set; } = new Dictionary<string, int>
-    {
-        { "cferp_test_1", 1000 },
-        { "cferp_test_2", 2000 },
-        { "cferp_test_3", int.MaxValue },
-    };
-
-    public static string GetDbNameByTenantKeyValue(int tenantKeyValue)
-    {
-        // var dictionary = new SortedDictionary<int, string>();
-        // DbList.ForEach(pair => dictionary.Add(pair.Value, pair.Key));
-        // 注意配置顺序
-        var dbName = DbList.Cast<KeyValuePair<string, int>?>()
-            .FirstOrDefault(pair => pair?.Value != null && pair.Value.Value > tenantKeyValue)!.Value.Key;
-        return dbName ??
-               throw new ArgumentOutOfRangeException(nameof(tenantKeyValue),
-                   $"已配置的数据库中没有任何符合'{nameof(tenantKeyValue)}'值的对象");
-    }
-}

TestProject1/TestProject1.csproj

@@ -1,29 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-    <PropertyGroup>
-        <TargetFramework>net8.0</TargetFramework>
-        <ImplicitUsings>enable</ImplicitUsings>
-        <Nullable>enable</Nullable>
-
-        <IsPackable>false</IsPackable>
-        <IsTestProject>true</IsTestProject>
-    </PropertyGroup>
-
-    <ItemGroup>
-        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.0"/>
-        <PackageReference Include="xunit" Version="2.4.2"/>
-        <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
-            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-            <PrivateAssets>all</PrivateAssets>
-        </PackageReference>
-        <PackageReference Include="coverlet.collector" Version="6.0.0">
-            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-            <PrivateAssets>all</PrivateAssets>
-        </PackageReference>
-    </ItemGroup>
-
-    <ItemGroup>
-      <ProjectReference Include="..\ConsoleApp2\ConsoleApp2.csproj" />
-    </ItemGroup>
-
-</Project>