整理代码

output使用ThreadPool.QueueUserWorkItem

ConsoleApp2.sln

@@ -0,0 +1,16 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ConsoleApp2", "ConsoleApp2\ConsoleApp2.csproj", "{155E4B04-E88C-4BA4-AED2-B13E0A0432B5}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{155E4B04-E88C-4BA4-AED2-B13E0A0432B5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{155E4B04-E88C-4BA4-AED2-B13E0A0432B5}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{155E4B04-E88C-4BA4-AED2-B13E0A0432B5}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{155E4B04-E88C-4BA4-AED2-B13E0A0432B5}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+EndGlobal

ConsoleApp2/ConsoleApp2.csproj

@@ -1,13 +1,11 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-        <AssemblyName>MesETL</AssemblyName>
         <OutputType>Exe</OutputType>
-        <TargetFramework>net9.0</TargetFramework>
+        <TargetFramework>net8.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
         <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
-        <RootNamespace>MesETL.App</RootNamespace>
     </PropertyGroup>
 
     <ItemGroup>
@@ -21,18 +19,15 @@
     </ItemGroup>
 
     <ItemGroup>
-      <PackageReference Include="Microsoft.Extensions.Hosting" Version="9.0.0" />
-      <PackageReference Include="Microsoft.Extensions.ObjectPool" Version="9.0.0" />
-      <PackageReference Include="Serilog" Version="4.1.0" />
+      <PackageReference Include="Microsoft.Extensions.Caching.StackExchangeRedis" Version="8.0.0" />
+      <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
+      <PackageReference Include="MySqlConnector" Version="2.3.3" />
+      <PackageReference Include="Serilog" Version="3.1.2-dev-02097" />
       <PackageReference Include="Serilog.Extensions.Hosting" Version="8.0.0" />
-      <PackageReference Include="Serilog.Sinks.Console" Version="6.0.0" />
-      <PackageReference Include="Serilog.Sinks.File" Version="6.0.0" />
-      <PackageReference Include="StackExchange.Redis" Version="2.8.16" />
-      <PackageReference Include="ZstdSharp.Port" Version="0.8.2" />
-    </ItemGroup>
-
-    <ItemGroup>
-      <ProjectReference Include="..\MesETL.Shared\MesETL.Shared.csproj" />
+      <PackageReference Include="Serilog.Sinks.Console" Version="5.0.1" />
+      <PackageReference Include="Serilog.Sinks.File" Version="5.0.1-dev-00972" />
+      <PackageReference Include="ServiceStack.Text" Version="8.0.0" />
+      <PackageReference Include="ZstdSharp.Port" Version="0.7.4" />
     </ItemGroup>
 
 </Project>

ConsoleApp2/Const/ProcessStep.cs

@@ -0,0 +1,7 @@
+namespace ConsoleApp2.Const;
+
+public static class ProcessStep
+{
+    public const string Producer = "Producer";
+    public const string Consumer = "Consumer";
+}

ConsoleApp2/DataRecord.cs

@@ -0,0 +1,90 @@
+using System.ComponentModel.Design;
+
+namespace ConsoleApp2;
+
+
+public class DataRecord
+{
+    public static bool TryGetField(DataRecord record, string columnName, out string value)
+    {
+        value = string.Empty;
+        if (record.Headers is null)
+            throw new InvalidOperationException("Cannot get field when headers of a record have not been set.");
+        var idx = Array.IndexOf(record.Headers, columnName); //可能可以优化
+        if (idx == -1)
+            return false;
+        value = record.Fields[idx];
+        return true;
+    }
+
+    public static string GetField(DataRecord record, string columnName)
+    {
+        if (record.Headers is null)
+            throw new InvalidOperationException("Headers have not been set.");
+        var idx = Array.IndexOf(record.Headers, columnName);
+        if (idx is -1)
+            throw new IndexOutOfRangeException("Column name not found in this record.");
+        return record.Fields[idx];
+    }
+
+
+    public string[] Fields { get; }
+
+    public string[] Headers { get; }
+
+    public string TableName { get; }
+
+    public string? Database { get; set; }
+    public int CompanyID { get; set; }
+
+
+    public DataRecord(string[] fields, string tableName, string[] headers, int companyID=0)
+    {
+        if (fields.Length != headers.Length)
+            throw new ArgumentException(
+                $"The number of fields does not match the number of headers. Expected: {headers.Length} Got: {fields.Length} Fields: {string.Join(',', fields)}",
+                nameof(fields));
+
+        Fields = fields;
+        TableName = tableName;
+        Headers = headers;
+        CompanyID = companyID;
+    }
+
+    public string this[int index]
+    {
+        get => Fields[index];
+        set => Fields[index] = value;
+    }
+
+    public string this[string columnName] => GetField(this, columnName);
+
+    public int Count => Fields.Length;
+
+    public bool TryGetField(string columnName, out string value) => TryGetField(this, columnName, out value);
+
+    public bool SetField(string columnName,  string value) => SetField(this, columnName,value);
+
+    public string GetCacheKey(string columnName) => GetCacheKey(this, columnName);
+
+    public  bool SetField( DataRecord record,string columnName,string value)
+    {
+        if (record.Headers is null)
+            throw new InvalidOperationException("Headers have not been set.");
+        var idx = Array.IndexOf(record.Headers, columnName);
+        if (idx is -1)
+            throw new IndexOutOfRangeException("Column name not found in this record.");
+        record.Fields[idx] = value;
+        return true;
+    }
+    public string GetCacheKey(DataRecord record, string columnName)
+    {
+        if (TryGetField(record, columnName, out var  value))
+        {
+            return $"{TableName}_{value}";
+        }else
+            throw new IndexOutOfRangeException($"Column name:{columnName} not found in this record.");
+
+
+    }
+}

ConsoleApp2/Helpers/DumpDataHelper.cs

@@ -0,0 +1,95 @@
+using ConsoleApp2.Options;
+using System.Text;
+using System.Text.RegularExpressions;
+
+namespace ConsoleApp2.Helpers;
+
+public static partial class DumpDataHelper
+{
+    [GeneratedRegex(@"'.+\.dat'")]
+    private static partial Regex MatchDatFile();
+    [GeneratedRegex(@"\([^)]*\)")]
+    private static partial Regex MatchBrackets();
+    
+    
+    public static string[] GetCsvHeadersFromSqlFileAsync(string txt)
+    {
+        var match = MatchBrackets().Match(txt);
+        return ParseHeader(match.ValueSpan);
+    }
+
+    private static string[] ParseHeader(ReadOnlySpan<char> headerStr)
+    {
+        headerStr = headerStr[1..^1];
+        Span<Range> ranges = stackalloc Range[50];
+        var count = headerStr.Split(ranges, ',');
+        var arr = new string[count];
+
+        for (var i = 0; i < count; i++)
+        {
+            arr[i] = headerStr[ranges[i]].Trim("@`").ToString(); // 消除列名的反引号，如果是变量则消除@
+        }
+
+        return arr;
+    }
+    
+    
+    public static string GetTableName(ReadOnlySpan<char> filePath)
+    {
+        filePath = filePath[(filePath.LastIndexOf('\\') + 1)..];
+        var firstDotIdx = -1;
+        var secondDotIdx = -1;
+        var times = 0;
+        for (var i = 0; i < filePath.Length; i++)
+        {
+            if (filePath[i] == '.')
+            {
+                ++times;
+                if(times == 1)
+                    firstDotIdx = i;
+                if (times == 2)
+                {
+                    secondDotIdx = i;
+                    break;
+                }
+            }
+        }
+
+        return filePath[(firstDotIdx+1)..secondDotIdx].ToString();
+    }
+
+    public static  string[] GetCsvFileNamesFromSqlFileAsync(string txt,Regex regex)
+    {
+        var matches = regex.Matches(txt);
+        return matches.Select(match => match.ValueSpan[1..^1].ToString()).ToArray();
+    }
+    
+    public static bool CheckHexField(string? str)
+    {
+        if (string.IsNullOrWhiteSpace(str))
+            return false;
+        
+        if (str.StartsWith('\"'))
+            return false;
+
+        var isAllDigit = true; 
+
+        foreach (var c in str)
+        {
+            if (!char.IsAsciiHexDigit(c))
+                return false;
+            if (!char.IsNumber(c))
+                isAllDigit = false;
+        }
+
+        if (isAllDigit) //避免全数字
+            return false;
+
+        return true;
+    }
+
+    // public static string EliminateEscapeChars(ReadOnlySpan<char> str)
+    // {
+    //     char[] escapeChars = ['0','\''];
+    // }
+}

ConsoleApp2/Helpers/Extensions.Dictionary.cs

@@ -1,4 +1,4 @@
-namespace MesETL.Shared.Helper;
+namespace ConsoleApp2.Helpers;
 
 public static class DictionaryExtensions
 {

ConsoleApp2/Helpers/Extensions.String.cs

@@ -1,16 +1,10 @@
 using System.Globalization;
 using System.Text;
 
-namespace MesETL.Shared.Helper;
+namespace ConsoleApp2.Helpers;
 
 public static class StringExtensions
 {
-    /// <summary>
-    /// 截断字符串
-    /// </summary>
-    /// <param name="this"></param>
-    /// <param name="maxLength">截断长度</param>
-    /// <returns></returns>
     public static string Omit(this ReadOnlySpan<char> @this, int maxLength)
     {
         if (@this.Length > maxLength)
@@ -18,20 +12,8 @@ public static class StringExtensions
         return @this.ToString();
     }
 
-    /// <summary>
-    /// 截断字符串
-    /// </summary>
-    /// <param name="this"></param>
-    /// <param name="maxLength">截断长度</param>
-    /// <returns></returns>
     public static string Omit(this string @this, int maxLength) => Omit(@this.AsSpan(), maxLength);
 
-    /// <summary>
-    /// 将16进制字符串转换为字符串
-    /// </summary>
-    /// <param name="hexString"></param>
-    /// <param name="encoding"></param>
-    /// <returns></returns>
     public static string FromHex(ReadOnlySpan<char> hexString, Encoding? encoding = null)
     {
         encoding ??= Encoding.UTF8;
@@ -56,11 +38,6 @@ public static class StringExtensions
         return encoding.GetString(bytes);
     }
     
-    /// <summary>
-    /// 检查是否为JSON字符串
-    /// </summary>
-    /// <param name="hexStr"></param>
-    /// <returns></returns>
     public static bool CheckJsonHex(ReadOnlySpan<char> hexStr)
     {
         if (hexStr.Length < 2)

ConsoleApp2/Helpers/ValidateConsole.cs

@@ -0,0 +1,31 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace ConsoleApp2.Helpers
+{
+    public static class ValidateConsole
+    {
+      public static void ValidateInput<T>(Func<string,bool> converter,string message)
+        {
+            Console.Write(message);
+           string ? input = Console.ReadLine();
+            while (true)
+            {
+                if (!string.IsNullOrEmpty(input))
+                {
+                    var result = converter(input);
+                    if (result == false)
+                    {
+                        Console.WriteLine($"输入的内容不合法,请重新输入!");
+                        input = Console.ReadLine();
+                    }
+                    else break;
+                }
+                break;
+            }
+        }
+    }
+}

ConsoleApp2/HostedServices/Abstractions/IDataSource.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace ConsoleApp2.HostedServices.Abstractions
+{
+    public interface  IDataSource:IDisposable
+    {
+        public Task DoEnqueue(Action<DataRecord> action);
+    }
+}

ConsoleApp2/HostedServices/Abstractions/IInputService.cs

@@ -0,0 +1,9 @@
+using ConsoleApp2.Options;
+using ConsoleApp2.Services;
+
+namespace ConsoleApp2.HostedServices.Abstractions;
+
+public interface IInputService
+{
+    public Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, ProcessContext context, CancellationToken cancellationToken);
+}

ConsoleApp2/HostedServices/Abstractions/IOutputService.cs

@@ -0,0 +1,9 @@
+using ConsoleApp2.Options;
+using ConsoleApp2.Services;
+
+namespace ConsoleApp2.HostedServices.Abstractions;
+
+public interface IOutputService
+{
+    public void ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken);
+}

ConsoleApp2/HostedServices/Abstractions/ITransformService.cs

@@ -0,0 +1,9 @@
+using ConsoleApp2.Options;
+using ConsoleApp2.Services;
+
+namespace ConsoleApp2.HostedServices.Abstractions;
+
+public interface ITransformService
+{
+    public Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken);
+}

ConsoleApp2/HostedServices/InputService.cs

@@ -0,0 +1,66 @@
+using ConsoleApp2.Const;
+using ConsoleApp2.Helpers;
+using ConsoleApp2.HostedServices.Abstractions;
+using ConsoleApp2.Options;
+using ConsoleApp2.Services;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace ConsoleApp2.HostedServices;
+
+/// <summary>
+/// 从MyDumper导出的CSV文件中导入表头和数据
+/// </summary>
+public class InputService : IInputService
+{
+    private readonly ILogger _logger;
+    private readonly IOptions<DataInputOptions> _dataInputOptions;
+    private readonly ProcessContext _context;
+
+    public InputService(ILogger<InputService> logger,
+            IOptions<DataInputOptions> dataInputOptions,
+            ProcessContext context)
+    {
+        _logger = logger;
+        _dataInputOptions = dataInputOptions;
+        _context = context;
+    }
+
+    public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, ProcessContext context, CancellationToken cancellationToken)
+    {
+        var inputDir = _dataInputOptions.Value.InputDir;
+        _logger.LogInformation("***** Csv input service start, working dir: {InputDir}, thread id: {ThreadId} *****", inputDir, Environment.CurrentManagedThreadId);
+        var files = Directory.GetFiles(inputDir);
+        if (files.Length == 0)
+        {
+            _logger.LogInformation("No source files found in {InputDir}", inputDir);
+            return;
+        }
+        var count = 0;
+        foreach (var tableName in tasksOptions.TableInfoConfig.Keys)
+        {
+            _logger.LogInformation("Working table: {tableName}", tableName);
+            var source = _dataInputOptions.Value.CreateSource?.Invoke(tableName);
+            if (source != null)
+            {
+                await source.DoEnqueue((record) =>
+                {
+                    _context.AddInput();
+                    producerQueue.Enqueue(record);
+                    count++;
+
+                });
+            }
+            if (!_context.GetExceptions().IsEmpty)
+            {
+                _logger.LogInformation("***** Csv input service is canceled *****");
+                return;
+            }
+            _logger.LogInformation("table:'{tableName}' input completed", tableName);
+        }
+
+        context.CompleteInput();
+        _logger.LogInformation("***** Csv input service completed *****");
+    }
+}

ConsoleApp2/HostedServices/MainHostedService.cs

@@ -0,0 +1,125 @@
+using ConsoleApp2.HostedServices.Abstractions;
+using ConsoleApp2.Options;
+using ConsoleApp2.Services;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+namespace ConsoleApp2.HostedServices;
+
+public class MainHostedService : IHostedService
+{
+    private readonly ILogger _logger;
+    private readonly IInputService _input;
+    private readonly ITransformService _transform;
+    private readonly IOutputService _output;
+    private readonly ProcessContext _context;
+    private readonly Timer? _bigTableTimer=null;
+    private readonly Timer? _smallTableTimer=null;
+    public MainHostedService(ILogger<MainHostedService> logger, IInputService input, ITransformService transform, IOutputService output, ProcessContext context)
+    {
+        _logger = logger;
+        _input = input;
+        _transform = transform;
+        _output = output;
+        _context = context;
+    }
+
+    public  Task  StartAsync(CancellationToken cancellationToken)
+    {
+        var taskFun = (TasksOptions taskOp, DataRecordQueue producerQueue, DataRecordQueue consumerQueue, ProcessContext context,Timer? timer) =>
+        {
+            Task.Factory.StartNew(async () =>
+            {
+                try
+                {
+                    await _input.ExecuteAsync(taskOp, producerQueue, context, cancellationToken);
+                }
+                catch (Exception ex)
+                {
+                    _context.AddException(ex);
+                    _logger.LogError("Exception occurred on inputService:{Message},{StackTrace}", ex.Message, ex.StackTrace);
+                }
+
+            });
+            Task.Factory.StartNew(async () =>
+            {
+                try
+                {
+                    await _transform.ExecuteAsync(taskOp, producerQueue, consumerQueue, context, cancellationToken);
+                }
+                catch (Exception ex)
+                {
+                    _context.AddException(ex);
+                    _logger.LogError("Exception occurred on transformService:{Message},{StackTrace}", ex.Message, ex.StackTrace);
+                }
+
+            });
+            Task.Factory.StartNew(() =>
+            {
+                try
+                {
+                    timer = new Timer((object? state) =>
+                    {
+                         _output.ExecuteAsync(taskOp, consumerQueue, context, cancellationToken);
+                    },null, TimeSpan.Zero,TimeSpan.FromSeconds(0.5));
+
+                }
+                catch (Exception ex)
+                {
+                    _context.AddException(ex);
+                    _logger.LogError("Exception occurred on outputService:{Message},{StackTrace}", ex.Message, ex.StackTrace);
+                }
+
+            });
+        };
+
+        var bigTablesDic = new Dictionary<string, TableInfo>
+        {
+            {"order",new TableInfo{SimulaRowCount=5019216 }},
+            {"order_block_plan",new TableInfo{SimulaRowCount=2725553 }},//CreateTime < 202301的删除 
+            {"order_block_plan_result",new TableInfo{SimulaRowCount=1174096 }},
+            {"order_box_block",new TableInfo{SimulaRowCount=29755672 }},
+            {"order_item",new TableInfo{SimulaRowCount=1345520079 }},
+            {"simple_plan_order",new TableInfo{SimulaRowCount=351470 }},//CreateTime < 202301的删除
+        };
+        var bigTableContext = new ProcessContext();
+        var bigTableOptions = new TasksOptions { TableInfoConfig = bigTablesDic, OutPutOptions = new OutPutOptions { FlushCount = 20000, OutPutTaskCount = 2 } };
+        taskFun(bigTableOptions, new DataRecordQueue(), new DataRecordQueue(), bigTableContext,_bigTableTimer);
+        var smallTablesDic = new Dictionary<string, TableInfo>
+        {
+            {"machine",new TableInfo{SimulaRowCount=14655 }},
+            {"order_data_block",new TableInfo{SimulaRowCount=731800334 }},
+            {"order_data_goods",new TableInfo{SimulaRowCount=25803671 }},
+            {"order_data_parts",new TableInfo{SimulaRowCount=468517543 }},
+            {"order_module",new TableInfo{SimulaRowCount=103325385 }},
+            {"order_module_extra",new TableInfo{SimulaRowCount=54361321 }},
+            {"order_module_item",new TableInfo{SimulaRowCount=69173339 }},
+            {"order_package",new TableInfo{SimulaRowCount=16196195 }},
+            {"order_process",new TableInfo{SimulaRowCount=3892685 }},//orderNo < 202301的  
+            {"order_process_step",new TableInfo{SimulaRowCount=8050349 }},//orderNo < 202301的删除   
+            {"order_process_step_item",new TableInfo{SimulaRowCount=14538058 }},//orderNo < 202301的删除 
+            {"order_scrap_board",new TableInfo{SimulaRowCount=123998 }},
+            {"process_group",new TableInfo{SimulaRowCount=1253 }},
+            {"process_info",new TableInfo{SimulaRowCount=7839 }},
+            {"process_item_exp",new TableInfo{SimulaRowCount=28 }},
+            {"process_schdule_capacity",new TableInfo{SimulaRowCount=39736 }},
+            {"process_step_efficiency",new TableInfo{SimulaRowCount=8 }},
+            {"report_template",new TableInfo{SimulaRowCount=7337 }},
+            {"simple_package",new TableInfo{SimulaRowCount=130436 }},//orderNo < 202301的删除
+            {"sys_config",new TableInfo{SimulaRowCount=2296 }},
+            {"work_calendar",new TableInfo{SimulaRowCount=11 }},
+            {"work_shift",new TableInfo{SimulaRowCount=59 }},
+            {"work_time",new TableInfo{SimulaRowCount=62 }},
+        };
+        var smallTableContext = new ProcessContext();
+        taskFun(new TasksOptions { TableInfoConfig = smallTablesDic, OutPutOptions = new OutPutOptions { FlushCount = 20000, OutPutTaskCount = 4 } },
+            new DataRecordQueue(), new DataRecordQueue(), smallTableContext,_smallTableTimer);
+        return Task.CompletedTask;
+    }
+
+    public Task StopAsync(CancellationToken cancellationToken)
+    {
+        throw new NotImplementedException();
+    }
+
+
+}

ConsoleApp2/HostedServices/OutputService.cs

@@ -0,0 +1,94 @@
+
+using ConsoleApp2.HostedServices.Abstractions;
+using ConsoleApp2.Options;
+using ConsoleApp2.Services;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+namespace ConsoleApp2.HostedServices;
+
+/// <summary>
+/// 数据导出服务，将数据导出至MySql服务
+/// </summary>
+public class OutputService : IOutputService
+{
+    private readonly ILogger _logger;
+    private readonly IOptions<DatabaseOutputOptions> _outputOptions;
+    private readonly IOptions<DataTransformOptions> _transformOptions;
+    private readonly ProcessContext _context;
+    private readonly TaskManager _taskManager;
+    private readonly ErrorRecorder _errorRecorder;
+
+    public OutputService(ILogger<OutputService> logger,
+        IOptions<DatabaseOutputOptions> outputOptions,
+        ProcessContext context,
+        TaskManager taskManager, 
+        IOptions<DataTransformOptions> transformOptions, 
+        ErrorRecorder errorRecorder)
+    {
+        _logger = logger;
+        _outputOptions = outputOptions;
+        _context = context;
+        _taskManager = taskManager;
+        _transformOptions = transformOptions;
+        _errorRecorder = errorRecorder;
+    }
+    private int _runingTaskCount;
+    public int RuningTaskCount
+    {
+        get => _runingTaskCount;
+    }
+    public void DoTask() => Interlocked.Increment(ref _runingTaskCount);
+    public void FinishTask() => Interlocked.Decrement(ref _runingTaskCount);
+    public void ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken)
+    {
+        if (context.IsTransformCompleted == false && consumerQueue.Count < tasksOptions.OutPutOptions.FlushCount) return;
+        if (RuningTaskCount >= tasksOptions.OutPutOptions.OutPutTaskCount ) return;
+        var records = new List<DataRecord>();
+
+        for (int i = 0; i < tasksOptions.OutPutOptions.FlushCount; i++)
+        {
+            if (consumerQueue.TryDequeue(out var record)) records.Add(record);
+            else break;
+        }
+        if (records.Count > 0)
+        {
+            ThreadPool.QueueUserWorkItem(async (queueState) =>
+            {
+                DoTask();
+                await FlushAsync(records);
+                FinishTask();
+            });
+        }
+    }
+
+    private async Task FlushAsync(IEnumerable<DataRecord> records)
+    {
+        var count = 0;
+        await using var output = new MySqlDestination(
+            _outputOptions.Value.ConnectionString ?? throw new InvalidOperationException("Connection string is required"),
+            _logger, _context, _transformOptions, _errorRecorder);
+        //if (records == null || records.Count() == 0) return;
+        //var dbName = $"cferp_test_1";
+        //if (records != null && records.Count() > 0)
+        //{
+        //    dbName = $"cferp_test_{records.FirstOrDefault()?.CompanyID}";
+        //}
+
+        //await  using var output = new MySqlDestination(new MySqlConnectionStringBuilder
+        //{
+        //    Server = "127.0.0.1",
+        //    Port = 34309,
+        //    Database = dbName,
+        //    UserID = "root",
+        //    Password = "123456",
+        //    MaximumPoolSize = 50,
+        //}.ConnectionString, _logger,true);
+        foreach (var record in records)
+        {
+            await output.WriteRecordAsync(record);
+            count++;
+        }
+        await output.FlushAsync(_outputOptions.Value.MaxAllowedPacket);
+        _context.AddOutput(count);
+    }
+}

ConsoleApp2/HostedServices/TaskMonitorService.cs

@@ -0,0 +1,123 @@
+using System.Diagnostics;
+using ConsoleApp2.Const;
+using ConsoleApp2.Services;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+
+namespace ConsoleApp2.HostedServices;
+
+/// <summary>
+/// 任务监控
+/// </summary>
+public class TaskMonitorService : BackgroundService
+{
+    private readonly IHostApplicationLifetime _lifetime;
+    private readonly ILogger<TaskMonitorService> _logger;
+    private readonly ProcessContext _context;
+    private readonly DataRecordQueue _producerQueue;
+    private readonly DataRecordQueue _consumerQueue;
+
+    public TaskMonitorService(IHostApplicationLifetime lifetime,
+        ILogger<TaskMonitorService> logger,
+        ProcessContext context,
+        [FromKeyedServices(ProcessStep.Producer)]
+        DataRecordQueue producerQueue,
+        [FromKeyedServices(ProcessStep.Consumer)]
+        DataRecordQueue consumerQueue)
+    {
+        _lifetime = lifetime;
+        _logger = logger;
+        _context = context;
+        _producerQueue = producerQueue;
+        _consumerQueue = consumerQueue;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        await Task.Factory.StartNew(Monitor, stoppingToken);
+    }
+
+    private async Task Monitor()
+    {
+        var sw = Stopwatch.StartNew();
+        var lastTime = sw.ElapsedMilliseconds;
+        var lastInputCount = _context.InputCount;
+        var lastTransformCount = _context.TransformCount;
+        var lastOutputCount = _context.OutputCount;
+
+        bool endCheck = false;
+        while (true)
+        {
+            if (_context.GetExceptions().Count>0) return;
+            EndCheck:
+            // var running = 0;
+            // var error = 0;
+            // var completed = 0;
+            // var canceled = 0;
+            // foreach (var task in _taskManager.Tasks)
+            // {
+            //     switch (task.Status)
+            //     {
+            //         case TaskStatus.Canceled:
+            //             canceled++;
+            //             break;
+            //         case TaskStatus.Faulted:
+            //             error++;
+            //             break;
+            //         case TaskStatus.RanToCompletion:
+            //             completed++;
+            //             break;
+            //         default:
+            //             running++;
+            //             break;
+            //     }
+            // }
+
+            var time = sw.ElapsedMilliseconds;
+            var inputCount = _context.InputCount;
+            var transformCount = _context.TransformCount;
+            var outputCount = _context.OutputCount;
+            
+            var elapseTime = (time - lastTime) / 1000f;
+            var inputSpeed = (inputCount - lastInputCount) / elapseTime;
+            var transformSpeed = (transformCount - lastTransformCount) / elapseTime;
+            var outputSpeed = (outputCount - lastOutputCount) / elapseTime;
+
+            // _logger.LogInformation(
+            //     "Task monitor: running: {Running}, error: {Error}, completed: {Completed}, canceled: {Canceled}, outputSpeed: {Speed} records/s",
+            //     running, error, completed, canceled, outputSpeed);
+            _logger.LogInformation(
+                "Process monitor: input: {inputStatus}, transform: {transformStatus}, output: {outputStatus}\nInput: {InputCount}, Transform: {TransformCount}, Output: {OutputCount}",
+                _context.IsInputCompleted ? "completed" : $"running {inputSpeed:F2} records/s",
+                _context.IsTransformCompleted ? "completed" : $"running {transformSpeed:F2} records/s",
+                _context.IsOutputCompleted ? "completed" : $"running {outputSpeed:F2} records/s",
+                inputCount,
+                transformCount,
+                outputCount);
+            _logger.LogInformation("Queue monitor: producer queue: {ProducerQueue}, consumer queue: {ConsumerQueue}",
+                _producerQueue.Count, _consumerQueue.Count);
+
+            await Task.Delay(5000);
+
+            lastTime = time;
+            lastInputCount = inputCount;
+            lastTransformCount = transformCount;
+            lastOutputCount = outputCount;
+
+            if (_context is { IsInputCompleted: true, IsTransformCompleted: true, IsOutputCompleted: true })
+            {
+                if (!endCheck)
+                {
+                    endCheck = true;
+                    goto EndCheck;
+                }
+                break;
+            }
+        }
+        
+        _logger.LogInformation("***** All tasks completed *****");
+        _logger.LogInformation("***** ElapseTime: {Time}", (sw.ElapsedMilliseconds / 1000f).ToString("F3"));
+        // _lifetime.StopApplication();
+    }
+}

ConsoleApp2/HostedServices/TransformService.cs

@@ -0,0 +1,88 @@
+using ConsoleApp2.Const;
+using ConsoleApp2.HostedServices.Abstractions;
+using ConsoleApp2.Options;
+using ConsoleApp2.Services;
+using Microsoft.Extensions.Caching.Distributed;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using StackExchange.Redis;
+
+namespace ConsoleApp2.HostedServices;
+
+/// <summary>
+/// 数据处理服务，对导入后的数据进行处理
+/// </summary>
+public class TransformService : ITransformService
+{
+    private readonly ILogger _logger;
+    private readonly IOptions<DataTransformOptions> _options;
+    private readonly ProcessContext _context;
+    private readonly IDistributedCache _cache;
+
+
+    public TransformService(ILogger<TransformService> logger,
+        IOptions<DataTransformOptions> options,
+        ProcessContext context,
+        IDistributedCache cache)
+    {
+        _logger = logger;
+        _options = options;
+        _context = context;
+        _cache = cache;
+    }
+
+    public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken)
+    {
+        _logger.LogInformation("***** Data transform service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId);
+        while ((!context.IsInputCompleted || producerQueue.Count > 0))
+        {
+            if (_context.GetExceptions().Count > 0)
+            {
+                _logger.LogInformation("***** Csv transform service is canceled *****");
+                return;
+            }
+            if (!producerQueue.TryDequeue(out var record)) continue;
+
+            //过滤不要的record
+
+            if (_options.Value.RecordFilter != null)
+            {
+                var result = await _options.Value.RecordFilter.Invoke(record, _cache);
+                if (result == false) continue;
+            }
+            record.Database = _options.Value.DatabaseFilter?.Invoke(record);
+            //修改record
+            _options.Value.RecordModify?.Invoke(record);
+            //缓存record
+            if (_options.Value.RecordCache != null)
+            {
+                await _options.Value.RecordCache.Invoke(record, _cache);
+            }
+            //替换record
+            if (_options.Value.RecordReplace != null)
+            {
+                var result = await _options.Value.RecordReplace.Invoke(record, _cache);
+                if (result != null)
+                {
+                    record = result;
+                }
+            }
+            consumerQueue.Enqueue(record);
+            _context.AddTransform();
+            //数据增加
+            var addRecords = _options.Value.RecordAdd?.Invoke(record);
+            if (addRecords != null && addRecords.Count > 0)
+            {
+                foreach (var rc in addRecords)
+                {
+                    consumerQueue.Enqueue(rc);
+                    _context.AddTransform();
+                }
+            }
+        }
+        context.CompleteTransform();
+
+        _logger.LogInformation("***** Data transformation service completed *****");
+    }
+}

ConsoleApp2/HostedServices/VoidOutputService.cs

@@ -0,0 +1,38 @@
+using ConsoleApp2.Const;
+using ConsoleApp2.HostedServices.Abstractions;
+using ConsoleApp2.Options;
+using ConsoleApp2.Services;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+
+namespace ConsoleApp2.HostedServices;
+
+// 空输出服务，测试用
+public class VoidOutputService : IOutputService
+{
+    private readonly ILogger _logger;
+    private readonly DataRecordQueue _consumerQueue;
+    private readonly ProcessContext _context;
+
+    public VoidOutputService([FromKeyedServices(ProcessStep.Consumer)] DataRecordQueue consumerQueue,
+        ProcessContext context, ILogger<VoidOutputService> logger)
+    {
+        _consumerQueue = consumerQueue;
+        _context = context;
+        _logger = logger;
+    }
+
+    public void ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken)
+    {
+        _logger.LogInformation("***** Void output service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId);
+        while (!_context.IsTransformCompleted || _consumerQueue.Count > 0)
+        {
+            if (_consumerQueue.TryDequeue(out var record))
+                _context.AddOutput();
+        }
+
+        _context.CompleteOutput();
+        _logger.LogInformation("***** Void output service completed *****");
+    }
+}

ConsoleApp2/Options/CommandOptions.cs

@@ -0,0 +1,18 @@
+using System;
+using System.ComponentModel;
+using System.Configuration;
+namespace ConsoleApp2.Options
+{
+    public class CommandOptions
+    {
+
+        public string InputDir { get; set; } = "./MyDumper";
+        public bool IsMock { get; set; } = false;
+        public string NoFilterTables { get; set; }="";//不需要过滤的表列表
+        public bool Isutf8mb4 { get; set; } = true;
+
+        public short OldestShardKey { get; set; } = 23010;
+        public string OldestTime { get; set; } = "202301";
+
+    }
+}

ConsoleApp2/Options/CsvOptions.cs

@@ -0,0 +1,19 @@
+namespace ConsoleApp2.Options;
+
+public class CsvOptions
+{
+    /// <summary>
+    /// MyDumper导出的CSV文件目录
+    /// </summary>
+    //public string InputDir { get; set; } = "./";
+
+    /// <summary>
+    /// 字符串的包围符号，默认为双引号"
+    /// </summary>
+    public char QuoteChar { get; set; } = '"';
+
+    /// <summary>
+    /// 每个字段的分割符，默认逗号,
+    /// </summary>
+    public string Delimiter { get; set; } = ",";
+}

ConsoleApp2/Options/DataInputOptions.cs

@@ -0,0 +1,18 @@
+using ConsoleApp2.Services;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection.PortableExecutable;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace ConsoleApp2.Options
+{
+    public enum InputFileType { CSV, JWT, JSV }
+    public class DataInputOptions
+    {
+        public string InputDir { get; set; } = "./";
+
+        public Func<string, CsvSource>? CreateSource {  get; set; }
+    }
+}

ConsoleApp2/Options/DataTransformOptions.cs

@@ -0,0 +1,36 @@
+using Microsoft.Extensions.Caching.Distributed;
+using StackExchange.Redis;
+
+namespace ConsoleApp2.Options;
+
+public enum ColumnType
+{
+    Blob,
+    Text,
+    Json,
+    UnDefine,
+}
+
+public class DataTransformOptions
+{
+
+    public Func<DataRecord, string>? DatabaseFilter { get; set; }
+
+    public Func<string, string>? TransformBinary { get; set; }//Binary转字符串方法
+
+    public Func<DataRecord, IDistributedCache, Task<bool>>? RecordFilter { get; set; }//数据过滤方法
+    public Action<DataRecord>? RecordModify { get; set; }//数据修改
+    public Func<DataRecord, IDistributedCache, Task<DataRecord?>>? RecordReplace { get; set; }//数据替换
+    public Func<DataRecord, IList<DataRecord>?>? RecordAdd { get; set; }//数据替换
+    public Func<DataRecord, IDistributedCache, Task>? RecordCache { get; set; }//数据缓存
+
+    /// <summary>
+    /// 配置导入数据的特殊列
+    /// </summary>
+    public Dictionary<string, ColumnType> ColumnTypeConfig { get; set; } = new(); // "table.column" -> type
+
+    public ColumnType GetColumnType(string table, string column)
+    {
+        return ColumnTypeConfig.GetValueOrDefault($"{table}.{column}", ColumnType.UnDefine);
+    }
+}

ConsoleApp2/Options/DatabaseOutputOptions.cs

@@ -0,0 +1,10 @@
+namespace ConsoleApp2.Options;
+
+public class DatabaseOutputOptions
+{
+    /// <summary>
+    /// 数据库连接字符串
+    /// </summary>
+    public string? ConnectionString { get; set; }
+    public int MaxAllowedPacket { get; set; } = 64*1024*1024;
+}

ConsoleApp2/Options/TasksOptions.cs

@@ -0,0 +1,24 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace ConsoleApp2.Options
+{
+    public class TableInfo
+    {
+        public long SimulaRowCount { get; set; }//模拟的记录条数
+    }
+    public class TasksOptions
+    {
+        public Dictionary<string, TableInfo> TableInfoConfig { get; set; } = new();
+        public int TransformTaskCount { get; set; } = 1;
+        public OutPutOptions OutPutOptions { get; set; }=new();
+    }
+    public class OutPutOptions
+    {
+        public int FlushCount { get; set; } = 10000;
+        public int OutPutTaskCount { get; set; } = 1;
+    }
+}

ConsoleApp2/Program.cs

@@ -0,0 +1,460 @@
+using ConsoleApp2;
+using ConsoleApp2.Const;
+using ConsoleApp2.HostedServices;
+using ConsoleApp2.HostedServices.Abstractions;
+using ConsoleApp2.Options;
+using ConsoleApp2.Services;
+using Microsoft.Extensions.Caching.StackExchangeRedis;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using MySqlConnector;
+using Serilog;
+using Microsoft.Extensions.Caching.Distributed;
+using Serilog.Events;
+
+await RunProgram();
+return;
+
+async Task RunProgram()
+{
+
+
+    //var inputDir= "D:\\MyDumper";
+    //ValidateConsole.ValidateInput<string>((_inputDir) =>
+    //{
+    //    if (Directory.Exists(_inputDir))
+    //     {
+    //        inputDir = _inputDir;
+    //        return true;
+    //    }
+    //    else return false;
+    //}, "请输入读取csv文件的目录(默认为当前目录下MyDumper文件夹):");
+
+    //var maxTask = 16;
+    //ValidateConsole.ValidateInput<string>((_inputDir) =>
+    //{
+
+    //    _ = int.TryParse(_inputDir.ToString(), out var _taskCount);
+    //    if (_taskCount > 0) {
+    //        maxTask = _taskCount;
+    //        return true;
+    //    }
+    //    else return false;
+    //}, "请输入执行输出的线程数量(默认为16):");
+
+    //var flushCount = 2_0000;
+    //ValidateConsole.ValidateInput<string>((_inputDir) =>
+    //{
+    //    _ = int.TryParse(_inputDir.ToString(), out var _flashCount);
+    //    if (_flashCount > 0)
+    //    {
+    //        flushCount = _flashCount;
+    //        return true;
+    //    } else return false; 
+
+    //}, "请输入单次插入的行数(默认为20000):");
+
+    ThreadPool.SetMaxThreads(8, 4);
+    var host = Host.CreateApplicationBuilder(args);
+    var commandOptions = host.Configuration.GetSection("CmdOptions").Get<CommandOptions>() ?? new CommandOptions();
+    Console.WriteLine($"InputDir:{commandOptions?.InputDir}");
+    if (commandOptions == null) throw new ArgumentNullException("commandOptions is null");
+    var oldestTime = DateTime.ParseExact(commandOptions.OldestTime, "yyyyMM", System.Globalization.DateTimeFormatInfo.InvariantInfo);
+    host.Services.Configure<CsvOptions>(option =>
+    {
+        option.Delimiter = ",";
+        option.QuoteChar = '"';
+    });
+    host.Services.Configure<DataInputOptions>(options =>
+    {
+        options.InputDir = commandOptions.InputDir;
+        var _csvOptions = new CsvOptions { Delimiter = ",", QuoteChar = '"' };
+        options.CreateSource = (string tableName) =>
+        {
+            var source = new ZstSource(commandOptions.InputDir, tableName, _csvOptions.Delimiter, _csvOptions.QuoteChar);
+            return source;
+        };
+    });
+
+    host.Services.Configure<DataTransformOptions>(options =>
+    {
+        if (commandOptions.IsMock) return;
+        options.DatabaseFilter = record => "cferp_test";
+
+        options.TransformBinary = field => commandOptions != null && commandOptions.Isutf8mb4 ? $"_utf8mb4 0x{field}" : $"0x{field}";
+        var noFilterTables = commandOptions.NoFilterTables.Split(",");
+        //数据过滤
+        options.RecordFilter = async (record, cache) =>
+        {
+           
+            if (noFilterTables.Contains(record.TableName)) return true;
+
+            if (record.TryGetField("ShardKey", out var skStr))
+            {
+                short.TryParse(skStr, out var sk);
+                if (sk < commandOptions.OldestShardKey) return false;
+            }
+
+            if (record.TryGetField("CreateTime", out var createTime))
+            {
+                _ = DateTime.TryParse(createTime.Replace("\"", ""), out var time);
+                if (time < oldestTime) return false;
+            }
+            if (record.TryGetField("OrderNo", out var orderNo))
+            {
+                try
+                {
+                    var yearMonth = orderNo.Substring(0, 6);
+
+                    var dt = DateTime.ParseExact(yearMonth, "yyyyMM", System.Globalization.DateTimeFormatInfo.InvariantInfo);
+                    if (dt < oldestTime) return false;
+
+                }
+                catch (Exception)
+                {
+                    return false;//订单号转换失败，跳过
+
+                }
+                
+            }
+
+            if (record.TableName == "order_package")
+            {
+                if (record.TryGetField("PakageNo", out var pkNo))
+                {
+                    if (pkNo.Length <= 2) return false;
+                }
+            }
+            if (record.TableName == "order_block_plan")
+            {
+                if (record.TryGetField("OrderNos", out var nos))
+                {
+                    if (nos.Length <= 2) return false;
+                }
+              
+            }
+            if (record.TableName == "order_process_step" || record.TableName == "order_process_step_item")
+            {
+                //如果缓存中不存在OrderProcessID,则丢弃
+
+                if(record.TryGetField("OrderProcessID",out string orderProcessID))
+                {
+                    var value = await cache.GetStringAsync($"order_process_{orderProcessID}");
+                    if (string.IsNullOrEmpty(value)) return false;
+
+                }
+                   
+                
+            }
+            if (record.TableName == "order_block_plan_result" )
+            {
+                //如果缓存中不存在ID,则丢弃(ID 对应order_block_plan中的ID)
+                if (record.TryGetField("ID", out string id))
+                {
+                    var value = await cache.GetStringAsync($"order_block_plan_{id}");
+                    if (string.IsNullOrEmpty(value)) return false;
+                }
+            }
+            return true;
+
+        };
+        //数据修改
+        options.RecordModify = (record) =>
+        {
+            if (record.TableName == "order_block_plan")
+            {
+                if (record.TryGetField("OrderNos", out var nos))
+                {
+                    if (nos.Length <= 2) record.SetField("OrderNos", "");
+                }
+
+            }
+            if (record.TableName == "order_process")//修改order_process.NextStepID的默认值为0
+            {
+                 
+                if (record.TryGetField("NextStepID", out var idStr))
+                {
+
+                    if (idStr == "\\N")
+                    {
+                        record.SetField("NextStepID", "0");
+                    }
+                }
+            }
+
+        };
+        //数据缓存
+        options.RecordCache = async (record, cache) =>
+        {
+            if (record.TableName == "order")
+            {
+                if (record.TryGetField("OrderNo", out var orderNo))
+                {
+                    if (record.TryGetField("CompanyID", out var companyid))
+                    {
+                        await cache.SetStringAsync(record.GetCacheKey("OrderNo"), companyid);
+                    }
+                    
+                }
+            }
+            if (record.TableName == "order_process")
+            {
+                if (record.TryGetField("OrderNo", out var orderNo))
+                {
+                    var yearMonth = orderNo.Substring(2, 4);
+                    var sk = yearMonth + "0";
+                   
+                    if( record.TryGetField("ID", out var id))
+                    {
+                        await cache.SetStringAsync(record.GetCacheKey("ID"), sk);
+                    }
+          
+                }
+            }
+            if (record.TableName == "order_block_plan")
+            {
+                if (record.TryGetField("CompanyID", out var companyid))
+                {
+                    record.TryGetField("ID", out var id);
+                    await cache.SetStringAsync(record.GetCacheKey("ID"), companyid);
+                }
+            }
+        };
+        //数据替换
+        options.RecordReplace = async (record, cache) =>
+        {
+            //删除数据源里simple_plan_order.ProcessState 字段和值
+
+            if (record.TableName == "simple_plan_order")//修改order_process.NextStepID的默认值为0
+            {
+                var nextStepIdIndex = Array.IndexOf(record.Headers, "ProcessState");
+                if (nextStepIdIndex > -1)
+                {
+                    var headers = record.Headers.Where(t => t != "ProcessState").ToArray();
+                    var fs = record.Fields.ToList();
+                    fs.RemoveAt(nextStepIdIndex);
+                    var fields = fs.ToArray();
+                    return new DataRecord(fields, record.TableName, headers, record.CompanyID);
+                }
+            }
+            if (record.TableName == "order")//修改order_process.NextStepID的默认值为0
+            {
+                var nextStepIdIndex = Array.IndexOf(record.Headers, "IsBatch");
+                if (nextStepIdIndex > -1)
+                {
+                    var headers = record.Headers.Where(t => t != "IsBatch").ToArray();
+                    var fs = record.Fields.ToList();
+                    fs.RemoveAt(nextStepIdIndex);
+                    var fields = fs.ToArray();
+                    return new DataRecord(fields, record.TableName, headers, record.CompanyID);
+                }
+            }
+            if (record.TableName == "order_block_plan_result")//修改order_process.NextStepID的默认值为0
+            {
+                if (record.TryGetField("ID", out var id))
+                {
+                    var headers = new List<string>(record.Headers);
+                    var fields =new List<string>(record.Fields);
+                    headers.Add("CompanyID");
+                    var companyidResult =await cache.GetStringAsync($"order_block_plan_{id}");
+
+                    _ = int.TryParse(companyidResult, out var companyid);
+                    fields.Add(companyid.ToString());
+                    return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), companyid);
+                }
+            }
+            if(record.TableName == "order_box_block")
+            {
+                if (!record.TryGetField("CompanyID", out var companyid))
+                {
+                    if (record.TryGetField("OrderNo", out var orderNo))
+                    {
+                        var headers = new List<string>(record.Headers);
+                        var fields = new List<string>(record.Fields);
+                        headers.Add("CompanyID");
+                        var companyidResult = await cache.GetStringAsync($"order_{orderNo}");
+                        _ = int.TryParse(companyidResult, out var cpid);
+                        fields.Add(cpid.ToString());
+                        return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), cpid);
+                    }
+                }
+            }
+            if (record.TableName == "order_module")
+            {
+                if (record.TryGetField("ViewFileName",out var value))
+                {
+                    var index=Array.IndexOf(record.Headers, "ViewFileName");
+                    var headers = new List<string>(record.Headers);
+                    headers.RemoveAt(index); 
+                    var fields = new List<string>(record.Fields);
+                    fields.RemoveAt(index);
+                    return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), record.CompanyID);
+                }
+
+            }
+            if (record.TableName == "order_process")
+            {
+                if (!record.TryGetField("ShardKey", out var skStr))
+                {
+                    if(record.TryGetField("OrderNo", out var orderNo))
+                    {
+                        var yearMonth = orderNo.Substring(2, 4);
+                        var sk = yearMonth + "0";
+                        var headers = new List<string>(record.Headers);
+                        var fields = new List<string>(record.Fields);
+                        headers.Add("ShardKey");
+                        fields.Add(sk);
+                        return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), record.CompanyID);
+                    }
+                }
+            }
+            if(record.TableName == "order_process_step"|| record.TableName == "order_process_step_item")
+            {
+                if (!record.TryGetField("ShardKey",out var sk))
+                {
+                    if (record.TryGetField("OrderProcessID",out var processID))
+                    {
+                        var shardKey =await cache.GetStringAsync($"order_process_{processID}");
+                        var headers = new List<string>(record.Headers);
+                        var fields = new List<string>(record.Fields);
+                        headers.Add("ShardKey");
+                        fields.Add(shardKey??"0");
+                        return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), record.CompanyID);
+                    }
+                }
+            }
+            if(record.TableName == "order_moudle")
+            {
+                if (!record.TryGetField("ShardKey", out var skStr))
+                {
+                    if (record.TryGetField("OrderNo", out var orderNo))
+                    {
+                        var yearMonth = orderNo.Substring(2, 4);
+                        var sk = yearMonth + "0";
+                        var headers = new List<string>(record.Headers);
+                        var fields = new List<string>(record.Fields);
+                        headers.Add("ShardKey");
+                        fields.Add(sk);
+                        return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), record.CompanyID);
+                    }
+                }
+            }
+            return null;
+        };
+        //数据生成
+        options.RecordAdd = (record) =>
+        {
+            var resultList = new List<DataRecord>();
+            if (record.TableName == "order_item")
+            {
+                record.TryGetField("ID", out var itemID);
+                record.TryGetField("ShardKey", out var shardKey);
+                record.TryGetField("PlanID", out var planID);
+                record.TryGetField("PackageID", out var packageID);
+                record.TryGetField("CompanyID", out var companyID);
+                _=int.TryParse(planID, out var pid);
+                if (pid > 0)
+                {
+                    resultList.Add(new DataRecord(new[] { itemID, shardKey, planID, companyID },
+                        "order_block_plan_item",
+                        new[] { "ItemID", "ShardKey", "PlanID", "CompanyID" }));
+                }
+                _ = int.TryParse(packageID, out var pkid);
+                if(pkid > 0)
+                {
+                    resultList.Add(new DataRecord(new[] { itemID, shardKey, packageID, companyID },
+                        "order_package_item",
+                        new[] { "ItemID", "ShardKey", "PackageID", "CompanyID" }
+                        ));
+                }
+            }
+            return resultList;
+
+        };
+        options.ColumnTypeConfig = new()
+        {
+            { "simple_plan_order.PlaceData", ColumnType.Blob },
+            { "order_block_plan_result.PlaceData", ColumnType.Blob },
+            { "order_box_block.Data", ColumnType.Blob },
+            { "order_data_goods.ExtraProp", ColumnType.Text },
+            { "order_module_extra.JsonStr", ColumnType.Text },
+            { "process_info.Users", ColumnType.Text },
+            { "order_process_schdule.CustomOrderNo", ColumnType.Text },
+            { "order_process_schdule.OrderProcessStepName", ColumnType.Text },
+            { "order_process_schdule.AreaName", ColumnType.Text },
+            { "order_process_schdule.ConsigneeAddress", ColumnType.Text },
+            { "order_process_schdule.ConsigneePhone", ColumnType.Text },
+            { "report_source.Sql", ColumnType.Text },
+            { "report_source.KeyValue", ColumnType.Text },
+            { "report_source.Setting", ColumnType.Text },
+            { "order_data_block.RemarkJson", ColumnType.Text },
+            { "order_patch_detail.BlockDetail", ColumnType.Text },
+            { "order_scrap_board.OutLineJson", ColumnType.Text },
+            { "simple_package.Items", ColumnType.Text },
+            { "order_batch_pack_config.Setting", ColumnType.Text },
+            { "machine.Settings", ColumnType.Text },
+            { "sys_config.Value", ColumnType.Text },
+            { "sys_config.JsonStr", ColumnType.Text },
+            { "process_item_exp.ItemJson", ColumnType.Text },
+            { "report_template.Template", ColumnType.Text },
+            { "report_template.SourceConfig", ColumnType.Text },
+            { "order_block_plan.OrderNos", ColumnType.Json },
+            { "order_block_plan.BlockInfo", ColumnType.Text },
+        };
+    });
+    host.Services.Configure<DatabaseOutputOptions>(options =>
+    {
+        //options.ConnectionString = new MySqlConnectionStringBuilder
+        //{
+        //    Server = "127.0.0.1",
+        //    Port = 33309,
+        //    Database = "cferp_test",
+        //    UserID = "root",
+        //    Password = "123456",
+        //    MaximumPoolSize = 50, // 这个值应当小于 max_connections
+        //}.ConnectionString;
+        options.ConnectionString = new MySqlConnectionStringBuilder(host.Configuration.GetConnectionString("MySqlMaster")??"")
+        {
+            CharacterSet = "utf8",
+            AllowUserVariables = true,
+            IgnoreCommandTransaction = true,
+            TreatTinyAsBoolean = false,
+            MaximumPoolSize = 50,
+            SslMode = MySqlSslMode.None,
+        }.ConnectionString;
+    });
+    
+    host.Services.AddLogging(builder =>
+    {
+        builder.ClearProviders();
+        builder.AddSerilog(new LoggerConfiguration()
+            .WriteTo.Console()
+            .WriteTo.File("./log/error.log", restrictedToMinimumLevel:LogEventLevel.Error)
+            // .WriteTo.File("./log/info.log", restrictedToMinimumLevel:LogEventLevel.Information) //性能考虑暂不使用
+            .CreateLogger()
+        );
+    });
+
+    host.Services.AddScoped<ProcessContext>();
+    host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Producer);
+    host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Consumer);
+    host.Services.AddTransient<TaskManager>();
+    host.Services.AddSingleton<ErrorRecorder>();
+
+    host.Services.AddHostedService<MainHostedService>();
+    host.Services.AddHostedService<TaskMonitorService>();
+    if(commandOptions.IsMock)host.Services.AddSingleton<IInputService,InputService>();
+    else host.Services.AddSingleton<IInputService, InputService>();
+    host.Services.AddSingleton<ITransformService, TransformService>();
+    host.Services.AddSingleton<IOutputService, OutputService>();
+    var redisOptions = host.Configuration.GetSection("RedisCacheOptions").Get<RedisCacheOptions>() ?? new RedisCacheOptions();
+    host.Services.AddStackExchangeRedisCache(options =>
+    {
+        options.Configuration = redisOptions.Configuration;
+        options.InstanceName = redisOptions.InstanceName;
+    });
+    var app = host.Build();
+    await app.RunAsync();
+}

ConsoleApp2/README.md

@@ -0,0 +1,49 @@
+## 说明
+使用该程序来对MyDumper导出的CSV数据进行读取，转换，然后导出到其他数据库中。
+
+1. 用MyDumper从数据库导出CSV数据  
+
+   使用MyDumper Docker镜像
+    ```sh
+    docker run --rm --net=host -v D:/Dump:/home/backup mydumper/mydumper:v0.15.2-6 mydumper `
+   -h 127.0.0.1 -P 33306 -u root -p 123456 `
+   -B cferp_test --no-schemas --csv --hex-blob `
+   -o /home/backup
+    ```
+   将挂载卷，数据库连接和输出目录替换  
+   不导出数据库结构(--no-schemas)，
+   导出完的目录下应当包含.sql文件以及.dat文件
+
+2. 在Program.cs中修改`CsvOptions`配置
+   ```cs
+   host.Services.Configure<CsvOptions>(option =>
+    {
+        option.Delimiter = ",";
+        option.QuoteChar = '"';
+        option.InputDir = "D:/Dump/Test";
+    });
+   ```
+   将`option.InputDir`配置为MyDumper导出的数据目录
+
+3. 在Program.cs中修改`DatabaseOutputOptions`配置
+   ```cs
+   host.Services.Configure<DatabaseOutputOptions>(options =>
+    {
+        options.ConnectionString = new MySqlConnectionStringBuilder
+        {
+            Server = "127.0.0.1",
+            Port = 33306,
+            Database = "cferp_test_1",
+            UserID = "root",
+            Password = "123456",
+            MaximumPoolSize = 50, 
+        }.ConnectionString;
+        options.MaxTask = 16;
+        options.FlushCount = 200;
+    });
+   ```
+   将`MySqlConnectionStringBuilder`的属性修改为程序要导出至的数据库
+   > 后续将这些配置通过命令行传递
+
+4. 运行程序
+   > 注意，测试数据库`cferp_test`中的`order_process_step`表存在外键，如果要导出到和测试库同结构的数据库，记得先把外键删除。

ConsoleApp2/Services/CsvSource.cs

@@ -0,0 +1,183 @@
+using System.Reflection.PortableExecutable;
+using System.Text;
+using System.Text.RegularExpressions;
+using ConsoleApp2.Helpers;
+using ConsoleApp2.HostedServices.Abstractions;
+using Microsoft.Extensions.Logging;
+
+namespace ConsoleApp2.Services;
+
+/// <summary>
+/// CSV文件读取
+/// </summary>
+public class CsvSource:IDataSource
+{
+    protected readonly string _inputDir;
+    //protected readonly StreamReader _reader;
+    private readonly ILogger? _logger;
+    protected readonly string _tableName;
+    protected  string _sqlFilePath=string.Empty;
+    protected readonly string? _sqlFileText;
+    protected string[] headers=Array.Empty<string>();
+    protected string[] csvFiles = Array.Empty<string>();
+    public string? CurrentRaw { get; protected set; }
+    public string Delimiter { get; private set; }
+    public char QuoteChar { get; private set; }
+    public CsvSource(string inputDir,string tableName,string delimiter = ",", char quoteChar = '"',
+        ILogger? logger = null)
+    {
+        _inputDir = inputDir;
+        _tableName = tableName;
+        _logger = logger;
+        Delimiter = delimiter;
+        QuoteChar = quoteChar;
+
+    }
+
+
+    public string[] ParseRow(string row, char quoteChar, string delimiter)
+    {
+        var span = row.AsSpan();
+        var result = new List<string>();
+
+        if (span.Length == 0)
+            throw new ArgumentException("The row is empty", nameof(row));
+
+        var isInQuote = span[0] == quoteChar;
+        var start = 0;
+        for (var i = 1; i < span.Length; i++)
+        {
+            if (span[i] == quoteChar)
+            {
+                isInQuote = !isInQuote;
+            }
+            // delimiter需要足够复杂
+            else if (/*!isInQuote && */span.Length > i + delimiter.Length && span[i..(i + delimiter.Length)].Equals(delimiter, StringComparison.CurrentCulture)) // field matched
+            {
+                string field;
+                if (span[start] == quoteChar && span[i - 1] == quoteChar) // enclosed by quoteChar
+                    field = span[(start + 1)..(i - 1)].ToString(); // escape quoteChar
+                else
+                    field = span[start..i].ToString();
+
+                start = i + delimiter.Length;
+
+                if (field == "\\N")
+                    field = "NULL";
+
+                result.Add(field);
+                
+                continue;
+            }
+        }
+
+        result.Add(span[start..].ToString());
+
+
+        for (var i = 0; i < result.Count; i++)
+        {
+            var field = result[i];
+            if (DumpDataHelper.CheckHexField(field) && StringExtensions.CheckJsonHex(field))
+            {
+                result[i] = StringExtensions.FromHex(field);
+            }
+        }
+
+        return result.ToArray();
+    }
+    
+    public string[] ParseRow2(ReadOnlySpan<char> source, char quoteChar, string delimiter)
+    {
+        var result = new List<string>();
+        var index = -1;
+        StringBuilder current = new StringBuilder();
+        bool hasQuote = false;
+        bool hasSlash = false;
+        while (index < source.Length-1)
+        {
+            index++;
+            if (hasSlash == false && source[index] == '\\')
+            {
+                hasSlash = true;
+                current.Append('\\');
+                continue;
+            }
+            if (hasSlash ==false && source[index] == quoteChar)
+            {
+                hasQuote = !hasQuote;
+                current.Append(source[index]);
+                continue;
+            }
+            if (hasQuote==false && source[index] == delimiter[0])
+            {
+                result.Add(current.ToString());
+                current.Clear();
+            }
+            else
+            {
+                current.Append(source[index]);
+            }
+
+            hasSlash = false;
+        }
+
+        result.Add(current.ToString());
+        return result.ToArray();
+    }
+    public virtual async Task GetHeaderAndCsvFiles()
+    {
+        string pattern = $"^.*\\.{_tableName}\\..*\\.sql$";
+        _sqlFilePath = Directory.GetFiles(_inputDir).FirstOrDefault(s => Regex.Match(s, pattern).Success) ?? "";
+        var text = await File.ReadAllTextAsync(_sqlFilePath);
+        headers =  DumpDataHelper.GetCsvHeadersFromSqlFileAsync(text);
+        csvFiles =  DumpDataHelper.GetCsvFileNamesFromSqlFileAsync(text, new Regex(@"'.+\.dat.zst'"));
+
+    }
+    public virtual async Task DoEnqueue(Action<DataRecord> action)
+    {
+        await GetHeaderAndCsvFiles();
+        foreach (var file in csvFiles)
+        {
+            var filePath= Path.Combine(_inputDir, file);
+            using var fs = File.OpenRead(filePath);
+            {
+                using StreamReader sr = new (fs);
+                {
+                    while (!sr.EndOfStream)
+                    {
+                        var line = await sr.ReadLineAsync();
+                        var fields = ParseRow2(line, QuoteChar, Delimiter);
+                        var record = new DataRecord(fields, _tableName, headers);
+                        action?.Invoke(record);
+                    }
+                }
+            }
+
+          
+        }
+    }
+    public virtual async Task<DataRecord?> GetTestRecord()
+    {
+        await GetHeaderAndCsvFiles();
+        var file = csvFiles.FirstOrDefault();
+        if (file != null)
+        {
+            var filePath = Path.Combine(_inputDir, file);
+            using var fs = File.OpenRead(filePath);
+            {
+                using StreamReader sr = new(fs);
+                {
+                    var line = await sr.ReadLineAsync();
+                    var fields = ParseRow2(line, QuoteChar, Delimiter);
+                    var record = new DataRecord(fields, _tableName, headers);
+                    return record;
+                }
+            } 
+        }
+        return null;
+    }
+    public void Dispose()
+    {
+       // _reader.Dispose();
+    }
+}

ConsoleApp2/Services/DataRecordQueue.cs

@@ -0,0 +1,51 @@
+using System.Collections.Concurrent;
+using System.Diagnostics.CodeAnalysis;
+
+namespace ConsoleApp2.Services;
+
+/// <summary>
+/// 数据队列
+/// </summary>
+public class DataRecordQueue : IDisposable
+{
+    private readonly BlockingCollection<DataRecord> _queue;
+
+    public int Count => _queue.Count;
+    public bool IsCompleted => _queue.IsCompleted;
+    public bool IsAddingCompleted => _queue.IsAddingCompleted;
+
+    public event Action? OnRecordWrite;
+    public event Action? OnRecordRead;
+
+    public DataRecordQueue()
+    {
+        _queue = new BlockingCollection<DataRecord>(2000_000); // 队列最长为20W条记录
+    }
+    
+    public bool TryDequeue([MaybeNullWhen(false)] out DataRecord record)
+    {
+        if (_queue.TryTake(out record))
+        {
+            OnRecordRead?.Invoke();
+            return true;
+        }
+
+        return false;
+    }
+
+    public DataRecord Dequeue() => _queue.Take();
+
+    public void CompleteAdding() => _queue.CompleteAdding();
+
+    public void Enqueue(DataRecord record)
+    {
+        _queue.Add(record);
+        OnRecordWrite?.Invoke();
+    }
+
+
+    public void Dispose()
+    {
+        _queue.Dispose();
+    }
+}

ConsoleApp2/Services/ErrorRecorder.cs

@@ -1,20 +1,27 @@
 using System.Text;
 using Microsoft.Extensions.Logging;
 
-namespace MesETL.App.Services.ErrorRecorder;
+namespace ConsoleApp2.Services;
 
-
-public sealed class OutputErrorRecorder : ErrorRecorder
+public class ErrorRecorder
 {
-    private readonly string _outputDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"ErrorRecords/{UID}/Output");
-    private readonly string _database;
+    private readonly string _outputDir = "./ErrorRecords";
+    private readonly ILogger _logger;
     private readonly Dictionary<string, int> _logIndex = new();
-    private static readonly object Lock = new();
 
-    public OutputErrorRecorder(string database, ILogger logger) : base(logger)
+    /// <summary>
+    /// 当次执行标识
+    /// </summary>
+    private static readonly string UID = DateTime.Now.ToString("yyyy-MM-dd HH-mm-ss");
+
+    public ErrorRecorder(ILogger<ErrorRecorder> logger)
     {
-        _database = database;
-        Logger = logger;
+        _logger = logger;
+        var dir = Path.Combine(_outputDir, UID);
+        if (!Directory.Exists(dir))
+        {
+            Directory.CreateDirectory(dir);
+        }
     }
 
     /// <summary>
@@ -25,43 +32,30 @@ public sealed class OutputErrorRecorder : ErrorRecorder
     /// <param name="exception"></param>
     public async Task LogErrorSqlAsync(string commandText, string tableName, Exception exception)
     {
-        if (!Directory.Exists(_outputDir))
-            Directory.CreateDirectory(_outputDir);
         if (!_logIndex.TryGetValue(tableName, out var idx))
         {
             idx = 0;
             _logIndex.Add(tableName, idx);
         }
-        var filePath = Path.Combine(_outputDir, $"{tableName}-{idx}.errlog");
+        var filePath = Path.Combine(_outputDir, UID, $"{tableName}-{idx}.errlog");
             
         if (File.Exists(filePath) && new FileInfo(filePath).Length > 10 * 1024 * 1024)
         {
             ++idx;
             _logIndex[tableName] = idx;
-            filePath = Path.Combine(_outputDir, $"{tableName}-{idx}.errlog");
+            filePath = Path.Combine(_outputDir, UID, $"{tableName}-{idx}.errlog");
         }
         var content = $"""
                       /* [{DateTime.Now:yyyy-MM-dd HH:mm:ss}]
-                       * Error occurred when export table '{_database}.{tableName}':
+                       * Error occurred when export table '{tableName}':
                        * {exception.Message}
                        */
                       
-                      USE `{_database}`;
                       {commandText}
 
 
                       """;
-        try
-        {
-            lock (Lock)
-            {
-                File.AppendAllText(filePath, content, Encoding.UTF8);
-            }
-        }
-        catch(Exception e)
-        {
-            Logger.LogError(e, "输出错误日志时发生错误");   
-        }
+        await File.AppendAllTextAsync(filePath, content, Encoding.UTF8);
     }
 
     /// <summary>
@@ -71,9 +65,7 @@ public sealed class OutputErrorRecorder : ErrorRecorder
     /// <param name="exception"></param>
     public async Task LogErrorSqlAsync(string commandText, Exception exception)
     {
-        if (!Directory.Exists(_outputDir))
-            Directory.CreateDirectory(_outputDir);
-        var filePath = Path.Combine(_outputDir, "UnknownTables.errlog");
+        var filePath = Path.Combine(_outputDir, UID, "UnknownTables.errlog");
         var content = $"""
                        /* [{DateTime.Now:yyyy-MM-dd HH:mm:ss}]
                         * Error occurred when export table with unknown table name:
@@ -86,4 +78,27 @@ public sealed class OutputErrorRecorder : ErrorRecorder
         await File.AppendAllTextAsync(filePath, content, Encoding.UTF8);
     }
 
+    public async Task LogErrorRecordsAsync(IDictionary<string, DataRecord> records, Exception exception)
+    {
+        var pathDict = new Dictionary<string, string>();
+        foreach (var pair in records)
+        {
+            if(!pathDict.TryGetValue(pair.Key, out var path))
+            {
+                path = Path.Combine(_outputDir, UID, "ErrorRecords", $"{pair.Key}.errlog");
+                pathDict.Add(pair.Key, path);
+            }
+            //
+            await File.AppendAllTextAsync(path, string.Join(',', pair.Value.Fields));
+        }
+    }
+    
+    public void ClearErrorRecords()
+    {
+        _logger.LogInformation("***** Clear error records *****");
+        foreach (var file in Directory.GetFiles(_outputDir, "*.errlog", SearchOption.AllDirectories))
+        {
+            File.Delete(file);
+        }
+    }
 }

ConsoleApp2/Services/JsvSource.cs

@@ -0,0 +1,41 @@
+using ConsoleApp2.Helpers;
+using ConsoleApp2.HostedServices.Abstractions;
+using Microsoft.Extensions.Logging;
+using ServiceStack.Text;
+
+namespace ConsoleApp2.Services;
+
+/// <summary>
+/// 读取Jsv格式文件
+/// </summary>
+[Obsolete]
+public class JsvSource:IDataSource
+{
+    //private readonly string _inputDir;
+    //private readonly JsvStringSerializer _jsv;
+    //private readonly StreamReader? _reader;
+    // ReSharper disable once PrivateFieldCanBeConvertedToLocalVariable
+    //private readonly ILogger? _logger;
+    //private readonly string _tableName;
+
+    public DataRecord Current { get; protected set; } = null!;
+    public string[]? Headers { get; }
+    //public bool EndOfSource => _reader.EndOfStream;
+
+    public JsvSource(string inputDir,string tableName, ILogger? logger = null)
+    {
+        //_inputDir = inputDir;
+        //_tableName = tableName;
+        //_jsv = new JsvStringSerializer();
+        //_logger = logger;
+    }
+    public  Task DoEnqueue(Action<DataRecord> action)
+    {
+        return Task.CompletedTask;
+    }
+
+    public  void Dispose()
+    {
+       // _reader?.Dispose();
+    }
+}

ConsoleApp2/Services/MySqlDestination.cs

@@ -0,0 +1,214 @@
+using System.Data.Common;
+using System.Text;
+using System.Text.RegularExpressions;
+using ConsoleApp2.Helpers;
+using ConsoleApp2.Options;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using MySqlConnector;
+using ServiceStack;
+
+namespace ConsoleApp2.Services;
+
+/// <summary>
+/// Mysql导出
+/// </summary>
+public partial class MySqlDestination : IDisposable, IAsyncDisposable
+{
+    private readonly Dictionary<string, IList<DataRecord>> _recordCache;
+    private readonly MySqlConnection _conn;
+    private readonly ILogger _logger;
+    private readonly ProcessContext _context;
+    private readonly IOptions<DataTransformOptions> _transformOptions;
+    private readonly ErrorRecorder _errorRecorder;
+
+    public MySqlDestination(
+        string connStr, 
+        ILogger logger, 
+        ProcessContext context, 
+        IOptions<DataTransformOptions> transformOptions,
+        ErrorRecorder errorRecorder)
+    {
+        _conn = new MySqlConnection(connStr);
+        _conn.Open();
+        _recordCache = new Dictionary<string, IList<DataRecord>>();
+        _logger = logger;
+        _context = context;
+        _transformOptions = transformOptions;
+        _errorRecorder = errorRecorder;
+    }
+    
+    public Task WriteRecordAsync(DataRecord record)
+    {
+        _recordCache.AddOrUpdate(record.TableName, [record], (key, value) =>
+        {
+            value.Add(record);
+            return value;
+        });
+        return Task.CompletedTask;
+    }
+
+    public async Task WriteRecordsAsync(IEnumerable<DataRecord> records)
+    {
+        foreach (var record in records)
+        {
+            await WriteRecordAsync(record);
+        }
+    }
+
+    public async Task FlushAsync(int maxAllowPacket)
+    {
+        if (_recordCache.Count == 0)
+            return;
+
+        var cmd = _conn.CreateCommand();
+        cmd.CommandTimeout = 3 * 60;
+       
+        try
+        {
+            var excuseList = GetExcuseList(_recordCache, maxAllowPacket).ToList();
+            foreach (var insertSql in excuseList)
+            {
+                cmd.CommandText = insertSql;
+                try
+                {
+                    await cmd.ExecuteNonQueryAsync();
+                }
+                catch (Exception e)
+                {
+                    _logger.LogCritical(e, "Error when flushing records, sql: {Sql}", cmd.CommandText.Omit(1000));
+                    _context.AddException(e);
+                    
+                    var match = MatchTableName().Match(cmd.CommandText);
+                    if (match is { Success: true, Groups.Count: > 1 })
+                    {
+                        var tableName = match.Groups[1].Value;
+                        await _errorRecorder.LogErrorSqlAsync(cmd.CommandText, tableName, e);
+                    }
+                    else await _errorRecorder.LogErrorSqlAsync(cmd.CommandText, e);
+                }
+            }
+            _recordCache.Clear();
+        }
+        catch (Exception e)
+        {
+            _logger.LogCritical(e, "Error when serialize records, record:");
+            _context.AddException(e);
+        }
+        finally
+        {
+            await cmd.DisposeAsync();
+        }
+    }
+    
+    [GeneratedRegex("INSERT INTO `([^`]+)`")]
+    private static partial Regex MatchTableName();
+
+    public IEnumerable<string> GetExcuseList(IDictionary<string, IList<DataRecord>> tableRecords,int maxAllowPacket)
+    {
+        var sb = new StringBuilder();
+        foreach (var (tableName, records) in tableRecords)
+        {
+            if (records.Count == 0)
+                continue;
+            
+            var recordIdx = 0;
+            StartBuild:
+            var noCommas = true;
+            
+            // INSERT INTO ... VALUES >>>
+            sb.Append($"INSERT INTO `{tableName}`(");
+            for (var i = 0; i < records[0].Headers.Length; i++)
+            {
+                var header = records[0].Headers[i];
+                sb.Append($"`{header}`");
+                if (i != records[0].Headers.Length - 1)
+                    sb.Append(',');
+            }
+
+            sb.Append(") VALUES ");
+            
+            // ([FIELDS]), >>>
+            for (;recordIdx < records.Count; recordIdx++)
+            {
+                var record = records[recordIdx];
+                var recordSb = new StringBuilder();
+                recordSb.Append('(');
+                for (var fieldIdx = 0; fieldIdx < record.Fields.Length; fieldIdx++)
+                {
+                    var field = record.Fields[fieldIdx];
+
+                    // 在这里处理特殊列
+                    #region HandleFields
+                    if (field == "\\N")
+                    {
+                        recordSb.Append("NULL");
+                        goto Escape;
+                    }
+                    
+                    switch (_transformOptions.Value.GetColumnType(record.TableName, record.Headers[fieldIdx]))
+                    {
+                        case ColumnType.Text:
+                            recordSb.Append(string.IsNullOrEmpty(field)
+                                ? "''"
+                                : _transformOptions.Value.TransformBinary?.Invoke(field) ?? field);
+                            break;
+                        case ColumnType.Blob:
+                            if (string.IsNullOrEmpty(field))
+                                recordSb.Append("''");
+                            else recordSb.Append($"0x{field}");
+                            break;
+                        case ColumnType.Json:
+                            recordSb.Append(string.IsNullOrEmpty(field)
+                                ? "\"[]\""
+                                : _transformOptions.Value.TransformBinary?.Invoke(field) ?? field);
+                            break;
+                        case ColumnType.UnDefine:
+                        default:
+                            recordSb.Append(field);
+                            break;
+                    }
+                    
+                    Escape:
+
+                    #endregion
+                    if (fieldIdx != record.Fields.Length - 1)
+                        recordSb.Append(',');
+                }
+
+                recordSb.Append(')');
+
+                // 若字符数量即将大于限制，则返回SQL，清空StringBuilder，保留当前记录的索引值，然后转到StartBuild标签重新开始一轮INSERT
+                if (sb.Length + recordSb.Length + 1 > maxAllowPacket)
+                {
+                    sb.Append(';');
+                    yield return sb.ToString();
+                    sb.Clear();
+                    goto StartBuild;
+                }
+
+                if (!noCommas)
+                    sb.Append(',').AppendLine();
+                noCommas = false;
+                sb.Append(recordSb); // StringBuilder.Append(StringBuilder)不会分配多余的内存
+            }
+
+            sb.Append(';');
+            yield return sb.ToString();
+            sb.Clear();
+        }
+    }
+
+
+    public void Dispose()
+    {
+        _conn.Close();
+        _conn.Dispose();
+    }
+
+    public async ValueTask DisposeAsync()
+    {
+        await _conn.CloseAsync();
+        await _conn.DisposeAsync();
+    }
+}

ConsoleApp2/Services/ProcessContext.cs

@@ -0,0 +1,57 @@
+using System.Collections.Concurrent;
+
+namespace ConsoleApp2.Services;
+
+/// <summary>
+/// 处理上下文类，标识处理进度
+/// </summary>
+public class ProcessContext
+{
+    private int _inputCount;
+    private int _transformCount;
+    private int _outputCount;
+    private ConcurrentBag<Exception> _exceptionList = new ConcurrentBag<Exception>();
+    public bool IsInputCompleted { get; private set; }
+    public bool IsTransformCompleted { get; private set; }
+    public bool IsOutputCompleted { get; private set; }
+
+    public int InputCount
+    {
+        get => _inputCount;
+        private set => _inputCount = value;
+    }
+
+    public int TransformCount
+    {
+        get => _transformCount;
+        private set => _transformCount = value;
+    }
+
+    public int OutputCount
+    {
+        get => _outputCount;
+        private set => _outputCount = value;
+    }
+    public void AddException(Exception ex) 
+    {
+        _exceptionList.Add(ex);
+    }
+    public ConcurrentBag<Exception> GetExceptions()
+    {
+        return _exceptionList;
+    }
+    public void CompleteInput() => IsInputCompleted = true;
+
+    public void CompleteTransform() => IsTransformCompleted = true;
+    public void CompleteOutput() => IsOutputCompleted = true;
+
+    public void AddInput() => Interlocked.Increment(ref _inputCount);
+
+    public void AddInput(int count) => Interlocked.Add(ref _inputCount, count);
+
+    public void AddTransform() => Interlocked.Increment(ref _transformCount);
+    public void AddTransform(int count) => Interlocked.Add(ref _transformCount, count);
+
+    public void AddOutput() => Interlocked.Increment(ref _outputCount);
+    public void AddOutput(int count) => Interlocked.Add(ref _outputCount, count);
+}

ConsoleApp2/Services/TaskManager.cs

@@ -0,0 +1,48 @@
+using System.Collections.Concurrent;
+using Microsoft.Extensions.Logging;
+
+namespace ConsoleApp2.Services;
+
+/// <summary>
+/// 快速批量创建和等待任务
+/// </summary>
+public class TaskManager
+{
+    private readonly ConcurrentBag<Task> _tasks;
+    private readonly ILogger _logger;
+
+    public int TaskCount => _tasks.Count;
+    public int RunningTaskCount => _tasks.Count(task => !task.IsCompleted);
+    public IReadOnlyCollection<Task> Tasks => _tasks;
+
+    public TaskManager(ILogger<TaskManager> logger)
+    {
+        _tasks = new ConcurrentBag<Task>();
+        _logger = logger;
+    }
+
+    public void CreateTask<TResult>(Func<TResult> func, CancellationToken cancellationToken = default)
+    {
+        var task = Task.Factory.StartNew(func, cancellationToken);
+        _tasks.Add(task);
+        _logger.LogDebug("New task created");
+    }
+    public void CreateTasks<TResult>(Func<TResult> func,int taskCount, CancellationToken cancellationToken = default)
+    {
+        for (int i = 0; i < taskCount; i++)
+        {
+            CreateTask(func, cancellationToken);
+        }
+    }
+    public async Task WaitAll()
+    {
+        await Task.WhenAll(_tasks);
+    }
+
+    public void ClearTask()
+    {
+        if(RunningTaskCount != 0)
+            throw new InvalidOperationException("Unable to clear task. There are still running tasks");
+        _tasks.Clear();
+    }
+}

ConsoleApp2/Services/ZstSource.cs

@@ -0,0 +1,90 @@
+using ConsoleApp2.Helpers;
+using Microsoft.Extensions.Logging;
+using System.Text.RegularExpressions;
+using ZstdSharp;
+namespace ConsoleApp2.Services
+{
+    public class ZstSource : CsvSource
+    {
+        public ZstSource(string inputDir, string tableName, string delimiter = ",", char quoteChar = '"',
+        ILogger? logger = null) : base(inputDir, tableName, delimiter = ",", quoteChar = '"', logger = null)
+        {
+
+        }
+        private static async Task<string> DecompressFile(string filePath)
+        {
+            using var input = File.OpenRead(filePath);
+            {
+                using var decopress = new DecompressionStream(input);
+                {
+
+                    var ms = new MemoryStream();
+                    decopress.CopyTo(ms);
+                    ms.Seek(0, SeekOrigin.Begin);
+                    StreamReader reader = new(ms);
+                    var text = await reader.ReadToEndAsync();
+                    return text;
+
+                }
+            }
+        }
+        public override async Task GetHeaderAndCsvFiles()
+        {
+            string pattern = $"^.*\\.{_tableName}\\..*\\.sql.zst$";
+            _sqlFilePath = Directory.GetFiles(_inputDir).FirstOrDefault(s => Regex.Match(s, pattern).Success) ?? "";
+            var text = await DecompressFile(_sqlFilePath);
+            headers= DumpDataHelper.GetCsvHeadersFromSqlFileAsync(text);
+            csvFiles= DumpDataHelper.GetCsvFileNamesFromSqlFileAsync(text, new Regex(@"'.+\.dat.zst'"));
+
+        }
+        public override async Task DoEnqueue(Action<DataRecord> action)
+        {
+           await GetHeaderAndCsvFiles();
+            foreach (var file in csvFiles)
+            {
+                var filePath = Path.Combine(_inputDir, file);
+                using var input = File.OpenRead(filePath);
+                {
+                    using var decopress = new DecompressionStream(input);
+                    {
+                        using var reader = new StreamReader(decopress);
+                        {
+                            while (!reader.EndOfStream)
+                            {
+                                var line = await reader.ReadLineAsync();
+                                var fields = ParseRow2(line, QuoteChar, Delimiter);
+                                var record = new DataRecord(fields, _tableName, headers);
+                                action?.Invoke(record);
+                            }
+                        }
+                    }
+                }
+
+            }
+        }
+        public override async Task<DataRecord?> GetTestRecord()
+        {
+            await GetHeaderAndCsvFiles();
+            var file = csvFiles?.FirstOrDefault();
+            if (file != null)
+            {
+                var filePath = Path.Combine(_inputDir, file);
+                using var input = File.OpenRead(filePath);
+                {
+                    using var decopress = new DecompressionStream(input);
+                    {
+                        using var reader = new StreamReader(decopress);
+                        {
+                            var line = await reader.ReadLineAsync();
+                            var fields = ParseRow2(line, QuoteChar, Delimiter);
+                            var record = new DataRecord(fields, _tableName, headers);
+                            return record;
+                        }
+
+                    }
+                }
+            }
+            return null;
+        }
+    }
+}

ConsoleApp2/SimulationService/DataHelper.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Text.RegularExpressions;
+using System.Threading.Tasks;
+
+namespace ConsoleApp2.SimulationService
+{
+    public static partial class DataHelper
+    {
+        public static short[] shareKeys = {23000, 23040, 23070, 23100, 24000, 24040, 24070, 24100, 25000, 25040, 25070, 25100 };
+        public static int[] companyIds = { 1, 2, 3, 4 };
+        private static T getArrayValue<T>(int index, T[] array)//按index取数据,超过数组长度,index从0开始再取
+        {
+            return array[index % array.Length];
+        }
+        public static short GetShareKey(int index)
+        {
+            return getArrayValue(index, shareKeys);
+        }
+        public static int GetCompanyId(int index)
+        {
+            return getArrayValue(index, companyIds);
+        }
+       
+    }
+}

ConsoleApp2/SimulationService/SimulationInputService.cs

@@ -0,0 +1,136 @@
+using ConsoleApp2.Const;
+using ConsoleApp2.Helpers;
+using ConsoleApp2.HostedServices;
+using ConsoleApp2.HostedServices.Abstractions;
+using ConsoleApp2.Options;
+using ConsoleApp2.Services;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using System.Text.RegularExpressions;
+
+namespace ConsoleApp2.SimulationService
+{
+
+    public class SimulationInputService : IInputService
+    {
+        private readonly ILogger _logger;
+        private readonly IOptions<DataInputOptions> _dataInputOptions;
+
+        private readonly ProcessContext _context;
+
+        public SimulationInputService(ILogger<InputService> logger,
+            IOptions<DataInputOptions> dataInputOptions,
+            ProcessContext context)
+        {
+            _logger = logger;
+            _dataInputOptions = dataInputOptions;
+            _context = context;
+        }
+        public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, ProcessContext context, CancellationToken cancellationToken)
+        {
+            var inputDir = _dataInputOptions.Value.InputDir;
+            _logger.LogInformation("***** simulation input service start, working dir: {InputDir}, thread id: {ThreadId} *****", inputDir, Environment.CurrentManagedThreadId);
+            var files = Directory.GetFiles(inputDir);
+            if (files.Length == 0)
+            {
+                _logger.LogInformation("No source files found in {InputDir}", inputDir);
+                return;
+            }
+            foreach (var tableName in tasksOptions.TableInfoConfig.Keys)
+            {
+                _logger.LogInformation("Working table: {tableName}", tableName);
+
+                var dataCount = tasksOptions.TableInfoConfig[tableName].SimulaRowCount;//当前表要生成的总数据量
+                var companyTotallCount = 1000;//当前表每个公司生成的总数据量
+                var tempRecords = new List<DataRecord>();
+                var sk = DataHelper.shareKeys.First();
+                var companyID = DataHelper.companyIds.First();
+
+                var shareKeyInterval = 20000;//每个sharekey的数据量 
+                var getShareKeyTimes = 0;//sharekey生成的次数,每生成一次，改变sharekey的值
+                var getCompanyIDTimes = 0;//公司生成的次数,每生成一次，改变companyID的值
+                var shareKeyIntervalCount = 0;
+
+                var source = _dataInputOptions.Value.CreateSource?.Invoke(tableName);
+                if (source == null) throw new NullReferenceException($"create table source:{tableName} failed!");
+                var testRecord = await source.GetTestRecord();
+                if(testRecord == null) throw new NullReferenceException($"create testRecord failed, tableName:{tableName}");
+                for (long i = 1; i <= dataCount; i++)
+                {
+                    shareKeyIntervalCount++;
+                    if (shareKeyIntervalCount > shareKeyInterval)
+                    {
+                        sk = DataHelper.GetShareKey(getShareKeyTimes);
+                        getShareKeyTimes++;
+                        shareKeyIntervalCount = 0;
+                    }
+                    var fields = new string[testRecord.Fields.Length];
+                    Array.Copy(testRecord.Fields, fields, testRecord.Fields.Length);
+                    var record = new DataRecord(fields, testRecord.TableName, testRecord.Headers, companyID);
+                    //更新record的ID、OrderNo,ShardKey值
+                    if (record.Headers.Contains("ID"))
+                    {
+                        var index = Array.IndexOf(record.Headers, "ID");
+                        if (index > -1)
+                        {
+                            record.Fields[index] = i.ToString();
+                        }
+                    }
+                    if (record.TableName == "order_box_block" && record.Headers.Contains("BoxID"))
+                    {
+                        var index = Array.IndexOf(record.Headers, "BoxID");
+                        if (index > -1)
+                        {
+                            record.Fields[index] = i.ToString();
+                        }
+                    }
+                    if ((record.TableName == "order_block_plan_item" || record.TableName == "order_package_item") && record.Headers.Contains("ItemID"))
+                    {
+                        var index = Array.IndexOf(record.Headers, "ItemID");
+                        if (index > -1)
+                        {
+                            record.Fields[index] = i.ToString();
+                        }
+                    }
+                    if (record.TableName == "order" && record.Headers.Contains("OrderNo"))
+                    {
+                        var index = Array.IndexOf(record.Headers, "OrderNo");
+                        if (index > -1)
+                        {
+                            record.Fields[index] = i.ToString();
+                        }
+                    }
+                    if (record.Headers.Contains("ShardKey"))
+                    {
+                        var index = Array.IndexOf(record.Headers, "ShardKey");
+                        if (index > -1)
+                        {
+                            record.Fields[index] = sk.ToString();
+                        }
+                    }
+                    tempRecords.Add(record);
+                    if (tempRecords.Count >= companyTotallCount || i >= dataCount - 1)
+                    {
+                        foreach (var rc in tempRecords)
+                        {
+                            _context.AddInput();
+                            producerQueue.Enqueue(rc);
+                            if (cancellationToken.IsCancellationRequested)
+                                return;
+                        }
+                        tempRecords.Clear();
+                        companyID = DataHelper.GetCompanyId(getCompanyIDTimes);
+                        getCompanyIDTimes++;
+                    }
+                }
+                _logger.LogInformation("table:'{tableName}' simulation input completed", tableName);
+                //}
+                //_logger.LogInformation("File '{File}' input completed", Path.GetFileName(sqlPath));
+            }
+
+            context.CompleteInput();
+            _logger.LogInformation("***** Csv input service completed *****");
+        }
+    }
+}

ConsoleApp2/appsettings.json

@@ -0,0 +1,18 @@
+{
+  "CmdOptions": {
+    "InputFileType": "CSV",
+    "InputDir": "D:/MyDumper-ZST",
+    "TaskCount": 6,
+    "FlushCount": 10000,
+    "Isutf8mb4": true,
+    "OldestShardKey": 23000,
+    "OldestTime": "202301"
+  },
+  "ConnectionStrings": {
+    "MySqlMaster": "Server=127.0.0.1;Port=33309;UserId=root;Password=123456;Database=cferp_test;"
+  },
+  "RedisCacheOptions": {
+    "Configuration": "192.168.1.246:6380",
+    "InstanceName" : "mes-etl:"
+  }
+}

MesETL.App/Cache/CacheKeysFunc.cs

@@ -1,33 +0,0 @@
-using MesETL.App.Const;
-
-namespace MesETL.App.Cache;
-
-#nullable disable
-public static class CacheKeysFunc
-{
-    /// <summary>
-    /// Order表 由OrderNo获取对应的CompanyID
-    /// </summary>
-    /// <param name="orderNo"></param>
-    /// <returns></returns>
-    public static string Order_OrderNo_CompanyID(string orderNo) => BuildCacheKey(TableNames.Order, "OrderNo", orderNo, "CompanyID");
-
-    /// <summary>
-    /// OrderBlockPlan表 由ID获取对应的CompanyID
-    /// </summary>
-    /// <param name="id"></param>
-    /// <returns></returns>
-    public static string OrderBlockPlan_ID_CompanyID(string id) => BuildCacheKey(TableNames.OrderBlockPlan, "ID", id, "CompanyID");
-    
-    /// <summary>
-    /// OrderProcess表 由ID 获取对应的ShardKey
-    /// </summary>
-    /// <param name="id"></param>
-    /// <returns></returns>
-    public static string OrderProcess_ID_ShardKey(string id) => BuildCacheKey(TableNames.OrderProcess, "ID", id, "ShardKey");
-    
-    
-    // 数据缓存键格式为[TableName]-[ColumnName@ColumnValue]-[CacheColumnName]
-    static string BuildCacheKey(string tableName, string columnName, string columnValue, string cacheColumnName) 
-        => $"{tableName}-{columnName}@{columnValue}-{cacheColumnName}";
-}

MesETL.App/Cache/ICacher.cs

@@ -1,10 +0,0 @@
-namespace MesETL.App.Cache;
-
-public interface ICacher
-{
-    Task<string?> GetStringAsync(string key);
-    Task SetStringAsync(string key, string value);
-    Task<bool> ExistsAsync(string key);
-    Task SetHashAsync(string key, IReadOnlyDictionary<string, string> hash);
-    Task<Dictionary<string, string>> GetHashAsync(string key);
-}

MesETL.App/Cache/MemoryCache.cs

@@ -1,51 +0,0 @@
-using System.Collections.Concurrent;
-
-namespace MesETL.App.Cache;
-
-public class MemoryCache : ICacher
-{
-    private readonly ConcurrentDictionary<string, string> _stringCache = new();
-    private readonly ConcurrentDictionary<string, Dictionary<string, string>> _hashCache = new();
-    
-    public static MemoryCache? Instance { get; private set; }
-
-    public MemoryCache()
-    {
-        Instance = this;
-    }
-    
-    public Task<string?> GetStringAsync(string key)
-    {
-        return _stringCache.TryGetValue(key, out var value) ? Task.FromResult<string?>(value) : Task.FromResult((string?)null);
-    }
-
-    public Task SetStringAsync(string key, string value)
-    {
-        _stringCache[key] = value;
-        return Task.CompletedTask;
-    }
-
-    public Task<bool> ExistsAsync(string key)
-    {
-        return Task.FromResult(_stringCache.ContainsKey(key));
-    }
-
-    public Task SetHashAsync(string key, IReadOnlyDictionary<string, string> hash)
-    {
-        _hashCache[key] = hash.ToDictionary(x => x.Key, x => x.Value);
-        return Task.CompletedTask;
-    }
-
-    public Task<Dictionary<string, string>> GetHashAsync(string key)
-    {
-        return Task.FromResult(_hashCache[key]);
-    }
-
-    public void Delete(Func<string,bool> keySelector)
-    {
-        foreach (var k in _stringCache.Keys.Where(keySelector))
-        {
-            _stringCache.TryRemove(k, out _);
-        }
-    }
-}

MesETL.App/Cache/RedisCache.cs

@@ -1,64 +0,0 @@
-using MesETL.App.Options;
-using Microsoft.Extensions.DependencyInjection;
-using StackExchange.Redis;
-
-namespace MesETL.App.Cache;
-
-public class RedisCache : ICacher
-{
-    private readonly IDatabase _db;
-
-    public string KeyPrefix { get; set; }
-
-    public RedisCache(IConnectionMultiplexer conn, int dataBase, string keyPrefix = "")
-    {
-        _db = conn.GetDatabase(dataBase);
-        KeyPrefix = keyPrefix;
-    }
-
-    public async Task<string?> GetStringAsync(string key)
-    {
-        var value = await _db.StringGetAsync($"{KeyPrefix}{key}");
-        return !value.HasValue ? null : value.ToString();
-    }
-
-    public async Task SetStringAsync(string key, string value)
-    {
-        if (!await _db.StringSetAsync($"{KeyPrefix}{key}", value))
-            throw new RedisCommandException("设置Redis缓存失败");
-    }
-
-    public Task<bool> ExistsAsync(string key)
-    {
-        return _db.KeyExistsAsync($"{KeyPrefix}{key}");
-    }
-
-    public Task SetHashAsync(string key, IReadOnlyDictionary<string, string> hash)
-    {
-        return _db.HashSetAsync($"{KeyPrefix}{key}", hash.Select(pair => new HashEntry(pair.Key, pair.Value)).ToArray());
-    }
-
-    public async Task<Dictionary<string, string>> GetHashAsync(string key)
-    {
-        var entries = await _db.HashGetAllAsync($"{KeyPrefix}{key}");
-        var result = new Dictionary<string, string>();
-        foreach (var entry in entries)
-        {
-            result.Add(entry.Name.ToString(), entry.Value.ToString());
-        }
-
-        return result;
-    }
-}
-
-public static class RedisCacheExtensions
-{
-    public static IServiceCollection AddRedisCache(this IServiceCollection services, RedisCacheOptions options)
-    {
-        var conn = ConnectionMultiplexer.Connect(options.Configuration 
-                                                 ?? throw new ApplicationException("未配置Redis连接字符串"));
-        services.AddSingleton(conn);
-        services.AddSingleton<ICacher>(new RedisCache(conn, options.Database, options.InstanceName));
-        return services;
-    }
-}

MesETL.App/Const/ConstVar.cs

@@ -1,8 +0,0 @@
-namespace MesETL.App.Const;
-
-public static class ConstVar
-{
-    public const string Producer = "Producer";
-    public const string Null = "NULL";
-    public const string MyDumperNull = @"\N";
-}

MesETL.App/Const/TableNames.cs

@@ -1,41 +0,0 @@
-namespace MesETL.App.Const;
-
-public static class TableNames
-{
-    public const string Machine = "machine";
-    public const string Order = "order";
-    public const string OrderBlockPlan = "order_block_plan";
-    public const string OrderBlockPlanItem = "order_block_plan_item";
-    public const string OrderBlockPlanResult = "order_block_plan_result";
-    public const string OrderBoxBlock = "order_box_block";
-    public const string OrderBoxViewConfig = "order_box_view_config";
-    public const string OrderDataBlock = "order_data_block";
-    public const string OrderDataGoods = "order_data_goods";
-    public const string OrderDataParts = "order_data_parts";
-    public const string OrderExtra = "order_extra";
-    public const string OrderItem = "order_item";
-    public const string OrderModule = "order_module";
-    public const string OrderModuleExtra = "order_module_extra";
-    public const string OrderModuleItem = "order_module_item";
-    public const string OrderPackage = "order_package";
-    public const string OrderPackageItem = "order_package_item";
-    public const string OrderPatchDetail = "order_patch_detail";
-    public const string OrderProcess = "order_process";
-    public const string OrderProcessSchedule = "order_process_schdule";
-    public const string OrderProcessStep = "order_process_step";
-    public const string OrderProcessStepItem = "order_process_step_item";
-    public const string OrderScrapBoard = "order_scrap_board";
-    public const string OrderExtraList = "order_extra_list";
-    public const string ProcessGroup = "process_group";
-    public const string ProcessInfo = "process_info";
-    public const string ProcessItemExp = "process_item_exp";
-    public const string ProcessScheduleCapacity = "process_schdule_capacity";
-    public const string ProcessStepEfficiency = "process_step_efficiency";
-    public const string ReportTemplate = "report_template";
-    public const string SimplePackage = "simple_package";
-    public const string SimplePlanOrder = "simple_plan_order";
-    public const string SysConfig = "sys_config";
-    public const string WorkCalendar = "work_calendar";
-    public const string WorkShift = "work_shift";
-    public const string WorkTime = "work_time";
-}

MesETL.App/DataRecord.cs

@@ -1,183 +0,0 @@
-namespace MesETL.App;
-
-public class DataRecord : ICloneable
-{
-    private static int IndexOfIgnoreCase(IList<string> list, string value)
-    {
-        var idx = -1;
-        for (var i = 0; i < list.Count; i++)
-        {
-            if (list[i].Equals(value, StringComparison.OrdinalIgnoreCase))
-            {
-                idx = i;
-                break;
-            }
-        }
-
-        return idx;
-    }
-
-    private readonly List<string> _fields;
-    private readonly List<string> _headers;
-
-    /// <summary>
-    /// 字段列表
-    /// </summary>
-    public IReadOnlyList<string> Fields => _fields;
-
-    /// <summary>
-    /// 表头列表
-    /// </summary>
-    public IReadOnlyList<string> Headers => _headers;
-
-    /// <summary>
-    /// 来源表名
-    /// </summary>
-    public string TableName { get; }
-
-    /// <summary>
-    /// 需要输出的数据库
-    /// </summary>
-    public string? Database { get; set; }
-
-    /// <summary>
-    /// 所有字段的总字符数量
-    /// </summary>
-    public long FieldCharCount { get; }
-
-    /// <summary>
-    /// 忽略这个记录，不会被输出
-    /// </summary>
-    public bool Ignore { get; set; }
-
-
-    public DataRecord(IEnumerable<string> fields, string tableName, IEnumerable<string> headers,
-        string? database = null)
-    {
-        _fields = fields.ToList();
-        TableName = tableName;
-        _headers = headers.ToList();
-        Database = database;
-
-        if (_fields.Count != _headers.Count)
-            throw new ArgumentException(
-                $"The number of fields does not match the number of headers. Expected: {_headers.Count} Got: {_fields.Count} Fields: {string.Join(',', _fields)}",
-                nameof(fields));
-
-        FieldCharCount = _fields.Sum(x => (long)x.Length);
-    }
-
-    /// <summary>
-    /// 使用索引访问字段
-    /// </summary>
-    /// <param name="index"></param>
-    public string this[int index]
-    {
-        get => _fields[index];
-        set => _fields[index] = value;
-    }
-
-    /// <summary>
-    /// 使用列名访问字段，不区分大小写
-    /// </summary>
-    /// <param name="columnName"></param>
-    public string this[string columnName]
-    {
-        get => GetField(columnName);
-        set => SetField(columnName, value);
-    }
-
-    /// <summary>
-    /// 尝试获取某个字段值
-    /// </summary>
-    /// <param name="columnName"></param>
-    /// <param name="value"></param>
-    /// <returns></returns>
-    /// <exception cref="InvalidOperationException"></exception>
-    public bool TryGetField(string columnName, out string value)
-    {
-        value = string.Empty;
-        if (_headers is null)
-            throw new InvalidOperationException("Cannot get field when headers of a record have not been set.");
-        var idx = IndexOfIgnoreCase(_headers, columnName);
-        if (idx == -1)
-            return false;
-        value = _fields[idx];
-        return true;
-    }
-
-    /// <summary>
-    /// 获取一条记录的某个字段值
-    /// TODO: 最好能优化至O(1)
-    /// </summary>
-    /// <param name="columnName"></param>
-    /// <returns></returns>
-    /// <exception cref="InvalidOperationException"></exception>
-    /// <exception cref="IndexOutOfRangeException"></exception>
-    public string GetField(string columnName)
-    {
-        if (_headers is null)
-            throw new InvalidOperationException("记录的表头尚未设置，无法赋值");
-        var idx = IndexOfIgnoreCase(_headers, columnName);
-        if (idx is -1)
-            throw new IndexOutOfRangeException(
-                $"列 '{columnName}' 不存在于该纪录中，表名 '{TableName}");
-        return _fields[idx];
-    }
-
-    /// <summary>
-    /// 为记录的一个字段赋值，如果该字段名不存在则会抛出异常
-    /// </summary>
-    /// <param name="columnName">列名</param>
-    /// <param name="value">值</param>
-    /// <returns></returns>
-    /// <exception cref="InvalidOperationException">该记录的表头尚未初始化，你可能在错误的阶段调用了该方法</exception>
-    /// <exception cref="IndexOutOfRangeException">输入的字段名不存在于该记录中</exception>
-    public void SetField(string columnName, string value)
-    {
-        // 表头检查
-        if (_headers is null)
-            throw new InvalidOperationException("记录的表头尚未设置，无法赋值");
-        var idx = IndexOfIgnoreCase(_headers, columnName);
-        if (idx is -1)
-            throw new IndexOutOfRangeException(
-                $"列 '{columnName}' 不存在于该纪录中，表名 '{TableName}");
-        _fields[idx] = value;
-    }
-
-    /// <summary>
-    /// 在记录中追加一个字段
-    /// </summary>
-    /// <param name="columnName">字段名</param>
-    /// <param name="value">字段值</param>
-    /// <exception cref="InvalidOperationException">记录的表头尚未初始化，你可能在错误的阶段调用了此方法</exception>
-    /// <exception cref="ArgumentException">提供的字段名已存在于该记录中</exception>
-    public void AppendField(string columnName, string value)
-    {
-        if (_headers is null)
-            throw new InvalidOperationException("记录的表头尚未设置，无法赋值");
-        var idx = IndexOfIgnoreCase(_headers, columnName);
-        if (idx is > 0)
-            throw new ArgumentException($"字段名 '{columnName}' 已存在于该记录中，无法重复添加", nameof(columnName));
-
-        _headers.Add(columnName);
-        _fields.Add(value);
-    }
-
-    public void RemoveField(string columnName)
-    {
-        var idx = IndexOfIgnoreCase(_headers, columnName);
-        if (idx == -1)
-            throw new InvalidOperationException($"{TableName}: 列名 '{columnName}' 不存在");
-
-        _fields.RemoveAt(idx);
-        _headers.Remove(columnName);
-    }
-
-    public bool HeaderExists(string columnName) => IndexOfIgnoreCase(_headers, columnName) != -1;
-
-    public object Clone()
-    {
-        return new DataRecord(new List<string>(_fields), TableName, new List<string>(_headers), Database);
-    }
-}

MesETL.App/Helpers/DumpDataHelper.cs

@@ -1,191 +0,0 @@
-using System.Text.Json;
-using System.Text.RegularExpressions;
-using MesETL.App.HostedServices;
-using Serilog;
-using ZstdSharp;
-
-namespace MesETL.App.Helpers;
-
-public static partial class DumpDataHelper
-{
-    [GeneratedRegex(@"'.+\.dat'")]
-    private static partial Regex MatchDatFile();
-    [GeneratedRegex(@"\([^)]*\)")]
-    private static partial Regex MatchBrackets();
-    
-    /// <summary>
-    /// 从MyDumper导出的SQL文件内容中读取表头
-    /// </summary>
-    /// <param name="content"></param>
-    /// <returns></returns>
-    /// <exception cref="ArgumentException"></exception>
-    public static string[] GetCsvHeadersFromSqlFile(string content)
-    {
-        var match = MatchBrackets().Match(content);
-        if (!match.Success)
-            throw new ArgumentException("输入的SQL内容有误，无法提取表头", nameof(content));
-        return ParseHeader(match.ValueSpan);
-        
-        string[] ParseHeader(ReadOnlySpan<char> headerStr)
-        {
-            headerStr = headerStr[1..^1];
-            var headers = new List<string>();
-            foreach (var range in headerStr.Split(','))
-            {
-                headers.Add(headerStr[range].Trim("@`").ToString()); // 消除列名的反引号，如果是变量则消除@
-            }
-
-            return headers.ToArray();
-        }
-    }
-    
-    /// <summary>
-    /// 从MyDumper导出的Csv文件名解析出表名
-    /// </summary>
-    /// <param name="filePath"></param>
-    /// <returns></returns>
-    [Obsolete("用ParseMyDumperFile替代")]
-    public static string GetTableNameFromCsvFileName(ReadOnlySpan<char> filePath)
-    {
-        filePath = filePath[(filePath.LastIndexOf('\\') + 1)..];
-        var firstDotIdx = -1;
-        var secondDotIdx = -1;
-        var times = 0;
-        for (var i = 0; i < filePath.Length; i++)
-        {
-            if (filePath[i] == '.')
-            {
-                ++times;
-                if(times == 1)
-                    firstDotIdx = i;
-                if (times == 2)
-                {
-                    secondDotIdx = i;
-                    break;
-                }
-            }
-        }
-
-        return filePath[(firstDotIdx+1)..secondDotIdx].ToString();
-    }
-    
-    public enum MyDumperFileType { Dat, Sql }
-
-    public record MyDumperFileMeta(string Path, string Database, string TableName, int Index, MyDumperFileType Type);
-
-    public static MyDumperFileMeta ParseMyDumperFile(ReadOnlySpan<char> path)
-    {
-        try
-        {
-            var fileName = Path.GetFileName(path).ToString();
-            var parts = fileName.Split('.');
-            var type = parts[3] switch
-            {
-                "dat" => MyDumperFileType.Dat,
-                "sql" => MyDumperFileType.Sql,
-                _ => throw new ArgumentException("不支持的MyDumper文件类型", nameof(path))
-            };
-            return new MyDumperFileMeta(path.ToString(), parts[0], parts[1], int.Parse(parts[2]), type);
-        }
-        catch (Exception e)
-        {
-            throw new ArgumentException($"此文件不是MyDumper导出的文件 {path}", nameof(path), e);
-        }
-    }
-
-    /// <summary>
-    /// 从MyDumper导出的SQL文件内容中读取CSV文件名
-    /// </summary>
-    /// <param name="txt"></param>
-    /// <param name="regex"></param>
-    /// <returns></returns>
-    public static Task<string[]> GetCsvFileNamesFromSqlFileAsync(string txt, Regex regex)
-    {
-        //var txt = await File.ReadAllTextAsync(filePath);
-        var matches = regex.Matches(txt);
-        return Task.FromResult(matches.Select(match => match.ValueSpan[1..^1].ToString()).ToArray());
-    }
-    
-    /// <summary>
-    /// 检查字符串是否为16进制
-    /// </summary>
-    /// <param name="str"></param>
-    /// <returns></returns>
-    public static bool CheckHexField(string? str)
-    {
-        if (string.IsNullOrWhiteSpace(str))
-            return false;
-        
-        if (str.StartsWith('\"'))
-            return false;
-
-        var isAllDigit = true; 
-
-        foreach (var c in str)
-        {
-            if (!char.IsAsciiHexDigit(c))
-                return false;
-            if (!char.IsNumber(c))
-                isAllDigit = false;
-        }
-
-        if (isAllDigit) //避免全数字
-            return false;
-
-        return true;
-    }
-    
-    /// <summary>
-    /// 将输入流以ZSTD标准解压为字符串
-    /// </summary>
-    /// <param name="stream"></param>
-    /// <returns></returns>
-    public static async Task<string> DecompressZstAsStringAsync(Stream stream)
-    {
-        await using var ds = new DecompressionStream(stream);
-        var reader = new StreamReader(ds);
-        return await reader.ReadToEndAsync();
-    }
-
-    /// <summary>
-    /// 适用于文件输入服务以及MyDumper Zst导出目录的文件元数据构建函数
-    /// </summary>
-    /// <param name="filePath"></param>
-    /// <returns></returns>
-    /// <exception cref="ApplicationException"></exception>
-    public static FileInputInfo? MyDumperFileInputMetaBuilder(string filePath)
-    {
-        // 只查找后缀为.dat.zst的文件
-        if (!filePath.EndsWith(".dat.zst")) return null;
-            
-        var fileMeta = ParseMyDumperFile(filePath);
-        var inputDir = Path.GetDirectoryName(filePath);
-        string[]? headers;
-        try
-        {
-            // 查找同目录下同表的SQL文件
-            var sqlFile = Directory.GetFiles(inputDir!)
-                .SingleOrDefault(f => f.Equals(filePath.Replace(".dat.zst", ".sql.zst")));
-            if (sqlFile is null)
-            {
-                Log.Debug("{TableName}表的SQL文件不存在", fileMeta.TableName);
-                return null;
-            }
-            headers = GetCsvHeadersFromSqlFile(
-                DecompressZstAsStringAsync(File.OpenRead(sqlFile)).Result);
-        }
-        catch (InvalidOperationException e)
-        {
-            throw new ApplicationException($"目录下不止一个{fileMeta.TableName}表的SQL文件", e);
-        }
-
-        return new FileInputInfo
-        {
-            FileName = filePath,
-            TableName = fileMeta.TableName,
-            Headers = headers,
-            Database = fileMeta.Database,
-            Part = fileMeta.Index
-        };
-    }
-}

MesETL.App/HostedServices/Abstractions/IDataReader.cs

@@ -1,8 +0,0 @@
-namespace MesETL.App.HostedServices.Abstractions
-{
-    public interface IDataReader : IDisposable
-    {
-        DataRecord Current { get; }
-        ValueTask<bool> ReadAsync();
-    }
-}

MesETL.App/HostedServices/Abstractions/IInputService.cs

@@ -1,6 +0,0 @@
-namespace MesETL.App.HostedServices.Abstractions;
-
-public interface IInputService
-{
-    public Task ExecuteAsync(CancellationToken cancellationToken);
-}

MesETL.App/HostedServices/Abstractions/IOutputService.cs

@@ -1,6 +0,0 @@
-namespace MesETL.App.HostedServices.Abstractions;
-
-public interface IOutputService
-{
-    public Task ExecuteAsync(CancellationToken ct);
-}

MesETL.App/HostedServices/Abstractions/ITransformService.cs

@@ -1,6 +0,0 @@
-namespace MesETL.App.HostedServices.Abstractions;
-
-public interface ITransformService
-{
-    public Task ExecuteAsync(CancellationToken cancellationToken);
-}

MesETL.App/HostedServices/FileInputService.cs

@@ -1,159 +0,0 @@
-using System.Runtime;
-using MesETL.App.Const;
-using MesETL.App.HostedServices.Abstractions;
-using MesETL.App.Options;
-using MesETL.App.Services;
-using MesETL.App.Services.ETL;
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Options;
-
-namespace MesETL.App.HostedServices;
-
-public record FileInputInfo
-{
-    public required string FileName { get; init; }
-    public required string TableName { get; init; }
-    public required string Database { get; init; }
-    public required int Part { get; init; }
-    public required string[] Headers { get; init; }
-}
-
-/// <summary>
-/// 从输入目录中导入文件
-/// </summary>
-public class FileInputService : IInputService
-{
-    private readonly ILogger _logger;
-    private readonly DataRecordQueue _producerQueue;
-    private readonly IOptions<DataInputOptions> _dataInputOptions;
-    private readonly ProcessContext _context;
-    private readonly DataReaderFactory _dataReaderFactory;
-    private readonly long _memoryThreshold;
-    private readonly bool _dryRun;
-    private readonly int _dryRunCount;
-
-    public FileInputService(ILogger<FileInputService> logger,
-            IOptions<DataInputOptions> dataInputOptions,
-            ProcessContext context, 
-            [FromKeyedServices(Const.ConstVar.Producer)] DataRecordQueue producerQueue, 
-            DataReaderFactory dataReaderFactory, 
-            IConfiguration configuration)
-    {
-        _logger = logger;
-        _dataInputOptions = dataInputOptions;
-        _context = context;
-        _producerQueue = producerQueue;
-        _dataReaderFactory = dataReaderFactory;
-        _memoryThreshold = (long)(configuration.GetValue<double>("MemoryThreshold", 8) * 1024 * 1024 * 1024);
-        _dryRun = configuration.GetValue("DryRun", false);
-        _dryRunCount = configuration.GetValue("DryRunCount", 100_000);
-    }
-
-    public async Task ExecuteAsync(CancellationToken cancellationToken)
-    {
-        var inputDir = _dataInputOptions.Value.InputDir ?? throw new ApplicationException("未配置文件输入目录");
-        _logger.LogInformation("***** 输入服务已启动，工作目录为：{InputDir} *****", inputDir);
-        if (_dryRun)
-            _logger.LogInformation("***** 试运行模式已开启，只读取前 {Count} 行数据 *****", _dryRunCount);
-
-        var orderedInfo = GetOrderedInputInfo(inputDir);
-        
-        foreach (var info in orderedInfo)
-        {
-            var file = Path.GetFileName(info.FileName);
-            _logger.LogInformation("正在读取文件：{FileName}, 对应的数据表：{TableName}", file, info.TableName);
-            using var source = _dataReaderFactory.CreateReader(info.FileName, info.TableName, info.Headers);
-            var countBuffer = 0;
-            
-            if (_dryRun && _context.TableProgress.GetValueOrDefault(info.TableName, (input: 0, output: 0)).input >= _dryRunCount)
-                continue;
-            
-            while (await source.ReadAsync())
-            {
-                if (GC.GetTotalMemory(false) > _memoryThreshold)
-                {
-                    _logger.LogWarning("内存使用率过高，暂缓输入");
-                    GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
-                    GC.Collect();
-                    await Task.Delay(3000, cancellationToken);
-                }
-                var record = source.Current;
-                await _producerQueue.EnqueueAsync(record);
-                countBuffer++;
-                _context.AddInput();
-                
-                // 避免影响性能，每1000条更新一次表输入进度
-                if (countBuffer >= 1000)
-                {
-                    _context.AddTableInput(info.TableName, countBuffer);
-                    countBuffer = 0;
-                    // 试运行模式下，超出了指定行数则停止输入
-                    if (_dryRun && _context.TableProgress[info.TableName].input >= _dryRunCount)
-                    {
-                        break;
-                    }
-                }
-            }
-
-            _context.AddTableInput(info.TableName, countBuffer);
-            _logger.LogInformation("文件 {File} 输入完成", file);
-            _dataInputOptions.Value.OnTableInputCompleted?.Invoke(info.TableName);
-        }
-
-        _context.CompleteInput();
-        _logger.LogInformation("***** 输入服务{DryRun}已执行完毕 *****", _dryRun ? " (试运行)" : "");
-    }
-
-    public IEnumerable<FileInputInfo> GetOrderedInputInfo(string dir)
-    {
-        var metaBuilder = _dataInputOptions.Value.FileInputMetaBuilder;
-        if(metaBuilder is null) throw new ApplicationException("未配置文件名->表名的映射委托函数");
-        var files = Directory.GetFiles(dir);
-        FileInputInfo[] infoArr = files
-            .Select(f => metaBuilder(f))
-            .Where(info => info is not null).ToArray()!;
-        
-        var orderedInfo = GetFilesInOrder(infoArr).ToArray();
-        
-        _logger.LogInformation("***** 输入目录中发现 {Count} 个文件， {InfoCount} 个文件可用，{OrderedCount} 个文件符合当前输入配置 *****", 
-            files.Length, infoArr.Length, orderedInfo.Length);
-        foreach (var info in orderedInfo.GroupBy(i => i.TableName))
-        {
-            _logger.LogDebug("表 {TableName} 发现 {FileCount} 个对应文件：\n{FileName}", 
-                info.Key, info.Count(), string.Join('\n', info.Select(f => f.FileName)));
-        }
-
-        return orderedInfo;
-    }
-
-    /// <summary>
-    /// 读取配置，按照配置的表顺序来返回
-    /// </summary>
-    /// <returns></returns>
-    private IEnumerable<FileInputInfo> GetFilesInOrder(FileInputInfo[] inputFiles)
-    {
-        var tableOrder = _dataInputOptions.Value.TableOrder ?? typeof(TableNames).GetFields().Select(f => f.GetValue(null) as string).ToArray();
-        var ignoreTable = _dataInputOptions.Value.TableIgnoreList;
-        if (tableOrder is null or { Length: 0 })
-            return inputFiles;
-
-        return Yield();
-
-        IEnumerable<FileInputInfo> Yield()
-        {
-            foreach (var tableName in tableOrder)
-            {
-                var targets = inputFiles.Where(f =>
-                    f.TableName.Equals(tableName, StringComparison.OrdinalIgnoreCase) &&
-                    !ignoreTable.Contains(f.TableName));
-
-                foreach (var target in targets)
-                {
-                    yield return target;
-                }
-            }
-        }
-    }
-}

MesETL.App/HostedServices/MainHostedService.cs

@@ -1,206 +0,0 @@
-using System.Diagnostics;
-using System.Text;
-using MesETL.App.Helpers;
-using MesETL.App.HostedServices.Abstractions;
-using MesETL.App.Options;
-using MesETL.App.Services;
-using MesETL.App.Services.ErrorRecorder;
-using MesETL.Shared.Helper;
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.Hosting;
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Options;
-using MySqlConnector;
-
-namespace MesETL.App.HostedServices;
-
-public class MainHostedService : BackgroundService
-{
-    private Stopwatch? _stopwatch;
-    private readonly IInputService _input;
-    private readonly ITransformService _transform;
-    private readonly IOutputService _output;
-    private readonly TaskMonitorService _taskMonitor;
-    private readonly ILogger _logger;
-    private readonly ProcessContext _context;
-    
-    private readonly IOptions<DatabaseOutputOptions> _databaseOptions;
-    private readonly IOptions<TenantDbOptions> _tenantDbOptions;
-    private readonly IConfiguration _config;
-
-    public MainHostedService(IInputService input, 
-        ITransformService transform,
-        IOutputService output, 
-        ILogger<MainHostedService> logger, 
-        IOptions<TenantDbOptions> tenantDbOptions, 
-        IOptions<DatabaseOutputOptions> databaseOptions, 
-        IConfiguration config, 
-        ProcessContext context, 
-        TaskMonitorService taskMonitor)
-    {
-        _input = input;
-        _transform = transform;
-        _output = output;
-        _logger = logger;
-        _tenantDbOptions = tenantDbOptions;
-        _databaseOptions = databaseOptions;
-        _config = config;
-        _context = context;
-        _taskMonitor = taskMonitor;
-    }
-
-    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
-    {
-        var command = _config["Command"];
-        if (!string.IsNullOrEmpty(command))
-        {
-            _logger.LogInformation("检测到命令参数传入，将对所有配置的数据库执行输入的命令。。。");
-            _logger.LogInformation("***** 执行SQL命令 *****");
-            await ExecuteEachDatabase(command, stoppingToken);
-            _logger.LogInformation("***** 执行完成 *****");
-            Environment.Exit(0);
-        }
-        
-        _stopwatch = Stopwatch.StartNew();
-        var enableUnsafeVar = _config.GetValue<bool>("UnsafeVariable", false);
-        if (enableUnsafeVar)
-            await SetVariableAsync(); // 开启延迟写入，禁用重做日志 >>> 重做日志处于禁用状态时不要关闭数据库服务！
-
-        var monitorTask = Task.Run(async () => await _taskMonitor.Monitor(stoppingToken), stoppingToken);
-        var inputTask = ExecuteAndCatch(
-            async () => await _input.ExecuteAsync(stoppingToken), "文件输入程序出现异常", stoppingToken);
-        var transformTask = ExecuteAndCatch(
-            async () => await _transform.ExecuteAsync(stoppingToken), "转换程序出现异常", stoppingToken);
-        var outputTask = ExecuteAndCatch(
-            async () => await _output.ExecuteAsync(stoppingToken), "输出程序出现异常", stoppingToken);
-
-        await Task.WhenAll(inputTask, transformTask, outputTask);
-        _stopwatch.Stop();
-        _logger.LogInformation("***** 所有传输任务均已完成 *****");
-        if (_context.HasException)
-            _logger.LogError("***** 传输过程中有错误发生 *****");
-        _logger.LogInformation("***** 耗时：{Time}", (_stopwatch.ElapsedMilliseconds / 1000f).ToString("F3"));
-        await Task.Delay(5000, stoppingToken);
-
-        if(enableUnsafeVar)
-            await SetVariableAsync(false); // 关闭延迟写入，开启重做日志
-        if (!stoppingToken.IsCancellationRequested)
-        {
-            await ExportResultAsync();
-            _logger.LogInformation("传输结果已保存至 {Path}",
-                Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"Result-{ErrorRecorder.UID}.md"));
-
-            Environment.Exit(0);
-        }
-        else Environment.Exit(1);
-    }
-    
-    private Task ExecuteAndCatch(Func<Task> func, string message, CancellationToken ct)
-    {
-        return Task.Run(async () =>
-        {
-            try
-            {
-                await func();
-            }
-            catch (Exception e)
-            {
-                _logger.LogCritical(e, "{Msg}\t{ErrMsg}", message, e.Message);
-                _context.AddException(e);
-                Environment.Exit(1);
-            }
-        }, ct);
-    }
-
-    private async Task SetVariableAsync(bool enable = true)
-    {
-        var connStr = _databaseOptions.Value.ConnectionString
-                      ?? throw new ApplicationException("分库配置中没有配置数据库");
-        if (enable)
-            _logger.LogWarning("已开启MySQL延迟写入功能并禁用重做日志，请注意数据安全");
-        else _logger.LogInformation("不安全变量已关闭");
-        
-        if (enable)
-        {
-            await DatabaseHelper.NonQueryAsync(connStr,
-                """
-                SET GLOBAL innodb_flush_log_at_trx_commit = 0;
-                ALTER INSTANCE DISABLE INNODB REDO_LOG;
-                """);
-        }
-        else
-        {
-            await DatabaseHelper.NonQueryAsync(connStr,
-                """
-                SET GLOBAL innodb_flush_log_at_trx_commit = 1;
-                ALTER INSTANCE ENABLE INNODB REDO_LOG;
-                """);
-        }
-    }
-
-    private async Task ExecuteEachDatabase(string command, CancellationToken cancellationToken = default)
-    {
-        var databases = _tenantDbOptions.Value.DbGroup?.Keys 
-                        ?? throw new ApplicationException("分库配置中没有配置数据库");
-        var list = new List<Task>();
-        foreach (var db in databases)
-        {
-            var connStr = new MySqlConnectionStringBuilder(_databaseOptions.Value.ConnectionString
-                                                           ?? throw new ApplicationException("没有配置数据库连接字符串"))
-            {
-                ConnectionTimeout = 60,
-                DefaultCommandTimeout = 0,
-                Database = db
-            }.ConnectionString;
-            var task = Task.Run(async () => await DatabaseHelper.NonQueryAsync(connStr, command),
-                cancellationToken);
-            list.Add(task);
-        }
-
-        await Task.WhenAll(list);
-    }
-
-    private async Task ExportResultAsync()
-    {
-        var sb = new StringBuilder();
-
-        var title = (_config.GetValue("DryRun", false), _context.HasException) switch
-        {
-            (true, true) => "# 试运行结束，**请注意处理异常**",
-            (true, false) => "# 试运行结束，没有发生异常",
-            (false, true) => "# 程序执行完毕，**但中途发生了异常**",
-            (false, false) => "# 程序执行完毕，没有发生错误"
-        };
-        sb.AppendLine(title);
-          
-        sb.AppendLine("## 处理计数");
-        var processCount = new[]
-        {
-            new { 操作 = "输入", 数量 = _context.InputCount },
-            new { 操作 = "转换", 数量 = _context.TransformCount },
-            new { 操作 = "输出", 数量 = _context.OutputCount }
-        };
-        sb.AppendLine(processCount.ToMarkdownTable());
-        sb.AppendLine("\n---\n");
-        sb.AppendLine("## 表输入/输出计数");
-        var tableOutputProgress = _context.TableProgress.Select(pair =>
-            new { 表名 = pair.Key, 计数 = pair.Value }).OrderBy(s => s.表名);
-        sb.AppendLine(tableOutputProgress.ToMarkdownTable());
-        sb.AppendLine("\n---\n");
-        sb.AppendLine("## 总览");
-        var elapsedTime = (_stopwatch!.ElapsedMilliseconds / 1000f);
-        var result = new[]
-        {
-            new { 条目 = "耗时", 值 = elapsedTime.ToString("F2") + " 秒" },
-            new
-            {
-                条目 = "平均处理速度",
-                值 = (_context.OutputCount / elapsedTime).ToString("F2") + " 条记录/秒"
-            },
-            new { 条目 = "内存占用峰值", 值 = _context.MaxMemoryUsage + " 兆字节" }
-        };
-        sb.AppendLine(result.ToMarkdownTable());
-        await File.WriteAllTextAsync(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"Result-{ErrorRecorder.UID}.md"),
-            sb.ToString());
-    }
-}

MesETL.App/HostedServices/OutputService.cs

@@ -1,169 +0,0 @@
-using System.Buffers;
-using System.Text;
-using MesETL.App.Const;
-using MesETL.App.Helpers;
-using MesETL.App.HostedServices.Abstractions;
-using MesETL.App.Options;
-using MesETL.App.Services;
-using MesETL.App.Services.ErrorRecorder;
-using MesETL.Shared.Helper;
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Options;
-using MySqlConnector;
-using MySqlDestination = MesETL.App.Services.ETL.MySqlDestination;
-using TaskExtensions = MesETL.Shared.Helper.TaskExtensions;
-
-namespace MesETL.App.HostedServices;
-
-public record DataOutputContext(IServiceProvider Serivces);
-
-/// <summary>
-/// 数据导出服务，将数据导出至MySql服务
-/// </summary>
-public class OutputService : IOutputService
-{
-    private readonly ILogger _logger;
-    private readonly IOptions<DatabaseOutputOptions> _outputOptions;
-    private readonly ProcessContext _context;
-    private readonly ErrorRecorderFactory _errorRecorderFactory;
-    private readonly RecordQueuePool _queuePool;
-    private readonly IServiceProvider _services;
-
-    public OutputService(ILogger<OutputService> logger,
-        IOptions<DatabaseOutputOptions> outputOptions,
-        ProcessContext context,
-        RecordQueuePool queuePool, 
-        ErrorRecorderFactory errorRecorderFactory, 
-        IServiceProvider services)
-    {
-        _logger = logger;
-        _outputOptions = outputOptions;
-        _context = context;
-        _queuePool = queuePool;
-        _errorRecorderFactory = errorRecorderFactory;
-        _services = services;
-    }
-
-    public async Task ExecuteAsync(CancellationToken ct)
-    {
-        _logger.LogInformation("***** 输出服务已启动 *****");
-        var dbTaskManager = new TaskManager(_queuePool.Queues.Count);
-        var dbTasks = new Dictionary<string, Task>();
-        while (!_context.IsTransformCompleted)
-        {
-            foreach (var (db, queue) in _queuePool.Queues)
-            {
-                if (!dbTasks.ContainsKey(db))
-                {
-                    dbTasks.Add(db, await dbTaskManager.CreateTaskAsync(
-                        async () =>
-                        {
-                            try
-                            {
-                                await StartDatabaseWorker(db, queue, ct);
-                            }
-                            catch (Exception e)
-                            {
-                                _logger.LogError(e, "输出线程发生错误");
-                                _queuePool.RemoveQueue(db);
-                            }
-                        }, ct));
-                }
-            }
-
-            await Task.Delay(500, ct);
-        }
-
-        await TaskExtensions.WaitUntil(() => dbTaskManager.RunningTaskCount == 0, 25, ct);
-        
-        _context.CompleteOutput();
-        _outputOptions.Value.OutputFinished?.Invoke(new DataOutputContext(_services));
-        _logger.LogInformation("***** 输出服务执行完毕 *****");
-    }
-
-    private async Task StartDatabaseWorker(string db, DataRecordQueue queue, CancellationToken ct = default)
-    {
-        _logger.LogInformation("*****开启输出线程，数据库: {db} *****", db);
-        var taskManager = new TaskManager(_outputOptions.Value.MaxDatabaseOutputTask);
-        var ignoreOutput = new HashSet<string>(_outputOptions.Value.NoOutput);
-        var tmp = new List<DataRecord>(_outputOptions.Value.FlushCount);
-        while (!_context.IsTransformCompleted || queue.Count > 0)
-        {
-            if (ct.IsCancellationRequested)
-                break;
-
-            if (!queue.TryDequeue(out var record) || record.Ignore || ignoreOutput.Contains(record.TableName))
-                continue;
-            
-            var dbName = record.Database ?? throw new ApplicationException("输出的记录缺少数据库名");
-            if(dbName != db)
-                throw new ApplicationException($"输出记录的数据与当前输出线程不匹配，记录：{dbName}, 输出线程：{db}");
-            tmp.Add(record);
-
-            if (tmp.Count >= _outputOptions.Value.FlushCount)
-            {
-                var list = tmp;
-                tmp = [];
-                await taskManager.CreateTaskAsync(async arg => // 转换为方法组
-                {
-                    var tuple = arg as Tuple<string, List<DataRecord>>;
-                    try
-                    {
-                        await FlushAsync(tuple!.Item1, tuple.Item2);
-                    }
-                    catch (Exception e)
-                    {
-                        _logger.LogError(e, "输出记录时发生错误");
-                        throw;
-                    }
-                }, Tuple.Create(dbName, list), ct);
-            }
-        }
-
-        // 等待所有子任务完成
-        await TaskExtensions.WaitUntil(() => taskManager.RunningTaskCount == 0, 10, ct);
-        
-        _logger.LogDebug("输出线程结束，清理剩余记录[{Count}]", tmp.Count);
-        
-        // 清理剩余记录
-        if (tmp.Count > 0)
-        {
-            await FlushAsync(db, tmp);
-        }
-
-        _logger.LogInformation("***** 输出线程结束，数据库: {db} *****", db);
-    }
-
-    private async Task FlushAsync(string dbName, IEnumerable<DataRecord> records)
-    {
-        var connStr = new MySqlConnectionStringBuilder(_outputOptions.Value.ConnectionString 
-                                                       ?? throw new ApplicationException("未配置数据库连接字符串"))
-            {
-                CharacterSet = "utf8mb4",
-                AllowUserVariables = true,
-                IgnoreCommandTransaction = true,
-                TreatTinyAsBoolean = false,
-                ConnectionTimeout = 60,
-                DefaultCommandTimeout = 0,
-                SslMode = MySqlSslMode.None,
-                Database = dbName
-            }.ConnectionString;
-        await using var output = new MySqlDestination(connStr, _logger,
-            _outputOptions, _errorRecorderFactory.CreateOutput(dbName), _context);
-
-        var tableOutput = new Dictionary<string, int>();
-        foreach (var record in records)
-        {
-            await output.WriteRecordAsync(record);
-            tableOutput.AddOrUpdate(record.TableName, 1, (_, v) => v + 1);
-        }
-
-        await output.FlushAsync(_outputOptions.Value.MaxAllowedPacket);
-        foreach (var (key, value) in tableOutput)
-        {
-            _context.AddOutput(value);
-            _context.AddTableOutput(key, value);
-        }
-        // _logger.LogTrace("输出任务：刷新了 {Count} 条记录", tableOutput.Values.Sum(i => i));
-    }
-}

MesETL.App/HostedServices/TaskMonitorService.cs

@@ -1,140 +0,0 @@
-using System.Diagnostics;
-using System.Text;
-using MesETL.App.Services;
-using MesETL.App.Services.Loggers;
-using Microsoft.Extensions.DependencyInjection;
-
-namespace MesETL.App.HostedServices;
-
-/// <summary>
-/// 任务监控
-/// </summary>
-public class TaskMonitorService
-{
-    private readonly IEnumerable<ITaskMonitorLogger> _monitorLoggers;
-    private readonly ProcessContext _context;
-    private readonly DataRecordQueue _producerQueue;
-    private readonly RecordQueuePool _queuePool;
-    
-    private string _outputPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Log/progress.txt");
-
-    public TaskMonitorService(ProcessContext context,
-        [FromKeyedServices(Const.ConstVar.Producer)]
-        DataRecordQueue producerQueue,
-        RecordQueuePool queuePool,
-        IEnumerable<ITaskMonitorLogger> monitorLoggers)
-    {
-        _context = context;
-        _producerQueue = producerQueue;
-        _queuePool = queuePool;
-        _monitorLoggers = monitorLoggers;
-    }
-
-    public async Task Monitor(CancellationToken stoppingToken)
-    {
-        var sw = Stopwatch.StartNew();
-        var lastTime = sw.ElapsedMilliseconds;
-        var lastInputCount = _context.InputCount;
-        var lastTransformCount = _context.TransformCount;
-        var lastOutputCount = _context.OutputCount;
-
-        bool endCheck = false;
-        while (!stoppingToken.IsCancellationRequested)
-        {
-            EndCheck:
-            // var running = 0;
-            // var error = 0;
-            // var completed = 0;
-            // var canceled = 0;
-            // foreach (var task in _taskManager.Tasks)
-            // {
-            //     switch (task.Status)
-            //     {
-            //         case TaskStatus.Canceled:
-            //             canceled++;
-            //             break;
-            //         case TaskStatus.Faulted:
-            //             error++;
-            //             break;
-            //         case TaskStatus.RanToCompletion:
-            //             completed++;
-            //             break;
-            //         default:
-            //             running++;
-            //             break;
-            //     }
-            // }
-
-            var time = sw.ElapsedMilliseconds;
-            var inputCount = _context.InputCount;
-            var transformCount = _context.TransformCount;
-            var outputCount = _context.OutputCount;
-            
-            var elapseTime = (time - lastTime) / 1000f;
-            var inputSpeed = (inputCount - lastInputCount) / elapseTime;
-            var transformSpeed = (transformCount - lastTransformCount) / elapseTime;
-            var outputSpeed = (outputCount - lastOutputCount) / elapseTime;
-            
-            // _logger.LogInformation(
-            //     "Task monitor: running: {Running}, error: {Error}, completed: {Completed}, canceled: {Canceled}, outputSpeed: {Speed} records/s",
-            //     running, error, completed, canceled, outputSpeed);
-            foreach (var logger in _monitorLoggers)
-            {
-                var memory = GC.GetTotalMemory(false) / 1024 / 1024;
-                _context.MaxMemoryUsage = Math.Max(_context.MaxMemoryUsage, memory);
-                logger.LogStatus("系统监控", new Dictionary<string, string>
-                {
-                    { "输入速度", _context.IsInputCompleted ? "OK" : $"{inputSpeed:F2}/s" },
-                    { "转换速度", _context.IsTransformCompleted ? "OK" : $"{transformSpeed:F2}/s" },
-                    { "输出速度", _context.IsOutputCompleted ? "OK" : $"{outputSpeed:F2}/s" },
-
-                    { "| 输入队列长度", _producerQueue.Count.ToString() },
-                    { "输出队列长度", _queuePool.Queues.Values.Sum(queue => queue.Count).ToString() },
-                    { "内存使用", $"{memory} MiB" },
-                });
-
-                var dict = _context.TableProgress
-                    .ToDictionary(kv => kv.Key, kv => $"{kv.Value.input}:{kv.Value.output}");
-                logger.LogStatus("系统监控: 表处理进度(I:O)", dict, ITaskMonitorLogger.LogLevel.Progress);
-                logger.LogStatus("系统监控：输出队列状态", 
-                    _queuePool.Queues.ToDictionary(q => q.Key, q => q.Value.Count.ToString()), 
-                    ITaskMonitorLogger.LogLevel.Progress);
-                var sb = new StringBuilder("表处理进度：\n");
-                foreach (var kv in dict)
-                {
-                    sb.Append(kv.Key).AppendLine(kv.Value);
-                }
-
-                sb.AppendLine($"数据记录字段的最大长度：{_producerQueue.LongestFieldCharCount}");
-                await File.WriteAllTextAsync(_outputPath, sb.ToString(), CancellationToken.None);
-                
-            
-                // logger.LogStatus("Monitor: Process count", new Dictionary<string, string>
-                // {
-                //     {"Input", inputCount.ToString()},
-                //     {"Transform", transformCount.ToString()},
-                //     {"Output", outputCount.ToString()}
-                // }, ITaskMonitorLogger.LogLevel.Progress);
-            }
-
-            
-            await Task.Delay(5000, stoppingToken);
-
-            lastTime = time;
-            lastInputCount = inputCount;
-            lastTransformCount = transformCount;
-            lastOutputCount = outputCount;
-
-            if (_context is { IsInputCompleted: true, IsTransformCompleted: true, IsOutputCompleted: true })
-            {
-                if (!endCheck)
-                {
-                    endCheck = true;
-                    goto EndCheck;
-                }
-                break;
-            }
-        }
-    }
-
-}

MesETL.App/HostedServices/TransformService.cs

@@ -1,201 +0,0 @@
-using MesETL.App.Cache;
-using MesETL.App.Const;
-using MesETL.App.HostedServices.Abstractions;
-using MesETL.App.Options;
-using MesETL.App.Services;
-using MesETL.App.Services.ErrorRecorder;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Options;
-
-namespace MesETL.App.HostedServices;
-
-public record DataTransformContext(DataRecord Record, ICacher Cacher, ILogger Logger, IServiceProvider Services);
-
-/// <summary>
-/// 数据处理服务，对导入后的数据进行处理
-/// </summary>
-public class TransformService : ITransformService
-{
-    private readonly ILogger _logger;
-    private readonly IOptions<DataTransformOptions> _options;
-    private readonly DataRecordQueue _producerQueue;
-    private readonly RecordQueuePool _queuePool;
-    private readonly ProcessContext _context;
-    private readonly ICacher _cache;
-    private readonly ErrorRecorderFactory _errorRecorderFactory;
-    private readonly IServiceProvider _services;
-
-
-    public TransformService(ILogger<TransformService> logger, 
-        IOptions<DataTransformOptions> options,
-        [FromKeyedServices(Const.ConstVar.Producer)] DataRecordQueue producerQueue,
-        RecordQueuePool queuePool,
-        ProcessContext context,
-        ICacher cache, 
-        ErrorRecorderFactory errorRecorderFactory,
-        IServiceProvider services)
-    {
-        _logger = logger;
-        _options = options;
-        _producerQueue = producerQueue;
-        _queuePool = queuePool;
-        _context = context;
-        _cache = cache;
-        _errorRecorderFactory = errorRecorderFactory;
-        _services = services;
-    }
-
-    public async Task ExecuteAsync(CancellationToken cancellationToken)
-    {
-        _logger.LogInformation("***** 数据转换服务已启动 *****");
-        
-        await TransformWorker2();
-        
-        _context.CompleteTransform();
-        _logger.LogInformation("***** 数据转换服务执行完毕 *****");
-    }
-
-    public async Task TransformWorker(DataRecordQueue queue)
-    {
-        while (!_context.IsInputCompleted || _producerQueue.Count > 0)
-        {
-            if (!_producerQueue.TryDequeue(out var record))
-            {
-                await Task.Delay(100);
-                continue;
-            }
-            
-            try
-            {
-                var context = new DataTransformContext(record, _cache, _logger, _services);
-                if (_options.Value.EnableFilter)
-                {
-                    // 数据过滤
-                    var filter = _options.Value.RecordFilter;
-                    if (filter is not null && await filter(context) == false) continue;
-                }
-
-                if (_options.Value.EnableReplacer)
-                {
-                    // 数据替换
-                    var replacer = _options.Value.RecordModify;
-                    if (replacer is not null)
-                    {
-                        record = await replacer(context);
-                    }
-                }
-                
-                // 字段缓存
-                var cacher = _options.Value.RecordCache;
-                if(cacher is not null)
-                    await cacher.Invoke(context);
-            
-                //计算需要分流的数据库
-                var dbFilter = _options.Value.DatabaseFilter 
-                               ?? throw new ApplicationException("未配置数据库过滤器");
-                record.Database = dbFilter(record);
-
-                if (_options.Value.EnableReBuilder)
-                {
-                    //数据重建
-                    var addRecords = _options.Value.RecordReBuild?.Invoke(context);
-                    if (addRecords is { Count: > 0 })
-                    {
-                        foreach (var rc in addRecords)
-                        {
-                            if(dbFilter is not null)
-                                rc.Database = dbFilter.Invoke(record);
-                            await queue.EnqueueAsync(rc);
-                            _context.AddTransform();
-                        }
-                    }
-                }
-
-                await queue.EnqueueAsync(record);
-                _context.AddTransform();
-            }
-            catch (Exception e)
-            {
-                _context.AddException(e);
-                var errorRecorder = _errorRecorderFactory.CreateTransform();
-                await errorRecorder.LogErrorRecordAsync(record, e);
-                if (!_options.Value.StrictMode)
-                    _logger.LogError(e, "数据转换时发生错误");
-                else throw;
-            }
-        }
-    }
-    
-    public async Task TransformWorker2()
-    {
-        while (!_context.IsInputCompleted || _producerQueue.Count > 0)
-        {
-            if (!_producerQueue.TryDequeue(out var record))
-            {
-                await Task.Delay(100);
-                continue;
-            }
-            
-            try
-            {
-                var context = new DataTransformContext(record, _cache, _logger, _services);
-                if (_options.Value.EnableFilter)
-                {
-                    // 数据过滤
-                    var filter = _options.Value.RecordFilter;
-                    if (filter is not null && await filter(context) == false) continue;
-                }
-
-                if (_options.Value.EnableReplacer)
-                {
-                    // 数据替换
-                    var replacer = _options.Value.RecordModify;
-                    if (replacer is not null)
-                    {
-                        record = await replacer(context);
-                    }
-                }
-                
-                // 字段缓存
-                var cacher = _options.Value.RecordCache;
-                if(cacher is not null)
-                    await cacher.Invoke(context);
-            
-                //计算需要分流的数据库
-                var dbFilter = _options.Value.DatabaseFilter 
-                               ?? throw new ApplicationException("未配置数据库过滤器");
-                record.Database = dbFilter(record);
-
-                if (_options.Value.EnableReBuilder)
-                {
-                    //数据重建
-                    var addRecords = _options.Value.RecordReBuild?.Invoke(context);
-                    if (addRecords is { Count: > 0 })
-                    {
-                        foreach (var rc in addRecords)
-                        {
-                            if(dbFilter is not null)
-                                rc.Database = dbFilter.Invoke(record);
-                            await _queuePool[record.Database].EnqueueAsync(rc);
-                            _context.AddTransform();
-                        }
-                    }
-                }
-
-                await _queuePool[record.Database].EnqueueAsync(record);
-                _context.AddTransform();
-            }
-            catch (Exception e)
-            {
-                _context.AddException(e);
-                var errorRecorder = _errorRecorderFactory.CreateTransform();
-                await errorRecorder.LogErrorRecordAsync(record, e);
-                if (!_options.Value.StrictMode)
-                    _logger.LogError(e, "数据转换时发生错误");
-                else throw;
-            }
-        }
-        _context.CompleteTransform();
-    }
-}

MesETL.App/HostedServices/VoidOutputService.cs

@@ -1,51 +0,0 @@
-using System.Numerics;
-using MesETL.App.HostedServices.Abstractions;
-using MesETL.App.Services;
-using Microsoft.Extensions.Logging;
-
-namespace MesETL.App.HostedServices;
-
-// 空输出服务，测试用
-public class VoidOutputService : IOutputService
-{
-    private readonly ILogger _logger;
-    private readonly RecordQueuePool _queuePool;
-    private readonly ProcessContext _context;
-
-    private BigInteger _total;
-
-    public VoidOutputService(
-        ProcessContext context, ILogger<VoidOutputService> logger, RecordQueuePool queuePool)
-    {
-        _context = context;
-        _logger = logger;
-        _queuePool = queuePool;
-    }
-
-    public Task ExecuteAsync(CancellationToken ct)
-    {
-        _logger.LogInformation("***** Void Output Service Started *****");
-        while (!_context.IsTransformCompleted || _queuePool.Queues.Count > 0)
-        {
-            foreach (var pair in _queuePool.Queues)
-            {
-                if (_context.IsTransformCompleted && pair.Value.Count == 0)
-                {
-                    _queuePool.RemoveQueue(pair.Key);
-                    continue;
-                }
-
-                if (!pair.Value.TryDequeue(out var record))
-                    continue;
-                
-                _total += record.FieldCharCount;
-                _context.AddOutput();
-            }
-        }
-
-        _context.CompleteOutput();
-        _logger.LogInformation("***** Void Output Service Stopped *****");
-        _logger.LogInformation("平均列字符数：{Number}", _total / _context.OutputCount);
-        return Task.CompletedTask;
-    }
-}

MesETL.App/Options/DataInputOptions.cs

@@ -1,72 +0,0 @@
-using MesETL.App.HostedServices;
-
-namespace MesETL.App.Options
-{
-    public class DataInputOptions
-    {
-        /// <summary>
-        /// 文件输入的目录
-        /// </summary>
-        public string? InputDir { get; set; }
-
-        #region CSV
-        
-        /// <summary>
-        /// 字符串的包围符号，默认为双引号"
-        /// </summary>
-        public char QuoteChar { get; set; } = '"';
-
-        /// <summary>
-        /// 每个字段的分割符，默认逗号,
-        /// </summary>
-        public string Delimiter { get; set; } = ",";
-
-        #endregion
-
-        #region Mock
-
-        /// <summary>
-        /// <para>生成模拟数据进行测试</para>
-        /// <para>启用后在读取数据时会截取ZST文件中的CSV文件的第一条记录，然后复制成指定数量的数据</para>
-        /// </summary>
-        public bool UseMock { get; set; }
-
-        /// <summary>
-        /// 当开启模拟数据生成时，模拟数据的倍数
-        /// </summary>
-        public double MockCountMultiplier { get; set; } = 1;
-        
-        /// <summary>
-        /// 配置每张表生成模拟数据的规则，此属性暂时在程序中配置
-        /// </summary>
-        public Dictionary<string, TableMockConfig>? TableMockConfig { get; set; }
-        
-        #endregion
-
-
-        #region Reader
-        
-        /// <summary>
-        /// <para>配置输入表及其顺序，如果为空则按照程序默认的顺序。</para>
-        /// <para>该值如果存在，程序会按照集合中表的顺序来读取数据，不在集合中的表将被忽略！</para>
-        /// </summary>
-        public string[]? TableOrder { get; set; }
-
-        /// <summary>
-        /// 忽略集合中配置的表，不进行读取
-        /// </summary>
-        public string[] TableIgnoreList { get; set; } = [];
-        
-        /// <summary>
-        /// 配置如何从文件名转换为表名和表头
-        /// </summary>
-        public Func<string, FileInputInfo?>? FileInputMetaBuilder { get; set; }
-        
-        /// <summary>
-        /// 表输入完成事件
-        /// </summary>
-        public Action<string>? OnTableInputCompleted { get; set; }
-
-        #endregion
-    }
-}

MesETL.App/Options/DataTransformOptions.cs

@@ -1,50 +0,0 @@
-using MesETL.App.HostedServices;
-
-namespace MesETL.App.Options;
-
-public enum ColumnType
-{
-    Blob,
-    Text,
-    Json,
-    UnDefine,
-}
-
-public class DataTransformOptions
-{
-    public bool StrictMode { get; set; } = true;
-    public bool EnableFilter { get; set; } = true;
-    public bool EnableReplacer { get; set; } = true;
-    public bool EnableReBuilder { get; set; } = true;
-    
-    /// <summary>
-    /// yyyyMM
-    /// </summary>
-    public string CleanDate { get; set; } = "202401";
-    
-    /// <summary>
-    /// Record -> Database name
-    /// 决定记录应当被插入到哪一个数据库中
-    /// </summary>
-    public Func<DataRecord, string>? DatabaseFilter { get; set; }
-    /// <summary>
-    /// Context -> Should output
-    /// 对记录进行过滤，返回false则不输出
-    /// </summary>
-    public Func<DataTransformContext, Task<bool>>? RecordFilter { get; set; }//数据过滤方法
-    /// <summary>
-    /// Context -> New record
-    /// 对当前记录进行修改或完整替换，你可以在这里修改记录中的字段，或者新增/删除字段
-    /// </summary>
-    public Func<DataTransformContext, Task<DataRecord>>? RecordModify { get; set; }//数据替换
-    /// <summary>
-    /// Context -> New rebuild records
-    /// 基于当前记录新增多个记录
-    /// </summary>
-    public Func<DataTransformContext, IList<DataRecord>?>? RecordReBuild { get; set; }//新增数据
-    /// <summary>
-    /// Context -> void
-    /// 对数据的某些字段进行缓存
-    /// </summary>
-    public Func<DataTransformContext, Task>? RecordCache { get; set; }//数据缓存
-}

MesETL.App/Options/DatabaseOutputOptions.cs

@@ -1,73 +0,0 @@
-using MesETL.App.HostedServices;
-
-namespace MesETL.App.Options;
-
-public class DatabaseOutputOptions
-{
-    /// <summary>
-    /// 输出数据库的连接字符串
-    /// </summary>
-    public string? ConnectionString { get; set; }
-
-    /// <summary>
-    /// MySql max_allowed_packet变量值大小
-    /// </summary>
-    public int MaxAllowedPacket { get; set; } = 32 * 1024 * 1024;
-
-    /// <summary>
-    /// 每次Insert提交的数据量
-    /// </summary>
-    public int FlushCount { get; set; } = 10000;
-
-    /// <summary>
-    /// 每个数据库最大提交任务数
-    /// </summary>
-    public int MaxDatabaseOutputTask { get; set; } = 4;
-
-    /// <summary>
-    /// 将json列作为16进制格式输出(0x前缀)
-    /// </summary>
-    public bool TreatJsonAsHex { get; set; } = true;
-
-    /// <summary>
-    /// 不对某些表进行输出
-    /// </summary>
-    public string[] NoOutput { get; set; } = [];
-    
-    /// <summary>
-    /// <para>当某张表的键出现重复时，在输出时使用ON DUPLICATE KEY UPDATE更新该条记录</para>
-    /// <para>表名为键，更新的字段为值</para>
-    /// <example>
-    /// <code>
-    /// {
-    ///     // 当order_data_parts表的键出现重复时，使用ON DUPLICATE KEY UPDATE更新已存在记录的CompanyID为新插入记录的值
-    ///     "order_data_parts": "CompanyID = new.CompanyID"
-    /// }
-    /// </code>
-    /// </example>
-    /// </summary>
-    public Dictionary<string, string>? ForUpdate { get; set; }
-
-    /// <summary>
-    /// 配置导入数据的特殊列，请在代码中配置
-    /// </summary>
-    public Dictionary<string, ColumnType> ColumnTypeConfig { get; set; } = new(); // "table.column" -> type
-    
-    /// <summary>
-    /// 所有数据都输出完毕时的事件，请在代码中配置
-    /// </summary>
-    public Action<DataOutputContext>? OutputFinished { get; set; } 
-
-    public ColumnType GetColumnType(string table, string column)
-    {
-        return ColumnTypeConfig.GetValueOrDefault(string.Concat(table, ".", column), ColumnType.UnDefine);
-    }
-
-    public bool TryGetForUpdate(string table, out string? forUpdate)
-    {
-        forUpdate = null;
-        if (ForUpdate is null || !ForUpdate.TryGetValue(table, out forUpdate))
-            return false;
-        return true;
-    }
-}

MesETL.App/Options/RedisCacheOptions.cs

@@ -1,20 +0,0 @@
-namespace MesETL.App.Options;
-
-/// <summary>
-/// Redis缓存选项
-/// </summary>
-public class RedisCacheOptions
-{
-    /// <summary>
-    /// Redis连接字符串
-    /// </summary>
-    public string? Configuration { get; init; }
-    /// <summary>
-    /// Redis实例名称
-    /// </summary>
-    public string InstanceName { get; init; } = "";
-    /// <summary>
-    /// 使用的数据库序号
-    /// </summary>
-    public int Database { get; init; } = 0;
-}

MesETL.App/Options/TableMockConfig.cs

@@ -1,35 +0,0 @@
-namespace MesETL.App.Options;
-
-/// <summary>
-/// 表模拟数据生成规则
-/// </summary>
-public struct TableMockConfig
-{
-    /// <summary>
-    /// 使用深拷贝
-    /// </summary>
-    public bool UseDeepCopy { get; set; }
-    /// <summary>
-    /// 模拟数据量
-    /// </summary>
-    public long MockCount { get; set; }
-
-    /// <summary>
-    /// 需要开启MockCount
-    /// </summary>
-    public string[]? AutoIncrementColumn { get; set; } = null; // TODO: 换为自定义委托
-    
-    public void Deconstruct(out bool useDeepCopy, out long mockCount, out string[]? autoIncrementColumn)
-    {
-        useDeepCopy = UseDeepCopy;
-        mockCount = MockCount;
-        autoIncrementColumn = AutoIncrementColumn;
-    }
-
-    public TableMockConfig(bool useDeepCopy, long mockCount, string[]? autoIncrementColumn)
-    {
-        UseDeepCopy = useDeepCopy;
-        MockCount = mockCount;
-        AutoIncrementColumn = autoIncrementColumn;
-    }
-}

MesETL.App/Options/TenantDbOptions.cs

@@ -1,44 +0,0 @@
-namespace MesETL.App.Options;
-
-/// <summary>
-/// 多租户分库配置
-/// </summary>
-public class TenantDbOptions
-{
-    public string? TenantKey { get; set; }
-    public string? UseDbGroup { get; set; }
-
-    /// <summary>
-    /// Key-Value: {DbName}-{TenantKeyLessThan}
-    /// </summary>
-    public Dictionary<string, int>? DbGroup { get; set; }
-
-    public string GetDbNameByTenantKeyValue(int tenantKeyValue)
-    {
-        // var dictionary = new SortedDictionary<int, string>();
-        // DbList.ForEach(pair => dictionary.Add(pair.Value, pair.Key));
-        // 注意配置顺序
-        if(DbGroup is null) throw new ApplicationException("分库配置中没有发现任何数据库");
-
-        #region 性能较低，不使用
-        
-        // var dbName = DbGroup.Cast<KeyValuePair<string, int>?>()
-        //     .FirstOrDefault(pair => pair?.Value != null && pair.Value.Value > tenantKeyValue)!.Value.Key;
-
-        #endregion
-        
-        string? dbName = null;
-        foreach (var (key, value) in DbGroup)
-        {
-            if (value > tenantKeyValue)
-            {
-                dbName = key;
-                break;
-            }
-        }
-        
-        return dbName ??
-               throw new ArgumentOutOfRangeException(nameof(tenantKeyValue),
-                   $"分库配置中没有任何符合'{nameof(tenantKeyValue)}'值的数据库");
-    }
-}

MesETL.App/Program.cs

@@ -1,416 +0,0 @@
-// #define FIX_PLAN_ITEM // 测试环境对OrderBlockPlanItem表进行修复时使用
-
-using System.Text;
-using MesETL.App;
-using MesETL.App.Services;
-using MesETL.App.Services.ETL;
-using MesETL.App.Cache;
-using MesETL.App.Const;
-using MesETL.App.HostedServices;
-using MesETL.App.HostedServices.Abstractions;
-using MesETL.App.Options;
-using MesETL.App.Services.ErrorRecorder;
-using MesETL.App.Services.Loggers;
-using MesETL.App.Services.Seq;
-using MesETL.Shared.Compression;
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Hosting;
-using Microsoft.Extensions.Logging;
-using Serilog;
-using Serilog.Events;
-using DumpDataHelper = MesETL.App.Helpers.DumpDataHelper;
-
-await RunProgram();
-return;
-
-async Task RunProgram()
-{
-    ThreadPool.SetMaxThreads(200, 200);
-    var host = Host.CreateApplicationBuilder(args);
-    host.Configuration.AddJsonFile(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "appsettings.json"), false, false);
-    host.Configuration.AddCommandLine(args, new Dictionary<string, string>
-    {
-        { "-d", "Input:InputDir" },
-        { "--InputDir", "Input:InputDir" },
-        { "-s", "Output:ConnectionString" },
-        { "--ConnectionString", "Output:ConnectionString" },
-        { "-r", "RedisCache:Configuration" },
-        { "--Redis", "RedisCache:Configuration" },
-        { "-g", "TenantDb:UseDbGroup" },
-        { "--UseDbGroup", "TenantDb:UseDbGroup" },
-        { "-c", "Command" },
-        { "--Command", "Command" }
-    }); 
-
-    var inputOptions = host.Configuration.GetRequiredSection("Input").Get<DataInputOptions>() 
-                       ?? throw new ApplicationException("缺少Input配置");
-
-    var transformOptions = host.Configuration.GetRequiredSection("Transform").Get<DataTransformOptions>()
-                        ?? throw new ApplicationException("缺少Transform配置");
-
-    var outputOptions = host.Configuration.GetRequiredSection("Output").Get<DatabaseOutputOptions>() 
-                        ?? throw new ApplicationException("缺少Output配置");
-
-    var redisSection = host.Configuration.GetRequiredSection("RedisCache");
-    var redisOptions = redisSection.Get<RedisCacheOptions>() ?? throw new ApplicationException("缺少RedisCache配置");
-
-    var tenantDbSection = host.Configuration.GetRequiredSection("TenantDb");
-    var tenantDbOptions = new TenantDbOptions()
-    {
-        TenantKey = tenantDbSection.GetValue<string>(nameof(TenantDbOptions.TenantKey)) ?? throw new ApplicationException("分库配置缺少分库键TenantKey"),
-        UseDbGroup = tenantDbSection.GetValue<string>(nameof(TenantDbOptions.UseDbGroup)) ?? throw new ApplicationException("分库配置缺少使用分库组UseDbGroup")
-    };
-    tenantDbOptions.DbGroup = tenantDbSection.GetRequiredSection($"DbGroups:{tenantDbOptions.UseDbGroup}").Get<Dictionary<string,int>>()
-        ?? throw new ApplicationException($"分库配置无法解析分库组{tenantDbOptions.UseDbGroup}，请检查配置");
-    
-    host.Services.Configure<TenantDbOptions>(options =>
-    {
-        options.TenantKey = tenantDbOptions.TenantKey;
-        options.UseDbGroup = tenantDbOptions.UseDbGroup;
-        options.DbGroup = tenantDbOptions.DbGroup;
-    });
-    host.Services.Configure<RedisCacheOptions>(redisSection);
-
-    var oldestTime = DateTime.ParseExact(transformOptions.CleanDate, "yyyyMM", System.Globalization.DateTimeFormatInfo.InvariantInfo);
-    var oldestTimeInt_yyyyMM = int.Parse(transformOptions.CleanDate);
-    var oldestTimeInt_yyMM = int.Parse(transformOptions.CleanDate[2..]);
-    
-    // 输入配置
-    host.Services.Configure<DataInputOptions>(options =>
-    {
-        options.InputDir = inputOptions.InputDir ?? throw new ApplicationException("未配置输入目录");
-        options.UseMock = inputOptions.UseMock;
-        options.TableMockConfig = inputOptions.TableMockConfig;
-        options.MockCountMultiplier = inputOptions.MockCountMultiplier;
-        options.TableIgnoreList = inputOptions.TableIgnoreList;
-        options.TableOrder = inputOptions.TableOrder;
-        
-        // 配置文件元数据构建方法
-        options.FileInputMetaBuilder = DumpDataHelper.MyDumperFileInputMetaBuilder;
-        
-        // 配置表输入完成事件
-        options.OnTableInputCompleted = null;
-
-        // 配置表模拟数据
-        options.TableMockConfig = new Dictionary<string, TableMockConfig>
-            { };
-    });
-
-    host.Services.Configure<DataTransformOptions>(options =>
-    {
-        static string CalculateShardKeyByOrderNo(ReadOnlySpan<char> orderNo)
-            => $"{orderNo[2..6]}0";
-        
-        options.StrictMode = transformOptions.StrictMode;
-        options.EnableFilter = transformOptions.EnableFilter;
-        options.EnableReplacer = transformOptions.EnableReplacer;
-        options.EnableReBuilder = transformOptions.EnableReBuilder;
-        
-        // order_block_plan_item和order_package_item表不导入，根据order_item数据直接重建
-        
-        // 数据清理
-        options.RecordFilter = async context => // TODO: OPT: oldestTime等外部变量会产生闭包
-        {
-            var record = context.Record;
-            switch (record.TableName)
-            {
-                // 清理CreateTime < 202401的
-                case TableNames.OrderBlockPlan:
-                {
-                    var creationTime = DateTime.Parse(record["CreateTime"].AsSpan().Trim(['"', '\'']));
-                    if (creationTime < oldestTime)
-                    {
-                        return false;
-                    }
-                    break;
-                }
-                // 忽略OrderBlockPlanItem
-                case TableNames.OrderBlockPlanItem:
-                {
-                    return false;
-                }
-                // 清理(Status != 0 || Deleted = 1) && ID前四位 < 2401的
-                case TableNames.OrderScrapBoard:
-                {
-                    var status = record["Status"].AsSpan();
-                    var deleted = record["Deleted"].AsSpan();
-                    var idPref = int.Parse(record["ID"].AsSpan()[..4]);
-                    if ((status is not "0" || deleted is "1") && idPref < oldestTimeInt_yyMM)
-                    {
-                        return false;
-                    }
-                    break;
-                }
-                // 清理OrderNo < 202401的
-                case TableNames.SimplePackage:
-                {
-                    var orderNo = int.Parse(record["OrderNo"].AsSpan()[..4]);
-                    if (orderNo < oldestTimeInt_yyMM)
-                    {
-                        return false;
-                    }
-                    break;
-                }
-                // 清理CreateTime < 202401的
-                case TableNames.SimplePlanOrder:
-                {
-                    var creationTime = DateTime.Parse(record["CreateTime"].AsSpan().Trim(['"', '\'']));
-                    if (creationTime < oldestTime)
-                    {
-                        return false;
-                    }
-                    break;
-                }
-                default: break;
-            }
-
-            return true;
-        };
-        
-        // 数据替换
-        /*
-         * 空数据处理：
-         * 某些列生产库为可空，而测试库变为了不可空，则需要根据列的类型对这些列做单独处理
-         * int或任何非无符号整型 -> -1
-         * varchar -> ''（空字符串）
-         * datetime -> '1000-01-01'（datetime最小值）
-         * text -> 0 （16进制0，MyDumper中的text是为16进制）
-         */
-        const string DefaultInt = "0";
-        const string DefaultStr = "''";
-        const string DefaultDateTime = "'1000-01-01'";
-        const string DefaultText = "0";
-        
-        options.RecordModify = async context =>
-        {
-            void ReplaceIfMyDumperNull(DataRecord record, string fieldName, string replaceValue)
-            {
-                if (record[fieldName] is ConstVar.MyDumperNull)
-                {
-                    context.Logger.LogWarning("发现不可空的字段为空({TableName}.{FieldName})，填充默认值: {DefaultValue}", 
-                        record.TableName, fieldName, replaceValue);
-                    record[fieldName] = replaceValue;
-                }
-            }
-            
-            var record = context.Record;
-            var cache = context.Cacher;
-            switch (record.TableName)
-            {
-                // 重构Data列二进制数据
-                case TableNames.OrderBoxBlock:
-                {
-                    var data = record["Data"];
-                    if (data is not ConstVar.MyDumperNull and ConstVar.Null)
-                    {
-                        var hex = Encoding.UTF8.GetString(Convert.FromHexString(data));
-                        record["Data"] = hex;
-                    }
-
-                    break;
-                }
-                // 将JsonStr列转换为Data列，添加CompressionType列
-                case TableNames.OrderModuleExtra:
-                {
-                    record.AppendField("CompressionType", "1");
-                    record.AppendField("Data",
-                        Convert.ToHexString(DeflateArchive.Compress(Convert.FromHexString(record["JsonStr"]))));
-                    record.RemoveField("JsonStr");
-                    break;
-                }
-                // 删除ID列，让数据库自行递增
-                // TODO: 数据表改进，删除ID列或是替换为流水号
-                case TableNames.ProcessStepEfficiency:
-                {
-                    record.RemoveField("ID");
-                    break;
-                }
-                case TableNames.ProcessScheduleCapacity:
-                {
-                    record.RemoveField("ID");
-                    break;
-                }
-                case TableNames.SysConfig:
-                {
-                    record.RemoveField("Key");
-                    break;
-                }
-                // 移除PlaceData列（如果存在的话，生产库已经删除）
-                case TableNames.SimplePlanOrder:
-                {
-                    if(record.HeaderExists("PlaceData"))
-                        record.RemoveField("PlaceData");
-                    break;
-                }
-                default: break;
-            }
-
-            return record;
-
-            string ThrowIfNoCached(string? cached, string tableName, string cachedTableName, string cachedColumn, string appendMessage = "")
-            {
-                if (cached is null)
-                    throw new InvalidDataException(
-                        $"{tableName}数据异常，在缓存中未找到对应{cachedTableName}.{cachedColumn}\t{appendMessage}");
-                return cached;
-            }
-        };
-        
-        // 数据缓存
-        options.RecordCache = null;
-        
-        // 数据库过滤
-        options.DatabaseFilter = record =>
-        {
-            var companyId = int.Parse(record[tenantDbOptions.TenantKey]); // 每个实体都应存在CompanyID，否则异常
-            return tenantDbOptions.GetDbNameByTenantKeyValue(companyId);
-        };
-        
-        // 数据重建
-        options.RecordReBuild = context =>
-        {
-            var record = context.Record;
-            
-            // 将OrderExtra表迁移至OrderWaveGroup表
-            if (record.TableName == TableNames.OrderExtra)
-            {
-                record.Ignore = true;
-                var resultList = new List<DataRecord>();
-                var seq = context.Services.GetRequiredService<SeqService>();
-                string[] headers = ["OrderNo", "ShardKey", "ConfigType", "ConfigJson", "CompanyID"];
-                var id = seq.AddCachedSeq(SeqConfig.OrderWaveGroupID);
-                var orderWaveGroup = new DataRecord(
-                    [id.ToString(), ..headers.Select(c => record[c])],
-                    TableNames.OrderExtraList,
-                    ["ID", "OrderNo", "ShardKey", "Type", "ConfigJson", "CompanyID"]);
-                resultList.Add(orderWaveGroup);
-                return resultList;
-            }
-            
-            // 通过OrderItem重建OrderBlockPlanItem表
-            if (record.TableName == TableNames.OrderItem)
-            {
-#if FIX_PLAN_ITEM
-                record.Ignore = true; 
-#endif
-                var resultList = new List<DataRecord>();
-                record.TryGetField("ID", out var itemId);
-                record.TryGetField("ShardKey", out var shardKey);
-                record.TryGetField("PlanID", out var planId);
-                record.TryGetField("PackageID", out var packageId);
-                record.TryGetField("CompanyID", out var companyId);
-                if(!int.TryParse(planId, out var pid))
-                    throw new ApplicationException($"数据发生异常：OrderItem.PlanID，值: {(string.IsNullOrWhiteSpace(planId) ? "NULL" : planId)}"); 
-                if (pid > 0)
-                {
-                    resultList.Add(new DataRecord([itemId, shardKey, planId, companyId],
-                        TableNames.OrderBlockPlanItem,
-                        ["ItemID", "ShardKey", "PlanID", "CompanyID"]
-                    ));
-                }
-                if(!int.TryParse(packageId, out var pkid))
-                    throw new ApplicationException($"数据发生异常：OrderItem.PackageID，值: {(string.IsNullOrWhiteSpace(packageId) ? "NULL" : packageId)}"); 
-                if(pkid > 0)
-                {
-                    resultList.Add(new DataRecord([itemId, shardKey, packageId, companyId],
-                        TableNames.OrderPackageItem,
-                        [ "ItemID", "ShardKey", "PackageID", "CompanyID" ]
-                    ));
-                }
-                
-                record.RemoveField("PlanID");
-                record.RemoveField("PackageID");
-
-                return resultList;
-            }
-            
-
-            return ArraySegment<DataRecord>.Empty;
-        };
-    });
-    
-    host.Services.Configure<DatabaseOutputOptions>(options =>
-    {
-        options.ConnectionString = outputOptions.ConnectionString;
-        options.FlushCount = outputOptions.FlushCount;
-        options.MaxAllowedPacket = outputOptions.MaxAllowedPacket / 2;
-        options.MaxDatabaseOutputTask = outputOptions.MaxDatabaseOutputTask;
-        options.TreatJsonAsHex = outputOptions.TreatJsonAsHex;
-        options.NoOutput = outputOptions.NoOutput;
-        options.ForUpdate = outputOptions.ForUpdate;
-        
-        // 配置列的类型以便于在输出时区分二进制内容
-        // Prod server
-        options.ColumnTypeConfig = new Dictionary<string, ColumnType>
-        {
-            {"machine.Settings", ColumnType.Text},
-            {"order_block_plan.BlockInfo", ColumnType.Text},
-            {"order_block_plan.OrderNos", ColumnType.Json},
-            {"order_block_plan_result.PlaceData", ColumnType.Blob},
-            {"order_box_block.Data", ColumnType.Blob},
-            {"order_data_block.RemarkJson", ColumnType.Text},
-            {"order_data_goods.ExtraProp", ColumnType.Json},
-            {"order_extra.ConfigJson", ColumnType.Json},
-            {"order_module_extra.Data", ColumnType.Blob},
-            {"order_module_extra.JsonStr", ColumnType.Text},
-            {"order_patch_detail.BlockDetail", ColumnType.Json},
-            {"order_process_schdule.AreaName", ColumnType.Text},
-            {"order_process_schdule.ConsigneeAddress", ColumnType.Text},
-            {"order_process_schdule.ConsigneePhone", ColumnType.Text},
-            {"order_process_schdule.CustomOrderNo", ColumnType.Text},
-            {"order_process_schdule.OrderProcessStepName", ColumnType.Text},
-            {"order_scrap_board.OutLineJson", ColumnType.Text},
-            {"order_wave_group.ConfigJson", ColumnType.Json},
-            {"process_info.Users", ColumnType.Text},
-            {"process_item_exp.ItemJson", ColumnType.Text},
-            {"report_template.SourceConfig", ColumnType.Text},
-            {"report_template.Template", ColumnType.Text},
-            {"simple_package.Items", ColumnType.Json},
-            {"sys_config.JsonStr", ColumnType.Text},
-            {"sys_config.Value", ColumnType.Text}
-        };
-
-        options.OutputFinished += ctx =>
-        {
-            var seq = ctx.Serivces.GetRequiredService<SeqService>();
-            seq.ApplyToDatabaseAsync().GetAwaiter().GetResult();
-        };
-    });
-    
-    host.Services.AddLogging(builder =>
-    {
-        builder.ClearProviders();
-        var logger = new LoggerConfiguration()
-            .MinimumLevel.Verbose()
-            .WriteTo.Console()
-            .WriteTo.File(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"./Log/Error/{ErrorRecorder.UID}.log"),
-                restrictedToMinimumLevel: LogEventLevel.Error)
-            // .WriteTo.File("./Log/Info/{ErrorRecorder.UID}.log", restrictedToMinimumLevel:LogEventLevel.Information) //性能考虑暂不使用
-            .CreateLogger();
-        builder.AddSerilog(logger);
-        Log.Logger = logger;
-    });
-
-    host.Services.AddDataSourceFactory();
-    host.Services.AddErrorRecorderFactory();
-    host.Services.AddSingleton<ProcessContext>();
-    host.Services.AddSingleton<SeqService>();
-    var prodLen = host.Configuration.GetRequiredSection("RecordQueue").GetValue<int>("ProducerQueueLength");
-    var consLen = host.Configuration.GetRequiredSection("RecordQueue").GetValue<int>("ConsumerQueueLength");
-    var maxCharCount = host.Configuration.GetRequiredSection("RecordQueue").GetValue<long>("MaxByteCount") / 2;
-    host.Services.AddKeyedSingleton<DataRecordQueue>(ConstVar.Producer, new DataRecordQueue(prodLen, maxCharCount));
-    host.Services.AddRecordQueuePool(tenantDbOptions.DbGroup.Keys.Select(key => (key:key, queue:new DataRecordQueue(consLen, maxCharCount))).ToArray());
-    // host.Services.AddSingleton<ITaskMonitorLogger, CacheTaskMonitorLogger>();
-    host.Services.AddSingleton<ITaskMonitorLogger, LoggerTaskMonitorLogger>();
-
-    host.Services.AddHostedService<MainHostedService>();
-    host.Services.AddSingleton<IInputService, FileInputService>();
-    host.Services.AddSingleton<ITransformService, TransformService>();
-    host.Services.AddSingleton<IOutputService, OutputService>();
-    host.Services.AddSingleton<TaskMonitorService>();
-    // host.Services.AddRedisCache(redisOptions);
-    host.Services.AddSingleton<ICacher, MemoryCache>();
-    var app = host.Build();
-    await app.RunAsync();
-}

MesETL.App/Services/DataRecordQueue.cs

@@ -1,77 +0,0 @@
-using System.Collections.Concurrent;
-using System.Diagnostics.CodeAnalysis;
-using TaskExtensions = MesETL.Shared.Helper.TaskExtensions;
-
-namespace MesETL.App.Services;
-
-/// <summary>
-/// 数据队列
-/// </summary>
-public class DataRecordQueue : IDisposable
-{
-    private readonly BlockingCollection<DataRecord> _queue;
-
-    private long _currentCharCount;
-    private readonly long _maxCharCount = 2_147_483_648; // 4GiB
-
-    public int Count => _queue.Count;
-    public bool IsCompleted => _queue.IsCompleted;
-    public bool IsAddingCompleted => _queue.IsAddingCompleted;
-
-    public long LongestFieldCharCount { get; private set; }
-
-    public event Action? OnRecordWrite;
-    public event Action? OnRecordRead;
-
-    public DataRecordQueue() : this(500_000, 2_147_483_648) // 默认容量最大500K
-    {
-    }
-    
-    public DataRecordQueue(int boundedCapacity, long maxCharCount)
-    {
-        _queue = new BlockingCollection<DataRecord>(boundedCapacity); 
-        _maxCharCount = maxCharCount;
-    }
-    
-    public void CompleteAdding() => _queue.CompleteAdding();
-    
-    public bool TryDequeue([MaybeNullWhen(false)] out DataRecord record)
-    {
-        if (_queue.TryTake(out record))
-        {
-            // if (record.Database is not null)
-            // {
-            //     Console.WriteLine("out " + record.Database);
-            // }
-            Interlocked.Add(ref _currentCharCount, -record.FieldCharCount);
-            OnRecordRead?.Invoke();
-            return true;
-        }
-
-        return false;
-    }
-
-
-    public async Task EnqueueAsync(DataRecord record)
-    {
-        var charCount = record.FieldCharCount;
-        LongestFieldCharCount = Math.Max(LongestFieldCharCount, charCount);
-        if (_currentCharCount + charCount > _maxCharCount)
-        {
-            // 不用Task.WaitUntil是为了防止产生Lambda闭包
-            while (!(_currentCharCount + charCount < _maxCharCount))
-            {
-                await Task.Delay(50);
-            }
-        }
-        _queue.Add(record);
-        Interlocked.Add(ref _currentCharCount, charCount);
-        OnRecordWrite?.Invoke();
-    }
-
-
-    public void Dispose()
-    {
-        _queue.Dispose();
-    }
-}

MesETL.App/Services/ETL/CsvReader.cs

@@ -1,160 +0,0 @@
-using System.Text;
-using MesETL.App.HostedServices.Abstractions;
-using Microsoft.Extensions.Logging;
-
-namespace MesETL.App.Services.ETL;
-
-/// <summary>
-/// CSV文件读取
-/// </summary>
-public class CsvReader : IDataReader
-{
-    protected readonly string? FilePath;
-    protected readonly Lazy<StreamReader> Reader;
-    private Stream? _stream;
-    protected readonly ILogger? Logger;
-    protected readonly string TableName;
-
-    public DataRecord Current { get; protected set; } = default!;
-    public string[] Headers { get; }
-    public string Delimiter { get; }
-    public char QuoteChar { get; }
-    
-    public CsvReader(Stream stream, string tableName, string[] headers, string delimiter = ",", char quoteChar = '"', ILogger? logger = null)
-    : this(tableName, headers, delimiter, quoteChar, logger)
-    {
-        Reader = new Lazy<StreamReader>(() => new StreamReader(stream),false);
-    }
-
-    public CsvReader(string filePath, string tableName, string[] headers, string delimiter = ",", char quoteChar = '"', ILogger? logger = null)
-    : this(tableName, headers, delimiter, quoteChar, logger)
-    {
-        FilePath = filePath;
-        Reader = new Lazy<StreamReader>(() =>
-        {
-            _stream = File.OpenRead(filePath);
-            return new StreamReader(_stream);
-        });
-    }
-
-    private CsvReader(string tableName, string[] headers, string delimiter = ",", char quoteChar = '"', ILogger? logger = null)
-    {
-        TableName = tableName;
-        Headers = headers;
-        Logger = logger;
-        Delimiter = delimiter;
-        QuoteChar = quoteChar;
-        Reader = null!;
-    }
-
-    public virtual async ValueTask<bool> ReadAsync()
-    {
-        var str = await Reader.Value.ReadLineAsync();
-        if (string.IsNullOrWhiteSpace(str))
-            return false;
-
-        var fields = ParseRowFaster(str, QuoteChar, Delimiter[0]);
-        Current = new DataRecord(fields, TableName, Headers);
-        return true;
-    }
-
-    public static string[] ParseRow(ReadOnlySpan<char> source, char quoteChar, char delimiter)
-    {
-        var result = new List<string>();
-        var index = -1;
-        var current = new StringBuilder(source.Length);
-        var hasQuote = false;
-        var hasSlash = false;
-        while (index < source.Length - 1)
-        {
-            index++;
-            var currChar = source[index];
-            if (hasSlash == false && currChar == '\\')
-            {
-                hasSlash = true;
-                current.Append('\\');
-                continue;
-            }
-
-            if (hasSlash == false && currChar == quoteChar)
-            {
-                hasQuote = !hasQuote;
-                current.Append(currChar);
-                continue;
-            }
-
-            if (hasQuote == false && currChar == delimiter)
-            {
-                result.Add(current.ToString());
-                current.Clear();
-            }
-            else
-            {
-                current.Append(currChar);
-            }
-
-            hasSlash = false;
-        }
-
-        result.Add(current.ToString());
-        return result.ToArray();
-    }
-    
-    public static List<string> ParseRowFaster(ReadOnlySpan<char> source, char quoteChar, char delimiter, int columnCount = 10)
-    {
-        var result = new List<string>(columnCount);
-        var index = -1;
-        var hasQuote = false;
-        var hasSlash = false;
-        var start = 0;
-        var end = 0;
-        var len = source.Length - 1;
-        while (index < len)
-        {
-            ++index;
-            var currChar = source[index];
-            
-            if (!hasSlash)
-            {
-                if (currChar is '\\')
-                {
-                    hasSlash = true;
-                    ++end;
-                    continue;
-                }
-
-                if (currChar == quoteChar)
-                {
-                    hasQuote = !hasQuote;
-                    ++end;
-                    continue;
-                }
-            }
-            
-            if (!hasQuote && currChar == delimiter)
-            {
-                result.Add(source[start..(end)].ToString()); // 超大型字符串会在LOH中分配内存，没救
-                start = end + 1;
-                ++end;
-            }
-            else
-            {
-                ++end;
-            }
-
-            hasSlash = false;
-        }
-
-        result.Add(source[start..end].ToString());
-        return result;
-    }
-
-    public virtual void Dispose()
-    {
-        if (Reader.IsValueCreated)
-        {
-            Reader.Value.Dispose();
-            _stream?.Dispose();
-        }
-    }
-}

MesETL.App/Services/ETL/DataReaderFactory.cs

@@ -1,46 +0,0 @@
-using MesETL.App.HostedServices.Abstractions;
-using MesETL.App.Options;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Options;
-
-namespace MesETL.App.Services.ETL;
-
-public class DataReaderFactory
-{
-    private readonly ILogger<DataReaderFactory> _logger;
-    private readonly IOptions<DataInputOptions> _options;
-
-    public DataReaderFactory(ILogger<DataReaderFactory> logger, IOptions<DataInputOptions> options)
-    {
-        _logger = logger;
-        _options = options;
-    }
-
-    public IDataReader CreateReader(string filePath, string tableName, string[] headers)
-    {
-        if (_options.Value.UseMock)
-        {
-            if (_options.Value.TableMockConfig is null)
-                throw new ApplicationException("未配置表模拟数据量级");
-            _logger.LogDebug("***** Using {Type} data source *****", "ZSTD mock");
-            var mockConfig = _options.Value.TableMockConfig.GetValueOrDefault(tableName, 
-                new TableMockConfig { MockCount = 1, UseDeepCopy = false });
-            mockConfig.MockCount = (long)Math.Ceiling(mockConfig.MockCount * _options.Value.MockCountMultiplier); 
-            return new ZstMockReader(mockConfig, filePath,
-                tableName, headers, _options.Value.Delimiter, _options.Value.QuoteChar, _logger);
-        }
-
-        _logger.LogDebug("***** Using {Type} data source *****", "ZSTD");
-        return new ZstReader(filePath, tableName, headers, _options.Value.Delimiter, _options.Value.QuoteChar, _logger);
-    }
-}
-
-public static class DataSourceFactoryExtensions
-{
-    public static IServiceCollection AddDataSourceFactory(this IServiceCollection services)
-    {
-        services.AddSingleton<DataReaderFactory>();
-        return services;
-    }
-}

MesETL.App/Services/ETL/MySqlDestination.cs

@@ -1,268 +0,0 @@
-using System.Text;
-using System.Text.RegularExpressions;
-using MesETL.App.Const;
-using MesETL.App.Helpers;
-using MesETL.App.Options;
-using MesETL.Shared.Helper;
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Options;
-using MySqlConnector;
-
-namespace MesETL.App.Services.ETL;
-
-/// <summary>
-/// Mysql导出
-/// </summary>
-public partial class MySqlDestination : IDisposable, IAsyncDisposable
-{
-    /// <summary>
-    /// table => records
-    /// </summary>
-    private readonly Dictionary<string, IList<DataRecord>> _recordCache;
-    private readonly MySqlConnection _conn;
-    private readonly ILogger _logger;
-    private readonly IOptions<DatabaseOutputOptions> _options;
-    private readonly ErrorRecorder.OutputErrorRecorder _outputErrorRecorder;
-    private readonly ProcessContext _context;
-
-    public MySqlDestination(
-        string connStr, 
-        ILogger logger, 
-        IOptions<DatabaseOutputOptions> options,
-        ErrorRecorder.OutputErrorRecorder outputErrorRecorder, 
-        ProcessContext context)
-    {
-        _conn = new MySqlConnection(connStr);
-        _conn.Open();
-        _recordCache = new Dictionary<string, IList<DataRecord>>();
-        _logger = logger;
-        _options = options;
-        _outputErrorRecorder = outputErrorRecorder;
-        _context = context;
-    }
-    
-    public Task WriteRecordAsync(DataRecord record)
-    {
-        _recordCache.AddOrUpdate(record.TableName, [record], (_, value) =>
-        {
-            value.Add(record);
-            return value;
-        });
-        return Task.CompletedTask;
-    }
-
-    public async Task WriteRecordsAsync(IEnumerable<DataRecord> records)
-    {
-        foreach (var record in records)
-        {
-            await WriteRecordAsync(record);
-        }
-    }
-
-    public async Task FlushAsync(int maxAllowPacket)
-    {
-        if (_recordCache.Count == 0)
-            return;
-
-        var cmd = _conn.CreateCommand();
-        cmd.CommandTimeout = 0;
-       
-        try
-        {
-            var executionList = GetExecutionList(_recordCache, maxAllowPacket);
-            foreach (var insertSql in executionList)
-            {
-                cmd.CommandText = insertSql;
-                try
-                {
-                    await cmd.ExecuteNonQueryAsync();
-                }
-                catch (Exception e)
-                {
-                    _logger.LogError(e, "插入数据库时发生错误, sql: {Sql}", cmd.CommandText.Omit(1000));
-                    _context.AddException(e);
-                    var match = MatchTableName().Match(cmd.CommandText);
-                    if (match is { Success: true, Groups.Count: > 1 })
-                    {
-                        var tableName = match.Groups[1].Value;
-                        await _outputErrorRecorder.LogErrorSqlAsync(cmd.CommandText, tableName, e);
-                    }
-                    else await _outputErrorRecorder.LogErrorSqlAsync(cmd.CommandText, e);
-                }
-            }
-            _recordCache.Clear();
-        }
-        catch (Exception e)
-        {
-            _logger.LogError(e, "序列化记录时发生错误");
-            throw;
-        }
-        finally
-        {
-            await cmd.DisposeAsync();
-        }
-    }
-    
-    [GeneratedRegex("INSERT INTO `([^`]+)`")]
-    private static partial Regex MatchTableName();
-
-    public IEnumerable<string> GetExecutionList(IDictionary<string, IList<DataRecord>> tableRecords, int maxAllowPacket)
-    {
-        var sb = new StringBuilder(_options.Value.FlushCount * 128);
-        var appendCount = 0;
-        foreach (var (tableName, records) in tableRecords)
-        {
-            if (records.Count == 0)
-                continue;
-            
-            var recordIdx = 0;
-            StartBuild:
-            sb.AppendLine("SET AUTOCOMMIT = 0;\n");
-            var noCommas = true;
-            
-            // 标准列顺序，插入时的字段需要按照该顺序排列
-            var headers = records[0].Headers;
-            
-            // INSERT INTO ... VALUES >>>
-            sb.Append($"INSERT INTO `{tableName}`(");
-            for (var i = 0; i < headers.Count; i++)
-            {
-                var header = records[0].Headers[i];
-                sb.Append($"`{header}`");
-                if (i != headers.Count - 1)
-                    sb.Append(',');
-            }
-
-            sb.Append(") VALUES ");
-            
-            // ([FIELDS]), >>>
-            for (;recordIdx < records.Count; recordIdx++)
-            {
-                var record = records[recordIdx];
-                
-                // 数据列校验
-                if (record.Headers.Count != headers.Count)
-                {
-                    throw new InvalidOperationException($"数据异常，数据列数量出现冲突，表名：{tableName}");
-                }
-                
-                var recordSb = new StringBuilder();
-                recordSb.Append('(');
-                for (var idx = 0; idx < headers.Count; idx++)
-                {
-                    var header = headers[idx];
-                    // TODO: 可进行性能优化
-                    var field = record[header];
-
-                    // 在这里处理特殊列
-                    #region HandleFields
-
-                    if (field.Length == 2 && field == ConstVar.MyDumperNull) // MyDumper导出的NULL为'\N'（'\'不是转义字符）
-                    {
-                        recordSb.Append(ConstVar.Null);
-                        goto Escape;
-                    }
-                    
-                    switch (_options.Value.GetColumnType(record.TableName, header))
-                    {
-                        case ColumnType.Text:
-                            if(string.IsNullOrEmpty(field))
-                                recordSb.Append("''");
-                            else if (field == ConstVar.Null)
-                                recordSb.Append(ConstVar.Null);
-                            else recordSb.Append($"_utf8mb4 0x{field}");
-                            break;
-                        case ColumnType.Blob:
-                            if (string.IsNullOrEmpty(field))
-                                recordSb.Append("''");
-                            else if (field == ConstVar.Null)
-                                recordSb.Append(ConstVar.Null);
-                            else recordSb.Append($"0x{field}");
-                            break;
-                        case ColumnType.Json: // Mydumper v0.16.7-5导出的Json为字符串，且会将逗号转义，需要适配
-                            if(string.IsNullOrEmpty(field)) 
-                                recordSb.Append(ConstVar.Null);
-                            else if (_options.Value.TreatJsonAsHex)
-                                recordSb.Append($"_utf8mb4 0x{field}");
-                            else recordSb.AppendLine(field.Replace("\\,", ","));
-                            break;
-                        case ColumnType.UnDefine:
-                        default:
-                            recordSb.Append(field);
-                            break;
-                    }
-                    
-                    Escape:
-
-                    #endregion
-                    if (idx != headers.Count - 1)
-                        recordSb.Append(',');
-                }
-
-                recordSb.Append(')');
-
-                // 若字符数量即将大于限制，则返回SQL，清空StringBuilder，保留当前记录的索引值，然后转到StartBuild标签重新开始一轮INSERT
-                if (sb.Length + recordSb.Length + 23 > maxAllowPacket)
-                {
-                    if (appendCount == 0) // 如果单条记录超出maxAllowedPacket
-                    {
-                        sb.Append(recordSb);
-                        _logger.LogWarning("{Table}表单条数据的SQL超出了配置的MaxAllowedPacket，字符数{Count}", tableName,
-                            sb.Length + recordSb.Length + 23);
-                    }
-
-                    TryAddForUpdateSuffix(tableName, sb);
-                    sb.Append(';').AppendLine();
-                    sb.Append("COMMIT;");
-                    yield return sb.ToString();
-                    sb.Clear();
-                    goto StartBuild;
-                }
-
-                if (!noCommas)
-                    sb.Append(',').AppendLine();
-                noCommas = false;
-                sb.Append(recordSb); // StringBuilder.Append(StringBuilder)不会分配多余的内存
-                appendCount++;
-            }
-
-            TryAddForUpdateSuffix(tableName, sb);
-            sb.Append(';');
-            sb.Append("COMMIT;");
-            yield return sb.ToString();
-            sb.Clear();
-        }
-    }
-
-    /// <summary>
-    /// 数据必须是同一张表
-    /// </summary>
-    /// <param name="tableName"></param>
-    /// <param name="sb"></param>
-    private void TryAddForUpdateSuffix(string tableName, StringBuilder sb)
-    {
-        var forUpdate = _options.Value.TryGetForUpdate(tableName, out var forUpdateSql);
-        if (forUpdate)
-        {
-            sb.AppendLine($"""
-                           AS new
-                           ON DUPLICATE KEY UPDATE
-                           {forUpdateSql}
-                           """);
-        }
-    }
-    
-    public void Dispose()
-    {
-        _conn.Close();
-        _conn.Dispose();
-        _recordCache.Clear();
-    }
-
-    public async ValueTask DisposeAsync()
-    {
-        await _conn.CloseAsync();
-        await _conn.DisposeAsync();
-        _recordCache.Clear();
-    }
-}

MesETL.App/Services/ETL/ZstMockReader.cs

@@ -1,64 +0,0 @@
-using MesETL.App.Options;
-using Microsoft.Extensions.Logging;
-
-namespace MesETL.App.Services.ETL;
-
-/// <summary>
-/// 截取提供ZST文件中的第一行，然后复制成指定数量的数据
-/// </summary>
-public class ZstMockReader : ZstReader
-{
-    private long _currentCount;
-    private readonly long _mockCount;
-    private DataRecord? _template;
-    private readonly bool _deepCopy;
-    private readonly string[]? _autoIncrementColumn;
-    
-    static readonly IReadOnlyList<int> Range = [500, 1500, 2500];
-    
-    public ZstMockReader(TableMockConfig mockConfig, string filePath, string tableName, string[] headers, string delimiter = ",", char quoteChar = '\"', ILogger? logger = null) : base(filePath, tableName, headers, delimiter, quoteChar, logger)
-    {
-        _mockCount = mockConfig.MockCount;
-        _deepCopy = mockConfig.UseDeepCopy;
-        _autoIncrementColumn = mockConfig.AutoIncrementColumn;
-    }
-
-    public ZstMockReader(TableMockConfig mockConfig, Stream stream, string tableName, string[] headers, string delimiter = ",", char quoteChar = '\"', ILogger? logger = null) : base(stream, tableName, headers, delimiter, quoteChar, logger)
-    {
-        _mockCount = mockConfig.MockCount;
-        _deepCopy = mockConfig.UseDeepCopy;
-        _autoIncrementColumn = mockConfig.AutoIncrementColumn;
-    }
-
-    public override async ValueTask<bool> ReadAsync()
-    {
-        if (_template is null)
-        {
-            if (!await base.ReadAsync())
-                throw new InvalidOperationException("所提供的ZST源为空，无法生成模板数据");
-            _template = Current.Clone() as DataRecord;
-            if (_template is null)
-                throw new ApplicationException("记录拷贝失败");
-            _currentCount++;
-            return true;
-        }
-
-        if (_deepCopy)
-        {
-            Current = _template.Clone() as DataRecord ?? throw new ApplicationException("记录拷贝失败");
-            if(_autoIncrementColumn is not null)
-            {
-                foreach (var column in _autoIncrementColumn)
-                {
-                    Current[column] = (Convert.ToInt64(Current[column]) + 1).ToString();
-                    _template = Current;
-                }
-            }
-
-            Current["CompanyID"] = Range[Random.Shared.Next(0, Range.Count)].ToString();//随机CompanyID
-        }
-        else Current = _template;
-        _currentCount++;
-        return _currentCount < _mockCount;
-    }
-}

MesETL.App/Services/ETL/ZstReader.cs

@@ -1,101 +0,0 @@
-using System.Runtime.InteropServices;
-using System.Text;
-using Microsoft.Extensions.Logging;
-using ZstdSharp;
-
-namespace MesETL.App.Services.ETL;
-
-/// <summary>
-/// 解压ZST文件，从中读取CSV数据
-/// </summary>
-public class ZstReader : CsvReader
-{
-    protected new readonly Lazy<StreamReader> Reader;
-    private Stream? _stream;
-
-    private readonly List<char> _str = new(1024);
-    private readonly char[] _charBuffer = new char[1024];
-    private int _charLen = 0;
-    private int _charPos = 0;
-    
-    
-    public ZstReader(string filePath, string tableName, string[] headers, string delimiter = ",", char quoteChar = '\"', ILogger? logger = null) 
-        : base(filePath, tableName, headers, delimiter, quoteChar, logger)
-    {
-        Reader = new Lazy<StreamReader>(() =>
-        {
-             _stream = new DecompressionStream(File.OpenRead(filePath));
-            return new StreamReader(_stream);
-        }, false);
-        ReadBuffer();
-    }
-        
-    public ZstReader(Stream stream, string tableName, string[] headers, string delimiter = ",", char quoteChar = '\"', ILogger? logger = null) 
-        : base(stream, tableName, headers, delimiter, quoteChar, logger)
-    {
-        var ds = new DecompressionStream(stream);
-        Reader = new Lazy<StreamReader>(() => new StreamReader(ds), false);
-        ReadBuffer();
-    }
-
-    private int ReadBuffer()
-    {
-        _charLen = _charPos = 0;
-        _charLen = Reader.Value.ReadBlock(_charBuffer);
-        return _charLen;
-    }
-
-    public override async ValueTask<bool> ReadAsync()
-    {
-        // 缓冲区已经读取完毕，并且流状态为EOF
-        if (_charPos == _charLen && ReadBuffer() == 0)
-            return false;
-
-        do
-        {
-            // 读取缓冲区
-            var span = _charBuffer.AsSpan(_charPos, _charLen - _charPos);
-            var newLineIdx = span.IndexOfAny('\r', '\n');
-            // 读取到行，结合当前构建字符串转换进行转换
-            if (newLineIdx >= 0)
-            {
-                if (_str.Count == 0) // => 可以提高一点性能...
-                {
-                    var fields = ParseRowFaster(span[..newLineIdx], QuoteChar, Delimiter[0]);
-                    Current = new DataRecord(fields, TableName, Headers);
-                }
-                else
-                {
-                    _str.AddRange(span[..newLineIdx]);
-                    var fields = ParseRowFaster(CollectionsMarshal.AsSpan(_str), QuoteChar, Delimiter[0]);
-                    Current = new DataRecord(fields, TableName, Headers);
-                }
-                _str.Clear();
-
-                var ch = span[newLineIdx];
-                _charPos += newLineIdx + 1;
-                if (ch == '\r' && (_charPos < _charLen || ReadBuffer() > 0) && _charBuffer[_charPos] == '\n') // 跳过CRLF
-                    ++_charPos;
-                return true;
-            }
-            // 未读取到行，将缓冲区插入构建字符串
-            _str.AddRange(span);
-        } while (ReadBuffer() > 0);
-
-        var f = ParseRowFaster(CollectionsMarshal.AsSpan(_str), QuoteChar, Delimiter[0]);
-        Current = new DataRecord(f, TableName, Headers);
-        _str.Clear();
-        return true;
-    }
-
-    public override void Dispose()
-    {
-        base.Dispose();
-        if (Reader.IsValueCreated)
-        {
-            Reader.Value.Dispose();
-            _stream?.Dispose();
-        }
-    }
-
-}

MesETL.App/Services/ErrorRecorder/ErrorRecorder.cs

@@ -1,80 +0,0 @@
-using MesETL.App.Helpers;
-using MesETL.Shared.Helper;
-using Microsoft.Extensions.Logging;
-
-namespace MesETL.App.Services.ErrorRecorder;
-
-public class ErrorRecorder
-{
-    protected ILogger Logger;
-
-    /// <summary>
-    /// 当次执行标识
-    /// </summary>
-    public static readonly string UID = DateTime.Now.ToString("yyyy-MM-dd HH-mm-ss");
-
-    public ErrorRecorder(ILogger logger)
-    {
-        Logger = logger;
-    }
-    
-    
-     public static async Task LogErrorRecordAsync(string outputDir, DataRecord record, Exception exception)
-     {
-         if(!Directory.Exists(outputDir))
-             Directory.CreateDirectory(outputDir);
-         var content = $"""
-                        ### {exception.Message}
-                        {string.Join(',', record.Fields)}
-                        """;
-         var path = Path.Combine(outputDir, $"{record.TableName}.errlog");
-         await File.AppendAllTextAsync(path, content);
-     }
-     
-     public static async Task LogErrorRecordAsync(string outputDir, IEnumerable<DataRecord> records, Exception exception)
-     {
-         if(!Directory.Exists(outputDir))
-             Directory.CreateDirectory(outputDir);
-         var tableMapping = new Dictionary<string, Tuple<List<DataRecord>, StreamWriter>>();
-         foreach (var record in records)
-         {
-             tableMapping.AddOrUpdate(record.TableName,
-                 Tuple.Create((List<DataRecord>) [record], new StreamWriter(File.OpenRead(record.TableName))),
-                 (_, tuple) =>
-                 {
-                     tuple.Item1.Add(record);
-                     return tuple;
-                 });
-         }
-
-         var maxParallelism = 5;
-         for (var i = 0; i < tableMapping.Count; i+=maxParallelism)
-         {
-             await Parallel.ForEachAsync(tableMapping.Take(maxParallelism), async (pair, token) =>
-             {
-                 var (records, writer) = pair.Value;
-                 foreach (var record in records)
-                 {
-                     var content =
-                         $"""
-                          ### {exception.Message}
-                          {string.Join(',', record.Fields)}
-                          """;
-                     await writer.WriteLineAsync(content);
-                     if (token.IsCancellationRequested)
-                         break;
-                 }
-                 await writer.DisposeAsync();
-             });
-         }
-     }
-     
-     public void ClearErrorRecords(string dir)
-     {
-         Logger.LogInformation("***** Clear error records *****");
-         foreach (var file in Directory.GetFiles(dir, "*.errlog", SearchOption.AllDirectories))
-         {
-             File.Delete(file);
-         }
-     }
-}

MesETL.App/Services/ErrorRecorder/ErrorRecorderFactory.cs

@@ -1,27 +0,0 @@
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Logging;
-
-namespace MesETL.App.Services.ErrorRecorder;
-
-public class ErrorRecorderFactory
-{
-    private readonly ILogger<ErrorRecorderFactory> _logger;
-
-    public ErrorRecorderFactory(ILogger<ErrorRecorderFactory> logger)
-    {
-        _logger = logger;
-    }
-    
-    public OutputErrorRecorder CreateOutput(string database) => new(database, _logger);
-    public TransformErrorRecorder CreateTransform() => new(_logger);
-    public InputErrorRecorder CreateInput() => new(_logger);
-}
-
-public static class ErrorRecorderFactoryExtensions
-{
-    public static IServiceCollection AddErrorRecorderFactory(this IServiceCollection services)
-    {
-        services.AddSingleton<ErrorRecorderFactory>();
-        return services;
-    }
-}

MesETL.App/Services/ErrorRecorder/InputErrorRecorder.cs

@@ -1,19 +0,0 @@
-using Microsoft.Extensions.Logging;
-
-namespace MesETL.App.Services.ErrorRecorder;
-
-public sealed class InputErrorRecorder : ErrorRecorder
-{
-    private readonly string _outputDir =
-        Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"ErrorRecords/{UID}/Input");
-    
-    public InputErrorRecorder(ILogger logger) : base(logger)
-    {
-    }
-    
-    public Task LogErrorRecordAsync(DataRecord record, Exception exception) =>
-        LogErrorRecordAsync(_outputDir, record, exception);
-
-    public Task LogErrorRecordAsync(IEnumerable<DataRecord> records, Exception exception) =>
-        LogErrorRecordAsync(_outputDir, records, exception);
-}

MesETL.App/Services/ErrorRecorder/TransformErrorRecorder.cs

@@ -1,20 +0,0 @@
-using Microsoft.Extensions.Logging;
-
-namespace MesETL.App.Services.ErrorRecorder;
-
-public sealed class TransformErrorRecorder : ErrorRecorder
-{
-    private readonly string _outputDir =
-        Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"ErrorRecords/{UID}/Transform");
-
-
-    public TransformErrorRecorder(ILogger logger) : base(logger)
-    {
-    }
-
-    public Task LogErrorRecordAsync(DataRecord record, Exception exception) =>
-        LogErrorRecordAsync(_outputDir, record, exception);
-
-    public Task LogErrorRecordAsync(IEnumerable<DataRecord> records, Exception exception) =>
-        LogErrorRecordAsync(_outputDir, records, exception);
-}

MesETL.App/Services/Loggers/CacheTaskMonitorLogger.cs

@@ -1,20 +0,0 @@
-using System.Text;
-using MesETL.App.Cache;
-
-namespace MesETL.App.Services.Loggers;
-
-public class CacheTaskMonitorLogger : ITaskMonitorLogger
-{
-    private readonly ICacher _cacher;
-
-    public CacheTaskMonitorLogger(ICacher cacher)
-    {
-        _cacher = cacher;
-    }
-
-    public void LogStatus(string name, IReadOnlyDictionary<string, string> properties, ITaskMonitorLogger.LogLevel logLevel)
-    {
-        if(logLevel is ITaskMonitorLogger.LogLevel.Progress)
-            _cacher.SetHashAsync(name, properties);
-    }
-}

MesETL.App/Services/Loggers/ITaskMonitorLogger.cs

@@ -1,12 +0,0 @@
-namespace MesETL.App.Services.Loggers;
-
-public interface ITaskMonitorLogger
-{
-    public enum LogLevel
-    {
-        Info,
-        Debug,
-        Progress,
-    }
-    void LogStatus(string name, IReadOnlyDictionary<string, string> properties, LogLevel logLevel = LogLevel.Info);
-}

MesETL.App/Services/Loggers/LoggerTaskMonitorLogger.cs

@@ -1,41 +0,0 @@
-using System.Text;
-using Microsoft.Extensions.Logging;
-
-namespace MesETL.App.Services.Loggers;
-
-public class LoggerTaskMonitorLogger : ITaskMonitorLogger
-{
-    private readonly ILogger _logger;
-
-    public LoggerTaskMonitorLogger(ILogger<LoggerTaskMonitorLogger> logger)
-    {
-        _logger = logger;
-    }
-
-    public void LogStatus(string name, IReadOnlyDictionary<string, string> properties, ITaskMonitorLogger.LogLevel logLevel)
-    {
-        var sb = new StringBuilder();
-        sb.Append($"{name}: {{");
-        sb.AppendJoin(',', properties.Select((pair, i) => $" {pair.Key}: {pair.Value}"));
-        sb.Append([' ', '}']);
-        // var args = new List<string> { name };
-        // properties.Aggregate(args, (args, pair) =>
-        // {
-        //     args.Add(pair.Key);
-        //     args.Add(pair.Value);
-        //     return args;
-        // });
-        switch (logLevel)
-        {
-            case ITaskMonitorLogger.LogLevel.Info:
-                _logger.LogInformation("{message}", sb.ToString());
-                break;
-            case ITaskMonitorLogger.LogLevel.Progress:
-            case ITaskMonitorLogger.LogLevel.Debug:
-                _logger.LogDebug("{message}", sb.ToString());
-                break;
-            default:
-                throw new ArgumentOutOfRangeException(nameof(logLevel), logLevel, null);
-        }
-    }
-}

MesETL.App/Services/ProcessContext.cs

@@ -1,77 +0,0 @@
-using System.Collections.Concurrent;
-
-namespace MesETL.App.Services;
-
-/// <summary>
-/// 处理上下文类，标识处理进度
-/// </summary>
-public class ProcessContext
-{
-    private bool _hasException;
-    private long _inputCount;
-    private long _transformCount;
-    private long _outputCount;
-    private readonly ConcurrentDictionary<string, (long input, long output)> _tableProgress = new();
-    public bool HasException => _hasException;
-    public bool IsInputCompleted { get; private set; }
-    public bool IsTransformCompleted { get; private set; }
-    public bool IsOutputCompleted { get; private set; }
-
-    public long InputCount
-    {
-        get => _inputCount;
-        set => Interlocked.Exchange(ref _inputCount, value);
-    }
-
-    public long TransformCount
-    {
-        get => _transformCount;
-        set => Interlocked.Exchange(ref _transformCount, value);
-    }
-
-    public long OutputCount
-    {
-        get => _outputCount;
-        set => Interlocked.Exchange(ref _outputCount, value);
-    }
-
-    public long MaxMemoryUsage { get; set; }
-
-
-    // TableName -> Count
-    public IReadOnlyDictionary<string, (long input, long output)> TableProgress => _tableProgress;
-
-    public void CompleteInput() => IsInputCompleted = true;
-
-    public void CompleteTransform() => IsTransformCompleted = true;
-    public void CompleteOutput() => IsOutputCompleted = true;
-    public bool AddException(Exception e) => _hasException = true; // 没打算存起来，暂时先加个标记
-
-    public void AddInput() => Interlocked.Increment(ref _inputCount);
-
-    public void AddInput(int count) => Interlocked.Add(ref _inputCount, count);
-
-    public void AddTransform() => Interlocked.Increment(ref _transformCount);
-    public void AddTransform(int count) => Interlocked.Add(ref _transformCount, count);
-
-    public void AddOutput() => Interlocked.Increment(ref _outputCount);
-    public void AddOutput(int count) => Interlocked.Add(ref _outputCount, count);
-
-    public void AddTableInput(string table, int count)
-    {
-        _tableProgress.AddOrUpdate(table, (input: count, output: 0), (k, tuple) =>
-        {
-            tuple.input += count;
-            return tuple;
-        });
-    }
-
-    public void AddTableOutput(string table, int count)
-    {
-        _tableProgress.AddOrUpdate(table, (input: 0, output: count), (k, tuple) =>
-        {
-            tuple.output += count;
-            return tuple;
-        });
-    }
-}

MesETL.App/Services/RecordQueuePool.cs

@@ -1,66 +0,0 @@
-using System.Collections.Concurrent;
-using Microsoft.Extensions.DependencyInjection;
-
-namespace MesETL.App.Services;
-
-public class RecordQueuePool
-{
-    private readonly ConcurrentDictionary<string, DataRecordQueue> _queues = new();
-
-    public IReadOnlyDictionary<string, DataRecordQueue> Queues => _queues;
-
-    public void AddQueue(string key, int boundedCapacity = 200_0000, long maxCharCount = 2_147_483_648) 
-        => AddQueue(key, new DataRecordQueue(boundedCapacity, maxCharCount));
-    
-    public void AddQueue(string key, DataRecordQueue queue)
-    {
-        if (!_queues.TryAdd(key, queue))
-            throw new InvalidOperationException($"请勿添加重复的队列，队列名: {key}");
-    }
-    
-    public void RemoveQueue(string key, bool dispose = true)
-    {
-        if (!_queues.Remove(key, out var queue))
-            throw new InvalidOperationException($"未找到对应的队列，队列名：{key}");
-        if (dispose) queue.Dispose();
-    }
-    
-    public DataRecordQueue GetQueue(string key)
-    {
-        return _queues[key];
-    }
-
-    public DataRecordQueue this[string key]
-    {
-        get => GetQueue(key);
-        set => AddQueue(key, value);
-    }
-}
-
-public static class MultiRecordQueueExtensions
-{
-    public static IServiceCollection AddRecordQueuePool(this IServiceCollection services, params string[] keys)
-    {
-        var pool = new RecordQueuePool();
-        foreach (var key in keys)
-        {
-            pool.AddQueue(key);
-        }
-
-        services.AddSingleton(pool);
-        return services;
-    }
-
-    public static IServiceCollection AddRecordQueuePool(this IServiceCollection services,
-        params (string key, DataRecordQueue queue)[] queues)
-    {
-        var pool = new RecordQueuePool();
-        foreach (var (key, queue) in queues)
-        {
-            pool.AddQueue(key, queue);
-        }
-
-        services.AddSingleton(pool);
-        return services;
-    }
-}

MesETL.App/Services/Seq/SeqConfig.cs

@@ -1,42 +0,0 @@
-// ReSharper disable InconsistentNaming
-namespace MesETL.App.Services.Seq;
-
-public class SeqConfig(string Name, bool Recycle = true, int Step = 1, long Max = 999_999_999)
-{
-    public string Name { get; init; } = Name;
-    public bool Recycle { get; init; } = Recycle;
-    public int Step { get; init; } = Step;
-    public long Max { get; init; } = Max;
-
-    public static readonly SeqConfig ItemNo = new("seq_ItemNo", true, 1, 999_999_999);
-    public static readonly SeqConfig OrderModuleID = new("seq_order_module_id", false);
-    public static readonly SeqConfig OrderDataID = new("seq_order_data_id", false);
-    public static readonly SeqConfig OrderItemID = new("seq_order_item", false);
-    public static readonly SeqConfig ProcessStepID = new("seq_step_id", false);
-    public static readonly SeqConfig PackageNo = new("seq_pack_no", true, 1, 9_999_999);
-    public static readonly SeqConfig PlanNo = new("seq_plan_order", true, 1, 999_999);
-    public static readonly SeqConfig SimplePlanNo = new("seq_simple_plan_order", true, 1, 999_999);
-
-    // 下面这些类型的流水号在BaseService添加实体时进行生成
-    public static readonly SeqConfig MachineID = new("seq_machine_id", false);
-    public static readonly SeqConfig OrderBlockPlanID = new("seq_order_block_plan_id", false);
-    public static readonly SeqConfig OrderDataGoodsID = new("seq_order_data_goods_id", false);
-    public static readonly SeqConfig OrderPackageID = new("seq_order_pack_id", false);
-    public static readonly SeqConfig OrderProcessID = new("seq_order_process_id", false);
-    public static readonly SeqConfig OrderProcessStepItemID = new("seq_order_process_step_item_id", false);
-    public static readonly SeqConfig ProcessGroupID = new("seq_process_group_id", false);
-    public static readonly SeqConfig ProcessInfoID = new("seq_process_info_id", false);
-    public static readonly SeqConfig ProcessItemExpID = new("seq_process_item_exp_id", false);
-    public static readonly SeqConfig ProcessScheduleCapacityID = new("seq_process_schedule_capacity_id", false);
-    public static readonly SeqConfig ProcessStepEfficiencyID = new("seq_process_step_efficiency_id", false);
-    public static readonly SeqConfig ReportTemplateID = new("seq_report_template_id", false);
-    public static readonly SeqConfig SysConfigKey = new("seq_sys_config_key", false);
-    public static readonly SeqConfig WorkCalendarID = new("seq_work_calendar_id", false);
-    public static readonly SeqConfig WorkShiftID = new("seq_work_shift_id", false);
-    public static readonly SeqConfig WorkTimeID = new("seq_work_time_id", false);
-    public static readonly SeqConfig OrderPatchDetailID = new("seq_order_patch_detail_id", false);
-    public static readonly SeqConfig OrderModuleExtraID = new("seq_order_module_extra_id", false);
-    public static readonly SeqConfig SimplePackageID = new("seq_simple_pack_id", false);
-    public static readonly SeqConfig OrderModuleItemID = new("seq_order_module_item_id", false);
-    public static readonly SeqConfig OrderWaveGroupID = new("seq_order_wave_group_id", false);
-}

MesETL.App/Services/Seq/SeqService.cs

@@ -1,136 +0,0 @@
-using System.Collections.Concurrent;
-using System.Diagnostics;
-using System.Runtime.CompilerServices;
-using System.Text;
-using MesETL.App.Options;
-using MesETL.Shared.Helper;
-using Microsoft.Extensions.Options;
-using MySqlConnector;
-
-namespace MesETL.App.Services.Seq;
-
-public class SeqService
-{
-    private readonly string _connectionString;
-    private readonly ConcurrentDictionary<SeqConfig, long> _cachedSequence;
-
-    public IReadOnlyDictionary<SeqConfig, long> CachedSequence => _cachedSequence;
-
-    public SeqService(IOptions<DatabaseOutputOptions> options)
-    {
-        var connStr = options.Value.ConnectionString ?? throw new ApplicationException("未配置输出数据库连接字符串");
-        var builder = new MySqlConnectionStringBuilder(connStr)
-        {
-            Database = "mes_global"
-        };
-        _connectionString = builder.ConnectionString;
-
-        _cachedSequence = new ConcurrentDictionary<SeqConfig, long>();
-    }
-
-    private async Task<long> UpdateSequenceID(string name,int step,long max,bool recycle, int add)
-    {
-        var sql = new StringBuilder(
-            $"""
-            INSERT INTO seq (SeqName,CurrentVal,Increment,MinVal,MaxVal,UpdateTime)
-            VALUES ({name},{add},{step},1,{max},NOW()) 
-            ON DUPLICATE KEY UPDATE UpdateTime = NOW(),
-            """);
-        if (recycle)
-        {
-            sql.Append($"CurrentVal = (@updatedVal := IF(CurrentVal + {add} >= MaxVal, {add}, CurrentVal + {add}));");
-        }
-        else
-        {
-            sql.Append($"CurrentVal = (@updatedVal := CurrentVal + {add});");
-        }
-        sql.Append("SELECT @updatedVal;");
-        var result = await DatabaseHelper.QueryScalarAsync(_connectionString, sql.ToString());
-        return Convert.ToInt64(result);
-    }
-
-    public async Task<long> PeekKey(SeqConfig config)
-    {
-        var sql = $"SELECT CurrentVal FROM seq WHERE SeqName = '{config.Name}' LIMIT 1;";
-        return Convert.ToInt64(await DatabaseHelper.QueryScalarAsync(_connectionString, sql));
-    }
-    
-    public async Task<long[]> GetKeys(SeqConfig config, int count)
-    {
-        if (count < 1) return [];
-
-        var list = new long[count];
-        var add = config.Step * count;
-        var lastId = await UpdateSequenceID(config.Name, config.Step, config.Max, config.Recycle, add);
-        var step = Convert.ToInt64(config.Step);
-        for (var i = count - 1; i > -1; i--)
-        {
-            list[i] = lastId;
-            lastId -= step;
-        }
-
-        return list;
-    }
-
-    /// <summary>
-    /// 添加并取得一个缓存的流水号
-    /// </summary>
-    /// <param name="config"></param>
-    /// <returns></returns>
-    public long AddCachedSeq(SeqConfig config)
-    {
-        if (!_cachedSequence.TryGetValue(config, out var val))
-        {
-            var seq = PeekKey(config).GetAwaiter().GetResult();
-            val = seq;
-            _cachedSequence[config] = val;
-        }
-
-        var step = config.Step;
-        if (config.Recycle)
-        {
-            val = val + step >= config.Max ? val : val + step;
-        }
-        else val += step;
-        _cachedSequence[config] = val;
-        return val;
-    }
-    
-    /// <summary>
-    /// 移除一个缓存的流水号
-    /// </summary>
-    /// <param name="config"></param>
-    public bool RemoveCachedSeq(SeqConfig config)
-    {
-        return _cachedSequence.Remove(config, out _);
-    }
-    
-    /// <summary>
-    /// 清空所有缓存的流水号
-    /// </summary>
-    public void ClearCache()
-    {
-        _cachedSequence.Clear();
-    }
-
-    /// <summary>
-    /// 将缓存的流水号应用至数据库
-    /// </summary>
-    public async Task ApplyToDatabaseAsync()
-    {
-        if (_cachedSequence.Count == 0) return;
-        var sql = GenerateCachedSeqSql();
-        await DatabaseHelper.NonQueryAsync(_connectionString, sql);
-    }
-
-    private string GenerateCachedSeqSql()
-    {
-        var sb = new StringBuilder();
-        foreach (var kv in _cachedSequence)
-        {
-            sb.AppendLine($"UPDATE seq SET CurrentVal = {kv.Value} WHERE SeqName = '{kv.Key.Name}';");
-        }
-
-        return sb.ToString();
-    }
-}

MesETL.App/Services/TaskManager.cs

@@ -1,86 +0,0 @@
-using Serilog;
-using ApplicationException = System.ApplicationException;
-using TaskExtensions = MesETL.Shared.Helper.TaskExtensions;
-
-namespace MesETL.App.Services;
-
-/// <summary>
-/// 快速批量创建和等待任务
-/// </summary>
-public class TaskManager
-{
-    private int _runningTaskCount;
-
-    public int RunningTaskCount => _runningTaskCount;
-    public int MaxTaskCount { get; }
-
-    public event Action<Exception>? OnException;
-    public event Action? OnTaskCompleteSuccessfully;
-
-    public TaskManager(int maxTaskCount)
-    {
-        MaxTaskCount = maxTaskCount;
-    }
-
-    public async ValueTask<Task> CreateTaskAsync(Func<Task> func, CancellationToken cancellationToken = default)
-    {
-        await TaskExtensions.WaitUntil(() => _runningTaskCount < MaxTaskCount, 25, cancellationToken);
-        return RunTask(func, cancellationToken);
-    }
-
-    public async ValueTask<Task> CreateTaskAsync(Func<object?, Task> func, object? arg, CancellationToken ct = default)
-    {
-        await TaskExtensions.WaitUntil(() => _runningTaskCount < MaxTaskCount, 25, ct);
-        return RunTaskNoClosure(func, arg, ct);
-    }
-    
-    private Task RunTask(Func<Task> func, CancellationToken cancellationToken = default)
-    {
-        var task = Task.Run(async () =>
-        {
-            // Log.Logger.Verbose("[任务管理器] 新的任务已创建");
-            Interlocked.Increment(ref _runningTaskCount);
-            try
-            {
-                await func();
-                OnTaskCompleteSuccessfully?.Invoke();
-            }
-            catch(Exception ex)
-            {
-                OnException?.Invoke(ex);
-                Log.Logger.Error(ex, "[任务管理器] 执行任务时出错");
-            }
-            finally
-            {
-                Interlocked.Decrement(ref _runningTaskCount);
-            }
-        }, cancellationToken);
-        return task;
-    }
-
-    private Task RunTaskNoClosure(Func<object?, Task> func, object? arg, CancellationToken cancellationToken = default)
-    {
-        var task = Task.Factory.StartNew(async obj => // 性能考虑，这个lambda中不要捕获任何外部变量!
-        {
-            // Log.Logger.Verbose("[任务管理器] 新的任务已创建");
-            if (obj is not Tuple<Func<object?, Task>, object?> tuple)
-                throw new ApplicationException("这个异常不该出现");
-            Interlocked.Increment(ref _runningTaskCount);
-            try
-            {
-                await tuple.Item1(tuple.Item2);
-                OnTaskCompleteSuccessfully?.Invoke();
-            }
-            catch(Exception ex)
-            {
-                OnException?.Invoke(ex);
-                Log.Logger.Error(ex, "[任务管理器] 执行任务时出错");
-            }
-            finally
-            {
-                Interlocked.Decrement(ref _runningTaskCount);
-            }
-        }, Tuple.Create(func, arg), cancellationToken).Unwrap();
-        return task;
-    }
-}

MesETL.App/appsettings.json

@@ -1,72 +0,0 @@
-{
-  "MemoryThreshold": 6,
-  "GCIntervalMilliseconds": -1,
-  "UnsafeVariable": true,
-  "DryRun": true, // 试运行，仅输入每张表的前100000条数据
-  "Logging": {
-    "LogLevel": {
-      "Default": "Trace"
-    }
-  },
-  "Input":{
-    "InputDir": "D:\\Data\\DatabaseDump\\Prod_Mock_CSV_2024-12-31", // Csv数据输入目录
-    "UseMock": false, // 使用模拟数据进行测试
-    "MockCountMultiplier": 1, // 模拟数据量级的乘数
-//    "TableOrder": ["order_item"], // 按顺序输入的表
-    "TableIgnoreList": [] // 忽略输入的表
-  },
-  "Transform":{
-    "StrictMode": true, // 设为true时如果数据转换发生错误，立刻停止程序
-    "EnableFilter": true, // 启用数据过滤
-    "EnableReplacer": true, // 启用数据修改
-    "EnableReBuilder": true, // 启用数据重建
-    "CleanDate": "202401" // 当数据过滤开启时，删除这个时间之前的数据
-  },
-  "Output":{
-    "ConnectionString": "Server=127.0.0.1;Port=3306;UserId=root;Password=123456;", // 要分库，不用加'Database='了
-    "MaxAllowedPacket": 67108864, 
-    "FlushCount": 10000, // 每次提交记录条数
-    "MaxDatabaseOutputTask" : 4, // 每个数据库最大提交任务数
-    "TreatJsonAsHex": false, // 使Json列输出时带上"0x"前缀
-    "NoOutput": [], // 不输出的表
-    "ForUpdate":
-    {
-    }
-  },  
-  "RecordQueue":{
-    "ProducerQueueLength": 20000, // 输入队列最大长度
-    "ConsumerQueueLength": 20000, // 每个输出队列最大长度
-    "MaxByteCount": 3221225472 // 队列最大字节数
-  },
-  "RedisCache": {
-    "Configuration": "192.168.1.246:6380",
-    "InstanceName" : "mes-etl:" 
-  },
-  "TenantDb": // 分库配置
-  {
-    "TenantKey" : "CompanyID",
-    "UseDbGroup": "prod",
-    "DbGroups": {
-      "test": {
-        "cferp_test_1": 1000,
-        "cferp_test_2": 2000,
-        "cferp_test_3": 2147483647
-      },
-      "prod":{
-        "mesdb_1": 5000,
-        "mesdb_2": 7500,
-        "mesdb_3": 10000,
-        "mesdb_4": 15000,
-        "mesdb_5": 20000,
-        "mesdb_6": 2147483647
-      },
-      "mock_void":{
-        "mesdb_1_void": 5000,
-        "mesdb_2_void": 10000,
-        "mesdb_3_void": 15000,
-        "mesdb_4_void": 20000,
-        "mesdb_5_void": 2147483647
-      }
-    }
-  }
-}

MesETL.Clean/MesETL.Clean.csproj

@@ -1,14 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-    <PropertyGroup>
-        <OutputType>Exe</OutputType>
-        <TargetFramework>net8.0</TargetFramework>
-        <ImplicitUsings>enable</ImplicitUsings>
-        <Nullable>enable</Nullable>
-    </PropertyGroup>
-
-    <ItemGroup>
-      <ProjectReference Include="..\MesETL.Shared\MesETL.Shared.csproj" />
-    </ItemGroup>
-
-</Project>

MesETL.Clean/Program.cs

@@ -1,55 +0,0 @@
-using MesETL.Shared.Helper;
-
-var connStr = GetArg("-s") ?? throw new ApplicationException("未配置数据库连接字符串");
-var eachLimit = int.Parse(GetArg("-l") ?? "1000");
-var parallelTask = int.Parse(GetArg("-p") ?? "4");
-
-var deletionCount = 0;
-
-Console.WriteLine("Running Deletion...");
-
-_ = Task.Run(async () =>
-{
-    while (true)
-    {
-        await Task.Delay(5000);
-        Console.WriteLine($"[{DateTime.Now}] DELETE COUNT: {deletionCount}");
-    }
-});
-
-await Parallel.ForAsync(0, parallelTask, async (i, token) =>
-{
-    while (true)
-    {
-        var effectRows = await DatabaseHelper.NonQueryAsync(connStr,
-            $"DELETE FROM `order_data_block` WHERE CompanyID = 0 ORDER BY ID LIMIT {eachLimit};", token);
-        if(effectRows == 0)
-            break;
-        Interlocked.Add(ref deletionCount, effectRows);
-    }
-});
-
-Console.WriteLine($"[{DateTime.Now}] DELETE COUNT: {deletionCount}");
-return;
-string? GetArg(string instruct)
-{
-    var idx = Array.IndexOf(args, instruct);
-    if (idx == -1)
-        return null;
-    if (args[idx + 1].StartsWith('-'))
-        throw new ArgumentException("Argument Lost", nameof(instruct));
-    return args[idx + 1];
-}
-
-// var match = await DatabaseHelper.QueryTableAsync(connStr, 
-//     $"SELECT `ID` FROM `order_data_block` WHERE CompanyID = 0 LIMIT {eachLimit};", 
-//     token);
-// var rows = match.Tables[0].Rows;
-// if (rows.Count == 0)
-//     return;
-//     
-// foreach (DataRow row in rows)
-// {
-//     var id = row["ID"].ToString();
-//     await DatabaseHelper.NonQueryAsync(connStr, $"DELETE FROM `order_data_block` WHERE `ID` = {id}", token);
-// }

MesETL.Shared/Compression/DeflateArchive.cs

@@ -1,39 +0,0 @@
-using System.IO.Compression;
-
-namespace MesETL.Shared.Compression;
-
-/// <summary>
-/// Deflate压缩工具类
-/// </summary>
-public static class DeflateArchive
-{
-    /// <summary>
-    /// 解压Deflate
-    /// </summary>
-    /// <param name="input"></param>
-    /// <returns></returns>
-    public static byte[] Decompress(byte[] input)
-    {
-        using var msi = new MemoryStream(input);
-        using var mso = new MemoryStream();
-        using var ds = new DeflateStream(msi, CompressionMode.Decompress);
-        ds.CopyTo(mso);
-        ds.Flush();
-        return mso.ToArray();
-    }
-
-    /// <summary>
-    /// 压缩Deflate
-    /// </summary>
-    /// <param name="input"></param>
-    /// <returns></returns>
-    public static byte[] Compress(byte[] input)
-    {
-        using var msi = new MemoryStream(input);
-        using var mso = new MemoryStream();
-        using var ds = new DeflateStream(mso, CompressionMode.Compress);
-        msi.CopyTo(ds);
-        ds.Flush();
-        return mso.ToArray();
-    }
-}

MesETL.Shared/Helper/DatabaseHelper.cs

@@ -1,97 +0,0 @@
-using System.Data;
-using MySqlConnector;
-
-namespace MesETL.Shared.Helper;
-
-public static class DatabaseHelper
-{
-    /// <summary>
-    /// 创建一个MySql连接
-    /// </summary>
-    /// <param name="connStr"></param>
-    /// <returns></returns>
-    public static MySqlConnection CreateConnection(string connStr)
-    {
-        var newConnStr = new MySqlConnectionStringBuilder(connStr)
-        {
-            ConnectionTimeout = 30,
-            DefaultCommandTimeout = 0,
-        }.ConnectionString;
-        return new MySqlConnection(newConnStr);
-    }
-    
-    /// <summary>
-    /// 使用语句查询数据库
-    /// </summary>
-    /// <param name="connStr"></param>
-    /// <param name="sql"></param>
-    /// <param name="ct"></param>
-    /// <returns></returns>
-    public static async Task<DataSet> QueryTableAsync(string connStr, string sql, CancellationToken ct = default)
-    {
-        await using var conn = CreateConnection(connStr);
-        if(conn.State is not ConnectionState.Open)
-            await conn.OpenAsync(ct);
-        await using var cmd = conn.CreateCommand();
-        cmd.CommandText = sql;
-        var ds = new DataSet();
-        new MySqlDataAdapter(cmd).Fill(ds);
-        return ds;
-    }
-
-    /// <summary>
-    /// 使用语句进行标量查询
-    /// </summary>
-    /// <param name="connStr"></param>
-    /// <param name="sql"></param>
-    /// <param name="ct"></param>
-    /// <returns></returns>
-    public static async Task<object?> QueryScalarAsync(string connStr, string sql, CancellationToken ct = default)
-    {
-        await using var conn = CreateConnection(connStr);
-        if(conn.State is not ConnectionState.Open)
-            await conn.OpenAsync(ct);
-        await using var cmd = conn.CreateCommand();
-        cmd.CommandText = sql;
-        return await cmd.ExecuteScalarAsync(ct);
-    }
-
-    /// <summary>
-    /// 执行非查询语句
-    /// </summary>
-    /// <param name="connStr"></param>
-    /// <param name="sql"></param>
-    /// <param name="ct"></param>
-    /// <returns></returns>
-    public static async Task<int> NonQueryAsync(string connStr, string sql, CancellationToken ct = default)
-    {
-        await using var conn = CreateConnection(connStr);
-        if(conn.State is not ConnectionState.Open)
-            await conn.OpenAsync(ct);
-        await using var cmd = conn.CreateCommand();
-        cmd.CommandText = sql;
-        return await cmd.ExecuteNonQueryAsync(ct);
-    }
-
-    /// <summary>
-    /// 在事务中执行语句
-    /// </summary>
-    /// <param name="connStr"></param>
-    /// <param name="sql"></param>
-    /// <param name="parameters"></param>
-    /// <returns></returns>
-    public static async Task<int> TransactionAsync(string connStr, string sql, params MySqlParameter[] parameters)
-    {
-        await using var conn = CreateConnection(connStr);
-        if(conn.State is not ConnectionState.Open)
-            await conn.OpenAsync();
-        await using var trans = await conn.BeginTransactionAsync();
-        await using var cmd = conn.CreateCommand();
-        cmd.CommandText = sql;
-        cmd.Transaction = trans;
-        cmd.Parameters.AddRange(parameters);
-        var rows = await cmd.ExecuteNonQueryAsync();
-        await trans.CommitAsync();
-        return rows;
-    }
-}

MesETL.Shared/Helper/Extensions.IEnumerable.cs

@@ -1,71 +0,0 @@
-using System.Diagnostics.CodeAnalysis;
-using System.Reflection;
-
-namespace MesETL.Shared.Helper;
-#nullable disable
-public static class EnumerableExtensions
-{
-    public static string ToMarkdownTable<
-        [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicProperties | 
-                                    DynamicallyAccessedMemberTypes.NonPublicProperties |
-                                    DynamicallyAccessedMemberTypes.PublicFields |
-                                    DynamicallyAccessedMemberTypes.NonPublicFields)] T>(this IEnumerable<T> source)
-    {
-        var properties = typeof(T).GetRuntimeProperties();
-        var fields = typeof(T)
-            .GetRuntimeFields()
-            .Where(f => f.IsPublic);
-
-        var gettables = Enumerable.Union(
-            properties.Select(p => new { p.Name, GetValue = (Func<object, object>)p.GetValue, Type = p.PropertyType }),
-            fields.Select(p => new { p.Name, GetValue = (Func<object, object>)p.GetValue, Type = p.FieldType }));
-
-        var maxColumnValues = source
-            .Select(x => gettables.Select(p => p.GetValue(x)?.ToString()?.Length ?? 0))
-            .Union(new[] { gettables.Select(p => p.Name.Length) }) // Include header in column sizes
-            .Aggregate(
-                new int[gettables.Count()].AsEnumerable(),
-                (accumulate, x) => accumulate.Zip(x, Math.Max))
-            .ToArray();
-
-        var columnNames = gettables.Select(p => p.Name);
-
-        var headerLine = "| " + string.Join(" | ", columnNames.Select((n, i) => n.PadRight(maxColumnValues[i]))) + " |";
-
-        var isNumeric = new Func<Type, bool>(type =>
-            type == typeof(Byte) ||
-            type == typeof(SByte) ||
-            type == typeof(UInt16) ||
-            type == typeof(UInt32) ||
-            type == typeof(UInt64) ||
-            type == typeof(Int16) ||
-            type == typeof(Int32) ||
-            type == typeof(Int64) ||
-            type == typeof(Decimal) ||
-            type == typeof(Double) ||
-            type == typeof(Single));
-
-        var rightAlign = new Func<Type, char>(type => isNumeric(type) ? ':' : ' ');
-
-        var headerDataDividerLine =
-            "| " +
-            string.Join(
-                "| ",
-                gettables.Select((g, i) => new string('-', maxColumnValues[i]) + rightAlign(g.Type))) +
-            "|";
-
-        var lines = new[]
-        {
-            headerLine,
-            headerDataDividerLine,
-        }.Union(
-            source
-                .Select(s =>
-                    "| " + string.Join(" | ",
-                        gettables.Select((n, i) => (n.GetValue(s)?.ToString() ?? "").PadRight(maxColumnValues[i]))) +
-                    " |"));
-
-        return lines
-            .Aggregate((p, c) => p + Environment.NewLine + c);
-    }
-}

MesETL.Shared/Helper/Extensions.Lang.cs

@@ -1,63 +0,0 @@
-namespace Azusa.Shared.Extensions;
-
-/// <summary>
-/// 使用Range作为参数的迭代器方法
-/// <br/>
-/// 扩展foreach关键字来实现类似<c>foreach (var i in 1..5)</c>的效果
-/// </summary>
-public static class ForeachExtensions
-{
-    /// <summary>
-    /// 拓展Range结构实现GetEnumerator方法供foreach读取，实现foreach(var i in x..y)
-    /// </summary>
-    /// <returns></returns>
-    public static CustomIntEnumerator GetEnumerator(this Range range)
-    {
-        return new CustomIntEnumerator(range);
-    }
-
-    /// <summary>
-    /// 拓展int类实现GetEnumerator方法供foreach读取，实现foreach(var i in x)
-    /// </summary>
-    /// <param name="end"></param>
-    /// <returns></returns>
-    public static CustomIntEnumerator GetEnumerator(this int end)
-    {
-        return new CustomIntEnumerator(end);
-        return new CustomIntEnumerator(new Range(0, end));//在执行空函数时性能比上一句低10倍，Why
-    }
-}
-
-//使用引用结构体增强性能
-public ref struct CustomIntEnumerator
-{
-    private int _current;
-    private readonly int _end;
-
-    public CustomIntEnumerator(Range range)
-    {
-        //避免某些时候从结尾开始编制
-        // x.. 时会产生Range(x,^0)
-        if (range.End.IsFromEnd)
-        {
-            throw new NotSupportedException("不支持从结尾编制索引");
-        }
-        _current = range.Start.Value - 1;
-        _end = range.End.Value - 1;//迭代器不包含范围的尾部
-    }
-
-    public CustomIntEnumerator(int end)
-    {
-        _current = -1;
-        _end = end;
-    }
-
-    /* 注意，供foeach使用的迭代器不需要实现IEnumerator接口，只需要提供Current属性以及MoveNext方法即可，*/
-    public int Current => _current;
-
-    public bool MoveNext()
-    {
-        _current++;
-        return _current <= _end;
-    }
-}

MesETL.Shared/Helper/Extensions.Task.cs

@@ -1,19 +0,0 @@
-namespace MesETL.Shared.Helper;
-
-public static class TaskExtensions
-{
-    public static async Task WaitUntil(Func<bool> condition, int pollDelay = 25, CancellationToken ct = default)
-    {
-        try
-        {
-            while (!condition())
-            {
-                await Task.Delay(pollDelay, ct);
-            }
-        }
-        catch(TaskCanceledException)
-        {
-            // CancellationToken激活时Task.Delay通过抛异常来结束停止等待，不用管它
-        }
-    }
-}

MesETL.Shared/MesETL.Shared.csproj

@@ -1,13 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-    <PropertyGroup>
-        <TargetFramework>net8.0</TargetFramework>
-        <ImplicitUsings>enable</ImplicitUsings>
-        <Nullable>enable</Nullable>
-    </PropertyGroup>
-
-    <ItemGroup>
-      <PackageReference Include="MySqlConnector" Version="2.4.0" />
-    </ItemGroup>
-
-</Project>

MesETL.Test/DataFix.cs

@@ -1,142 +0,0 @@
-using System.Data;
-using MesETL.App.Helpers;
-using MesETL.Shared.Helper;
-using MySqlConnector;
-using Xunit.Abstractions;
-
-namespace TestProject1;
-
-public class DataFix
-{
-    public const string ConnStr = "Server=192.168.1.245;Port=3306;UserId=root;Password=ruixinjie!@#123;";
-    public static string[] DbNames = ["cferp_test_1", "cferp_test_2", "cferp_test_3"];
-    
-    private readonly ITestOutputHelper _output;
-    
-    public DataFix(ITestOutputHelper output)
-    {
-        _output = output;
-    }
-    
-    private string MakeConnStr(string dbName) => ConnStr + $"Database={dbName};";
-
-    /// <summary>
-    /// 查找cferp_test_1中CompanyID = 0的order_box_block，根据OrderNo查找对应订单的CompanyID，然后删除重新插入相应的数据库
-    /// 如果没有找到对应订单，则保留CompanyID为0
-    /// </summary>
-    [Fact]
-    public async Task FixOrderBoxBlockCompanyID()
-    {
-        var ds = await DatabaseHelper.QueryTableAsync(MakeConnStr(DbNames[0]), "SELECT * FROM `order_box_block` WHERE COMPANYID = 0");
-        var dict = new Dictionary<long, int>();//orderNo -> CompanyID
-        foreach (DataRow row in ds.Tables[0].Rows)
-        {
-            // foreach (var column in row.ItemArray)
-            // {
-            //     Console.Write(column.ToString() + '\t');
-            // }
-
-            var orderNo = Convert.ToInt64(row["OrderNo"]);
-            var boxId = Convert.ToInt64(row["BoxID"]);
-
-            int? companyId = null;
-            foreach (var db in DbNames)
-            {
-                if(!dict.TryGetValue(orderNo, out var cid)) // 可以提到外面
-                {
-                    var result = await DatabaseHelper.QueryScalarAsync(ConnStr + $"Database={db};",
-                        $"SELECT CompanyID FROM `order` WHERE OrderNo = {orderNo}");
-                    if (result is null or 0) continue;
-                    
-                    companyId = Convert.ToInt32(result);
-                    dict.Add(orderNo, companyId.Value);
-                    break;
-                }
-                else
-                {
-                    companyId = cid;
-                }
-            }
-
-            if (companyId is null or 0)
-            {
-                _output.WriteLine($"OrderBoxBlock:{boxId} - OrderNo {orderNo} not found");
-                continue;
-            }
-
-            row["CompanyID"] = companyId;
-            await DatabaseHelper.TransactionAsync(ConnStr,
-                $"""
-                 DELETE FROM cferp_test_1.`order_box_block` WHERE BoxID = {boxId};
-                 INSERT INTO {TenantDbHelper.GetDbNameByTenantKeyValue(companyId.Value)}.`order_box_block`
-                 VALUES(@c1, @c2, @c3, @c4, @c5);
-                 """, [new MySqlParameter("c1", row[0]),
-                    new MySqlParameter("c2", row[1]),
-                    new MySqlParameter("c3", row[2]),
-                    new MySqlParameter("c4", row[3]),
-                    new MySqlParameter("c5", row[4])]);
-            _output.WriteLine($"OrderBoxBock:{boxId} CompanyID -> {companyId}");
-        }
-    }
-
-    [Theory]
-    [InlineData(["order_data_block", "ID", 0])]
-    [InlineData(["order_data_block", "ID", 1])]
-    [InlineData(["order_data_block", "ID", 2])]
-    public async Task FixCompanyIdWithOwnOrderNo(string tableName, string keyName, int dbNameIndex)
-    {
-        var ds = await DatabaseHelper.QueryTableAsync(MakeConnStr(DbNames[dbNameIndex]), 
-            $"SELECT * FROM `{tableName}` WHERE COMPANYID = 0");
-        var dict = new Dictionary<long, int>();//orderNo -> CompanyID
-        foreach (DataRow row in ds.Tables[0].Rows)
-        {
-            // foreach (var column in row.ItemArray)
-            // {
-            //     Console.Write(column.ToString() + '\t');
-            // }
-
-            var orderNo = Convert.ToInt64(row["OrderNo"]);
-            var key = Convert.ToInt32(row[keyName]);
-
-            int? companyId = null;
-            if(!dict.TryGetValue(orderNo, out var cid))
-            {
-                companyId = await MesDatabaseHelper.TrySearchCompanyId(ConnStr, DbNames,
-                    $"SELECT CompanyID FROM `order` WHERE OrderNo = {orderNo}");
-            }
-            else
-            {
-                companyId = cid;
-            }
-
-            if (companyId is null or 0)
-            {
-                _output.WriteLine($"{tableName}:{key} - OrderNo {orderNo} not found");
-                continue;
-            }
-
-            row["CompanyID"] = companyId;
-            // ON DUPLICATE KEY UPDATE
-            await DatabaseHelper.TransactionAsync(ConnStr,
-                $"""
-                 DELETE FROM {DbNames[dbNameIndex]}.`{tableName}` WHERE ID = {key};
-                 INSERT INTO {TenantDbHelper.GetDbNameByTenantKeyValue(companyId.Value)}.`{tableName}`
-                 VALUES({string.Join(',', Enumerable.Range(0, row.ItemArray.Length).Select(i => $"@c{i}"))})
-                 ON DUPLICATE KEY UPDATE CompanyID = {companyId};
-                 """, 
-                Enumerable.Range(0, row.ItemArray.Length).Select(i => new MySqlParameter($"c{i}", row[i])).ToArray());
-            _output.WriteLine($"{tableName}:{key} CompanyID -> {companyId}");
-        }
-    }
-
-    [Theory]
-    [InlineData("order", 0)]
-    [InlineData("order", 1)]
-    [InlineData("order", 2)]
-    public async Task FixShardKeyWithOwnOrderNo(string tableName, int dbNameIndex)
-    {
-        var r = await DatabaseHelper.NonQueryAsync(MakeConnStr(DbNames[dbNameIndex]),
-            $"UPDATE `{tableName}` SET ShardKey = CONCAT(SUBSTR(`order`.OrderNo,3,4),'0') WHERE ShardKey = 0");
-        _output.WriteLine($"Affect Rows: {r}");
-    }
-}

MesETL.Test/DatabaseToolBox.cs

@@ -1,224 +0,0 @@
-using System.Data;
-using System.Text;
-using MesETL.App.Helpers;
-using MesETL.Shared.Helper;
-using MySqlConnector;
-using Newtonsoft.Json;
-using Newtonsoft.Json.Linq;
-using Xunit.Abstractions;
-
-namespace TestProject1;
-
-public class DatabaseToolBox
-{
-    private readonly ITestOutputHelper _output;
-    public const string ConnStr = "Server=localhost;Port=3306;UserId=root;Password=123456;";
-
-    public DatabaseToolBox(ITestOutputHelper output)
-    {
-        _output = output;
-    }
-
-    [Theory]
-    [InlineData("cferp_void_1")]
-    public async Task AlterAllTableToBlackHole(string database)
-    {
-        var connStr = ConnStr + $"Database={database}";
-        var tables = await DatabaseHelper.QueryTableAsync(connStr,
-            $"""
-             SELECT TABLE_NAME FROM information_schema.`TABLES` WHERE TABLE_SCHEMA = '{database}';
-             """);
-
-        foreach (DataRow row in tables.Tables[0].Rows)
-        {
-            var tableName = row["TABLE_NAME"].ToString();
-            var sql = $"""
-                       ALTER TABLE `{tableName}` REMOVE PARTITIONING;
-                       """;
-            try
-            {
-                await DatabaseHelper.NonQueryAsync(connStr, sql);
-            }
-            catch (MySqlException e) when (e.ErrorCode == MySqlErrorCode.PartitionManagementOnNoPartitioned)
-            {
-            }
-
-            sql = $"""
-                   ALTER TABLE `{tableName}` ENGINE=BLACKHOLE;
-                   """;
-            await DatabaseHelper.NonQueryAsync(connStr, sql);
-        }
-    }
-
-    public async Task<long> CountAllTable(string connStr, string database)
-    {
-        var count = 0L;
-        var set = await DatabaseHelper.QueryTableAsync(connStr,
-            $"""
-             SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = '{database}';
-             """);
-        foreach (DataRow row in set.Tables[0].Rows)
-        {
-            count += Convert.ToInt64(await DatabaseHelper.QueryScalarAsync(connStr,
-                $"""
-                 SELECT COUNT(1) FROM `{database}`.`{row[0]}`;
-                 """));
-        }
-        _output.WriteLine($"Record count: {count} records");
-        return count;
-    }
-
-    [Theory]
-    [InlineData([new[]{"cferp_test_1", "cferp_test_2", "cferp_test_3"}])]
-    public async Task CountAllDatabase(IEnumerable<string> databases)
-    {
-        var count = 0L;
-        var connStr = "Server=192.168.1.245;Port=3306;UserId=root;Password=ruixinjie!@#123;";
-        foreach (var db in databases)
-        {
-            count += await CountAllTable(connStr, db);
-        }
-
-        _output.WriteLine(count.ToString());
-    }
-
-    public async Task<TableIndex[]> GetAllTableIndexes(string database)
-    {
-        var data = await DatabaseHelper.QueryTableAsync(ConnStr,
-            $"""
-            SELECT TABLE_NAME, INDEX_NAME, NON_UNIQUE, COLUMN_NAME, INDEX_TYPE FROM information_schema.STATISTICS
-                     WHERE TABLE_SCHEMA = '{database}'
-                       AND INDEX_NAME != 'PRIMARY'
-                       AND INDEX_TYPE = 'BTREE'
-            """);
-        return data.Tables[0].Rows.Cast<DataRow>().Select(row =>
-        {
-            return new TableIndex(row["TABLE_NAME"].ToString()!,
-                row["INDEX_NAME"].ToString()!,
-                !Convert.ToBoolean(row["NON_UNIQUE"]),
-                row["COLUMN_NAME"].ToString()!,
-                row["INDEX_TYPE"] switch
-                {
-                    "BTREE" => TableIndex.TableIndexType.BTree,
-                    "PRIMARY" => TableIndex.TableIndexType.Primary,
-                    "HASH" => TableIndex.TableIndexType.Hash,
-                    _ => throw new ArgumentOutOfRangeException(nameof(row))
-                }
-            );
-        }).ToArray();
-    }
-
-    [Theory]
-    [InlineData(["mesdb_1"])]
-    [InlineData(["mesdb_2"])]
-    [InlineData(["mesdb_3"])]
-    [InlineData(["mesdb_4"])]
-    [InlineData(["mesdb_5"])]
-    public async Task ShowIndex(string database)
-    {
-        var indexes = await GetAllTableIndexes(database);
-        var sb = new StringBuilder();
-        foreach (var (tableName, indexName, isUnique, columnName, tableIndexType) in indexes!)
-        {
-            sb.AppendLine($"Drop {(isUnique ? "UNIQUE" : string.Empty)} INDEX `{indexName}` ON `{database}`.`{tableName}`;");
-        }
-
-        _output.WriteLine(sb.ToString());
-    }
-
-    [Theory]
-    [InlineData(["cferp_test_1", "D:/Indexes_cferp_test_1.json"])]
-    [InlineData(["cferp_test_2", "D:/Indexes_cferp_test_2.json"])]
-    [InlineData(["cferp_test_3", "D:/Indexes_cferp_test_3.json"])]
-    public async Task ExportAllIndexes(string database, string outputPath)
-    {
-        var indexes = await GetAllTableIndexes(database);
-        var json = JArray.FromObject(indexes);
-        await File.WriteAllTextAsync(outputPath, json.ToString());
-        _output.WriteLine($"Exported {indexes.Length} indexes to '{outputPath}'");
-    }
-
-    [Theory]
-    [InlineData("cferp_test_1", "D:/Indexes_cferp_test_1.json")]
-    [InlineData("cferp_test_2", "D:/Indexes_cferp_test_2.json")]
-    [InlineData("cferp_test_3", "D:/Indexes_cferp_test_3.json")]
-    public async Task ImportAllIndexes(string database, string importPath)
-    {
-        var json = await File.ReadAllTextAsync(importPath);
-        var indexes = JsonConvert.DeserializeObject<TableIndex[]>(json);
-        var sb = new StringBuilder();
-        foreach (var (tableName, indexName, isUnique, columnName, tableIndexType) in indexes!)
-        {
-            sb.AppendLine($"CREATE {(isUnique ? "UNIQUE" : string.Empty)} INDEX `{indexName}` ON `{database}`.`{tableName}` (`{columnName}`) USING {tableIndexType};");
-        }
-        await DatabaseHelper.NonQueryAsync(ConnStr, sb.ToString());
-        _output.WriteLine($"Import {indexes.Length} indexes to '{database}' from '{importPath}'");
-    }
-
-    [Theory]
-    [InlineData(["mesdb_1"])]
-    [InlineData(["mesdb_2"])]
-    [InlineData(["mesdb_3"])]
-    [InlineData(["mesdb_4"])]
-    [InlineData(["mesdb_5"])]
-    public async Task DropAllIndex(string database)
-    {
-        var indexes = await GetAllTableIndexes(database);
-        var sb = new StringBuilder();
-        foreach (var (tableName, indexName) in indexes)
-        {
-            sb.AppendLine($"DROP INDEX `{indexName}` ON `{database}`.`{tableName}`;");
-        }
-        await DatabaseHelper.NonQueryAsync(ConnStr, sb.ToString());
-        _output.WriteLine($"Dropped {indexes.Length} indexes from {database}");
-    }
-
-    [Theory]
-    [InlineData("mesdb_1")]
-    [InlineData("mesdb_2")]
-    [InlineData("mesdb_3")]
-    [InlineData("mesdb_4")]
-    [InlineData("mesdb_5")]
-    [InlineData("mesdb_6")]
-    public async Task TruncateAllTable(string database)
-    {
-        var tables = await DatabaseHelper.QueryTableAsync(ConnStr,
-            $"""
-             SELECT TABLE_NAME FROM information_schema.`TABLES` WHERE TABLE_SCHEMA = '{database}';
-             """);
-        var sb = new StringBuilder();
-        sb.AppendLine($"USE `{database}`;");
-        foreach (DataRow row in tables.Tables[0].Rows)
-        {
-            var tableName = row["TABLE_NAME"].ToString();
-            var sql = $"""
-                       TRUNCATE TABLE `{tableName}`;
-                       """;
-            sb.AppendLine(sql);
-        }
-        await DatabaseHelper.NonQueryAsync(ConnStr, sb.ToString());
-    }
-
-    [Theory]
-    [InlineData("cferp_test_1")]
-    [InlineData("cferp_test_2")]
-    [InlineData("cferp_test_3")]
-    public async Task AnalyzeAllTable(string database)
-    {
-        var tables = await DatabaseHelper.QueryTableAsync(ConnStr,
-            $"""
-             SELECT TABLE_NAME FROM information_schema.`TABLES` WHERE TABLE_SCHEMA = '{database}';
-             """);
-        var sb = new StringBuilder();
-        sb.AppendLine($"USE `{database}`;");
-        foreach (DataRow row in tables.Tables[0].Rows)
-        {
-            var tableName = row["TABLE_NAME"].ToString();
-            var sql = $"""
-                       ANALYZE TABLE `{tableName}`;
-                       """;
-            sb.AppendLine(sql);
-        }
-        await DatabaseHelper.NonQueryAsync(ConnStr, sb.ToString());
-    }
-}

MesETL.Test/GlobalUsings.cs

@@ -1 +0,0 @@
-global using Xunit;

MesETL.Test/InputServiceTest.cs

@@ -1,54 +0,0 @@
-using MesETL.App.Helpers;
-using MesETL.App.HostedServices;
-using MesETL.App.Options;
-using MesETL.App.Services;
-using MesETL.App.Services.ETL;
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.Options;
-using TestProject1.XUnit;
-using Xunit.Abstractions;
-
-namespace TestProject1;
-
-public class InputServiceTest : TestBase
-{
-    private readonly ITestOutputHelper _output;
-
-    public InputServiceTest(ITestOutputHelper output) : base(output)
-    {
-        _output = output;
-    }
-
-    /// <summary>
-    /// 测试文件输入服务是否能正确的认到应有的文件
-    /// </summary>
-    /// <param name="inputDir"></param>
-    /// <param name="tableOrder"></param>
-    /// <param name="ignored"></param>
-    /// <param name="assertCount"></param>
-    [Theory]
-    [InlineData(@"D:\Data\DatabaseDump\MyDumper-ZST 2024-12-3", null, new string[0], 152)] // 没有seq和三个库的efmigration
-    [InlineData(@"D:\Data\DatabaseDump\MyDumper-ZST 2024-12-3", new[] { "order", "machine" }, new string[0],
-        11)] // 只有order和machine
-    [InlineData(@"D:\Data\DatabaseDump\MyDumper-ZST 2024-12-3", null, new[] { "order", "machine" },
-        152 - 11)] // 忽略order和machine
-    public void Test_InputInfo_Get_And_Order(string inputDir, string[]? tableOrder, string[] ignored, int assertCount)
-    {
-        var options = new OptionsWrapper<DataInputOptions>(new DataInputOptions()
-        {
-            InputDir = inputDir,
-            FileInputMetaBuilder = DumpDataHelper.MyDumperFileInputMetaBuilder,
-            TableOrder = tableOrder,
-            TableIgnoreList = ignored
-        });
-        var ctx = new ProcessContext();
-        var queue = new DataRecordQueue();
-        var dataReaderFactory = new DataReaderFactory(CreateXUnitLogger<DataReaderFactory>(), options);
-        var sut = new FileInputService(CreateXUnitLogger<FileInputService>(), options, ctx, queue, dataReaderFactory,
-            new ConfigurationManager());
-
-        var result = sut.GetOrderedInputInfo(inputDir).ToArray();
-        WriteJson(result);
-        Assert.True(assertCount == result.Length);
-    }
-}

MesETL.Test/MesDatabaseHelper.cs

@@ -1,21 +0,0 @@
-using MesETL.App.Helpers;
-using MesETL.Shared.Helper;
-
-namespace TestProject1;
-
-public static class MesDatabaseHelper
-{
-    public static async Task<int?> TrySearchCompanyId(string connStr, IEnumerable<string> dbNames, string scalarQuery)
-    {
-        foreach (var db in dbNames)
-        {
-            var result = await DatabaseHelper.QueryScalarAsync(connStr + $"Database={db};", scalarQuery);
-            if (result is null or 0) continue;
-                
-            var companyId = Convert.ToInt32(result);
-            return companyId;
-        }
-
-        return null;
-    }
-}

MesETL.Test/MesETL.Test.csproj

@@ -1,31 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-    <PropertyGroup>
-        <TargetFramework>net9.0</TargetFramework>
-        <ImplicitUsings>enable</ImplicitUsings>
-        <Nullable>enable</Nullable>
-
-        <IsPackable>false</IsPackable>
-        <IsTestProject>true</IsTestProject>
-        <RootNamespace>TestProject1</RootNamespace>
-    </PropertyGroup>
-
-    <ItemGroup>
-        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.12.0" />
-        <PackageReference Include="NSubstitute" Version="5.3.0" />
-        <PackageReference Include="xunit" Version="2.9.2" />
-        <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
-            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-            <PrivateAssets>all</PrivateAssets>
-        </PackageReference>
-        <PackageReference Include="coverlet.collector" Version="6.0.2">
-            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-            <PrivateAssets>all</PrivateAssets>
-        </PackageReference>
-    </ItemGroup>
-
-    <ItemGroup>
-      <ProjectReference Include="..\MesETL.App\MesETL.App.csproj" />
-    </ItemGroup>
-
-</Project>

MesETL.Test/Services/SeqServiceTests.cs

@@ -1,33 +0,0 @@
-using System.Reflection;
-using Azusa.Shared.Extensions;
-using MesETL.App.Options;
-using MesETL.App.Services.Seq;
-using Microsoft.Extensions.Options;
-using TestProject1.XUnit;
-using Xunit.Abstractions;
-
-namespace TestProject1.Services;
-
-public class SeqServiceTests : TestBase
-{
-    public SeqServiceTests(ITestOutputHelper output) : base(output)
-    {
-    }
-
-    [Fact]
-    public void Test_Sequence_Sql_Generation()
-    {
-        var sut = new SeqService(new OptionsWrapper<DatabaseOutputOptions>(new DatabaseOutputOptions()
-        {
-            ConnectionString = "Server=127.0.0.1;Port=3306;UserId=root;Password=123456;"
-        }));
-
-        foreach (var i in 10)
-        {
-            Write("Seq: " + sut.AddCachedSeq(SeqConfig.OrderWaveGroupID));
-        }
-
-        var sql = typeof(SeqService).GetMethod("GenerateCachedSeqSql", BindingFlags.Instance | BindingFlags.NonPublic)!.Invoke(sut, []);
-        Write(sql ?? "null");
-    }
-}

MesETL.Test/TableIndex.cs

@@ -1,24 +0,0 @@
-namespace TestProject1;
-
-public record TableIndex(string TableName, string IndexName, bool IsUnique, string ColumnName, TableIndex.TableIndexType IndexType)
-{
-    public enum TableIndexType
-    {
-        BTree,
-        Hash,
-        Primary
-    }
-
-    public void Deconstruct(out string tableName, out string indexName)
-    {
-        tableName = TableName;
-        indexName = IndexName;
-    }
-
-    public void Deconstruct(out string tableName, out string indexName, out string columnName)
-    {
-        tableName = TableName;
-        indexName = IndexName;
-        columnName = ColumnName;
-    }
-}

MesETL.Test/TenantDbHelper.cs

@@ -1,26 +0,0 @@
-namespace TestProject1;
-
-public static class TenantDbHelper
-{
-    /// <summary>
-    /// Key-Value: {DbName}-{TenantKeyLessThan}
-    /// </summary>
-    public static Dictionary<string, int> DbList { get; set; } = new Dictionary<string, int>
-    {
-        { "cferp_test_1", 1000 },
-        { "cferp_test_2", 2000 },
-        { "cferp_test_3", int.MaxValue },
-    };
-
-    public static string GetDbNameByTenantKeyValue(int tenantKeyValue)
-    {
-        // var dictionary = new SortedDictionary<int, string>();
-        // DbList.ForEach(pair => dictionary.Add(pair.Value, pair.Key));
-        // 注意配置顺序
-        var dbName = DbList.Cast<KeyValuePair<string, int>?>()
-            .FirstOrDefault(pair => pair?.Value != null && pair.Value.Value > tenantKeyValue)!.Value.Key;
-        return dbName ??
-               throw new ArgumentOutOfRangeException(nameof(tenantKeyValue),
-                   $"已配置的数据库中没有任何符合'{nameof(tenantKeyValue)}'值的对象");
-    }
-}

MesETL.Test/Test.cs

@@ -1,152 +0,0 @@
-using System.Collections.Concurrent;
-using System.Diagnostics;
-using MesETL.App.Services.ETL;
-using MesETL.Shared.Helper;
-using Xunit.Abstractions;
-using ZstdSharp;
-
-namespace TestProject1;
-
-public class Test
-{
-    private readonly ITestOutputHelper _output;
-
-    public Test(ITestOutputHelper output)
-    {
-        _output = output;
-    }
-
-    [Theory]
-    [InlineData([@"D:\Dump\NewMockData2\cferp.order_box_block.00000.dat.zst"])]
-    public async Task ZstdDecompressTest(string inputFile)
-    {
-        var count = 0;
-        var flag = true;
-        var sw = Stopwatch.StartNew();
-        var reader = new StreamReader(new DecompressionStream(File.OpenRead(inputFile)));
-        var monitor = Task.Run(async () =>
-        {
-            var lastElapse = sw.ElapsedMilliseconds;
-            var lastCount = 0;
-            while (flag)
-            {
-                await Task.Delay(2000);
-
-                _output.WriteLine($"speed: {(count - lastCount) / ((sw.ElapsedMilliseconds - lastElapse) / 1000f)}");
-                lastElapse = sw.ElapsedMilliseconds;
-                lastCount = count;
-            }
-        });
-        while (!reader.EndOfStream)
-        {
-            var str = await reader.ReadLineAsync();
-            char a;
-            // foreach (var c in str)
-            // {
-            //     a = c;
-            // }
-            CsvReader.ParseRowFaster(str, '"', ',');
-            count++;
-        }
-
-        flag = false;
-        monitor.Wait();
-    }
-
-    
-    public static IEnumerable<object[]> ParseRowData()
-    {
-        yield return
-            [@"20220104020855,""2022-01-04 10:06:46"",1455,""0001-01-01 00:00:00"",""1"",0,""2"",""0"",\N,""0"",22010"];
-        yield return 
-            [@"20220104020858,""2022-01-04 15:08:22"",1455,""0001-01-01 00:00:00"",""1"",838,""2"",""0"",""5"",""0"",22010"];
-        yield return
-            [@"5586326,20220104020855,220105981029,""1"",482278,482279,3768774,0,0,""1.000"",1455,22010"];
-        yield return
-            [@"130658,""PD220104002302"",3,4616,""2022-01-04 15:10:40"",1443,""2022-01-04 15:10:40"",""2022-01-04 15:10:51"",0,"""",0,1455,""0001-01-01 00:00:00"",1,5B32303232303130343032303835385D,E590B8E5A1912D2DE590B8E5A1912D2D31382D2D323030302A3630302D2D3130E789872D2D352E3936333B6361696C69616F2D2D79616E73652D2D392D2D323031302A313137342D2D31E789872D2D322E3336,""0"",0"];
-    }
-    [Theory]
-    [MemberData(nameof(ParseRowData))]
-    public void ParseRowFasterTest(string row)
-    {
-        var fields = CsvReader.ParseRowFaster(row, '"', ',');
-        _output.WriteLine(string.Join(',', fields));
-    }
-
-    [Fact]
-    public void DictMemoryTest()
-    {
-        var dict = new ConcurrentDictionary<string, string>();
-        for (int i = 0; i < 3000000; i++)
-        {
-            dict.AddOrUpdate(Guid.NewGuid().ToString(), Random.Shared.NextInt64(1000000000L, 9999999999L).ToString(), (_, __) => Random.Shared.NextInt64(1000000000L, 9999999999L).ToString());
-        }
-
-        while (true)
-        {
-            
-        }
-    }
-
-    [Fact]
-    public void GetResult()
-    {
-        var input =
-            """
-            machine: 19303/19061
-            order: 3416759/3415192
-            order_block_plan: 2934281/1968850
-            order_block_plan_item: 0/235927707
-            order_block_plan_result: 1375479/277667
-            order_box_block: 23457666/23450841
-            order_data_block: 513012248/513012248
-            order_data_goods: 18655270/18655270
-            order_data_parts: 353139066/353139066
-            order_item: 955274320/955274320
-            order_module: 102907480/56935691
-            order_module_extra: 40044077/40044077
-            order_module_item: 49209022/49209022
-            order_package: 12012712/12012712
-            order_package_item: 0/80605124
-            order_process: 4045309/2682043
-            order_process_step: 8343418/5505158
-            order_process_step_item: 14856509/9787696
-            order_scrap_board: 136096/136090
-            process_group: 1577/1543
-            process_info: 9212/9008
-            process_item_exp: 30/30
-            process_schdule_capacity: 42442/42442
-            process_step_efficiency: 8/8
-            report_template: 7358/7338
-            simple_package: 142861/137730
-            simple_plan_order: 1167004/854699
-            simple_plan_order: 0/55677
-            sys_config: 2608/2608
-            work_calendar: 11/11
-            work_shift: 73/73
-            work_time: 77/77
-            order_process_step_item: 14856509/9790701
-            order_process_step: 8343418/5506925
-            order_module: 102907480/56935691
-            order_process: 4045309/2682043
-            report_template: 7358/7358
-            process_info: 9212/9212
-            process_group: 1577/1577
-            order_block_plan_result: 1375479/277667
-            order_box_block: 23457666/23457666
-            order_block_plan: 2934281/1968850
-            order: 3416759/3416759
-            machine: 19303/19303
-            order_scrap_board: 136096/136096
-            """;
-
-        var arr = input.Split('\n').Select(s => 
-        {
-            var x = s.Split(':');
-            var y = x[1].Split('/').Select(i => long.Parse(i)).ToArray();
-            return new {TABLE_NAME = x[0], INPUT = y[0], OUTPUT = y[1], FILTER = y[0] - y[1]};
-        }).OrderBy(s => s.TABLE_NAME);
-
-        _output.WriteLine(arr.ToMarkdownTable());
-    }
-}

MesETL.Test/XUnit/Configuration/XUnitConfiguration.cs

@@ -1,16 +0,0 @@
-using Microsoft.Extensions.Configuration;
-
-namespace TestProject1.XUnit.Configuration;
-
-public static class XUnitConfiguration
-{
-    public static IConfiguration Configuration { get; }
-    
-    static XUnitConfiguration()
-    {
-        Configuration = new ConfigurationBuilder()
-            .SetBasePath(AppContext.BaseDirectory)
-            .AddJsonFile("appsettings.json", false, true)
-            .Build();
-    }
-}