Update

ConsoleApp2/Const/ProcessStep.cs

@@ -1,4 +1,4 @@
-namespace ConsoleApp2;
+namespace ConsoleApp2.Const;
 
 public static class ProcessStep
 {

ConsoleApp2/DataRecord.cs

@@ -1,6 +1,5 @@
-using System.Text;
+namespace ConsoleApp2;
 
-namespace ConsoleApp2.Entities;
 
 public class DataRecord
 {
@@ -9,7 +8,7 @@ public class DataRecord
         value = string.Empty;
         if (record.Headers is null)
             throw new InvalidOperationException("Cannot get field when headers of a record have not been set.");
-        var idx = Array.IndexOf(record.Headers, columnName);
+        var idx = Array.IndexOf(record.Headers, columnName); //可能可以优化
         if (idx == -1)
             return false;
         value = record.Fields[idx];

ConsoleApp2/Helpers/DumpDataHelper.cs

@@ -1,6 +1,5 @@
 using System.Text;
 using System.Text.RegularExpressions;
-using ConsoleApp2.Entities;
 
 namespace ConsoleApp2.Helpers;
 

ConsoleApp2/Helpers/HashExtension.cs

@@ -1,249 +0,0 @@
-using System.Security.Cryptography;
-using System.Text;
-
-namespace ConsoleApp2.Helpers;
-
-public static class HashExtensions
-{
-    /// <summary>
-    /// 计算32位MD5码
-    /// </summary>
-    /// <param name="word">字符串</param>
-    /// <param name="toUpper">返回哈希值格式 true：英文大写，false：英文小写</param>
-    /// <returns></returns>
-    public static string ToMd5Hash(this string word, bool toUpper = true)
-    {
-        try
-        {
-            var MD5CSP = MD5.Create();
-            var bytValue = Encoding.UTF8.GetBytes(word);
-            var bytHash = MD5CSP.ComputeHash(bytValue);
-            MD5CSP.Clear();
-            //根据计算得到的Hash码翻译为MD5码
-            var sHash = "";
-            foreach (var t in bytHash)
-            {
-                long i = t / 16;
-                var sTemp = i > 9 ? ((char)(i - 10 + 0x41)).ToString() : ((char)(i + 0x30)).ToString();
-                i = t % 16;
-                if (i > 9)
-                {
-                    sTemp += ((char)(i - 10 + 0x41)).ToString();
-                }
-                else
-                {
-                    sTemp += ((char)(i + 0x30)).ToString();
-                }
-
-                sHash += sTemp;
-            }
-
-            //根据大小写规则决定返回的字符串
-            return toUpper ? sHash : sHash.ToLower();
-        }
-        catch (System.Exception ex)
-        {
-            throw new System.Exception(ex.Message);
-        }
-    }
-
-    public static string ToMd5Hash(this Stream stream, bool toUpper = true)
-    {
-        using var md5Hash = MD5.Create();
-        var bytes = md5Hash.ComputeHash(stream);
-        return ToHashString(bytes, toUpper);
-    }
-
-    /// <summary>
-    /// 计算SHA-1码
-    /// </summary>
-    /// <param name="word">字符串</param>
-    /// <param name="toUpper">返回哈希值格式 true：英文大写，false：英文小写</param>
-    /// <returns></returns>
-    public static string ToSHA1Hash(this string word, bool toUpper = true)
-    {
-        try
-        {
-            var SHA1CSP = SHA1.Create();
-            var bytValue = Encoding.UTF8.GetBytes(word);
-            var bytHash = SHA1CSP.ComputeHash(bytValue);
-            SHA1CSP.Clear();
-            //根据计算得到的Hash码翻译为SHA-1码
-            var sHash = "";
-            foreach (var t in bytHash)
-            {
-                long i = t / 16;
-                var sTemp = i > 9 ? ((char)(i - 10 + 0x41)).ToString() : ((char)(i + 0x30)).ToString();
-                i = t % 16;
-                if (i > 9)
-                {
-                    sTemp += ((char)(i - 10 + 0x41)).ToString();
-                }
-                else
-                {
-                    sTemp += ((char)(i + 0x30)).ToString();
-                }
-
-                sHash += sTemp;
-            }
-
-            //根据大小写规则决定返回的字符串
-            return toUpper ? sHash : sHash.ToLower();
-        }
-        catch (System.Exception ex)
-        {
-            throw new System.Exception(ex.Message);
-        }
-    }
-
-    /// <summary>
-    /// 计算SHA-256码
-    /// </summary>
-    /// <param name="word">字符串</param>
-    /// <param name="toUpper">返回哈希值格式 true：英文大写，false：英文小写</param>
-    /// <returns></returns>
-    public static string ToSHA256Hash(this string word, bool toUpper = true)
-    {
-        try
-        {
-            var SHA256CSP = SHA256.Create();
-            var bytValue = Encoding.UTF8.GetBytes(word);
-            var bytHash = SHA256CSP.ComputeHash(bytValue);
-            SHA256CSP.Clear();
-            //根据计算得到的Hash码翻译为SHA-1码
-            var sHash = "";
-            foreach (var t in bytHash)
-            {
-                long i = t / 16;
-                var sTemp = i > 9 ? ((char)(i - 10 + 0x41)).ToString() : ((char)(i + 0x30)).ToString();
-                i = t % 16;
-                if (i > 9)
-                {
-                    sTemp += ((char)(i - 10 + 0x41)).ToString();
-                }
-                else
-                {
-                    sTemp += ((char)(i + 0x30)).ToString();
-                }
-
-                sHash += sTemp;
-            }
-
-            //根据大小写规则决定返回的字符串
-            return toUpper ? sHash : sHash.ToLower();
-        }
-        catch (System.Exception ex)
-        {
-            throw new System.Exception(ex.Message);
-        }
-    }
-
-    /// <summary>
-    /// 计算SHA-256码
-    /// </summary>
-    /// <param name="stream"></param>
-    /// <param name="toUpper"></param>
-    /// <returns></returns>
-    public static string ToSHA256Hash(this Stream stream, bool toUpper = true)
-    {
-        using var sha256Hash = SHA256.Create();
-        var bytes = sha256Hash.ComputeHash(stream);
-        return ToHashString(bytes, toUpper);
-    }
-
-    /// <summary>
-    /// 计算SHA-384码
-    /// </summary>
-    /// <param name="word">字符串</param>
-    /// <param name="toUpper">返回哈希值格式 true：英文大写，false：英文小写</param>
-    /// <returns></returns>
-    public static string ToSHA384Hash(this string word, bool toUpper = true)
-    {
-        try
-        {
-            var SHA384CSP = SHA384.Create();
-            var bytValue = Encoding.UTF8.GetBytes(word);
-            var bytHash = SHA384CSP.ComputeHash(bytValue);
-            SHA384CSP.Clear();
-            //根据计算得到的Hash码翻译为SHA-1码
-            var sHash = "";
-            foreach (var t in bytHash)
-            {
-                long i = t / 16;
-                var sTemp = i > 9 ? ((char)(i - 10 + 0x41)).ToString() : ((char)(i + 0x30)).ToString();
-                i = t % 16;
-                if (i > 9)
-                {
-                    sTemp += ((char)(i - 10 + 0x41)).ToString();
-                }
-                else
-                {
-                    sTemp += ((char)(i + 0x30)).ToString();
-                }
-
-                sHash += sTemp;
-            }
-
-            //根据大小写规则决定返回的字符串
-            return toUpper ? sHash : sHash.ToLower();
-        }
-        catch (System.Exception ex)
-        {
-            throw new System.Exception(ex.Message);
-        }
-    }
-
-    /// <summary>
-    /// 计算SHA-512码
-    /// </summary>
-    /// <param name="word">字符串</param>
-    /// <param name="toUpper">返回哈希值格式 true：英文大写，false：英文小写</param>
-    /// <returns></returns>
-    public static string ToSHA512Hash(this string word, bool toUpper = true)
-    {
-        try
-        {
-            var SHA512CSP = SHA512.Create();
-            var bytValue = Encoding.UTF8.GetBytes(word);
-            var bytHash = SHA512CSP.ComputeHash(bytValue);
-            SHA512CSP.Clear();
-            //根据计算得到的Hash码翻译为SHA-1码
-            var sHash = "";
-            foreach (var t in bytHash)
-            {
-                long i = t / 16;
-                var sTemp = i > 9 ? ((char)(i - 10 + 0x41)).ToString() : ((char)(i + 0x30)).ToString();
-                i = t % 16;
-                if (i > 9)
-                {
-                    sTemp += ((char)(i - 10 + 0x41)).ToString();
-                }
-                else
-                {
-                    sTemp += ((char)(i + 0x30)).ToString();
-                }
-
-                sHash += sTemp;
-            }
-
-            //根据大小写规则决定返回的字符串
-            return toUpper ? sHash : sHash.ToLower();
-        }
-        catch (System.Exception ex)
-        {
-            throw new System.Exception(ex.Message);
-        }
-    }
-
-    private static string ToHashString(byte[] bytes, bool toUpper = true)
-    {
-        var builder = new StringBuilder();
-        foreach (var t in bytes)
-        {
-            builder.Append(t.ToString("x2"));
-        }
-
-        var str = builder.ToString();
-        return toUpper ? str.ToUpper() : str.ToLower();
-    }
-}

ConsoleApp2/HostedServices/Abstractions/IInputService.cs

@@ -0,0 +1,6 @@
+namespace ConsoleApp2.HostedServices.Abstractions;
+
+public interface IInputService
+{
+    public Task ExecuteAsync(CancellationToken cancellationToken);
+}

ConsoleApp2/HostedServices/Abstractions/IOutputService.cs

@@ -0,0 +1,6 @@
+namespace ConsoleApp2.HostedServices.Abstractions;
+
+public interface IOutputService
+{
+    public Task ExecuteAsync(CancellationToken cancellationToken);
+}

ConsoleApp2/HostedServices/Abstractions/ITransformService.cs

@@ -0,0 +1,6 @@
+namespace ConsoleApp2.HostedServices.Abstractions;
+
+public interface ITransformService
+{
+    public Task ExecuteAsync(CancellationToken cancellationToken);
+}

ConsoleApp2/HostedServices/InputService.cs

@@ -1,35 +1,36 @@
-using ConsoleApp2.Helpers;
+using ConsoleApp2.Const;
+using ConsoleApp2.Helpers;
+using ConsoleApp2.HostedServices.Abstractions;
 using ConsoleApp2.Options;
 using ConsoleApp2.Services;
 using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 
 namespace ConsoleApp2.HostedServices;
 
-public class CsvInputService : BackgroundService
+/// <summary>
+/// 从MyDumper导出的CSV文件中导入表头和数据
+/// </summary>
+public class InputService : IInputService
 {
     private readonly ILogger _logger;
     private readonly IOptions<CsvOptions> _csvOptions;
-    private readonly TaskManager _taskManager; // TBD
     private readonly DataRecordQueue _producerQueue;
     private readonly ProcessContext _context;
 
-    public CsvInputService(ILogger<CsvInputService> logger,
+    public InputService(ILogger<InputService> logger,
         IOptions<CsvOptions> csvOptions, 
-        [FromKeyedServices(ProcessStep.Producer)]TaskManager taskManager, 
         [FromKeyedServices(ProcessStep.Producer)]DataRecordQueue producerQueue, 
         ProcessContext context)
     {
         _logger = logger;
         _csvOptions = csvOptions;
-        _taskManager = taskManager;
         _producerQueue = producerQueue;
         _context = context;
     }
 
-    protected override async Task ExecuteAsync(CancellationToken cancellationToken)
+    public async Task ExecuteAsync(CancellationToken cancellationToken)
     {
         var inputDir = _csvOptions.Value.InputDir;
         _logger.LogInformation("***** Csv input service start, working dir: {InputDir}, thread id: {ThreadId} *****", inputDir, Environment.CurrentManagedThreadId);
@@ -50,7 +51,7 @@ public class CsvInputService : BackgroundService
             {
                 var csvPath = Path.Combine(inputDir, csvFile);
                 // var source = new JsvSource(csvPath, headers, _logger);
-                var source = new NewCsvSource(csvPath, headers,  logger: _logger);
+                var source = new CsvSource(csvPath, headers, _csvOptions.Value.Delimiter, _csvOptions.Value.QuoteChar, _logger);
 
                 while (await source.ReadAsync())
                 {

ConsoleApp2/HostedServices/MainHostedService.cs

@@ -0,0 +1,30 @@
+using ConsoleApp2.HostedServices.Abstractions;
+using Microsoft.Extensions.Hosting;
+
+namespace ConsoleApp2.HostedServices;
+
+public class MainHostedService : BackgroundService
+{
+    private readonly IInputService _input;
+    private readonly ITransformService _transform;
+    private readonly IOutputService _output;
+
+    public MainHostedService(IInputService input, ITransformService transform, IOutputService output)
+    {
+        _input = input;
+        _transform = transform;
+        _output = output;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        var tasks = new List<Task>()
+        {
+            Task.Run(async () => await _input.ExecuteAsync(stoppingToken), stoppingToken),
+            Task.Run(async () => await _transform.ExecuteAsync(stoppingToken), stoppingToken),
+            Task.Run(async () => await _output.ExecuteAsync(stoppingToken), stoppingToken),
+        };
+        await Task.WhenAll(tasks);
+        // await Task.Run(async () => await _output.ExecuteAsync(stoppingToken), stoppingToken);
+    }
+}

ConsoleApp2/HostedServices/OutputService.cs

@@ -1,58 +1,62 @@
-using ConsoleApp2.Entities;
+using ConsoleApp2.Const;
+using ConsoleApp2.HostedServices.Abstractions;
 using ConsoleApp2.Options;
 using ConsoleApp2.Services;
 using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
-using MySqlConnector;
 
 namespace ConsoleApp2.HostedServices;
 
-public class MysqlOutputService : BackgroundService
+/// <summary>
+/// 数据导出服务，将数据导出至MySql服务
+/// </summary>
+public class OutputService : IOutputService
 {
     private readonly ILogger _logger;
     private readonly DataRecordQueue _consumerQueue;
-    private readonly IOptions<DatabaseOptions> _options;
+    private readonly IOptions<DatabaseOutputOptions> _options;
     private readonly ProcessContext _context;
+    private readonly TaskManager _taskManager;
 
-    public MysqlOutputService(ILogger<MysqlOutputService> logger, 
+    public OutputService(ILogger<OutputService> logger,
-        [FromKeyedServices(ProcessStep.Consumer)]DataRecordQueue consumerQueue, 
+        [FromKeyedServices(ProcessStep.Consumer)] DataRecordQueue consumerQueue,
-        IOptions<DatabaseOptions> options, 
+        IOptions<DatabaseOutputOptions> options,
-        ProcessContext context)
+        ProcessContext context,
+        TaskManager taskManager)
     {
         _logger = logger;
         _consumerQueue = consumerQueue;
         _options = options;
         _context = context;
+        _taskManager = taskManager;
     }
 
-    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    public async Task ExecuteAsync(CancellationToken stoppingToken)
     {
         _logger.LogInformation("***** Mysql output service started *****");
 
-        var tasks = new List<Task>();
         var records = new List<DataRecord>();
         while (!_context.IsTransformCompleted || _consumerQueue.Count > 0)
         {
             if (!_consumerQueue.TryDequeue(out var record)) continue;
             records.Add(record);
 
-            if (records.Count >= 200)
+            if (records.Count >= _options.Value.FlushCount)
             {
                 var recordsCopy = records;
-                tasks.Add(Task.Run(async () => await FlushAsync(recordsCopy), stoppingToken));
+                _taskManager.CreateTask(async () => await FlushAsync(recordsCopy), stoppingToken);
                 records = [];
             }
 
-            if (tasks.Count >= 10)
+            if (_taskManager.TaskCount >= _options.Value.MaxTask)
             {
-                await Task.WhenAll(tasks);
+                await _taskManager.WaitAll();
-                tasks.Clear();
+                _taskManager.ClearTask();
             }
         }
 
-        await Task.WhenAll(tasks);
+        await _taskManager.WaitAll();
         await FlushAsync(records);
 
         _context.CompleteOutput();
@@ -63,15 +67,9 @@ public class MysqlOutputService : BackgroundService
     private async Task FlushAsync(IEnumerable<DataRecord> records)
     {
         var count = 0;
-        await using var output = new MySqlDestination(new MySqlConnectionStringBuilder
+        await using var output = new MySqlDestination(
-        {
+            _options.Value.ConnectionString ?? throw new InvalidOperationException("Connection string is required"),
-            Server = _options.Value.Host,
+            _logger, true);
-            Port = _options.Value.Port,
-            Database = _options.Value.Database,
-            UserID = _options.Value.User,
-            Password = _options.Value.Password,
-            ConnectionTimeout = 180,
-        }.ConnectionString, _logger, true);
         
         foreach (var record in records)
         {

ConsoleApp2/HostedServices/SqlFileOutputService.cs

@@ -1,64 +0,0 @@
-using ConsoleApp2.Entities;
-using ConsoleApp2.Helpers;
-using ConsoleApp2.Services;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Hosting;
-using Microsoft.Extensions.Logging;
-
-namespace ConsoleApp2.HostedServices;
-
-public class SqlFileOutputService : BackgroundService
-{
-    private readonly string _outputFile = "D:/DumpOutput/cferp_test_1.sql"; //
-    private readonly DataRecordQueue _consumerQueue;
-    private readonly ILogger _logger;
-    private readonly ProcessContext _context;
-
-    public SqlFileOutputService(
-        ILogger<SqlFileOutputService> logger,
-        [FromKeyedServices(ProcessStep.Consumer)]
-        DataRecordQueue consumerQueue,
-        ProcessContext context)
-    {
-        _logger = logger;
-        _consumerQueue = consumerQueue;
-        _context = context;
-    }
-
-
-    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
-    {
-        _logger.LogInformation("***** Sql file output service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId);
-        var count = 0;
-        var tableRecords = new Dictionary<string, IList<DataRecord>>();
-        while (!_context.IsTransformCompleted || _consumerQueue.Count > 0)
-        {
-            if (!_consumerQueue.TryDequeue(out var record)) continue;
-
-            tableRecords.AddOrUpdate(record.TableName, [record], (key, value) =>
-            {
-                value.Add(record);
-                return value;
-            });
-
-            ++count;
-
-            if (count >= 200)
-            {
-                await File.AppendAllTextAsync(_outputFile,
-                    MySqlDestination.SerializeRecords(tableRecords), stoppingToken);
-                tableRecords.Clear();
-                _context.AddOutput(count);
-                count = 0;
-            }
-        }
-        await File.AppendAllTextAsync(_outputFile,
-            MySqlDestination.SerializeRecords(tableRecords), stoppingToken);
-        tableRecords.Clear();
-        _context.AddOutput(count);
-        _context.CompleteOutput();
-        
-        _logger.LogInformation("***** Sql file output service completed *****");
-    }
-   
-}

ConsoleApp2/HostedServices/TaskMonitorService.cs

@@ -1,4 +1,5 @@
 using System.Diagnostics;
+using ConsoleApp2.Const;
 using ConsoleApp2.Services;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
@@ -6,6 +7,9 @@ using Microsoft.Extensions.Logging;
 
 namespace ConsoleApp2.HostedServices;
 
+/// <summary>
+/// 任务监控
+/// </summary>
 public class TaskMonitorService : BackgroundService
 {
     private readonly IHostApplicationLifetime _lifetime;
@@ -93,7 +97,7 @@ public class TaskMonitorService : BackgroundService
             _logger.LogInformation("Queue monitor: producer queue: {ProducerQueue}, consumer queue: {ConsumerQueue}",
                 _producerQueue.Count, _consumerQueue.Count);
 
-            await Task.Delay(2000);
+            await Task.Delay(5000);
 
             lastTime = time;
             lastInputCount = inputCount;

ConsoleApp2/HostedServices/TransformService.cs

@@ -1,14 +1,17 @@
-using ConsoleApp2.Helpers;
+using ConsoleApp2.Const;
+using ConsoleApp2.HostedServices.Abstractions;
 using ConsoleApp2.Options;
 using ConsoleApp2.Services;
 using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 
 namespace ConsoleApp2.HostedServices;
 
-public class DataTransformService : BackgroundService
+/// <summary>
+/// 数据处理服务，对导入后的数据进行处理
+/// </summary>
+public class TransformService : ITransformService
 {
     private readonly ILogger _logger;
     private readonly IOptions<DataTransformOptions> _options;
@@ -17,21 +20,20 @@ public class DataTransformService : BackgroundService
     private readonly ProcessContext _context;
 
 
-    public DataTransformService(ILogger<DataTransformService> logger, 
+    public TransformService(ILogger<TransformService> logger, 
-        IOptions<DataTransformOptions> options, // TBD: database filter
+        IOptions<DataTransformOptions> options,
         [FromKeyedServices(ProcessStep.Producer)]DataRecordQueue producerQueue,
         [FromKeyedServices(ProcessStep.Consumer)]DataRecordQueue consumerQueue, 
         ProcessContext context)
     {
         _logger = logger;
-        // _taskManager = taskManager;
         _options = options;
         _producerQueue = producerQueue;
         _consumerQueue = consumerQueue;
         _context = context;
     }
 
-    protected override async Task ExecuteAsync(CancellationToken cancellationToken)
+    public async Task ExecuteAsync(CancellationToken cancellationToken)
     {
         _logger.LogInformation("***** Data transform service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId);
         while (!_context.IsInputCompleted || _producerQueue.Count > 0)
@@ -58,7 +60,6 @@ public class DataTransformService : BackgroundService
                         field = string.IsNullOrEmpty(field) ? "''" : $"0x{field}";
                         break;
                     default:
-                        field = field;
                         break;
                 }
 
@@ -66,6 +67,8 @@ public class DataTransformService : BackgroundService
                 record[i] = field;
             }
             
+            // TODO: 数据处理/过滤/复制
+            
             _consumerQueue.Enqueue(record);
             _context.AddTransform();
         }

ConsoleApp2/HostedServices/VoidOutputService.cs

@@ -1,11 +1,14 @@
-using ConsoleApp2.Services;
+using ConsoleApp2.Const;
+using ConsoleApp2.HostedServices.Abstractions;
+using ConsoleApp2.Services;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 
 namespace ConsoleApp2.HostedServices;
 
-public class VoidOutputService : BackgroundService
+// 空输出服务，测试用
+public class VoidOutputService : IOutputService
 {
     private readonly ILogger _logger;
     private readonly DataRecordQueue _consumerQueue;
@@ -19,7 +22,7 @@ public class VoidOutputService : BackgroundService
         _logger = logger;
     }
 
-    protected override Task ExecuteAsync(CancellationToken stoppingToken)
+    public Task ExecuteAsync(CancellationToken stoppingToken)
     {
         _logger.LogInformation("***** Void output service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId);
         while (!_context.IsTransformCompleted || _consumerQueue.Count > 0)

ConsoleApp2/Options/CsvOptions.cs

@@ -3,26 +3,17 @@
 public class CsvOptions
 {
     /// <summary>
-    /// The directory to input csv and sql file.
+    /// MyDumper导出的CSV文件目录
     /// </summary>
     public string InputDir { get; set; } = "./";
-    /// <summary>
-    /// The output directory.
-    /// </summary>
-    public string OutputDir { get; set; } = "./Output";
 
     /// <summary>
-    /// The ASCII char that fields are enclosed by. Default is '"'.
+    /// 字符串的包围符号，默认为双引号"
     /// </summary>
     public char QuoteChar { get; set; } = '"';
 
     /// <summary>
-    /// The ASCII char that fields are separated by. Default is ','.
+    /// 每个字段的分割符，默认逗号,
     /// </summary>
-    public char DelimiterChar { get; set; } = ',';
+    public string Delimiter { get; set; } = ",";
-
-    /// <summary>
-    /// The max number of threads to use.
-    /// </summary>
-    public int MaxThreads { get; set; } = 12;
 }

ConsoleApp2/Options/DataTransformOptions.cs

@@ -1,6 +1,4 @@
-using ConsoleApp2.Entities;
+namespace ConsoleApp2.Options;
-
-namespace ConsoleApp2.Options;
 
 public enum ColumnType
 {
@@ -12,6 +10,10 @@ public enum ColumnType
 public class DataTransformOptions
 {
     public Func<DataRecord, string>? DatabaseFilter { get; set; }
+    
+    /// <summary>
+    /// 配置导入数据的特殊列
+    /// </summary>
     public Dictionary<string, ColumnType> ColumnTypeConfig { get; set; } = new(); // "table.column" -> type
 
     public ColumnType GetColumnType(string table, string column)

ConsoleApp2/Options/DatabaseOptions.cs

@@ -1,10 +0,0 @@
-namespace ConsoleApp2.Options;
-
-public class DatabaseOptions
-{
-    public string Host { get; set; }
-    public uint Port { get; set; }
-    public string Database { get; set; }
-    public string User { get; set; }
-    public string Password { get; set; }
-}

ConsoleApp2/Options/DatabaseOutputOptions.cs

@@ -0,0 +1,17 @@
+namespace ConsoleApp2.Options;
+
+public class DatabaseOutputOptions
+{
+    /// <summary>
+    /// 数据库连接字符串
+    /// </summary>
+    public string? ConnectionString { get; set; }
+    /// <summary>
+    /// 输出服务的最大任务(Task)数
+    /// </summary>
+    public int MaxTask { get; set; }
+    /// <summary>
+    /// 每个任务每次提交到数据库的记录数量（每N条构建一次SQL语句）
+    /// </summary>
+    public int FlushCount { get; set; }
+}

ConsoleApp2/Program.cs

@@ -1,26 +1,17 @@
-using ConsoleApp2;
+using ConsoleApp2.Const;
 using ConsoleApp2.HostedServices;
+using ConsoleApp2.HostedServices.Abstractions;
 using ConsoleApp2.Options;
 using ConsoleApp2.Services;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
+using MySqlConnector;
 using Serilog;
 
-
+// 运行之前把Mysql max_allowed_packets 调大
-// 加入数据库过滤
+// 运行之前把process_step表的外键删掉
-// HostedService不是并行的，完善TaskManager手动开启线程
-// 测试BlockingCollection对速度的影响？
-
-// 重新同步数据
-// Json列和Blob列不一致
-/* JSV导出带转义的列有误
- * order_data_block表id 4153969
- * order_data_parts表Spec列
- * order_module表Name列
- * process_group表Items列
- */
 
 await RunProgram();
 return;
@@ -32,15 +23,12 @@ async Task RunProgram()
     host.Configuration.AddCommandLine(args); 
     host.Services.Configure<CsvOptions>(option =>
     {
-        option.DelimiterChar = ',';
+        option.Delimiter = ",";
         option.QuoteChar = '"';
-        option.InputDir = "D:/Dump/MyDumper";
+        option.InputDir = "D:/Dump";
-        option.OutputDir = "D:/DumpOutput";
-        option.MaxThreads = 12;
     });
     host.Services.Configure<DataTransformOptions>(options =>
     {
-        //TODO: Database Filter
         options.DatabaseFilter = record => "cferp_test_1";
         options.ColumnTypeConfig = new()
         {
@@ -73,14 +61,20 @@ async Task RunProgram()
             { "order_block_plan.BlockInfo", ColumnType.Text },
         };
     });
-// 加入数据库过滤后删除
+    
-    host.Services.Configure<DatabaseOptions>(options =>
+    host.Services.Configure<DatabaseOutputOptions>(options =>
     {
-        options.Host = "localhost";
+        options.ConnectionString = new MySqlConnectionStringBuilder
-        options.Port = 33306;
+        {
-        options.Database = "cferp_test_1";
+            Server = "127.0.0.1",
-        options.User = "root";
+            Port = 33306,
-        options.Password = "123456";
+            Database = "cferp_test_1",
+            UserID = "root",
+            Password = "123456",
+            MaximumPoolSize = 50, // 这个值应当小于 max_connections
+        }.ConnectionString;
+        options.MaxTask = 16;
+        options.FlushCount = 200;
     });
     host.Services.AddLogging(builder =>
     {
@@ -91,14 +85,13 @@ async Task RunProgram()
     host.Services.AddSingleton<ProcessContext>();
     host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Producer);
     host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Consumer);
-    host.Services.AddKeyedSingleton<TaskManager>(ProcessStep.Producer);
+    host.Services.AddTransient<TaskManager>();
-    host.Services.AddKeyedSingleton<TaskManager>(ProcessStep.Consumer);
 
+    host.Services.AddHostedService<MainHostedService>();
     host.Services.AddHostedService<TaskMonitorService>();
-    host.Services.AddHostedService<CsvInputService>();
+    host.Services.AddSingleton<IInputService, InputService>();
-    host.Services.AddHostedService<DataTransformService>();
+    host.Services.AddSingleton<ITransformService, TransformService>();
-    host.Services.AddHostedService<MysqlOutputService>();
+    host.Services.AddSingleton<IOutputService, OutputService>();
-
 
     var app = host.Build();
     await app.RunAsync();

ConsoleApp2/README.md

@@ -0,0 +1,49 @@
+## 说明
+使用该程序来对MyDumper导出的CSV数据进行读取，转换，然后导出到其他数据库中。
+
+1. 用MyDumper从数据库导出CSV数据  
+
+   使用MyDumper Docker镜像
+    ```sh
+    docker run --rm --net=host -v D:/Dump:/home/backup mydumper/mydumper:v0.15.2-6 mydumper `
+   -h 127.0.0.1 -P 33306 -u root -p 123456 `
+   -B cferp_test --no-schemas --csv --hex-blob `
+   -o /home/backup
+    ```
+   将挂载卷，数据库连接和输出目录替换  
+   不导出数据库结构(--no-schemas)，
+   导出完的目录下应当包含.sql文件以及.dat文件
+
+2. 在Program.cs中修改`CsvOptions`配置
+   ```cs
+   host.Services.Configure<CsvOptions>(option =>
+    {
+        option.Delimiter = ",";
+        option.QuoteChar = '"';
+        option.InputDir = "D:/Dump/Test";
+    });
+   ```
+   将`option.InputDir`配置为MyDumper导出的数据目录
+
+3. 在Program.cs中修改`DatabaseOutputOptions`配置
+   ```cs
+   host.Services.Configure<DatabaseOutputOptions>(options =>
+    {
+        options.ConnectionString = new MySqlConnectionStringBuilder
+        {
+            Server = "127.0.0.1",
+            Port = 33306,
+            Database = "cferp_test_1",
+            UserID = "root",
+            Password = "123456",
+            MaximumPoolSize = 50, 
+        }.ConnectionString;
+        options.MaxTask = 16;
+        options.FlushCount = 200;
+    });
+   ```
+   将`MySqlConnectionStringBuilder`的属性修改为程序要导出至的数据库
+   > 后续将这些配置通过命令行传递
+
+4. 运行程序
+   > 注意，测试数据库`cferp_test`中的`order_process_step`表存在外键，如果要导出到和测试库同结构的数据库，记得先把外键删除。

ConsoleApp2/Services/CsvSource.cs

@@ -1,24 +1,26 @@
 using System.Text;
-using ConsoleApp2.Entities;
 using ConsoleApp2.Helpers;
 using Microsoft.Extensions.Logging;
 
 namespace ConsoleApp2.Services;
 
-public class NewCsvSource
+/// <summary>
+/// CSV文件读取
+/// </summary>
+public class CsvSource
 {
     private readonly string _filePath;
     private readonly StreamReader _reader;
     private readonly ILogger? _logger;
     private readonly string _tableName;
 
-    public DataRecord Current { get; protected set; }
+    public DataRecord Current { get; private set; }
     public string[]? Headers { get; }
     public string? CurrentRaw { get; private set; }
     public string Delimiter { get; private set; }
     public char QuoteChar { get; private set; }
 
-    public NewCsvSource(string filePath, string[]? headers = null, string delimiter = ",", char quoteChar = '"',
+    public CsvSource(string filePath, string[]? headers = null, string delimiter = ",", char quoteChar = '"',
         ILogger? logger = null)
     {
         _filePath = filePath;

ConsoleApp2/Services/DataRecordQueue.cs

@@ -1,9 +1,11 @@
 using System.Collections.Concurrent;
 using System.Diagnostics.CodeAnalysis;
-using ConsoleApp2.Entities;
 
 namespace ConsoleApp2.Services;
 
+/// <summary>
+/// 数据队列
+/// </summary>
 public class DataRecordQueue : IDisposable
 {
     private readonly BlockingCollection<DataRecord> _queue;
@@ -17,7 +19,7 @@ public class DataRecordQueue : IDisposable
 
     public DataRecordQueue()
     {
-        _queue = new BlockingCollection<DataRecord>();
+        _queue = new BlockingCollection<DataRecord>(200_000); // 队列最长为20W条记录
     }
     
     public bool TryDequeue([MaybeNullWhen(false)] out DataRecord record)

ConsoleApp2/Services/JsvSource.cs

@@ -1,10 +1,13 @@
-using ConsoleApp2.Entities;
+using ConsoleApp2.Helpers;
-using ConsoleApp2.Helpers;
 using Microsoft.Extensions.Logging;
 using ServiceStack.Text;
 
 namespace ConsoleApp2.Services;
 
+/// <summary>
+/// 读取Jsv格式文件
+/// </summary>
+[Obsolete]
 public class JsvSource : IDisposable
 {
     private readonly string _filePath;

ConsoleApp2/Services/MySqlDestination.cs

@@ -1,11 +1,13 @@
 using System.Text;
-using ConsoleApp2.Entities;
 using ConsoleApp2.Helpers;
 using Microsoft.Extensions.Logging;
 using MySqlConnector;
 
 namespace ConsoleApp2.Services;
 
+/// <summary>
+/// Mysql导出
+/// </summary>
 public class MySqlDestination : IDisposable, IAsyncDisposable
 {
     private readonly Dictionary<string, IList<DataRecord>> _recordCache;
@@ -13,8 +15,6 @@ public class MySqlDestination : IDisposable, IAsyncDisposable
     private readonly ILogger _logger;
     private readonly bool _prettyOutput;
 
-    public static int AddCount;
-
     public MySqlDestination(string connStr, ILogger logger, bool prettyOutput = false)
     {
         _conn = new MySqlConnection(connStr);
@@ -29,7 +29,6 @@ public class MySqlDestination : IDisposable, IAsyncDisposable
         _recordCache.AddOrUpdate(record.TableName, [record], (key, value) =>
         {
             value.Add(record);
-            Interlocked.Increment(ref AddCount);
             return value;
         });
         return Task.CompletedTask;
@@ -60,6 +59,10 @@ public class MySqlDestination : IDisposable, IAsyncDisposable
             _logger.LogCritical(e, "Error when flushing records, sql: {Sql}", cmd.CommandText.Omit(1000));
             throw;
         }
+        finally
+        {
+            await cmd.DisposeAsync();
+        }
     }
 
     public static string SerializeRecords(IDictionary<string, IList<DataRecord>> tableRecords,
@@ -91,22 +94,7 @@ public class MySqlDestination : IDisposable, IAsyncDisposable
                 for (var j = 0; j < record.Fields.Length; j++)
                 {
                     var field = record.Fields[j];
-
-                    #region HandleFields
-
-                    // if (field == "\\N")
-                    //     sb.Append("NULL");
-                    // else if (DumpDataHelper.CheckHexField(field))
-                    // {
-                    //     // if (StringExtensions.CheckJsonHex(field))
-                    //         sb.Append($"0x{field}");
-                    // }
-                    // else
-                    //     sb.Append($"'{field}'");
-
                     sb.Append(field);
-                    #endregion
-
                     if (j != record.Fields.Length - 1)
                         sb.Append(',');
                 }
@@ -127,11 +115,13 @@ public class MySqlDestination : IDisposable, IAsyncDisposable
 
     public void Dispose()
     {
+        _conn.Close();
         _conn.Dispose();
     }
 
     public async ValueTask DisposeAsync()
     {
+        await _conn.CloseAsync();
         await _conn.DisposeAsync();
     }
 }

ConsoleApp2/Services/ProcessContext.cs

@@ -1,5 +1,8 @@
 namespace ConsoleApp2.Services;
 
+/// <summary>
+/// 处理上下文类，标识处理进度
+/// </summary>
 public class ProcessContext
 {
     private int _inputCount;

ConsoleApp2/Services/TaskManager.cs

@@ -3,14 +3,17 @@ using Microsoft.Extensions.Logging;
 
 namespace ConsoleApp2.Services;
 
+/// <summary>
+/// 快速批量创建和等待任务
+/// </summary>
 public class TaskManager
 {
     private readonly ConcurrentBag<Task> _tasks;
     private readonly ILogger _logger;
 
+    public int TaskCount => _tasks.Count;
     public int RunningTaskCount => _tasks.Count(task => !task.IsCompleted);
     public IReadOnlyCollection<Task> Tasks => _tasks;
-    public bool MainTaskCompleted { get; set; }
 
     public TaskManager(ILogger<TaskManager> logger)
     {
@@ -18,11 +21,22 @@ public class TaskManager
         _logger = logger;
     }
 
-    public Task<TResult> CreateTask<TResult>(Func<TResult> func)
+    public void CreateTask<TResult>(Func<TResult> func, CancellationToken cancellationToken = default)
     {
-        var task = Task.Factory.StartNew(func);
+        var task = Task.Factory.StartNew(func, cancellationToken);
         _tasks.Add(task);
-        _logger.LogDebug("New task created.");
+        _logger.LogDebug("New task created");
-        return task;
+    }
+
+    public async Task WaitAll()
+    {
+        await Task.WhenAll(_tasks);
+    }
+
+    public void ClearTask()
+    {
+        if(RunningTaskCount != 0)
+            throw new InvalidOperationException("Unable to clear task. There are still running tasks");
+        _tasks.Clear();
     }
 }

ConsoleApp2/Services/TsvSource.cs

@@ -1,6 +0,0 @@
-namespace ConsoleApp2.Services;
-
-public class TsvSource
-{
-    
-}