Update

ConsoleApp2/Services/CsvSource.cs

@@ -1,24 +1,26 @@
 using System.Text;
-using ConsoleApp2.Entities;
 using ConsoleApp2.Helpers;
 using Microsoft.Extensions.Logging;
 
 namespace ConsoleApp2.Services;
 
-public class NewCsvSource
+/// <summary>
+/// CSV文件读取
+/// </summary>
+public class CsvSource
 {
     private readonly string _filePath;
     private readonly StreamReader _reader;
     private readonly ILogger? _logger;
     private readonly string _tableName;
 
-    public DataRecord Current { get; protected set; }
+    public DataRecord Current { get; private set; }
     public string[]? Headers { get; }
     public string? CurrentRaw { get; private set; }
     public string Delimiter { get; private set; }
     public char QuoteChar { get; private set; }
 
-    public NewCsvSource(string filePath, string[]? headers = null, string delimiter = ",", char quoteChar = '"',
+    public CsvSource(string filePath, string[]? headers = null, string delimiter = ",", char quoteChar = '"',
         ILogger? logger = null)
     {
         _filePath = filePath;

ConsoleApp2/Services/DataRecordQueue.cs

@@ -1,9 +1,11 @@
 using System.Collections.Concurrent;
 using System.Diagnostics.CodeAnalysis;
-using ConsoleApp2.Entities;
 
 namespace ConsoleApp2.Services;
 
+/// <summary>
+/// 数据队列
+/// </summary>
 public class DataRecordQueue : IDisposable
 {
     private readonly BlockingCollection<DataRecord> _queue;
@@ -17,7 +19,7 @@ public class DataRecordQueue : IDisposable
 
     public DataRecordQueue()
     {
-        _queue = new BlockingCollection<DataRecord>();
+        _queue = new BlockingCollection<DataRecord>(200_000); // 队列最长为20W条记录
     }
     
     public bool TryDequeue([MaybeNullWhen(false)] out DataRecord record)

ConsoleApp2/Services/JsvSource.cs

@@ -1,10 +1,13 @@
-using ConsoleApp2.Entities;
-using ConsoleApp2.Helpers;
+using ConsoleApp2.Helpers;
 using Microsoft.Extensions.Logging;
 using ServiceStack.Text;
 
 namespace ConsoleApp2.Services;
 
+/// <summary>
+/// 读取Jsv格式文件
+/// </summary>
+[Obsolete]
 public class JsvSource : IDisposable
 {
     private readonly string _filePath;

ConsoleApp2/Services/MySqlDestination.cs

@@ -1,11 +1,13 @@
 using System.Text;
-using ConsoleApp2.Entities;
 using ConsoleApp2.Helpers;
 using Microsoft.Extensions.Logging;
 using MySqlConnector;
 
 namespace ConsoleApp2.Services;
 
+/// <summary>
+/// Mysql导出
+/// </summary>
 public class MySqlDestination : IDisposable, IAsyncDisposable
 {
     private readonly Dictionary<string, IList<DataRecord>> _recordCache;
@@ -13,8 +15,6 @@ public class MySqlDestination : IDisposable, IAsyncDisposable
     private readonly ILogger _logger;
     private readonly bool _prettyOutput;
 
-    public static int AddCount;
-
     public MySqlDestination(string connStr, ILogger logger, bool prettyOutput = false)
     {
         _conn = new MySqlConnection(connStr);
@@ -29,7 +29,6 @@ public class MySqlDestination : IDisposable, IAsyncDisposable
         _recordCache.AddOrUpdate(record.TableName, [record], (key, value) =>
         {
             value.Add(record);
-            Interlocked.Increment(ref AddCount);
             return value;
         });
         return Task.CompletedTask;
@@ -60,6 +59,10 @@ public class MySqlDestination : IDisposable, IAsyncDisposable
             _logger.LogCritical(e, "Error when flushing records, sql: {Sql}", cmd.CommandText.Omit(1000));
             throw;
         }
+        finally
+        {
+            await cmd.DisposeAsync();
+        }
     }
 
     public static string SerializeRecords(IDictionary<string, IList<DataRecord>> tableRecords,
@@ -91,22 +94,7 @@ public class MySqlDestination : IDisposable, IAsyncDisposable
                 for (var j = 0; j < record.Fields.Length; j++)
                 {
                     var field = record.Fields[j];
-
-                    #region HandleFields
-
-                    // if (field == "\\N")
-                    //     sb.Append("NULL");
-                    // else if (DumpDataHelper.CheckHexField(field))
-                    // {
-                    //     // if (StringExtensions.CheckJsonHex(field))
-                    //         sb.Append($"0x{field}");
-                    // }
-                    // else
-                    //     sb.Append($"'{field}'");
-
                     sb.Append(field);
-                    #endregion
-
                     if (j != record.Fields.Length - 1)
                         sb.Append(',');
                 }
@@ -127,11 +115,13 @@ public class MySqlDestination : IDisposable, IAsyncDisposable
 
     public void Dispose()
     {
+        _conn.Close();
         _conn.Dispose();
     }
 
     public async ValueTask DisposeAsync()
     {
+        await _conn.CloseAsync();
         await _conn.DisposeAsync();
     }
 }

ConsoleApp2/Services/ProcessContext.cs

@@ -1,5 +1,8 @@
 namespace ConsoleApp2.Services;
 
+/// <summary>
+/// 处理上下文类，标识处理进度
+/// </summary>
 public class ProcessContext
 {
     private int _inputCount;

ConsoleApp2/Services/TaskManager.cs

@@ -3,14 +3,17 @@ using Microsoft.Extensions.Logging;
 
 namespace ConsoleApp2.Services;
 
+/// <summary>
+/// 快速批量创建和等待任务
+/// </summary>
 public class TaskManager
 {
     private readonly ConcurrentBag<Task> _tasks;
     private readonly ILogger _logger;
 
+    public int TaskCount => _tasks.Count;
     public int RunningTaskCount => _tasks.Count(task => !task.IsCompleted);
     public IReadOnlyCollection<Task> Tasks => _tasks;
-    public bool MainTaskCompleted { get; set; }
 
     public TaskManager(ILogger<TaskManager> logger)
     {
@@ -18,11 +21,22 @@ public class TaskManager
         _logger = logger;
     }
 
-    public Task<TResult> CreateTask<TResult>(Func<TResult> func)
+    public void CreateTask<TResult>(Func<TResult> func, CancellationToken cancellationToken = default)
     {
-        var task = Task.Factory.StartNew(func);
+        var task = Task.Factory.StartNew(func, cancellationToken);
         _tasks.Add(task);
-        _logger.LogDebug("New task created.");
-        return task;
+        _logger.LogDebug("New task created");
+    }
+
+    public async Task WaitAll()
+    {
+        await Task.WhenAll(_tasks);
+    }
+
+    public void ClearTask()
+    {
+        if(RunningTaskCount != 0)
+            throw new InvalidOperationException("Unable to clear task. There are still running tasks");
+        _tasks.Clear();
     }
 }

ConsoleApp2/Services/TsvSource.cs

@@ -1,6 +0,0 @@
-namespace ConsoleApp2.Services;
-
-public class TsvSource
-{
-    
-}