Optimize structure

MesETL.App/Services/DataRecordQueue.cs

@@ -1,6 +1,6 @@
 using System.Collections.Concurrent;
 using System.Diagnostics.CodeAnalysis;
-using TaskExtensions = MesETL.App.Helpers.TaskExtensions;
+using TaskExtensions = MesETL.Shared.Helper.TaskExtensions;
 
 namespace MesETL.App.Services;
 
@@ -50,6 +50,9 @@ public class DataRecordQueue : IDisposable
 
     public async Task EnqueueAsync(DataRecord record)
     {
+        if (_queue.Count >= _queue.BoundedCapacity)
+            await Task.Delay(500);
+        
         var charCount = record.FieldCharCount;
         LongestFieldCharCount = Math.Max(LongestFieldCharCount, charCount);
         if(_currentCharCount + charCount > _maxCharCount)

MesETL.App/Services/ETL/MySqlDestination.cs

@@ -3,6 +3,7 @@ using System.Text.RegularExpressions;
 using MesETL.App.Const;
 using MesETL.App.Helpers;
 using MesETL.App.Options;
+using MesETL.Shared.Helper;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using MySqlConnector;

MesETL.App/Services/ErrorRecorder/ErrorRecorder.cs

@@ -1,4 +1,5 @@
 using MesETL.App.Helpers;
+using MesETL.Shared.Helper;
 using Microsoft.Extensions.Logging;
 
 namespace MesETL.App.Services.ErrorRecorder;

MesETL.App/Services/ProcessContext.cs

@@ -11,7 +11,7 @@ public class ProcessContext
     private long _inputCount;
     private long _transformCount;
     private long _outputCount;
-    private readonly ConcurrentDictionary<string, long> _tableProgress = new();
+    private readonly ConcurrentDictionary<string, (long input, long output)> _tableProgress = new();
     public bool HasException => _hasException;
     public bool IsInputCompleted { get; private set; }
     public bool IsTransformCompleted { get; private set; }
@@ -37,7 +37,7 @@ public class ProcessContext
 
 
     // TableName -> Count
-    public IReadOnlyDictionary<string, long> TableProgress => _tableProgress;
+    public IReadOnlyDictionary<string, (long input, long output)> TableProgress => _tableProgress;
 
     public void CompleteInput() => IsInputCompleted = true;
 
@@ -55,16 +55,21 @@ public class ProcessContext
     public void AddOutput() => Interlocked.Increment(ref _outputCount);
     public void AddOutput(int count) => Interlocked.Add(ref _outputCount, count);
 
+    public void AddTableInput(string table, int count)
+    {
+        _tableProgress.AddOrUpdate(table, (input:count, output:0), (k, tuple) =>
+        {
+            tuple.input += count;
+            return tuple;
+        });
+    }
+    
     public void AddTableOutput(string table, int count)
     {
-        _tableProgress.AddOrUpdate(table, count, (k, v) => v + count);
-        AddOutput(count);
-    }
-
-    public long GetTableOutput(string table)
-    {
-        if(!_tableProgress.TryGetValue(table, out var count))
-            throw new ApplicationException($"未找到表{table}输出记录");
-        return count;
+        _tableProgress.AddOrUpdate(table, (input:0, output:count), (k, tuple) =>
+        {
+            tuple.output += count;
+            return tuple;
+        });
     }
 }

MesETL.App/Services/TaskManager.cs

@@ -1,5 +1,5 @@
 using ApplicationException = System.ApplicationException;
-using TaskExtensions = MesETL.App.Helpers.TaskExtensions;
+using TaskExtensions = MesETL.Shared.Helper.TaskExtensions;
 
 namespace MesETL.App.Services;