Update

ConsoleApp2/Program.cs

@@ -1,26 +1,17 @@
-using ConsoleApp2;
+using ConsoleApp2.Const;
 using ConsoleApp2.HostedServices;
+using ConsoleApp2.HostedServices.Abstractions;
 using ConsoleApp2.Options;
 using ConsoleApp2.Services;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
+using MySqlConnector;
 using Serilog;
 
-
-// 加入数据库过滤
-// HostedService不是并行的，完善TaskManager手动开启线程
-// 测试BlockingCollection对速度的影响？
-
-// 重新同步数据
-// Json列和Blob列不一致
-/* JSV导出带转义的列有误
- * order_data_block表id 4153969
- * order_data_parts表Spec列
- * order_module表Name列
- * process_group表Items列
- */
+// 运行之前把Mysql max_allowed_packets 调大
+// 运行之前把process_step表的外键删掉
 
 await RunProgram();
 return;
@@ -29,18 +20,15 @@ async Task RunProgram()
 {
     ThreadPool.SetMaxThreads(200, 200);
     var host = Host.CreateApplicationBuilder();
-    host.Configuration.AddCommandLine(args);
+    host.Configuration.AddCommandLine(args); 
     host.Services.Configure<CsvOptions>(option =>
     {
-        option.DelimiterChar = ',';
+        option.Delimiter = ",";
         option.QuoteChar = '"';
-        option.InputDir = "D:/Dump/MyDumper";
-        option.OutputDir = "D:/DumpOutput";
-        option.MaxThreads = 12;
+        option.InputDir = "D:/Dump";
     });
     host.Services.Configure<DataTransformOptions>(options =>
     {
-        //TODO: Database Filter
         options.DatabaseFilter = record => "cferp_test_1";
         options.ColumnTypeConfig = new()
         {
@@ -73,14 +61,20 @@ async Task RunProgram()
             { "order_block_plan.BlockInfo", ColumnType.Text },
         };
     });
-// 加入数据库过滤后删除
-    host.Services.Configure<DatabaseOptions>(options =>
+    
+    host.Services.Configure<DatabaseOutputOptions>(options =>
     {
-        options.Host = "localhost";
-        options.Port = 33306;
-        options.Database = "cferp_test_1";
-        options.User = "root";
-        options.Password = "123456";
+        options.ConnectionString = new MySqlConnectionStringBuilder
+        {
+            Server = "127.0.0.1",
+            Port = 33306,
+            Database = "cferp_test_1",
+            UserID = "root",
+            Password = "123456",
+            MaximumPoolSize = 50, // 这个值应当小于 max_connections
+        }.ConnectionString;
+        options.MaxTask = 16;
+        options.FlushCount = 200;
     });
     host.Services.AddLogging(builder =>
     {
@@ -91,14 +85,13 @@ async Task RunProgram()
     host.Services.AddSingleton<ProcessContext>();
     host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Producer);
     host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Consumer);
-    host.Services.AddKeyedSingleton<TaskManager>(ProcessStep.Producer);
-    host.Services.AddKeyedSingleton<TaskManager>(ProcessStep.Consumer);
+    host.Services.AddTransient<TaskManager>();
 
+    host.Services.AddHostedService<MainHostedService>();
     host.Services.AddHostedService<TaskMonitorService>();
-    host.Services.AddHostedService<CsvInputService>();
-    host.Services.AddHostedService<DataTransformService>();
-    host.Services.AddHostedService<MysqlOutputService>();
-
+    host.Services.AddSingleton<IInputService, InputService>();
+    host.Services.AddSingleton<ITransformService, TransformService>();
+    host.Services.AddSingleton<IOutputService, OutputService>();
 
     var app = host.Build();
     await app.RunAsync();