改进模拟数据生成器，代码质量优化

2024-12-25 15:09:16 +08:00 · 2024-12-25 15:09:16 +08:00 · 77a3909160
commit 77a3909160
parent 4986c60416
10 changed files with 102 additions and 58 deletions
--- a/MesETL.App/HostedServices/MainHostedService.cs
+++ b/MesETL.App/HostedServices/MainHostedService.cs
@ -114,6 +114,7 @@ public class MainHostedService : BackgroundService
    {
        var connStr = _databaseOptions.Value.ConnectionString
                      ?? throw new ApplicationException("分库配置中没有配置数据库");
+        _logger.LogWarning("已开启MySQL延迟写入功能并禁用重做日志，请注意数据安全");
        if (enable)
        {
            await DatabaseHelper.NonQueryAsync(connStr,
--- a/MesETL.App/HostedServices/OutputService.cs
+++ b/MesETL.App/HostedServices/OutputService.cs
@ -164,6 +164,6 @@ public class OutputService : IOutputService
            _context.AddOutput(value);
            _context.AddTableOutput(key, value);
        }
-        _logger.LogTrace("输出任务：刷新了 {Count} 条记录", tableOutput.Values.Sum(i => i));
+        // _logger.LogTrace("输出任务：刷新了 {Count} 条记录", tableOutput.Values.Sum(i => i));
    }
 }
--- a/MesETL.App/Program.cs
+++ b/MesETL.App/Program.cs
@ -124,6 +124,8 @@ async Task RunProgram()
                    }
                    break;
                }
+#if USE_TEST_DB
+                // 测试环境的OrderExtra表没有分区，故按照SharedKey清理
                // 清理ShardKey < 24010的
                case TableNames.OrderExtra:
                {
@ -134,6 +136,7 @@ async Task RunProgram()
                    }
                    break;
                }   
+#endif
                // 清理(Status != 0 || Deleted = 1) && ID前四位 < 2401的
                case TableNames.OrderScrapBoard:
                {
@ -214,16 +217,6 @@ async Task RunProgram()

                    break;
                }
-                // 移除ViewFileName列
-                case TableNames.OrderModule:
-                {
-#if USE_TEST_DB
-                    if (record.HeaderExists("ViewFileName"))
-#endif
-                        record.RemoveField("ViewFileName");
-                    break;
-                }
-#if USE_TEST_DB
                // 删除ID列，让数据库自行递增
                // TODO: 数据表改进，删除ID列或是替换为流水号
                case TableNames.ProcessStepEfficiency:
@ -236,17 +229,18 @@ async Task RunProgram()
                    record.RemoveField("ID");
                    break;
                }
+                case TableNames.SysConfig:
+                {
+                    record.RemoveField("Key");
+                    break;
+                }
+#if USE_TEST_DB
                // 测试环境忽略PlaceData列，生产环境会提前将其移除
                case TableNames.SimplePlanOrder:
                {
                    record.RemoveField("PlaceData");
                    break;
                }
-                case TableNames.SysConfig:
-                {
-                    record.RemoveField("Key");
-                    break;
-                }
 #endif
                default: break;
            }
@ -348,13 +342,15 @@ async Task RunProgram()
    host.Services.AddLogging(builder =>
    {
        builder.ClearProviders();
-        builder.AddSerilog(new LoggerConfiguration()
-            .WriteTo.Console().MinimumLevel.Debug()
+        var logger = new LoggerConfiguration()
+            .MinimumLevel.Verbose()
+            .WriteTo.Console()
            .WriteTo.File(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"./Log/Error/{ErrorRecorder.UID}.log"),
-                restrictedToMinimumLevel:LogEventLevel.Error)
+                restrictedToMinimumLevel: LogEventLevel.Error)
            // .WriteTo.File("./Log/Info/{ErrorRecorder.UID}.log", restrictedToMinimumLevel:LogEventLevel.Information) //性能考虑暂不使用
-            .CreateLogger()
-        );
+            .CreateLogger();
+        builder.AddSerilog(logger);
+        Log.Logger = logger;
    });

    host.Services.AddDataSourceFactory();
@ -372,7 +368,7 @@ async Task RunProgram()
    host.Services.AddHostedService<MainHostedService>();
    host.Services.AddSingleton<IInputService, FileInputService>();
    host.Services.AddSingleton<ITransformService, TransformService>();
-    host.Services.AddSingleton<IOutputService, OutputService>();
+    host.Services.AddSingleton<IOutputService, VoidOutputService>();
    host.Services.AddSingleton<TaskMonitorService>();
    // host.Services.AddRedisCache(redisOptions);
    host.Services.AddSingleton<ICacher, MemoryCache>();
--- a/MesETL.App/Services/ETL/MySqlDestination.cs
+++ b/MesETL.App/Services/ETL/MySqlDestination.cs
@ -64,7 +64,7 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
        if (_recordCache.Count == 0)
            return;

-        var cmd = _conn.CreateCommand();
+        await using var cmd = _conn.CreateCommand();
        cmd.CommandTimeout = 0;
       
        try
@ -255,11 +255,13 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
    {
        _conn.Close();
        _conn.Dispose();
+        _recordCache.Clear();
    }

    public async ValueTask DisposeAsync()
    {
        await _conn.CloseAsync();
        await _conn.DisposeAsync();
+        _recordCache.Clear();
    }
 }
--- a/MesETL.App/Services/TaskManager.cs
+++ b/MesETL.App/Services/TaskManager.cs
@ -1,4 +1,5 @@
-using ApplicationException = System.ApplicationException;
+using Serilog;
+using ApplicationException = System.ApplicationException;
 using TaskExtensions = MesETL.Shared.Helper.TaskExtensions;

 namespace MesETL.App.Services;
@ -37,6 +38,8 @@ public class TaskManager
    {
        var task = Task.Run(async () =>
        {
+            // Log.Logger.Verbose("[任务管理器] 新的任务已创建");
+            Interlocked.Increment(ref _runningTaskCount);
            try
            {
                await func();
@ -45,13 +48,13 @@ public class TaskManager
            catch(Exception ex)
            {
                OnException?.Invoke(ex);
+                Log.Logger.Error(ex, "[任务管理器] 执行任务时出错");
            }
            finally
            {
                Interlocked.Decrement(ref _runningTaskCount);
            }
        }, cancellationToken);
-        Interlocked.Increment(ref _runningTaskCount);
        return task;
    }

@ -59,8 +62,10 @@ public class TaskManager
    {
        var task = Task.Factory.StartNew(async obj => // 性能考虑，这个lambda中不要捕获任何外部变量!
        {
+            // Log.Logger.Verbose("[任务管理器] 新的任务已创建");
            if (obj is not Tuple<Func<object?, Task>, object?> tuple)
                throw new ApplicationException("这个异常不该出现");
+            Interlocked.Increment(ref _runningTaskCount);
            try
            {
                await tuple.Item1(tuple.Item2);
@ -69,13 +74,13 @@ public class TaskManager
            catch(Exception ex)
            {
                OnException?.Invoke(ex);
+                Log.Logger.Error(ex, "[任务管理器] 执行任务时出错");
            }
            finally
            {
                Interlocked.Decrement(ref _runningTaskCount);
            }
        }, Tuple.Create(func, arg), cancellationToken).Unwrap();
-        Interlocked.Increment(ref _runningTaskCount);
        return task;
    }
 }
--- a/MesETL.App/appsettings.json
+++ b/MesETL.App/appsettings.json
@ -1,7 +1,7 @@
 {
-  "MemoryThreshold": 8,
+  "MemoryThreshold": 6,
  "GCIntervalMilliseconds": -1,
-  "UnsafeVariable": false,
+  "UnsafeVariable": true,
  "Logging": {
    "LogLevel": {
      "Default": "Trace"
@ -33,7 +33,7 @@
    }
  },  
  "RecordQueue":{
-    "ProducerQueueLength": 40000, // 输入队列最大长度
+    "ProducerQueueLength": 20000, // 输入队列最大长度
    "ConsumerQueueLength": 20000, // 每个输出队列最大长度
    "MaxByteCount": 3221225472 // 队列最大字节数
  },
@ -44,7 +44,7 @@
  "TenantDb": // 分库配置
  {
    "TenantKey" : "CompanyID",
-    "UseDbGroup": "prod",
+    "UseDbGroup": "mock",
    "DbGroups": {
      "test": {
        "cferp_test_1": 1000,
@ -58,6 +58,20 @@
        "mesdb_4": 15000,
        "mesdb_5": 20000,
        "mesdb_6": 2147483647
+      },
+      "mock":{
+        "mesdb_1": 5000,
+        "mesdb_2": 10000,
+        "mesdb_3": 15000,
+        "mesdb_4": 20000,
+        "mesdb_5": 2147483647
+      },
+      "mock_void":{
+        "mesdb_1_void": 5000,
+        "mesdb_2_void": 10000,
+        "mesdb_3_void": 15000,
+        "mesdb_4_void": 20000,
+        "mesdb_5_void": 2147483647
      }
    }
  }
--- a/MesETL.Test/DatabaseToolBox.cs
+++ b/MesETL.Test/DatabaseToolBox.cs
@ -177,7 +177,6 @@ public class DatabaseToolBox
    [InlineData("mesdb_3")]
    [InlineData("mesdb_4")]
    [InlineData("mesdb_5")]
-    [InlineData("mesdb_6")]
    public async Task TruncateAllTable(string database)
    {
        var tables = await DatabaseHelper.QueryTableAsync(ConnStr,
--- a/Mesdb.DataGenerator/MockInputService.cs
+++ b/Mesdb.DataGenerator/MockInputService.cs
@ -1,6 +1,8 @@
+using System.Runtime;
 using MesETL.App.Const;
 using MesETL.App.HostedServices.Abstractions;
 using MesETL.App.Services;
+using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
@ -13,14 +15,16 @@ public class MockInputService : IInputService
    private readonly ProcessContext _context;
    private readonly IOptions<MockInputOptions> _options;
    private readonly ILogger _logger;
+    private readonly long _memoryThreshold;

    public MockInputService([FromKeyedServices(ConstVar.Producer)]DataRecordQueue producerQueue, ProcessContext context, IOptions<MockInputOptions> options,
-        ILogger<MockInputService> logger)
+        ILogger<MockInputService> logger, IConfiguration configuration)
    {
        _producerQueue = producerQueue;
        _context = context;
        _options = options;
        _logger = logger;
+        _memoryThreshold = (long)(configuration.GetValue<double>("MemoryThreshold", 8) * 1024 * 1024 * 1024);
    }

    public async Task ExecuteAsync(CancellationToken cancellationToken)
@ -32,6 +36,13 @@ public class MockInputService : IInputService

            for (int i = 0; i < options.Amount; i++)
            {
+                if (GC.GetTotalMemory(false) > _memoryThreshold)
+                {
+                    _logger.LogWarning("内存使用率过高，暂缓输入");
+                    GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
+                    GC.Collect();
+                    await Task.Delay(3000, cancellationToken);
+                }
                var ctx = new TableMockContext()
                {
                    Index = i,
--- a/Mesdb.DataGenerator/OrderModuleExtraHelper.cs
+++ b/Mesdb.DataGenerator/OrderModuleExtraHelper.cs
--- a/Mesdb.DataGenerator/Program.cs
+++ b/Mesdb.DataGenerator/Program.cs
@ -28,20 +28,22 @@ async Task RunProgram()
    ThreadPool.SetMaxThreads(200, 200);
    var host = Host.CreateApplicationBuilder(args);

-    var dbGroup = new Dictionary<string, int>()
-    {
-        { "mesdb_1", 5000 },
-        { "mesdb_2", 10000 },
-        { "mesdb_3", 15000 },
-        { "mesdb_4", 20000 },
-        { "mesdb_5", 2147483647 },
-    };
+    var inputOptions = host.Configuration.GetRequiredSection("Input").Get<DataInputOptions>()
+                       ?? throw new ApplicationException("缺少Input配置");

+    var transformOptions = host.Configuration.GetRequiredSection("Transform").Get<DataTransformOptions>()
+                           ?? throw new ApplicationException("缺少Transform配置");
+
+    var outputOptions = host.Configuration.GetRequiredSection("Output").Get<DatabaseOutputOptions>()
+                        ?? throw new ApplicationException("缺少Output配置");
+
+    var tenantDbSection = host.Configuration.GetRequiredSection("TenantDb");
    var tenantDbOptions = new TenantDbOptions()
    {
-        TenantKey = "CompanyID",
-        DbGroup = dbGroup,
-        UseDbGroup = "Prod",
+        TenantKey = tenantDbSection.GetValue<string>(nameof(TenantDbOptions.TenantKey)) ??
+                    throw new ApplicationException("分库配置缺少分库键TenantKey"),
+        UseDbGroup = tenantDbSection.GetValue<string>(nameof(TenantDbOptions.UseDbGroup)) ??
+                     throw new ApplicationException("分库配置缺少使用分库组UseDbGroup")
    };
    host.Services.Configure<TenantDbOptions>(options =>
    {
@ -49,6 +51,9 @@ async Task RunProgram()
        options.DbGroup = tenantDbOptions.DbGroup;
        options.UseDbGroup = tenantDbOptions.UseDbGroup;
    });
+    tenantDbOptions.DbGroup = tenantDbSection.GetRequiredSection($"DbGroups:{tenantDbOptions.UseDbGroup}")
+                                  .Get<Dictionary<string, int>>()
+                              ?? throw new ApplicationException($"分库配置无法解析分库组{tenantDbOptions.UseDbGroup}，请检查配置");

    host.Services.Configure<MockInputOptions>(options =>
    {
@ -206,6 +211,11 @@ async Task RunProgram()

    host.Services.Configure<DataTransformOptions>(options =>
    {
+        options.StrictMode = transformOptions.StrictMode;
+        options.EnableFilter = transformOptions.EnableFilter;
+        options.EnableReplacer = transformOptions.EnableReplacer;
+        options.EnableReBuilder = transformOptions.EnableReBuilder;
+        
        options.DatabaseFilter = record =>
        {
            var companyId = int.Parse(record[tenantDbOptions.TenantKey]); // 每个实体都应存在CompanyID，否则异常
@ -215,11 +225,16 @@ async Task RunProgram()

    host.Services.Configure<DatabaseOutputOptions>(options =>
    {
-        options.ConnectionString = "Server=192.168.1.246;Port=3333;UserId=root;Password=123456;";
-        options.FlushCount = 10000;
-        options.MaxAllowedPacket = 67108864;
-        options.MaxDatabaseOutputTask = 4;
+        options.ConnectionString = outputOptions.ConnectionString;
+        options.FlushCount = outputOptions.FlushCount;
+        options.MaxAllowedPacket = outputOptions.MaxAllowedPacket / 2;
+        options.MaxDatabaseOutputTask = outputOptions.MaxDatabaseOutputTask;
+        options.TreatJsonAsHex = outputOptions.TreatJsonAsHex;
+        options.NoOutput = outputOptions.NoOutput;
+        options.ForUpdate = outputOptions.ForUpdate;

+        // 配置列的类型以便于在输出时区分二进制内容
+        // Prod server
        options.ColumnTypeConfig = new Dictionary<string, ColumnType>
        {
            { "machine.Settings", ColumnType.Text },
@ -253,14 +268,15 @@ async Task RunProgram()
    host.Services.AddLogging(builder =>
    {
        builder.ClearProviders();
-        builder.AddSerilog(new LoggerConfiguration()
-            .MinimumLevel.Debug()
+        var logger = new LoggerConfiguration()
+            .MinimumLevel.Verbose()
            .WriteTo.Console()
            .WriteTo.File(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"./Log/Error/{ErrorRecorder.UID}.log"),
                restrictedToMinimumLevel: LogEventLevel.Error)
            // .WriteTo.File("./Log/Info/{ErrorRecorder.UID}.log", restrictedToMinimumLevel:LogEventLevel.Information) //性能考虑暂不使用
-            .CreateLogger()
-        );
+            .CreateLogger();
+        builder.AddSerilog(logger);
+        Log.Logger = logger;
    });

    host.Services.AddDataSourceFactory();
@ -271,7 +287,7 @@ async Task RunProgram()
    var consLen = host.Configuration.GetRequiredSection("RecordQueue").GetValue<int>("ConsumerQueueLength");
    var maxCharCount = host.Configuration.GetRequiredSection("RecordQueue").GetValue<long>("MaxByteCount") / 2;
    host.Services.AddKeyedSingleton<DataRecordQueue>(ConstVar.Producer, new DataRecordQueue(prodLen, maxCharCount));
-    host.Services.AddRecordQueuePool(dbGroup.Keys
+    host.Services.AddRecordQueuePool(tenantDbOptions.DbGroup.Keys
        .Select(key => (key: key, queue: new DataRecordQueue(consLen, maxCharCount))).ToArray());
    // host.Services.AddSingleton<ITaskMonitorLogger, CacheTaskMonitorLogger>();
    host.Services.AddSingleton<ITaskMonitorLogger, LoggerTaskMonitorLogger>();