CZY/MES-ETL
1
0
Fork 0
Code Pull Requests Releases Activity

改进模拟数据生成器，代码质量优化

Browse Source
This commit is contained in:
陈梓阳 2024-12-25 15:09:16 +08:00
parent 4986c60416
commit 77a3909160
10 changed files with 102 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
MesETL.App/HostedServices/MainHostedService.cs
View File

@ -114,6 +114,7 @@ public class MainHostedService : BackgroundService
{ {
var connStr = _databaseOptions.Value.ConnectionString var connStr = _databaseOptions.Value.ConnectionString
?? throw new ApplicationException("分库配置中没有配置数据库"); ?? throw new ApplicationException("分库配置中没有配置数据库");
_logger.LogWarning("已开启MySQL延迟写入功能并禁用重做日志，请注意数据安全");
if (enable) if (enable)
{ {
await DatabaseHelper.NonQueryAsync(connStr, await DatabaseHelper.NonQueryAsync(connStr,

2
MesETL.App/HostedServices/OutputService.cs
View File

@ -164,6 +164,6 @@ public class OutputService : IOutputService
_context.AddOutput(value); _context.AddOutput(value);
_context.AddTableOutput(key, value); _context.AddTableOutput(key, value);
} }
_logger.LogTrace("输出任务：刷新了 {Count} 条记录", tableOutput.Values.Sum(i => i)); // _logger.LogTrace("输出任务：刷新了 {Count} 条记录", tableOutput.Values.Sum(i => i));
} }
} }

42
MesETL.App/Program.cs
View File

@ -124,6 +124,8 @@ async Task RunProgram()
} }
break; break;
} }
#if USE_TEST_DB
// 测试环境的OrderExtra表没有分区，故按照SharedKey清理
// 清理ShardKey < 24010的 // 清理ShardKey < 24010的
case TableNames.OrderExtra: case TableNames.OrderExtra:
{ {
@ -133,7 +135,8 @@ async Task RunProgram()
return false; return false;
} }
break; break;
} }
#endif
// 清理(Status != 0 || Deleted = 1) && ID前四位 < 2401的 // 清理(Status != 0 || Deleted = 1) && ID前四位 < 2401的
case TableNames.OrderScrapBoard: case TableNames.OrderScrapBoard:
{ {
@ -214,16 +217,6 @@ async Task RunProgram()
break; break;
} }
// 移除ViewFileName列
case TableNames.OrderModule:
{
#if USE_TEST_DB
if (record.HeaderExists("ViewFileName"))
#endif
record.RemoveField("ViewFileName");
break;
}
#if USE_TEST_DB
// 删除ID列，让数据库自行递增 // 删除ID列，让数据库自行递增
// TODO: 数据表改进，删除ID列或是替换为流水号 // TODO: 数据表改进，删除ID列或是替换为流水号
case TableNames.ProcessStepEfficiency: case TableNames.ProcessStepEfficiency:
@ -236,17 +229,18 @@ async Task RunProgram()
record.RemoveField("ID"); record.RemoveField("ID");
break; break;
} }
case TableNames.SysConfig:
{
record.RemoveField("Key");
break;
}
#if USE_TEST_DB
// 测试环境忽略PlaceData列，生产环境会提前将其移除 // 测试环境忽略PlaceData列，生产环境会提前将其移除
case TableNames.SimplePlanOrder: case TableNames.SimplePlanOrder:
{ {
record.RemoveField("PlaceData"); record.RemoveField("PlaceData");
break; break;
} }
case TableNames.SysConfig:
{
record.RemoveField("Key");
break;
}
#endif #endif
default: break; default: break;
} }
@ -348,13 +342,15 @@ async Task RunProgram()
host.Services.AddLogging(builder => host.Services.AddLogging(builder =>
{ {
builder.ClearProviders(); builder.ClearProviders();
builder.AddSerilog(new LoggerConfiguration() var logger = new LoggerConfiguration()
.WriteTo.Console().MinimumLevel.Debug() .MinimumLevel.Verbose()
.WriteTo.File(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"./Log/Error/{ErrorRecorder.UID}.log"), .WriteTo.Console()
restrictedToMinimumLevel:LogEventLevel.Error) .WriteTo.File(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"./Log/Error/{ErrorRecorder.UID}.log"),
restrictedToMinimumLevel: LogEventLevel.Error)
// .WriteTo.File("./Log/Info/{ErrorRecorder.UID}.log", restrictedToMinimumLevel:LogEventLevel.Information) //性能考虑暂不使用 // .WriteTo.File("./Log/Info/{ErrorRecorder.UID}.log", restrictedToMinimumLevel:LogEventLevel.Information) //性能考虑暂不使用
.CreateLogger() .CreateLogger();
); builder.AddSerilog(logger);
Log.Logger = logger;
}); });
host.Services.AddDataSourceFactory(); host.Services.AddDataSourceFactory();
@ -372,7 +368,7 @@ async Task RunProgram()
host.Services.AddHostedService<MainHostedService>(); host.Services.AddHostedService<MainHostedService>();
host.Services.AddSingleton<IInputService, FileInputService>(); host.Services.AddSingleton<IInputService, FileInputService>();
host.Services.AddSingleton<ITransformService, TransformService>(); host.Services.AddSingleton<ITransformService, TransformService>();
host.Services.AddSingleton<IOutputService, OutputService>(); host.Services.AddSingleton<IOutputService, VoidOutputService>();
host.Services.AddSingleton<TaskMonitorService>(); host.Services.AddSingleton<TaskMonitorService>();
// host.Services.AddRedisCache(redisOptions); // host.Services.AddRedisCache(redisOptions);
host.Services.AddSingleton<ICacher, MemoryCache>(); host.Services.AddSingleton<ICacher, MemoryCache>();

4
MesETL.App/Services/ETL/MySqlDestination.cs
View File

@ -64,7 +64,7 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
if (_recordCache.Count == 0) if (_recordCache.Count == 0)
return; return;
var cmd = _conn.CreateCommand(); await using var cmd = _conn.CreateCommand();
cmd.CommandTimeout = 0; cmd.CommandTimeout = 0;
try try
@ -255,11 +255,13 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
{ {
_conn.Close(); _conn.Close();
_conn.Dispose(); _conn.Dispose();
_recordCache.Clear();
} }
public async ValueTask DisposeAsync() public async ValueTask DisposeAsync()
{ {
await _conn.CloseAsync(); await _conn.CloseAsync();
await _conn.DisposeAsync(); await _conn.DisposeAsync();
_recordCache.Clear();
} }
} }

11
MesETL.App/Services/TaskManager.cs
View File

@ -1,4 +1,5 @@
using ApplicationException = System.ApplicationException; using Serilog;
using ApplicationException = System.ApplicationException;
using TaskExtensions = MesETL.Shared.Helper.TaskExtensions; using TaskExtensions = MesETL.Shared.Helper.TaskExtensions;
namespace MesETL.App.Services; namespace MesETL.App.Services;
@ -37,6 +38,8 @@ public class TaskManager
{ {
var task = Task.Run(async () => var task = Task.Run(async () =>
{ {
// Log.Logger.Verbose("[任务管理器] 新的任务已创建");
Interlocked.Increment(ref _runningTaskCount);
try try
{ {
await func(); await func();
@ -45,13 +48,13 @@ public class TaskManager
catch(Exception ex) catch(Exception ex)
{ {
OnException?.Invoke(ex); OnException?.Invoke(ex);
Log.Logger.Error(ex, "[任务管理器] 执行任务时出错");
} }
finally finally
{ {
Interlocked.Decrement(ref _runningTaskCount); Interlocked.Decrement(ref _runningTaskCount);
} }
}, cancellationToken); }, cancellationToken);
Interlocked.Increment(ref _runningTaskCount);
return task; return task;
} }
@ -59,8 +62,10 @@ public class TaskManager
{ {
var task = Task.Factory.StartNew(async obj => // 性能考虑，这个lambda中不要捕获任何外部变量! var task = Task.Factory.StartNew(async obj => // 性能考虑，这个lambda中不要捕获任何外部变量!
{ {
// Log.Logger.Verbose("[任务管理器] 新的任务已创建");
if (obj is not Tuple<Func<object?, Task>, object?> tuple) if (obj is not Tuple<Func<object?, Task>, object?> tuple)
throw new ApplicationException("这个异常不该出现"); throw new ApplicationException("这个异常不该出现");
Interlocked.Increment(ref _runningTaskCount);
try try
{ {
await tuple.Item1(tuple.Item2); await tuple.Item1(tuple.Item2);
@ -69,13 +74,13 @@ public class TaskManager
catch(Exception ex) catch(Exception ex)
{ {
OnException?.Invoke(ex); OnException?.Invoke(ex);
Log.Logger.Error(ex, "[任务管理器] 执行任务时出错");
} }
finally finally
{ {
Interlocked.Decrement(ref _runningTaskCount); Interlocked.Decrement(ref _runningTaskCount);
} }
}, Tuple.Create(func, arg), cancellationToken).Unwrap(); }, Tuple.Create(func, arg), cancellationToken).Unwrap();
Interlocked.Increment(ref _runningTaskCount);
return task; return task;
} }
} }

22
MesETL.App/appsettings.json
View File

@ -1,7 +1,7 @@
{ {
"MemoryThreshold": 8, "MemoryThreshold": 6,
"GCIntervalMilliseconds": -1, "GCIntervalMilliseconds": -1,
"UnsafeVariable": false, "UnsafeVariable": true,
"Logging": { "Logging": {
"LogLevel": { "LogLevel": {
"Default": "Trace" "Default": "Trace"
@ -33,7 +33,7 @@
} }
}, },
"RecordQueue":{ "RecordQueue":{
"ProducerQueueLength": 40000, // 输入队列最大长度 "ProducerQueueLength": 20000, // 输入队列最大长度
"ConsumerQueueLength": 20000, // 每个输出队列最大长度 "ConsumerQueueLength": 20000, // 每个输出队列最大长度
"MaxByteCount": 3221225472 // 队列最大字节数 "MaxByteCount": 3221225472 // 队列最大字节数
}, },
@ -44,7 +44,7 @@
"TenantDb": // 分库配置 "TenantDb": // 分库配置
{ {
"TenantKey" : "CompanyID", "TenantKey" : "CompanyID",
"UseDbGroup": "prod", "UseDbGroup": "mock",
"DbGroups": { "DbGroups": {
"test": { "test": {
"cferp_test_1": 1000, "cferp_test_1": 1000,
@ -58,6 +58,20 @@
"mesdb_4": 15000, "mesdb_4": 15000,
"mesdb_5": 20000, "mesdb_5": 20000,
"mesdb_6": 2147483647 "mesdb_6": 2147483647
},
"mock":{
"mesdb_1": 5000,
"mesdb_2": 10000,
"mesdb_3": 15000,
"mesdb_4": 20000,
"mesdb_5": 2147483647
},
"mock_void":{
"mesdb_1_void": 5000,
"mesdb_2_void": 10000,
"mesdb_3_void": 15000,
"mesdb_4_void": 20000,
"mesdb_5_void": 2147483647
} }
} }
} }

1
MesETL.Test/DatabaseToolBox.cs
View File

@ -177,7 +177,6 @@ public class DatabaseToolBox
[InlineData("mesdb_3")] [InlineData("mesdb_3")]
[InlineData("mesdb_4")] [InlineData("mesdb_4")]
[InlineData("mesdb_5")] [InlineData("mesdb_5")]
[InlineData("mesdb_6")]
public async Task TruncateAllTable(string database) public async Task TruncateAllTable(string database)
{ {
var tables = await DatabaseHelper.QueryTableAsync(ConnStr, var tables = await DatabaseHelper.QueryTableAsync(ConnStr,

13
Mesdb.DataGenerator/MockInputService.cs
View File

@ -1,6 +1,8 @@
using System.Runtime;
using MesETL.App.Const; using MesETL.App.Const;
using MesETL.App.HostedServices.Abstractions; using MesETL.App.HostedServices.Abstractions;
using MesETL.App.Services; using MesETL.App.Services;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
@ -13,14 +15,16 @@ public class MockInputService : IInputService
private readonly ProcessContext _context; private readonly ProcessContext _context;
private readonly IOptions<MockInputOptions> _options; private readonly IOptions<MockInputOptions> _options;
private readonly ILogger _logger; private readonly ILogger _logger;
private readonly long _memoryThreshold;
public MockInputService([FromKeyedServices(ConstVar.Producer)]DataRecordQueue producerQueue, ProcessContext context, IOptions<MockInputOptions> options, public MockInputService([FromKeyedServices(ConstVar.Producer)]DataRecordQueue producerQueue, ProcessContext context, IOptions<MockInputOptions> options,
ILogger<MockInputService> logger) ILogger<MockInputService> logger, IConfiguration configuration)
{ {
_producerQueue = producerQueue; _producerQueue = producerQueue;
_context = context; _context = context;
_options = options; _options = options;
_logger = logger; _logger = logger;
_memoryThreshold = (long)(configuration.GetValue<double>("MemoryThreshold", 8) * 1024 * 1024 * 1024);
} }
public async Task ExecuteAsync(CancellationToken cancellationToken) public async Task ExecuteAsync(CancellationToken cancellationToken)
@ -32,6 +36,13 @@ public class MockInputService : IInputService
for (int i = 0; i < options.Amount; i++) for (int i = 0; i < options.Amount; i++)
{ {
if (GC.GetTotalMemory(false) > _memoryThreshold)
{
_logger.LogWarning("内存使用率过高，暂缓输入");
GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
GC.Collect();
await Task.Delay(3000, cancellationToken);
}
var ctx = new TableMockContext() var ctx = new TableMockContext()
{ {
Index = i, Index = i,

2
Mesdb.DataGenerator/OrderModuleExtraHelper.cs
View File

File diff suppressed because one or more lines are too long

62
Mesdb.DataGenerator/Program.cs
View File

@ -27,21 +27,23 @@ async Task RunProgram()
{ {
ThreadPool.SetMaxThreads(200, 200); ThreadPool.SetMaxThreads(200, 200);
var host = Host.CreateApplicationBuilder(args); var host = Host.CreateApplicationBuilder(args);
var dbGroup = new Dictionary<string, int>() var inputOptions = host.Configuration.GetRequiredSection("Input").Get<DataInputOptions>()
{ ?? throw new ApplicationException("缺少Input配置");
{ "mesdb_1", 5000 },
{ "mesdb_2", 10000 }, var transformOptions = host.Configuration.GetRequiredSection("Transform").Get<DataTransformOptions>()
{ "mesdb_3", 15000 }, ?? throw new ApplicationException("缺少Transform配置");
{ "mesdb_4", 20000 },
{ "mesdb_5", 2147483647 }, var outputOptions = host.Configuration.GetRequiredSection("Output").Get<DatabaseOutputOptions>()
}; ?? throw new ApplicationException("缺少Output配置");
var tenantDbSection = host.Configuration.GetRequiredSection("TenantDb");
var tenantDbOptions = new TenantDbOptions() var tenantDbOptions = new TenantDbOptions()
{ {
TenantKey = "CompanyID", TenantKey = tenantDbSection.GetValue<string>(nameof(TenantDbOptions.TenantKey)) ??
DbGroup = dbGroup, throw new ApplicationException("分库配置缺少分库键TenantKey"),
UseDbGroup = "Prod", UseDbGroup = tenantDbSection.GetValue<string>(nameof(TenantDbOptions.UseDbGroup)) ??
throw new ApplicationException("分库配置缺少使用分库组UseDbGroup")
}; };
host.Services.Configure<TenantDbOptions>(options => host.Services.Configure<TenantDbOptions>(options =>
{ {
@ -49,7 +51,10 @@ async Task RunProgram()
options.DbGroup = tenantDbOptions.DbGroup; options.DbGroup = tenantDbOptions.DbGroup;
options.UseDbGroup = tenantDbOptions.UseDbGroup; options.UseDbGroup = tenantDbOptions.UseDbGroup;
}); });
tenantDbOptions.DbGroup = tenantDbSection.GetRequiredSection($"DbGroups:{tenantDbOptions.UseDbGroup}")
.Get<Dictionary<string, int>>()
?? throw new ApplicationException($"分库配置无法解析分库组{tenantDbOptions.UseDbGroup}，请检查配置");
host.Services.Configure<MockInputOptions>(options => host.Services.Configure<MockInputOptions>(options =>
{ {
const float Multiplexer = 0.01F; const float Multiplexer = 0.01F;
@ -206,6 +211,11 @@ async Task RunProgram()
host.Services.Configure<DataTransformOptions>(options => host.Services.Configure<DataTransformOptions>(options =>
{ {
options.StrictMode = transformOptions.StrictMode;
options.EnableFilter = transformOptions.EnableFilter;
options.EnableReplacer = transformOptions.EnableReplacer;
options.EnableReBuilder = transformOptions.EnableReBuilder;
options.DatabaseFilter = record => options.DatabaseFilter = record =>
{ {
var companyId = int.Parse(record[tenantDbOptions.TenantKey]); // 每个实体都应存在CompanyID，否则异常 var companyId = int.Parse(record[tenantDbOptions.TenantKey]); // 每个实体都应存在CompanyID，否则异常
@ -215,11 +225,16 @@ async Task RunProgram()
host.Services.Configure<DatabaseOutputOptions>(options => host.Services.Configure<DatabaseOutputOptions>(options =>
{ {
options.ConnectionString = "Server=192.168.1.246;Port=3333;UserId=root;Password=123456;"; options.ConnectionString = outputOptions.ConnectionString;
options.FlushCount = 10000; options.FlushCount = outputOptions.FlushCount;
options.MaxAllowedPacket = 67108864; options.MaxAllowedPacket = outputOptions.MaxAllowedPacket / 2;
options.MaxDatabaseOutputTask = 4; options.MaxDatabaseOutputTask = outputOptions.MaxDatabaseOutputTask;
options.TreatJsonAsHex = outputOptions.TreatJsonAsHex;
options.NoOutput = outputOptions.NoOutput;
options.ForUpdate = outputOptions.ForUpdate;
// 配置列的类型以便于在输出时区分二进制内容
// Prod server
options.ColumnTypeConfig = new Dictionary<string, ColumnType> options.ColumnTypeConfig = new Dictionary<string, ColumnType>
{ {
{ "machine.Settings", ColumnType.Text }, { "machine.Settings", ColumnType.Text },
@ -253,14 +268,15 @@ async Task RunProgram()
host.Services.AddLogging(builder => host.Services.AddLogging(builder =>
{ {
builder.ClearProviders(); builder.ClearProviders();
builder.AddSerilog(new LoggerConfiguration() var logger = new LoggerConfiguration()
.MinimumLevel.Debug() .MinimumLevel.Verbose()
.WriteTo.Console() .WriteTo.Console()
.WriteTo.File(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"./Log/Error/{ErrorRecorder.UID}.log"), .WriteTo.File(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"./Log/Error/{ErrorRecorder.UID}.log"),
restrictedToMinimumLevel: LogEventLevel.Error) restrictedToMinimumLevel: LogEventLevel.Error)
// .WriteTo.File("./Log/Info/{ErrorRecorder.UID}.log", restrictedToMinimumLevel:LogEventLevel.Information) //性能考虑暂不使用 // .WriteTo.File("./Log/Info/{ErrorRecorder.UID}.log", restrictedToMinimumLevel:LogEventLevel.Information) //性能考虑暂不使用
.CreateLogger() .CreateLogger();
); builder.AddSerilog(logger);
Log.Logger = logger;
}); });
host.Services.AddDataSourceFactory(); host.Services.AddDataSourceFactory();
@ -271,7 +287,7 @@ async Task RunProgram()
var consLen = host.Configuration.GetRequiredSection("RecordQueue").GetValue<int>("ConsumerQueueLength"); var consLen = host.Configuration.GetRequiredSection("RecordQueue").GetValue<int>("ConsumerQueueLength");
var maxCharCount = host.Configuration.GetRequiredSection("RecordQueue").GetValue<long>("MaxByteCount") / 2; var maxCharCount = host.Configuration.GetRequiredSection("RecordQueue").GetValue<long>("MaxByteCount") / 2;
host.Services.AddKeyedSingleton<DataRecordQueue>(ConstVar.Producer, new DataRecordQueue(prodLen, maxCharCount)); host.Services.AddKeyedSingleton<DataRecordQueue>(ConstVar.Producer, new DataRecordQueue(prodLen, maxCharCount));
host.Services.AddRecordQueuePool(dbGroup.Keys host.Services.AddRecordQueuePool(tenantDbOptions.DbGroup.Keys
.Select(key => (key: key, queue: new DataRecordQueue(consLen, maxCharCount))).ToArray()); .Select(key => (key: key, queue: new DataRecordQueue(consLen, maxCharCount))).ToArray());
// host.Services.AddSingleton<ITaskMonitorLogger, CacheTaskMonitorLogger>(); // host.Services.AddSingleton<ITaskMonitorLogger, CacheTaskMonitorLogger>();
host.Services.AddSingleton<ITaskMonitorLogger, LoggerTaskMonitorLogger>(); host.Services.AddSingleton<ITaskMonitorLogger, LoggerTaskMonitorLogger>();