CZY/MES-ETL
1
0
Fork 0
Code Pull Requests Releases Activity

改进模拟数据生成器，代码质量优化

Browse Source
This commit is contained in:
陈梓阳
2024-12-25 15:09:16 +08:00
parent 4986c60416
commit 77a3909160
10 changed files with 102 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
Mesdb.DataGenerator/MockInputService.cs
View File

@@ -1,6 +1,8 @@
using System.Runtime;
using MesETL.App.Const;
using MesETL.App.HostedServices.Abstractions;
using MesETL.App.Services;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@@ -13,14 +15,16 @@ public class MockInputService : IInputService
private readonly ProcessContext _context;
private readonly IOptions<MockInputOptions> _options;
private readonly ILogger _logger;
private readonly long _memoryThreshold;
public MockInputService([FromKeyedServices(ConstVar.Producer)]DataRecordQueue producerQueue, ProcessContext context, IOptions<MockInputOptions> options,
ILogger<MockInputService> logger)
ILogger<MockInputService> logger, IConfiguration configuration)
{
_producerQueue = producerQueue;
_context = context;
_options = options;
_logger = logger;
_memoryThreshold = (long)(configuration.GetValue<double>("MemoryThreshold", 8) * 1024 * 1024 * 1024);
}
public async Task ExecuteAsync(CancellationToken cancellationToken)
@@ -32,6 +36,13 @@ public class MockInputService : IInputService
for (int i = 0; i < options.Amount; i++)
{
if (GC.GetTotalMemory(false) > _memoryThreshold)
{
_logger.LogWarning("内存使用率过高，暂缓输入");
GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
GC.Collect();
await Task.Delay(3000, cancellationToken);
}
var ctx = new TableMockContext()
{
Index = i,

2
Mesdb.DataGenerator/OrderModuleExtraHelper.cs
View File

File diff suppressed because one or more lines are too long

62
Mesdb.DataGenerator/Program.cs
View File

@@ -27,21 +27,23 @@ async Task RunProgram()
{
ThreadPool.SetMaxThreads(200, 200);
var host = Host.CreateApplicationBuilder(args);
var dbGroup = new Dictionary<string, int>()
{
{ "mesdb_1", 5000 },
{ "mesdb_2", 10000 },
{ "mesdb_3", 15000 },
{ "mesdb_4", 20000 },
{ "mesdb_5", 2147483647 },
};
var inputOptions = host.Configuration.GetRequiredSection("Input").Get<DataInputOptions>()
?? throw new ApplicationException("缺少Input配置");
var transformOptions = host.Configuration.GetRequiredSection("Transform").Get<DataTransformOptions>()
?? throw new ApplicationException("缺少Transform配置");
var outputOptions = host.Configuration.GetRequiredSection("Output").Get<DatabaseOutputOptions>()
?? throw new ApplicationException("缺少Output配置");
var tenantDbSection = host.Configuration.GetRequiredSection("TenantDb");
var tenantDbOptions = new TenantDbOptions()
{
TenantKey = "CompanyID",
DbGroup = dbGroup,
UseDbGroup = "Prod",
TenantKey = tenantDbSection.GetValue<string>(nameof(TenantDbOptions.TenantKey)) ??
throw new ApplicationException("分库配置缺少分库键TenantKey"),
UseDbGroup = tenantDbSection.GetValue<string>(nameof(TenantDbOptions.UseDbGroup)) ??
throw new ApplicationException("分库配置缺少使用分库组UseDbGroup")
};
host.Services.Configure<TenantDbOptions>(options =>
{
@@ -49,7 +51,10 @@ async Task RunProgram()
options.DbGroup = tenantDbOptions.DbGroup;
options.UseDbGroup = tenantDbOptions.UseDbGroup;
});
tenantDbOptions.DbGroup = tenantDbSection.GetRequiredSection($"DbGroups:{tenantDbOptions.UseDbGroup}")
.Get<Dictionary<string, int>>()
?? throw new ApplicationException($"分库配置无法解析分库组{tenantDbOptions.UseDbGroup}，请检查配置");
host.Services.Configure<MockInputOptions>(options =>
{
const float Multiplexer = 0.01F;
@@ -206,6 +211,11 @@ async Task RunProgram()
host.Services.Configure<DataTransformOptions>(options =>
{
options.StrictMode = transformOptions.StrictMode;
options.EnableFilter = transformOptions.EnableFilter;
options.EnableReplacer = transformOptions.EnableReplacer;
options.EnableReBuilder = transformOptions.EnableReBuilder;
options.DatabaseFilter = record =>
{
var companyId = int.Parse(record[tenantDbOptions.TenantKey]); // 每个实体都应存在CompanyID，否则异常
@@ -215,11 +225,16 @@ async Task RunProgram()
host.Services.Configure<DatabaseOutputOptions>(options =>
{
options.ConnectionString = "Server=192.168.1.246;Port=3333;UserId=root;Password=123456;";
options.FlushCount = 10000;
options.MaxAllowedPacket = 67108864;
options.MaxDatabaseOutputTask = 4;
options.ConnectionString = outputOptions.ConnectionString;
options.FlushCount = outputOptions.FlushCount;
options.MaxAllowedPacket = outputOptions.MaxAllowedPacket / 2;
options.MaxDatabaseOutputTask = outputOptions.MaxDatabaseOutputTask;
options.TreatJsonAsHex = outputOptions.TreatJsonAsHex;
options.NoOutput = outputOptions.NoOutput;
options.ForUpdate = outputOptions.ForUpdate;
// 配置列的类型以便于在输出时区分二进制内容
// Prod server
options.ColumnTypeConfig = new Dictionary<string, ColumnType>
{
{ "machine.Settings", ColumnType.Text },
@@ -253,14 +268,15 @@ async Task RunProgram()
host.Services.AddLogging(builder =>
{
builder.ClearProviders();
builder.AddSerilog(new LoggerConfiguration()
.MinimumLevel.Debug()
var logger = new LoggerConfiguration()
.MinimumLevel.Verbose()
.WriteTo.Console()
.WriteTo.File(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"./Log/Error/{ErrorRecorder.UID}.log"),
restrictedToMinimumLevel: LogEventLevel.Error)
// .WriteTo.File("./Log/Info/{ErrorRecorder.UID}.log", restrictedToMinimumLevel:LogEventLevel.Information) //性能考虑暂不使用
.CreateLogger()
);
.CreateLogger();
builder.AddSerilog(logger);
Log.Logger = logger;
});
host.Services.AddDataSourceFactory();
@@ -271,7 +287,7 @@ async Task RunProgram()
var consLen = host.Configuration.GetRequiredSection("RecordQueue").GetValue<int>("ConsumerQueueLength");
var maxCharCount = host.Configuration.GetRequiredSection("RecordQueue").GetValue<long>("MaxByteCount") / 2;
host.Services.AddKeyedSingleton<DataRecordQueue>(ConstVar.Producer, new DataRecordQueue(prodLen, maxCharCount));
host.Services.AddRecordQueuePool(dbGroup.Keys
host.Services.AddRecordQueuePool(tenantDbOptions.DbGroup.Keys
.Select(key => (key: key, queue: new DataRecordQueue(consLen, maxCharCount))).ToArray());
// host.Services.AddSingleton<ITaskMonitorLogger, CacheTaskMonitorLogger>();
host.Services.AddSingleton<ITaskMonitorLogger, LoggerTaskMonitorLogger>();