MES-ETL/ConsoleApp2/Program.cs
2024-02-01 10:04:00 +08:00

566 lines
27 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#define USE_TEST_DB // 测试库的结构与生产库不一样如果使用测试库运行则加上USE_TEST_DB预处理器指令
using ConsoleApp2;
using ConsoleApp2.Cache;
using ConsoleApp2.Const;
using ConsoleApp2.Helpers;
using ConsoleApp2.HostedServices;
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
using ConsoleApp2.Services.ErrorRecorder;
using ConsoleApp2.Services.ETL;
using ConsoleApp2.Services.Loggers;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using MySqlConnector;
using Serilog;
using Serilog.Events;
await RunProgram();
return;
async Task RunProgram()
{
ThreadPool.SetMaxThreads(200, 200);
var host = Host.CreateApplicationBuilder(args);
host.Configuration.AddJsonFile(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "appsettings.json"), false, false);
host.Configuration.AddCommandLine(args, new Dictionary<string, string>
{
{"-d", "Input:InputDir"},
{"--InputDir", "Input:InputDir"},
{"-s", "Output:ConnectionString"},
{"--ConnectionString", "Output:ConnectionString"},
{"-r", "RedisCache:Configuration"},
{"--Redis", "RedisCache:Configuration"},
{"-i", "RestoreIndex"},
{"--RestoreIndex", "RestoreIndex"}
});
var inputOptions = host.Configuration.GetRequiredSection("Input").Get<DataInputOptions>()
?? throw new ApplicationException("缺少Input配置");
var transformOptions = host.Configuration.GetRequiredSection("Transform").Get<DataTransformOptions>()
?? throw new ApplicationException("缺少Transform配置");
var outputOptions = host.Configuration.GetRequiredSection("Output").Get<DatabaseOutputOptions>()
?? throw new ApplicationException("缺少Output配置");
var redisSection = host.Configuration.GetRequiredSection("RedisCache");
var redisOptions = redisSection.Get<RedisCacheOptions>() ?? throw new ApplicationException("缺少RedisCache配置");
var tenantDbSection = host.Configuration.GetRequiredSection("TenantDb");
var tenantDbOptions = tenantDbSection.Get<TenantDbOptions>() ?? throw new ApplicationException("缺少TenantDb配置");
if(tenantDbOptions.TenantKey is null) throw new ApplicationException("分库配置中缺少分库键");
if(tenantDbOptions.DbList is null) throw new ApplicationException("分库配置中没有发现任何数据库");
host.Services.Configure<TenantDbOptions>(tenantDbSection);
host.Services.Configure<RedisCacheOptions>(redisSection);
var oldestTime = DateTime.ParseExact(inputOptions.CleanDate, "yyyyMM", System.Globalization.DateTimeFormatInfo.InvariantInfo);
var oldestTimeInt = int.Parse(inputOptions.CleanDate);
// 输入配置
host.Services.Configure<DataInputOptions>(options =>
{
options.InputDir = inputOptions.InputDir ?? throw new ApplicationException("未配置输入目录");
options.UseMock = inputOptions.UseMock;
options.TableMockConfig = inputOptions.TableMockConfig;
options.MockCountMultiplier = inputOptions.MockCountMultiplier;
// 配置文件输入方法
options.FileInputMetaBuilder = fileName =>
{
if (fileName.EndsWith(".dat.zst"))
{
var tableName = DumpDataHelper.GetTableNameFromCsvFileName(
Path.GetFileNameWithoutExtension(fileName)); // 去除.zst
string[]? headers;
try
{
// 查找同目录下同表的SQL文件
var sqlFile = Directory.GetFiles(options.InputDir)
.SingleOrDefault(f => f.Equals(fileName.Replace(".dat.zst",".sql.zst")));
if (sqlFile is null)
return null;
headers = DumpDataHelper.GetCsvHeadersFromSqlFile(
DumpDataHelper.DecompressZstAsStringAsync(File.OpenRead(sqlFile)).Result);
}
catch (InvalidOperationException e)
{
throw new ApplicationException($"目录下不止一个{tableName}表的SQL文件", e);
}
return new FileInputInfo
{
FileName = fileName,
TableName = tableName,
Headers = headers
};
}
return null;
};
options.TableOrder =
[
TableNames.Machine,
TableNames.Order,
TableNames.OrderBoxBlock, // 依赖Order.CompanyID
TableNames.OrderBlockPlan,
TableNames.OrderBlockPlanResult,// 依赖OrderBlockPlan.CompanyID / 删除
TableNames.OrderItem,
TableNames.OrderDataBlock,
TableNames.OrderDataGoods,
TableNames.OrderDataParts,
TableNames.OrderModule,
TableNames.OrderModuleExtra,
TableNames.OrderModuleItem,
TableNames.OrderPackage,
TableNames.OrderPatchDetail,
TableNames.OrderProcess,
TableNames.OrderProcessStep,
TableNames.OrderProcessStepItem,// 依赖OrderProcess.ShardKey / 删除
TableNames.OrderProcessSchedule,
TableNames.OrderScrapBoard,
TableNames.ProcessGroup,
TableNames.ProcessInfo,
TableNames.ProcessItemExp,
TableNames.ProcessScheduleCapacity,
TableNames.ProcessStepEfficiency,
TableNames.ReportTemplate,
TableNames.SimplePackage,
TableNames.SimplePlanOrder,
TableNames.SysConfig,
TableNames.WorkCalendar,
TableNames.WorkShift,
TableNames.WorkTime
];
// options.TableMockConfig = new Dictionary<string, TableMockConfig>
// {
// { TableNames.Machine, new TableMockConfig(true, 14655, ["ID"]) },
// { TableNames.Order, new TableMockConfig(true, 5019216, ["OrderNo"]) },
// { TableNames.OrderDataBlock, new TableMockConfig(true, 731800334, ["ID"]) },
// { TableNames.OrderDataGoods, new TableMockConfig(true, 25803671, ["ID"]) },
// { TableNames.OrderDataParts, new TableMockConfig(true, 468517543, ["ID"]) },
// { TableNames.OrderModule, new TableMockConfig(true, 103325385, ["ID"]) },
// { TableNames.OrderModuleExtra, new TableMockConfig(true, 54361321, ["ID"]) },
// { TableNames.OrderModuleItem, new TableMockConfig(true, 69173339, ["ID"]) },
// { TableNames.OrderPackage, new TableMockConfig(true, 16196195, ["ID"]) },
// { TableNames.OrderProcess, new TableMockConfig(true, 3892685, ["ID"]) },
// { TableNames.OrderProcessStep, new TableMockConfig(true, 8050349, ["ID"]) },
// { TableNames.OrderProcessStepItem, new TableMockConfig(true, 14538058, ["ID"]) },
// { TableNames.OrderScrapBoard, new TableMockConfig(true, 123998, ["ID"]) },
// { TableNames.ProcessGroup, new TableMockConfig(true, 1253, ["ID"]) },
// { TableNames.ProcessInfo, new TableMockConfig(true, 7839, ["ID"]) },
// { TableNames.ProcessItemExp, new TableMockConfig(true, 28, ["ID"]) },
// { TableNames.ProcessScheduleCapacity, new TableMockConfig(true, 39736, ["ID"]) },
// { TableNames.ProcessStepEfficiency, new TableMockConfig(true, 8, ["ID"]) },
// { TableNames.ReportTemplate, new TableMockConfig(true, 7337, ["ID"]) },
// { TableNames.SimplePackage, new TableMockConfig(true, 130436, ["ID"]) },
// { TableNames.SysConfig, new TableMockConfig(true, 2296, ["ID"]) },
// { TableNames.WorkCalendar, new TableMockConfig(true, 11, ["ID"]) },
// { TableNames.WorkShift, new TableMockConfig(true, 59, ["ID"]) },
// { TableNames.WorkTime, new TableMockConfig(true, 62, ["ID"]) }
// };
options.TableMockConfig = new Dictionary<string, TableMockConfig>
{
{ TableNames.Machine, new TableMockConfig(true, 14655, ["ID"]) },
{ TableNames.Order, new TableMockConfig(true, 50192, ["OrderNo"]) },
{ TableNames.OrderDataBlock, new TableMockConfig(true, 7318003, ["ID"]) },
{ TableNames.OrderDataGoods, new TableMockConfig(true, 258036, ["ID"]) },
{ TableNames.OrderDataParts, new TableMockConfig(true, 4685175, ["ID"]) },
{ TableNames.OrderItem, new TableMockConfig(true, 13298896, ["ID"])},
{ TableNames.OrderModule, new TableMockConfig(true, 1033253, ["ID"]) },
{ TableNames.OrderModuleExtra, new TableMockConfig(true, 543613, ["ID"]) },
{ TableNames.OrderModuleItem, new TableMockConfig(true, 691733, ["ID"]) },
{ TableNames.OrderPackage, new TableMockConfig(true, 161961, ["ID"]) },
{ TableNames.OrderProcess, new TableMockConfig(true, 38926, ["ID"]) },
{ TableNames.OrderProcessStep, new TableMockConfig(true, 80503, ["ID"]) },
{ TableNames.OrderProcessStepItem, new TableMockConfig(true, 145380, ["ID"]) },
{ TableNames.OrderScrapBoard, new TableMockConfig(true, 1239, ["ID"]) },
{ TableNames.ProcessGroup, new TableMockConfig(true, 125, ["ID"]) },
{ TableNames.ProcessInfo, new TableMockConfig(true, 783, ["ID"]) },
{ TableNames.ProcessItemExp, new TableMockConfig(true, 28, ["ID"]) },
{ TableNames.ProcessScheduleCapacity, new TableMockConfig(true, 39736, ["ID"]) },
{ TableNames.ProcessStepEfficiency, new TableMockConfig(true, 8, ["ID"]) },
{ TableNames.ReportTemplate, new TableMockConfig(true, 7337, ["ID"]) },
{ TableNames.SimplePackage, new TableMockConfig(true, 130436, ["ID"]) },
{ TableNames.SysConfig, new TableMockConfig(true, 2296, ["Key"]) },
{ TableNames.WorkCalendar, new TableMockConfig(true, 11, ["ID"]) },
{ TableNames.WorkShift, new TableMockConfig(true, 59, ["ID"]) },
{ TableNames.WorkTime, new TableMockConfig(true, 62, ["ID"]) }
};
});
host.Services.Configure<DataTransformOptions>(options =>
{
// 数据缓存键格式为[TableName]-[ColumnName@ColumnValue]-[CacheColumnName]
static string BuildCacheKey(string tableName, string columnName, string columnValue, string cacheColumnName)
=> $"{tableName}-{columnName}@{columnValue}-{cacheColumnName}";
static string CalculateShardKeyByOrderNo(ReadOnlySpan<char> orderNo)
=> $"{orderNo[2..6]}0";
options.StrictMode = transformOptions.StrictMode;
options.EnableFilter = transformOptions.EnableFilter;
options.EnableReplacer = transformOptions.EnableReplacer;
options.EnableReBuilder = transformOptions.EnableReBuilder;
// order_block_plan_item和order_package_item表不导入根据order_item数据直接重建
// 数据清理
options.RecordFilter = async context =>
{
var record = context.Record;
var cache = context.Cacher;
switch (record.TableName)
{
// OrderBoxBlock删除对应Order.OrderNo不存在的对象
case TableNames.OrderBoxBlock:
{
if (!await cache.ExistsAsync(CacheKeys.Order_OrderNo_CompanyID(record["OrderNo"])))
return false;
break;
}
// OrderBlockPlan删除CreateTime < 202301的
case TableNames.OrderBlockPlan:
{
var time = DateTime.Parse(record["CreateTime"].Trim('"'));
if (time < oldestTime)
return false;
break;
}
// OrderBlockPlanResult删除对应order_block_plan.ID不存在的对象
case TableNames.OrderBlockPlanResult:
{
if (!await cache.ExistsAsync(CacheKeys.OrderBlockPlan_ID_CompanyID(record["ID"])))
return false;
break;
}
// OrderModule删除OrderNo < 202301的
case TableNames.OrderModule:
{
var orderNo = record["OrderNo"];
if(int.Parse(orderNo.AsSpan(0, 6).ToString()) < oldestTimeInt)
return false;
break;
}
// OrderProcess删除OrderNo < 202301的
case TableNames.OrderProcess:
{
var orderNo = record["OrderNo"];
if(int.Parse(orderNo.AsSpan(0, 6).ToString()) < oldestTimeInt)
return false;
break;
}
// OrderProcessStep删除OrderNo < 202301的
case TableNames.OrderProcessStep:
{
var orderNo = record["OrderNo"];
if(int.Parse(orderNo.AsSpan(0, 6).ToString()) < oldestTimeInt)
return false;
break;
}
// OrderProcessStepStep删除对应OrderProcess.ID不存在的对象
case TableNames.OrderProcessStepItem:
{
if (!await cache.ExistsAsync(CacheKeys.OrderProcess_ID_ShardKey(record["OrderProcessID"])))
return false;
break;
}
// SimplePackage删除OrderNo < 202301的
case TableNames.SimplePackage:
{
var orderNo = record["OrderNo"];
if(int.Parse(orderNo.AsSpan(0, 6).ToString()) < oldestTimeInt)
return false;
break;
}
// SimplePlanOrder删除CreateTime < 202301的
case TableNames.SimplePlanOrder:
{
var time = DateTime.Parse(record["CreateTime"].Trim('"'));
if (time < oldestTime)
return false;
break;
}
}
return true;
};
// 数据替换
options.RecordModify = async context =>
{
var record = context.Record;
var cache = context.Cacher;
switch (record.TableName)
{
#if USE_TEST_DB
// Order表移除IsBatch列
case TableNames.Order:
record.RemoveField("IsBatch");
break;
#endif
//OrderBlockPlan将OrderNo长度<2的置空
case TableNames.OrderBlockPlan:
if (record["OrderNos"].Length < 2)
record["OrderNos"] = "";
break;
// OrderBlockPlanResult表添加CompanyID列
case TableNames.OrderBlockPlanResult:
record.AddField("CompanyID",
// 获取OrderBlockPlan.ID -> CompanyID
ThrowIfNoCached(await cache.GetStringAsync(CacheKeys.OrderBlockPlan_ID_CompanyID(record["ID"])),
TableNames.OrderBlockPlanResult, TableNames.OrderBlockPlan, "ID", "脏数据未处理"));
break;
// OrderBoxBlock添加CompanyID列
case TableNames.OrderBoxBlock:
record.AddField("CompanyID",
// 获取Order.OrderNo -> CompanyID
ThrowIfNoCached(await cache.GetStringAsync(CacheKeys.Order_OrderNo_CompanyID(record["OrderNo"])),
TableNames.OrderBoxBlock, TableNames.Order, "OrderNo", "脏数据未处理"));
break;
// OrderModule添加ShardKey列移除ViewFileName列
case TableNames.OrderModule:
record.AddField("ShardKey", CalculateShardKeyByOrderNo(record["OrderNo"]));
record.RemoveField("ViewFileName");
break;
// OrderProcess添加ShardKey列NextStepID的空值转换为0
case TableNames.OrderProcess:
record.AddField("ShardKey", CalculateShardKeyByOrderNo(record["OrderNo"]));
#if USE_TEST_DB
if(record["NextStepID"] == "\\N")
record["NextStepID"] = "0";
#endif
break;
// OrderProcessStepOrderProcessStepItem添加ShardKey列
case TableNames.OrderProcessStep:
record.AddField("ShardKey", CalculateShardKeyByOrderNo(record["OrderNo"]));
break;
case TableNames.OrderProcessStepItem:
record.AddField("ShardKey",
// 获取OrderProcess.ID -> ShardKey
ThrowIfNoCached(await cache.GetStringAsync(CacheKeys.OrderProcess_ID_ShardKey(record["OrderProcessID"])),
TableNames.OrderProcessStepItem, TableNames.OrderProcessStep, "OrderProcessID", "脏数据未处理"));
break;
case TableNames.SimplePlanOrder:
#if USE_TEST_DB
record.RemoveField("ProcessState");
#endif
record.AddField("Deleted", "0");
break;
}
return record;
string ThrowIfNoCached(string? cached, string tableName, string cachedTableName, string cachedColumn, string appendMessage = "")
{
if (cached is null)
throw new InvalidDataException(
$"{tableName}数据异常,在缓存中未找到对应{cachedTableName}.{cachedColumn}\t{appendMessage}");
return cached;
}
};
// 数据缓存
options.RecordCache = async context =>
{
var record = context.Record;
var cache = context.Cacher;
switch (record.TableName)
{
// 缓存Order.OrderNo -> CompanyID
case TableNames.Order:
CacheKeys.Order_OrderNo_CompanyID = orderNo =>
BuildCacheKey(TableNames.Order, "OrderNo", orderNo, "CompanyID");
await cache.SetStringAsync(
CacheKeys.Order_OrderNo_CompanyID(record["OrderNo"]),
record["CompanyID"]);
break;
// 缓存OrderBlockPlan.ID -> CompanyID
case TableNames.OrderBlockPlan:
CacheKeys.OrderBlockPlan_ID_CompanyID = id =>
BuildCacheKey(TableNames.OrderBlockPlan, "ID", id, "CompanyID");
await cache.SetStringAsync(
CacheKeys.OrderBlockPlan_ID_CompanyID(record["ID"]),
record["CompanyID"]);
break;
// 缓存OrderProcess.ID -> ShardKey
case TableNames.OrderProcess:
CacheKeys.OrderProcess_ID_ShardKey = id =>
BuildCacheKey(TableNames.OrderProcess, "ID", id, "ShardKey");
await cache.SetStringAsync(
CacheKeys.OrderProcess_ID_ShardKey(record["ID"]),
record["ShardKey"]);
break;
}
};
// 数据库过滤
options.DatabaseFilter = record =>
{
var companyId = int.Parse(record[tenantDbOptions.TenantKey]); // 每个实体都应存在CompanyID否则异常
return tenantDbOptions.GetDbNameByTenantKeyValue(companyId);
};
// 数据重建
options.RecordReBuild = context =>
{
var record = context.Record;
var resultList = new List<DataRecord>();
// 分流OrderItem表
if (record.TableName == TableNames.OrderItem)
{
record.TryGetField("ID", out var itemId);
record.TryGetField("ShardKey", out var shardKey);
record.TryGetField("PlanID", out var planId);
record.TryGetField("PackageID", out var packageId);
record.TryGetField("CompanyID", out var companyId);
if(!int.TryParse(planId, out var pid))
throw new ApplicationException($"数据发生异常OrderItem.PlanID值: {planId}");
if (pid > 0)
{
resultList.Add(new DataRecord(new[] { itemId, shardKey, planId, companyId },
TableNames.OrderBlockPlanItem,
["ItemID", "ShardKey", "PlanID", "CompanyID"]
));
}
if(!int.TryParse(packageId, out var pkid))
throw new ApplicationException($"数据发生异常OrderItem.PackageID值: {packageId}");
if(pkid > 0)
{
resultList.Add(new DataRecord(new[] { itemId, shardKey, packageId, companyId },
TableNames.OrderPackageItem,
[ "ItemID", "ShardKey", "PackageID", "CompanyID" ]
));
}
}
return resultList;
};
});
host.Services.Configure<DatabaseOutputOptions>(options =>
{
options.ConnectionString = new MySqlConnectionStringBuilder(outputOptions.ConnectionString ?? throw new ApplicationException("未配置数据库连接字符串"))
{
CharacterSet = "utf8mb4",
AllowUserVariables = true,
IgnoreCommandTransaction = true,
TreatTinyAsBoolean = false,
ConnectionTimeout = 60,
DefaultCommandTimeout = 0,
SslMode = MySqlSslMode.None,
}.ConnectionString;
options.FlushCount = outputOptions.FlushCount;
options.MaxAllowedPacket = outputOptions.MaxAllowedPacket;
options.MaxDatabaseOutputTask = outputOptions.MaxDatabaseOutputTask;
options.TreatJsonAsHex = outputOptions.TreatJsonAsHex;
#if USE_TEST_DB
// Test Server
options.ColumnTypeConfig = new Dictionary<string, ColumnType>
{
{ "simple_plan_order.PlaceData", ColumnType.Blob },
{ "order_block_plan_result.PlaceData", ColumnType.Blob },
{ "order_box_block.Data", ColumnType.Blob },
{ "order_data_goods.ExtraProp", ColumnType.Json },
{ "order_module_extra.JsonStr", ColumnType.Text },
{ "process_info.Users", ColumnType.Text },
{ "order_process_schdule.CustomOrderNo", ColumnType.Text },
{ "order_process_schdule.OrderProcessStepName", ColumnType.Text },
{ "order_process_schdule.AreaName", ColumnType.Text },
{ "order_process_schdule.ConsigneeAddress", ColumnType.Text },
{ "order_process_schdule.ConsigneePhone", ColumnType.Text },
{ "report_source.Sql", ColumnType.Text },
{ "report_source.KeyValue", ColumnType.Text },
{ "report_source.Setting", ColumnType.Text },
{ "order_data_block.RemarkJson", ColumnType.Text },
{ "order_patch_detail.BlockDetail", ColumnType.Json },
{ "order_scrap_board.OutLineJson", ColumnType.Text },
{ "simple_package.Items", ColumnType.Json },
{ "order_batch_pack_config.Setting", ColumnType.Text },
{ "machine.Settings", ColumnType.Text },
{ "sys_config.Value", ColumnType.Text },
{ "sys_config.JsonStr", ColumnType.Text },
{ "process_item_exp.ItemJson", ColumnType.Text },
{ "report_template.Template", ColumnType.Text },
{ "report_template.SourceConfig", ColumnType.Text },
{ "order_block_plan.OrderNos", ColumnType.Json },
{ "order_block_plan.BlockInfo", ColumnType.Text },
};
#else
// 配置列类型
// Prod server
options.ColumnTypeConfig = new Dictionary<string, ColumnType>
{
{ "simple_plan_order.PlaceData", ColumnType.Blob },
{ "order_block_plan_result.PlaceData", ColumnType.Blob },
{ "order_box_block.Data", ColumnType.Blob },
{ "order_data_goods.ExtraProp", ColumnType.Text },
{ "order_module_extra.JsonStr", ColumnType.Text },
{ "process_info.Users", ColumnType.Text },
{ "order_process_schdule.CustomOrderNo", ColumnType.Text },
{ "order_process_schdule.OrderProcessStepName", ColumnType.Text },
{ "order_process_schdule.AreaName", ColumnType.Text },
{ "order_process_schdule.ConsigneeAddress", ColumnType.Text },
{ "order_process_schdule.ConsigneePhone", ColumnType.Text },
{ "report_source.Sql", ColumnType.Text },
{ "report_source.KeyValue", ColumnType.Text },
{ "report_source.Setting", ColumnType.Text },
{ "order_data_block.RemarkJson", ColumnType.Text },
{ "order_patch_detail.BlockDetail", ColumnType.Text },
{ "order_scrap_board.OutLineJson", ColumnType.Text },
{ "simple_package.Items", ColumnType.Text },
{ "order_batch_pack_config.Setting", ColumnType.Text },
{ "machine.Settings", ColumnType.Text },
{ "sys_config.Value", ColumnType.Text },
{ "sys_config.JsonStr", ColumnType.Text },
{ "process_item_exp.ItemJson", ColumnType.Text },
{ "report_template.Template", ColumnType.Text },
{ "report_template.SourceConfig", ColumnType.Text },
{ "order_block_plan.OrderNos", ColumnType.Json },
{ "order_block_plan.BlockInfo", ColumnType.Text },
};
#endif
});
host.Services.AddLogging(builder =>
{
builder.ClearProviders();
builder.AddSerilog(new LoggerConfiguration()
.WriteTo.Console()
.WriteTo.File($"./Log/Error/{ErrorRecorder.UID}.log", restrictedToMinimumLevel:LogEventLevel.Error)
// .WriteTo.File("./Log/Info/{ErrorRecorder.UID}.log", restrictedToMinimumLevel:LogEventLevel.Information) //性能考虑暂不使用
.CreateLogger()
);
});
host.Services.AddDataSourceFactory();
host.Services.AddErrorRecorderFactory();
host.Services.AddSingleton<ProcessContext>();
host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Produce);
host.Services.AddRecordQueuePool(tenantDbOptions.DbList.Keys.Select(key => (key:key, queue:new DataRecordQueue(500_000))).ToArray());
host.Services.AddSingleton<ITaskMonitorLogger, CacheTaskMonitorLogger>();
host.Services.AddSingleton<ITaskMonitorLogger, LoggerTaskMonitorLogger>();
host.Services.AddHostedService<MainHostedService>();
host.Services.AddHostedService<TaskMonitorService>();
host.Services.AddSingleton<IInputService, FileInputService>();
host.Services.AddSingleton<ITransformService, TransformService>();
host.Services.AddSingleton<IOutputService, OutputService>();
host.Services.AddRedisCache(redisOptions);
var app = host.Build();
await app.RunAsync();
}