MES-ETL/MesETL.App/Program.cs

// #define USE_TEST_DB // 测试库的结构与生产库不一样，如果使用测试库运行，则加上USE_TEST_DB预处理器指令

using MesETL.App;
using MesETL.App.Services;
using MesETL.App.Services.ETL;
using MesETL.App.Cache;
using MesETL.App.Const;
using MesETL.App.HostedServices;
using MesETL.App.HostedServices.Abstractions;
using MesETL.App.Options;
using MesETL.App.Services.ErrorRecorder;
using MesETL.App.Services.Loggers;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Serilog;
using Serilog.Events;
using DumpDataHelper = MesETL.App.Helpers.DumpDataHelper;

await RunProgram();
return;

async Task RunProgram()
{
    ThreadPool.SetMaxThreads(200, 200);
    var host = Host.CreateApplicationBuilder(args);
    host.Configuration.AddJsonFile(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "appsettings.json"), false, false);
    host.Configuration.AddCommandLine(args, new Dictionary<string, string>
    {
        { "-d", "Input:InputDir" },
        { "--InputDir", "Input:InputDir" },
        { "-s", "Output:ConnectionString" },
        { "--ConnectionString", "Output:ConnectionString" },
        { "-r", "RedisCache:Configuration" },
        { "--Redis", "RedisCache:Configuration" },
        { "-g", "TenantDb:UseDbGroup" },
        { "--UseDbGroup", "TenantDb:UseDbGroup" },
        { "-c", "Command" },
        { "--Command", "Command" }
    }); 

    var inputOptions = host.Configuration.GetRequiredSection("Input").Get<DataInputOptions>() 
                       ?? throw new ApplicationException("缺少Input配置");

    var transformOptions = host.Configuration.GetRequiredSection("Transform").Get<DataTransformOptions>()
                        ?? throw new ApplicationException("缺少Transform配置");

    var outputOptions = host.Configuration.GetRequiredSection("Output").Get<DatabaseOutputOptions>() 
                        ?? throw new ApplicationException("缺少Output配置");

    var redisSection = host.Configuration.GetRequiredSection("RedisCache");
    var redisOptions = redisSection.Get<RedisCacheOptions>() ?? throw new ApplicationException("缺少RedisCache配置");

    var tenantDbSection = host.Configuration.GetRequiredSection("TenantDb");
    var tenantDbOptions = new TenantDbOptions()
    {
        TenantKey = tenantDbSection.GetValue<string>(nameof(TenantDbOptions.TenantKey)) ?? throw new ApplicationException("分库配置缺少分库键TenantKey"),
        UseDbGroup = tenantDbSection.GetValue<string>(nameof(TenantDbOptions.UseDbGroup)) ?? throw new ApplicationException("分库配置缺少使用分库组UseDbGroup")
    };
    tenantDbOptions.DbGroup = tenantDbSection.GetRequiredSection($"DbGroups:{tenantDbOptions.UseDbGroup}").Get<Dictionary<string,int>>()
        ?? throw new ApplicationException($"分库配置无法解析分库组{tenantDbOptions.UseDbGroup}，请检查配置");
    
    host.Services.Configure<TenantDbOptions>(options =>
    {
        options.TenantKey = tenantDbOptions.TenantKey;
        options.UseDbGroup = tenantDbOptions.UseDbGroup;
        options.DbGroup = tenantDbOptions.DbGroup;
    });
    host.Services.Configure<RedisCacheOptions>(redisSection);
    
    var oldestTime = DateTime.ParseExact(transformOptions.CleanDate, "yyyyMM", System.Globalization.DateTimeFormatInfo.InvariantInfo);
    var oldestTimeInt = int.Parse(transformOptions.CleanDate);
    
    // 输入配置
    host.Services.Configure<DataInputOptions>(options =>
    {
        options.InputDir = inputOptions.InputDir ?? throw new ApplicationException("未配置输入目录");
        options.UseMock = inputOptions.UseMock;
        options.TableMockConfig = inputOptions.TableMockConfig;
        options.MockCountMultiplier = inputOptions.MockCountMultiplier;
        
        // 配置文件输入方法
        options.FileInputMetaBuilder = fileName =>
        {
            if (fileName.EndsWith(".dat.zst"))
            {
                var tableName = DumpDataHelper.GetTableNameFromCsvFileName(
                    Path.GetFileNameWithoutExtension(fileName)); // 去除.zst
                string[]? headers;
                try
                {
                    // 查找同目录下同表的SQL文件
                    var sqlFile = Directory.GetFiles(options.InputDir)
                        .SingleOrDefault(f => f.Equals(fileName.Replace(".dat.zst",".sql.zst")));
                    if (sqlFile is null)
                        return null;
                    headers = DumpDataHelper.GetCsvHeadersFromSqlFile(
                        DumpDataHelper.DecompressZstAsStringAsync(File.OpenRead(sqlFile)).Result);
                }
                catch (InvalidOperationException e)
                {
                    throw new ApplicationException($"目录下不止一个{tableName}表的SQL文件", e);
                }
                
                return new FileInputInfo
                {
                    FileName = fileName,
                    TableName = tableName,
                    Headers = headers
                };
            }
            return null;
        };

        options.TableOrder =
        [
            TableNames.Machine,
            
            TableNames.Order,
            TableNames.OrderBoxBlock,       // 依赖Order.CompanyID
            
            TableNames.OrderBlockPlan,      
            TableNames.OrderBlockPlanResult,// 依赖OrderBlockPlan.CompanyID / 删除
            
            TableNames.OrderItem,
            TableNames.OrderDataBlock,
            TableNames.OrderDataGoods,
            TableNames.OrderDataParts,
            TableNames.OrderModule,
            TableNames.OrderModuleExtra,
            TableNames.OrderModuleItem,
            TableNames.OrderPackage,
            TableNames.OrderPatchDetail,
            
            TableNames.OrderProcess,
            TableNames.OrderProcessStep,
            TableNames.OrderProcessStepItem,// 依赖OrderProcess.ShardKey / 删除
            
            TableNames.OrderProcessSchedule,
            TableNames.OrderScrapBoard,
            TableNames.ProcessGroup,
            TableNames.ProcessInfo,
            TableNames.ProcessItemExp,
            TableNames.ProcessScheduleCapacity,
            TableNames.ProcessStepEfficiency,
            TableNames.ReportTemplate,
            TableNames.SimplePackage,
            TableNames.SimplePlanOrder,
            TableNames.SysConfig,
            TableNames.WorkCalendar,
            TableNames.WorkShift,
            TableNames.WorkTime
        ];

        // options.TableMockConfig = new Dictionary<string, TableMockConfig>
        // {
        //     { TableNames.Machine, new TableMockConfig(true, 14655, ["ID"]) },
        //     { TableNames.Order, new TableMockConfig(true, 5019216, ["OrderNo"]) },
        //     { TableNames.OrderDataBlock, new TableMockConfig(true, 731800334, ["ID"]) },
        //     { TableNames.OrderDataGoods, new TableMockConfig(true, 25803671, ["ID"]) },
        //     { TableNames.OrderDataParts, new TableMockConfig(true, 468517543, ["ID"]) },
        //     { TableNames.OrderModule, new TableMockConfig(true, 103325385, ["ID"]) },
        //     { TableNames.OrderModuleExtra, new TableMockConfig(true, 54361321, ["ID"]) },
        //     { TableNames.OrderModuleItem, new TableMockConfig(true, 69173339, ["ID"]) },
        //     { TableNames.OrderPackage, new TableMockConfig(true, 16196195, ["ID"]) },
        //     { TableNames.OrderProcess, new TableMockConfig(true, 3892685, ["ID"]) },
        //     { TableNames.OrderProcessStep, new TableMockConfig(true, 8050349, ["ID"]) },
        //     { TableNames.OrderProcessStepItem, new TableMockConfig(true, 14538058, ["ID"]) },
        //     { TableNames.OrderScrapBoard, new TableMockConfig(true, 123998, ["ID"]) },
        //     { TableNames.ProcessGroup, new TableMockConfig(true, 1253, ["ID"]) },
        //     { TableNames.ProcessInfo, new TableMockConfig(true, 7839, ["ID"]) },
        //     { TableNames.ProcessItemExp, new TableMockConfig(true, 28, ["ID"]) },
        //     { TableNames.ProcessScheduleCapacity, new TableMockConfig(true, 39736, ["ID"]) },
        //     { TableNames.ProcessStepEfficiency, new TableMockConfig(true, 8, ["ID"]) },
        //     { TableNames.ReportTemplate, new TableMockConfig(true, 7337, ["ID"]) },
        //     { TableNames.SimplePackage, new TableMockConfig(true, 130436, ["ID"]) },
        //     { TableNames.SysConfig, new TableMockConfig(true, 2296, ["ID"]) },
        //     { TableNames.WorkCalendar, new TableMockConfig(true, 11, ["ID"]) },
        //     { TableNames.WorkShift, new TableMockConfig(true, 59, ["ID"]) },
        //     { TableNames.WorkTime, new TableMockConfig(true, 62, ["ID"]) }
        // };
        options.TableMockConfig = new Dictionary<string, TableMockConfig>
        {
            { TableNames.Machine, new TableMockConfig(true, 14655, ["ID"]) },
            { TableNames.Order, new TableMockConfig(true, 50192, ["OrderNo"]) },
            { TableNames.OrderDataBlock, new TableMockConfig(true, 7318003, ["ID"]) },
            { TableNames.OrderDataGoods, new TableMockConfig(true, 258036, ["ID"]) },
            { TableNames.OrderDataParts, new TableMockConfig(true, 4685175, ["ID"]) },
            { TableNames.OrderItem, new TableMockConfig(true, 13298896, ["ID"])},
            { TableNames.OrderModule, new TableMockConfig(true, 1033253, ["ID"]) },
            { TableNames.OrderModuleExtra, new TableMockConfig(true, 543613, ["ID"]) },
            { TableNames.OrderModuleItem, new TableMockConfig(true, 691733, ["ID"]) },
            { TableNames.OrderPackage, new TableMockConfig(true, 161961, ["ID"]) },
            { TableNames.OrderProcess, new TableMockConfig(true, 38926, ["ID"]) },
            { TableNames.OrderProcessStep, new TableMockConfig(true, 80503, ["ID"]) },
            { TableNames.OrderProcessStepItem, new TableMockConfig(true, 145380, ["ID"]) },
            { TableNames.OrderScrapBoard, new TableMockConfig(true, 1239, ["ID"]) },
            { TableNames.ProcessGroup, new TableMockConfig(true, 125, ["ID"]) },
            { TableNames.ProcessInfo, new TableMockConfig(true, 783, ["ID"]) },
            { TableNames.ProcessItemExp, new TableMockConfig(true, 28, ["ID"]) },
            { TableNames.ProcessScheduleCapacity, new TableMockConfig(true, 39736, ["ID"]) },
            { TableNames.ProcessStepEfficiency, new TableMockConfig(true, 8, ["ID"]) },
            { TableNames.ReportTemplate, new TableMockConfig(true, 7337, ["ID"]) },
            { TableNames.SimplePackage, new TableMockConfig(true, 130436, ["ID"]) },
            { TableNames.SysConfig, new TableMockConfig(true, 2296, ["Key"]) },
            { TableNames.WorkCalendar, new TableMockConfig(true, 11, ["ID"]) },
            { TableNames.WorkShift, new TableMockConfig(true, 59, ["ID"]) },
            { TableNames.WorkTime, new TableMockConfig(true, 62, ["ID"]) }
        };
    });

    host.Services.Configure<DataTransformOptions>(options =>
    {
        // 数据缓存键格式为[TableName]-[ColumnName@ColumnValue]-[CacheColumnName]
        static string BuildCacheKey(string tableName, string columnName, string columnValue, string cacheColumnName) 
            => $"{tableName}-{columnName}@{columnValue}-{cacheColumnName}";

        static string CalculateShardKeyByOrderNo(ReadOnlySpan<char> orderNo)
            => $"{orderNo[2..6]}0";
        
        options.StrictMode = transformOptions.StrictMode;
        options.EnableFilter = transformOptions.EnableFilter;
        options.EnableReplacer = transformOptions.EnableReplacer;
        options.EnableReBuilder = transformOptions.EnableReBuilder;
        
        // order_block_plan_item和order_package_item表不导入，根据order_item数据直接重建
        
        // 数据清理
        options.RecordFilter = async context =>
        {
            var record = context.Record;
            var cache = context.Cacher;
            switch (record.TableName)
            {
                // OrderBoxBlock删除对应Order.OrderNo不存在的对象
                case TableNames.OrderBoxBlock:
                {
                    if (!await cache.ExistsAsync(CacheKeys.Order_OrderNo_CompanyID(record["OrderNo"])))
                        return false;
                    break;
                }
                // OrderBlockPlan删除CreateTime < 202301的
                case TableNames.OrderBlockPlan:
                {
                    var time = DateTime.Parse(record["CreateTime"].Trim('"'));
                    if (time < oldestTime)
                        return false;
                    break;
                }
                // OrderBlockPlanResult删除对应order_block_plan.ID不存在的对象
                case TableNames.OrderBlockPlanResult:
                {
                    if (!await cache.ExistsAsync(CacheKeys.OrderBlockPlan_ID_CompanyID(record["ID"])))
                        return false;
                    break;
                }
                // OrderModule删除OrderNo < 202301的
                case TableNames.OrderModule:
                {
                    var orderNo = record["OrderNo"];
                    if(int.Parse(orderNo.AsSpan(0, 6).ToString()) < oldestTimeInt)
                        return false;
                    break;
                }
                // OrderProcess删除OrderNo < 202301的
                case TableNames.OrderProcess:
                {
                    var orderNo = record["OrderNo"];
                    if(int.Parse(orderNo.AsSpan(0, 6).ToString()) < oldestTimeInt)
                        return false;
                    break;
                }
                // OrderProcessStep删除OrderNo < 202301的
                case TableNames.OrderProcessStep:
                {
                    var orderNo = record["OrderNo"];
                    if(int.Parse(orderNo.AsSpan(0, 6).ToString()) < oldestTimeInt)
                        return false;
                    break;
                }
                // OrderProcessStepStep删除对应OrderProcess.ID不存在的对象
                case TableNames.OrderProcessStepItem:
                {
                    if (!await cache.ExistsAsync(CacheKeys.OrderProcess_ID_ShardKey(record["OrderProcessID"])))
                        return false;
                    break;
                }
                // SimplePackage删除OrderNo < 202301的
                case TableNames.SimplePackage:
                {
                    var orderNo = record["OrderNo"];
                    if(int.Parse(orderNo.AsSpan(0, 6).ToString()) < oldestTimeInt)
                        return false;
                    break;
                }
                // SimplePlanOrder删除CreateTime < 202301的
                case TableNames.SimplePlanOrder:
                {
                    var time = DateTime.Parse(record["CreateTime"].Trim('"'));
                    if (time < oldestTime)
                        return false;
                    break;
                }
            }

            return true;
        };
        
        // 数据替换
        options.RecordModify = async context =>
        {
            var record = context.Record;
            var cache = context.Cacher;
            switch (record.TableName)
            {
#if USE_TEST_DB
                // Order表移除IsBatch列
                case TableNames.Order:
                    record.RemoveField("IsBatch");
                    break;
#endif
                //OrderBlockPlan将OrderNo长度<2的置空
                case TableNames.OrderBlockPlan:
                    if (record["OrderNos"].Length < 2)
                        record["OrderNos"] = "NULL";
                    break;
                // OrderBlockPlanResult表添加CompanyID列
                case TableNames.OrderBlockPlanResult:
                    record.AddField("CompanyID",
                        // 获取OrderBlockPlan.ID -> CompanyID
                        ThrowIfNoCached(await cache.GetStringAsync(CacheKeys.OrderBlockPlan_ID_CompanyID(record["ID"])),
                            TableNames.OrderBlockPlanResult, TableNames.OrderBlockPlan, "ID", "脏数据未处理"));
                    break;
                // OrderBoxBlock添加CompanyID列
                case TableNames.OrderBoxBlock:
                    record.AddField("CompanyID", 
                        // 获取Order.OrderNo -> CompanyID
                        ThrowIfNoCached(await cache.GetStringAsync(CacheKeys.Order_OrderNo_CompanyID(record["OrderNo"])),
                            TableNames.OrderBoxBlock, TableNames.Order, "OrderNo", "脏数据未处理"));
                    break;
                // OrderModule添加ShardKey列，移除ViewFileName列
                case TableNames.OrderModule:
                    record.AddField("ShardKey", CalculateShardKeyByOrderNo(record["OrderNo"]));
                    record.RemoveField("ViewFileName");
                    break;
                // OrderProcess添加ShardKey列，NextStepID的空值转换为0
                case TableNames.OrderProcess:
                    record.AddField("ShardKey", CalculateShardKeyByOrderNo(record["OrderNo"]));
#if USE_TEST_DB
                    if(record["NextStepID"] == "\\N")
                        record["NextStepID"] = "0";
#endif
                    break;
                // OrderProcessStep，OrderProcessStepItem添加ShardKey列
                case TableNames.OrderProcessStep:
                    record.AddField("ShardKey", CalculateShardKeyByOrderNo(record["OrderNo"]));
                    break;
                case TableNames.OrderProcessStepItem:
                    record.AddField("ShardKey", 
                        // 获取OrderProcess.ID -> ShardKey
                    ThrowIfNoCached(await cache.GetStringAsync(CacheKeys.OrderProcess_ID_ShardKey(record["OrderProcessID"])), 
                        TableNames.OrderProcessStepItem, TableNames.OrderProcessStep, "OrderProcessID", "脏数据未处理"));
                    break;
                case TableNames.SimplePlanOrder:
#if USE_TEST_DB
                    record.RemoveField("ProcessState");
#endif
                    record.AddField("Deleted", "0");
                    break;
            }

            return record;

            string ThrowIfNoCached(string? cached, string tableName, string cachedTableName, string cachedColumn, string appendMessage = "")
            {
                if (cached is null)
                    throw new InvalidDataException(
                        $"{tableName}数据异常，在缓存中未找到对应{cachedTableName}.{cachedColumn}\t{appendMessage}");
                return cached;
            }
        };
        
        // 数据缓存
        options.RecordCache = async context =>
        {
            var record = context.Record;
            var cache = context.Cacher;
            switch (record.TableName)
            {
                // 缓存Order.OrderNo -> CompanyID
                case TableNames.Order:
                    CacheKeys.Order_OrderNo_CompanyID = orderNo =>
                        BuildCacheKey(TableNames.Order, "OrderNo", orderNo, "CompanyID");
                    await cache.SetStringAsync(
                        CacheKeys.Order_OrderNo_CompanyID(record["OrderNo"]),
                        record["CompanyID"]);
                    break;
                
                // 缓存OrderBlockPlan.ID -> CompanyID
                case TableNames.OrderBlockPlan:
                    CacheKeys.OrderBlockPlan_ID_CompanyID = id =>
                        BuildCacheKey(TableNames.OrderBlockPlan, "ID", id, "CompanyID");
                    await cache.SetStringAsync(
                        CacheKeys.OrderBlockPlan_ID_CompanyID(record["ID"]),
                        record["CompanyID"]);
                    break;
                
                // 缓存OrderProcess.ID -> ShardKey
                case TableNames.OrderProcess:
                    CacheKeys.OrderProcess_ID_ShardKey = id =>
                        BuildCacheKey(TableNames.OrderProcess, "ID", id, "ShardKey");
                    await cache.SetStringAsync(
                        CacheKeys.OrderProcess_ID_ShardKey(record["ID"]),
                        record["ShardKey"]);
                    break;
            }
        };
        
        // 数据库过滤
        options.DatabaseFilter = record =>
        {
            var companyId = int.Parse(record[tenantDbOptions.TenantKey]); // 每个实体都应存在CompanyID，否则异常
            return tenantDbOptions.GetDbNameByTenantKeyValue(companyId);
        };
        
        // 数据重建
        options.RecordReBuild = context =>
        {
            var record = context.Record;
            var resultList = new List<DataRecord>();
            // 分流OrderItem表
            if (record.TableName == TableNames.OrderItem)
            {
                record.TryGetField("ID", out var itemId);
                record.TryGetField("ShardKey", out var shardKey);
                record.TryGetField("PlanID", out var planId);
                record.TryGetField("PackageID", out var packageId);
                record.TryGetField("CompanyID", out var companyId);
                if(!int.TryParse(planId, out var pid))
                    throw new ApplicationException($"数据发生异常：OrderItem.PlanID，值: {planId}"); 
                if (pid > 0)
                {
                    resultList.Add(new DataRecord(new[] { itemId, shardKey, planId, companyId },
                        TableNames.OrderBlockPlanItem,
                        ["ItemID", "ShardKey", "PlanID", "CompanyID"]
                        ));
                }
                if(!int.TryParse(packageId, out var pkid))
                    throw new ApplicationException($"数据发生异常：OrderItem.PackageID，值: {packageId}"); 
                if(pkid > 0)
                {
                    resultList.Add(new DataRecord(new[] { itemId, shardKey, packageId, companyId },
                        TableNames.OrderPackageItem,
                        [ "ItemID", "ShardKey", "PackageID", "CompanyID" ]
                    ));
                }
            }
            return resultList;

        };
    });
    
    host.Services.Configure<DatabaseOutputOptions>(options =>
    {
        options.ConnectionString = outputOptions.ConnectionString;
        options.FlushCount = outputOptions.FlushCount;
        options.MaxAllowedPacket = outputOptions.MaxAllowedPacket;
        options.MaxDatabaseOutputTask = outputOptions.MaxDatabaseOutputTask;
        options.TreatJsonAsHex = outputOptions.TreatJsonAsHex;

#if USE_TEST_DB
        // Test Server
         options.ColumnTypeConfig = new Dictionary<string, ColumnType>
         {
             { "simple_plan_order.PlaceData", ColumnType.Blob },
             { "order_block_plan_result.PlaceData", ColumnType.Blob },
             { "order_box_block.Data", ColumnType.Blob },
             { "order_data_goods.ExtraProp", ColumnType.Json },
             { "order_module_extra.JsonStr", ColumnType.Text },
             { "process_info.Users", ColumnType.Text },
             { "order_process_schdule.CustomOrderNo", ColumnType.Text },
             { "order_process_schdule.OrderProcessStepName", ColumnType.Text },
             { "order_process_schdule.AreaName", ColumnType.Text },
             { "order_process_schdule.ConsigneeAddress", ColumnType.Text },
             { "order_process_schdule.ConsigneePhone", ColumnType.Text },
             { "report_source.Sql", ColumnType.Text },
             { "report_source.KeyValue", ColumnType.Text },
             { "report_source.Setting", ColumnType.Text },
             { "order_data_block.RemarkJson", ColumnType.Text },
             { "order_patch_detail.BlockDetail", ColumnType.Json },
             { "order_scrap_board.OutLineJson", ColumnType.Text },
             { "simple_package.Items", ColumnType.Json },
             { "order_batch_pack_config.Setting", ColumnType.Text },
             { "machine.Settings", ColumnType.Text },
             { "sys_config.Value", ColumnType.Text },
             { "sys_config.JsonStr", ColumnType.Text },
             { "process_item_exp.ItemJson", ColumnType.Text },
             { "report_template.Template", ColumnType.Text },
             { "report_template.SourceConfig", ColumnType.Text },
             { "order_block_plan.OrderNos", ColumnType.Json },
             { "order_block_plan.BlockInfo", ColumnType.Text },
         };
#else
        // 配置列类型
        // Prod server
        options.ColumnTypeConfig = new Dictionary<string, ColumnType>
        {
            { "simple_plan_order.PlaceData", ColumnType.Blob },
            { "order_block_plan_result.PlaceData", ColumnType.Blob },
            { "order_box_block.Data", ColumnType.Blob },
            { "order_data_goods.ExtraProp", ColumnType.Text },
            { "order_module_extra.JsonStr", ColumnType.Text },
            { "process_info.Users", ColumnType.Text },
            { "order_process_schdule.CustomOrderNo", ColumnType.Text },
            { "order_process_schdule.OrderProcessStepName", ColumnType.Text },
            { "order_process_schdule.AreaName", ColumnType.Text },
            { "order_process_schdule.ConsigneeAddress", ColumnType.Text },
            { "order_process_schdule.ConsigneePhone", ColumnType.Text },
            { "report_source.Sql", ColumnType.Text },
            { "report_source.KeyValue", ColumnType.Text },
            { "report_source.Setting", ColumnType.Text },
            { "order_data_block.RemarkJson", ColumnType.Text },
            { "order_patch_detail.BlockDetail", ColumnType.Text },
            { "order_scrap_board.OutLineJson", ColumnType.Text },
            { "simple_package.Items", ColumnType.Text },
            { "order_batch_pack_config.Setting", ColumnType.Text },
            { "machine.Settings", ColumnType.Text },
            { "sys_config.Value", ColumnType.Text },
            { "sys_config.JsonStr", ColumnType.Text },
            { "process_item_exp.ItemJson", ColumnType.Text },
            { "report_template.Template", ColumnType.Text },
            { "report_template.SourceConfig", ColumnType.Text },
            { "order_block_plan.OrderNos", ColumnType.Json },
            { "order_block_plan.BlockInfo", ColumnType.Text },
        };
#endif
    });
    
    host.Services.AddLogging(builder =>
    {
        builder.ClearProviders();
        builder.AddSerilog(new LoggerConfiguration()
            .WriteTo.Console()
            .WriteTo.File(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"./Log/Error/{ErrorRecorder.UID}.log"), 
                restrictedToMinimumLevel:LogEventLevel.Error)
            // .WriteTo.File("./Log/Info/{ErrorRecorder.UID}.log", restrictedToMinimumLevel:LogEventLevel.Information) //性能考虑暂不使用
            .CreateLogger()
        );
    });

    host.Services.AddDataSourceFactory();
    host.Services.AddErrorRecorderFactory();
    host.Services.AddSingleton<ProcessContext>();
    host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Produce);
    host.Services.AddRecordQueuePool(tenantDbOptions.DbGroup.Keys.Select(key => (key:key, queue:new DataRecordQueue(500_000))).ToArray());
    host.Services.AddSingleton<ITaskMonitorLogger, CacheTaskMonitorLogger>();
    host.Services.AddSingleton<ITaskMonitorLogger, LoggerTaskMonitorLogger>();

    host.Services.AddHostedService<MainHostedService>();
    host.Services.AddSingleton<IInputService, FileInputService>();
    host.Services.AddSingleton<ITransformService, TransformService>();
    host.Services.AddSingleton<IOutputService, OutputService>();
    host.Services.AddSingleton<TaskMonitorService>();
    host.Services.AddRedisCache(redisOptions);
    var app = host.Build();
    await app.RunAsync();
}