// See https://aka.ms/new-console-template for more information using System.Text; using Mesdb.DataGenerator; using MesETL.App; using MesETL.App.Cache; using MesETL.App.Const; using MesETL.App.HostedServices; using MesETL.App.HostedServices.Abstractions; using MesETL.App.Options; using MesETL.App.Services; using MesETL.App.Services.ErrorRecorder; using MesETL.App.Services.ETL; using MesETL.App.Services.Loggers; using MesETL.App.Services.Seq; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Serilog; using Serilog.Events; await RunProgram(); return; async Task RunProgram() { ThreadPool.SetMaxThreads(200, 200); var host = Host.CreateApplicationBuilder(args); var inputOptions = host.Configuration.GetRequiredSection("Input").Get() ?? throw new ApplicationException("缺少Input配置"); var transformOptions = host.Configuration.GetRequiredSection("Transform").Get() ?? throw new ApplicationException("缺少Transform配置"); var outputOptions = host.Configuration.GetRequiredSection("Output").Get() ?? throw new ApplicationException("缺少Output配置"); var tenantDbSection = host.Configuration.GetRequiredSection("TenantDb"); var tenantDbOptions = new TenantDbOptions() { TenantKey = tenantDbSection.GetValue(nameof(TenantDbOptions.TenantKey)) ?? throw new ApplicationException("分库配置缺少分库键TenantKey"), UseDbGroup = tenantDbSection.GetValue(nameof(TenantDbOptions.UseDbGroup)) ?? throw new ApplicationException("分库配置缺少使用分库组UseDbGroup") }; host.Services.Configure(options => { options.TenantKey = tenantDbOptions.TenantKey; options.DbGroup = tenantDbOptions.DbGroup; options.UseDbGroup = tenantDbOptions.UseDbGroup; }); tenantDbOptions.DbGroup = tenantDbSection.GetRequiredSection($"DbGroups:{tenantDbOptions.UseDbGroup}") .Get>() ?? throw new ApplicationException($"分库配置无法解析分库组{tenantDbOptions.UseDbGroup},请检查配置"); host.Services.Configure(options => { const float Multiplexer = 1F; var SampleSharedKeys = Enumerable.Range(0, 11).Select(i => (23010 + i * 10).ToString()).Concat( Enumerable.Range(0, 11).Select(i => (24010 + i * 10).ToString())).ToArray(); options.Rules = new Dictionary() { { TableNames.Order, new TableMockOptions((long)(2912406 * Multiplexer), context => { string[] headers = [ "OrderNo", "ShardKey", "CreateTime", "CompanyID", "SchduleDeliveryDate", "OrderType", "OrderSort", "CadDataType", "Deleted", "ProcessState" ]; string[] fields = [ (20241210000000 + context.Index).ToString(), SampleSharedKeys[Random.Shared.Next(SampleSharedKeys.Length)], $"\"{DateTime.Now:yyyy-MM-dd HH:mm:ss}\"", Random.Shared.Next(1, 28888).ToString(), $"\"{DateTime.Now.AddDays(Random.Shared.Next(1, 30)):yyyy-MM-dd HH:mm:ss}\"", Random.Shared.Next(0, 3).ToString(), Random.Shared.Next(0, 3).ToString(), Random.Shared.Next(0, 3).ToString(), Random.Shared.Next(0, 2).ToString(), Random.Shared.Next(0, 2).ToString() ]; return new DataRecord(fields, TableNames.Order, headers); }) }, { TableNames.OrderItem, new TableMockOptions((long)(820241144 * Multiplexer), context => { string[] headers = [ "ID", "ShardKey", "OrderNo", "ItemNo", "ItemType", "RoomID", "BoxID", "DataID", "PlanID", "PackageID", "Num", "CompanyID" ]; string[] fields = [ context.Index.ToString(), SampleSharedKeys[Random.Shared.Next(SampleSharedKeys.Length)], (20241210000000 + Random.Shared.Next(0, 2912406)).ToString(), (2412000000000 + context.Index).ToString(), Random.Shared.Next(0, 3).ToString(), Random.Shared.Next(1, 1000000).ToString(), Random.Shared.Next(1, 1000000).ToString(), Random.Shared.Next(1, 1000000).ToString(), Random.Shared.Next(1, 1306670366).ToString(), Random.Shared.Next(1, 10000000).ToString(), (Random.Shared.Next(1, 2000000) / 1000).ToString("F3"), Random.Shared.Next(1, 28888).ToString(), ]; return new DataRecord(fields, TableNames.OrderItem, headers); }) }, { TableNames.OrderDataBlock, new TableMockOptions((long)(428568719 * Multiplexer), context => { string[] headers = [ "ID", "ShardKey", "OrderNo", "BoardName", "BoardType", "GoodsID", "Width", "Height", "Thickness", "SpliteWidth", "SpliteHeight", "SpliteThickness", "SealedLeft", "SealedRight", "SealedUp", "SealedDown", "Area", "Wave", "HoleFace", "PaiKong", "RemarkJson", "UnRegularPointCount", "FrontHoleCount", "BackHoleCount", "SideHoleCount", "FrontModelCount", "BackModelCount", "IsDoor", "OpenDoorType", "CompanyID", "BorderLengthHeavy", "BorderLengthLight", "IsHXDJX", "ModuleTypeID", "PlanFilterType" ]; string[] BoardNames = ["左侧板", "右侧板", "左开门板", "右开门板", "薄背板", "顶板", "底板", "地脚线", "后地脚", "层板", "立板"]; string[] fields = [ context.Index.ToString(), SampleSharedKeys[Random.Shared.Next(SampleSharedKeys.Length)], (20241210000000 + Random.Shared.Next(0, 2912406)).ToString(), $"\"{BoardNames[Random.Shared.Next(0, BoardNames.Length)]}\"", Random.Shared.Next(0, 3).ToString(), Random.Shared.Next(1, 1000000).ToString(), (Random.Shared.Next(1, 2000000) / 1000).ToString("F3"), (Random.Shared.Next(1, 1200000) / 1000).ToString("F3"), (Random.Shared.Next(1, 18000) / 1000).ToString("F3"), (Random.Shared.Next(1, 2000000) / 1000).ToString("F3"), (Random.Shared.Next(1, 1200000) / 1000).ToString("F3"), (Random.Shared.Next(1, 18000) / 1000).ToString("F3"), (Random.Shared.Next(1, 2000) / 1000).ToString("F2"), (Random.Shared.Next(1, 2000) / 1000).ToString("F2"), (Random.Shared.Next(1, 2000) / 1000).ToString("F2"), (Random.Shared.Next(1, 2000) / 1000).ToString("F2"), (Random.Shared.Next(1, 3000) / 1000).ToString("F3"), Random.Shared.Next(0, 3).ToString(), Random.Shared.Next(0, 3).ToString(), Random.Shared.Next(0, 3).ToString(), Convert.ToHexString(Encoding.UTF8.GetBytes(MockHelper.RandomString(100))), Random.Shared.Next(0, 4).ToString(), Random.Shared.Next(0, 4).ToString(), Random.Shared.Next(0, 4).ToString(), Random.Shared.Next(0, 4).ToString(), Random.Shared.Next(0, 4).ToString(), Random.Shared.Next(0, 4).ToString(), Random.Shared.Next(0, 2).ToString(), Random.Shared.Next(0, 6).ToString(), Random.Shared.Next(1, 28888).ToString(), (Random.Shared.Next(1, 2000000) / 1000).ToString("F3"), (Random.Shared.Next(1, 2000000) / 1000).ToString("F3"), Random.Shared.Next(0, 2).ToString(), Random.Shared.Next(0, 100000).ToString(), Random.Shared.Next(0, 5).ToString(), ]; return new DataRecord(fields, TableNames.OrderDataBlock, headers); }) }, { TableNames.OrderBoxBlock, new TableMockOptions((long)(20163038 * Multiplexer), context => { string[] headers = ["BoxID", "ShardKey", "OrderNo", "Data", "CompanyID"]; string[] fields = [ context.Index.ToString(), SampleSharedKeys[Random.Shared.Next(SampleSharedKeys.Length)], (20241210000000 + Random.Shared.Next(0, 2912406)).ToString(), OrderBoxBlockHelper.RandomPick(), Random.Shared.Next(1, 28888).ToString(), ]; return new DataRecord(fields, TableNames.OrderBoxBlock, headers); }) }, { TableNames.OrderModuleExtra, new TableMockOptions((long)(33853580 * Multiplexer), context => { string[] headers = [ "ID", "ShardKey", "OrderNo", "RoomID", "BoxID", "PropType", "JsonStr", "CompanyID" ]; string[] fields = [ context.Index.ToString(), SampleSharedKeys[Random.Shared.Next(SampleSharedKeys.Length)], (20241210000000 + Random.Shared.Next(0, 2912406)).ToString(), Random.Shared.Next(1, 1000000).ToString(), Random.Shared.Next(1, 1000000).ToString(), Random.Shared.Next(0, 14).ToString(), OrderModuleExtraHelper.PickJson(), Random.Shared.Next(1, 28888).ToString(), ]; return new DataRecord(fields, TableNames.OrderModuleExtra, headers); }) } }; }); host.Services.Configure(options => { options.StrictMode = transformOptions.StrictMode; options.EnableFilter = transformOptions.EnableFilter; options.EnableReplacer = transformOptions.EnableReplacer; options.EnableReBuilder = transformOptions.EnableReBuilder; options.DatabaseFilter = record => { var companyId = int.Parse(record[tenantDbOptions.TenantKey]); // 每个实体都应存在CompanyID,否则异常 return tenantDbOptions.GetDbNameByTenantKeyValue(companyId); }; }); host.Services.Configure(options => { options.ConnectionString = outputOptions.ConnectionString; options.FlushCount = outputOptions.FlushCount; options.MaxAllowedPacket = outputOptions.MaxAllowedPacket / 2; options.MaxDatabaseOutputTask = outputOptions.MaxDatabaseOutputTask; options.TreatJsonAsHex = outputOptions.TreatJsonAsHex; options.NoOutput = outputOptions.NoOutput; options.ForUpdate = outputOptions.ForUpdate; // 配置列的类型以便于在输出时区分二进制内容 // Prod server options.ColumnTypeConfig = new Dictionary { { "machine.Settings", ColumnType.Text }, { "order_block_plan.BlockInfo", ColumnType.Text }, { "order_block_plan.OrderNos", ColumnType.Json }, { "order_block_plan_result.PlaceData", ColumnType.Blob }, { "order_box_block.Data", ColumnType.Blob }, { "order_data_block.RemarkJson", ColumnType.Text }, { "order_data_goods.ExtraProp", ColumnType.Json }, { "order_extra.ConfigJson", ColumnType.Json }, { "order_module_extra.Data", ColumnType.Blob }, { "order_module_extra.JsonStr", ColumnType.Text }, { "order_patch_detail.BlockDetail", ColumnType.Json }, { "order_process_schdule.AreaName", ColumnType.Text }, { "order_process_schdule.ConsigneeAddress", ColumnType.Text }, { "order_process_schdule.ConsigneePhone", ColumnType.Text }, { "order_process_schdule.CustomOrderNo", ColumnType.Text }, { "order_process_schdule.OrderProcessStepName", ColumnType.Text }, { "order_scrap_board.OutLineJson", ColumnType.Text }, { "order_wave_group.ConfigJson", ColumnType.Json }, { "process_info.Users", ColumnType.Text }, { "process_item_exp.ItemJson", ColumnType.Text }, { "report_template.SourceConfig", ColumnType.Text }, { "report_template.Template", ColumnType.Text }, { "simple_package.Items", ColumnType.Json }, { "sys_config.JsonStr", ColumnType.Text }, { "sys_config.Value", ColumnType.Text } }; }); host.Services.AddLogging(builder => { builder.ClearProviders(); var logger = new LoggerConfiguration() .MinimumLevel.Verbose() .WriteTo.Console() .WriteTo.File(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"./Log/Error/{ErrorRecorder.UID}.log"), restrictedToMinimumLevel: LogEventLevel.Error) // .WriteTo.File("./Log/Info/{ErrorRecorder.UID}.log", restrictedToMinimumLevel:LogEventLevel.Information) //性能考虑暂不使用 .CreateLogger(); builder.AddSerilog(logger); Log.Logger = logger; }); host.Services.AddDataSourceFactory(); host.Services.AddErrorRecorderFactory(); host.Services.AddSingleton(); host.Services.AddSingleton(); var prodLen = host.Configuration.GetRequiredSection("RecordQueue").GetValue("ProducerQueueLength"); var consLen = host.Configuration.GetRequiredSection("RecordQueue").GetValue("ConsumerQueueLength"); var maxCharCount = host.Configuration.GetRequiredSection("RecordQueue").GetValue("MaxByteCount") / 2; host.Services.AddKeyedSingleton(ConstVar.Producer, new DataRecordQueue(prodLen, maxCharCount)); host.Services.AddRecordQueuePool(tenantDbOptions.DbGroup.Keys .Select(key => (key: key, queue: new DataRecordQueue(consLen, maxCharCount))).ToArray()); // host.Services.AddSingleton(); host.Services.AddSingleton(); host.Services.AddHostedService(); host.Services.AddSingleton(); host.Services.AddSingleton(); host.Services.AddSingleton(); host.Services.AddSingleton(); // host.Services.AddRedisCache(redisOptions); host.Services.AddSingleton(); var app = host.Build(); await app.RunAsync(); }