using ConsoleApp2; using ConsoleApp2.Const; using ConsoleApp2.Helpers; using ConsoleApp2.HostedServices; using ConsoleApp2.HostedServices.Abstractions; using ConsoleApp2.Options; using ConsoleApp2.Services; using ConsoleApp2.SimulationService; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using MySqlConnector; using Serilog; using Serilog.Core; using System.Reflection.PortableExecutable; // 运行之前把Mysql max_allowed_packets 调大 // 运行之前把process_step表的外键删掉 await RunProgram(); return; async Task RunProgram() { //var inputDir= "D:\\MyDumper"; //ValidateConsole.ValidateInput((_inputDir) => //{ // if (Directory.Exists(_inputDir)) // { // inputDir = _inputDir; // return true; // } // else return false; //}, "请输入读取csv文件的目录(默认为当前目录下MyDumper文件夹):"); //var maxTask = 16; //ValidateConsole.ValidateInput((_inputDir) => //{ // _ = int.TryParse(_inputDir.ToString(), out var _taskCount); // if (_taskCount > 0) { // maxTask = _taskCount; // return true; // } // else return false; //}, "请输入执行输出的线程数量(默认为16):"); //var flushCount = 2_0000; //ValidateConsole.ValidateInput((_inputDir) => //{ // _ = int.TryParse(_inputDir.ToString(), out var _flashCount); // if (_flashCount > 0) // { // flushCount = _flashCount; // return true; // } else return false; //}, "请输入单次插入的行数(默认为20000):"); ThreadPool.SetMaxThreads(200, 200); var host = Host.CreateApplicationBuilder(args); var commandOptions = host.Configuration.GetSection("CmdOptions").Get() ?? new CommandOptions(); Console.WriteLine($"InputDir:{commandOptions?.InputDir}"); Console.WriteLine($"OutPutFlushCount:{commandOptions?.FlushCount}"); Console.WriteLine($"OutPutTaskCount:{commandOptions?.TaskCount}"); host.Services.Configure(option => { option.TableInfoConfig = new Dictionary { //{"order_block_plan_item",new TableInfo{SimulaRowCount=136323566 }},//从order_item表查询,然后程序插入 //{"order_package_item",new TableInfo{SimulaRowCount=52525224 }},//从order_item表查询,然后程序插入 //{"order_patch_detail",new TableInfo{SimulaRowCount=10 }},//生产没有这个表,不处理 //{"machine",new TableInfo{SimulaRowCount=14655 }}, //{"order",new TableInfo{SimulaRowCount=5019216 }}, //{"order_block_plan",new TableInfo{SimulaRowCount=2725553 }},//CreateTime < 202301的删除 {"order_block_plan_result",new TableInfo{SimulaRowCount=1174096 }}, //{"order_box_block",new TableInfo{SimulaRowCount=29755672 }}, //{"order_data_block",new TableInfo{SimulaRowCount=731800334 }}, //{"order_data_goods",new TableInfo{SimulaRowCount=25803671 }}, //{"order_data_parts",new TableInfo{SimulaRowCount=468517543 }}, //{"order_item",new TableInfo{SimulaRowCount=1345520079 }}, //{"order_module",new TableInfo{SimulaRowCount=103325385 }}, //{"order_module_extra",new TableInfo{SimulaRowCount=54361321 }}, //{"order_module_item",new TableInfo{SimulaRowCount=69173339 }}, //{"order_package",new TableInfo{SimulaRowCount=16196195 }}, //{"order_process",new TableInfo{SimulaRowCount=3892685 }},//orderNo < 202301的 //{"order_process_step",new TableInfo{SimulaRowCount=8050349 }},//orderNo < 202301的删除 //{"order_process_step_item",new TableInfo{SimulaRowCount=14538058 }},//orderNo < 202301的删除 //{"order_scrap_board",new TableInfo{SimulaRowCount=123998 }}, //{"process_group",new TableInfo{SimulaRowCount=1253 }}, //{"process_info",new TableInfo{SimulaRowCount=7839 }}, //{"process_item_exp",new TableInfo{SimulaRowCount=28 }}, //{"process_schdule_capacity",new TableInfo{SimulaRowCount=39736 }}, //{"process_step_efficiency",new TableInfo{SimulaRowCount=8 }}, //{"report_template",new TableInfo{SimulaRowCount=7337 }}, //{"simple_package",new TableInfo{SimulaRowCount=130436 }},//orderNo < 202301的删除 //{"simple_plan_order",new TableInfo{SimulaRowCount=351470 }},//CreateTime < 202301的删除 //{"sys_config",new TableInfo{SimulaRowCount=2296 }}, //{"work_calendar",new TableInfo{SimulaRowCount=11 }}, //{"work_shift",new TableInfo{SimulaRowCount=59 }}, //{"work_time",new TableInfo{SimulaRowCount=62 }}, }; }); host.Services.Configure(option => { option.Delimiter = ","; option.QuoteChar = '"'; }); host.Services.Configure(options => { options.InputDir = commandOptions.InputDir; var _csvOptions = new CsvOptions { Delimiter = ",", QuoteChar = '"' }; options.CreateSource = (string tableName) => { var source = new ZstSource(commandOptions.InputDir, tableName, _csvOptions.Delimiter, _csvOptions.QuoteChar); return source; }; }); host.Services.Configure(options => { options.DatabaseFilter = record => "cferp_test"; options.TransformBinary = field => commandOptions != null && commandOptions.Isutf8mb4 ? $"_utf8mb4 0x{field}" : $"0x{field}"; //数据过滤 options.RecordFilter = record => { var index = Array.IndexOf(record.Headers, "ShardKey"); if (index > -1) { var skString = record.Fields[index]; short.TryParse(skString, out var sk); if (sk < commandOptions.OldestShardKey) return false; } if (record.TableName == "order_package") { var pkNoIndex = Array.IndexOf(record.Headers, "PakageNo"); if (pkNoIndex > -1) { var pkNo = record.Fields[pkNoIndex]; if (pkNo.Length <= 2) return false; } } if (record.TableName == "order_block_plan") { var orderNosIndex = Array.IndexOf(record.Headers, "OrderNos"); if (orderNosIndex > -1) { var pkNo = record.Fields[orderNosIndex]; if (pkNo.Length <= 2) return false; } } return true; }; //数据修改 options.RecordModify = (record) => { if (record.TableName == "order_process")//修改order_process.NextStepID的默认值为0 { var nextStepIdIndex = Array.IndexOf(record.Headers, "NextStepID"); if (nextStepIdIndex > -1) { var idString = record.Fields[nextStepIdIndex]; if (idString == "\\N") { record.Fields[nextStepIdIndex] = "0"; } } } }; //数据替换 options.RecordReplace = (record) => { //删除数据源里simple_plan_order.ProcessState 字段和值 if (record.TableName == "simple_plan_order")//修改order_process.NextStepID的默认值为0 { var nextStepIdIndex = Array.IndexOf(record.Headers, "ProcessState"); if (nextStepIdIndex > -1) { var headers = record.Headers.Where(t => t != "ProcessState").ToArray(); var fs = record.Fields.ToList(); fs.RemoveAt(nextStepIdIndex); var fields = fs.ToArray(); return new DataRecord(fields, record.TableName, headers, record.CompanyID); } } return null; }; //数据生成 options.RecordAdd = (record) => { var resultList = new List(); if(record.TableName == "order_item") { var itemIDIndex = Array.IndexOf(record.Headers, "ItemID"); var shardKeyIndex = Array.IndexOf(record.Headers, "ShardKey"); var planIDIndex = Array.IndexOf(record.Headers, "PlanID"); var packageIDIndex = Array.IndexOf(record.Headers, "PackageID"); var companyIDIndex = Array.IndexOf(record.Headers, "CompanyID"); //resultList.Add(new DataRecord( // new[] { "ItemID", "ShardKey", "PlanID","CompanyID" }, "order_block_plan_item", // new[] { record.Fields[itemIDIndex], record.Fields[shardKeyIndex], record.Fields[planIDIndex], record.Fields[companyIDIndex] })); //resultList.Add( // new DataRecord(new[] { "ItemID", "ShardKey", "PackageID", "CompanyID" }, "order_package_item", // new[] { record.Fields[itemIDIndex], record.Fields[shardKeyIndex], record.Fields[packageIDIndex], record.Fields[companyIDIndex] })); } return resultList; }; options.ColumnTypeConfig = new() { { "simple_plan_order.PlaceData", ColumnType.Blob }, { "order_block_plan_result.PlaceData", ColumnType.Blob }, { "order_box_block.Data", ColumnType.Blob }, { "order_data_goods.ExtraProp", ColumnType.Text }, { "order_module_extra.JsonStr", ColumnType.Text }, { "process_info.Users", ColumnType.Text }, { "order_process_schdule.CustomOrderNo", ColumnType.Text }, { "order_process_schdule.OrderProcessStepName", ColumnType.Text }, { "order_process_schdule.AreaName", ColumnType.Text }, { "order_process_schdule.ConsigneeAddress", ColumnType.Text }, { "order_process_schdule.ConsigneePhone", ColumnType.Text }, { "report_source.Sql", ColumnType.Text }, { "report_source.KeyValue", ColumnType.Text }, { "report_source.Setting", ColumnType.Text }, { "order_data_block.RemarkJson", ColumnType.Text }, { "order_patch_detail.BlockDetail", ColumnType.Text }, { "order_scrap_board.OutLineJson", ColumnType.Text }, { "simple_package.Items", ColumnType.Text }, { "order_batch_pack_config.Setting", ColumnType.Text }, { "machine.Settings", ColumnType.Text }, { "sys_config.Value", ColumnType.Text }, { "sys_config.JsonStr", ColumnType.Text }, { "process_item_exp.ItemJson", ColumnType.Text }, { "report_template.Template", ColumnType.Text }, { "report_template.SourceConfig", ColumnType.Text }, { "order_block_plan.OrderNos", ColumnType.Text }, { "order_block_plan.BlockInfo", ColumnType.Text }, }; }); host.Services.Configure(options => { options.ConnectionString = new MySqlConnectionStringBuilder { Server = "127.0.0.1", Port = 33309, Database = "cferp_test", UserID = "root", Password = "123456", MaximumPoolSize = 50, // 这个值应当小于 max_connections }.ConnectionString; options.TaskCount = commandOptions.TaskCount; options.FlushCount = commandOptions.FlushCount; }); host.Services.AddLogging(builder => { builder.ClearProviders(); builder.AddSerilog(new LoggerConfiguration().WriteTo.Console().CreateLogger()); }); host.Services.AddSingleton(); host.Services.AddKeyedSingleton(ProcessStep.Producer); host.Services.AddKeyedSingleton(ProcessStep.Consumer); host.Services.AddTransient(); host.Services.AddHostedService(); host.Services.AddHostedService(); host.Services.AddSingleton(); host.Services.AddSingleton(); host.Services.AddSingleton(); host.Services.AddStackExchangeRedisCache(options => { options.Configuration = "localhost:6379"; }); var app = host.Build(); await app.RunAsync(); }