313 lines
13 KiB
C#
313 lines
13 KiB
C#
using ConsoleApp2;
|
||
using ConsoleApp2.Const;
|
||
using ConsoleApp2.Helpers;
|
||
using ConsoleApp2.HostedServices;
|
||
using ConsoleApp2.HostedServices.Abstractions;
|
||
using ConsoleApp2.Options;
|
||
using ConsoleApp2.Services;
|
||
using ConsoleApp2.SimulationService;
|
||
using Microsoft.Extensions.Caching.StackExchangeRedis;
|
||
using Microsoft.Extensions.Configuration;
|
||
using Microsoft.Extensions.DependencyInjection;
|
||
using Microsoft.Extensions.Hosting;
|
||
using Microsoft.Extensions.Logging;
|
||
using MySqlConnector;
|
||
using Serilog;
|
||
using Serilog.Core;
|
||
using StackExchange.Redis;
|
||
using System.Reflection.PortableExecutable;
|
||
|
||
|
||
// 运行之前把Mysql max_allowed_packets 调大
|
||
// 运行之前把process_step表的外键删掉
|
||
await RunProgram();
|
||
return;
|
||
|
||
async Task RunProgram()
|
||
{
|
||
|
||
|
||
//var inputDir= "D:\\MyDumper";
|
||
//ValidateConsole.ValidateInput<string>((_inputDir) =>
|
||
//{
|
||
// if (Directory.Exists(_inputDir))
|
||
// {
|
||
// inputDir = _inputDir;
|
||
// return true;
|
||
// }
|
||
// else return false;
|
||
//}, "请输入读取csv文件的目录(默认为当前目录下MyDumper文件夹):");
|
||
|
||
//var maxTask = 16;
|
||
//ValidateConsole.ValidateInput<string>((_inputDir) =>
|
||
//{
|
||
|
||
// _ = int.TryParse(_inputDir.ToString(), out var _taskCount);
|
||
// if (_taskCount > 0) {
|
||
// maxTask = _taskCount;
|
||
// return true;
|
||
// }
|
||
// else return false;
|
||
//}, "请输入执行输出的线程数量(默认为16):");
|
||
|
||
//var flushCount = 2_0000;
|
||
//ValidateConsole.ValidateInput<string>((_inputDir) =>
|
||
//{
|
||
// _ = int.TryParse(_inputDir.ToString(), out var _flashCount);
|
||
// if (_flashCount > 0)
|
||
// {
|
||
// flushCount = _flashCount;
|
||
// return true;
|
||
// } else return false;
|
||
|
||
//}, "请输入单次插入的行数(默认为20000):");
|
||
|
||
ThreadPool.SetMaxThreads(200, 200);
|
||
var host = Host.CreateApplicationBuilder(args);
|
||
var commandOptions = host.Configuration.GetSection("CmdOptions").Get<CommandOptions>() ?? new CommandOptions();
|
||
Console.WriteLine($"InputDir:{commandOptions?.InputDir}");
|
||
Console.WriteLine($"OutPutFlushCount:{commandOptions?.FlushCount}");
|
||
Console.WriteLine($"OutPutTaskCount:{commandOptions?.TaskCount}");
|
||
|
||
|
||
host.Services.Configure<InputTableOptions>(option =>
|
||
{
|
||
option.TableInfoConfig = new Dictionary<string, TableInfo>
|
||
{
|
||
|
||
//{"order_block_plan_item",new TableInfo{SimulaRowCount=136323566 }},//从order_item表查询，然后程序插入
|
||
//{"order_package_item",new TableInfo{SimulaRowCount=52525224 }},//从order_item表查询，然后程序插入
|
||
//{"order_patch_detail",new TableInfo{SimulaRowCount=10 }},//生产没有这个表，不处理
|
||
|
||
|
||
//{"machine",new TableInfo{SimulaRowCount=14655 }},
|
||
//{"order",new TableInfo{SimulaRowCount=5019216 }},
|
||
//{"order_block_plan",new TableInfo{SimulaRowCount=2725553 }},//CreateTime < 202301的删除
|
||
{"order_block_plan_result",new TableInfo{SimulaRowCount=1174096 }},
|
||
//{"order_box_block",new TableInfo{SimulaRowCount=29755672 }},
|
||
//{"order_data_block",new TableInfo{SimulaRowCount=731800334 }},
|
||
//{"order_data_goods",new TableInfo{SimulaRowCount=25803671 }},
|
||
//{"order_data_parts",new TableInfo{SimulaRowCount=468517543 }},
|
||
//{"order_item",new TableInfo{SimulaRowCount=1345520079 }},
|
||
//{"order_module",new TableInfo{SimulaRowCount=103325385 }},
|
||
//{"order_module_extra",new TableInfo{SimulaRowCount=54361321 }},
|
||
//{"order_module_item",new TableInfo{SimulaRowCount=69173339 }},
|
||
//{"order_package",new TableInfo{SimulaRowCount=16196195 }},
|
||
//{"order_process",new TableInfo{SimulaRowCount=3892685 }},//orderNo < 202301的
|
||
//{"order_process_step",new TableInfo{SimulaRowCount=8050349 }},//orderNo < 202301的删除
|
||
//{"order_process_step_item",new TableInfo{SimulaRowCount=14538058 }},//orderNo < 202301的删除
|
||
//{"order_scrap_board",new TableInfo{SimulaRowCount=123998 }},
|
||
//{"process_group",new TableInfo{SimulaRowCount=1253 }},
|
||
//{"process_info",new TableInfo{SimulaRowCount=7839 }},
|
||
//{"process_item_exp",new TableInfo{SimulaRowCount=28 }},
|
||
//{"process_schdule_capacity",new TableInfo{SimulaRowCount=39736 }},
|
||
//{"process_step_efficiency",new TableInfo{SimulaRowCount=8 }},
|
||
//{"report_template",new TableInfo{SimulaRowCount=7337 }},
|
||
//{"simple_package",new TableInfo{SimulaRowCount=130436 }},//orderNo < 202301的删除
|
||
//{"simple_plan_order",new TableInfo{SimulaRowCount=351470 }},//CreateTime < 202301的删除
|
||
//{"sys_config",new TableInfo{SimulaRowCount=2296 }},
|
||
//{"work_calendar",new TableInfo{SimulaRowCount=11 }},
|
||
//{"work_shift",new TableInfo{SimulaRowCount=59 }},
|
||
//{"work_time",new TableInfo{SimulaRowCount=62 }},
|
||
};
|
||
});
|
||
host.Services.Configure<CsvOptions>(option =>
|
||
{
|
||
option.Delimiter = ",";
|
||
option.QuoteChar = '"';
|
||
});
|
||
host.Services.Configure<DataInputOptions>(options =>
|
||
{
|
||
options.InputDir = commandOptions.InputDir;
|
||
var _csvOptions = new CsvOptions { Delimiter = ",", QuoteChar = '"' };
|
||
options.CreateSource = (string tableName) =>
|
||
{
|
||
var source = new ZstSource(commandOptions.InputDir, tableName, _csvOptions.Delimiter, _csvOptions.QuoteChar);
|
||
return source;
|
||
};
|
||
});
|
||
|
||
host.Services.Configure<DataTransformOptions>(options =>
|
||
{
|
||
options.DatabaseFilter = record => "cferp_test";
|
||
|
||
options.TransformBinary = field => commandOptions != null && commandOptions.Isutf8mb4 ? $"_utf8mb4 0x{field}" : $"0x{field}";
|
||
//数据过滤
|
||
options.RecordFilter = record =>
|
||
{
|
||
var index = Array.IndexOf(record.Headers, "ShardKey");
|
||
if (index > -1)
|
||
{
|
||
var skString = record.Fields[index];
|
||
short.TryParse(skString, out var sk);
|
||
if (sk < commandOptions.OldestShardKey) return false;
|
||
}
|
||
if (record.TableName == "order_package")
|
||
{
|
||
var pkNoIndex = Array.IndexOf(record.Headers, "PakageNo");
|
||
if (pkNoIndex > -1)
|
||
{
|
||
var pkNo = record.Fields[pkNoIndex];
|
||
if (pkNo.Length <= 2) return false;
|
||
}
|
||
}
|
||
if (record.TableName == "order_block_plan")
|
||
{
|
||
var orderNosIndex = Array.IndexOf(record.Headers, "OrderNos");
|
||
if (orderNosIndex > -1)
|
||
{
|
||
var pkNo = record.Fields[orderNosIndex];
|
||
if (pkNo.Length <= 2) return false;
|
||
}
|
||
}
|
||
return true;
|
||
|
||
};
|
||
//数据修改
|
||
options.RecordModify = (record) =>
|
||
{
|
||
if (record.TableName == "order_process")//修改order_process.NextStepID的默认值为0
|
||
{
|
||
var nextStepIdIndex = Array.IndexOf(record.Headers, "NextStepID");
|
||
if (nextStepIdIndex > -1)
|
||
{
|
||
var idString = record.Fields[nextStepIdIndex];
|
||
|
||
if (idString == "\\N")
|
||
{
|
||
record.Fields[nextStepIdIndex] = "0";
|
||
}
|
||
|
||
}
|
||
}
|
||
|
||
|
||
};
|
||
//数据替换
|
||
options.RecordReplace = (record) =>
|
||
{
|
||
//删除数据源里simple_plan_order.ProcessState 字段和值
|
||
|
||
if (record.TableName == "simple_plan_order")//修改order_process.NextStepID的默认值为0
|
||
{
|
||
var nextStepIdIndex = Array.IndexOf(record.Headers, "ProcessState");
|
||
if (nextStepIdIndex > -1)
|
||
{
|
||
var headers = record.Headers.Where(t => t != "ProcessState").ToArray();
|
||
var fs = record.Fields.ToList();
|
||
fs.RemoveAt(nextStepIdIndex);
|
||
var fields = fs.ToArray();
|
||
return new DataRecord(fields, record.TableName, headers, record.CompanyID);
|
||
}
|
||
}
|
||
if (record.TableName == "order_process_step")
|
||
{
|
||
|
||
}
|
||
return null;
|
||
};
|
||
//数据生成
|
||
options.RecordAdd = (record) =>
|
||
{
|
||
var resultList = new List<DataRecord>();
|
||
if(record.TableName == "order_item")
|
||
{
|
||
|
||
var itemIDIndex = Array.IndexOf(record.Headers, "ItemID");
|
||
var shardKeyIndex = Array.IndexOf(record.Headers, "ShardKey");
|
||
var planIDIndex = Array.IndexOf(record.Headers, "PlanID");
|
||
var packageIDIndex = Array.IndexOf(record.Headers, "PackageID");
|
||
var companyIDIndex = Array.IndexOf(record.Headers, "CompanyID");
|
||
|
||
resultList.Add(new DataRecord(
|
||
new[] { "ItemID", "ShardKey", "PlanID", "CompanyID" }, "order_block_plan_item",
|
||
new[] { record.Fields[itemIDIndex], record.Fields[shardKeyIndex], record.Fields[planIDIndex], record.Fields[companyIDIndex] }));
|
||
resultList.Add(
|
||
new DataRecord(new[] { "ItemID", "ShardKey", "PackageID", "CompanyID" }, "order_package_item",
|
||
new[] { record.Fields[itemIDIndex], record.Fields[shardKeyIndex], record.Fields[packageIDIndex], record.Fields[companyIDIndex] }));
|
||
}
|
||
return resultList;
|
||
|
||
};
|
||
options.RecordCache = async (record, db) =>
|
||
{
|
||
if(record.TableName == "order_process")
|
||
{
|
||
var skIndex = Array.IndexOf(record.Headers, "ShardKey");
|
||
if(skIndex > -1)
|
||
{
|
||
var sk = record.Fields[skIndex];
|
||
var idIndex = Array.IndexOf(record.Headers, "ID");
|
||
var id = record.Fields[idIndex];
|
||
await db.SetAddAsync(id, sk);
|
||
}
|
||
}
|
||
|
||
};
|
||
options.ColumnTypeConfig = new()
|
||
{
|
||
{ "simple_plan_order.PlaceData", ColumnType.Blob },
|
||
{ "order_block_plan_result.PlaceData", ColumnType.Blob },
|
||
{ "order_box_block.Data", ColumnType.Blob },
|
||
{ "order_data_goods.ExtraProp", ColumnType.Text },
|
||
{ "order_module_extra.JsonStr", ColumnType.Text },
|
||
{ "process_info.Users", ColumnType.Text },
|
||
{ "order_process_schdule.CustomOrderNo", ColumnType.Text },
|
||
{ "order_process_schdule.OrderProcessStepName", ColumnType.Text },
|
||
{ "order_process_schdule.AreaName", ColumnType.Text },
|
||
{ "order_process_schdule.ConsigneeAddress", ColumnType.Text },
|
||
{ "order_process_schdule.ConsigneePhone", ColumnType.Text },
|
||
{ "report_source.Sql", ColumnType.Text },
|
||
{ "report_source.KeyValue", ColumnType.Text },
|
||
{ "report_source.Setting", ColumnType.Text },
|
||
{ "order_data_block.RemarkJson", ColumnType.Text },
|
||
{ "order_patch_detail.BlockDetail", ColumnType.Text },
|
||
{ "order_scrap_board.OutLineJson", ColumnType.Text },
|
||
{ "simple_package.Items", ColumnType.Text },
|
||
{ "order_batch_pack_config.Setting", ColumnType.Text },
|
||
{ "machine.Settings", ColumnType.Text },
|
||
{ "sys_config.Value", ColumnType.Text },
|
||
{ "sys_config.JsonStr", ColumnType.Text },
|
||
{ "process_item_exp.ItemJson", ColumnType.Text },
|
||
{ "report_template.Template", ColumnType.Text },
|
||
{ "report_template.SourceConfig", ColumnType.Text },
|
||
{ "order_block_plan.OrderNos", ColumnType.Text },
|
||
{ "order_block_plan.BlockInfo", ColumnType.Text },
|
||
};
|
||
});
|
||
host.Services.Configure<DatabaseOutputOptions>(options =>
|
||
{
|
||
options.ConnectionString = new MySqlConnectionStringBuilder
|
||
{
|
||
Server = "127.0.0.1",
|
||
Port = 33309,
|
||
Database = "cferp_test",
|
||
UserID = "root",
|
||
Password = "123456",
|
||
MaximumPoolSize = 50, // 这个值应当小于 max_connections
|
||
}.ConnectionString;
|
||
options.TaskCount = commandOptions.TaskCount;
|
||
options.FlushCount = commandOptions.FlushCount;
|
||
});
|
||
host.Services.AddLogging(builder =>
|
||
{
|
||
builder.ClearProviders();
|
||
builder.AddSerilog(new LoggerConfiguration().WriteTo.Console().CreateLogger());
|
||
});
|
||
|
||
host.Services.AddSingleton<ProcessContext>();
|
||
host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Producer);
|
||
host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Consumer);
|
||
host.Services.AddTransient<TaskManager>();
|
||
|
||
host.Services.AddHostedService<MainHostedService>();
|
||
host.Services.AddHostedService<TaskMonitorService>();
|
||
host.Services.AddSingleton<IInputService, SimulationInputService>();
|
||
host.Services.AddSingleton<ITransformService, TransformService>();
|
||
host.Services.AddSingleton<IOutputService, OutputService>();
|
||
var redisOptions = host.Configuration.GetSection("RedisCacheOptions").Get<RedisCacheOptions>()??new RedisCacheOptions();
|
||
var redis = ConnectionMultiplexer.Connect(redisOptions.Configuration);
|
||
host.Services.AddSingleton(redis);
|
||
var app = host.Build();
|
||
await app.RunAsync();
|
||
} |