CZY/MES-ETL
1
0
Fork 0
Code Pull Requests Releases Activity

修改

Browse Source
This commit is contained in:
lindj
2024-01-12 16:50:37 +08:00
parent eab3695f53
commit 0984853c79
28 changed files with 1115 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files

220
ConsoleApp2/Program.cs
View File

@@ -1,35 +1,227 @@
using ConsoleApp2.Const;
using ConsoleApp2;
using ConsoleApp2.Const;
using ConsoleApp2.Helpers;
using ConsoleApp2.HostedServices;
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
using ConsoleApp2.SimulationService;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using MySqlConnector;
using Serilog;
using Serilog.Core;
using System.Reflection.PortableExecutable;
// 运行之前把Mysql max_allowed_packets 调大
// 运行之前把process_step表的外键删掉
await RunProgram();
return;
async Task RunProgram()
{
//var inputDir= "D:\\MyDumper";
//ValidateConsole.ValidateInput<string>((_inputDir) =>
//{
// if (Directory.Exists(_inputDir))
// {
// inputDir = _inputDir;
// return true;
// }
// else return false;
//}, "请输入读取csv文件的目录(默认为当前目录下MyDumper文件夹):");
//var maxTask = 16;
//ValidateConsole.ValidateInput<string>((_inputDir) =>
//{
// _ = int.TryParse(_inputDir.ToString(), out var _taskCount);
// if (_taskCount > 0) {
// maxTask = _taskCount;
// return true;
// }
// else return false;
//}, "请输入执行输出的线程数量(默认为16):");
//var flushCount = 2_0000;
//ValidateConsole.ValidateInput<string>((_inputDir) =>
//{
// _ = int.TryParse(_inputDir.ToString(), out var _flashCount);
// if (_flashCount > 0)
// {
// flushCount = _flashCount;
// return true;
// } else return false;
//}, "请输入单次插入的行数(默认为20000):");
ThreadPool.SetMaxThreads(200, 200);
var host = Host.CreateApplicationBuilder();
host.Configuration.AddCommandLine(args);
var host = Host.CreateApplicationBuilder(args);
var commandOptions = host.Configuration.GetSection("CmdOptions").Get<CommandOptions>() ?? new CommandOptions();
Console.WriteLine($"InputDir:{commandOptions?.InputDir}");
Console.WriteLine($"OutPutFlushCount:{commandOptions?.FlushCount}");
Console.WriteLine($"OutPutTaskCount:{commandOptions?.TaskCount}");
host.Services.Configure<InputTableOptions>(option =>
{
option.TableInfoConfig = new Dictionary<string, TableInfo>
{
//{"order_block_plan_item",new TableInfo{SimulaRowCount=136323566 }},//从order_item表查询，然后程序插入
//{"order_package_item",new TableInfo{SimulaRowCount=52525224 }},//从order_item表查询，然后程序插入
//{"order_patch_detail",new TableInfo{SimulaRowCount=10 }},//生产没有这个表，不处理
//{"machine",new TableInfo{SimulaRowCount=14655 }},
//{"order",new TableInfo{SimulaRowCount=5019216 }},
//{"order_block_plan",new TableInfo{SimulaRowCount=2725553 }},//CreateTime < 202301的删除
{"order_block_plan_result",new TableInfo{SimulaRowCount=1174096 }},
//{"order_box_block",new TableInfo{SimulaRowCount=29755672 }},
//{"order_data_block",new TableInfo{SimulaRowCount=731800334 }},
//{"order_data_goods",new TableInfo{SimulaRowCount=25803671 }},
//{"order_data_parts",new TableInfo{SimulaRowCount=468517543 }},
//{"order_item",new TableInfo{SimulaRowCount=1345520079 }},
//{"order_module",new TableInfo{SimulaRowCount=103325385 }},
//{"order_module_extra",new TableInfo{SimulaRowCount=54361321 }},
//{"order_module_item",new TableInfo{SimulaRowCount=69173339 }},
//{"order_package",new TableInfo{SimulaRowCount=16196195 }},
//{"order_process",new TableInfo{SimulaRowCount=3892685 }},//orderNo < 202301的
//{"order_process_step",new TableInfo{SimulaRowCount=8050349 }},//orderNo < 202301的删除
//{"order_process_step_item",new TableInfo{SimulaRowCount=14538058 }},//orderNo < 202301的删除
//{"order_scrap_board",new TableInfo{SimulaRowCount=123998 }},
//{"process_group",new TableInfo{SimulaRowCount=1253 }},
//{"process_info",new TableInfo{SimulaRowCount=7839 }},
//{"process_item_exp",new TableInfo{SimulaRowCount=28 }},
//{"process_schdule_capacity",new TableInfo{SimulaRowCount=39736 }},
//{"process_step_efficiency",new TableInfo{SimulaRowCount=8 }},
//{"report_template",new TableInfo{SimulaRowCount=7337 }},
//{"simple_package",new TableInfo{SimulaRowCount=130436 }},//orderNo < 202301的删除
//{"simple_plan_order",new TableInfo{SimulaRowCount=351470 }},//CreateTime < 202301的删除
//{"sys_config",new TableInfo{SimulaRowCount=2296 }},
//{"work_calendar",new TableInfo{SimulaRowCount=11 }},
//{"work_shift",new TableInfo{SimulaRowCount=59 }},
//{"work_time",new TableInfo{SimulaRowCount=62 }},
};
});
host.Services.Configure<CsvOptions>(option =>
{
option.Delimiter = ",";
option.QuoteChar = '"';
option.InputDir = "D:/Dump";
});
host.Services.Configure<DataInputOptions>(options =>
{
options.InputDir = commandOptions.InputDir;
var _csvOptions = new CsvOptions { Delimiter = ",", QuoteChar = '"' };
options.CreateSource = (string tableName) =>
{
var source = new ZstSource(commandOptions.InputDir, tableName, _csvOptions.Delimiter, _csvOptions.QuoteChar);
return source;
};
});
host.Services.Configure<DataTransformOptions>(options =>
{
options.DatabaseFilter = record => "cferp_test_1";
options.DatabaseFilter = record => "cferp_test";
options.TransformBinary = field => commandOptions != null && commandOptions.Isutf8mb4 ? $"_utf8mb4 0x{field}" : $"0x{field}";
//数据过滤
options.RecordFilter = record =>
{
var index = Array.IndexOf(record.Headers, "ShardKey");
if (index > -1)
{
var skString = record.Fields[index];
short.TryParse(skString, out var sk);
if (sk < commandOptions.OldestShardKey) return false;
}
if (record.TableName == "order_package")
{
var pkNoIndex = Array.IndexOf(record.Headers, "PakageNo");
if (pkNoIndex > -1)
{
var pkNo = record.Fields[pkNoIndex];
if (pkNo.Length <= 2) return false;
}
}
if (record.TableName == "order_block_plan")
{
var orderNosIndex = Array.IndexOf(record.Headers, "OrderNos");
if (orderNosIndex > -1)
{
var pkNo = record.Fields[orderNosIndex];
if (pkNo.Length <= 2) return false;
}
}
return true;
};
//数据修改
options.RecordModify = (record) =>
{
if (record.TableName == "order_process")//修改order_process.NextStepID的默认值为0
{
var nextStepIdIndex = Array.IndexOf(record.Headers, "NextStepID");
if (nextStepIdIndex > -1)
{
var idString = record.Fields[nextStepIdIndex];
if (idString == "\\N")
{
record.Fields[nextStepIdIndex] = "0";
}
}
}
};
//数据替换
options.RecordReplace = (record) =>
{
//删除数据源里simple_plan_order.ProcessState 字段和值
if (record.TableName == "simple_plan_order")//修改order_process.NextStepID的默认值为0
{
var nextStepIdIndex = Array.IndexOf(record.Headers, "ProcessState");
if (nextStepIdIndex > -1)
{
var headers = record.Headers.Where(t => t != "ProcessState").ToArray();
var fs = record.Fields.ToList();
fs.RemoveAt(nextStepIdIndex);
var fields = fs.ToArray();
return new DataRecord(fields, record.TableName, headers, record.CompanyID);
}
}
return null;
};
//数据生成
options.RecordAdd = (record) =>
{
var resultList = new List<DataRecord>();
if(record.TableName == "order_item")
{
var itemIDIndex = Array.IndexOf(record.Headers, "ItemID");
var shardKeyIndex = Array.IndexOf(record.Headers, "ShardKey");
var planIDIndex = Array.IndexOf(record.Headers, "PlanID");
var packageIDIndex = Array.IndexOf(record.Headers, "PackageID");
var companyIDIndex = Array.IndexOf(record.Headers, "CompanyID");
//resultList.Add(new DataRecord(
// new[] { "ItemID", "ShardKey", "PlanID","CompanyID" }, "order_block_plan_item",
// new[] { record.Fields[itemIDIndex], record.Fields[shardKeyIndex], record.Fields[planIDIndex], record.Fields[companyIDIndex] }));
//resultList.Add(
// new DataRecord(new[] { "ItemID", "ShardKey", "PackageID", "CompanyID" }, "order_package_item",
// new[] { record.Fields[itemIDIndex], record.Fields[shardKeyIndex], record.Fields[packageIDIndex], record.Fields[companyIDIndex] }));
}
return resultList;
};
options.ColumnTypeConfig = new()
{
{ "simple_plan_order.PlaceData", ColumnType.Blob },
@@ -61,20 +253,19 @@ async Task RunProgram()
{ "order_block_plan.BlockInfo", ColumnType.Text },
};
});
host.Services.Configure<DatabaseOutputOptions>(options =>
{
options.ConnectionString = new MySqlConnectionStringBuilder
{
Server = "127.0.0.1",
Port = 33306,
Database = "cferp_test_1",
Port = 33309,
Database = "cferp_test",
UserID = "root",
Password = "123456",
MaximumPoolSize = 50, // 这个值应当小于 max_connections
}.ConnectionString;
options.MaxTask = 16;
options.FlushCount = 200;
options.TaskCount = commandOptions.TaskCount;
options.FlushCount = commandOptions.FlushCount;
});
host.Services.AddLogging(builder =>
{
@@ -89,10 +280,13 @@ async Task RunProgram()
host.Services.AddHostedService<MainHostedService>();
host.Services.AddHostedService<TaskMonitorService>();
host.Services.AddSingleton<IInputService, InputService>();
host.Services.AddSingleton<IInputService, SimulationInputService>();
host.Services.AddSingleton<ITransformService, TransformService>();
host.Services.AddSingleton<IOutputService, OutputService>();
host.Services.AddStackExchangeRedisCache(options =>
{
options.Configuration = "localhost:6379";
});
var app = host.Build();
await app.RunAsync();
}