CZY/MES-ETL
1
0
Fork 0
Code Pull Requests Releases Activity

支持按多个表开立线程

Browse Source
This commit is contained in:
lindj
2024-01-18 14:36:36 +08:00
parent 629a4d2fb5
commit 97e359468f
16 changed files with 232 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files

127
ConsoleApp2/Program.cs
View File

@@ -60,51 +60,49 @@ async Task RunProgram()
var host = Host.CreateApplicationBuilder(args);
var commandOptions = host.Configuration.GetSection("CmdOptions").Get<CommandOptions>() ?? new CommandOptions();
Console.WriteLine($"InputDir:{commandOptions?.InputDir}");
Console.WriteLine($"OutPutFlushCount:{commandOptions?.FlushCount}");
Console.WriteLine($"OutPutTaskCount:{commandOptions?.TaskCount}");
var oldestTime = DateTime.ParseExact(commandOptions.OldestTime, "yyyyMM", System.Globalization.DateTimeFormatInfo.InvariantInfo);
host.Services.Configure<InputTableOptions>(option =>
{
option.TableInfoConfig = new Dictionary<string, TableInfo>
{
//host.Services.Configure<InputTableOptions>(option =>
//{
// option.TableInfoConfig = new Dictionary<string, TableInfo>
// {
//order_block_plan_item从order_item表查询，然后程序插入
//order_package_item从order_item表查询，然后程序插入
//order_patch_detail生产没有这个表，不处理
// //order_block_plan_item从order_item表查询，然后程序插入
// //order_package_item从order_item表查询，然后程序插入
// //order_patch_detail生产没有这个表，不处理
{"machine",new TableInfo{SimulaRowCount=14655 }},
{"order",new TableInfo{SimulaRowCount=5019216 }},
{"order_block_plan",new TableInfo{SimulaRowCount=2725553 }},//CreateTime < 202301的删除
{"order_block_plan_result",new TableInfo{SimulaRowCount=1174096 }},
{"order_box_block",new TableInfo{SimulaRowCount=29755672 }},
{"order_data_block",new TableInfo{SimulaRowCount=731800334 }},
{"order_data_goods",new TableInfo{SimulaRowCount=25803671 }},
{"order_data_parts",new TableInfo{SimulaRowCount=468517543 }},
{"order_item",new TableInfo{SimulaRowCount=1345520079 }},
{"order_module",new TableInfo{SimulaRowCount=103325385 }},
{"order_module_extra",new TableInfo{SimulaRowCount=54361321 }},
{"order_module_item",new TableInfo{SimulaRowCount=69173339 }},
{"order_package",new TableInfo{SimulaRowCount=16196195 }},
{"order_process",new TableInfo{SimulaRowCount=3892685 }},//orderNo < 202301的
{"order_process_step",new TableInfo{SimulaRowCount=8050349 }},//orderNo < 202301的删除
{"order_process_step_item",new TableInfo{SimulaRowCount=14538058 }},//orderNo < 202301的删除
{"order_scrap_board",new TableInfo{SimulaRowCount=123998 }},
{"process_group",new TableInfo{SimulaRowCount=1253 }},
{"process_info",new TableInfo{SimulaRowCount=7839 }},
{"process_item_exp",new TableInfo{SimulaRowCount=28 }},
{"process_schdule_capacity",new TableInfo{SimulaRowCount=39736 }},
{"process_step_efficiency",new TableInfo{SimulaRowCount=8 }},
{"report_template",new TableInfo{SimulaRowCount=7337 }},
{"simple_package",new TableInfo{SimulaRowCount=130436 }},//orderNo < 202301的删除
{"simple_plan_order",new TableInfo{SimulaRowCount=351470 }},//CreateTime < 202301的删除
{"sys_config",new TableInfo{SimulaRowCount=2296 }},
{"work_calendar",new TableInfo{SimulaRowCount=11 }},
{"work_shift",new TableInfo{SimulaRowCount=59 }},
{"work_time",new TableInfo{SimulaRowCount=62 }},
};
});
// {"machine",new TableInfo{SimulaRowCount=14655 }},
// {"order",new TableInfo{SimulaRowCount=5019216 }},
// {"order_block_plan",new TableInfo{SimulaRowCount=2725553 }},//CreateTime < 202301的删除
// {"order_block_plan_result",new TableInfo{SimulaRowCount=1174096 }},
// {"order_box_block",new TableInfo{SimulaRowCount=29755672 }},
// {"order_data_block",new TableInfo{SimulaRowCount=731800334 }},
// {"order_data_goods",new TableInfo{SimulaRowCount=25803671 }},
// {"order_data_parts",new TableInfo{SimulaRowCount=468517543 }},
// {"order_item",new TableInfo{SimulaRowCount=1345520079 }},
// {"order_module",new TableInfo{SimulaRowCount=103325385 }},
// {"order_module_extra",new TableInfo{SimulaRowCount=54361321 }},
// {"order_module_item",new TableInfo{SimulaRowCount=69173339 }},
// {"order_package",new TableInfo{SimulaRowCount=16196195 }},
// {"order_process",new TableInfo{SimulaRowCount=3892685 }},//orderNo < 202301的
// {"order_process_step",new TableInfo{SimulaRowCount=8050349 }},//orderNo < 202301的删除
// {"order_process_step_item",new TableInfo{SimulaRowCount=14538058 }},//orderNo < 202301的删除
// {"order_scrap_board",new TableInfo{SimulaRowCount=123998 }},
// {"process_group",new TableInfo{SimulaRowCount=1253 }},
// {"process_info",new TableInfo{SimulaRowCount=7839 }},
// {"process_item_exp",new TableInfo{SimulaRowCount=28 }},
// {"process_schdule_capacity",new TableInfo{SimulaRowCount=39736 }},
// {"process_step_efficiency",new TableInfo{SimulaRowCount=8 }},
// {"report_template",new TableInfo{SimulaRowCount=7337 }},
// {"simple_package",new TableInfo{SimulaRowCount=130436 }},//orderNo < 202301的删除
// {"simple_plan_order",new TableInfo{SimulaRowCount=351470 }},//CreateTime < 202301的删除
// {"sys_config",new TableInfo{SimulaRowCount=2296 }},
// {"work_calendar",new TableInfo{SimulaRowCount=11 }},
// {"work_shift",new TableInfo{SimulaRowCount=59 }},
// {"work_time",new TableInfo{SimulaRowCount=62 }},
// };
//});
host.Services.Configure<CsvOptions>(option =>
{
option.Delimiter = ",";
@@ -177,20 +175,20 @@ async Task RunProgram()
if (record.TableName == "order_process_step" || record.TableName == "order_process_step_item")
{
//如果缓存中不存在OrderProcessID,则丢弃
if(record.TryGetField("OrderProcessID",out var orderProcessID))
{
var value = await cache.GetStringAsync(orderProcessID);
//if(record.TryGetField("OrderProcessID",out var orderProcessID))
var value = await cache.GetStringAsync(record.GetCacheKey("OrderProcessID"));
if (string.IsNullOrEmpty(value))return false;
}
}
if (record.TableName == "order_block_plan_result" )
{
//如果缓存中不存在ID,则丢弃(ID 对应order_block_plan中的ID)
if (record.TryGetField("ID", out var id))
{
var value = await cache.GetStringAsync(id);
var value = await cache.GetStringAsync(record.GetCacheKey("ID"));
if (string.IsNullOrEmpty(value)) return false;
}
}
return true;
@@ -202,7 +200,7 @@ async Task RunProgram()
{
if (record.TryGetField("OrderNos", out var nos))
{
if (nos.Length <= 2) record.SetField("OrderNos", "\"[]\"");
if (nos.Length <= 2) record.SetField("OrderNos", "");
}
}
@@ -229,7 +227,7 @@ async Task RunProgram()
{
if (record.TryGetField("CompanyID", out var companyid))
{
await cache.SetStringAsync(orderNo, companyid);
await cache.SetStringAsync(record.GetCacheKey("OrderNo"), companyid);
}
}
@@ -243,7 +241,7 @@ async Task RunProgram()
if( record.TryGetField("ID", out var id))
{
await cache.SetStringAsync(id, sk);
await cache.SetStringAsync(record.GetCacheKey("ID"), sk);
}
}
@@ -253,7 +251,7 @@ async Task RunProgram()
if (record.TryGetField("CompanyID", out var companyid))
{
record.TryGetField("ID", out var id);
await cache.SetStringAsync(id, companyid);
await cache.SetStringAsync(record.GetCacheKey("ID"), companyid);
}
}
};
@@ -293,8 +291,9 @@ async Task RunProgram()
var headers = new List<string>(record.Headers);
var fields =new List<string>(record.Fields);
headers.Add("CompanyID");
var companyidResult =await cache.GetStringAsync(id);
_ = int.TryParse(companyidResult.ToString(), out var companyid);
var companyidResult =await cache.GetStringAsync(record.GetCacheKey("ID"));
_ = int.TryParse(companyidResult, out var companyid);
fields.Add(companyid.ToString());
return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), companyid);
}
@@ -308,7 +307,7 @@ async Task RunProgram()
var headers = new List<string>(record.Headers);
var fields = new List<string>(record.Fields);
headers.Add("CompanyID");
var companyidResult = await cache.GetStringAsync(orderNo);
var companyidResult = await cache.GetStringAsync(record.GetCacheKey("OrderNo"));
_ = int.TryParse(companyidResult, out var cpid);
fields.Add(cpid.ToString());
return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), cpid);
@@ -350,20 +349,12 @@ async Task RunProgram()
{
if (record.TryGetField("OrderProcessID",out var processID))
{
try
{
var shardKey =await cache.GetStringAsync(processID);
var shardKey =await cache.GetStringAsync(record.GetCacheKey("OrderProcessID"));
var headers = new List<string>(record.Headers);
var fields = new List<string>(record.Fields);
headers.Add("ShardKey");
fields.Add(shardKey);
fields.Add(shardKey??"0");
return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), record.CompanyID);
}
catch (Exception ex)
{
}
}
}
}
@@ -442,7 +433,7 @@ async Task RunProgram()
{ "process_item_exp.ItemJson", ColumnType.Text },
{ "report_template.Template", ColumnType.Text },
{ "report_template.SourceConfig", ColumnType.Text },
{ "order_block_plan.OrderNos", ColumnType.Text },
{ "order_block_plan.OrderNos", ColumnType.Json },
{ "order_block_plan.BlockInfo", ColumnType.Text },
};
});
@@ -465,8 +456,6 @@ async Task RunProgram()
TreatTinyAsBoolean = false,
MaximumPoolSize = 50
}.ConnectionString;
options.TaskCount = commandOptions.TaskCount;
options.FlushCount = commandOptions.FlushCount;
});
host.Services.AddLogging(builder =>
{
@@ -474,7 +463,7 @@ async Task RunProgram()
builder.AddSerilog(new LoggerConfiguration().WriteTo.Console().CreateLogger());
});
host.Services.AddSingleton<ProcessContext>();
host.Services.AddScoped<ProcessContext>();
host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Producer);
host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Consumer);
host.Services.AddTransient<TaskManager>();