CZY/MES-ETL
1
0
Fork 0
Code Pull Requests Releases Activity

整理代码

Browse Source
This commit is contained in:
lindj
2024-01-16 18:00:23 +08:00
parent 78cd833617
commit a74cf5ddb7
7 changed files with 344 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files

426
ConsoleApp2/Program.cs
View File

@@ -69,46 +69,46 @@ async Task RunProgram()
Console.WriteLine($"OutPutFlushCount:{commandOptions?.FlushCount}");
Console.WriteLine($"OutPutTaskCount:{commandOptions?.TaskCount}");
var oldestTime = DateTime.ParseExact(commandOptions.OldestTime, "yyyyMM", System.Globalization.DateTimeFormatInfo.InvariantInfo);
host.Services.Configure<InputTableOptions>(option =>
{
option.TableInfoConfig = new Dictionary<string, TableInfo>
{
//{"order_block_plan_item",new TableInfo{SimulaRowCount=136323566 }},//从order_item表查询，然后程序插入
//{"order_package_item",new TableInfo{SimulaRowCount=52525224 }},//从order_item表查询，然后程序插入
//{"order_patch_detail",new TableInfo{SimulaRowCount=10 }},//生产没有这个表，不处理
//order_block_plan_item从order_item表查询，然后程序插入
//order_package_item从order_item表查询，然后程序插入
//order_patch_detail生产没有这个表，不处理
//{"machine",new TableInfo{SimulaRowCount=14655 }},
//{"order",new TableInfo{SimulaRowCount=5019216 }},
//{"order_block_plan",new TableInfo{SimulaRowCount=2725553 }},//CreateTime < 202301的删除
{"machine",new TableInfo{SimulaRowCount=14655 }},
{"order",new TableInfo{SimulaRowCount=5019216 }},
{"order_block_plan",new TableInfo{SimulaRowCount=2725553 }},//CreateTime < 202301的删除
{"order_block_plan_result",new TableInfo{SimulaRowCount=1174096 }},
//{"order_box_block",new TableInfo{SimulaRowCount=29755672 }},
//{"order_data_block",new TableInfo{SimulaRowCount=731800334 }},
//{"order_data_goods",new TableInfo{SimulaRowCount=25803671 }},
//{"order_data_parts",new TableInfo{SimulaRowCount=468517543 }},
//{"order_item",new TableInfo{SimulaRowCount=1345520079 }},
//{"order_module",new TableInfo{SimulaRowCount=103325385 }},
//{"order_module_extra",new TableInfo{SimulaRowCount=54361321 }},
//{"order_module_item",new TableInfo{SimulaRowCount=69173339 }},
//{"order_package",new TableInfo{SimulaRowCount=16196195 }},
//{"order_process",new TableInfo{SimulaRowCount=3892685 }},//orderNo < 202301的
//{"order_process_step",new TableInfo{SimulaRowCount=8050349 }},//orderNo < 202301的删除
//{"order_process_step_item",new TableInfo{SimulaRowCount=14538058 }},//orderNo < 202301的删除
//{"order_scrap_board",new TableInfo{SimulaRowCount=123998 }},
//{"process_group",new TableInfo{SimulaRowCount=1253 }},
//{"process_info",new TableInfo{SimulaRowCount=7839 }},
//{"process_item_exp",new TableInfo{SimulaRowCount=28 }},
//{"process_schdule_capacity",new TableInfo{SimulaRowCount=39736 }},
//{"process_step_efficiency",new TableInfo{SimulaRowCount=8 }},
//{"report_template",new TableInfo{SimulaRowCount=7337 }},
//{"simple_package",new TableInfo{SimulaRowCount=130436 }},//orderNo < 202301的删除
//{"simple_plan_order",new TableInfo{SimulaRowCount=351470 }},//CreateTime < 202301的删除
//{"sys_config",new TableInfo{SimulaRowCount=2296 }},
//{"work_calendar",new TableInfo{SimulaRowCount=11 }},
//{"work_shift",new TableInfo{SimulaRowCount=59 }},
//{"work_time",new TableInfo{SimulaRowCount=62 }},
{"order_box_block",new TableInfo{SimulaRowCount=29755672 }},
{"order_data_block",new TableInfo{SimulaRowCount=731800334 }},
{"order_data_goods",new TableInfo{SimulaRowCount=25803671 }},
{"order_data_parts",new TableInfo{SimulaRowCount=468517543 }},
{"order_item",new TableInfo{SimulaRowCount=1345520079 }},
{"order_module",new TableInfo{SimulaRowCount=103325385 }},
{"order_module_extra",new TableInfo{SimulaRowCount=54361321 }},
{"order_module_item",new TableInfo{SimulaRowCount=69173339 }},
{"order_package",new TableInfo{SimulaRowCount=16196195 }},
{"order_process",new TableInfo{SimulaRowCount=3892685 }},//orderNo < 202301的
{"order_process_step",new TableInfo{SimulaRowCount=8050349 }},//orderNo < 202301的删除
{"order_process_step_item",new TableInfo{SimulaRowCount=14538058 }},//orderNo < 202301的删除
{"order_scrap_board",new TableInfo{SimulaRowCount=123998 }},
{"process_group",new TableInfo{SimulaRowCount=1253 }},
{"process_info",new TableInfo{SimulaRowCount=7839 }},
{"process_item_exp",new TableInfo{SimulaRowCount=28 }},
{"process_schdule_capacity",new TableInfo{SimulaRowCount=39736 }},
{"process_step_efficiency",new TableInfo{SimulaRowCount=8 }},
{"report_template",new TableInfo{SimulaRowCount=7337 }},
{"simple_package",new TableInfo{SimulaRowCount=130436 }},//orderNo < 202301的删除
{"simple_plan_order",new TableInfo{SimulaRowCount=351470 }},//CreateTime < 202301的删除
{"sys_config",new TableInfo{SimulaRowCount=2296 }},
{"work_calendar",new TableInfo{SimulaRowCount=11 }},
{"work_shift",new TableInfo{SimulaRowCount=59 }},
{"work_time",new TableInfo{SimulaRowCount=62 }},
};
});
host.Services.Configure<CsvOptions>(option =>
@@ -129,119 +129,295 @@ async Task RunProgram()
host.Services.Configure<DataTransformOptions>(options =>
{
options.DatabaseFilter = record => "cferp_test";
options.DatabaseFilter = record => "cferp_test";
options.TransformBinary = field => commandOptions != null && commandOptions.Isutf8mb4 ? $"_utf8mb4 0x{field}" : $"0x{field}";
//数据过滤
options.RecordFilter = record =>
{
var index = Array.IndexOf(record.Headers, "ShardKey");
if (index > -1)
options.TransformBinary = field => commandOptions != null && commandOptions.Isutf8mb4 ? $"_utf8mb4 0x{field}" : $"0x{field}";
//数据过滤
options.RecordFilter = async (record, db) =>
{
var skString = record.Fields[index];
short.TryParse(skString, out var sk);
if (sk < commandOptions.OldestShardKey) return false;
}
if (record.TableName == "order_package")
{
var pkNoIndex = Array.IndexOf(record.Headers, "PakageNo");
if (pkNoIndex > -1)
//var index = Array.IndexOf(record.Headers, "ShardKey");
if (record.TryGetField("ShardKey", out var skStr))
{
var pkNo = record.Fields[pkNoIndex];
if (pkNo.Length <= 2) return false;
short.TryParse(skStr, out var sk);
if (sk < commandOptions.OldestShardKey) return false;
}
}
if (record.TableName == "order_block_plan")
{
var orderNosIndex = Array.IndexOf(record.Headers, "OrderNos");
if (orderNosIndex > -1)
if (record.TryGetField("CreateTime", out var createTime))
{
var pkNo = record.Fields[orderNosIndex];
if (pkNo.Length <= 2) return false;
_ = DateTime.TryParse(createTime.Replace("\"", ""), out var time);
if (time < oldestTime) return false;
}
}
return true;
};
//数据修改
options.RecordModify = (record) =>
{
if (record.TableName == "order_process")//修改order_process.NextStepID的默认值为0
{
var nextStepIdIndex = Array.IndexOf(record.Headers, "NextStepID");
if (nextStepIdIndex > -1)
if (record.TryGetField("OrderNo", out var orderNo))
{
var idString = record.Fields[nextStepIdIndex];
if (idString == "\\N")
try
{
record.Fields[nextStepIdIndex] = "0";
var yearMonth = orderNo.Substring(0, 6);
var dt = DateTime.ParseExact(yearMonth, "yyyyMM", System.Globalization.DateTimeFormatInfo.InvariantInfo);
if (dt < oldestTime) return false;
}
catch (Exception ex)
{
return false;//订单号转换失败，跳过
}
}
}
};
//数据替换
options.RecordReplace = (record) =>
{
//删除数据源里simple_plan_order.ProcessState 字段和值
if (record.TableName == "simple_plan_order")//修改order_process.NextStepID的默认值为0
{
var nextStepIdIndex = Array.IndexOf(record.Headers, "ProcessState");
if (nextStepIdIndex > -1)
if (record.TableName == "order_package")
{
var headers = record.Headers.Where(t => t != "ProcessState").ToArray();
var fs = record.Fields.ToList();
fs.RemoveAt(nextStepIdIndex);
var fields = fs.ToArray();
return new DataRecord(fields, record.TableName, headers, record.CompanyID);
if (record.TryGetField("PakageNo", out var pkNo))
{
if (pkNo.Length <= 2) return false;
}
}
}
if (record.TableName == "order_process_step")
{
if (record.TableName == "order_block_plan")
{
if (record.TryGetField("OrderNos", out var nos))
{
if (nos.Length <= 2) return false;
}
}
if (record.TableName == "order_process_step" || record.TableName == "order_process_step_item")
{
//如果缓存中不存在OrderProcessID,则丢弃
if(record.TryGetField("OrderProcessID",out var orderProcessID))
{
var value = await db.StringGetAsync(orderProcessID);
if (string.IsNullOrEmpty(value.ToString()))return false;
}
}
if (record.TableName == "order_block_plan_result" )
{
//如果缓存中不存在ID,则丢弃(ID 对应order_block_plan中的ID)
if (record.TryGetField("ID", out var id))
{
var value = await db.StringGetAsync(id);
if (string.IsNullOrEmpty(value.ToString())) return false;
}
}
return true;
}
return null;
};
//数据生成
options.RecordAdd = (record) =>
{
var resultList = new List<DataRecord>();
if(record.TableName == "order_item")
};
//数据修改
options.RecordModify = (record) =>
{
var itemIDIndex = Array.IndexOf(record.Headers, "ItemID");
var shardKeyIndex = Array.IndexOf(record.Headers, "ShardKey");
var planIDIndex = Array.IndexOf(record.Headers, "PlanID");
var packageIDIndex = Array.IndexOf(record.Headers, "PackageID");
var companyIDIndex = Array.IndexOf(record.Headers, "CompanyID");
if (record.TableName == "order_process")//修改order_process.NextStepID的默认值为0
{
if (record.TryGetField("NextStepID", out var idStr))
{
resultList.Add(new DataRecord(
new[] { "ItemID", "ShardKey", "PlanID", "CompanyID" }, "order_block_plan_item",
new[] { record.Fields[itemIDIndex], record.Fields[shardKeyIndex], record.Fields[planIDIndex], record.Fields[companyIDIndex] }));
resultList.Add(
new DataRecord(new[] { "ItemID", "ShardKey", "PackageID", "CompanyID" }, "order_package_item",
new[] { record.Fields[itemIDIndex], record.Fields[shardKeyIndex], record.Fields[packageIDIndex], record.Fields[companyIDIndex] }));
}
return resultList;
};
if (idStr == "NULL")
{
record.SetField("NextStepID", "0");
}
}
}
};
//数据缓存
options.RecordCache = async (record, db) =>
{
if(record.TableName == "order_process")
if (record.TableName == "order")
{
var skIndex = Array.IndexOf(record.Headers, "ShardKey");
if(skIndex > -1)
if (record.TryGetField("OrderNo", out var orderNo))
{
var sk = record.Fields[skIndex];
var idIndex = Array.IndexOf(record.Headers, "ID");
var id = record.Fields[idIndex];
await db.SetAddAsync(id, sk);
if (record.TryGetField("CompanyID", out var companyid))
{
await db.StringSetAsync(orderNo, companyid);
}
}
}
if (record.TableName == "order_process")
{
if (record.TryGetField("OrderNo", out var orderNo))
{
var yearMonth = orderNo.Substring(2, 4);
var sk = yearMonth + "0";
if( record.TryGetField("ID", out var id))
{
try
{
await db.StringSetAsync(id, sk);
}
catch (Exception ex)
{
}
}
}
}
if (record.TableName == "order_block_plan")
{
if (record.TryGetField("CompanyID", out var companyid))
{
record.TryGetField("ID", out var id);
await db.StringSetAsync(id, companyid);
}
}
};
//数据替换
options.RecordReplace = async (record, db) =>
{
//删除数据源里simple_plan_order.ProcessState 字段和值
if (record.TableName == "simple_plan_order")//修改order_process.NextStepID的默认值为0
{
var nextStepIdIndex = Array.IndexOf(record.Headers, "ProcessState");
if (nextStepIdIndex > -1)
{
var headers = record.Headers.Where(t => t != "ProcessState").ToArray();
var fs = record.Fields.ToList();
fs.RemoveAt(nextStepIdIndex);
var fields = fs.ToArray();
return new DataRecord(fields, record.TableName, headers, record.CompanyID);
}
}
if (record.TableName == "order")//修改order_process.NextStepID的默认值为0
{
var nextStepIdIndex = Array.IndexOf(record.Headers, "IsBatch");
if (nextStepIdIndex > -1)
{
var headers = record.Headers.Where(t => t != "IsBatch").ToArray();
var fs = record.Fields.ToList();
fs.RemoveAt(nextStepIdIndex);
var fields = fs.ToArray();
return new DataRecord(fields, record.TableName, headers, record.CompanyID);
}
}
if (record.TableName == "order_block_plan_result")//修改order_process.NextStepID的默认值为0
{
if (record.TryGetField("ID", out var id))
{
var headers = new List<string>(record.Headers);
var fields =new List<string>(record.Fields);
headers.Add("CompanyID");
var companyidResult =await db.StringGetAsync(id);
_ = int.TryParse(companyidResult.ToString(), out var companyid);
fields.Add(companyid.ToString());
return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), companyid);
}
}
if(record.TableName == "order_box_block")
{
if (!record.TryGetField("CompanyID", out var companyid))
{
if (record.TryGetField("OrderNo", out var orderNo))
{
var headers = new List<string>(record.Headers);
var fields = new List<string>(record.Fields);
headers.Add("CompanyID");
var companyidResult = await db.StringGetAsync(orderNo);
_ = int.TryParse(companyidResult.ToString(), out var cpid);
fields.Add(cpid.ToString());
return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), cpid);
}
}
}
if (record.TableName == "order_module")
{
if (record.TryGetField("ViewFileName",out var value))
{
var index=Array.IndexOf(record.Headers, "ViewFileName");
var headers = new List<string>(record.Headers);
headers.RemoveAt(index);
var fields = new List<string>(record.Fields);
fields.RemoveAt(index);
return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), record.CompanyID);
}
}
if (record.TableName == "order_process")
{
if (!record.TryGetField("ShardKey", out var skStr))
{
if(record.TryGetField("OrderNo", out var orderNo))
{
var yearMonth = orderNo.Substring(2, 4);
var sk = yearMonth + "0";
var headers = new List<string>(record.Headers);
var fields = new List<string>(record.Fields);
headers.Add("ShardKey");
fields.Add(sk);
return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), record.CompanyID);
}
}
}
if(record.TableName == "order_process_step"|| record.TableName == "order_process_step_item")
{
if (!record.TryGetField("ShardKey",out var sk))
{
if (record.TryGetField("OrderProcessID",out var processID))
{
try
{
var shardKey =await db.StringGetAsync(processID);
var headers = new List<string>(record.Headers);
var fields = new List<string>(record.Fields);
headers.Add("ShardKey");
fields.Add(shardKey.ToString());
return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), record.CompanyID);
}
catch (Exception ex)
{
}
}
}
}
if(record.TableName == "order_moudle")
{
if (!record.TryGetField("ShardKey", out var skStr))
{
if (record.TryGetField("OrderNo", out var orderNo))
{
var yearMonth = orderNo.Substring(2, 4);
var sk = yearMonth + "0";
var headers = new List<string>(record.Headers);
var fields = new List<string>(record.Fields);
headers.Add("ShardKey");
fields.Add(sk);
return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), record.CompanyID);
}
}
}
return null;
};
//数据生成
options.RecordAdd = (record) =>
{
var resultList = new List<DataRecord>();
if (record.TableName == "order_item")
{
record.TryGetField("ID", out var itemID);
record.TryGetField("ShardKey", out var shardKey);
record.TryGetField("PlanID", out var planID);
record.TryGetField("PackageID", out var packageID);
record.TryGetField("CompanyID", out var companyID);
_=int.TryParse(planID, out var pid);
if (pid > 0)
{
resultList.Add(new DataRecord(new[] { itemID, shardKey, planID, companyID },
"order_block_plan_item",
new[] { "ItemID", "ShardKey", "PlanID", "CompanyID" }));
}
_ = int.TryParse(packageID, out var pkid);
if(pkid > 0)
{
resultList.Add(new DataRecord(new[] { itemID, shardKey, packageID, companyID },
"order_package_item",
new[] { "ItemID", "ShardKey", "PackageID", "CompanyID" }
));
}
}
return resultList;
};
options.ColumnTypeConfig = new()
@@ -302,10 +478,10 @@ async Task RunProgram()
host.Services.AddHostedService<MainHostedService>();
host.Services.AddHostedService<TaskMonitorService>();
host.Services.AddSingleton<IInputService, SimulationInputService>();
host.Services.AddSingleton<IInputService, InputService>();
host.Services.AddSingleton<ITransformService, TransformService>();
host.Services.AddSingleton<IOutputService, OutputService>();
var redisOptions = host.Configuration.GetSection("RedisCacheOptions").Get<RedisCacheOptions>()??new RedisCacheOptions();
var redisOptions = host.Configuration.GetSection("RedisCacheOptions").Get<RedisCacheOptions>() ?? new RedisCacheOptions();
var redis = ConnectionMultiplexer.Connect(redisOptions.Configuration);
host.Services.AddSingleton(redis);
var app = host.Build();