CZY/MES-ETL
1
0
Fork 0
Code Pull Requests Releases Activity

修复Linux环境下appsettings.json配置文件可能因控制台工作目录不同而无法读取的问题;

Browse Source 
修复数据替换时order_module表的ShardKey无法正确赋值的问题;
旧数据库order_data_block.CompanyID数据有误，在数据替换时重新计算;
This commit is contained in:
陈梓阳 2024-01-22 15:49:32 +08:00
parent 8da3110ecd
commit 8e5efa83f1
3 changed files with 34 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
ConsoleApp2/DataRecord.cs
View File

@ -57,7 +57,11 @@ public class DataRecord
set => Fields[index] = value; set => Fields[index] = value;
} }
public string this[string columnName] => GetField(this, columnName); public string this[string columnName]
{
get => GetField(this, columnName);
set => SetField(columnName, value);
}
public int Count => Fields.Length; public int Count => Fields.Length;

6
ConsoleApp2/HostedServices/MainHostedService.cs
View File

@ -26,7 +26,7 @@ public class MainHostedService : BackgroundService
{ {
var taskFun = (TasksOptions taskOp, DataRecordQueue producerQueue, DataRecordQueue consumerQueue, ProcessContext context) => var taskFun = (TasksOptions taskOp, DataRecordQueue producerQueue, DataRecordQueue consumerQueue, ProcessContext context) =>
{ {
var inputTask = Task.Factory.StartNew(async () => var inputTask = Task.Run(async () =>
{ {
try try
{ {
@ -39,7 +39,7 @@ public class MainHostedService : BackgroundService
} }
}); });
var transformTask = Task.Factory.StartNew(async () => var transformTask = Task.Run(async () =>
{ {
try try
{ {
@ -52,7 +52,7 @@ public class MainHostedService : BackgroundService
} }
}); });
var outputTask = Task.Factory.StartNew(async () => var outputTask = Task.Run(async () =>
{ {
try try
{ {

42
ConsoleApp2/Program.cs
View File

@ -59,7 +59,9 @@ async Task RunProgram()
ThreadPool.SetMaxThreads(200, 200); ThreadPool.SetMaxThreads(200, 200);
var host = Host.CreateApplicationBuilder(args); var host = Host.CreateApplicationBuilder(args);
var commandOptions = host.Configuration.GetSection("CmdOptions").Get<CommandOptions>() ?? new CommandOptions(); host.Configuration.AddJsonFile(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "appsettings.json"), false, false);
var commandOptions = host.Configuration.GetSection("CmdOptions").Get<CommandOptions>() ??
throw new ApplicationException("未发现配置文件");
Console.WriteLine($"InputDir:{commandOptions?.InputDir}"); Console.WriteLine($"InputDir:{commandOptions?.InputDir}");
var oldestTime = DateTime.ParseExact(commandOptions.OldestTime, "yyyyMM", System.Globalization.DateTimeFormatInfo.InvariantInfo); var oldestTime = DateTime.ParseExact(commandOptions.OldestTime, "yyyyMM", System.Globalization.DateTimeFormatInfo.InvariantInfo);
@ -332,7 +334,20 @@ async Task RunProgram()
fields.RemoveAt(index); fields.RemoveAt(index);
return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), record.CompanyID); return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), record.CompanyID);
} }
if (!record.TryGetField("ShardKey", out var skStr))
{
if (record.TryGetField("OrderNo", out var orderNo))
{
var yearMonth = orderNo.Substring(2, 4);
var sk = yearMonth + "0";
var headers = new List<string>(record.Headers);
var fields = new List<string>(record.Fields);
headers.Add("ShardKey");
fields.Add(sk);
return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), record.CompanyID);
}
}
} }
if (record.TableName == "order_process") if (record.TableName == "order_process")
{ {
@ -365,21 +380,16 @@ async Task RunProgram()
} }
} }
} }
if(record.TableName == "order_moudle")
// 旧数据库order_data_block.CompanyID数据有误，重新计算
if (record.TableName == "order_data_block")
{ {
if (!record.TryGetField("ShardKey", out var skStr)) var companyID = await cache.GetStringAsync($"order_{record["OrderNo"]}");
{ if (string.IsNullOrEmpty(companyID))
if (record.TryGetField("OrderNo", out var orderNo)) // throw new InvalidDataException($"没有获取到OrderNo={record["OrderNo"]}的CompanyID");
{ return record;
var yearMonth = orderNo.Substring(2, 4); record["CompanyID"] = companyID;
var sk = yearMonth + "0"; return record;
var headers = new List<string>(record.Headers);
var fields = new List<string>(record.Fields);
headers.Add("ShardKey");
fields.Add(sk);
return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), record.CompanyID);
}
}
} }
return null; return null;
}; };