CZY/MES-ETL
1
0
Fork 0
Code Pull Requests Releases Activity

修复获取cachekey错误问题

Browse Source 
output使用ThreadPool.QueueUserWorkItem
This commit is contained in:
lindj 2024-01-19 15:14:01 +08:00
parent e3f6ecbd91
commit f167256082
5 changed files with 45 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
ConsoleApp2/HostedServices/MainHostedService.cs
View File

@ -77,33 +77,33 @@ public class MainHostedService : BackgroundService
}; };
var bigTableContext = new ProcessContext(); var bigTableContext = new ProcessContext();
var bigTableOptions = new TasksOptions { TableInfoConfig = bigTablesDic, OutPutOptions = new OutPutOptions { FlushCount = 10000, OutPutTaskCount = 2 } }; var bigTableOptions = new TasksOptions { TableInfoConfig = bigTablesDic, OutPutOptions = new OutPutOptions { FlushCount = 10000, OutPutTaskCount = 2 } };
//taskFun(bigTableOptions,new DataRecordQueue(), new DataRecordQueue(), bigTableContext); taskFun(bigTableOptions,new DataRecordQueue(), new DataRecordQueue(), bigTableContext);
var smallTablesDic = new Dictionary<string, TableInfo> var smallTablesDic = new Dictionary<string, TableInfo>
{ {
//{"machine",new TableInfo{SimulaRowCount=14655 }}, {"machine",new TableInfo{SimulaRowCount=14655 }},
//{"order",new TableInfo{SimulaRowCount=5019216 }}, {"order",new TableInfo{SimulaRowCount=5019216 }},
//{"order_data_block",new TableInfo{SimulaRowCount=731800334 }}, {"order_data_block",new TableInfo{SimulaRowCount=731800334 }},
//{"order_data_goods",new TableInfo{SimulaRowCount=25803671 }}, {"order_data_goods",new TableInfo{SimulaRowCount=25803671 }},
//{"order_data_parts",new TableInfo{SimulaRowCount=468517543 }}, {"order_data_parts",new TableInfo{SimulaRowCount=468517543 }},
//{"order_module",new TableInfo{SimulaRowCount=103325385 }}, {"order_module",new TableInfo{SimulaRowCount=103325385 }},
//{"order_module_extra",new TableInfo{SimulaRowCount=54361321 }}, {"order_module_extra",new TableInfo{SimulaRowCount=54361321 }},
//{"order_module_item",new TableInfo{SimulaRowCount=69173339 }}, {"order_module_item",new TableInfo{SimulaRowCount=69173339 }},
//{"order_package",new TableInfo{SimulaRowCount=16196195 }}, {"order_package",new TableInfo{SimulaRowCount=16196195 }},
{"order_process",new TableInfo{SimulaRowCount=3892685 }},//orderNo < 202301的 {"order_process",new TableInfo{SimulaRowCount=3892685 }},//orderNo < 202301的
{"order_process_step",new TableInfo{SimulaRowCount=8050349 }},//orderNo < 202301的删除 {"order_process_step",new TableInfo{SimulaRowCount=8050349 }},//orderNo < 202301的删除
{"order_process_step_item",new TableInfo{SimulaRowCount=14538058 }},//orderNo < 202301的删除 {"order_process_step_item",new TableInfo{SimulaRowCount=14538058 }},//orderNo < 202301的删除
//{"order_scrap_board",new TableInfo{SimulaRowCount=123998 }}, {"order_scrap_board",new TableInfo{SimulaRowCount=123998 }},
//{"process_group",new TableInfo{SimulaRowCount=1253 }}, {"process_group",new TableInfo{SimulaRowCount=1253 }},
//{"process_info",new TableInfo{SimulaRowCount=7839 }}, {"process_info",new TableInfo{SimulaRowCount=7839 }},
//{"process_item_exp",new TableInfo{SimulaRowCount=28 }}, {"process_item_exp",new TableInfo{SimulaRowCount=28 }},
//{"process_schdule_capacity",new TableInfo{SimulaRowCount=39736 }}, {"process_schdule_capacity",new TableInfo{SimulaRowCount=39736 }},
//{"process_step_efficiency",new TableInfo{SimulaRowCount=8 }}, {"process_step_efficiency",new TableInfo{SimulaRowCount=8 }},
//{"report_template",new TableInfo{SimulaRowCount=7337 }}, {"report_template",new TableInfo{SimulaRowCount=7337 }},
//{"simple_package",new TableInfo{SimulaRowCount=130436 }},//orderNo < 202301的删除 {"simple_package",new TableInfo{SimulaRowCount=130436 }},//orderNo < 202301的删除
//{"sys_config",new TableInfo{SimulaRowCount=2296 }}, {"sys_config",new TableInfo{SimulaRowCount=2296 }},
//{"work_calendar",new TableInfo{SimulaRowCount=11 }}, {"work_calendar",new TableInfo{SimulaRowCount=11 }},
//{"work_shift",new TableInfo{SimulaRowCount=59 }}, {"work_shift",new TableInfo{SimulaRowCount=59 }},
//{"work_time",new TableInfo{SimulaRowCount=62 }}, {"work_time",new TableInfo{SimulaRowCount=62 }},
}; };
var smallTableContext = new ProcessContext(); var smallTableContext = new ProcessContext();
taskFun(new TasksOptions { TableInfoConfig = smallTablesDic, OutPutOptions = new OutPutOptions { FlushCount = 20000, OutPutTaskCount = 4 } }, taskFun(new TasksOptions { TableInfoConfig = smallTablesDic, OutPutOptions = new OutPutOptions { FlushCount = 20000, OutPutTaskCount = 4 } },

15
ConsoleApp2/HostedServices/OutputService.cs
View File

@ -47,13 +47,11 @@ public class OutputService : IOutputService
records.Add(record); records.Add(record);
if (records.Count >= tasksOptions.OutPutOptions.FlushCount) if (records.Count >= tasksOptions.OutPutOptions.FlushCount)
{ {
var temp= new List<DataRecord>(); var temp= new List<DataRecord>(records);
temp.AddRange(records);
ThreadPool.QueueUserWorkItem(async (queueState) => ThreadPool.QueueUserWorkItem(async (queueState) =>
{ {
await FlushAsync(temp); await FlushAsync(temp);
}); });
records.Clear(); records.Clear();
} }
@ -65,15 +63,12 @@ public class OutputService : IOutputService
} }
if (records.Count > 0) if (records.Count > 0)
{ {
var temp = new List<DataRecord>(); var temp = new List<DataRecord>(records);
temp.AddRange(records);
ThreadPool.QueueUserWorkItem(async (queueState) => ThreadPool.QueueUserWorkItem(async (queueState) =>
{ {
await FlushAsync(temp); await FlushAsync(temp);
}); });
records.Clear(); records.Clear();
context.AddFinishTask();
_logger.LogInformation("***** Finished Tasks Count:{FinishTaskCount} *****", context.FinishTaskCount);
_logger.LogInformation("***** Mysql output thread completed *****"); _logger.LogInformation("***** Mysql output thread completed *****");
} }
} }

24
ConsoleApp2/Program.cs
View File

@ -179,16 +179,24 @@ async Task RunProgram()
//if(record.TryGetField("OrderProcessID",out var orderProcessID)) //if(record.TryGetField("OrderProcessID",out var orderProcessID))
var value = await cache.GetStringAsync(record.GetCacheKey("OrderProcessID")); if(record.TryGetField("OrderProcessID",out string orderProcessID))
if (string.IsNullOrEmpty(value))return false; {
var value = await cache.GetStringAsync($"order_process_{orderProcessID}");
if (string.IsNullOrEmpty(value)) return false;
}
} }
if (record.TableName == "order_block_plan_result" ) if (record.TableName == "order_block_plan_result" )
{ {
//如果缓存中不存在ID,则丢弃(ID 对应order_block_plan中的ID) //如果缓存中不存在ID,则丢弃(ID 对应order_block_plan中的ID)
if (record.TryGetField("ID", out string id))
var value = await cache.GetStringAsync(record.GetCacheKey("ID")); {
var value = await cache.GetStringAsync($"order_block_plan_{id}");
if (string.IsNullOrEmpty(value)) return false; if (string.IsNullOrEmpty(value)) return false;
}
} }
return true; return true;
@ -292,7 +300,7 @@ async Task RunProgram()
var headers = new List<string>(record.Headers); var headers = new List<string>(record.Headers);
var fields =new List<string>(record.Fields); var fields =new List<string>(record.Fields);
headers.Add("CompanyID"); headers.Add("CompanyID");
var companyidResult =await cache.GetStringAsync(record.GetCacheKey("ID")); var companyidResult =await cache.GetStringAsync($"order_block_plan_{id}");
_ = int.TryParse(companyidResult, out var companyid); _ = int.TryParse(companyidResult, out var companyid);
fields.Add(companyid.ToString()); fields.Add(companyid.ToString());
@ -308,7 +316,7 @@ async Task RunProgram()
var headers = new List<string>(record.Headers); var headers = new List<string>(record.Headers);
var fields = new List<string>(record.Fields); var fields = new List<string>(record.Fields);
headers.Add("CompanyID"); headers.Add("CompanyID");
var companyidResult = await cache.GetStringAsync(record.GetCacheKey("OrderNo")); var companyidResult = await cache.GetStringAsync($"order_{orderNo}");
_ = int.TryParse(companyidResult, out var cpid); _ = int.TryParse(companyidResult, out var cpid);
fields.Add(cpid.ToString()); fields.Add(cpid.ToString());
return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), cpid); return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), cpid);
@ -350,7 +358,7 @@ async Task RunProgram()
{ {
if (record.TryGetField("OrderProcessID",out var processID)) if (record.TryGetField("OrderProcessID",out var processID))
{ {
var shardKey =await cache.GetStringAsync(record.GetCacheKey("OrderProcessID")); var shardKey =await cache.GetStringAsync($"order_process_{processID}");
var headers = new List<string>(record.Headers); var headers = new List<string>(record.Headers);
var fields = new List<string>(record.Fields); var fields = new List<string>(record.Fields);
headers.Add("ShardKey"); headers.Add("ShardKey");
@ -478,7 +486,7 @@ async Task RunProgram()
host.Services.AddHostedService<MainHostedService>(); host.Services.AddHostedService<MainHostedService>();
host.Services.AddHostedService<TaskMonitorService>(); host.Services.AddHostedService<TaskMonitorService>();
host.Services.AddSingleton<IInputService, InputService>(); host.Services.AddSingleton<IInputService, InputService>();
host.Services.AddSingleton<ITransformService, TransformService>(); host.Services.AddSingleton<ITransformService, TransformService>();
host.Services.AddSingleton<IOutputService, OutputService>(); host.Services.AddSingleton<IOutputService, OutputService>();
var redisOptions = host.Configuration.GetSection("RedisCacheOptions").Get<RedisCacheOptions>() ?? new RedisCacheOptions(); var redisOptions = host.Configuration.GetSection("RedisCacheOptions").Get<RedisCacheOptions>() ?? new RedisCacheOptions();

6
ConsoleApp2/Services/ProcessContext.cs
View File

@ -37,12 +37,6 @@ public class ProcessContext
{ {
_exceptionList.Add(ex); _exceptionList.Add(ex);
} }
public void AddFinishTask() => Interlocked.Increment(ref _finishedTaskCount);
public int FinishTaskCount
{
get => _finishedTaskCount;
}
public ConcurrentBag<Exception> GetExceptions() public ConcurrentBag<Exception> GetExceptions()
{ {
return _exceptionList; return _exceptionList;

6
ConsoleApp2/SimulationService/SimulationInputService.cs
View File

@ -16,17 +16,15 @@ namespace ConsoleApp2.SimulationService
{ {
private readonly ILogger _logger; private readonly ILogger _logger;
private readonly IOptions<DataInputOptions> _dataInputOptions; private readonly IOptions<DataInputOptions> _dataInputOptions;
private readonly DataRecordQueue _producerQueue;
private readonly ProcessContext _context; private readonly ProcessContext _context;
public SimulationInputService(ILogger<InputService> logger, public SimulationInputService(ILogger<InputService> logger,
IOptions<DataInputOptions> dataInputOptions, IOptions<DataInputOptions> dataInputOptions,
[FromKeyedServices(ProcessStep.Producer)] DataRecordQueue producerQueue,
ProcessContext context) ProcessContext context)
{ {
_logger = logger; _logger = logger;
_dataInputOptions = dataInputOptions; _dataInputOptions = dataInputOptions;
_producerQueue = producerQueue;
_context = context; _context = context;
} }
public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, ProcessContext context, CancellationToken cancellationToken) public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, ProcessContext context, CancellationToken cancellationToken)
@ -113,7 +111,7 @@ namespace ConsoleApp2.SimulationService
foreach (var rc in tempRecords) foreach (var rc in tempRecords)
{ {
_context.AddInput(); _context.AddInput();
_producerQueue.Enqueue(rc); producerQueue.Enqueue(rc);
if (cancellationToken.IsCancellationRequested) if (cancellationToken.IsCancellationRequested)
return; return;
} }