CZY/MES-ETL
1
0
Fork 0
Code Pull Requests Releases Activity

Merge remote-tracking branch 'origin/multTasks'

Browse Source 
# Conflicts:
#	ConsoleApp2/HostedServices/OutputService.cs
#	ConsoleApp2/HostedServices/TransformService.cs
#	ConsoleApp2/Program.cs
This commit is contained in:
陈梓阳 2024-01-18 15:18:28 +08:00
commit 45ad15a065
16 changed files with 232 additions and 192 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
ConsoleApp2/DataRecord.cs
View File

@ -65,6 +65,8 @@ public class DataRecord
public bool SetField(string columnName, string value) => SetField(this, columnName,value); public bool SetField(string columnName, string value) => SetField(this, columnName,value);
public string GetCacheKey(string columnName) => GetCacheKey(this, columnName);
public bool SetField( DataRecord record,string columnName,string value) public bool SetField( DataRecord record,string columnName,string value)
{ {
if (record.Headers is null) if (record.Headers is null)
@ -75,4 +77,14 @@ public class DataRecord
record.Fields[idx] = value; record.Fields[idx] = value;
return true; return true;
} }
public string GetCacheKey(DataRecord record, string columnName)
{
if (TryGetField(record, columnName, out var value))
{
return $"{TableName}_{value}";
}else
throw new IndexOutOfRangeException($"Column name:{columnName} not found in this record.");
}
} }

7
ConsoleApp2/HostedServices/Abstractions/IInputService.cs
View File

@ -1,6 +1,9 @@
namespace ConsoleApp2.HostedServices.Abstractions; using ConsoleApp2.Options;
using ConsoleApp2.Services;
namespace ConsoleApp2.HostedServices.Abstractions;
public interface IInputService public interface IInputService
{ {
public Task ExecuteAsync(CancellationToken cancellationToken); public Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, ProcessContext context, CancellationToken cancellationToken);
} }

7
ConsoleApp2/HostedServices/Abstractions/IOutputService.cs
View File

@ -1,6 +1,9 @@
namespace ConsoleApp2.HostedServices.Abstractions; using ConsoleApp2.Options;
using ConsoleApp2.Services;
namespace ConsoleApp2.HostedServices.Abstractions;
public interface IOutputService public interface IOutputService
{ {
public Task ExecuteAsync(CancellationToken cancellationToken); public Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken);
} }

7
ConsoleApp2/HostedServices/Abstractions/ITransformService.cs
View File

@ -1,6 +1,9 @@
namespace ConsoleApp2.HostedServices.Abstractions; using ConsoleApp2.Options;
using ConsoleApp2.Services;
namespace ConsoleApp2.HostedServices.Abstractions;
public interface ITransformService public interface ITransformService
{ {
public Task ExecuteAsync(CancellationToken cancellationToken); public Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken);
} }

14
ConsoleApp2/HostedServices/InputService.cs
View File

@ -16,24 +16,18 @@ public class InputService : IInputService
{ {
private readonly ILogger _logger; private readonly ILogger _logger;
private readonly IOptions<DataInputOptions> _dataInputOptions; private readonly IOptions<DataInputOptions> _dataInputOptions;
private readonly IOptions<InputTableOptions> _tableOptions;
private readonly DataRecordQueue _producerQueue;
private readonly ProcessContext _context; private readonly ProcessContext _context;
public InputService(ILogger<InputService> logger, public InputService(ILogger<InputService> logger,
IOptions<DataInputOptions> dataInputOptions, IOptions<DataInputOptions> dataInputOptions,
IOptions<InputTableOptions> tableOptions,
[FromKeyedServices(ProcessStep.Producer)] DataRecordQueue producerQueue,
ProcessContext context) ProcessContext context)
{ {
_logger = logger; _logger = logger;
_dataInputOptions = dataInputOptions; _dataInputOptions = dataInputOptions;
_tableOptions = tableOptions;
_producerQueue = producerQueue;
_context = context; _context = context;
} }
public async Task ExecuteAsync(CancellationToken cancellationToken) public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, ProcessContext context,CancellationToken cancellationToken)
{ {
var inputDir = _dataInputOptions.Value.InputDir; var inputDir = _dataInputOptions.Value.InputDir;
_logger.LogInformation("***** Csv input service start, working dir: {InputDir}, thread id: {ThreadId} *****", inputDir, Environment.CurrentManagedThreadId); _logger.LogInformation("***** Csv input service start, working dir: {InputDir}, thread id: {ThreadId} *****", inputDir, Environment.CurrentManagedThreadId);
@ -44,14 +38,14 @@ public class InputService : IInputService
return; return;
} }
var count = 0; var count = 0;
foreach (var tableName in _tableOptions.Value.TableInfoConfig.Keys) foreach (var tableName in tasksOptions.TableInfoConfig.Keys)
{ {
_logger.LogInformation("Working table: {tableName}", tableName); _logger.LogInformation("Working table: {tableName}", tableName);
var source = _dataInputOptions.Value.CreateSource?.Invoke(tableName); var source = _dataInputOptions.Value.CreateSource?.Invoke(tableName);
await source.DoEnqueue((record) => await source.DoEnqueue((record) =>
{ {
_context.AddInput(); _context.AddInput();
_producerQueue.Enqueue(record); producerQueue.Enqueue(record);
count++; count++;
}); });
@ -63,7 +57,7 @@ public class InputService : IInputService
_logger.LogInformation("table:'{tableName}' input completed", tableName); _logger.LogInformation("table:'{tableName}' input completed", tableName);
} }
_context.CompleteInput(); context.CompleteInput();
_logger.LogInformation("***** Csv input service completed *****"); _logger.LogInformation("***** Csv input service completed *****");
} }
} }

55
ConsoleApp2/HostedServices/MainHostedService.cs
View File

@ -1,9 +1,8 @@
using ConsoleApp2.HostedServices.Abstractions; using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services; using ConsoleApp2.Services;
using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using System.Threading.Tasks;
namespace ConsoleApp2.HostedServices; namespace ConsoleApp2.HostedServices;
public class MainHostedService : BackgroundService public class MainHostedService : BackgroundService
@ -25,13 +24,13 @@ public class MainHostedService : BackgroundService
protected override async Task ExecuteAsync(CancellationToken stoppingToken) protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{ {
var taskFun = (TasksOptions taskOp, DataRecordQueue producerQueue, DataRecordQueue consumerQueue, ProcessContext context) =>
{
var inputTask = Task.Factory.StartNew(async () => var inputTask = Task.Factory.StartNew(async () =>
{ {
try try
{ {
await _input.ExecuteAsync(stoppingToken); await _input.ExecuteAsync(taskOp, producerQueue, context, stoppingToken);
} }
catch (Exception ex) catch (Exception ex)
{ {
@ -44,7 +43,7 @@ public class MainHostedService : BackgroundService
{ {
try try
{ {
await _transform.ExecuteAsync(stoppingToken); await _transform.ExecuteAsync(taskOp, producerQueue, consumerQueue, context, stoppingToken);
} }
catch (Exception ex) catch (Exception ex)
{ {
@ -53,12 +52,11 @@ public class MainHostedService : BackgroundService
} }
}); });
var outputTask = Task.Factory.StartNew(async () => var outputTask = Task.Factory.StartNew(async () =>
{ {
try try
{ {
await _output.ExecuteAsync(stoppingToken); await _output.ExecuteAsync(taskOp, consumerQueue, context,stoppingToken);
} }
catch (Exception ex) catch (Exception ex)
{ {
@ -67,7 +65,46 @@ public class MainHostedService : BackgroundService
} }
}); });
};
// await Task.Run(async () => await _output.ExecuteAsync(stoppingToken), stoppingToken); var bigTablesDic = new Dictionary<string, TableInfo>
{
{"order_block_plan",new TableInfo{SimulaRowCount=2725553 }},//CreateTime < 202301的删除
{"order_block_plan_result",new TableInfo{SimulaRowCount=1174096 }},
{"order_box_block",new TableInfo{SimulaRowCount=29755672 }},
{"order_item",new TableInfo{SimulaRowCount=1345520079 }},
{"simple_plan_order",new TableInfo{SimulaRowCount=351470 }},//CreateTime < 202301的删除
};
taskFun(new TasksOptions { TableInfoConfig = bigTablesDic, OutPutOptions = new OutPutOptions { FlushCount = 10000, OutPutTaskCount = 2 } },
new DataRecordQueue(), new DataRecordQueue(),new ProcessContext());
var smallTablesDic = new Dictionary<string, TableInfo>
{
{"machine",new TableInfo{SimulaRowCount=14655 }},
{"order",new TableInfo{SimulaRowCount=5019216 }},
{"order_data_block",new TableInfo{SimulaRowCount=731800334 }},
{"order_data_goods",new TableInfo{SimulaRowCount=25803671 }},
{"order_data_parts",new TableInfo{SimulaRowCount=468517543 }},
{"order_module",new TableInfo{SimulaRowCount=103325385 }},
{"order_module_extra",new TableInfo{SimulaRowCount=54361321 }},
{"order_module_item",new TableInfo{SimulaRowCount=69173339 }},
{"order_package",new TableInfo{SimulaRowCount=16196195 }},
{"order_process",new TableInfo{SimulaRowCount=3892685 }},//orderNo < 202301的
{"order_process_step",new TableInfo{SimulaRowCount=8050349 }},//orderNo < 202301的删除
{"order_process_step_item",new TableInfo{SimulaRowCount=14538058 }},//orderNo < 202301的删除
{"order_scrap_board",new TableInfo{SimulaRowCount=123998 }},
{"process_group",new TableInfo{SimulaRowCount=1253 }},
{"process_info",new TableInfo{SimulaRowCount=7839 }},
{"process_item_exp",new TableInfo{SimulaRowCount=28 }},
{"process_schdule_capacity",new TableInfo{SimulaRowCount=39736 }},
{"process_step_efficiency",new TableInfo{SimulaRowCount=8 }},
{"report_template",new TableInfo{SimulaRowCount=7337 }},
{"simple_package",new TableInfo{SimulaRowCount=130436 }},//orderNo < 202301的删除
{"sys_config",new TableInfo{SimulaRowCount=2296 }},
{"work_calendar",new TableInfo{SimulaRowCount=11 }},
{"work_shift",new TableInfo{SimulaRowCount=59 }},
{"work_time",new TableInfo{SimulaRowCount=62 }},
};
taskFun(new TasksOptions { TableInfoConfig = smallTablesDic, OutPutOptions = new OutPutOptions { FlushCount = 20000, OutPutTaskCount = 4 } },
new DataRecordQueue(), new DataRecordQueue(), new ProcessContext());
} }
} }

13
ConsoleApp2/HostedServices/OutputService.cs
View File

@ -14,7 +14,6 @@ namespace ConsoleApp2.HostedServices;
public class OutputService : IOutputService public class OutputService : IOutputService
{ {
private readonly ILogger _logger; private readonly ILogger _logger;
private readonly DataRecordQueue _consumerQueue;
private readonly IOptions<DatabaseOutputOptions> _outputOptions; private readonly IOptions<DatabaseOutputOptions> _outputOptions;
private readonly IOptions<DataTransformOptions> _transformOptions; private readonly IOptions<DataTransformOptions> _transformOptions;
private readonly ProcessContext _context; private readonly ProcessContext _context;
@ -22,7 +21,6 @@ public class OutputService : IOutputService
private readonly ErrorRecorder _errorRecorder; private readonly ErrorRecorder _errorRecorder;
public OutputService(ILogger<OutputService> logger, public OutputService(ILogger<OutputService> logger,
[FromKeyedServices(ProcessStep.Consumer)] DataRecordQueue consumerQueue,
IOptions<DatabaseOutputOptions> outputOptions, IOptions<DatabaseOutputOptions> outputOptions,
ProcessContext context, ProcessContext context,
TaskManager taskManager, TaskManager taskManager,
@ -30,7 +28,6 @@ public class OutputService : IOutputService
ErrorRecorder errorRecorder) ErrorRecorder errorRecorder)
{ {
_logger = logger; _logger = logger;
_consumerQueue = consumerQueue;
_outputOptions = outputOptions; _outputOptions = outputOptions;
_context = context; _context = context;
_taskManager = taskManager; _taskManager = taskManager;
@ -38,18 +35,18 @@ public class OutputService : IOutputService
_errorRecorder = errorRecorder; _errorRecorder = errorRecorder;
} }
public async Task ExecuteAsync(CancellationToken cancellationToken) public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue consumerQueue, ProcessContext context,CancellationToken cancellationToken)
{ {
_logger.LogInformation("***** Mysql output service started *****"); _logger.LogInformation("***** Mysql output service started *****");
_taskManager.CreateTasks(async () => _taskManager.CreateTasks(async () =>
{ {
var records = new List<DataRecord>(); var records = new List<DataRecord>();
while (!_context.IsTransformCompleted || _consumerQueue.Count > 0) while (!context.IsTransformCompleted || consumerQueue.Count > 0)
{ {
if (!_consumerQueue.TryDequeue(out var record)) continue; if (!consumerQueue.TryDequeue(out var record)) continue;
records.Add(record); records.Add(record);
//_logger.LogInformation(@"*****OutputCount: {count} *****",count); //_logger.LogInformation(@"*****OutputCount: {count} *****",count);
if (records.Count >= _outputOptions.Value.FlushCount) if (records.Count >= tasksOptions.OutPutOptions.FlushCount)
{ {
await FlushAsync(records); await FlushAsync(records);
records.Clear(); records.Clear();
@ -66,7 +63,7 @@ public class OutputService : IOutputService
records.Clear(); records.Clear();
_logger.LogInformation("***** Mysql output thread completed *****"); _logger.LogInformation("***** Mysql output thread completed *****");
} }
}, _outputOptions.Value.TaskCount); }, tasksOptions.OutPutOptions.OutPutTaskCount);
await _taskManager.WaitAll(); await _taskManager.WaitAll();
//_context.CompleteOutput(); //_context.CompleteOutput();

2
ConsoleApp2/HostedServices/TestInputService.cs
View File

@ -32,7 +32,7 @@ namespace ConsoleApp2.HostedServices
_producerQueue = producerQueue; _producerQueue = producerQueue;
_context = context; _context = context;
} }
public async Task ExecuteAsync(CancellationToken cancellationToken) public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, ProcessContext context, CancellationToken cancellationToken)
{ {
var tableName = "order_item"; var tableName = "order_item";
var headers = new string[] { "ID","OrderNo","ItemNo","ItemType","RoomID","BoxID","DataID","PlanID","PackageID","Num","CompanyID","ShardKey" }; var headers = new string[] { "ID","OrderNo","ItemNo","ItemType","RoomID","BoxID","DataID","PlanID","PackageID","Num","CompanyID","ShardKey" };

30
ConsoleApp2/HostedServices/TransformService.cs
View File

@ -17,39 +17,38 @@ public class TransformService : ITransformService
{ {
private readonly ILogger _logger; private readonly ILogger _logger;
private readonly IOptions<DataTransformOptions> _options; private readonly IOptions<DataTransformOptions> _options;
private readonly DataRecordQueue _producerQueue;
private readonly DataRecordQueue _consumerQueue;
private readonly ProcessContext _context; private readonly ProcessContext _context;
private readonly IDistributedCache _cache; private readonly IDistributedCache _cache;
private readonly TaskManager _taskManager;
public TransformService(ILogger<TransformService> logger, public TransformService(ILogger<TransformService> logger,
IOptions<DataTransformOptions> options, IOptions<DataTransformOptions> options,
[FromKeyedServices(ProcessStep.Producer)]DataRecordQueue producerQueue,
[FromKeyedServices(ProcessStep.Consumer)]DataRecordQueue consumerQueue,
ProcessContext context, ProcessContext context,
IDistributedCache cache) IDistributedCache cache,
TaskManager taskManager)
{ {
_logger = logger; _logger = logger;
_options = options; _options = options;
_producerQueue = producerQueue;
_consumerQueue = consumerQueue;
_context = context; _context = context;
_cache = cache; _cache = cache;
_taskManager = taskManager;
} }
public async Task ExecuteAsync(CancellationToken cancellationToken) public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken)
{ {
_logger.LogInformation("***** Data transform service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId); _logger.LogInformation("***** Data transform service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId);
while ((!_context.IsInputCompleted || _producerQueue.Count > 0))
_taskManager.CreateTasks(async () =>
{
while ((!context.IsInputCompleted || producerQueue.Count > 0))
{ {
if (_context.GetExceptions().Count > 0) if (_context.GetExceptions().Count > 0)
{ {
_logger.LogInformation("***** Csv transform service is canceled *****"); _logger.LogInformation("***** Csv transform service is canceled *****");
return; return;
} }
// var dbOptions = _options.Value.DatabaseFilter(record); if (!producerQueue.TryDequeue(out var record)) continue;
if (!_producerQueue.TryDequeue(out var record)) continue;
//过滤不要的record //过滤不要的record
if (await _options.Value.RecordFilter?.Invoke(record, _cache) == false) continue; if (await _options.Value.RecordFilter?.Invoke(record, _cache) == false) continue;
@ -64,7 +63,7 @@ public class TransformService : ITransformService
{ {
record = replaceRecord; record = replaceRecord;
} }
_consumerQueue.Enqueue(record); consumerQueue.Enqueue(record);
_context.AddTransform(); _context.AddTransform();
//数据增加 //数据增加
var addRecords = _options.Value.RecordAdd?.Invoke(record); var addRecords = _options.Value.RecordAdd?.Invoke(record);
@ -72,14 +71,13 @@ public class TransformService : ITransformService
{ {
foreach (var rc in addRecords) foreach (var rc in addRecords)
{ {
_consumerQueue.Enqueue(rc); consumerQueue.Enqueue(rc);
_context.AddTransform(); _context.AddTransform();
} }
} }
} }
context.CompleteTransform();
_context.CompleteTransform(); },tasksOptions.TransformTaskCount,cancellationToken);
_logger.LogInformation("***** Data transformation service completed *****"); _logger.LogInformation("***** Data transformation service completed *****");
} }
} }

3
ConsoleApp2/HostedServices/VoidOutputService.cs
View File

@ -1,5 +1,6 @@
using ConsoleApp2.Const; using ConsoleApp2.Const;
using ConsoleApp2.HostedServices.Abstractions; using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services; using ConsoleApp2.Services;
using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Hosting;
@ -22,7 +23,7 @@ public class VoidOutputService : IOutputService
_logger = logger; _logger = logger;
} }
public Task ExecuteAsync(CancellationToken stoppingToken) public Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken)
{ {
_logger.LogInformation("***** Void output service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId); _logger.LogInformation("***** Void output service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId);
while (!_context.IsTransformCompleted || _consumerQueue.Count > 0) while (!_context.IsTransformCompleted || _consumerQueue.Count > 0)

1
ConsoleApp2/Options/DataTransformOptions.cs
View File

@ -7,6 +7,7 @@ public enum ColumnType
{ {
Blob, Blob,
Text, Text,
Json,
UnDefine, UnDefine,
} }

9
ConsoleApp2/Options/DatabaseOutputOptions.cs
View File

@ -6,14 +6,5 @@ public class DatabaseOutputOptions
/// 数据库连接字符串 /// 数据库连接字符串
/// </summary> /// </summary>
public string? ConnectionString { get; set; } public string? ConnectionString { get; set; }
/// <summary>
/// 输出服务的任务(Task)数
/// </summary>
public int TaskCount { get; set; }
/// <summary>
/// 每个任务每次提交到数据库的记录数量（每N条构建一次SQL语句）
/// </summary>
public int FlushCount { get; set; }
public int MaxAllowedPacket { get; set; } = 64*1024*1024; public int MaxAllowedPacket { get; set; } = 64*1024*1024;
} }

9
ConsoleApp2/Options/InputTableOptions.cs → ConsoleApp2/Options/TasksOptions.cs
View File

@ -10,8 +10,15 @@ namespace ConsoleApp2.Options
{ {
public long SimulaRowCount { get; set; }//模拟的记录条数 public long SimulaRowCount { get; set; }//模拟的记录条数
} }
public class InputTableOptions public class TasksOptions
{ {
public Dictionary<string, TableInfo> TableInfoConfig { get; set; } = new(); public Dictionary<string, TableInfo> TableInfoConfig { get; set; } = new();
public int TransformTaskCount { get; set; } = 1;
public OutPutOptions OutPutOptions { get; set; }=new();
}
public class OutPutOptions
{
public int FlushCount { get; set; } = 10000;
public int OutPutTaskCount { get; set; } = 2;
} }
} }

125
ConsoleApp2/Program.cs
View File

@ -61,51 +61,49 @@ async Task RunProgram()
var host = Host.CreateApplicationBuilder(args); var host = Host.CreateApplicationBuilder(args);
var commandOptions = host.Configuration.GetSection("CmdOptions").Get<CommandOptions>() ?? new CommandOptions(); var commandOptions = host.Configuration.GetSection("CmdOptions").Get<CommandOptions>() ?? new CommandOptions();
Console.WriteLine($"InputDir:{commandOptions?.InputDir}"); Console.WriteLine($"InputDir:{commandOptions?.InputDir}");
Console.WriteLine($"OutPutFlushCount:{commandOptions?.FlushCount}");
Console.WriteLine($"OutPutTaskCount:{commandOptions?.TaskCount}");
var oldestTime = DateTime.ParseExact(commandOptions.OldestTime, "yyyyMM", System.Globalization.DateTimeFormatInfo.InvariantInfo); var oldestTime = DateTime.ParseExact(commandOptions.OldestTime, "yyyyMM", System.Globalization.DateTimeFormatInfo.InvariantInfo);
host.Services.Configure<InputTableOptions>(option => //host.Services.Configure<InputTableOptions>(option =>
{ //{
option.TableInfoConfig = new Dictionary<string, TableInfo> // option.TableInfoConfig = new Dictionary<string, TableInfo>
{ // {
//order_block_plan_item从order_item表查询，然后程序插入 // //order_block_plan_item从order_item表查询，然后程序插入
//order_package_item从order_item表查询，然后程序插入 // //order_package_item从order_item表查询，然后程序插入
//order_patch_detail生产没有这个表，不处理 // //order_patch_detail生产没有这个表，不处理
{"machine",new TableInfo{SimulaRowCount=14655 }}, // {"machine",new TableInfo{SimulaRowCount=14655 }},
{"order",new TableInfo{SimulaRowCount=5019216 }}, // {"order",new TableInfo{SimulaRowCount=5019216 }},
{"order_block_plan",new TableInfo{SimulaRowCount=2725553 }},//CreateTime < 202301的删除 // {"order_block_plan",new TableInfo{SimulaRowCount=2725553 }},//CreateTime < 202301的删除
{"order_block_plan_result",new TableInfo{SimulaRowCount=1174096 }}, // {"order_block_plan_result",new TableInfo{SimulaRowCount=1174096 }},
{"order_box_block",new TableInfo{SimulaRowCount=29755672 }}, // {"order_box_block",new TableInfo{SimulaRowCount=29755672 }},
{"order_data_block",new TableInfo{SimulaRowCount=731800334 }}, // {"order_data_block",new TableInfo{SimulaRowCount=731800334 }},
{"order_data_goods",new TableInfo{SimulaRowCount=25803671 }}, // {"order_data_goods",new TableInfo{SimulaRowCount=25803671 }},
{"order_data_parts",new TableInfo{SimulaRowCount=468517543 }}, // {"order_data_parts",new TableInfo{SimulaRowCount=468517543 }},
{"order_item",new TableInfo{SimulaRowCount=1345520079 }}, // {"order_item",new TableInfo{SimulaRowCount=1345520079 }},
{"order_module",new TableInfo{SimulaRowCount=103325385 }}, // {"order_module",new TableInfo{SimulaRowCount=103325385 }},
{"order_module_extra",new TableInfo{SimulaRowCount=54361321 }}, // {"order_module_extra",new TableInfo{SimulaRowCount=54361321 }},
{"order_module_item",new TableInfo{SimulaRowCount=69173339 }}, // {"order_module_item",new TableInfo{SimulaRowCount=69173339 }},
{"order_package",new TableInfo{SimulaRowCount=16196195 }}, // {"order_package",new TableInfo{SimulaRowCount=16196195 }},
{"order_process",new TableInfo{SimulaRowCount=3892685 }},//orderNo < 202301的 // {"order_process",new TableInfo{SimulaRowCount=3892685 }},//orderNo < 202301的
{"order_process_step",new TableInfo{SimulaRowCount=8050349 }},//orderNo < 202301的删除 // {"order_process_step",new TableInfo{SimulaRowCount=8050349 }},//orderNo < 202301的删除
{"order_process_step_item",new TableInfo{SimulaRowCount=14538058 }},//orderNo < 202301的删除 // {"order_process_step_item",new TableInfo{SimulaRowCount=14538058 }},//orderNo < 202301的删除
{"order_scrap_board",new TableInfo{SimulaRowCount=123998 }}, // {"order_scrap_board",new TableInfo{SimulaRowCount=123998 }},
{"process_group",new TableInfo{SimulaRowCount=1253 }}, // {"process_group",new TableInfo{SimulaRowCount=1253 }},
{"process_info",new TableInfo{SimulaRowCount=7839 }}, // {"process_info",new TableInfo{SimulaRowCount=7839 }},
{"process_item_exp",new TableInfo{SimulaRowCount=28 }}, // {"process_item_exp",new TableInfo{SimulaRowCount=28 }},
{"process_schdule_capacity",new TableInfo{SimulaRowCount=39736 }}, // {"process_schdule_capacity",new TableInfo{SimulaRowCount=39736 }},
{"process_step_efficiency",new TableInfo{SimulaRowCount=8 }}, // {"process_step_efficiency",new TableInfo{SimulaRowCount=8 }},
{"report_template",new TableInfo{SimulaRowCount=7337 }}, // {"report_template",new TableInfo{SimulaRowCount=7337 }},
{"simple_package",new TableInfo{SimulaRowCount=130436 }},//orderNo < 202301的删除 // {"simple_package",new TableInfo{SimulaRowCount=130436 }},//orderNo < 202301的删除
{"simple_plan_order",new TableInfo{SimulaRowCount=351470 }},//CreateTime < 202301的删除 // {"simple_plan_order",new TableInfo{SimulaRowCount=351470 }},//CreateTime < 202301的删除
{"sys_config",new TableInfo{SimulaRowCount=2296 }}, // {"sys_config",new TableInfo{SimulaRowCount=2296 }},
{"work_calendar",new TableInfo{SimulaRowCount=11 }}, // {"work_calendar",new TableInfo{SimulaRowCount=11 }},
{"work_shift",new TableInfo{SimulaRowCount=59 }}, // {"work_shift",new TableInfo{SimulaRowCount=59 }},
{"work_time",new TableInfo{SimulaRowCount=62 }}, // {"work_time",new TableInfo{SimulaRowCount=62 }},
}; // };
}); //});
host.Services.Configure<CsvOptions>(option => host.Services.Configure<CsvOptions>(option =>
{ {
option.Delimiter = ","; option.Delimiter = ",";
@ -178,20 +176,20 @@ async Task RunProgram()
if (record.TableName == "order_process_step" || record.TableName == "order_process_step_item") if (record.TableName == "order_process_step" || record.TableName == "order_process_step_item")
{ {
//如果缓存中不存在OrderProcessID,则丢弃 //如果缓存中不存在OrderProcessID,则丢弃
if(record.TryGetField("OrderProcessID",out var orderProcessID)) //if(record.TryGetField("OrderProcessID",out var orderProcessID))
{
var value = await cache.GetStringAsync(orderProcessID);
var value = await cache.GetStringAsync(record.GetCacheKey("OrderProcessID"));
if (string.IsNullOrEmpty(value))return false; if (string.IsNullOrEmpty(value))return false;
}
} }
if (record.TableName == "order_block_plan_result" ) if (record.TableName == "order_block_plan_result" )
{ {
//如果缓存中不存在ID,则丢弃(ID 对应order_block_plan中的ID) //如果缓存中不存在ID,则丢弃(ID 对应order_block_plan中的ID)
if (record.TryGetField("ID", out var id))
{ var value = await cache.GetStringAsync(record.GetCacheKey("ID"));
var value = await cache.GetStringAsync(id);
if (string.IsNullOrEmpty(value)) return false; if (string.IsNullOrEmpty(value)) return false;
}
} }
return true; return true;
@ -203,7 +201,7 @@ async Task RunProgram()
{ {
if (record.TryGetField("OrderNos", out var nos)) if (record.TryGetField("OrderNos", out var nos))
{ {
if (nos.Length <= 2) record.SetField("OrderNos", "\"[]\""); if (nos.Length <= 2) record.SetField("OrderNos", "");
} }
} }
@ -230,7 +228,7 @@ async Task RunProgram()
{ {
if (record.TryGetField("CompanyID", out var companyid)) if (record.TryGetField("CompanyID", out var companyid))
{ {
await cache.SetStringAsync(orderNo, companyid); await cache.SetStringAsync(record.GetCacheKey("OrderNo"), companyid);
} }
} }
@ -244,7 +242,7 @@ async Task RunProgram()
if( record.TryGetField("ID", out var id)) if( record.TryGetField("ID", out var id))
{ {
await cache.SetStringAsync(id, sk); await cache.SetStringAsync(record.GetCacheKey("ID"), sk);
} }
} }
@ -254,7 +252,7 @@ async Task RunProgram()
if (record.TryGetField("CompanyID", out var companyid)) if (record.TryGetField("CompanyID", out var companyid))
{ {
record.TryGetField("ID", out var id); record.TryGetField("ID", out var id);
await cache.SetStringAsync(id, companyid); await cache.SetStringAsync(record.GetCacheKey("ID"), companyid);
} }
} }
}; };
@ -294,7 +292,8 @@ async Task RunProgram()
var headers = new List<string>(record.Headers); var headers = new List<string>(record.Headers);
var fields =new List<string>(record.Fields); var fields =new List<string>(record.Fields);
headers.Add("CompanyID"); headers.Add("CompanyID");
var companyidResult =await cache.GetStringAsync(id); var companyidResult =await cache.GetStringAsync(record.GetCacheKey("ID"));
_ = int.TryParse(companyidResult, out var companyid); _ = int.TryParse(companyidResult, out var companyid);
fields.Add(companyid.ToString()); fields.Add(companyid.ToString());
return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), companyid); return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), companyid);
@ -309,7 +308,7 @@ async Task RunProgram()
var headers = new List<string>(record.Headers); var headers = new List<string>(record.Headers);
var fields = new List<string>(record.Fields); var fields = new List<string>(record.Fields);
headers.Add("CompanyID"); headers.Add("CompanyID");
var companyidResult = await cache.GetStringAsync(orderNo); var companyidResult = await cache.GetStringAsync(record.GetCacheKey("OrderNo"));
_ = int.TryParse(companyidResult, out var cpid); _ = int.TryParse(companyidResult, out var cpid);
fields.Add(cpid.ToString()); fields.Add(cpid.ToString());
return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), cpid); return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), cpid);
@ -351,21 +350,13 @@ async Task RunProgram()
{ {
if (record.TryGetField("OrderProcessID",out var processID)) if (record.TryGetField("OrderProcessID",out var processID))
{ {
try var shardKey =await cache.GetStringAsync(record.GetCacheKey("OrderProcessID"));
{
var shardKey =await cache.GetStringAsync(processID);
var headers = new List<string>(record.Headers); var headers = new List<string>(record.Headers);
var fields = new List<string>(record.Fields); var fields = new List<string>(record.Fields);
headers.Add("ShardKey"); headers.Add("ShardKey");
fields.Add(shardKey); fields.Add(shardKey??"0");
return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), record.CompanyID); return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), record.CompanyID);
} }
catch (Exception ex)
{
}
}
} }
} }
if(record.TableName == "order_moudle") if(record.TableName == "order_moudle")
@ -443,7 +434,7 @@ async Task RunProgram()
{ "process_item_exp.ItemJson", ColumnType.Text }, { "process_item_exp.ItemJson", ColumnType.Text },
{ "report_template.Template", ColumnType.Text }, { "report_template.Template", ColumnType.Text },
{ "report_template.SourceConfig", ColumnType.Text }, { "report_template.SourceConfig", ColumnType.Text },
{ "order_block_plan.OrderNos", ColumnType.Text }, { "order_block_plan.OrderNos", ColumnType.Json },
{ "order_block_plan.BlockInfo", ColumnType.Text }, { "order_block_plan.BlockInfo", ColumnType.Text },
}; };
}); });
@ -466,8 +457,6 @@ async Task RunProgram()
TreatTinyAsBoolean = false, TreatTinyAsBoolean = false,
MaximumPoolSize = 50 MaximumPoolSize = 50
}.ConnectionString; }.ConnectionString;
options.TaskCount = commandOptions.TaskCount;
options.FlushCount = commandOptions.FlushCount;
}); });
host.Services.AddLogging(builder => host.Services.AddLogging(builder =>
@ -481,7 +470,7 @@ async Task RunProgram()
); );
}); });
host.Services.AddSingleton<ProcessContext>(); host.Services.AddScoped<ProcessContext>();
host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Producer); host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Producer);
host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Consumer); host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Consumer);
host.Services.AddTransient<TaskManager>(); host.Services.AddTransient<TaskManager>();

7
ConsoleApp2/Services/MySqlDestination.cs
View File

@ -77,6 +77,7 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
catch (Exception e) catch (Exception e)
{ {
_logger.LogCritical(e, "Error when flushing records, sql: {Sql}", cmd.CommandText.Omit(1000)); _logger.LogCritical(e, "Error when flushing records, sql: {Sql}", cmd.CommandText.Omit(1000));
_context.AddException(e);
var match = MatchTableName().Match(cmd.CommandText); var match = MatchTableName().Match(cmd.CommandText);
if (match is { Success: true, Groups.Count: > 1 }) if (match is { Success: true, Groups.Count: > 1 })
@ -92,6 +93,7 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
catch (Exception e) catch (Exception e)
{ {
_logger.LogCritical(e, "Error when serialize records, record:"); _logger.LogCritical(e, "Error when serialize records, record:");
_context.AddException(e);
} }
finally finally
{ {
@ -156,6 +158,11 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
recordSb.Append("''"); recordSb.Append("''");
else recordSb.Append($"0x{field}"); else recordSb.Append($"0x{field}");
break; break;
case ColumnType.Json:
recordSb.Append(string.IsNullOrEmpty(field)
? "\"[]\""
: _transformOptions.Value.TransformBinary?.Invoke(field) ?? field);
break;
case ColumnType.UnDefine: case ColumnType.UnDefine:
default: default:
recordSb.Append(field); recordSb.Append(field);

9
ConsoleApp2/SimulationService/SimulationInputService.cs
View File

@ -16,23 +16,20 @@ namespace ConsoleApp2.SimulationService
{ {
private readonly ILogger _logger; private readonly ILogger _logger;
private readonly IOptions<DataInputOptions> _dataInputOptions; private readonly IOptions<DataInputOptions> _dataInputOptions;
private readonly IOptions<InputTableOptions> _tableOptions;
private readonly DataRecordQueue _producerQueue; private readonly DataRecordQueue _producerQueue;
private readonly ProcessContext _context; private readonly ProcessContext _context;
public SimulationInputService(ILogger<InputService> logger, public SimulationInputService(ILogger<InputService> logger,
IOptions<DataInputOptions> dataInputOptions, IOptions<DataInputOptions> dataInputOptions,
IOptions<InputTableOptions> tableOptions,
[FromKeyedServices(ProcessStep.Producer)] DataRecordQueue producerQueue, [FromKeyedServices(ProcessStep.Producer)] DataRecordQueue producerQueue,
ProcessContext context) ProcessContext context)
{ {
_logger = logger; _logger = logger;
_dataInputOptions = dataInputOptions; _dataInputOptions = dataInputOptions;
_tableOptions = tableOptions;
_producerQueue = producerQueue; _producerQueue = producerQueue;
_context = context; _context = context;
} }
public async Task ExecuteAsync(CancellationToken cancellationToken) public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, ProcessContext context, CancellationToken cancellationToken)
{ {
var inputDir = _dataInputOptions.Value.InputDir; var inputDir = _dataInputOptions.Value.InputDir;
_logger.LogInformation("***** simulation input service start, working dir: {InputDir}, thread id: {ThreadId} *****", inputDir, Environment.CurrentManagedThreadId); _logger.LogInformation("***** simulation input service start, working dir: {InputDir}, thread id: {ThreadId} *****", inputDir, Environment.CurrentManagedThreadId);
@ -42,9 +39,9 @@ namespace ConsoleApp2.SimulationService
_logger.LogInformation("No source files found in {InputDir}", inputDir); _logger.LogInformation("No source files found in {InputDir}", inputDir);
return; return;
} }
foreach (var tableName in _tableOptions.Value.TableInfoConfig.Keys) foreach (var tableName in tasksOptions.TableInfoConfig.Keys)
{ {
var dataCount = _tableOptions.Value.TableInfoConfig[tableName].SimulaRowCount;//当前表要生成的总数据量 var dataCount = tasksOptions.TableInfoConfig[tableName].SimulaRowCount;//当前表要生成的总数据量
var companyTotallCount = 1000;//当前表每个公司生成的总数据量 var companyTotallCount = 1000;//当前表每个公司生成的总数据量
var tempRecords = new List<DataRecord>(); var tempRecords = new List<DataRecord>();
var sk = DataHelper.shareKeys.First(); var sk = DataHelper.shareKeys.First();