CZY/MES-ETL
1
0
Fork 0
Code Pull Requests Releases Activity

支持按多个表开立线程

Browse Source
This commit is contained in:
lindj 2024-01-18 14:36:36 +08:00
parent 629a4d2fb5
commit 97e359468f
16 changed files with 232 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
ConsoleApp2/DataRecord.cs
View File

@ -65,6 +65,8 @@ public class DataRecord
public bool SetField(string columnName, string value) => SetField(this, columnName,value);
public string GetCacheKey(string columnName) => GetCacheKey(this, columnName);
public bool SetField( DataRecord record,string columnName,string value)
{
if (record.Headers is null)
@ -75,4 +77,14 @@ public class DataRecord
record.Fields[idx] = value;
return true;
}
public string GetCacheKey(DataRecord record, string columnName)
{
if (TryGetField(record, columnName, out var value))
{
return $"{TableName}_{columnName}_{value}";
}else
throw new IndexOutOfRangeException($"Column name:{columnName} not found in this record.");
}
}

7
ConsoleApp2/HostedServices/Abstractions/IInputService.cs
View File

@ -1,6 +1,9 @@
namespace ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
namespace ConsoleApp2.HostedServices.Abstractions;
public interface IInputService
{
public Task ExecuteAsync(CancellationToken cancellationToken);
public Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, ProcessContext context, CancellationToken cancellationToken);
}

7
ConsoleApp2/HostedServices/Abstractions/IOutputService.cs
View File

@ -1,6 +1,9 @@
namespace ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
namespace ConsoleApp2.HostedServices.Abstractions;
public interface IOutputService
{
public Task ExecuteAsync(CancellationToken cancellationToken);
public Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken);
}

7
ConsoleApp2/HostedServices/Abstractions/ITransformService.cs
View File

@ -1,6 +1,9 @@
namespace ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
namespace ConsoleApp2.HostedServices.Abstractions;
public interface ITransformService
{
public Task ExecuteAsync(CancellationToken cancellationToken);
public Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken);
}

14
ConsoleApp2/HostedServices/InputService.cs
View File

@ -16,24 +16,18 @@ public class InputService : IInputService
{
private readonly ILogger _logger;
private readonly IOptions<DataInputOptions> _dataInputOptions;
private readonly IOptions<InputTableOptions> _tableOptions;
private readonly DataRecordQueue _producerQueue;
private readonly ProcessContext _context;
public InputService(ILogger<InputService> logger,
IOptions<DataInputOptions> dataInputOptions,
IOptions<InputTableOptions> tableOptions,
[FromKeyedServices(ProcessStep.Producer)] DataRecordQueue producerQueue,
ProcessContext context)
{
_logger = logger;
_dataInputOptions = dataInputOptions;
_tableOptions = tableOptions;
_producerQueue = producerQueue;
_context = context;
}
public async Task ExecuteAsync(CancellationToken cancellationToken)
public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, ProcessContext context,CancellationToken cancellationToken)
{
var inputDir = _dataInputOptions.Value.InputDir;
_logger.LogInformation("***** Csv input service start, working dir: {InputDir}, thread id: {ThreadId} *****", inputDir, Environment.CurrentManagedThreadId);
@ -44,14 +38,14 @@ public class InputService : IInputService
return;
}
var count = 0;
foreach (var tableName in _tableOptions.Value.TableInfoConfig.Keys)
foreach (var tableName in tasksOptions.TableInfoConfig.Keys)
{
_logger.LogInformation("Working table: {tableName}", tableName);
var source = _dataInputOptions.Value.CreateSource?.Invoke(tableName);
await source.DoEnqueue((record) =>
{
_context.AddInput();
_producerQueue.Enqueue(record);
producerQueue.Enqueue(record);
count++;
});
@ -63,7 +57,7 @@ public class InputService : IInputService
_logger.LogInformation("table:'{tableName}' input completed", tableName);
}
_context.CompleteInput();
context.CompleteInput();
_logger.LogInformation("***** Csv input service completed *****");
}
}

117
ConsoleApp2/HostedServices/MainHostedService.cs
View File

@ -1,9 +1,8 @@
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using System.Threading.Tasks;
namespace ConsoleApp2.HostedServices;
public class MainHostedService : BackgroundService
@ -25,49 +24,87 @@ public class MainHostedService : BackgroundService
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var inputTask = Task.Factory.StartNew(async () =>
{
try
{
await _input.ExecuteAsync(stoppingToken);
}
catch (Exception ex)
{
_context.AddException(ex);
_logger.LogError("Exception occurred on inputService:{Message},{StackTrace}", ex.Message, ex.StackTrace);
}
});
var transformTask = Task.Factory.StartNew(async () =>
var taskFun = (TasksOptions taskOp, DataRecordQueue producerQueue, DataRecordQueue consumerQueue, ProcessContext context) =>
{
try
var inputTask = Task.Factory.StartNew(async () =>
{
await _transform.ExecuteAsync(stoppingToken);
}
catch (Exception ex)
try
{
await _input.ExecuteAsync(taskOp, producerQueue, context, stoppingToken);
}
catch (Exception ex)
{
_context.AddException(ex);
_logger.LogError("Exception occurred on inputService:{Message},{StackTrace}", ex.Message, ex.StackTrace);
}
});
var transformTask = Task.Factory.StartNew(async () =>
{
_context.AddException(ex);
_logger.LogError("Exception occurred on transformService:{Message},{StackTrace}", ex.Message, ex.StackTrace);
}
try
{
await _transform.ExecuteAsync(taskOp, producerQueue, consumerQueue, context, stoppingToken);
}
catch (Exception ex)
{
_context.AddException(ex);
_logger.LogError("Exception occurred on transformService:{Message},{StackTrace}", ex.Message, ex.StackTrace);
}
});
});
var outputTask = Task.Factory.StartNew(async () =>
{
try
{
await _output.ExecuteAsync(taskOp, consumerQueue, context,stoppingToken);
}
catch (Exception ex)
{
_context.AddException(ex);
_logger.LogError("Exception occurred on outputService:{Message},{StackTrace}", ex.Message, ex.StackTrace);
}
var outputTask = Task.Factory.StartNew(async () =>
});
};
var bigTablesDic = new Dictionary<string, TableInfo>
{
try
{
await _output.ExecuteAsync(stoppingToken);
}
catch (Exception ex)
{
_context.AddException(ex);
_logger.LogError("Exception occurred on outputService:{Message},{StackTrace}", ex.Message, ex.StackTrace);
}
});
// await Task.Run(async () => await _output.ExecuteAsync(stoppingToken), stoppingToken);
{"order_block_plan",new TableInfo{SimulaRowCount=2725553 }},//CreateTime < 202301的删除
{"order_block_plan_result",new TableInfo{SimulaRowCount=1174096 }},
{"order_box_block",new TableInfo{SimulaRowCount=29755672 }},
{"order_item",new TableInfo{SimulaRowCount=1345520079 }},
{"simple_plan_order",new TableInfo{SimulaRowCount=351470 }},//CreateTime < 202301的删除
};
taskFun(new TasksOptions { TableInfoConfig = bigTablesDic, OutPutOptions = new OutPutOptions { FlushCount = 10000, OutPutTaskCount = 2 } },
new DataRecordQueue(), new DataRecordQueue(),new ProcessContext());
var smallTablesDic = new Dictionary<string, TableInfo>
{
{"machine",new TableInfo{SimulaRowCount=14655 }},
{"order",new TableInfo{SimulaRowCount=5019216 }},
{"order_data_block",new TableInfo{SimulaRowCount=731800334 }},
{"order_data_goods",new TableInfo{SimulaRowCount=25803671 }},
{"order_data_parts",new TableInfo{SimulaRowCount=468517543 }},
{"order_module",new TableInfo{SimulaRowCount=103325385 }},
{"order_module_extra",new TableInfo{SimulaRowCount=54361321 }},
{"order_module_item",new TableInfo{SimulaRowCount=69173339 }},
{"order_package",new TableInfo{SimulaRowCount=16196195 }},
{"order_process",new TableInfo{SimulaRowCount=3892685 }},//orderNo < 202301的
{"order_process_step",new TableInfo{SimulaRowCount=8050349 }},//orderNo < 202301的删除
{"order_process_step_item",new TableInfo{SimulaRowCount=14538058 }},//orderNo < 202301的删除
{"order_scrap_board",new TableInfo{SimulaRowCount=123998 }},
{"process_group",new TableInfo{SimulaRowCount=1253 }},
{"process_info",new TableInfo{SimulaRowCount=7839 }},
{"process_item_exp",new TableInfo{SimulaRowCount=28 }},
{"process_schdule_capacity",new TableInfo{SimulaRowCount=39736 }},
{"process_step_efficiency",new TableInfo{SimulaRowCount=8 }},
{"report_template",new TableInfo{SimulaRowCount=7337 }},
{"simple_package",new TableInfo{SimulaRowCount=130436 }},//orderNo < 202301的删除
{"sys_config",new TableInfo{SimulaRowCount=2296 }},
{"work_calendar",new TableInfo{SimulaRowCount=11 }},
{"work_shift",new TableInfo{SimulaRowCount=59 }},
{"work_time",new TableInfo{SimulaRowCount=62 }},
};
taskFun(new TasksOptions { TableInfoConfig = smallTablesDic, OutPutOptions = new OutPutOptions { FlushCount = 20000, OutPutTaskCount = 4 } },
new DataRecordQueue(), new DataRecordQueue(), new ProcessContext());
}
}

15
ConsoleApp2/HostedServices/OutputService.cs
View File

@ -16,39 +16,36 @@ namespace ConsoleApp2.HostedServices;
public class OutputService : IOutputService
{
private readonly ILogger _logger;
private readonly DataRecordQueue _consumerQueue;
private readonly IOptions<DatabaseOutputOptions> _outputOptions;
private readonly IOptions<DataTransformOptions> _transformOptions;
private readonly ProcessContext _context;
private readonly TaskManager _taskManager;
private readonly IOptions<DatabaseOutputOptions> _outputOptions;
public OutputService(ILogger<OutputService> logger,
[FromKeyedServices(ProcessStep.Consumer)] DataRecordQueue consumerQueue,
IOptions<DatabaseOutputOptions> outputOptions,
ProcessContext context,
TaskManager taskManager,
IOptions<DataTransformOptions> transformOptions)
{
_logger = logger;
_consumerQueue = consumerQueue;
_outputOptions = outputOptions;
_context = context;
_taskManager = taskManager;
_transformOptions = transformOptions;
}
public async Task ExecuteAsync(CancellationToken cancellationToken)
public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue consumerQueue, ProcessContext context,CancellationToken cancellationToken)
{
_logger.LogInformation("***** Mysql output service started *****");
_taskManager.CreateTasks(async () =>
{
var records = new List<DataRecord>();
while (!_context.IsTransformCompleted || _consumerQueue.Count > 0)
while (!context.IsTransformCompleted || consumerQueue.Count > 0)
{
if (!_consumerQueue.TryDequeue(out var record)) continue;
if (!consumerQueue.TryDequeue(out var record)) continue;
records.Add(record);
//_logger.LogInformation(@"*****OutputCount: {count} *****",count);
if (records.Count >= _outputOptions.Value.FlushCount)
if (records.Count >= tasksOptions.OutPutOptions.FlushCount)
{
await FlushAsync(records);
records.Clear();
@ -65,7 +62,7 @@ public class OutputService : IOutputService
records.Clear();
_logger.LogInformation("***** Mysql output thread completed *****");
}
}, _outputOptions.Value.TaskCount);
}, tasksOptions.OutPutOptions.OutPutTaskCount);
await _taskManager.WaitAll();
//_context.CompleteOutput();

2
ConsoleApp2/HostedServices/TestInputService.cs
View File

@ -32,7 +32,7 @@ namespace ConsoleApp2.HostedServices
_producerQueue = producerQueue;
_context = context;
}
public async Task ExecuteAsync(CancellationToken cancellationToken)
public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, ProcessContext context, CancellationToken cancellationToken)
{
var tableName = "order_item";
var headers = new string[] { "ID","OrderNo","ItemNo","ItemType","RoomID","BoxID","DataID","PlanID","PackageID","Num","CompanyID","ShardKey" };

82
ConsoleApp2/HostedServices/TransformService.cs
View File

@ -17,69 +17,67 @@ public class TransformService : ITransformService
{
private readonly ILogger _logger;
private readonly IOptions<DataTransformOptions> _options;
private readonly DataRecordQueue _producerQueue;
private readonly DataRecordQueue _consumerQueue;
private readonly ProcessContext _context;
private readonly IDistributedCache _cache;
private readonly TaskManager _taskManager;
public TransformService(ILogger<TransformService> logger,
IOptions<DataTransformOptions> options,
[FromKeyedServices(ProcessStep.Producer)]DataRecordQueue producerQueue,
[FromKeyedServices(ProcessStep.Consumer)]DataRecordQueue consumerQueue,
ProcessContext context,
IDistributedCache cache)
IDistributedCache cache,
TaskManager taskManager)
{
_logger = logger;
_options = options;
_producerQueue = producerQueue;
_consumerQueue = consumerQueue;
_context = context;
_cache = cache;
_taskManager = taskManager;
}
public async Task ExecuteAsync(CancellationToken cancellationToken)
public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken)
{
_logger.LogInformation("***** Data transform service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId);
while ((!_context.IsInputCompleted || _producerQueue.Count > 0))
_taskManager.CreateTasks(async () =>
{
if (_context.GetExceptions().Count > 0)
while ((!context.IsInputCompleted || producerQueue.Count > 0))
{
_logger.LogInformation("***** Csv transform service is canceled *****");
return;
}
// var dbOptions = _options.Value.DatabaseFilter(record);
if (!_producerQueue.TryDequeue(out var record)) continue;
//过滤不要的record
if ( await _options.Value.RecordFilter?.Invoke(record, _cache) == false) continue;
record.Database = _options.Value.DatabaseFilter?.Invoke(record);
//修改record
_options.Value.RecordModify?.Invoke(record);
//缓存record
_options.Value.RecordCache?.Invoke(record, _cache);
//替换record
var replaceRecord =await _options.Value.RecordReplace?.Invoke(record, _cache);
if (replaceRecord != null)
{
record = replaceRecord;
}
_consumerQueue.Enqueue(record);
_context.AddTransform();
//数据增加
var addRecords=_options.Value.RecordAdd?.Invoke(record);
if(addRecords != null&& addRecords.Count>0)
{
foreach(var rc in addRecords)
if (_context.GetExceptions().Count > 0)
{
_consumerQueue.Enqueue(rc);
_context.AddTransform();
_logger.LogInformation("***** Csv transform service is canceled *****");
return;
}
if (!producerQueue.TryDequeue(out var record)) continue;
//过滤不要的record
if (await _options.Value.RecordFilter?.Invoke(record, _cache) == false) continue;
record.Database = _options.Value.DatabaseFilter?.Invoke(record);
//修改record
_options.Value.RecordModify?.Invoke(record);
//缓存record
_options.Value.RecordCache?.Invoke(record, _cache);
//替换record
var replaceRecord = await _options.Value.RecordReplace?.Invoke(record, _cache);
if (replaceRecord != null)
{
record = replaceRecord;
}
consumerQueue.Enqueue(record);
_context.AddTransform();
//数据增加
var addRecords = _options.Value.RecordAdd?.Invoke(record);
if (addRecords != null && addRecords.Count > 0)
{
foreach (var rc in addRecords)
{
consumerQueue.Enqueue(rc);
_context.AddTransform();
}
}
}
}
_context.CompleteTransform();
context.CompleteTransform();
},tasksOptions.TransformTaskCount,cancellationToken);
_logger.LogInformation("***** Data transformation service completed *****");
}
}

3
ConsoleApp2/HostedServices/VoidOutputService.cs
View File

@ -1,5 +1,6 @@
using ConsoleApp2.Const;
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
@ -22,7 +23,7 @@ public class VoidOutputService : IOutputService
_logger = logger;
}
public Task ExecuteAsync(CancellationToken stoppingToken)
public Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken)
{
_logger.LogInformation("***** Void output service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId);
while (!_context.IsTransformCompleted || _consumerQueue.Count > 0)

1
ConsoleApp2/Options/DataTransformOptions.cs
View File

@ -7,6 +7,7 @@ public enum ColumnType
{
Blob,
Text,
Json,
UnDefine,
}

9
ConsoleApp2/Options/DatabaseOutputOptions.cs
View File

@ -6,14 +6,5 @@ public class DatabaseOutputOptions
/// 数据库连接字符串
/// </summary>
public string? ConnectionString { get; set; }
/// <summary>
/// 输出服务的任务(Task)数
/// </summary>
public int TaskCount { get; set; }
/// <summary>
/// 每个任务每次提交到数据库的记录数量（每N条构建一次SQL语句）
/// </summary>
public int FlushCount { get; set; }
public int MaxAllowedPacket { get; set; } = 64*1024*1024;
}

13
ConsoleApp2/Options/InputTableOptions.cs → ConsoleApp2/Options/TasksOptions.cs
View File

@ -9,13 +9,16 @@ namespace ConsoleApp2.Options
public class TableInfo
{
public long SimulaRowCount { get; set; }//模拟的记录条数
public int InputTaskCount { get; set; } = 1;
public int TransformTaskCount { get; set; } = 1;
public int OutPutTaskCount { get; set; } = 1;
}
public class InputTableOptions
public class TasksOptions
{
public Dictionary<string, TableInfo> TableInfoConfig { get; set; } = new();
public int TransformTaskCount { get; set; } = 1;
public OutPutOptions OutPutOptions { get; set; }=new();
}
public class OutPutOptions
{
public int FlushCount { get; set; } = 10000;
public int OutPutTaskCount { get; set; } = 2;
}
}

127
ConsoleApp2/Program.cs
View File

@ -60,51 +60,49 @@ async Task RunProgram()
var host = Host.CreateApplicationBuilder(args);
var commandOptions = host.Configuration.GetSection("CmdOptions").Get<CommandOptions>() ?? new CommandOptions();
Console.WriteLine($"InputDir:{commandOptions?.InputDir}");
Console.WriteLine($"OutPutFlushCount:{commandOptions?.FlushCount}");
Console.WriteLine($"OutPutTaskCount:{commandOptions?.TaskCount}");
var oldestTime = DateTime.ParseExact(commandOptions.OldestTime, "yyyyMM", System.Globalization.DateTimeFormatInfo.InvariantInfo);
host.Services.Configure<InputTableOptions>(option =>
{
option.TableInfoConfig = new Dictionary<string, TableInfo>
{
//host.Services.Configure<InputTableOptions>(option =>
//{
// option.TableInfoConfig = new Dictionary<string, TableInfo>
// {
//order_block_plan_item从order_item表查询，然后程序插入
//order_package_item从order_item表查询，然后程序插入
//order_patch_detail生产没有这个表，不处理
// //order_block_plan_item从order_item表查询，然后程序插入
// //order_package_item从order_item表查询，然后程序插入
// //order_patch_detail生产没有这个表，不处理
{"machine",new TableInfo{SimulaRowCount=14655 }},
{"order",new TableInfo{SimulaRowCount=5019216 }},
{"order_block_plan",new TableInfo{SimulaRowCount=2725553 }},//CreateTime < 202301的删除
{"order_block_plan_result",new TableInfo{SimulaRowCount=1174096 }},
{"order_box_block",new TableInfo{SimulaRowCount=29755672 }},
{"order_data_block",new TableInfo{SimulaRowCount=731800334 }},
{"order_data_goods",new TableInfo{SimulaRowCount=25803671 }},
{"order_data_parts",new TableInfo{SimulaRowCount=468517543 }},
{"order_item",new TableInfo{SimulaRowCount=1345520079 }},
{"order_module",new TableInfo{SimulaRowCount=103325385 }},
{"order_module_extra",new TableInfo{SimulaRowCount=54361321 }},
{"order_module_item",new TableInfo{SimulaRowCount=69173339 }},
{"order_package",new TableInfo{SimulaRowCount=16196195 }},
{"order_process",new TableInfo{SimulaRowCount=3892685 }},//orderNo < 202301的
{"order_process_step",new TableInfo{SimulaRowCount=8050349 }},//orderNo < 202301的删除
{"order_process_step_item",new TableInfo{SimulaRowCount=14538058 }},//orderNo < 202301的删除
{"order_scrap_board",new TableInfo{SimulaRowCount=123998 }},
{"process_group",new TableInfo{SimulaRowCount=1253 }},
{"process_info",new TableInfo{SimulaRowCount=7839 }},
{"process_item_exp",new TableInfo{SimulaRowCount=28 }},
{"process_schdule_capacity",new TableInfo{SimulaRowCount=39736 }},
{"process_step_efficiency",new TableInfo{SimulaRowCount=8 }},
{"report_template",new TableInfo{SimulaRowCount=7337 }},
{"simple_package",new TableInfo{SimulaRowCount=130436 }},//orderNo < 202301的删除
{"simple_plan_order",new TableInfo{SimulaRowCount=351470 }},//CreateTime < 202301的删除
{"sys_config",new TableInfo{SimulaRowCount=2296 }},
{"work_calendar",new TableInfo{SimulaRowCount=11 }},
{"work_shift",new TableInfo{SimulaRowCount=59 }},
{"work_time",new TableInfo{SimulaRowCount=62 }},
};
});
// {"machine",new TableInfo{SimulaRowCount=14655 }},
// {"order",new TableInfo{SimulaRowCount=5019216 }},
// {"order_block_plan",new TableInfo{SimulaRowCount=2725553 }},//CreateTime < 202301的删除
// {"order_block_plan_result",new TableInfo{SimulaRowCount=1174096 }},
// {"order_box_block",new TableInfo{SimulaRowCount=29755672 }},
// {"order_data_block",new TableInfo{SimulaRowCount=731800334 }},
// {"order_data_goods",new TableInfo{SimulaRowCount=25803671 }},
// {"order_data_parts",new TableInfo{SimulaRowCount=468517543 }},
// {"order_item",new TableInfo{SimulaRowCount=1345520079 }},
// {"order_module",new TableInfo{SimulaRowCount=103325385 }},
// {"order_module_extra",new TableInfo{SimulaRowCount=54361321 }},
// {"order_module_item",new TableInfo{SimulaRowCount=69173339 }},
// {"order_package",new TableInfo{SimulaRowCount=16196195 }},
// {"order_process",new TableInfo{SimulaRowCount=3892685 }},//orderNo < 202301的
// {"order_process_step",new TableInfo{SimulaRowCount=8050349 }},//orderNo < 202301的删除
// {"order_process_step_item",new TableInfo{SimulaRowCount=14538058 }},//orderNo < 202301的删除
// {"order_scrap_board",new TableInfo{SimulaRowCount=123998 }},
// {"process_group",new TableInfo{SimulaRowCount=1253 }},
// {"process_info",new TableInfo{SimulaRowCount=7839 }},
// {"process_item_exp",new TableInfo{SimulaRowCount=28 }},
// {"process_schdule_capacity",new TableInfo{SimulaRowCount=39736 }},
// {"process_step_efficiency",new TableInfo{SimulaRowCount=8 }},
// {"report_template",new TableInfo{SimulaRowCount=7337 }},
// {"simple_package",new TableInfo{SimulaRowCount=130436 }},//orderNo < 202301的删除
// {"simple_plan_order",new TableInfo{SimulaRowCount=351470 }},//CreateTime < 202301的删除
// {"sys_config",new TableInfo{SimulaRowCount=2296 }},
// {"work_calendar",new TableInfo{SimulaRowCount=11 }},
// {"work_shift",new TableInfo{SimulaRowCount=59 }},
// {"work_time",new TableInfo{SimulaRowCount=62 }},
// };
//});
host.Services.Configure<CsvOptions>(option =>
{
option.Delimiter = ",";
@ -177,20 +175,20 @@ async Task RunProgram()
if (record.TableName == "order_process_step" || record.TableName == "order_process_step_item")
{
//如果缓存中不存在OrderProcessID,则丢弃
if(record.TryGetField("OrderProcessID",out var orderProcessID))
{
var value = await cache.GetStringAsync(orderProcessID);
//if(record.TryGetField("OrderProcessID",out var orderProcessID))
var value = await cache.GetStringAsync(record.GetCacheKey("OrderProcessID"));
if (string.IsNullOrEmpty(value))return false;
}
}
if (record.TableName == "order_block_plan_result" )
{
//如果缓存中不存在ID,则丢弃(ID 对应order_block_plan中的ID)
if (record.TryGetField("ID", out var id))
{
var value = await cache.GetStringAsync(id);
var value = await cache.GetStringAsync(record.GetCacheKey("ID"));
if (string.IsNullOrEmpty(value)) return false;
}
}
return true;
@ -202,7 +200,7 @@ async Task RunProgram()
{
if (record.TryGetField("OrderNos", out var nos))
{
if (nos.Length <= 2) record.SetField("OrderNos", "\"[]\"");
if (nos.Length <= 2) record.SetField("OrderNos", "");
}
}
@ -229,7 +227,7 @@ async Task RunProgram()
{
if (record.TryGetField("CompanyID", out var companyid))
{
await cache.SetStringAsync(orderNo, companyid);
await cache.SetStringAsync(record.GetCacheKey("OrderNo"), companyid);
}
}
@ -243,7 +241,7 @@ async Task RunProgram()
if( record.TryGetField("ID", out var id))
{
await cache.SetStringAsync(id, sk);
await cache.SetStringAsync(record.GetCacheKey("ID"), sk);
}
}
@ -253,7 +251,7 @@ async Task RunProgram()
if (record.TryGetField("CompanyID", out var companyid))
{
record.TryGetField("ID", out var id);
await cache.SetStringAsync(id, companyid);
await cache.SetStringAsync(record.GetCacheKey("ID"), companyid);
}
}
};
@ -293,8 +291,9 @@ async Task RunProgram()
var headers = new List<string>(record.Headers);
var fields =new List<string>(record.Fields);
headers.Add("CompanyID");
var companyidResult =await cache.GetStringAsync(id);
_ = int.TryParse(companyidResult.ToString(), out var companyid);
var companyidResult =await cache.GetStringAsync(record.GetCacheKey("ID"));
_ = int.TryParse(companyidResult, out var companyid);
fields.Add(companyid.ToString());
return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), companyid);
}
@ -308,7 +307,7 @@ async Task RunProgram()
var headers = new List<string>(record.Headers);
var fields = new List<string>(record.Fields);
headers.Add("CompanyID");
var companyidResult = await cache.GetStringAsync(orderNo);
var companyidResult = await cache.GetStringAsync(record.GetCacheKey("OrderNo"));
_ = int.TryParse(companyidResult, out var cpid);
fields.Add(cpid.ToString());
return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), cpid);
@ -350,20 +349,12 @@ async Task RunProgram()
{
if (record.TryGetField("OrderProcessID",out var processID))
{
try
{
var shardKey =await cache.GetStringAsync(processID);
var shardKey =await cache.GetStringAsync(record.GetCacheKey("OrderProcessID"));
var headers = new List<string>(record.Headers);
var fields = new List<string>(record.Fields);
headers.Add("ShardKey");
fields.Add(shardKey);
fields.Add(shardKey??"0");
return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), record.CompanyID);
}
catch (Exception ex)
{
}
}
}
}
@ -442,7 +433,7 @@ async Task RunProgram()
{ "process_item_exp.ItemJson", ColumnType.Text },
{ "report_template.Template", ColumnType.Text },
{ "report_template.SourceConfig", ColumnType.Text },
{ "order_block_plan.OrderNos", ColumnType.Text },
{ "order_block_plan.OrderNos", ColumnType.Json },
{ "order_block_plan.BlockInfo", ColumnType.Text },
};
});
@ -465,8 +456,6 @@ async Task RunProgram()
TreatTinyAsBoolean = false,
MaximumPoolSize = 50
}.ConnectionString;
options.TaskCount = commandOptions.TaskCount;
options.FlushCount = commandOptions.FlushCount;
});
host.Services.AddLogging(builder =>
{
@ -474,7 +463,7 @@ async Task RunProgram()
builder.AddSerilog(new LoggerConfiguration().WriteTo.Console().CreateLogger());
});
host.Services.AddSingleton<ProcessContext>();
host.Services.AddScoped<ProcessContext>();
host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Producer);
host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Consumer);
host.Services.AddTransient<TaskManager>();

5
ConsoleApp2/Services/MySqlDestination.cs
View File

@ -131,6 +131,11 @@ public class MySqlDestination : IDisposable, IAsyncDisposable
recordSb.Append("NULL");
else recordSb.Append($"0x{field}");
break;
case ColumnType.Json:
recordSb.Append(string.IsNullOrEmpty(field)
? "\"[]\""
: _transformOptions.Value.TransformBinary?.Invoke(field) ?? field);
break;
case ColumnType.UnDefine:
default:
recordSb.Append(field);

9
ConsoleApp2/SimulationService/SimulationInputService.cs
View File

@ -16,23 +16,20 @@ namespace ConsoleApp2.SimulationService
{
private readonly ILogger _logger;
private readonly IOptions<DataInputOptions> _dataInputOptions;
private readonly IOptions<InputTableOptions> _tableOptions;
private readonly DataRecordQueue _producerQueue;
private readonly ProcessContext _context;
public SimulationInputService(ILogger<InputService> logger,
IOptions<DataInputOptions> dataInputOptions,
IOptions<InputTableOptions> tableOptions,
[FromKeyedServices(ProcessStep.Producer)] DataRecordQueue producerQueue,
ProcessContext context)
{
_logger = logger;
_dataInputOptions = dataInputOptions;
_tableOptions = tableOptions;
_producerQueue = producerQueue;
_context = context;
}
public async Task ExecuteAsync(CancellationToken cancellationToken)
public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, ProcessContext context, CancellationToken cancellationToken)
{
var inputDir = _dataInputOptions.Value.InputDir;
_logger.LogInformation("***** simulation input service start, working dir: {InputDir}, thread id: {ThreadId} *****", inputDir, Environment.CurrentManagedThreadId);
@ -42,9 +39,9 @@ namespace ConsoleApp2.SimulationService
_logger.LogInformation("No source files found in {InputDir}", inputDir);
return;
}
foreach (var tableName in _tableOptions.Value.TableInfoConfig.Keys)
foreach (var tableName in tasksOptions.TableInfoConfig.Keys)
{
var dataCount = _tableOptions.Value.TableInfoConfig[tableName].SimulaRowCount;//当前表要生成的总数据量
var dataCount = tasksOptions.TableInfoConfig[tableName].SimulaRowCount;//当前表要生成的总数据量
var companyTotallCount = 1000;//当前表每个公司生成的总数据量
var tempRecords = new List<DataRecord>();
var sk = DataHelper.shareKeys.First();