CZY/MES-ETL
1
0
Fork 0
Code Pull Requests Releases Activity

Update

Browse Source
This commit is contained in:
陈梓阳
2023-12-29 16:16:05 +08:00
parent 6b88f5bd40
commit c53d2927bb
24 changed files with 909 additions and 386 deletions
Download Patch File Download Diff File Expand all files Collapse all files

122
ConsoleApp2/Program.cs
View File

@@ -1,4 +1,5 @@
using ConsoleApp2;
using ConsoleApp2.HostedServices;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
using Microsoft.Extensions.Configuration;
@@ -7,31 +8,98 @@ using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Serilog;
ThreadPool.SetMaxThreads(200, 200);
var host = Host.CreateApplicationBuilder();
host.Configuration.AddCommandLine(args);
host.Services.Configure<CsvOptions>(option =>
// 加入数据库过滤
// HostedService不是并行的，完善TaskManager手动开启线程
// 测试BlockingCollection对速度的影响？
// 重新同步数据
// Json列和Blob列不一致
/* JSV导出带转义的列有误
* order_data_block表id 4153969
* order_data_parts表Spec列
* order_module表Name列
* process_group表Items列
*/
await RunProgram();
return;
async Task RunProgram()
{
option.DelimiterChar = ',';
option.QuoteChar = '"';
option.InputDir = "D:/Dump/MyDumper-Csv";
option.OutputDir = "D:/DumpOutput";
option.MaxThreads = 12;
});
host.Services.Configure<DataTransformOptions>(options =>
{
var dbOption = new DatabaseOptions("localhost", 33306, "cferp_test_1", "root", "123456");
options.DatabaseFilter = record => dbOption;
});
host.Services.AddLogging(builder =>
{
builder.ClearProviders();
builder.AddSerilog(new LoggerConfiguration().WriteTo.Console().CreateLogger());
});
host.Services.AddHostedService<CsvConversion>();
host.Services.AddHostedService<TaskMonitorService>();
host.Services.AddSingleton<TaskManager>();
host.Services.AddSingleton<DatabaseOutputService>();
host.Services.AddSingleton<DataTransformService>();
var app = host.Build();
await app.RunAsync();
ThreadPool.SetMaxThreads(200, 200);
var host = Host.CreateApplicationBuilder();
host.Configuration.AddCommandLine(args);
host.Services.Configure<CsvOptions>(option =>
{
option.DelimiterChar = ',';
option.QuoteChar = '"';
option.InputDir = "D:/Dump/MyDumper";
option.OutputDir = "D:/DumpOutput";
option.MaxThreads = 12;
});
host.Services.Configure<DataTransformOptions>(options =>
{
//TODO: Database Filter
options.DatabaseFilter = record => "cferp_test_1";
options.ColumnTypeConfig = new()
{
{ "simple_plan_order.PlaceData", ColumnType.Blob },
{ "order_block_plan_result.PlaceData", ColumnType.Blob },
{ "order_box_block.Data", ColumnType.Blob },
{ "order_data_goods.ExtraProp", ColumnType.Text },
{ "order_module_extra.JsonStr", ColumnType.Text },
{ "process_info.Users", ColumnType.Text },
{ "order_process_schdule.CustomOrderNo", ColumnType.Text },
{ "order_process_schdule.OrderProcessStepName", ColumnType.Text },
{ "order_process_schdule.AreaName", ColumnType.Text },
{ "order_process_schdule.ConsigneeAddress", ColumnType.Text },
{ "order_process_schdule.ConsigneePhone", ColumnType.Text },
{ "report_source.Sql", ColumnType.Text },
{ "report_source.KeyValue", ColumnType.Text },
{ "report_source.Setting", ColumnType.Text },
{ "order_data_block.RemarkJson", ColumnType.Text },
{ "order_patch_detail.BlockDetail", ColumnType.Text },
{ "order_scrap_board.OutLineJson", ColumnType.Text },
{ "simple_package.Items", ColumnType.Text },
{ "order_batch_pack_config.Setting", ColumnType.Text },
{ "machine.Settings", ColumnType.Text },
{ "sys_config.Value", ColumnType.Text },
{ "sys_config.JsonStr", ColumnType.Text },
{ "process_item_exp.ItemJson", ColumnType.Text },
{ "report_template.Template", ColumnType.Text },
{ "report_template.SourceConfig", ColumnType.Text },
{ "order_block_plan.OrderNos", ColumnType.Text },
{ "order_block_plan.BlockInfo", ColumnType.Text },
};
});
// 加入数据库过滤后删除
host.Services.Configure<DatabaseOptions>(options =>
{
options.Host = "localhost";
options.Port = 33306;
options.Database = "cferp_test_1";
options.User = "root";
options.Password = "123456";
});
host.Services.AddLogging(builder =>
{
builder.ClearProviders();
builder.AddSerilog(new LoggerConfiguration().WriteTo.Console().CreateLogger());
});
host.Services.AddSingleton<ProcessContext>();
host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Producer);
host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Consumer);
host.Services.AddKeyedSingleton<TaskManager>(ProcessStep.Producer);
host.Services.AddKeyedSingleton<TaskManager>(ProcessStep.Consumer);
host.Services.AddHostedService<TaskMonitorService>();
host.Services.AddHostedService<CsvInputService>();
host.Services.AddHostedService<DataTransformService>();
host.Services.AddHostedService<MysqlOutputService>();
var app = host.Build();
await app.RunAsync();
}