288 lines
14 KiB
C#
288 lines
14 KiB
C#
|
// See https://aka.ms/new-console-template for more information
|
|||
|
|
|
|||
|
|
using System.Text;
|
|||
|
|
using Mesdb.DataGenerator;
|
|||
|
|
using MesETL.App;
|
|||
|
|
using MesETL.App.Cache;
|
|||
|
|
using MesETL.App.Const;
|
|||
|
|
using MesETL.App.HostedServices;
|
|||
|
|
using MesETL.App.HostedServices.Abstractions;
|
|||
|
|
using MesETL.App.Options;
|
|||
|
|
using MesETL.App.Services;
|
|||
|
|
using MesETL.App.Services.ErrorRecorder;
|
|||
|
|
using MesETL.App.Services.ETL;
|
|||
|
|
using MesETL.App.Services.Loggers;
|
|||
|
|
using MesETL.App.Services.Seq;
|
|||
|
|
using Microsoft.Extensions.Configuration;
|
|||
|
|
using Microsoft.Extensions.DependencyInjection;
|
|||
|
|
using Microsoft.Extensions.Hosting;
|
|||
|
|
using Microsoft.Extensions.Logging;
|
|||
|
|
using Serilog;
|
|||
|
|
using Serilog.Events;
|
|||
|
|
|
|||
|
|
await RunProgram();
|
|||
|
|
return;
|
|||
|
|
|
|||
|
|
async Task RunProgram()
|
|||
|
|
{
|
|||
|
|
ThreadPool.SetMaxThreads(200, 200);
|
|||
|
|
var host = Host.CreateApplicationBuilder(args);
|
|||
|
|
|
|||
|
|
var dbGroup = new Dictionary<string, int>()
|
|||
|
|
{
|
|||
|
|
{ "mesdb_1", 5000 },
|
|||
|
|
{ "mesdb_2", 10000 },
|
|||
|
|
{ "mesdb_3", 15000 },
|
|||
|
|
{ "mesdb_4", 20000 },
|
|||
|
|
{ "mesdb_5", 2147483647 },
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
var tenantDbOptions = new TenantDbOptions()
|
|||
|
|
{
|
|||
|
|
TenantKey = "CompanyID",
|
|||
|
|
DbGroup = dbGroup,
|
|||
|
|
UseDbGroup = "Prod",
|
|||
|
|
};
|
|||
|
|
host.Services.Configure<TenantDbOptions>(options =>
|
|||
|
|
{
|
|||
|
|
options.TenantKey = tenantDbOptions.TenantKey;
|
|||
|
|
options.DbGroup = tenantDbOptions.DbGroup;
|
|||
|
|
options.UseDbGroup = tenantDbOptions.UseDbGroup;
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
host.Services.Configure<MockInputOptions>(options =>
|
|||
|
|
{
|
|||
|
|
const float Multiplexer = 0.01F;
|
|||
|
|
var SampleSharedKeys = Enumerable.Range(0, 11).Select(i => (23010 + i * 10).ToString()).Concat(
|
|||
|
|
Enumerable.Range(0, 11).Select(i => (24010 + i * 10).ToString())).ToArray();
|
|||
|
|
options.Rules = new Dictionary<string, TableMockOptions>()
|
|||
|
|
{
|
|||
|
|
{
|
|||
|
|
TableNames.Order, new TableMockOptions((long)(2912406 * Multiplexer), context =>
|
|||
|
|
{
|
|||
|
|
string[] headers =
|
|||
|
|
[
|
|||
|
|
"OrderNo", "ShardKey", "CreateTime", "CompanyID", "SchduleDeliveryDate", "OrderType",
|
|||
|
|
"OrderSort",
|
|||
|
|
"CadDataType", "Deleted", "ProcessState"
|
|||
|
|
];
|
|||
|
|
string[] fields =
|
|||
|
|
[
|
|||
|
|
(20241210000000 + context.Index).ToString(),
|
|||
|
|
SampleSharedKeys[Random.Shared.Next(SampleSharedKeys.Length)],
|
|||
|
|
$"\"{DateTime.Now:yyyy-MM-dd HH:mm:ss}\"",
|
|||
|
|
Random.Shared.Next(1, 28888).ToString(),
|
|||
|
|
$"\"{DateTime.Now.AddDays(Random.Shared.Next(1, 30)):yyyy-MM-dd HH:mm:ss}\"",
|
|||
|
|
Random.Shared.Next(0, 3).ToString(),
|
|||
|
|
Random.Shared.Next(0, 3).ToString(),
|
|||
|
|
Random.Shared.Next(0, 3).ToString(),
|
|||
|
|
Random.Shared.Next(0, 2).ToString(),
|
|||
|
|
Random.Shared.Next(0, 2).ToString()
|
|||
|
|
];
|
|||
|
|
return new DataRecord(fields, TableNames.Order, headers);
|
|||
|
|
})
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
TableNames.OrderItem, new TableMockOptions((long)(820241144 * Multiplexer), context =>
|
|||
|
|
{
|
|||
|
|
string[] headers =
|
|||
|
|
[
|
|||
|
|
"ID", "ShardKey", "OrderNo", "ItemNo", "ItemType", "RoomID", "BoxID", "DataID", "PlanID",
|
|||
|
|
"PackageID", "Num", "CompanyID"
|
|||
|
|
];
|
|||
|
|
string[] fields =
|
|||
|
|
[
|
|||
|
|
context.Index.ToString(),
|
|||
|
|
SampleSharedKeys[Random.Shared.Next(SampleSharedKeys.Length)],
|
|||
|
|
(20241210000000 + Random.Shared.Next(0, 2912406)).ToString(),
|
|||
|
|
(2412000000000 + context.Index).ToString(),
|
|||
|
|
Random.Shared.Next(0, 3).ToString(),
|
|||
|
|
Random.Shared.Next(1, 1000000).ToString(),
|
|||
|
|
Random.Shared.Next(1, 1000000).ToString(),
|
|||
|
|
Random.Shared.Next(1, 1000000).ToString(),
|
|||
|
|
Random.Shared.Next(1, 1306670366).ToString(),
|
|||
|
|
Random.Shared.Next(1, 10000000).ToString(),
|
|||
|
|
(Random.Shared.Next(1, 2000000) / 1000).ToString("F3"),
|
|||
|
|
Random.Shared.Next(1, 28888).ToString(),
|
|||
|
|
];
|
|||
|
|
return new DataRecord(fields, TableNames.OrderItem, headers);
|
|||
|
|
})
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
TableNames.OrderDataBlock, new TableMockOptions((long)(428568719 * Multiplexer), context =>
|
|||
|
|
{
|
|||
|
|
string[] headers =
|
|||
|
|
[
|
|||
|
|
"ID", "ShardKey", "OrderNo", "BoardName", "BoardType", "GoodsID", "Width", "Height",
|
|||
|
|
"Thickness",
|
|||
|
|
"SpliteWidth", "SpliteHeight", "SpliteThickness", "SealedLeft", "SealedRight", "SealedUp",
|
|||
|
|
"SealedDown", "Area", "Wave", "HoleFace", "PaiKong", "RemarkJson", "UnRegularPointCount",
|
|||
|
|
"FrontHoleCount", "BackHoleCount", "SideHoleCount", "FrontModelCount", "BackModelCount",
|
|||
|
|
"IsDoor",
|
|||
|
|
"OpenDoorType", "CompanyID", "BorderLengthHeavy", "BorderLengthLight", "IsHXDJX",
|
|||
|
|
"ModuleTypeID",
|
|||
|
|
"PlanFilterType"
|
|||
|
|
];
|
|||
|
|
string[] BoardNames = ["左侧板", "右侧板", "左开门板", "右开门板", "薄背板", "顶板", "底板", "地脚线", "后地脚", "层板", "立板"];
|
|||
|
|
string[] fields =
|
|||
|
|
[
|
|||
|
|
context.Index.ToString(),
|
|||
|
|
SampleSharedKeys[Random.Shared.Next(SampleSharedKeys.Length)],
|
|||
|
|
(20241210000000 + Random.Shared.Next(0, 2912406)).ToString(),
|
|||
|
|
$"\"{BoardNames[Random.Shared.Next(0, BoardNames.Length)]}\"",
|
|||
|
|
Random.Shared.Next(0, 3).ToString(),
|
|||
|
|
Random.Shared.Next(1, 1000000).ToString(),
|
|||
|
|
(Random.Shared.Next(1, 2000000) / 1000).ToString("F3"),
|
|||
|
|
(Random.Shared.Next(1, 1200000) / 1000).ToString("F3"),
|
|||
|
|
(Random.Shared.Next(1, 18000) / 1000).ToString("F3"),
|
|||
|
|
(Random.Shared.Next(1, 2000000) / 1000).ToString("F3"),
|
|||
|
|
(Random.Shared.Next(1, 1200000) / 1000).ToString("F3"),
|
|||
|
|
(Random.Shared.Next(1, 18000) / 1000).ToString("F3"),
|
|||
|
|
(Random.Shared.Next(1, 2000) / 1000).ToString("F2"),
|
|||
|
|
(Random.Shared.Next(1, 2000) / 1000).ToString("F2"),
|
|||
|
|
(Random.Shared.Next(1, 2000) / 1000).ToString("F2"),
|
|||
|
|
(Random.Shared.Next(1, 2000) / 1000).ToString("F2"),
|
|||
|
|
(Random.Shared.Next(1, 3000) / 1000).ToString("F3"),
|
|||
|
|
Random.Shared.Next(0, 3).ToString(),
|
|||
|
|
Random.Shared.Next(0, 3).ToString(),
|
|||
|
|
Random.Shared.Next(0, 3).ToString(),
|
|||
|
|
Convert.ToHexString(Encoding.UTF8.GetBytes(MockHelper.RandomString(100))),
|
|||
|
|
Random.Shared.Next(0, 4).ToString(),
|
|||
|
|
Random.Shared.Next(0, 4).ToString(),
|
|||
|
|
Random.Shared.Next(0, 4).ToString(),
|
|||
|
|
Random.Shared.Next(0, 4).ToString(),
|
|||
|
|
Random.Shared.Next(0, 4).ToString(),
|
|||
|
|
Random.Shared.Next(0, 4).ToString(),
|
|||
|
|
Random.Shared.Next(0, 2).ToString(),
|
|||
|
|
Random.Shared.Next(0, 6).ToString(),
|
|||
|
|
Random.Shared.Next(1, 28888).ToString(),
|
|||
|
|
(Random.Shared.Next(1, 2000000) / 1000).ToString("F3"),
|
|||
|
|
(Random.Shared.Next(1, 2000000) / 1000).ToString("F3"),
|
|||
|
|
Random.Shared.Next(0, 2).ToString(),
|
|||
|
|
Random.Shared.Next(0, 100000).ToString(),
|
|||
|
|
Random.Shared.Next(0, 5).ToString(),
|
|||
|
|
];
|
|||
|
|
return new DataRecord(fields, TableNames.OrderDataBlock, headers);
|
|||
|
|
})
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
TableNames.OrderBoxBlock, new TableMockOptions((long)(20163038 * Multiplexer), context =>
|
|||
|
|
{
|
|||
|
|
string[] headers = ["BoxID", "ShardKey", "OrderNo", "Data", "CompanyID"];
|
|||
|
|
string[] fields =
|
|||
|
|
[
|
|||
|
|
context.Index.ToString(),
|
|||
|
|
SampleSharedKeys[Random.Shared.Next(SampleSharedKeys.Length)],
|
|||
|
|
(20241210000000 + Random.Shared.Next(0, 2912406)).ToString(),
|
|||
|
|
OrderBoxBlockHelper.RandomPick(),
|
|||
|
|
Random.Shared.Next(1, 28888).ToString(),
|
|||
|
|
];
|
|||
|
|
return new DataRecord(fields, TableNames.OrderBoxBlock, headers);
|
|||
|
|
})
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
TableNames.OrderModuleExtra, new TableMockOptions((long)(33853580 * Multiplexer), context =>
|
|||
|
|
{
|
|||
|
|
string[] headers =
|
|||
|
|
[
|
|||
|
|
"ID", "ShardKey", "OrderNo", "RoomID", "BoxID", "PropType", "JsonStr", "CompanyID"
|
|||
|
|
];
|
|||
|
|
string[] fields =
|
|||
|
|
[
|
|||
|
|
context.Index.ToString(),
|
|||
|
|
SampleSharedKeys[Random.Shared.Next(SampleSharedKeys.Length)],
|
|||
|
|
(20241210000000 + Random.Shared.Next(0, 2912406)).ToString(),
|
|||
|
|
Random.Shared.Next(1, 1000000).ToString(),
|
|||
|
|
Random.Shared.Next(1, 1000000).ToString(),
|
|||
|
|
Random.Shared.Next(0, 14).ToString(),
|
|||
|
|
OrderModuleExtraHelper.PickJson(),
|
|||
|
|
Random.Shared.Next(1, 28888).ToString(),
|
|||
|
|
];
|
|||
|
|
return new DataRecord(fields, TableNames.OrderModuleExtra, headers);
|
|||
|
|
})
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
host.Services.Configure<DataTransformOptions>(options =>
|
|||
|
|
{
|
|||
|
|
options.DatabaseFilter = record =>
|
|||
|
|
{
|
|||
|
|
var companyId = int.Parse(record[tenantDbOptions.TenantKey]); // 每个实体都应存在CompanyID，否则异常
|
|||
|
|
return tenantDbOptions.GetDbNameByTenantKeyValue(companyId);
|
|||
|
|
};
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
host.Services.Configure<DatabaseOutputOptions>(options =>
|
|||
|
|
{
|
|||
|
|
options.ConnectionString = "Server=192.168.1.246;Port=3333;UserId=root;Password=123456;";
|
|||
|
|
options.FlushCount = 10000;
|
|||
|
|
options.MaxAllowedPacket = 67108864;
|
|||
|
|
options.MaxDatabaseOutputTask = 4;
|
|||
|
|
|
|||
|
|
options.ColumnTypeConfig = new Dictionary<string, ColumnType>
|
|||
|
|
{
|
|||
|
|
{ "machine.Settings", ColumnType.Text },
|
|||
|
|
{ "order_block_plan.BlockInfo", ColumnType.Text },
|
|||
|
|
{ "order_block_plan.OrderNos", ColumnType.Json },
|
|||
|
|
{ "order_block_plan_result.PlaceData", ColumnType.Blob },
|
|||
|
|
{ "order_box_block.Data", ColumnType.Blob },
|
|||
|
|
{ "order_data_block.RemarkJson", ColumnType.Text },
|
|||
|
|
{ "order_data_goods.ExtraProp", ColumnType.Json },
|
|||
|
|
{ "order_extra.ConfigJson", ColumnType.Json },
|
|||
|
|
{ "order_module_extra.Data", ColumnType.Blob },
|
|||
|
|
{ "order_module_extra.JsonStr", ColumnType.Text },
|
|||
|
|
{ "order_patch_detail.BlockDetail", ColumnType.Json },
|
|||
|
|
{ "order_process_schdule.AreaName", ColumnType.Text },
|
|||
|
|
{ "order_process_schdule.ConsigneeAddress", ColumnType.Text },
|
|||
|
|
{ "order_process_schdule.ConsigneePhone", ColumnType.Text },
|
|||
|
|
{ "order_process_schdule.CustomOrderNo", ColumnType.Text },
|
|||
|
|
{ "order_process_schdule.OrderProcessStepName", ColumnType.Text },
|
|||
|
|
{ "order_scrap_board.OutLineJson", ColumnType.Text },
|
|||
|
|
{ "order_wave_group.ConfigJson", ColumnType.Json },
|
|||
|
|
{ "process_info.Users", ColumnType.Text },
|
|||
|
|
{ "process_item_exp.ItemJson", ColumnType.Text },
|
|||
|
|
{ "report_template.SourceConfig", ColumnType.Text },
|
|||
|
|
{ "report_template.Template", ColumnType.Text },
|
|||
|
|
{ "simple_package.Items", ColumnType.Json },
|
|||
|
|
{ "sys_config.JsonStr", ColumnType.Text },
|
|||
|
|
{ "sys_config.Value", ColumnType.Text }
|
|||
|
|
};
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
host.Services.AddLogging(builder =>
|
|||
|
|
{
|
|||
|
|
builder.ClearProviders();
|
|||
|
|
builder.AddSerilog(new LoggerConfiguration()
|
|||
|
|
.MinimumLevel.Debug()
|
|||
|
|
.WriteTo.Console()
|
|||
|
|
.WriteTo.File(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"./Log/Error/{ErrorRecorder.UID}.log"),
|
|||
|
|
restrictedToMinimumLevel: LogEventLevel.Error)
|
|||
|
|
// .WriteTo.File("./Log/Info/{ErrorRecorder.UID}.log", restrictedToMinimumLevel:LogEventLevel.Information) //性能考虑暂不使用
|
|||
|
|
.CreateLogger()
|
|||
|
|
);
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
host.Services.AddDataSourceFactory();
|
|||
|
|
host.Services.AddErrorRecorderFactory();
|
|||
|
|
host.Services.AddSingleton<ProcessContext>();
|
|||
|
|
host.Services.AddSingleton<SeqService>();
|
|||
|
|
var prodLen = host.Configuration.GetRequiredSection("RecordQueue").GetValue<int>("ProducerQueueLength");
|
|||
|
|
var consLen = host.Configuration.GetRequiredSection("RecordQueue").GetValue<int>("ConsumerQueueLength");
|
|||
|
|
var maxCharCount = host.Configuration.GetRequiredSection("RecordQueue").GetValue<long>("MaxByteCount") / 2;
|
|||
|
|
host.Services.AddKeyedSingleton<DataRecordQueue>(ConstVar.Producer, new DataRecordQueue(prodLen, maxCharCount));
|
|||
|
|
host.Services.AddRecordQueuePool(dbGroup.Keys
|
|||
|
|
.Select(key => (key: key, queue: new DataRecordQueue(consLen, maxCharCount))).ToArray());
|
|||
|
|
// host.Services.AddSingleton<ITaskMonitorLogger, CacheTaskMonitorLogger>();
|
|||
|
|
host.Services.AddSingleton<ITaskMonitorLogger, LoggerTaskMonitorLogger>();
|
|||
|
|
|
|||
|
|
host.Services.AddHostedService<MainHostedService>();
|
|||
|
|
host.Services.AddSingleton<IInputService, MockInputService>();
|
|||
|
|
host.Services.AddSingleton<ITransformService, TransformService>();
|
|||
|
|
host.Services.AddSingleton<IOutputService, OutputService>();
|
|||
|
|
host.Services.AddSingleton<TaskMonitorService>();
|
|||
|
|
// host.Services.AddRedisCache(redisOptions);
|
|||
|
|
host.Services.AddSingleton<ICacher, MemoryCache>();
|
|||
|
|
var app = host.Build();
|
|||
|
|
await app.RunAsync();
|
|||
|
|
}
|