566 lines
27 KiB
C#
566 lines
27 KiB
C#
#define USE_TEST_DB // 测试库的结构与生产库不一样,如果使用测试库运行,则加上USE_TEST_DB预处理器指令
|
||
|
||
using ConsoleApp2;
|
||
using ConsoleApp2.Cache;
|
||
using ConsoleApp2.Const;
|
||
using ConsoleApp2.Helpers;
|
||
using ConsoleApp2.HostedServices;
|
||
using ConsoleApp2.HostedServices.Abstractions;
|
||
using ConsoleApp2.Options;
|
||
using ConsoleApp2.Services;
|
||
using ConsoleApp2.Services.ErrorRecorder;
|
||
using ConsoleApp2.Services.ETL;
|
||
using ConsoleApp2.Services.Loggers;
|
||
using Microsoft.Extensions.Configuration;
|
||
using Microsoft.Extensions.DependencyInjection;
|
||
using Microsoft.Extensions.Hosting;
|
||
using Microsoft.Extensions.Logging;
|
||
using MySqlConnector;
|
||
using Serilog;
|
||
using Serilog.Events;
|
||
|
||
await RunProgram();
|
||
return;
|
||
|
||
async Task RunProgram()
|
||
{
|
||
ThreadPool.SetMaxThreads(200, 200);
|
||
var host = Host.CreateApplicationBuilder(args);
|
||
host.Configuration.AddJsonFile(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "appsettings.json"), false, false);
|
||
host.Configuration.AddCommandLine(args, new Dictionary<string, string>
|
||
{
|
||
{"-d", "Input:InputDir"},
|
||
{"--InputDir", "Input:InputDir"},
|
||
{"-s", "Output:ConnectionString"},
|
||
{"--ConnectionString", "Output:ConnectionString"},
|
||
{"-r", "RedisCache:Configuration"},
|
||
{"--Redis", "RedisCache:Configuration"},
|
||
{"-i", "RestoreIndex"},
|
||
{"--RestoreIndex", "RestoreIndex"}
|
||
});
|
||
|
||
var inputOptions = host.Configuration.GetRequiredSection("Input").Get<DataInputOptions>()
|
||
?? throw new ApplicationException("缺少Input配置");
|
||
|
||
var transformOptions = host.Configuration.GetRequiredSection("Transform").Get<DataTransformOptions>()
|
||
?? throw new ApplicationException("缺少Transform配置");
|
||
|
||
var outputOptions = host.Configuration.GetRequiredSection("Output").Get<DatabaseOutputOptions>()
|
||
?? throw new ApplicationException("缺少Output配置");
|
||
|
||
var redisSection = host.Configuration.GetRequiredSection("RedisCache");
|
||
var redisOptions = redisSection.Get<RedisCacheOptions>() ?? throw new ApplicationException("缺少RedisCache配置");
|
||
|
||
var tenantDbSection = host.Configuration.GetRequiredSection("TenantDb");
|
||
var tenantDbOptions = tenantDbSection.Get<TenantDbOptions>() ?? throw new ApplicationException("缺少TenantDb配置");
|
||
if(tenantDbOptions.TenantKey is null) throw new ApplicationException("分库配置中缺少分库键");
|
||
if(tenantDbOptions.DbList is null) throw new ApplicationException("分库配置中没有发现任何数据库");
|
||
|
||
host.Services.Configure<TenantDbOptions>(tenantDbSection);
|
||
host.Services.Configure<RedisCacheOptions>(redisSection);
|
||
|
||
var oldestTime = DateTime.ParseExact(inputOptions.CleanDate, "yyyyMM", System.Globalization.DateTimeFormatInfo.InvariantInfo);
|
||
var oldestTimeInt = int.Parse(inputOptions.CleanDate);
|
||
|
||
// 输入配置
|
||
host.Services.Configure<DataInputOptions>(options =>
|
||
{
|
||
options.InputDir = inputOptions.InputDir ?? throw new ApplicationException("未配置输入目录");
|
||
options.UseMock = inputOptions.UseMock;
|
||
options.TableMockConfig = inputOptions.TableMockConfig;
|
||
options.MockCountMultiplier = inputOptions.MockCountMultiplier;
|
||
|
||
// 配置文件输入方法
|
||
options.FileInputMetaBuilder = fileName =>
|
||
{
|
||
if (fileName.EndsWith(".dat.zst"))
|
||
{
|
||
var tableName = DumpDataHelper.GetTableNameFromCsvFileName(
|
||
Path.GetFileNameWithoutExtension(fileName)); // 去除.zst
|
||
string[]? headers;
|
||
try
|
||
{
|
||
// 查找同目录下同表的SQL文件
|
||
var sqlFile = Directory.GetFiles(options.InputDir)
|
||
.SingleOrDefault(f => f.Equals(fileName.Replace(".dat.zst",".sql.zst")));
|
||
if (sqlFile is null)
|
||
return null;
|
||
headers = DumpDataHelper.GetCsvHeadersFromSqlFile(
|
||
DumpDataHelper.DecompressZstAsStringAsync(File.OpenRead(sqlFile)).Result);
|
||
}
|
||
catch (InvalidOperationException e)
|
||
{
|
||
throw new ApplicationException($"目录下不止一个{tableName}表的SQL文件", e);
|
||
}
|
||
|
||
return new FileInputInfo
|
||
{
|
||
FileName = fileName,
|
||
TableName = tableName,
|
||
Headers = headers
|
||
};
|
||
}
|
||
return null;
|
||
};
|
||
|
||
options.TableOrder =
|
||
[
|
||
TableNames.Machine,
|
||
|
||
TableNames.Order,
|
||
TableNames.OrderBoxBlock, // 依赖Order.CompanyID
|
||
|
||
TableNames.OrderBlockPlan,
|
||
TableNames.OrderBlockPlanResult,// 依赖OrderBlockPlan.CompanyID / 删除
|
||
|
||
TableNames.OrderItem,
|
||
TableNames.OrderDataBlock,
|
||
TableNames.OrderDataGoods,
|
||
TableNames.OrderDataParts,
|
||
TableNames.OrderModule,
|
||
TableNames.OrderModuleExtra,
|
||
TableNames.OrderModuleItem,
|
||
TableNames.OrderPackage,
|
||
TableNames.OrderPatchDetail,
|
||
|
||
TableNames.OrderProcess,
|
||
TableNames.OrderProcessStep,
|
||
TableNames.OrderProcessStepItem,// 依赖OrderProcess.ShardKey / 删除
|
||
|
||
TableNames.OrderProcessSchedule,
|
||
TableNames.OrderScrapBoard,
|
||
TableNames.ProcessGroup,
|
||
TableNames.ProcessInfo,
|
||
TableNames.ProcessItemExp,
|
||
TableNames.ProcessScheduleCapacity,
|
||
TableNames.ProcessStepEfficiency,
|
||
TableNames.ReportTemplate,
|
||
TableNames.SimplePackage,
|
||
TableNames.SimplePlanOrder,
|
||
TableNames.SysConfig,
|
||
TableNames.WorkCalendar,
|
||
TableNames.WorkShift,
|
||
TableNames.WorkTime
|
||
];
|
||
|
||
// options.TableMockConfig = new Dictionary<string, TableMockConfig>
|
||
// {
|
||
// { TableNames.Machine, new TableMockConfig(true, 14655, ["ID"]) },
|
||
// { TableNames.Order, new TableMockConfig(true, 5019216, ["OrderNo"]) },
|
||
// { TableNames.OrderDataBlock, new TableMockConfig(true, 731800334, ["ID"]) },
|
||
// { TableNames.OrderDataGoods, new TableMockConfig(true, 25803671, ["ID"]) },
|
||
// { TableNames.OrderDataParts, new TableMockConfig(true, 468517543, ["ID"]) },
|
||
// { TableNames.OrderModule, new TableMockConfig(true, 103325385, ["ID"]) },
|
||
// { TableNames.OrderModuleExtra, new TableMockConfig(true, 54361321, ["ID"]) },
|
||
// { TableNames.OrderModuleItem, new TableMockConfig(true, 69173339, ["ID"]) },
|
||
// { TableNames.OrderPackage, new TableMockConfig(true, 16196195, ["ID"]) },
|
||
// { TableNames.OrderProcess, new TableMockConfig(true, 3892685, ["ID"]) },
|
||
// { TableNames.OrderProcessStep, new TableMockConfig(true, 8050349, ["ID"]) },
|
||
// { TableNames.OrderProcessStepItem, new TableMockConfig(true, 14538058, ["ID"]) },
|
||
// { TableNames.OrderScrapBoard, new TableMockConfig(true, 123998, ["ID"]) },
|
||
// { TableNames.ProcessGroup, new TableMockConfig(true, 1253, ["ID"]) },
|
||
// { TableNames.ProcessInfo, new TableMockConfig(true, 7839, ["ID"]) },
|
||
// { TableNames.ProcessItemExp, new TableMockConfig(true, 28, ["ID"]) },
|
||
// { TableNames.ProcessScheduleCapacity, new TableMockConfig(true, 39736, ["ID"]) },
|
||
// { TableNames.ProcessStepEfficiency, new TableMockConfig(true, 8, ["ID"]) },
|
||
// { TableNames.ReportTemplate, new TableMockConfig(true, 7337, ["ID"]) },
|
||
// { TableNames.SimplePackage, new TableMockConfig(true, 130436, ["ID"]) },
|
||
// { TableNames.SysConfig, new TableMockConfig(true, 2296, ["ID"]) },
|
||
// { TableNames.WorkCalendar, new TableMockConfig(true, 11, ["ID"]) },
|
||
// { TableNames.WorkShift, new TableMockConfig(true, 59, ["ID"]) },
|
||
// { TableNames.WorkTime, new TableMockConfig(true, 62, ["ID"]) }
|
||
// };
|
||
options.TableMockConfig = new Dictionary<string, TableMockConfig>
|
||
{
|
||
{ TableNames.Machine, new TableMockConfig(true, 14655, ["ID"]) },
|
||
{ TableNames.Order, new TableMockConfig(true, 50192, ["OrderNo"]) },
|
||
{ TableNames.OrderDataBlock, new TableMockConfig(true, 7318003, ["ID"]) },
|
||
{ TableNames.OrderDataGoods, new TableMockConfig(true, 258036, ["ID"]) },
|
||
{ TableNames.OrderDataParts, new TableMockConfig(true, 4685175, ["ID"]) },
|
||
{ TableNames.OrderItem, new TableMockConfig(true, 13298896, ["ID"])},
|
||
{ TableNames.OrderModule, new TableMockConfig(true, 1033253, ["ID"]) },
|
||
{ TableNames.OrderModuleExtra, new TableMockConfig(true, 543613, ["ID"]) },
|
||
{ TableNames.OrderModuleItem, new TableMockConfig(true, 691733, ["ID"]) },
|
||
{ TableNames.OrderPackage, new TableMockConfig(true, 161961, ["ID"]) },
|
||
{ TableNames.OrderProcess, new TableMockConfig(true, 38926, ["ID"]) },
|
||
{ TableNames.OrderProcessStep, new TableMockConfig(true, 80503, ["ID"]) },
|
||
{ TableNames.OrderProcessStepItem, new TableMockConfig(true, 145380, ["ID"]) },
|
||
{ TableNames.OrderScrapBoard, new TableMockConfig(true, 1239, ["ID"]) },
|
||
{ TableNames.ProcessGroup, new TableMockConfig(true, 125, ["ID"]) },
|
||
{ TableNames.ProcessInfo, new TableMockConfig(true, 783, ["ID"]) },
|
||
{ TableNames.ProcessItemExp, new TableMockConfig(true, 28, ["ID"]) },
|
||
{ TableNames.ProcessScheduleCapacity, new TableMockConfig(true, 39736, ["ID"]) },
|
||
{ TableNames.ProcessStepEfficiency, new TableMockConfig(true, 8, ["ID"]) },
|
||
{ TableNames.ReportTemplate, new TableMockConfig(true, 7337, ["ID"]) },
|
||
{ TableNames.SimplePackage, new TableMockConfig(true, 130436, ["ID"]) },
|
||
{ TableNames.SysConfig, new TableMockConfig(true, 2296, ["Key"]) },
|
||
{ TableNames.WorkCalendar, new TableMockConfig(true, 11, ["ID"]) },
|
||
{ TableNames.WorkShift, new TableMockConfig(true, 59, ["ID"]) },
|
||
{ TableNames.WorkTime, new TableMockConfig(true, 62, ["ID"]) }
|
||
};
|
||
});
|
||
|
||
host.Services.Configure<DataTransformOptions>(options =>
|
||
{
|
||
// 数据缓存键格式为[TableName]-[ColumnName@ColumnValue]-[CacheColumnName]
|
||
static string BuildCacheKey(string tableName, string columnName, string columnValue, string cacheColumnName)
|
||
=> $"{tableName}-{columnName}@{columnValue}-{cacheColumnName}";
|
||
|
||
static string CalculateShardKeyByOrderNo(ReadOnlySpan<char> orderNo)
|
||
=> $"{orderNo[2..6]}0";
|
||
|
||
options.StrictMode = transformOptions.StrictMode;
|
||
options.EnableFilter = transformOptions.EnableFilter;
|
||
options.EnableReplacer = transformOptions.EnableReplacer;
|
||
options.EnableReBuilder = transformOptions.EnableReBuilder;
|
||
|
||
// order_block_plan_item和order_package_item表不导入,根据order_item数据直接重建
|
||
|
||
// 数据清理
|
||
options.RecordFilter = async context =>
|
||
{
|
||
var record = context.Record;
|
||
var cache = context.Cacher;
|
||
switch (record.TableName)
|
||
{
|
||
// OrderBoxBlock删除对应Order.OrderNo不存在的对象
|
||
case TableNames.OrderBoxBlock:
|
||
{
|
||
if (!await cache.ExistsAsync(CacheKeys.Order_OrderNo_CompanyID(record["OrderNo"])))
|
||
return false;
|
||
break;
|
||
}
|
||
// OrderBlockPlan删除CreateTime < 202301的
|
||
case TableNames.OrderBlockPlan:
|
||
{
|
||
var time = DateTime.Parse(record["CreateTime"].Trim('"'));
|
||
if (time < oldestTime)
|
||
return false;
|
||
break;
|
||
}
|
||
// OrderBlockPlanResult删除对应order_block_plan.ID不存在的对象
|
||
case TableNames.OrderBlockPlanResult:
|
||
{
|
||
if (!await cache.ExistsAsync(CacheKeys.OrderBlockPlan_ID_CompanyID(record["ID"])))
|
||
return false;
|
||
break;
|
||
}
|
||
// OrderModule删除OrderNo < 202301的
|
||
case TableNames.OrderModule:
|
||
{
|
||
var orderNo = record["OrderNo"];
|
||
if(int.Parse(orderNo.AsSpan(0, 6).ToString()) < oldestTimeInt)
|
||
return false;
|
||
break;
|
||
}
|
||
// OrderProcess删除OrderNo < 202301的
|
||
case TableNames.OrderProcess:
|
||
{
|
||
var orderNo = record["OrderNo"];
|
||
if(int.Parse(orderNo.AsSpan(0, 6).ToString()) < oldestTimeInt)
|
||
return false;
|
||
break;
|
||
}
|
||
// OrderProcessStep删除OrderNo < 202301的
|
||
case TableNames.OrderProcessStep:
|
||
{
|
||
var orderNo = record["OrderNo"];
|
||
if(int.Parse(orderNo.AsSpan(0, 6).ToString()) < oldestTimeInt)
|
||
return false;
|
||
break;
|
||
}
|
||
// OrderProcessStepStep删除对应OrderProcess.ID不存在的对象
|
||
case TableNames.OrderProcessStepItem:
|
||
{
|
||
if (!await cache.ExistsAsync(CacheKeys.OrderProcess_ID_ShardKey(record["OrderProcessID"])))
|
||
return false;
|
||
break;
|
||
}
|
||
// SimplePackage删除OrderNo < 202301的
|
||
case TableNames.SimplePackage:
|
||
{
|
||
var orderNo = record["OrderNo"];
|
||
if(int.Parse(orderNo.AsSpan(0, 6).ToString()) < oldestTimeInt)
|
||
return false;
|
||
break;
|
||
}
|
||
// SimplePlanOrder删除CreateTime < 202301的
|
||
case TableNames.SimplePlanOrder:
|
||
{
|
||
var time = DateTime.Parse(record["CreateTime"].Trim('"'));
|
||
if (time < oldestTime)
|
||
return false;
|
||
break;
|
||
}
|
||
}
|
||
|
||
return true;
|
||
};
|
||
|
||
// 数据替换
|
||
options.RecordModify = async context =>
|
||
{
|
||
var record = context.Record;
|
||
var cache = context.Cacher;
|
||
switch (record.TableName)
|
||
{
|
||
#if USE_TEST_DB
|
||
// Order表移除IsBatch列
|
||
case TableNames.Order:
|
||
record.RemoveField("IsBatch");
|
||
break;
|
||
#endif
|
||
//OrderBlockPlan将OrderNo长度<2的置空
|
||
case TableNames.OrderBlockPlan:
|
||
if (record["OrderNos"].Length < 2)
|
||
record["OrderNos"] = "";
|
||
break;
|
||
// OrderBlockPlanResult表添加CompanyID列
|
||
case TableNames.OrderBlockPlanResult:
|
||
record.AddField("CompanyID",
|
||
// 获取OrderBlockPlan.ID -> CompanyID
|
||
ThrowIfNoCached(await cache.GetStringAsync(CacheKeys.OrderBlockPlan_ID_CompanyID(record["ID"])),
|
||
TableNames.OrderBlockPlanResult, TableNames.OrderBlockPlan, "ID", "脏数据未处理"));
|
||
break;
|
||
// OrderBoxBlock添加CompanyID列
|
||
case TableNames.OrderBoxBlock:
|
||
record.AddField("CompanyID",
|
||
// 获取Order.OrderNo -> CompanyID
|
||
ThrowIfNoCached(await cache.GetStringAsync(CacheKeys.Order_OrderNo_CompanyID(record["OrderNo"])),
|
||
TableNames.OrderBoxBlock, TableNames.Order, "OrderNo", "脏数据未处理"));
|
||
break;
|
||
// OrderModule添加ShardKey列,移除ViewFileName列
|
||
case TableNames.OrderModule:
|
||
record.AddField("ShardKey", CalculateShardKeyByOrderNo(record["OrderNo"]));
|
||
record.RemoveField("ViewFileName");
|
||
break;
|
||
// OrderProcess添加ShardKey列,NextStepID的空值转换为0
|
||
case TableNames.OrderProcess:
|
||
record.AddField("ShardKey", CalculateShardKeyByOrderNo(record["OrderNo"]));
|
||
#if USE_TEST_DB
|
||
if(record["NextStepID"] == "\\N")
|
||
record["NextStepID"] = "0";
|
||
#endif
|
||
break;
|
||
// OrderProcessStep,OrderProcessStepItem添加ShardKey列
|
||
case TableNames.OrderProcessStep:
|
||
record.AddField("ShardKey", CalculateShardKeyByOrderNo(record["OrderNo"]));
|
||
break;
|
||
case TableNames.OrderProcessStepItem:
|
||
record.AddField("ShardKey",
|
||
// 获取OrderProcess.ID -> ShardKey
|
||
ThrowIfNoCached(await cache.GetStringAsync(CacheKeys.OrderProcess_ID_ShardKey(record["OrderProcessID"])),
|
||
TableNames.OrderProcessStepItem, TableNames.OrderProcessStep, "OrderProcessID", "脏数据未处理"));
|
||
break;
|
||
case TableNames.SimplePlanOrder:
|
||
#if USE_TEST_DB
|
||
record.RemoveField("ProcessState");
|
||
#endif
|
||
record.AddField("Deleted", "0");
|
||
break;
|
||
}
|
||
|
||
return record;
|
||
|
||
string ThrowIfNoCached(string? cached, string tableName, string cachedTableName, string cachedColumn, string appendMessage = "")
|
||
{
|
||
if (cached is null)
|
||
throw new InvalidDataException(
|
||
$"{tableName}数据异常,在缓存中未找到对应{cachedTableName}.{cachedColumn}\t{appendMessage}");
|
||
return cached;
|
||
}
|
||
};
|
||
|
||
// 数据缓存
|
||
options.RecordCache = async context =>
|
||
{
|
||
var record = context.Record;
|
||
var cache = context.Cacher;
|
||
switch (record.TableName)
|
||
{
|
||
// 缓存Order.OrderNo -> CompanyID
|
||
case TableNames.Order:
|
||
CacheKeys.Order_OrderNo_CompanyID = orderNo =>
|
||
BuildCacheKey(TableNames.Order, "OrderNo", orderNo, "CompanyID");
|
||
await cache.SetStringAsync(
|
||
CacheKeys.Order_OrderNo_CompanyID(record["OrderNo"]),
|
||
record["CompanyID"]);
|
||
break;
|
||
|
||
// 缓存OrderBlockPlan.ID -> CompanyID
|
||
case TableNames.OrderBlockPlan:
|
||
CacheKeys.OrderBlockPlan_ID_CompanyID = id =>
|
||
BuildCacheKey(TableNames.OrderBlockPlan, "ID", id, "CompanyID");
|
||
await cache.SetStringAsync(
|
||
CacheKeys.OrderBlockPlan_ID_CompanyID(record["ID"]),
|
||
record["CompanyID"]);
|
||
break;
|
||
|
||
// 缓存OrderProcess.ID -> ShardKey
|
||
case TableNames.OrderProcess:
|
||
CacheKeys.OrderProcess_ID_ShardKey = id =>
|
||
BuildCacheKey(TableNames.OrderProcess, "ID", id, "ShardKey");
|
||
await cache.SetStringAsync(
|
||
CacheKeys.OrderProcess_ID_ShardKey(record["ID"]),
|
||
record["ShardKey"]);
|
||
break;
|
||
}
|
||
};
|
||
|
||
// 数据库过滤
|
||
options.DatabaseFilter = record =>
|
||
{
|
||
var companyId = int.Parse(record[tenantDbOptions.TenantKey]); // 每个实体都应存在CompanyID,否则异常
|
||
return tenantDbOptions.GetDbNameByTenantKeyValue(companyId);
|
||
};
|
||
|
||
// 数据重建
|
||
options.RecordReBuild = context =>
|
||
{
|
||
var record = context.Record;
|
||
var resultList = new List<DataRecord>();
|
||
// 分流OrderItem表
|
||
if (record.TableName == TableNames.OrderItem)
|
||
{
|
||
record.TryGetField("ID", out var itemId);
|
||
record.TryGetField("ShardKey", out var shardKey);
|
||
record.TryGetField("PlanID", out var planId);
|
||
record.TryGetField("PackageID", out var packageId);
|
||
record.TryGetField("CompanyID", out var companyId);
|
||
if(!int.TryParse(planId, out var pid))
|
||
throw new ApplicationException($"数据发生异常:OrderItem.PlanID,值: {planId}");
|
||
if (pid > 0)
|
||
{
|
||
resultList.Add(new DataRecord(new[] { itemId, shardKey, planId, companyId },
|
||
TableNames.OrderBlockPlanItem,
|
||
["ItemID", "ShardKey", "PlanID", "CompanyID"]
|
||
));
|
||
}
|
||
if(!int.TryParse(packageId, out var pkid))
|
||
throw new ApplicationException($"数据发生异常:OrderItem.PackageID,值: {packageId}");
|
||
if(pkid > 0)
|
||
{
|
||
resultList.Add(new DataRecord(new[] { itemId, shardKey, packageId, companyId },
|
||
TableNames.OrderPackageItem,
|
||
[ "ItemID", "ShardKey", "PackageID", "CompanyID" ]
|
||
));
|
||
}
|
||
}
|
||
return resultList;
|
||
|
||
};
|
||
});
|
||
|
||
host.Services.Configure<DatabaseOutputOptions>(options =>
|
||
{
|
||
options.ConnectionString = new MySqlConnectionStringBuilder(outputOptions.ConnectionString ?? throw new ApplicationException("未配置数据库连接字符串"))
|
||
{
|
||
CharacterSet = "utf8mb4",
|
||
AllowUserVariables = true,
|
||
IgnoreCommandTransaction = true,
|
||
TreatTinyAsBoolean = false,
|
||
ConnectionTimeout = 60,
|
||
DefaultCommandTimeout = 0,
|
||
SslMode = MySqlSslMode.None,
|
||
}.ConnectionString;
|
||
options.FlushCount = outputOptions.FlushCount;
|
||
options.MaxAllowedPacket = outputOptions.MaxAllowedPacket;
|
||
options.MaxDatabaseOutputTask = outputOptions.MaxDatabaseOutputTask;
|
||
options.TreatJsonAsHex = outputOptions.TreatJsonAsHex;
|
||
|
||
#if USE_TEST_DB
|
||
// Test Server
|
||
options.ColumnTypeConfig = new Dictionary<string, ColumnType>
|
||
{
|
||
{ "simple_plan_order.PlaceData", ColumnType.Blob },
|
||
{ "order_block_plan_result.PlaceData", ColumnType.Blob },
|
||
{ "order_box_block.Data", ColumnType.Blob },
|
||
{ "order_data_goods.ExtraProp", ColumnType.Json },
|
||
{ "order_module_extra.JsonStr", ColumnType.Text },
|
||
{ "process_info.Users", ColumnType.Text },
|
||
{ "order_process_schdule.CustomOrderNo", ColumnType.Text },
|
||
{ "order_process_schdule.OrderProcessStepName", ColumnType.Text },
|
||
{ "order_process_schdule.AreaName", ColumnType.Text },
|
||
{ "order_process_schdule.ConsigneeAddress", ColumnType.Text },
|
||
{ "order_process_schdule.ConsigneePhone", ColumnType.Text },
|
||
{ "report_source.Sql", ColumnType.Text },
|
||
{ "report_source.KeyValue", ColumnType.Text },
|
||
{ "report_source.Setting", ColumnType.Text },
|
||
{ "order_data_block.RemarkJson", ColumnType.Text },
|
||
{ "order_patch_detail.BlockDetail", ColumnType.Json },
|
||
{ "order_scrap_board.OutLineJson", ColumnType.Text },
|
||
{ "simple_package.Items", ColumnType.Json },
|
||
{ "order_batch_pack_config.Setting", ColumnType.Text },
|
||
{ "machine.Settings", ColumnType.Text },
|
||
{ "sys_config.Value", ColumnType.Text },
|
||
{ "sys_config.JsonStr", ColumnType.Text },
|
||
{ "process_item_exp.ItemJson", ColumnType.Text },
|
||
{ "report_template.Template", ColumnType.Text },
|
||
{ "report_template.SourceConfig", ColumnType.Text },
|
||
{ "order_block_plan.OrderNos", ColumnType.Json },
|
||
{ "order_block_plan.BlockInfo", ColumnType.Text },
|
||
};
|
||
#else
|
||
// 配置列类型
|
||
// Prod server
|
||
options.ColumnTypeConfig = new Dictionary<string, ColumnType>
|
||
{
|
||
{ "simple_plan_order.PlaceData", ColumnType.Blob },
|
||
{ "order_block_plan_result.PlaceData", ColumnType.Blob },
|
||
{ "order_box_block.Data", ColumnType.Blob },
|
||
{ "order_data_goods.ExtraProp", ColumnType.Text },
|
||
{ "order_module_extra.JsonStr", ColumnType.Text },
|
||
{ "process_info.Users", ColumnType.Text },
|
||
{ "order_process_schdule.CustomOrderNo", ColumnType.Text },
|
||
{ "order_process_schdule.OrderProcessStepName", ColumnType.Text },
|
||
{ "order_process_schdule.AreaName", ColumnType.Text },
|
||
{ "order_process_schdule.ConsigneeAddress", ColumnType.Text },
|
||
{ "order_process_schdule.ConsigneePhone", ColumnType.Text },
|
||
{ "report_source.Sql", ColumnType.Text },
|
||
{ "report_source.KeyValue", ColumnType.Text },
|
||
{ "report_source.Setting", ColumnType.Text },
|
||
{ "order_data_block.RemarkJson", ColumnType.Text },
|
||
{ "order_patch_detail.BlockDetail", ColumnType.Text },
|
||
{ "order_scrap_board.OutLineJson", ColumnType.Text },
|
||
{ "simple_package.Items", ColumnType.Text },
|
||
{ "order_batch_pack_config.Setting", ColumnType.Text },
|
||
{ "machine.Settings", ColumnType.Text },
|
||
{ "sys_config.Value", ColumnType.Text },
|
||
{ "sys_config.JsonStr", ColumnType.Text },
|
||
{ "process_item_exp.ItemJson", ColumnType.Text },
|
||
{ "report_template.Template", ColumnType.Text },
|
||
{ "report_template.SourceConfig", ColumnType.Text },
|
||
{ "order_block_plan.OrderNos", ColumnType.Json },
|
||
{ "order_block_plan.BlockInfo", ColumnType.Text },
|
||
};
|
||
#endif
|
||
});
|
||
|
||
host.Services.AddLogging(builder =>
|
||
{
|
||
builder.ClearProviders();
|
||
builder.AddSerilog(new LoggerConfiguration()
|
||
.WriteTo.Console()
|
||
.WriteTo.File($"./Log/Error/{ErrorRecorder.UID}.log", restrictedToMinimumLevel:LogEventLevel.Error)
|
||
// .WriteTo.File("./Log/Info/{ErrorRecorder.UID}.log", restrictedToMinimumLevel:LogEventLevel.Information) //性能考虑暂不使用
|
||
.CreateLogger()
|
||
);
|
||
});
|
||
|
||
host.Services.AddDataSourceFactory();
|
||
host.Services.AddErrorRecorderFactory();
|
||
host.Services.AddSingleton<ProcessContext>();
|
||
host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Produce);
|
||
host.Services.AddRecordQueuePool(tenantDbOptions.DbList.Keys.Select(key => (key:key, queue:new DataRecordQueue(500_000))).ToArray());
|
||
host.Services.AddSingleton<ITaskMonitorLogger, CacheTaskMonitorLogger>();
|
||
host.Services.AddSingleton<ITaskMonitorLogger, LoggerTaskMonitorLogger>();
|
||
|
||
host.Services.AddHostedService<MainHostedService>();
|
||
host.Services.AddHostedService<TaskMonitorService>();
|
||
host.Services.AddSingleton<IInputService, FileInputService>();
|
||
host.Services.AddSingleton<ITransformService, TransformService>();
|
||
host.Services.AddSingleton<IOutputService, OutputService>();
|
||
host.Services.AddRedisCache(redisOptions);
|
||
var app = host.Build();
|
||
await app.RunAsync();
|
||
} |