项目重命名
This commit is contained in:
parent
70cf0322e4
commit
e0de5d1c58
@ -1,28 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace ConsoleApp2.SimulationService
|
||||
{
|
||||
public static partial class DataHelper
|
||||
{
|
||||
public static short[] shareKeys = {23000, 23040, 23070, 23100, 24000, 24040, 24070, 24100, 25000, 25040, 25070, 25100 };
|
||||
public static int[] companyIds = { 1, 2, 3, 4 };
|
||||
private static T getArrayValue<T>(int index, T[] array)//按index取数据,超过数组长度,index从0开始再取
|
||||
{
|
||||
return array[index % array.Length];
|
||||
}
|
||||
public static short GetShareKey(int index)
|
||||
{
|
||||
return getArrayValue(index, shareKeys);
|
||||
}
|
||||
public static int GetCompanyId(int index)
|
||||
{
|
||||
return getArrayValue(index, companyIds);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
@ -1,136 +0,0 @@
|
||||
using ConsoleApp2.Const;
|
||||
using ConsoleApp2.Helpers;
|
||||
using ConsoleApp2.HostedServices;
|
||||
using ConsoleApp2.HostedServices.Abstractions;
|
||||
using ConsoleApp2.Options;
|
||||
using ConsoleApp2.Services;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace ConsoleApp2.SimulationService
|
||||
{
|
||||
|
||||
#if false
|
||||
public class SimulationInputService : IInputService
|
||||
{
|
||||
private readonly ILogger _logger;
|
||||
private readonly IOptions<DataInputOptions> _dataInputOptions;
|
||||
private readonly DataRecordQueue _producerQueue;
|
||||
private readonly ProcessContext _context;
|
||||
|
||||
public SimulationInputService(ILogger<InputService> logger,
|
||||
IOptions<DataInputOptions> dataInputOptions,
|
||||
[FromKeyedServices(ProcessStep.Producer)] DataRecordQueue producerQueue,
|
||||
ProcessContext context)
|
||||
{
|
||||
_logger = logger;
|
||||
_dataInputOptions = dataInputOptions;
|
||||
_producerQueue = producerQueue;
|
||||
_context = context;
|
||||
}
|
||||
public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, ProcessContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
var inputDir = _dataInputOptions.Value.InputDir;
|
||||
_logger.LogInformation("***** simulation input service start, working dir: {InputDir}, thread id: {ThreadId} *****", inputDir, Environment.CurrentManagedThreadId);
|
||||
var files = Directory.GetFiles(inputDir);
|
||||
if (files.Length == 0)
|
||||
{
|
||||
_logger.LogInformation("No source files found in {InputDir}", inputDir);
|
||||
return;
|
||||
}
|
||||
foreach (var tableName in tasksOptions.TableInfoConfig.Keys)
|
||||
{
|
||||
var dataCount = tasksOptions.TableInfoConfig[tableName].SimulaRowCount;//当前表要生成的总数据量
|
||||
var companyTotallCount = 1000;//当前表每个公司生成的总数据量
|
||||
var tempRecords = new List<DataRecord>();
|
||||
var sk = DataHelper.shareKeys.First();
|
||||
var companyID = DataHelper.companyIds.First();
|
||||
|
||||
var shareKeyInterval = 20000;//每个sharekey的数据量
|
||||
var getShareKeyTimes = 0;//sharekey生成的次数,每生成一次,改变sharekey的值
|
||||
var getCompanyIDTimes = 0;//公司生成的次数,每生成一次,改变companyID的值
|
||||
var shareKeyIntervalCount = 0;
|
||||
|
||||
var source = _dataInputOptions.Value.CreateSource?.Invoke(tableName);
|
||||
var testRecord =await source.GetTestRecord();
|
||||
for (long i = 1; i <= dataCount; i++)
|
||||
{
|
||||
shareKeyIntervalCount++;
|
||||
if (shareKeyIntervalCount > shareKeyInterval)
|
||||
{
|
||||
sk = DataHelper.GetShareKey(getShareKeyTimes);
|
||||
getShareKeyTimes++;
|
||||
shareKeyIntervalCount = 0;
|
||||
}
|
||||
var fields = new string[testRecord.Fields.Length];
|
||||
Array.Copy(testRecord.Fields, fields, testRecord.Fields.Length);
|
||||
var record = new DataRecord(fields, testRecord.TableName, testRecord.Headers, companyID);
|
||||
//更新record的ID、OrderNo,ShardKey值
|
||||
if (record.Headers.Contains("ID"))
|
||||
{
|
||||
var index = Array.IndexOf(record.Headers, "ID");
|
||||
if (index > -1)
|
||||
{
|
||||
record.Fields[index] = i.ToString();
|
||||
}
|
||||
}
|
||||
if (record.TableName == "order_box_block" && record.Headers.Contains("BoxID"))
|
||||
{
|
||||
var index = Array.IndexOf(record.Headers, "BoxID");
|
||||
if (index > -1)
|
||||
{
|
||||
record.Fields[index] = i.ToString();
|
||||
}
|
||||
}
|
||||
if ((record.TableName == "order_block_plan_item" || record.TableName == "order_package_item") && record.Headers.Contains("ItemID"))
|
||||
{
|
||||
var index = Array.IndexOf(record.Headers, "ItemID");
|
||||
if (index > -1)
|
||||
{
|
||||
record.Fields[index] = i.ToString();
|
||||
}
|
||||
}
|
||||
if (record.TableName == "order" && record.Headers.Contains("OrderNo"))
|
||||
{
|
||||
var index = Array.IndexOf(record.Headers, "OrderNo");
|
||||
if (index > -1)
|
||||
{
|
||||
record.Fields[index] = i.ToString();
|
||||
}
|
||||
}
|
||||
if (record.Headers.Contains("ShardKey"))
|
||||
{
|
||||
var index = Array.IndexOf(record.Headers, "ShardKey");
|
||||
if (index > -1)
|
||||
{
|
||||
record.Fields[index] = sk.ToString();
|
||||
}
|
||||
}
|
||||
tempRecords.Add(record);
|
||||
if (tempRecords.Count >= companyTotallCount || i >= dataCount - 1)
|
||||
{
|
||||
foreach (var rc in tempRecords)
|
||||
{
|
||||
_context.AddInput();
|
||||
_producerQueue.Enqueue(rc);
|
||||
if (cancellationToken.IsCancellationRequested)
|
||||
return;
|
||||
}
|
||||
tempRecords.Clear();
|
||||
companyID = DataHelper.GetCompanyId(getCompanyIDTimes);
|
||||
getCompanyIDTimes++;
|
||||
}
|
||||
}
|
||||
_logger.LogInformation("table:'{tableName}' simulation input completed", tableName);
|
||||
//}
|
||||
//_logger.LogInformation("File '{File}' input completed", Path.GetFileName(sqlPath));
|
||||
}
|
||||
|
||||
_context.CompleteInput();
|
||||
_logger.LogInformation("***** Csv input service completed *****");
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
@ -1,7 +1,7 @@
|
||||
using System.Data;
|
||||
using MySqlConnector;
|
||||
|
||||
namespace ConsoleApp2.Helpers.Database;
|
||||
namespace ConsoleApp2.Helpers;
|
||||
|
||||
public static class DatabaseHelper
|
||||
{
|
||||
@ -13,7 +13,7 @@ public static class DatabaseHelper
|
||||
await using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = sql;
|
||||
var ds = new DataSet();
|
||||
var adapter = new MySqlDataAdapter(cmd).Fill(ds);
|
||||
new MySqlDataAdapter(cmd).Fill(ds);
|
||||
return ds;
|
||||
}
|
||||
|
@ -2,7 +2,7 @@
|
||||
using System.Reflection;
|
||||
|
||||
namespace ConsoleApp2.Helpers;
|
||||
|
||||
#nullable disable
|
||||
public static class EnumerableExtensions
|
||||
{
|
||||
public static string ToMarkdownTable<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicMethods)]T>(this IEnumerable<T> source)
|
@ -1,7 +1,6 @@
|
||||
using System.Diagnostics;
|
||||
using System.Text;
|
||||
using ConsoleApp2.Helpers;
|
||||
using ConsoleApp2.Helpers.Database;
|
||||
using ConsoleApp2.HostedServices.Abstractions;
|
||||
using ConsoleApp2.Options;
|
||||
using ConsoleApp2.Services;
|
||||
@ -10,6 +9,7 @@ using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MySqlConnector;
|
||||
|
||||
namespace ConsoleApp2.HostedServices;
|
||||
|
||||
@ -19,6 +19,7 @@ public class MainHostedService : BackgroundService
|
||||
private readonly IInputService _input;
|
||||
private readonly ITransformService _transform;
|
||||
private readonly IOutputService _output;
|
||||
private readonly TaskMonitorService _taskMonitor;
|
||||
private readonly ILogger _logger;
|
||||
private readonly ProcessContext _context;
|
||||
|
||||
@ -33,7 +34,8 @@ public class MainHostedService : BackgroundService
|
||||
IOptions<TenantDbOptions> tenantDbOptions,
|
||||
IOptions<DatabaseOutputOptions> databaseOptions,
|
||||
IConfiguration config,
|
||||
ProcessContext context)
|
||||
ProcessContext context,
|
||||
TaskMonitorService taskMonitor)
|
||||
{
|
||||
_input = input;
|
||||
_transform = transform;
|
||||
@ -43,13 +45,23 @@ public class MainHostedService : BackgroundService
|
||||
_databaseOptions = databaseOptions;
|
||||
_config = config;
|
||||
_context = context;
|
||||
_taskMonitor = taskMonitor;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
_stopwatch = Stopwatch.StartNew();
|
||||
await SetVariableAsync(); // 开启延迟写入,禁用重做日志 >>> 重做日志处于禁用状态时绝对不要关闭数据库服务!
|
||||
var command = _config["Command"];
|
||||
if (!string.IsNullOrEmpty(command))
|
||||
{
|
||||
_logger.LogInformation("***** Running Sql Command *****");
|
||||
await ExecuteEachDatabase(command, stoppingToken);
|
||||
Environment.Exit(0);
|
||||
}
|
||||
|
||||
_stopwatch = Stopwatch.StartNew();
|
||||
await SetVariableAsync(); // 开启延迟写入,禁用重做日志 >>> 重做日志处于禁用状态时不要关闭数据库服务!
|
||||
|
||||
var monitorTask = Task.Run(async () => await _taskMonitor.Monitor(stoppingToken), stoppingToken);
|
||||
var inputTask = ExecuteAndCatch(
|
||||
async () => await _input.ExecuteAsync(stoppingToken), "文件输入程序出现异常", stoppingToken);
|
||||
var transformTask = ExecuteAndCatch(
|
||||
@ -69,8 +81,6 @@ public class MainHostedService : BackgroundService
|
||||
await ExportResultAsync();
|
||||
_logger.LogInformation("The execution result export to {Path}",
|
||||
Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"Result-{ErrorRecorder.UID}.md"));
|
||||
if (_config["RestoreIndex"] is not null)
|
||||
await RestoreIndexAsync();
|
||||
|
||||
Environment.Exit(0);
|
||||
}
|
||||
@ -115,24 +125,23 @@ public class MainHostedService : BackgroundService
|
||||
""");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 还原所有数据库的索引...
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
/// <exception cref="ApplicationException"></exception>
|
||||
private async Task RestoreIndexAsync()
|
||||
|
||||
private async Task ExecuteEachDatabase(string command, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var databases = _tenantDbOptions.Value.DbList?.Keys
|
||||
?? throw new ApplicationException("无法还原索引,因为分库配置中没有配置数据库");
|
||||
var connStr = _databaseOptions.Value.ConnectionString
|
||||
?? throw new ApplicationException("无法还原索引,因为没有配置数据库连接字符串");
|
||||
var list = new List<Task>();
|
||||
foreach(var db in databases)
|
||||
foreach (var db in databases)
|
||||
{
|
||||
var task = DatabaseHelper.NonQueryAsync(connStr + $";Database={db};",
|
||||
await File.ReadAllTextAsync(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "RestoreIndex.sql"))
|
||||
);
|
||||
var connStr = new MySqlConnectionStringBuilder(_databaseOptions.Value.ConnectionString
|
||||
?? throw new ApplicationException("无法还原索引,因为没有配置数据库连接字符串"))
|
||||
{
|
||||
ConnectionTimeout = 60,
|
||||
DefaultCommandTimeout = 0,
|
||||
Database = db
|
||||
}.ConnectionString;
|
||||
var task = Task.Run(async () => await DatabaseHelper.NonQueryAsync(connStr, command),
|
||||
cancellationToken);
|
||||
list.Add(task);
|
||||
}
|
||||
|
@ -3,43 +3,32 @@ using ConsoleApp2.Const;
|
||||
using ConsoleApp2.Services;
|
||||
using ConsoleApp2.Services.Loggers;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace ConsoleApp2.HostedServices;
|
||||
|
||||
/// <summary>
|
||||
/// 任务监控
|
||||
/// </summary>
|
||||
public class TaskMonitorService : BackgroundService
|
||||
public class TaskMonitorService
|
||||
{
|
||||
private readonly ILogger<TaskMonitorService> _logger;
|
||||
private readonly IEnumerable<ITaskMonitorLogger> _monitorLoggers;
|
||||
private readonly ProcessContext _context;
|
||||
private readonly DataRecordQueue _producerQueue;
|
||||
private readonly RecordQueuePool _queuePool;
|
||||
|
||||
public TaskMonitorService(
|
||||
ILogger<TaskMonitorService> logger,
|
||||
ProcessContext context,
|
||||
public TaskMonitorService(ProcessContext context,
|
||||
[FromKeyedServices(ProcessStep.Produce)]
|
||||
DataRecordQueue producerQueue,
|
||||
RecordQueuePool queuePool,
|
||||
IEnumerable<ITaskMonitorLogger> monitorLoggers)
|
||||
{
|
||||
_logger = logger;
|
||||
_context = context;
|
||||
_producerQueue = producerQueue;
|
||||
_queuePool = queuePool;
|
||||
_monitorLoggers = monitorLoggers;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
await Task.Run(() => Monitor(stoppingToken), stoppingToken);
|
||||
}
|
||||
|
||||
private async Task Monitor(CancellationToken stoppingToken)
|
||||
public async Task Monitor(CancellationToken stoppingToken)
|
||||
{
|
||||
var sw = Stopwatch.StartNew();
|
||||
var lastTime = sw.ElapsedMilliseconds;
|
@ -6,6 +6,7 @@
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
|
||||
<RootNamespace>ConsoleApp2</RootNamespace>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
@ -35,8 +35,8 @@ async Task RunProgram()
|
||||
{"--ConnectionString", "Output:ConnectionString"},
|
||||
{"-r", "RedisCache:Configuration"},
|
||||
{"--Redis", "RedisCache:Configuration"},
|
||||
{"-i", "RestoreIndex"},
|
||||
{"--RestoreIndex", "RestoreIndex"}
|
||||
{"-c", "Command"},
|
||||
{"--Command", "Command"}
|
||||
});
|
||||
|
||||
var inputOptions = host.Configuration.GetRequiredSection("Input").Get<DataInputOptions>()
|
||||
@ -547,10 +547,10 @@ async Task RunProgram()
|
||||
host.Services.AddSingleton<ITaskMonitorLogger, LoggerTaskMonitorLogger>();
|
||||
|
||||
host.Services.AddHostedService<MainHostedService>();
|
||||
host.Services.AddHostedService<TaskMonitorService>();
|
||||
host.Services.AddSingleton<IInputService, FileInputService>();
|
||||
host.Services.AddSingleton<ITransformService, TransformService>();
|
||||
host.Services.AddSingleton<IOutputService, OutputService>();
|
||||
host.Services.AddSingleton<TaskMonitorService>();
|
||||
host.Services.AddRedisCache(redisOptions);
|
||||
var app = host.Build();
|
||||
await app.RunAsync();
|
@ -12,7 +12,7 @@
|
||||
"Transform":{
|
||||
"StrictMode": true, // 设为true时如果数据转换发生错误,立刻停止程序
|
||||
"EnableFilter": false, // 启用数据过滤
|
||||
"EnableReplacer": false, // 启用数据修改
|
||||
"EnableReplacer": false, // 启用数据修改ma
|
||||
"EnableReBuilder": false, // 启用数据重建
|
||||
"CleanDate": "202301" // 当数据过滤开启时,删除这个时间之前的数据
|
||||
},
|
@ -1,6 +1,5 @@
|
||||
using System.Data;
|
||||
using ConsoleApp2.Helpers;
|
||||
using ConsoleApp2.Helpers.Database;
|
||||
using MySqlConnector;
|
||||
using Xunit.Abstractions;
|
||||
|
@ -1,6 +1,6 @@
|
||||
using System.Data;
|
||||
using System.Text;
|
||||
using ConsoleApp2.Helpers.Database;
|
||||
using ConsoleApp2.Helpers;
|
||||
using MySqlConnector;
|
||||
using Newtonsoft.Json;
|
||||
using Newtonsoft.Json.Linq;
|
||||
@ -11,7 +11,7 @@ namespace TestProject1;
|
||||
public class DatabaseToolBox
|
||||
{
|
||||
private readonly ITestOutputHelper _output;
|
||||
public const string ConnStr = "Server=127.0.0.1;Port=33309;UserId=root;Password=123456;";
|
||||
public const string ConnStr = "Server=127.0.0.1;Port=3306;UserId=root;Password=cfmes123456;";
|
||||
|
||||
public DatabaseToolBox(ITestOutputHelper output)
|
||||
{
|
@ -1,4 +1,4 @@
|
||||
using ConsoleApp2.Helpers.Database;
|
||||
using ConsoleApp2.Helpers;
|
||||
|
||||
namespace TestProject1;
|
||||
|
@ -7,6 +7,7 @@
|
||||
|
||||
<IsPackable>false</IsPackable>
|
||||
<IsTestProject>true</IsTestProject>
|
||||
<RootNamespace>TestProject1</RootNamespace>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
@ -23,7 +24,7 @@
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\ConsoleApp2\ConsoleApp2.csproj" />
|
||||
<ProjectReference Include="..\MesETL.App\MesETL.App.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
@ -1,8 +1,8 @@
|
||||
|
||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ConsoleApp2", "ConsoleApp2\ConsoleApp2.csproj", "{155E4B04-E88C-4BA4-AED2-B13E0A0432B5}"
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MesETL.App", "MesETL.App\MesETL.App.csproj", "{155E4B04-E88C-4BA4-AED2-B13E0A0432B5}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestProject1", "TestProject1\TestProject1.csproj", "{8679D5B6-5853-446E-9882-7B7A8E270500}"
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MesETL.Test", "MesETL.Test\MesETL.Test.csproj", "{8679D5B6-5853-446E-9882-7B7A8E270500}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
Loading…
Reference in New Issue
Block a user