项目重命名

ConsoleApp2/SimulationService/DataHelper.cs

@@ -1,28 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Text.RegularExpressions;
-using System.Threading.Tasks;
-
-namespace ConsoleApp2.SimulationService
-{
-    public static partial class DataHelper
-    {
-        public static short[] shareKeys = {23000, 23040, 23070, 23100, 24000, 24040, 24070, 24100, 25000, 25040, 25070, 25100 };
-        public static int[] companyIds = { 1, 2, 3, 4 };
-        private static T getArrayValue<T>(int index, T[] array)//按index取数据,超过数组长度,index从0开始再取
-        {
-            return array[index % array.Length];
-        }
-        public static short GetShareKey(int index)
-        {
-            return getArrayValue(index, shareKeys);
-        }
-        public static int GetCompanyId(int index)
-        {
-            return getArrayValue(index, companyIds);
-        }
-       
-    }
-}

ConsoleApp2/SimulationService/SimulationInputService.cs

@@ -1,136 +0,0 @@
-using ConsoleApp2.Const;
-using ConsoleApp2.Helpers;
-using ConsoleApp2.HostedServices;
-using ConsoleApp2.HostedServices.Abstractions;
-using ConsoleApp2.Options;
-using ConsoleApp2.Services;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Options;
-using System.Text.RegularExpressions;
-
-namespace ConsoleApp2.SimulationService
-{
-
-    #if false
-    public class SimulationInputService : IInputService
-    {
-        private readonly ILogger _logger;
-        private readonly IOptions<DataInputOptions> _dataInputOptions;
-        private readonly DataRecordQueue _producerQueue;
-        private readonly ProcessContext _context;
-
-        public SimulationInputService(ILogger<InputService> logger,
-            IOptions<DataInputOptions> dataInputOptions,
-            [FromKeyedServices(ProcessStep.Producer)] DataRecordQueue producerQueue,
-            ProcessContext context)
-        {
-            _logger = logger;
-            _dataInputOptions = dataInputOptions;
-            _producerQueue = producerQueue;
-            _context = context;
-        }
-        public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, ProcessContext context, CancellationToken cancellationToken)
-        {
-            var inputDir = _dataInputOptions.Value.InputDir;
-            _logger.LogInformation("***** simulation input service start, working dir: {InputDir}, thread id: {ThreadId} *****", inputDir, Environment.CurrentManagedThreadId);
-            var files = Directory.GetFiles(inputDir);
-            if (files.Length == 0)
-            {
-                _logger.LogInformation("No source files found in {InputDir}", inputDir);
-                return;
-            }
-            foreach (var tableName in tasksOptions.TableInfoConfig.Keys)
-            {
-                    var dataCount = tasksOptions.TableInfoConfig[tableName].SimulaRowCount;//当前表要生成的总数据量
-                    var companyTotallCount = 1000;//当前表每个公司生成的总数据量
-                    var tempRecords = new List<DataRecord>();
-                    var sk = DataHelper.shareKeys.First();
-                    var companyID = DataHelper.companyIds.First();
-
-                    var shareKeyInterval = 20000;//每个sharekey的数据量 
-                    var getShareKeyTimes = 0;//sharekey生成的次数,每生成一次，改变sharekey的值
-                    var getCompanyIDTimes = 0;//公司生成的次数,每生成一次，改变companyID的值
-                    var shareKeyIntervalCount = 0;
-
-                    var source = _dataInputOptions.Value.CreateSource?.Invoke(tableName);
-                var testRecord =await source.GetTestRecord();
-                    for (long i = 1; i <= dataCount; i++)
-                    {
-                        shareKeyIntervalCount++;
-                        if (shareKeyIntervalCount > shareKeyInterval)
-                        {
-                            sk = DataHelper.GetShareKey(getShareKeyTimes);
-                            getShareKeyTimes++;
-                            shareKeyIntervalCount = 0;
-                        }
-                        var fields = new string[testRecord.Fields.Length];
-                        Array.Copy(testRecord.Fields, fields, testRecord.Fields.Length);
-                        var record = new DataRecord(fields, testRecord.TableName, testRecord.Headers, companyID);
-                        //更新record的ID、OrderNo,ShardKey值
-                        if (record.Headers.Contains("ID"))
-                        {
-                            var index = Array.IndexOf(record.Headers, "ID");
-                            if (index > -1)
-                            {
-                                record.Fields[index] = i.ToString();
-                            }
-                        }
-                        if (record.TableName == "order_box_block" && record.Headers.Contains("BoxID"))
-                        {
-                            var index = Array.IndexOf(record.Headers, "BoxID");
-                            if (index > -1)
-                            {
-                                record.Fields[index] = i.ToString();
-                            }
-                        }
-                        if ((record.TableName == "order_block_plan_item" || record.TableName == "order_package_item") && record.Headers.Contains("ItemID"))
-                        {
-                            var index = Array.IndexOf(record.Headers, "ItemID");
-                            if (index > -1)
-                            {
-                                record.Fields[index] = i.ToString();
-                            }
-                        }
-                        if (record.TableName == "order" && record.Headers.Contains("OrderNo"))
-                        {
-                            var index = Array.IndexOf(record.Headers, "OrderNo");
-                            if (index > -1)
-                            {
-                                record.Fields[index] = i.ToString();
-                            }
-                        }
-                        if (record.Headers.Contains("ShardKey"))
-                        {
-                            var index = Array.IndexOf(record.Headers, "ShardKey");
-                            if (index > -1)
-                            {
-                                record.Fields[index] = sk.ToString();
-                            }
-                        }
-                        tempRecords.Add(record);
-                        if (tempRecords.Count >= companyTotallCount || i >= dataCount - 1)
-                        {
-                            foreach (var rc in tempRecords)
-                            {
-                                _context.AddInput();
-                                _producerQueue.Enqueue(rc);
-                                if (cancellationToken.IsCancellationRequested)
-                                    return;
-                            }
-                            tempRecords.Clear();
-                            companyID = DataHelper.GetCompanyId(getCompanyIDTimes);
-                            getCompanyIDTimes++;
-                        }
-                    }
-                    _logger.LogInformation("table:'{tableName}' simulation input completed", tableName);
-                //}
-                //_logger.LogInformation("File '{File}' input completed", Path.GetFileName(sqlPath));
-            }
-
-            _context.CompleteInput();
-            _logger.LogInformation("***** Csv input service completed *****");
-        }
-    }
-#endif
-}

MesETL.App/Helpers/DatabaseHelper.cs

@@ -1,7 +1,7 @@
 using System.Data;
 using MySqlConnector;
 
-namespace ConsoleApp2.Helpers.Database;
+namespace ConsoleApp2.Helpers;
 
 public static class DatabaseHelper
 {
@@ -13,7 +13,7 @@ public static class DatabaseHelper
         await using var cmd = conn.CreateCommand();
         cmd.CommandText = sql;
         var ds = new DataSet();
-        var adapter = new MySqlDataAdapter(cmd).Fill(ds);
+        new MySqlDataAdapter(cmd).Fill(ds);
         return ds;
     }
 

MesETL.App/Helpers/Extensions.IEnumerable.cs

@@ -2,7 +2,7 @@
 using System.Reflection;
 
 namespace ConsoleApp2.Helpers;
-
+#nullable disable
 public static class EnumerableExtensions
 {
     public static string ToMarkdownTable<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicMethods)]T>(this IEnumerable<T> source)

MesETL.App/HostedServices/MainHostedService.cs

@@ -1,7 +1,6 @@
 using System.Diagnostics;
 using System.Text;
 using ConsoleApp2.Helpers;
-using ConsoleApp2.Helpers.Database;
 using ConsoleApp2.HostedServices.Abstractions;
 using ConsoleApp2.Options;
 using ConsoleApp2.Services;
@@ -10,6 +9,7 @@ using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using MySqlConnector;
 
 namespace ConsoleApp2.HostedServices;
 
@@ -19,6 +19,7 @@ public class MainHostedService : BackgroundService
     private readonly IInputService _input;
     private readonly ITransformService _transform;
     private readonly IOutputService _output;
+    private readonly TaskMonitorService _taskMonitor;
     private readonly ILogger _logger;
     private readonly ProcessContext _context;
     
@@ -33,7 +34,8 @@ public class MainHostedService : BackgroundService
         IOptions<TenantDbOptions> tenantDbOptions, 
         IOptions<DatabaseOutputOptions> databaseOptions, 
         IConfiguration config, 
-        ProcessContext context)
+        ProcessContext context, 
+        TaskMonitorService taskMonitor)
     {
         _input = input;
         _transform = transform;
@@ -43,13 +45,23 @@ public class MainHostedService : BackgroundService
         _databaseOptions = databaseOptions;
         _config = config;
         _context = context;
+        _taskMonitor = taskMonitor;
     }
 
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
     {
-        _stopwatch = Stopwatch.StartNew();
+        var command = _config["Command"];
-        await SetVariableAsync(); // 开启延迟写入，禁用重做日志 >>> 重做日志处于禁用状态时绝对不要关闭数据库服务！
+        if (!string.IsNullOrEmpty(command))
+        {
+            _logger.LogInformation("***** Running Sql Command *****");
+            await ExecuteEachDatabase(command, stoppingToken);
+            Environment.Exit(0);
+        }
         
+        _stopwatch = Stopwatch.StartNew();
+        await SetVariableAsync(); // 开启延迟写入，禁用重做日志 >>> 重做日志处于禁用状态时不要关闭数据库服务！
+
+        var monitorTask = Task.Run(async () => await _taskMonitor.Monitor(stoppingToken), stoppingToken);
         var inputTask = ExecuteAndCatch(
             async () => await _input.ExecuteAsync(stoppingToken), "文件输入程序出现异常", stoppingToken);
         var transformTask = ExecuteAndCatch(
@@ -69,8 +81,6 @@ public class MainHostedService : BackgroundService
             await ExportResultAsync();
             _logger.LogInformation("The execution result export to {Path}",
                 Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"Result-{ErrorRecorder.UID}.md"));
-            if (_config["RestoreIndex"] is not null)
-                await RestoreIndexAsync();
 
             Environment.Exit(0);
         }
@@ -116,23 +126,22 @@ public class MainHostedService : BackgroundService
         }
     }
 
-    /// <summary>
+    private async Task ExecuteEachDatabase(string command, CancellationToken cancellationToken = default)
-    /// 还原所有数据库的索引...
-    /// </summary>
-    /// <returns></returns>
-    /// <exception cref="ApplicationException"></exception>
-    private async Task RestoreIndexAsync()
     {
         var databases = _tenantDbOptions.Value.DbList?.Keys 
                         ?? throw new ApplicationException("无法还原索引，因为分库配置中没有配置数据库");
-        var connStr = _databaseOptions.Value.ConnectionString 
-                      ?? throw new ApplicationException("无法还原索引，因为没有配置数据库连接字符串");
         var list = new List<Task>();
         foreach (var db in databases)
         {
-            var task = DatabaseHelper.NonQueryAsync(connStr + $";Database={db};",
+            var connStr = new MySqlConnectionStringBuilder(_databaseOptions.Value.ConnectionString
-                await File.ReadAllTextAsync(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "RestoreIndex.sql"))
+                                                           ?? throw new ApplicationException("无法还原索引，因为没有配置数据库连接字符串"))
-                );
+            {
+                ConnectionTimeout = 60,
+                DefaultCommandTimeout = 0,
+                Database = db
+            }.ConnectionString;
+            var task = Task.Run(async () => await DatabaseHelper.NonQueryAsync(connStr, command),
+                cancellationToken);
             list.Add(task);
         }
 

MesETL.App/HostedServices/TaskMonitorService.cs

@@ -3,43 +3,32 @@ using ConsoleApp2.Const;
 using ConsoleApp2.Services;
 using ConsoleApp2.Services.Loggers;
 using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Hosting;
-using Microsoft.Extensions.Logging;
 
 namespace ConsoleApp2.HostedServices;
 
 /// <summary>
 /// 任务监控
 /// </summary>
-public class TaskMonitorService : BackgroundService
+public class TaskMonitorService
 {
-    private readonly ILogger<TaskMonitorService> _logger;
     private readonly IEnumerable<ITaskMonitorLogger> _monitorLoggers;
     private readonly ProcessContext _context;
     private readonly DataRecordQueue _producerQueue;
     private readonly RecordQueuePool _queuePool;
 
-    public TaskMonitorService(
+    public TaskMonitorService(ProcessContext context,
-        ILogger<TaskMonitorService> logger,
-        ProcessContext context,
         [FromKeyedServices(ProcessStep.Produce)]
         DataRecordQueue producerQueue,
         RecordQueuePool queuePool,
         IEnumerable<ITaskMonitorLogger> monitorLoggers)
     {
-        _logger = logger;
         _context = context;
         _producerQueue = producerQueue;
         _queuePool = queuePool;
         _monitorLoggers = monitorLoggers;
     }
 
-    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    public async Task Monitor(CancellationToken stoppingToken)
-    {
-        await Task.Run(() => Monitor(stoppingToken), stoppingToken);
-    }
-
-    private async Task Monitor(CancellationToken stoppingToken)
     {
         var sw = Stopwatch.StartNew();
         var lastTime = sw.ElapsedMilliseconds;

MesETL.App/MesETL.App.csproj

@@ -6,6 +6,7 @@
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
         <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
+        <RootNamespace>ConsoleApp2</RootNamespace>
     </PropertyGroup>
 
     <ItemGroup>

MesETL.App/Program.cs

@@ -35,8 +35,8 @@ async Task RunProgram()
         {"--ConnectionString", "Output:ConnectionString"},
         {"-r", "RedisCache:Configuration"},
         {"--Redis", "RedisCache:Configuration"},
-        {"-i", "RestoreIndex"},
+        {"-c", "Command"},
-        {"--RestoreIndex", "RestoreIndex"}
+        {"--Command", "Command"}
     }); 
 
     var inputOptions = host.Configuration.GetRequiredSection("Input").Get<DataInputOptions>() 
@@ -547,10 +547,10 @@ async Task RunProgram()
     host.Services.AddSingleton<ITaskMonitorLogger, LoggerTaskMonitorLogger>();
 
     host.Services.AddHostedService<MainHostedService>();
-    host.Services.AddHostedService<TaskMonitorService>();
     host.Services.AddSingleton<IInputService, FileInputService>();
     host.Services.AddSingleton<ITransformService, TransformService>();
     host.Services.AddSingleton<IOutputService, OutputService>();
+    host.Services.AddSingleton<TaskMonitorService>();
     host.Services.AddRedisCache(redisOptions);
     var app = host.Build();
     await app.RunAsync();

MesETL.App/appsettings.json

@@ -12,7 +12,7 @@
   "Transform":{
     "StrictMode": true, // 设为true时如果数据转换发生错误，立刻停止程序
     "EnableFilter": false, // 启用数据过滤
-    "EnableReplacer": false, // 启用数据修改
+    "EnableReplacer": false, // 启用数据修改ma
     "EnableReBuilder": false, // 启用数据重建
     "CleanDate": "202301" // 当数据过滤开启时，删除这个时间之前的数据
   },

MesETL.Test/DataFix.cs

@@ -1,6 +1,5 @@
 using System.Data;
 using ConsoleApp2.Helpers;
-using ConsoleApp2.Helpers.Database;
 using MySqlConnector;
 using Xunit.Abstractions;
 

MesETL.Test/DatabaseToolBox.cs

@@ -1,6 +1,6 @@
 using System.Data;
 using System.Text;
-using ConsoleApp2.Helpers.Database;
+using ConsoleApp2.Helpers;
 using MySqlConnector;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
@@ -11,7 +11,7 @@ namespace TestProject1;
 public class DatabaseToolBox
 {
     private readonly ITestOutputHelper _output;
-    public const string ConnStr = "Server=127.0.0.1;Port=33309;UserId=root;Password=123456;";
+    public const string ConnStr = "Server=127.0.0.1;Port=3306;UserId=root;Password=cfmes123456;";
 
     public DatabaseToolBox(ITestOutputHelper output)
     {

MesETL.Test/MesDatabaseHelper.cs

@@ -1,4 +1,4 @@
-using ConsoleApp2.Helpers.Database;
+using ConsoleApp2.Helpers;
 
 namespace TestProject1;
 

MesETL.Test/MesETL.Test.csproj

@@ -7,6 +7,7 @@
 
         <IsPackable>false</IsPackable>
         <IsTestProject>true</IsTestProject>
+        <RootNamespace>TestProject1</RootNamespace>
     </PropertyGroup>
 
     <ItemGroup>
@@ -23,7 +24,7 @@
     </ItemGroup>
 
     <ItemGroup>
-      <ProjectReference Include="..\ConsoleApp2\ConsoleApp2.csproj" />
+      <ProjectReference Include="..\MesETL.App\MesETL.App.csproj" />
     </ItemGroup>
 
 </Project>

MesETL.sln

@@ -1,8 +1,8 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ConsoleApp2", "ConsoleApp2\ConsoleApp2.csproj", "{155E4B04-E88C-4BA4-AED2-B13E0A0432B5}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MesETL.App", "MesETL.App\MesETL.App.csproj", "{155E4B04-E88C-4BA4-AED2-B13E0A0432B5}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestProject1", "TestProject1\TestProject1.csproj", "{8679D5B6-5853-446E-9882-7B7A8E270500}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MesETL.Test", "MesETL.Test\MesETL.Test.csproj", "{8679D5B6-5853-446E-9882-7B7A8E270500}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution