Update

2024-02-01 13:41:59 +08:00 · 2024-02-01 13:41:59 +08:00 · 70cf0322e4
commit 70cf0322e4
parent 083090c62b
8 changed files with 68 additions and 113 deletions
--- a/ConsoleApp2/ConsoleApp2.csproj
+++ b/ConsoleApp2/ConsoleApp2.csproj
@ -10,6 +10,9 @@

    <ItemGroup>
      <None Remove="appsettings.json" />
+      <None Update="RestoreIndex.sql">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      </None>
    </ItemGroup>

    <ItemGroup>
--- a/ConsoleApp2/HostedServices/MainHostedService.cs
+++ b/ConsoleApp2/HostedServices/MainHostedService.cs
@ -48,6 +48,8 @@ public class MainHostedService : BackgroundService
    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        _stopwatch = Stopwatch.StartNew();
+        await SetVariableAsync(); // 开启延迟写入，禁用重做日志 >>> 重做日志处于禁用状态时绝对不要关闭数据库服务！
+        
        var inputTask = ExecuteAndCatch(
            async () => await _input.ExecuteAsync(stoppingToken), "文件输入程序出现异常", stoppingToken);
        var transformTask = ExecuteAndCatch(
@ -61,7 +63,7 @@ public class MainHostedService : BackgroundService
        _logger.LogInformation("***** ElapseTime: {Time}", (_stopwatch.ElapsedMilliseconds / 1000f).ToString("F3"));
        await Task.Delay(5000, stoppingToken);

-
+        await SetVariableAsync(false); // 关闭延迟写入，开启重做日志
        if (!stoppingToken.IsCancellationRequested)
        {
            await ExportResultAsync();
@ -92,12 +94,34 @@ public class MainHostedService : BackgroundService
        }, ct);
    }

+    private async Task SetVariableAsync(bool enable = true)
+    {
+        var connStr = _databaseOptions.Value.ConnectionString
+                      ?? throw new ApplicationException("无法还原索引，因为分库配置中没有配置数据库");
+        if (enable)
+        {
+            await DatabaseHelper.NonQueryAsync(connStr,
+                """
+                SET GLOBAL innodb_flush_log_at_trx_commit = 0;
+                ALTER INSTANCE DISABLE INNODB REDO_LOG;
+                """);
+        }
+        else
+        {
+            await DatabaseHelper.NonQueryAsync(connStr,
+                """
+                SET GLOBAL innodb_flush_log_at_trx_commit = 1;
+                ALTER INSTANCE ENABLE INNODB REDO_LOG;
+                """);
+        }
+    }
+    
    /// <summary>
    /// 还原所有数据库的索引...
    /// </summary>
    /// <returns></returns>
    /// <exception cref="ApplicationException"></exception>
-    private Task RestoreIndexAsync()
+    private async Task RestoreIndexAsync()
    {
        var databases = _tenantDbOptions.Value.DbList?.Keys 
                        ?? throw new ApplicationException("无法还原索引，因为分库配置中没有配置数据库");
@ -107,85 +131,12 @@ public class MainHostedService : BackgroundService
        foreach(var db in databases)
        {
            var task = DatabaseHelper.NonQueryAsync(connStr + $";Database={db};",
-                """
-                CREATE INDEX `idx_CompanyID` ON `machine` (`CompanyID`);
-
-                CREATE INDEX `idx_companyid` ON `order` (`CompanyID`);
-
-                CREATE INDEX `idx_CompanyID` ON `order_block_plan` (`CompanyID`);
-
-                CREATE INDEX `idx_PlanID` ON `order_block_plan_item` (`PlanID`);
-
-                CREATE INDEX `idx_orderno` ON `order_box_block` (`OrderNo`);
-
-                CREATE INDEX `index_OrderNo` ON `order_data_block` (`OrderNo`);
-
-                CREATE INDEX `index_OrderNo` ON `order_data_goods` (`OrderNo`);
-
-                CREATE INDEX `index_OrderNo` ON `order_data_parts` (`OrderNo`);
-
-                CREATE INDEX `index_ItemNo` ON `order_item` (`ItemNo`);
-
-                CREATE INDEX `index_OrderNo` ON `order_item` (`OrderNo`);
-
-                CREATE INDEX `index_PackageID` ON `order_item` (`PackageID`);
-
-                CREATE INDEX `index_PlanID` ON `order_item` (`PlanID`);
-
-                CREATE INDEX `idx_OrderNo` ON `order_module` (`OrderNo`);
-
-                CREATE INDEX `index_OrderNo` ON `order_module_extra` (`OrderNo`);
-
-                CREATE INDEX `index_OrderNo` ON `order_module_item` (`OrderNo`);
-
-                CREATE INDEX `idx_OrderNo` ON `order_package` (`OrderNo`);
-
-                CREATE INDEX `idx_PakageNo` ON `order_package` (`PakageNo`);
-
-                CREATE INDEX `idx_PackageID` ON `order_package_item` (`PackageID`);
-
-                CREATE INDEX `idx_companyid` ON `order_patch_detail` (`CompanyID`);
-
-                CREATE INDEX `idx_OrderNo` ON `order_process` (`OrderNo`);
-
-                CREATE INDEX `index_CompanyID` ON `order_process_schdule` (`CompanyID`);
-
-                CREATE INDEX `IX_order_process_step_OrderProcessID` ON `order_process_step` (`OrderProcessID`);
-
-                CREATE INDEX `idx_OrderProcessID` ON `order_process_step_item` (`OrderProcessID`);
-
-                CREATE INDEX `idx_OrderProcessStepID` ON `order_process_step_item` (`OrderProcessStepID`);
-
-                CREATE INDEX `idx_CompanyID` ON `order_scrap_board` (`CompanyID`);
-
-                CREATE INDEX `idx_CompanyID` ON `process_group` (`CompanyID`);
-
-                CREATE INDEX `idx_CompanyID` ON `process_info` (`CompanyID`);
-
-                CREATE INDEX `index_CompanyID` ON `process_item_exp` (`CompanyID`);
-
-                CREATE INDEX `idx_CompanyID` ON `process_schdule_capacity` (`CompanyID`);
-
-                CREATE INDEX `idx_CompanyID` ON `process_step_efficiency` (`CompanyID`);
-
-                CREATE INDEX `idx_CompanyID` ON `report_template` (`CompanyID`);
-
-                CREATE INDEX `indx_OrderNo` ON `simple_package` (`OrderNo`);
-
-                CREATE INDEX `idx_CompanyID` ON `simple_plan_order` (`CompanyID`);
-
-                CREATE INDEX `idx_CompanyID` ON `sys_config` (`CompanyID`);
-
-                CREATE INDEX `idx` ON `work_calendar` (`CompanyID`);
-
-                CREATE INDEX `idx_CompanyID` ON `work_shift` (`CompanyID`);
-
-                CREATE INDEX `IX_work_time_ShiftID` ON `work_time` (`ShiftID`);
-                """);
+                await File.ReadAllTextAsync(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "RestoreIndex.sql"))
+                );
            list.Add(task);
        }

-        return Task.WhenAll(list);
+        await Task.WhenAll(list);
    }

    private async Task ExportResultAsync()
--- a/ConsoleApp2/HostedServices/OutputService.cs
+++ b/ConsoleApp2/HostedServices/OutputService.cs
@ -5,6 +5,7 @@ using ConsoleApp2.Services;
 using ConsoleApp2.Services.ErrorRecorder;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using MySqlConnector;
 using MySqlDestination = ConsoleApp2.Services.ETL.MySqlDestination;
 using TaskExtensions = ConsoleApp2.Helpers.TaskExtensions;

@ -110,9 +111,19 @@ public class OutputService : IOutputService

    private async Task FlushAsync(string dbName, IEnumerable<DataRecord> records)
    {
-        var connStr = _outputOptions.Value.ConnectionString ??
-                      throw new InvalidOperationException("连接字符串为空");
-        await using var output = new MySqlDestination($"{connStr};Database={dbName};", _logger,
+        var connStr = new MySqlConnectionStringBuilder(_outputOptions.Value.ConnectionString 
+                                                       ?? throw new ApplicationException("未配置数据库连接字符串"))
+            {
+                CharacterSet = "utf8mb4",
+                AllowUserVariables = true,
+                IgnoreCommandTransaction = true,
+                TreatTinyAsBoolean = false,
+                ConnectionTimeout = 60,
+                DefaultCommandTimeout = 0,
+                SslMode = MySqlSslMode.None,
+                Database = dbName
+            }.ConnectionString;
+        await using var output = new MySqlDestination(connStr, _logger,
            _outputOptions, _errorRecorderFactory.CreateOutput(dbName), _context);

        var tableOutput = new Dictionary<string, int>();
--- a/ConsoleApp2/Options/DataInputOptions.cs
+++ b/ConsoleApp2/Options/DataInputOptions.cs
@ -19,11 +19,6 @@ namespace ConsoleApp2.Options
        public string Delimiter { get; set; } = ",";

        #endregion
-        
-        /// <summary>
-        /// yyyyMM
-        /// </summary>
-        public string CleanDate { get; set; } = "202301";

        #region Mock

--- a/ConsoleApp2/Options/DataTransformOptions.cs
+++ b/ConsoleApp2/Options/DataTransformOptions.cs
@ -19,6 +19,11 @@ public class DataTransformOptions
    public bool EnableReplacer { get; set; } = true;
    public bool EnableReBuilder { get; set; } = true;
    
+    /// <summary>
+    /// yyyyMM
+    /// </summary>
+    public string CleanDate { get; set; } = "202301";
+    
    /// <summary>
    /// Record -> Database name
    /// 对记录进行数据库过滤
--- a/ConsoleApp2/Program.cs
+++ b/ConsoleApp2/Program.cs
@ -59,8 +59,8 @@ async Task RunProgram()
    host.Services.Configure<TenantDbOptions>(tenantDbSection);
    host.Services.Configure<RedisCacheOptions>(redisSection);
    
-    var oldestTime = DateTime.ParseExact(inputOptions.CleanDate, "yyyyMM", System.Globalization.DateTimeFormatInfo.InvariantInfo);
-    var oldestTimeInt = int.Parse(inputOptions.CleanDate);
+    var oldestTime = DateTime.ParseExact(transformOptions.CleanDate, "yyyyMM", System.Globalization.DateTimeFormatInfo.InvariantInfo);
+    var oldestTimeInt = int.Parse(transformOptions.CleanDate);
    
    // 输入配置
    host.Services.Configure<DataInputOptions>(options =>
@ -453,16 +453,7 @@ async Task RunProgram()
    
    host.Services.Configure<DatabaseOutputOptions>(options =>
    {
-        options.ConnectionString = new MySqlConnectionStringBuilder(outputOptions.ConnectionString ?? throw new ApplicationException("未配置数据库连接字符串"))
-        {
-            CharacterSet = "utf8mb4",
-            AllowUserVariables = true,
-            IgnoreCommandTransaction = true,
-            TreatTinyAsBoolean = false,
-            ConnectionTimeout = 60,
-            DefaultCommandTimeout = 0,
-            SslMode = MySqlSslMode.None,
-        }.ConnectionString;
+        options.ConnectionString = outputOptions.ConnectionString;
        options.FlushCount = outputOptions.FlushCount;
        options.MaxAllowedPacket = outputOptions.MaxAllowedPacket;
        options.MaxDatabaseOutputTask = outputOptions.MaxDatabaseOutputTask;
--- a/ConsoleApp2/Services/ETL/MySqlDestination.cs
+++ b/ConsoleApp2/Services/ETL/MySqlDestination.cs
@ -137,7 +137,7 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable

                    // 在这里处理特殊列
                    #region HandleFields
-                    if (field == "\\N")
+                    if (field.Length == 2 && field == @"\N") // MyDumper NULL
                    {
                        recordSb.Append("NULL");
                        goto Escape;
--- a/ConsoleApp2/appsettings.json
+++ b/ConsoleApp2/appsettings.json
@ -5,25 +5,24 @@
    }
  },
  "Input":{
-    "InputDir": "D:\\Dump\\MockData",
-    "CleanDate": "202301",
-    "StrictMode": false,
-    "UseMock": true,
-    "MockCountMultiplier": 0.05
+    "InputDir": "D:\\Dump\\MockData", // Csv数据输入目录
+    "UseMock": true, // 使用模拟数据进行测试
+    "MockCountMultiplier": 0.5 // 模拟数据量级的乘数
  },
  "Transform":{
-    "StrictMode": true,
-    "EnableFilter": false,
-    "EnableReplacer": false,
-    "EnableReBuilder": false
+    "StrictMode": true, // 设为true时如果数据转换发生错误，立刻停止程序
+    "EnableFilter": false, // 启用数据过滤
+    "EnableReplacer": false, // 启用数据修改
+    "EnableReBuilder": false, // 启用数据重建
+    "CleanDate": "202301" // 当数据过滤开启时，删除这个时间之前的数据
  },
  "Output":{
    "ConnectionString": "Server=127.0.0.1;Port=3306;UserId=root;Password=cfmes123456;", // 要分库，不用加'Database='了
-    "MaxAllowedPacket": 67108864,
-    "FlushCount": 20000,
-    "MaxDatabaseOutputTask" : 4,
-    "TreatJsonAsHex": false,
-    "RestoreIndex": true
+    "MaxAllowedPacket": 67108864, 
+    "FlushCount": 10000, // 每次提交记录条数
+    "MaxDatabaseOutputTask" : 4, // 每个数据库最大提交任务数
+    "TreatJsonAsHex": false, // 将json列作为16进制格式输出(0x前缀)
+    "RestoreIndex": true // 在输出结束以后还原数据库的索引(调用RestoreIndex.sql)
  },
  "RedisCache": {
    "Configuration": "192.168.1.246:6380",