2025迁移版本，多项规则修改

2024-12-10 14:03:09 +08:00
parent dc239c776e
commit 0e28d639c1
34 changed files with 1075 additions and 564 deletions
--- a/MesETL.App/Services/ETL/MySqlDestination.cs
+++ b/MesETL.App/Services/ETL/MySqlDestination.cs
@@ -15,6 +15,9 @@ namespace MesETL.App.Services.ETL;
 /// </summary>
 public partial class MySqlDestination : IDisposable, IAsyncDisposable
 {
+    /// <summary>
+    /// table => records
+    /// </summary>
    private readonly Dictionary<string, IList<DataRecord>> _recordCache;
    private readonly MySqlConnection _conn;
    private readonly ILogger _logger;
@@ -66,8 +69,8 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
       
        try
        {
-            var excuseList = GetExcuseList(_recordCache, maxAllowPacket);
-            foreach (var insertSql in excuseList)
+            var executionList = GetExecutionList(_recordCache, maxAllowPacket);
+            foreach (var insertSql in executionList)
            {
                cmd.CommandText = insertSql;
                try
@@ -103,7 +106,7 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
    [GeneratedRegex("INSERT INTO `([^`]+)`")]
    private static partial Regex MatchTableName();

-    public IEnumerable<string> GetExcuseList(IDictionary<string, IList<DataRecord>> tableRecords,int maxAllowPacket)
+    public IEnumerable<string> GetExecutionList(IDictionary<string, IList<DataRecord>> tableRecords, int maxAllowPacket)
    {
        var sb = new StringBuilder("SET AUTOCOMMIT = 1;\n");
        var appendCount = 0;
@@ -116,13 +119,16 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
            StartBuild:
            var noCommas = true;
            
+            // 标准列顺序，插入时的字段需要按照该顺序排列
+            var headers = records[0].Headers;
+            
            // INSERT INTO ... VALUES >>>
            sb.Append($"INSERT INTO `{tableName}`(");
-            for (var i = 0; i < records[0].Headers.Count; i++)
+            for (var i = 0; i < headers.Count; i++)
            {
                var header = records[0].Headers[i];
                sb.Append($"`{header}`");
-                if (i != records[0].Headers.Count - 1)
+                if (i != headers.Count - 1)
                    sb.Append(',');
            }

@@ -132,11 +138,20 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
            for (;recordIdx < records.Count; recordIdx++)
            {
                var record = records[recordIdx];
+                
+                // 数据列校验
+                if (record.Headers.Count != headers.Count)
+                {
+                    throw new InvalidOperationException($"数据异常，数据列数量出现冲突，表名：{tableName}");
+                }
+                
                var recordSb = new StringBuilder();
                recordSb.Append('(');
-                for (var fieldIdx = 0; fieldIdx < record.Fields.Count; fieldIdx++)
+                for (var idx = 0; idx < headers.Count; idx++)
                {
-                    var field = record.Fields[fieldIdx];
+                    var header = headers[idx];
+                    // TODO: 可进行性能优化
+                    var field = record[header];

                    // 在这里处理特殊列
                    #region HandleFields
@@ -147,7 +162,7 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
                        goto Escape;
                    }
                    
-                    switch (_options.Value.GetColumnType(record.TableName, record.Headers[fieldIdx]))
+                    switch (_options.Value.GetColumnType(record.TableName, header))
                    {
                        case ColumnType.Text:
                            if(string.IsNullOrEmpty(field))
@@ -163,12 +178,12 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
                                recordSb.Append(ConstVar.Null);
                            else recordSb.Append($"0x{field}");
                            break;
-                        case ColumnType.Json:// 生产库没有JSON列，仅用于测试库进行测试
-                            if(string.IsNullOrEmpty(field))
-                                recordSb.Append("'[]'"); // JObject or JArray?
+                        case ColumnType.Json: // Mydumper v0.16.7-5导出的Json为字符串，且会将逗号转义，需要适配
+                            if(string.IsNullOrEmpty(field)) 
+                                recordSb.Append(ConstVar.Null);
                            else if (_options.Value.TreatJsonAsHex)
                                recordSb.Append($"_utf8mb4 0x{field}");
-                            else recordSb.AppendLine(field);
+                            else recordSb.AppendLine(field.Replace("\\,", ","));
                            break;
                        case ColumnType.UnDefine:
                        default:
@@ -179,7 +194,7 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
                    Escape:

                    #endregion
-                    if (fieldIdx != record.Fields.Count - 1)
+                    if (idx != headers.Count - 1)
                        recordSb.Append(',');
                }

--- a/MesETL.App/Services/Loggers/LoggerTaskMonitorLogger.cs
+++ b/MesETL.App/Services/Loggers/LoggerTaskMonitorLogger.cs
@@ -17,7 +17,7 @@ public class LoggerTaskMonitorLogger : ITaskMonitorLogger
        var sb = new StringBuilder();
        sb.Append($"{name}: {{");
        sb.AppendJoin(',', properties.Select((pair, i) => $" {pair.Key}: {pair.Value}"));
-        sb.Append('}');
+        sb.Append([' ', '}']);
        // var args = new List<string> { name };
        // properties.Aggregate(args, (args, pair) =>
        // {
--- a/MesETL.App/Services/ProcessContext.cs
+++ b/MesETL.App/Services/ProcessContext.cs
@@ -35,6 +35,8 @@ public class ProcessContext
        set => Interlocked.Exchange(ref _outputCount, value);
    }

+    public long MaxMemoryUsage { get; set; }
+

    // TableName -> Count
    public IReadOnlyDictionary<string, (long input, long output)> TableProgress => _tableProgress;
--- a/MesETL.App/Services/Seq/SeqConfig.cs
+++ b/MesETL.App/Services/Seq/SeqConfig.cs
@@ -0,0 +1,42 @@
+// ReSharper disable InconsistentNaming
+namespace MesETL.App.Services.Seq;
+
+public class SeqConfig(string Name, bool Recycle = true, int Step = 1, long Max = 999_999_999)
+{
+    public string Name { get; init; } = Name;
+    public bool Recycle { get; init; } = Recycle;
+    public int Step { get; init; } = Step;
+    public long Max { get; init; } = Max;
+
+    public static readonly SeqConfig ItemNo = new("seq_ItemNo", true, 1, 999_999_999);
+    public static readonly SeqConfig OrderModuleID = new("seq_order_module_id", false);
+    public static readonly SeqConfig OrderDataID = new("seq_order_data_id", false);
+    public static readonly SeqConfig OrderItemID = new("seq_order_item", false);
+    public static readonly SeqConfig ProcessStepID = new("seq_step_id", false);
+    public static readonly SeqConfig PackageNo = new("seq_pack_no", true, 1, 9_999_999);
+    public static readonly SeqConfig PlanNo = new("seq_plan_order", true, 1, 999_999);
+    public static readonly SeqConfig SimplePlanNo = new("seq_simple_plan_order", true, 1, 999_999);
+
+    // 下面这些类型的流水号在BaseService添加实体时进行生成
+    public static readonly SeqConfig MachineID = new("seq_machine_id", false);
+    public static readonly SeqConfig OrderBlockPlanID = new("seq_order_block_plan_id", false);
+    public static readonly SeqConfig OrderDataGoodsID = new("seq_order_data_goods_id", false);
+    public static readonly SeqConfig OrderPackageID = new("seq_order_pack_id", false);
+    public static readonly SeqConfig OrderProcessID = new("seq_order_process_id", false);
+    public static readonly SeqConfig OrderProcessStepItemID = new("seq_order_process_step_item_id", false);
+    public static readonly SeqConfig ProcessGroupID = new("seq_process_group_id", false);
+    public static readonly SeqConfig ProcessInfoID = new("seq_process_info_id", false);
+    public static readonly SeqConfig ProcessItemExpID = new("seq_process_item_exp_id", false);
+    public static readonly SeqConfig ProcessScheduleCapacityID = new("seq_process_schedule_capacity_id", false);
+    public static readonly SeqConfig ProcessStepEfficiencyID = new("seq_process_step_efficiency_id", false);
+    public static readonly SeqConfig ReportTemplateID = new("seq_report_template_id", false);
+    public static readonly SeqConfig SysConfigKey = new("seq_sys_config_key", false);
+    public static readonly SeqConfig WorkCalendarID = new("seq_work_calendar_id", false);
+    public static readonly SeqConfig WorkShiftID = new("seq_work_shift_id", false);
+    public static readonly SeqConfig WorkTimeID = new("seq_work_time_id", false);
+    public static readonly SeqConfig OrderPatchDetailID = new("seq_order_patch_detail_id", false);
+    public static readonly SeqConfig OrderModuleExtraID = new("seq_order_module_extra_id", false);
+    public static readonly SeqConfig SimplePackageID = new("seq_simple_pack_id", false);
+    public static readonly SeqConfig OrderModuleItemID = new("seq_order_module_item_id", false);
+    public static readonly SeqConfig OrderWaveGroupID = new("seq_order_wave_group_id", false);
+}
--- a/MesETL.App/Services/Seq/SeqService.cs
+++ b/MesETL.App/Services/Seq/SeqService.cs
@@ -0,0 +1,134 @@
+using System.Diagnostics;
+using System.Runtime.CompilerServices;
+using System.Text;
+using MesETL.App.Options;
+using MesETL.Shared.Helper;
+using Microsoft.Extensions.Options;
+using MySqlConnector;
+
+namespace MesETL.App.Services.Seq;
+
+public class SeqService
+{
+    private readonly string _connectionString;
+    private readonly Dictionary<SeqConfig, long> _cachedSequence;
+
+    public IReadOnlyDictionary<SeqConfig, long> CachedSequence => _cachedSequence;
+
+    public SeqService(IOptions<DatabaseOutputOptions> options)
+    {
+        var connStr = options.Value.ConnectionString ?? throw new ApplicationException("未配置输出数据库连接字符串");
+        var builder = new MySqlConnectionStringBuilder(connStr)
+        {
+            Database = "mes_global"
+        };
+        _connectionString = builder.ConnectionString;
+
+        _cachedSequence = new Dictionary<SeqConfig, long>();
+    }
+
+    private async Task<long> UpdateSequenceID(string name,int step,long max,bool recycle, int add)
+    {
+        var sql = new StringBuilder(
+            $"""
+            INSERT INTO seq (SeqName,CurrentVal,Increment,MinVal,MaxVal,UpdateTime)
+            VALUES ({name},{add},{step},1,{max},NOW()) 
+            ON DUPLICATE KEY UPDATE UpdateTime = NOW(),
+            """);
+        if (recycle)
+        {
+            sql.Append($"CurrentVal = (@updatedVal := IF(CurrentVal + {add} >= MaxVal, {add}, CurrentVal + {add}));");
+        }
+        else
+        {
+            sql.Append($"CurrentVal = (@updatedVal := CurrentVal + {add});");
+        }
+        sql.Append("SELECT @updatedVal;");
+        var result = await DatabaseHelper.QueryScalarAsync(_connectionString, sql.ToString());
+        return Convert.ToInt64(result);
+    }
+
+    public async Task<long> PeekKey(SeqConfig config)
+    {
+        var sql = $"SELECT CurrentVal FROM seq WHERE SeqName = '{config.Name}' LIMIT 1;";
+        return Convert.ToInt64(await DatabaseHelper.QueryScalarAsync(_connectionString, sql));
+    }
+    
+    public async Task<long[]> GetKeys(SeqConfig config, int count)
+    {
+        if (count < 1) return [];
+
+        var list = new long[count];
+        var add = config.Step * count;
+        var lastId = await UpdateSequenceID(config.Name, config.Step, config.Max, config.Recycle, add);
+        var step = Convert.ToInt64(config.Step);
+        for (var i = count - 1; i > -1; i--)
+        {
+            list[i] = lastId;
+            lastId -= step;
+        }
+
+        return list;
+    }
+
+    /// <summary>
+    /// 添加并取得一个缓存的流水号
+    /// </summary>
+    /// <param name="config"></param>
+    /// <returns></returns>
+    public long AddCachedSeq(SeqConfig config)
+    {
+        if (!_cachedSequence.TryGetValue(config, out var val))
+        {
+            var seq = PeekKey(config).GetAwaiter().GetResult();
+            val = seq;
+            _cachedSequence[config] = val;
+        }
+
+        var step = config.Step;
+        if (config.Recycle)
+        {
+            val = val + step >= config.Max ? val : val + step;
+        }
+        else val += step;
+        _cachedSequence[config] = val;
+        return val;
+    }
+    
+    /// <summary>
+    /// 移除一个缓存的流水号
+    /// </summary>
+    /// <param name="config"></param>
+    public void RemoveCachedSeq(SeqConfig config)
+    {
+        _cachedSequence.Remove(config);
+    }
+    
+    /// <summary>
+    /// 清空所有缓存的流水号
+    /// </summary>
+    public void ClearCache()
+    {
+        _cachedSequence.Clear();
+    }
+
+    /// <summary>
+    /// 将缓存的流水号应用至数据库
+    /// </summary>
+    public async Task ApplyToDatabaseAsync()
+    {
+        var sql = GenerateCachedSeqSql();
+        await DatabaseHelper.NonQueryAsync(_connectionString, sql);
+    }
+
+    private string GenerateCachedSeqSql()
+    {
+        var sb = new StringBuilder();
+        foreach (var kv in _cachedSequence)
+        {
+            sb.AppendLine($"UPDATE seq SET CurrentVal = {kv.Value} WHERE SeqName = '{kv.Key.Name}';");
+        }
+
+        return sb.ToString();
+    }
+}