CZY/MES-ETL
1
0
Fork 0
Code Pull Requests Releases Activity

2025迁移版本，多项规则修改

Browse Source
This commit is contained in:
陈梓阳
2024-12-10 14:03:09 +08:00
parent dc239c776e
commit 0e28d639c1
34 changed files with 1075 additions and 564 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
MesETL.App/Services/ETL/MySqlDestination.cs
View File

@@ -15,6 +15,9 @@ namespace MesETL.App.Services.ETL;
/// </summary>
public partial class MySqlDestination : IDisposable, IAsyncDisposable
{
/// <summary>
/// table => records
/// </summary>
private readonly Dictionary<string, IList<DataRecord>> _recordCache;
private readonly MySqlConnection _conn;
private readonly ILogger _logger;
@@ -66,8 +69,8 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
try
{
var excuseList = GetExcuseList(_recordCache, maxAllowPacket);
foreach (var insertSql in excuseList)
var executionList = GetExecutionList(_recordCache, maxAllowPacket);
foreach (var insertSql in executionList)
{
cmd.CommandText = insertSql;
try
@@ -103,7 +106,7 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
[GeneratedRegex("INSERT INTO `([^`]+)`")]
private static partial Regex MatchTableName();
public IEnumerable<string> GetExcuseList(IDictionary<string, IList<DataRecord>> tableRecords,int maxAllowPacket)
public IEnumerable<string> GetExecutionList(IDictionary<string, IList<DataRecord>> tableRecords, int maxAllowPacket)
{
var sb = new StringBuilder("SET AUTOCOMMIT = 1;\n");
var appendCount = 0;
@@ -116,13 +119,16 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
StartBuild:
var noCommas = true;
// 标准列顺序，插入时的字段需要按照该顺序排列
var headers = records[0].Headers;
// INSERT INTO ... VALUES >>>
sb.Append($"INSERT INTO `{tableName}`(");
for (var i = 0; i < records[0].Headers.Count; i++)
for (var i = 0; i < headers.Count; i++)
{
var header = records[0].Headers[i];
sb.Append($"`{header}`");
if (i != records[0].Headers.Count - 1)
if (i != headers.Count - 1)
sb.Append(',');
}
@@ -132,11 +138,20 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
for (;recordIdx < records.Count; recordIdx++)
{
var record = records[recordIdx];
// 数据列校验
if (record.Headers.Count != headers.Count)
{
throw new InvalidOperationException($"数据异常，数据列数量出现冲突，表名：{tableName}");
}
var recordSb = new StringBuilder();
recordSb.Append('(');
for (var fieldIdx = 0; fieldIdx < record.Fields.Count; fieldIdx++)
for (var idx = 0; idx < headers.Count; idx++)
{
var field = record.Fields[fieldIdx];
var header = headers[idx];
// TODO: 可进行性能优化
var field = record[header];
// 在这里处理特殊列
#region HandleFields
@@ -147,7 +162,7 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
goto Escape;
}
switch (_options.Value.GetColumnType(record.TableName, record.Headers[fieldIdx]))
switch (_options.Value.GetColumnType(record.TableName, header))
{
case ColumnType.Text:
if(string.IsNullOrEmpty(field))
@@ -163,12 +178,12 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
recordSb.Append(ConstVar.Null);
else recordSb.Append($"0x{field}");
break;
case ColumnType.Json:// 生产库没有JSON列，仅用于测试库进行测试
if(string.IsNullOrEmpty(field))
recordSb.Append("'[]'"); // JObject or JArray?
case ColumnType.Json: // Mydumper v0.16.7-5导出的Json为字符串，且会将逗号转义，需要适配
if(string.IsNullOrEmpty(field))
recordSb.Append(ConstVar.Null);
else if (_options.Value.TreatJsonAsHex)
recordSb.Append($"_utf8mb4 0x{field}");
else recordSb.AppendLine(field);
else recordSb.AppendLine(field.Replace("\\,", ","));
break;
case ColumnType.UnDefine:
default:
@@ -179,7 +194,7 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
Escape:
#endregion
if (fieldIdx != record.Fields.Count - 1)
if (idx != headers.Count - 1)
recordSb.Append(',');
}