优化内存分配
This commit is contained in:
@@ -1,5 +1,4 @@
|
||||
using System.Reflection.Metadata;
|
||||
using System.Text;
|
||||
using System.Text;
|
||||
using ConsoleApp2.Helpers;
|
||||
using ConsoleApp2.Options;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -17,22 +16,18 @@ public class MySqlDestination : IDisposable, IAsyncDisposable
|
||||
private readonly Dictionary<string, IList<DataRecord>> _recordCache;
|
||||
private readonly MySqlConnection _conn;
|
||||
private readonly ILogger _logger;
|
||||
private readonly bool _prettyOutput;
|
||||
private readonly int _maxAllowPacket;
|
||||
private readonly ProcessContext _context;
|
||||
|
||||
public MySqlDestination(string connStr, ILogger logger, ProcessContext context,bool prettyOutput = false)
|
||||
private readonly IOptions<DataTransformOptions> _transformOptions;
|
||||
public MySqlDestination(string connStr, ILogger logger, ProcessContext context, IOptions<DataTransformOptions> transformOptions)
|
||||
{
|
||||
_conn = new MySqlConnection(connStr);
|
||||
_conn.Open();
|
||||
_recordCache = new Dictionary<string, IList<DataRecord>>();
|
||||
_logger = logger;
|
||||
_context = context;
|
||||
_prettyOutput = prettyOutput;
|
||||
|
||||
|
||||
|
||||
_transformOptions = transformOptions;
|
||||
}
|
||||
|
||||
public Task WriteRecordAsync(DataRecord record)
|
||||
{
|
||||
_recordCache.AddOrUpdate(record.TableName, [record], (key, value) =>
|
||||
@@ -51,20 +46,22 @@ public class MySqlDestination : IDisposable, IAsyncDisposable
|
||||
}
|
||||
}
|
||||
|
||||
public async Task FlushAsync(int maxAllowPacket, IOptions<DataTransformOptions> transOptions)
|
||||
public async Task FlushAsync(int maxAllowPacket)
|
||||
{
|
||||
if (_recordCache.Count == 0)
|
||||
return;
|
||||
|
||||
var cmd = _conn.CreateCommand();
|
||||
cmd.CommandTimeout = 3 * 60;
|
||||
var excuseList = GetExcuseList(_recordCache, maxAllowPacket, transOptions, _prettyOutput);
|
||||
|
||||
try
|
||||
{
|
||||
var excuseList = GetExcuseList(_recordCache, maxAllowPacket).ToList();
|
||||
foreach (var insertSql in excuseList)
|
||||
{
|
||||
cmd.CommandText = insertSql;
|
||||
await cmd.ExecuteNonQueryAsync();
|
||||
_logger.LogInformation(@"do insert completed!size:{Length}", cmd.CommandText.Length);
|
||||
}
|
||||
_recordCache.Clear();
|
||||
}
|
||||
@@ -80,89 +77,94 @@ public class MySqlDestination : IDisposable, IAsyncDisposable
|
||||
}
|
||||
}
|
||||
|
||||
public static IList<string> GetExcuseList(IDictionary<string, IList<DataRecord>> tableRecords,int maxAllowPacket, IOptions<DataTransformOptions> transOptions,
|
||||
bool prettyOutput = false)
|
||||
public IEnumerable<string> GetExcuseList(IDictionary<string, IList<DataRecord>> tableRecords,int maxAllowPacket)
|
||||
{
|
||||
var resultList = new List<string>();
|
||||
var headerSb = new StringBuilder();
|
||||
var recordSb = new StringBuilder();
|
||||
var sb = new StringBuilder();
|
||||
foreach (var (tableName, records) in tableRecords)
|
||||
{
|
||||
if (records.Count == 0)
|
||||
continue;
|
||||
headerSb.Append($"INSERT INTO `{tableName}`(");
|
||||
|
||||
var recordIdx = 0;
|
||||
StartBuild:
|
||||
var noCommas = true;
|
||||
|
||||
// INSERT INTO ... VALUES >>>
|
||||
sb.Append($"INSERT INTO `{tableName}`(");
|
||||
for (var i = 0; i < records[0].Headers.Length; i++)
|
||||
{
|
||||
var header = records[0].Headers[i];
|
||||
headerSb.Append($"`{header}`");
|
||||
sb.Append($"`{header}`");
|
||||
if (i != records[0].Headers.Length - 1)
|
||||
headerSb.Append(',');
|
||||
sb.Append(',');
|
||||
}
|
||||
|
||||
headerSb.Append(") VALUES ");
|
||||
if (prettyOutput)
|
||||
headerSb.AppendLine();
|
||||
|
||||
var sbList = new List<string>();
|
||||
var currentLength = headerSb.Length;
|
||||
for (var i = 0; i < records.Count; i++)
|
||||
sb.Append(") VALUES ");
|
||||
|
||||
// ([FIELDS]), >>>
|
||||
for (;recordIdx < records.Count; recordIdx++)
|
||||
{
|
||||
var record = records[i];
|
||||
var record = records[recordIdx];
|
||||
var recordSb = new StringBuilder();
|
||||
recordSb.Append('(');
|
||||
for (var j = 0; j < record.Fields.Length; j++)
|
||||
for (var fieldIdx = 0; fieldIdx < record.Fields.Length; fieldIdx++)
|
||||
{
|
||||
var field = record.Fields[j];
|
||||
var header = record.Headers[j];
|
||||
if (transOptions.Value.GetColumnType(record.TableName, header) ==ColumnType.Blob)
|
||||
var field = record.Fields[fieldIdx];
|
||||
|
||||
// 在这里处理特殊列
|
||||
#region HandleFields
|
||||
if (field == "\\N")
|
||||
{
|
||||
if (string.IsNullOrEmpty(field))
|
||||
{
|
||||
recordSb.Append("NULL");
|
||||
}
|
||||
else
|
||||
recordSb.Append("0x"+field);
|
||||
recordSb.Append("NULL");
|
||||
goto Escape;
|
||||
}
|
||||
else
|
||||
recordSb.Append(field);
|
||||
if (j != record.Fields.Length - 1)
|
||||
|
||||
switch (_transformOptions.Value.GetColumnType(record.TableName, record.Headers[fieldIdx]))
|
||||
{
|
||||
case ColumnType.Text:
|
||||
recordSb.Append(string.IsNullOrEmpty(field)
|
||||
? "''"
|
||||
: _transformOptions.Value.TransformBinary?.Invoke(field) ?? field);
|
||||
break;
|
||||
case ColumnType.Blob:
|
||||
if (string.IsNullOrEmpty(field))
|
||||
recordSb.Append("NULL");
|
||||
else recordSb.Append($"0x{field}");
|
||||
break;
|
||||
case ColumnType.UnDefine:
|
||||
default:
|
||||
recordSb.Append(field);
|
||||
break;
|
||||
}
|
||||
|
||||
Escape:
|
||||
|
||||
#endregion
|
||||
if (fieldIdx != record.Fields.Length - 1)
|
||||
recordSb.Append(',');
|
||||
}
|
||||
|
||||
recordSb.Append(')');
|
||||
|
||||
//if (i != records.Count - 1) // not last field
|
||||
// recordSb.Append(',');
|
||||
if (prettyOutput) recordSb.AppendLine();
|
||||
// 若字符数量即将大于限制,则返回SQL,清空StringBuilder,保留当前记录的索引值,然后转到StartBuild标签重新开始一轮INSERT
|
||||
if (sb.Length + recordSb.Length + 1 > maxAllowPacket)
|
||||
{
|
||||
sb.Append(';');
|
||||
yield return sb.ToString();
|
||||
sb.Clear();
|
||||
goto StartBuild;
|
||||
}
|
||||
|
||||
if (currentLength + recordSb.Length >= maxAllowPacket)
|
||||
{
|
||||
var insertSb = new StringBuilder(headerSb.ToString());
|
||||
insertSb.Append(string.Join(",", sbList));
|
||||
insertSb.Append(";");
|
||||
resultList.Add(insertSb.ToString());
|
||||
insertSb.Clear();
|
||||
sbList.Clear();
|
||||
sbList.Add(recordSb.ToString());
|
||||
currentLength = headerSb.Length + 1;//逗号长度加1
|
||||
}
|
||||
else
|
||||
{
|
||||
sbList.Add(recordSb.ToString());
|
||||
}
|
||||
currentLength += recordSb.Length;
|
||||
recordSb.Clear();
|
||||
if (!noCommas)
|
||||
sb.Append(',').AppendLine();
|
||||
noCommas = false;
|
||||
sb.Append(recordSb); // StringBuilder.Append(StringBuilder)不会分配多余的内存
|
||||
}
|
||||
if (sbList.Count > 0)
|
||||
{
|
||||
var insertSb = new StringBuilder(headerSb.ToString());
|
||||
insertSb.Append(string.Join(",", sbList));
|
||||
insertSb.Append(";");
|
||||
resultList.Add(insertSb.ToString());
|
||||
insertSb.Clear();
|
||||
}
|
||||
headerSb.Clear();
|
||||
|
||||
sb.Append(';');
|
||||
yield return sb.ToString();
|
||||
sb.Clear();
|
||||
}
|
||||
return resultList;
|
||||
}
|
||||
|
||||
|
||||
|
Reference in New Issue
Block a user