using System.Reflection.Metadata;
using System.Text;
using ConsoleApp2.Helpers;
using ConsoleApp2.Options;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MySqlConnector;
using ServiceStack;
namespace ConsoleApp2.Services;
///
/// Mysql导出
///
public class MySqlDestination : IDisposable, IAsyncDisposable
{
private readonly Dictionary> _recordCache;
private readonly MySqlConnection _conn;
private readonly ILogger _logger;
private readonly bool _prettyOutput;
private readonly int _maxAllowPacket;
private readonly ProcessContext _context;
public MySqlDestination(string connStr, ILogger logger, ProcessContext context,bool prettyOutput = false)
{
_conn = new MySqlConnection(connStr);
_conn.Open();
_recordCache = new Dictionary>();
_logger = logger;
_context = context;
_prettyOutput = prettyOutput;
}
public Task WriteRecordAsync(DataRecord record)
{
_recordCache.AddOrUpdate(record.TableName, [record], (key, value) =>
{
value.Add(record);
return value;
});
return Task.CompletedTask;
}
public async Task WriteRecordsAsync(IEnumerable records)
{
foreach (var record in records)
{
await WriteRecordAsync(record);
}
}
public async Task FlushAsync(int maxAllowPacket, IOptions transOptions)
{
if (_recordCache.Count == 0)
return;
var cmd = _conn.CreateCommand();
cmd.CommandTimeout = 3 * 60;
var excuseList = GetExcuseList(_recordCache, maxAllowPacket, transOptions, _prettyOutput);
try
{
foreach (var insertSql in excuseList)
{
cmd.CommandText = insertSql;
await cmd.ExecuteNonQueryAsync();
}
_recordCache.Clear();
}
catch (Exception e)
{
_logger.LogCritical(e, "Error when flushing records, sql: {Sql}", cmd.CommandText.Omit(1000));
_context.AddException(e);
throw;
}
finally
{
await cmd.DisposeAsync();
}
}
public static IList GetExcuseList(IDictionary> tableRecords,int maxAllowPacket, IOptions transOptions,
bool prettyOutput = false)
{
var resultList = new List();
var headerSb = new StringBuilder();
var recordSb = new StringBuilder();
foreach (var (tableName, records) in tableRecords)
{
if (records.Count == 0)
continue;
headerSb.Append($"INSERT INTO `{tableName}`(");
for (var i = 0; i < records[0].Headers.Length; i++)
{
var header = records[0].Headers[i];
headerSb.Append($"`{header}`");
if (i != records[0].Headers.Length - 1)
headerSb.Append(',');
}
headerSb.Append(") VALUES ");
if (prettyOutput)
headerSb.AppendLine();
var sbList = new List();
var currentLength = headerSb.Length;
for (var i = 0; i < records.Count; i++)
{
var record = records[i];
recordSb.Append('(');
for (var j = 0; j < record.Fields.Length; j++)
{
var field = record.Fields[j];
var header = record.Headers[j];
if (transOptions.Value.GetColumnType(record.TableName, header) ==ColumnType.Blob)
{
if (string.IsNullOrEmpty(field))
{
recordSb.Append("NULL");
}
else
recordSb.Append("0x"+field);
}
else
recordSb.Append(field);
if (j != record.Fields.Length - 1)
recordSb.Append(',');
}
recordSb.Append(')');
//if (i != records.Count - 1) // not last field
// recordSb.Append(',');
if (prettyOutput) recordSb.AppendLine();
if (currentLength + recordSb.Length >= maxAllowPacket)
{
var insertSb = new StringBuilder(headerSb.ToString());
insertSb.Append(string.Join(",", sbList));
insertSb.Append(";");
resultList.Add(insertSb.ToString());
insertSb.Clear();
sbList.Clear();
sbList.Add(recordSb.ToString());
currentLength = headerSb.Length + 1;//逗号长度加1
}
else
{
sbList.Add(recordSb.ToString());
}
currentLength += recordSb.Length;
recordSb.Clear();
}
if (sbList.Count > 0)
{
var insertSb = new StringBuilder(headerSb.ToString());
insertSb.Append(string.Join(",", sbList));
insertSb.Append(";");
resultList.Add(insertSb.ToString());
insertSb.Clear();
}
headerSb.Clear();
}
return resultList;
}
public void Dispose()
{
_conn.Close();
_conn.Dispose();
}
public async ValueTask DisposeAsync()
{
await _conn.CloseAsync();
await _conn.DisposeAsync();
}
}