using System.Text; using ConsoleApp2.Helpers; using Microsoft.Extensions.Logging; using MySqlConnector; using ServiceStack; namespace ConsoleApp2.Services; /// /// Mysql导出 /// public class MySqlDestination : IDisposable, IAsyncDisposable { private readonly Dictionary> _recordCache; private readonly MySqlConnection _conn; private readonly ILogger _logger; private readonly bool _prettyOutput; private readonly int _maxAllowPacket; private readonly ProcessContext _context; private static StringBuilder recordSb = new StringBuilder(); public MySqlDestination(string connStr, ILogger logger, ProcessContext context,bool prettyOutput = false) { _conn = new MySqlConnection(connStr); _conn.Open(); _recordCache = new Dictionary>(); _logger = logger; _context = context; _prettyOutput = prettyOutput; } public Task WriteRecordAsync(DataRecord record) { _recordCache.AddOrUpdate(record.TableName, [record], (key, value) => { value.Add(record); return value; }); return Task.CompletedTask; } public async Task WriteRecordsAsync(IEnumerable records) { foreach (var record in records) { await WriteRecordAsync(record); } } public async Task FlushAsync(int maxAllowPacket) { if (_recordCache.Count == 0) return; //var cmd = _conn.CreateCommand(); //cmd.CommandTimeout = 3 * 60; try { var excuseList = GetExcuseList(_recordCache, maxAllowPacket, _prettyOutput); //foreach (var insertSql in excuseList) //{ // //cmd.CommandText = insertSql; // //await cmd.ExecuteNonQueryAsync(); // //_logger.LogInformation(@"do insert completed!size:{Length}", cmd.CommandText.Length); //} _recordCache.Clear(); } catch (Exception e) { //_logger.LogCritical(e, "Error when flushing records, sql: {Sql}", cmd.CommandText.Omit(1000)); _context.AddException(e); throw; } finally { //await cmd.DisposeAsync(); } } public static IList GetExcuseList(IDictionary> tableRecords,int maxAllowPacket, bool prettyOutput = false) { var resultList = new List(); var headerSb = string.Empty; //var recordSb = new StringBuilder(); recordSb.Clear(); foreach (var (tableName, records) in tableRecords) { if (records.Count == 0) continue; headerSb=$"INSERT INTO `{tableName}`("; for (var i = 0; i < records[0].Headers.Length; i++) { var header = records[0].Headers[i]; headerSb+=$"`{header}`"; if (i != records[0].Headers.Length - 1) headerSb.Append(','); } headerSb+=") VALUES "; if (prettyOutput) headerSb+="/r/n"; var sbList = new List(); var currentLength = headerSb.Length; for (var i = 0; i < records.Count; i++) { var record = records[i]; recordSb.Append('('); for (var j = 0; j < record.Fields.Length; j++) { var field = record.Fields[j]; recordSb.Append(field); if (j != record.Fields.Length - 1) recordSb.Append(','); } recordSb.Append(')'); //if (i != records.Count - 1) // not last field // recordSb.Append(','); if (prettyOutput) recordSb.AppendLine(); if (currentLength + recordSb.Length >= maxAllowPacket) { var insertSb = headerSb; insertSb+=string.Join(",", sbList); insertSb += ";"; resultList.Add(insertSb); insertSb=String.Empty; sbList.Clear(); currentLength = headerSb.Length; sbList.Add(recordSb.ToString()); } else { sbList.Add(recordSb.ToString()); } currentLength += recordSb.Length; recordSb.Clear(); } if (sbList.Count > 0) { var insertSb = headerSb.ToString(); insertSb += string.Join(",", sbList); insertSb += ";"; resultList.Add(insertSb.ToString()); insertSb=string.Empty; } headerSb=string.Empty; } return resultList; } public void Dispose() { _conn.Close(); _conn.Dispose(); } public async ValueTask DisposeAsync() { await _conn.CloseAsync(); await _conn.DisposeAsync(); } }