MES-ETL/ConsoleApp2/Services/MySqlDestination.cs

using System.Text;
using ConsoleApp2.Helpers;
using Microsoft.Extensions.Logging;
using MySqlConnector;
using ServiceStack;

namespace ConsoleApp2.Services;

/// <summary>
/// Mysql导出
/// </summary>
public class MySqlDestination : IDisposable, IAsyncDisposable
{
    private readonly Dictionary<string, IList<DataRecord>> _recordCache;
    private readonly MySqlConnection _conn;
    private readonly ILogger _logger;
    private readonly bool _prettyOutput;
    private readonly int _maxAllowPacket;
    private readonly ProcessContext _context;
    private static StringBuilder recordSb = new StringBuilder();
    public MySqlDestination(string connStr, ILogger logger, ProcessContext context,bool prettyOutput = false)
    {
        _conn = new MySqlConnection(connStr);
        _conn.Open();
        _recordCache = new Dictionary<string, IList<DataRecord>>();
        _logger = logger;
        _context = context;
        _prettyOutput = prettyOutput;



    }
    public Task WriteRecordAsync(DataRecord record)
    {
        _recordCache.AddOrUpdate(record.TableName, [record], (key, value) =>
        {
            value.Add(record);
            return value;
        });
        return Task.CompletedTask;
    }

    public async Task WriteRecordsAsync(IEnumerable<DataRecord> records)
    {
        foreach (var record in records)
        {
            await WriteRecordAsync(record);
        }
    }

    public async Task FlushAsync(int maxAllowPacket)
    {
        if (_recordCache.Count == 0)
            return;

        //var cmd = _conn.CreateCommand();
        //cmd.CommandTimeout = 3 * 60;
       
        try
        {
            var excuseList = GetExcuseList(_recordCache, maxAllowPacket, _prettyOutput);
            //foreach (var insertSql in excuseList)
            //{
            //    //cmd.CommandText = insertSql;
            //    //await cmd.ExecuteNonQueryAsync();
            //    //_logger.LogInformation(@"do insert completed!size:{Length}", cmd.CommandText.Length);
            //}
            _recordCache.Clear();
        }
        catch (Exception e)
        {
            //_logger.LogCritical(e, "Error when flushing records, sql: {Sql}", cmd.CommandText.Omit(1000));
            _context.AddException(e);
            throw;
        }
        finally
        {
            //await cmd.DisposeAsync();
        }
    }

    public static IList<string> GetExcuseList(IDictionary<string, IList<DataRecord>> tableRecords,int maxAllowPacket,
           bool prettyOutput = false)
    {
        var resultList = new List<string>();
        var headerSb = string.Empty;
        //var recordSb = new StringBuilder();
        recordSb.Clear();
        foreach (var (tableName, records) in tableRecords)
        {
            if (records.Count == 0)
                continue;
            headerSb=$"INSERT INTO `{tableName}`(";
            for (var i = 0; i < records[0].Headers.Length; i++)
            {
                var header = records[0].Headers[i];
                headerSb+=$"`{header}`";
                if (i != records[0].Headers.Length - 1)
                    headerSb.Append(',');
            }

            headerSb+=") VALUES ";
            if (prettyOutput)
                headerSb+="/r/n";

            var sbList = new List<string>();
            var currentLength = headerSb.Length;
            for (var i = 0; i < records.Count; i++)
            {
                var record = records[i];
                recordSb.Append('(');
                for (var j = 0; j < record.Fields.Length; j++)
                {
                    var field = record.Fields[j];
                    recordSb.Append(field);
                    if (j != record.Fields.Length - 1)
                        recordSb.Append(',');
                }

                recordSb.Append(')');

                //if (i != records.Count - 1) // not last field
                //    recordSb.Append(',');
                if (prettyOutput) recordSb.AppendLine();

                if (currentLength + recordSb.Length >= maxAllowPacket)
                {

                    var insertSb = headerSb;

                    insertSb+=string.Join(",", sbList);
                    insertSb += ";";
                    resultList.Add(insertSb);
                    insertSb=String.Empty;
                    sbList.Clear();
                    currentLength = headerSb.Length;
                    sbList.Add(recordSb.ToString());
                }
                else
                {
                    sbList.Add(recordSb.ToString());
                }
                currentLength += recordSb.Length;
                recordSb.Clear();
            }
            if (sbList.Count > 0)
            {
                var insertSb = headerSb.ToString();
                insertSb += string.Join(",", sbList);
                insertSb += ";";
                resultList.Add(insertSb.ToString());
                insertSb=string.Empty;
            }
            headerSb=string.Empty;
        }

        return resultList;
    }


    public void Dispose()
    {
        _conn.Close();
        _conn.Dispose();
    }

    public async ValueTask DisposeAsync()
    {
        await _conn.CloseAsync();
        await _conn.DisposeAsync();
    }
}