using ConsoleApp2.Entities; using ConsoleApp2.Options; using ConsoleApp2.Services; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MySqlConnector; namespace ConsoleApp2.HostedServices; public class MysqlOutputService : BackgroundService { private readonly ILogger _logger; private readonly DataRecordQueue _consumerQueue; private readonly IOptions _options; private readonly ProcessContext _context; public MysqlOutputService(ILogger logger, [FromKeyedServices(ProcessStep.Consumer)]DataRecordQueue consumerQueue, IOptions options, ProcessContext context) { _logger = logger; _consumerQueue = consumerQueue; _options = options; _context = context; } protected override async Task ExecuteAsync(CancellationToken stoppingToken) { _logger.LogInformation("***** Mysql output service started *****"); var tasks = new List(); var records = new List(); while (!_context.IsTransformCompleted || _consumerQueue.Count > 0) { if (!_consumerQueue.TryDequeue(out var record)) continue; records.Add(record); if (records.Count >= 200) { var recordsCopy = records; tasks.Add(Task.Run(async () => await FlushAsync(recordsCopy), stoppingToken)); records = []; } if (tasks.Count >= 10) { await Task.WhenAll(tasks); tasks.Clear(); } } await Task.WhenAll(tasks); await FlushAsync(records); _context.CompleteOutput(); _logger.LogInformation("***** Mysql output service completed *****"); } private async Task FlushAsync(IEnumerable records) { var count = 0; await using var output = new MySqlDestination(new MySqlConnectionStringBuilder { Server = _options.Value.Host, Port = _options.Value.Port, Database = _options.Value.Database, UserID = _options.Value.User, Password = _options.Value.Password, ConnectionTimeout = 180, }.ConnectionString, _logger, true); foreach (var record in records) { await output.WriteRecordAsync(record); count++; } await output.FlushAsync(); _context.AddOutput(count); } }