85 lines
2.6 KiB
C#
85 lines
2.6 KiB
C#
using ConsoleApp2.Entities;
|
|
using ConsoleApp2.Options;
|
|
using ConsoleApp2.Services;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.Hosting;
|
|
using Microsoft.Extensions.Logging;
|
|
using Microsoft.Extensions.Options;
|
|
using MySqlConnector;
|
|
|
|
namespace ConsoleApp2.HostedServices;
|
|
|
|
public class MysqlOutputService : BackgroundService
|
|
{
|
|
private readonly ILogger _logger;
|
|
private readonly DataRecordQueue _consumerQueue;
|
|
private readonly IOptions<DatabaseOptions> _options;
|
|
private readonly ProcessContext _context;
|
|
|
|
public MysqlOutputService(ILogger<MysqlOutputService> logger,
|
|
[FromKeyedServices(ProcessStep.Consumer)]DataRecordQueue consumerQueue,
|
|
IOptions<DatabaseOptions> options,
|
|
ProcessContext context)
|
|
{
|
|
_logger = logger;
|
|
_consumerQueue = consumerQueue;
|
|
_options = options;
|
|
_context = context;
|
|
}
|
|
|
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
|
{
|
|
_logger.LogInformation("***** Mysql output service started *****");
|
|
|
|
var tasks = new List<Task>();
|
|
var records = new List<DataRecord>();
|
|
while (!_context.IsTransformCompleted || _consumerQueue.Count > 0)
|
|
{
|
|
if (!_consumerQueue.TryDequeue(out var record)) continue;
|
|
records.Add(record);
|
|
|
|
if (records.Count >= 200)
|
|
{
|
|
var recordsCopy = records;
|
|
tasks.Add(Task.Run(async () => await FlushAsync(recordsCopy), stoppingToken));
|
|
records = [];
|
|
}
|
|
|
|
if (tasks.Count >= 10)
|
|
{
|
|
await Task.WhenAll(tasks);
|
|
tasks.Clear();
|
|
}
|
|
}
|
|
|
|
await Task.WhenAll(tasks);
|
|
await FlushAsync(records);
|
|
|
|
_context.CompleteOutput();
|
|
|
|
_logger.LogInformation("***** Mysql output service completed *****");
|
|
}
|
|
|
|
private async Task FlushAsync(IEnumerable<DataRecord> records)
|
|
{
|
|
var count = 0;
|
|
await using var output = new MySqlDestination(new MySqlConnectionStringBuilder
|
|
{
|
|
Server = _options.Value.Host,
|
|
Port = _options.Value.Port,
|
|
Database = _options.Value.Database,
|
|
UserID = _options.Value.User,
|
|
Password = _options.Value.Password,
|
|
ConnectionTimeout = 180,
|
|
}.ConnectionString, _logger, true);
|
|
|
|
foreach (var record in records)
|
|
{
|
|
await output.WriteRecordAsync(record);
|
|
count++;
|
|
}
|
|
|
|
await output.FlushAsync();
|
|
_context.AddOutput(count);
|
|
}
|
|
} |