支持按多个表开立线程

This commit is contained in:
2024-01-18 14:36:36 +08:00
parent 629a4d2fb5
commit 97e359468f
16 changed files with 232 additions and 198 deletions

View File

@@ -16,39 +16,36 @@ namespace ConsoleApp2.HostedServices;
public class OutputService : IOutputService
{
private readonly ILogger _logger;
private readonly DataRecordQueue _consumerQueue;
private readonly IOptions<DatabaseOutputOptions> _outputOptions;
private readonly IOptions<DataTransformOptions> _transformOptions;
private readonly ProcessContext _context;
private readonly TaskManager _taskManager;
private readonly IOptions<DatabaseOutputOptions> _outputOptions;
public OutputService(ILogger<OutputService> logger,
[FromKeyedServices(ProcessStep.Consumer)] DataRecordQueue consumerQueue,
IOptions<DatabaseOutputOptions> outputOptions,
ProcessContext context,
TaskManager taskManager,
IOptions<DataTransformOptions> transformOptions)
{
_logger = logger;
_consumerQueue = consumerQueue;
_outputOptions = outputOptions;
_context = context;
_taskManager = taskManager;
_transformOptions = transformOptions;
}
public async Task ExecuteAsync(CancellationToken cancellationToken)
public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue consumerQueue, ProcessContext context,CancellationToken cancellationToken)
{
_logger.LogInformation("***** Mysql output service started *****");
_taskManager.CreateTasks(async () =>
{
var records = new List<DataRecord>();
while (!_context.IsTransformCompleted || _consumerQueue.Count > 0)
while (!context.IsTransformCompleted || consumerQueue.Count > 0)
{
if (!_consumerQueue.TryDequeue(out var record)) continue;
if (!consumerQueue.TryDequeue(out var record)) continue;
records.Add(record);
//_logger.LogInformation(@"*****OutputCount: {count} *****",count);
if (records.Count >= _outputOptions.Value.FlushCount)
if (records.Count >= tasksOptions.OutPutOptions.FlushCount)
{
await FlushAsync(records);
records.Clear();
@@ -65,7 +62,7 @@ public class OutputService : IOutputService
records.Clear();
_logger.LogInformation("***** Mysql output thread completed *****");
}
}, _outputOptions.Value.TaskCount);
}, tasksOptions.OutPutOptions.OutPutTaskCount);
await _taskManager.WaitAll();
//_context.CompleteOutput();