修改
This commit is contained in:
@@ -35,40 +35,47 @@ public class OutputService : IOutputService
|
||||
_errorRecorder = errorRecorder;
|
||||
}
|
||||
|
||||
public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue consumerQueue, ProcessContext context,CancellationToken cancellationToken)
|
||||
public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
|
||||
_logger.LogInformation("***** Mysql output service started *****");
|
||||
_taskManager.CreateTasks(async () =>
|
||||
|
||||
var records = new List<DataRecord>();
|
||||
while (!context.IsTransformCompleted || consumerQueue.Count > 0)
|
||||
{
|
||||
var records = new List<DataRecord>();
|
||||
while (!context.IsTransformCompleted || consumerQueue.Count > 0)
|
||||
if (!consumerQueue.TryDequeue(out var record)) continue;
|
||||
records.Add(record);
|
||||
if (records.Count >= tasksOptions.OutPutOptions.FlushCount)
|
||||
{
|
||||
if (!consumerQueue.TryDequeue(out var record)) continue;
|
||||
records.Add(record);
|
||||
//_logger.LogInformation(@"*****OutputCount: {count} *****",count);
|
||||
if (records.Count >= tasksOptions.OutPutOptions.FlushCount)
|
||||
{
|
||||
await FlushAsync(records);
|
||||
records.Clear();
|
||||
}
|
||||
if (_context.GetExceptions().Count>0)
|
||||
{
|
||||
_logger.LogInformation("***** Csv output thread is canceled *****");
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (records.Count > 0)
|
||||
{
|
||||
await FlushAsync(records);
|
||||
var temp= new List<DataRecord>();
|
||||
temp.AddRange(records);
|
||||
|
||||
ThreadPool.QueueUserWorkItem(async (queueState) =>
|
||||
{
|
||||
await FlushAsync(temp);
|
||||
});
|
||||
|
||||
records.Clear();
|
||||
_logger.LogInformation("***** Mysql output thread completed *****");
|
||||
}
|
||||
}, tasksOptions.OutPutOptions.OutPutTaskCount);
|
||||
|
||||
await _taskManager.WaitAll();
|
||||
//_context.CompleteOutput();
|
||||
_logger.LogInformation(@"***** Mysql output service completed *****");
|
||||
|
||||
if (_context.GetExceptions().Count > 0)
|
||||
{
|
||||
_logger.LogInformation("***** Csv output thread is canceled *****");
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (records.Count > 0)
|
||||
{
|
||||
var temp = new List<DataRecord>();
|
||||
temp.AddRange(records);
|
||||
ThreadPool.QueueUserWorkItem(async (queueState) =>
|
||||
{
|
||||
await FlushAsync(temp);
|
||||
});
|
||||
records.Clear();
|
||||
context.AddFinishTask();
|
||||
_logger.LogInformation("***** Finished Tasks Count:{FinishTaskCount} *****", context.FinishTaskCount);
|
||||
_logger.LogInformation("***** Mysql output thread completed *****");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task FlushAsync(IEnumerable<DataRecord> records)
|
||||
|
Reference in New Issue
Block a user