整理代码
This commit is contained in:
@@ -5,5 +5,5 @@ namespace ConsoleApp2.HostedServices.Abstractions;
|
||||
|
||||
public interface IOutputService
|
||||
{
|
||||
public Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken);
|
||||
public void ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken);
|
||||
}
|
@@ -42,14 +42,17 @@ public class InputService : IInputService
|
||||
{
|
||||
_logger.LogInformation("Working table: {tableName}", tableName);
|
||||
var source = _dataInputOptions.Value.CreateSource?.Invoke(tableName);
|
||||
await source.DoEnqueue((record) =>
|
||||
if (source != null)
|
||||
{
|
||||
_context.AddInput();
|
||||
producerQueue.Enqueue(record);
|
||||
count++;
|
||||
await source.DoEnqueue((record) =>
|
||||
{
|
||||
_context.AddInput();
|
||||
producerQueue.Enqueue(record);
|
||||
count++;
|
||||
|
||||
});
|
||||
if (_context.GetExceptions().Count > 0)
|
||||
});
|
||||
}
|
||||
if (!_context.GetExceptions().IsEmpty)
|
||||
{
|
||||
_logger.LogInformation("***** Csv input service is canceled *****");
|
||||
return;
|
||||
|
@@ -5,14 +5,15 @@ using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
namespace ConsoleApp2.HostedServices;
|
||||
|
||||
public class MainHostedService : BackgroundService
|
||||
public class MainHostedService : IHostedService
|
||||
{
|
||||
private readonly ILogger _logger;
|
||||
private readonly IInputService _input;
|
||||
private readonly ITransformService _transform;
|
||||
private readonly IOutputService _output;
|
||||
private readonly ProcessContext _context;
|
||||
|
||||
private readonly Timer? _bigTableTimer;
|
||||
private readonly Timer? _smallTableTimer;
|
||||
public MainHostedService(ILogger<MainHostedService> logger, IInputService input, ITransformService transform, IOutputService output, ProcessContext context)
|
||||
{
|
||||
_logger = logger;
|
||||
@@ -22,15 +23,15 @@ public class MainHostedService : BackgroundService
|
||||
_context = context;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
public Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var taskFun = (TasksOptions taskOp, DataRecordQueue producerQueue, DataRecordQueue consumerQueue, ProcessContext context) =>
|
||||
var taskFun = (TasksOptions taskOp, DataRecordQueue producerQueue, DataRecordQueue consumerQueue, ProcessContext context,Timer? timer) =>
|
||||
{
|
||||
var inputTask = Task.Factory.StartNew(async () =>
|
||||
Task.Factory.StartNew(async () =>
|
||||
{
|
||||
try
|
||||
{
|
||||
await _input.ExecuteAsync(taskOp, producerQueue, context, stoppingToken);
|
||||
await _input.ExecuteAsync(taskOp, producerQueue, context, cancellationToken);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -39,11 +40,11 @@ public class MainHostedService : BackgroundService
|
||||
}
|
||||
|
||||
});
|
||||
var transformTask = Task.Factory.StartNew(async () =>
|
||||
Task.Factory.StartNew(async () =>
|
||||
{
|
||||
try
|
||||
{
|
||||
await _transform.ExecuteAsync(taskOp, producerQueue, consumerQueue, context, stoppingToken);
|
||||
await _transform.ExecuteAsync(taskOp, producerQueue, consumerQueue, context, cancellationToken);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -52,11 +53,15 @@ public class MainHostedService : BackgroundService
|
||||
}
|
||||
|
||||
});
|
||||
var outputTask = Task.Factory.StartNew(async () =>
|
||||
Task.Factory.StartNew(() =>
|
||||
{
|
||||
try
|
||||
{
|
||||
await _output.ExecuteAsync(taskOp, consumerQueue, context,stoppingToken);
|
||||
timer = new Timer((object? state) =>
|
||||
{
|
||||
_output.ExecuteAsync(taskOp, consumerQueue, context, cancellationToken);
|
||||
},null, TimeSpan.Zero,TimeSpan.FromSeconds(0.5));
|
||||
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -77,8 +82,8 @@ public class MainHostedService : BackgroundService
|
||||
{"simple_plan_order",new TableInfo{SimulaRowCount=351470 }},//CreateTime < 202301的删除
|
||||
};
|
||||
var bigTableContext = new ProcessContext();
|
||||
var bigTableOptions = new TasksOptions { TableInfoConfig = bigTablesDic, OutPutOptions = new OutPutOptions { FlushCount = 10000, OutPutTaskCount = 2 } };
|
||||
taskFun(bigTableOptions,new DataRecordQueue(), new DataRecordQueue(), bigTableContext);
|
||||
var bigTableOptions = new TasksOptions { TableInfoConfig = bigTablesDic, OutPutOptions = new OutPutOptions { FlushCount = 20000, OutPutTaskCount = 2 } };
|
||||
taskFun(bigTableOptions, new DataRecordQueue(), new DataRecordQueue(), bigTableContext,_bigTableTimer);
|
||||
var smallTablesDic = new Dictionary<string, TableInfo>
|
||||
{
|
||||
{"machine",new TableInfo{SimulaRowCount=14655 }},
|
||||
@@ -107,6 +112,14 @@ public class MainHostedService : BackgroundService
|
||||
};
|
||||
var smallTableContext = new ProcessContext();
|
||||
taskFun(new TasksOptions { TableInfoConfig = smallTablesDic, OutPutOptions = new OutPutOptions { FlushCount = 20000, OutPutTaskCount = 4 } },
|
||||
new DataRecordQueue(), new DataRecordQueue(), smallTableContext);
|
||||
new DataRecordQueue(), new DataRecordQueue(), smallTableContext,_smallTableTimer);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
|
||||
}
|
@@ -1,11 +1,9 @@
|
||||
using ConsoleApp2.Const;
|
||||
|
||||
using ConsoleApp2.HostedServices.Abstractions;
|
||||
using ConsoleApp2.Options;
|
||||
using ConsoleApp2.Services;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace ConsoleApp2.HostedServices;
|
||||
|
||||
/// <summary>
|
||||
@@ -34,42 +32,32 @@ public class OutputService : IOutputService
|
||||
_transformOptions = transformOptions;
|
||||
_errorRecorder = errorRecorder;
|
||||
}
|
||||
|
||||
public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken)
|
||||
private int _runingTaskCount;
|
||||
public int RuningTaskCount
|
||||
{
|
||||
|
||||
_logger.LogInformation("***** Mysql output service started *****");
|
||||
|
||||
get => _runingTaskCount;
|
||||
}
|
||||
public void DoTask() => Interlocked.Increment(ref _runingTaskCount);
|
||||
public void FinishTask() => Interlocked.Decrement(ref _runingTaskCount);
|
||||
public void ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
if (context.IsTransformCompleted == false && consumerQueue.Count < tasksOptions.OutPutOptions.FlushCount) return;
|
||||
if (RuningTaskCount >= tasksOptions.OutPutOptions.OutPutTaskCount ) return;
|
||||
var records = new List<DataRecord>();
|
||||
while (!context.IsTransformCompleted || consumerQueue.Count > 0)
|
||||
{
|
||||
if (!consumerQueue.TryDequeue(out var record)) continue;
|
||||
records.Add(record);
|
||||
if (records.Count >= tasksOptions.OutPutOptions.FlushCount)
|
||||
{
|
||||
var temp= new List<DataRecord>(records);
|
||||
ThreadPool.QueueUserWorkItem(async (queueState) =>
|
||||
{
|
||||
await FlushAsync(temp);
|
||||
});
|
||||
|
||||
records.Clear();
|
||||
}
|
||||
if (_context.GetExceptions().Count > 0)
|
||||
{
|
||||
_logger.LogInformation("***** Csv output thread is canceled *****");
|
||||
return;
|
||||
}
|
||||
for (int i = 0; i < tasksOptions.OutPutOptions.FlushCount; i++)
|
||||
{
|
||||
if (consumerQueue.TryDequeue(out var record)) records.Add(record);
|
||||
else break;
|
||||
}
|
||||
if (records.Count > 0)
|
||||
{
|
||||
var temp = new List<DataRecord>(records);
|
||||
ThreadPool.QueueUserWorkItem(async (queueState) =>
|
||||
{
|
||||
await FlushAsync(temp);
|
||||
DoTask();
|
||||
await FlushAsync(records);
|
||||
FinishTask();
|
||||
});
|
||||
records.Clear();
|
||||
_logger.LogInformation("***** Mysql output thread completed *****");
|
||||
}
|
||||
}
|
||||
|
||||
|
@@ -19,20 +19,17 @@ public class TransformService : ITransformService
|
||||
private readonly IOptions<DataTransformOptions> _options;
|
||||
private readonly ProcessContext _context;
|
||||
private readonly IDistributedCache _cache;
|
||||
private readonly TaskManager _taskManager;
|
||||
|
||||
|
||||
public TransformService(ILogger<TransformService> logger,
|
||||
IOptions<DataTransformOptions> options,
|
||||
ProcessContext context,
|
||||
IDistributedCache cache,
|
||||
TaskManager taskManager)
|
||||
IDistributedCache cache)
|
||||
{
|
||||
_logger = logger;
|
||||
_options = options;
|
||||
_context = context;
|
||||
_cache = cache;
|
||||
_taskManager = taskManager;
|
||||
}
|
||||
|
||||
public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken)
|
||||
|
@@ -23,7 +23,7 @@ public class VoidOutputService : IOutputService
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken)
|
||||
public void ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
_logger.LogInformation("***** Void output service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId);
|
||||
while (!_context.IsTransformCompleted || _consumerQueue.Count > 0)
|
||||
@@ -34,6 +34,5 @@ public class VoidOutputService : IOutputService
|
||||
|
||||
_context.CompleteOutput();
|
||||
_logger.LogInformation("***** Void output service completed *****");
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
Reference in New Issue
Block a user