支持按多个表开立线程
This commit is contained in:
@@ -17,69 +17,67 @@ public class TransformService : ITransformService
|
||||
{
|
||||
private readonly ILogger _logger;
|
||||
private readonly IOptions<DataTransformOptions> _options;
|
||||
private readonly DataRecordQueue _producerQueue;
|
||||
private readonly DataRecordQueue _consumerQueue;
|
||||
private readonly ProcessContext _context;
|
||||
private readonly IDistributedCache _cache;
|
||||
private readonly TaskManager _taskManager;
|
||||
|
||||
|
||||
public TransformService(ILogger<TransformService> logger,
|
||||
IOptions<DataTransformOptions> options,
|
||||
[FromKeyedServices(ProcessStep.Producer)]DataRecordQueue producerQueue,
|
||||
[FromKeyedServices(ProcessStep.Consumer)]DataRecordQueue consumerQueue,
|
||||
ProcessContext context,
|
||||
IDistributedCache cache)
|
||||
IDistributedCache cache,
|
||||
TaskManager taskManager)
|
||||
{
|
||||
_logger = logger;
|
||||
_options = options;
|
||||
_producerQueue = producerQueue;
|
||||
_consumerQueue = consumerQueue;
|
||||
_context = context;
|
||||
_cache = cache;
|
||||
_taskManager = taskManager;
|
||||
}
|
||||
|
||||
public async Task ExecuteAsync(CancellationToken cancellationToken)
|
||||
public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
_logger.LogInformation("***** Data transform service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId);
|
||||
while ((!_context.IsInputCompleted || _producerQueue.Count > 0))
|
||||
|
||||
_taskManager.CreateTasks(async () =>
|
||||
{
|
||||
if (_context.GetExceptions().Count > 0)
|
||||
while ((!context.IsInputCompleted || producerQueue.Count > 0))
|
||||
{
|
||||
_logger.LogInformation("***** Csv transform service is canceled *****");
|
||||
return;
|
||||
}
|
||||
// var dbOptions = _options.Value.DatabaseFilter(record);
|
||||
if (!_producerQueue.TryDequeue(out var record)) continue;
|
||||
|
||||
//过滤不要的record
|
||||
if ( await _options.Value.RecordFilter?.Invoke(record, _cache) == false) continue;
|
||||
record.Database = _options.Value.DatabaseFilter?.Invoke(record);
|
||||
//修改record
|
||||
_options.Value.RecordModify?.Invoke(record);
|
||||
//缓存record
|
||||
_options.Value.RecordCache?.Invoke(record, _cache);
|
||||
//替换record
|
||||
var replaceRecord =await _options.Value.RecordReplace?.Invoke(record, _cache);
|
||||
if (replaceRecord != null)
|
||||
{
|
||||
record = replaceRecord;
|
||||
}
|
||||
_consumerQueue.Enqueue(record);
|
||||
_context.AddTransform();
|
||||
//数据增加
|
||||
var addRecords=_options.Value.RecordAdd?.Invoke(record);
|
||||
if(addRecords != null&& addRecords.Count>0)
|
||||
{
|
||||
foreach(var rc in addRecords)
|
||||
if (_context.GetExceptions().Count > 0)
|
||||
{
|
||||
_consumerQueue.Enqueue(rc);
|
||||
_context.AddTransform();
|
||||
_logger.LogInformation("***** Csv transform service is canceled *****");
|
||||
return;
|
||||
}
|
||||
if (!producerQueue.TryDequeue(out var record)) continue;
|
||||
|
||||
//过滤不要的record
|
||||
if (await _options.Value.RecordFilter?.Invoke(record, _cache) == false) continue;
|
||||
record.Database = _options.Value.DatabaseFilter?.Invoke(record);
|
||||
//修改record
|
||||
_options.Value.RecordModify?.Invoke(record);
|
||||
//缓存record
|
||||
_options.Value.RecordCache?.Invoke(record, _cache);
|
||||
//替换record
|
||||
var replaceRecord = await _options.Value.RecordReplace?.Invoke(record, _cache);
|
||||
if (replaceRecord != null)
|
||||
{
|
||||
record = replaceRecord;
|
||||
}
|
||||
consumerQueue.Enqueue(record);
|
||||
_context.AddTransform();
|
||||
//数据增加
|
||||
var addRecords = _options.Value.RecordAdd?.Invoke(record);
|
||||
if (addRecords != null && addRecords.Count > 0)
|
||||
{
|
||||
foreach (var rc in addRecords)
|
||||
{
|
||||
consumerQueue.Enqueue(rc);
|
||||
_context.AddTransform();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
_context.CompleteTransform();
|
||||
context.CompleteTransform();
|
||||
},tasksOptions.TransformTaskCount,cancellationToken);
|
||||
_logger.LogInformation("***** Data transformation service completed *****");
|
||||
}
|
||||
}
|
Reference in New Issue
Block a user