using MesETL.App.Cache; using MesETL.App.Const; using MesETL.App.HostedServices.Abstractions; using MesETL.App.Options; using MesETL.App.Services; using MesETL.App.Services.ErrorRecorder; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; namespace MesETL.App.HostedServices; public record DataTransformContext(DataRecord Record, ICacher Cacher, ILogger Logger, IServiceProvider Services); /// /// 数据处理服务,对导入后的数据进行处理 /// public class TransformService : ITransformService { private readonly ILogger _logger; private readonly IOptions _options; private readonly DataRecordQueue _producerQueue; private readonly RecordQueuePool _queuePool; private readonly ProcessContext _context; private readonly ICacher _cache; private readonly ErrorRecorderFactory _errorRecorderFactory; private readonly IServiceProvider _services; public TransformService(ILogger logger, IOptions options, [FromKeyedServices(Const.ConstVar.Producer)] DataRecordQueue producerQueue, RecordQueuePool queuePool, ProcessContext context, ICacher cache, ErrorRecorderFactory errorRecorderFactory, IServiceProvider services) { _logger = logger; _options = options; _producerQueue = producerQueue; _queuePool = queuePool; _context = context; _cache = cache; _errorRecorderFactory = errorRecorderFactory; _services = services; } public async Task ExecuteAsync(CancellationToken cancellationToken) { _logger.LogInformation("***** 数据转换服务已启动, 当前线程ID: {ThreadId} *****", Environment.CurrentManagedThreadId); // var tasks = new List(); // for (int i = 0; i < 4; i++) // { // tasks.Add(Task.Run(TransformWorker, cancellationToken)); // } // // await Task.WhenAll(tasks); await TransformWorker(); _logger.LogInformation("***** 数据转换服务执行完毕 *****"); } public async Task TransformWorker() { while (!_context.IsInputCompleted || _producerQueue.Count > 0) { if (!_producerQueue.TryDequeue(out var record)) { continue; } try { var context = new DataTransformContext(record, _cache, _logger, _services); if (_options.Value.EnableFilter) { // 数据过滤 var filter = _options.Value.RecordFilter; if (filter is not null && await filter(context) == false) continue; } if (_options.Value.EnableReplacer) { // 数据替换 var replacer = _options.Value.RecordModify; if (replacer is not null) { record = await replacer(context); } } // 字段缓存 var cacher = _options.Value.RecordCache; if(cacher is not null) await cacher.Invoke(context); //计算需要分流的数据库 var dbFilter = _options.Value.DatabaseFilter ?? throw new ApplicationException("未配置数据库过滤器"); record.Database = dbFilter(record); if (_options.Value.EnableReBuilder) { //数据重建 var addRecords = _options.Value.RecordReBuild?.Invoke(context); if (addRecords is { Count: > 0 }) { foreach (var rc in addRecords) { if(dbFilter is not null) rc.Database = dbFilter.Invoke(record); await _queuePool[record.Database].EnqueueAsync(rc); _context.AddTransform(); } } } await _queuePool[record.Database].EnqueueAsync(record); _context.AddTransform(); } catch (Exception e) { _context.AddException(e); var errorRecorder = _errorRecorderFactory.CreateTransform(); await errorRecorder.LogErrorRecordAsync(record, e); if (!_options.Value.StrictMode) _logger.LogError(e, "数据转换时发生错误"); else throw; } } _context.CompleteTransform(); } }