MES-ETL/MesETL.App/HostedServices/TransformService.cs

using MesETL.App.Cache;
using MesETL.App.Const;
using MesETL.App.HostedServices.Abstractions;
using MesETL.App.Options;
using MesETL.App.Services;
using MesETL.App.Services.ErrorRecorder;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;

namespace MesETL.App.HostedServices;

public record DataTransformContext(DataRecord Record, ICacher Cacher, ILogger Logger);

/// <summary>
/// 数据处理服务，对导入后的数据进行处理
/// </summary>
public class TransformService : ITransformService
{
    private readonly ILogger _logger;
    private readonly IOptions<DataTransformOptions> _options;
    private readonly DataRecordQueue _producerQueue;
    private readonly RecordQueuePool _queuePool;
    private readonly ProcessContext _context;
    private readonly ICacher _cache;
    private readonly ErrorRecorderFactory _errorRecorderFactory;


    public TransformService(ILogger<TransformService> logger, 
        IOptions<DataTransformOptions> options,
        [FromKeyedServices(ProcessStep.Produce)] DataRecordQueue producerQueue,
        RecordQueuePool queuePool,
        ProcessContext context,
        ICacher cache, 
        ErrorRecorderFactory errorRecorderFactory)
    {
        _logger = logger;
        _options = options;
        _producerQueue = producerQueue;
        _queuePool = queuePool;
        _context = context;
        _cache = cache;
        _errorRecorderFactory = errorRecorderFactory;
    }

    public async Task ExecuteAsync(CancellationToken cancellationToken)
    {
        _logger.LogInformation("***** Data transform service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId);

        while (!_context.IsInputCompleted || _producerQueue.Count > 0)
        {
            if (!_producerQueue.TryDequeue(out var record)) continue;

            try
            {
                var context = new DataTransformContext(record, _cache, _logger);
                if (_options.Value.EnableFilter)
                {
                    // 数据过滤
                    var filter = _options.Value.RecordFilter;
                    if (filter is not null && await filter(context) == false) continue;
                }

                if (_options.Value.EnableReplacer)
                {
                    // 数据替换
                    var replacer = _options.Value.RecordModify;
                    if (replacer is not null)
                    {
                        record = await replacer(context);
                    }
                }    
                
                // 字段缓存
                var cacher = _options.Value.RecordCache;
                if(cacher is not null)
                    await cacher.Invoke(context);
            
                //计算需要分流的数据库
                var dbFilter = _options.Value.DatabaseFilter 
                               ?? throw new ApplicationException("未配置数据库过滤器");
                record.Database = dbFilter(record);
            
                _queuePool[record.Database].Enqueue(record);
                _context.AddTransform();

                if (_options.Value.EnableReBuilder)
                {
                    //数据重建
                    var addRecords = _options.Value.RecordReBuild?.Invoke(context);
                    if (addRecords is { Count: > 0 })
                    {
                        foreach (var rc in addRecords)
                        {
                            if(dbFilter is not null)
                                rc.Database =dbFilter.Invoke(record);
                            _queuePool[record.Database].Enqueue(rc);
                            _context.AddTransform();
                        }
                    }
                }
            }
            catch (Exception e)
            {
                _context.AddException(e);
                var errorRecorder = _errorRecorderFactory.CreateTransform();
                await errorRecorder.LogErrorRecordAsync(record, e);
                if (!_options.Value.StrictMode)
                    _logger.LogError(e, "数据转换时发生错误");
                else throw;
            }
        }
        _context.CompleteTransform();
        
        _logger.LogInformation("***** Data transformation service finished *****");
    }
}