MES-ETL/MesETL.App/HostedServices/FileInputService.cs

using ConsoleApp2.Const;
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
using ConsoleApp2.Services.ETL;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;

namespace ConsoleApp2.HostedServices;

public record FileInputInfo
{
    public required string FileName { get; init; }
    public required string TableName { get; init; }
    public required string[] Headers { get; init; }
}

public enum FileInputType
{
    MyDumperCsv,
    MyDumperZst,
    ErrorLog,
}

/// <summary>
/// 从输入目录中导入文件
/// </summary>
public class FileInputService : IInputService
{
    private readonly ILogger _logger;
    private readonly DataRecordQueue _producerQueue;
    private readonly IOptions<DataInputOptions> _dataInputOptions;
    private readonly ProcessContext _context;
    private readonly DataReaderFactory _dataReaderFactory;

    public FileInputService(ILogger<FileInputService> logger,
            IOptions<DataInputOptions> dataInputOptions,
            ProcessContext context, 
            [FromKeyedServices(ProcessStep.Produce)] DataRecordQueue producerQueue, 
            DataReaderFactory dataReaderFactory)
    {
        _logger = logger;
        _dataInputOptions = dataInputOptions;
        _context = context;
        _producerQueue = producerQueue;
        _dataReaderFactory = dataReaderFactory;
    }

    public async Task ExecuteAsync(CancellationToken cancellationToken)
    {
        var inputDir = _dataInputOptions.Value.InputDir ?? throw new ApplicationException("未配置文件输入目录");
        _logger.LogInformation("***** Input service started, working directory: {InputDir} *****", inputDir);
        
        var trans = _dataInputOptions.Value.FileInputMetaBuilder;
        if(trans is null) throw new ApplicationException("未配置文件名-表名映射委托");
        FileInputInfo[] infoArr = Directory.GetFiles(inputDir)
            .Select(f => trans(f))
            .Where(info => info is not null).ToArray()!;
        
        var orderedInfo = GetFilesInOrder(infoArr).ToArray();
        
        _logger.LogInformation("***** {Count} files founded in directory，{OrderedCount} files is matched with configuration *****", infoArr.Length, orderedInfo.Length);
        foreach (var info in orderedInfo)
        {
            _logger.LogDebug("Table {TableName}: {FileName}", info.TableName, info.FileName);
        }

        foreach (var info in orderedInfo)
        {
            _logger.LogInformation("Reading file: {FileName}, table: {TableName}", info.FileName, info.TableName);
            var source = _dataReaderFactory.CreateReader(info.FileName,info.TableName,info.Headers);

            while (await source.ReadAsync())
            {
                var record = source.Current;
                _producerQueue.Enqueue(record);
                _context.AddInput();
            }
            
            _logger.LogInformation("Input of table: '{TableName}' finished", info.TableName);
        }

        _context.CompleteInput();
        _logger.LogInformation("***** Input service finished *****");
    }

    /// <summary>
    /// 读取配置，按照配置的表顺序来返回
    /// </summary>
    /// <returns></returns>
    private IEnumerable<FileInputInfo> GetFilesInOrder(FileInputInfo[] inputFiles)
    {
        var tableOrder = _dataInputOptions.Value.TableOrder;
        if (tableOrder is null or { Length: 0 })
            return inputFiles;

        return Yield();

        IEnumerable<FileInputInfo> Yield()
        {
            foreach (var tableName in tableOrder)
            {
                var target = inputFiles.FirstOrDefault(f =>
                    f.TableName.Equals(tableName, StringComparison.OrdinalIgnoreCase));
                if (target is not null)
                    yield return target;
            }
        }
    }
}