using ConsoleApp2.Const; using ConsoleApp2.HostedServices.Abstractions; using ConsoleApp2.Options; using ConsoleApp2.Services; using ConsoleApp2.Services.ETL; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; namespace ConsoleApp2.HostedServices; public record FileInputInfo { public required string FileName { get; init; } public required string TableName { get; init; } public required string[] Headers { get; init; } } public enum FileInputType { MyDumperCsv, MyDumperZst, ErrorLog, } /// /// 从输入目录中导入文件 /// public class FileInputService : IInputService { private readonly ILogger _logger; private readonly DataRecordQueue _producerQueue; private readonly IOptions _dataInputOptions; private readonly ProcessContext _context; private readonly DataReaderFactory _dataReaderFactory; public FileInputService(ILogger logger, IOptions dataInputOptions, ProcessContext context, [FromKeyedServices(ProcessStep.Produce)] DataRecordQueue producerQueue, DataReaderFactory dataReaderFactory) { _logger = logger; _dataInputOptions = dataInputOptions; _context = context; _producerQueue = producerQueue; _dataReaderFactory = dataReaderFactory; } public async Task ExecuteAsync(CancellationToken cancellationToken) { var inputDir = _dataInputOptions.Value.InputDir ?? throw new ApplicationException("未配置文件输入目录"); _logger.LogInformation("***** Input service started, working directory: {InputDir} *****", inputDir); var trans = _dataInputOptions.Value.FileInputMetaBuilder; if(trans is null) throw new ApplicationException("未配置文件名-表名映射委托"); FileInputInfo[] infoArr = Directory.GetFiles(inputDir) .Select(f => trans(f)) .Where(info => info is not null).ToArray()!; var orderedInfo = GetFilesInOrder(infoArr).ToArray(); _logger.LogInformation("***** {Count} files founded in directory,{OrderedCount} files is matched with configuration *****", infoArr.Length, orderedInfo.Length); foreach (var info in orderedInfo) { _logger.LogDebug("Table {TableName}: {FileName}", info.TableName, info.FileName); } foreach (var info in orderedInfo) { _logger.LogInformation("Reading file: {FileName}, table: {TableName}", info.FileName, info.TableName); var source = _dataReaderFactory.CreateReader(info.FileName,info.TableName,info.Headers); while (await source.ReadAsync()) { var record = source.Current; _producerQueue.Enqueue(record); _context.AddInput(); } _logger.LogInformation("Input of table: '{TableName}' finished", info.TableName); } _context.CompleteInput(); _logger.LogInformation("***** Input service finished *****"); } /// /// 读取配置,按照配置的表顺序来返回 /// /// private IEnumerable GetFilesInOrder(FileInputInfo[] inputFiles) { var tableOrder = _dataInputOptions.Value.TableOrder; if (tableOrder is null or { Length: 0 }) return inputFiles; return Yield(); IEnumerable Yield() { foreach (var tableName in tableOrder) { var target = inputFiles.FirstOrDefault(f => f.TableName.Equals(tableName, StringComparison.OrdinalIgnoreCase)); if (target is not null) yield return target; } } } }