MES-ETL/ConsoleApp2/HostedServices/InputService.cs
2024-01-04 09:00:44 +08:00

71 lines
2.6 KiB
C#

using ConsoleApp2.Const;
using ConsoleApp2.Helpers;
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace ConsoleApp2.HostedServices;
/// <summary>
/// 从MyDumper导出的CSV文件中导入表头和数据
/// </summary>
public class InputService : IInputService
{
private readonly ILogger _logger;
private readonly IOptions<CsvOptions> _csvOptions;
private readonly DataRecordQueue _producerQueue;
private readonly ProcessContext _context;
public InputService(ILogger<InputService> logger,
IOptions<CsvOptions> csvOptions,
[FromKeyedServices(ProcessStep.Producer)]DataRecordQueue producerQueue,
ProcessContext context)
{
_logger = logger;
_csvOptions = csvOptions;
_producerQueue = producerQueue;
_context = context;
}
public async Task ExecuteAsync(CancellationToken cancellationToken)
{
var inputDir = _csvOptions.Value.InputDir;
_logger.LogInformation("***** Csv input service start, working dir: {InputDir}, thread id: {ThreadId} *****", inputDir, Environment.CurrentManagedThreadId);
var files = Directory.GetFiles(inputDir).Where(s => s.EndsWith(".sql") && !s.Contains("schema")).ToArray();
if (files.Length == 0)
{
_logger.LogInformation("No sql files found in {InputDir}", inputDir);
return;
}
foreach (var sqlPath in files)
{
_logger.LogInformation("Working sql file: {SqlPath}", sqlPath);
var headers = await DumpDataHelper.GetCsvHeadersFromSqlFileAsync(sqlPath);
var csvFiles = await DumpDataHelper.GetCsvFileNamesFromSqlFileAsync(sqlPath);
foreach (var csvFile in csvFiles)
{
var csvPath = Path.Combine(inputDir, csvFile);
// var source = new JsvSource(csvPath, headers, _logger);
var source = new CsvSource(csvPath, headers, _csvOptions.Value.Delimiter, _csvOptions.Value.QuoteChar, _logger);
while (await source.ReadAsync())
{
_context.AddInput();
_producerQueue.Enqueue(source.Current);
if (cancellationToken.IsCancellationRequested)
return;
}
}
_logger.LogInformation("File '{File}' input completed", Path.GetFileName(sqlPath));
}
_context.CompleteInput();
_logger.LogInformation("***** Csv input service completed *****");
}
}