MES-ETL/ConsoleApp2/HostedServices/CsvInputService.cs

using ConsoleApp2.Helpers;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;

namespace ConsoleApp2.HostedServices;

public class CsvInputService : BackgroundService
{
    private readonly ILogger _logger;
    private readonly IOptions<CsvOptions> _csvOptions;
    private readonly TaskManager _taskManager; // TBD
    private readonly DataRecordQueue _producerQueue;
    private readonly ProcessContext _context;

    public CsvInputService(ILogger<CsvInputService> logger,
        IOptions<CsvOptions> csvOptions,
        [FromKeyedServices(ProcessStep.Producer)]TaskManager taskManager, 
        [FromKeyedServices(ProcessStep.Producer)]DataRecordQueue producerQueue, 
        ProcessContext context)
    {
        _logger = logger;
        _csvOptions = csvOptions;
        _taskManager = taskManager;
        _producerQueue = producerQueue;
        _context = context;
    }

    protected override async Task ExecuteAsync(CancellationToken cancellationToken)
    {
        var inputDir = _csvOptions.Value.InputDir;
        _logger.LogInformation("***** Csv input service start, working dir: {InputDir}, thread id: {ThreadId} *****", inputDir, Environment.CurrentManagedThreadId);
        var files = Directory.GetFiles(inputDir).Where(s => s.EndsWith(".sql") && !s.Contains("schema")).ToArray();
        if (files.Length == 0)
        {
            _logger.LogInformation("No sql files found in {InputDir}", inputDir);
            return;
        }

        foreach (var sqlPath in files)
        {
            _logger.LogInformation("Working sql file: {SqlPath}", sqlPath);
            var headers = await DumpDataHelper.GetCsvHeadersFromSqlFileAsync(sqlPath);
            var csvFiles = await DumpDataHelper.GetCsvFileNamesFromSqlFileAsync(sqlPath);

            foreach (var csvFile in csvFiles)
            {
                var csvPath = Path.Combine(inputDir, csvFile);
                // var source = new JsvSource(csvPath, headers, _logger);
                var source = new NewCsvSource(csvPath, headers,  logger: _logger);

                while (await source.ReadAsync())
                {
                    _context.AddInput();
                    _producerQueue.Enqueue(source.Current);
                    if (cancellationToken.IsCancellationRequested)
                        return;
                }
            }

            _logger.LogInformation("File '{File}' input completed", Path.GetFileName(sqlPath));
        }

        _context.CompleteInput();
        _logger.LogInformation("***** Csv input service completed *****");
    }
}