Update

ConsoleApp2/HostedServices/CsvInputService.cs

@@ -0,0 +1,70 @@
+using ConsoleApp2.Helpers;
+using ConsoleApp2.Options;
+using ConsoleApp2.Services;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace ConsoleApp2.HostedServices;
+
+public class CsvInputService : BackgroundService
+{
+    private readonly ILogger _logger;
+    private readonly IOptions<CsvOptions> _csvOptions;
+    private readonly TaskManager _taskManager; // TBD
+    private readonly DataRecordQueue _producerQueue;
+    private readonly ProcessContext _context;
+
+    public CsvInputService(ILogger<CsvInputService> logger,
+        IOptions<CsvOptions> csvOptions,
+        [FromKeyedServices(ProcessStep.Producer)]TaskManager taskManager, 
+        [FromKeyedServices(ProcessStep.Producer)]DataRecordQueue producerQueue, 
+        ProcessContext context)
+    {
+        _logger = logger;
+        _csvOptions = csvOptions;
+        _taskManager = taskManager;
+        _producerQueue = producerQueue;
+        _context = context;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken cancellationToken)
+    {
+        var inputDir = _csvOptions.Value.InputDir;
+        _logger.LogInformation("***** Csv input service start, working dir: {InputDir}, thread id: {ThreadId} *****", inputDir, Environment.CurrentManagedThreadId);
+        var files = Directory.GetFiles(inputDir).Where(s => s.EndsWith(".sql") && !s.Contains("schema")).ToArray();
+        if (files.Length == 0)
+        {
+            _logger.LogInformation("No sql files found in {InputDir}", inputDir);
+            return;
+        }
+
+        foreach (var sqlPath in files)
+        {
+            _logger.LogInformation("Working sql file: {SqlPath}", sqlPath);
+            var headers = await DumpDataHelper.GetCsvHeadersFromSqlFileAsync(sqlPath);
+            var csvFiles = await DumpDataHelper.GetCsvFileNamesFromSqlFileAsync(sqlPath);
+
+            foreach (var csvFile in csvFiles)
+            {
+                var csvPath = Path.Combine(inputDir, csvFile);
+                // var source = new JsvSource(csvPath, headers, _logger);
+                var source = new NewCsvSource(csvPath, headers,  logger: _logger);
+
+                while (await source.ReadAsync())
+                {
+                    _context.AddInput();
+                    _producerQueue.Enqueue(source.Current);
+                    if (cancellationToken.IsCancellationRequested)
+                        return;
+                }
+            }
+
+            _logger.LogInformation("File '{File}' input completed", Path.GetFileName(sqlPath));
+        }
+
+        _context.CompleteInput();
+        _logger.LogInformation("***** Csv input service completed *****");
+    }
+}

ConsoleApp2/HostedServices/DataTransformService.cs

@@ -0,0 +1,76 @@
+using ConsoleApp2.Helpers;
+using ConsoleApp2.Options;
+using ConsoleApp2.Services;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace ConsoleApp2.HostedServices;
+
+public class DataTransformService : BackgroundService
+{
+    private readonly ILogger _logger;
+    private readonly IOptions<DataTransformOptions> _options;
+    private readonly DataRecordQueue _producerQueue;
+    private readonly DataRecordQueue _consumerQueue;
+    private readonly ProcessContext _context;
+
+
+    public DataTransformService(ILogger<DataTransformService> logger, 
+        IOptions<DataTransformOptions> options, // TBD: database filter
+        [FromKeyedServices(ProcessStep.Producer)]DataRecordQueue producerQueue,
+        [FromKeyedServices(ProcessStep.Consumer)]DataRecordQueue consumerQueue, 
+        ProcessContext context)
+    {
+        _logger = logger;
+        // _taskManager = taskManager;
+        _options = options;
+        _producerQueue = producerQueue;
+        _consumerQueue = consumerQueue;
+        _context = context;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken cancellationToken)
+    {
+        _logger.LogInformation("***** Data transform service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId);
+        while (!_context.IsInputCompleted || _producerQueue.Count > 0)
+        {
+            // var dbOptions = _options.Value.DatabaseFilter(record);
+            if (!_producerQueue.TryDequeue(out var record)) continue;
+            record.Database = _options.Value.DatabaseFilter?.Invoke(record);
+
+            for (var i = 0; i < record.Fields.Length; i++)
+            {
+                var field = record[i];
+
+                if (field == "\\N")
+                {
+                    field = "NULL";
+                    goto Escape;
+                }
+                // else if(DumpDataHelper.CheckHexField(field))
+                //     field = $"0x{field}";
+
+                switch (_options.Value.GetColumnType(record.TableName, record.Headers[i]))
+                {
+                    case ColumnType.Blob or ColumnType.Text:
+                        field = string.IsNullOrEmpty(field) ? "''" : $"0x{field}";
+                        break;
+                    default:
+                        field = field;
+                        break;
+                }
+
+                Escape:
+                record[i] = field;
+            }
+            
+            _consumerQueue.Enqueue(record);
+            _context.AddTransform();
+        }
+
+        _context.CompleteTransform();
+        _logger.LogInformation("***** Data transformation service completed *****");
+    }
+}

ConsoleApp2/HostedServices/MysqlOutputService.cs

@@ -0,0 +1,85 @@
+using ConsoleApp2.Entities;
+using ConsoleApp2.Options;
+using ConsoleApp2.Services;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using MySqlConnector;
+
+namespace ConsoleApp2.HostedServices;
+
+public class MysqlOutputService : BackgroundService
+{
+    private readonly ILogger _logger;
+    private readonly DataRecordQueue _consumerQueue;
+    private readonly IOptions<DatabaseOptions> _options;
+    private readonly ProcessContext _context;
+
+    public MysqlOutputService(ILogger<MysqlOutputService> logger, 
+        [FromKeyedServices(ProcessStep.Consumer)]DataRecordQueue consumerQueue, 
+        IOptions<DatabaseOptions> options, 
+        ProcessContext context)
+    {
+        _logger = logger;
+        _consumerQueue = consumerQueue;
+        _options = options;
+        _context = context;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        _logger.LogInformation("***** Mysql output service started *****");
+
+        var tasks = new List<Task>();
+        var records = new List<DataRecord>();
+        while (!_context.IsTransformCompleted || _consumerQueue.Count > 0)
+        {
+            if (!_consumerQueue.TryDequeue(out var record)) continue;
+            records.Add(record);
+
+            if (records.Count >= 200)
+            {
+                var recordsCopy = records;
+                tasks.Add(Task.Run(async () => await FlushAsync(recordsCopy), stoppingToken));
+                records = [];
+            }
+
+            if (tasks.Count >= 10)
+            {
+                await Task.WhenAll(tasks);
+                tasks.Clear();
+            }
+        }
+
+        await Task.WhenAll(tasks);
+        await FlushAsync(records);
+        
+        _context.CompleteOutput();
+        
+        _logger.LogInformation("***** Mysql output service completed *****");
+    }
+
+    private async Task FlushAsync(IEnumerable<DataRecord> records)
+    {
+        var count = 0;
+        await using var output = new MySqlDestination(new MySqlConnectionStringBuilder
+        {
+            Server = _options.Value.Host,
+            Port = _options.Value.Port,
+            Database = _options.Value.Database,
+            UserID = _options.Value.User,
+            Password = _options.Value.Password,
+            ConnectionTimeout = 180,
+        }.ConnectionString, _logger, true);
+
+        foreach (var record in records)
+        {
+            await output.WriteRecordAsync(record);
+            count++;
+        }
+
+        await output.FlushAsync();
+        _context.AddOutput(count);
+    }
+}

ConsoleApp2/HostedServices/SqlFileOutputService.cs

@@ -0,0 +1,64 @@
+using ConsoleApp2.Entities;
+using ConsoleApp2.Helpers;
+using ConsoleApp2.Services;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+
+namespace ConsoleApp2.HostedServices;
+
+public class SqlFileOutputService : BackgroundService
+{
+    private readonly string _outputFile = "D:/DumpOutput/cferp_test_1.sql"; //
+    private readonly DataRecordQueue _consumerQueue;
+    private readonly ILogger _logger;
+    private readonly ProcessContext _context;
+
+    public SqlFileOutputService(
+        ILogger<SqlFileOutputService> logger,
+        [FromKeyedServices(ProcessStep.Consumer)]
+        DataRecordQueue consumerQueue,
+        ProcessContext context)
+    {
+        _logger = logger;
+        _consumerQueue = consumerQueue;
+        _context = context;
+    }
+
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        _logger.LogInformation("***** Sql file output service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId);
+        var count = 0;
+        var tableRecords = new Dictionary<string, IList<DataRecord>>();
+        while (!_context.IsTransformCompleted || _consumerQueue.Count > 0)
+        {
+            if (!_consumerQueue.TryDequeue(out var record)) continue;
+
+            tableRecords.AddOrUpdate(record.TableName, [record], (key, value) =>
+            {
+                value.Add(record);
+                return value;
+            });
+
+            ++count;
+
+            if (count >= 200)
+            {
+                await File.AppendAllTextAsync(_outputFile,
+                    MySqlDestination.SerializeRecords(tableRecords), stoppingToken);
+                tableRecords.Clear();
+                _context.AddOutput(count);
+                count = 0;
+            }
+        }
+        await File.AppendAllTextAsync(_outputFile,
+            MySqlDestination.SerializeRecords(tableRecords), stoppingToken);
+        tableRecords.Clear();
+        _context.AddOutput(count);
+        _context.CompleteOutput();
+        
+        _logger.LogInformation("***** Sql file output service completed *****");
+    }
+   
+}

ConsoleApp2/HostedServices/TaskMonitorService.cs

@@ -0,0 +1,118 @@
+using System.Diagnostics;
+using ConsoleApp2.Services;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+
+namespace ConsoleApp2.HostedServices;
+
+public class TaskMonitorService : BackgroundService
+{
+    private readonly IHostApplicationLifetime _lifetime;
+    private readonly ILogger<TaskMonitorService> _logger;
+    private readonly ProcessContext _context;
+    private readonly DataRecordQueue _producerQueue;
+    private readonly DataRecordQueue _consumerQueue;
+
+    public TaskMonitorService(IHostApplicationLifetime lifetime,
+        ILogger<TaskMonitorService> logger,
+        ProcessContext context,
+        [FromKeyedServices(ProcessStep.Producer)]
+        DataRecordQueue producerQueue,
+        [FromKeyedServices(ProcessStep.Consumer)]
+        DataRecordQueue consumerQueue)
+    {
+        _lifetime = lifetime;
+        _logger = logger;
+        _context = context;
+        _producerQueue = producerQueue;
+        _consumerQueue = consumerQueue;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        await Task.Factory.StartNew(Monitor, stoppingToken);
+    }
+
+    private async Task Monitor()
+    {
+        var sw = Stopwatch.StartNew();
+        var lastTime = sw.ElapsedMilliseconds;
+        var lastInputCount = _context.InputCount;
+        var lastTransformCount = _context.TransformCount;
+        var lastOutputCount = _context.OutputCount;
+
+        bool endCheck = false;
+        while (true)
+        {
+            EndCheck:
+            // var running = 0;
+            // var error = 0;
+            // var completed = 0;
+            // var canceled = 0;
+            // foreach (var task in _taskManager.Tasks)
+            // {
+            //     switch (task.Status)
+            //     {
+            //         case TaskStatus.Canceled:
+            //             canceled++;
+            //             break;
+            //         case TaskStatus.Faulted:
+            //             error++;
+            //             break;
+            //         case TaskStatus.RanToCompletion:
+            //             completed++;
+            //             break;
+            //         default:
+            //             running++;
+            //             break;
+            //     }
+            // }
+
+            var time = sw.ElapsedMilliseconds;
+            var inputCount = _context.InputCount;
+            var transformCount = _context.TransformCount;
+            var outputCount = _context.OutputCount;
+            
+            var elapseTime = (time - lastTime) / 1000f;
+            var inputSpeed = (inputCount - lastInputCount) / elapseTime;
+            var transformSpeed = (transformCount - lastTransformCount) / elapseTime;
+            var outputSpeed = (outputCount - lastOutputCount) / elapseTime;
+
+            // _logger.LogInformation(
+            //     "Task monitor: running: {Running}, error: {Error}, completed: {Completed}, canceled: {Canceled}, outputSpeed: {Speed} records/s",
+            //     running, error, completed, canceled, outputSpeed);
+            _logger.LogInformation(
+                "Process monitor: input: {inputStatus}, transform: {transformStatus}, output: {outputStatus}\nInput: {InputCount}, Transform: {TransformCount}, Output: {OutputCount}",
+                _context.IsInputCompleted ? "completed" : $"running {inputSpeed:F2} records/s",
+                _context.IsTransformCompleted ? "completed" : $"running {transformSpeed:F2} records/s",
+                _context.IsOutputCompleted ? "completed" : $"running {outputSpeed:F2} records/s",
+                inputCount,
+                transformCount,
+                outputCount);
+            _logger.LogInformation("Queue monitor: producer queue: {ProducerQueue}, consumer queue: {ConsumerQueue}",
+                _producerQueue.Count, _consumerQueue.Count);
+
+            await Task.Delay(2000);
+
+            lastTime = time;
+            lastInputCount = inputCount;
+            lastTransformCount = transformCount;
+            lastOutputCount = outputCount;
+
+            if (_context is { IsInputCompleted: true, IsTransformCompleted: true, IsOutputCompleted: true })
+            {
+                if (!endCheck)
+                {
+                    endCheck = true;
+                    goto EndCheck;
+                }
+                break;
+            }
+        }
+        
+        _logger.LogInformation("***** All tasks completed *****");
+        _logger.LogInformation("***** ElapseTime: {Time}", (sw.ElapsedMilliseconds / 1000f).ToString("F3"));
+        // _lifetime.StopApplication();
+    }
+}

ConsoleApp2/HostedServices/VoidOutputService.cs

@@ -0,0 +1,35 @@
+using ConsoleApp2.Services;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+
+namespace ConsoleApp2.HostedServices;
+
+public class VoidOutputService : BackgroundService
+{
+    private readonly ILogger _logger;
+    private readonly DataRecordQueue _consumerQueue;
+    private readonly ProcessContext _context;
+
+    public VoidOutputService([FromKeyedServices(ProcessStep.Consumer)] DataRecordQueue consumerQueue,
+        ProcessContext context, ILogger<VoidOutputService> logger)
+    {
+        _consumerQueue = consumerQueue;
+        _context = context;
+        _logger = logger;
+    }
+
+    protected override Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        _logger.LogInformation("***** Void output service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId);
+        while (!_context.IsTransformCompleted || _consumerQueue.Count > 0)
+        {
+            if (_consumerQueue.TryDequeue(out var record))
+                _context.AddOutput();
+        }
+
+        _context.CompleteOutput();
+        _logger.LogInformation("***** Void output service completed *****");
+        return Task.CompletedTask;
+    }
+}