Update

2024-01-04 09:00:44 +08:00
parent c53d2927bb
commit eab3695f53
27 changed files with 258 additions and 466 deletions
--- a/ConsoleApp2/HostedServices/InputService.cs
+++ b/ConsoleApp2/HostedServices/InputService.cs
@@ -0,0 +1,71 @@
+using ConsoleApp2.Const;
+using ConsoleApp2.Helpers;
+using ConsoleApp2.HostedServices.Abstractions;
+using ConsoleApp2.Options;
+using ConsoleApp2.Services;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace ConsoleApp2.HostedServices;
+
+/// <summary>
+/// 从MyDumper导出的CSV文件中导入表头和数据
+/// </summary>
+public class InputService : IInputService
+{
+    private readonly ILogger _logger;
+    private readonly IOptions<CsvOptions> _csvOptions;
+    private readonly DataRecordQueue _producerQueue;
+    private readonly ProcessContext _context;
+
+    public InputService(ILogger<InputService> logger,
+        IOptions<CsvOptions> csvOptions, 
+        [FromKeyedServices(ProcessStep.Producer)]DataRecordQueue producerQueue, 
+        ProcessContext context)
+    {
+        _logger = logger;
+        _csvOptions = csvOptions;
+        _producerQueue = producerQueue;
+        _context = context;
+    }
+
+    public async Task ExecuteAsync(CancellationToken cancellationToken)
+    {
+        var inputDir = _csvOptions.Value.InputDir;
+        _logger.LogInformation("***** Csv input service start, working dir: {InputDir}, thread id: {ThreadId} *****", inputDir, Environment.CurrentManagedThreadId);
+        var files = Directory.GetFiles(inputDir).Where(s => s.EndsWith(".sql") && !s.Contains("schema")).ToArray();
+        if (files.Length == 0)
+        {
+            _logger.LogInformation("No sql files found in {InputDir}", inputDir);
+            return;
+        }
+
+        foreach (var sqlPath in files)
+        {
+            _logger.LogInformation("Working sql file: {SqlPath}", sqlPath);
+            var headers = await DumpDataHelper.GetCsvHeadersFromSqlFileAsync(sqlPath);
+            var csvFiles = await DumpDataHelper.GetCsvFileNamesFromSqlFileAsync(sqlPath);
+
+            foreach (var csvFile in csvFiles)
+            {
+                var csvPath = Path.Combine(inputDir, csvFile);
+                // var source = new JsvSource(csvPath, headers, _logger);
+                var source = new CsvSource(csvPath, headers, _csvOptions.Value.Delimiter, _csvOptions.Value.QuoteChar, _logger);
+
+                while (await source.ReadAsync())
+                {
+                    _context.AddInput();
+                    _producerQueue.Enqueue(source.Current);
+                    if (cancellationToken.IsCancellationRequested)
+                        return;
+                }
+            }
+
+            _logger.LogInformation("File '{File}' input completed", Path.GetFileName(sqlPath));
+        }
+
+        _context.CompleteInput();
+        _logger.LogInformation("***** Csv input service completed *****");
+    }
+}