修改

ConsoleApp2/HostedServices/InputService.cs

@@ -15,54 +15,52 @@ namespace ConsoleApp2.HostedServices;
 public class InputService : IInputService
 {
     private readonly ILogger _logger;
-    private readonly IOptions<CsvOptions> _csvOptions;
+    private readonly IOptions<DataInputOptions> _dataInputOptions;
+    private readonly IOptions<InputTableOptions> _tableOptions;
     private readonly DataRecordQueue _producerQueue;
     private readonly ProcessContext _context;
 
     public InputService(ILogger<InputService> logger,
-        IOptions<CsvOptions> csvOptions, 
-        [FromKeyedServices(ProcessStep.Producer)]DataRecordQueue producerQueue, 
-        ProcessContext context)
+            IOptions<DataInputOptions> dataInputOptions,
+            IOptions<InputTableOptions> tableOptions,
+            [FromKeyedServices(ProcessStep.Producer)] DataRecordQueue producerQueue,
+            ProcessContext context)
     {
         _logger = logger;
-        _csvOptions = csvOptions;
+        _dataInputOptions = dataInputOptions;
+        _tableOptions = tableOptions;
         _producerQueue = producerQueue;
         _context = context;
     }
 
     public async Task ExecuteAsync(CancellationToken cancellationToken)
     {
-        var inputDir = _csvOptions.Value.InputDir;
+        var inputDir = _dataInputOptions.Value.InputDir;
         _logger.LogInformation("***** Csv input service start, working dir: {InputDir}, thread id: {ThreadId} *****", inputDir, Environment.CurrentManagedThreadId);
-        var files = Directory.GetFiles(inputDir).Where(s => s.EndsWith(".sql") && !s.Contains("schema")).ToArray();
+        var files = Directory.GetFiles(inputDir);
         if (files.Length == 0)
         {
-            _logger.LogInformation("No sql files found in {InputDir}", inputDir);
+            _logger.LogInformation("No source files found in {InputDir}", inputDir);
             return;
         }
-
-        foreach (var sqlPath in files)
+        var count = 0;
+        foreach (var tableName in _tableOptions.Value.TableInfoConfig.Keys)
         {
-            _logger.LogInformation("Working sql file: {SqlPath}", sqlPath);
-            var headers = await DumpDataHelper.GetCsvHeadersFromSqlFileAsync(sqlPath);
-            var csvFiles = await DumpDataHelper.GetCsvFileNamesFromSqlFileAsync(sqlPath);
-
-            foreach (var csvFile in csvFiles)
+            _logger.LogInformation("Working table: {tableName}", tableName);
+            var source = _dataInputOptions.Value.CreateSource?.Invoke(tableName);
+            await source.DoEnqueue((record) =>
             {
-                var csvPath = Path.Combine(inputDir, csvFile);
-                // var source = new JsvSource(csvPath, headers, _logger);
-                var source = new CsvSource(csvPath, headers, _csvOptions.Value.Delimiter, _csvOptions.Value.QuoteChar, _logger);
+                _context.AddInput();
+                _producerQueue.Enqueue(record);
+                count++;
 
-                while (await source.ReadAsync())
-                {
-                    _context.AddInput();
-                    _producerQueue.Enqueue(source.Current);
-                    if (cancellationToken.IsCancellationRequested)
-                        return;
-                }
+            });
+            if (_context.GetExceptions().Count > 0)
+            {
+                _logger.LogInformation("***** Csv input service is canceled *****");
+                return;
             }
-
-            _logger.LogInformation("File '{File}' input completed", Path.GetFileName(sqlPath));
+            _logger.LogInformation("table:'{tableName}' input completed", tableName);
         }
 
         _context.CompleteInput();