Init

ConsoleApp2/Services/DataRecordQueue.cs

@@ -0,0 +1,60 @@
+using System.Collections.Concurrent;
+using System.Diagnostics.CodeAnalysis;
+using ConsoleApp2.Entities;
+using Microsoft.Extensions.Logging;
+
+namespace ConsoleApp2.Services;
+
+public class DataRecordQueue
+{
+    /// <summary>
+    /// Indicate that the queue is completed adding.
+    /// </summary>
+    public bool IsCompletedAdding { get; private set; }
+    
+    /// <summary>
+    /// Remark that the queue is completed for adding and empty;
+    /// </summary>
+    public bool IsCompleted => IsCompletedAdding && _queue.IsEmpty;
+    
+    private readonly ConcurrentQueue<DataRecord> _queue;
+
+    public DataRecordQueue()
+    {
+        _queue = new ConcurrentQueue<DataRecord>();
+    }
+
+    public DataRecordQueue(IEnumerable<DataRecord> records)
+    {
+        _queue = new ConcurrentQueue<DataRecord>(records);
+    }
+    
+    /// <inheritdoc cref="ConcurrentQueue{T}.Enqueue"/>
+    public void Enqueue(DataRecord item)
+    {
+        _queue.Enqueue(item);
+    }
+
+    /// <inheritdoc cref="ConcurrentQueue{T}.TryDequeue"/>
+    public bool TryDequeue([MaybeNullWhen(false)] out DataRecord result)
+    {
+        return _queue.TryDequeue(out result);
+    }
+
+    /// <inheritdoc cref="ConcurrentQueue{T}.TryPeek"/>
+    public bool TryPeek([MaybeNullWhen(false)] out DataRecord result)
+    {
+        return _queue.TryPeek(out result);
+    }
+
+    /// <inheritdoc cref="ConcurrentQueue{T}.Count"/>
+    public int Count => _queue.Count;
+
+    /// <inheritdoc cref="ConcurrentQueue{T}.IsEmpty"/>
+    public bool IsEmpty => _queue.IsEmpty;
+
+    /// <summary>
+    /// Indicate that the queue is completed adding.
+    /// </summary>
+    public void CompleteAdding() => IsCompletedAdding = true;
+}

ConsoleApp2/Services/DataTransformService.cs

@@ -0,0 +1,46 @@
+using ConsoleApp2.Entities;
+using ConsoleApp2.Helpers;
+using ConsoleApp2.Options;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace ConsoleApp2.Services;
+
+public class DataTransformService
+{
+    private readonly ILogger _logger;
+    private readonly TaskManager _taskManager;
+    private readonly DatabaseOutputService _output;
+    private readonly IOptions<DataTransformOptions> _options;
+
+    public DataTransformService(ILogger<DataTransformService> logger, TaskManager taskManager, DatabaseOutputService output, IOptions<DataTransformOptions> options)
+    {
+        _logger = logger;
+        _taskManager = taskManager;
+        _output = output;
+        _options = options;
+    }
+
+    public async Task ExecuteAsync(DataRecordQueue records, CancellationToken cancellationToken = default)
+    {
+        _logger.LogInformation("Start transforming data.");
+        var map = new Dictionary<DatabaseOptions, DataRecordQueue>();
+        while (records.TryDequeue(out var record))
+        {
+            var dbOptions = _options.Value.DatabaseFilter(record);
+            map.AddOrUpdate(dbOptions, new DataRecordQueue([record]), (options, queue) =>
+            {
+                queue.Enqueue(record);
+                return queue;
+            });
+        }
+
+        foreach (var (dbOptions, queue) in map)
+        {
+            await _taskManager.CreateTask(async () =>
+            {
+                await _output.ExecuteAsync(queue, dbOptions, cancellationToken);
+            });
+        }
+    }
+}

ConsoleApp2/Services/DatabaseOutputService.cs

@@ -0,0 +1,39 @@
+using ConsoleApp2.Entities;
+using ConsoleApp2.Options;
+using Microsoft.Extensions.Logging;
+using MySqlConnector;
+
+namespace ConsoleApp2.Services;
+
+public class DatabaseOutputService
+{
+    private readonly ILogger _logger;
+
+    public DatabaseOutputService(ILogger<DatabaseOutputService> logger)
+    {
+        _logger = logger;
+    }
+
+    public async Task ExecuteAsync(DataRecordQueue records, DatabaseOptions options, CancellationToken stoppingToken = default)
+    {
+        var count = records.Count;
+        var output = new MySqlDestination(new MySqlConnectionStringBuilder()
+        {
+            Server = options.Host,
+            Port = options.Port,
+            Database = options.Database,
+            UserID = options.User,
+            Password = options.Password,
+            ConnectionTimeout = 120,
+        }.ConnectionString, _logger); // TODO: 加入DI
+        
+        while (records.TryDequeue(out var record) && !stoppingToken.IsCancellationRequested)
+        {
+            await output.WriteRecordAsync(record);
+        }
+        
+        await output.FlushAsync();
+
+        _logger.LogInformation("Flush {Count} records to database.", count);
+    }
+}

ConsoleApp2/Services/TaskManager.cs

@@ -0,0 +1,28 @@
+using System.Collections.Concurrent;
+using Microsoft.Extensions.Logging;
+
+namespace ConsoleApp2.Services;
+
+public class TaskManager
+{
+    private readonly ConcurrentBag<Task> _tasks;
+    private readonly ILogger _logger;
+
+    public int RunningTaskCount => _tasks.Count(task => !task.IsCompleted);
+    public IReadOnlyCollection<Task> Tasks => _tasks;
+    public bool MainTaskCompleted { get; set; }
+
+    public TaskManager(ILogger<TaskManager> logger)
+    {
+        _tasks = new ConcurrentBag<Task>();
+        _logger = logger;
+    }
+
+    public Task<TResult> CreateTask<TResult>(Func<TResult> func)
+    {
+        var task = Task.Factory.StartNew(func);
+        _tasks.Add(task);
+        _logger.LogInformation("New task created.");
+        return task;
+    }
+}

ConsoleApp2/Services/TaskMonitorService.cs

@@ -0,0 +1,57 @@
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+
+namespace ConsoleApp2.Services;
+
+public class TaskMonitorService : BackgroundService
+{
+    private readonly IHostApplicationLifetime _lifetime;
+    private readonly TaskManager _taskManager;
+    private readonly ILogger<TaskMonitorService> _logger;
+
+    public TaskMonitorService(IHostApplicationLifetime lifetime, TaskManager taskManager,
+        ILogger<TaskMonitorService> logger)
+    {
+        _lifetime = lifetime;
+        _taskManager = taskManager;
+        _logger = logger;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        while (!_taskManager.MainTaskCompleted || _taskManager.RunningTaskCount != 0)
+        {
+            var running = 0;
+            var error = 0;
+            var completed = 0;
+            var canceled = 0;
+            foreach (var task in _taskManager.Tasks)
+            {
+                switch (task.Status)
+                {
+                    case TaskStatus.Running:
+                        running++;
+                        break;
+                    case TaskStatus.Canceled:
+                        canceled++;
+                        break;
+                    case TaskStatus.Faulted:
+                        error++;
+                        break;
+                    case TaskStatus.RanToCompletion:
+                        completed++;
+                        break;
+                    default:
+                        throw new ArgumentOutOfRangeException();
+                }
+            }
+
+            _logger.LogInformation(
+                "Task monitor: running: {Running}, error: {Error}, completed: {Completed}, canceled: {Canceled}",
+                running, error, completed, canceled);
+            await Task.Delay(2000);
+        }
+
+        _logger.LogInformation("***** All tasks completed *****");
+    }
+}