CZY/MES-ETL
1
0
Fork 0
Code Pull Requests Releases Activity

Update

Browse Source
This commit is contained in:
陈梓阳
2024-01-04 09:00:44 +08:00
parent c53d2927bb
commit eab3695f53
27 changed files with 258 additions and 466 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
ConsoleApp2/HostedServices/Abstractions/IInputService.cs Normal file
View File

@@ -0,0 +1,6 @@
namespace ConsoleApp2.HostedServices.Abstractions;
public interface IInputService
{
public Task ExecuteAsync(CancellationToken cancellationToken);
}

6
ConsoleApp2/HostedServices/Abstractions/IOutputService.cs Normal file
View File

@@ -0,0 +1,6 @@
namespace ConsoleApp2.HostedServices.Abstractions;
public interface IOutputService
{
public Task ExecuteAsync(CancellationToken cancellationToken);
}

6
ConsoleApp2/HostedServices/Abstractions/ITransformService.cs Normal file
View File

@@ -0,0 +1,6 @@
namespace ConsoleApp2.HostedServices.Abstractions;
public interface ITransformService
{
public Task ExecuteAsync(CancellationToken cancellationToken);
}

21
ConsoleApp2/HostedServices/CsvInputService.cs → ConsoleApp2/HostedServices/InputService.cs
View File

@@ -1,35 +1,36 @@
using ConsoleApp2.Helpers;
using ConsoleApp2.Const;
using ConsoleApp2.Helpers;
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace ConsoleApp2.HostedServices;
public class CsvInputService : BackgroundService
/// <summary>
/// 从MyDumper导出的CSV文件中导入表头和数据
/// </summary>
public class InputService : IInputService
{
private readonly ILogger _logger;
private readonly IOptions<CsvOptions> _csvOptions;
private readonly TaskManager _taskManager; // TBD
private readonly DataRecordQueue _producerQueue;
private readonly ProcessContext _context;
public CsvInputService(ILogger<CsvInputService> logger,
IOptions<CsvOptions> csvOptions,
[FromKeyedServices(ProcessStep.Producer)]TaskManager taskManager,
public InputService(ILogger<InputService> logger,
IOptions<CsvOptions> csvOptions,
[FromKeyedServices(ProcessStep.Producer)]DataRecordQueue producerQueue,
ProcessContext context)
{
_logger = logger;
_csvOptions = csvOptions;
_taskManager = taskManager;
_producerQueue = producerQueue;
_context = context;
}
protected override async Task ExecuteAsync(CancellationToken cancellationToken)
public async Task ExecuteAsync(CancellationToken cancellationToken)
{
var inputDir = _csvOptions.Value.InputDir;
_logger.LogInformation("***** Csv input service start, working dir: {InputDir}, thread id: {ThreadId} *****", inputDir, Environment.CurrentManagedThreadId);
@@ -50,7 +51,7 @@ public class CsvInputService : BackgroundService
{
var csvPath = Path.Combine(inputDir, csvFile);
// var source = new JsvSource(csvPath, headers, _logger);
var source = new NewCsvSource(csvPath, headers, logger: _logger);
var source = new CsvSource(csvPath, headers, _csvOptions.Value.Delimiter, _csvOptions.Value.QuoteChar, _logger);
while (await source.ReadAsync())
{

30
ConsoleApp2/HostedServices/MainHostedService.cs Normal file
View File

@@ -0,0 +1,30 @@
using ConsoleApp2.HostedServices.Abstractions;
using Microsoft.Extensions.Hosting;
namespace ConsoleApp2.HostedServices;
public class MainHostedService : BackgroundService
{
private readonly IInputService _input;
private readonly ITransformService _transform;
private readonly IOutputService _output;
public MainHostedService(IInputService input, ITransformService transform, IOutputService output)
{
_input = input;
_transform = transform;
_output = output;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var tasks = new List<Task>()
{
Task.Run(async () => await _input.ExecuteAsync(stoppingToken), stoppingToken),
Task.Run(async () => await _transform.ExecuteAsync(stoppingToken), stoppingToken),
Task.Run(async () => await _output.ExecuteAsync(stoppingToken), stoppingToken),
};
await Task.WhenAll(tasks);
// await Task.Run(async () => await _output.ExecuteAsync(stoppingToken), stoppingToken);
}
}

56
ConsoleApp2/HostedServices/MysqlOutputService.cs → ConsoleApp2/HostedServices/OutputService.cs
View File

@@ -1,78 +1,76 @@
using ConsoleApp2.Entities;
using ConsoleApp2.Const;
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MySqlConnector;
namespace ConsoleApp2.HostedServices;
public class MysqlOutputService : BackgroundService
/// <summary>
/// 数据导出服务，将数据导出至MySql服务
/// </summary>
public class OutputService : IOutputService
{
private readonly ILogger _logger;
private readonly DataRecordQueue _consumerQueue;
private readonly IOptions<DatabaseOptions> _options;
private readonly IOptions<DatabaseOutputOptions> _options;
private readonly ProcessContext _context;
private readonly TaskManager _taskManager;
public MysqlOutputService(ILogger<MysqlOutputService> logger,
[FromKeyedServices(ProcessStep.Consumer)]DataRecordQueue consumerQueue,
IOptions<DatabaseOptions> options,
ProcessContext context)
public OutputService(ILogger<OutputService> logger,
[FromKeyedServices(ProcessStep.Consumer)] DataRecordQueue consumerQueue,
IOptions<DatabaseOutputOptions> options,
ProcessContext context,
TaskManager taskManager)
{
_logger = logger;
_consumerQueue = consumerQueue;
_options = options;
_context = context;
_taskManager = taskManager;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
public async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("***** Mysql output service started *****");
var tasks = new List<Task>();
var records = new List<DataRecord>();
while (!_context.IsTransformCompleted || _consumerQueue.Count > 0)
{
if (!_consumerQueue.TryDequeue(out var record)) continue;
records.Add(record);
if (records.Count >= 200)
if (records.Count >= _options.Value.FlushCount)
{
var recordsCopy = records;
tasks.Add(Task.Run(async () => await FlushAsync(recordsCopy), stoppingToken));
_taskManager.CreateTask(async () => await FlushAsync(recordsCopy), stoppingToken);
records = [];
}
if (tasks.Count >= 10)
if (_taskManager.TaskCount >= _options.Value.MaxTask)
{
await Task.WhenAll(tasks);
tasks.Clear();
await _taskManager.WaitAll();
_taskManager.ClearTask();
}
}
await Task.WhenAll(tasks);
await _taskManager.WaitAll();
await FlushAsync(records);
_context.CompleteOutput();
_logger.LogInformation("***** Mysql output service completed *****");
}
private async Task FlushAsync(IEnumerable<DataRecord> records)
{
var count = 0;
await using var output = new MySqlDestination(new MySqlConnectionStringBuilder
{
Server = _options.Value.Host,
Port = _options.Value.Port,
Database = _options.Value.Database,
UserID = _options.Value.User,
Password = _options.Value.Password,
ConnectionTimeout = 180,
}.ConnectionString, _logger, true);
await using var output = new MySqlDestination(
_options.Value.ConnectionString ?? throw new InvalidOperationException("Connection string is required"),
_logger, true);
foreach (var record in records)
{
await output.WriteRecordAsync(record);

64
ConsoleApp2/HostedServices/SqlFileOutputService.cs
View File

@@ -1,64 +0,0 @@
using ConsoleApp2.Entities;
using ConsoleApp2.Helpers;
using ConsoleApp2.Services;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace ConsoleApp2.HostedServices;
public class SqlFileOutputService : BackgroundService
{
private readonly string _outputFile = "D:/DumpOutput/cferp_test_1.sql"; //
private readonly DataRecordQueue _consumerQueue;
private readonly ILogger _logger;
private readonly ProcessContext _context;
public SqlFileOutputService(
ILogger<SqlFileOutputService> logger,
[FromKeyedServices(ProcessStep.Consumer)]
DataRecordQueue consumerQueue,
ProcessContext context)
{
_logger = logger;
_consumerQueue = consumerQueue;
_context = context;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("***** Sql file output service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId);
var count = 0;
var tableRecords = new Dictionary<string, IList<DataRecord>>();
while (!_context.IsTransformCompleted || _consumerQueue.Count > 0)
{
if (!_consumerQueue.TryDequeue(out var record)) continue;
tableRecords.AddOrUpdate(record.TableName, [record], (key, value) =>
{
value.Add(record);
return value;
});
++count;
if (count >= 200)
{
await File.AppendAllTextAsync(_outputFile,
MySqlDestination.SerializeRecords(tableRecords), stoppingToken);
tableRecords.Clear();
_context.AddOutput(count);
count = 0;
}
}
await File.AppendAllTextAsync(_outputFile,
MySqlDestination.SerializeRecords(tableRecords), stoppingToken);
tableRecords.Clear();
_context.AddOutput(count);
_context.CompleteOutput();
_logger.LogInformation("***** Sql file output service completed *****");
}
}

6
ConsoleApp2/HostedServices/TaskMonitorService.cs
View File

@@ -1,4 +1,5 @@
using System.Diagnostics;
using ConsoleApp2.Const;
using ConsoleApp2.Services;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
@@ -6,6 +7,9 @@ using Microsoft.Extensions.Logging;
namespace ConsoleApp2.HostedServices;
/// <summary>
/// 任务监控
/// </summary>
public class TaskMonitorService : BackgroundService
{
private readonly IHostApplicationLifetime _lifetime;
@@ -93,7 +97,7 @@ public class TaskMonitorService : BackgroundService
_logger.LogInformation("Queue monitor: producer queue: {ProducerQueue}, consumer queue: {ConsumerQueue}",
_producerQueue.Count, _consumerQueue.Count);
await Task.Delay(2000);
await Task.Delay(5000);
lastTime = time;
lastInputCount = inputCount;

19
ConsoleApp2/HostedServices/DataTransformService.cs → ConsoleApp2/HostedServices/TransformService.cs
View File

@@ -1,14 +1,17 @@
using ConsoleApp2.Helpers;
using ConsoleApp2.Const;
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace ConsoleApp2.HostedServices;
public class DataTransformService : BackgroundService
/// <summary>
/// 数据处理服务，对导入后的数据进行处理
/// </summary>
public class TransformService : ITransformService
{
private readonly ILogger _logger;
private readonly IOptions<DataTransformOptions> _options;
@@ -17,21 +20,20 @@ public class DataTransformService : BackgroundService
private readonly ProcessContext _context;
public DataTransformService(ILogger<DataTransformService> logger,
IOptions<DataTransformOptions> options, // TBD: database filter
public TransformService(ILogger<TransformService> logger,
IOptions<DataTransformOptions> options,
[FromKeyedServices(ProcessStep.Producer)]DataRecordQueue producerQueue,
[FromKeyedServices(ProcessStep.Consumer)]DataRecordQueue consumerQueue,
ProcessContext context)
{
_logger = logger;
// _taskManager = taskManager;
_options = options;
_producerQueue = producerQueue;
_consumerQueue = consumerQueue;
_context = context;
}
protected override async Task ExecuteAsync(CancellationToken cancellationToken)
public async Task ExecuteAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("***** Data transform service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId);
while (!_context.IsInputCompleted || _producerQueue.Count > 0)
@@ -58,7 +60,6 @@ public class DataTransformService : BackgroundService
field = string.IsNullOrEmpty(field) ? "''" : $"0x{field}";
break;
default:
field = field;
break;
}
@@ -66,6 +67,8 @@ public class DataTransformService : BackgroundService
record[i] = field;
}
// TODO: 数据处理/过滤/复制
_consumerQueue.Enqueue(record);
_context.AddTransform();
}

9
ConsoleApp2/HostedServices/VoidOutputService.cs
View File

@@ -1,11 +1,14 @@
using ConsoleApp2.Services;
using ConsoleApp2.Const;
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Services;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace ConsoleApp2.HostedServices;
public class VoidOutputService : BackgroundService
// 空输出服务，测试用
public class VoidOutputService : IOutputService
{
private readonly ILogger _logger;
private readonly DataRecordQueue _consumerQueue;
@@ -19,7 +22,7 @@ public class VoidOutputService : BackgroundService
_logger = logger;
}
protected override Task ExecuteAsync(CancellationToken stoppingToken)
public Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("***** Void output service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId);
while (!_context.IsTransformCompleted || _consumerQueue.Count > 0)