CZY/MES-ETL
1
0
Fork 0
Code Pull Requests Releases Activity

添加配置项

Browse Source
This commit is contained in:
陈梓阳
2024-02-15 16:18:50 +08:00
parent f6af04bfcd
commit f689e1b659
10 changed files with 156 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
MesETL.App/HostedServices/FileInputService.cs
View File

@@ -2,6 +2,7 @@
using MesETL.App.Options;
using MesETL.App.Services;
using MesETL.App.Services.ETL;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@@ -32,18 +33,21 @@ public class FileInputService : IInputService
private readonly IOptions<DataInputOptions> _dataInputOptions;
private readonly ProcessContext _context;
private readonly DataReaderFactory _dataReaderFactory;
private readonly long _memoryThreshold;
public FileInputService(ILogger<FileInputService> logger,
IOptions<DataInputOptions> dataInputOptions,
ProcessContext context,
[FromKeyedServices(Const.ConstVar.Producer)] DataRecordQueue producerQueue,
DataReaderFactory dataReaderFactory)
DataReaderFactory dataReaderFactory,
IConfiguration configuration)
{
_logger = logger;
_dataInputOptions = dataInputOptions;
_context = context;
_producerQueue = producerQueue;
_dataReaderFactory = dataReaderFactory;
_memoryThreshold = (long)(configuration.GetValue<double>("MemoryThreshold", 8) * 1024 * 1024 * 1024);
}
public async Task ExecuteAsync(CancellationToken cancellationToken)
@@ -73,6 +77,12 @@ public class FileInputService : IInputService
while (await source.ReadAsync())
{
if (GC.GetTotalMemory(false) > _memoryThreshold)
{
_logger.LogWarning("内存过高，暂缓输入");
GC.Collect();
await Task.Delay(3000, cancellationToken);
}
var record = source.Current;
await _producerQueue.EnqueueAsync(record);
count++;

9
MesETL.App/HostedServices/MainHostedService.cs
View File

@@ -61,7 +61,9 @@ public class MainHostedService : BackgroundService
}
_stopwatch = Stopwatch.StartNew();
await SetVariableAsync(); // 开启延迟写入，禁用重做日志 >>> 重做日志处于禁用状态时不要关闭数据库服务！
var enableUnsafeVar = _config.GetValue<bool>("UnsafeVariable", false);
if (enableUnsafeVar)
await SetVariableAsync(); // 开启延迟写入，禁用重做日志 >>> 重做日志处于禁用状态时不要关闭数据库服务！
var monitorTask = Task.Run(async () => await _taskMonitor.Monitor(stoppingToken), stoppingToken);
var inputTask = ExecuteAndCatch(
@@ -77,7 +79,8 @@ public class MainHostedService : BackgroundService
_logger.LogInformation("***** ElapseTime: {Time}", (_stopwatch.ElapsedMilliseconds / 1000f).ToString("F3"));
await Task.Delay(5000, stoppingToken);
await SetVariableAsync(false); // 关闭延迟写入，开启重做日志
if(enableUnsafeVar)
await SetVariableAsync(false); // 关闭延迟写入，开启重做日志
if (!stoppingToken.IsCancellationRequested)
{
await ExportResultAsync();
@@ -167,7 +170,7 @@ public class MainHostedService : BackgroundService
sb.AppendLine("\n---\n");
sb.AppendLine("## Table Output Progress");
var tableOutputProgress = _context.TableProgress.Select(pair =>
new { Table = pair.Key, Count = pair.Value });
new { Table = pair.Key, Count = pair.Value }).OrderBy(s => s.Table);
sb.AppendLine(tableOutputProgress.ToMarkdownTable());
sb.AppendLine("\n---\n");
sb.AppendLine("## Result");

6
MesETL.App/HostedServices/OutputService.cs
View File

@@ -1,4 +1,5 @@
using MesETL.App.Helpers;
using System.Buffers;
using MesETL.App.Helpers;
using MesETL.App.HostedServices.Abstractions;
using MesETL.App.Options;
using MesETL.App.Services;
@@ -65,13 +66,14 @@ public class OutputService : IOutputService
{
_logger.LogInformation("*****开启输出线程，数据库: {db} *****", db);
var taskManager = new TaskManager(_outputOptions.Value.MaxDatabaseOutputTask);
var ignoreOutput = new HashSet<string>(_outputOptions.Value.NoOutput);
var tmp = new List<DataRecord>();
while (!_context.IsTransformCompleted || queue.Count > 0)
{
if (ct.IsCancellationRequested)
break;
if (!queue.TryDequeue(out var record)) continue;
if (!queue.TryDequeue(out var record) || ignoreOutput.Contains(record.TableName)) continue;
var dbName = record.Database ?? throw new ApplicationException("输出的记录缺少数据库名");
if(dbName != db)

17
MesETL.App/HostedServices/TaskMonitorService.cs
View File

@@ -2,7 +2,6 @@
using System.Text;
using MesETL.App.Services;
using MesETL.App.Services.Loggers;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
namespace MesETL.App.HostedServices;
@@ -16,32 +15,24 @@ public class TaskMonitorService
private readonly ProcessContext _context;
private readonly DataRecordQueue _producerQueue;
private readonly RecordQueuePool _queuePool;
private readonly IConfiguration _configuration;
private string _outputPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Log/progress.txt");
private readonly int _gcInterval;
public TaskMonitorService(ProcessContext context,
[FromKeyedServices(Const.ConstVar.Producer)]
DataRecordQueue producerQueue,
RecordQueuePool queuePool,
IEnumerable<ITaskMonitorLogger> monitorLoggers,
IConfiguration configuration)
IEnumerable<ITaskMonitorLogger> monitorLoggers)
{
_context = context;
_producerQueue = producerQueue;
_queuePool = queuePool;
_monitorLoggers = monitorLoggers;
_configuration = configuration;
_gcInterval = _configuration.GetValue<int>("GCIntervalMilliseconds)");
}
public async Task Monitor(CancellationToken stoppingToken)
{
var sw = Stopwatch.StartNew();
var lastGCTime = sw.ElapsedMilliseconds;
var lastTime = sw.ElapsedMilliseconds;
var lastInputCount = _context.InputCount;
var lastTransformCount = _context.TransformCount;
@@ -83,12 +74,6 @@ public class TaskMonitorService
var inputSpeed = (inputCount - lastInputCount) / elapseTime;
var transformSpeed = (transformCount - lastTransformCount) / elapseTime;
var outputSpeed = (outputCount - lastOutputCount) / elapseTime;
if(_gcInterval > 0 && time - lastGCTime > _gcInterval)
{
GC.Collect();
lastGCTime = time;
}
// _logger.LogInformation(
// "Task monitor: running: {Running}, error: {Error}, completed: {Completed}, canceled: {Canceled}, outputSpeed: {Speed} records/s",