CZY/MES-ETL
1
0
Fork 0
Code Pull Requests Releases Activity

Update

Browse Source
This commit is contained in:
陈梓阳
2024-02-09 19:08:57 +08:00
parent 913c725fe1
commit 73895fbce4
17 changed files with 216 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
MesETL.App/HostedServices/FileInputService.cs
View File

@@ -73,11 +73,12 @@ public class FileInputService : IInputService
while (await source.ReadAsync())
{
var record = source.Current;
_producerQueue.Enqueue(record);
await _producerQueue.EnqueueAsync(record);
_context.AddInput();
}
_logger.LogInformation("Input of table: '{TableName}' finished", info.TableName);
_dataInputOptions.Value.OnTableInputCompleted?.Invoke(info.TableName);
}
_context.CompleteInput();
@@ -91,6 +92,7 @@ public class FileInputService : IInputService
private IEnumerable<FileInputInfo> GetFilesInOrder(FileInputInfo[] inputFiles)
{
var tableOrder = _dataInputOptions.Value.TableOrder;
var ignoreTable = _dataInputOptions.Value.TableIgnoreList;
if (tableOrder is null or { Length: 0 })
return inputFiles;
@@ -102,7 +104,7 @@ public class FileInputService : IInputService
{
var target = inputFiles.FirstOrDefault(f =>
f.TableName.Equals(tableName, StringComparison.OrdinalIgnoreCase));
if (target is not null)
if (target is not null && !ignoreTable.Contains(target.TableName))
yield return target;
}
}

34
MesETL.App/HostedServices/TaskMonitorService.cs
View File

@@ -1,4 +1,5 @@
using System.Diagnostics;
using System.Text;
using MesETL.App.Services;
using MesETL.App.Services.Loggers;
using Microsoft.Extensions.DependencyInjection;
@@ -14,6 +15,8 @@ public class TaskMonitorService
private readonly ProcessContext _context;
private readonly DataRecordQueue _producerQueue;
private readonly RecordQueuePool _queuePool;
private string _outputPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Log/progress.txt");
public TaskMonitorService(ProcessContext context,
[FromKeyedServices(Const.ConstVar.Producer)]
@@ -79,15 +82,24 @@ public class TaskMonitorService
{
logger.LogStatus("Monitor: Progress status", new Dictionary<string, string>
{
{"Input",_context.IsInputCompleted ? "completed" : $"running {inputSpeed:F2} records/s" },
{"Transform", _context.IsTransformCompleted ? "completed" : $"running {transformSpeed:F2} records/s" },
{"Output", _context.IsOutputCompleted ? "completed" : $"running {outputSpeed:F2} records/s" }
{"Input",_context.IsInputCompleted ? "OK" : $"{inputSpeed:F2}/s" },
{"Transform", _context.IsTransformCompleted ? "OK" : $"{transformSpeed:F2}/s" },
{"Output", _context.IsOutputCompleted ? "OK" : $"{outputSpeed:F2}/s" },
{"| Input Queue", _producerQueue.Count.ToString() },
{"Output Queue", _queuePool.Queues.Values.Sum(queue => queue.Count).ToString()},
});
logger.LogStatus("Monitor: Table output progress",
_context.TableProgress
.ToDictionary(kv => kv.Key, kv => kv.Value.ToString()),
ITaskMonitorLogger.LogLevel.Progress);
var dict = _context.TableProgress
.ToDictionary(kv => kv.Key, kv => kv.Value.ToString());
logger.LogStatus("Monitor: Table output progress", dict, ITaskMonitorLogger.LogLevel.Progress);
var sb = new StringBuilder("Table Progress: \n");
foreach (var kv in _context.TableProgress)
{
sb.AppendLine($"{kv.Key}: {kv.Value}");
}
await File.WriteAllTextAsync(_outputPath, sb.ToString(), CancellationToken.None);
logger.LogStatus("Monitor: Process count", new Dictionary<string, string>
{
@@ -95,14 +107,8 @@ public class TaskMonitorService
{"Transform", transformCount.ToString()},
{"Output", outputCount.ToString()}
}, ITaskMonitorLogger.LogLevel.Progress);
logger.LogStatus("Monitor: Queue", new Dictionary<string, string>
{
{"Producer queue records", _producerQueue.Count.ToString() },
{"Output queues", _queuePool.Queues.Count.ToString() },
{"Output queue records", _queuePool.Queues.Values.Sum(queue => queue.Count).ToString()},
});
}
await Task.Delay(5000, stoppingToken);

25
MesETL.App/HostedServices/TransformService.cs
View File

@@ -46,9 +46,26 @@ public class TransformService : ITransformService
{
_logger.LogInformation("***** Data transform service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId);
// var tasks = new List<Task>();
// for (int i = 0; i < 4; i++)
// {
// tasks.Add(Task.Run(TransformWorker, cancellationToken));
// }
//
// await Task.WhenAll(tasks);
await TransformWorker();
_logger.LogInformation("***** Data transformation service finished *****");
}
public async Task TransformWorker()
{
while (!_context.IsInputCompleted || _producerQueue.Count > 0)
{
if (!_producerQueue.TryDequeue(out var record)) continue;
if (!_producerQueue.TryDequeue(out var record))
{
continue;
}
try
{
@@ -80,7 +97,7 @@ public class TransformService : ITransformService
?? throw new ApplicationException("未配置数据库过滤器");
record.Database = dbFilter(record);
_queuePool[record.Database].Enqueue(record);
await _queuePool[record.Database].EnqueueAsync(record);
_context.AddTransform();
if (_options.Value.EnableReBuilder)
@@ -93,7 +110,7 @@ public class TransformService : ITransformService
{
if(dbFilter is not null)
rc.Database =dbFilter.Invoke(record);
_queuePool[record.Database].Enqueue(rc);
await _queuePool[record.Database].EnqueueAsync(rc);
_context.AddTransform();
}
}
@@ -110,7 +127,5 @@ public class TransformService : ITransformService
}
}
_context.CompleteTransform();
_logger.LogInformation("***** Data transformation service finished *****");
}
}