CZY/MES-ETL
1
0
Fork 0
Code Pull Requests Releases Activity

优化内存分配

Browse Source
This commit is contained in:
陈梓阳
2024-01-16 15:35:54 +08:00
committed by lindj
parent dadb36b1c9
commit 1de3603afe
5 changed files with 89 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
ConsoleApp2/HostedServices/OutputService.cs
View File

@@ -17,24 +17,24 @@ public class OutputService : IOutputService
{
private readonly ILogger _logger;
private readonly DataRecordQueue _consumerQueue;
private readonly IOptions<DataTransformOptions> _transOptions;
private readonly IOptions<DatabaseOutputOptions> _options;
private readonly IOptions<DatabaseOutputOptions> _outputOptions;
private readonly IOptions<DataTransformOptions> _transformOptions;
private readonly ProcessContext _context;
private readonly TaskManager _taskManager;
public OutputService(ILogger<OutputService> logger,
[FromKeyedServices(ProcessStep.Consumer)] DataRecordQueue consumerQueue,
IOptions<DatabaseOutputOptions> options,
IOptions<DataTransformOptions> transOptions,
IOptions<DatabaseOutputOptions> outputOptions,
ProcessContext context,
TaskManager taskManager)
TaskManager taskManager,
IOptions<DataTransformOptions> transformOptions)
{
_logger = logger;
_consumerQueue = consumerQueue;
_transOptions = transOptions;
_options = options;
_outputOptions = outputOptions;
_context = context;
_taskManager = taskManager;
_transformOptions = transformOptions;
}
public async Task ExecuteAsync(CancellationToken cancellationToken)
@@ -50,9 +50,10 @@ public class OutputService : IOutputService
records.Add(record);
count++;
//_logger.LogInformation(@"*****OutputCount: {count} *****",count);
if (records.Count >= _options.Value.FlushCount)
if (records.Count >= _outputOptions.Value.FlushCount)
{
await FlushAsync(records);
_context.AddOutput(count);
records.Clear();
}
if (_context.GetExceptions().Count>0)
@@ -67,7 +68,7 @@ public class OutputService : IOutputService
records.Clear();
_logger.LogInformation("***** Mysql output thread completed *****");
}
}, _options.Value.TaskCount);
}, _outputOptions.Value.TaskCount);
await _taskManager.WaitAll();
//_context.CompleteOutput();
@@ -79,8 +80,8 @@ public class OutputService : IOutputService
{
var count = 0;
await using var output = new MySqlDestination(
_options.Value.ConnectionString ?? throw new InvalidOperationException("Connection string is required"),
_logger, _context,true);
_outputOptions.Value.ConnectionString ?? throw new InvalidOperationException("Connection string is required"),
_logger, _context, _transformOptions);
//if (records == null || records.Count() == 0) return;
//var dbName = $"cferp_test_1";
//if (records != null && records.Count() > 0)
@@ -102,7 +103,7 @@ public class OutputService : IOutputService
await output.WriteRecordAsync(record);
count++;
}
await output.FlushAsync(_options.Value.MaxAllowedPacket, _transOptions);
await output.FlushAsync(_outputOptions.Value.MaxAllowedPacket);
_context.AddOutput(count);
}
}

28
ConsoleApp2/HostedServices/TransformService.cs
View File

@@ -49,34 +49,6 @@ public class TransformService : ITransformService
// var dbOptions = _options.Value.DatabaseFilter(record);
if (!_producerQueue.TryDequeue(out var record)) continue;
for (var i = 0; i < record.Fields.Length; i++)
{
var field = record[i];
if (field == "\\N")
{
field = "NULL";
goto Escape;
}
// else if(DumpDataHelper.CheckHexField(field))
// field = $"0x{field}";
switch (_options.Value.GetColumnType(record.TableName, record.Headers[i]))
{
case ColumnType.Text:
field = string.IsNullOrEmpty(field) ? "''" : _options.Value.TransformBinary?.Invoke(field) ?? field; ;
break;
case ColumnType.Blob:
//field = string.IsNullOrEmpty(field) ? "NULL" : $"0x{field}";
break;
default:
break;
}
Escape:
record[i] = field;
}
//过滤不要的record
if ( await _options.Value.RecordFilter?.Invoke(record,_db) == false) continue;
record.Database = _options.Value.DatabaseFilter?.Invoke(record);