79 lines
2.7 KiB
C#
79 lines
2.7 KiB
C#
using ConsoleApp2.Const;
|
|
using ConsoleApp2.HostedServices.Abstractions;
|
|
using ConsoleApp2.Options;
|
|
using ConsoleApp2.Services;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.Logging;
|
|
using Microsoft.Extensions.Options;
|
|
|
|
namespace ConsoleApp2.HostedServices;
|
|
|
|
/// <summary>
|
|
/// 数据处理服务，对导入后的数据进行处理
|
|
/// </summary>
|
|
public class TransformService : ITransformService
|
|
{
|
|
private readonly ILogger _logger;
|
|
private readonly IOptions<DataTransformOptions> _options;
|
|
private readonly DataRecordQueue _producerQueue;
|
|
private readonly DataRecordQueue _consumerQueue;
|
|
private readonly ProcessContext _context;
|
|
|
|
|
|
public TransformService(ILogger<TransformService> logger,
|
|
IOptions<DataTransformOptions> options,
|
|
[FromKeyedServices(ProcessStep.Producer)]DataRecordQueue producerQueue,
|
|
[FromKeyedServices(ProcessStep.Consumer)]DataRecordQueue consumerQueue,
|
|
ProcessContext context)
|
|
{
|
|
_logger = logger;
|
|
_options = options;
|
|
_producerQueue = producerQueue;
|
|
_consumerQueue = consumerQueue;
|
|
_context = context;
|
|
}
|
|
|
|
public async Task ExecuteAsync(CancellationToken cancellationToken)
|
|
{
|
|
_logger.LogInformation("***** Data transform service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId);
|
|
while (!_context.IsInputCompleted || _producerQueue.Count > 0)
|
|
{
|
|
// var dbOptions = _options.Value.DatabaseFilter(record);
|
|
if (!_producerQueue.TryDequeue(out var record)) continue;
|
|
record.Database = _options.Value.DatabaseFilter?.Invoke(record);
|
|
|
|
for (var i = 0; i < record.Fields.Length; i++)
|
|
{
|
|
var field = record[i];
|
|
|
|
if (field == "\\N")
|
|
{
|
|
field = "NULL";
|
|
goto Escape;
|
|
}
|
|
// else if(DumpDataHelper.CheckHexField(field))
|
|
// field = $"0x{field}";
|
|
|
|
switch (_options.Value.GetColumnType(record.TableName, record.Headers[i]))
|
|
{
|
|
case ColumnType.Blob or ColumnType.Text:
|
|
field = string.IsNullOrEmpty(field) ? "''" : $"0x{field}";
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
|
|
Escape:
|
|
record[i] = field;
|
|
}
|
|
|
|
// TODO: 数据处理/过滤/复制
|
|
|
|
_consumerQueue.Enqueue(record);
|
|
_context.AddTransform();
|
|
}
|
|
|
|
_context.CompleteTransform();
|
|
_logger.LogInformation("***** Data transformation service completed *****");
|
|
}
|
|
} |