CZY/MES-ETL
1
0
Fork 0
Code Pull Requests Releases Activity

添加数据分库;

Browse Source 
修复taskManager中异步方法没有正常等待的错误;
删除无用的异常捕获;
This commit is contained in:
陈梓阳
2024-01-19 11:17:22 +08:00
parent 45ad15a065
commit 8da3110ecd
12 changed files with 94 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
ConsoleApp2/HostedServices/InputService.cs
View File

@@ -49,11 +49,7 @@ public class InputService : IInputService
count++;
});
if (_context.GetExceptions().Count > 0)
{
_logger.LogInformation("***** Csv input service is canceled *****");
return;
}
_logger.LogInformation("table:'{tableName}' input completed", tableName);
}

12
ConsoleApp2/HostedServices/MainHostedService.cs
View File

@@ -34,8 +34,8 @@ public class MainHostedService : BackgroundService
}
catch (Exception ex)
{
_context.AddException(ex);
_logger.LogError("Exception occurred on inputService:{Message},{StackTrace}", ex.Message, ex.StackTrace);
_logger.LogCritical("Exception occurred on inputService:{Message},{StackTrace}", ex.Message, ex.StackTrace);
throw;
}
});
@@ -47,8 +47,8 @@ public class MainHostedService : BackgroundService
}
catch (Exception ex)
{
_context.AddException(ex);
_logger.LogError("Exception occurred on transformService:{Message},{StackTrace}", ex.Message, ex.StackTrace);
_logger.LogCritical("Exception occurred on transformService:{Message},{StackTrace}", ex.Message, ex.StackTrace);
throw;
}
});
@@ -60,8 +60,8 @@ public class MainHostedService : BackgroundService
}
catch (Exception ex)
{
_context.AddException(ex);
_logger.LogError("Exception occurred on outputService:{Message},{StackTrace}", ex.Message, ex.StackTrace);
_logger.LogCritical("Exception occurred on outputService:{Message},{StackTrace}", ex.Message, ex.StackTrace);
throw;
}
});

47
ConsoleApp2/HostedServices/OutputService.cs
View File

@@ -1,8 +1,7 @@
using ConsoleApp2.Const;
using ConsoleApp2.Helpers;
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@@ -35,48 +34,54 @@ public class OutputService : IOutputService
_errorRecorder = errorRecorder;
}
public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue consumerQueue, ProcessContext context,CancellationToken cancellationToken)
public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken)
{
_logger.LogInformation("***** Mysql output service started *****");
_taskManager.CreateTasks(async () =>
{
var records = new List<DataRecord>();
//k: database v: records，按照要导出的数据库名分组
var databaseDict = new Dictionary<string, List<DataRecord>>();
while (!context.IsTransformCompleted || consumerQueue.Count > 0)
{
if (!consumerQueue.TryDequeue(out var record)) continue;
records.Add(record);
//_logger.LogInformation(@"*****OutputCount: {count} *****",count);
var dbName = record.Database;
var records = databaseDict.AddOrUpdate(dbName, [record], (_, list) =>
{
list.Add(record);
return list;
});
if (records.Count >= tasksOptions.OutPutOptions.FlushCount)
{
await FlushAsync(records);
await FlushAsync(dbName, records);
records.Clear();
}
if (_context.GetExceptions().Count>0)
}
foreach (var (db, records) in databaseDict)
{
if (records.Count > 0)
{
_logger.LogInformation("***** Csv output thread is canceled *****");
return;
await FlushAsync(db, records);
records.Clear();
}
}
if (records.Count > 0)
{
await FlushAsync(records);
records.Clear();
_logger.LogInformation("***** Mysql output thread completed *****");
}
databaseDict.Clear();
_logger.LogInformation("***** Mysql output thread completed *****");
}, tasksOptions.OutPutOptions.OutPutTaskCount);
await _taskManager.WaitAll();
//_context.CompleteOutput();
_logger.LogInformation(@"***** Mysql output service completed *****");
_logger.LogInformation("***** Mysql output service completed *****");
}
private async Task FlushAsync(IEnumerable<DataRecord> records)
private async Task FlushAsync(string dbName, IEnumerable<DataRecord> records)
{
var count = 0;
await using var output = new MySqlDestination(
_outputOptions.Value.ConnectionString ?? throw new InvalidOperationException("Connection string is required"),
_logger, _context, _transformOptions, _errorRecorder);
var connStr = _outputOptions.Value.ConnectionString ?? throw new InvalidOperationException("ConnectionString is null");
await using var output = new MySqlDestination($"{connStr};Database={dbName};", _logger, _context, _transformOptions, _errorRecorder);
//if (records == null || records.Count() == 0) return;
//var dbName = $"cferp_test_1";
//if (records != null && records.Count() > 0)

1
ConsoleApp2/HostedServices/TaskMonitorService.cs
View File

@@ -49,7 +49,6 @@ public class TaskMonitorService : BackgroundService
bool endCheck = false;
while (true)
{
if (_context.GetExceptions().Count>0) return;
EndCheck:
// var running = 0;
// var error = 0;

12
ConsoleApp2/HostedServices/TransformService.cs
View File

@@ -43,16 +43,10 @@ public class TransformService : ITransformService
{
while ((!context.IsInputCompleted || producerQueue.Count > 0))
{
if (_context.GetExceptions().Count > 0)
{
_logger.LogInformation("***** Csv transform service is canceled *****");
return;
}
if (!producerQueue.TryDequeue(out var record)) continue;
//过滤不要的record
if (await _options.Value.RecordFilter?.Invoke(record, _cache) == false) continue;
record.Database = _options.Value.DatabaseFilter?.Invoke(record);
//修改record
_options.Value.RecordModify?.Invoke(record);
//缓存record
@@ -63,14 +57,17 @@ public class TransformService : ITransformService
{
record = replaceRecord;
}
//计算需要分流的数据库
record.Database = _options.Value.DatabaseFilter.Invoke(record);
consumerQueue.Enqueue(record);
_context.AddTransform();
//数据增加
var addRecords = _options.Value.RecordAdd?.Invoke(record);
if (addRecords != null && addRecords.Count > 0)
if (addRecords is { Count: > 0 })
{
foreach (var rc in addRecords)
{
rc.Database = _options.Value.DatabaseFilter.Invoke(record);
consumerQueue.Enqueue(rc);
_context.AddTransform();
}
@@ -78,6 +75,7 @@ public class TransformService : ITransformService
}
context.CompleteTransform();
},tasksOptions.TransformTaskCount,cancellationToken);
await _taskManager.WaitAll();
_logger.LogInformation("***** Data transformation service completed *****");
}
}