CZY/MES-ETL
1
0
Fork 0
Code Pull Requests Releases Activity

Update

Browse Source
This commit is contained in:
陈梓阳 2024-01-04 09:00:44 +08:00
parent c53d2927bb
commit eab3695f53
27 changed files with 258 additions and 466 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ConsoleApp2/ProcessStep.cs → ConsoleApp2/Const/ProcessStep.cs
View File

@ -1,4 +1,4 @@
namespace ConsoleApp2;
namespace ConsoleApp2.Const;
public static class ProcessStep
{

5
ConsoleApp2/Entities/DataRecord.cs → ConsoleApp2/DataRecord.cs
View File

@ -1,6 +1,5 @@
using System.Text;
namespace ConsoleApp2;
namespace ConsoleApp2.Entities;
public class DataRecord
{
@ -9,7 +8,7 @@ public class DataRecord
value = string.Empty;
if (record.Headers is null)
throw new InvalidOperationException("Cannot get field when headers of a record have not been set.");
var idx = Array.IndexOf(record.Headers, columnName);
var idx = Array.IndexOf(record.Headers, columnName); //可能可以优化
if (idx == -1)
return false;
value = record.Fields[idx];

1
ConsoleApp2/Helpers/DumpDataHelper.cs
View File

@ -1,6 +1,5 @@
using System.Text;
using System.Text.RegularExpressions;
using ConsoleApp2.Entities;
namespace ConsoleApp2.Helpers;

249
ConsoleApp2/Helpers/HashExtension.cs
View File

@ -1,249 +0,0 @@
using System.Security.Cryptography;
using System.Text;
namespace ConsoleApp2.Helpers;
public static class HashExtensions
{
/// <summary>
/// 计算32位MD5码
/// </summary>
/// <param name="word">字符串</param>
/// <param name="toUpper">返回哈希值格式 true：英文大写，false：英文小写</param>
/// <returns></returns>
public static string ToMd5Hash(this string word, bool toUpper = true)
{
try
{
var MD5CSP = MD5.Create();
var bytValue = Encoding.UTF8.GetBytes(word);
var bytHash = MD5CSP.ComputeHash(bytValue);
MD5CSP.Clear();
//根据计算得到的Hash码翻译为MD5码
var sHash = "";
foreach (var t in bytHash)
{
long i = t / 16;
var sTemp = i > 9 ? ((char)(i - 10 + 0x41)).ToString() : ((char)(i + 0x30)).ToString();
i = t % 16;
if (i > 9)
{
sTemp += ((char)(i - 10 + 0x41)).ToString();
}
else
{
sTemp += ((char)(i + 0x30)).ToString();
}
sHash += sTemp;
}
//根据大小写规则决定返回的字符串
return toUpper ? sHash : sHash.ToLower();
}
catch (System.Exception ex)
{
throw new System.Exception(ex.Message);
}
}
public static string ToMd5Hash(this Stream stream, bool toUpper = true)
{
using var md5Hash = MD5.Create();
var bytes = md5Hash.ComputeHash(stream);
return ToHashString(bytes, toUpper);
}
/// <summary>
/// 计算SHA-1码
/// </summary>
/// <param name="word">字符串</param>
/// <param name="toUpper">返回哈希值格式 true：英文大写，false：英文小写</param>
/// <returns></returns>
public static string ToSHA1Hash(this string word, bool toUpper = true)
{
try
{
var SHA1CSP = SHA1.Create();
var bytValue = Encoding.UTF8.GetBytes(word);
var bytHash = SHA1CSP.ComputeHash(bytValue);
SHA1CSP.Clear();
//根据计算得到的Hash码翻译为SHA-1码
var sHash = "";
foreach (var t in bytHash)
{
long i = t / 16;
var sTemp = i > 9 ? ((char)(i - 10 + 0x41)).ToString() : ((char)(i + 0x30)).ToString();
i = t % 16;
if (i > 9)
{
sTemp += ((char)(i - 10 + 0x41)).ToString();
}
else
{
sTemp += ((char)(i + 0x30)).ToString();
}
sHash += sTemp;
}
//根据大小写规则决定返回的字符串
return toUpper ? sHash : sHash.ToLower();
}
catch (System.Exception ex)
{
throw new System.Exception(ex.Message);
}
}
/// <summary>
/// 计算SHA-256码
/// </summary>
/// <param name="word">字符串</param>
/// <param name="toUpper">返回哈希值格式 true：英文大写，false：英文小写</param>
/// <returns></returns>
public static string ToSHA256Hash(this string word, bool toUpper = true)
{
try
{
var SHA256CSP = SHA256.Create();
var bytValue = Encoding.UTF8.GetBytes(word);
var bytHash = SHA256CSP.ComputeHash(bytValue);
SHA256CSP.Clear();
//根据计算得到的Hash码翻译为SHA-1码
var sHash = "";
foreach (var t in bytHash)
{
long i = t / 16;
var sTemp = i > 9 ? ((char)(i - 10 + 0x41)).ToString() : ((char)(i + 0x30)).ToString();
i = t % 16;
if (i > 9)
{
sTemp += ((char)(i - 10 + 0x41)).ToString();
}
else
{
sTemp += ((char)(i + 0x30)).ToString();
}
sHash += sTemp;
}
//根据大小写规则决定返回的字符串
return toUpper ? sHash : sHash.ToLower();
}
catch (System.Exception ex)
{
throw new System.Exception(ex.Message);
}
}
/// <summary>
/// 计算SHA-256码
/// </summary>
/// <param name="stream"></param>
/// <param name="toUpper"></param>
/// <returns></returns>
public static string ToSHA256Hash(this Stream stream, bool toUpper = true)
{
using var sha256Hash = SHA256.Create();
var bytes = sha256Hash.ComputeHash(stream);
return ToHashString(bytes, toUpper);
}
/// <summary>
/// 计算SHA-384码
/// </summary>
/// <param name="word">字符串</param>
/// <param name="toUpper">返回哈希值格式 true：英文大写，false：英文小写</param>
/// <returns></returns>
public static string ToSHA384Hash(this string word, bool toUpper = true)
{
try
{
var SHA384CSP = SHA384.Create();
var bytValue = Encoding.UTF8.GetBytes(word);
var bytHash = SHA384CSP.ComputeHash(bytValue);
SHA384CSP.Clear();
//根据计算得到的Hash码翻译为SHA-1码
var sHash = "";
foreach (var t in bytHash)
{
long i = t / 16;
var sTemp = i > 9 ? ((char)(i - 10 + 0x41)).ToString() : ((char)(i + 0x30)).ToString();
i = t % 16;
if (i > 9)
{
sTemp += ((char)(i - 10 + 0x41)).ToString();
}
else
{
sTemp += ((char)(i + 0x30)).ToString();
}
sHash += sTemp;
}
//根据大小写规则决定返回的字符串
return toUpper ? sHash : sHash.ToLower();
}
catch (System.Exception ex)
{
throw new System.Exception(ex.Message);
}
}
/// <summary>
/// 计算SHA-512码
/// </summary>
/// <param name="word">字符串</param>
/// <param name="toUpper">返回哈希值格式 true：英文大写，false：英文小写</param>
/// <returns></returns>
public static string ToSHA512Hash(this string word, bool toUpper = true)
{
try
{
var SHA512CSP = SHA512.Create();
var bytValue = Encoding.UTF8.GetBytes(word);
var bytHash = SHA512CSP.ComputeHash(bytValue);
SHA512CSP.Clear();
//根据计算得到的Hash码翻译为SHA-1码
var sHash = "";
foreach (var t in bytHash)
{
long i = t / 16;
var sTemp = i > 9 ? ((char)(i - 10 + 0x41)).ToString() : ((char)(i + 0x30)).ToString();
i = t % 16;
if (i > 9)
{
sTemp += ((char)(i - 10 + 0x41)).ToString();
}
else
{
sTemp += ((char)(i + 0x30)).ToString();
}
sHash += sTemp;
}
//根据大小写规则决定返回的字符串
return toUpper ? sHash : sHash.ToLower();
}
catch (System.Exception ex)
{
throw new System.Exception(ex.Message);
}
}
private static string ToHashString(byte[] bytes, bool toUpper = true)
{
var builder = new StringBuilder();
foreach (var t in bytes)
{
builder.Append(t.ToString("x2"));
}
var str = builder.ToString();
return toUpper ? str.ToUpper() : str.ToLower();
}
}

6
ConsoleApp2/HostedServices/Abstractions/IInputService.cs Normal file
View File

@ -0,0 +1,6 @@
namespace ConsoleApp2.HostedServices.Abstractions;
public interface IInputService
{
public Task ExecuteAsync(CancellationToken cancellationToken);
}

6
ConsoleApp2/HostedServices/Abstractions/IOutputService.cs Normal file
View File

@ -0,0 +1,6 @@
namespace ConsoleApp2.HostedServices.Abstractions;
public interface IOutputService
{
public Task ExecuteAsync(CancellationToken cancellationToken);
}

6
ConsoleApp2/HostedServices/Abstractions/ITransformService.cs Normal file
View File

@ -0,0 +1,6 @@
namespace ConsoleApp2.HostedServices.Abstractions;
public interface ITransformService
{
public Task ExecuteAsync(CancellationToken cancellationToken);
}

19
ConsoleApp2/HostedServices/CsvInputService.cs → ConsoleApp2/HostedServices/InputService.cs
View File

@ -1,35 +1,36 @@
using ConsoleApp2.Helpers;
using ConsoleApp2.Const;
using ConsoleApp2.Helpers;
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace ConsoleApp2.HostedServices;
public class CsvInputService : BackgroundService
/// <summary>
/// 从MyDumper导出的CSV文件中导入表头和数据
/// </summary>
public class InputService : IInputService
{
private readonly ILogger _logger;
private readonly IOptions<CsvOptions> _csvOptions;
private readonly TaskManager _taskManager; // TBD
private readonly DataRecordQueue _producerQueue;
private readonly ProcessContext _context;
public CsvInputService(ILogger<CsvInputService> logger,
public InputService(ILogger<InputService> logger,
IOptions<CsvOptions> csvOptions,
[FromKeyedServices(ProcessStep.Producer)]TaskManager taskManager,
[FromKeyedServices(ProcessStep.Producer)]DataRecordQueue producerQueue,
ProcessContext context)
{
_logger = logger;
_csvOptions = csvOptions;
_taskManager = taskManager;
_producerQueue = producerQueue;
_context = context;
}
protected override async Task ExecuteAsync(CancellationToken cancellationToken)
public async Task ExecuteAsync(CancellationToken cancellationToken)
{
var inputDir = _csvOptions.Value.InputDir;
_logger.LogInformation("***** Csv input service start, working dir: {InputDir}, thread id: {ThreadId} *****", inputDir, Environment.CurrentManagedThreadId);
@ -50,7 +51,7 @@ public class CsvInputService : BackgroundService
{
var csvPath = Path.Combine(inputDir, csvFile);
// var source = new JsvSource(csvPath, headers, _logger);
var source = new NewCsvSource(csvPath, headers, logger: _logger);
var source = new CsvSource(csvPath, headers, _csvOptions.Value.Delimiter, _csvOptions.Value.QuoteChar, _logger);
while (await source.ReadAsync())
{

30
ConsoleApp2/HostedServices/MainHostedService.cs Normal file
View File

@ -0,0 +1,30 @@
using ConsoleApp2.HostedServices.Abstractions;
using Microsoft.Extensions.Hosting;
namespace ConsoleApp2.HostedServices;
public class MainHostedService : BackgroundService
{
private readonly IInputService _input;
private readonly ITransformService _transform;
private readonly IOutputService _output;
public MainHostedService(IInputService input, ITransformService transform, IOutputService output)
{
_input = input;
_transform = transform;
_output = output;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var tasks = new List<Task>()
{
Task.Run(async () => await _input.ExecuteAsync(stoppingToken), stoppingToken),
Task.Run(async () => await _transform.ExecuteAsync(stoppingToken), stoppingToken),
Task.Run(async () => await _output.ExecuteAsync(stoppingToken), stoppingToken),
};
await Task.WhenAll(tasks);
// await Task.Run(async () => await _output.ExecuteAsync(stoppingToken), stoppingToken);
}
}

48
ConsoleApp2/HostedServices/MysqlOutputService.cs → ConsoleApp2/HostedServices/OutputService.cs
View File

@ -1,58 +1,62 @@
using ConsoleApp2.Entities;
using ConsoleApp2.Const;
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MySqlConnector;
namespace ConsoleApp2.HostedServices;
public class MysqlOutputService : BackgroundService
/// <summary>
/// 数据导出服务，将数据导出至MySql服务
/// </summary>
public class OutputService : IOutputService
{
private readonly ILogger _logger;
private readonly DataRecordQueue _consumerQueue;
private readonly IOptions<DatabaseOptions> _options;
private readonly IOptions<DatabaseOutputOptions> _options;
private readonly ProcessContext _context;
private readonly TaskManager _taskManager;
public MysqlOutputService(ILogger<MysqlOutputService> logger,
public OutputService(ILogger<OutputService> logger,
[FromKeyedServices(ProcessStep.Consumer)] DataRecordQueue consumerQueue,
IOptions<DatabaseOptions> options,
ProcessContext context)
IOptions<DatabaseOutputOptions> options,
ProcessContext context,
TaskManager taskManager)
{
_logger = logger;
_consumerQueue = consumerQueue;
_options = options;
_context = context;
_taskManager = taskManager;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
public async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("***** Mysql output service started *****");
var tasks = new List<Task>();
var records = new List<DataRecord>();
while (!_context.IsTransformCompleted || _consumerQueue.Count > 0)
{
if (!_consumerQueue.TryDequeue(out var record)) continue;
records.Add(record);
if (records.Count >= 200)
if (records.Count >= _options.Value.FlushCount)
{
var recordsCopy = records;
tasks.Add(Task.Run(async () => await FlushAsync(recordsCopy), stoppingToken));
_taskManager.CreateTask(async () => await FlushAsync(recordsCopy), stoppingToken);
records = [];
}
if (tasks.Count >= 10)
if (_taskManager.TaskCount >= _options.Value.MaxTask)
{
await Task.WhenAll(tasks);
tasks.Clear();
await _taskManager.WaitAll();
_taskManager.ClearTask();
}
}
await Task.WhenAll(tasks);
await _taskManager.WaitAll();
await FlushAsync(records);
_context.CompleteOutput();
@ -63,15 +67,9 @@ public class MysqlOutputService : BackgroundService
private async Task FlushAsync(IEnumerable<DataRecord> records)
{
var count = 0;
await using var output = new MySqlDestination(new MySqlConnectionStringBuilder
{
Server = _options.Value.Host,
Port = _options.Value.Port,
Database = _options.Value.Database,
UserID = _options.Value.User,
Password = _options.Value.Password,
ConnectionTimeout = 180,
}.ConnectionString, _logger, true);
await using var output = new MySqlDestination(
_options.Value.ConnectionString ?? throw new InvalidOperationException("Connection string is required"),
_logger, true);
foreach (var record in records)
{

64
ConsoleApp2/HostedServices/SqlFileOutputService.cs
View File

@ -1,64 +0,0 @@
using ConsoleApp2.Entities;
using ConsoleApp2.Helpers;
using ConsoleApp2.Services;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace ConsoleApp2.HostedServices;
public class SqlFileOutputService : BackgroundService
{
private readonly string _outputFile = "D:/DumpOutput/cferp_test_1.sql"; //
private readonly DataRecordQueue _consumerQueue;
private readonly ILogger _logger;
private readonly ProcessContext _context;
public SqlFileOutputService(
ILogger<SqlFileOutputService> logger,
[FromKeyedServices(ProcessStep.Consumer)]
DataRecordQueue consumerQueue,
ProcessContext context)
{
_logger = logger;
_consumerQueue = consumerQueue;
_context = context;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("***** Sql file output service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId);
var count = 0;
var tableRecords = new Dictionary<string, IList<DataRecord>>();
while (!_context.IsTransformCompleted || _consumerQueue.Count > 0)
{
if (!_consumerQueue.TryDequeue(out var record)) continue;
tableRecords.AddOrUpdate(record.TableName, [record], (key, value) =>
{
value.Add(record);
return value;
});
++count;
if (count >= 200)
{
await File.AppendAllTextAsync(_outputFile,
MySqlDestination.SerializeRecords(tableRecords), stoppingToken);
tableRecords.Clear();
_context.AddOutput(count);
count = 0;
}
}
await File.AppendAllTextAsync(_outputFile,
MySqlDestination.SerializeRecords(tableRecords), stoppingToken);
tableRecords.Clear();
_context.AddOutput(count);
_context.CompleteOutput();
_logger.LogInformation("***** Sql file output service completed *****");
}
}

6
ConsoleApp2/HostedServices/TaskMonitorService.cs
View File

@ -1,4 +1,5 @@
using System.Diagnostics;
using ConsoleApp2.Const;
using ConsoleApp2.Services;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
@ -6,6 +7,9 @@ using Microsoft.Extensions.Logging;
namespace ConsoleApp2.HostedServices;
/// <summary>
/// 任务监控
/// </summary>
public class TaskMonitorService : BackgroundService
{
private readonly IHostApplicationLifetime _lifetime;
@ -93,7 +97,7 @@ public class TaskMonitorService : BackgroundService
_logger.LogInformation("Queue monitor: producer queue: {ProducerQueue}, consumer queue: {ConsumerQueue}",
_producerQueue.Count, _consumerQueue.Count);
await Task.Delay(2000);
await Task.Delay(5000);
lastTime = time;
lastInputCount = inputCount;

19
ConsoleApp2/HostedServices/DataTransformService.cs → ConsoleApp2/HostedServices/TransformService.cs
View File

@ -1,14 +1,17 @@
using ConsoleApp2.Helpers;
using ConsoleApp2.Const;
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace ConsoleApp2.HostedServices;
public class DataTransformService : BackgroundService
/// <summary>
/// 数据处理服务，对导入后的数据进行处理
/// </summary>
public class TransformService : ITransformService
{
private readonly ILogger _logger;
private readonly IOptions<DataTransformOptions> _options;
@ -17,21 +20,20 @@ public class DataTransformService : BackgroundService
private readonly ProcessContext _context;
public DataTransformService(ILogger<DataTransformService> logger,
IOptions<DataTransformOptions> options, // TBD: database filter
public TransformService(ILogger<TransformService> logger,
IOptions<DataTransformOptions> options,
[FromKeyedServices(ProcessStep.Producer)]DataRecordQueue producerQueue,
[FromKeyedServices(ProcessStep.Consumer)]DataRecordQueue consumerQueue,
ProcessContext context)
{
_logger = logger;
// _taskManager = taskManager;
_options = options;
_producerQueue = producerQueue;
_consumerQueue = consumerQueue;
_context = context;
}
protected override async Task ExecuteAsync(CancellationToken cancellationToken)
public async Task ExecuteAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("***** Data transform service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId);
while (!_context.IsInputCompleted || _producerQueue.Count > 0)
@ -58,7 +60,6 @@ public class DataTransformService : BackgroundService
field = string.IsNullOrEmpty(field) ? "''" : $"0x{field}";
break;
default:
field = field;
break;
}
@ -66,6 +67,8 @@ public class DataTransformService : BackgroundService
record[i] = field;
}
// TODO: 数据处理/过滤/复制
_consumerQueue.Enqueue(record);
_context.AddTransform();
}

9
ConsoleApp2/HostedServices/VoidOutputService.cs
View File

@ -1,11 +1,14 @@
using ConsoleApp2.Services;
using ConsoleApp2.Const;
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Services;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace ConsoleApp2.HostedServices;
public class VoidOutputService : BackgroundService
// 空输出服务，测试用
public class VoidOutputService : IOutputService
{
private readonly ILogger _logger;
private readonly DataRecordQueue _consumerQueue;
@ -19,7 +22,7 @@ public class VoidOutputService : BackgroundService
_logger = logger;
}
protected override Task ExecuteAsync(CancellationToken stoppingToken)
public Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("***** Void output service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId);
while (!_context.IsTransformCompleted || _consumerQueue.Count > 0)

17
ConsoleApp2/Options/CsvOptions.cs
View File

@ -3,26 +3,17 @@
public class CsvOptions
{
/// <summary>
/// The directory to input csv and sql file.
/// MyDumper导出的CSV文件目录
/// </summary>
public string InputDir { get; set; } = "./";
/// <summary>
/// The output directory.
/// </summary>
public string OutputDir { get; set; } = "./Output";
/// <summary>
/// The ASCII char that fields are enclosed by. Default is '"'.
/// 字符串的包围符号，默认为双引号"
/// </summary>
public char QuoteChar { get; set; } = '"';
/// <summary>
/// The ASCII char that fields are separated by. Default is ','.
/// 每个字段的分割符，默认逗号,
/// </summary>
public char DelimiterChar { get; set; } = ',';
/// <summary>
/// The max number of threads to use.
/// </summary>
public int MaxThreads { get; set; } = 12;
public string Delimiter { get; set; } = ",";
}

8
ConsoleApp2/Options/DataTransformOptions.cs
View File

@ -1,6 +1,4 @@
using ConsoleApp2.Entities;
namespace ConsoleApp2.Options;
namespace ConsoleApp2.Options;
public enum ColumnType
{
@ -12,6 +10,10 @@ public enum ColumnType
public class DataTransformOptions
{
public Func<DataRecord, string>? DatabaseFilter { get; set; }
/// <summary>
/// 配置导入数据的特殊列
/// </summary>
public Dictionary<string, ColumnType> ColumnTypeConfig { get; set; } = new(); // "table.column" -> type
public ColumnType GetColumnType(string table, string column)

10
ConsoleApp2/Options/DatabaseOptions.cs
View File

@ -1,10 +0,0 @@
namespace ConsoleApp2.Options;
public class DatabaseOptions
{
public string Host { get; set; }
public uint Port { get; set; }
public string Database { get; set; }
public string User { get; set; }
public string Password { get; set; }
}

17
ConsoleApp2/Options/DatabaseOutputOptions.cs Normal file
View File

@ -0,0 +1,17 @@
namespace ConsoleApp2.Options;
public class DatabaseOutputOptions
{
/// <summary>
/// 数据库连接字符串
/// </summary>
public string? ConnectionString { get; set; }
/// <summary>
/// 输出服务的最大任务(Task)数
/// </summary>
public int MaxTask { get; set; }
/// <summary>
/// 每个任务每次提交到数据库的记录数量（每N条构建一次SQL语句）
/// </summary>
public int FlushCount { get; set; }
}

57
ConsoleApp2/Program.cs
View File

@ -1,26 +1,17 @@
using ConsoleApp2;
using ConsoleApp2.Const;
using ConsoleApp2.HostedServices;
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using MySqlConnector;
using Serilog;
// 加入数据库过滤
// HostedService不是并行的，完善TaskManager手动开启线程
// 测试BlockingCollection对速度的影响？
// 重新同步数据
// Json列和Blob列不一致
/* JSV导出带转义的列有误
* order_data_block表id 4153969
* order_data_parts表Spec列
* order_module表Name列
* process_group表Items列
*/
// 运行之前把Mysql max_allowed_packets 调大
// 运行之前把process_step表的外键删掉
await RunProgram();
return;
@ -32,15 +23,12 @@ async Task RunProgram()
host.Configuration.AddCommandLine(args);
host.Services.Configure<CsvOptions>(option =>
{
option.DelimiterChar = ',';
option.Delimiter = ",";
option.QuoteChar = '"';
option.InputDir = "D:/Dump/MyDumper";
option.OutputDir = "D:/DumpOutput";
option.MaxThreads = 12;
option.InputDir = "D:/Dump";
});
host.Services.Configure<DataTransformOptions>(options =>
{
//TODO: Database Filter
options.DatabaseFilter = record => "cferp_test_1";
options.ColumnTypeConfig = new()
{
@ -73,14 +61,20 @@ async Task RunProgram()
{ "order_block_plan.BlockInfo", ColumnType.Text },
};
});
// 加入数据库过滤后删除
host.Services.Configure<DatabaseOptions>(options =>
host.Services.Configure<DatabaseOutputOptions>(options =>
{
options.Host = "localhost";
options.Port = 33306;
options.Database = "cferp_test_1";
options.User = "root";
options.Password = "123456";
options.ConnectionString = new MySqlConnectionStringBuilder
{
Server = "127.0.0.1",
Port = 33306,
Database = "cferp_test_1",
UserID = "root",
Password = "123456",
MaximumPoolSize = 50, // 这个值应当小于 max_connections
}.ConnectionString;
options.MaxTask = 16;
options.FlushCount = 200;
});
host.Services.AddLogging(builder =>
{
@ -91,14 +85,13 @@ async Task RunProgram()
host.Services.AddSingleton<ProcessContext>();
host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Producer);
host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Consumer);
host.Services.AddKeyedSingleton<TaskManager>(ProcessStep.Producer);
host.Services.AddKeyedSingleton<TaskManager>(ProcessStep.Consumer);
host.Services.AddTransient<TaskManager>();
host.Services.AddHostedService<MainHostedService>();
host.Services.AddHostedService<TaskMonitorService>();
host.Services.AddHostedService<CsvInputService>();
host.Services.AddHostedService<DataTransformService>();
host.Services.AddHostedService<MysqlOutputService>();
host.Services.AddSingleton<IInputService, InputService>();
host.Services.AddSingleton<ITransformService, TransformService>();
host.Services.AddSingleton<IOutputService, OutputService>();
var app = host.Build();
await app.RunAsync();

49
ConsoleApp2/README.md Normal file
View File

@ -0,0 +1,49 @@
## 说明
使用该程序来对MyDumper导出的CSV数据进行读取，转换，然后导出到其他数据库中。
1. 用MyDumper从数据库导出CSV数据
使用MyDumper Docker镜像
```sh
docker run --rm --net=host -v D:/Dump:/home/backup mydumper/mydumper:v0.15.2-6 mydumper `
-h 127.0.0.1 -P 33306 -u root -p 123456 `
-B cferp_test --no-schemas --csv --hex-blob `
-o /home/backup
```
将挂载卷，数据库连接和输出目录替换
不导出数据库结构(--no-schemas)，
导出完的目录下应当包含.sql文件以及.dat文件
2. 在Program.cs中修改`CsvOptions`配置
```cs
host.Services.Configure<CsvOptions>(option =>
{
option.Delimiter = ",";
option.QuoteChar = '"';
option.InputDir = "D:/Dump/Test";
});
```
将`option.InputDir`配置为MyDumper导出的数据目录
3. 在Program.cs中修改`DatabaseOutputOptions`配置
```cs
host.Services.Configure<DatabaseOutputOptions>(options =>
{
options.ConnectionString = new MySqlConnectionStringBuilder
{
Server = "127.0.0.1",
Port = 33306,
Database = "cferp_test_1",
UserID = "root",
Password = "123456",
MaximumPoolSize = 50,
}.ConnectionString;
options.MaxTask = 16;
options.FlushCount = 200;
});
```
将`MySqlConnectionStringBuilder`的属性修改为程序要导出至的数据库
> 后续将这些配置通过命令行传递
4. 运行程序
> 注意，测试数据库`cferp_test`中的`order_process_step`表存在外键，如果要导出到和测试库同结构的数据库，记得先把外键删除。

10
ConsoleApp2/Services/NewCsvSource.cs → ConsoleApp2/Services/CsvSource.cs
View File

@ -1,24 +1,26 @@
using System.Text;
using ConsoleApp2.Entities;
using ConsoleApp2.Helpers;
using Microsoft.Extensions.Logging;
namespace ConsoleApp2.Services;
public class NewCsvSource
/// <summary>
/// CSV文件读取
/// </summary>
public class CsvSource
{
private readonly string _filePath;
private readonly StreamReader _reader;
private readonly ILogger? _logger;
private readonly string _tableName;
public DataRecord Current { get; protected set; }
public DataRecord Current { get; private set; }
public string[]? Headers { get; }
public string? CurrentRaw { get; private set; }
public string Delimiter { get; private set; }
public char QuoteChar { get; private set; }
public NewCsvSource(string filePath, string[]? headers = null, string delimiter = ",", char quoteChar = '"',
public CsvSource(string filePath, string[]? headers = null, string delimiter = ",", char quoteChar = '"',
ILogger? logger = null)
{
_filePath = filePath;

6
ConsoleApp2/Services/DataRecordQueue.cs
View File

@ -1,9 +1,11 @@
using System.Collections.Concurrent;
using System.Diagnostics.CodeAnalysis;
using ConsoleApp2.Entities;
namespace ConsoleApp2.Services;
/// <summary>
/// 数据队列
/// </summary>
public class DataRecordQueue : IDisposable
{
private readonly BlockingCollection<DataRecord> _queue;
@ -17,7 +19,7 @@ public class DataRecordQueue : IDisposable
public DataRecordQueue()
{
_queue = new BlockingCollection<DataRecord>();
_queue = new BlockingCollection<DataRecord>(200_000); // 队列最长为20W条记录
}
public bool TryDequeue([MaybeNullWhen(false)] out DataRecord record)

7
ConsoleApp2/Services/JsvSource.cs
View File

@ -1,10 +1,13 @@
using ConsoleApp2.Entities;
using ConsoleApp2.Helpers;
using ConsoleApp2.Helpers;
using Microsoft.Extensions.Logging;
using ServiceStack.Text;
namespace ConsoleApp2.Services;
/// <summary>
/// 读取Jsv格式文件
/// </summary>
[Obsolete]
public class JsvSource : IDisposable
{
private readonly string _filePath;

28
ConsoleApp2/Services/MySqlDestination.cs
View File

@ -1,11 +1,13 @@
using System.Text;
using ConsoleApp2.Entities;
using ConsoleApp2.Helpers;
using Microsoft.Extensions.Logging;
using MySqlConnector;
namespace ConsoleApp2.Services;
/// <summary>
/// Mysql导出
/// </summary>
public class MySqlDestination : IDisposable, IAsyncDisposable
{
private readonly Dictionary<string, IList<DataRecord>> _recordCache;
@ -13,8 +15,6 @@ public class MySqlDestination : IDisposable, IAsyncDisposable
private readonly ILogger _logger;
private readonly bool _prettyOutput;
public static int AddCount;
public MySqlDestination(string connStr, ILogger logger, bool prettyOutput = false)
{
_conn = new MySqlConnection(connStr);
@ -29,7 +29,6 @@ public class MySqlDestination : IDisposable, IAsyncDisposable
_recordCache.AddOrUpdate(record.TableName, [record], (key, value) =>
{
value.Add(record);
Interlocked.Increment(ref AddCount);
return value;
});
return Task.CompletedTask;
@ -60,6 +59,10 @@ public class MySqlDestination : IDisposable, IAsyncDisposable
_logger.LogCritical(e, "Error when flushing records, sql: {Sql}", cmd.CommandText.Omit(1000));
throw;
}
finally
{
await cmd.DisposeAsync();
}
}
public static string SerializeRecords(IDictionary<string, IList<DataRecord>> tableRecords,
@ -91,22 +94,7 @@ public class MySqlDestination : IDisposable, IAsyncDisposable
for (var j = 0; j < record.Fields.Length; j++)
{
var field = record.Fields[j];
#region HandleFields
// if (field == "\\N")
// sb.Append("NULL");
// else if (DumpDataHelper.CheckHexField(field))
// {
// // if (StringExtensions.CheckJsonHex(field))
// sb.Append($"0x{field}");
// }
// else
// sb.Append($"'{field}'");
sb.Append(field);
#endregion
if (j != record.Fields.Length - 1)
sb.Append(',');
}
@ -127,11 +115,13 @@ public class MySqlDestination : IDisposable, IAsyncDisposable
public void Dispose()
{
_conn.Close();
_conn.Dispose();
}
public async ValueTask DisposeAsync()
{
await _conn.CloseAsync();
await _conn.DisposeAsync();
}
}

3
ConsoleApp2/Services/ProcessContext.cs
View File

@ -1,5 +1,8 @@
namespace ConsoleApp2.Services;
/// <summary>
/// 处理上下文类，标识处理进度
/// </summary>
public class ProcessContext
{
private int _inputCount;

24
ConsoleApp2/Services/TaskManager.cs
View File

@ -3,14 +3,17 @@ using Microsoft.Extensions.Logging;
namespace ConsoleApp2.Services;
/// <summary>
/// 快速批量创建和等待任务
/// </summary>
public class TaskManager
{
private readonly ConcurrentBag<Task> _tasks;
private readonly ILogger _logger;
public int TaskCount => _tasks.Count;
public int RunningTaskCount => _tasks.Count(task => !task.IsCompleted);
public IReadOnlyCollection<Task> Tasks => _tasks;
public bool MainTaskCompleted { get; set; }
public TaskManager(ILogger<TaskManager> logger)
{
@ -18,11 +21,22 @@ public class TaskManager
_logger = logger;
}
public Task<TResult> CreateTask<TResult>(Func<TResult> func)
public void CreateTask<TResult>(Func<TResult> func, CancellationToken cancellationToken = default)
{
var task = Task.Factory.StartNew(func);
var task = Task.Factory.StartNew(func, cancellationToken);
_tasks.Add(task);
_logger.LogDebug("New task created.");
return task;
_logger.LogDebug("New task created");
}
public async Task WaitAll()
{
await Task.WhenAll(_tasks);
}
public void ClearTask()
{
if(RunningTaskCount != 0)
throw new InvalidOperationException("Unable to clear task. There are still running tasks");
_tasks.Clear();
}
}

6
ConsoleApp2/Services/TsvSource.cs
View File

@ -1,6 +0,0 @@
namespace ConsoleApp2.Services;
public class TsvSource
{
}