CZY/MES-ETL
1
0
Fork 0
Code Pull Requests Releases Activity

Update

Browse Source
This commit is contained in:
陈梓阳 2024-01-29 09:29:16 +08:00
parent 4f96b77e55
commit 083090c62b
63 changed files with 2479 additions and 1491 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
ConsoleApp2/Cache/CacheKeys.cs Normal file
View File

@ -0,0 +1,9 @@
namespace ConsoleApp2.Cache;
#nullable disable
public static class CacheKeys
{
public static Func<string, string> Order_OrderNo_CompanyID { get; set; }
public static Func<string, string> OrderBlockPlan_ID_CompanyID { get; set; }
public static Func<string, string> OrderProcess_ID_ShardKey { get; set; }
}

10
ConsoleApp2/Cache/ICacher.cs Normal file
View File

@ -0,0 +1,10 @@
namespace ConsoleApp2.Cache;
public interface ICacher
{
Task<string?> GetStringAsync(string key);
Task SetStringAsync(string key, string value);
Task<bool> ExistsAsync(string key);
Task SetHashAsync(string key, IReadOnlyDictionary<string, string> hash);
Task<Dictionary<string, string>> GetHashAsync(string key);
}

63
ConsoleApp2/Cache/RedisCache.cs Normal file
View File

@ -0,0 +1,63 @@
using ConsoleApp2.Options;
using Microsoft.Extensions.DependencyInjection;
using StackExchange.Redis;
namespace ConsoleApp2.Cache;
public class RedisCache : ICacher
{
private readonly IDatabase _db;
public string KeyPrefix { get; set; }
public RedisCache(IConnectionMultiplexer conn, int dataBase, string keyPrefix = "")
{
_db = conn.GetDatabase(dataBase);
KeyPrefix = keyPrefix;
}
public async Task<string?> GetStringAsync(string key)
{
var value = await _db.StringGetAsync($"{KeyPrefix}{key}");
return !value.HasValue ? null : value.ToString();
}
public async Task SetStringAsync(string key, string value)
{
if (!await _db.StringSetAsync($"{KeyPrefix}{key}", value))
throw new RedisCommandException("设置Redis缓存失败");
}
public Task<bool> ExistsAsync(string key)
{
return _db.KeyExistsAsync($"{KeyPrefix}{key}");
}
public Task SetHashAsync(string key, IReadOnlyDictionary<string, string> hash)
{
return _db.HashSetAsync($"{KeyPrefix}{key}", hash.Select(pair => new HashEntry(pair.Key, pair.Value)).ToArray());
}
public async Task<Dictionary<string, string>> GetHashAsync(string key)
{
var entries = await _db.HashGetAllAsync($"{KeyPrefix}{key}");
var result = new Dictionary<string, string>();
foreach (var entry in entries)
{
result.Add(entry.Name.ToString(), entry.Value.ToString());
}
return result;
}
}
public static class RedisCacheExtensions
{
public static IServiceCollection AddRedisCache(this IServiceCollection services, RedisCacheOptions options)
{
var conn = ConnectionMultiplexer.Connect(options.Configuration
?? throw new ApplicationException("未配置Redis连接字符串"));
services.AddSingleton<ICacher>(new RedisCache(conn, options.Database, options.InstanceName));
return services;
}
}

2
ConsoleApp2/ConsoleApp2.csproj
View File

@ -19,7 +19,6 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Caching.StackExchangeRedis" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
<PackageReference Include="MySqlConnector" Version="2.3.3" />
<PackageReference Include="Serilog" Version="3.1.2-dev-02097" />
@ -27,6 +26,7 @@
<PackageReference Include="Serilog.Sinks.Console" Version="5.0.1" />
<PackageReference Include="Serilog.Sinks.File" Version="5.0.1-dev-00972" />
<PackageReference Include="ServiceStack.Text" Version="8.0.0" />
<PackageReference Include="StackExchange.Redis" Version="2.7.17" />
<PackageReference Include="ZstdSharp.Port" Version="0.7.4" />
</ItemGroup>

3
ConsoleApp2/Const/ProcessStep.cs
View File

@ -2,6 +2,5 @@
public static class ProcessStep
{
public const string Producer = "Producer";
public const string Consumer = "Consumer";
public const string Produce = "Producer";
}

38
ConsoleApp2/Const/TableNames.cs Normal file
View File

@ -0,0 +1,38 @@
namespace ConsoleApp2.Const;
public static class TableNames
{
public const string Machine = "machine";
public const string Order = "order";
public const string OrderBlockPlan = "order_block_plan";
public const string OrderBlockPlanItem = "order_block_plan_item";
public const string OrderBlockPlanResult = "order_block_plan_result";
public const string OrderBoxBlock = "order_box_block";
public const string OrderDataBlock = "order_data_block";
public const string OrderDataGoods = "order_data_goods";
public const string OrderDataParts = "order_data_parts";
public const string OrderItem = "order_item";
public const string OrderModule = "order_module";
public const string OrderModuleExtra = "order_module_extra";
public const string OrderModuleItem = "order_module_item";
public const string OrderPackage = "order_package";
public const string OrderPackageItem = "order_package_item";
public const string OrderPatchDetail = "order_patch_detail";
public const string OrderProcess = "order_process";
public const string OrderProcessSchedule = "order_process_schdule";
public const string OrderProcessStep = "order_process_step";
public const string OrderProcessStepItem = "order_process_step_item";
public const string OrderScrapBoard = "order_scrap_board";
public const string ProcessGroup = "process_group";
public const string ProcessInfo = "process_info";
public const string ProcessItemExp = "process_item_exp";
public const string ProcessScheduleCapacity = "process_schdule_capacity";
public const string ProcessStepEfficiency = "process_step_efficiency";
public const string ReportTemplate = "report_template";
public const string SimplePackage = "simple_package";
public const string SimplePlanOrder = "simple_plan_order";
public const string SysConfig = "sys_config";
public const string WorkCalendar = "work_calendar";
public const string WorkShift = "work_shift";
public const string WorkTime = "work_time";
}

105
ConsoleApp2/DataRecord.cs
View File

@ -1,16 +1,13 @@
using System.ComponentModel.Design;
namespace ConsoleApp2;
namespace ConsoleApp2;
public class DataRecord
public class DataRecord : ICloneable
{
public static bool TryGetField(DataRecord record, string columnName, out string value)
{
value = string.Empty;
if (record.Headers is null)
throw new InvalidOperationException("Cannot get field when headers of a record have not been set.");
var idx = Array.IndexOf(record.Headers, columnName);
var idx = IndexOfIgnoreCase(record.Headers, columnName);
if (idx == -1)
return false;
value = record.Fields[idx];
@ -21,34 +18,46 @@ public class DataRecord
{
if (record.Headers is null)
throw new InvalidOperationException("Headers have not been set.");
var idx = Array.IndexOf(record.Headers, columnName);
var idx = IndexOfIgnoreCase(record.Headers, columnName);
if (idx is -1)
throw new IndexOutOfRangeException($"Column name {columnName} not found in this record, table name {record.TableName}.");
throw new IndexOutOfRangeException(
$"Column name '{columnName}' not found in this record, table name '{record.TableName}'.");
return record.Fields[idx];
}
public string[] Fields { get; }
public string[] Headers { get; }
public string TableName { get; }
public string Database { get; set; }
public int CompanyID { get; set; }
public DataRecord(string[] fields, string tableName, string[] headers, int companyID=0)
private static int IndexOfIgnoreCase(IList<string> list, string value)
{
if (fields.Length != headers.Length)
throw new ArgumentException(
$"The number of fields does not match the number of headers. Expected: {headers.Length} Got: {fields.Length} Fields: {string.Join(',', fields)}",
nameof(fields));
var idx = -1;
for (var i = 0; i < list.Count; i++)
{
if (list[i].Equals(value, StringComparison.OrdinalIgnoreCase))
{
idx = i;
break;
}
}
Fields = fields;
return idx;
}
public string? RawField { get; set; }
public IList<string> Fields { get; }
public IList<string> Headers { get; }
public string TableName { get; }
public string? Database { get; set; }
public DataRecord(IEnumerable<string> fields, string tableName, IEnumerable<string> headers, string? database = null)
{
Fields = fields.ToList();
TableName = tableName;
Headers = headers;
CompanyID = companyID;
Headers = headers.ToList();
Database = database;
if (Fields.Count != Headers.Count)
throw new ArgumentException(
$"The number of fields does not match the number of headers. Expected: {Headers.Count} Got: {Fields.Count} Fields: {string.Join(',', Fields)}",
nameof(fields));
}
public string this[int index]
@ -63,32 +72,46 @@ public class DataRecord
set => SetField(columnName, value);
}
public int Count => Fields.Length;
public int FieldCount => Fields.Count;
public bool TryGetField(string columnName, out string value) => TryGetField(this, columnName, out value);
public bool SetField(string columnName, string value) => SetField(this, columnName,value);
public bool SetField(string columnName, string value) => SetField(this, columnName, value);
public string GetCacheKey(string columnName) => GetCacheKey(this, columnName);
public bool SetField( DataRecord record,string columnName,string value)
public bool SetField(DataRecord record, string columnName, string value)
{
if (record.Headers is null)
throw new InvalidOperationException("Headers have not been set.");
var idx = Array.IndexOf(record.Headers, columnName);
var idx = IndexOfIgnoreCase(record.Headers, columnName);
if (idx is -1)
throw new IndexOutOfRangeException("Column name not found in this record.");
throw new IndexOutOfRangeException(
$"Column name '{columnName}' not found in this record, table name '{record.TableName}");
record.Fields[idx] = value;
return true;
}
public string GetCacheKey(DataRecord record, string columnName)
public void AddField(string columnName, string value)
{
if (TryGetField(record, columnName, out var value))
{
return $"{TableName}_{value}";
}else
throw new IndexOutOfRangeException($"Column name:{columnName} not found in this record.");
if (IndexOfIgnoreCase(Headers, columnName) != -1)
throw new InvalidOperationException($"{TableName}: 列名 '{columnName}' 已存在");
Fields.Add(value);
Headers.Add(columnName);
}
public void RemoveField(string columnName)
{
var idx = IndexOfIgnoreCase(Headers, columnName);
if (idx == -1)
throw new InvalidOperationException($"{TableName}: 列名 '{columnName}' 不存在");
Fields.RemoveAt(idx);
Headers.Remove(columnName);
}
public bool HeaderExists(string columnName) => IndexOfIgnoreCase(Headers, columnName) != -1;
public object Clone()
{
return new DataRecord(new List<string>(Fields), TableName, new List<string>(Headers), Database);
}
}

9
TestProject1/DatabaseHelper.cs → ConsoleApp2/Helpers/Database/DatabaseHelper.cs
View File

@ -1,12 +1,11 @@
using System.Data;
using System.Data.Common;
using MySqlConnector;
namespace TestProject1;
namespace ConsoleApp2.Helpers.Database;
public static class DatabaseHelper
{
public static async Task<DataSet> QueryTable(string connStr, string sql)
public static async Task<DataSet> QueryTableAsync(string connStr, string sql)
{
await using var conn = new MySqlConnection(connStr);
if(conn.State is not ConnectionState.Open)
@ -18,7 +17,7 @@ public static class DatabaseHelper
return ds;
}
public static async Task<object?> QueryScalar(string connStr, string sql)
public static async Task<object?> QueryScalarAsync(string connStr, string sql)
{
await using var conn = new MySqlConnection(connStr);
if(conn.State is not ConnectionState.Open)
@ -28,7 +27,7 @@ public static class DatabaseHelper
return await cmd.ExecuteScalarAsync();
}
public static async Task<int> NonQuery(string connStr, string sql)
public static async Task<int> NonQueryAsync(string connStr, string sql)
{
await using var conn = new MySqlConnection(connStr);
if(conn.State is not ConnectionState.Open)

85
ConsoleApp2/Helpers/DumpDataHelper.cs
View File

@ -1,6 +1,5 @@
using ConsoleApp2.Options;
using System.Text;
using System.Text.RegularExpressions;
using System.Text.RegularExpressions;
using ZstdSharp;
namespace ConsoleApp2.Helpers;
@ -11,32 +10,41 @@ public static partial class DumpDataHelper
[GeneratedRegex(@"\([^)]*\)")]
private static partial Regex MatchBrackets();
public static async Task<string[]> GetCsvHeadersFromSqlFileAsync(string txt)
/// <summary>
/// 从MyDumper导出的SQL文件内容中读取表头
/// </summary>
/// <param name="content"></param>
/// <returns></returns>
/// <exception cref="ArgumentException"></exception>
public static string[] GetCsvHeadersFromSqlFile(string content)
{
//var txt = await File.ReadAllTextAsync(filePath);
var match = MatchBrackets().Match(txt);
var match = MatchBrackets().Match(content);
if (!match.Success)
throw new ArgumentException("输入的SQL内容有误，无法提取表头", nameof(content));
return ParseHeader(match.ValueSpan);
}
private static string[] ParseHeader(ReadOnlySpan<char> headerStr)
{
headerStr = headerStr[1..^1];
Span<Range> ranges = stackalloc Range[50];
var count = headerStr.Split(ranges, ',');
var arr = new string[count];
for (var i = 0; i < count; i++)
string[] ParseHeader(ReadOnlySpan<char> headerStr)
{
arr[i] = headerStr[ranges[i]].Trim("@`").ToString(); // 消除列名的反引号，如果是变量则消除@
}
headerStr = headerStr[1..^1];
Span<Range> ranges = stackalloc Range[50];
var count = headerStr.Split(ranges, ',');
var arr = new string[count];
return arr;
for (var i = 0; i < count; i++)
{
arr[i] = headerStr[ranges[i]].Trim("@`").ToString(); // 消除列名的反引号，如果是变量则消除@
}
return arr;
}
}
public static string GetTableName(ReadOnlySpan<char> filePath)
/// <summary>
/// 从MyDumper导出的Csv文件名解析出表名
/// </summary>
/// <param name="filePath"></param>
/// <returns></returns>
public static string GetTableNameFromCsvFileName(ReadOnlySpan<char> filePath)
{
filePath = filePath[(filePath.LastIndexOf('\\') + 1)..];
var firstDotIdx = -1;
@ -60,13 +68,24 @@ public static partial class DumpDataHelper
return filePath[(firstDotIdx+1)..secondDotIdx].ToString();
}
public static async Task<string[]> GetCsvFileNamesFromSqlFileAsync(string txt,Regex regex)
/// <summary>
/// 从MyDumper导出的SQL文件内容中读取CSV文件名
/// </summary>
/// <param name="txt"></param>
/// <param name="regex"></param>
/// <returns></returns>
public static Task<string[]> GetCsvFileNamesFromSqlFileAsync(string txt, Regex regex)
{
//var txt = await File.ReadAllTextAsync(filePath);
var matches = regex.Matches(txt);
return matches.Select(match => match.ValueSpan[1..^1].ToString()).ToArray();
return Task.FromResult(matches.Select(match => match.ValueSpan[1..^1].ToString()).ToArray());
}
/// <summary>
/// 检查字符串是否为16进制
/// </summary>
/// <param name="str"></param>
/// <returns></returns>
public static bool CheckHexField(string? str)
{
if (string.IsNullOrWhiteSpace(str))
@ -90,9 +109,17 @@ public static partial class DumpDataHelper
return true;
}
/// <summary>
/// 将输入流以ZSTD标准解压为字符串
/// </summary>
/// <param name="stream"></param>
/// <returns></returns>
public static async Task<string> DecompressZstAsStringAsync(Stream stream)
{
await using var ds = new DecompressionStream(stream);
var reader = new StreamReader(ds);
return await reader.ReadToEndAsync();
}
// public static string EliminateEscapeChars(ReadOnlySpan<char> str)
// {
// char[] escapeChars = ['0','\''];
// }
}

67
ConsoleApp2/Helpers/Extensions.IEnumerable.cs Normal file
View File

@ -0,0 +1,67 @@
using System.Diagnostics.CodeAnalysis;
using System.Reflection;
namespace ConsoleApp2.Helpers;
public static class EnumerableExtensions
{
public static string ToMarkdownTable<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicMethods)]T>(this IEnumerable<T> source)
{
var properties = typeof(T).GetRuntimeProperties();
var fields = typeof(T)
.GetRuntimeFields()
.Where(f => f.IsPublic);
var gettables = Enumerable.Union(
properties.Select(p => new { p.Name, GetValue = (Func<object, object>)p.GetValue, Type = p.PropertyType }),
fields.Select(p => new { p.Name, GetValue = (Func<object, object>)p.GetValue, Type = p.FieldType }));
var maxColumnValues = source
.Select(x => gettables.Select(p => p.GetValue(x)?.ToString()?.Length ?? 0))
.Union(new[] { gettables.Select(p => p.Name.Length) }) // Include header in column sizes
.Aggregate(
new int[gettables.Count()].AsEnumerable(),
(accumulate, x) => accumulate.Zip(x, Math.Max))
.ToArray();
var columnNames = gettables.Select(p => p.Name);
var headerLine = "| " + string.Join(" | ", columnNames.Select((n, i) => n.PadRight(maxColumnValues[i]))) + " |";
var isNumeric = new Func<Type, bool>(type =>
type == typeof(Byte) ||
type == typeof(SByte) ||
type == typeof(UInt16) ||
type == typeof(UInt32) ||
type == typeof(UInt64) ||
type == typeof(Int16) ||
type == typeof(Int32) ||
type == typeof(Int64) ||
type == typeof(Decimal) ||
type == typeof(Double) ||
type == typeof(Single));
var rightAlign = new Func<Type, char>(type => isNumeric(type) ? ':' : ' ');
var headerDataDividerLine =
"| " +
string.Join(
"| ",
gettables.Select((g, i) => new string('-', maxColumnValues[i]) + rightAlign(g.Type))) +
"|";
var lines = new[]
{
headerLine,
headerDataDividerLine,
}.Union(
source
.Select(s =>
"| " + string.Join(" | ",
gettables.Select((n, i) => (n.GetValue(s)?.ToString() ?? "").PadRight(maxColumnValues[i]))) +
" |"));
return lines
.Aggregate((p, c) => p + Environment.NewLine + c);
}
}

23
ConsoleApp2/Helpers/Extensions.String.cs
View File

@ -5,6 +5,12 @@ namespace ConsoleApp2.Helpers;
public static class StringExtensions
{
/// <summary>
/// 截断字符串
/// </summary>
/// <param name="this"></param>
/// <param name="maxLength">截断长度</param>
/// <returns></returns>
public static string Omit(this ReadOnlySpan<char> @this, int maxLength)
{
if (@this.Length > maxLength)
@ -12,8 +18,20 @@ public static class StringExtensions
return @this.ToString();
}
/// <summary>
/// 截断字符串
/// </summary>
/// <param name="this"></param>
/// <param name="maxLength">截断长度</param>
/// <returns></returns>
public static string Omit(this string @this, int maxLength) => Omit(@this.AsSpan(), maxLength);
/// <summary>
/// 将16进制字符串转换为字符串
/// </summary>
/// <param name="hexString"></param>
/// <param name="encoding"></param>
/// <returns></returns>
public static string FromHex(ReadOnlySpan<char> hexString, Encoding? encoding = null)
{
encoding ??= Encoding.UTF8;
@ -38,6 +56,11 @@ public static class StringExtensions
return encoding.GetString(bytes);
}
/// <summary>
/// 检查是否为JSON字符串
/// </summary>
/// <param name="hexStr"></param>
/// <returns></returns>
public static bool CheckJsonHex(ReadOnlySpan<char> hexStr)
{
if (hexStr.Length < 2)

19
ConsoleApp2/Helpers/Extensions.Task.cs Normal file
View File

@ -0,0 +1,19 @@
namespace ConsoleApp2.Helpers;
public static class TaskExtensions
{
public static async Task WaitUntil(Func<bool> condition, int pollDelay = 25, CancellationToken ct = default)
{
try
{
while (!condition())
{
await Task.Delay(pollDelay, ct);
}
}
catch(TaskCanceledException)
{
// CancellationToken激活时Task.Delay通过抛异常来结束停止等待，不用管它
}
}
}

31
ConsoleApp2/Helpers/ValidateConsole.cs
View File

@ -1,31 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace ConsoleApp2.Helpers
{
public static class ValidateConsole
{
public static void ValidateInput<T>(Func<string,bool> converter,string message)
{
Console.Write(message);
string ? input = Console.ReadLine();
while (true)
{
if (!string.IsNullOrEmpty(input))
{
var result = converter(input);
if (result == false)
{
Console.WriteLine($"输入的内容不合法,请重新输入!");
input = Console.ReadLine();
}
else break;
}
break;
}
}
}
}

5
ConsoleApp2/HostedServices/Abstractions/IDataSource.cs → ConsoleApp2/HostedServices/Abstractions/IDataReader.cs
View File

@ -6,8 +6,9 @@ using System.Threading.Tasks;
namespace ConsoleApp2.HostedServices.Abstractions
{
public interface IDataSource:IDisposable
public interface IDataReader : IDisposable
{
public Task DoEnqueue(Action<DataRecord> action);
DataRecord Current { get; }
ValueTask<bool> ReadAsync();
}
}

2
ConsoleApp2/HostedServices/Abstractions/IInputService.cs
View File

@ -5,5 +5,5 @@ namespace ConsoleApp2.HostedServices.Abstractions;
public interface IInputService
{
public Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, ProcessContext context, CancellationToken cancellationToken);
public Task ExecuteAsync(CancellationToken cancellationToken);
}

2
ConsoleApp2/HostedServices/Abstractions/IOutputService.cs
View File

@ -5,5 +5,5 @@ namespace ConsoleApp2.HostedServices.Abstractions;
public interface IOutputService
{
public Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken);
public Task ExecuteAsync(CancellationToken ct);
}

2
ConsoleApp2/HostedServices/Abstractions/ITransformService.cs
View File

@ -5,5 +5,5 @@ namespace ConsoleApp2.HostedServices.Abstractions;
public interface ITransformService
{
public Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken);
public Task ExecuteAsync(CancellationToken cancellationToken);
}

111
ConsoleApp2/HostedServices/FileInputService.cs Normal file
View File

@ -0,0 +1,111 @@
using ConsoleApp2.Const;
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
using ConsoleApp2.Services.ETL;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace ConsoleApp2.HostedServices;
public record FileInputInfo
{
public required string FileName { get; init; }
public required string TableName { get; init; }
public required string[] Headers { get; init; }
}
public enum FileInputType
{
MyDumperCsv,
MyDumperZst,
ErrorLog,
}
/// <summary>
/// 从输入目录中导入文件
/// </summary>
public class FileInputService : IInputService
{
private readonly ILogger _logger;
private readonly DataRecordQueue _producerQueue;
private readonly IOptions<DataInputOptions> _dataInputOptions;
private readonly ProcessContext _context;
private readonly DataReaderFactory _dataReaderFactory;
public FileInputService(ILogger<FileInputService> logger,
IOptions<DataInputOptions> dataInputOptions,
ProcessContext context,
[FromKeyedServices(ProcessStep.Produce)] DataRecordQueue producerQueue,
DataReaderFactory dataReaderFactory)
{
_logger = logger;
_dataInputOptions = dataInputOptions;
_context = context;
_producerQueue = producerQueue;
_dataReaderFactory = dataReaderFactory;
}
public async Task ExecuteAsync(CancellationToken cancellationToken)
{
var inputDir = _dataInputOptions.Value.InputDir ?? throw new ApplicationException("未配置文件输入目录");
_logger.LogInformation("***** Input service started, working directory: {InputDir} *****", inputDir);
var trans = _dataInputOptions.Value.FileInputMetaBuilder;
if(trans is null) throw new ApplicationException("未配置文件名-表名映射委托");
FileInputInfo[] infoArr = Directory.GetFiles(inputDir)
.Select(f => trans(f))
.Where(info => info is not null).ToArray()!;
var orderedInfo = GetFilesInOrder(infoArr).ToArray();
_logger.LogInformation("***** {Count} files founded in directory，{OrderedCount} files is matched with configuration *****", infoArr.Length, orderedInfo.Length);
foreach (var info in orderedInfo)
{
_logger.LogDebug("Table {TableName}: {FileName}", info.TableName, info.FileName);
}
foreach (var info in orderedInfo)
{
_logger.LogInformation("Reading file: {FileName}, table: {TableName}", info.FileName, info.TableName);
var source = _dataReaderFactory.CreateReader(info.FileName,info.TableName,info.Headers);
while (await source.ReadAsync())
{
var record = source.Current;
_producerQueue.Enqueue(record);
_context.AddInput();
}
_logger.LogInformation("Input of table: '{TableName}' finished", info.TableName);
}
_context.CompleteInput();
_logger.LogInformation("***** Input service finished *****");
}
/// <summary>
/// 读取配置，按照配置的表顺序来返回
/// </summary>
/// <returns></returns>
private IEnumerable<FileInputInfo> GetFilesInOrder(FileInputInfo[] inputFiles)
{
var tableOrder = _dataInputOptions.Value.TableOrder;
if (tableOrder is null or { Length: 0 })
return inputFiles;
return Yield();
IEnumerable<FileInputInfo> Yield()
{
foreach (var tableName in tableOrder)
{
var target = inputFiles.FirstOrDefault(f =>
f.TableName.Equals(tableName, StringComparison.OrdinalIgnoreCase));
if (target is not null)
yield return target;
}
}
}
}

59
ConsoleApp2/HostedServices/InputService.cs
View File

@ -1,59 +0,0 @@
using ConsoleApp2.Const;
using ConsoleApp2.Helpers;
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace ConsoleApp2.HostedServices;
/// <summary>
/// 从MyDumper导出的CSV文件中导入表头和数据
/// </summary>
public class InputService : IInputService
{
private readonly ILogger _logger;
private readonly IOptions<DataInputOptions> _dataInputOptions;
private readonly ProcessContext _context;
public InputService(ILogger<InputService> logger,
IOptions<DataInputOptions> dataInputOptions,
ProcessContext context)
{
_logger = logger;
_dataInputOptions = dataInputOptions;
_context = context;
}
public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, ProcessContext context,CancellationToken cancellationToken)
{
var inputDir = _dataInputOptions.Value.InputDir;
_logger.LogInformation("***** Csv input service start, working dir: {InputDir}, thread id: {ThreadId} *****", inputDir, Environment.CurrentManagedThreadId);
var files = Directory.GetFiles(inputDir);
if (files.Length == 0)
{
_logger.LogInformation("No source files found in {InputDir}", inputDir);
return;
}
var count = 0;
foreach (var tableName in tasksOptions.TableInfoConfig.Keys)
{
_logger.LogInformation("Working table: {tableName}", tableName);
var source = _dataInputOptions.Value.CreateSource?.Invoke(tableName);
await source.DoEnqueue((record) =>
{
_context.AddInput();
producerQueue.Enqueue(record);
count++;
});
_logger.LogInformation("table:'{tableName}' input completed", tableName);
}
context.CompleteInput();
_logger.LogInformation("***** Csv input service completed *****");
}
}

278
ConsoleApp2/HostedServices/MainHostedService.cs
View File

@ -1,110 +1,226 @@
using ConsoleApp2.HostedServices.Abstractions;
using System.Diagnostics;
using System.Text;
using ConsoleApp2.Helpers;
using ConsoleApp2.Helpers.Database;
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
using ConsoleApp2.Services.ErrorRecorder;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace ConsoleApp2.HostedServices;
public class MainHostedService : BackgroundService
{
private readonly ILogger _logger;
private Stopwatch? _stopwatch;
private readonly IInputService _input;
private readonly ITransformService _transform;
private readonly IOutputService _output;
private readonly ILogger _logger;
private readonly ProcessContext _context;
private readonly IOptions<DatabaseOutputOptions> _databaseOptions;
private readonly IOptions<TenantDbOptions> _tenantDbOptions;
private readonly IConfiguration _config;
public MainHostedService(ILogger<MainHostedService> logger, IInputService input, ITransformService transform, IOutputService output, ProcessContext context)
public MainHostedService(IInputService input,
ITransformService transform,
IOutputService output,
ILogger<MainHostedService> logger,
IOptions<TenantDbOptions> tenantDbOptions,
IOptions<DatabaseOutputOptions> databaseOptions,
IConfiguration config,
ProcessContext context)
{
_logger = logger;
_input = input;
_transform = transform;
_output = output;
_logger = logger;
_tenantDbOptions = tenantDbOptions;
_databaseOptions = databaseOptions;
_config = config;
_context = context;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var taskFun = (TasksOptions taskOp, DataRecordQueue producerQueue, DataRecordQueue consumerQueue, ProcessContext context) =>
_stopwatch = Stopwatch.StartNew();
var inputTask = ExecuteAndCatch(
async () => await _input.ExecuteAsync(stoppingToken), "文件输入程序出现异常", stoppingToken);
var transformTask = ExecuteAndCatch(
async () => await _transform.ExecuteAsync(stoppingToken), "转换程序出现异常", stoppingToken);
var outputTask = ExecuteAndCatch(
async () => await _output.ExecuteAsync(stoppingToken), "输出程序出现异常", stoppingToken);
await Task.WhenAll(inputTask, transformTask, outputTask);
_stopwatch.Stop();
_logger.LogInformation("***** All tasks completed *****");
_logger.LogInformation("***** ElapseTime: {Time}", (_stopwatch.ElapsedMilliseconds / 1000f).ToString("F3"));
await Task.Delay(5000, stoppingToken);
if (!stoppingToken.IsCancellationRequested)
{
var inputTask = Task.Run(async () =>
{
try
{
await _input.ExecuteAsync(taskOp, producerQueue, context, stoppingToken);
}
catch (Exception ex)
{
_logger.LogCritical("Exception occurred on inputService:{Message},{StackTrace}", ex.Message, ex.StackTrace);
throw;
}
await ExportResultAsync();
_logger.LogInformation("The execution result export to {Path}",
Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"Result-{ErrorRecorder.UID}.md"));
if (_config["RestoreIndex"] is not null)
await RestoreIndexAsync();
});
var transformTask = Task.Run(async () =>
{
try
{
await _transform.ExecuteAsync(taskOp, producerQueue, consumerQueue, context, stoppingToken);
}
catch (Exception ex)
{
_logger.LogCritical("Exception occurred on transformService:{Message},{StackTrace}", ex.Message, ex.StackTrace);
throw;
}
});
var outputTask = Task.Run(async () =>
{
try
{
await _output.ExecuteAsync(taskOp, consumerQueue, context,stoppingToken);
}
catch (Exception ex)
{
_logger.LogCritical("Exception occurred on outputService:{Message},{StackTrace}", ex.Message, ex.StackTrace);
throw;
}
});
};
var bigTablesDic = new Dictionary<string, TableInfo>
Environment.Exit(0);
}
else Environment.Exit(1);
}
private Task ExecuteAndCatch(Func<Task> func, string message, CancellationToken ct)
{
return Task.Run(async () =>
{
{"order_block_plan",new TableInfo{SimulaRowCount=2725553 }},//CreateTime < 202301的删除
{"order_block_plan_result",new TableInfo{SimulaRowCount=1174096 }},
{"order_box_block",new TableInfo{SimulaRowCount=29755672 }},
{"order_item",new TableInfo{SimulaRowCount=1345520079 }},
{"simple_plan_order",new TableInfo{SimulaRowCount=351470 }},//CreateTime < 202301的删除
};
taskFun(new TasksOptions { TableInfoConfig = bigTablesDic, OutPutOptions = new OutPutOptions { FlushCount = 10000, OutPutTaskCount = 2 } },
new DataRecordQueue(), new DataRecordQueue(),new ProcessContext());
var smallTablesDic = new Dictionary<string, TableInfo>
try
{
await func();
}
catch (Exception e)
{
_logger.LogCritical(e, "{Msg}\t{ErrMsg}", message, e.Message);
_context.AddException(e);
Environment.Exit(1);
}
}, ct);
}
/// <summary>
/// 还原所有数据库的索引...
/// </summary>
/// <returns></returns>
/// <exception cref="ApplicationException"></exception>
private Task RestoreIndexAsync()
{
var databases = _tenantDbOptions.Value.DbList?.Keys
?? throw new ApplicationException("无法还原索引，因为分库配置中没有配置数据库");
var connStr = _databaseOptions.Value.ConnectionString
?? throw new ApplicationException("无法还原索引，因为没有配置数据库连接字符串");
var list = new List<Task>();
foreach(var db in databases)
{
{"machine",new TableInfo{SimulaRowCount=14655 }},
{"order",new TableInfo{SimulaRowCount=5019216 }},
{"order_data_block",new TableInfo{SimulaRowCount=731800334 }},
{"order_data_goods",new TableInfo{SimulaRowCount=25803671 }},
{"order_data_parts",new TableInfo{SimulaRowCount=468517543 }},
{"order_module",new TableInfo{SimulaRowCount=103325385 }},
{"order_module_extra",new TableInfo{SimulaRowCount=54361321 }},
{"order_module_item",new TableInfo{SimulaRowCount=69173339 }},
{"order_package",new TableInfo{SimulaRowCount=16196195 }},
{"order_process",new TableInfo{SimulaRowCount=3892685 }},//orderNo < 202301的
{"order_process_step",new TableInfo{SimulaRowCount=8050349 }},//orderNo < 202301的删除
{"order_process_step_item",new TableInfo{SimulaRowCount=14538058 }},//orderNo < 202301的删除
{"order_scrap_board",new TableInfo{SimulaRowCount=123998 }},
{"process_group",new TableInfo{SimulaRowCount=1253 }},
{"process_info",new TableInfo{SimulaRowCount=7839 }},
{"process_item_exp",new TableInfo{SimulaRowCount=28 }},
{"process_schdule_capacity",new TableInfo{SimulaRowCount=39736 }},
{"process_step_efficiency",new TableInfo{SimulaRowCount=8 }},
{"report_template",new TableInfo{SimulaRowCount=7337 }},
{"simple_package",new TableInfo{SimulaRowCount=130436 }},//orderNo < 202301的删除
{"sys_config",new TableInfo{SimulaRowCount=2296 }},
{"work_calendar",new TableInfo{SimulaRowCount=11 }},
{"work_shift",new TableInfo{SimulaRowCount=59 }},
{"work_time",new TableInfo{SimulaRowCount=62 }},
var task = DatabaseHelper.NonQueryAsync(connStr + $";Database={db};",
"""
CREATE INDEX `idx_CompanyID` ON `machine` (`CompanyID`);
CREATE INDEX `idx_companyid` ON `order` (`CompanyID`);
CREATE INDEX `idx_CompanyID` ON `order_block_plan` (`CompanyID`);
CREATE INDEX `idx_PlanID` ON `order_block_plan_item` (`PlanID`);
CREATE INDEX `idx_orderno` ON `order_box_block` (`OrderNo`);
CREATE INDEX `index_OrderNo` ON `order_data_block` (`OrderNo`);
CREATE INDEX `index_OrderNo` ON `order_data_goods` (`OrderNo`);
CREATE INDEX `index_OrderNo` ON `order_data_parts` (`OrderNo`);
CREATE INDEX `index_ItemNo` ON `order_item` (`ItemNo`);
CREATE INDEX `index_OrderNo` ON `order_item` (`OrderNo`);
CREATE INDEX `index_PackageID` ON `order_item` (`PackageID`);
CREATE INDEX `index_PlanID` ON `order_item` (`PlanID`);
CREATE INDEX `idx_OrderNo` ON `order_module` (`OrderNo`);
CREATE INDEX `index_OrderNo` ON `order_module_extra` (`OrderNo`);
CREATE INDEX `index_OrderNo` ON `order_module_item` (`OrderNo`);
CREATE INDEX `idx_OrderNo` ON `order_package` (`OrderNo`);
CREATE INDEX `idx_PakageNo` ON `order_package` (`PakageNo`);
CREATE INDEX `idx_PackageID` ON `order_package_item` (`PackageID`);
CREATE INDEX `idx_companyid` ON `order_patch_detail` (`CompanyID`);
CREATE INDEX `idx_OrderNo` ON `order_process` (`OrderNo`);
CREATE INDEX `index_CompanyID` ON `order_process_schdule` (`CompanyID`);
CREATE INDEX `IX_order_process_step_OrderProcessID` ON `order_process_step` (`OrderProcessID`);
CREATE INDEX `idx_OrderProcessID` ON `order_process_step_item` (`OrderProcessID`);
CREATE INDEX `idx_OrderProcessStepID` ON `order_process_step_item` (`OrderProcessStepID`);
CREATE INDEX `idx_CompanyID` ON `order_scrap_board` (`CompanyID`);
CREATE INDEX `idx_CompanyID` ON `process_group` (`CompanyID`);
CREATE INDEX `idx_CompanyID` ON `process_info` (`CompanyID`);
CREATE INDEX `index_CompanyID` ON `process_item_exp` (`CompanyID`);
CREATE INDEX `idx_CompanyID` ON `process_schdule_capacity` (`CompanyID`);
CREATE INDEX `idx_CompanyID` ON `process_step_efficiency` (`CompanyID`);
CREATE INDEX `idx_CompanyID` ON `report_template` (`CompanyID`);
CREATE INDEX `indx_OrderNo` ON `simple_package` (`OrderNo`);
CREATE INDEX `idx_CompanyID` ON `simple_plan_order` (`CompanyID`);
CREATE INDEX `idx_CompanyID` ON `sys_config` (`CompanyID`);
CREATE INDEX `idx` ON `work_calendar` (`CompanyID`);
CREATE INDEX `idx_CompanyID` ON `work_shift` (`CompanyID`);
CREATE INDEX `IX_work_time_ShiftID` ON `work_time` (`ShiftID`);
""");
list.Add(task);
}
return Task.WhenAll(list);
}
private async Task ExportResultAsync()
{
var sb = new StringBuilder();
if (_context.HasException)
sb.AppendLine("# Program Completed With Error");
else sb.AppendLine("# Program Completed Successfully");
sb.AppendLine("## Process Count");
var processCount = new[]
{
new { State = "Input", Count = _context.InputCount },
new { State = "Transform", Count = _context.TransformCount },
new { State = "Output", Count = _context.OutputCount }
};
taskFun(new TasksOptions { TableInfoConfig = smallTablesDic, OutPutOptions = new OutPutOptions { FlushCount = 20000, OutPutTaskCount = 4 } },
new DataRecordQueue(), new DataRecordQueue(), new ProcessContext());
sb.AppendLine(processCount.ToMarkdownTable());
sb.AppendLine("\n---\n");
sb.AppendLine("## Table Output Progress");
var tableOutputProgress = _context.TableProgress.Select(pair =>
new { Table = pair.Key, Count = pair.Value });
sb.AppendLine(tableOutputProgress.ToMarkdownTable());
sb.AppendLine("\n---\n");
sb.AppendLine("## Result");
var elapsedTime = (_stopwatch!.ElapsedMilliseconds / 1000f);
var result = new[]
{
new { Field = "ElapsedTime", Value = elapsedTime.ToString("F2") },
new
{
Field = "Average Output Speed",
Value = (_context.OutputCount / elapsedTime).ToString("F2") + "records/s"
}
};
sb.AppendLine(result.ToMarkdownTable());
await File.WriteAllTextAsync(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"Result-{ErrorRecorder.UID}.md"),
sb.ToString());
}
}

139
ConsoleApp2/HostedServices/OutputService.cs
View File

@ -2,8 +2,11 @@
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
using ConsoleApp2.Services.ErrorRecorder;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MySqlDestination = ConsoleApp2.Services.ETL.MySqlDestination;
using TaskExtensions = ConsoleApp2.Helpers.TaskExtensions;
namespace ConsoleApp2.HostedServices;
@ -14,96 +17,116 @@ public class OutputService : IOutputService
{
private readonly ILogger _logger;
private readonly IOptions<DatabaseOutputOptions> _outputOptions;
private readonly IOptions<DataTransformOptions> _transformOptions;
private readonly ProcessContext _context;
private readonly TaskManager _taskManager;
private readonly ErrorRecorder _errorRecorder;
private readonly ErrorRecorderFactory _errorRecorderFactory;
private readonly RecordQueuePool _queuePool;
public OutputService(ILogger<OutputService> logger,
IOptions<DatabaseOutputOptions> outputOptions,
ProcessContext context,
TaskManager taskManager,
IOptions<DataTransformOptions> transformOptions,
ErrorRecorder errorRecorder)
RecordQueuePool queuePool,
ErrorRecorderFactory errorRecorderFactory)
{
_logger = logger;
_outputOptions = outputOptions;
_context = context;
_taskManager = taskManager;
_transformOptions = transformOptions;
_errorRecorder = errorRecorder;
_queuePool = queuePool;
_errorRecorderFactory = errorRecorderFactory;
}
public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken)
public async Task ExecuteAsync(CancellationToken ct)
{
_logger.LogInformation("***** Mysql output service started *****");
_taskManager.CreateTasks(async () =>
_logger.LogInformation("***** Output service started *****");
var dbTaskManager = new TaskManager(5);
var dbTasks = new Dictionary<string, Task>();
while (!_context.IsTransformCompleted)
{
//k: database v: records，按照要导出的数据库名分组
var databaseDict = new Dictionary<string, List<DataRecord>>();
while (!context.IsTransformCompleted || consumerQueue.Count > 0)
foreach (var (db, queue) in _queuePool.Queues)
{
if (!consumerQueue.TryDequeue(out var record)) continue;
var dbName = record.Database;
var records = databaseDict.AddOrUpdate(dbName, [record], (_, list) =>
if (!dbTasks.ContainsKey(db))
{
list.Add(record);
return list;
});
if (records.Count >= tasksOptions.OutPutOptions.FlushCount)
{
await FlushAsync(dbName, records);
records.Clear();
dbTasks.Add(db, await dbTaskManager.CreateTaskAsync(
async () => await StartDatabaseWorker(db, queue, ct), ct));
}
}
foreach (var (db, records) in databaseDict)
await Task.Delay(500, ct);
}
await TaskExtensions.WaitUntil(() => dbTaskManager.RunningTaskCount == 0, 25, ct);
_context.CompleteOutput();
_logger.LogInformation("***** Output service finished *****");
}
private async Task StartDatabaseWorker(string db, DataRecordQueue queue, CancellationToken ct = default)
{
_logger.LogInformation("*****开启输出线程，数据库: {db} *****", db);
var taskManager = new TaskManager(_outputOptions.Value.MaxDatabaseOutputTask);
var tmp = new List<DataRecord>();
while (!_context.IsTransformCompleted || queue.Count > 0)
{
if (ct.IsCancellationRequested)
break;
if (!queue.TryDequeue(out var record)) continue;
var dbName = record.Database ?? throw new ApplicationException("输出的记录缺少数据库名");
if(dbName != db)
throw new ApplicationException($"输出记录的数据与当前输出线程不匹配，记录：{dbName}, 输出线程：{db}");
tmp.Add(record);
if (tmp.Count >= _outputOptions.Value.FlushCount)
{
if (records.Count > 0)
var list = tmp;
tmp = [];
await taskManager.CreateTaskAsync(async arg => // 转换为方法组
{
await FlushAsync(db, records);
records.Clear();
}
var tuple = arg as Tuple<string, List<DataRecord>>;
try
{
await FlushAsync(tuple!.Item1, tuple.Item2);
}
catch (Exception e)
{
_logger.LogError(e, "输出记录时发生错误");
throw;
}
}, Tuple.Create(dbName, list), ct);
}
}
databaseDict.Clear();
_logger.LogInformation("***** Mysql output thread completed *****");
}, tasksOptions.OutPutOptions.OutPutTaskCount);
await _taskManager.WaitAll();
//_context.CompleteOutput();
_logger.LogInformation("***** Mysql output service completed *****");
// 等待所有子任务完成
await TaskExtensions.WaitUntil(() => taskManager.RunningTaskCount == 0, 10, ct);
// 清理剩余记录
if (tmp.Count > 0)
{
await FlushAsync(db, tmp);
}
_logger.LogInformation("*****输出线程结束，数据库: {db} *****", db);
}
private async Task FlushAsync(string dbName, IEnumerable<DataRecord> records)
{
var count = 0;
var connStr = _outputOptions.Value.ConnectionString ?? throw new InvalidOperationException("ConnectionString is null");
await using var output = new MySqlDestination($"{connStr};Database={dbName};", _logger, _context, _transformOptions, _errorRecorder);
//if (records == null || records.Count() == 0) return;
//var dbName = $"cferp_test_1";
//if (records != null && records.Count() > 0)
//{
// dbName = $"cferp_test_{records.FirstOrDefault()?.CompanyID}";
//}
var connStr = _outputOptions.Value.ConnectionString ??
throw new InvalidOperationException("连接字符串为空");
await using var output = new MySqlDestination($"{connStr};Database={dbName};", _logger,
_outputOptions, _errorRecorderFactory.CreateOutput(dbName), _context);
//await using var output = new MySqlDestination(new MySqlConnectionStringBuilder
//{
// Server = "127.0.0.1",
// Port = 34309,
// Database = dbName,
// UserID = "root",
// Password = "123456",
// MaximumPoolSize = 50,
//}.ConnectionString, _logger,true);
var tableOutput = new Dictionary<string, int>();
foreach (var record in records)
{
await output.WriteRecordAsync(record);
count++;
tableOutput.AddOrUpdate(record.TableName, 1, (_, v) => v + 1);
}
await output.FlushAsync(_outputOptions.Value.MaxAllowedPacket);
_context.AddOutput(count);
foreach (var (key, value) in tableOutput)
{
_context.AddTableOutput(key, value);
}
_logger.LogTrace("Flushed {Count} records", tableOutput.Values.Sum(i => i));
}
}

70
ConsoleApp2/HostedServices/TaskMonitorService.cs
View File

@ -1,6 +1,7 @@
using System.Diagnostics;
using ConsoleApp2.Const;
using ConsoleApp2.Services;
using ConsoleApp2.Services.Loggers;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
@ -12,33 +13,33 @@ namespace ConsoleApp2.HostedServices;
/// </summary>
public class TaskMonitorService : BackgroundService
{
private readonly IHostApplicationLifetime _lifetime;
private readonly ILogger<TaskMonitorService> _logger;
private readonly IEnumerable<ITaskMonitorLogger> _monitorLoggers;
private readonly ProcessContext _context;
private readonly DataRecordQueue _producerQueue;
private readonly DataRecordQueue _consumerQueue;
private readonly RecordQueuePool _queuePool;
public TaskMonitorService(IHostApplicationLifetime lifetime,
public TaskMonitorService(
ILogger<TaskMonitorService> logger,
ProcessContext context,
[FromKeyedServices(ProcessStep.Producer)]
[FromKeyedServices(ProcessStep.Produce)]
DataRecordQueue producerQueue,
[FromKeyedServices(ProcessStep.Consumer)]
DataRecordQueue consumerQueue)
RecordQueuePool queuePool,
IEnumerable<ITaskMonitorLogger> monitorLoggers)
{
_lifetime = lifetime;
_logger = logger;
_context = context;
_producerQueue = producerQueue;
_consumerQueue = consumerQueue;
_queuePool = queuePool;
_monitorLoggers = monitorLoggers;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
await Task.Factory.StartNew(Monitor, stoppingToken);
await Task.Run(() => Monitor(stoppingToken), stoppingToken);
}
private async Task Monitor()
private async Task Monitor(CancellationToken stoppingToken)
{
var sw = Stopwatch.StartNew();
var lastTime = sw.ElapsedMilliseconds;
@ -47,7 +48,7 @@ public class TaskMonitorService : BackgroundService
var lastOutputCount = _context.OutputCount;
bool endCheck = false;
while (true)
while (!stoppingToken.IsCancellationRequested)
{
EndCheck:
// var running = 0;
@ -82,22 +83,40 @@ public class TaskMonitorService : BackgroundService
var inputSpeed = (inputCount - lastInputCount) / elapseTime;
var transformSpeed = (transformCount - lastTransformCount) / elapseTime;
var outputSpeed = (outputCount - lastOutputCount) / elapseTime;
// _logger.LogInformation(
// "Task monitor: running: {Running}, error: {Error}, completed: {Completed}, canceled: {Canceled}, outputSpeed: {Speed} records/s",
// running, error, completed, canceled, outputSpeed);
_logger.LogInformation(
"Process monitor: input: {inputStatus}, transform: {transformStatus}, output: {outputStatus}\nInput: {InputCount}, Transform: {TransformCount}, Output: {OutputCount}",
_context.IsInputCompleted ? "completed" : $"running {inputSpeed:F2} records/s",
_context.IsTransformCompleted ? "completed" : $"running {transformSpeed:F2} records/s",
_context.IsOutputCompleted ? "completed" : $"running {outputSpeed:F2} records/s",
inputCount,
transformCount,
outputCount);
_logger.LogInformation("Queue monitor: producer queue: {ProducerQueue}, consumer queue: {ConsumerQueue}",
_producerQueue.Count, _consumerQueue.Count);
foreach (var logger in _monitorLoggers)
{
logger.LogStatus("Monitor: Progress status", new Dictionary<string, string>
{
{"Input",_context.IsInputCompleted ? "completed" : $"running {inputSpeed:F2} records/s" },
{"Transform", _context.IsTransformCompleted ? "completed" : $"running {transformSpeed:F2} records/s" },
{"Output", _context.IsOutputCompleted ? "completed" : $"running {outputSpeed:F2} records/s" }
});
await Task.Delay(5000);
logger.LogStatus("Monitor: Table output progress",
_context.TableProgress
.ToDictionary(kv => kv.Key, kv => kv.Value.ToString()),
ITaskMonitorLogger.LogLevel.Progress);
logger.LogStatus("Monitor: Process count", new Dictionary<string, string>
{
{"Input", inputCount.ToString()},
{"Transform", transformCount.ToString()},
{"Output", outputCount.ToString()}
}, ITaskMonitorLogger.LogLevel.Progress);
logger.LogStatus("Monitor: Queue", new Dictionary<string, string>
{
{"Producer queue records", _producerQueue.Count.ToString() },
{"Output queues", _queuePool.Queues.Count.ToString() },
{"Output queue records", _queuePool.Queues.Values.Sum(queue => queue.Count).ToString()},
});
}
await Task.Delay(5000, stoppingToken);
lastTime = time;
lastInputCount = inputCount;
@ -114,9 +133,6 @@ public class TaskMonitorService : BackgroundService
break;
}
}
_logger.LogInformation("***** All tasks completed *****");
_logger.LogInformation("***** ElapseTime: {Time}", (sw.ElapsedMilliseconds / 1000f).ToString("F3"));
// _lifetime.StopApplication();
}
}

83
ConsoleApp2/HostedServices/TestInputService.cs
View File

@ -1,83 +0,0 @@
using ConsoleApp2.HostedServices.Abstractions;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using ConsoleApp2.Const;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using System.Reflection.PortableExecutable;
using System.Collections.Concurrent;
using ConsoleApp2.SimulationService;
namespace ConsoleApp2.HostedServices
{
public class TestInputService : IInputService
{
private readonly ILogger _logger;
private readonly IOptions<CsvOptions> _csvOptions;
private readonly DataRecordQueue _producerQueue;
private readonly ProcessContext _context;
public TestInputService(ILogger<TestInputService> logger,
IOptions<CsvOptions> csvOptions,
[FromKeyedServices(ProcessStep.Producer)] DataRecordQueue producerQueue,
ProcessContext context)
{
_logger = logger;
_csvOptions = csvOptions;
_producerQueue = producerQueue;
_context = context;
}
public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, ProcessContext context, CancellationToken cancellationToken)
{
var tableName = "order_item";
var headers = new string[] { "ID","OrderNo","ItemNo","ItemType","RoomID","BoxID","DataID","PlanID","PackageID","Num","CompanyID","ShardKey" };
var dataCount = 1200000000L;
var tempCount = 80000;
var tempRecords=new List<DataRecord>();
var comanyID = 1;
short[] shareKeys = { 23040, 23070, 23100, 24000, 24040, 24070, 24100, 25000, 25040, 25070, 25100 };
int[] companyIds = { 1, 2, 3, 4 };
var sk = shareKeys.First();
var companyID = companyIds.First();
var shareKeyInterval = 20000;
var getShareKeyTimes = 0;
var getCompanyIDTimes = 0;
var shareKeyIntervalCount = 0;
for (long i = 1; i <= dataCount; i++)
{
shareKeyIntervalCount++;
if (shareKeyIntervalCount > shareKeyInterval) {
sk=DataHelper.GetShareKey(getShareKeyTimes);
getShareKeyTimes++;
shareKeyIntervalCount = 0;
}
var fields = new string[] { i.ToString(), "20220104020855", (220105981029 + i).ToString(), "1", "482278", "482279", "3768774", "0", "0", "1", companyID.ToString(), sk.ToString() };
var record = new DataRecord(fields, tableName, headers, comanyID);
tempRecords.Add(record);
if (tempRecords.Count >= tempCount)
{
foreach (var rc in tempRecords)
{
_context.AddInput();
_producerQueue.Enqueue(rc);
if (cancellationToken.IsCancellationRequested)
return;
}
tempRecords.Clear();
companyID = DataHelper. GetCompanyId(getCompanyIDTimes);
getCompanyIDTimes++;
}
}
_context.CompleteInput();
_logger.LogInformation("***** Csv input service completed *****");
}
}
}

112
ConsoleApp2/HostedServices/TransformService.cs
View File

@ -1,15 +1,17 @@
using ConsoleApp2.Const;
using ConsoleApp2.Cache;
using ConsoleApp2.Const;
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
using Microsoft.Extensions.Caching.Distributed;
using ConsoleApp2.Services.ErrorRecorder;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StackExchange.Redis;
namespace ConsoleApp2.HostedServices;
public record DataTransformContext(DataRecord Record, ICacher Cacher, ILogger Logger);
/// <summary>
/// 数据处理服务，对导入后的数据进行处理
/// </summary>
@ -17,65 +19,99 @@ public class TransformService : ITransformService
{
private readonly ILogger _logger;
private readonly IOptions<DataTransformOptions> _options;
private readonly DataRecordQueue _producerQueue;
private readonly RecordQueuePool _queuePool;
private readonly ProcessContext _context;
private readonly IDistributedCache _cache;
private readonly TaskManager _taskManager;
private readonly ICacher _cache;
private readonly ErrorRecorderFactory _errorRecorderFactory;
public TransformService(ILogger<TransformService> logger,
IOptions<DataTransformOptions> options,
[FromKeyedServices(ProcessStep.Produce)] DataRecordQueue producerQueue,
RecordQueuePool queuePool,
ProcessContext context,
IDistributedCache cache,
TaskManager taskManager)
ICacher cache,
ErrorRecorderFactory errorRecorderFactory)
{
_logger = logger;
_options = options;
_producerQueue = producerQueue;
_queuePool = queuePool;
_context = context;
_cache = cache;
_taskManager = taskManager;
_errorRecorderFactory = errorRecorderFactory;
}
public async Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue producerQueue, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken)
public async Task ExecuteAsync(CancellationToken cancellationToken)
{
_logger.LogInformation("***** Data transform service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId);
_taskManager.CreateTasks(async () =>
while (!_context.IsInputCompleted || _producerQueue.Count > 0)
{
while ((!context.IsInputCompleted || producerQueue.Count > 0))
{
if (!producerQueue.TryDequeue(out var record)) continue;
if (!_producerQueue.TryDequeue(out var record)) continue;
//过滤不要的record
if (await _options.Value.RecordFilter?.Invoke(record, _cache) == false) continue;
//修改record
_options.Value.RecordModify?.Invoke(record);
//缓存record
await _options.Value.RecordCache?.Invoke(record, _cache);
//替换record
var replaceRecord = await _options.Value.RecordReplace?.Invoke(record, _cache);
if (replaceRecord != null)
try
{
var context = new DataTransformContext(record, _cache, _logger);
if (_options.Value.EnableFilter)
{
record = replaceRecord;
// 数据过滤
var filter = _options.Value.RecordFilter;
if (filter is not null && await filter(context) == false) continue;
}
//计算需要分流的数据库
record.Database = _options.Value.DatabaseFilter.Invoke(record);
consumerQueue.Enqueue(record);
_context.AddTransform();
//数据增加
var addRecords = _options.Value.RecordAdd?.Invoke(record);
if (addRecords is { Count: > 0 })
if (_options.Value.EnableReplacer)
{
foreach (var rc in addRecords)
// 数据替换
var replacer = _options.Value.RecordModify;
if (replacer is not null)
{
rc.Database = _options.Value.DatabaseFilter.Invoke(record);
consumerQueue.Enqueue(rc);
_context.AddTransform();
record = await replacer(context);
}
}
// 字段缓存
var cacher = _options.Value.RecordCache;
if(cacher is not null)
await cacher.Invoke(context);
//计算需要分流的数据库
var dbFilter = _options.Value.DatabaseFilter
?? throw new ApplicationException("未配置数据库过滤器");
record.Database = dbFilter(record);
_queuePool[record.Database].Enqueue(record);
_context.AddTransform();
if (_options.Value.EnableReBuilder)
{
//数据重建
var addRecords = _options.Value.RecordReBuild?.Invoke(context);
if (addRecords is { Count: > 0 })
{
foreach (var rc in addRecords)
{
if(dbFilter is not null)
rc.Database =dbFilter.Invoke(record);
_queuePool[record.Database].Enqueue(rc);
_context.AddTransform();
}
}
}
}
context.CompleteTransform();
},tasksOptions.TransformTaskCount,cancellationToken);
await _taskManager.WaitAll();
_logger.LogInformation("***** Data transformation service completed *****");
catch (Exception e)
{
_context.AddException(e);
var errorRecorder = _errorRecorderFactory.CreateTransform();
await errorRecorder.LogErrorRecordAsync(record, e);
if (!_options.Value.StrictMode)
_logger.LogError(e, "数据转换时发生错误");
else throw;
}
}
_context.CompleteTransform();
_logger.LogInformation("***** Data transformation service finished *****");
}
}

32
ConsoleApp2/HostedServices/VoidOutputService.cs
View File

@ -1,9 +1,5 @@
using ConsoleApp2.Const;
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Services;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace ConsoleApp2.HostedServices;
@ -12,28 +8,36 @@ namespace ConsoleApp2.HostedServices;
public class VoidOutputService : IOutputService
{
private readonly ILogger _logger;
private readonly DataRecordQueue _consumerQueue;
private readonly RecordQueuePool _queuePool;
private readonly ProcessContext _context;
public VoidOutputService([FromKeyedServices(ProcessStep.Consumer)] DataRecordQueue consumerQueue,
ProcessContext context, ILogger<VoidOutputService> logger)
public VoidOutputService(
ProcessContext context, ILogger<VoidOutputService> logger, RecordQueuePool queuePool)
{
_consumerQueue = consumerQueue;
_context = context;
_logger = logger;
_queuePool = queuePool;
}
public Task ExecuteAsync(TasksOptions tasksOptions, DataRecordQueue consumerQueue, ProcessContext context, CancellationToken cancellationToken)
public Task ExecuteAsync(CancellationToken ct)
{
_logger.LogInformation("***** Void output service started, thread id: {ThreadId} *****", Environment.CurrentManagedThreadId);
while (!_context.IsTransformCompleted || _consumerQueue.Count > 0)
_logger.LogInformation("***** Void Output Service Started *****");
while (!_context.IsTransformCompleted || _queuePool.Queues.Count > 0)
{
if (_consumerQueue.TryDequeue(out var record))
foreach (var pair in _queuePool.Queues) // 内存优化
{
if (_context.IsTransformCompleted && pair.Value.Count == 0)
{
_queuePool.RemoveQueue(pair.Key);
continue;
}
if(!pair.Value.TryDequeue(out var record)) continue;
_context.AddOutput();
}
}
_context.CompleteOutput();
_logger.LogInformation("***** Void output service completed *****");
_logger.LogInformation("***** Void Output Service Stopped *****");
return Task.CompletedTask;
}
}

21
ConsoleApp2/Options/CommandOptions.cs
View File

@ -1,21 +0,0 @@
using System;
using System.ComponentModel;
using System.Configuration;
namespace ConsoleApp2.Options
{
public class CommandOptions
{
public string InputDir { get; set; } = "./MyDumper";
public int TaskCount { get; set; } = 16;
public int FlushCount { get; set; } = 20000;
public bool Isutf8mb4 { get; set; } = true;
public short OldestShardKey { get; set; } = 23010;
public string OldestTime { get; set; } = "202301";
}
}

19
ConsoleApp2/Options/CsvOptions.cs
View File

@ -1,19 +0,0 @@
namespace ConsoleApp2.Options;
public class CsvOptions
{
/// <summary>
/// MyDumper导出的CSV文件目录
/// </summary>
//public string InputDir { get; set; } = "./";
/// <summary>
/// 字符串的包围符号，默认为双引号"
/// </summary>
public char QuoteChar { get; set; } = '"';
/// <summary>
/// 每个字段的分割符，默认逗号,
/// </summary>
public string Delimiter { get; set; } = ",";
}

55
ConsoleApp2/Options/DataInputOptions.cs
View File

@ -1,18 +1,53 @@
using ConsoleApp2.Services;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection.PortableExecutable;
using System.Text;
using System.Threading.Tasks;
using ConsoleApp2.HostedServices;
namespace ConsoleApp2.Options
{
public enum InputFileType { CSV, JWT, JSV }
public class DataInputOptions
{
public string InputDir { get; set; } = "./";
public string? InputDir { get; set; }
public Func<string, CsvSource>? CreateSource { get; set; }
#region CSV
/// <summary>
/// 字符串的包围符号，默认为双引号"
/// </summary>
public char QuoteChar { get; set; } = '"';
/// <summary>
/// 每个字段的分割符，默认逗号,
/// </summary>
public string Delimiter { get; set; } = ",";
#endregion
/// <summary>
/// yyyyMM
/// </summary>
public string CleanDate { get; set; } = "202301";
#region Mock
public bool UseMock { get; set; }
public double MockCountMultiplier { get; set; } = 1;
/// <summary>
/// Table -> Mock Count 暂时为手动配置
/// </summary>
public Dictionary<string, TableMockConfig>? TableMockConfig { get; set; }
#endregion
#region ManualSet
public string[]? TableOrder { get; set; }
/// <summary>
/// 配置如何从文件名转换为表名和表头
/// </summary>
public Func<string, FileInputInfo?>? FileInputMetaBuilder { get; set; } //TODO: 抽离
#endregion
}
}

52
ConsoleApp2/Options/DataTransformOptions.cs
View File

@ -1,5 +1,6 @@
using Microsoft.Extensions.Caching.Distributed;
using StackExchange.Redis;
using ConsoleApp2.Cache;
using ConsoleApp2.HostedServices;
using Microsoft.Extensions.Logging;
namespace ConsoleApp2.Options;
@ -13,23 +14,34 @@ public enum ColumnType
public class DataTransformOptions
{
public Func<DataRecord, string>? DatabaseFilter { get; set; }
public Func<string, string>? TransformBinary { get; set; }//Binary转字符串方法
public Func<DataRecord, IDistributedCache, Task<bool>>? RecordFilter { get; set; }//数据过滤方法
public Action<DataRecord>? RecordModify { get; set; }//数据修改
public Func<DataRecord, IDistributedCache, Task<DataRecord?>>? RecordReplace { get; set; }//数据替换
public Func<DataRecord, IList<DataRecord>?>? RecordAdd { get; set; }//数据替换
public Func<DataRecord, IDistributedCache, Task>? RecordCache { get; set; }//数据缓存
public bool StrictMode { get; set; } = true;
public bool EnableFilter { get; set; } = true;
public bool EnableReplacer { get; set; } = true;
public bool EnableReBuilder { get; set; } = true;
/// <summary>
/// 配置导入数据的特殊列
/// Record -> Database name
/// 对记录进行数据库过滤
/// </summary>
public Dictionary<string, ColumnType> ColumnTypeConfig { get; set; } = new(); // "table.column" -> type
public ColumnType GetColumnType(string table, string column)
{
return ColumnTypeConfig.GetValueOrDefault($"{table}.{column}", ColumnType.UnDefine);
}
}
public Func<DataRecord, string>? DatabaseFilter { get; set; }
/// <summary>
/// Context -> Should output
/// 配置对数据过滤的条件
/// </summary>
public Func<DataTransformContext, Task<bool>>? RecordFilter { get; set; }//数据过滤方法
/// <summary>
/// Context -> New record
/// 对当前记录进行修改或完整替换
/// </summary>
public Func<DataTransformContext, Task<DataRecord>>? RecordModify { get; set; }//数据替换
/// <summary>
/// Context -> New rebuild records
/// 使用当前记录对某些数据进行重建
/// </summary>
public Func<DataTransformContext, IList<DataRecord>?>? RecordReBuild { get; set; }//新增数据
/// <summary>
/// Context -> void
/// 对数据的某些字段进行缓存
/// </summary>
public Func<DataTransformContext, Task>? RecordCache { get; set; }//数据缓存
}

23
ConsoleApp2/Options/DatabaseOutputOptions.cs
View File

@ -2,9 +2,24 @@
public class DatabaseOutputOptions
{
/// <summary>
/// 数据库连接字符串
/// </summary>
public string? ConnectionString { get; set; }
public int MaxAllowedPacket { get; set; } = 64*1024*1024;
public int MaxAllowedPacket { get; set; } = 64 * 1024 * 1024;
public int FlushCount { get; set; } = 10000;
public int MaxDatabaseOutputTask { get; set; } = 4;
public bool TreatJsonAsHex { get; set; } = true;
/// <summary>
/// 配置导入数据的特殊列
/// </summary>
public Dictionary<string, ColumnType> ColumnTypeConfig { get; set; } = new(); // "table.column" -> type
public ColumnType GetColumnType(string table, string column)
{
return ColumnTypeConfig.GetValueOrDefault($"{table}.{column}", ColumnType.UnDefine);
}
}

8
ConsoleApp2/Options/RedisCacheOptions.cs Normal file
View File

@ -0,0 +1,8 @@
namespace ConsoleApp2.Options;
public class RedisCacheOptions
{
public string? Configuration { get; init; }
public string InstanceName { get; init; } = "";
public int Database { get; init; } = 0;
}

32
ConsoleApp2/Options/TableMockConfig.cs Normal file
View File

@ -0,0 +1,32 @@
namespace ConsoleApp2.Options;
public struct TableMockConfig
{
/// <summary>
/// 使用深拷贝
/// </summary>
public bool UseDeepCopy { get; set; }
/// <summary>
/// 模拟数据量
/// </summary>
public long MockCount { get; set; }
/// <summary>
/// 需要开启MockCount
/// </summary>
public string[]? AutoIncrementColumn { get; set; } = null; // TODO: 换为自定义委托
public void Deconstruct(out bool useDeepCopy, out long mockCount, out string[]? autoIncrementColumn)
{
useDeepCopy = UseDeepCopy;
mockCount = MockCount;
autoIncrementColumn = AutoIncrementColumn;
}
public TableMockConfig(bool useDeepCopy, long mockCount, string[]? autoIncrementColumn)
{
UseDeepCopy = useDeepCopy;
MockCount = mockCount;
AutoIncrementColumn = autoIncrementColumn;
}
}

24
ConsoleApp2/Options/TasksOptions.cs
View File

@ -1,24 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace ConsoleApp2.Options
{
public class TableInfo
{
public long SimulaRowCount { get; set; }//模拟的记录条数
}
public class TasksOptions
{
public Dictionary<string, TableInfo> TableInfoConfig { get; set; } = new();
public int TransformTaskCount { get; set; } = 1;
public OutPutOptions OutPutOptions { get; set; }=new();
}
public class OutPutOptions
{
public int FlushCount { get; set; } = 10000;
public int OutPutTaskCount { get; set; } = 2;
}
}

7
ConsoleApp2/Options/TenantDbOptions.cs
View File

@ -2,22 +2,23 @@ namespace ConsoleApp2.Options;
public class TenantDbOptions
{
public string TenantKey { get; set; }
public string? TenantKey { get; set; }
/// <summary>
/// Key-Value: {DbName}-{TenantKeyLessThan}
/// </summary>
public Dictionary<string, int> DbList { get; set; }
public Dictionary<string, int>? DbList { get; set; }
public string GetDbNameByTenantKeyValue(int tenantKeyValue)
{
// var dictionary = new SortedDictionary<int, string>();
// DbList.ForEach(pair => dictionary.Add(pair.Value, pair.Key));
// 注意配置顺序
if(DbList is null) throw new ApplicationException("分库配置中没有发现任何数据库");
var dbName = DbList.Cast<KeyValuePair<string, int>?>()
.FirstOrDefault(pair => pair?.Value != null && pair.Value.Value > tenantKeyValue)!.Value.Key;
return dbName ??
throw new ArgumentOutOfRangeException(nameof(tenantKeyValue),
$"已配置的数据库中没有任何符合'{nameof(tenantKeyValue)}'值的对象");
$"分库配置中没有任何符合'{nameof(tenantKeyValue)}'值的数据库");
}
}

880
ConsoleApp2/Program.cs
View File

@ -1,429 +1,509 @@
using ConsoleApp2;
#define USE_TEST_DB // 测试库的结构与生产库不一样，如果使用测试库运行，则加上USE_TEST_DB预处理器指令
using ConsoleApp2;
using ConsoleApp2.Cache;
using ConsoleApp2.Const;
using ConsoleApp2.Helpers;
using ConsoleApp2.HostedServices;
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
using Microsoft.Extensions.Caching.StackExchangeRedis;
using ConsoleApp2.Services.ErrorRecorder;
using ConsoleApp2.Services.ETL;
using ConsoleApp2.Services.Loggers;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using MySqlConnector;
using Serilog;
using Microsoft.Extensions.Caching.Distributed;
using Serilog.Events;
await RunProgram();
return;
async Task RunProgram()
{
//var inputDir= "D:\\MyDumper";
//ValidateConsole.ValidateInput<string>((_inputDir) =>
//{
// if (Directory.Exists(_inputDir))
// {
// inputDir = _inputDir;
// return true;
// }
// else return false;
//}, "请输入读取csv文件的目录(默认为当前目录下MyDumper文件夹):");
//var maxTask = 16;
//ValidateConsole.ValidateInput<string>((_inputDir) =>
//{
// _ = int.TryParse(_inputDir.ToString(), out var _taskCount);
// if (_taskCount > 0) {
// maxTask = _taskCount;
// return true;
// }
// else return false;
//}, "请输入执行输出的线程数量(默认为16):");
//var flushCount = 2_0000;
//ValidateConsole.ValidateInput<string>((_inputDir) =>
//{
// _ = int.TryParse(_inputDir.ToString(), out var _flashCount);
// if (_flashCount > 0)
// {
// flushCount = _flashCount;
// return true;
// } else return false;
//}, "请输入单次插入的行数(默认为20000):");
ThreadPool.SetMaxThreads(200, 200);
var host = Host.CreateApplicationBuilder(args);
host.Configuration.AddJsonFile(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "appsettings.json"), false, false);
var commandOptions = host.Configuration.GetSection("CmdOptions").Get<CommandOptions>() ??
throw new ApplicationException("未发现配置文件");
Console.WriteLine($"InputDir:{commandOptions?.InputDir}");
var oldestTime = DateTime.ParseExact(commandOptions.OldestTime, "yyyyMM", System.Globalization.DateTimeFormatInfo.InvariantInfo);
//host.Services.Configure<InputTableOptions>(option =>
//{
// option.TableInfoConfig = new Dictionary<string, TableInfo>
// {
// //order_block_plan_item从order_item表查询，然后程序插入
// //order_package_item从order_item表查询，然后程序插入
// //order_patch_detail生产没有这个表，不处理
// {"machine",new TableInfo{SimulaRowCount=14655 }},
// {"order",new TableInfo{SimulaRowCount=5019216 }},
// {"order_block_plan",new TableInfo{SimulaRowCount=2725553 }},//CreateTime < 202301的删除
// {"order_block_plan_result",new TableInfo{SimulaRowCount=1174096 }},
// {"order_box_block",new TableInfo{SimulaRowCount=29755672 }},
// {"order_data_block",new TableInfo{SimulaRowCount=731800334 }},
// {"order_data_goods",new TableInfo{SimulaRowCount=25803671 }},
// {"order_data_parts",new TableInfo{SimulaRowCount=468517543 }},
// {"order_item",new TableInfo{SimulaRowCount=1345520079 }},
// {"order_module",new TableInfo{SimulaRowCount=103325385 }},
// {"order_module_extra",new TableInfo{SimulaRowCount=54361321 }},
// {"order_module_item",new TableInfo{SimulaRowCount=69173339 }},
// {"order_package",new TableInfo{SimulaRowCount=16196195 }},
// {"order_process",new TableInfo{SimulaRowCount=3892685 }},//orderNo < 202301的
// {"order_process_step",new TableInfo{SimulaRowCount=8050349 }},//orderNo < 202301的删除
// {"order_process_step_item",new TableInfo{SimulaRowCount=14538058 }},//orderNo < 202301的删除
// {"order_scrap_board",new TableInfo{SimulaRowCount=123998 }},
// {"process_group",new TableInfo{SimulaRowCount=1253 }},
// {"process_info",new TableInfo{SimulaRowCount=7839 }},
// {"process_item_exp",new TableInfo{SimulaRowCount=28 }},
// {"process_schdule_capacity",new TableInfo{SimulaRowCount=39736 }},
// {"process_step_efficiency",new TableInfo{SimulaRowCount=8 }},
// {"report_template",new TableInfo{SimulaRowCount=7337 }},
// {"simple_package",new TableInfo{SimulaRowCount=130436 }},//orderNo < 202301的删除
// {"simple_plan_order",new TableInfo{SimulaRowCount=351470 }},//CreateTime < 202301的删除
// {"sys_config",new TableInfo{SimulaRowCount=2296 }},
// {"work_calendar",new TableInfo{SimulaRowCount=11 }},
// {"work_shift",new TableInfo{SimulaRowCount=59 }},
// {"work_time",new TableInfo{SimulaRowCount=62 }},
// };
//});
host.Services.Configure<CsvOptions>(option =>
host.Configuration.AddCommandLine(args, new Dictionary<string, string>
{
option.Delimiter = ",";
option.QuoteChar = '"';
});
{"-d", "Input:InputDir"},
{"--InputDir", "Input:InputDir"},
{"-s", "Output:ConnectionString"},
{"--ConnectionString", "Output:ConnectionString"},
{"-r", "RedisCache:Configuration"},
{"--Redis", "RedisCache:Configuration"},
{"-i", "RestoreIndex"},
{"--RestoreIndex", "RestoreIndex"}
});
var inputOptions = host.Configuration.GetRequiredSection("Input").Get<DataInputOptions>()
?? throw new ApplicationException("缺少Input配置");
var transformOptions = host.Configuration.GetRequiredSection("Transform").Get<DataTransformOptions>()
?? throw new ApplicationException("缺少Transform配置");
var outputOptions = host.Configuration.GetRequiredSection("Output").Get<DatabaseOutputOptions>()
?? throw new ApplicationException("缺少Output配置");
var redisSection = host.Configuration.GetRequiredSection("RedisCache");
var redisOptions = redisSection.Get<RedisCacheOptions>() ?? throw new ApplicationException("缺少RedisCache配置");
var tenantDbSection = host.Configuration.GetRequiredSection("TenantDb");
var tenantDbOptions = tenantDbSection.Get<TenantDbOptions>() ?? throw new ApplicationException("缺少TenantDb配置");
if(tenantDbOptions.TenantKey is null) throw new ApplicationException("分库配置中缺少分库键");
if(tenantDbOptions.DbList is null) throw new ApplicationException("分库配置中没有发现任何数据库");
host.Services.Configure<TenantDbOptions>(tenantDbSection);
host.Services.Configure<RedisCacheOptions>(redisSection);
var oldestTime = DateTime.ParseExact(inputOptions.CleanDate, "yyyyMM", System.Globalization.DateTimeFormatInfo.InvariantInfo);
var oldestTimeInt = int.Parse(inputOptions.CleanDate);
// 输入配置
host.Services.Configure<DataInputOptions>(options =>
{
options.InputDir = commandOptions.InputDir;
var _csvOptions = new CsvOptions { Delimiter = ",", QuoteChar = '"' };
options.CreateSource = (string tableName) =>
options.InputDir = inputOptions.InputDir ?? throw new ApplicationException("未配置输入目录");
options.UseMock = inputOptions.UseMock;
options.TableMockConfig = inputOptions.TableMockConfig;
options.MockCountMultiplier = inputOptions.MockCountMultiplier;
// 配置文件输入方法
options.FileInputMetaBuilder = fileName =>
{
var source = new ZstSource(commandOptions.InputDir, tableName, _csvOptions.Delimiter, _csvOptions.QuoteChar);
return source;
if (fileName.EndsWith(".dat.zst"))
{
var tableName = DumpDataHelper.GetTableNameFromCsvFileName(
Path.GetFileNameWithoutExtension(fileName)); // 去除.zst
string[]? headers;
try
{
// 查找同目录下同表的SQL文件
var sqlFile = Directory.GetFiles(options.InputDir)
.SingleOrDefault(f => f.Equals(fileName.Replace(".dat.zst",".sql.zst")));
if (sqlFile is null)
return null;
headers = DumpDataHelper.GetCsvHeadersFromSqlFile(
DumpDataHelper.DecompressZstAsStringAsync(File.OpenRead(sqlFile)).Result);
}
catch (InvalidOperationException e)
{
throw new ApplicationException($"目录下不止一个{tableName}表的SQL文件", e);
}
return new FileInputInfo
{
FileName = fileName,
TableName = tableName,
Headers = headers
};
}
return null;
};
options.TableOrder =
[
TableNames.Machine,
TableNames.Order,
TableNames.OrderBoxBlock, // 依赖Order.CompanyID
TableNames.OrderBlockPlan,
TableNames.OrderBlockPlanResult,// 依赖OrderBlockPlan.CompanyID / 删除
TableNames.OrderItem,
TableNames.OrderDataBlock,
TableNames.OrderDataGoods,
TableNames.OrderDataParts,
TableNames.OrderModule,
TableNames.OrderModuleExtra,
TableNames.OrderModuleItem,
TableNames.OrderPackage,
TableNames.OrderPatchDetail,
TableNames.OrderProcess,
TableNames.OrderProcessStep,
TableNames.OrderProcessStepItem,// 依赖OrderProcess.ShardKey / 删除
TableNames.OrderProcessSchedule,
TableNames.OrderScrapBoard,
TableNames.ProcessGroup,
TableNames.ProcessInfo,
TableNames.ProcessItemExp,
TableNames.ProcessScheduleCapacity,
TableNames.ProcessStepEfficiency,
TableNames.ReportTemplate,
TableNames.SimplePackage,
TableNames.SimplePlanOrder,
TableNames.SysConfig,
TableNames.WorkCalendar,
TableNames.WorkShift,
TableNames.WorkTime
];
// options.TableMockConfig = new Dictionary<string, TableMockConfig>
// {
// { TableNames.Machine, new TableMockConfig(true, 14655, ["ID"]) },
// { TableNames.Order, new TableMockConfig(true, 5019216, ["OrderNo"]) },
// { TableNames.OrderDataBlock, new TableMockConfig(true, 731800334, ["ID"]) },
// { TableNames.OrderDataGoods, new TableMockConfig(true, 25803671, ["ID"]) },
// { TableNames.OrderDataParts, new TableMockConfig(true, 468517543, ["ID"]) },
// { TableNames.OrderModule, new TableMockConfig(true, 103325385, ["ID"]) },
// { TableNames.OrderModuleExtra, new TableMockConfig(true, 54361321, ["ID"]) },
// { TableNames.OrderModuleItem, new TableMockConfig(true, 69173339, ["ID"]) },
// { TableNames.OrderPackage, new TableMockConfig(true, 16196195, ["ID"]) },
// { TableNames.OrderProcess, new TableMockConfig(true, 3892685, ["ID"]) },
// { TableNames.OrderProcessStep, new TableMockConfig(true, 8050349, ["ID"]) },
// { TableNames.OrderProcessStepItem, new TableMockConfig(true, 14538058, ["ID"]) },
// { TableNames.OrderScrapBoard, new TableMockConfig(true, 123998, ["ID"]) },
// { TableNames.ProcessGroup, new TableMockConfig(true, 1253, ["ID"]) },
// { TableNames.ProcessInfo, new TableMockConfig(true, 7839, ["ID"]) },
// { TableNames.ProcessItemExp, new TableMockConfig(true, 28, ["ID"]) },
// { TableNames.ProcessScheduleCapacity, new TableMockConfig(true, 39736, ["ID"]) },
// { TableNames.ProcessStepEfficiency, new TableMockConfig(true, 8, ["ID"]) },
// { TableNames.ReportTemplate, new TableMockConfig(true, 7337, ["ID"]) },
// { TableNames.SimplePackage, new TableMockConfig(true, 130436, ["ID"]) },
// { TableNames.SysConfig, new TableMockConfig(true, 2296, ["ID"]) },
// { TableNames.WorkCalendar, new TableMockConfig(true, 11, ["ID"]) },
// { TableNames.WorkShift, new TableMockConfig(true, 59, ["ID"]) },
// { TableNames.WorkTime, new TableMockConfig(true, 62, ["ID"]) }
// };
options.TableMockConfig = new Dictionary<string, TableMockConfig>
{
{ TableNames.Machine, new TableMockConfig(true, 14655, ["ID"]) },
{ TableNames.Order, new TableMockConfig(true, 50192, ["OrderNo"]) },
{ TableNames.OrderDataBlock, new TableMockConfig(true, 7318003, ["ID"]) },
{ TableNames.OrderDataGoods, new TableMockConfig(true, 258036, ["ID"]) },
{ TableNames.OrderDataParts, new TableMockConfig(true, 4685175, ["ID"]) },
{ TableNames.OrderItem, new TableMockConfig(true, 13298896, ["ID"])},
{ TableNames.OrderModule, new TableMockConfig(true, 1033253, ["ID"]) },
{ TableNames.OrderModuleExtra, new TableMockConfig(true, 543613, ["ID"]) },
{ TableNames.OrderModuleItem, new TableMockConfig(true, 691733, ["ID"]) },
{ TableNames.OrderPackage, new TableMockConfig(true, 161961, ["ID"]) },
{ TableNames.OrderProcess, new TableMockConfig(true, 38926, ["ID"]) },
{ TableNames.OrderProcessStep, new TableMockConfig(true, 80503, ["ID"]) },
{ TableNames.OrderProcessStepItem, new TableMockConfig(true, 145380, ["ID"]) },
{ TableNames.OrderScrapBoard, new TableMockConfig(true, 1239, ["ID"]) },
{ TableNames.ProcessGroup, new TableMockConfig(true, 125, ["ID"]) },
{ TableNames.ProcessInfo, new TableMockConfig(true, 783, ["ID"]) },
{ TableNames.ProcessItemExp, new TableMockConfig(true, 28, ["ID"]) },
{ TableNames.ProcessScheduleCapacity, new TableMockConfig(true, 39736, ["ID"]) },
{ TableNames.ProcessStepEfficiency, new TableMockConfig(true, 8, ["ID"]) },
{ TableNames.ReportTemplate, new TableMockConfig(true, 7337, ["ID"]) },
{ TableNames.SimplePackage, new TableMockConfig(true, 130436, ["ID"]) },
{ TableNames.SysConfig, new TableMockConfig(true, 2296, ["Key"]) },
{ TableNames.WorkCalendar, new TableMockConfig(true, 11, ["ID"]) },
{ TableNames.WorkShift, new TableMockConfig(true, 59, ["ID"]) },
{ TableNames.WorkTime, new TableMockConfig(true, 62, ["ID"]) }
};
});
host.Services.Configure<DataTransformOptions>(options =>
{
var tenantDbOptions = host.Configuration.GetRequiredSection("TenantDb").Get<TenantDbOptions>()
?? throw new ApplicationException("分库配置项不存在");
// 数据缓存键格式为[TableName]-[ColumnName@ColumnValue]-[CacheColumnName]
static string BuildCacheKey(string tableName, string columnName, string columnValue, string cacheColumnName)
=> $"{tableName}-{columnName}@{columnValue}-{cacheColumnName}";
static string CalculateShardKeyByOrderNo(ReadOnlySpan<char> orderNo)
=> $"{orderNo[2..6]}0";
options.StrictMode = transformOptions.StrictMode;
options.EnableFilter = transformOptions.EnableFilter;
options.EnableReplacer = transformOptions.EnableReplacer;
options.EnableReBuilder = transformOptions.EnableReBuilder;
// order_block_plan_item和order_package_item表不导入，根据order_item数据直接重建
// 数据清理
options.RecordFilter = async context =>
{
var record = context.Record;
var cache = context.Cacher;
switch (record.TableName)
{
// OrderBoxBlock删除对应Order.OrderNo不存在的对象
case TableNames.OrderBoxBlock:
{
if (!await cache.ExistsAsync(CacheKeys.Order_OrderNo_CompanyID(record["OrderNo"])))
return false;
break;
}
// OrderBlockPlan删除CreateTime < 202301的
case TableNames.OrderBlockPlan:
{
var time = DateTime.Parse(record["CreateTime"].Trim('"'));
if (time < oldestTime)
return false;
break;
}
// OrderBlockPlanResult删除对应order_block_plan.ID不存在的对象
case TableNames.OrderBlockPlanResult:
{
if (!await cache.ExistsAsync(CacheKeys.OrderBlockPlan_ID_CompanyID(record["ID"])))
return false;
break;
}
// OrderModule删除OrderNo < 202301的
case TableNames.OrderModule:
{
var orderNo = record["OrderNo"];
if(int.Parse(orderNo.AsSpan(0, 6).ToString()) < oldestTimeInt)
return false;
break;
}
// OrderProcess删除OrderNo < 202301的
case TableNames.OrderProcess:
{
var orderNo = record["OrderNo"];
if(int.Parse(orderNo.AsSpan(0, 6).ToString()) < oldestTimeInt)
return false;
break;
}
// OrderProcessStep删除OrderNo < 202301的
case TableNames.OrderProcessStep:
{
var orderNo = record["OrderNo"];
if(int.Parse(orderNo.AsSpan(0, 6).ToString()) < oldestTimeInt)
return false;
break;
}
// OrderProcessStepStep删除对应OrderProcess.ID不存在的对象
case TableNames.OrderProcessStepItem:
{
if (!await cache.ExistsAsync(CacheKeys.OrderProcess_ID_ShardKey(record["OrderProcessID"])))
return false;
break;
}
// SimplePackage删除OrderNo < 202301的
case TableNames.SimplePackage:
{
var orderNo = record["OrderNo"];
if(int.Parse(orderNo.AsSpan(0, 6).ToString()) < oldestTimeInt)
return false;
break;
}
// SimplePlanOrder删除CreateTime < 202301的
case TableNames.SimplePlanOrder:
{
var time = DateTime.Parse(record["CreateTime"].Trim('"'));
if (time < oldestTime)
return false;
break;
}
}
return true;
};
// 数据替换
options.RecordModify = async context =>
{
var record = context.Record;
var cache = context.Cacher;
switch (record.TableName)
{
#if USE_TEST_DB
// Order表移除IsBatch列
case TableNames.Order:
record.RemoveField("IsBatch");
break;
#endif
//OrderBlockPlan将OrderNo长度<2的置空
case TableNames.OrderBlockPlan:
if (record["OrderNos"].Length < 2)
record["OrderNos"] = "";
break;
// OrderBlockPlanResult表添加CompanyID列
case TableNames.OrderBlockPlanResult:
record.AddField("CompanyID",
// 获取OrderBlockPlan.ID -> CompanyID
ThrowIfNoCached(await cache.GetStringAsync(CacheKeys.OrderBlockPlan_ID_CompanyID(record["ID"])),
TableNames.OrderBlockPlanResult, TableNames.OrderBlockPlan, "ID", "脏数据未处理"));
break;
// OrderBoxBlock添加CompanyID列
case TableNames.OrderBoxBlock:
record.AddField("CompanyID",
// 获取Order.OrderNo -> CompanyID
ThrowIfNoCached(await cache.GetStringAsync(CacheKeys.Order_OrderNo_CompanyID(record["OrderNo"])),
TableNames.OrderBoxBlock, TableNames.Order, "OrderNo", "脏数据未处理"));
break;
// OrderModule添加ShardKey列，移除ViewFileName列
case TableNames.OrderModule:
record.AddField("ShardKey", CalculateShardKeyByOrderNo(record["OrderNo"]));
record.RemoveField("ViewFileName");
break;
// OrderProcess添加ShardKey列，NextStepID的空值转换为0
case TableNames.OrderProcess:
record.AddField("ShardKey", CalculateShardKeyByOrderNo(record["OrderNo"]));
#if USE_TEST_DB
if(record["NextStepID"] == "\\N")
record["NextStepID"] = "0";
#endif
break;
// OrderProcessStep，OrderProcessStepItem添加ShardKey列
case TableNames.OrderProcessStep:
record.AddField("ShardKey", CalculateShardKeyByOrderNo(record["OrderNo"]));
break;
case TableNames.OrderProcessStepItem:
record.AddField("ShardKey",
// 获取OrderProcess.ID -> ShardKey
ThrowIfNoCached(await cache.GetStringAsync(CacheKeys.OrderProcess_ID_ShardKey(record["OrderProcessID"])),
TableNames.OrderProcessStepItem, TableNames.OrderProcessStep, "OrderProcessID", "脏数据未处理"));
break;
case TableNames.SimplePlanOrder:
#if USE_TEST_DB
record.RemoveField("ProcessState");
#endif
record.AddField("Deleted", "0");
break;
}
return record;
string ThrowIfNoCached(string? cached, string tableName, string cachedTableName, string cachedColumn, string appendMessage = "")
{
if (cached is null)
throw new InvalidDataException(
$"{tableName}数据异常，在缓存中未找到对应{cachedTableName}.{cachedColumn}\t{appendMessage}");
return cached;
}
};
// 数据缓存
options.RecordCache = async context =>
{
var record = context.Record;
var cache = context.Cacher;
switch (record.TableName)
{
// 缓存Order.OrderNo -> CompanyID
case TableNames.Order:
CacheKeys.Order_OrderNo_CompanyID = orderNo =>
BuildCacheKey(TableNames.Order, "OrderNo", orderNo, "CompanyID");
await cache.SetStringAsync(
CacheKeys.Order_OrderNo_CompanyID(record["OrderNo"]),
record["CompanyID"]);
break;
// 缓存OrderBlockPlan.ID -> CompanyID
case TableNames.OrderBlockPlan:
CacheKeys.OrderBlockPlan_ID_CompanyID = id =>
BuildCacheKey(TableNames.OrderBlockPlan, "ID", id, "CompanyID");
await cache.SetStringAsync(
CacheKeys.OrderBlockPlan_ID_CompanyID(record["ID"]),
record["CompanyID"]);
break;
// 缓存OrderProcess.ID -> ShardKey
case TableNames.OrderProcess:
CacheKeys.OrderProcess_ID_ShardKey = id =>
BuildCacheKey(TableNames.OrderProcess, "ID", id, "ShardKey");
await cache.SetStringAsync(
CacheKeys.OrderProcess_ID_ShardKey(record["ID"]),
record["ShardKey"]);
break;
}
};
// 数据库过滤
options.DatabaseFilter = record =>
{
var companyId = int.Parse(record[tenantDbOptions.TenantKey]); // 每个实体都应存在CompanyID，否则异常
return tenantDbOptions.GetDbNameByTenantKeyValue(companyId);
};
options.TransformBinary = field => commandOptions != null && commandOptions.Isutf8mb4 ? $"_utf8mb4 0x{field}" : $"0x{field}";
//数据过滤
options.RecordFilter = async (record, cache) =>
{
//var index = Array.IndexOf(record.Headers, "ShardKey");
if (record.TryGetField("ShardKey", out var skStr))
{
short.TryParse(skStr, out var sk);
if (sk < commandOptions.OldestShardKey) return false;
}
if (record.TryGetField("CreateTime", out var createTime))
{
_ = DateTime.TryParse(createTime.Replace("\"", ""), out var time);
if (time < oldestTime) return false;
}
if (record.TryGetField("OrderNo", out var orderNo))
{
try
{
var yearMonth = orderNo.Substring(0, 6);
var dt = DateTime.ParseExact(yearMonth, "yyyyMM", System.Globalization.DateTimeFormatInfo.InvariantInfo);
if (dt < oldestTime) return false;
}
catch (Exception ex)
{
return false;//订单号转换失败，跳过
}
}
if (record.TableName == "order_package")
{
if (record.TryGetField("PakageNo", out var pkNo))
{
if (pkNo.Length <= 2) return false;
}
}
if (record.TableName == "order_block_plan")
{
if (record.TryGetField("OrderNos", out var nos))
{
if (nos.Length <= 2) return false;
}
}
if (record.TableName == "order_process_step" || record.TableName == "order_process_step_item")
{
//如果缓存中不存在OrderProcessID,则丢弃
//if(record.TryGetField("OrderProcessID",out var orderProcessID))
var value = await cache.GetStringAsync(record.GetCacheKey("OrderProcessID"));
if (string.IsNullOrEmpty(value))return false;
}
if (record.TableName == "order_block_plan_result" )
{
//如果缓存中不存在ID,则丢弃(ID 对应order_block_plan中的ID)
var value = await cache.GetStringAsync(record.GetCacheKey("ID"));
if (string.IsNullOrEmpty(value)) return false;
}
return true;
};
//数据修改
options.RecordModify = (record) =>
{
if (record.TableName == "order_block_plan")
{
if (record.TryGetField("OrderNos", out var nos))
{
if (nos.Length <= 2) record.SetField("OrderNos", "");
}
}
if (record.TableName == "order_process")//修改order_process.NextStepID的默认值为0
{
if (record.TryGetField("NextStepID", out var idStr))
{
if (idStr == "\\N")
{
record.SetField("NextStepID", "0");
}
}
}
};
//数据缓存
options.RecordCache = async (record, cache) =>
{
if (record.TableName == "order")
{
if (record.TryGetField("OrderNo", out var orderNo))
{
if (record.TryGetField("CompanyID", out var companyid))
{
await cache.SetStringAsync(record.GetCacheKey("OrderNo"), companyid);
}
}
}
if (record.TableName == "order_process")
{
if (record.TryGetField("OrderNo", out var orderNo))
{
var yearMonth = orderNo.Substring(2, 4);
var sk = yearMonth + "0";
if( record.TryGetField("ID", out var id))
{
await cache.SetStringAsync(record.GetCacheKey("ID"), sk);
}
}
}
if (record.TableName == "order_block_plan")
{
if (record.TryGetField("CompanyID", out var companyid))
{
record.TryGetField("ID", out var id);
await cache.SetStringAsync(record.GetCacheKey("ID"), companyid);
}
}
};
//数据替换
options.RecordReplace = async (record, cache) =>
{
//删除数据源里simple_plan_order.ProcessState 字段和值
if (record.TableName == "simple_plan_order")//修改order_process.NextStepID的默认值为0
{
var nextStepIdIndex = Array.IndexOf(record.Headers, "ProcessState");
if (nextStepIdIndex > -1)
{
var headers = record.Headers.Where(t => t != "ProcessState").ToArray();
var fs = record.Fields.ToList();
fs.RemoveAt(nextStepIdIndex);
var fields = fs.ToArray();
return new DataRecord(fields, record.TableName, headers, record.CompanyID);
}
}
if (record.TableName == "order")//修改order_process.NextStepID的默认值为0
{
var nextStepIdIndex = Array.IndexOf(record.Headers, "IsBatch");
if (nextStepIdIndex > -1)
{
var headers = record.Headers.Where(t => t != "IsBatch").ToArray();
var fs = record.Fields.ToList();
fs.RemoveAt(nextStepIdIndex);
var fields = fs.ToArray();
return new DataRecord(fields, record.TableName, headers, record.CompanyID);
}
}
if (record.TableName == "order_block_plan_result")//修改order_process.NextStepID的默认值为0
{
if (record.TryGetField("ID", out var id))
{
var headers = new List<string>(record.Headers);
var fields =new List<string>(record.Fields);
headers.Add("CompanyID");
var companyidResult =await cache.GetStringAsync(record.GetCacheKey("ID"));
_ = int.TryParse(companyidResult, out var companyid);
fields.Add(companyid.ToString());
return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), companyid);
}
}
if(record.TableName == "order_box_block")
{
if (!record.TryGetField("CompanyID", out var companyid))
{
if (record.TryGetField("OrderNo", out var orderNo))
{
var headers = new List<string>(record.Headers);
var fields = new List<string>(record.Fields);
headers.Add("CompanyID");
var companyidResult = await cache.GetStringAsync(record.GetCacheKey("OrderNo"));
_ = int.TryParse(companyidResult, out var cpid);
fields.Add(cpid.ToString());
return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), cpid);
}
}
}
if (record.TableName == "order_module")
{
if (record.TryGetField("ViewFileName",out var value))
{
var index=Array.IndexOf(record.Headers, "ViewFileName");
var headers = new List<string>(record.Headers);
headers.RemoveAt(index);
var fields = new List<string>(record.Fields);
fields.RemoveAt(index);
return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), record.CompanyID);
}
if (!record.TryGetField("ShardKey", out var skStr))
{
if (record.TryGetField("OrderNo", out var orderNo))
{
var yearMonth = orderNo.Substring(2, 4);
var sk = yearMonth + "0";
var headers = new List<string>(record.Headers);
var fields = new List<string>(record.Fields);
headers.Add("ShardKey");
fields.Add(sk);
return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), record.CompanyID);
}
}
}
if (record.TableName == "order_process")
{
if (!record.TryGetField("ShardKey", out var skStr))
{
if(record.TryGetField("OrderNo", out var orderNo))
{
var yearMonth = orderNo.Substring(2, 4);
var sk = yearMonth + "0";
var headers = new List<string>(record.Headers);
var fields = new List<string>(record.Fields);
headers.Add("ShardKey");
fields.Add(sk);
return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), record.CompanyID);
}
}
}
if(record.TableName == "order_process_step"|| record.TableName == "order_process_step_item")
{
if (!record.TryGetField("ShardKey",out var sk))
{
if (record.TryGetField("OrderProcessID",out var processID))
{
var shardKey =await cache.GetStringAsync(record.GetCacheKey("OrderProcessID"));
var headers = new List<string>(record.Headers);
var fields = new List<string>(record.Fields);
headers.Add("ShardKey");
fields.Add(shardKey??"0");
return new DataRecord(fields.ToArray(), record.TableName, headers.ToArray(), record.CompanyID);
}
}
}
// 旧数据库order_data_block.CompanyID数据有误，重新计算
if (record.TableName == "order_data_block")
{
var companyID = await cache.GetStringAsync($"order_{record["OrderNo"]}");
if (string.IsNullOrEmpty(companyID))
// throw new InvalidDataException($"没有获取到OrderNo={record["OrderNo"]}的CompanyID");
return record;
record["CompanyID"] = companyID;
return record;
}
return null;
};
//数据生成
options.RecordAdd = (record) =>
// 数据重建
options.RecordReBuild = context =>
{
var record = context.Record;
var resultList = new List<DataRecord>();
if (record.TableName == "order_item")
// 分流OrderItem表
if (record.TableName == TableNames.OrderItem)
{
record.TryGetField("ID", out var itemID);
record.TryGetField("ID", out var itemId);
record.TryGetField("ShardKey", out var shardKey);
record.TryGetField("PlanID", out var planID);
record.TryGetField("PackageID", out var packageID);
record.TryGetField("CompanyID", out var companyID);
_=int.TryParse(planID, out var pid);
record.TryGetField("PlanID", out var planId);
record.TryGetField("PackageID", out var packageId);
record.TryGetField("CompanyID", out var companyId);
if(!int.TryParse(planId, out var pid))
throw new ApplicationException($"数据发生异常：OrderItem.PlanID，值: {planId}");
if (pid > 0)
{
resultList.Add(new DataRecord(new[] { itemID, shardKey, planID, companyID },
"order_block_plan_item",
new[] { "ItemID", "ShardKey", "PlanID", "CompanyID" }));
resultList.Add(new DataRecord(new[] { itemId, shardKey, planId, companyId },
TableNames.OrderBlockPlanItem,
["ItemID", "ShardKey", "PlanID", "CompanyID"]
));
}
_ = int.TryParse(packageID, out var pkid);
if(!int.TryParse(packageId, out var pkid))
throw new ApplicationException($"数据发生异常：OrderItem.PackageID，值: {packageId}");
if(pkid > 0)
{
resultList.Add(new DataRecord(new[] { itemID, shardKey, packageID, companyID },
"order_package_item",
new[] { "ItemID", "ShardKey", "PackageID", "CompanyID" }
));
resultList.Add(new DataRecord(new[] { itemId, shardKey, packageId, companyId },
TableNames.OrderPackageItem,
[ "ItemID", "ShardKey", "PackageID", "CompanyID" ]
));
}
}
return resultList;
};
options.ColumnTypeConfig = new()
});
host.Services.Configure<DatabaseOutputOptions>(options =>
{
options.ConnectionString = new MySqlConnectionStringBuilder(outputOptions.ConnectionString ?? throw new ApplicationException("未配置数据库连接字符串"))
{
CharacterSet = "utf8mb4",
AllowUserVariables = true,
IgnoreCommandTransaction = true,
TreatTinyAsBoolean = false,
ConnectionTimeout = 60,
DefaultCommandTimeout = 0,
SslMode = MySqlSslMode.None,
}.ConnectionString;
options.FlushCount = outputOptions.FlushCount;
options.MaxAllowedPacket = outputOptions.MaxAllowedPacket;
options.MaxDatabaseOutputTask = outputOptions.MaxDatabaseOutputTask;
options.TreatJsonAsHex = outputOptions.TreatJsonAsHex;
#if USE_TEST_DB
// Test Server
options.ColumnTypeConfig = new Dictionary<string, ColumnType>
{
{ "simple_plan_order.PlaceData", ColumnType.Blob },
{ "order_block_plan_result.PlaceData", ColumnType.Blob },
{ "order_box_block.Data", ColumnType.Blob },
{ "order_data_goods.ExtraProp", ColumnType.Json },
{ "order_module_extra.JsonStr", ColumnType.Text },
{ "process_info.Users", ColumnType.Text },
{ "order_process_schdule.CustomOrderNo", ColumnType.Text },
{ "order_process_schdule.OrderProcessStepName", ColumnType.Text },
{ "order_process_schdule.AreaName", ColumnType.Text },
{ "order_process_schdule.ConsigneeAddress", ColumnType.Text },
{ "order_process_schdule.ConsigneePhone", ColumnType.Text },
{ "report_source.Sql", ColumnType.Text },
{ "report_source.KeyValue", ColumnType.Text },
{ "report_source.Setting", ColumnType.Text },
{ "order_data_block.RemarkJson", ColumnType.Text },
{ "order_patch_detail.BlockDetail", ColumnType.Json },
{ "order_scrap_board.OutLineJson", ColumnType.Text },
{ "simple_package.Items", ColumnType.Json },
{ "order_batch_pack_config.Setting", ColumnType.Text },
{ "machine.Settings", ColumnType.Text },
{ "sys_config.Value", ColumnType.Text },
{ "sys_config.JsonStr", ColumnType.Text },
{ "process_item_exp.ItemJson", ColumnType.Text },
{ "report_template.Template", ColumnType.Text },
{ "report_template.SourceConfig", ColumnType.Text },
{ "order_block_plan.OrderNos", ColumnType.Json },
{ "order_block_plan.BlockInfo", ColumnType.Text },
};
#else
// 配置列类型
// Prod server
options.ColumnTypeConfig = new Dictionary<string, ColumnType>
{
{ "simple_plan_order.PlaceData", ColumnType.Blob },
{ "order_block_plan_result.PlaceData", ColumnType.Blob },
@ -453,26 +533,7 @@ async Task RunProgram()
{ "order_block_plan.OrderNos", ColumnType.Json },
{ "order_block_plan.BlockInfo", ColumnType.Text },
};
});
host.Services.Configure<DatabaseOutputOptions>(options =>
{
//options.ConnectionString = new MySqlConnectionStringBuilder
//{
// Server = "127.0.0.1",
// Port = 33309,
// Database = "cferp_test",
// UserID = "root",
// Password = "123456",
// MaximumPoolSize = 50, // 这个值应当小于 max_connections
//}.ConnectionString;
options.ConnectionString = new MySqlConnectionStringBuilder(host.Configuration.GetConnectionString("MySqlMaster"))
{
CharacterSet = "utf8",
AllowUserVariables = true,
IgnoreCommandTransaction = true,
TreatTinyAsBoolean = false,
MaximumPoolSize = 50
}.ConnectionString;
#endif
});
host.Services.AddLogging(builder =>
@ -480,29 +541,26 @@ async Task RunProgram()
builder.ClearProviders();
builder.AddSerilog(new LoggerConfiguration()
.WriteTo.Console()
.WriteTo.File("./log/error.log", restrictedToMinimumLevel:LogEventLevel.Error)
// .WriteTo.File("./log/info.log", restrictedToMinimumLevel:LogEventLevel.Information) //性能考虑暂不使用
.WriteTo.File($"./Log/Error/{ErrorRecorder.UID}.log", restrictedToMinimumLevel:LogEventLevel.Error)
// .WriteTo.File("./Log/Info/{ErrorRecorder.UID}.log", restrictedToMinimumLevel:LogEventLevel.Information) //性能考虑暂不使用
.CreateLogger()
);
});
host.Services.AddScoped<ProcessContext>();
host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Producer);
host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Consumer);
host.Services.AddTransient<TaskManager>();
host.Services.AddSingleton<ErrorRecorder>();
host.Services.AddDataSourceFactory();
host.Services.AddErrorRecorderFactory();
host.Services.AddSingleton<ProcessContext>();
host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Produce);
host.Services.AddRecordQueuePool(tenantDbOptions.DbList.Keys.Select(key => (key:key, queue:new DataRecordQueue(500_000))).ToArray());
host.Services.AddSingleton<ITaskMonitorLogger, CacheTaskMonitorLogger>();
host.Services.AddSingleton<ITaskMonitorLogger, LoggerTaskMonitorLogger>();
host.Services.AddHostedService<MainHostedService>();
host.Services.AddHostedService<TaskMonitorService>();
host.Services.AddSingleton<IInputService, InputService>();
host.Services.AddSingleton<IInputService, FileInputService>();
host.Services.AddSingleton<ITransformService, TransformService>();
host.Services.AddSingleton<IOutputService, OutputService>();
var redisOptions = host.Configuration.GetSection("RedisCacheOptions").Get<RedisCacheOptions>() ?? new RedisCacheOptions();
host.Services.AddStackExchangeRedisCache(options =>
{
options.Configuration = redisOptions.Configuration;
options.InstanceName = redisOptions.InstanceName;
});
host.Services.AddRedisCache(redisOptions);
var app = host.Build();
await app.RunAsync();
}

49
ConsoleApp2/README.md
View File

@ -1,49 +0,0 @@
## 说明
使用该程序来对MyDumper导出的CSV数据进行读取，转换，然后导出到其他数据库中。
1. 用MyDumper从数据库导出CSV数据
使用MyDumper Docker镜像
```sh
docker run --rm --net=host -v D:/Dump:/home/backup mydumper/mydumper:v0.15.2-6 mydumper `
-h 127.0.0.1 -P 33306 -u root -p 123456 `
-B cferp_test --no-schemas --csv --hex-blob `
-o /home/backup
```
将挂载卷，数据库连接和输出目录替换
不导出数据库结构(--no-schemas)，
导出完的目录下应当包含.sql文件以及.dat文件
2. 在Program.cs中修改`CsvOptions`配置
```cs
host.Services.Configure<CsvOptions>(option =>
{
option.Delimiter = ",";
option.QuoteChar = '"';
option.InputDir = "D:/Dump/Test";
});
```
将`option.InputDir`配置为MyDumper导出的数据目录
3. 在Program.cs中修改`DatabaseOutputOptions`配置
```cs
host.Services.Configure<DatabaseOutputOptions>(options =>
{
options.ConnectionString = new MySqlConnectionStringBuilder
{
Server = "127.0.0.1",
Port = 33306,
Database = "cferp_test_1",
UserID = "root",
Password = "123456",
MaximumPoolSize = 50,
}.ConnectionString;
options.MaxTask = 16;
options.FlushCount = 200;
});
```
将`MySqlConnectionStringBuilder`的属性修改为程序要导出至的数据库
> 后续将这些配置通过命令行传递
4. 运行程序
> 注意，测试数据库`cferp_test`中的`order_process_step`表存在外键，如果要导出到和测试库同结构的数据库，记得先把外键删除。

73
ConsoleApp2/RestoreIndex.sql Normal file
View File

@ -0,0 +1,73 @@
CREATE INDEX `idx_CompanyID` ON `machine` (`CompanyID`);
CREATE INDEX `idx_companyid` ON `order` (`CompanyID`);
CREATE INDEX `idx_CompanyID` ON `order_block_plan` (`CompanyID`);
CREATE INDEX `idx_PlanID` ON `order_block_plan_item` (`PlanID`);
CREATE INDEX `idx_orderno` ON `order_box_block` (`OrderNo`);
CREATE INDEX `index_OrderNo` ON `order_data_block` (`OrderNo`);
CREATE INDEX `index_OrderNo` ON `order_data_goods` (`OrderNo`);
CREATE INDEX `index_OrderNo` ON `order_data_parts` (`OrderNo`);
CREATE INDEX `index_ItemNo` ON `order_item` (`ItemNo`);
CREATE INDEX `index_OrderNo` ON `order_item` (`OrderNo`);
CREATE INDEX `index_PackageID` ON `order_item` (`PackageID`);
CREATE INDEX `index_PlanID` ON `order_item` (`PlanID`);
CREATE INDEX `idx_OrderNo` ON `order_module` (`OrderNo`);
CREATE INDEX `index_OrderNo` ON `order_module_extra` (`OrderNo`);
CREATE INDEX `index_OrderNo` ON `order_module_item` (`OrderNo`);
CREATE INDEX `idx_OrderNo` ON `order_package` (`OrderNo`);
CREATE INDEX `idx_PakageNo` ON `order_package` (`PakageNo`);
CREATE INDEX `idx_PackageID` ON `order_package_item` (`PackageID`);
CREATE INDEX `idx_companyid` ON `order_patch_detail` (`CompanyID`);
CREATE INDEX `idx_OrderNo` ON `order_process` (`OrderNo`);
CREATE INDEX `index_CompanyID` ON `order_process_schdule` (`CompanyID`);
CREATE INDEX `IX_order_process_step_OrderProcessID` ON `order_process_step` (`OrderProcessID`);
CREATE INDEX `idx_OrderProcessID` ON `order_process_step_item` (`OrderProcessID`);
CREATE INDEX `idx_OrderProcessStepID` ON `order_process_step_item` (`OrderProcessStepID`);
CREATE INDEX `idx_CompanyID` ON `order_scrap_board` (`CompanyID`);
CREATE INDEX `idx_CompanyID` ON `process_group` (`CompanyID`);
CREATE INDEX `idx_CompanyID` ON `process_info` (`CompanyID`);
CREATE INDEX `index_CompanyID` ON `process_item_exp` (`CompanyID`);
CREATE INDEX `idx_CompanyID` ON `process_schdule_capacity` (`CompanyID`);
CREATE INDEX `idx_CompanyID` ON `process_step_efficiency` (`CompanyID`);
CREATE INDEX `idx_CompanyID` ON `report_template` (`CompanyID`);
CREATE INDEX `indx_OrderNo` ON `simple_package` (`OrderNo`);
CREATE INDEX `idx_CompanyID` ON `simple_plan_order` (`CompanyID`);
CREATE INDEX `idx_CompanyID` ON `sys_config` (`CompanyID`);
CREATE INDEX `idx` ON `work_calendar` (`CompanyID`);
CREATE INDEX `idx_CompanyID` ON `work_shift` (`CompanyID`);
CREATE INDEX `IX_work_time_ShiftID` ON `work_time` (`ShiftID`);

184
ConsoleApp2/Services/CsvSource.cs
View File

@ -1,184 +0,0 @@
using System.Reflection.PortableExecutable;
using System.Text;
using System.Text.RegularExpressions;
using ConsoleApp2.Helpers;
using ConsoleApp2.HostedServices.Abstractions;
using Microsoft.Extensions.Logging;
namespace ConsoleApp2.Services;
/// <summary>
/// CSV文件读取
/// </summary>
public class CsvSource:IDataSource
{
protected readonly string _inputDir;
//protected readonly StreamReader _reader;
private readonly ILogger? _logger;
protected readonly string _tableName;
protected string? _sqlFilePath;
protected readonly string? _sqlFileText;
protected string[]? headers;
protected string[]? csvFiles;
public string? CurrentRaw { get; protected set; }
public string Delimiter { get; private set; }
public char QuoteChar { get; private set; }
public CsvSource(string inputDir,string tableName,string delimiter = ",", char quoteChar = '"',
ILogger? logger = null)
{
_inputDir = inputDir;
_tableName = tableName;
_logger = logger;
Delimiter = delimiter;
QuoteChar = quoteChar;
string pattern = $"^.*\\.{tableName}\\..*\\.sql$";
_sqlFilePath = Directory.GetFiles(_inputDir).FirstOrDefault(s => Regex.Match(s, pattern).Success);
}
public string[] ParseRow(string row, char quoteChar, string delimiter)
{
var span = row.AsSpan();
var result = new List<string>();
if (span.Length == 0)
throw new ArgumentException("The row is empty", nameof(row));
var isInQuote = span[0] == quoteChar;
var start = 0;
for (var i = 1; i < span.Length; i++)
{
if (span[i] == quoteChar)
{
isInQuote = !isInQuote;
}
// delimiter需要足够复杂
else if (/*!isInQuote && */span.Length > i + delimiter.Length && span[i..(i + delimiter.Length)].Equals(delimiter, StringComparison.CurrentCulture)) // field matched
{
string field;
if (span[start] == quoteChar && span[i - 1] == quoteChar) // enclosed by quoteChar
field = span[(start + 1)..(i - 1)].ToString(); // escape quoteChar
else
field = span[start..i].ToString();
start = i + delimiter.Length;
if (field == "\\N")
field = "NULL";
result.Add(field);
continue;
}
}
result.Add(span[start..].ToString());
for (var i = 0; i < result.Count; i++)
{
var field = result[i];
if (DumpDataHelper.CheckHexField(field) && StringExtensions.CheckJsonHex(field))
{
result[i] = StringExtensions.FromHex(field);
}
}
return result.ToArray();
}
public string[] ParseRow2(ReadOnlySpan<char> source, char quoteChar, string delimiter)
{
var result = new List<string>();
var index = -1;
StringBuilder current = new StringBuilder();
bool hasQuote = false;
bool hasSlash = false;
while (index < source.Length-1)
{
index++;
if (hasSlash == false && source[index] == '\\')
{
hasSlash = true;
current.Append('\\');
continue;
}
if (hasSlash ==false && source[index] == quoteChar)
{
hasQuote = !hasQuote;
current.Append(source[index]);
continue;
}
if (hasQuote==false && source[index] == delimiter[0])
{
result.Add(current.ToString());
current.Clear();
}
else
{
current.Append(source[index]);
}
hasSlash = false;
}
result.Add(current.ToString());
return result.ToArray();
}
public virtual async Task GetHeaderAndCsvFiles()
{
var text = await File.ReadAllTextAsync(_sqlFilePath);
headers = await DumpDataHelper.GetCsvHeadersFromSqlFileAsync(text);
csvFiles = await DumpDataHelper.GetCsvFileNamesFromSqlFileAsync(text, new Regex(@"'.+\.dat.zst'"));
}
public virtual async Task DoEnqueue(Action<DataRecord> action)
{
await GetHeaderAndCsvFiles();
foreach (var file in csvFiles)
{
var filePath= Path.Combine(_inputDir, file);
using (var fs = File.OpenRead(filePath))
{
using (StreamReader sr = new StreamReader(fs))
{
while (!sr.EndOfStream)
{
var line = await sr.ReadLineAsync();
var fields = ParseRow2(line, QuoteChar, Delimiter);
var record = new DataRecord(fields, _tableName, headers);
action?.Invoke(record);
}
}
}
}
}
public virtual async Task<DataRecord?> GetTestRecord()
{
await GetHeaderAndCsvFiles();
var file = csvFiles.FirstOrDefault();
if (file != null)
{
var filePath = Path.Combine(_inputDir, file);
using (var fs = File.OpenRead(filePath))
{
using (StreamReader sr = new StreamReader(fs))
{
var line = await sr.ReadLineAsync();
var fields = ParseRow2(line, QuoteChar, Delimiter);
var record = new DataRecord(fields, _tableName, headers);
return record;
}
}
}
return null;
}
public void Dispose()
{
// _reader.Dispose();
}
}

8
ConsoleApp2/Services/DataRecordQueue.cs
View File

@ -17,9 +17,13 @@ public class DataRecordQueue : IDisposable
public event Action? OnRecordWrite;
public event Action? OnRecordRead;
public DataRecordQueue()
public DataRecordQueue() : this(1000000) // 默认容量最大1M
{
_queue = new BlockingCollection<DataRecord>(2000_000); // 队列最长为20W条记录
}
public DataRecordQueue(int boundedCapacity)
{
_queue = new BlockingCollection<DataRecord>(boundedCapacity);
}
public bool TryDequeue([MaybeNullWhen(false)] out DataRecord record)

106
ConsoleApp2/Services/ETL/CsvReader.cs Normal file
View File

@ -0,0 +1,106 @@
using System.Text;
using ConsoleApp2.HostedServices.Abstractions;
using Microsoft.Extensions.Logging;
namespace ConsoleApp2.Services.ETL;
/// <summary>
/// CSV文件读取
/// </summary>
public class CsvReader : IDataReader
{
protected readonly string? FilePath;
protected readonly Lazy<StreamReader> Reader;
protected readonly ILogger? Logger;
protected readonly string TableName;
public DataRecord Current { get; protected set; } = null!;
public string[] Headers { get; }
public string? CurrentRaw { get; protected set; }
public string Delimiter { get; }
public char QuoteChar { get; }
public CsvReader(Stream stream, string tableName, string[] headers, string delimiter = ",", char quoteChar = '"', ILogger? logger = null)
: this(tableName, headers, delimiter, quoteChar, logger)
{
Reader = new Lazy<StreamReader>(() => new StreamReader(stream));
}
public CsvReader(string filePath, string tableName, string[] headers, string delimiter = ",", char quoteChar = '"', ILogger? logger = null)
: this(tableName, headers, delimiter, quoteChar, logger)
{
var fs = File.OpenRead(filePath);
FilePath = filePath;
Reader = new Lazy<StreamReader>(() => new StreamReader(fs));
}
private CsvReader(string tableName, string[] headers, string delimiter = ",", char quoteChar = '"', ILogger? logger = null)
{
TableName = tableName;
Headers = headers;
Logger = logger;
Delimiter = delimiter;
QuoteChar = quoteChar;
Reader = null!;
}
public virtual async ValueTask<bool> ReadAsync()
{
var str = await Reader.Value.ReadLineAsync();
if (string.IsNullOrWhiteSpace(str))
return false;
CurrentRaw = str;
var fields = ParseRow(str, QuoteChar, Delimiter);
Current = new DataRecord(fields, TableName, Headers){RawField = str};
return true;
}
public string[] ParseRow(ReadOnlySpan<char> source, char quoteChar, string delimiter)
{
var result = new List<string>();
var index = -1;
var current = new StringBuilder();
var hasQuote = false;
var hasSlash = false;
while (index < source.Length - 1)
{
index++;
if (hasSlash == false && source[index] == '\\')
{
hasSlash = true;
current.Append('\\');
continue;
}
if (hasSlash == false && source[index] == quoteChar)
{
hasQuote = !hasQuote;
current.Append(source[index]);
continue;
}
if (hasQuote == false && source[index] == delimiter[0])
{
result.Add(current.ToString());
current.Clear();
}
else
{
current.Append(source[index]);
}
hasSlash = false;
}
result.Add(current.ToString());
return result.ToArray();
}
public virtual void Dispose()
{
if(Reader.IsValueCreated)
Reader.Value.Dispose();
}
}

46
ConsoleApp2/Services/ETL/DataReaderFactory.cs Normal file
View File

@ -0,0 +1,46 @@
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace ConsoleApp2.Services.ETL;
public class DataReaderFactory
{
private readonly ILogger<DataReaderFactory> _logger;
private readonly IOptions<DataInputOptions> _options;
public DataReaderFactory(ILogger<DataReaderFactory> logger, IOptions<DataInputOptions> options)
{
_logger = logger;
_options = options;
}
public IDataReader CreateReader(string filePath, string tableName, string[] headers)
{
if (_options.Value.UseMock)
{
if (_options.Value.TableMockConfig is null)
throw new ApplicationException("未配置表模拟数据量级");
_logger.LogDebug("***** Using {Type} data source *****", "ZSTD mock");
var mockConfig = _options.Value.TableMockConfig.GetValueOrDefault(tableName,
new TableMockConfig { MockCount = 1, UseDeepCopy = false });
mockConfig.MockCount = (long)Math.Ceiling(mockConfig.MockCount * _options.Value.MockCountMultiplier);
return new ZstMockReader(mockConfig, filePath,
tableName, headers, _options.Value.Delimiter, _options.Value.QuoteChar, _logger);
}
_logger.LogDebug("***** Using {Type} data source *****", "ZSTD");
return new ZstReader(filePath, tableName, headers, _options.Value.Delimiter, _options.Value.QuoteChar, _logger);
}
}
public static class DataSourceFactoryExtensions
{
public static IServiceCollection AddDataSourceFactory(this IServiceCollection services)
{
services.AddSingleton<DataReaderFactory>();
return services;
}
}

65
ConsoleApp2/Services/MySqlDestination.cs → ConsoleApp2/Services/ETL/MySqlDestination.cs
View File

@ -1,14 +1,12 @@
using System.Data.Common;
using System.Text;
using System.Text;
using System.Text.RegularExpressions;
using ConsoleApp2.Helpers;
using ConsoleApp2.Options;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MySqlConnector;
using ServiceStack;
namespace ConsoleApp2.Services;
namespace ConsoleApp2.Services.ETL;
/// <summary>
/// Mysql导出
@ -18,29 +16,29 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
private readonly Dictionary<string, IList<DataRecord>> _recordCache;
private readonly MySqlConnection _conn;
private readonly ILogger _logger;
private readonly IOptions<DatabaseOutputOptions> _options;
private readonly ErrorRecorder.OutputErrorRecorder _outputErrorRecorder;
private readonly ProcessContext _context;
private readonly IOptions<DataTransformOptions> _transformOptions;
private readonly ErrorRecorder _errorRecorder;
public MySqlDestination(
string connStr,
ILogger logger,
ProcessContext context,
IOptions<DataTransformOptions> transformOptions,
ErrorRecorder errorRecorder)
IOptions<DatabaseOutputOptions> options,
ErrorRecorder.OutputErrorRecorder outputErrorRecorder,
ProcessContext context)
{
_conn = new MySqlConnection(connStr);
_conn.Open();
_recordCache = new Dictionary<string, IList<DataRecord>>();
_logger = logger;
_options = options;
_outputErrorRecorder = outputErrorRecorder;
_context = context;
_transformOptions = transformOptions;
_errorRecorder = errorRecorder;
}
public Task WriteRecordAsync(DataRecord record)
{
_recordCache.AddOrUpdate(record.TableName, [record], (key, value) =>
_recordCache.AddOrUpdate(record.TableName, [record], (_, value) =>
{
value.Add(record);
return value;
@ -76,22 +74,23 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
}
catch (Exception e)
{
_logger.LogError(e, "Error when flushing records, sql: {Sql}", cmd.CommandText.Omit(1000));
_logger.LogError(e, "插入数据库时发生错误, sql: {Sql}", cmd.CommandText.Omit(1000));
_context.AddException(e);
var match = MatchTableName().Match(cmd.CommandText);
if (match is { Success: true, Groups.Count: > 1 })
{
var tableName = match.Groups[1].Value;
await _errorRecorder.LogErrorSqlAsync(cmd.CommandText, tableName, e);
await _outputErrorRecorder.LogErrorSqlAsync(cmd.CommandText, tableName, e);
}
else await _errorRecorder.LogErrorSqlAsync(cmd.CommandText, e);
else await _outputErrorRecorder.LogErrorSqlAsync(cmd.CommandText, e);
}
}
_recordCache.Clear();
}
catch (Exception e)
{
_logger.LogError(e, "Error when serialize records, record:");
_logger.LogError(e, "序列化记录时发生错误");
throw;
}
finally
{
@ -104,7 +103,7 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
public IEnumerable<string> GetExcuseList(IDictionary<string, IList<DataRecord>> tableRecords,int maxAllowPacket)
{
var sb = new StringBuilder();
var sb = new StringBuilder("SET AUTOCOMMIT = 1;\n");
foreach (var (tableName, records) in tableRecords)
{
if (records.Count == 0)
@ -116,11 +115,11 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
// INSERT INTO ... VALUES >>>
sb.Append($"INSERT INTO `{tableName}`(");
for (var i = 0; i < records[0].Headers.Length; i++)
for (var i = 0; i < records[0].Headers.Count; i++)
{
var header = records[0].Headers[i];
sb.Append($"`{header}`");
if (i != records[0].Headers.Length - 1)
if (i != records[0].Headers.Count - 1)
sb.Append(',');
}
@ -132,7 +131,7 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
var record = records[recordIdx];
var recordSb = new StringBuilder();
recordSb.Append('(');
for (var fieldIdx = 0; fieldIdx < record.Fields.Length; fieldIdx++)
for (var fieldIdx = 0; fieldIdx < record.Fields.Count; fieldIdx++)
{
var field = record.Fields[fieldIdx];
@ -144,12 +143,12 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
goto Escape;
}
switch (_transformOptions.Value.GetColumnType(record.TableName, record.Headers[fieldIdx]))
switch (_options.Value.GetColumnType(record.TableName, record.Headers[fieldIdx]))
{
case ColumnType.Text:
recordSb.Append(string.IsNullOrEmpty(field)
? "''"
: _transformOptions.Value.TransformBinary?.Invoke(field) ?? field);
if(string.IsNullOrEmpty(field))
recordSb.Append("''");
else recordSb.Append($"_utf8mb4 0x{field}");
break;
case ColumnType.Blob:
if (string.IsNullOrEmpty(field))
@ -157,9 +156,11 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
else recordSb.Append($"0x{field}");
break;
case ColumnType.Json:
recordSb.Append(string.IsNullOrEmpty(field)
? "\"[]\""
: _transformOptions.Value.TransformBinary?.Invoke(field) ?? field);
if(string.IsNullOrEmpty(field))
recordSb.Append("'[]'"); // JObject or JArray?
else if (_options.Value.TreatJsonAsHex)
recordSb.Append($"_utf8mb4 0x{field}");
else recordSb.AppendLine(field);
break;
case ColumnType.UnDefine:
default:
@ -170,16 +171,17 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
Escape:
#endregion
if (fieldIdx != record.Fields.Length - 1)
if (fieldIdx != record.Fields.Count - 1)
recordSb.Append(',');
}
recordSb.Append(')');
// 若字符数量即将大于限制，则返回SQL，清空StringBuilder，保留当前记录的索引值，然后转到StartBuild标签重新开始一轮INSERT
if (sb.Length + recordSb.Length + 1 > maxAllowPacket)
if (sb.Length + recordSb.Length + 23 > maxAllowPacket)
{
sb.Append(';');
sb.Append(';').AppendLine();
sb.Append("SET AUTOCOMMIT = 1;");
yield return sb.ToString();
sb.Clear();
goto StartBuild;
@ -192,6 +194,7 @@ public partial class MySqlDestination : IDisposable, IAsyncDisposable
}
sb.Append(';');
sb.Append("COMMIT;");
yield return sb.ToString();
sb.Clear();
}

64
ConsoleApp2/Services/ETL/ZstMockReader.cs Normal file
View File

@ -0,0 +1,64 @@
using ConsoleApp2.Options;
using Microsoft.Extensions.Logging;
namespace ConsoleApp2.Services.ETL;
/// <summary>
/// 截取提供ZST文件中的第一行，然后复制成指定数量的数据
/// </summary>
public class ZstMockReader : ZstReader
{
private long _currentCount;
private readonly long _mockCount;
private DataRecord? _template;
private readonly bool _deepCopy;
private readonly string[]? _autoIncrementColumn;
static readonly IReadOnlyList<int> Range = [500, 1500, 2500];
public ZstMockReader(TableMockConfig mockConfig, string filePath, string tableName, string[] headers, string delimiter = ",", char quoteChar = '\"', ILogger? logger = null) : base(filePath, tableName, headers, delimiter, quoteChar, logger)
{
_mockCount = mockConfig.MockCount;
_deepCopy = mockConfig.UseDeepCopy;
_autoIncrementColumn = mockConfig.AutoIncrementColumn;
}
public ZstMockReader(TableMockConfig mockConfig, Stream stream, string tableName, string[] headers, string delimiter = ",", char quoteChar = '\"', ILogger? logger = null) : base(stream, tableName, headers, delimiter, quoteChar, logger)
{
_mockCount = mockConfig.MockCount;
_deepCopy = mockConfig.UseDeepCopy;
_autoIncrementColumn = mockConfig.AutoIncrementColumn;
}
public override async ValueTask<bool> ReadAsync()
{
if (_template is null)
{
if (!await base.ReadAsync())
throw new InvalidOperationException("所提供的ZST源为空，无法生成模板数据");
_template = Current.Clone() as DataRecord;
if (_template is null)
throw new ApplicationException("记录拷贝失败");
_currentCount++;
return true;
}
if (_deepCopy)
{
Current = _template.Clone() as DataRecord ?? throw new ApplicationException("记录拷贝失败");
if(_autoIncrementColumn is not null)
{
foreach (var column in _autoIncrementColumn)
{
Current[column] = (Convert.ToInt64(Current[column]) + 1).ToString();
_template = Current;
}
}
Current["CompanyID"] = Range[Random.Shared.Next(0, Range.Count)].ToString();//随机CompanyID
}
else Current = _template;
_currentCount++;
return _currentCount < _mockCount;
}
}

48
ConsoleApp2/Services/ETL/ZstReader.cs Normal file
View File

@ -0,0 +1,48 @@
using Microsoft.Extensions.Logging;
using ZstdSharp;
namespace ConsoleApp2.Services.ETL;
/// <summary>
/// 解压ZST文件，从中读取CSV数据
/// </summary>
public class ZstReader : CsvReader
{
protected new readonly Lazy<StreamReader> Reader;
public ZstReader(string filePath, string tableName, string[] headers, string delimiter = ",", char quoteChar = '\"', ILogger? logger = null)
: base(filePath, tableName, headers, delimiter, quoteChar, logger)
{
var ds = new DecompressionStream(File.OpenRead(filePath));
Reader = new Lazy<StreamReader>(() => new StreamReader(ds));
}
public ZstReader(Stream stream, string tableName, string[] headers, string delimiter = ",", char quoteChar = '\"', ILogger? logger = null)
: base(stream, tableName, headers, delimiter, quoteChar, logger)
{
var ds = new DecompressionStream(stream);
Reader = new Lazy<StreamReader>(() => new StreamReader(ds));
}
public override async ValueTask<bool> ReadAsync()
{
var str = await Reader.Value.ReadLineAsync();
if (string.IsNullOrWhiteSpace(str))
return false;
CurrentRaw = str;
var fields = ParseRow(str, QuoteChar, Delimiter);
Current = new DataRecord(fields, TableName, Headers) {RawField = str};
return true;
}
public override void Dispose()
{
base.Dispose();
if(Reader.IsValueCreated)
Reader.Value.Dispose();
}
}

79
ConsoleApp2/Services/ErrorRecorder/ErrorRecorder.cs Normal file
View File

@ -0,0 +1,79 @@
using ConsoleApp2.Helpers;
using Microsoft.Extensions.Logging;
namespace ConsoleApp2.Services.ErrorRecorder;
public class ErrorRecorder
{
protected ILogger Logger;
/// <summary>
/// 当次执行标识
/// </summary>
public static readonly string UID = DateTime.Now.ToString("yyyy-MM-dd HH-mm-ss");
public ErrorRecorder(ILogger logger)
{
Logger = logger;
}
public static async Task LogErrorRecordAsync(string outputDir, DataRecord record, Exception exception)
{
if(!Directory.Exists(outputDir))
Directory.CreateDirectory(outputDir);
var content = $"""
### {exception.Message}
{record.RawField}
""";
var path = Path.Combine(outputDir, $"{record.TableName}.errlog");
await File.AppendAllTextAsync(path, content);
}
public static async Task LogErrorRecordAsync(string outputDir, IEnumerable<DataRecord> records, Exception exception)
{
if(!Directory.Exists(outputDir))
Directory.CreateDirectory(outputDir);
var tableMapping = new Dictionary<string, Tuple<List<DataRecord>, StreamWriter>>();
foreach (var record in records)
{
tableMapping.AddOrUpdate(record.TableName,
Tuple.Create((List<DataRecord>) [record], new StreamWriter(File.OpenRead(record.TableName))),
(_, tuple) =>
{
tuple.Item1.Add(record);
return tuple;
});
}
var maxParallelism = 5;
for (var i = 0; i < tableMapping.Count; i+=maxParallelism)
{
await Parallel.ForEachAsync(tableMapping.Take(maxParallelism), async (pair, token) =>
{
var (records, writer) = pair.Value;
foreach (var record in records)
{
var content =
$"""
### {exception.Message}
{record.RawField}
""";
await writer.WriteLineAsync(content);
if (token.IsCancellationRequested)
break;
}
await writer.DisposeAsync();
});
}
}
public void ClearErrorRecords(string dir)
{
Logger.LogInformation("***** Clear error records *****");
foreach (var file in Directory.GetFiles(dir, "*.errlog", SearchOption.AllDirectories))
{
File.Delete(file);
}
}
}

27
ConsoleApp2/Services/ErrorRecorder/ErrorRecorderFactory.cs Normal file
View File

@ -0,0 +1,27 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
namespace ConsoleApp2.Services.ErrorRecorder;
public class ErrorRecorderFactory
{
private readonly ILogger<ErrorRecorderFactory> _logger;
public ErrorRecorderFactory(ILogger<ErrorRecorderFactory> logger)
{
_logger = logger;
}
public OutputErrorRecorder CreateOutput(string database) => new(database, _logger);
public TransformErrorRecorder CreateTransform() => new(_logger);
public InputErrorRecorder CreateInput() => new(_logger);
}
public static class ErrorRecorderFactoryExtensions
{
public static IServiceCollection AddErrorRecorderFactory(this IServiceCollection services)
{
services.AddSingleton<ErrorRecorderFactory>();
return services;
}
}

19
ConsoleApp2/Services/ErrorRecorder/InputErrorRecorder.cs Normal file
View File

@ -0,0 +1,19 @@
using Microsoft.Extensions.Logging;
namespace ConsoleApp2.Services.ErrorRecorder;
public sealed class InputErrorRecorder : ErrorRecorder
{
private readonly string _outputDir =
Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"ErrorRecords/{UID}/Input");
public InputErrorRecorder(ILogger logger) : base(logger)
{
}
public Task LogErrorRecordAsync(DataRecord record, Exception exception) =>
LogErrorRecordAsync(_outputDir, record, exception);
public Task LogErrorRecordAsync(IEnumerable<DataRecord> records, Exception exception) =>
LogErrorRecordAsync(_outputDir, records, exception);
}

60
ConsoleApp2/Services/ErrorRecorder.cs → ConsoleApp2/Services/ErrorRecorder/OutputErrorRecorder.cs
View File

@ -1,27 +1,19 @@
using System.Text;
using Microsoft.Extensions.Logging;
namespace ConsoleApp2.Services;
namespace ConsoleApp2.Services.ErrorRecorder;
public class ErrorRecorder
public sealed class OutputErrorRecorder : ErrorRecorder
{
private readonly string _outputDir = "./ErrorRecords";
private readonly ILogger _logger;
private readonly string _outputDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"/ErrorRecords/{UID}/Output");
private readonly string _database;
private readonly Dictionary<string, int> _logIndex = new();
/// <summary>
/// 当次执行标识
/// </summary>
private static readonly string UID = DateTime.Now.ToString("yyyy-MM-dd HH-mm-ss");
public ErrorRecorder(ILogger<ErrorRecorder> logger)
public OutputErrorRecorder(string database, ILogger logger) : base(logger)
{
_logger = logger;
var dir = Path.Combine(_outputDir, UID);
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
_database = database;
Logger = logger;
}
/// <summary>
@ -32,25 +24,28 @@ public class ErrorRecorder
/// <param name="exception"></param>
public async Task LogErrorSqlAsync(string commandText, string tableName, Exception exception)
{
if (!Directory.Exists(_outputDir))
Directory.CreateDirectory(_outputDir);
if (!_logIndex.TryGetValue(tableName, out var idx))
{
idx = 0;
_logIndex.Add(tableName, idx);
}
var filePath = Path.Combine(_outputDir, UID, $"{tableName}-{idx}.errlog");
var filePath = Path.Combine(_outputDir, $"{tableName}-{idx}.errlog");
if (File.Exists(filePath) && new FileInfo(filePath).Length > 10 * 1024 * 1024)
{
++idx;
_logIndex[tableName] = idx;
filePath = Path.Combine(_outputDir, UID, $"{tableName}-{idx}.errlog");
filePath = Path.Combine(_outputDir, $"{tableName}-{idx}.errlog");
}
var content = $"""
/* [{DateTime.Now:yyyy-MM-dd HH:mm:ss}]
* Error occurred when export table '{tableName}':
* Error occurred when export table '{_database}.{tableName}':
* {exception.Message}
*/
USE `{_database}`;
{commandText}
@ -65,7 +60,9 @@ public class ErrorRecorder
/// <param name="exception"></param>
public async Task LogErrorSqlAsync(string commandText, Exception exception)
{
var filePath = Path.Combine(_outputDir, UID, "UnknownTables.errlog");
if (!Directory.Exists(_outputDir))
Directory.CreateDirectory(_outputDir);
var filePath = Path.Combine(_outputDir, "UnknownTables.errlog");
var content = $"""
/* [{DateTime.Now:yyyy-MM-dd HH:mm:ss}]
* Error occurred when export table with unknown table name:
@ -78,27 +75,4 @@ public class ErrorRecorder
await File.AppendAllTextAsync(filePath, content, Encoding.UTF8);
}
public async Task LogErrorRecordsAsync(IDictionary<string, DataRecord> records, Exception exception)
{
var pathDict = new Dictionary<string, string>();
foreach (var pair in records)
{
if(!pathDict.TryGetValue(pair.Key, out var path))
{
path = Path.Combine(_outputDir, UID, "ErrorRecords", $"{pair.Key}.errlog");
pathDict.Add(pair.Key, path);
}
//
await File.AppendAllTextAsync(path, string.Join(',', pair.Value.Fields));
}
}
public void ClearErrorRecords()
{
_logger.LogInformation("***** Clear error records *****");
foreach (var file in Directory.GetFiles(_outputDir, "*.errlog", SearchOption.AllDirectories))
{
File.Delete(file);
}
}
}

20
ConsoleApp2/Services/ErrorRecorder/TransformErrorRecorder.cs Normal file
View File

@ -0,0 +1,20 @@
using Microsoft.Extensions.Logging;
namespace ConsoleApp2.Services.ErrorRecorder;
public sealed class TransformErrorRecorder : ErrorRecorder
{
private readonly string _outputDir =
Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"ErrorRecords/{UID}/Transform");
public TransformErrorRecorder(ILogger logger) : base(logger)
{
}
public Task LogErrorRecordAsync(DataRecord record, Exception exception) =>
LogErrorRecordAsync(_outputDir, record, exception);
public Task LogErrorRecordAsync(IEnumerable<DataRecord> records, Exception exception) =>
LogErrorRecordAsync(_outputDir, records, exception);
}

44
ConsoleApp2/Services/JsvSource.cs
View File

@ -1,44 +0,0 @@
using ConsoleApp2.Helpers;
using ConsoleApp2.HostedServices.Abstractions;
using Microsoft.Extensions.Logging;
using ServiceStack.Text;
namespace ConsoleApp2.Services;
/// <summary>
/// 读取Jsv格式文件
/// </summary>
[Obsolete]
public class JsvSource:IDataSource
{
private readonly string _inputDir;
private readonly JsvStringSerializer _jsv;
private readonly StreamReader _reader;
// ReSharper disable once PrivateFieldCanBeConvertedToLocalVariable
private readonly ILogger? _logger;
private readonly string _tableName;
public DataRecord Current { get; protected set; } = null!;
public string[]? Headers { get; }
public bool EndOfSource => _reader.EndOfStream;
public JsvSource(string inputDir,string tableName, ILogger? logger = null)
{
_inputDir = inputDir;
_tableName = tableName;
_jsv = new JsvStringSerializer();
// _reader = new StreamReader(filePath);
//Headers = headers;
_logger = logger;
// _logger?.LogInformation("Reading file: {FilePath}", filePath);
//_tableName = DumpDataHelper.GetTableName(filePath);
}
public async Task DoEnqueue(Action<DataRecord> action)
{
}
public void Dispose()
{
_reader.Dispose();
}
}

19
ConsoleApp2/Services/Loggers/CacheTaskMonitorLogger.cs Normal file
View File

@ -0,0 +1,19 @@
using ConsoleApp2.Cache;
namespace ConsoleApp2.Services.Loggers;
public class CacheTaskMonitorLogger : ITaskMonitorLogger
{
private readonly ICacher _cacher;
public CacheTaskMonitorLogger(ICacher cacher)
{
_cacher = cacher;
}
public void LogStatus(string name, IReadOnlyDictionary<string, string> properties, ITaskMonitorLogger.LogLevel logLevel)
{
if(logLevel is ITaskMonitorLogger.LogLevel.Progress)
_cacher.SetHashAsync(name, properties);
}
}

12
ConsoleApp2/Services/Loggers/ITaskMonitorLogger.cs Normal file
View File

@ -0,0 +1,12 @@
namespace ConsoleApp2.Services.Loggers;
public interface ITaskMonitorLogger
{
public enum LogLevel
{
Info,
Debug,
Progress,
}
void LogStatus(string name, IReadOnlyDictionary<string, string> properties, LogLevel logLevel = LogLevel.Info);
}

41
ConsoleApp2/Services/Loggers/LoggerTaskMonitorLogger.cs Normal file
View File

@ -0,0 +1,41 @@
using System.Text;
using Microsoft.Extensions.Logging;
namespace ConsoleApp2.Services.Loggers;
public class LoggerTaskMonitorLogger : ITaskMonitorLogger
{
private readonly ILogger _logger;
public LoggerTaskMonitorLogger(ILogger<LoggerTaskMonitorLogger> logger)
{
_logger = logger;
}
public void LogStatus(string name, IReadOnlyDictionary<string, string> properties, ITaskMonitorLogger.LogLevel logLevel)
{
var sb = new StringBuilder();
sb.Append($"{name}: {{");
sb.AppendJoin(',', properties.Select((pair, i) => $" {pair.Key}: {pair.Value}"));
sb.Append('}');
// var args = new List<string> { name };
// properties.Aggregate(args, (args, pair) =>
// {
// args.Add(pair.Key);
// args.Add(pair.Value);
// return args;
// });
switch (logLevel)
{
case ITaskMonitorLogger.LogLevel.Info:
_logger.LogInformation("{message}", sb.ToString());
break;
case ITaskMonitorLogger.LogLevel.Progress:
case ITaskMonitorLogger.LogLevel.Debug:
_logger.LogDebug("{message}", sb.ToString());
break;
default:
throw new ArgumentOutOfRangeException(nameof(logLevel), logLevel, null);
}
}
}

45
ConsoleApp2/Services/ProcessContext.cs
View File

@ -1,39 +1,49 @@
namespace ConsoleApp2.Services;
using System.Collections.Concurrent;
namespace ConsoleApp2.Services;
/// <summary>
/// 处理上下文类，标识处理进度
/// </summary>
public class ProcessContext
{
private int _inputCount;
private int _transformCount;
private int _outputCount;
private bool _hasException;
private long _inputCount;
private long _transformCount;
private long _outputCount;
private readonly ConcurrentDictionary<string, long> _tableProgress = new();
public bool HasException => _hasException;
public bool IsInputCompleted { get; private set; }
public bool IsTransformCompleted { get; private set; }
public bool IsOutputCompleted { get; private set; }
public int InputCount
public long InputCount
{
get => _inputCount;
private set => _inputCount = value;
set => Interlocked.Exchange(ref _inputCount, value);
}
public int TransformCount
public long TransformCount
{
get => _transformCount;
private set => _transformCount = value;
set => Interlocked.Exchange(ref _transformCount, value);
}
public int OutputCount
public long OutputCount
{
get => _outputCount;
private set => _outputCount = value;
set => Interlocked.Exchange(ref _outputCount, value);
}
// TableName -> Count
public IReadOnlyDictionary<string, long> TableProgress => _tableProgress;
public void CompleteInput() => IsInputCompleted = true;
public void CompleteTransform() => IsTransformCompleted = true;
public void CompleteOutput() => IsOutputCompleted = true;
public bool AddException(Exception e) => _hasException = true;
public void AddInput() => Interlocked.Increment(ref _inputCount);
@ -44,4 +54,17 @@ public class ProcessContext
public void AddOutput() => Interlocked.Increment(ref _outputCount);
public void AddOutput(int count) => Interlocked.Add(ref _outputCount, count);
public void AddTableOutput(string table, int count)
{
_tableProgress.AddOrUpdate(table, count, (k, v) => v + count);
AddOutput(count);
}
public long GetTableOutput(string table)
{
if(!_tableProgress.TryGetValue(table, out var count))
throw new ApplicationException($"未找到表{table}输出记录");
return count;
}
}

65
ConsoleApp2/Services/RecordQueuePool.cs Normal file
View File

@ -0,0 +1,65 @@
using System.Collections.Concurrent;
using Microsoft.Extensions.DependencyInjection;
namespace ConsoleApp2.Services;
public class RecordQueuePool
{
private readonly ConcurrentDictionary<string, DataRecordQueue> _queues = new();
public IReadOnlyDictionary<string, DataRecordQueue> Queues => _queues;
public void AddQueue(string key, int boundedCapacity = 200_0000) => AddQueue(key, new DataRecordQueue(boundedCapacity));
public void AddQueue(string key, DataRecordQueue queue)
{
if (!_queues.TryAdd(key, queue))
throw new InvalidOperationException($"请勿添加重复的队列，队列名: {key}");
}
public void RemoveQueue(string key, bool dispose = true)
{
if (!_queues.Remove(key, out var queue))
throw new InvalidOperationException($"未找到对应的队列，队列名：{key}");
if (dispose) queue.Dispose();
}
public DataRecordQueue GetQueue(string key)
{
return _queues[key];
}
public DataRecordQueue this[string key]
{
get => GetQueue(key);
set => AddQueue(key, value);
}
}
public static class MultiRecordQueueExtensions
{
public static IServiceCollection AddRecordQueuePool(this IServiceCollection services, params string[] keys)
{
var pool = new RecordQueuePool();
foreach (var key in keys)
{
pool.AddQueue(key);
}
services.AddSingleton(pool);
return services;
}
public static IServiceCollection AddRecordQueuePool(this IServiceCollection services,
params (string key, DataRecordQueue queue)[] queues)
{
var pool = new RecordQueuePool();
foreach (var (key, queue) in queues)
{
pool.AddQueue(key, queue);
}
services.AddSingleton(pool);
return services;
}
}

84
ConsoleApp2/Services/TaskManager.cs
View File

@ -1,5 +1,5 @@
using System.Collections.Concurrent;
using Microsoft.Extensions.Logging;
using ApplicationException = System.ApplicationException;
using TaskExtensions = ConsoleApp2.Helpers.TaskExtensions;
namespace ConsoleApp2.Services;
@ -8,42 +8,74 @@ namespace ConsoleApp2.Services;
/// </summary>
public class TaskManager
{
private readonly ConcurrentBag<Task> _tasks;
private readonly ILogger _logger;
private int _runningTaskCount;
public int TaskCount => _tasks.Count;
public int RunningTaskCount => _tasks.Count(task => !task.IsCompleted);
public IReadOnlyCollection<Task> Tasks => _tasks;
public int RunningTaskCount => _runningTaskCount;
public int MaxTaskCount { get; }
public TaskManager(ILogger<TaskManager> logger)
public event Action<Exception>? OnException;
public event Action? OnTaskCompleteSuccessfully;
public TaskManager(int maxTaskCount)
{
_tasks = new ConcurrentBag<Task>();
_logger = logger;
MaxTaskCount = maxTaskCount;
}
public void CreateTask(Func<Task> func, CancellationToken cancellationToken = default)
public async ValueTask<Task> CreateTaskAsync(Func<Task> func, CancellationToken cancellationToken = default)
{
var task = Task.Run(func, cancellationToken);
_tasks.Add(task);
_logger.LogDebug("New task created");
await TaskExtensions.WaitUntil(() => _runningTaskCount < MaxTaskCount, 25, cancellationToken);
return RunTask(func, cancellationToken);
}
public async ValueTask<Task> CreateTaskAsync(Func<object?, Task> func, object? arg, CancellationToken ct = default)
{
await TaskExtensions.WaitUntil(() => _runningTaskCount < MaxTaskCount, 25, ct);
return RunTaskNoClosure(func, arg, ct);
}
public void CreateTasks(Func<Task> func,int taskCount, CancellationToken cancellationToken = default)
private Task RunTask(Func<Task> func, CancellationToken cancellationToken = default)
{
for (int i = 0; i < taskCount; i++)
var task = Task.Run(async () =>
{
CreateTask(func, cancellationToken);
}
}
public async Task WaitAll()
{
await Task.WhenAll(_tasks);
try
{
await func();
OnTaskCompleteSuccessfully?.Invoke();
}
catch(Exception ex)
{
OnException?.Invoke(ex);
}
finally
{
Interlocked.Decrement(ref _runningTaskCount);
}
}, cancellationToken);
Interlocked.Increment(ref _runningTaskCount);
return task;
}
public void ClearTask()
private Task RunTaskNoClosure(Func<object?, Task> func, object? arg, CancellationToken cancellationToken = default)
{
if(RunningTaskCount != 0)
throw new InvalidOperationException("Unable to clear task. There are still running tasks");
_tasks.Clear();
var task = Task.Factory.StartNew(async obj => // 性能考虑，这个lambda中不要捕获任何外部变量!
{
if (obj is not Tuple<Func<object?, Task>, object?> tuple)
throw new ApplicationException("这个异常不该出现");
try
{
await tuple.Item1(tuple.Item2);
OnTaskCompleteSuccessfully?.Invoke();
}
catch(Exception ex)
{
OnException?.Invoke(ex);
}
finally
{
Interlocked.Decrement(ref _runningTaskCount);
}
}, Tuple.Create(func, arg), cancellationToken).Unwrap();
Interlocked.Increment(ref _runningTaskCount);
return task;
}
}

96
ConsoleApp2/Services/ZstSource.cs
View File

@ -1,96 +0,0 @@
using ConsoleApp2.Helpers;
using Microsoft.Extensions.Logging;
using System.IO;
using System.Text.RegularExpressions;
using ZstdSharp;
namespace ConsoleApp2.Services
{
public class ZstSource : CsvSource
{
public ZstSource(string inputDir, string tableName, string delimiter = ",", char quoteChar = '"',
ILogger? logger = null) : base(inputDir, tableName, delimiter = ",", quoteChar = '"', logger = null)
{
//throw new Exception("aaa");
string pattern = $"^.*\\.{tableName}\\..*\\.sql.zst$";
_sqlFilePath = Directory.GetFiles(_inputDir).FirstOrDefault(s => Regex.Match(s, pattern).Success);
}
private async Task<string> DecompressFile(string filePath)
{
using (var input = File.OpenRead(filePath))
{
using (var decopress = new DecompressionStream(input))
{
var ms = new MemoryStream();
decopress.CopyTo(ms);
ms.Seek(0, SeekOrigin.Begin);
StreamReader reader = new StreamReader(ms);
var text = await reader.ReadToEndAsync();
return text;
}
}
}
public override async Task GetHeaderAndCsvFiles()
{
var text = await DecompressFile(_sqlFilePath);
headers=await DumpDataHelper.GetCsvHeadersFromSqlFileAsync(text);
csvFiles=await DumpDataHelper.GetCsvFileNamesFromSqlFileAsync(text, new Regex(@"'.+\.dat.zst'"));
}
public override async Task DoEnqueue(Action<DataRecord> action)
{
await GetHeaderAndCsvFiles();
foreach (var file in csvFiles)
{
var filePath = Path.Combine(_inputDir, file);
using (var input = File.OpenRead(filePath))
{
using (var decopress = new DecompressionStream(input))
{
using( var reader = new StreamReader(decopress))
{
while (!reader.EndOfStream)
{
var line = await reader.ReadLineAsync();
var fields = ParseRow2(line, QuoteChar, Delimiter);
var record = new DataRecord(fields, _tableName, headers);
action?.Invoke(record);
}
}
}
}
}
}
public override async Task<DataRecord?> GetTestRecord()
{
await GetHeaderAndCsvFiles();
var file = csvFiles.FirstOrDefault();
if (file != null)
{
var filePath = Path.Combine(_inputDir, file);
using (var input = File.OpenRead(filePath))
{
using (var decopress = new DecompressionStream(input))
{
using (var reader = new StreamReader(decopress))
{
var line = await reader.ReadLineAsync();
var fields = ParseRow2(line, QuoteChar, Delimiter);
var record = new DataRecord(fields, _tableName, headers);
return record;
}
}
}
}
return null;
}
public void Dispose()
{
//_reader.Dispose();
}
}
}

2
ConsoleApp2/SimulationService/SimulationInputService.cs
View File

@ -12,6 +12,7 @@ using System.Text.RegularExpressions;
namespace ConsoleApp2.SimulationService
{
#if false
public class SimulationInputService : IInputService
{
private readonly ILogger _logger;
@ -131,4 +132,5 @@ namespace ConsoleApp2.SimulationService
_logger.LogInformation("***** Csv input service completed *****");
}
}
#endif
}

37
ConsoleApp2/appsettings.json
View File

@ -1,17 +1,31 @@
{
"CmdOptions": {
"InputFileType": "CSV",
"InputDir": "D:/MyDumper-ZST",
"TaskCount": 6,
"FlushCount": 10000,
"Isutf8mb4": true,
"OldestShardKey": 23000,
"OldestTime": "202301"
"Logging": {
"LogLevel": {
"Default": "Debug"
}
},
"ConnectionStrings": {
"MySqlMaster": "Server=127.0.0.1;Port=33309;UserId=root;Password=123456;" // 要分库，不用加'Database='了
"Input":{
"InputDir": "D:\\Dump\\MockData",
"CleanDate": "202301",
"StrictMode": false,
"UseMock": true,
"MockCountMultiplier": 0.05
},
"RedisCacheOptions": {
"Transform":{
"StrictMode": true,
"EnableFilter": false,
"EnableReplacer": false,
"EnableReBuilder": false
},
"Output":{
"ConnectionString": "Server=127.0.0.1;Port=3306;UserId=root;Password=cfmes123456;", // 要分库，不用加'Database='了
"MaxAllowedPacket": 67108864,
"FlushCount": 20000,
"MaxDatabaseOutputTask" : 4,
"TreatJsonAsHex": false,
"RestoreIndex": true
},
"RedisCache": {
"Configuration": "192.168.1.246:6380",
"InstanceName" : "mes-etl:"
},
@ -28,6 +42,7 @@
"cferp_test_1": 1000,
"cferp_test_2": 2000,
"cferp_test_3": 2147483647
// "cferp_void_1": 2147483647
}
}
}

13
TestProject1/UnitTest1.cs → TestProject1/DataFix.cs
View File

@ -1,18 +1,19 @@
using System.Data;
using ConsoleApp2.Helpers;
using ConsoleApp2.Helpers.Database;
using MySqlConnector;
using Xunit.Abstractions;
namespace TestProject1;
public class UnitTest1
public class DataFix
{
public const string ConnStr = "Server=192.168.1.245;Port=3306;UserId=root;Password=ruixinjie!@#123;";
public static string[] DbNames = ["cferp_test_1", "cferp_test_2", "cferp_test_3"];
private readonly ITestOutputHelper _output;
public UnitTest1(ITestOutputHelper output)
public DataFix(ITestOutputHelper output)
{
_output = output;
}
@ -26,7 +27,7 @@ public class UnitTest1
[Fact]
public async Task FixOrderBoxBlockCompanyID()
{
var ds = await DatabaseHelper.QueryTable(MakeConnStr(DbNames[0]), "SELECT * FROM `order_box_block` WHERE COMPANYID = 0");
var ds = await DatabaseHelper.QueryTableAsync(MakeConnStr(DbNames[0]), "SELECT * FROM `order_box_block` WHERE COMPANYID = 0");
var dict = new Dictionary<long, int>();//orderNo -> CompanyID
foreach (DataRow row in ds.Tables[0].Rows)
{
@ -43,7 +44,7 @@ public class UnitTest1
{
if(!dict.TryGetValue(orderNo, out var cid)) // 可以提到外面
{
var result = await DatabaseHelper.QueryScalar(ConnStr + $"Database={db};",
var result = await DatabaseHelper.QueryScalarAsync(ConnStr + $"Database={db};",
$"SELECT CompanyID FROM `order` WHERE OrderNo = {orderNo}");
if (result is null or 0) continue;
@ -84,7 +85,7 @@ public class UnitTest1
[InlineData(["order_data_block", "ID", 2])]
public async Task FixCompanyIdWithOwnOrderNo(string tableName, string keyName, int dbNameIndex)
{
var ds = await DatabaseHelper.QueryTable(MakeConnStr(DbNames[dbNameIndex]),
var ds = await DatabaseHelper.QueryTableAsync(MakeConnStr(DbNames[dbNameIndex]),
$"SELECT * FROM `{tableName}` WHERE COMPANYID = 0");
var dict = new Dictionary<long, int>();//orderNo -> CompanyID
foreach (DataRow row in ds.Tables[0].Rows)
@ -134,7 +135,7 @@ public class UnitTest1
[InlineData("order", 2)]
public async Task FixShardKeyWithOwnOrderNo(string tableName, int dbNameIndex)
{
var r = await DatabaseHelper.NonQuery(MakeConnStr(DbNames[dbNameIndex]),
var r = await DatabaseHelper.NonQueryAsync(MakeConnStr(DbNames[dbNameIndex]),
$"UPDATE `{tableName}` SET ShardKey = CONCAT(SUBSTR(`order`.OrderNo,3,4),'0') WHERE ShardKey = 0");
_output.WriteLine($"Affect Rows: {r}");
}

154
TestProject1/DatabaseToolBox.cs Normal file
View File

@ -0,0 +1,154 @@
using System.Data;
using System.Text;
using ConsoleApp2.Helpers.Database;
using MySqlConnector;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Xunit.Abstractions;
namespace TestProject1;
public class DatabaseToolBox
{
private readonly ITestOutputHelper _output;
public const string ConnStr = "Server=127.0.0.1;Port=33309;UserId=root;Password=123456;";
public DatabaseToolBox(ITestOutputHelper output)
{
_output = output;
}
[Theory]
[InlineData("cferp_void_1")]
public async Task AlterAllTableToBlackHole(string database)
{
var connStr = ConnStr + $"Database={database}";
var tables = await DatabaseHelper.QueryTableAsync(connStr,
$"""
SELECT TABLE_NAME FROM information_schema.`TABLES` WHERE TABLE_SCHEMA = '{database}';
""");
foreach (DataRow row in tables.Tables[0].Rows)
{
var tableName = row["TABLE_NAME"].ToString();
var sql = $"""
ALTER TABLE `{tableName}` REMOVE PARTITIONING;
""";
try
{
await DatabaseHelper.NonQueryAsync(connStr, sql);
}
catch (MySqlException e) when (e.ErrorCode == MySqlErrorCode.PartitionManagementOnNoPartitioned)
{
}
sql = $"""
ALTER TABLE `{tableName}` ENGINE=BLACKHOLE;
""";
await DatabaseHelper.NonQueryAsync(connStr, sql);
}
}
public async Task<long> CountAllTable(string connStr, string database)
{
var count = 0L;
var set = await DatabaseHelper.QueryTableAsync(connStr,
$"""
SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = '{database}';
""");
foreach (DataRow row in set.Tables[0].Rows)
{
count += Convert.ToInt64(await DatabaseHelper.QueryScalarAsync(connStr,
$"""
SELECT COUNT(1) FROM `{database}`.`{row[0]}`;
"""));
}
_output.WriteLine($"Record count: {count} records");
return count;
}
[Theory]
[InlineData([new[]{"cferp_test_1", "cferp_test_2", "cferp_test_3"}])]
public async Task CountAllDatabase(IEnumerable<string> databases)
{
var count = 0L;
var connStr = "Server=192.168.1.245;Port=3306;UserId=root;Password=ruixinjie!@#123;";
foreach (var db in databases)
{
count += await CountAllTable(connStr, db);
}
_output.WriteLine(count.ToString());
}
public async Task<TableIndex[]> GetAllTableIndexes(string database)
{
var data = await DatabaseHelper.QueryTableAsync(ConnStr,
$"""
SELECT TABLE_NAME, INDEX_NAME, NON_UNIQUE, COLUMN_NAME, INDEX_TYPE FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = '{database}'
AND INDEX_NAME != 'PRIMARY'
AND INDEX_TYPE = 'BTREE'
""");
return data.Tables[0].Rows.Cast<DataRow>().Select(row =>
{
return new TableIndex(row["TABLE_NAME"].ToString()!,
row["INDEX_NAME"].ToString()!,
!Convert.ToBoolean(row["NON_UNIQUE"]),
row["COLUMN_NAME"].ToString()!,
row["INDEX_TYPE"] switch
{
"BTREE" => TableIndex.TableIndexType.BTree,
"PRIMARY" => TableIndex.TableIndexType.Primary,
"HASH" => TableIndex.TableIndexType.Hash,
_ => throw new ArgumentOutOfRangeException(nameof(row))
}
);
}).ToArray();
}
[Theory]
[InlineData(["cferp_test_1", "D:/Indexes_cferp_test_1.json"])]
[InlineData(["cferp_test_2", "D:/Indexes_cferp_test_2.json"])]
[InlineData(["cferp_test_3", "D:/Indexes_cferp_test_3.json"])]
public async Task ExportAllIndexes(string database, string outputPath)
{
var indexes = await GetAllTableIndexes(database);
var json = JArray.FromObject(indexes);
await File.WriteAllTextAsync(outputPath, json.ToString());
_output.WriteLine($"Exported {indexes.Length} indexes to '{outputPath}'");
}
[Theory]
[InlineData("cferp_test_1", "D:/Indexes_cferp_test_1.json")]
[InlineData("cferp_test_2", "D:/Indexes_cferp_test_2.json")]
[InlineData("cferp_test_3", "D:/Indexes_cferp_test_3.json")]
public async Task ImportAllIndexes(string database, string importPath)
{
var json = await File.ReadAllTextAsync(importPath);
var indexes = JsonConvert.DeserializeObject<TableIndex[]>(json);
var sb = new StringBuilder();
foreach (var (tableName, indexName, isUnique, columnName, tableIndexType) in indexes!)
{
sb.AppendLine($"CREATE {(isUnique ? "UNIQUE" : string.Empty)} INDEX `{indexName}` ON `{database}`.`{tableName}` (`{columnName}`) USING {tableIndexType};");
}
await DatabaseHelper.NonQueryAsync(ConnStr, sb.ToString());
_output.WriteLine($"Import {indexes.Length} indexes to '{database}' from '{importPath}'");
}
[Theory]
[InlineData(["cferp_test_1"])]
[InlineData(["cferp_test_2"])]
[InlineData(["cferp_test_3"])]
public async Task DropAllIndex(string database)
{
var indexes = await GetAllTableIndexes(database);
var sb = new StringBuilder();
foreach (var (tableName, indexName) in indexes)
{
sb.AppendLine($"DROP INDEX `{indexName}` ON `{database}`.`{tableName}`;");
}
await DatabaseHelper.NonQueryAsync(ConnStr, sb.ToString());
_output.WriteLine($"Dropped {indexes.Length} indexes from {database}");
}
}

6
TestProject1/MesDatabaseHelper.cs
View File

@ -1,4 +1,6 @@
namespace TestProject1;
using ConsoleApp2.Helpers.Database;
namespace TestProject1;
public static class MesDatabaseHelper
{
@ -6,7 +8,7 @@ public static class MesDatabaseHelper
{
foreach (var db in dbNames)
{
var result = await DatabaseHelper.QueryScalar(connStr + $"Database={db};", scalarQuery);
var result = await DatabaseHelper.QueryScalarAsync(connStr + $"Database={db};", scalarQuery);
if (result is null or 0) continue;
var companyId = Convert.ToInt32(result);

24
TestProject1/TableIndex.cs Normal file
View File

@ -0,0 +1,24 @@
namespace TestProject1;
public record TableIndex(string TableName, string IndexName, bool IsUnique, string ColumnName, TableIndex.TableIndexType IndexType)
{
public enum TableIndexType
{
BTree,
Hash,
Primary
}
public void Deconstruct(out string tableName, out string indexName)
{
tableName = TableName;
indexName = IndexName;
}
public void Deconstruct(out string tableName, out string indexName, out string columnName)
{
tableName = TableName;
indexName = IndexName;
columnName = ColumnName;
}
}