This commit is contained in:
2024-01-29 09:29:16 +08:00
parent 4f96b77e55
commit 083090c62b
63 changed files with 2479 additions and 1491 deletions

View File

@@ -1,21 +0,0 @@
using System;
using System.ComponentModel;
using System.Configuration;
namespace ConsoleApp2.Options
{
public class CommandOptions
{
public string InputDir { get; set; } = "./MyDumper";
public int TaskCount { get; set; } = 16;
public int FlushCount { get; set; } = 20000;
public bool Isutf8mb4 { get; set; } = true;
public short OldestShardKey { get; set; } = 23010;
public string OldestTime { get; set; } = "202301";
}
}

View File

@@ -1,19 +0,0 @@
namespace ConsoleApp2.Options;
public class CsvOptions
{
/// <summary>
/// MyDumper导出的CSV文件目录
/// </summary>
//public string InputDir { get; set; } = "./";
/// <summary>
/// 字符串的包围符号,默认为双引号"
/// </summary>
public char QuoteChar { get; set; } = '"';
/// <summary>
/// 每个字段的分割符,默认逗号,
/// </summary>
public string Delimiter { get; set; } = ",";
}

View File

@@ -1,18 +1,53 @@
using ConsoleApp2.Services;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection.PortableExecutable;
using System.Text;
using System.Threading.Tasks;
using ConsoleApp2.HostedServices;
namespace ConsoleApp2.Options
{
public enum InputFileType { CSV, JWT, JSV }
public class DataInputOptions
{
public string InputDir { get; set; } = "./";
public string? InputDir { get; set; }
public Func<string, CsvSource>? CreateSource { get; set; }
#region CSV
/// <summary>
/// 字符串的包围符号,默认为双引号"
/// </summary>
public char QuoteChar { get; set; } = '"';
/// <summary>
/// 每个字段的分割符,默认逗号,
/// </summary>
public string Delimiter { get; set; } = ",";
#endregion
/// <summary>
/// yyyyMM
/// </summary>
public string CleanDate { get; set; } = "202301";
#region Mock
public bool UseMock { get; set; }
public double MockCountMultiplier { get; set; } = 1;
/// <summary>
/// Table -> Mock Count 暂时为手动配置
/// </summary>
public Dictionary<string, TableMockConfig>? TableMockConfig { get; set; }
#endregion
#region ManualSet
public string[]? TableOrder { get; set; }
/// <summary>
/// 配置如何从文件名转换为表名和表头
/// </summary>
public Func<string, FileInputInfo?>? FileInputMetaBuilder { get; set; } //TODO: 抽离
#endregion
}
}

View File

@@ -1,5 +1,6 @@
using Microsoft.Extensions.Caching.Distributed;
using StackExchange.Redis;
using ConsoleApp2.Cache;
using ConsoleApp2.HostedServices;
using Microsoft.Extensions.Logging;
namespace ConsoleApp2.Options;
@@ -13,23 +14,34 @@ public enum ColumnType
public class DataTransformOptions
{
public Func<DataRecord, string>? DatabaseFilter { get; set; }
public Func<string, string>? TransformBinary { get; set; }//Binary转字符串方法
public Func<DataRecord, IDistributedCache, Task<bool>>? RecordFilter { get; set; }//数据过滤方法
public Action<DataRecord>? RecordModify { get; set; }//数据修改
public Func<DataRecord, IDistributedCache, Task<DataRecord?>>? RecordReplace { get; set; }//数据替换
public Func<DataRecord, IList<DataRecord>?>? RecordAdd { get; set; }//数据替换
public Func<DataRecord, IDistributedCache, Task>? RecordCache { get; set; }//数据缓存
public bool StrictMode { get; set; } = true;
public bool EnableFilter { get; set; } = true;
public bool EnableReplacer { get; set; } = true;
public bool EnableReBuilder { get; set; } = true;
/// <summary>
/// 配置导入数据的特殊列
/// Record -> Database name
/// 对记录进行数据库过滤
/// </summary>
public Dictionary<string, ColumnType> ColumnTypeConfig { get; set; } = new(); // "table.column" -> type
public ColumnType GetColumnType(string table, string column)
{
return ColumnTypeConfig.GetValueOrDefault($"{table}.{column}", ColumnType.UnDefine);
}
}
public Func<DataRecord, string>? DatabaseFilter { get; set; }
/// <summary>
/// Context -> Should output
/// 配置对数据过滤的条件
/// </summary>
public Func<DataTransformContext, Task<bool>>? RecordFilter { get; set; }//数据过滤方法
/// <summary>
/// Context -> New record
/// 对当前记录进行修改或完整替换
/// </summary>
public Func<DataTransformContext, Task<DataRecord>>? RecordModify { get; set; }//数据替换
/// <summary>
/// Context -> New rebuild records
/// 使用当前记录对某些数据进行重建
/// </summary>
public Func<DataTransformContext, IList<DataRecord>?>? RecordReBuild { get; set; }//新增数据
/// <summary>
/// Context -> void
/// 对数据的某些字段进行缓存
/// </summary>
public Func<DataTransformContext, Task>? RecordCache { get; set; }//数据缓存
}

View File

@@ -2,9 +2,24 @@
public class DatabaseOutputOptions
{
/// <summary>
/// 数据库连接字符串
/// </summary>
public string? ConnectionString { get; set; }
public int MaxAllowedPacket { get; set; } = 64*1024*1024;
public int MaxAllowedPacket { get; set; } = 64 * 1024 * 1024;
public int FlushCount { get; set; } = 10000;
public int MaxDatabaseOutputTask { get; set; } = 4;
public bool TreatJsonAsHex { get; set; } = true;
/// <summary>
/// 配置导入数据的特殊列
/// </summary>
public Dictionary<string, ColumnType> ColumnTypeConfig { get; set; } = new(); // "table.column" -> type
public ColumnType GetColumnType(string table, string column)
{
return ColumnTypeConfig.GetValueOrDefault($"{table}.{column}", ColumnType.UnDefine);
}
}

View File

@@ -0,0 +1,8 @@
namespace ConsoleApp2.Options;
public class RedisCacheOptions
{
public string? Configuration { get; init; }
public string InstanceName { get; init; } = "";
public int Database { get; init; } = 0;
}

View File

@@ -0,0 +1,32 @@
namespace ConsoleApp2.Options;
public struct TableMockConfig
{
/// <summary>
/// 使用深拷贝
/// </summary>
public bool UseDeepCopy { get; set; }
/// <summary>
/// 模拟数据量
/// </summary>
public long MockCount { get; set; }
/// <summary>
/// 需要开启MockCount
/// </summary>
public string[]? AutoIncrementColumn { get; set; } = null; // TODO: 换为自定义委托
public void Deconstruct(out bool useDeepCopy, out long mockCount, out string[]? autoIncrementColumn)
{
useDeepCopy = UseDeepCopy;
mockCount = MockCount;
autoIncrementColumn = AutoIncrementColumn;
}
public TableMockConfig(bool useDeepCopy, long mockCount, string[]? autoIncrementColumn)
{
UseDeepCopy = useDeepCopy;
MockCount = mockCount;
AutoIncrementColumn = autoIncrementColumn;
}
}

View File

@@ -1,24 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace ConsoleApp2.Options
{
public class TableInfo
{
public long SimulaRowCount { get; set; }//模拟的记录条数
}
public class TasksOptions
{
public Dictionary<string, TableInfo> TableInfoConfig { get; set; } = new();
public int TransformTaskCount { get; set; } = 1;
public OutPutOptions OutPutOptions { get; set; }=new();
}
public class OutPutOptions
{
public int FlushCount { get; set; } = 10000;
public int OutPutTaskCount { get; set; } = 2;
}
}

View File

@@ -2,22 +2,23 @@ namespace ConsoleApp2.Options;
public class TenantDbOptions
{
public string TenantKey { get; set; }
public string? TenantKey { get; set; }
/// <summary>
/// Key-Value: {DbName}-{TenantKeyLessThan}
/// </summary>
public Dictionary<string, int> DbList { get; set; }
public Dictionary<string, int>? DbList { get; set; }
public string GetDbNameByTenantKeyValue(int tenantKeyValue)
{
// var dictionary = new SortedDictionary<int, string>();
// DbList.ForEach(pair => dictionary.Add(pair.Value, pair.Key));
// 注意配置顺序
if(DbList is null) throw new ApplicationException("分库配置中没有发现任何数据库");
var dbName = DbList.Cast<KeyValuePair<string, int>?>()
.FirstOrDefault(pair => pair?.Value != null && pair.Value.Value > tenantKeyValue)!.Value.Key;
return dbName ??
throw new ArgumentOutOfRangeException(nameof(tenantKeyValue),
$"已配置的数据库中没有任何符合'{nameof(tenantKeyValue)}'值的对象");
$"分库配置中没有任何符合'{nameof(tenantKeyValue)}'值的数据库");
}
}