Update
This commit is contained in:
@@ -1,21 +0,0 @@
|
||||
using System;
|
||||
using System.ComponentModel;
|
||||
using System.Configuration;
|
||||
namespace ConsoleApp2.Options
|
||||
{
|
||||
public class CommandOptions
|
||||
{
|
||||
|
||||
public string InputDir { get; set; } = "./MyDumper";
|
||||
|
||||
public int TaskCount { get; set; } = 16;
|
||||
|
||||
public int FlushCount { get; set; } = 20000;
|
||||
|
||||
public bool Isutf8mb4 { get; set; } = true;
|
||||
|
||||
public short OldestShardKey { get; set; } = 23010;
|
||||
public string OldestTime { get; set; } = "202301";
|
||||
|
||||
}
|
||||
}
|
@@ -1,19 +0,0 @@
|
||||
namespace ConsoleApp2.Options;
|
||||
|
||||
public class CsvOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// MyDumper导出的CSV文件目录
|
||||
/// </summary>
|
||||
//public string InputDir { get; set; } = "./";
|
||||
|
||||
/// <summary>
|
||||
/// 字符串的包围符号,默认为双引号"
|
||||
/// </summary>
|
||||
public char QuoteChar { get; set; } = '"';
|
||||
|
||||
/// <summary>
|
||||
/// 每个字段的分割符,默认逗号,
|
||||
/// </summary>
|
||||
public string Delimiter { get; set; } = ",";
|
||||
}
|
@@ -1,18 +1,53 @@
|
||||
using ConsoleApp2.Services;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using ConsoleApp2.HostedServices;
|
||||
|
||||
namespace ConsoleApp2.Options
|
||||
{
|
||||
public enum InputFileType { CSV, JWT, JSV }
|
||||
public class DataInputOptions
|
||||
{
|
||||
public string InputDir { get; set; } = "./";
|
||||
public string? InputDir { get; set; }
|
||||
|
||||
public Func<string, CsvSource>? CreateSource { get; set; }
|
||||
#region CSV
|
||||
|
||||
/// <summary>
|
||||
/// 字符串的包围符号,默认为双引号"
|
||||
/// </summary>
|
||||
public char QuoteChar { get; set; } = '"';
|
||||
|
||||
/// <summary>
|
||||
/// 每个字段的分割符,默认逗号,
|
||||
/// </summary>
|
||||
public string Delimiter { get; set; } = ",";
|
||||
|
||||
#endregion
|
||||
|
||||
/// <summary>
|
||||
/// yyyyMM
|
||||
/// </summary>
|
||||
public string CleanDate { get; set; } = "202301";
|
||||
|
||||
#region Mock
|
||||
|
||||
public bool UseMock { get; set; }
|
||||
|
||||
public double MockCountMultiplier { get; set; } = 1;
|
||||
|
||||
/// <summary>
|
||||
/// Table -> Mock Count 暂时为手动配置
|
||||
/// </summary>
|
||||
public Dictionary<string, TableMockConfig>? TableMockConfig { get; set; }
|
||||
|
||||
#endregion
|
||||
|
||||
|
||||
#region ManualSet
|
||||
|
||||
public string[]? TableOrder { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// 配置如何从文件名转换为表名和表头
|
||||
/// </summary>
|
||||
public Func<string, FileInputInfo?>? FileInputMetaBuilder { get; set; } //TODO: 抽离
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
|
@@ -1,5 +1,6 @@
|
||||
using Microsoft.Extensions.Caching.Distributed;
|
||||
using StackExchange.Redis;
|
||||
using ConsoleApp2.Cache;
|
||||
using ConsoleApp2.HostedServices;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace ConsoleApp2.Options;
|
||||
|
||||
@@ -13,23 +14,34 @@ public enum ColumnType
|
||||
|
||||
public class DataTransformOptions
|
||||
{
|
||||
public Func<DataRecord, string>? DatabaseFilter { get; set; }
|
||||
|
||||
public Func<string, string>? TransformBinary { get; set; }//Binary转字符串方法
|
||||
|
||||
public Func<DataRecord, IDistributedCache, Task<bool>>? RecordFilter { get; set; }//数据过滤方法
|
||||
public Action<DataRecord>? RecordModify { get; set; }//数据修改
|
||||
public Func<DataRecord, IDistributedCache, Task<DataRecord?>>? RecordReplace { get; set; }//数据替换
|
||||
public Func<DataRecord, IList<DataRecord>?>? RecordAdd { get; set; }//数据替换
|
||||
public Func<DataRecord, IDistributedCache, Task>? RecordCache { get; set; }//数据缓存
|
||||
|
||||
public bool StrictMode { get; set; } = true;
|
||||
public bool EnableFilter { get; set; } = true;
|
||||
public bool EnableReplacer { get; set; } = true;
|
||||
public bool EnableReBuilder { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// 配置导入数据的特殊列
|
||||
/// Record -> Database name
|
||||
/// 对记录进行数据库过滤
|
||||
/// </summary>
|
||||
public Dictionary<string, ColumnType> ColumnTypeConfig { get; set; } = new(); // "table.column" -> type
|
||||
|
||||
public ColumnType GetColumnType(string table, string column)
|
||||
{
|
||||
return ColumnTypeConfig.GetValueOrDefault($"{table}.{column}", ColumnType.UnDefine);
|
||||
}
|
||||
}
|
||||
public Func<DataRecord, string>? DatabaseFilter { get; set; }
|
||||
/// <summary>
|
||||
/// Context -> Should output
|
||||
/// 配置对数据过滤的条件
|
||||
/// </summary>
|
||||
public Func<DataTransformContext, Task<bool>>? RecordFilter { get; set; }//数据过滤方法
|
||||
/// <summary>
|
||||
/// Context -> New record
|
||||
/// 对当前记录进行修改或完整替换
|
||||
/// </summary>
|
||||
public Func<DataTransformContext, Task<DataRecord>>? RecordModify { get; set; }//数据替换
|
||||
/// <summary>
|
||||
/// Context -> New rebuild records
|
||||
/// 使用当前记录对某些数据进行重建
|
||||
/// </summary>
|
||||
public Func<DataTransformContext, IList<DataRecord>?>? RecordReBuild { get; set; }//新增数据
|
||||
/// <summary>
|
||||
/// Context -> void
|
||||
/// 对数据的某些字段进行缓存
|
||||
/// </summary>
|
||||
public Func<DataTransformContext, Task>? RecordCache { get; set; }//数据缓存
|
||||
}
|
||||
|
@@ -2,9 +2,24 @@
|
||||
|
||||
public class DatabaseOutputOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// 数据库连接字符串
|
||||
/// </summary>
|
||||
public string? ConnectionString { get; set; }
|
||||
public int MaxAllowedPacket { get; set; } = 64*1024*1024;
|
||||
|
||||
public int MaxAllowedPacket { get; set; } = 64 * 1024 * 1024;
|
||||
|
||||
public int FlushCount { get; set; } = 10000;
|
||||
|
||||
public int MaxDatabaseOutputTask { get; set; } = 4;
|
||||
|
||||
public bool TreatJsonAsHex { get; set; } = true;
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// 配置导入数据的特殊列
|
||||
/// </summary>
|
||||
public Dictionary<string, ColumnType> ColumnTypeConfig { get; set; } = new(); // "table.column" -> type
|
||||
|
||||
public ColumnType GetColumnType(string table, string column)
|
||||
{
|
||||
return ColumnTypeConfig.GetValueOrDefault($"{table}.{column}", ColumnType.UnDefine);
|
||||
}
|
||||
}
|
8
ConsoleApp2/Options/RedisCacheOptions.cs
Normal file
8
ConsoleApp2/Options/RedisCacheOptions.cs
Normal file
@@ -0,0 +1,8 @@
|
||||
namespace ConsoleApp2.Options;
|
||||
|
||||
public class RedisCacheOptions
|
||||
{
|
||||
public string? Configuration { get; init; }
|
||||
public string InstanceName { get; init; } = "";
|
||||
public int Database { get; init; } = 0;
|
||||
}
|
32
ConsoleApp2/Options/TableMockConfig.cs
Normal file
32
ConsoleApp2/Options/TableMockConfig.cs
Normal file
@@ -0,0 +1,32 @@
|
||||
namespace ConsoleApp2.Options;
|
||||
|
||||
public struct TableMockConfig
|
||||
{
|
||||
/// <summary>
|
||||
/// 使用深拷贝
|
||||
/// </summary>
|
||||
public bool UseDeepCopy { get; set; }
|
||||
/// <summary>
|
||||
/// 模拟数据量
|
||||
/// </summary>
|
||||
public long MockCount { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// 需要开启MockCount
|
||||
/// </summary>
|
||||
public string[]? AutoIncrementColumn { get; set; } = null; // TODO: 换为自定义委托
|
||||
|
||||
public void Deconstruct(out bool useDeepCopy, out long mockCount, out string[]? autoIncrementColumn)
|
||||
{
|
||||
useDeepCopy = UseDeepCopy;
|
||||
mockCount = MockCount;
|
||||
autoIncrementColumn = AutoIncrementColumn;
|
||||
}
|
||||
|
||||
public TableMockConfig(bool useDeepCopy, long mockCount, string[]? autoIncrementColumn)
|
||||
{
|
||||
UseDeepCopy = useDeepCopy;
|
||||
MockCount = mockCount;
|
||||
AutoIncrementColumn = autoIncrementColumn;
|
||||
}
|
||||
}
|
@@ -1,24 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace ConsoleApp2.Options
|
||||
{
|
||||
public class TableInfo
|
||||
{
|
||||
public long SimulaRowCount { get; set; }//模拟的记录条数
|
||||
}
|
||||
public class TasksOptions
|
||||
{
|
||||
public Dictionary<string, TableInfo> TableInfoConfig { get; set; } = new();
|
||||
public int TransformTaskCount { get; set; } = 1;
|
||||
public OutPutOptions OutPutOptions { get; set; }=new();
|
||||
}
|
||||
public class OutPutOptions
|
||||
{
|
||||
public int FlushCount { get; set; } = 10000;
|
||||
public int OutPutTaskCount { get; set; } = 2;
|
||||
}
|
||||
}
|
@@ -2,22 +2,23 @@ namespace ConsoleApp2.Options;
|
||||
|
||||
public class TenantDbOptions
|
||||
{
|
||||
public string TenantKey { get; set; }
|
||||
public string? TenantKey { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Key-Value: {DbName}-{TenantKeyLessThan}
|
||||
/// </summary>
|
||||
public Dictionary<string, int> DbList { get; set; }
|
||||
public Dictionary<string, int>? DbList { get; set; }
|
||||
|
||||
public string GetDbNameByTenantKeyValue(int tenantKeyValue)
|
||||
{
|
||||
// var dictionary = new SortedDictionary<int, string>();
|
||||
// DbList.ForEach(pair => dictionary.Add(pair.Value, pair.Key));
|
||||
// 注意配置顺序
|
||||
if(DbList is null) throw new ApplicationException("分库配置中没有发现任何数据库");
|
||||
var dbName = DbList.Cast<KeyValuePair<string, int>?>()
|
||||
.FirstOrDefault(pair => pair?.Value != null && pair.Value.Value > tenantKeyValue)!.Value.Key;
|
||||
return dbName ??
|
||||
throw new ArgumentOutOfRangeException(nameof(tenantKeyValue),
|
||||
$"已配置的数据库中没有任何符合'{nameof(tenantKeyValue)}'值的对象");
|
||||
$"分库配置中没有任何符合'{nameof(tenantKeyValue)}'值的数据库");
|
||||
}
|
||||
}
|
Reference in New Issue
Block a user