CZY/MES-ETL
1
0
Fork 0
Code Pull Requests Releases Activity

2025迁移版本，多项规则修改

Browse Source
This commit is contained in:
陈梓阳
2024-12-10 14:03:09 +08:00
parent dc239c776e
commit 0e28d639c1
34 changed files with 1075 additions and 564 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
MesETL.App/Options/DataInputOptions.cs
View File

@@ -4,6 +4,9 @@ namespace MesETL.App.Options
{
public class DataInputOptions
{
/// <summary>
/// 文件输入的目录
/// </summary>
public string? InputDir { get; set; }
#region CSV
@@ -22,29 +25,46 @@ namespace MesETL.App.Options
#region Mock
/// <summary>
/// <para>生成模拟数据进行测试</para>
/// <para>启用后在读取数据时会截取ZST文件中的CSV文件的第一条记录，然后复制成指定数量的数据</para>
/// </summary>
public bool UseMock { get; set; }
/// <summary>
/// 当开启模拟数据生成时，模拟数据的倍数
/// </summary>
public double MockCountMultiplier { get; set; } = 1;
/// <summary>
/// Table -> Mock Count 暂时为手动配置
/// 配置每张表生成模拟数据的规则，此属性暂时在程序中配置
/// </summary>
public Dictionary<string, TableMockConfig>? TableMockConfig { get; set; }
#endregion
#region ManualSet
#region Reader
/// <summary>
/// <para>配置输入表及其顺序，如果为空则按照程序默认的顺序。</para>
/// <para>该值如果存在，程序会按照集合中表的顺序来读取数据，不在集合中的表将被忽略！</para>
/// </summary>
public string[]? TableOrder { get; set; }
/// <summary>
/// 忽略集合中配置的表，不进行读取
/// </summary>
public string[] TableIgnoreList { get; set; } = [];
/// <summary>
/// 配置如何从文件名转换为表名和表头
/// </summary>
public Func<string, FileInputInfo?>? FileInputMetaBuilder { get; set; } //TODO: 抽离
public Func<string, FileInputInfo?>? FileInputMetaBuilder { get; set; }
/// <summary>
/// 表输入完成事件
/// </summary>
public Action<string>? OnTableInputCompleted { get; set; }
#endregion

10
MesETL.App/Options/DataTransformOptions.cs
View File

@@ -20,26 +20,26 @@ public class DataTransformOptions
/// <summary>
/// yyyyMM
/// </summary>
public string CleanDate { get; set; } = "202301";
public string CleanDate { get; set; } = "202401";
/// <summary>
/// Record -> Database name
/// 对记录进行数据库过滤
/// 决定记录应当被插入到哪一个数据库中
/// </summary>
public Func<DataRecord, string>? DatabaseFilter { get; set; }
/// <summary>
/// Context -> Should output
/// 配置对数据过滤的条件
/// 对记录进行过滤，返回false则不输出
/// </summary>
public Func<DataTransformContext, Task<bool>>? RecordFilter { get; set; }//数据过滤方法
/// <summary>
/// Context -> New record
/// 对当前记录进行修改或完整替换
/// 对当前记录进行修改或完整替换，你可以在这里修改记录中的字段，或者新增/删除字段
/// </summary>
public Func<DataTransformContext, Task<DataRecord>>? RecordModify { get; set; }//数据替换
/// <summary>
/// Context -> New rebuild records
/// 使用当前记录对某些数据进行重建
/// 基于当前记录新增多个记录
/// </summary>
public Func<DataTransformContext, IList<DataRecord>?>? RecordReBuild { get; set; }//新增数据
/// <summary>

43
MesETL.App/Options/DatabaseOutputOptions.cs
View File

@@ -1,29 +1,66 @@
namespace MesETL.App.Options;
using MesETL.App.HostedServices;
namespace MesETL.App.Options;
public class DatabaseOutputOptions
{
/// <summary>
/// 输出数据库的连接字符串
/// </summary>
public string? ConnectionString { get; set; }
/// <summary>
/// MySql max_allowed_packet变量值大小
/// </summary>
public int MaxAllowedPacket { get; set; } = 32 * 1024 * 1024;
/// <summary>
/// 每次Insert提交的数据量
/// </summary>
public int FlushCount { get; set; } = 10000;
/// <summary>
/// 每个数据库最大提交任务数
/// </summary>
public int MaxDatabaseOutputTask { get; set; } = 4;
/// <summary>
/// 将json列作为16进制格式输出(0x前缀)
/// </summary>
public bool TreatJsonAsHex { get; set; } = true;
/// <summary>
/// 不对某些表进行输出
/// </summary>
public string[] NoOutput { get; set; } = [];
/// <summary>
/// <para>当某张表的键出现重复时，在输出时使用ON DUPLICATE KEY UPDATE更新该条记录</para>
/// <para>表名为键，更新的字段为值</para>
/// <example>
/// <code>
/// {
/// // 当order_data_parts表的键出现重复时，使用ON DUPLICATE KEY UPDATE更新已存在记录的CompanyID为新插入记录的值
/// "order_data_parts": "CompanyID = new.CompanyID"
/// }
/// </code>
/// </example>
/// </summary>
public Dictionary<string, string>? ForUpdate { get; set; }
/// <summary>
/// 配置导入数据的特殊列
/// 配置导入数据的特殊列，请在代码中配置
/// </summary>
public Dictionary<string, ColumnType> ColumnTypeConfig { get; set; } = new(); // "table.column" -> type
/// <summary>
/// 所有数据都输出完毕时的事件，请在代码中配置
/// </summary>
public Action<DataOutputContext>? OutputFinished { get; set; }
public ColumnType GetColumnType(string table, string column)
{
return ColumnTypeConfig.GetValueOrDefault($"{table}.{column}", ColumnType.UnDefine);
return ColumnTypeConfig.GetValueOrDefault(string.Concat(table, ".", column), ColumnType.UnDefine);
}
public bool TryGetForUpdate(string table, out string? forUpdate)

12
MesETL.App/Options/RedisCacheOptions.cs
View File

@@ -1,8 +1,20 @@
namespace MesETL.App.Options;
/// <summary>
/// Redis缓存选项
/// </summary>
public class RedisCacheOptions
{
/// <summary>
/// Redis连接字符串
/// </summary>
public string? Configuration { get; init; }
/// <summary>
/// Redis实例名称
/// </summary>
public string InstanceName { get; init; } = "";
/// <summary>
/// 使用的数据库序号
/// </summary>
public int Database { get; init; } = 0;
}

3
MesETL.App/Options/TableMockConfig.cs
View File

@@ -1,5 +1,8 @@
namespace MesETL.App.Options;
/// <summary>
/// 表模拟数据生成规则
/// </summary>
public struct TableMockConfig
{
/// <summary>

20
MesETL.App/Options/TenantDbOptions.cs
View File

@@ -1,5 +1,8 @@
namespace MesETL.App.Options;
/// <summary>
/// 多租户分库配置
/// </summary>
public class TenantDbOptions
{
public string? TenantKey { get; set; }
@@ -16,8 +19,21 @@ public class TenantDbOptions
// DbList.ForEach(pair => dictionary.Add(pair.Value, pair.Key));
// 注意配置顺序
if(DbGroup is null) throw new ApplicationException("分库配置中没有发现任何数据库");
var dbName = DbGroup.Cast<KeyValuePair<string, int>?>()
.FirstOrDefault(pair => pair?.Value != null && pair.Value.Value > tenantKeyValue)!.Value.Key;
#region 性能较低，不使用
// var dbName = DbGroup.Cast<KeyValuePair<string, int>?>()
// .FirstOrDefault(pair => pair?.Value != null && pair.Value.Value > tenantKeyValue)!.Value.Key;
#endregion
string? dbName = null;
foreach (var (key, value) in DbGroup)
{
if (value > tenantKeyValue)
dbName = key;
}
return dbName ??
throw new ArgumentOutOfRangeException(nameof(tenantKeyValue),
$"分库配置中没有任何符合'{nameof(tenantKeyValue)}'值的数据库");