项目重命名

MesETL.App/Options/DataInputOptions.cs

@@ -0,0 +1,48 @@
+using ConsoleApp2.HostedServices;
+
+namespace ConsoleApp2.Options
+{
+    public class DataInputOptions
+    {
+        public string? InputDir { get; set; }
+
+        #region CSV
+        
+        /// <summary>
+        /// 字符串的包围符号，默认为双引号"
+        /// </summary>
+        public char QuoteChar { get; set; } = '"';
+
+        /// <summary>
+        /// 每个字段的分割符，默认逗号,
+        /// </summary>
+        public string Delimiter { get; set; } = ",";
+
+        #endregion
+
+        #region Mock
+
+        public bool UseMock { get; set; }
+
+        public double MockCountMultiplier { get; set; } = 1;
+        
+        /// <summary>
+        /// Table -> Mock Count 暂时为手动配置
+        /// </summary>
+        public Dictionary<string, TableMockConfig>? TableMockConfig { get; set; }
+        
+        #endregion
+
+
+        #region ManualSet
+        
+        public string[]? TableOrder { get; set; }
+        
+        /// <summary>
+        /// 配置如何从文件名转换为表名和表头
+        /// </summary>
+        public Func<string, FileInputInfo?>? FileInputMetaBuilder { get; set; } //TODO: 抽离
+
+        #endregion
+    }
+}

MesETL.App/Options/DataTransformOptions.cs

@@ -0,0 +1,52 @@
+using ConsoleApp2.Cache;
+using ConsoleApp2.HostedServices;
+using Microsoft.Extensions.Logging;
+
+namespace ConsoleApp2.Options;
+
+public enum ColumnType
+{
+    Blob,
+    Text,
+    Json,
+    UnDefine,
+}
+
+public class DataTransformOptions
+{
+    public bool StrictMode { get; set; } = true;
+    public bool EnableFilter { get; set; } = true;
+    public bool EnableReplacer { get; set; } = true;
+    public bool EnableReBuilder { get; set; } = true;
+    
+    /// <summary>
+    /// yyyyMM
+    /// </summary>
+    public string CleanDate { get; set; } = "202301";
+    
+    /// <summary>
+    /// Record -> Database name
+    /// 对记录进行数据库过滤
+    /// </summary>
+    public Func<DataRecord, string>? DatabaseFilter { get; set; }
+    /// <summary>
+    /// Context -> Should output
+    /// 配置对数据过滤的条件
+    /// </summary>
+    public Func<DataTransformContext, Task<bool>>? RecordFilter { get; set; }//数据过滤方法
+    /// <summary>
+    /// Context -> New record
+    /// 对当前记录进行修改或完整替换
+    /// </summary>
+    public Func<DataTransformContext, Task<DataRecord>>? RecordModify { get; set; }//数据替换
+    /// <summary>
+    /// Context -> New rebuild records
+    /// 使用当前记录对某些数据进行重建
+    /// </summary>
+    public Func<DataTransformContext, IList<DataRecord>?>? RecordReBuild { get; set; }//新增数据
+    /// <summary>
+    /// Context -> void
+    /// 对数据的某些字段进行缓存
+    /// </summary>
+    public Func<DataTransformContext, Task>? RecordCache { get; set; }//数据缓存
+}

MesETL.App/Options/DatabaseOutputOptions.cs

@@ -0,0 +1,25 @@
+namespace ConsoleApp2.Options;
+
+public class DatabaseOutputOptions
+{
+    public string? ConnectionString { get; set; }
+
+    public int MaxAllowedPacket { get; set; } = 64 * 1024 * 1024;
+
+    public int FlushCount { get; set; } = 10000;
+
+    public int MaxDatabaseOutputTask { get; set; } = 4;
+
+    public bool TreatJsonAsHex { get; set; } = true;
+    
+
+    /// <summary>
+    /// 配置导入数据的特殊列
+    /// </summary>
+    public Dictionary<string, ColumnType> ColumnTypeConfig { get; set; } = new(); // "table.column" -> type
+
+    public ColumnType GetColumnType(string table, string column)
+    {
+        return ColumnTypeConfig.GetValueOrDefault($"{table}.{column}", ColumnType.UnDefine);
+    }
+}

MesETL.App/Options/RedisCacheOptions.cs

@@ -0,0 +1,8 @@
+namespace ConsoleApp2.Options;
+
+public class RedisCacheOptions
+{
+    public string? Configuration { get; init; }
+    public string InstanceName { get; init; } = "";
+    public int Database { get; init; } = 0;
+}

MesETL.App/Options/TableMockConfig.cs

@@ -0,0 +1,32 @@
+namespace ConsoleApp2.Options;
+
+public struct TableMockConfig
+{
+    /// <summary>
+    /// 使用深拷贝
+    /// </summary>
+    public bool UseDeepCopy { get; set; }
+    /// <summary>
+    /// 模拟数据量
+    /// </summary>
+    public long MockCount { get; set; }
+
+    /// <summary>
+    /// 需要开启MockCount
+    /// </summary>
+    public string[]? AutoIncrementColumn { get; set; } = null; // TODO: 换为自定义委托
+    
+    public void Deconstruct(out bool useDeepCopy, out long mockCount, out string[]? autoIncrementColumn)
+    {
+        useDeepCopy = UseDeepCopy;
+        mockCount = MockCount;
+        autoIncrementColumn = AutoIncrementColumn;
+    }
+
+    public TableMockConfig(bool useDeepCopy, long mockCount, string[]? autoIncrementColumn)
+    {
+        UseDeepCopy = useDeepCopy;
+        MockCount = mockCount;
+        AutoIncrementColumn = autoIncrementColumn;
+    }
+}

MesETL.App/Options/TenantDbOptions.cs

@@ -0,0 +1,24 @@
+namespace ConsoleApp2.Options;
+
+public class TenantDbOptions
+{
+    public string? TenantKey { get; set; }
+
+    /// <summary>
+    /// Key-Value: {DbName}-{TenantKeyLessThan}
+    /// </summary>
+    public Dictionary<string, int>? DbList { get; set; }
+
+    public string GetDbNameByTenantKeyValue(int tenantKeyValue)
+    {
+        // var dictionary = new SortedDictionary<int, string>();
+        // DbList.ForEach(pair => dictionary.Add(pair.Value, pair.Key));
+        // 注意配置顺序
+        if(DbList is null) throw new ApplicationException("分库配置中没有发现任何数据库");
+        var dbName = DbList.Cast<KeyValuePair<string, int>?>()
+            .FirstOrDefault(pair => pair?.Value != null && pair.Value.Value > tenantKeyValue)!.Value.Key;
+        return dbName ??
+               throw new ArgumentOutOfRangeException(nameof(tenantKeyValue),
+                   $"分库配置中没有任何符合'{nameof(tenantKeyValue)}'值的数据库");
+    }
+}