CZY/MES-ETL
1
0
Fork 0
Code Pull Requests Releases Activity

修改根命名空间，修改分库配置。

Browse Source
This commit is contained in:
陈梓阳 2024-02-02 17:14:41 +08:00
parent e0de5d1c58
commit 5cda84797b
52 changed files with 163 additions and 237 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
MesETL.App/Cache/CacheKeys.cs
View File

@ -1,4 +1,4 @@
namespace ConsoleApp2.Cache;
namespace MesETL.App.Cache;
#nullable disable
public static class CacheKeys

2
MesETL.App/Cache/ICacher.cs
View File

@ -1,4 +1,4 @@
namespace ConsoleApp2.Cache;
namespace MesETL.App.Cache;
public interface ICacher
{

4
MesETL.App/Cache/RedisCache.cs
View File

@ -1,8 +1,8 @@
using ConsoleApp2.Options;
using MesETL.App.Options;
using Microsoft.Extensions.DependencyInjection;
using StackExchange.Redis;
namespace ConsoleApp2.Cache;
namespace MesETL.App.Cache;
public class RedisCache : ICacher
{

2
MesETL.App/Const/ProcessStep.cs
View File

@ -1,4 +1,4 @@
namespace ConsoleApp2.Const;
namespace MesETL.App.Const;
public static class ProcessStep
{

2
MesETL.App/Const/TableNames.cs
View File

@ -1,4 +1,4 @@
namespace ConsoleApp2.Const;
namespace MesETL.App.Const;
public static class TableNames
{

2
MesETL.App/DataRecord.cs
View File

@ -1,4 +1,4 @@
namespace ConsoleApp2;
namespace MesETL.App;
public class DataRecord : ICloneable
{

2
MesETL.App/Helpers/DatabaseHelper.cs
View File

@ -1,7 +1,7 @@
using System.Data;
using MySqlConnector;
namespace ConsoleApp2.Helpers;
namespace MesETL.App.Helpers;
public static class DatabaseHelper
{

2
MesETL.App/Helpers/DumpDataHelper.cs
View File

@ -1,7 +1,7 @@
using System.Text.RegularExpressions;
using ZstdSharp;
namespace ConsoleApp2.Helpers;
namespace MesETL.App.Helpers;
public static partial class DumpDataHelper
{

2
MesETL.App/Helpers/Extensions.Dictionary.cs
View File

@ -1,4 +1,4 @@
namespace ConsoleApp2.Helpers;
namespace MesETL.App.Helpers;
public static class DictionaryExtensions
{

8
MesETL.App/Helpers/Extensions.IEnumerable.cs
View File

@ -1,11 +1,15 @@
using System.Diagnostics.CodeAnalysis;
using System.Reflection;
namespace ConsoleApp2.Helpers;
namespace MesETL.App.Helpers;
#nullable disable
public static class EnumerableExtensions
{
public static string ToMarkdownTable<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicMethods)]T>(this IEnumerable<T> source)
public static string ToMarkdownTable<
[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicProperties |
DynamicallyAccessedMemberTypes.NonPublicProperties |
DynamicallyAccessedMemberTypes.PublicFields |
DynamicallyAccessedMemberTypes.NonPublicFields)] T>(this IEnumerable<T> source)
{
var properties = typeof(T).GetRuntimeProperties();
var fields = typeof(T)

2
MesETL.App/Helpers/Extensions.String.cs
View File

@ -1,7 +1,7 @@
using System.Globalization;
using System.Text;
namespace ConsoleApp2.Helpers;
namespace MesETL.App.Helpers;
public static class StringExtensions
{

2
MesETL.App/Helpers/Extensions.Task.cs
View File

@ -1,4 +1,4 @@
namespace ConsoleApp2.Helpers;
namespace MesETL.App.Helpers;
public static class TaskExtensions
{

8
MesETL.App/HostedServices/Abstractions/IDataReader.cs
View File

@ -1,10 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace ConsoleApp2.HostedServices.Abstractions
namespace MesETL.App.HostedServices.Abstractions
{
public interface IDataReader : IDisposable
{

5
MesETL.App/HostedServices/Abstractions/IInputService.cs
View File

@ -1,7 +1,4 @@
using ConsoleApp2.Options;
using ConsoleApp2.Services;
namespace ConsoleApp2.HostedServices.Abstractions;
namespace MesETL.App.HostedServices.Abstractions;
public interface IInputService
{

5
MesETL.App/HostedServices/Abstractions/IOutputService.cs
View File

@ -1,7 +1,4 @@
using ConsoleApp2.Options;
using ConsoleApp2.Services;
namespace ConsoleApp2.HostedServices.Abstractions;
namespace MesETL.App.HostedServices.Abstractions;
public interface IOutputService
{

5
MesETL.App/HostedServices/Abstractions/ITransformService.cs
View File

@ -1,7 +1,4 @@
using ConsoleApp2.Options;
using ConsoleApp2.Services;
namespace ConsoleApp2.HostedServices.Abstractions;
namespace MesETL.App.HostedServices.Abstractions;
public interface ITransformService
{

12
MesETL.App/HostedServices/FileInputService.cs
View File

@ -1,13 +1,13 @@
using ConsoleApp2.Const;
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
using ConsoleApp2.Services.ETL;
using MesETL.App.Const;
using MesETL.App.HostedServices.Abstractions;
using MesETL.App.Options;
using MesETL.App.Services;
using MesETL.App.Services.ETL;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace ConsoleApp2.HostedServices;
namespace MesETL.App.HostedServices;
public record FileInputInfo
{

20
MesETL.App/HostedServices/MainHostedService.cs
View File

@ -1,17 +1,17 @@
using System.Diagnostics;
using System.Text;
using ConsoleApp2.Helpers;
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
using ConsoleApp2.Services.ErrorRecorder;
using MesETL.App.Helpers;
using MesETL.App.HostedServices.Abstractions;
using MesETL.App.Options;
using MesETL.App.Services;
using MesETL.App.Services.ErrorRecorder;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MySqlConnector;
namespace ConsoleApp2.HostedServices;
namespace MesETL.App.HostedServices;
public class MainHostedService : BackgroundService
{
@ -107,7 +107,7 @@ public class MainHostedService : BackgroundService
private async Task SetVariableAsync(bool enable = true)
{
var connStr = _databaseOptions.Value.ConnectionString
?? throw new ApplicationException("无法还原索引，因为分库配置中没有配置数据库");
?? throw new ApplicationException("分库配置中没有配置数据库");
if (enable)
{
await DatabaseHelper.NonQueryAsync(connStr,
@ -128,13 +128,13 @@ public class MainHostedService : BackgroundService
private async Task ExecuteEachDatabase(string command, CancellationToken cancellationToken = default)
{
var databases = _tenantDbOptions.Value.DbList?.Keys
?? throw new ApplicationException("无法还原索引，因为分库配置中没有配置数据库");
var databases = _tenantDbOptions.Value.DbGroup?.Keys
?? throw new ApplicationException("分库配置中没有配置数据库");
var list = new List<Task>();
foreach (var db in databases)
{
var connStr = new MySqlConnectionStringBuilder(_databaseOptions.Value.ConnectionString
?? throw new ApplicationException("无法还原索引，因为没有配置数据库连接字符串"))
?? throw new ApplicationException("没有配置数据库连接字符串"))
{
ConnectionTimeout = 60,
DefaultCommandTimeout = 0,

16
MesETL.App/HostedServices/OutputService.cs
View File

@ -1,15 +1,15 @@
using ConsoleApp2.Helpers;
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
using ConsoleApp2.Services.ErrorRecorder;
using MesETL.App.Helpers;
using MesETL.App.HostedServices.Abstractions;
using MesETL.App.Options;
using MesETL.App.Services;
using MesETL.App.Services.ErrorRecorder;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MySqlConnector;
using MySqlDestination = ConsoleApp2.Services.ETL.MySqlDestination;
using TaskExtensions = ConsoleApp2.Helpers.TaskExtensions;
using MySqlDestination = MesETL.App.Services.ETL.MySqlDestination;
using TaskExtensions = MesETL.App.Helpers.TaskExtensions;
namespace ConsoleApp2.HostedServices;
namespace MesETL.App.HostedServices;
/// <summary>
/// 数据导出服务，将数据导出至MySql服务

8
MesETL.App/HostedServices/TaskMonitorService.cs
View File

@ -1,10 +1,10 @@
using System.Diagnostics;
using ConsoleApp2.Const;
using ConsoleApp2.Services;
using ConsoleApp2.Services.Loggers;
using MesETL.App.Const;
using MesETL.App.Services;
using MesETL.App.Services.Loggers;
using Microsoft.Extensions.DependencyInjection;
namespace ConsoleApp2.HostedServices;
namespace MesETL.App.HostedServices;
/// <summary>
/// 任务监控

14
MesETL.App/HostedServices/TransformService.cs
View File

@ -1,14 +1,14 @@
using ConsoleApp2.Cache;
using ConsoleApp2.Const;
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
using ConsoleApp2.Services.ErrorRecorder;
using MesETL.App.Cache;
using MesETL.App.Const;
using MesETL.App.HostedServices.Abstractions;
using MesETL.App.Options;
using MesETL.App.Services;
using MesETL.App.Services.ErrorRecorder;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace ConsoleApp2.HostedServices;
namespace MesETL.App.HostedServices;
public record DataTransformContext(DataRecord Record, ICacher Cacher, ILogger Logger);

6
MesETL.App/HostedServices/VoidOutputService.cs
View File

@ -1,8 +1,8 @@
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Services;
using MesETL.App.HostedServices.Abstractions;
using MesETL.App.Services;
using Microsoft.Extensions.Logging;
namespace ConsoleApp2.HostedServices;
namespace MesETL.App.HostedServices;
// 空输出服务，测试用
public class VoidOutputService : IOutputService

6
MesETL.App/MesETL.App.csproj
View File

@ -1,19 +1,17 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<AssemblyName>MesETL</AssemblyName>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
<RootNamespace>ConsoleApp2</RootNamespace>
<RootNamespace>MesETL.App</RootNamespace>
</PropertyGroup>
<ItemGroup>
<None Remove="appsettings.json" />
<None Update="RestoreIndex.sql">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
<ItemGroup>

4
MesETL.App/Options/DataInputOptions.cs
View File

@ -1,6 +1,6 @@
using ConsoleApp2.HostedServices;
using MesETL.App.HostedServices;
namespace ConsoleApp2.Options
namespace MesETL.App.Options
{
public class DataInputOptions
{

6
MesETL.App/Options/DataTransformOptions.cs
View File

@ -1,8 +1,6 @@
using ConsoleApp2.Cache;
using ConsoleApp2.HostedServices;
using Microsoft.Extensions.Logging;
using MesETL.App.HostedServices;
namespace ConsoleApp2.Options;
namespace MesETL.App.Options;
public enum ColumnType
{

2
MesETL.App/Options/DatabaseOutputOptions.cs
View File

@ -1,4 +1,4 @@
namespace ConsoleApp2.Options;
namespace MesETL.App.Options;
public class DatabaseOutputOptions
{

2
MesETL.App/Options/RedisCacheOptions.cs
View File

@ -1,4 +1,4 @@
namespace ConsoleApp2.Options;
namespace MesETL.App.Options;
public class RedisCacheOptions
{

2
MesETL.App/Options/TableMockConfig.cs
View File

@ -1,4 +1,4 @@
namespace ConsoleApp2.Options;
namespace MesETL.App.Options;
public struct TableMockConfig
{

9
MesETL.App/Options/TenantDbOptions.cs
View File

@ -1,21 +1,22 @@
namespace ConsoleApp2.Options;
namespace MesETL.App.Options;
public class TenantDbOptions
{
public string? TenantKey { get; set; }
public string? UseDbGroup { get; set; }
/// <summary>
/// Key-Value: {DbName}-{TenantKeyLessThan}
/// </summary>
public Dictionary<string, int>? DbList { get; set; }
public Dictionary<string, int>? DbGroup { get; set; }
public string GetDbNameByTenantKeyValue(int tenantKeyValue)
{
// var dictionary = new SortedDictionary<int, string>();
// DbList.ForEach(pair => dictionary.Add(pair.Value, pair.Key));
// 注意配置顺序
if(DbList is null) throw new ApplicationException("分库配置中没有发现任何数据库");
var dbName = DbList.Cast<KeyValuePair<string, int>?>()
if(DbGroup is null) throw new ApplicationException("分库配置中没有发现任何数据库");
var dbName = DbGroup.Cast<KeyValuePair<string, int>?>()
.FirstOrDefault(pair => pair?.Value != null && pair.Value.Value > tenantKeyValue)!.Value.Key;
return dbName ??
throw new ArgumentOutOfRangeException(nameof(tenantKeyValue),

69
MesETL.App/Program.cs
View File

@ -1,23 +1,22 @@
#define USE_TEST_DB // 测试库的结构与生产库不一样，如果使用测试库运行，则加上USE_TEST_DB预处理器指令
// #define USE_TEST_DB // 测试库的结构与生产库不一样，如果使用测试库运行，则加上USE_TEST_DB预处理器指令
using ConsoleApp2;
using ConsoleApp2.Cache;
using ConsoleApp2.Const;
using ConsoleApp2.Helpers;
using ConsoleApp2.HostedServices;
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
using ConsoleApp2.Services.ErrorRecorder;
using ConsoleApp2.Services.ETL;
using ConsoleApp2.Services.Loggers;
using MesETL.App;
using MesETL.App.Services;
using MesETL.App.Services.ETL;
using MesETL.App.Cache;
using MesETL.App.Const;
using MesETL.App.HostedServices;
using MesETL.App.HostedServices.Abstractions;
using MesETL.App.Options;
using MesETL.App.Services.ErrorRecorder;
using MesETL.App.Services.Loggers;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using MySqlConnector;
using Serilog;
using Serilog.Events;
using DumpDataHelper = MesETL.App.Helpers.DumpDataHelper;
await RunProgram();
return;
@ -29,14 +28,16 @@ async Task RunProgram()
host.Configuration.AddJsonFile(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "appsettings.json"), false, false);
host.Configuration.AddCommandLine(args, new Dictionary<string, string>
{
{"-d", "Input:InputDir"},
{"--InputDir", "Input:InputDir"},
{"-s", "Output:ConnectionString"},
{"--ConnectionString", "Output:ConnectionString"},
{"-r", "RedisCache:Configuration"},
{"--Redis", "RedisCache:Configuration"},
{"-c", "Command"},
{"--Command", "Command"}
{ "-d", "Input:InputDir" },
{ "--InputDir", "Input:InputDir" },
{ "-s", "Output:ConnectionString" },
{ "--ConnectionString", "Output:ConnectionString" },
{ "-r", "RedisCache:Configuration" },
{ "--Redis", "RedisCache:Configuration" },
{ "-g", "TenantDb:UseDbGroup" },
{ "--UseDbGroup", "TenantDb:UseDbGroup" },
{ "-c", "Command" },
{ "--Command", "Command" }
});
var inputOptions = host.Configuration.GetRequiredSection("Input").Get<DataInputOptions>()
@ -50,13 +51,22 @@ async Task RunProgram()
var redisSection = host.Configuration.GetRequiredSection("RedisCache");
var redisOptions = redisSection.Get<RedisCacheOptions>() ?? throw new ApplicationException("缺少RedisCache配置");
var tenantDbSection = host.Configuration.GetRequiredSection("TenantDb");
var tenantDbOptions = tenantDbSection.Get<TenantDbOptions>() ?? throw new ApplicationException("缺少TenantDb配置");
if(tenantDbOptions.TenantKey is null) throw new ApplicationException("分库配置中缺少分库键");
if(tenantDbOptions.DbList is null) throw new ApplicationException("分库配置中没有发现任何数据库");
var tenantDbOptions = new TenantDbOptions()
{
TenantKey = tenantDbSection.GetValue<string>(nameof(TenantDbOptions.TenantKey)) ?? throw new ApplicationException("分库配置缺少分库键TenantKey"),
UseDbGroup = tenantDbSection.GetValue<string>(nameof(TenantDbOptions.UseDbGroup)) ?? throw new ApplicationException("分库配置缺少使用分库组UseDbGroup")
};
tenantDbOptions.DbGroup = tenantDbSection.GetRequiredSection($"DbGroups:{tenantDbOptions.UseDbGroup}").Get<Dictionary<string,int>>()
?? throw new ApplicationException($"分库配置无法解析分库组{tenantDbOptions.UseDbGroup}，请检查配置");
host.Services.Configure<TenantDbOptions>(tenantDbSection);
host.Services.Configure<TenantDbOptions>(options =>
{
options.TenantKey = tenantDbOptions.TenantKey;
options.UseDbGroup = tenantDbOptions.UseDbGroup;
options.DbGroup = tenantDbOptions.DbGroup;
});
host.Services.Configure<RedisCacheOptions>(redisSection);
var oldestTime = DateTime.ParseExact(transformOptions.CleanDate, "yyyyMM", System.Globalization.DateTimeFormatInfo.InvariantInfo);
@ -313,7 +323,7 @@ async Task RunProgram()
//OrderBlockPlan将OrderNo长度<2的置空
case TableNames.OrderBlockPlan:
if (record["OrderNos"].Length < 2)
record["OrderNos"] = "";
record["OrderNos"] = "NULL";
break;
// OrderBlockPlanResult表添加CompanyID列
case TableNames.OrderBlockPlanResult:
@ -532,7 +542,8 @@ async Task RunProgram()
builder.ClearProviders();
builder.AddSerilog(new LoggerConfiguration()
.WriteTo.Console()
.WriteTo.File($"./Log/Error/{ErrorRecorder.UID}.log", restrictedToMinimumLevel:LogEventLevel.Error)
.WriteTo.File(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"./Log/Error/{ErrorRecorder.UID}.log"),
restrictedToMinimumLevel:LogEventLevel.Error)
// .WriteTo.File("./Log/Info/{ErrorRecorder.UID}.log", restrictedToMinimumLevel:LogEventLevel.Information) //性能考虑暂不使用
.CreateLogger()
);
@ -542,7 +553,7 @@ async Task RunProgram()
host.Services.AddErrorRecorderFactory();
host.Services.AddSingleton<ProcessContext>();
host.Services.AddKeyedSingleton<DataRecordQueue>(ProcessStep.Produce);
host.Services.AddRecordQueuePool(tenantDbOptions.DbList.Keys.Select(key => (key:key, queue:new DataRecordQueue(500_000))).ToArray());
host.Services.AddRecordQueuePool(tenantDbOptions.DbGroup.Keys.Select(key => (key:key, queue:new DataRecordQueue(500_000))).ToArray());
host.Services.AddSingleton<ITaskMonitorLogger, CacheTaskMonitorLogger>();
host.Services.AddSingleton<ITaskMonitorLogger, LoggerTaskMonitorLogger>();

73
MesETL.App/RestoreIndex.sql
View File

@ -1,73 +0,0 @@
CREATE INDEX `idx_CompanyID` ON `machine` (`CompanyID`);
CREATE INDEX `idx_companyid` ON `order` (`CompanyID`);
CREATE INDEX `idx_CompanyID` ON `order_block_plan` (`CompanyID`);
CREATE INDEX `idx_PlanID` ON `order_block_plan_item` (`PlanID`);
CREATE INDEX `idx_orderno` ON `order_box_block` (`OrderNo`);
CREATE INDEX `index_OrderNo` ON `order_data_block` (`OrderNo`);
CREATE INDEX `index_OrderNo` ON `order_data_goods` (`OrderNo`);
CREATE INDEX `index_OrderNo` ON `order_data_parts` (`OrderNo`);
CREATE INDEX `index_ItemNo` ON `order_item` (`ItemNo`);
CREATE INDEX `index_OrderNo` ON `order_item` (`OrderNo`);
CREATE INDEX `index_PackageID` ON `order_item` (`PackageID`);
CREATE INDEX `index_PlanID` ON `order_item` (`PlanID`);
CREATE INDEX `idx_OrderNo` ON `order_module` (`OrderNo`);
CREATE INDEX `index_OrderNo` ON `order_module_extra` (`OrderNo`);
CREATE INDEX `index_OrderNo` ON `order_module_item` (`OrderNo`);
CREATE INDEX `idx_OrderNo` ON `order_package` (`OrderNo`);
CREATE INDEX `idx_PakageNo` ON `order_package` (`PakageNo`);
CREATE INDEX `idx_PackageID` ON `order_package_item` (`PackageID`);
CREATE INDEX `idx_companyid` ON `order_patch_detail` (`CompanyID`);
CREATE INDEX `idx_OrderNo` ON `order_process` (`OrderNo`);
CREATE INDEX `index_CompanyID` ON `order_process_schdule` (`CompanyID`);
CREATE INDEX `IX_order_process_step_OrderProcessID` ON `order_process_step` (`OrderProcessID`);
CREATE INDEX `idx_OrderProcessID` ON `order_process_step_item` (`OrderProcessID`);
CREATE INDEX `idx_OrderProcessStepID` ON `order_process_step_item` (`OrderProcessStepID`);
CREATE INDEX `idx_CompanyID` ON `order_scrap_board` (`CompanyID`);
CREATE INDEX `idx_CompanyID` ON `process_group` (`CompanyID`);
CREATE INDEX `idx_CompanyID` ON `process_info` (`CompanyID`);
CREATE INDEX `index_CompanyID` ON `process_item_exp` (`CompanyID`);
CREATE INDEX `idx_CompanyID` ON `process_schdule_capacity` (`CompanyID`);
CREATE INDEX `idx_CompanyID` ON `process_step_efficiency` (`CompanyID`);
CREATE INDEX `idx_CompanyID` ON `report_template` (`CompanyID`);
CREATE INDEX `indx_OrderNo` ON `simple_package` (`OrderNo`);
CREATE INDEX `idx_CompanyID` ON `simple_plan_order` (`CompanyID`);
CREATE INDEX `idx_CompanyID` ON `sys_config` (`CompanyID`);
CREATE INDEX `idx` ON `work_calendar` (`CompanyID`);
CREATE INDEX `idx_CompanyID` ON `work_shift` (`CompanyID`);
CREATE INDEX `IX_work_time_ShiftID` ON `work_time` (`ShiftID`);

2
MesETL.App/Services/DataRecordQueue.cs
View File

@ -1,7 +1,7 @@
using System.Collections.Concurrent;
using System.Diagnostics.CodeAnalysis;
namespace ConsoleApp2.Services;
namespace MesETL.App.Services;
/// <summary>
/// 数据队列

4
MesETL.App/Services/ETL/CsvReader.cs
View File

@ -1,8 +1,8 @@
using System.Text;
using ConsoleApp2.HostedServices.Abstractions;
using MesETL.App.HostedServices.Abstractions;
using Microsoft.Extensions.Logging;
namespace ConsoleApp2.Services.ETL;
namespace MesETL.App.Services.ETL;
/// <summary>
/// CSV文件读取

6
MesETL.App/Services/ETL/DataReaderFactory.cs
View File

@ -1,10 +1,10 @@
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using MesETL.App.HostedServices.Abstractions;
using MesETL.App.Options;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace ConsoleApp2.Services.ETL;
namespace MesETL.App.Services.ETL;
public class DataReaderFactory
{

6
MesETL.App/Services/ETL/MySqlDestination.cs
View File

@ -1,12 +1,12 @@
using System.Text;
using System.Text.RegularExpressions;
using ConsoleApp2.Helpers;
using ConsoleApp2.Options;
using MesETL.App.Helpers;
using MesETL.App.Options;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MySqlConnector;
namespace ConsoleApp2.Services.ETL;
namespace MesETL.App.Services.ETL;
/// <summary>
/// Mysql导出

4
MesETL.App/Services/ETL/ZstMockReader.cs
View File

@ -1,7 +1,7 @@
using ConsoleApp2.Options;
using MesETL.App.Options;
using Microsoft.Extensions.Logging;
namespace ConsoleApp2.Services.ETL;
namespace MesETL.App.Services.ETL;
/// <summary>
/// 截取提供ZST文件中的第一行，然后复制成指定数量的数据

2
MesETL.App/Services/ETL/ZstReader.cs
View File

@ -1,7 +1,7 @@
using Microsoft.Extensions.Logging;
using ZstdSharp;
namespace ConsoleApp2.Services.ETL;
namespace MesETL.App.Services.ETL;
/// <summary>
/// 解压ZST文件，从中读取CSV数据

4
MesETL.App/Services/ErrorRecorder/ErrorRecorder.cs
View File

@ -1,7 +1,7 @@
using ConsoleApp2.Helpers;
using MesETL.App.Helpers;
using Microsoft.Extensions.Logging;
namespace ConsoleApp2.Services.ErrorRecorder;
namespace MesETL.App.Services.ErrorRecorder;
public class ErrorRecorder
{

2
MesETL.App/Services/ErrorRecorder/ErrorRecorderFactory.cs
View File

@ -1,7 +1,7 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
namespace ConsoleApp2.Services.ErrorRecorder;
namespace MesETL.App.Services.ErrorRecorder;
public class ErrorRecorderFactory
{

2
MesETL.App/Services/ErrorRecorder/InputErrorRecorder.cs
View File

@ -1,6 +1,6 @@
using Microsoft.Extensions.Logging;
namespace ConsoleApp2.Services.ErrorRecorder;
namespace MesETL.App.Services.ErrorRecorder;
public sealed class InputErrorRecorder : ErrorRecorder
{

4
MesETL.App/Services/ErrorRecorder/OutputErrorRecorder.cs
View File

@ -1,12 +1,12 @@
using System.Text;
using Microsoft.Extensions.Logging;
namespace ConsoleApp2.Services.ErrorRecorder;
namespace MesETL.App.Services.ErrorRecorder;
public sealed class OutputErrorRecorder : ErrorRecorder
{
private readonly string _outputDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"/ErrorRecords/{UID}/Output");
private readonly string _outputDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, $"ErrorRecords/{UID}/Output");
private readonly string _database;
private readonly Dictionary<string, int> _logIndex = new();

2
MesETL.App/Services/ErrorRecorder/TransformErrorRecorder.cs
View File

@ -1,6 +1,6 @@
using Microsoft.Extensions.Logging;
namespace ConsoleApp2.Services.ErrorRecorder;
namespace MesETL.App.Services.ErrorRecorder;
public sealed class TransformErrorRecorder : ErrorRecorder
{

4
MesETL.App/Services/Loggers/CacheTaskMonitorLogger.cs
View File

@ -1,6 +1,6 @@
using ConsoleApp2.Cache;
using MesETL.App.Cache;
namespace ConsoleApp2.Services.Loggers;
namespace MesETL.App.Services.Loggers;
public class CacheTaskMonitorLogger : ITaskMonitorLogger
{

2
MesETL.App/Services/Loggers/ITaskMonitorLogger.cs
View File

@ -1,4 +1,4 @@
namespace ConsoleApp2.Services.Loggers;
namespace MesETL.App.Services.Loggers;
public interface ITaskMonitorLogger
{

2
MesETL.App/Services/Loggers/LoggerTaskMonitorLogger.cs
View File

@ -1,7 +1,7 @@
using System.Text;
using Microsoft.Extensions.Logging;
namespace ConsoleApp2.Services.Loggers;
namespace MesETL.App.Services.Loggers;
public class LoggerTaskMonitorLogger : ITaskMonitorLogger
{

2
MesETL.App/Services/ProcessContext.cs
View File

@ -1,6 +1,6 @@
using System.Collections.Concurrent;
namespace ConsoleApp2.Services;
namespace MesETL.App.Services;
/// <summary>
/// 处理上下文类，标识处理进度

2
MesETL.App/Services/RecordQueuePool.cs
View File

@ -1,7 +1,7 @@
using System.Collections.Concurrent;
using Microsoft.Extensions.DependencyInjection;
namespace ConsoleApp2.Services;
namespace MesETL.App.Services;
public class RecordQueuePool
{

4
MesETL.App/Services/TaskManager.cs
View File

@ -1,7 +1,7 @@
using ApplicationException = System.ApplicationException;
using TaskExtensions = ConsoleApp2.Helpers.TaskExtensions;
using TaskExtensions = MesETL.App.Helpers.TaskExtensions;
namespace ConsoleApp2.Services;
namespace MesETL.App.Services;
/// <summary>
/// 快速批量创建和等待任务

36
MesETL.App/appsettings.json
View File

@ -6,14 +6,14 @@
},
"Input":{
"InputDir": "D:\\Dump\\MockData", // Csv数据输入目录
"UseMock": true, // 使用模拟数据进行测试
"UseMock": false, // 使用模拟数据进行测试
"MockCountMultiplier": 0.5 // 模拟数据量级的乘数
},
"Transform":{
"StrictMode": true, // 设为true时如果数据转换发生错误，立刻停止程序
"EnableFilter": false, // 启用数据过滤
"EnableReplacer": false, // 启用数据修改ma
"EnableReBuilder": false, // 启用数据重建
"StrictMode": false, // 设为true时如果数据转换发生错误，立刻停止程序
"EnableFilter": true, // 启用数据过滤
"EnableReplacer": true, // 启用数据修改
"EnableReBuilder": true, // 启用数据重建
"CleanDate": "202301" // 当数据过滤开启时，删除这个时间之前的数据
},
"Output":{
@ -22,7 +22,6 @@
"FlushCount": 10000, // 每次提交记录条数
"MaxDatabaseOutputTask" : 4, // 每个数据库最大提交任务数
"TreatJsonAsHex": false, // 将json列作为16进制格式输出(0x前缀)
"RestoreIndex": true // 在输出结束以后还原数据库的索引(调用RestoreIndex.sql)
},
"RedisCache": {
"Configuration": "192.168.1.246:6380",
@ -31,17 +30,20 @@
"TenantDb": // 分库配置
{
"TenantKey" : "CompanyID",
"DbList": {
/*
* 相当于
* (CompanyId < 1000) -> cferp_test_1
* (CompanyId < 2000) -> cferp_test_2
* (CompanyId < 2147483647) -> cferp_test_3
*/
"cferp_test_1": 1000,
"cferp_test_2": 2000,
"cferp_test_3": 2147483647
// "cferp_void_1": 2147483647
"UseDbGroup": "test",
"DbGroups": {
"test": {
"cferp_test_1": 1000,
"cferp_test_2": 2000,
"cferp_test_3": 2147483647
},
"prod":{
"mesdb_1": 5000,
"mesdb_2": 10000,
"mesdb_3": 15000,
"mesdb_4": 20000,
"mesdb_5": 2147483647
}
}
}
}

2
MesETL.Test/DataFix.cs
View File

@ -1,5 +1,5 @@
using System.Data;
using ConsoleApp2.Helpers;
using MesETL.App.Helpers;
using MySqlConnector;
using Xunit.Abstractions;

2
MesETL.Test/DatabaseToolBox.cs
View File

@ -1,6 +1,6 @@
using System.Data;
using System.Text;
using ConsoleApp2.Helpers;
using MesETL.App.Helpers;
using MySqlConnector;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;

2
MesETL.Test/MesDatabaseHelper.cs
View File

@ -1,4 +1,4 @@
using ConsoleApp2.Helpers;
using MesETL.App.Helpers;
namespace TestProject1;