CZY/MES-ETL
1
0
Fork 0
Code Pull Requests Releases Activity

添加数据修复程序

Browse Source
This commit is contained in:
陈梓阳 2024-01-19 11:17:22 +08:00
parent 8e5efa83f1
commit 4f96b77e55
7 changed files with 276 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
ConsoleApp2.sln
View File

@ -2,6 +2,8 @@
Microsoft Visual Studio Solution File, Format Version 12.00 Microsoft Visual Studio Solution File, Format Version 12.00
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ConsoleApp2", "ConsoleApp2\ConsoleApp2.csproj", "{155E4B04-E88C-4BA4-AED2-B13E0A0432B5}" Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ConsoleApp2", "ConsoleApp2\ConsoleApp2.csproj", "{155E4B04-E88C-4BA4-AED2-B13E0A0432B5}"
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestProject1", "TestProject1\TestProject1.csproj", "{8679D5B6-5853-446E-9882-7B7A8E270500}"
EndProject
Global Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU Debug|Any CPU = Debug|Any CPU
@ -12,5 +14,9 @@ Global
{155E4B04-E88C-4BA4-AED2-B13E0A0432B5}.Debug|Any CPU.Build.0 = Debug|Any CPU {155E4B04-E88C-4BA4-AED2-B13E0A0432B5}.Debug|Any CPU.Build.0 = Debug|Any CPU
{155E4B04-E88C-4BA4-AED2-B13E0A0432B5}.Release|Any CPU.ActiveCfg = Release|Any CPU {155E4B04-E88C-4BA4-AED2-B13E0A0432B5}.Release|Any CPU.ActiveCfg = Release|Any CPU
{155E4B04-E88C-4BA4-AED2-B13E0A0432B5}.Release|Any CPU.Build.0 = Release|Any CPU {155E4B04-E88C-4BA4-AED2-B13E0A0432B5}.Release|Any CPU.Build.0 = Release|Any CPU
{8679D5B6-5853-446E-9882-7B7A8E270500}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{8679D5B6-5853-446E-9882-7B7A8E270500}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8679D5B6-5853-446E-9882-7B7A8E270500}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8679D5B6-5853-446E-9882-7B7A8E270500}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection EndGlobalSection
EndGlobal EndGlobal

55
TestProject1/DatabaseHelper.cs Normal file
View File

@ -0,0 +1,55 @@
using System.Data;
using System.Data.Common;
using MySqlConnector;
namespace TestProject1;
public static class DatabaseHelper
{
public static async Task<DataSet> QueryTable(string connStr, string sql)
{
await using var conn = new MySqlConnection(connStr);
if(conn.State is not ConnectionState.Open)
await conn.OpenAsync();
await using var cmd = conn.CreateCommand();
cmd.CommandText = sql;
var ds = new DataSet();
var adapter = new MySqlDataAdapter(cmd).Fill(ds);
return ds;
}
public static async Task<object?> QueryScalar(string connStr, string sql)
{
await using var conn = new MySqlConnection(connStr);
if(conn.State is not ConnectionState.Open)
await conn.OpenAsync();
await using var cmd = conn.CreateCommand();
cmd.CommandText = sql;
return await cmd.ExecuteScalarAsync();
}
public static async Task<int> NonQuery(string connStr, string sql)
{
await using var conn = new MySqlConnection(connStr);
if(conn.State is not ConnectionState.Open)
await conn.OpenAsync();
await using var cmd = conn.CreateCommand();
cmd.CommandText = sql;
return await cmd.ExecuteNonQueryAsync();
}
public static async Task<int> TransactionAsync(string connStr, string sql, params MySqlParameter[] parameters)
{
await using var conn = new MySqlConnection(connStr);
if(conn.State is not ConnectionState.Open)
await conn.OpenAsync();
await using var trans = await conn.BeginTransactionAsync();
await using var cmd = conn.CreateCommand();
cmd.CommandText = sql;
cmd.Transaction = trans;
cmd.Parameters.AddRange(parameters);
var rows = await cmd.ExecuteNonQueryAsync();
await trans.CommitAsync();
return rows;
}
}

1
TestProject1/GlobalUsings.cs Normal file
View File

@ -0,0 +1 @@
global using Xunit;

18
TestProject1/MesDatabaseHelper.cs Normal file
View File

@ -0,0 +1,18 @@
namespace TestProject1;
public static class MesDatabaseHelper
{
public static async Task<int?> TrySearchCompanyId(string connStr, IEnumerable<string> dbNames, string scalarQuery)
{
foreach (var db in dbNames)
{
var result = await DatabaseHelper.QueryScalar(connStr + $"Database={db};", scalarQuery);
if (result is null or 0) continue;
var companyId = Convert.ToInt32(result);
return companyId;
}
return null;
}
}

26
TestProject1/TenantDbHelper.cs Normal file
View File

@ -0,0 +1,26 @@
namespace TestProject1;
public static class TenantDbHelper
{
/// <summary>
/// Key-Value: {DbName}-{TenantKeyLessThan}
/// </summary>
public static Dictionary<string, int> DbList { get; set; } = new Dictionary<string, int>
{
{ "cferp_test_1", 1000 },
{ "cferp_test_2", 2000 },
{ "cferp_test_3", int.MaxValue },
};
public static string GetDbNameByTenantKeyValue(int tenantKeyValue)
{
// var dictionary = new SortedDictionary<int, string>();
// DbList.ForEach(pair => dictionary.Add(pair.Value, pair.Key));
// 注意配置顺序
var dbName = DbList.Cast<KeyValuePair<string, int>?>()
.FirstOrDefault(pair => pair?.Value != null && pair.Value.Value > tenantKeyValue)!.Value.Key;
return dbName ??
throw new ArgumentOutOfRangeException(nameof(tenantKeyValue),
$"已配置的数据库中没有任何符合'{nameof(tenantKeyValue)}'值的对象");
}
}

29
TestProject1/TestProject1.csproj Normal file
View File

@ -0,0 +1,29 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<IsPackable>false</IsPackable>
<IsTestProject>true</IsTestProject>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.0"/>
<PackageReference Include="xunit" Version="2.4.2"/>
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="coverlet.collector" Version="6.0.0">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\ConsoleApp2\ConsoleApp2.csproj" />
</ItemGroup>
</Project>

141
TestProject1/UnitTest1.cs Normal file
View File

@ -0,0 +1,141 @@
using System.Data;
using ConsoleApp2.Helpers;
using MySqlConnector;
using Xunit.Abstractions;
namespace TestProject1;
public class UnitTest1
{
public const string ConnStr = "Server=192.168.1.245;Port=3306;UserId=root;Password=ruixinjie!@#123;";
public static string[] DbNames = ["cferp_test_1", "cferp_test_2", "cferp_test_3"];
private readonly ITestOutputHelper _output;
public UnitTest1(ITestOutputHelper output)
{
_output = output;
}
private string MakeConnStr(string dbName) => ConnStr + $"Database={dbName};";
/// <summary>
/// 查找cferp_test_1中CompanyID = 0的order_box_block，根据OrderNo查找对应订单的CompanyID，然后删除重新插入相应的数据库
/// 如果没有找到对应订单，则保留CompanyID为0
/// </summary>
[Fact]
public async Task FixOrderBoxBlockCompanyID()
{
var ds = await DatabaseHelper.QueryTable(MakeConnStr(DbNames[0]), "SELECT * FROM `order_box_block` WHERE COMPANYID = 0");
var dict = new Dictionary<long, int>();//orderNo -> CompanyID
foreach (DataRow row in ds.Tables[0].Rows)
{
// foreach (var column in row.ItemArray)
// {
// Console.Write(column.ToString() + '\t');
// }
var orderNo = Convert.ToInt64(row["OrderNo"]);
var boxId = Convert.ToInt64(row["BoxID"]);
int? companyId = null;
foreach (var db in DbNames)
{
if(!dict.TryGetValue(orderNo, out var cid)) // 可以提到外面
{
var result = await DatabaseHelper.QueryScalar(ConnStr + $"Database={db};",
$"SELECT CompanyID FROM `order` WHERE OrderNo = {orderNo}");
if (result is null or 0) continue;
companyId = Convert.ToInt32(result);
dict.Add(orderNo, companyId.Value);
break;
}
else
{
companyId = cid;
}
}
if (companyId is null or 0)
{
_output.WriteLine($"OrderBoxBlock:{boxId} - OrderNo {orderNo} not found");
continue;
}
row["CompanyID"] = companyId;
await DatabaseHelper.TransactionAsync(ConnStr,
$"""
DELETE FROM cferp_test_1.`order_box_block` WHERE BoxID = {boxId};
INSERT INTO {TenantDbHelper.GetDbNameByTenantKeyValue(companyId.Value)}.`order_box_block`
VALUES(@c1, @c2, @c3, @c4, @c5);
""", [new MySqlParameter("c1", row[0]),
new MySqlParameter("c2", row[1]),
new MySqlParameter("c3", row[2]),
new MySqlParameter("c4", row[3]),
new MySqlParameter("c5", row[4])]);
_output.WriteLine($"OrderBoxBock:{boxId} CompanyID -> {companyId}");
}
}
[Theory]
[InlineData(["order_data_block", "ID", 0])]
[InlineData(["order_data_block", "ID", 1])]
[InlineData(["order_data_block", "ID", 2])]
public async Task FixCompanyIdWithOwnOrderNo(string tableName, string keyName, int dbNameIndex)
{
var ds = await DatabaseHelper.QueryTable(MakeConnStr(DbNames[dbNameIndex]),
$"SELECT * FROM `{tableName}` WHERE COMPANYID = 0");
var dict = new Dictionary<long, int>();//orderNo -> CompanyID
foreach (DataRow row in ds.Tables[0].Rows)
{
// foreach (var column in row.ItemArray)
// {
// Console.Write(column.ToString() + '\t');
// }
var orderNo = Convert.ToInt64(row["OrderNo"]);
var key = Convert.ToInt32(row[keyName]);
int? companyId = null;
if(!dict.TryGetValue(orderNo, out var cid))
{
companyId = await MesDatabaseHelper.TrySearchCompanyId(ConnStr, DbNames,
$"SELECT CompanyID FROM `order` WHERE OrderNo = {orderNo}");
}
else
{
companyId = cid;
}
if (companyId is null or 0)
{
_output.WriteLine($"{tableName}:{key} - OrderNo {orderNo} not found");
continue;
}
row["CompanyID"] = companyId;
// ON DUPLICATE KEY UPDATE
await DatabaseHelper.TransactionAsync(ConnStr,
$"""
DELETE FROM {DbNames[dbNameIndex]}.`{tableName}` WHERE ID = {key};
INSERT INTO {TenantDbHelper.GetDbNameByTenantKeyValue(companyId.Value)}.`{tableName}`
VALUES({string.Join(',', Enumerable.Range(0, row.ItemArray.Length).Select(i => $"@c{i}"))})
ON DUPLICATE KEY UPDATE CompanyID = {companyId};
""",
Enumerable.Range(0, row.ItemArray.Length).Select(i => new MySqlParameter($"c{i}", row[i])).ToArray());
_output.WriteLine($"{tableName}:{key} CompanyID -> {companyId}");
}
}
[Theory]
[InlineData("order", 0)]
[InlineData("order", 1)]
[InlineData("order", 2)]
public async Task FixShardKeyWithOwnOrderNo(string tableName, int dbNameIndex)
{
var r = await DatabaseHelper.NonQuery(MakeConnStr(DbNames[dbNameIndex]),
$"UPDATE `{tableName}` SET ShardKey = CONCAT(SUBSTR(`order`.OrderNo,3,4),'0') WHERE ShardKey = 0");
_output.WriteLine($"Affect Rows: {r}");
}
}