CZY/MES-ETL
1
0
Fork 0
Code Pull Requests Releases Activity

Update

Browse Source
This commit is contained in:
陈梓阳
2024-01-29 09:29:16 +08:00
parent 4f96b77e55
commit 083090c62b
63 changed files with 2479 additions and 1491 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
TestProject1/UnitTest1.cs → TestProject1/DataFix.cs
View File

@@ -1,18 +1,19 @@
using System.Data;
using ConsoleApp2.Helpers;
using ConsoleApp2.Helpers.Database;
using MySqlConnector;
using Xunit.Abstractions;
namespace TestProject1;
public class UnitTest1
public class DataFix
{
public const string ConnStr = "Server=192.168.1.245;Port=3306;UserId=root;Password=ruixinjie!@#123;";
public static string[] DbNames = ["cferp_test_1", "cferp_test_2", "cferp_test_3"];
private readonly ITestOutputHelper _output;
public UnitTest1(ITestOutputHelper output)
public DataFix(ITestOutputHelper output)
{
_output = output;
}
@@ -26,7 +27,7 @@ public class UnitTest1
[Fact]
public async Task FixOrderBoxBlockCompanyID()
{
var ds = await DatabaseHelper.QueryTable(MakeConnStr(DbNames[0]), "SELECT * FROM `order_box_block` WHERE COMPANYID = 0");
var ds = await DatabaseHelper.QueryTableAsync(MakeConnStr(DbNames[0]), "SELECT * FROM `order_box_block` WHERE COMPANYID = 0");
var dict = new Dictionary<long, int>();//orderNo -> CompanyID
foreach (DataRow row in ds.Tables[0].Rows)
{
@@ -43,7 +44,7 @@ public class UnitTest1
{
if(!dict.TryGetValue(orderNo, out var cid)) // 可以提到外面
{
var result = await DatabaseHelper.QueryScalar(ConnStr + $"Database={db};",
var result = await DatabaseHelper.QueryScalarAsync(ConnStr + $"Database={db};",
$"SELECT CompanyID FROM `order` WHERE OrderNo = {orderNo}");
if (result is null or 0) continue;
@@ -84,7 +85,7 @@ public class UnitTest1
[InlineData(["order_data_block", "ID", 2])]
public async Task FixCompanyIdWithOwnOrderNo(string tableName, string keyName, int dbNameIndex)
{
var ds = await DatabaseHelper.QueryTable(MakeConnStr(DbNames[dbNameIndex]),
var ds = await DatabaseHelper.QueryTableAsync(MakeConnStr(DbNames[dbNameIndex]),
$"SELECT * FROM `{tableName}` WHERE COMPANYID = 0");
var dict = new Dictionary<long, int>();//orderNo -> CompanyID
foreach (DataRow row in ds.Tables[0].Rows)
@@ -134,7 +135,7 @@ public class UnitTest1
[InlineData("order", 2)]
public async Task FixShardKeyWithOwnOrderNo(string tableName, int dbNameIndex)
{
var r = await DatabaseHelper.NonQuery(MakeConnStr(DbNames[dbNameIndex]),
var r = await DatabaseHelper.NonQueryAsync(MakeConnStr(DbNames[dbNameIndex]),
$"UPDATE `{tableName}` SET ShardKey = CONCAT(SUBSTR(`order`.OrderNo,3,4),'0') WHERE ShardKey = 0");
_output.WriteLine($"Affect Rows: {r}");
}

55
TestProject1/DatabaseHelper.cs
View File

@@ -1,55 +0,0 @@
using System.Data;
using System.Data.Common;
using MySqlConnector;
namespace TestProject1;
public static class DatabaseHelper
{
public static async Task<DataSet> QueryTable(string connStr, string sql)
{
await using var conn = new MySqlConnection(connStr);
if(conn.State is not ConnectionState.Open)
await conn.OpenAsync();
await using var cmd = conn.CreateCommand();
cmd.CommandText = sql;
var ds = new DataSet();
var adapter = new MySqlDataAdapter(cmd).Fill(ds);
return ds;
}
public static async Task<object?> QueryScalar(string connStr, string sql)
{
await using var conn = new MySqlConnection(connStr);
if(conn.State is not ConnectionState.Open)
await conn.OpenAsync();
await using var cmd = conn.CreateCommand();
cmd.CommandText = sql;
return await cmd.ExecuteScalarAsync();
}
public static async Task<int> NonQuery(string connStr, string sql)
{
await using var conn = new MySqlConnection(connStr);
if(conn.State is not ConnectionState.Open)
await conn.OpenAsync();
await using var cmd = conn.CreateCommand();
cmd.CommandText = sql;
return await cmd.ExecuteNonQueryAsync();
}
public static async Task<int> TransactionAsync(string connStr, string sql, params MySqlParameter[] parameters)
{
await using var conn = new MySqlConnection(connStr);
if(conn.State is not ConnectionState.Open)
await conn.OpenAsync();
await using var trans = await conn.BeginTransactionAsync();
await using var cmd = conn.CreateCommand();
cmd.CommandText = sql;
cmd.Transaction = trans;
cmd.Parameters.AddRange(parameters);
var rows = await cmd.ExecuteNonQueryAsync();
await trans.CommitAsync();
return rows;
}
}

154
TestProject1/DatabaseToolBox.cs Normal file
View File

@@ -0,0 +1,154 @@
using System.Data;
using System.Text;
using ConsoleApp2.Helpers.Database;
using MySqlConnector;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Xunit.Abstractions;
namespace TestProject1;
public class DatabaseToolBox
{
private readonly ITestOutputHelper _output;
public const string ConnStr = "Server=127.0.0.1;Port=33309;UserId=root;Password=123456;";
public DatabaseToolBox(ITestOutputHelper output)
{
_output = output;
}
[Theory]
[InlineData("cferp_void_1")]
public async Task AlterAllTableToBlackHole(string database)
{
var connStr = ConnStr + $"Database={database}";
var tables = await DatabaseHelper.QueryTableAsync(connStr,
$"""
SELECT TABLE_NAME FROM information_schema.`TABLES` WHERE TABLE_SCHEMA = '{database}';
""");
foreach (DataRow row in tables.Tables[0].Rows)
{
var tableName = row["TABLE_NAME"].ToString();
var sql = $"""
ALTER TABLE `{tableName}` REMOVE PARTITIONING;
""";
try
{
await DatabaseHelper.NonQueryAsync(connStr, sql);
}
catch (MySqlException e) when (e.ErrorCode == MySqlErrorCode.PartitionManagementOnNoPartitioned)
{
}
sql = $"""
ALTER TABLE `{tableName}` ENGINE=BLACKHOLE;
""";
await DatabaseHelper.NonQueryAsync(connStr, sql);
}
}
public async Task<long> CountAllTable(string connStr, string database)
{
var count = 0L;
var set = await DatabaseHelper.QueryTableAsync(connStr,
$"""
SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = '{database}';
""");
foreach (DataRow row in set.Tables[0].Rows)
{
count += Convert.ToInt64(await DatabaseHelper.QueryScalarAsync(connStr,
$"""
SELECT COUNT(1) FROM `{database}`.`{row[0]}`;
"""));
}
_output.WriteLine($"Record count: {count} records");
return count;
}
[Theory]
[InlineData([new[]{"cferp_test_1", "cferp_test_2", "cferp_test_3"}])]
public async Task CountAllDatabase(IEnumerable<string> databases)
{
var count = 0L;
var connStr = "Server=192.168.1.245;Port=3306;UserId=root;Password=ruixinjie!@#123;";
foreach (var db in databases)
{
count += await CountAllTable(connStr, db);
}
_output.WriteLine(count.ToString());
}
public async Task<TableIndex[]> GetAllTableIndexes(string database)
{
var data = await DatabaseHelper.QueryTableAsync(ConnStr,
$"""
SELECT TABLE_NAME, INDEX_NAME, NON_UNIQUE, COLUMN_NAME, INDEX_TYPE FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = '{database}'
AND INDEX_NAME != 'PRIMARY'
AND INDEX_TYPE = 'BTREE'
""");
return data.Tables[0].Rows.Cast<DataRow>().Select(row =>
{
return new TableIndex(row["TABLE_NAME"].ToString()!,
row["INDEX_NAME"].ToString()!,
!Convert.ToBoolean(row["NON_UNIQUE"]),
row["COLUMN_NAME"].ToString()!,
row["INDEX_TYPE"] switch
{
"BTREE" => TableIndex.TableIndexType.BTree,
"PRIMARY" => TableIndex.TableIndexType.Primary,
"HASH" => TableIndex.TableIndexType.Hash,
_ => throw new ArgumentOutOfRangeException(nameof(row))
}
);
}).ToArray();
}
[Theory]
[InlineData(["cferp_test_1", "D:/Indexes_cferp_test_1.json"])]
[InlineData(["cferp_test_2", "D:/Indexes_cferp_test_2.json"])]
[InlineData(["cferp_test_3", "D:/Indexes_cferp_test_3.json"])]
public async Task ExportAllIndexes(string database, string outputPath)
{
var indexes = await GetAllTableIndexes(database);
var json = JArray.FromObject(indexes);
await File.WriteAllTextAsync(outputPath, json.ToString());
_output.WriteLine($"Exported {indexes.Length} indexes to '{outputPath}'");
}
[Theory]
[InlineData("cferp_test_1", "D:/Indexes_cferp_test_1.json")]
[InlineData("cferp_test_2", "D:/Indexes_cferp_test_2.json")]
[InlineData("cferp_test_3", "D:/Indexes_cferp_test_3.json")]
public async Task ImportAllIndexes(string database, string importPath)
{
var json = await File.ReadAllTextAsync(importPath);
var indexes = JsonConvert.DeserializeObject<TableIndex[]>(json);
var sb = new StringBuilder();
foreach (var (tableName, indexName, isUnique, columnName, tableIndexType) in indexes!)
{
sb.AppendLine($"CREATE {(isUnique ? "UNIQUE" : string.Empty)} INDEX `{indexName}` ON `{database}`.`{tableName}` (`{columnName}`) USING {tableIndexType};");
}
await DatabaseHelper.NonQueryAsync(ConnStr, sb.ToString());
_output.WriteLine($"Import {indexes.Length} indexes to '{database}' from '{importPath}'");
}
[Theory]
[InlineData(["cferp_test_1"])]
[InlineData(["cferp_test_2"])]
[InlineData(["cferp_test_3"])]
public async Task DropAllIndex(string database)
{
var indexes = await GetAllTableIndexes(database);
var sb = new StringBuilder();
foreach (var (tableName, indexName) in indexes)
{
sb.AppendLine($"DROP INDEX `{indexName}` ON `{database}`.`{tableName}`;");
}
await DatabaseHelper.NonQueryAsync(ConnStr, sb.ToString());
_output.WriteLine($"Dropped {indexes.Length} indexes from {database}");
}
}

6
TestProject1/MesDatabaseHelper.cs
View File

@@ -1,4 +1,6 @@
namespace TestProject1;
using ConsoleApp2.Helpers.Database;
namespace TestProject1;
public static class MesDatabaseHelper
{
@@ -6,7 +8,7 @@ public static class MesDatabaseHelper
{
foreach (var db in dbNames)
{
var result = await DatabaseHelper.QueryScalar(connStr + $"Database={db};", scalarQuery);
var result = await DatabaseHelper.QueryScalarAsync(connStr + $"Database={db};", scalarQuery);
if (result is null or 0) continue;
var companyId = Convert.ToInt32(result);

24
TestProject1/TableIndex.cs Normal file
View File

@@ -0,0 +1,24 @@
namespace TestProject1;
public record TableIndex(string TableName, string IndexName, bool IsUnique, string ColumnName, TableIndex.TableIndexType IndexType)
{
public enum TableIndexType
{
BTree,
Hash,
Primary
}
public void Deconstruct(out string tableName, out string indexName)
{
tableName = TableName;
indexName = IndexName;
}
public void Deconstruct(out string tableName, out string indexName, out string columnName)
{
tableName = TableName;
indexName = IndexName;
columnName = ColumnName;
}
}