MES/MES-Toolkit
6
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

优化项目结构，新增export命令

Browse Source
This commit is contained in:
xief 2022-01-18 09:37:36 +08:00
parent 66569ddec1
commit de141de3bd
10 changed files with 396 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.gitignore vendored
View File

@ -360,4 +360,8 @@ MigrationBackup/
.ionide/
# Fody - auto-generated XML schema
FodyWeavers.xsd
FodyWeavers.xsd
.vscode
/src/Archiver/Properties/launchSettings.json

13
Chenfeng.MES.Archiver/Properties/launchSettings.json
View File

@ -1,13 +0,0 @@
{
"profiles": {
"WSL": {
"commandName": "WSL2",
"environmentVariables": {},
"distributionName": ""
},
"copy-table": {
"commandName": "Project",
"commandLineArgs": "Server=192.168.1.127;UserId=root;Password=ruixinjie!@#123;Database=cferp_test;Port=3306;"
}
}
}

2
MES-Toolkit.sln
View File

@ -3,7 +3,7 @@ Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17
VisualStudioVersion = 17.0.32014.148
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Chenfeng.MES.Archiver", "Chenfeng.MES.Archiver\Chenfeng.MES.Archiver.csproj", "{D42E639F-0E4D-4CE3-9793-230D6B07C3FE}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Archiver", "src\Archiver\Archiver.csproj", "{D42E639F-0E4D-4CE3-9793-230D6B07C3FE}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution

12
Chenfeng.MES.Archiver/Chenfeng.MES.Archiver.csproj → src/Archiver/Archiver.csproj
View File

@ -9,16 +9,10 @@
<ItemGroup>
<PackageReference Include="CliFx" Version="2.1.0" />
<PackageReference Include="CliFx" Version="2.2.1" />
<PackageReference Include="Dapper" Version="2.0.123" />
<PackageReference Include="MySqlConnector" Version="2.1.2" />
</ItemGroup>
<ItemGroup>
<Content Update="appsettings.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
</Project>

91
Chenfeng.MES.Archiver/Commands/MysqlCommand.cs → src/Archiver/Commands/CopyTableCommand.cs
View File

@ -10,11 +10,13 @@ using CliFx.Infrastructure;
using CliFx.Attributes;
using System.Data;
using System.Data.Common;
using Chenfeng.MES.Archiver.Core;
using Chenfeng.MES.Archiver.Data;
namespace Chenfeng.MES.Archiver.Commands
{
[Command()]
public class MysqlCommand : ICommand
[Command("copy")]
public class CopyTableCommand : ICommand,IArchiveReader
{
[CommandParameter(0, Description = "数据库连接")]
public string Connection { get; set; } = "";
@ -74,11 +76,11 @@ namespace Chenfeng.MES.Archiver.Commands
Db.Open();
if (Legacy)
{
CopyMesLegacyTables(starMonth);
TableQueryHelper.MesLegacyTables(this,starMonth);
}
else
{
CopyMesTables(starMonth);
TableQueryHelper.MesTables(this,starMonth);
}
@ -97,82 +99,20 @@ namespace Chenfeng.MES.Archiver.Commands
return default;
}
private void CopyMesTables(string starMonth)
public void ExecuteData(string table, string where, object param)
{
CopyStep((table, newTable) =>
{
CopyData(table, newTable, "LEFT(orderno,6) >= @starMonth", new { starMonth });
}, "order", "order_box_block", "order_data_block", "order_data_goods", "order_data_parts", "order_item", "order_module_extra", "order_module_item", "order_package");
CopyStep((tProcess, newProcess) =>
{
CopyData(tProcess, newProcess, "LEFT(orderno,6) >= @starMonth", new { starMonth });
CopyStep((table, newTable) =>
{
CopyChildData(table, newTable, newProcess, "parent.ID = child.OrderProcessID");
}, "order_process_step_item", "order_process_step");
}, "order_process");
CopyStep((table, newTable) =>
{
CopyData(table, newTable, "LEFT(DATE_FORMAT(CreateTime,'%Y%m'),6) >= @starMonth", new { starMonth });
}, "simple_plan_order");
CopyStep((parent, newParent) =>
{
CopyData(parent, newParent, "LEFT(DATE_FORMAT(CreateTime,'%Y%m'),6) >= @starMonth", new { starMonth });
CopyStep((child, newChild) =>
{
CopyChildData(child, newChild, newParent, "parent.ID = child.ID");
}, "order_block_plan_result");
}, "order_block_plan");
//CopyStep((table, newTable) =>
//{
// CopyData(table, newTable, "concat('20',left(id,4)) >= @starMonth ", new { starMonth });
//}, "order_scrap_board");
Db.Execute($"INSERT INTO `{TableDic[table]}` SELECT * FROM `{table}` WHERE {where}", param);
}
private void CopyMesLegacyTables(string starMonth)
public void ExecuteChildData(string table, string parentTable, string condition)
{
CopyStep((order, newOrder) =>
{
CopyData(order, newOrder, "LEFT(DATE_FORMAT(SaleDate,'%Y%m'),6) >= @starMonth", new { starMonth });
CopyStep((child, newChild) =>
{
CopyChildData(child, newChild, newOrder, "parent.orderno=child.orderno");
}, "SaleOrderItem", "saleblock", "saleobject", "saleorderobject", "saleorderoffer", "saleorderpackage", "saleblockobjids", "saleobjectobjids", "saleorder_block_cadmodelinfo", "saleorderblockbaseposition",
"saleordergoodsinfo", "saleorder_block_point", "saleorder_block_pointinfo", "orderprocess", "orderprocessstep", "orderprocessstepitem");
}, "saleorder");
CopyStep((plan, newPlan) =>
{
CopyData(plan, newPlan, "LEFT(DATE_FORMAT(createtime,'%Y%m'),6) >= @starMonth", new { starMonth });
CopyStep((child, newChild) =>
{
CopyChildData(child, newChild, newPlan, "parent.id=child.id");
}, "orderblockprocessplanplaceresult");
}, "orderblockprocessplan");
Db.Execute($"INSERT INTO `{TableDic[table]}` SELECT child.* FROM `{TableDic[parentTable]}` parent join `{table}` child on {condition} ");
}
private void CopyData(string table, string newTable, string where, object param)
{
Db.Execute($"INSERT INTO `{newTable}` SELECT * FROM `{table}` WHERE {where}", param);
}
private void CopyChildData(string table, string newTable, string parentTable, string condition)
{
Db.Execute($"INSERT INTO `{newTable}` SELECT child.* FROM `{parentTable}` parent join `{table}` child on {condition} ");
}
private void CopyStep(Action<string, string> step, params string[] tables)
public Dictionary<string,string> TableDic { get; set; } = new Dictionary<string,string>();
public void Step(Action<string> action, params string[] tables)
{
var list = tables.Select(table => new
{
@ -193,11 +133,13 @@ namespace Chenfeng.MES.Archiver.Commands
}
else
{
TableDic[item.Table] = item.NewTable;
Db.Execute($"DROP TABLE IF EXISTS `{item.NewTable}`");
Db.Execute($"CREATE TABLE `{item.NewTable}` LIKE `{item.Table}`"); // IF NOT EXISTS
Print(item.NewTable + " 创建完成");
step(item.Table, item.NewTable);
action(item.Table);
Print(item.NewTable + " 拷贝完成");
if (Replace)
@ -207,7 +149,6 @@ namespace Chenfeng.MES.Archiver.Commands
}
}
}
}
}
}

251
src/Archiver/Commands/ExportTableCommand.cs Normal file
View File

@ -0,0 +1,251 @@
using MySqlConnector;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Dapper;
using CliFx;
using CliFx.Infrastructure;
using CliFx.Attributes;
using System.Data;
using System.Data.Common;
using Chenfeng.MES.Archiver.Core;
using Chenfeng.MES.Archiver.Data;
using System.Diagnostics;
namespace Chenfeng.MES.Archiver.Commands
{
[Command("export")]
public class ExportTableCommand : ICommand, IArchiveReader
{
[CommandParameter(0, Description = "数据库连接")]
public string Connection { get; set; } = "";
public MySqlConnection? Db { get; private set; }
public Action<string> Print { get; set; } = (text) => { };
[CommandOption("legacy", Description = "旧版MES")]
public bool Legacy { get; set; } = false;
[CommandOption("timeout", Description = "sql命令超时时间")]
public int Timeout { get; set; } = 3;
[CommandOption("compress", Description = "文件压缩,默认gzip")]
public string Compress { get; set; } = "gz";
[CommandOption("pagesize", Description = "查询分页")]
public int PageSize { get; set; } = 500_000;
[CommandOption("output", Description = "导出路径")]
public string Output { get; set; } = "./output";
public ValueTask ExecuteAsync(IConsole console)
{
var startTime = DateTime.Now;
Print = (text) => console.Output.WriteLine($"{DateTime.Now.ToLongTimeString()} {text}");
var connBuilder = new MySqlConnectionStringBuilder(this.Connection);
connBuilder.SslMode = MySqlSslMode.None;
connBuilder.CharacterSet = "utf8";
var starMonth = DateTime.Now.AddMonths(-12).ToString("yyyyMM");
SqlMapper.Settings.CommandTimeout = 60 * Timeout;
console.Output.WriteLine($"拷贝" + (Legacy ? "旧" : "新") + "版表数据");
if (Directory.Exists(Output) == false)
{
Directory.CreateDirectory(Output);
}
try
{
Db = new MySqlConnection(connBuilder.ConnectionString);
Db.Open();
if (Legacy)
{
TableQueryHelper.MesLegacyTables(this, starMonth);
}
else
{
TableQueryHelper.MesTables(this, starMonth);
}
Print("操作成功");
}
catch (Exception ex)
{
console.Output.WriteLine(ex.ToString());
}
finally
{
Db?.Close();
}
console.Output.WriteLine("执行耗时:" + (DateTime.Now - startTime).TotalMilliseconds+"ms");
return default;
}
public void ExecuteData(string table, string where, object param)
{
var sql = $"SELECT * FROM `{table}` WHERE {where} ";
if (PageSize > 0)
{
sql += "limit @pageIndex,@pageSize";
}
DumpData(table, sql, param);
}
public void ExecuteChildData(string table, string parentTable, string condition)
{
var parentSqlInfo = tableSqlDic[parentTable];
DumpData(table, $"SELECT child.* FROM ({parentSqlInfo.Item1}) parent join `{table}` child on {condition} ", parentSqlInfo.Item2);
}
public Dictionary<string, Tuple<string, object?>> tableSqlDic { get; set; } = new();
private void DumpData(string table, string sql, object? param = null)
{
var watch = new Stopwatch();
watch.Start();
Print(table);
Print("开始导出");
tableSqlDic[table] = new Tuple<string, object?>(sql, param);
var cmd = Db.CreateCommand();
cmd.CommandTimeout = 60 * Timeout;
cmd.CommandText = sql;
if (param != null)
{
switch (param)
{
case Dictionary<string, object> dic:
foreach (var kv in dic.AsEnumerable())
{
cmd.Parameters.AddWithValue("@" + kv.Key, kv.Value);
}
break;
default:
foreach (var property in param.GetType().GetProperties())
{
cmd.Parameters.AddWithValue("@" + property.Name, property.GetValue(param));
}
break;
}
}
var columns = new List<string>();
var columnString = new StringBuilder();
var pageIndex = 1;
cmd.Parameters.AddWithValue("@pageSize", PageSize);
cmd.Parameters.AddWithValue("@pageIndex", 0);
while (true)
{
cmd.Parameters["@pageIndex"].Value = (pageIndex - 1) * PageSize;
var valString = new StringBuilder();
using (var reader = cmd.ExecuteReader())
{
if (columnString.Length ==0)
{
for (int i = 0; i < reader.FieldCount; i++)
{
//columns.Add(reader.GetName(i));
columnString.Append($"`reader.GetName(i)`,");
}
columnString.Length--;
}
while (reader.Read())
{
var row = new List<object>();
for (int i = 0; i < reader.FieldCount; i++)
{
var val = reader.GetValue(i);
switch (val)
{
case string s:
row.Add($"'{s}'");
break;
case DateTime dt:
row.Add(dt.ToString("'yyyy-MM-dd HH:mm:ss'"));
break;
case byte[] byteList:
row.Add("0x" + string.Concat(byteList.Select(i => i.ToString("X2"))));
break;
default:
row.Add(val);
break;
}
}
valString.Append($"({string.Join(",", row)}),");
}
if (valString.Length == 0) break;
valString.Length--; // 移除最后一个逗号
var distFile = $"{Output}/{table}-p{pageIndex}.{Compress}.sql";
using var fileStream = File.OpenWrite(distFile);
Stream compressStream;
switch (Compress)
{
case "gz":
case "gzip":
compressStream = new System.IO.Compression.GZipStream(fileStream, System.IO.Compression.CompressionLevel.Fastest);
break;
case "deflate":
compressStream = new System.IO.Compression.DeflateStream(fileStream, System.IO.Compression.CompressionLevel.Fastest);
break;
case "br":
compressStream = new System.IO.Compression.BrotliStream(fileStream, System.IO.Compression.CompressionLevel.Fastest);
break;
case "none": // 不压缩
default:
compressStream = fileStream;
break;
}
using (compressStream)
{
using (StreamWriter streamWriter = new StreamWriter(compressStream))
{
streamWriter.Write($"INSERT INTO `{table}` (");
streamWriter.Write(columnString);
streamWriter.Write(") VALUES ");
streamWriter.Write(valString);
//streamWriter.Flush();
}
}
pageIndex++;
}
}
watch.Stop();
Print("导出完成,耗时" + watch.ElapsedMilliseconds+"ms");
}
public void Step(Action<string> action, params string[] tables)
{
foreach (var item in tables)
{
action(item);
}
}
}
}

26
src/Archiver/Commands/ImportSqlCommand.cs Normal file
View File

@ -0,0 +1,26 @@
using CliFx;
using CliFx.Attributes;
using CliFx.Infrastructure;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Chenfeng.MES.Archiver.Commands
{
[Command("import")]
public class ImportSqlCommand : ICommand
{
[CommandParameter(0, Description = "路径")]
public string Source { get; set; } = "";
public string Connection { get; set; } = "";
public ValueTask ExecuteAsync(IConsole console)
{
throw new NotImplementedException();
}
}
}

17
src/Archiver/Core/IArchiveReader.cs Normal file
View File

@ -0,0 +1,17 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Chenfeng.MES.Archiver.Core
{
public interface IArchiveReader
{
void ExecuteData(string table, string where, object param);
void ExecuteChildData(string table,string parentTable, string condition);
void Step(Action<string> action, params string[] tables);
}
}

77
src/Archiver/Data/TableQueryHelper.cs Normal file
View File

@ -0,0 +1,77 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Chenfeng.MES.Archiver.Data
{
public class TableQueryHelper
{
public static void MesTables(Core.IArchiveReader executor, string starMonth)
{
executor.Step((table) =>
{
executor.ExecuteData(table, "LEFT(orderno,6) >= @starMonth", new { starMonth });
}, "order", "order_box_block", "order_data_block", "order_data_goods", "order_data_parts", "order_item", "order_module_extra", "order_module_item", "order_package");
executor.Step((tProcess) =>
{
executor.ExecuteData(tProcess, "LEFT(orderno,6) >= @starMonth", new { starMonth });
executor.Step((table) =>
{
executor.ExecuteChildData(table, tProcess, "parent.ID = child.OrderProcessID");
}, "order_process_step_item", "order_process_step");
}, "order_process");
executor.Step((table) =>
{
executor.ExecuteData(table, "LEFT(DATE_FORMAT(CreateTime,'%Y%m'),6) >= @starMonth", new { starMonth });
}, "simple_plan_order");
executor.Step((parent) =>
{
executor.ExecuteData(parent, "LEFT(DATE_FORMAT(CreateTime,'%Y%m'),6) >= @starMonth", new { starMonth });
executor.Step((child) =>
{
executor.ExecuteChildData(child, parent, "parent.ID = child.ID");
}, "order_block_plan_result");
}, "order_block_plan");
//CopyStep((table, newTable) =>
//{
// CopyData(table, newTable, "concat('20',left(id,4)) >= @starMonth ", new { starMonth });
//}, "order_scrap_board");
}
public static void MesLegacyTables(Core.IArchiveReader executor, string starMonth)
{
executor.Step((order) =>
{
executor.ExecuteData(order, "LEFT(DATE_FORMAT(SaleDate,'%Y%m'),6) >= @starMonth", new { starMonth });
executor.Step((child) =>
{
executor.ExecuteChildData(child, order, "parent.orderno=child.orderno");
}, "SaleOrderItem", "saleblock", "saleobject", "saleorderobject", "saleorderoffer", "saleorderpackage", "saleblockobjids", "saleobjectobjids", "saleorder_block_cadmodelinfo", "saleorderblockbaseposition",
"saleordergoodsinfo", "saleorder_block_point", "saleorder_block_pointinfo", "orderprocess", "orderprocessstep", "orderprocessstepitem");
}, "saleorder");
executor.Step((plan) =>
{
executor.ExecuteData(plan, "LEFT(DATE_FORMAT(createtime,'%Y%m'),6) >= @starMonth", new { starMonth });
executor.Step((child) =>
{
executor.ExecuteChildData(child, plan, "parent.id=child.id");
}, "orderblockprocessplanplaceresult");
}, "orderblockprocessplan");
}
}
}

0
Chenfeng.MES.Archiver/Program.cs → src/Archiver/Program.cs
View File