优化项目结构，新增export命令

.gitignore

@@ -360,4 +360,8 @@ MigrationBackup/
 .ionide/
 
 # Fody - auto-generated XML schema
-FodyWeavers.xsd
+FodyWeavers.xsd
+
+.vscode
+
+/src/Archiver/Properties/launchSettings.json

Chenfeng.MES.Archiver/Properties/launchSettings.json

@@ -1,13 +0,0 @@
-{
-  "profiles": {
-    "WSL": {
-      "commandName": "WSL2",
-      "environmentVariables": {},
-      "distributionName": ""
-    },
-    "copy-table": {
-      "commandName": "Project",
-      "commandLineArgs": "Server=192.168.1.127;UserId=root;Password=ruixinjie!@#123;Database=cferp_test;Port=3306;"
-    }
-  }
-}

MES-Toolkit.sln

@@ -3,7 +3,7 @@ Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
 VisualStudioVersion = 17.0.32014.148
 MinimumVisualStudioVersion = 10.0.40219.1
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Chenfeng.MES.Archiver", "Chenfeng.MES.Archiver\Chenfeng.MES.Archiver.csproj", "{D42E639F-0E4D-4CE3-9793-230D6B07C3FE}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Archiver", "src\Archiver\Archiver.csproj", "{D42E639F-0E4D-4CE3-9793-230D6B07C3FE}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution

src/Archiver/Archiver.csproj

@@ -9,16 +9,10 @@
 
 
 	<ItemGroup>
-		<PackageReference Include="CliFx" Version="2.1.0" />
+		<PackageReference Include="CliFx" Version="2.2.1" />
 		<PackageReference Include="Dapper" Version="2.0.123" />
 		<PackageReference Include="MySqlConnector" Version="2.1.2" />
 	</ItemGroup>
-
+	
-
+	
-	<ItemGroup>
-		<Content Update="appsettings.json">
-			<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-		</Content>
-	</ItemGroup>
-
 </Project>

src/Archiver/Commands/CopyTableCommand.cs

@@ -10,11 +10,13 @@ using CliFx.Infrastructure;
 using CliFx.Attributes;
 using System.Data;
 using System.Data.Common;
+using Chenfeng.MES.Archiver.Core;
+using Chenfeng.MES.Archiver.Data;
 
 namespace Chenfeng.MES.Archiver.Commands
 {
-    [Command()]
+    [Command("copy")]
-    public class MysqlCommand : ICommand
+    public class CopyTableCommand : ICommand,IArchiveReader
     {
         [CommandParameter(0, Description = "数据库连接")]
         public string Connection { get; set; } = "";
@@ -74,11 +76,11 @@ namespace Chenfeng.MES.Archiver.Commands
                 Db.Open();
                 if (Legacy)
                 {
-                    CopyMesLegacyTables(starMonth);
+                    TableQueryHelper.MesLegacyTables(this,starMonth);
                 }
                 else
                 {
-                    CopyMesTables(starMonth);
+                    TableQueryHelper.MesTables(this,starMonth);
                 }
 
 
@@ -97,82 +99,20 @@ namespace Chenfeng.MES.Archiver.Commands
             return default;
         }
 
-        private void CopyMesTables(string starMonth)
+        public void ExecuteData(string table, string where, object param)
         {
-            CopyStep((table, newTable) =>
+            Db.Execute($"INSERT INTO `{TableDic[table]}` SELECT * FROM `{table}` WHERE {where}", param);
-            {
-                CopyData(table, newTable, "LEFT(orderno,6) >= @starMonth", new { starMonth });
-            }, "order", "order_box_block", "order_data_block", "order_data_goods", "order_data_parts", "order_item", "order_module_extra", "order_module_item", "order_package");
-
-
-            CopyStep((tProcess, newProcess) =>
-            {
-                CopyData(tProcess, newProcess, "LEFT(orderno,6) >= @starMonth", new { starMonth });
-                CopyStep((table, newTable) =>
-                {
-                    CopyChildData(table, newTable, newProcess, "parent.ID = child.OrderProcessID");
-                }, "order_process_step_item", "order_process_step");
-            }, "order_process");
-
-
-
-            CopyStep((table, newTable) =>
-            {
-                CopyData(table, newTable, "LEFT(DATE_FORMAT(CreateTime,'%Y%m'),6) >= @starMonth", new { starMonth });
-            }, "simple_plan_order");
-
-
-            CopyStep((parent, newParent) =>
-            {
-                CopyData(parent, newParent, "LEFT(DATE_FORMAT(CreateTime,'%Y%m'),6) >= @starMonth", new { starMonth });
-                CopyStep((child, newChild) =>
-                {
-                    CopyChildData(child, newChild, newParent, "parent.ID = child.ID");
-                }, "order_block_plan_result");
-            }, "order_block_plan");
-
-            //CopyStep((table, newTable) =>
-            //{
-            //    CopyData(table, newTable, "concat('20',left(id,4)) >= @starMonth ", new { starMonth });
-            //}, "order_scrap_board");
         }
 
-        private void CopyMesLegacyTables(string starMonth)
+        public void ExecuteChildData(string table, string parentTable, string condition)
         {
-            CopyStep((order, newOrder) =>
+            Db.Execute($"INSERT INTO `{TableDic[table]}`  SELECT child.* FROM `{TableDic[parentTable]}` parent join `{table}` child on {condition} ");
-            {
-                CopyData(order, newOrder, "LEFT(DATE_FORMAT(SaleDate,'%Y%m'),6) >= @starMonth", new { starMonth });
-
-                CopyStep((child, newChild) =>
-                {
-                    CopyChildData(child, newChild, newOrder, "parent.orderno=child.orderno");
-                }, "SaleOrderItem", "saleblock", "saleobject", "saleorderobject", "saleorderoffer", "saleorderpackage", "saleblockobjids", "saleobjectobjids", "saleorder_block_cadmodelinfo", "saleorderblockbaseposition",
-            "saleordergoodsinfo", "saleorder_block_point", "saleorder_block_pointinfo", "orderprocess", "orderprocessstep", "orderprocessstepitem");
-
-            }, "saleorder");
-
-
-            CopyStep((plan, newPlan) =>
-            {
-                CopyData(plan, newPlan, "LEFT(DATE_FORMAT(createtime,'%Y%m'),6) >= @starMonth", new { starMonth });
-
-                CopyStep((child, newChild) =>
-                {
-                    CopyChildData(child, newChild, newPlan, "parent.id=child.id");
-                }, "orderblockprocessplanplaceresult");
-            }, "orderblockprocessplan");
         }
 
-        private void CopyData(string table, string newTable, string where, object param)
-        {
-            Db.Execute($"INSERT INTO `{newTable}` SELECT * FROM `{table}` WHERE {where}", param);
-        }
-        private void CopyChildData(string table, string newTable, string parentTable, string condition)
-        {
-            Db.Execute($"INSERT INTO `{newTable}`  SELECT child.* FROM `{parentTable}` parent join `{table}` child on {condition} ");
-        }
 
-        private void CopyStep(Action<string, string> step, params string[] tables)
+        public Dictionary<string,string> TableDic { get; set; } = new Dictionary<string,string>();
+
+        public void Step(Action<string> action, params string[] tables)
         {
             var list = tables.Select(table => new
             {
@@ -193,11 +133,13 @@ namespace Chenfeng.MES.Archiver.Commands
                 }
                 else
                 {
+                    TableDic[item.Table] = item.NewTable; 
+
                     Db.Execute($"DROP TABLE IF EXISTS `{item.NewTable}`");
                     Db.Execute($"CREATE TABLE `{item.NewTable}` LIKE `{item.Table}`"); // IF NOT EXISTS
 
                     Print(item.NewTable + " 创建完成");
-                    step(item.Table, item.NewTable);
+                    action(item.Table);
                     Print(item.NewTable + " 拷贝完成");
 
                     if (Replace)
@@ -207,7 +149,6 @@ namespace Chenfeng.MES.Archiver.Commands
                     }
                 }
             }
-
         }
     }
 }

src/Archiver/Commands/ExportTableCommand.cs

@@ -0,0 +1,251 @@
+using MySqlConnector;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Dapper;
+using CliFx;
+using CliFx.Infrastructure;
+using CliFx.Attributes;
+using System.Data;
+using System.Data.Common;
+using Chenfeng.MES.Archiver.Core;
+using Chenfeng.MES.Archiver.Data;
+using System.Diagnostics;
+
+namespace Chenfeng.MES.Archiver.Commands
+{
+    [Command("export")]
+    public class ExportTableCommand : ICommand, IArchiveReader
+    {
+        [CommandParameter(0, Description = "数据库连接")]
+        public string Connection { get; set; } = "";
+
+        public MySqlConnection? Db { get; private set; }
+
+        public Action<string> Print { get; set; } = (text) => { };
+
+        [CommandOption("legacy", Description = "旧版MES")]
+        public bool Legacy { get; set; } = false;
+
+        [CommandOption("timeout", Description = "sql命令超时时间")]
+        public int Timeout { get; set; } = 3;
+
+        [CommandOption("compress", Description = "文件压缩,默认gzip")]
+        public string Compress { get; set; } = "gz";
+
+        [CommandOption("pagesize", Description = "查询分页")]
+        public int PageSize { get; set; } = 500_000;
+
+        [CommandOption("output", Description = "导出路径")]
+        public string Output { get; set; } = "./output";
+
+
+
+        public ValueTask ExecuteAsync(IConsole console)
+        {
+            var startTime = DateTime.Now;
+            Print = (text) => console.Output.WriteLine($"{DateTime.Now.ToLongTimeString()} {text}");
+
+            var connBuilder = new MySqlConnectionStringBuilder(this.Connection);
+            connBuilder.SslMode = MySqlSslMode.None;
+            connBuilder.CharacterSet = "utf8";
+
+            var starMonth = DateTime.Now.AddMonths(-12).ToString("yyyyMM");
+
+            SqlMapper.Settings.CommandTimeout = 60 * Timeout;
+
+            console.Output.WriteLine($"拷贝" + (Legacy ? "旧" : "新") + "版表数据");
+
+            if (Directory.Exists(Output) == false)
+            {
+                Directory.CreateDirectory(Output);
+            }
+
+            try
+            {
+                Db = new MySqlConnection(connBuilder.ConnectionString);
+
+                Db.Open();
+                if (Legacy)
+                {
+                    TableQueryHelper.MesLegacyTables(this, starMonth);
+                }
+                else
+                {
+                    TableQueryHelper.MesTables(this, starMonth);
+                }
+
+
+                Print("操作成功");
+
+            }
+            catch (Exception ex)
+            {
+                console.Output.WriteLine(ex.ToString());
+            }
+            finally
+            {
+                Db?.Close();
+            }
+            console.Output.WriteLine("执行耗时:" + (DateTime.Now - startTime).TotalMilliseconds+"ms");
+            return default;
+        }
+
+        public void ExecuteData(string table, string where, object param)
+        {
+            var sql = $"SELECT * FROM `{table}` WHERE {where} ";
+            if (PageSize > 0)
+            {
+                sql += "limit @pageIndex,@pageSize";
+            }
+            DumpData(table, sql, param);
+        }
+
+        public void ExecuteChildData(string table, string parentTable, string condition)
+        {
+            var parentSqlInfo = tableSqlDic[parentTable];
+            DumpData(table, $"SELECT child.* FROM ({parentSqlInfo.Item1}) parent join `{table}` child on {condition} ", parentSqlInfo.Item2);
+        }
+
+        public Dictionary<string, Tuple<string, object?>> tableSqlDic { get; set; } = new();
+
+        private void DumpData(string table, string sql, object? param = null)
+        {
+            var watch = new Stopwatch();
+            watch.Start();
+            Print(table);
+            Print("开始导出");
+
+            tableSqlDic[table] = new Tuple<string, object?>(sql, param);
+            var cmd = Db.CreateCommand();
+            cmd.CommandTimeout = 60 * Timeout;
+            cmd.CommandText = sql;
+
+
+            if (param != null)
+            {
+                switch (param)
+                {
+                    case Dictionary<string, object> dic:
+                        foreach (var kv in dic.AsEnumerable())
+                        {
+                            cmd.Parameters.AddWithValue("@" + kv.Key, kv.Value);
+                        }
+                        break;
+                    default:
+                        foreach (var property in param.GetType().GetProperties())
+                        {
+                            cmd.Parameters.AddWithValue("@" + property.Name, property.GetValue(param));
+                        }
+                        break;
+                }
+            }
+
+            var columns = new List<string>();
+            var columnString = new StringBuilder();
+
+
+            var pageIndex = 1;
+            cmd.Parameters.AddWithValue("@pageSize", PageSize);
+            cmd.Parameters.AddWithValue("@pageIndex", 0);
+
+            while (true)
+            {
+                cmd.Parameters["@pageIndex"].Value = (pageIndex - 1) * PageSize;
+
+                var valString = new StringBuilder();
+
+                using (var reader = cmd.ExecuteReader())
+                {
+                    if (columnString.Length ==0)
+                    {
+                        for (int i = 0; i < reader.FieldCount; i++)
+                        {
+                            //columns.Add(reader.GetName(i));
+                            columnString.Append($"`reader.GetName(i)`,");
+                        }
+                        columnString.Length--; 
+                    }
+
+                    while (reader.Read())
+                    {
+                        var row = new List<object>();
+                        for (int i = 0; i < reader.FieldCount; i++)
+                        {
+                            var val = reader.GetValue(i);
+                            switch (val)
+                            {
+                                case string s:
+                                    row.Add($"'{s}'");
+                                    break;
+                                case DateTime dt:
+                                    row.Add(dt.ToString("'yyyy-MM-dd HH:mm:ss'"));
+                                    break;
+                                case byte[] byteList:
+                                    row.Add("0x" + string.Concat(byteList.Select(i => i.ToString("X2"))));
+                                    break;
+                                default:
+                                    row.Add(val);
+                                    break;
+                            }
+
+                        }
+                        valString.Append($"({string.Join(",", row)}),");
+                    }
+                    if (valString.Length == 0) break;
+
+                    valString.Length--; // 移除最后一个逗号
+
+                    var distFile = $"{Output}/{table}-p{pageIndex}.{Compress}.sql";
+
+                    using var fileStream = File.OpenWrite(distFile);
+                    Stream compressStream;
+                    switch (Compress)
+                    {
+                        case "gz":
+                        case "gzip":
+                            compressStream = new System.IO.Compression.GZipStream(fileStream, System.IO.Compression.CompressionLevel.Fastest);
+                            break;
+                        case "deflate":
+                            compressStream = new System.IO.Compression.DeflateStream(fileStream, System.IO.Compression.CompressionLevel.Fastest);
+                            break;
+                        case "br":
+                            compressStream = new System.IO.Compression.BrotliStream(fileStream, System.IO.Compression.CompressionLevel.Fastest);
+                            break;
+                        case "none": // 不压缩
+                        default:
+                            compressStream = fileStream;
+                            break;
+                    }
+                    using (compressStream)
+                    {
+                        using (StreamWriter streamWriter = new StreamWriter(compressStream))
+                        {
+                            streamWriter.Write($"INSERT INTO `{table}` (");
+                            streamWriter.Write(columnString);
+                            streamWriter.Write(") VALUES ");
+                            streamWriter.Write(valString);
+                            //streamWriter.Flush();
+                        }
+                    }
+
+                    pageIndex++;
+                }
+
+            }
+
+            watch.Stop();
+            Print("导出完成,耗时" + watch.ElapsedMilliseconds+"ms");
+        }
+
+        public void Step(Action<string> action, params string[] tables)
+        {
+            foreach (var item in tables)
+            {
+                action(item);
+            }
+        }
+    }
+}

src/Archiver/Commands/ImportSqlCommand.cs

@@ -0,0 +1,26 @@
+using CliFx;
+using CliFx.Attributes;
+using CliFx.Infrastructure;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Chenfeng.MES.Archiver.Commands
+{
+    [Command("import")]
+    public class ImportSqlCommand : ICommand
+    {
+        [CommandParameter(0, Description = "路径")]
+        public string Source { get; set; } = "";
+
+        public string Connection { get; set; } = "";
+
+
+        public ValueTask ExecuteAsync(IConsole console)
+        {
+            throw new NotImplementedException();
+        }
+    }
+}

src/Archiver/Core/IArchiveReader.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Chenfeng.MES.Archiver.Core
+{
+    public interface IArchiveReader
+    {
+        void ExecuteData(string table, string where, object param);
+
+        void ExecuteChildData(string table,string parentTable, string condition);
+
+        void Step(Action<string> action, params string[] tables);
+    }
+}

src/Archiver/Data/TableQueryHelper.cs

@@ -0,0 +1,77 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Chenfeng.MES.Archiver.Data
+{
+    public class TableQueryHelper
+    {
+        public static void MesTables(Core.IArchiveReader executor, string starMonth)
+        {
+            executor.Step((table) =>
+            {
+                executor.ExecuteData(table, "LEFT(orderno,6) >= @starMonth", new { starMonth });
+            }, "order", "order_box_block", "order_data_block", "order_data_goods", "order_data_parts", "order_item", "order_module_extra", "order_module_item", "order_package");
+
+
+            executor.Step((tProcess) =>
+            {
+                executor.ExecuteData(tProcess, "LEFT(orderno,6) >= @starMonth", new { starMonth });
+                executor.Step((table) =>
+                {
+                    executor.ExecuteChildData(table, tProcess, "parent.ID = child.OrderProcessID");
+                }, "order_process_step_item", "order_process_step");
+            }, "order_process");
+
+
+
+            executor.Step((table) =>
+            {
+                executor.ExecuteData(table, "LEFT(DATE_FORMAT(CreateTime,'%Y%m'),6) >= @starMonth", new { starMonth });
+            }, "simple_plan_order");
+
+
+            executor.Step((parent) =>
+            {
+                executor.ExecuteData(parent, "LEFT(DATE_FORMAT(CreateTime,'%Y%m'),6) >= @starMonth", new { starMonth });
+                executor.Step((child) =>
+                {
+                    executor.ExecuteChildData(child, parent, "parent.ID = child.ID");
+                }, "order_block_plan_result");
+            }, "order_block_plan");
+
+            //CopyStep((table, newTable) =>
+            //{
+            //    CopyData(table, newTable, "concat('20',left(id,4)) >= @starMonth ", new { starMonth });
+            //}, "order_scrap_board");
+        }
+
+        public static void MesLegacyTables(Core.IArchiveReader executor, string starMonth)
+        {
+            executor.Step((order) =>
+            {
+                executor.ExecuteData(order, "LEFT(DATE_FORMAT(SaleDate,'%Y%m'),6) >= @starMonth", new { starMonth });
+
+                executor.Step((child) =>
+                {
+                    executor.ExecuteChildData(child, order, "parent.orderno=child.orderno");
+                }, "SaleOrderItem", "saleblock", "saleobject", "saleorderobject", "saleorderoffer", "saleorderpackage", "saleblockobjids", "saleobjectobjids", "saleorder_block_cadmodelinfo", "saleorderblockbaseposition",
+            "saleordergoodsinfo", "saleorder_block_point", "saleorder_block_pointinfo", "orderprocess", "orderprocessstep", "orderprocessstepitem");
+
+            }, "saleorder");
+
+
+            executor.Step((plan) =>
+            {
+                executor.ExecuteData(plan, "LEFT(DATE_FORMAT(createtime,'%Y%m'),6) >= @starMonth", new { starMonth });
+
+                executor.Step((child) =>
+                {
+                    executor.ExecuteChildData(child, plan, "parent.id=child.id");
+                }, "orderblockprocessplanplaceresult");
+            }, "orderblockprocessplan");
+        }
+    }
+}