修改
This commit is contained in:
28
ConsoleApp2/SimulationService/DataHelper.cs
Normal file
28
ConsoleApp2/SimulationService/DataHelper.cs
Normal file
@@ -0,0 +1,28 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace ConsoleApp2.SimulationService
|
||||
{
|
||||
public static partial class DataHelper
|
||||
{
|
||||
public static short[] shareKeys = {23000, 23040, 23070, 23100, 24000, 24040, 24070, 24100, 25000, 25040, 25070, 25100 };
|
||||
public static int[] companyIds = { 1, 2, 3, 4 };
|
||||
private static T getArrayValue<T>(int index, T[] array)//按index取数据,超过数组长度,index从0开始再取
|
||||
{
|
||||
return array[index % array.Length];
|
||||
}
|
||||
public static short GetShareKey(int index)
|
||||
{
|
||||
return getArrayValue(index, shareKeys);
|
||||
}
|
||||
public static int GetCompanyId(int index)
|
||||
{
|
||||
return getArrayValue(index, companyIds);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
173
ConsoleApp2/SimulationService/SimulationInputService.cs
Normal file
173
ConsoleApp2/SimulationService/SimulationInputService.cs
Normal file
@@ -0,0 +1,173 @@
|
||||
using ConsoleApp2.Const;
|
||||
using ConsoleApp2.Helpers;
|
||||
using ConsoleApp2.HostedServices;
|
||||
using ConsoleApp2.HostedServices.Abstractions;
|
||||
using ConsoleApp2.Options;
|
||||
using ConsoleApp2.Services;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace ConsoleApp2.SimulationService
|
||||
{
|
||||
|
||||
public class SimulationInputService : IInputService
|
||||
{
|
||||
private readonly ILogger _logger;
|
||||
private readonly IOptions<DataInputOptions> _dataInputOptions;
|
||||
private readonly IOptions<InputTableOptions> _tableOptions;
|
||||
private readonly DataRecordQueue _producerQueue;
|
||||
private readonly ProcessContext _context;
|
||||
|
||||
public SimulationInputService(ILogger<InputService> logger,
|
||||
IOptions<DataInputOptions> dataInputOptions,
|
||||
IOptions<InputTableOptions> tableOptions,
|
||||
[FromKeyedServices(ProcessStep.Producer)] DataRecordQueue producerQueue,
|
||||
ProcessContext context)
|
||||
{
|
||||
_logger = logger;
|
||||
_dataInputOptions = dataInputOptions;
|
||||
_tableOptions = tableOptions;
|
||||
_producerQueue = producerQueue;
|
||||
_context = context;
|
||||
}
|
||||
public async Task ExecuteAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var inputDir = _dataInputOptions.Value.InputDir;
|
||||
_logger.LogInformation("***** simulation input service start, working dir: {InputDir}, thread id: {ThreadId} *****", inputDir, Environment.CurrentManagedThreadId);
|
||||
var files = Directory.GetFiles(inputDir);
|
||||
if (files.Length == 0)
|
||||
{
|
||||
_logger.LogInformation("No source files found in {InputDir}", inputDir);
|
||||
return;
|
||||
}
|
||||
foreach (var tableName in _tableOptions.Value.TableInfoConfig.Keys)
|
||||
{
|
||||
//_logger.LogInformation("Working sql file: {SqlPath}", sqlPath);
|
||||
//var headers = await DumpDataHelper.GetCsvHeadersFromSqlFileAsync(sqlPath);
|
||||
//var sqlFileSource = _dataInputOptions.Value.CreateSource?.Invoke(sqlPath,null);
|
||||
//var headers =await sqlFileSource?.GetHeaders();
|
||||
//var csvFiles = await DumpDataHelper.GetCsvFileNamesFromSqlFileAsync(sqlPath);
|
||||
//var csvFiles =await sqlFileSource?.GetCsvFiles();
|
||||
|
||||
//foreach (var csvFile in csvFiles)
|
||||
//{
|
||||
//var csvPath = Path.Combine(inputDir, csvFile);
|
||||
//// var source = new JsvSource(csvPath, headers, _logger);
|
||||
//var source = new CsvSource(csvPath, headers, _csvOptions.Value.Delimiter, _csvOptions.Value.QuoteChar, _logger);
|
||||
|
||||
//while (await source.ReadAsync())
|
||||
//{
|
||||
// _context.AddInput();
|
||||
// _producerQueue.Enqueue(source.Current);
|
||||
// if (cancellationToken.IsCancellationRequested)
|
||||
// return;
|
||||
//}
|
||||
//var csvPath = Path.Combine(inputDir, csvFile);
|
||||
//var tableName = DumpDataHelper.GetTableName(csvPath);
|
||||
|
||||
//var dataCount = 1200000000L;//当前表要生成的总数据量
|
||||
var dataCount = _tableOptions.Value.TableInfoConfig[tableName].SimulaRowCount;//当前表要生成的总数据量
|
||||
var companyTotallCount = 1000;//当前表每个公司生成的总数据量
|
||||
var tempRecords = new List<DataRecord>();
|
||||
var sk = DataHelper.shareKeys.First();
|
||||
var companyID = DataHelper.companyIds.First();
|
||||
|
||||
var shareKeyInterval = 20000;//每个sharekey的数据量
|
||||
var getShareKeyTimes = 0;//sharekey生成的次数,每生成一次，改变sharekey的值
|
||||
var getCompanyIDTimes = 0;//公司生成的次数,每生成一次，改变companyID的值
|
||||
var shareKeyIntervalCount = 0;
|
||||
//CsvSource source;
|
||||
//switch (_dataInputOptions.Value.FileType)
|
||||
//{
|
||||
// case InputFileType.CSV:
|
||||
// source=new CsvSource(csvPath, headers, _csvOptions.Value.Delimiter, _csvOptions.Value.QuoteChar, _logger);
|
||||
// break;
|
||||
// case InputFileType.JWT:
|
||||
// source = new JwtSource(csvPath, headers, _csvOptions.Value.Delimiter, _csvOptions.Value.QuoteChar, _logger);
|
||||
// break;
|
||||
// default: break;
|
||||
|
||||
//}
|
||||
//var source = new JwtSource(csvPath, headers, _csvOptions.Value.Delimiter, _csvOptions.Value.QuoteChar, _logger);
|
||||
var source = _dataInputOptions.Value.CreateSource?.Invoke(tableName);
|
||||
var testRecord =await source.GetTestRecord();
|
||||
for (long i = 1; i <= dataCount; i++)
|
||||
{
|
||||
shareKeyIntervalCount++;
|
||||
if (shareKeyIntervalCount > shareKeyInterval)
|
||||
{
|
||||
sk = DataHelper.GetShareKey(getShareKeyTimes);
|
||||
getShareKeyTimes++;
|
||||
shareKeyIntervalCount = 0;
|
||||
}
|
||||
var fields = new string[testRecord.Fields.Length];
|
||||
Array.Copy(testRecord.Fields, fields, testRecord.Fields.Length);
|
||||
var record = new DataRecord(fields, testRecord.TableName, testRecord.Headers, companyID);
|
||||
//更新record的ID、OrderNo,ShardKey值
|
||||
if (record.Headers.Contains("ID"))
|
||||
{
|
||||
var index = Array.IndexOf(record.Headers, "ID");
|
||||
if (index > -1)
|
||||
{
|
||||
record.Fields[index] = i.ToString();
|
||||
}
|
||||
}
|
||||
if (record.TableName == "order_box_block" && record.Headers.Contains("BoxID"))
|
||||
{
|
||||
var index = Array.IndexOf(record.Headers, "BoxID");
|
||||
if (index > -1)
|
||||
{
|
||||
record.Fields[index] = i.ToString();
|
||||
}
|
||||
}
|
||||
if ((record.TableName == "order_block_plan_item" || record.TableName == "order_package_item") && record.Headers.Contains("ItemID"))
|
||||
{
|
||||
var index = Array.IndexOf(record.Headers, "ItemID");
|
||||
if (index > -1)
|
||||
{
|
||||
record.Fields[index] = i.ToString();
|
||||
}
|
||||
}
|
||||
if (record.TableName == "order" && record.Headers.Contains("OrderNo"))
|
||||
{
|
||||
var index = Array.IndexOf(record.Headers, "OrderNo");
|
||||
if (index > -1)
|
||||
{
|
||||
record.Fields[index] = i.ToString();
|
||||
}
|
||||
}
|
||||
if (record.Headers.Contains("ShardKey"))
|
||||
{
|
||||
var index = Array.IndexOf(record.Headers, "ShardKey");
|
||||
if (index > -1)
|
||||
{
|
||||
record.Fields[index] = sk.ToString();
|
||||
}
|
||||
}
|
||||
tempRecords.Add(record);
|
||||
if (tempRecords.Count >= companyTotallCount || i >= dataCount - 1)
|
||||
{
|
||||
foreach (var rc in tempRecords)
|
||||
{
|
||||
_context.AddInput();
|
||||
_producerQueue.Enqueue(rc);
|
||||
if (cancellationToken.IsCancellationRequested)
|
||||
return;
|
||||
}
|
||||
tempRecords.Clear();
|
||||
companyID = DataHelper.GetCompanyId(getCompanyIDTimes);
|
||||
getCompanyIDTimes++;
|
||||
}
|
||||
}
|
||||
_logger.LogInformation("table:'{tableName}' simulation input completed", tableName);
|
||||
//}
|
||||
//_logger.LogInformation("File '{File}' input completed", Path.GetFileName(sqlPath));
|
||||
}
|
||||
|
||||
_context.CompleteInput();
|
||||
_logger.LogInformation("***** Csv input service completed *****");
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user