CZY/MES-ETL
1
0
Fork 0
Code Pull Requests Releases Activity

目标数据库增加配置

Browse Source
This commit is contained in:
lindj
2024-01-17 11:40:16 +08:00
parent e1aa621a7d
commit adb31cdc6d
4 changed files with 30 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
ConsoleApp2/SimulationService/SimulationInputService.cs
View File

@@ -44,30 +44,6 @@ namespace ConsoleApp2.SimulationService
}
foreach (var tableName in _tableOptions.Value.TableInfoConfig.Keys)
{
//_logger.LogInformation("Working sql file: {SqlPath}", sqlPath);
//var headers = await DumpDataHelper.GetCsvHeadersFromSqlFileAsync(sqlPath);
//var sqlFileSource = _dataInputOptions.Value.CreateSource?.Invoke(sqlPath,null);
//var headers =await sqlFileSource?.GetHeaders();
//var csvFiles = await DumpDataHelper.GetCsvFileNamesFromSqlFileAsync(sqlPath);
//var csvFiles =await sqlFileSource?.GetCsvFiles();
//foreach (var csvFile in csvFiles)
//{
//var csvPath = Path.Combine(inputDir, csvFile);
//// var source = new JsvSource(csvPath, headers, _logger);
//var source = new CsvSource(csvPath, headers, _csvOptions.Value.Delimiter, _csvOptions.Value.QuoteChar, _logger);
//while (await source.ReadAsync())
//{
// _context.AddInput();
// _producerQueue.Enqueue(source.Current);
// if (cancellationToken.IsCancellationRequested)
// return;
//}
//var csvPath = Path.Combine(inputDir, csvFile);
//var tableName = DumpDataHelper.GetTableName(csvPath);
//var dataCount = 1200000000L;//当前表要生成的总数据量
var dataCount = _tableOptions.Value.TableInfoConfig[tableName].SimulaRowCount;//当前表要生成的总数据量
var companyTotallCount = 1000;//当前表每个公司生成的总数据量
var tempRecords = new List<DataRecord>();
@@ -78,19 +54,7 @@ namespace ConsoleApp2.SimulationService
var getShareKeyTimes = 0;//sharekey生成的次数,每生成一次，改变sharekey的值
var getCompanyIDTimes = 0;//公司生成的次数,每生成一次，改变companyID的值
var shareKeyIntervalCount = 0;
//CsvSource source;
//switch (_dataInputOptions.Value.FileType)
//{
// case InputFileType.CSV:
// source=new CsvSource(csvPath, headers, _csvOptions.Value.Delimiter, _csvOptions.Value.QuoteChar, _logger);
// break;
// case InputFileType.JWT:
// source = new JwtSource(csvPath, headers, _csvOptions.Value.Delimiter, _csvOptions.Value.QuoteChar, _logger);
// break;
// default: break;
//}
//var source = new JwtSource(csvPath, headers, _csvOptions.Value.Delimiter, _csvOptions.Value.QuoteChar, _logger);
var source = _dataInputOptions.Value.CreateSource?.Invoke(tableName);
var testRecord =await source.GetTestRecord();
for (long i = 1; i <= dataCount; i++)