using ConsoleApp2.Const; using ConsoleApp2.Helpers; using ConsoleApp2.HostedServices; using ConsoleApp2.HostedServices.Abstractions; using ConsoleApp2.Options; using ConsoleApp2.Services; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using System.Text.RegularExpressions; namespace ConsoleApp2.SimulationService { public class SimulationInputService : IInputService { private readonly ILogger _logger; private readonly IOptions _dataInputOptions; private readonly IOptions _tableOptions; private readonly DataRecordQueue _producerQueue; private readonly ProcessContext _context; public SimulationInputService(ILogger logger, IOptions dataInputOptions, IOptions tableOptions, [FromKeyedServices(ProcessStep.Producer)] DataRecordQueue producerQueue, ProcessContext context) { _logger = logger; _dataInputOptions = dataInputOptions; _tableOptions = tableOptions; _producerQueue = producerQueue; _context = context; } public async Task ExecuteAsync(CancellationToken cancellationToken) { var inputDir = _dataInputOptions.Value.InputDir; _logger.LogInformation("***** simulation input service start, working dir: {InputDir}, thread id: {ThreadId} *****", inputDir, Environment.CurrentManagedThreadId); var files = Directory.GetFiles(inputDir); if (files.Length == 0) { _logger.LogInformation("No source files found in {InputDir}", inputDir); return; } foreach (var tableName in _tableOptions.Value.TableInfoConfig.Keys) { //_logger.LogInformation("Working sql file: {SqlPath}", sqlPath); //var headers = await DumpDataHelper.GetCsvHeadersFromSqlFileAsync(sqlPath); //var sqlFileSource = _dataInputOptions.Value.CreateSource?.Invoke(sqlPath,null); //var headers =await sqlFileSource?.GetHeaders(); //var csvFiles = await DumpDataHelper.GetCsvFileNamesFromSqlFileAsync(sqlPath); //var csvFiles =await sqlFileSource?.GetCsvFiles(); //foreach (var csvFile in csvFiles) //{ //var csvPath = Path.Combine(inputDir, csvFile); //// var source = new JsvSource(csvPath, headers, _logger); //var source = new CsvSource(csvPath, headers, _csvOptions.Value.Delimiter, _csvOptions.Value.QuoteChar, _logger); //while (await source.ReadAsync()) //{ // _context.AddInput(); // _producerQueue.Enqueue(source.Current); // if (cancellationToken.IsCancellationRequested) // return; //} //var csvPath = Path.Combine(inputDir, csvFile); //var tableName = DumpDataHelper.GetTableName(csvPath); //var dataCount = 1200000000L;//当前表要生成的总数据量 var dataCount = _tableOptions.Value.TableInfoConfig[tableName].SimulaRowCount;//当前表要生成的总数据量 var companyTotallCount = 1000;//当前表每个公司生成的总数据量 var tempRecords = new List(); var sk = DataHelper.shareKeys.First(); var companyID = DataHelper.companyIds.First(); var shareKeyInterval = 20000;//每个sharekey的数据量 var getShareKeyTimes = 0;//sharekey生成的次数,每生成一次,改变sharekey的值 var getCompanyIDTimes = 0;//公司生成的次数,每生成一次,改变companyID的值 var shareKeyIntervalCount = 0; //CsvSource source; //switch (_dataInputOptions.Value.FileType) //{ // case InputFileType.CSV: // source=new CsvSource(csvPath, headers, _csvOptions.Value.Delimiter, _csvOptions.Value.QuoteChar, _logger); // break; // case InputFileType.JWT: // source = new JwtSource(csvPath, headers, _csvOptions.Value.Delimiter, _csvOptions.Value.QuoteChar, _logger); // break; // default: break; //} //var source = new JwtSource(csvPath, headers, _csvOptions.Value.Delimiter, _csvOptions.Value.QuoteChar, _logger); var source = _dataInputOptions.Value.CreateSource?.Invoke(tableName); var testRecord =await source.GetTestRecord(); for (long i = 1; i <= dataCount; i++) { shareKeyIntervalCount++; if (shareKeyIntervalCount > shareKeyInterval) { sk = DataHelper.GetShareKey(getShareKeyTimes); getShareKeyTimes++; shareKeyIntervalCount = 0; } var fields = new string[testRecord.Fields.Length]; Array.Copy(testRecord.Fields, fields, testRecord.Fields.Length); var record = new DataRecord(fields, testRecord.TableName, testRecord.Headers, companyID); //更新record的ID、OrderNo,ShardKey值 if (record.Headers.Contains("ID")) { var index = Array.IndexOf(record.Headers, "ID"); if (index > -1) { record.Fields[index] = i.ToString(); } } if (record.TableName == "order_box_block" && record.Headers.Contains("BoxID")) { var index = Array.IndexOf(record.Headers, "BoxID"); if (index > -1) { record.Fields[index] = i.ToString(); } } if ((record.TableName == "order_block_plan_item" || record.TableName == "order_package_item") && record.Headers.Contains("ItemID")) { var index = Array.IndexOf(record.Headers, "ItemID"); if (index > -1) { record.Fields[index] = i.ToString(); } } if (record.TableName == "order" && record.Headers.Contains("OrderNo")) { var index = Array.IndexOf(record.Headers, "OrderNo"); if (index > -1) { record.Fields[index] = i.ToString(); } } if (record.Headers.Contains("ShardKey")) { var index = Array.IndexOf(record.Headers, "ShardKey"); if (index > -1) { record.Fields[index] = sk.ToString(); } } tempRecords.Add(record); if (tempRecords.Count >= companyTotallCount || i >= dataCount - 1) { foreach (var rc in tempRecords) { _context.AddInput(); _producerQueue.Enqueue(rc); if (cancellationToken.IsCancellationRequested) return; } tempRecords.Clear(); companyID = DataHelper.GetCompanyId(getCompanyIDTimes); getCompanyIDTimes++; } } _logger.LogInformation("table:'{tableName}' simulation input completed", tableName); //} //_logger.LogInformation("File '{File}' input completed", Path.GetFileName(sqlPath)); } _context.CompleteInput(); _logger.LogInformation("***** Csv input service completed *****"); } } }