MES-ETL/ConsoleApp2/SimulationService/SimulationInputService.cs

using ConsoleApp2.Const;
using ConsoleApp2.Helpers;
using ConsoleApp2.HostedServices;
using ConsoleApp2.HostedServices.Abstractions;
using ConsoleApp2.Options;
using ConsoleApp2.Services;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using System.Text.RegularExpressions;

namespace ConsoleApp2.SimulationService
{

    public class SimulationInputService : IInputService
    {
        private readonly ILogger _logger;
        private readonly IOptions<DataInputOptions> _dataInputOptions;
        private readonly IOptions<InputTableOptions> _tableOptions;
        private readonly DataRecordQueue _producerQueue;
        private readonly ProcessContext _context;

        public SimulationInputService(ILogger<InputService> logger,
            IOptions<DataInputOptions> dataInputOptions,
            IOptions<InputTableOptions> tableOptions,
            [FromKeyedServices(ProcessStep.Producer)] DataRecordQueue producerQueue,
            ProcessContext context)
        {
            _logger = logger;
            _dataInputOptions = dataInputOptions;
            _tableOptions = tableOptions;
            _producerQueue = producerQueue;
            _context = context;
        }
        public async Task ExecuteAsync(CancellationToken cancellationToken)
        {
            var inputDir = _dataInputOptions.Value.InputDir;
            _logger.LogInformation("***** simulation input service start, working dir: {InputDir}, thread id: {ThreadId} *****", inputDir, Environment.CurrentManagedThreadId);
            var files = Directory.GetFiles(inputDir);
            if (files.Length == 0)
            {
                _logger.LogInformation("No source files found in {InputDir}", inputDir);
                return;
            }
            foreach (var tableName in _tableOptions.Value.TableInfoConfig.Keys)
            {
                //_logger.LogInformation("Working sql file: {SqlPath}", sqlPath);
                //var headers = await DumpDataHelper.GetCsvHeadersFromSqlFileAsync(sqlPath);
                //var sqlFileSource = _dataInputOptions.Value.CreateSource?.Invoke(sqlPath,null);
                //var headers =await sqlFileSource?.GetHeaders();
                //var csvFiles = await DumpDataHelper.GetCsvFileNamesFromSqlFileAsync(sqlPath);
                //var csvFiles =await sqlFileSource?.GetCsvFiles();

                //foreach (var csvFile in csvFiles)
                //{
                    //var csvPath = Path.Combine(inputDir, csvFile);
                    //// var source = new JsvSource(csvPath, headers, _logger);
                    //var source = new CsvSource(csvPath, headers, _csvOptions.Value.Delimiter, _csvOptions.Value.QuoteChar, _logger);

                    //while (await source.ReadAsync())
                    //{
                    //    _context.AddInput();
                    //    _producerQueue.Enqueue(source.Current);
                    //    if (cancellationToken.IsCancellationRequested)
                    //        return;
                    //}
                    //var csvPath = Path.Combine(inputDir, csvFile);
                    //var tableName = DumpDataHelper.GetTableName(csvPath);

                    //var dataCount = 1200000000L;//当前表要生成的总数据量
                    var dataCount = _tableOptions.Value.TableInfoConfig[tableName].SimulaRowCount;//当前表要生成的总数据量
                    var companyTotallCount = 1000;//当前表每个公司生成的总数据量
                    var tempRecords = new List<DataRecord>();
                    var sk = DataHelper.shareKeys.First();
                    var companyID = DataHelper.companyIds.First();

                    var shareKeyInterval = 20000;//每个sharekey的数据量 
                    var getShareKeyTimes = 0;//sharekey生成的次数,每生成一次，改变sharekey的值
                    var getCompanyIDTimes = 0;//公司生成的次数,每生成一次，改变companyID的值
                    var shareKeyIntervalCount = 0;
                    //CsvSource source;
                    //switch (_dataInputOptions.Value.FileType)
                    //{
                    //    case InputFileType.CSV:
                    //        source=new  CsvSource(csvPath, headers, _csvOptions.Value.Delimiter, _csvOptions.Value.QuoteChar, _logger);
                    //        break;
                    //    case InputFileType.JWT:
                    //        source = new JwtSource(csvPath, headers, _csvOptions.Value.Delimiter, _csvOptions.Value.QuoteChar, _logger);
                    //        break;
                    //    default: break;

                    //}
                    //var source = new JwtSource(csvPath, headers, _csvOptions.Value.Delimiter, _csvOptions.Value.QuoteChar, _logger);
                    var source = _dataInputOptions.Value.CreateSource?.Invoke(tableName);
                var testRecord =await source.GetTestRecord();
                    for (long i = 1; i <= dataCount; i++)
                    {
                        shareKeyIntervalCount++;
                        if (shareKeyIntervalCount > shareKeyInterval)
                        {
                            sk = DataHelper.GetShareKey(getShareKeyTimes);
                            getShareKeyTimes++;
                            shareKeyIntervalCount = 0;
                        }
                        var fields = new string[testRecord.Fields.Length];
                        Array.Copy(testRecord.Fields, fields, testRecord.Fields.Length);
                        var record = new DataRecord(fields, testRecord.TableName, testRecord.Headers, companyID);
                        //更新record的ID、OrderNo,ShardKey值
                        if (record.Headers.Contains("ID"))
                        {
                            var index = Array.IndexOf(record.Headers, "ID");
                            if (index > -1)
                            {
                                record.Fields[index] = i.ToString();
                            }
                        }
                        if (record.TableName == "order_box_block" && record.Headers.Contains("BoxID"))
                        {
                            var index = Array.IndexOf(record.Headers, "BoxID");
                            if (index > -1)
                            {
                                record.Fields[index] = i.ToString();
                            }
                        }
                        if ((record.TableName == "order_block_plan_item" || record.TableName == "order_package_item") && record.Headers.Contains("ItemID"))
                        {
                            var index = Array.IndexOf(record.Headers, "ItemID");
                            if (index > -1)
                            {
                                record.Fields[index] = i.ToString();
                            }
                        }
                        if (record.TableName == "order" && record.Headers.Contains("OrderNo"))
                        {
                            var index = Array.IndexOf(record.Headers, "OrderNo");
                            if (index > -1)
                            {
                                record.Fields[index] = i.ToString();
                            }
                        }
                        if (record.Headers.Contains("ShardKey"))
                        {
                            var index = Array.IndexOf(record.Headers, "ShardKey");
                            if (index > -1)
                            {
                                record.Fields[index] = sk.ToString();
                            }
                        }
                        tempRecords.Add(record);
                        if (tempRecords.Count >= companyTotallCount || i >= dataCount - 1)
                        {
                            foreach (var rc in tempRecords)
                            {
                                _context.AddInput();
                                _producerQueue.Enqueue(rc);
                                if (cancellationToken.IsCancellationRequested)
                                    return;
                            }
                            tempRecords.Clear();
                            companyID = DataHelper.GetCompanyId(getCompanyIDTimes);
                            getCompanyIDTimes++;
                        }
                    }
                    _logger.LogInformation("table:'{tableName}' simulation input completed", tableName);
                //}
                //_logger.LogInformation("File '{File}' input completed", Path.GetFileName(sqlPath));
            }

            _context.CompleteInput();
            _logger.LogInformation("***** Csv input service completed *****");
        }
    }
}