添加试运行模式,修改生产环境数据重建规则

This commit is contained in:
2025-01-20 14:29:43 +08:00
parent 1f7213a75c
commit c049e092e4
5 changed files with 26 additions and 23 deletions

View File

@@ -31,6 +31,8 @@ public class FileInputService : IInputService
private readonly ProcessContext _context;
private readonly DataReaderFactory _dataReaderFactory;
private readonly long _memoryThreshold;
private readonly bool _dryRun;
private readonly int _dryRunCount;
public FileInputService(ILogger<FileInputService> logger,
IOptions<DataInputOptions> dataInputOptions,
@@ -45,12 +47,16 @@ public class FileInputService : IInputService
_producerQueue = producerQueue;
_dataReaderFactory = dataReaderFactory;
_memoryThreshold = (long)(configuration.GetValue<double>("MemoryThreshold", 8) * 1024 * 1024 * 1024);
_dryRun = configuration.GetValue("DryRun", false);
_dryRunCount = configuration.GetValue("DryRunCount", 100_000);
}
public async Task ExecuteAsync(CancellationToken cancellationToken)
{
var inputDir = _dataInputOptions.Value.InputDir ?? throw new ApplicationException("未配置文件输入目录");
_logger.LogInformation("***** 输入服务已启动,工作目录为:{InputDir} *****", inputDir);
if (_dryRun)
_logger.LogInformation("***** 试运行模式已开启,只读取前 {Count} 行数据 *****", _dryRunCount);
var orderedInfo = GetOrderedInputInfo(inputDir);
@@ -74,6 +80,8 @@ public class FileInputService : IInputService
await _producerQueue.EnqueueAsync(record);
count++;
_context.AddInput();
if (_dryRun && count >= _dryRunCount)
break;
}
_context.AddTableInput(info.TableName, count);
@@ -82,7 +90,7 @@ public class FileInputService : IInputService
}
_context.CompleteInput();
_logger.LogInformation("***** 输入服务已执行完毕 *****");
_logger.LogInformation("***** 输入服务{DryRun}已执行完毕 *****", _dryRun ? " (试运行)" : "");
}
public IEnumerable<FileInputInfo> GetOrderedInputInfo(string dir)

View File

@@ -163,9 +163,16 @@ public class MainHostedService : BackgroundService
private async Task ExportResultAsync()
{
var sb = new StringBuilder();
if (_context.HasException)
sb.AppendLine("# 程序执行完毕,**但中途发生了异常**");
else sb.AppendLine("# 程序执行完毕,没有发生错误");
var title = (_config.GetValue("DryRun", false), _context.HasException) switch
{
(true, true) => "# 试运行结束,**请注意处理异常**",
(true, false) => "# 试运行结束,没有发生异常",
(false, true) => "# 程序执行完毕,**但中途发生了异常**",
(false, false) => "# 程序执行完毕,没有发生错误"
};
sb.AppendLine(title);
sb.AppendLine("## 处理计数");
var processCount = new[]
{