整理代码
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
using System.Text;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using ConsoleApp2.Helpers;
|
||||
using ConsoleApp2.HostedServices.Abstractions;
|
||||
@@ -15,46 +16,27 @@ public class CsvSource:IDataSource
|
||||
//protected readonly StreamReader _reader;
|
||||
private readonly ILogger? _logger;
|
||||
protected readonly string _tableName;
|
||||
protected string _sqlFilePath;
|
||||
protected readonly string _sqlFileText;
|
||||
|
||||
//public DataRecord Current { get; protected set; }
|
||||
//public string[]? Headers { get; }
|
||||
protected string? _sqlFilePath;
|
||||
protected readonly string? _sqlFileText;
|
||||
protected string[]? headers;
|
||||
protected string[]? csvFiles;
|
||||
public string? CurrentRaw { get; protected set; }
|
||||
public string Delimiter { get; private set; }
|
||||
public char QuoteChar { get; private set; }
|
||||
|
||||
public CsvSource(string inputDir,string tableName,string delimiter = ",", char quoteChar = '"',
|
||||
ILogger? logger = null)
|
||||
{
|
||||
_inputDir = inputDir;
|
||||
_tableName = tableName;
|
||||
//Headers = headers;
|
||||
_logger = logger;
|
||||
Delimiter = delimiter;
|
||||
QuoteChar = quoteChar;
|
||||
|
||||
//var fs = File.OpenRead(filePath);
|
||||
//_reader = new StreamReader(fs);
|
||||
//_tableName = DumpDataHelper.GetTableName(filePath);
|
||||
string pattern = $"^.*\\.{tableName}\\..*\\.sql$";
|
||||
_sqlFilePath = Directory.GetFiles(_inputDir).FirstOrDefault(s => Regex.Match(s, pattern).Success);
|
||||
|
||||
|
||||
}
|
||||
|
||||
//public virtual async ValueTask<bool> ReadAsync()
|
||||
//{
|
||||
// var str = await _reader.ReadLineAsync();
|
||||
// if (string.IsNullOrWhiteSpace(str))
|
||||
// return false;
|
||||
|
||||
// CurrentRaw = str;
|
||||
|
||||
// var fields = ParseRow2(str, QuoteChar, Delimiter);
|
||||
// Current = new DataRecord(fields, _tableName, Headers);
|
||||
// return true;
|
||||
//}
|
||||
|
||||
public string[] ParseRow(string row, char quoteChar, string delimiter)
|
||||
{
|
||||
@@ -145,23 +127,18 @@ public class CsvSource:IDataSource
|
||||
result.Add(current.ToString());
|
||||
return result.ToArray();
|
||||
}
|
||||
public virtual async Task<string[]> GetHeaders()
|
||||
public virtual async Task GetHeaderAndCsvFiles()
|
||||
{
|
||||
var text = await File.ReadAllTextAsync(_sqlFilePath);
|
||||
return await DumpDataHelper.GetCsvHeadersFromSqlFileAsync(text);
|
||||
|
||||
}
|
||||
public virtual async Task<string[]> GetCsvFiles()
|
||||
{
|
||||
var text= await File.ReadAllTextAsync(_sqlFilePath);
|
||||
return await DumpDataHelper.GetCsvFileNamesFromSqlFileAsync(text,new Regex(@"'.+\.dat'"));
|
||||
headers = await DumpDataHelper.GetCsvHeadersFromSqlFileAsync(text);
|
||||
csvFiles = await DumpDataHelper.GetCsvFileNamesFromSqlFileAsync(text, new Regex(@"'.+\.dat.zst'"));
|
||||
|
||||
}
|
||||
public virtual async Task DoEnqueue(Action<DataRecord> action)
|
||||
{
|
||||
var sourceFiles =await GetCsvFiles();
|
||||
foreach (var file in sourceFiles)
|
||||
await GetHeaderAndCsvFiles();
|
||||
foreach (var file in csvFiles)
|
||||
{
|
||||
var headers = await GetHeaders();
|
||||
var filePath= Path.Combine(_inputDir, file);
|
||||
using (var fs = File.OpenRead(filePath))
|
||||
{
|
||||
@@ -182,11 +159,10 @@ public class CsvSource:IDataSource
|
||||
}
|
||||
public virtual async Task<DataRecord?> GetTestRecord()
|
||||
{
|
||||
var sourceFiles = await GetCsvFiles();
|
||||
var file = sourceFiles.FirstOrDefault();
|
||||
await GetHeaderAndCsvFiles();
|
||||
var file = csvFiles.FirstOrDefault();
|
||||
if (file != null)
|
||||
{
|
||||
var headers = await GetHeaders();
|
||||
var filePath = Path.Combine(_inputDir, file);
|
||||
using (var fs = File.OpenRead(filePath))
|
||||
{
|
||||
|
Reference in New Issue
Block a user