Writing multiple CSV files programmatically
I am writing an application that will write database tables into CSV Files. Since many tables are over a million records, I am giving the user an option of writing large tables into files of 25,000 lines each. I want the user to specify the initial file name in a SaveFileDialog and then append "-part1", "-part2", etc. for each new file written. How can one programmatically write multiple files until all the data is written? The current code I have to write the 25,000 line file is listed below.
public void ExportPartition(SaveFileDialog saveFile,开发者_开发知识库 DataTable table)
{
TextWriter writer = new StreamWriter(saveFile.FileName, true, System.Text.Encoding.ASCII, 1048576);
for (int i = 0; i <= 25000; i++)
{
for (int j = 0; j < table.Columns.Count; j++)
{
writer.Write(table.Rows[i][j].ToString() + ",");
}
writer.Write("\r\n");
}
writer.Flush();
DisposeObjects(saveFile, writer);
}
bool ExportPartition(string fileName, DataTable table, int batchSize, int batchNum)
{
string fn = string.Format("{0}-{1}{2}",
Path.GetFileNameWithoutExtension(fileName),
batchNum,
Path.GetExtension(fileName));
fn = Path.Combine(Path.GetDirectoryName(fileName), fn);
using (TextWriter writer = new StreamWriter(fn))
{
int start = batchNum * batchSize;
int end = start + batchSize;
for (int i = start; i < end; i++)
{
if (i >= table.Rows.Count)
break;
for (int j = 0; j < table.Columns.Count; j++)
{
writer.Write(table.Rows[i][j] + ",");
}
writer.Write("\r\n");
}
return table.Rows.Count <= end;
}
}
Usage:
void WriteFiles(DataTable table, String fileName, int batchSize)
{
int batchNum = 0;
bool done = false;
while (!done)
{
done = ExportPartition(fileName, table, batchSize, batchNum++);
}
}
void Main()
{
DataTable dt = GetData();
string fileName = GetFileNameWithSaveDialog();
int batchSize = 25000;
WriteFiles(dt, fileName, batchSize);
}
An alternative solution:
class Program
{
static void Main(string[] args)
{
DataTable dt = new DataTable();
dt.Columns.Add("Col1");
dt.Columns.Add("Col2");
for (int i = 0; i < 103; ++i)
{
var r = dt.NewRow();
r[0] = Guid.NewGuid().ToString();
r[1] = i.ToString();
dt.Rows.Add(r);
}
WriteCsvFile(dt, 25, @"C:\temp\test.txt");
}
public static string[] ToStringArray(DataRow row)
{
var arr = new string[row.Table.Columns.Count];
for (int j = 0; j < arr.Length; j++)
{
arr[j] = row[j].ToString();
if((arr[j]??"").Contains(","))
throw new Exception("This will end badly...");
}
return arr;
}
public static void WriteCsvFile(DataTable table, int maxCount, string fileName)
{
if (table.Rows.Count <= maxCount)
WriteCsvFile(table, maxCount, fileName, 0);
else
for (int i = 0; i < (table.Rows.Count / maxCount + 1); ++i)
{
var partFileName = Path.Combine(Path.GetDirectoryName(fileName), string.Format("{0}-part{1}{2}", Path.GetFileNameWithoutExtension(fileName), i+1, Path.GetExtension(fileName)));
WriteCsvFile(table, maxCount, partFileName, i * maxCount);
}
}
public static void WriteCsvFile(DataTable table, int maxCount, string fileName, int startIndex)
{
using(var fs = File.Create(fileName))
using(var w = new StreamWriter(fs, Encoding.ASCII))
{
for (int i = startIndex; i < Math.Min(table.Rows.Count, startIndex + maxCount); i++)
w.WriteLine(String.Join(",", ToStringArray(table.Rows[i])));
w.Flush();
}
}
}
精彩评论