I have done StreamReaders using sr.ReadLine() for over 4 million records and it works fine. In this instance though, the lines have comma separated values with quote delimiters and I need to use them. So, I found a function that will split the lines and return a list. I have remarked out the routine and the ReadLine() spins through fine. So, I believe it has something to do with either my variables or the routine or both.
C#:
while ((sLine = sr.ReadLine()) != null)
{
//Pass the line to the parser routine. Get back a string[] variable.
iCurrLine++;
string[] sItems = new string[30]; //declare a new array of strings (blank the old one)
sItems = SplitCSV(sLine); //This splits the line using RegEx on commas and quotes.
//Process the line. Not even doing anything here yet.
//Tried the following to clean up some memory. It helped a bit (ran further), but still does not complete.
if (iCurrLine % 25 == 0) //Runs every 25 lines (At least this is what i intended.
{
System.GC.Collect();
}
}
public static string[] SplitCSV(string input)
{
//Function to split up the CSV row.
Regex csvSplit = new Regex("(?:^|,)(\"(?:[^\"]+|\"\")*\"|[^,]*)", RegexOptions.Compiled);
List<string> list = new List<string>();
string curr = null;
foreach (Match match in csvSplit.Matches(input))
{
curr = match.Value;
if (0 == curr.Length)
{
list.Add("");
}
list.Add(curr.TrimStart(','));
}
return list.ToArray<string>();
}
