Gokul's Blog

File Helpers to handle large text file

Leave a comment

File Helpers to handle large text file

Sample Type
 public class SampleType
	public string Field1;
	public int    Field2;

For small files

FileHelperEngine&lt;SampleType&gt; engine = new FileHelperEngine&lt;SampleType&gt;();
engine.ErrorManager.ErrorMode = ErrorMode.SaveAndContinue;
_dataSet = engine.ReadFile(sourceFilePath).ToDataSet();

For large files
                    FileHelperAsyncEngine engine2 = new FileHelperAsyncEngine(typeof(SampleType));

                    List&lt;SampleType&gt; records = new List&lt;SampleType&gt;();
                    int iCount = 0;
                    while (engine2.ReadNext() != null)
                        if (iCount++ &gt; 10000)
                            //Put in a queue here and call the method to create dataset as follows
                            records = new List&lt;SampleType&gt;(); //rinse and repeat till all records are processed.
                        // put your code here !!!!
                        Console.WriteLine("Data " + records[0].Field1 + " , " + records[0].Field2.ToString());

Extension method to create Dataset from a list.
 #region Extension Classes
    static class Extensions
        public static DataSet ToDataSet&lt;T&gt;(this IList&lt;T&gt; list)
            Type elementType = typeof(T);
            DataSet ds = new DataSet();
            DataTable t = new DataTable(elementType.Name);

            //add a column to table for each public property on T
            foreach (var propInfo in elementType.GetProperties())
                Type ColType = Nullable.GetUnderlyingType(propInfo.PropertyType) ?? propInfo.PropertyType;

                t.Columns.Add(propInfo.Name, ColType);

            //go through each property on T and add each value to the table
            foreach (T item in list)
                DataRow row = t.NewRow();

                foreach (var propInfo in elementType.GetProperties())
                    row[propInfo.Name] = propInfo.GetValue(item, null) ?? DBNull.Value;


            return ds;


Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out /  Change )

Google+ photo

You are commenting using your Google+ account. Log Out /  Change )

Twitter picture

You are commenting using your Twitter account. Log Out /  Change )

Facebook photo

You are commenting using your Facebook account. Log Out /  Change )


Connecting to %s