Gokul's Blog

File Helpers to handle large text file

Leave a comment

File Helpers to handle large text file

Sample Type
<pre>[Delimitedrecord("|")]
 public class SampleType
 {
	public string Field1;
	public int    Field2;
 }

For small files

FileHelperEngine&lt;SampleType&gt; engine = new FileHelperEngine&lt;SampleType&gt;();
engine.ErrorManager.ErrorMode = ErrorMode.SaveAndContinue;
_dataSet = engine.ReadFile(sourceFilePath).ToDataSet();

For large files
                    FileHelperAsyncEngine engine2 = new FileHelperAsyncEngine(typeof(SampleType));
                    engine2.BeginReadFile(sourceFilePath);

                    List&lt;SampleType&gt; records = new List&lt;SampleType&gt;();
                    int iCount = 0;
                    while (engine2.ReadNext() != null)
                    {
                        records.Add((SampleType)engine2.LastRecord);
                        if (iCount++ &gt; 10000)
                        {
                            //Put in a queue here and call the method to create dataset as follows
                            ///records.ToDataSet();
                            records = new List&lt;SampleType&gt;(); //rinse and repeat till all records are processed.
                        }
                        // put your code here !!!!
                        Console.WriteLine("Data " + records[0].Field1 + " , " + records[0].Field2.ToString());

                    }
Extension method to create Dataset from a list.
 #region Extension Classes
    static class Extensions
    {
        public static DataSet ToDataSet&lt;T&gt;(this IList&lt;T&gt; list)
        {
            Type elementType = typeof(T);
            DataSet ds = new DataSet();
            DataTable t = new DataTable(elementType.Name);
            ds.Tables.Add(t);

            //add a column to table for each public property on T
            foreach (var propInfo in elementType.GetProperties())
            {
                Type ColType = Nullable.GetUnderlyingType(propInfo.PropertyType) ?? propInfo.PropertyType;

                t.Columns.Add(propInfo.Name, ColType);
            }

            //go through each property on T and add each value to the table
            foreach (T item in list)
            {
                DataRow row = t.NewRow();

                foreach (var propInfo in elementType.GetProperties())
                {
                    row[propInfo.Name] = propInfo.GetValue(item, null) ?? DBNull.Value;
                }

                t.Rows.Add(row);
            }

            return ds;
        }
    }
    #endregion

Advertisements

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out / Change )

Twitter picture

You are commenting using your Twitter account. Log Out / Change )

Facebook photo

You are commenting using your Facebook account. Log Out / Change )

Google+ photo

You are commenting using your Google+ account. Log Out / Change )

Connecting to %s