Export Sql data to Excel very slow

Question

I am exporting Sql data to Excel. The code I am using currently is :

    DataTable dt = new DataTable();
            // Create sql connection string
            string conString = "Data Source=DELL\\SQLSERVER1;Trusted_Connection=True;DATABASE=Zelen;CONNECTION RESET=FALSE";

            SqlConnection sqlCon = new SqlConnection(conString);
            sqlCon.Open();


            SqlDataAdapter da = new SqlDataAdapter("select LocalSKU,ItemName, QOH,Price,Discontinued,CAST(Barcode As varchar(25)) As Barcode,Integer2,Integer3,ISNULL(SalePrice,0.0000)AS SalePrice,SaleOn,ISNULL(Price2,0.0000)AS Price2 from dbo.Inventory", sqlCon);

            System.Data.DataTable dtMainSQLData = new System.Data.DataTable();

            da.Fill(dtMainSQLData);
            DataColumnCollection dcCollection = dtMainSQLData.Columns;

            // Export Data into EXCEL Sheet
            Microsoft.Office.Interop.Excel.ApplicationClass ExcelApp = new Microsoft.Office.Interop.Excel.ApplicationClass();
            ExcelApp.Application.Workbooks.Add(Type.Missing);



            int i = 1;
            int j = 1;
            int s = 1;
            //header row
            foreach (DataColumn col in dtMainSQLData.Columns)
            {
                ExcelApp.Cells[i, j] = col.ColumnName;
                j++;

                ExcelApp.Rows.AutoFit();
                ExcelApp.Columns.AutoFit();
            }

            i++;

            //data rows
            foreach (DataRow row in dtMainSQLData.Rows)
            {
                for (int k = 1; k < dtMainSQLData.Columns.Count + 1; k++)
                {
                    ExcelApp.Cells[i, k] = "'" + row[k - 1].ToString();
                }

                i++;
                s++;
                Console.Write(s);
                Console.Write("\n\r");

                ExcelApp.Columns.AutoFit();
                ExcelApp.Rows.AutoFit();
            }

            var b = Environment.CurrentDirectory + @"\Sheet1.xlsx";
            ExcelApp.ActiveWorkbook.SaveCopyAs(b);

            ExcelApp.ActiveWorkbook.Saved = true;
            ExcelApp.Quit();

            Console.WriteLine(".xlsx file Exported succssessfully.");

Takes are 70000 rows in my sql database. I am running this script in Console application. It takes more then an hour to export it to excel file.

How can I use this to export it faster?

Examples would be appreciated.

Answer 1

Option 1:

See this answer. Use a library called ClosedXML to write the data to Excel.

Option 2:

Get a range big enough for all of the data and set the value equal to a 2 dimensional range. This works very fast without another referencing another library. I tried with 70000 records.

// Get an excel instance
Microsoft.Office.Interop.Excel.Application excel = new Microsoft.Office.Interop.Excel.Application();

// Get a workbook
Workbook wb = excel.Workbooks.Add();

// Get a worksheet
Worksheet ws = wb.Worksheets.Add();
ws.Name = "Test Export";

// Add column names to the first row
int col = 1;
foreach (DataColumn c in table.Columns) {
    ws.Cells[1, col] = c.ColumnName;
    col++;
}

// Create a 2D array with the data from the table
int i = 0;
string[,] data = new string[table.Rows.Count, table.Columns.Count];
foreach (DataRow row in table.Rows) {                
    int j = 0;
    foreach (DataColumn c in table.Columns) {
        data[i,j] = row[c].ToString();
        j++;
    }
    i++;            
}                   

// Set the range value to the 2D array
ws.Range[ws.Cells[2, 1], ws.Cells[table.Rows.Count + 1, table.Columns.Count]].value = data;

// Auto fit columns and rows, show excel, save.. etc
excel.Columns.AutoFit();
excel.Rows.AutoFit();
excel.Visible = true;

Edit: This version exported a million records on my machine it takes about a minute. This example uses Excel interop and breaks the rows in to chunks of 100,000.

// Start a stopwatch to time the process
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
sw.Start();

// Check if there are rows to process
if (table != null && table.Rows.Count > 0) {

    // Determine the number of chunks
    int chunkSize = 100000;
    double chunkCountD = (double)table.Rows.Count / (double)chunkSize;
    int chunkCount = table.Rows.Count / chunkSize;
    chunkCount = chunkCountD > chunkCount ? chunkCount + 1 : chunkCount;

    // Instantiate excel
    Microsoft.Office.Interop.Excel.Application excel = new Microsoft.Office.Interop.Excel.Application();

    // Get a workbook
    Workbook wb = excel.Workbooks.Add();

    // Get a worksheet
    Worksheet ws = wb.Worksheets.Add();
    ws.Name = "Test Export";

    // Add column names to excel
    int col = 1;                
    foreach (DataColumn c in table.Columns) {
        ws.Cells[1, col] = c.ColumnName;
        col++;
    }

    // Build 2D array
    int i = 0;
    string[,] data = new string[table.Rows.Count, table.Columns.Count];
    foreach (DataRow row in table.Rows) {
        int j = 0;
        foreach (DataColumn c in table.Columns) {
            data[i, j] = row[c].ToString();
            j++;
        }
        i++;
    }

    int processed = 0;
    int data2DLength = data.GetLength(1);
    for (int chunk = 1; chunk <= chunkCount; chunk++) {
        if (table.Rows.Count - processed < chunkSize) chunkSize = table.Rows.Count - processed;            
        string[,] chunkData = new string[chunkSize, data2DLength];
        int l = 0;
        for (int k = processed; k < chunkSize + processed; k++) {
            for (int m = 0; m < data2DLength; m++) {
                chunkData[l,m] = table.Rows[k][m].ToString();
            }
            l++;
        }
        // Set the range value to the chunk 2d array
        ws.Range[ws.Cells[2 + processed, 1], ws.Cells[processed + chunkSize + 1, data2DLength]].value = chunkData;
        processed += chunkSize;
    }

    // Auto fit columns and rows, show excel, save.. etc
    excel.Columns.AutoFit();
    excel.Rows.AutoFit();
    excel.Visible = true;                
}

// Stop the stopwatch and display the seconds elapsed
sw.Stop();
MessageBox.Show(sw.Elapsed.TotalSeconds.ToString());

Answer 2

If you save your data to as CSV formant you can load that into Excel, Here is some code i have modified from The Code Project site here http://www.codeproject.com/Tips/665519/Writing-a-DataTable-to-a-CSV-file

public class Program
{
    static void Main(string[] args)
    {
        Stopwatch stopwatch = new Stopwatch();
        stopwatch.Start();
        DataTable  dt = new DataTable();
        // Create Connection object
        using (SqlConnection conn = new SqlConnection(@"<Your Connection String>"))
        {
            // Create Command object
            conn.Open();
            using (SqlCommand cmd = new SqlCommand("SELECT * FROM <Your Table>", conn))
            {
                using (SqlDataReader reader = cmd.ExecuteReader())
                {
                    try
                    {
                        dt.Load(reader);

                        using (StreamWriter writer = new StreamWriter("C:\\Temp\\dump.csv"))
                        {
                            DataConvert.ToCSV(dt, writer, false);
                        }
                    }
                    catch (Exception)
                    {

                        throw;
                    }
                }
            }
        }

        // Stop timing
        stopwatch.Stop();

        // Write result
        Console.WriteLine("Time elapsed: {0}",
            stopwatch.Elapsed);
        Console.ReadKey();
    }
}


public static class DataConvert
{
    public static void ToCSV(DataTable sourceTable, TextWriter writer, bool includeHeaders)
    {
        if (includeHeaders)
        {
            List<string> headerValues = new List<string>();
            foreach (DataColumn column in sourceTable.Columns)
            {
                headerValues.Add(QuoteValue(column.ColumnName));
            }

            writer.WriteLine(String.Join(",", headerValues.ToArray()));
        }

        string[] items = null;
        foreach (DataRow row in sourceTable.Rows)
        {
            items = row.ItemArray.Select(o => QuoteValue(o.ToString())).ToArray();
            writer.WriteLine(String.Join(",", items));
        }

        writer.Flush();
    }

    private static string QuoteValue(string value)
    {
        return String.Concat("\"", value.Replace("\"", "\"\""), "\"");
    }
} 

}

On my PC this took 30 seconds to process 1 million records...

Answer 3

you can try this function:

After set your data in a datatable.

Public Shared Sub ExportDataSetToExcel(ByVal ds As DataTable, ByVal filename As String)
        Dim response As HttpResponse = HttpContext.Current.Response
        response.Clear()
        response.Buffer = True
        response.Charset = ""
        response.ContentType = "application/vnd.ms-excel"

        Using sw As New StringWriter()
            Using htw As New HtmlTextWriter(sw)
                Dim dg As New DataGrid()
                dg.DataSource = ds
                dg.DataBind()
                dg.RenderControl(htw)
                response.Charset = "UTF-8"
                response.ContentEncoding = System.Text.Encoding.UTF8
                response.BinaryWrite(System.Text.Encoding.UTF8.GetPreamble())
                response.Output.Write(sw.ToString())
                response.[End]()
            End Using
        End Using
    End Sub

Answer 4

I prefer Microsoft Open XML SDK's Open XML Writer. Open XML is the format all the new office files are in.

Export a large data query (60k+ rows) to Excel

Vincent Tan has a nice article on the topic.

http://polymathprogrammer.com/2012/08/06/how-to-properly-use-openxmlwriter-to-write-large-excel-files/