SqlBulkCopy Code

Posts   
 
    
psandler
User
Posts: 540
Joined: 22-Feb-2005
# Posted on: 27-Jan-2007 08:41:01   

Here is some simple code that seems to work pretty well, at least with the 3-4 entities I tested with in Northwind (I didn't test all of them). I'm pretty sure it could be tweaked to work with almost any entity.

Partial class to add functionality to adapter:


using SD.LLBLGen.Pro.ORMSupportClasses;

namespace SearchObjects.DAL.DatabaseSpecific
{
    public partial class DataAccessAdapter : DataAccessAdapterBase
    {
        public string GetFieldName(IEntityField2 field)
        {
            IFieldPersistenceInfo i = GetFieldPersistenceInfo(field);
            return i.SourceColumnName;
        }
        public string GetTableName(IEntityField2 field)
        {
            IFieldPersistenceInfo i = GetFieldPersistenceInfo(field);
            return i.SourceObjectName;
        }
    }
}

DAOGeneral class to do the actual saving:


using SearchObjects.DAL.HelperClasses;
using SearchObjects.DAL.EntityClasses;
using SD.LLBLGen.Pro.ORMSupportClasses;
using SearchObjects.DAL.DatabaseSpecific;
using System.Data;
using System.Data.SqlClient;

namespace SearchObjects.DAO
{
    public static class DAOGeneral
    {
        public static void SaveEntityCollection(IEntityCollection2 entities)
        {
            using (DataAccessAdapter adapter = new DataAccessAdapter())
            {
                adapter.SaveEntityCollection(entities, false, false);
            }
        }

        //these entities must all be of the same type, so it might be better to make
        //the parameter of type EntityCollection<ShippersEntity>
        public static void BulkSaveEntityCollection(IEntityCollection2 entitiesToSave)
        {
            if (entitiesToSave.Count > 0)
            {
                //create datatable
                DataTable dt = new DataTable();

                using (DataAccessAdapter adapter = new DataAccessAdapter())
                {
                    //get the name of the table from the first entity/field
                    //using the exposed method on the DataAccessAdapter
                    string tableName = adapter.GetTableName(entitiesToSave[0].Fields[0]);

                    //use the first entity to create columns in the datatable                   
                    foreach (IEntityField2 field in entitiesToSave[0].Fields)
                    {
                        dt.Columns.Add(adapter.GetFieldName(field));
                    }
                    //create a row in the datatable for each entity
                    foreach (IEntity2 entity in entitiesToSave)
                    {
                        DataRow dr = dt.NewRow();
                        foreach (IEntityField2 field in entity.Fields)
                        {
                            //get the field's actual database name
                            //using the exposed method on the DataAccessAdapter
                            dr[adapter.GetFieldName(field)] = field.CurrentValue;
                        }
                        dt.Rows.Add(dr);
                    }
                    using (SqlConnection conn = new SqlConnection(adapter.ConnectionString))
                    {
                        conn.Open();
                        SqlBulkCopyOptions options = SqlBulkCopyOptions.Default;
                        SqlBulkCopy bulk = new SqlBulkCopy(conn, options, null);
                        bulk.DestinationTableName = tableName;
                        bulk.WriteToServer(dt);
                    }
                }
            }
        }
    }
}

I created a simple form and some code for a unit test, since I don't have MbUnit or NUnit installed at home.

Form1.Designer.cs:


namespace SearchObjects.UI
{
    partial class Form1
    {
        /// <summary>
        /// Required designer variable.
        /// </summary>
        private System.ComponentModel.IContainer components = null;

        /// <summary>
        /// Clean up any resources being used.
        /// </summary>
        /// <param name="disposing">true if managed resources should be disposed; otherwise, false.</param>
        protected override void Dispose(bool disposing)
        {
            if (disposing && (components != null))
            {
                components.Dispose();
            }
            base.Dispose(disposing);
        }

        #region Windows Form Designer generated code

        /// <summary>
        /// Required method for Designer support - do not modify
        /// the contents of this method with the code editor.
        /// </summary>
        private void InitializeComponent()
        {
            this.btnStart = new System.Windows.Forms.Button();
            this.txtOutput = new System.Windows.Forms.TextBox();
            this.btnClear = new System.Windows.Forms.Button();
            this.SuspendLayout();
            // 
            // btnStart
            // 
            this.btnStart.Location = new System.Drawing.Point(61, 21);
            this.btnStart.Name = "btnStart";
            this.btnStart.Size = new System.Drawing.Size(75, 23);
            this.btnStart.TabIndex = 0;
            this.btnStart.Text = "Start";
            this.btnStart.UseVisualStyleBackColor = true;
            this.btnStart.Click += new System.EventHandler(this.btnStart_Click);
            // 
            // txtOutput
            // 
            this.txtOutput.Location = new System.Drawing.Point(21, 61);
            this.txtOutput.Multiline = true;
            this.txtOutput.Name = "txtOutput";
            this.txtOutput.ScrollBars = System.Windows.Forms.ScrollBars.Both;
            this.txtOutput.Size = new System.Drawing.Size(301, 583);
            this.txtOutput.TabIndex = 1;
            // 
            // btnClear
            // 
            this.btnClear.Location = new System.Drawing.Point(196, 21);
            this.btnClear.Name = "btnClear";
            this.btnClear.Size = new System.Drawing.Size(75, 23);
            this.btnClear.TabIndex = 2;
            this.btnClear.Text = "Clear";
            this.btnClear.UseVisualStyleBackColor = true;
            this.btnClear.Click += new System.EventHandler(this.btnClear_Click);
            // 
            // Form1
            // 
            this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F);
            this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font;
            this.ClientSize = new System.Drawing.Size(344, 667);
            this.Controls.Add(this.btnClear);
            this.Controls.Add(this.txtOutput);
            this.Controls.Add(this.btnStart);
            this.Name = "Form1";
            this.Text = "Form1";
            this.ResumeLayout(false);
            this.PerformLayout();

        }

        #endregion

        private System.Windows.Forms.Button btnStart;
        private System.Windows.Forms.TextBox txtOutput;
        private System.Windows.Forms.Button btnClear;
    }
}

Form1.cs:


using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Text;
using System.Windows.Forms;
using SearchObjects.DAO;
using SearchObjects.DAL.EntityClasses;
using SearchObjects.DAL.HelperClasses;

namespace SearchObjects.UI
{
    public partial class Form1 : Form
    {
        Random _random = new Random();

        public Form1()
        {
            InitializeComponent();
        }

        private void btnStart_Click(object sender, EventArgs e)
        {
            TestSetup();            
        }

        private void TestSetup()
        {
            //number if times to run tests
            const int outerLoopIterations = 5;
            //number of inserts for each test
            const int numberToTest = 5000;
            double normalTotalTime = 0;
            double bulkTotalTime = 0;
                        
            //do one bulk and one normal save to "warm up"
            //otherwise the first test is always significantly slower           
            txtOutput.Text += "\r\n";
            txtOutput.Text += "Warmup starting . . ." + "\r\n";
            EntityCollection<ShippersEntity> temp = GetShippersCollection(1);
            bulkTotalTime += TestBulkInsert(temp, 0);           
            temp = GetShippersCollection(1);
            normalTotalTime += TestNormalInsert(temp, 0);
            txtOutput.Text += "Warmup ending." + "\r\n";
            txtOutput.Text += "\r\n";
            
            for (int x = 0; x < outerLoopIterations; x++)
            {
                //get random entity collection
                EntityCollection<ShippersEntity> col = GetShippersCollection(numberToTest);
                bulkTotalTime += TestBulkInsert(col, x);

                //get random entity collection
                col = GetShippersCollection(numberToTest);
                normalTotalTime += TestNormalInsert(col, x);
            }
            txtOutput.Text += "\r\n";
            txtOutput.Text += "Total Bulk Time: " + bulkTotalTime + "\r\n";
            txtOutput.Text += "Average Bulk Time: " + bulkTotalTime / outerLoopIterations + "\r\n";
            txtOutput.Text += "Total Normal Time: " + normalTotalTime + "\r\n";
            txtOutput.Text += "Average Normal Time: " + normalTotalTime / outerLoopIterations + "\r\n";
            txtOutput.Text += "\r\n";           
        }

        private double TestNormalInsert(EntityCollection<ShippersEntity> entitiesToSave, int iteration)
        {
            DateTime start = DateTime.Now;
            DAOGeneral.SaveEntityCollection(entitiesToSave);
            DateTime end = DateTime.Now;
            double time = WriteTime(start, end, "Time for Normal Insert, #" + (iteration + 1) + " : ");
            return time;
        }

        private double TestBulkInsert(EntityCollection<ShippersEntity> entitiesToSave, int iteration)
        {
            DateTime start = DateTime.Now;
            DAOGeneral.BulkSaveEntityCollection(entitiesToSave);
            DateTime end = DateTime.Now;
            double time = WriteTime(start, end, "Time for Bulk Insert, #" + (iteration + 1) + " : ");
            return time;
        }

        private EntityCollection<ShippersEntity> GetShippersCollection(int collectionSize)
        {
            EntityCollection<ShippersEntity> col = new EntityCollection<ShippersEntity>();
            for (int x = 0; x < collectionSize; x++)
            {
                col.Add(CreateRandomShipper());
            }
            return col;
        }

        private ShippersEntity CreateRandomShipper()
        {
            ShippersEntity e = new ShippersEntity();            
            e.CompanyName = "++" + _random.Next(25000) + "++";
            e.Phone = "++" + _random.Next(5000) + "++";
            return e;
        }

        private double WriteTime(DateTime start, DateTime end, string message)
        {
            TimeSpan span = end - start;
            txtOutput.Text += message + span.TotalSeconds + "\r\n";
            Application.DoEvents();
            return span.TotalSeconds;
        }

        private void btnClear_Click(object sender, EventArgs e)
        {
            txtOutput.Clear();
        }
    }
}

Chester
Support Team
Posts: 223
Joined: 15-Jul-2005
# Posted on: 28-Jan-2007 02:23:21   

Interesting. Thanks for sharing. smile

psandler
User
Posts: 540
Joined: 22-Feb-2005
# Posted on: 28-Jan-2007 21:47:45   

Chester wrote:

Interesting. Thanks for sharing. smile

No sweat--thought some folks might find it useful. I don't believe bulk loading should be done from the app in most cases, but sometime a situation comes up when you need to create large amounts of data in the app layer. This code will speed up bulk inserts 100-fold.

OT: where do you live in Chicago, Chester?

Phil