- Home
- LLBLGen Pro
- LLBLGen Pro Runtime Framework
SqlBulkCopy Code
Joined: 22-Feb-2005
Here is some simple code that seems to work pretty well, at least with the 3-4 entities I tested with in Northwind (I didn't test all of them). I'm pretty sure it could be tweaked to work with almost any entity.
Partial class to add functionality to adapter:
using SD.LLBLGen.Pro.ORMSupportClasses;
namespace SearchObjects.DAL.DatabaseSpecific
{
public partial class DataAccessAdapter : DataAccessAdapterBase
{
public string GetFieldName(IEntityField2 field)
{
IFieldPersistenceInfo i = GetFieldPersistenceInfo(field);
return i.SourceColumnName;
}
public string GetTableName(IEntityField2 field)
{
IFieldPersistenceInfo i = GetFieldPersistenceInfo(field);
return i.SourceObjectName;
}
}
}
DAOGeneral class to do the actual saving:
using SearchObjects.DAL.HelperClasses;
using SearchObjects.DAL.EntityClasses;
using SD.LLBLGen.Pro.ORMSupportClasses;
using SearchObjects.DAL.DatabaseSpecific;
using System.Data;
using System.Data.SqlClient;
namespace SearchObjects.DAO
{
public static class DAOGeneral
{
public static void SaveEntityCollection(IEntityCollection2 entities)
{
using (DataAccessAdapter adapter = new DataAccessAdapter())
{
adapter.SaveEntityCollection(entities, false, false);
}
}
//these entities must all be of the same type, so it might be better to make
//the parameter of type EntityCollection<ShippersEntity>
public static void BulkSaveEntityCollection(IEntityCollection2 entitiesToSave)
{
if (entitiesToSave.Count > 0)
{
//create datatable
DataTable dt = new DataTable();
using (DataAccessAdapter adapter = new DataAccessAdapter())
{
//get the name of the table from the first entity/field
//using the exposed method on the DataAccessAdapter
string tableName = adapter.GetTableName(entitiesToSave[0].Fields[0]);
//use the first entity to create columns in the datatable
foreach (IEntityField2 field in entitiesToSave[0].Fields)
{
dt.Columns.Add(adapter.GetFieldName(field));
}
//create a row in the datatable for each entity
foreach (IEntity2 entity in entitiesToSave)
{
DataRow dr = dt.NewRow();
foreach (IEntityField2 field in entity.Fields)
{
//get the field's actual database name
//using the exposed method on the DataAccessAdapter
dr[adapter.GetFieldName(field)] = field.CurrentValue;
}
dt.Rows.Add(dr);
}
using (SqlConnection conn = new SqlConnection(adapter.ConnectionString))
{
conn.Open();
SqlBulkCopyOptions options = SqlBulkCopyOptions.Default;
SqlBulkCopy bulk = new SqlBulkCopy(conn, options, null);
bulk.DestinationTableName = tableName;
bulk.WriteToServer(dt);
}
}
}
}
}
}
I created a simple form and some code for a unit test, since I don't have MbUnit or NUnit installed at home.
Form1.Designer.cs:
namespace SearchObjects.UI
{
partial class Form1
{
/// <summary>
/// Required designer variable.
/// </summary>
private System.ComponentModel.IContainer components = null;
/// <summary>
/// Clean up any resources being used.
/// </summary>
/// <param name="disposing">true if managed resources should be disposed; otherwise, false.</param>
protected override void Dispose(bool disposing)
{
if (disposing && (components != null))
{
components.Dispose();
}
base.Dispose(disposing);
}
#region Windows Form Designer generated code
/// <summary>
/// Required method for Designer support - do not modify
/// the contents of this method with the code editor.
/// </summary>
private void InitializeComponent()
{
this.btnStart = new System.Windows.Forms.Button();
this.txtOutput = new System.Windows.Forms.TextBox();
this.btnClear = new System.Windows.Forms.Button();
this.SuspendLayout();
//
// btnStart
//
this.btnStart.Location = new System.Drawing.Point(61, 21);
this.btnStart.Name = "btnStart";
this.btnStart.Size = new System.Drawing.Size(75, 23);
this.btnStart.TabIndex = 0;
this.btnStart.Text = "Start";
this.btnStart.UseVisualStyleBackColor = true;
this.btnStart.Click += new System.EventHandler(this.btnStart_Click);
//
// txtOutput
//
this.txtOutput.Location = new System.Drawing.Point(21, 61);
this.txtOutput.Multiline = true;
this.txtOutput.Name = "txtOutput";
this.txtOutput.ScrollBars = System.Windows.Forms.ScrollBars.Both;
this.txtOutput.Size = new System.Drawing.Size(301, 583);
this.txtOutput.TabIndex = 1;
//
// btnClear
//
this.btnClear.Location = new System.Drawing.Point(196, 21);
this.btnClear.Name = "btnClear";
this.btnClear.Size = new System.Drawing.Size(75, 23);
this.btnClear.TabIndex = 2;
this.btnClear.Text = "Clear";
this.btnClear.UseVisualStyleBackColor = true;
this.btnClear.Click += new System.EventHandler(this.btnClear_Click);
//
// Form1
//
this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F);
this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font;
this.ClientSize = new System.Drawing.Size(344, 667);
this.Controls.Add(this.btnClear);
this.Controls.Add(this.txtOutput);
this.Controls.Add(this.btnStart);
this.Name = "Form1";
this.Text = "Form1";
this.ResumeLayout(false);
this.PerformLayout();
}
#endregion
private System.Windows.Forms.Button btnStart;
private System.Windows.Forms.TextBox txtOutput;
private System.Windows.Forms.Button btnClear;
}
}
Form1.cs:
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Text;
using System.Windows.Forms;
using SearchObjects.DAO;
using SearchObjects.DAL.EntityClasses;
using SearchObjects.DAL.HelperClasses;
namespace SearchObjects.UI
{
public partial class Form1 : Form
{
Random _random = new Random();
public Form1()
{
InitializeComponent();
}
private void btnStart_Click(object sender, EventArgs e)
{
TestSetup();
}
private void TestSetup()
{
//number if times to run tests
const int outerLoopIterations = 5;
//number of inserts for each test
const int numberToTest = 5000;
double normalTotalTime = 0;
double bulkTotalTime = 0;
//do one bulk and one normal save to "warm up"
//otherwise the first test is always significantly slower
txtOutput.Text += "\r\n";
txtOutput.Text += "Warmup starting . . ." + "\r\n";
EntityCollection<ShippersEntity> temp = GetShippersCollection(1);
bulkTotalTime += TestBulkInsert(temp, 0);
temp = GetShippersCollection(1);
normalTotalTime += TestNormalInsert(temp, 0);
txtOutput.Text += "Warmup ending." + "\r\n";
txtOutput.Text += "\r\n";
for (int x = 0; x < outerLoopIterations; x++)
{
//get random entity collection
EntityCollection<ShippersEntity> col = GetShippersCollection(numberToTest);
bulkTotalTime += TestBulkInsert(col, x);
//get random entity collection
col = GetShippersCollection(numberToTest);
normalTotalTime += TestNormalInsert(col, x);
}
txtOutput.Text += "\r\n";
txtOutput.Text += "Total Bulk Time: " + bulkTotalTime + "\r\n";
txtOutput.Text += "Average Bulk Time: " + bulkTotalTime / outerLoopIterations + "\r\n";
txtOutput.Text += "Total Normal Time: " + normalTotalTime + "\r\n";
txtOutput.Text += "Average Normal Time: " + normalTotalTime / outerLoopIterations + "\r\n";
txtOutput.Text += "\r\n";
}
private double TestNormalInsert(EntityCollection<ShippersEntity> entitiesToSave, int iteration)
{
DateTime start = DateTime.Now;
DAOGeneral.SaveEntityCollection(entitiesToSave);
DateTime end = DateTime.Now;
double time = WriteTime(start, end, "Time for Normal Insert, #" + (iteration + 1) + " : ");
return time;
}
private double TestBulkInsert(EntityCollection<ShippersEntity> entitiesToSave, int iteration)
{
DateTime start = DateTime.Now;
DAOGeneral.BulkSaveEntityCollection(entitiesToSave);
DateTime end = DateTime.Now;
double time = WriteTime(start, end, "Time for Bulk Insert, #" + (iteration + 1) + " : ");
return time;
}
private EntityCollection<ShippersEntity> GetShippersCollection(int collectionSize)
{
EntityCollection<ShippersEntity> col = new EntityCollection<ShippersEntity>();
for (int x = 0; x < collectionSize; x++)
{
col.Add(CreateRandomShipper());
}
return col;
}
private ShippersEntity CreateRandomShipper()
{
ShippersEntity e = new ShippersEntity();
e.CompanyName = "++" + _random.Next(25000) + "++";
e.Phone = "++" + _random.Next(5000) + "++";
return e;
}
private double WriteTime(DateTime start, DateTime end, string message)
{
TimeSpan span = end - start;
txtOutput.Text += message + span.TotalSeconds + "\r\n";
Application.DoEvents();
return span.TotalSeconds;
}
private void btnClear_Click(object sender, EventArgs e)
{
txtOutput.Clear();
}
}
}
Joined: 22-Feb-2005
Chester wrote:
Interesting. Thanks for sharing.
No sweat--thought some folks might find it useful. I don't believe bulk loading should be done from the app in most cases, but sometime a situation comes up when you need to create large amounts of data in the app layer. This code will speed up bulk inserts 100-fold.
OT: where do you live in Chicago, Chester?
Phil