Adapter Version 2.0.0.0 Final RTL version for ORMSupportClasses 2.0.7.219 SqlServer 2005 (sever is 64bit, clients 32bit) Inheritance is used on some referenced entities .NET 2.0

I have created a test app to load-up the Database with data for performance testing.

basically it is just

loop (1 to 1000000 step 1000) create new entity collection loop (1 to 1000) create new entity + all referenced entities add to collection end loop save entity collection (recursively, no reload) clear collection end loop

however the memory is not release until the app is terminated, this means that it runs out of memory pretty quickly.

I will post my c# code in the following message

using System;
using System.Threading;
using System.Windows.Forms;
using Cwc.Common.ModelClassLibrary;
using Cwc.WorkManagemant.WaterWorksServiceLayer;
using Cwc.WaterWorks;
using Cwc.WaterWorks.EntityClasses;
using Cwc.WaterWorks.FactoryClasses;
using Cwc.WaterWorks.HelperClasses;

namespace Cwc.WaterWorks.ProfileBuilder
{
    public partial class Form1 : Form
    {
        public Form1()
        {
            InitializeComponent();
        }

        private void CreateJobs()
        {
            WaterWorksServiceLayer profileStartWaterWorksServiceLayer = new WaterWorksServiceLayer("wmsPrototype3Profile");
            profileStartWaterWorksServiceLayer.adapter.KeepConnectionOpen = true;
            if (checkBox1.Checked)
            {
                profileStartWaterWorksServiceLayer.DeleteAllData();
                profileStartWaterWorksServiceLayer.CreateData();
            }

            JobTypeEntity JobType01 = profileStartWaterWorksServiceLayer.NewConnectionJobType(3, "Type01", "Type 01", "testing");
            profileStartWaterWorksServiceLayer.adapter.SaveEntity(JobType01, true, true);
            JobTypeEntity JobType02 = profileStartWaterWorksServiceLayer.NewConnectionJobType(3, "Type02", "Type 02", "testing");
            profileStartWaterWorksServiceLayer.adapter.SaveEntity(JobType02, true, true);
            JobTypeEntity JobType03 = profileStartWaterWorksServiceLayer.NewConnectionJobType(3, "Type03", "Type 03", "testing");
            profileStartWaterWorksServiceLayer.adapter.SaveEntity(JobType03, true, true);

            Random random = new Random(12345);
            DateTime currentDateTime = new DateTime(2006, 1, 1);

            int startIndex = 0;
            int endIndex = (int)numericUpDown1.Value;

            for (int index = startIndex; index < endIndex; index += 100)
            {
                CreateJobRange(index, index + 100, currentDateTime, JobType01, JobType02, JobType03, random);
                GC.Collect();
                GC.WaitForPendingFinalizers();
            }

            textBox1.Text += "Finished" + Environment.NewLine;
        }

        private void CreateJobRange(int startIndex, int endIndex, DateTime currentDateTime, JobTypeEntity JobType01, JobTypeEntity JobType02, JobTypeEntity JobType03, Random random)
        {
            DateTime start = DateTime.Now;
            EntityCollection<JobEntity> jobs = new EntityCollection<JobEntity>(new JobEntityFactory());
            WaterWorksServiceLayer profileWaterWorksServiceLayer = new WaterWorksServiceLayer("wmsPrototype3Profile");
            profileWaterWorksServiceLayer.adapter.KeepConnectionOpen = true;
            for (int index = startIndex; index < endIndex; index++)
            {
                JobTypeEntity jobType;
                switch (random.Next(0, 3))
                {
                case 0:
                {
                    jobType = JobType01;
                    break;
                }
                case 1:
                {
                    jobType = JobType02;
                    break;
                }
                default:
                {
                    jobType = JobType03;
                    break;
                }
                }

                JobEntity job = profileWaterWorksServiceLayer.CreateJob("Profile-" + index.ToString(Global.IdFormat), jobType, Customer.NewTest(), null, "ProfileTests");

                currentDateTime = currentDateTime.AddSeconds(random.Next(0, 60 * 60 * 8));
                job.CreatedAt = currentDateTime;
                if (random.Next(0, 10000) != 0)
                {
                    job.StartedAt = job.CreatedAt.AddSeconds(random.Next(1, 1000));
                    int finalState = random.Next(0, 1000);
                    if (finalState == 0)
                    {
                        job.CancelledAt = ((DateTime)job.StartedAt).AddSeconds(random.Next(1, 100000));
                        job.ReasonWaiting = "Cancelled";
                        job.CancelledBy = "A very naughty boy";
                        job.CancelledReason = "2B|!2B";
                    }
                    else if (finalState < 900)
                    {
                        job.ClosedAt = ((DateTime)job.StartedAt).AddSeconds(random.Next(1, 100000));
                        job.ReasonWaiting = "Closed";
                    }
                    else
                    {
                        switch (random.Next(0, 7))
                        {
                        case 0:
                        {
                            job.ReasonWaiting = "Train Late";
                            break;
                        }
                        case 1:
                        {
                            job.ReasonWaiting = "Stood up";
                            break;
                        }
                        case 2:
                        {
                            job.ReasonWaiting = "Dog ate my homework";
                            break;
                        }
                        case 4:
                        {
                            job.ReasonWaiting = "Overslept";
                            break;
                        }
                        case 5:
                        {
                            job.ReasonWaiting = "Overslept (again)";
                            break;
                        }
                        default:
                        {
                            job.ReasonWaiting = "Executing";
                            break;
                        }
                        }

                    }
                }
                jobs.Add(job);
            }
            profileWaterWorksServiceLayer.adapter.SaveEntityCollection(jobs, false, true);
            foreach(JobEntity job in jobs)
            {
                job.Parameters.Clear();
                job.JobSteps.Clear();
            }
            jobs.Clear();
            profileWaterWorksServiceLayer.adapter.CloseConnection();
            textBox1.Text += endIndex.ToString(Global.IdFormat) + " " + DateTime.Now.ToString(Global.IsoDate) + " (" + DateTime.Now.Subtract(start).TotalSeconds.ToString("0") + ")" + Environment.NewLine;
        }

        Thread workerThread = null;
        private void button1_Click(object sender, EventArgs e)
        {
            if (workerThread == null || !workerThread.IsAlive)
            {
                button1.Text = "Stop";
                workerThread = new Thread(new ThreadStart(CreateJobs));
                textBox1.Text = "Started" + Environment.NewLine;
                workerThread.Start();
                timer1.Enabled = true;
            }
            else
            {
                workerThread.Abort();
                textBox1.Text += "Interrupted!" + Environment.NewLine;
            }
        }

        private void Form1_Load(object sender, EventArgs e)
        {

        }

        private void timer1_Tick(object sender, EventArgs e)
        {
            if (workerThread == null || !workerThread.IsAlive)
            {
                workerThread = null;
                button1.Text = "Go";
                timer1.Enabled = false;
            }
        }


    }
}

        public void DeleteAllData()
        {
            adapter.StartTransaction(IsolationLevel.Unspecified, "Delete All Table Data");
            adapter.DeleteEntitiesDirectly("WorkItemEntity", null);
            adapter.DeleteEntitiesDirectly("AuditEntity", null);
            adapter.DeleteEntitiesDirectly("JobImageEntity", null);
            adapter.DeleteEntitiesDirectly("JobActionDataEntity", null);
            adapter.DeleteEntitiesDirectly("JobActionEntity", null);
            adapter.DeleteEntitiesDirectly("JobStepEntity", null);
            adapter.DeleteEntitiesDirectly("InvoiceLineEntity", null);
            adapter.DeleteEntitiesDirectly("InvoiceInfrastructureEntity", null);
            adapter.DeleteEntitiesDirectly("InvoiceEntity", null);
            adapter.DeleteEntitiesDirectly("ParameterBoolEntity", null);
            adapter.DeleteEntitiesDirectly("ParameterStringEntity", null);
            adapter.DeleteEntitiesDirectly("ParameterDecimalEntity", null);
            adapter.DeleteEntitiesDirectly("ParameterIntegerEntity", null);
            adapter.DeleteEntitiesDirectly("ParameterDatetimeEntity", null);
            adapter.DeleteEntitiesDirectly("ParameterEnumerationEntity", null);
            adapter.DeleteEntitiesDirectly("ParameterEntity", null);
            adapter.DeleteEntitiesDirectly("JobLockEntity", null);
            adapter.DeleteEntitiesDirectly("JobEntity", null);
            adapter.DeleteEntitiesDirectly("MeterEntity", null);
            adapter.DeleteEntitiesDirectly("CustomerEntity", null);
            adapter.DeleteEntitiesDirectly("AddressEntity", null);
            adapter.DeleteEntitiesDirectly("PrerequisiteParameterEntity", null);
            adapter.DeleteEntitiesDirectly("PrerequisiteEntity", null);
            adapter.DeleteEntitiesDirectly("PrintActionEntity", null);
            adapter.DeleteEntitiesDirectly("InstructionActionEntity", null);
            adapter.DeleteEntitiesDirectly("FileActionEntity", null);
            adapter.DeleteEntitiesDirectly("ActionEntity", null);
            adapter.DeleteEntitiesDirectly("JobTypeStepEntity", null);
            adapter.DeleteEntitiesDirectly("JobTypeParameterEntity", null);
            adapter.DeleteEntitiesDirectly("JobTypeEntity", null);
            adapter.Commit();
        }

        public void CreateData()
        {
            NewJobTypeParameter(null, "ApplicationReceived", 0, "Boolean", "false");
            NewJobTypeParameter(null, "EstimateCompleted", 0, "Boolean", "false");
            NewJobTypeParameter(null, "EstimateAccepted", 0, "Boolean", "false");
            NewJobTypeParameter(null, "InvoiceCompleted", 0, "Boolean", "false");
            NewJobTypeParameter(null, "WorkStatus", 0, "String", null);
            NewJobTypeParameter(null, "WorkStatusFriendly", 0, "String", null);
            NewJobTypeParameter(null, "WorkCompletionCode", 0, "String", null);
            NewJobTypeParameter(null, "WorkRejected", 0, "Boolean", "false");
            NewJobTypeParameter(null, "WorkReceived", 0, "Boolean", "false");
            NewJobTypeParameter(null, "WorkAccepted", 0, "Boolean", "false");
            NewJobTypeParameter(null, "WorkCompleted", 0, "Boolean", "false");
            NewJobTypeParameter(null, "WorkClosed", 0, "Boolean", "false");
            NewJobTypeParameter(null, "DateRequired", 0, "DateTime", null);
        }


        public void CreateJobTypeParameters(JobTypeEntity jobType)
        {
            jobType.JobTypeParameters.Add(NewJobTypeParameter(jobType, "DomesticFlag", 2, "Boolean", "false"));
        }

        public JobTypeParameterEntity NewJobTypeParameter(JobTypeEntity jobType, string parameterName, int sequence, string type, string defaultValue)
        {

            JobTypeParameterEntity jobTypeParameter = new JobTypeParameterEntity();
            jobTypeParameter.Name = parameterName;
            jobTypeParameter.JobType = jobType;
            jobTypeParameter.JobTypeCategory = null;
            adapter.FetchEntityUsingUniqueConstraint(jobTypeParameter, jobTypeParameter.ConstructFilterForUCNameJobTypeFkJobTypeCategoryFk());
            if (jobTypeParameter.IsNew)
            {
                jobTypeParameter.Type = type;
                jobTypeParameter.Description = "Temp for testing";
                jobTypeParameter.DefaultValue = defaultValue;
                jobTypeParameter.AuditTrail = false;
                jobTypeParameter.AskReasonWhenNotDefault = false;
                jobTypeParameter.Visible = true;
                jobTypeParameter.Editable = true;
                jobTypeParameter.MultiLine = false;
                jobTypeParameter.ShowOnCreate = false;
                jobTypeParameter.EssentialOnCreate = false;
                jobTypeParameter.EnumerationSet = null;
                jobTypeParameter.Sequence = sequence;
                adapter.SaveEntity(jobTypeParameter, true, false);
            }
            return jobTypeParameter;
        }


        public JobEntity CreateJobOnDatabase()
        {
            Logger testingLogger = TextWriterLogger.NewConsoleLogger();
            testingLogger.Enabled = true;
            JobTypeEntity newJobType = NewConnectionJobType(3, "C106", "New Connection", "connect");
            JobEntity newJob = CreateJob("New Connection Job", newJobType, Customer.NewTest(), testingLogger, "UnitTests");
            adapter.SaveEntity(newJob, true, true);
            return newJob;
        }

        public JobTypeEntity NewConnectionJobType(int catId, string code, string description, string work)
        {
            JobTypeCategoryEntity category = new JobTypeCategoryEntity(catId);
            adapter.FetchEntity(category);

            JobTypeEntity jobType = new JobTypeEntity();
            jobType.Code = code;
            adapter.FetchEntityUsingUniqueConstraint(jobType, jobType.ConstructFilterForUCCode());
            jobType.Description = description;
            jobType.WorkToBeDoneDefault = work;
            jobType.MeterWorkType = null;
            jobType.JobTypeCategory = category;

            bool isNew = jobType.IsNew;

            adapter.SaveEntity(jobType, true, false);

            if (!isNew)
            {
                return jobType;
            }

            CreateJobTypeParameters(jobType);

            EntityCollection<JobTypeParameterEntity> jobTypeParameters = GetAllParameters();
            EntityCollection<TemplateEntity> templates = TemplateEntity.GetAllLiveTemplates(adapter);

            JobTypeStepEntity newStep = new JobTypeStepEntity(1, "Create Application");
            newStep.AddAction(new PrintActionEntity("Connection Application Letter", GetTemplate(templates, "ConnectionApplicationLetterTemplate")));
            newStep.AddAction(new PrintActionEntity("Connection Application Form", GetTemplate(templates, "ConnectionApplicationFormTemplate")));
            newStep.AddAction(new InstructionActionEntity("Connection Application Form", "sam.mackrill@cambridge-water.co.uk", GetTemplate(templates, "ConnectionApplicationInstructionTemplate")));
            jobType.AddStep(newStep);

            newStep = new JobTypeStepEntity(2, "Do Estimate");
            newStep.AddPrerequisite(new PrerequisiteEntity("Waiting for Application to be received", JobTypeParameterEntity.GetParameter(jobTypeParameters, "ApplicationReceived")));
            newStep.AddAction(new InstructionActionEntity("Do Estimate", "sam.mackrill@cambridge-water.co.uk", GetTemplate(templates, "EstimateInstructionTemplate")));
            jobType.AddStep(newStep);

            newStep = new JobTypeStepEntity(3, "Give Estimate");
            newStep.AddPrerequisite(new PrerequisiteEntity("Waiting for Estimate to be done", JobTypeParameterEntity.GetParameter(jobTypeParameters, "EstimateCompleted")));
            newStep.AddAction(new PrintActionEntity("Connection Estimation Letter", GetTemplate(templates, "ConnectionEstimationLetterTemplate")));
            newStep.AddAction(new PrintActionEntity("Inspection Underground Form", GetTemplate(templates, "ConnectionInspectionUndergroundTemplate")));
            jobType.AddStep(newStep);

            newStep = new JobTypeStepEntity(4, "Issue Job");
            newStep.AddPrerequisite(new PrerequisiteEntity("Waiting for Customer to return Accepted Estimate", JobTypeParameterEntity.GetParameter(jobTypeParameters, "EstimateAccepted")));
            newStep.AddAction(new PrintActionEntity("Issue Job to May Gurney", GetTemplate(templates, "MayGurneyJobTemplate")));
            jobType.AddStep(newStep);

            newStep = new JobTypeStepEntity(5, "Investigation");
            newStep.AddPrerequisite(new PrerequisiteEntity("Waiting for Job Completion fromy May Guerney", JobTypeParameterEntity.GetParameter(jobTypeParameters, "WorkCompleted")));
            newStep.AddAction(new PrintActionEntity("Re-Issue Job to May Gurney", GetTemplate(templates, "MayGurneyJobTemplate")));
            newStep.SkipOnJobTypeParameter = JobTypeParameterEntity.GetParameter(jobTypeParameters, "ApplicationReceived");
            newStep.SkipFlag = false;
            jobType.AddStep(newStep);

            newStep = new JobTypeStepEntity(6, "Complete Job");
            newStep.AddPrerequisite(new PrerequisiteEntity("Waiting for Job Completion fromy May Guerney", JobTypeParameterEntity.GetParameter(jobTypeParameters, "WorkCompleted")));
            newStep.AddAction(new PrintActionEntity("Inspection Aboveground Form", GetTemplate(templates, "ConnectionInspectionAbovegroundTemplate")));
            newStep.AddAction(new PrintActionEntity("Water Supply (Water fittings) Regulations 1999 Infringement Record Form", GetTemplate(templates, "WaterSupplyInfringementTemplate")));
            newStep.AddAction(new InstructionActionEntity("Raise Invoice", "sam.mackrill@cambridge-water.co.uk", GetTemplate(templates, "InvoiceTemplate")));
            jobType.AddStep(newStep);

            newStep = new JobTypeStepEntity(7, "Finish Job");
            newStep.AddPrerequisite(new PrerequisiteEntity("Waiting for Invoice to be issued", JobTypeParameterEntity.GetParameter(jobTypeParameters, "InvoiceCompleted")));
            jobType.AddStep(newStep);

            return jobType;
        }

        public static TemplateEntity GetTemplate(EntityCollection<TemplateEntity> templates, string name)
        {
            List<int> templateIds = templates.FindMatches(TemplateFields.Name == name);
            if (templateIds.Count == 1)
            {
                return templates[templateIds[0]];
            }
            return null;
        }

        public JobEntity CreateJob(string description, JobTypeEntity jobType, Customer customer, Logger logger, string who)
        {
            JobEntity newJob = new JobEntity();
            newJob.Logger = logger;
            newJob.Description = description;
            newJob.Customer = new CustomerEntity(customer);
            newJob.WorkLocation = new AddressEntity();
            newJob.WorkLocation.SetFromAddress(customer.WorkLocation);

            newJob.JobType = jobType;
            foreach (JobTypeStepEntity jobTypeStep in jobType.Steps)
            {
                newJob.JobSteps.Add(new JobStepEntity(jobTypeStep));
            }
            // Add global parameters
            foreach (JobTypeParameterEntity jobTypeParameter in GetGlobalParameters())
            {
                newJob.Parameters.Add(ParameterEntity.CreateNewParameter(newJob, jobTypeParameter));
            }
            // Add custom parameters for Job Type Category
            foreach (JobTypeParameterEntity jobTypeParameter in GetCategoryParameters(jobType.JobTypeCategory))
            {
                newJob.Parameters.Add(ParameterEntity.CreateNewParameter(newJob, jobTypeParameter));
            }
            // Add custom parameters for Job Type
            foreach (JobTypeParameterEntity jobTypeParameter in jobType.JobTypeParameters)
            {
                newJob.Parameters.Add(ParameterEntity.CreateNewParameter(newJob, jobTypeParameter));
            }
            newJob.CreatedAt = DateTime.Now;
            newJob.CreatedBy = who;
            newJob.ModifiedAt = newJob.CreatedAt;
            newJob.ModifiedBy = who;
            newJob.ReasonWaiting = "Executing";
            newJob.Priority = 1;

            if (logger != null)
            {
                logger.LogInfo("JobEntity", "New Job Created: " + description);
            }
            return newJob;
        }

That's a big post and a lot of source code to look at

however the memory is not release until the app is terminated, this means that it runs out of memory pretty quickly.

I will post my c# code in the following message

The Garbage Collector may take time to clean things up. You can force GC to start cleaning up, if you want, by calling: System.GC.Collect method.

That's why I summerised it at the top I'm sure you don't need it all but don't know what is relevant...

I am doing a GC

           for (int index = startIndex; index < endIndex; index += 100)
            {
                CreateJobRange(index, index + 100, currentDateTime, JobType01, JobType02, JobType03, random);
                GC.Collect();
                GC.WaitForPendingFinalizers();
            }

1) why do you build and save at the same time? this takes more time on the db 2) I dont see you start a transaction, do you start a transaction somewhere that's not in the code snippet posted? 3) As soon as you set a reference to an entity which is hold somewhere in memory the entity in the pack to save is thus also held in memory as the counter reference is also set (ex: customer.Orders.Add(order) will make order.Customer be set to customer).

There's no known mem-leak in our code, we profile that a lot and are very keen on these kind of things. The last known mem-leak was fixed on 12-dec-2006 when event handlers weren't cleaned up in a Clear.

When you clear your collection, be sure that the collection is the only reference to the entities in that collection, otherwise the entities aren't removed from memory of course.

When you start a transaction up front, all entities are kept alive till Commit.

1) Because I am reusing code from the main app 2) No, I am not using a transaction here 3) I expect that is what is causing this but I am not sure exactyly how yet

I am not suggesting it is LLBLGen leaking memeory, just my poor usage. Really I am trying to understand what I am doing wrong here. I will double check, do I need to do a clear on every single collection that has been loaded? even in referenced entities?

All references are set to null before doing the GC, anyway I am doing the save in a thread so all references sould get freed when the thread ends, unless it is something static in LLBLGen I am not clearing....?

Thanks for your help

--Sam

Hi Sam,

I had something like this happen before, i would check this code

 public JobTypeParameterEntity NewJobTypeParameter(JobTypeEntity jobType, string parameterName, int sequence, string type, string defaultValue)
        {

            JobTypeParameterEntity jobTypeParameter = new JobTypeParameterEntity();
            jobTypeParameter.Name = parameterName;
            jobTypeParameter.JobType = jobType;
            jobTypeParameter.JobTypeCategory = null;
            adapter.FetchEntityUsingUniqueConstraint(jobTypeParameter, jobTypeParameter.ConstructFilterForUCNameJobTypeFkJobTypeCategoryFk());
            if (jobTypeParameter.IsNew)
            {
                jobTypeParameter.Type = type;
                jobTypeParameter.Description = "Temp for testing";
                jobTypeParameter.DefaultValue = defaultValue;
                jobTypeParameter.AuditTrail = false;
                jobTypeParameter.AskReasonWhenNotDefault = false;
                jobTypeParameter.Visible = true;
                jobTypeParameter.Editable = true;
                jobTypeParameter.MultiLine = false;
                jobTypeParameter.ShowOnCreate = false;
                jobTypeParameter.EssentialOnCreate = false;
                jobTypeParameter.EnumerationSet = null;
                jobTypeParameter.Sequence = sequence;
                adapter.SaveEntity(jobTypeParameter, true, false);
            }
            return jobTypeParameter;
        }

you are setting the new JobTypeParameter's JobType property to the passed in JobType parameter which is an entity. That entity will hold a referene to the JobTypeParameter even after the code is done, it will not release until you clear JobType. Try setting the id property of the new JobTypeParameter's JobType instead of the object property. Something like this:

jobTypeParameter.JobTypeId = jobType.JobTypeId;

I am not sure about the field names but that might point you in the right direction.

Good Luck,

Frank

Good point, Frank. I think that could be it.

Yep, excellent point. I am working on this but only inbetween other stuff. I will let you know what I find.

Hi Sam -

Were you able to find your problem?

I'm trying to force a Garbage Collection of one of my entities, and it's still hanging around for some reason. Even though after traversing the object graph (using ObjectGraphUtils.ProduceTopologyOrderedList), it's nowhere to be found. I'm using a WeakReference.IsAlive to test whether it's been collected or not.

Please let me know how you got your Entity to garbage collect. Much appreciated - Thanks!

Ryan D. Hatch

Ryan, would you please stop opening old threads? If you have a problem, please post a new thread, with explanation of your problem so we can help you. opening old threads isn't likely going to help you because it can be the person you ask the question to isn't browsing here if someone answered an old thread they posted in. So please if you have a problem we can help with (also with the couple of other threads you opened last week), please open a new thread, and specify exactly what the problem is. Thanks

Hi Frans -

Sorry about that. Didn't realize that was policy. I posted in the older forums because I was following up with people to see if they found a solution. Many times people don't come back with how they solved the problems. I simply thought it would be more efficient asking people who've solved the problems than opening new threads.

I wasn't attempting to describe my problems in full. Thanks...

Ryan

rdhatch wrote:

Hi Frans -

Sorry about that. Didn't realize that was policy.

http://www.llblgen.com/tinyforum/Messages.aspx?ThreadID=7717

Posting in old threads has the side effect that we have to decide if the older posts in the thread relate to the new question and how to help the customer better. It's best if you post a new thread, and if you want to refer to older threads, post links to older threads in your post.

I posted in the older forums because I was following up with people to see if they found a solution. Many times people don't come back with how they solved the problems. I simply thought it would be more efficient asking people who've solved the problems than opening new threads. I wasn't attempting to describe my problems in full. Thanks... Ryan

Ok Well, just in case: if you need help with getting the problems solved, just post a thread with a question and we'll look into it