How to optimize EF performance on foreach statement

Question

Here is my code; as you can see, I am saving changes for each row, but I want to make performance faster because I have large amount of data every time, like 50.000 or 100.000 rows or more. This takes a lot of time to finish.

How can I improve my EF SaveChanges performance? I tried bulksave, bulkupdate with some third party library, but it is not updating in the database. This takes 2 hours to update 50.000 rows. I want to improve time for this method.

private void TransferOrders()
{
    using (var context = new BbsfDbContext())
    {
        context.DisableFilter(AbpDataFilters.MayHaveTenant);
        context.DisableFilter("LanguageSpecificFilter");

        var sapOrders = context.SapOrders
                               .Where(p => p.VBTYP != null && 
                                           p.VBTYP.ToLower() == OrderDocumentType && 
                                           p.IsRead == false)
                                //.Where(p => p.VBTYP != null && p.VBTYP.ToLower() == OrderDocumentType && p.Id == 3025)
                               .Where(p => !ActiveUsersOnly || context.Users.Where(u => u.IsActive).Select(a => a.MainVendor.SapCode).Contains(p.KUNNR))
                               .OrderBy(p => p.CreatedDate)
                               .ToList();

        if (sapOrders.Any())
        {
            foreach (var item in sapOrders)
            {
                try
                {
                    var order = context.Orders.FirstOrDefault(p => p.SapCode == item.VBELN);

                    var isExist = context.SapOrderDetails.Any(p => p.DOCNUM == item.DOCNUM);

                    if (isExist)
                    {
                        var salesOrganization = context.SalesOrganizations.FirstOrDefault(p => p.SapCode == item.VKORG);

                        if (salesOrganization == null)
                            continue;

                        var distributionChannel = context.DistributionChannels.FirstOrDefault(p => p.SapCode == item.VTWEG);

                        if (distributionChannel == null)
                            continue;

                        var salesDepartment = context.SalesDepartments.FirstOrDefault(p => p.SapCode == item.SPART);

                        if (salesDepartment == null)
                            continue;

                        var salesOffice = context.SalesOffices
                                                 .FirstOrDefault(p => p.SapCode == item.VKBUR &&
                                                     p.SalesOrganization.Id == salesOrganization.Id &&
                                                     p.DistributionChannel.Id == distributionChannel.Id &&
                                                     p.SalesDepartment.Id == salesDepartment.Id);
                        if (salesOffice == null)
                            continue;

                        var ordererCustomer = context.Customers
                                .FirstOrDefault(p => p.SapCode == item.KUNNR &&
                                                     p.SalesOrganization.Id == salesOrganization.Id &&
                                                     p.DistributionChannel.Id == distributionChannel.Id &&
                                                     p.SalesDepartment.Id == salesDepartment.Id &&
                                                     p.SalesOffice.Id == salesOffice.Id);

                        var recipientCustomer = context.Customers
                                .FirstOrDefault(p => p.SapCode == item.KUNWE &&
                                                     p.SalesOrganization.Id == salesOrganization.Id &&
                                                     p.DistributionChannel.Id == distributionChannel.Id &&
                                                     p.SalesDepartment.Id == salesDepartment.Id &&
                                                     p.SalesOffice.Id == salesOffice.Id);

                        if (recipientCustomer == null)
                            recipientCustomer = context.Customers
                                    .FirstOrDefault(p => p.SapCode == item.KUNWE &&
                                                         p.SalesOrganization.Id == salesOrganization.Id &&
                                                         p.DistributionChannel.Id == distributionChannel.Id &&
                                                         p.SalesDepartment.Id == salesDepartment.Id &&
                                                         p.SalesOffice == null);

                        if (ordererCustomer == null || recipientCustomer == null)
                            continue;

                        if (order == null)
                        {
                            order = new Order
                                {
                                    SapCode = item.VBELN,
                                    SapOrderDate = item.AUDAT,
                                    DocumentType = context.DocumentTypes.FirstOrDefault(p => p.SapCode == item.VBTYP),
                                    SalesDocument = context.SalesDocuments.FirstOrDefault(p => p.SapCode == item.AUART),
                                    BaseAmount = item.NETWR,
                                    TotalTax = item.MWSBT,
                                    Currency = context.CurrencyDefinitions.FirstOrDefault(p => p.SapCode == item.WAERK),
                                    SalesOrganization = salesOrganization,
                                    DistributionChannel = distributionChannel,
                                    SalesDepartment = salesDepartment,
                                    SalesGroup = context.SalesGroups.FirstOrDefault(p => p.SapCode == item.VKGRP && p.SalesOffice.Id == salesOffice.Id),
                                    SalesOffice = salesOffice,
                                    RequestedDeliveryDate = item.VDATU,
                                    SASNo = item.BSTNK,
                                    SASOrderDate = item.BSTDK ?? item.AUDAT,
                                    OrdererCustomer = ordererCustomer,
                                    RecipientCustomer = recipientCustomer,
                                    //PRSDT
                                    Status = OrderStatus.Approved,
                                    Type = OrderType.MainVendor,
                                    DeliveryAddress = context.CustomerAddressBooks.FirstOrDefault(p => p.MainVendor.Id == ordererCustomer.Id && p.SubVendor.Id == recipientCustomer.Id),
                                    CreationTime = DateTime.Now,
                                    LastModificationTime = DateTime.Now,
                                    CreatorUserId = context.Users.First(p => p.UserName == AbpUserBase.AdminUserName).Id,
                                    LastModifierUserId = context.Users.First(p => p.UserName == AbpUserBase.AdminUserName).Id,
                                    IsSubVendorOrder = false,
                                    IsSameDayDelivery = false,
                                    RepresentativeId = context.Users.First(p => p.UserName == AbpUserBase.AdminUserName).Id
                                    //ProductionSite
                                    //RejectionReason =//todo:bu silinmeli iptal kalem bazında burada statu olmalı
                                };
                                var savedOrder = context.Orders.Add(order);
                                context.SaveChanges();

                                order.SASNo = BbsfConsts.KeasOrderNumberPrefix + savedOrder.Id;
                            }
                            else
                            {
                                order.SapOrderDate = item.AUDAT;
                                order.DocumentType = context.DocumentTypes.FirstOrDefault(p => p.SapCode == item.VBTYP);
                                order.SalesDocument = context.SalesDocuments.FirstOrDefault(p => p.SapCode == item.AUART);
                                order.BaseAmount = item.NETWR;
                                order.TotalTax = item.MWSBT;
                                order.Currency = context.CurrencyDefinitions.FirstOrDefault(p => p.SapCode == item.WAERK);
                                order.SalesOrganization = salesOrganization;
                                order.DistributionChannel = distributionChannel;
                                order.SalesDepartment = salesDepartment;
                                order.SalesGroup = context.SalesGroups.FirstOrDefault(p => p.SapCode == item.VKGRP && p.SalesOffice.Id == salesOffice.Id);
                                order.SalesOffice = salesOffice;
                                order.RequestedDeliveryDate = item.VDATU;
                                order.SASNo = BbsfConsts.KeasOrderNumberPrefix + order.Id;
                                //order.SASOrderDate = item.BSTDK.HasValue ? item.BSTDK : item.AUDAT;
                                order.OrdererCustomer = ordererCustomer;
                                order.RecipientCustomer = recipientCustomer;
                                //PRSDT
                                //order.Status = OrderStatus.Approved;
                                order.DeliveryAddress = context.CustomerAddressBooks.FirstOrDefault(p => p.MainVendor.Id == ordererCustomer.Id && p.SubVendor.Id == recipientCustomer.Id);
                                order.LastModifierUserId = context.Users.First(p => p.UserName == AbpUserBase.AdminUserName).Id;
                                order.LastModificationTime = DateTime.Now;
                                //ProductionSite
                                //RejectionReason =//todo:bu silinmeli iptal kalem bazında burada statu olmalı
                            }
                        }
                        else
                        {
                            if (order != null)
                            {
                                var orderDetails = context.OrderDetails.Where(p => p.OrderId == order.Id).ToList();
                                orderDetails?.ForEach(p => context.OrderDetails.Remove(p));
                                context.SaveChanges();

                                context.Orders.Remove(order);
                                context.SaveChanges();
                            }
                        }

                        item.IsRead = true;
                        item.ModifiedDate = DateTime.Now;

                        context.SaveChanges();
                    }
                    catch (Exception ex)
                    {
                        logger.Error(ex, MethodBase.GetCurrentMethod().Name + " Error During IDOCOperations " + ex.Message);
                        continue;
                    }
                }
            }
        }
    }

Answer 1

At first glance, it seems that the different actions could be grouped and then executed in a single step.

For example:

• Create a list of all items that must be updated with item.IsRead = true; item.ModifiedDate = DateTime.Now;, and execute everything together at the end in one step
• In the same way, create a list with all the orders to remove, and then execute in a single step at the end

I don't know if this will be applicable according to the context and/or the application, it's just an idea

Answer 2

First of all, check your configuration for Lazy Load feature, both global and by-property levels.

When you are executing ToList() in sapOrders, you're executing your query (and loading result in memory) and you might be executing side queries when you get Orders, SalesOrganizations, and so on...

Take a look at this article to improve your loop.

On the other hand, you could use async queries and parallel programming in your loop, so, for each non-dependant task in it, you can create an async Task method and run all of them concurrently.

Here and here you are some articles I hope help you.

Good luck!

Answer 3

If you can avoid calling .SaveChanges() in your loop, but rather do it at the very end, you'll be much better off, as you avoid multiple round trips to the DB. That being said, if you're looping over 50,000 items doing this you'd probably want to batch that to some degree, maybe every 1,000 you call it.

Unfortunately, you've got the complication of needing the result of an order creation on save to store on another column though. Perhaps if you keep track of the orders you are creating, when you do the batch save, straight after you could do it you do a batch set of SASNo for each just created and another .BulkSaveChanges()?

For your later save changes (where you remove the order details, save, remove order, save), I can't see the need to do that in multiple steps, but maybe my EF is rusty and it'll complain. Ideally, I would remove all those calls to .SaveChanges() and do that in the bulk op every 1000.

Above would significantly cut the number of DB network calls, assuming BulkSaveChanges can handle all this. Basically I'd be aiming for below, but at the end of the day, this could potentially be done better/faster without EF.

using (var context = new BbsfDbContext())
{
    var sapOrders = ...;
    var ordersCreated = new List<..>(); // might wanna initialized this with a size if you have a rough gauge on what % will need creation of loop

    //if (sapOrders.Any()) // not needed
    //{
        foreach (var item in sapOrders.Select((x, index) => new { x, index }))
        {
            try
            {
                var order = ...;
                var isExist = ...;

                if (isExist)
                {
                    // ...

                    if (order == null)
                    {
                        order = new Order { ... };
                            var savedOrder = context.Orders.Add(order);
                            //context.SaveChanges();

                            //order.SASNo = BbsfConsts.KeasOrderNumberPrefix + savedOrder.Id;
                            ordersCreated.Add(order);
                        }
                        else
                        {
                            // Do updates
                            // ...
                        }
                    }
                    else
                    {
                        //if (order != null) // shouldn't need this
                        //{
                            var orderDetails = context.OrderDetails.Where(p => p.OrderId == order.Id).ToList();
                            orderDetails?.ForEach(p => context.OrderDetails.Remove(p));
                            //context.SaveChanges();

                            context.Orders.Remove(order);
                            //context.SaveChanges();
                        //}
                    }

                    // ...

                    if (index % 1000 == 0)
                    {
                        context.BulkSaveChanges(); // bulk save of 1000 loops of changes

                        foreach (var orderCreated in ordersCreated)
                        {
                            orderCreated.SASNo = BbsfConsts.KeasOrderNumberPrefix + savedOrder.Id;
                        }
                        context.BulkSaveChanges(); // bulk save of x num of SASNo sets
                    }
                }
                catch (Exception ex)
                {
                    // ...
                }
            }
        }
    }
}