优化现金流计算速度

AutoTurnOver.DB/Base/DbHelper.cs

@@ -3,6 +3,7 @@ using Dapper;
 using MySql.Data.MySqlClient;
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
@@ -135,5 +136,63 @@ namespace AutoTurnOver.DB.Base
             return 0;
         }
 
+        /// <summary>
+        ///     批量插入
+        /// </summary>
+        /// <typeparam name="T"></typeparam>
+        /// <param name="entities"></param>
+        /// <param name="exclude">排除的字段</param>
+        public static int BatchInsert<T>(this MySqlConnection conn, List<T> entities, string[] exclude = null)
+        {
+            if (exclude == null || exclude.Length <= 0)
+                exclude = new[] { "id" };
+
+            for (var i = 0; i < exclude.Length; i++)
+            {
+                exclude[i] = exclude[i].ToLower();
+            }
+
+            if (entities != null && entities.Count > 0)
+            {
+                var tbname = typeof(T).Name.TrimEnd("Entity".ToCharArray());
+                var colms = new List<string>();
+                var ps = entities.First().GetType().GetProperties().Where(p => !exclude.Contains(p.Name.ToLower()))
+                    .ToList();
+                foreach (var p in ps)
+                {
+                    colms.Add($"`{p.Name}`");
+                }
+
+                var paramList = new DynamicParameters();
+                var paramsNames = new List<string>();
+
+
+                var s = 0;
+                var n = "a";
+                foreach (var item in entities)
+                {
+                    var toNames = new List<string>();
+
+                    foreach (var p in ps)
+                    {
+                        var pname = $"@{n}{s}";
+                        var pvalue = p.GetValue(item, null);
+
+                        toNames.Add(pname);
+                        paramList.Add(pname, pvalue);
+                        s++;
+                    }
+
+                    paramsNames.Add($"({string.Join(",", toNames)})");
+                }
+
+                var sql =
+                    $"INSERT INTO {tbname} ({string.Join(",", colms)}) VALUES {string.Join(",", paramsNames)}";
+                return conn.Execute(sql, paramList);
+            }
+
+            throw new Exception("传入参数先");
+        }
+
     }
 }

AutoTurnOver.DB/report_cash_flow_dao.cs

@@ -9,6 +9,7 @@ using Dapper;
 using AutoTurnOver.Utility;
 using AutoTurnOver.DB.Base;
 using AutoTurnOver.Models.Base;
+using MySql.Data.MySqlClient;
 
 namespace AutoTurnOver.DB
 {
@@ -26,7 +27,7 @@ namespace AutoTurnOver.DB
             {
                 var conn = _connection;
                 int page = 1;
-                int rows = 1000;
+                int rows = 5000;
                 List<dc_report_cash_flow_config> configs = conn.Query<dc_report_cash_flow_config>(" select * from dc_report_cash_flow_config ").AsList();
                 while (true)
                 {
@@ -46,6 +47,10 @@ namespace AutoTurnOver.DB
                         break;
                     }
                     page++;
+
+                    var addList = new List<dc_report_cash_flow_log>();
+                    var updateList = new List<dc_report_cash_flow_log>();
+
                     foreach (var order_item in order_list)
                     {
                         try
@@ -77,16 +82,18 @@ namespace AutoTurnOver.DB
                                 item_no = order_item.bailun_order_id
                             };
                             item8.occur_time_year_month_no = $"{item8.occur_time.Year}-{item8.occur_time.Month}-{item8.bailun_sku}";
-                            item8.pay_time_year_month_no = $"{item8.pay_time.Year}-{item8.pay_time.Month}-{item8.bailun_sku}";
                             item8.pay_time = CalculationPayTime(configs, item8.occur_time, item8.data_type, item8.platform_type);
-                            item8.id = (old_cash_flow_list.FirstOrDefault(s=>s.data_type==item8.data_type)??new old_cash_flow_dto()).id;
+                            item8.pay_time_year_month_no = $"{item8.pay_time.Year}-{item8.pay_time.Month}-{item8.bailun_sku}";
+                            item8.id = (old_cash_flow_list.FirstOrDefault(s => s.data_type == item8.data_type) ?? new old_cash_flow_dto()).id;
                             if (item8.id > 0)
                             {
-                                conn.Update(item8);
+                                updateList.Add(item8);
+                                //conn.Update(item8);
                             }
                             else
                             {
-                                conn.Insert(item8);
+                                addList.Add(item8);
+                                //conn.Insert(item8);
                             }
 
                             #endregion
@@ -110,16 +117,18 @@ namespace AutoTurnOver.DB
                                 item_no = order_item.bailun_order_id
                             };
                             item7.occur_time_year_month_no = $"{item7.occur_time.Year}-{item7.occur_time.Month}-{item7.bailun_sku}";
-                            item7.pay_time_year_month_no = $"{item7.pay_time.Year}-{item7.pay_time.Month}-{item7.bailun_sku}";
                             item7.pay_time = CalculationPayTime(configs, item7.occur_time, item7.data_type, item7.platform_type);
+                            item7.pay_time_year_month_no = $"{item7.pay_time.Year}-{item7.pay_time.Month}-{item7.bailun_sku}";
                             item7.id = (old_cash_flow_list.FirstOrDefault(s => s.data_type == item7.data_type) ?? new old_cash_flow_dto()).id;
                             if (item7.id > 0)
                             {
-                                conn.Update(item7);
+                                updateList.Add(item7);
+                                //conn.Update(item7);
                             }
                             else
                             {
-                                conn.Insert(item7);
+                                addList.Add(item7);
+                                //conn.Insert(item7);
                             }
 
                             #endregion
@@ -143,16 +152,18 @@ namespace AutoTurnOver.DB
                                 item_no = order_item.bailun_order_id
                             };
                             item6.occur_time_year_month_no = $"{item6.occur_time.Year}-{item6.occur_time.Month}-{item6.bailun_sku}";
-                            item6.pay_time_year_month_no = $"{item6.pay_time.Year}-{item6.pay_time.Month}-{item6.bailun_sku}";
                             item6.pay_time = CalculationPayTime(configs, item6.occur_time, item6.data_type, item6.platform_type);
+                            item6.pay_time_year_month_no = $"{item6.pay_time.Year}-{item6.pay_time.Month}-{item6.bailun_sku}";
                             item6.id = (old_cash_flow_list.FirstOrDefault(s => s.data_type == item6.data_type) ?? new old_cash_flow_dto()).id;
                             if (item6.id > 0)
                             {
-                                conn.Update(item6);
+                                updateList.Add(item6);
+                                //conn.Update(item6);
                             }
                             else
                             {
-                                conn.Insert(item6);
+                                addList.Add(item6);
+                                //conn.Insert(item6);
                             }
 
                             #endregion
@@ -176,16 +187,18 @@ namespace AutoTurnOver.DB
                                 item_no = order_item.bailun_order_id
                             };
                             item5.occur_time_year_month_no = $"{item5.occur_time.Year}-{item5.occur_time.Month}-{item5.bailun_sku}";
-                            item5.pay_time_year_month_no = $"{item5.pay_time.Year}-{item5.pay_time.Month}-{item5.bailun_sku}";
                             item5.pay_time = CalculationPayTime(configs, item5.occur_time, item5.data_type, item5.platform_type);
+                            item5.pay_time_year_month_no = $"{item5.pay_time.Year}-{item5.pay_time.Month}-{item5.bailun_sku}";
                             item5.id = (old_cash_flow_list.FirstOrDefault(s => s.data_type == item5.data_type) ?? new old_cash_flow_dto()).id;
                             if (item5.id > 0)
                             {
-                                conn.Update(item5);
+                                updateList.Add(item5);
+                                //conn.Update(item5);
                             }
                             else
                             {
-                                conn.Insert(item5);
+                                addList.Add(item5);
+                                //conn.Insert(item5);
                             }
 
                             #endregion
@@ -202,23 +215,25 @@ namespace AutoTurnOver.DB
                                 no = order_item.bailun_order_id,
                                 platform_type = order_item.platform_type,
                                 remarks = "",
-                                val =  0 - Math.Abs( order_item.cost_platform_fee * order_item.seller_order_exchange_rate * order_item.bailun_sku_quantity_ordered),
+                                val = 0 - Math.Abs(order_item.cost_platform_fee * order_item.seller_order_exchange_rate * order_item.bailun_sku_quantity_ordered),
                                 occur_time = order_item.paid_time ?? new DateTime(1991, 1, 1),
                                 warehouse_code = order_item.warehouse_code,
                                 pay_type = (int)dc_report_cash_flow_log_pay_type_enum.后付,
                                 item_no = order_item.bailun_order_id
                             };
                             item3.occur_time_year_month_no = $"{item3.occur_time.Year}-{item3.occur_time.Month}-{item3.bailun_sku}";
-                            item3.pay_time_year_month_no = $"{item3.pay_time.Year}-{item3.pay_time.Month}-{item3.bailun_sku}";
                             item3.pay_time = CalculationPayTime(configs, item3.occur_time, item3.data_type, item3.platform_type);
+                            item3.pay_time_year_month_no = $"{item3.pay_time.Year}-{item3.pay_time.Month}-{item3.bailun_sku}";
                             item3.id = (old_cash_flow_list.FirstOrDefault(s => s.data_type == item3.data_type) ?? new old_cash_flow_dto()).id;
                             if (item3.id > 0)
                             {
-                                conn.Update(item3);
+                                updateList.Add(item3);
+                                //conn.Update(item3);
                             }
                             else
                             {
-                                conn.Insert(item3);
+                                addList.Add(item3);
+                                //conn.Insert(item3);
                             }
 
                             #endregion
@@ -242,16 +257,18 @@ namespace AutoTurnOver.DB
                                 item_no = order_item.bailun_order_id
                             };
                             item2.occur_time_year_month_no = $"{item2.occur_time.Year}-{item2.occur_time.Month}-{item2.bailun_sku}";
-                            item2.pay_time_year_month_no = $"{item2.pay_time.Year}-{item2.pay_time.Month}-{item2.bailun_sku}";
                             item2.pay_time = CalculationPayTime(configs, item2.occur_time, item2.data_type, item2.platform_type);
+                            item2.pay_time_year_month_no = $"{item2.pay_time.Year}-{item2.pay_time.Month}-{item2.bailun_sku}";
                             item2.id = (old_cash_flow_list.FirstOrDefault(s => s.data_type == item2.data_type) ?? new old_cash_flow_dto()).id;
                             if (item2.id > 0)
                             {
-                                conn.Update(item2);
+                                updateList.Add(item2);
+                                //conn.Update(item2);
                             }
                             else
                             {
-                                conn.Insert(item2);
+                                addList.Add(item2);
+                                //conn.Insert(item2);
                             }
 
                             #endregion
@@ -275,16 +292,18 @@ namespace AutoTurnOver.DB
                                 item_no = order_item.bailun_order_id
                             };
                             item1.occur_time_year_month_no = $"{item1.occur_time.Year}-{item1.occur_time.Month}-{item1.bailun_sku}";
-                            item1.pay_time_year_month_no = $"{item1.pay_time.Year}-{item1.pay_time.Month}-{item1.bailun_sku}";
                             item1.pay_time = CalculationPayTime(configs, item1.occur_time, item1.data_type, item1.platform_type);
+                            item1.pay_time_year_month_no = $"{item1.pay_time.Year}-{item1.pay_time.Month}-{item1.bailun_sku}";
                             item1.id = (old_cash_flow_list.FirstOrDefault(s => s.data_type == item1.data_type) ?? new old_cash_flow_dto()).id;
                             if (item1.id > 0)
                             {
-                                conn.Update(item1);
+                                updateList.Add(item1);
+                                //conn.Update(item1);
                             }
                             else
                             {
-                                conn.Insert(item1);
+                                addList.Add(item1);
+                                //conn.Insert(item1);
                             }
 
                             #endregion
@@ -308,16 +327,18 @@ namespace AutoTurnOver.DB
                                 item_no = order_item.bailun_order_id
                             };
                             item.occur_time_year_month_no = $"{item.occur_time.Year}-{item.occur_time.Month}-{item.bailun_sku}";
-                            item.pay_time_year_month_no = $"{item.pay_time.Year}-{item.pay_time.Month}-{item.bailun_sku}";
                             item.pay_time = CalculationPayTime(configs, item.occur_time, item.data_type, item.platform_type);
+                            item.pay_time_year_month_no = $"{item.pay_time.Year}-{item.pay_time.Month}-{item.bailun_sku}";
                             item.id = (old_cash_flow_list.FirstOrDefault(s => s.data_type == item.data_type) ?? new old_cash_flow_dto()).id;
                             if (item.id > 0)
                             {
-                                conn.Update(item);
+                                updateList.Add(item);
+                                //conn.Update(item);
                             }
                             else
                             {
-                                conn.Insert(item);
+                                addList.Add(item);
+                                //conn.Insert(item);
                             }
 
                             #endregion
@@ -328,10 +349,17 @@ namespace AutoTurnOver.DB
                             Console.WriteLine("现金流-订单-异常" + ex.StackTrace);
                         }
 
+                    }
 
-
-
+                    if (addList != null && addList.Count >= 1)
+                    {
+                        conn.BatchInsert(addList);
                     }
+                    if (updateList != null && updateList.Count >= 1)
+                    {
+                        UpdateUsersTest(conn, updateList);
+                    }
+                        
                 }
             }
             catch (Exception ex)
@@ -343,6 +371,12 @@ namespace AutoTurnOver.DB
 
         }
 
+        public static void UpdateUsersTest(MySqlConnection conn, List<dc_report_cash_flow_log> datas)
+        {
+            var sql = @"UPDATE dc_report_cash_flow_log SET warehouse_code=@warehouse_code,platform_type=@platform_type,bailun_sku=@bailun_sku,data_type=@data_type,pay_type=@pay_type,val=@val,occur_time=@occur_time,pay_time=@pay_time,no=@no,item=@item,item_no=@item_no,remarks=@remarks,is_delete=@is_delete,update_time=@update_time,occur_time_year_month_no=@occur_time_year_month_no,pay_time_year_month_no=@pay_time_year_month_no WHERE id = @id ;";
+            var res = conn.Execute(sql, datas);
+        }
+
         /// <summary>
         /// 计算订单尾程费
         /// </summary>
@@ -354,7 +388,7 @@ namespace AutoTurnOver.DB
             {
                 var conn = _connection;
                 int page = 1;
-                int rows = 1000;
+                int rows = 5000;
                 var logisticsList = ApiUtility.RealTimeShipLogisticsList();
                 List<dc_report_logistics_company_config_dto> logistics_company_list = conn.Query<dc_report_logistics_company_config_dto>(" select * from dc_report_logistics_company_config ").AsList();
                 List<dc_report_cash_flow_config> configs = conn.Query<dc_report_cash_flow_config>(" select * from dc_report_cash_flow_config ").AsList();
@@ -376,6 +410,9 @@ namespace AutoTurnOver.DB
                         break;
                     }
                     page++;
+
+                    var addList = new List<dc_report_cash_flow_log>();
+                    var updateList = new List<dc_report_cash_flow_log>();
                     foreach (var order_item in order_list)
                     {
                         try
@@ -418,11 +455,13 @@ namespace AutoTurnOver.DB
                             }) ?? 0;
                             if (item4.id > 0)
                             {
-                                conn.Update(item4);
+                                addList.Add(item4);
+                                //conn.Update(item4);
                             }
                             else
                             {
-                                conn.Insert(item4);
+                                updateList.Add(item4);
+                                //conn.Insert(item4);
                             }
 
                             #endregion
@@ -434,6 +473,15 @@ namespace AutoTurnOver.DB
                         }
 
                     }
+
+                    if (addList != null && addList.Count >= 1)
+                    {
+                        conn.BatchInsert(addList);
+                    }
+                    if (updateList != null && updateList.Count >= 1)
+                    {
+                        UpdateUsersTest(conn, updateList);
+                    }
                 }
             }
             catch (Exception ex)
@@ -826,8 +874,8 @@ where t1.create_time>=@btime and t1.create_time<=@etime ";
         public static List<report_cash_flow_view_dto> GetView(report_cash_flow_view_search_dto search)
         {
             List<report_cash_flow_view_dto> datas = new List<report_cash_flow_view_dto>();
-            datas.Add(new report_cash_flow_view_dto { date_type = new List<int> { 1}, date_type_str = "销售数量", remarks = "", dates = new List<report_cash_flow_view_dto.date_dto>() });
-            datas.Add(new report_cash_flow_view_dto { date_type = new List<int> { 2}, date_type_str = "销售金额", remarks = "过滤掉刷单的订单 ", dates = new List<report_cash_flow_view_dto.date_dto>() });
+            datas.Add(new report_cash_flow_view_dto { date_type = new List<int> { 1 }, date_type_str = "销售数量", remarks = "", dates = new List<report_cash_flow_view_dto.date_dto>() });
+            datas.Add(new report_cash_flow_view_dto { date_type = new List<int> { 2 }, date_type_str = "销售金额", remarks = "过滤掉刷单的订单 ", dates = new List<report_cash_flow_view_dto.date_dto>() });
             datas.Add(new report_cash_flow_view_dto { date_type = new List<int> { 3 }, date_type_str = "退款", remarks = "crm 抓取的退款数据，未区分仓库", dates = new List<report_cash_flow_view_dto.date_dto>() });
             datas.Add(new report_cash_flow_view_dto { date_type = new List<int> { 4 }, date_type_str = "利润", remarks = "oms 中的订单，分摊到sku的利润，不包含广告费，上架费，退款", dates = new List<report_cash_flow_view_dto.date_dto>() });
             datas.Add(new report_cash_flow_view_dto { date_type = new List<int> { 5 }, date_type_str = "平台费用", dates = new List<report_cash_flow_view_dto.date_dto>() });
@@ -838,8 +886,8 @@ where t1.create_time>=@btime and t1.create_time<=@etime ";
             datas.Add(new report_cash_flow_view_dto { date_type = new List<int> { 10 }, date_type_str = "新增头程费用", remarks = " 调拨单的数据，支付时间 已 读取对应物流商的账期，注：调拨单需要付款之后才能获取到金额 ", dates = new List<report_cash_flow_view_dto.date_dto>() });
             datas.Add(new report_cash_flow_view_dto { date_type = new List<int> { 11 }, date_type_str = "fba费", remarks = "  ", dates = new List<report_cash_flow_view_dto.date_dto>() });
             datas.Add(new report_cash_flow_view_dto { date_type = new List<int> { 12 }, date_type_str = "paypal费", remarks = "  ", dates = new List<report_cash_flow_view_dto.date_dto>() });
-            datas.Add(new report_cash_flow_view_dto { date_type = new List<int> { 4,3 }, date_type_str = "汇总利润", remarks = " oms 中的订单，销售额 - 销售成本 - 尾程费 -  头程费 - 退款 - 平台费 - fba费 -paypal 费 （不包含广告费、上架费）", dates = new List<report_cash_flow_view_dto.date_dto>() });
-            datas.Add(new report_cash_flow_view_dto { date_type = new List<int> {2,9,10,5,8,11,12 }, date_type_str = "现金流结余", remarks = " 销售金额 - 新增采购费用 - 新增头程费用 - 平台费用 - 尾程费用  ", dates = new List<report_cash_flow_view_dto.date_dto>() });
+            datas.Add(new report_cash_flow_view_dto { date_type = new List<int> { 4, 3 }, date_type_str = "汇总利润", remarks = " oms 中的订单，销售额 - 销售成本 - 尾程费 -  头程费 - 退款 - 平台费 - fba费 -paypal 费 （不包含广告费、上架费）", dates = new List<report_cash_flow_view_dto.date_dto>() });
+            datas.Add(new report_cash_flow_view_dto { date_type = new List<int> { 2, 9, 10, 5, 8, 11, 12 }, date_type_str = "现金流结余", remarks = " 销售金额 - 新增采购费用 - 新增头程费用 - 平台费用 - 尾程费用  ", dates = new List<report_cash_flow_view_dto.date_dto>() });
             var btime = DateTime.Now.AddDays(-8).ToDayHome();
             var etime = DateTime.Now.AddDays(-1).ToDayEnd();
             if (search.btime != null)
@@ -858,7 +906,7 @@ where t1.create_time>=@btime and t1.create_time<=@etime ";
             parameters.Add("btime", btime);
             parameters.Add("etime", etime);
 
-            var occur_datas = conn.Query<dc_report_cash_flow_group_day>(occur_sql, parameters,commandTimeout:0).ToList();
+            var occur_datas = conn.Query<dc_report_cash_flow_group_day>(occur_sql, parameters, commandTimeout: 0).ToList();
             var pay_datas = conn.Query<dc_report_cash_flow_group_day>(pay_sql, parameters, commandTimeout: 0).ToList();
 
             foreach (var itemData in datas)
@@ -901,10 +949,11 @@ where t1.create_time>=@btime and t1.create_time<=@etime ";
         /// <returns></returns>
         public static List<dc_report_cash_flow_log_dto> Export(dc_report_cash_flow_log_export_search_dto search)
         {
-            if(search.btime==null || search.etime == null)
+            if (search.btime == null || search.etime == null)
             {
                 throw new Exception("时间范围必选");
-            }  if(search.type==null)
+            }
+            if (search.type == null)
             {
                 throw new Exception("时间类型必选");
             }
@@ -920,15 +969,15 @@ where t1.create_time>=@btime and t1.create_time<=@etime ";
                 sql += " and t1.pay_time>=@btime ";
                 sql += " and t1.pay_time<=@etime ";
             }
-            parameters.Add("btime",search.btime.Value.ToDayHome());
-            parameters.Add("etime",search.etime.Value.ToDayEnd());
+            parameters.Add("btime", search.btime.Value.ToDayHome());
+            parameters.Add("etime", search.etime.Value.ToDayEnd());
 
             if (!string.IsNullOrWhiteSpace(search.bailun_sku))
             {
                 sql += " and t1.bailun_sku = @bailun_sku ";
                 parameters.Add("bailun_sku", search.bailun_sku);
             }
-            if (search.data_type !=null &&search.data_type.Count>=0)
+            if (search.data_type != null && search.data_type.Count >= 0)
             {
                 sql += " and t1.data_type in @data_type ";
                 parameters.Add("data_type", search.data_type);
@@ -949,7 +998,7 @@ where t1.create_time>=@btime and t1.create_time<=@etime ";
             if (!string.IsNullOrWhiteSpace(search.bailun_sku))
             {
                 sql += " and t1.bailun_sku=@bailun_sku ";
-                parameters.Add("bailun_sku",search.bailun_sku);
+                parameters.Add("bailun_sku", search.bailun_sku);
             }
 
             return _connection.Page<dc_report_cash_flow_sku_group_dto>(sql, search, parameters);

ResetOutofstock/Program.cs

@@ -13,9 +13,8 @@ namespace ResetOutofstock
         static async Task Main(string[] args)
         {
             Console.WriteLine("刷新缺货数据任务启动...");
-            var now = DateTime.Now;
             //report_cash_flow_dao.CalculationOrderCostLogistics(now.AddMonths(-3), DateTime.Parse(now.AddDays(-1).ToString("yyyy-MM-dd 23:59:59")));
-            //report.ResetCashFlowData();
+
             var builder = new HostBuilder().ConfigureServices((hostContext, services) =>
             {
                 services.AddHostedService<ResetOutofstockBackgrounService>();

ResetOutofstock/ResetOutofstockBackgrounService.cs

@@ -35,6 +35,26 @@ namespace ResetOutofstock
                 }
             });
             
+            Task.Factory.StartNew(() => {
+                while (true)
+                {
+                    try
+                    {
+                        var now = DateTime.Now;
+                        Console.WriteLine($"开始 刷新订单尾程费  退款数据，线程Id：{Thread.CurrentThread.ManagedThreadId}，{DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")}");
+                        report_cash_flow_dao.CalculationOrderCostLogistics(now.AddMonths(-3), DateTime.Parse(now.AddDays(-1).ToString("yyyy-MM-dd 23:59:59")));
+                        Console.WriteLine($"结束 刷新订单尾程费  退款数据，线程Id：{Thread.CurrentThread.ManagedThreadId}，{DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")}");
+                    }
+                    catch (Exception ex)
+                    {
+                        Console.WriteLine(ex.Message);
+                    }
+
+
+                    Thread.Sleep(240 * 60 * 60 * 1000);
+                }
+            });
+
 
             Task.Factory.StartNew(() => {
                 while (true)