限流ip/oaid分别独立控制

molilian.api/Controllers/public/TaskController.cs

@@ -28,9 +28,9 @@ namespace molilian.api.Controllers
 
 
         [HttpGet]
-        public ActionResult BatchInsertLogDB(int limit = 100)
+        public async Task<ActionResult> BatchInsertLogDB(int limit = 100)
         {
-            int total = TkLogCore.BatchInsertLogDB(limit);
+            int total = await TkLogCore.BatchInsertLogDBAsync(limit);
             return new APIResult(new { success = true, message = "ok", limit, total });
         }
 
@@ -342,7 +342,7 @@ namespace molilian.api.Controllers
 
 
         [HttpGet]
-        public async Task<ActionResult> DailyTable(int days = 7)
+        public async Task<ActionResult> DailyTable(int days = 3)
         {
             try
             {
@@ -366,6 +366,9 @@ namespace molilian.api.Controllers
                     DBContext.Execute(sql, null);
                     sql = $"CREATE TABLE IF NOT EXISTS tk_promotion_logs_{table_suffix} LIKE tk_promotion_logs;";
                     DBContext.Execute(sql, null);
+
+                    sql = $"CREATE TABLE IF NOT EXISTS deeplink_parse_logs_{table_suffix} LIKE deeplink_parse_logs;";
+                    DBContext.Execute(sql, null);
                 }
                 TkLogCore.save_dailys_log = true;
                 RedisHelper.Set("turn:save_dailys_log", 1, 86400);

molilian.api/Properties/launchSettings.json

@@ -10,10 +10,10 @@
         "NtfyServer": "https://ntfy.yunhui800.com/5Sq9BytXXM5WDY3G",
         "ANPush": "",
         "DBType": "MySQL",
-        "DBConfig": "Server=rm-2zey1jqxnoqy9mcc0zo.rwlb.rds.aliyuncs.com; Port=3306; Database=taoke; Uid=taoke; Pwd=67ktWBmw5G4yMs4J;SslMode=None;CharSet=utf8mb4;ConnectionTimeout=60",
+        "DBConfig": "Server=rm-2zey1jqxnoqy9mcc0uo.rwlb.rds.aliyuncs.com; Port=3306; Database=taoke; Uid=taoke; Pwd=67ktWBmw5G4yMs4J;SslMode=None;CharSet=utf8mb4;ConnectionTimeout=60",
         "RedisConfig": "101.200.46.46:6379,password=pKBiS4ka2IpXayIdcx00,defaultDatabase=0,idleTimeout=20000,preheat=3,tryit=2,ssl=false,prefix=webhook",
         "CenterDB": "",
-        "CenterRedis": "",
+        "CenterRedis": ""
       },
       "environmentVariables2": {
         "ASPNETCORE_ENVIRONMENT": "Development",

molilian.core/Core/API/ApiReportCore.cs

@@ -121,7 +121,7 @@ namespace molilian.core
                     e.uclk_pv,
                     e.pay_ord_num,
                     e.order_ord_num,
-                    pay_ord_am = e.pay_ord_amt * 100,
+                    pay_ord_amt = e.pay_ord_amt * 100,
                     pay_tk_disp_tfee = e.pay_tk_disp_tfee * 100,
                     e.pay_ord_uv,
                     e.eff_ord_num,
@@ -160,7 +160,7 @@ namespace molilian.core
                             e.uclk_pv,
                             e.pay_ord_num,
                             e.order_ord_num,
-                            pay_ord_am = e.pay_ord_amt * 100,
+                            pay_ord_amt = e.pay_ord_amt * 100,
                             pay_tk_disp_tfee = e.pay_tk_disp_tfee * 100,
                             e.pay_ord_uv,
                             e.eff_ord_num,
@@ -183,7 +183,7 @@ namespace molilian.core
                             e.uclk_pv,
                             e.pay_ord_num,
                             e.order_ord_num,
-                            pay_ord_am = e.pay_ord_amt * 100,
+                            pay_ord_amt = e.pay_ord_amt * 100,
                             pay_tk_disp_tfee = e.pay_tk_disp_tfee * 100,
                             e.pay_ord_uv,
                             e.eff_ord_num,
@@ -206,7 +206,7 @@ namespace molilian.core
                             e.uclk_pv,
                             e.pay_ord_num,
                             e.order_ord_num,
-                            pay_ord_am = e.pay_ord_amt * 100,
+                            pay_ord_amt = e.pay_ord_amt * 100,
                             pay_tk_disp_tfee = e.pay_tk_disp_tfee * 100,
                             e.pay_ord_uv,
                             e.eff_ord_num,
@@ -229,7 +229,7 @@ namespace molilian.core
                             e.uclk_pv,
                             e.pay_ord_num,
                             e.order_ord_num,
-                            pay_ord_am = e.pay_ord_amt * 100,
+                            pay_ord_amt = e.pay_ord_amt * 100,
                             pay_tk_disp_tfee = e.pay_tk_disp_tfee * 100,
                             e.pay_ord_uv,
                             e.eff_ord_num,
@@ -252,7 +252,7 @@ namespace molilian.core
                             e.uclk_pv,
                             e.pay_ord_num,
                             e.order_ord_num,
-                            pay_ord_am = e.pay_ord_amt * 100,
+                            pay_ord_amt = e.pay_ord_amt * 100,
                             pay_tk_disp_tfee = e.pay_tk_disp_tfee * 100,
                             e.pay_ord_uv,
                             e.eff_ord_num,
@@ -276,7 +276,7 @@ namespace molilian.core
                             e.uclk_pv,
                             e.pay_ord_num,
                             e.order_ord_num,
-                            pay_ord_am = e.pay_ord_amt * 100,
+                            pay_ord_amt = e.pay_ord_amt * 100,
                             pay_tk_disp_tfee = e.pay_tk_disp_tfee * 100,
                             e.pay_ord_uv,
                             e.eff_ord_num,
@@ -301,7 +301,7 @@ namespace molilian.core
                             e.uclk_pv,
                             e.pay_ord_num,
                             e.order_ord_num,
-                            pay_ord_am = e.pay_ord_amt * 100,
+                            pay_ord_amt = e.pay_ord_amt * 100,
                             pay_tk_disp_tfee = e.pay_tk_disp_tfee * 100,
                             e.pay_ord_uv,
                             e.eff_ord_num,

molilian.core/Core/EndPointCore.cs

@@ -105,7 +105,18 @@ Server=rm-2ze74506m3gfsqe7mco.rwlb.rds.aliyuncs.com; Port=3306; Database=coupon;
             }
             return resultList;
         }
-
+        public static async Task<List<T>> ProcessEndPointNodesTaskAsync<T>(Func<EndPointDTO, Task<T>> nodeAction, bool force = false)
+        {
+            var resultList = new List<T>();
+            var list = List(force);
+            foreach (var node in list)
+            {
+                var result = await nodeAction(node);
+                if (result != null) resultList.Add(result);
+            }
+            return resultList;
+        }
+        
         public static async Task ProcessEndPointNodesAsync(Func<EndPointDTO, Task> nodeAction, bool force = false)
         {
             var list = List(force);
@@ -162,7 +173,7 @@ Server=rm-2ze74506m3gfsqe7mco.rwlb.rds.aliyuncs.com; Port=3306; Database=coupon;
                     var root = body.Convert2JsonElement();
                     var message = root.Read<string>("message", string.Empty);
                     return (node.name, message);
-                    
+
 
                 }
                 catch (Exception ex)

molilian.core/Core/cps/UnionCpsCore.cs

@@ -242,7 +242,12 @@ namespace molilian.core
                         reason = "IP控制";
                         return true;
                     }
-                    num = TkLogCore.getClientRequestTotalByOAID(channel, oaid);
+                }
+
+                limit_num = config.cps_limit_per_oaid_24h;
+                if (limit_num > 0)
+                {
+                    int num = TkLogCore.getClientRequestTotalByOAID(channel, oaid);
                     if (num > limit_num)
                     {
                         reason = "OAID控制";

molilian.core/Core/log/base.cs

@@ -19,6 +19,7 @@ using TencentCloud.Omics.V20221128.Models;
 using TencentCloud.Csip.V20221121.Models;
 using COSXML.Network;
 using System.Security.Policy;
+using System.Diagnostics;
 
 
 namespace molilian.core
@@ -29,35 +30,50 @@ namespace molilian.core
         public static bool save_dailys_log = false;
 
         public static string _test_oaid = "3B191CFA4C6B48F9BA459E915B57743BEC7D424979CC9C51BCAD0C245B1C7BA2";
+
         static TkLogCore()
         {
             int flag = RedisHelper.Get<int>("turn:save_dailys_log");
             if (flag == 1) save_dailys_log = true;
         }
-        public static int BatchInsertLogDB(int limit)
+        private static readonly SemaphoreSlim semaphore = new SemaphoreSlim(10, 10);
+
+        public static async Task<int> BatchInsertLogDBAsync(int limit)
         {
-            var result = EndPointCore.ProcessEndPointNodes<int>(node =>
+            await semaphore.WaitAsync(); // 等待获取锁
+            try
             {
-                if (!node.is_public_api) return 0;
-
-                if (CenterHub.IsCenter)
-                {
-                    if (node.is_coupon_api) return 0;
-                }
-                else
+                var result = await Task.Run(() =>
                 {
-                    if (!node.is_coupon_api) return 0;
-                }
-
-                if (string.IsNullOrEmpty(node.redis_server)) return 0;
-
-                var redis = RedisClientManager.GetRedisClient(node.redis_server);
-                return BatchInsertLogDB(limit, redis);
-            });
-            return result.Sum();
+                    return EndPointCore.ProcessEndPointNodes<int>(node =>
+                    {
+                        if (!node.is_public_api) return 0;
+
+                        if (CenterHub.IsCenter)
+                        {
+                            if (node.is_coupon_api) return 0;
+                        }
+                        else
+                        {
+                            if (!node.is_coupon_api) return 0;
+                        }
+
+                        if (string.IsNullOrEmpty(node.redis_server)) return 0;
+
+                        var redis = RedisClientManager.GetRedisClient(node.redis_server);
+                        return BatchInsertLogDB(limit, redis);
+                    });
+                });
+                return result.Sum();
+            }
+            finally
+            {
+                semaphore.Release(); // 释放锁，允许下一个任务执行
+            }
         }
 
-        public static int BatchInsertLogDB(int limit, CSRedisClient redis)
+
+        public static int BatchInsertLogDB2(int limit, CSRedisClient redis)
         {
             int total = 0;
             using var connection = DBContext.GetOpenConnection();
@@ -66,35 +82,164 @@ namespace molilian.core
 
             try
             {
-                //第三方接口写入日志
-                total += task_insert_tk_logs(connection, transaction, limit, redis);
-
-                total += task_insert_parse_tb_logs(connection, transaction, limit, redis);
-
-                total += task_insert_parse_jd_logs(connection, transaction, limit, redis);
+                Stopwatch stopwatch = new Stopwatch(); // 创建一个计时器
+                LoggerLibrary log = new LoggerLibrary("debug", "BatchInsertLogDB"); // 创建日志对象
 
-                total += task_insert_parse_pdd_logs(connection, transaction, limit, redis);
 
-                total += task_insert_parse_dy_logs(connection, transaction, limit, redis);
-
-                total += task_insert_parse_tool_logs(connection, transaction, limit, redis);
-
-                total += task_insert_parse_deeplink_logs(connection, transaction, limit, redis);
+                //第三方接口写入日志
+                stopwatch.Start();
+                int taskTotal = task_insert_tk_logs(limit, redis);
+                total += taskTotal;
+                stopwatch.Stop();
+                log.Info($"task_insert_tk_logs 耗时: {stopwatch.ElapsedMilliseconds} ms,\t插入记录数: {taskTotal}");
+                log.SaveAsync();
+                stopwatch.Reset();
+
+
+                stopwatch.Start();
+                taskTotal = task_insert_parse_tb_logs(connection, transaction, limit, redis);
+                total += taskTotal;
+                stopwatch.Stop();
+                log.Info($"task_insert_parse_tb_logs 耗时: {stopwatch.ElapsedMilliseconds} ms,\t插入记录数: {taskTotal}");
+                log.SaveAsync();
+                stopwatch.Reset();
+
+                stopwatch.Start();
+                taskTotal = task_insert_parse_jd_logs(connection, transaction, limit, redis);
+                total += taskTotal;
+                stopwatch.Stop();
+                log.Info($"task_insert_parse_jd_logs 耗时: {stopwatch.ElapsedMilliseconds} ms,\t插入记录数: {taskTotal}");
+                log.SaveAsync();
+                stopwatch.Reset();
+
+                stopwatch.Start();
+                taskTotal = task_insert_parse_pdd_logs(connection, transaction, limit, redis);
+                total += taskTotal;
+                stopwatch.Stop();
+                log.Info($"task_insert_parse_pdd_logs 耗时: {stopwatch.ElapsedMilliseconds} ms,\t插入记录数: {taskTotal}");
+                log.SaveAsync();
+                stopwatch.Reset();
+
+                stopwatch.Start();
+                taskTotal = task_insert_parse_dy_logs(connection, transaction, limit, redis);
+                total += taskTotal;
+                stopwatch.Stop();
+                log.Info($"task_insert_parse_dy_logs 耗时: {stopwatch.ElapsedMilliseconds} ms,\t插入记录数: {taskTotal}");
+                log.SaveAsync();
+                stopwatch.Reset();
+
+                stopwatch.Start();
+                taskTotal = task_insert_parse_tool_logs(connection, transaction, limit, redis);
+                total += taskTotal;
+                stopwatch.Stop();
+                log.Info($"task_insert_parse_tool_logs 耗时: {stopwatch.ElapsedMilliseconds} ms,\t插入记录数: {taskTotal}");
+                log.SaveAsync();
+                stopwatch.Reset();
+
+                stopwatch.Start();
+                taskTotal = task_insert_parse_deeplink_logs(limit, redis);
+                total += taskTotal;
+                stopwatch.Stop();
+                log.Info($"task_insert_parse_deeplink_logs  耗时: {stopwatch.ElapsedMilliseconds} ms,\t插入记录数: {taskTotal}");
+                log.SaveAsync();
+                stopwatch.Reset();
+
+                stopwatch.Start();
+                taskTotal = task_insert_parse_coupon_logs(connection, transaction, limit, redis);
+                total += taskTotal;
+                stopwatch.Stop();
+                log.Info($"task_insert_parse_coupon_logs 耗时: {stopwatch.ElapsedMilliseconds} ms,\t插入记录数: {taskTotal}");
+                log.SaveAsync();
+                stopwatch.Reset();
+
+                stopwatch.Start();
+                taskTotal = task_insert_parse_cps_logs(connection, transaction, limit, redis);
+                total += taskTotal;
+                stopwatch.Stop();
+                log.Info($"task_insert_parse_cps_logs 耗时: {stopwatch.ElapsedMilliseconds} ms,\t插入记录数: {taskTotal}");
+                log.SaveAsync();
+                stopwatch.Reset();
+
+                stopwatch.Start();
+                taskTotal = task_insert_promotion_img_logs(connection, transaction, limit, redis);
+                total += taskTotal;
+                stopwatch.Stop();
+                log.Info($"task_insert_promotion_img_logs 耗时: {stopwatch.ElapsedMilliseconds} ms,\t插入记录数: {taskTotal}");
+                log.SaveAsync();
+                stopwatch.Reset();
+
+                stopwatch.Start();
+                taskTotal = task_insert_parse_ks_logs(connection, transaction, limit, redis);
+                total += taskTotal;
+                stopwatch.Stop();
+                log.Info($"task_insert_parse_ks_logs 耗时: {stopwatch.ElapsedMilliseconds} ms,\t插入记录数: {taskTotal}");
+                log.SaveAsync();
+                stopwatch.Reset();
 
-                total += task_insert_parse_coupon_logs(connection, transaction, limit, redis);
+                transaction.Commit();
+            }
+            catch (Exception ex)
+            {
+                transaction.Rollback();
 
-                total += task_insert_parse_cps_logs(connection, transaction, limit, redis);
+                _ = new LoggerLibrary("database_error", "parse_log")
+                    .Info(ex.Message, ex.StackTrace)
+                    .SaveAsync();
 
-                total += task_insert_promotion_img_logs(connection, transaction, limit, redis);
+                NotifyCore.Notify(new NifyMessage
+                {
+                    message = $"【写入日志异常】\n{ex.Message}\n{ex.StackTrace}",
+                    priority = NifyMessagePriority.high,
+                    tags = ["red_circle"]
+                });
+            }
+            finally
+            {
+                connection.Close();
+            }
+            return total;
+        }
 
+        public static int BatchInsertLogDB(int limit, CSRedisClient redis)
+        {
+            int total = 0;
+            //using var connection = DBContext.GetOpenConnection();
+            //connection.Open();
+            //using var transaction = connection.BeginTransaction();
+            using IDbTransaction transaction = null;
+            using IDbConnection connection = null;
 
-                total += task_insert_parse_ks_logs(connection, transaction, limit, redis);
+            try
+            {
+                Stopwatch stopwatch = new Stopwatch(); // 创建一个计时器
+                LoggerLibrary log = new LoggerLibrary("debug", "BatchInsertLogDB"); // 创建日志对象
 
-                transaction.Commit();
+                var tasks = new List<Task<int>>
+        {
+            Task.Run(() => RunTaskWithLogging(() => task_insert_tk_logs(limit, redis), "task_insert_tk_logs", log)),
+            Task.Run(() => RunTaskWithLogging(() => task_insert_parse_tb_logs(connection, transaction, limit, redis), "task_insert_parse_tb_logs", log)),
+            Task.Run(() => RunTaskWithLogging(() => task_insert_parse_jd_logs(connection, transaction, limit, redis), "task_insert_parse_jd_logs", log)),
+            Task.Run(() => RunTaskWithLogging(() => task_insert_parse_pdd_logs(connection, transaction, limit, redis), "task_insert_parse_pdd_logs", log)),
+            Task.Run(() => RunTaskWithLogging(() => task_insert_parse_dy_logs(connection, transaction, limit, redis), "task_insert_parse_dy_logs", log)),
+            Task.Run(() => RunTaskWithLogging(() => task_insert_parse_tool_logs(connection, transaction, limit, redis), "task_insert_parse_tool_logs", log)),
+            Task.Run(() => RunTaskWithLogging(() => task_insert_parse_deeplink_logs(limit, redis), "task_insert_parse_deeplink_logs", log)),
+            Task.Run(() => RunTaskWithLogging(() => task_insert_parse_coupon_logs(connection, transaction, limit, redis), "task_insert_parse_coupon_logs", log)),
+            Task.Run(() => RunTaskWithLogging(() => task_insert_parse_cps_logs(connection, transaction, limit, redis), "task_insert_parse_cps_logs", log)),
+            Task.Run(() => RunTaskWithLogging(() => task_insert_promotion_img_logs(connection, transaction, limit, redis), "task_insert_promotion_img_logs", log)),
+            Task.Run(() => RunTaskWithLogging(() => task_insert_parse_ks_logs(connection, transaction, limit, redis), "task_insert_parse_ks_logs", log))
+        };
+
+                // 等待所有任务完成
+                Task.WhenAll(tasks).Wait();
+
+                // 计算所有任务的结果总和
+                total = tasks.Select(t => t.Result).Sum();
+
+                //transaction.Commit();
             }
             catch (Exception ex)
             {
-                transaction.Rollback();
+                //transaction.Rollback();
 
                 _ = new LoggerLibrary("database_error", "parse_log")
                     .Info(ex.Message, ex.StackTrace)
@@ -109,11 +254,47 @@ namespace molilian.core
             }
             finally
             {
-                connection.Close();
+                //connection.Close();
             }
             return total;
         }
 
+
+        // 新增一个辅助方法来处理日志记录和任务执行
+        private static int RunTaskWithLogging(Func<int> taskFunc, string taskName, LoggerLibrary log)
+        {
+            Stopwatch stopwatch = new Stopwatch();
+            stopwatch.Start();
+            int taskTotal = taskFunc();
+            stopwatch.Stop();
+            log.Info($"{taskName} 耗时: {stopwatch.ElapsedMilliseconds} ms,\t插入记录数: {taskTotal}");
+            log.SaveAsync();
+            return taskTotal;
+        }
+
+
+        public static int BatchInsertLogDB(int limit)
+        {
+            var result = EndPointCore.ProcessEndPointNodes<int>(node =>
+            {
+                if (!node.is_public_api) return 0;
+
+                if (CenterHub.IsCenter)
+                {
+                    if (node.is_coupon_api) return 0;
+                }
+                else
+                {
+                    if (!node.is_coupon_api) return 0;
+                }
+
+                if (string.IsNullOrEmpty(node.redis_server)) return 0;
+
+                var redis = RedisClientManager.GetRedisClient(node.redis_server);
+                return BatchInsertLogDB(limit, redis);
+            });
+            return result.Sum();
+        }
         private static void saveCache(string channel, int accountId, string accountName, bool success, string message, string reason)
         {
             saveAccountCache("all", success, message, reason);

molilian.core/Core/log/deeplink.cs

@@ -9,39 +9,92 @@ namespace molilian.core
 
         static string queue_deeplink_parse_key = "queue:parse_logs:deeplink";
 
-        public static int task_insert_parse_deeplink_logs(IDbConnection connection, IDbTransaction transaction, int limit, CSRedisClient redis)
+
+        public static int task_insert_parse_deeplink_logs(int limit, CSRedisClient redis)
         {
             int total = 0;
-
-            for (int i = 0; i < limit; i++)
+            using var connection = DBContext.GetOpenConnection();
+            connection.Open();
+            using var transaction = connection.BeginTransaction();
+            try
             {
-                var data = redis.LPop<DeeplinkParseDataDTO>(queue_deeplink_parse_key);
-                if (data == null) break;
 
-                if (_test_oaid.Equals(data.oaid) ||
-                     data.ip.Contains("127.0.0"))
+                for (int i = 0; i < limit; i++)
                 {
-                    var test_data = data.Convert2Json().Convert2Object<TestDeeplinkParseDataDTO>();
-                    connection.Insert(test_data);
-                }
-                else
-                {
-                    connection.Insert(data);
-                    if (!data.success)
+                    var data = redis.LPop<DeeplinkParseDataDTO>(queue_deeplink_parse_key);
+                    if (data == null) break;
+
+
+                    if (_test_oaid.Equals(data.oaid) ||
+                         data.ip.Contains("127.0.0"))
                     {
-                        var success_data = data.Convert2Json().Convert2Object<FilDeeplinkParseDataDTO>();
-                        connection.Insert(success_data);
+                        save_dp_parse_logs(data, "deeplink_parse_logs_test", connection, transaction);
                     }
                     else
                     {
-                        var success_data = data.Convert2Json().Convert2Object<SuccessDeeplinkParseDataDTO>();
-                        connection.Insert(success_data);
+                        //save_dp_parse_logs(data, "deeplink_parse_logs", connection, transaction);
+
+                        //if (save_dailys_log)
+                        {
+                            string daily_table = $"deeplink_parse_logs_{data.create_time:yyyyMMdd}";
+                            save_dp_parse_logs(data, daily_table, connection, transaction);
+                        }
+
+                        if (data.success)
+                        {
+                            save_dp_parse_logs(data, "deeplink_parse_logs_success", connection, transaction);
+                        }
                     }
+                    total++;
                 }
-                total++;
+                transaction.Commit();
+            }
+            catch
+            {
+                transaction.Rollback();
+                throw;
+            }
+            finally
+            {
+                connection.Close();
             }
             return total;
         }
+
+
+        //public static int task_insert_parse_deeplink_logs(IDbConnection connection, IDbTransaction transaction, int limit, CSRedisClient redis)
+        //{
+        //    int total = 0;
+
+        //    for (int i = 0; i < limit; i++)
+        //    {
+        //        var data = redis.LPop<DeeplinkParseDataDTO>(queue_deeplink_parse_key);
+        //        if (data == null) break;
+
+        //        if (_test_oaid.Equals(data.oaid) ||
+        //             data.ip.Contains("127.0.0"))
+        //        {
+        //            var test_data = data.Convert2Json().Convert2Object<TestDeeplinkParseDataDTO>();
+        //            connection.Insert(test_data);
+        //        }
+        //        else
+        //        {
+        //            connection.Insert(data);
+        //            if (!data.success)
+        //            {
+        //                var success_data = data.Convert2Json().Convert2Object<FilDeeplinkParseDataDTO>();
+        //                connection.Insert(success_data);
+        //            }
+        //            else
+        //            {
+        //                var success_data = data.Convert2Json().Convert2Object<SuccessDeeplinkParseDataDTO>();
+        //                connection.Insert(success_data);
+        //            }
+        //        }
+        //        total++;
+        //    }
+        //    return total;
+        //}
         public static async Task ParseLogAsync(DeeplinkParseDataDTO response)
         {
             try
@@ -60,6 +113,41 @@ namespace molilian.core
             catch (Exception ex) { }
         }
 
+
+        private static int save_dp_parse_logs(DeeplinkParseDataDTO data, string tablename, IDbConnection connection, IDbTransaction transaction)
+        {
+
+
+            //public int channel_id { get; set; } = 0;
+            //public string channel_name { get; set; } = "unknown";
+            //public bool success { get; set; } = true;
+            //public string message { get; set; } = string.Empty;
+            //public string reason { get; set; } = string.Empty;
+            //public string content { get; set; } = string.Empty;
+            //public string ip { get; set; } = string.Empty;
+            //public string oaid { get; set; } = string.Empty;
+            //public string deeplink_url { get; set; } = string.Empty;
+            //public string itemName { get; set; } = string.Empty;
+            //public int elapsedTime { get; set; } = 0;
+            //public DateTime create_time { get; set; } = DateTime.Now;
+            //public string end_point { get; set; } = string.Empty;
+
+            return new DBContext.Table(connection, tablename)
+                .Add("end_point", data.end_point)
+                .Add("channel_name", data.channel_name)
+                .Add("success", data.success)
+                .Add("message", data.message)
+                .Add("reason", data.reason)
+                .Add("content", data.content)
+                .Add("ip", data.ip)
+                .Add("oaid", data.oaid)
+                .Add("deeplink_url", data.deeplink_url)
+                .Add("itemName", data.itemName)
+                .Add("elapsedTime", data.elapsedTime)
+                .Add("create_time", data.create_time)
+                .Create(DBContext.InsertType.NORMAL, transaction);
+        }
+
     }
 
 }

molilian.core/Core/log/jd.cs

@@ -24,11 +24,11 @@ namespace molilian.core
                 }
                 else
                 {
-                    save_jd_parse_logs(data, "jd_parse_logs", connection, transaction);
+                    //save_jd_parse_logs(data, "jd_parse_logs", connection, transaction);
 
                     if (save_dailys_log)
                     {
-                        string daily_table = $"jd_parse_logs_{DateTime.Now:yyyyMMdd}";
+                        string daily_table = $"jd_parse_logs_{data.create_time:yyyyMMdd}";
                         save_jd_parse_logs(data, daily_table, connection, transaction);
                     }
 

molilian.core/Core/log/ks.cs

@@ -23,11 +23,11 @@ namespace molilian.core
                 }
                 else
                 {
-                    save_ks_parse_logs(data, "ks_parse_logs", connection, transaction);
+                    //save_ks_parse_logs(data, "ks_parse_logs", connection, transaction);
 
                     if (save_dailys_log)
                     {
-                        string daily_table = $"ks_parse_logs_{DateTime.Now:yyyyMMdd}";
+                        string daily_table = $"ks_parse_logs_{data.create_time:yyyyMMdd}";
                         save_ks_parse_logs(data, daily_table, connection, transaction);
                     }
 

molilian.core/Core/log/pdd.cs

@@ -25,10 +25,6 @@ namespace molilian.core
                 else
                 {
                     save_pdd_parse_logs(data, "pdd_parse_logs", connection, transaction);
-                    if (data.elapsedTime > 1000)
-                    {
-                        save_pdd_parse_logs(data, "pdd_parse_logs_test", connection, transaction);
-                    }
                     if (data.success)
                     {
                         save_pdd_parse_logs(data, "pdd_parse_logs_success", connection, transaction);
@@ -63,10 +59,6 @@ namespace molilian.core
                     else
                     {
                         save_pdd_parse_logs(data, "pdd_parse_logs", null, null);
-                        if (data.elapsedTime > 1000)
-                        {
-                            save_pdd_parse_logs(data, "pdd_parse_logs_test", null, null);
-                        }
                         if (data.success)
                         {
                             save_pdd_parse_logs(data, "pdd_parse_logs_success", null, null);
@@ -119,6 +111,7 @@ namespace molilian.core
                 .Add("accountId", data.accountId)
                 .Add("accountName", data.accountName)
                 .Add("rawContent", data.rawContent)
+                .Add("rawContent2", data.rawContent2)
                 .Add("success", data.success)
                 .Add("message", data.message)
                 .Add("reason", data.reason)
@@ -132,14 +125,15 @@ namespace molilian.core
                 .Add("shortLinkurl", data.shortLinkurl)
                 .Add("deeplink_url", data.deeplink_url)
                 .Add("elapsedTime", data.elapsedTime)
+                .Add("elapsedTime2", data.elapsedTime2)
+                .Add("elapsedTime3", data.elapsedTime3)
                 .Add("subCode", data.subCode)
                 .Add("ip", data.ip)
                 .Add("oaid", data.oaid)
                 .Add("create_time", data.create_time)
                 .Create(DBContext.InsertType.NORMAL, transaction);
-        }
-
 
+        }
 
     }
 

molilian.core/Core/log/promotion.cs

@@ -24,7 +24,7 @@ namespace molilian.core
                 }
 
                 //按日保存数据
-                string daily_table = $"tk_promotion_logs_{DateTime.Now:yyyyMMdd}";
+                string daily_table = $"tk_promotion_logs_{data.create_time:yyyyMMdd}";
                 save_tk_promotion_img_logs(data, daily_table, connection, transaction);
 
                 total++;

molilian.core/Core/log/taobao.cs

@@ -24,12 +24,12 @@ namespace molilian.core
                 }
                 else
                 {
-                    data.id = save_tk_parse_logs(data, "tk_parse_logs", connection, transaction);
+                    //data.id = save_tk_parse_logs(data, "tk_parse_logs", connection, transaction);
 
-                    //每日分表
-                    if (save_dailys_log)
+                    ////每日分表
+                    //if (save_dailys_log)
                     {
-                        string daily_table = $"tk_parse_logs_{DateTime.Now:yyyyMMdd}";
+                        string daily_table = $"tk_parse_logs_{data.create_time:yyyyMMdd}";
                         save_tk_parse_logs(data, daily_table, connection, transaction);
                     }
 

molilian.core/Core/log/第三方旧接口.cs

@@ -10,57 +10,73 @@ namespace molilian.core
         static string queue_tb_key = "queue:logs:tb";
         static string queue_jd_key = "queue:logs:jd";
 
-        public static int task_insert_tk_logs(IDbConnection connection, IDbTransaction transaction, int limit, CSRedisClient redis)
+        public static int task_insert_tk_logs(int limit, CSRedisClient redis)
         {
             int total = 0;
-            for (int i = 0; i < limit; i++)
+            using var connection = DBContext.GetOpenConnection();
+            connection.Open();
+            using var transaction = connection.BeginTransaction();
+            try
             {
-                var data = redis.LPop<TkDataDTO>(queue_tb_key);
-                if (data == null) break;
-
-                if (_test_oaid.Equals(data.oaid) ||
-                     data.ip.Contains("127.0.0"))
-                {
-                    save_tk_log(data, "tk_logs_test", connection, transaction);
-                }
-                else
+                for (int i = 0; i < limit; i++)
                 {
-                    data.id = save_tk_log(data, "tk_logs", connection, transaction);
-                    if (data.elapsedTime > 1000)
+                    var data = redis.LPop<TkDataDTO>(queue_tb_key);
+                    if (data == null) break;
+
+                    if (_test_oaid.Equals(data.oaid) ||
+                         data.ip.Contains("127.0.0"))
                     {
                         save_tk_log(data, "tk_logs_test", connection, transaction);
                     }
-                    if (data.success)
+                    else
                     {
-                        save_tk_log(data, "tk_success_logs", connection, transaction);
+                        data.id = save_tk_log(data, "tk_logs", connection, transaction);
+                        if (data.elapsedTime > 1000)
+                        {
+                            save_tk_log(data, "tk_logs_test", connection, transaction);
+                        }
+                        if (data.success)
+                        {
+                            save_tk_log(data, "tk_success_logs", connection, transaction);
+                        }
+                        if (!string.IsNullOrEmpty(data.itemId)) TkOrderTrackingCore.SaveLinkSummary(data);
                     }
-                    if (!string.IsNullOrEmpty(data.itemId)) TkOrderTrackingCore.SaveLinkSummary(data);
+                    total++;
                 }
-                total++;
-            }
 
-            for (int i = 0; i < limit; i++)
+                for (int i = 0; i < limit; i++)
+                {
+                    var data = redis.LPop<JdDataDTO>(queue_jd_key);
+                    if (data == null) break;
+                    new DBContext.Table(connection, "tk_logs")
+                        .Add("end_point", data.end_point)
+                        .Add("channel", (int)data.channel)
+                        .Add("accountId", data.accountId)
+                        .Add("accountName", data.accountName)
+                        .Add("rawContent", data.rawContent)
+                        .Add("rawContent2", data.rawContent2)
+                        .Add("success", data.success)
+                        .Add("message", data.message)
+                        .Add("reason", data.reason)
+                        .Add("shortLinkurl", data.shortLinkurl)
+                        .Add("deeplink_url", data.deeplink_url)
+                        .Add("elapsedTime", data.elapsedTime)
+                        .Add("ip", data.ip)
+                        .Add("oaid", data.oaid)
+                        .Add("create_time", data.create_time)
+                        .Create(DBContext.InsertType.NORMAL, transaction);
+                    total++;
+                }
+                transaction.Commit();
+            }
+            catch
+            {
+                transaction.Rollback();
+                throw;
+            }
+            finally
             {
-                var data = redis.LPop<JdDataDTO>(queue_jd_key);
-                if (data == null) break;
-                new DBContext.Table(connection, "tk_logs")
-                    .Add("end_point", data.end_point)
-                    .Add("channel", (int)data.channel)
-                    .Add("accountId", data.accountId)
-                    .Add("accountName", data.accountName)
-                    .Add("rawContent", data.rawContent)
-                    .Add("rawContent2", data.rawContent2)
-                    .Add("success", data.success)
-                    .Add("message", data.message)
-                    .Add("reason", data.reason)
-                    .Add("shortLinkurl", data.shortLinkurl)
-                    .Add("deeplink_url", data.deeplink_url)
-                    .Add("elapsedTime", data.elapsedTime)
-                    .Add("ip", data.ip)
-                    .Add("oaid", data.oaid)
-                    .Add("create_time", data.create_time)
-                    .Create(DBContext.InsertType.NORMAL, transaction);
-                total++;
+                connection.Close();
             }
             return total;
         }

molilian.core/Core/tool/DeeplinkParseCore.cs

@@ -5,6 +5,7 @@ using Microsoft.Extensions.FileSystemGlobbing.Internal;
 using Sayaka.Common;
 using Spire.Pdf.Annotations;
 using System.Text.RegularExpressions;
+using System.Web;
 using System.Xml.Linq;
 namespace molilian.core
 {
@@ -185,11 +186,43 @@ namespace molilian.core
                             string val = match.Groups[i].Value;
                             if (deeplink.Contains("{url:"))
                             {
-                                deeplink = deeplink.Replace($"{{url:{i - 1}}}", val.UrlEncode());
+                                string encodedUrl = HttpUtility.UrlEncode(val);
+                                deeplink = deeplink.Replace($"{{url:{i - 1}}}", encodedUrl);
+                                //deeplink = deeplink.Replace($"{{url:{i - 1}}}", val.UrlEncode());
                             }
                             if (prompt_text.Contains("{url:"))
                             {
-                                prompt_text = prompt_text.Replace($"{{url:{i - 1}}}", val.UrlEncode());
+                                string encodedUrl = HttpUtility.UrlEncode(val);
+                                prompt_text = prompt_text.Replace($"{{url:{i - 1}}}", encodedUrl);
+                                //prompt_text = prompt_text.Replace($"{{url:{i - 1}}}", val.UrlEncode());
+                            }
+
+
+                            if (deeplink.Contains("{url2:"))
+                            {
+                                string encodedUrl = HttpUtility.UrlEncode(val);
+                                encodedUrl = HttpUtility.UrlEncode(encodedUrl);
+                                deeplink = deeplink.Replace($"{{url2:{i - 1}}}", encodedUrl);
+                            }
+                            if (prompt_text.Contains("{url2:"))
+                            {
+                                string encodedUrl = HttpUtility.UrlEncode(val);
+                                encodedUrl = HttpUtility.UrlEncode(encodedUrl);
+                                prompt_text = prompt_text.Replace($"{{url2:{i - 1}}}", encodedUrl);
+                            }
+
+
+                            if (deeplink.Contains("{url3:"))
+                            {
+                                string encodedUrl = HttpUtility.UrlEncode(val);
+                                encodedUrl = HttpUtility.UrlEncode(encodedUrl);
+                                deeplink = deeplink.Replace($"{{url3:{i - 1}}}", encodedUrl);
+                            }
+                            if (prompt_text.Contains("{url3:"))
+                            {
+                                string encodedUrl = HttpUtility.UrlEncode(val);
+                                encodedUrl = HttpUtility.UrlEncode(encodedUrl);
+                                prompt_text = prompt_text.Replace($"{{url3:{i - 1}}}", encodedUrl);
                             }
 
                             //atob

molilian.core/DTO/alimama/TkConfigDTO.cs

@@ -27,17 +27,24 @@ namespace molilian.core
         public string multi_token_regular { get; set; } = string.Empty;
 
         public int jd_limit_per_ip_24h { get; set; } = 0;
+        public int jd_limit_per_oaid_24h { get; set; } = 0;
+
         public int tk_limit_per_ip_24h { get; set; } = 0;
+        public int tk_limit_per_oaid_24h { get; set; } = 0;
+
         public string cpsIgnorePercentageCity { get; set; } = string.Empty;
         public int cps_limit_per_ip_24h { get; set; } = 0;
+        public int cps_limit_per_oaid_24h { get; set; } = 0;
 
         public int pddIgnorePercentage { get; set; } = 0;
         public string pddIgnorePercentageCity { get; set; } = string.Empty;
         public int pdd_limit_per_ip_24h { get; set; } = 0;
+        public int pdd_limit_per_oaid_24h { get; set; } = 0;
 
 
         public int ksIgnorePercentage { get; set; } = 0;
         public string ksIgnorePercentageCity { get; set; } = string.Empty;
         public int ks_limit_per_ip_24h { get; set; } = 0;
+        public int ks_limit_per_oaid_24h { get; set; } = 0;
     }
 }

molilian.core/DTO/pdd/PddDataDTO.cs

@@ -26,6 +26,8 @@
         public string shortLinkurl { get; set; } = string.Empty;
         public string deeplink_url { get; set; } = string.Empty;
         public int elapsedTime { get; set; } = 0;
+        public int elapsedTime2 { get; set; } = 0;
+        public int elapsedTime3 { get; set; } = 0;
         public DateTime create_time { get; set; } = DateTime.Now;
         public string end_point { get; set; } = string.Empty;
         public int subCode { get; set; } = 0;

molilian.core/Plus/JDUnion/base.cs

@@ -131,7 +131,11 @@ namespace molilian.core
                         reason = "IP控制";
                         return true;
                     }
-                    num = TkLogCore.getClientRequestTotalByOAID(TkChannelEnum.jd, oaid);
+                }
+                limit_num = config.jd_limit_per_oaid_24h;
+                if (limit_num > 0)
+                {
+                    int num = TkLogCore.getClientRequestTotalByOAID(TkChannelEnum.jd, oaid);
                     if (num > limit_num)
                     {
                         reason = "OAID控制";

molilian.core/Plus/ks/base.cs

@@ -59,7 +59,12 @@ namespace molilian.core
                         reason = "IP控制";
                         return true;
                     }
-                    num = TkLogCore.getClientRequestTotalByOAID(TkChannelEnum.ks, oaid);
+                }
+
+                limit_num = config.ks_limit_per_oaid_24h;
+                if (limit_num > 0)
+                {
+                    int num = TkLogCore.getClientRequestTotalByOAID(TkChannelEnum.ks, oaid);
                     if (num > limit_num)
                     {
                         reason = "OAID控制";

molilian.core/Plus/pdd/PddUnionPlus.cs

@@ -1,5 +1,8 @@
 using dodohold.core;
+using Sayaka.Common;
+using System.Diagnostics;
 using System.Text.RegularExpressions;
+using System.Web;
 
 namespace molilian.core
 {
@@ -29,6 +32,61 @@ namespace molilian.core
             return result;
         }
 
+
+
+        private string ExtractGoodsIdUrl(string url)
+        {
+            var uri = new Uri(url);
+            var queryParams = HttpUtility.ParseQueryString(uri.Query);
+            string goodsId = queryParams["goods_id"];
+
+            if (!string.IsNullOrEmpty(goodsId)&& uri.AbsolutePath.StartsWith("/goods"))
+            {
+                // 保持域名和路径不变，仅保留 goods_id 参数
+                var baseUrl = $"{uri.Scheme}://{uri.Host}{uri.AbsolutePath}";
+                return $"{baseUrl}?goods_id={goodsId}";
+            }
+            return url; // 如果没有找到 goods_id 参数，则返回原始 URL
+        }
+
+        public async Task<string> GetRedirectedUrlAsync(string sourceUrl, CancellationToken cancellationToken = default)
+        {
+            string pattern = @"^https?://mobile\.yangkeduo\.com/goods(\d*)\.html\?ps=.*$";
+            Regex regex = new Regex(pattern);
+            if (!regex.IsMatch(sourceUrl)) return ExtractGoodsIdUrl(sourceUrl);
+            string redirectedUrl = sourceUrl;
+
+            try
+            {
+                string useragent = ProviderFakeUserAgent.RandomMobile;
+                WebClientUtility client = new()
+                {
+                    AllowAutoRedirect = false,
+                    UserAgent = useragent,
+                };
+                client.Proxy = _proxy;
+#if DEBUG
+                client.Proxy = null;
+#endif
+                var response = await client.RequestAsync(sourceUrl, "GET", cancellationToken);
+                if (!response.Successed) return null;
+
+                if (response.ResponseMessage.StatusCode == System.Net.HttpStatusCode.RedirectKeepVerb ||
+                  response.ResponseMessage.StatusCode == System.Net.HttpStatusCode.TemporaryRedirect)
+                {
+                    redirectedUrl = response.ResponseMessage.Headers.Location.ToString();
+                    redirectedUrl = ExtractGoodsIdUrl(redirectedUrl);
+                    return redirectedUrl;
+                }
+            }
+            catch (Exception ex)
+            {
+            }
+            return sourceUrl;
+        }
+
+
+
         public static PddDataDTO GetFormattedObject(string content, string ip = "", string oaid = "")
         {
             string shortLinkurl = GetLink(content);
@@ -102,6 +160,24 @@ namespace molilian.core
                 return result;
             }
 
+
+            //去追踪
+
+            Stopwatch stopwatch = Stopwatch.StartNew();
+            stopwatch.Start();
+            var redirectedUrl = await GetRedirectedUrlAsync(url, cancellationToken);
+            stopwatch.Stop();
+            if (redirectedUrl != url)
+            {
+                result.rawContent2 = redirectedUrl;
+                url = redirectedUrl;
+                result.elapsedTime2 = (int)stopwatch.ElapsedMilliseconds;
+            }
+
+
+            Stopwatch stopwatch2 = Stopwatch.StartNew();
+            stopwatch2.Start();
+
             switch (_account.work_mode)
             {
                 case PddUnionWorkMode.SiteApi:
@@ -115,6 +191,8 @@ namespace molilian.core
                     result = await transferUrl(result, url, cancellationToken);
                     break;
             }
+            stopwatch2.Stop();
+            result.elapsedTime3 = (int)stopwatch2.ElapsedMilliseconds;
             return result;
         }
     }

molilian.core/Plus/pdd/base.cs

@@ -48,7 +48,7 @@ namespace molilian.core
                 _proxy = ProxyNodesCore.GetOne(account.nodeName);
             }
         }
-           
+
         public static bool ShouldIgnoreRequest(TkConfigDTO config, string ip, string oaid, out string reason)
         {
             reason = string.Empty;
@@ -72,13 +72,19 @@ namespace molilian.core
                         reason = "IP控制";
                         return true;
                     }
-                    num = TkLogCore.getClientRequestTotalByOAID(TkChannelEnum.pdd, oaid);
+                }
+
+                limit_num = config.pdd_limit_per_oaid_24h;
+                if (limit_num > 0)
+                {
+                    int num = TkLogCore.getClientRequestTotalByOAID(TkChannelEnum.pdd, oaid);
                     if (num > limit_num)
                     {
                         reason = "OAID控制";
                         return true;
                     }
                 }
+
             }
             catch (Exception ex)
             {

molilian.core/Plus/pdd/open.cs

@@ -116,7 +116,6 @@ namespace molilian.core
             string body = response.Body();
             var root = body.Convert2Object<UnionGeneratelinkDTO>();
             string message = string.Empty;
-            _ = new LoggerLibrary("PddUnion", "debug").Info(urlWithQuery, body).SaveAsync();
 
             if (root.error_response != null || root.goods_zs_unit_generate_response == null)
             {
@@ -138,7 +137,15 @@ namespace molilian.core
             }
 
             result.itemId = root.goods_zs_unit_generate_response.url.GetContentPart("goods_id=", "&");
+            if (string.IsNullOrEmpty(root.goods_zs_unit_generate_response.short_url))
+            {
+                result.success = false;
+                result.message = "转链失败";
+                result.reason = "链接无效";
+                _ = new LoggerLibrary("PddUnion", "UnionGeneratelink").Info(urlWithQuery, body).SaveAsync();
+                return result;
 
+            }
             result.success = true;
             result.message = "OK";
             result.link_type = LinkTypeEnum.goods;

molilian.core/Worker/TaskWorker.cs

@@ -6,20 +6,20 @@ namespace molilian.core
     {
         protected override async Task ExecuteAsync(CancellationToken stoppingToken)
         {
-#if DEBUG
-#else
-            while (!stoppingToken.IsCancellationRequested)
-            {
-                //var tasks = new List<Task>();
-                try
-                {
-                    TkLogCore.BatchInsertLogDB(100);
-                }
-                catch { }
-                //await Task.WhenAll(tasks);
-                await Task.Delay(5 * 1000, stoppingToken);
-            }
-#endif
+//#if DEBUG
+//#else
+//            while (!stoppingToken.IsCancellationRequested)
+//            {
+//                //var tasks = new List<Task>();
+//                try
+//                {
+//                    TkLogCore.BatchInsertLogDB(100);
+//                }
+//                catch { }
+//                //await Task.WhenAll(tasks);
+//                await Task.Delay(5 * 1000, stoppingToken);
+//            }
+//#endif
         }
 
     }