批量写入mysql 增加每次任务总数；

修改task路由，增加干扰字符；
增加转脸任务的rt_max；网路过请（2次）求不超过rt-max；

molilian.api/Controllers/public/TaskController.cs

@@ -9,7 +9,7 @@ using System.Net;
 namespace molilian.api.Controllers
 {
     [ApiController]
-    [Route("[controller]/[action]")]
+    [Route("[controller]_70160bd632/[action]")]
     public class TaskController : ControllerBase
     {
 
@@ -20,10 +20,10 @@ namespace molilian.api.Controllers
         }
 
         [HttpGet]
-        public ActionResult BatchInsertLogDB()
+        public ActionResult BatchInsertLogDB(int limit = 100)
         {
-            TkLogCore.BatchInsertLogDB();
-            return new APIResult(new { success = true, message = "ok" });
+            int total = TkLogCore.BatchInsertLogDB(limit);
+            return new APIResult(new { success = true, message = "ok", limit, total });
         }
 
 

molilian.api/Properties/launchSettings.json

@@ -11,7 +11,7 @@
         "ANPush": "",
         "DBType": "MySQL",
         "DBConfig": "Server=rm-2zef2t13aq33dha43vo.mysql.rds.aliyuncs.com; Port=3306; Database=taoke; Uid=taoke; Pwd=67ktWBmw5G4yMs4J;SslMode=None;CharSet=utf8mb4;ConnectionTimeout=60",
-        "RedisConfig": "47.95.66.125:6379,password=pKBiS4ka2IpXayIdcx00,defaultDatabase=0,idleTimeout=20000,preheat=3,tryit=2,ssl=false,prefix=webhook"
+        "RedisConfig": "101.200.46.46:6379,password=pKBiS4ka2IpXayIdcx00,defaultDatabase=0,idleTimeout=20000,preheat=3,tryit=2,ssl=false,prefix=webhook"
       },
       "dotnetRunMessages": true,
       "applicationUrl": "http://localhost:8186"

molilian.core/Core/taoke/TkLogCore.cs

@@ -28,14 +28,15 @@ namespace molilian.core
 
         static string queue_coupon_key = "queue:coupon_logs";
 
-        public static void BatchInsertLogDB()
+        public static int BatchInsertLogDB(int limit)
         {
+            int total = 0;
             using var connection = DBContext.GetOpenConnection();
             connection.Open();
             using var transaction = connection.BeginTransaction();
             try
             {
-                for (int i = 0; i < 100; i++)
+                for (int i = 0; i < limit; i++)
                 {
                     var data = RedisHelper.LPop<TkDataDTO>(queue_tb_key);
                     if (data == null) break;
@@ -64,9 +65,10 @@ namespace molilian.core
                         .Add("create_time", data.create_time)
                         .Create(DBContext.InsertType.NORMAL, transaction);
                     if (!string.IsNullOrEmpty(data.itemId)) TkOrderTrackingCore.SaveLinkSummary(data);
+                    total++;
                 }
 
-                for (int i = 0; i < 100; i++)
+                for (int i = 0; i < limit; i++)
                 {
                     var data = RedisHelper.LPop<JdDataDTO>(queue_jd_key);
                     if (data == null) break;
@@ -86,9 +88,10 @@ namespace molilian.core
                        .Add("oaid", data.oaid)
                        .Add("create_time", data.create_time)
                        .Create(DBContext.InsertType.NORMAL, transaction);
+                    total++;
                 }
 
-                for (int i = 0; i < 100; i++)
+                for (int i = 0; i < limit; i++)
                 {
                     var data = RedisHelper.LPop<TkDataDTO>(queue_parse_tb_key);
                     if (data == null) break;
@@ -112,14 +115,17 @@ namespace molilian.core
                         .Add("deeplink_url", data.deeplink_url)
                         .Add("num_iid", data.num_iid)
                         .Add("elapsedTime", data.elapsedTime)
+                        .Add("elapsedTime2", data.elapsedTime2)
+                        .Add("elapsedTime3", data.elapsedTime3)
                         .Add("ip", data.ip)
                         .Add("oaid", data.oaid)
                         .Add("create_time", data.create_time)
                         .Create(DBContext.InsertType.NORMAL, transaction);
                     if (!string.IsNullOrEmpty(data.itemId)) TkOrderTrackingCore.SaveLinkSummary(data);
+                    total++;
                 }
 
-                for (int i = 0; i < 100; i++)
+                for (int i = 0; i < limit; i++)
                 {
                     var data = RedisHelper.LPop<JdDataDTO>(queue_parse_jd_key);
                     if (data == null) break;
@@ -145,8 +151,9 @@ namespace molilian.core
                         .Add("oaid", data.oaid)
                         .Add("create_time", data.create_time)
                         .Create(DBContext.InsertType.NORMAL, transaction);
+                    total++;
                 }
-                for (int i = 0; i < 100; i++)
+                for (int i = 0; i < limit; i++)
                 {
                     var data = RedisHelper.LPop<DyDataDTO>(queue_parse_dy_key);
                     if (data == null) break;
@@ -172,8 +179,9 @@ namespace molilian.core
                         .Add("oaid", data.oaid)
                         .Add("create_time", data.create_time)
                         .Create(DBContext.InsertType.NORMAL, transaction);
+                    total++;
                 }
-                for (int i = 0; i < 100; i++)
+                for (int i = 0; i < limit; i++)
                 {
                     var data = RedisHelper.LPop<ToolParseDataDTO>(queue_parse_tool_key);
                     if (data == null) break;
@@ -192,14 +200,16 @@ namespace molilian.core
                         .Add("oaid", data.oaid)
                         .Add("create_time", data.create_time)
                         .Create(DBContext.InsertType.NORMAL, transaction);
+                    total++;
                 }
 
 
-                for (int i = 0; i < 100; i++)
+                for (int i = 0; i < limit; i++)
                 {
                     var data = RedisHelper.LPop<UnionCouponDTO>(queue_coupon_key);
                     if (data == null) break;
                     connection.Insert(data);
+                    total++;
                 }
                 transaction.Commit();
             }
@@ -222,6 +232,7 @@ namespace molilian.core
             {
                 connection.Close();
             }
+            return total;
         }
 
         public static async Task LogAsync(JdDataDTO response)
@@ -281,7 +292,7 @@ namespace molilian.core
         public static async Task ParseLogAsync(TkDataDTO response, AlimamaPlus? alimamaPlus = null)
         {
 #if DEBUG
-            return;
+            //return;
 #endif
             try
             {

molilian.core/DTO/alimama/TkConfigDTO.cs

@@ -10,5 +10,6 @@
         public int fake_click_min { get; set; }
         public int fake_click_max { get; set; }
         public int fake_click_ttl { get; set; } = 86400;
+        public int rt_max { get; set; } = 0;
     }
 }

molilian.core/DTO/alimama/TkDataDTO.cs

@@ -20,6 +20,8 @@ namespace molilian.core
         public string shortLinkurl { get; set; } = string.Empty;
         public string deeplink_url { get; set; } = string.Empty;
         public int elapsedTime { get; set; } = 0;
+        public int elapsedTime2 { get; set; } = 0;
+        public int elapsedTime3 { get; set; } = 0;
         public string ip { get; set; } = string.Empty;
         public string oaid { get; set; } = string.Empty;
 

molilian.core/Plus/Alimama/parse.cs

@@ -22,11 +22,13 @@ namespace molilian.core
 
         private string _api;
         private int _api_id = 0;
+        private TkConfigDTO _config;
         public AlimamaPlus(TkPoolDTO account)
         {
             //    string accountName, string tb_token, string floorId, string refpid,
             //    string cookies, string user_agent, string nodeName
-
+            _config = TkConfigCore.Get();
+            
 
             _accountId = account.id;
             _accountName = account.name;
@@ -47,13 +49,17 @@ namespace molilian.core
 
         public bool ShouldIgnoreOtherAff(string content)
         {
-            (_, _, var link_type, _) = InternalTextProcessing(content);
-            return link_type == LinkTypeEnum.other_aff;
+            TkDataDTO result = new();
+
+            //(_, _, var link_type, _) = InternalTextProcessing(content, ref result);
+            InternalTextProcessing(content, ref result);
+            return result.link_type == LinkTypeEnum.other_aff;
         }
 
         public TkDataDTO UnionParse(string content, ref TkDataDTO result)
         {
-            (result.success, result.content, result.link_type, result.shortLinkurl) = InternalTextProcessing(content);
+            InternalTextProcessing(content, ref result);
+            //(result.success, result.content, result.link_type, result.shortLinkurl) = InternalTextProcessing(content,ref result);
             if (result.link_type == LinkTypeEnum.other_aff)
             {
                 result.channel = TkChannelEnum.tb;
@@ -80,7 +86,8 @@ namespace molilian.core
 
         public async Task<TkDataDTO> UnionParseAsync(string content, TkDataDTO result)
         {
-            (result.success, result.content, result.link_type, result.shortLinkurl) = InternalTextProcessing(content);
+            InternalTextProcessing(content, ref result);
+            //(result.success, result.content, result.link_type, result.shortLinkurl) = InternalTextProcessing(content);
             if (result.link_type == LinkTypeEnum.other_aff)
             {
                 result.channel = TkChannelEnum.tb;

molilian.core/Plus/Alimama/parse_2.cs

@@ -4,6 +4,7 @@ using System.Diagnostics;
 using System.Text.Json;
 using TencentCloud.Fmu.V20191213.Models;
 using System;
+using MySqlX.XDevAPI;
 
 namespace molilian.core
 {
@@ -131,6 +132,19 @@ namespace molilian.core
             return Regex.IsMatch(content, pattern);
         }
 
+        public TimeSpan GetRequestTimeout(int deduction = 0)
+        {
+#if DEBUG
+            return TimeSpan.FromMilliseconds(10 * 1000);
+#endif
+            if (_config.rt_max == 0) return TimeSpan.FromMilliseconds(1000);
+            int timeout = _config.rt_max;
+            if (deduction == 0) return TimeSpan.FromMilliseconds(timeout);
+            timeout = timeout - deduction;
+            if (timeout < 0) return TimeSpan.FromMilliseconds(200);
+            return TimeSpan.FromMilliseconds(timeout);
+        }
+
         /// <summary>
         /// 从http获取跳转的url
         /// </summary>
@@ -153,9 +167,8 @@ namespace molilian.core
                 };
 #if DEBUG
                 client.Proxy = null;
-#else
-                client.Timeout = TimeSpan.FromMilliseconds(3000);
 #endif
+                client.Timeout = GetRequestTimeout(0);
                 var response = client.Request(url);
 
                 var responseBody = response.Body();
@@ -193,7 +206,107 @@ namespace molilian.core
         }
 
 
-        private (bool, string, LinkTypeEnum, string) InternalTextProcessing(string content)
+        private void InternalTextProcessing(string content, ref TkDataDTO result)
+        {
+            //(result.success, result.content, result.link_type, result.shortLinkurl) = InternalTextProcessing(content,ref result);
+
+
+            bool success;
+            string resultText;
+            LinkTypeEnum link_type = LinkTypeEnum.unknown;
+
+            string url = GetLink(content);
+            if (!string.IsNullOrEmpty(url))
+            {
+                //排除已知的淘客链接
+                if (IsAffLink(url))
+                {
+                    result.success = false;
+                    result.content = "排除淘客链接";
+                    result.link_type = LinkTypeEnum.other_aff;
+                    result.shortLinkurl = url;
+                    return;
+                    //return (false, "排除淘客链接", LinkTypeEnum.other_aff, url);
+                }
+
+                //直接提取
+                (success, resultText) = IsTaobaoUrl(url);
+                if (success)
+                {
+                    result.success = true;
+                    result.content = resultText;
+                    result.link_type = LinkTypeEnum.goods;
+                    result.shortLinkurl = resultText;
+                    return;
+                    //return (true, result, LinkTypeEnum.goods, result);
+                }
+
+                //从http获取跳转的url
+                Stopwatch stopwatch = Stopwatch.StartNew();
+                stopwatch.Start();
+                (success, resultText) = GetDesiredUrl(url);
+                stopwatch.Stop();
+                // 获取执行时间
+                result.elapsedTime2 = (int)stopwatch.ElapsedMilliseconds;
+
+                if (success)
+                {
+                    //排除已知的淘客链接
+                    if (IsAffLink(resultText))
+                    {
+                        result.success = false;
+                        result.content = "排除淘客链接";
+                        result.link_type = LinkTypeEnum.other_aff;
+                        result.shortLinkurl = url;
+                        return;
+                        //return (false, "排除淘客链接", LinkTypeEnum.other_aff, url);
+                    }
+                    (success, resultText) = IsTaobaoUrl(resultText);
+                    if (success)
+                    {
+                        result.success = true;
+                        result.content = resultText;
+                        result.link_type = LinkTypeEnum.goods;
+                        result.shortLinkurl = resultText;
+                        return;
+                        //return (true, resultText, LinkTypeEnum.goods, resultText);
+                    }
+                    link_type = GetLinkType(resultText);
+                }
+                else
+                {
+                    link_type = GetLinkType(url);
+
+                }
+
+                result.success = false;
+                result.content = resultText;
+                result.link_type = link_type;
+                result.shortLinkurl = url;
+                return;
+
+                //return (false, result, link_type, url);
+            }
+
+            //判断淘客的￥羊角符号
+            if (ContainsSpecialFormat(content))
+            {
+                result.success = false;
+                result.content = "排除淘口令";
+                result.link_type = LinkTypeEnum.other_aff;
+                result.shortLinkurl = null;
+                return;
+                //return (false, "排除淘口令", LinkTypeEnum.other_aff, null);
+            }
+
+            //todo 临时应急  0614
+            //return (false, "纯口令 没链接", link_type, null);
+            result.success = false;
+            result.content = "纯口令 没链接";
+            result.link_type = LinkTypeEnum.other_aff;
+            result.shortLinkurl = null;
+        }
+        private (bool, string, LinkTypeEnum, string) InternalTextProcessing222(string content)
         {
             bool success;
             string result;
@@ -210,9 +323,9 @@ namespace molilian.core
                 if (success) return (true, result, LinkTypeEnum.goods, result);
                 //https://uland.taobao.com/coupon/edetail?af=4&e=%2B0aTAvEt16MoRnbUfHBdCarU9683KT3AsB67n92UtFJMmdsrkidbOWBzzpT26idJbwZX%2BHY%2B8QD9zJXNeOSc1euXSPKeLUBBWr02LqsxPV8Bf6pSMNy5GSw6iqMObXLLk3toOSlQXmUUcLJJYP0Uq5lvDADUDTeHRiuxU0i9W%2BtrC8Oh6piEfdwyUIuKQbeM8rUgQejRG616MoRnbUfHBdCarU9683KT3AsB67n92UtFv6uIFQszbmWWPZ8ka7%2BkV5xXydLt%2FcnuZ4%2BBFYnwwiUaCWC3TckcCwMW8xe3XqOGsb8uv9wbhxdkKJ0hxTrTkj7DNgliTXDluAYBRglsbQ%3D%3D&traceId=212cbaa417178512249145009e05a8&union_lens=lensId%3APUB%401717851219%402103c04a_0c89_18ff7e96612_9d39%40031w5YJSc7jl3U7gYjdvuCMM%40eyJmbG9vcklkIjo4NTAvEt16MoRnbUfHBdCarU9683JDaxNfPy83okP3kLScwCkGiuMcyC4PcynGV0YWlsX2h0bSIsInNyY0Zsb29ySWQiiOiiI4MDY3NCIsImNyZWF0aXZpdHlHcm91cElkIjowfQieie%3BtkScm%3AselectionPlaza_site_4358%3Bscm%3A1007.30148.329090.pub_search-item_c02df4bb-631f-4cc1-a9d5-abe802b8e134_&un=bfc196d23e155d289f2917200ca32a93&share_crt_v=1&un_site=0&ut_sk=1.utdid_null_1717851225204.TaoPassword-Outside.lianmeng-app&spm=a2159r.13376465.0.0&sp_tk=d2hLeFdDNHpKb3Y%3D&cpp=1&shareurl=true&short_name=h.gVx6mvI
 
-
                 //从http获取跳转的url
                 (success, result) = GetDesiredUrl(url);
+
                 if (success)
                 {
                     //排除已知的淘客链接
@@ -286,8 +399,7 @@ namespace molilian.core
             client.Proxy = null;
 #endif
             if (!string.IsNullOrEmpty(_user_agent)) client.UserAgent = _user_agent;
-
-            client.Timeout = TimeSpan.FromMilliseconds(1000);
+            client.Timeout = GetRequestTimeout(result.elapsedTime2);
             var response = client.Request(url);
 
             var body = string.Empty;
@@ -456,19 +568,23 @@ namespace molilian.core
             string url = "https://pub.alimama.com/openapi/param2/1/gateway.unionpub/xt.entry.json?" +
                 $"t={t}&_tb_token_={_tb_token}&floorId={_floorId}&refpid={_refpid}&variableMap={variableMap}";
 
+            Stopwatch stopwatch = Stopwatch.StartNew();
+            stopwatch.Start();
 
             var client = new WebClientUtility().SetContentType("application/json;charset=utf-8")
                         .AddHeaders("X-Requested-With", "XMLHttpRequest")
                         .AddHeaders("Cookie", _cookies);
             client.Proxy = _proxy;
             if (!string.IsNullOrEmpty(_user_agent)) client.UserAgent = _user_agent;
-            client.Timeout = TimeSpan.FromMilliseconds(1000);
 #if DEBUG
             client.Proxy = null;
-            client.Timeout = TimeSpan.FromMilliseconds(10000);
 #endif
+            client.Timeout = GetRequestTimeout(result.elapsedTime2);
             var response = client.Request(url);
 
+            stopwatch.Stop();
+            result.elapsedTime3 = (int)stopwatch.ElapsedMilliseconds;
+
             var body = string.Empty;
             try
             {

molilian.core/Worker/TaskWorker.cs

@@ -13,7 +13,7 @@ namespace molilian.core
                 //var tasks = new List<Task>();
                 try
                 {
-                    TkLogCore.BatchInsertLogDB();
+                    TkLogCore.BatchInsertLogDB(100);
                 }
                 catch { }
                 //await Task.WhenAll(tasks);