feat(淘宝解析): 增加七天缓存和错误通知限频统计

molilian.core/Core/taoke/TaobaoGeneralLinkParseCacheCore.cs

@@ -0,0 +1,97 @@
+using dodohold.core;
+using System.Security.Cryptography;
+using System.Text;
+
+namespace molilian.core
+{
+    public static class TaobaoGeneralLinkParseCacheCore
+    {
+        public const int CacheExpireSeconds = 7 * 86400;
+
+        // 统计跨天保留，每次通知领取上次通知后的全部次数；同一天只允许领取一次。
+        private const string ClaimNotificationScript = @"
+local count = redis.call('HINCRBY', KEYS[1], 'pending', 1)
+local lastDate = redis.call('HGET', KEYS[1], 'last_notify_date')
+if lastDate and lastDate >= ARGV[1] then
+    return 0
+end
+redis.call('HSET', KEYS[1], 'last_notify_date', ARGV[1])
+redis.call('HSET', KEYS[1], 'pending', 0)
+return count";
+
+        public static async Task<TaobaoGeneralLinkParseCacheDTO?> GetAsync(string content)
+        {
+            try
+            {
+                return await CenterHub.Redis.GetAsync<TaobaoGeneralLinkParseCacheDTO>(GetCacheKey(content));
+            }
+            catch (Exception ex)
+            {
+                LogFailure("cache_read_error", ex);
+                return null;
+            }
+        }
+
+        public static async Task SaveAsync(TaobaoGeneralLinkParseCacheDTO record)
+        {
+            try
+            {
+                await CenterHub.Redis.SetAsync(GetCacheKey(record.RequestContent), record, CacheExpireSeconds);
+            }
+            catch (Exception ex)
+            {
+                LogFailure("cache_write_error", ex);
+            }
+        }
+
+        public static async Task NotifyFailureOnceAsync(string reason, int parseAccountId, int consumerAccountId)
+        {
+            reason = string.IsNullOrWhiteSpace(reason) ? "未知错误" : reason.Trim();
+            string key = $"notify:taobao:general_link_parse:v1:{Hash(reason)}";
+            long count = 0;
+            try
+            {
+                count = Convert.ToInt64(await CenterHub.Redis.EvalAsync(ClaimNotificationScript, key, DateTime.Now.ToString("yyyyMMdd")));
+                if (count <= 0) return;
+
+                await NotifyCore.NotifyAsync(
+                    $"【淘宝万能解析失败】\n错误信息：{reason}\n" +
+                    $"自上次相同错误通知以来发生：{count} 次（含本次）\n" +
+                    $"使用账号：{consumerAccountId}，解析账号：{parseAccountId}\n已回退原短链解析逻辑。");
+            }
+            catch (Exception ex)
+            {
+                if (count > 0)
+                {
+                    try
+                    {
+                        // 发送失败的次数留给下一次通知，同一天不重复尝试发送。
+                        await CenterHub.Redis.HIncrByAsync(key, "pending", count);
+                    }
+                    catch (Exception restoreException)
+                    {
+                        LogFailure("notify_count_restore_error", restoreException);
+                    }
+                }
+                LogFailure("notify_error", ex);
+            }
+        }
+
+        private static string GetCacheKey(string content)
+        {
+            return $"cache:taobao:general_link_parse:v1:{Hash(content.Trim())}";
+        }
+
+        private static string Hash(string value)
+        {
+            return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(value)));
+        }
+
+        private static void LogFailure(string operation, Exception ex)
+        {
+            _ = new LoggerLibrary("tbk_dg_general_link_parse", operation)
+                .Info(ex.Message, ex.StackTrace)
+                .SaveAsync();
+        }
+    }
+}

molilian.core/Core/taoke/TaobaoShortLinkCacheCore.cs

@@ -0,0 +1,85 @@
+using dodohold.core;
+using System.Security.Cryptography;
+using System.Text;
+
+namespace molilian.core
+{
+    public static class TaobaoShortLinkCacheCore
+    {
+        public const int CacheExpireSeconds = 7 * 86400;
+
+        public static async Task<string?> GetAsync(string shortUrl)
+        {
+            try
+            {
+                string longUrl = await CenterHub.Redis.GetAsync<string>(GetCacheKey(shortUrl));
+                return IsLongUrl(longUrl) ? longUrl : null;
+            }
+            catch (Exception ex)
+            {
+                LogFailure("read", ex);
+                return null;
+            }
+        }
+
+        public static string? Get(string shortUrl)
+        {
+            try
+            {
+                string longUrl = CenterHub.Redis.Get<string>(GetCacheKey(shortUrl));
+                return IsLongUrl(longUrl) ? longUrl : null;
+            }
+            catch (Exception ex)
+            {
+                LogFailure("read", ex);
+                return null;
+            }
+        }
+
+        public static async Task SaveAsync(string shortUrl, string longUrl)
+        {
+            if (!IsLongUrl(longUrl)) return;
+            try
+            {
+                await CenterHub.Redis.SetAsync(GetCacheKey(shortUrl), longUrl, CacheExpireSeconds);
+            }
+            catch (Exception ex)
+            {
+                LogFailure("write", ex);
+            }
+        }
+
+        public static void Save(string shortUrl, string longUrl)
+        {
+            if (!IsLongUrl(longUrl)) return;
+            try
+            {
+                CenterHub.Redis.Set(GetCacheKey(shortUrl), longUrl, CacheExpireSeconds);
+            }
+            catch (Exception ex)
+            {
+                LogFailure("write", ex);
+            }
+        }
+
+        private static bool IsLongUrl(string? url)
+        {
+            return Uri.TryCreate(url, UriKind.Absolute, out var uri) &&
+                (uri.Scheme == Uri.UriSchemeHttp || uri.Scheme == Uri.UriSchemeHttps) &&
+                !AlimamaPlus.IsShortLink(url!);
+        }
+
+        private static string GetCacheKey(string shortUrl)
+        {
+            string hash = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(shortUrl.Trim())));
+            return $"cache:taobao:short_link_http:v1:{hash}";
+        }
+
+        private static void LogFailure(string operation, Exception ex)
+        {
+            _ = new LoggerLibrary("taobao_short_link_cache", operation)
+                .Info(ex.Message, ex.StackTrace)
+                .SaveAsync();
+        }
+    }
+}

molilian.core/DTO/alimama/TaobaoGeneralLinkParseCacheDTO.cs

@@ -0,0 +1,16 @@
+namespace molilian.core
+{
+    public sealed class TaobaoGeneralLinkParseCacheDTO
+    {
+        public string RequestContent { get; set; } = string.Empty;
+        public DateTime RequestedAtUtc { get; set; }
+        public int ParseAccountId { get; set; }
+        public bool Success { get; set; }
+        public string Reason { get; set; } = string.Empty;
+        public TkSubCodeEnum SubCode { get; set; }
+        public LinkTypeEnum LinkType { get; set; }
+        public string ItemId { get; set; } = string.Empty;
+        public string ItemUrl { get; set; } = string.Empty;
+        public string ResponseBody { get; set; } = string.Empty;
+    }
+}

molilian.core/Plus/Alimama/parse_2.cs

@@ -171,41 +171,43 @@ namespace molilian.core
         /// <param name="url"></param>
         /// <returns></returns>
         public async Task<(bool, string)> GetDesiredUrlAsync(WebProxy proxy, string url, CancellationToken cancellationToken = default)
-        {
-            var result = await GetDesiredUrlResultAsync(proxy, url, cancellationToken);
-            return (result.Success, result.Content);
-        }
-
-        private readonly record struct DesiredUrlResult(bool Success, string Content, bool UsedGeneralLinkParse);
-
-        private async Task<DesiredUrlResult> GetDesiredUrlResultAsync(WebProxy proxy, string url, CancellationToken cancellationToken)
         {
             string result;
-            bool usedGeneralLinkParse = false;
             try
             {
-                if (!IsShortLink(url)) return new(false, "不是淘宝短网址", false);
+                if (!IsShortLink(url)) return (false, "不是淘宝短网址");
 
                 if (_account.use_general_link_parse_pool)
                 {
-                    usedGeneralLinkParse = true;
-                    var parsed = await endpoint_tbk_dg_general_link_parse(url, new TkDataDTO { content = url }, cancellationToken);
-                    if (parsed.subCode != TkSubCodeEnum.GeneralLinkParsePoolUnavailable)
+                    try
                     {
-                        if (!parsed.success) return new(false, parsed.reason, true);
-
-                        string desiredUrl = IsDigitsOnly(parsed.itemId)
-                            ? $"https://item.taobao.com/item.htm?id={parsed.itemId}"
-                            : parsed.item_url;
-                        return new(true, desiredUrl, true);
+                        var parsed = await endpoint_tbk_dg_general_link_parse(url, new TkDataDTO { content = url }, cancellationToken);
+                        if (parsed.success)
+                        {
+                            string desiredUrl = IsDigitsOnly(parsed.itemId)
+                                ? $"https://item.taobao.com/item.htm?id={parsed.itemId}"
+                                : parsed.item_url;
+                            return (true, desiredUrl);
+                        }
                     }
-
-                    // 账号池无可用账号时，继续按下探网址设置执行原逻辑。
-                    usedGeneralLinkParse = false;
+                    catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+                    {
+                        throw;
+                    }
+                    catch (Exception ex)
+                    {
+                        _ = TaobaoGeneralLinkParseCacheCore.NotifyFailureOnceAsync($"{ex.GetType().Name}: {ex.Message}", 0, _accountId);
+                    }
+                    // 解析失败、接口异常或账号池无可用账号，均回退原 HTTP 逻辑。
                 }
 
                 cancellationToken.ThrowIfCancellationRequested();
-                if (!_account.enable_deep_url) return new(true, url, false);
+                if (!_account.enable_deep_url) return (true, url);
+
+                string shortUrl = url;
+                string? cachedUrl = await TaobaoShortLinkCacheCore.GetAsync(shortUrl);
+                cancellationToken.ThrowIfCancellationRequested();
+                if (cachedUrl != null) return (true, cachedUrl);
 
                 if (_account.use_general_link_parse_pool)
                 {
@@ -234,9 +236,11 @@ namespace molilian.core
 
                 if (response.ResponseMessage?.StatusCode == HttpStatusCode.Found &&
                     !string.IsNullOrEmpty(response.ResponseMessage?.Headers?.Location?.OriginalString))
-                {
+                {
                     string redirectUrl = response.ResponseMessage?.Headers?.Location?.OriginalString;
-                    return new(true, redirectUrl, false);
+                    if (redirectUrl.StartsWith("//")) redirectUrl = "https:" + redirectUrl;
+                    await TaobaoShortLinkCacheCore.SaveAsync(shortUrl, redirectUrl);
+                    return (true, redirectUrl);
                 }
                 var responseBody = response.Body();
                 result = responseBody;
@@ -250,10 +254,11 @@ namespace molilian.core
                         {
                             result = match.Groups[1].Value; // 返回匹配的 URL
 
-                            if (string.IsNullOrEmpty(result)) return new(false, "没有匹配的URL", false);
-                            if (result.StartsWith("//"))
-                                result = "https:" + result;
-                            return new(true, result, false);
+                            if (string.IsNullOrEmpty(result)) return (false, "没有匹配的URL");
+                            if (result.StartsWith("//"))
+                                result = "https:" + result;
+                            await TaobaoShortLinkCacheCore.SaveAsync(shortUrl, result);
+                            return (true, result);
                         }
                     }
                     else
@@ -276,7 +281,7 @@ namespace molilian.core
                 _ = new LoggerLibrary("api_error", "desiredUrl_error").Info(ex.Message, ex.StackTrace).SaveAsync();
                 result = "网络异常";
             }
-            return new(false, result, usedGeneralLinkParse);
+            return (false, result);
         }
 
         /// <summary>
@@ -284,14 +289,17 @@ namespace molilian.core
         /// </summary>
         /// <param name="url"></param>
         /// <returns></returns>
-        public (bool, string) GetDesiredUrl(string url)
+        public (bool, string) GetDesiredUrl(string url)
         {
             string result;
             try
             {
-                if (!IsShortLink(url)) return (false, "不是淘宝短网址");
-
-                string desiredUrlPattern = "var url = '(.*?)'";
+                if (!IsShortLink(url)) return (false, "不是淘宝短网址");
+
+                string? cachedUrl = TaobaoShortLinkCacheCore.Get(url);
+                if (cachedUrl != null) return (true, cachedUrl);
+
+                string desiredUrlPattern = "var url = '(.*?)'";
                 WebClientUtility client = new()
                 {
                     Proxy = ProxyNodesCore.RandomOne(_account.nodeName),
@@ -316,8 +324,10 @@ namespace molilian.core
                         {
                             result = match.Groups[1].Value; // 返回匹配的 URL
 
-                            if (string.IsNullOrEmpty(result)) return (false, "没有匹配的URL");
-                            return (true, result);
+                            if (string.IsNullOrEmpty(result)) return (false, "没有匹配的URL");
+                            if (result.StartsWith("//")) result = "https:" + result;
+                            TaobaoShortLinkCacheCore.Save(url, result);
+                            return (true, result);
                         }
                     }
                     else
@@ -338,14 +348,17 @@ namespace molilian.core
             return (false, result);
         }
 
-        public static (bool, string) GetItemUrl(string url, WebProxy proxy = null)
+        public static (bool, string) GetItemUrl(string url, WebProxy proxy = null)
         {
             string result;
             try
             {
-                if (!IsShortLink(url)) return (false, "不是淘宝短网址");
-
-                string desiredUrlPattern = "var url = '(.*?)'";
+                if (!IsShortLink(url)) return (false, "不是淘宝短网址");
+
+                string? cachedUrl = TaobaoShortLinkCacheCore.Get(url);
+                if (cachedUrl != null) return (true, cachedUrl);
+
+                string desiredUrlPattern = "var url = '(.*?)'";
                 WebClientUtility client = new()
                 {
                     Proxy = proxy,
@@ -368,8 +381,10 @@ namespace molilian.core
                         {
                             result = match.Groups[1].Value; // 返回匹配的 URL
 
-                            if (string.IsNullOrEmpty(result)) return (false, "没有匹配的URL");
-                            return (true, result);
+                            if (string.IsNullOrEmpty(result)) return (false, "没有匹配的URL");
+                            if (result.StartsWith("//")) result = "https:" + result;
+                            TaobaoShortLinkCacheCore.Save(url, result);
+                            return (true, result);
                         }
                     }
                     else
@@ -537,24 +552,12 @@ namespace molilian.core
                     result.proxy_node = proxy?.Address?.Authority;
                 }
 
-                var desiredUrlResult = await GetDesiredUrlResultAsync(proxy, url, cancellationToken);
-                success = desiredUrlResult.Success;
-                resultText = desiredUrlResult.Content;
+                (success, resultText) = await GetDesiredUrlAsync(proxy, url, cancellationToken);
                 sw.Stop();
                 // 获取执行时间
                 result.elapsedTime2 = (int)sw.ElapsedMilliseconds;
                 swData?.Add("\tGetDesiredUrlAsync", sw.ElapsedMilliseconds);
 
-                if (desiredUrlResult.UsedGeneralLinkParse && !success)
-                {
-                    result.success = false;
-                    result.message = "解析失败";
-                    result.reason = resultText;
-                    result.subCode = TkSubCodeEnum.NoConvert;
-                    result.link_type = LinkTypeEnum.unknown;
-                    result.content = content;
-                    return result;
-                }
                 //============================== 放弃转链-请求超时 ==============================
 #if DEBUG
 #else

molilian.core/Plus/Alimama/parse_endpoint/endpoint_tbk_dg_general_link_parse.cs

@@ -11,6 +11,11 @@ namespace molilian.core
             Stopwatch stopwatch = Stopwatch.StartNew();
             result.channel = TkChannelEnum.tb;
             result.commercial = false;
+            result.remote_response = string.Empty;
+            bool requestSent = false;
+            bool resultCompleted = false;
+            int parseAccountId = 0;
+            DateTime requestedAtUtc = default;
 
             TkDataDTO Fail(string reason, TkSubCodeEnum subCode = TkSubCodeEnum.Other)
             {
@@ -19,6 +24,7 @@ namespace molilian.core
                 result.message = "解析失败";
                 result.reason = reason;
                 result.subCode = subCode;
+                resultCompleted = true;
 
                 _ = new LoggerLibrary("endpoint", "tbk_dg_general_link_parse")
                     .Info(content, reason)
@@ -33,6 +39,28 @@ namespace molilian.core
                     return Fail("待解析内容为空", TkSubCodeEnum.NoConvert);
                 }
 
+                var cached = await TaobaoGeneralLinkParseCacheCore.GetAsync(content);
+                cancellationToken.ThrowIfCancellationRequested();
+                if (cached != null)
+                {
+                    parseAccountId = cached.ParseAccountId;
+                    result.success = cached.Success;
+                    result.message = cached.Success ? string.Empty : "解析失败";
+                    result.reason = cached.Reason;
+                    result.subCode = cached.SubCode;
+                    result.link_type = cached.LinkType;
+                    result.itemId = cached.ItemId;
+                    result.num_iid = cached.ItemId;
+                    result.shortLinkurl = cached.ItemUrl;
+                    result.item_url = cached.ItemUrl;
+                    result.deeplink_url = cached.Success ? GetDeeplink(cached.ItemUrl) : string.Empty;
+                    result.item_deeplink_url = result.deeplink_url;
+                    result.remote_response = cached.ResponseBody;
+                    result.content = cached.Success ? ReplaceUrls(content, cached.ItemUrl) : content;
+                    resultCompleted = true;
+                    return result;
+                }
+
                 var parseAccount = await TkPoolCore.GetGeneralLinkParseAccountAsync();
                 cancellationToken.ThrowIfCancellationRequested();
                 if (parseAccount == null)
@@ -40,6 +68,7 @@ namespace molilian.core
                     return Fail("万能解析账号池无可用账号", TkSubCodeEnum.GeneralLinkParsePoolUnavailable);
                 }
 
+                parseAccountId = parseAccount.id;
                 TaobaoOpenPlus taobao = new(parseAccount.appkey, parseAccount.appSecret, parseAccount.open_pid, false, null!);
                 try
                 {
@@ -52,8 +81,9 @@ namespace molilian.core
                         .SaveAsync();
                 }
                 cancellationToken.ThrowIfCancellationRequested();
+                requestedAtUtc = DateTime.UtcNow;
+                requestSent = true;
                 var response = taobao.GeneralLinkParseRequest(content);
-                cancellationToken.ThrowIfCancellationRequested();
                 result.remote_response = response?.Body ?? string.Empty;
 
                 if (response == null)
@@ -114,12 +144,43 @@ namespace molilian.core
                 result.deeplink_url = GetDeeplink(url);
                 result.item_url = url;
                 result.item_deeplink_url = result.deeplink_url;
+                resultCompleted = true;
                 return result;
             }
+            catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+            {
+                throw;
+            }
+            catch (Exception ex)
+            {
+                return Fail($"{ex.GetType().Name}: {ex.Message}", TkSubCodeEnum.NetError);
+            }
             finally
             {
+                if (requestSent && resultCompleted)
+                {
+                    await TaobaoGeneralLinkParseCacheCore.SaveAsync(new TaobaoGeneralLinkParseCacheDTO
+                    {
+                        RequestContent = content,
+                        RequestedAtUtc = requestedAtUtc,
+                        ParseAccountId = parseAccountId,
+                        Success = result.success,
+                        Reason = result.reason,
+                        SubCode = result.subCode,
+                        LinkType = result.success ? result.link_type : LinkTypeEnum.unknown,
+                        ItemId = result.success ? result.itemId : string.Empty,
+                        ItemUrl = result.success ? result.item_url : string.Empty,
+                        ResponseBody = result.remote_response
+                    });
+                }
+
+                if (resultCompleted && !result.success && !cancellationToken.IsCancellationRequested)
+                {
+                    _ = TaobaoGeneralLinkParseCacheCore.NotifyFailureOnceAsync(result.reason, parseAccountId, _accountId);
+                }
                 stopwatch.Stop();
                 result.elapsedTime2 = (int)stopwatch.ElapsedMilliseconds;
+                cancellationToken.ThrowIfCancellationRequested();
             }
         }
     }