|
@@ -5,20 +5,28 @@ using System;
|
|
|
using System.Collections.Concurrent;
|
|
using System.Collections.Concurrent;
|
|
|
using System.Linq;
|
|
using System.Linq;
|
|
|
|
|
|
|
|
-public partial class TkEndpointManager
|
|
|
|
|
-{
|
|
|
|
|
- // 只保留轮询索引
|
|
|
|
|
- private static readonly ConcurrentDictionary<int, AccountEndpointState> accountStates = new();
|
|
|
|
|
-
|
|
|
|
|
- // 记录每个账号最后一次挂起操作的时间
|
|
|
|
|
- private static readonly ConcurrentDictionary<int, DateTime> lastSuspendTimes = new();
|
|
|
|
|
-
|
|
|
|
|
-
|
|
|
|
|
- public static void Refresh()
|
|
|
|
|
- {
|
|
|
|
|
- accountStates.Clear();
|
|
|
|
|
- lastSuspendTimes.Clear();
|
|
|
|
|
- }
|
|
|
|
|
|
|
+public partial class TkEndpointManager
|
|
|
|
|
+{
|
|
|
|
|
+ // 只保留轮询索引
|
|
|
|
|
+ private static readonly ConcurrentDictionary<int, AccountEndpointState> accountStates = new();
|
|
|
|
|
+
|
|
|
|
|
+ // 记录每个账号最后一次挂起操作的时间
|
|
|
|
|
+ private static readonly ConcurrentDictionary<int, DateTime> lastSuspendTimes = new();
|
|
|
|
|
+
|
|
|
|
|
+ // 短时缓存挂起状态,削峰同一波请求对 Redis Exists 的放大访问
|
|
|
|
|
+ private static readonly ConcurrentDictionary<string, SuspendCacheEntry> suspendStateCache = new();
|
|
|
|
|
+ private static readonly ConcurrentDictionary<string, byte> suspendStateRefreshInFlight = new();
|
|
|
|
|
+ private static readonly TimeSpan SuspendStateCacheDuration = TimeSpan.FromSeconds(10);
|
|
|
|
|
+ private static readonly TimeSpan SuspendStateFailureBackoffDuration = TimeSpan.FromSeconds(3);
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ public static void Refresh()
|
|
|
|
|
+ {
|
|
|
|
|
+ accountStates.Clear();
|
|
|
|
|
+ lastSuspendTimes.Clear();
|
|
|
|
|
+ suspendStateCache.Clear();
|
|
|
|
|
+ suspendStateRefreshInFlight.Clear();
|
|
|
|
|
+ }
|
|
|
|
|
|
|
|
|
|
|
|
|
// 从数据库获取所有可用的API端点
|
|
// 从数据库获取所有可用的API端点
|
|
@@ -45,11 +53,20 @@ public partial class TkEndpointManager
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
return dict;
|
|
return dict;
|
|
|
- }
|
|
|
|
|
- public static bool IsEndpointSuspended(int accountId, string endpoint)
|
|
|
|
|
- {
|
|
|
|
|
- return RedisHelper.Exists($"tk_suspend:{accountId}:{endpoint}");
|
|
|
|
|
- }
|
|
|
|
|
|
|
+ }
|
|
|
|
|
+ public static bool IsEndpointSuspended(int accountId, string endpoint)
|
|
|
|
|
+ {
|
|
|
|
|
+ if (string.IsNullOrWhiteSpace(endpoint)) return false;
|
|
|
|
|
+
|
|
|
|
|
+ string key = BuildSuspendKey(accountId, endpoint);
|
|
|
|
|
+ if (TryGetCachedSuspendState(key, out var isSuspended))
|
|
|
|
|
+ return isSuspended;
|
|
|
|
|
+
|
|
|
|
|
+ // 热路径不再同步访问 Redis。缓存未命中时先降级放行,再异步探测 Redis 状态。
|
|
|
|
|
+ _ = RefreshSuspendStateAsync(key);
|
|
|
|
|
+ CacheSuspendState(key, false, SuspendStateFailureBackoffDuration);
|
|
|
|
|
+ return false;
|
|
|
|
|
+ }
|
|
|
|
|
|
|
|
/// <summary>
|
|
/// <summary>
|
|
|
/// 检查指定账号是否至少拥有一个可用的endpoint
|
|
/// 检查指定账号是否至少拥有一个可用的endpoint
|
|
@@ -61,12 +78,13 @@ public partial class TkEndpointManager
|
|
|
var availableApis = await GetAllAvailableApisAsync(accountId);
|
|
var availableApis = await GetAllAvailableApisAsync(accountId);
|
|
|
if (availableApis.Count == 0) return false;
|
|
if (availableApis.Count == 0) return false;
|
|
|
|
|
|
|
|
- if (!accountStates.TryGetValue(accountId, out var state))
|
|
|
|
|
- {
|
|
|
|
|
- return true; // 如果账号没有状态记录,所有节点都是正常的
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
- return availableApis.Any(api => !IsEndpointSuspended(accountId, api.Value));
|
|
|
|
|
|
|
+ if (!accountStates.TryGetValue(accountId, out var state))
|
|
|
|
|
+ {
|
|
|
|
|
+ return true; // 如果账号没有状态记录,所有节点都是正常的
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ var suspendStates = GetSuspendStates(accountId, availableApis.Values);
|
|
|
|
|
+ return availableApis.Any(api => !suspendStates.GetValueOrDefault(api.Value));
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
/// <summary>
|
|
/// <summary>
|
|
@@ -96,24 +114,25 @@ public partial class TkEndpointManager
|
|
|
return status;
|
|
return status;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- if (!accountStates.TryGetValue(accountId, out var state))
|
|
|
|
|
- {
|
|
|
|
|
- // 如果账号没有状态记录,所有节点都是正常的
|
|
|
|
|
- foreach ((var api_id, var api) in availableApis)
|
|
|
|
|
- {
|
|
|
|
|
|
|
+ if (!accountStates.TryGetValue(accountId, out var state))
|
|
|
|
|
+ {
|
|
|
|
|
+ // 如果账号没有状态记录,所有节点都是正常的
|
|
|
|
|
+ foreach ((var api_id, var api) in availableApis)
|
|
|
|
|
+ {
|
|
|
status[api] = "正常";
|
|
status[api] = "正常";
|
|
|
- }
|
|
|
|
|
- return status;
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
- foreach ((var api_id, var api) in availableApis)
|
|
|
|
|
- {
|
|
|
|
|
- if (IsEndpointSuspended(accountId, api))
|
|
|
|
|
- {
|
|
|
|
|
- // Redis没有挂起到期时间,展示"挂起"即可
|
|
|
|
|
- status[api] = $"挂起";
|
|
|
|
|
- }
|
|
|
|
|
- else
|
|
|
|
|
|
|
+ }
|
|
|
|
|
+ return status;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ var suspendStates = GetSuspendStates(accountId, availableApis.Values);
|
|
|
|
|
+ foreach ((var api_id, var api) in availableApis)
|
|
|
|
|
+ {
|
|
|
|
|
+ if (suspendStates.GetValueOrDefault(api))
|
|
|
|
|
+ {
|
|
|
|
|
+ // Redis没有挂起到期时间,展示"挂起"即可
|
|
|
|
|
+ status[api] = $"挂起";
|
|
|
|
|
+ }
|
|
|
|
|
+ else
|
|
|
{
|
|
{
|
|
|
status[api] = "正常";
|
|
status[api] = "正常";
|
|
|
}
|
|
}
|
|
@@ -126,18 +145,19 @@ public partial class TkEndpointManager
|
|
|
var endpoints = await TkEndpointCore.GetEndpointsByAccountAsync(accountId, isTaobaoUrl, parseEndpoint);
|
|
var endpoints = await TkEndpointCore.GetEndpointsByAccountAsync(accountId, isTaobaoUrl, parseEndpoint);
|
|
|
if (endpoints == null || !endpoints.Any(e => e.status)) return (0, string.Empty);
|
|
if (endpoints == null || !endpoints.Any(e => e.status)) return (0, string.Empty);
|
|
|
|
|
|
|
|
- // 1. 过滤可用端点(状态正常)
|
|
|
|
|
- var availableApis = endpoints.Where(e => e.status);
|
|
|
|
|
-
|
|
|
|
|
- var state = accountStates.GetOrAdd(accountId, _ => new AccountEndpointState());
|
|
|
|
|
- var now = DateTime.Now;
|
|
|
|
|
-
|
|
|
|
|
- // 2. 先找出所有未被挂起、未达限制的端点
|
|
|
|
|
- var availableEndpoints = availableApis
|
|
|
|
|
- .Where(e => !IsEndpointSuspended(accountId, e.endpoint))
|
|
|
|
|
- .Where(e => e.hourly_calls_limit <= 0 || e.current_hourly_calls < e.hourly_calls_limit)
|
|
|
|
|
- .Where(e => e.daily_calls_limit <= 0 || e.current_daily_calls < e.daily_calls_limit)
|
|
|
|
|
- .ToArray();
|
|
|
|
|
|
|
+ // 1. 过滤可用端点(状态正常)
|
|
|
|
|
+ var availableApis = endpoints.Where(e => e.status);
|
|
|
|
|
+ var suspendStates = GetSuspendStates(accountId, availableApis.Select(e => e.endpoint));
|
|
|
|
|
+
|
|
|
|
|
+ var state = accountStates.GetOrAdd(accountId, _ => new AccountEndpointState());
|
|
|
|
|
+ var now = DateTime.Now;
|
|
|
|
|
+
|
|
|
|
|
+ // 2. 先找出所有未被挂起、未达限制的端点
|
|
|
|
|
+ var availableEndpoints = availableApis
|
|
|
|
|
+ .Where(e => !suspendStates.GetValueOrDefault(e.endpoint))
|
|
|
|
|
+ .Where(e => e.hourly_calls_limit <= 0 || e.current_hourly_calls < e.hourly_calls_limit)
|
|
|
|
|
+ .Where(e => e.daily_calls_limit <= 0 || e.current_daily_calls < e.daily_calls_limit)
|
|
|
|
|
+ .ToArray();
|
|
|
|
|
|
|
|
// 3. 如果存在可用端点,直接返回
|
|
// 3. 如果存在可用端点,直接返回
|
|
|
if (availableEndpoints.Length > 0)
|
|
if (availableEndpoints.Length > 0)
|
|
@@ -154,27 +174,27 @@ public partial class TkEndpointManager
|
|
|
return (selectedEndpoint.ep_id, selectedEndpoint.endpoint);
|
|
return (selectedEndpoint.ep_id, selectedEndpoint.endpoint);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- // 4. 检查是否仅因时间间隔限制
|
|
|
|
|
- var intervalLimitedEndpoints = availableApis
|
|
|
|
|
- .Where(e => !IsEndpointSuspended(accountId, e.endpoint))
|
|
|
|
|
- .Where(e => e.hourly_calls_limit <= 0 || e.current_hourly_calls < e.hourly_calls_limit)
|
|
|
|
|
- .Where(e => e.daily_calls_limit <= 0 || e.current_daily_calls < e.daily_calls_limit)
|
|
|
|
|
- .Where(e => e.interval_seconds > 0 &&
|
|
|
|
|
- e.last_call_time != null &&
|
|
|
|
|
- (now - e.last_call_time).TotalSeconds < e.interval_seconds)
|
|
|
|
|
|
|
+ // 4. 检查是否仅因时间间隔限制
|
|
|
|
|
+ var intervalLimitedEndpoints = availableApis
|
|
|
|
|
+ .Where(e => !suspendStates.GetValueOrDefault(e.endpoint))
|
|
|
|
|
+ .Where(e => e.hourly_calls_limit <= 0 || e.current_hourly_calls < e.hourly_calls_limit)
|
|
|
|
|
+ .Where(e => e.daily_calls_limit <= 0 || e.current_daily_calls < e.daily_calls_limit)
|
|
|
|
|
+ .Where(e => e.interval_seconds > 0 &&
|
|
|
|
|
+ e.last_call_time != null &&
|
|
|
|
|
+ (now - e.last_call_time).TotalSeconds < e.interval_seconds)
|
|
|
.ToArray();
|
|
.ToArray();
|
|
|
|
|
|
|
|
if (intervalLimitedEndpoints.Length > 0)
|
|
if (intervalLimitedEndpoints.Length > 0)
|
|
|
{
|
|
{
|
|
|
// 如果是时间间隔限制,返回空字符串
|
|
// 如果是时间间隔限制,返回空字符串
|
|
|
return (0, string.Empty);
|
|
return (0, string.Empty);
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
- // 5. 检查所有端点的状态
|
|
|
|
|
- var allEndpointsSuspended = availableApis.All(e => IsEndpointSuspended(accountId, e.endpoint));
|
|
|
|
|
- var allEndpointsLimited = availableApis.All(e =>
|
|
|
|
|
- (e.hourly_calls_limit > 0 && e.current_hourly_calls >= e.hourly_calls_limit) ||
|
|
|
|
|
- (e.daily_calls_limit > 0 && e.current_daily_calls >= e.daily_calls_limit));
|
|
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ // 5. 检查所有端点的状态
|
|
|
|
|
+ var allEndpointsSuspended = availableApis.All(e => suspendStates.GetValueOrDefault(e.endpoint));
|
|
|
|
|
+ var allEndpointsLimited = availableApis.All(e =>
|
|
|
|
|
+ (e.hourly_calls_limit > 0 && e.current_hourly_calls >= e.hourly_calls_limit) ||
|
|
|
|
|
+ (e.daily_calls_limit > 0 && e.current_daily_calls >= e.daily_calls_limit));
|
|
|
|
|
|
|
|
// 如果所有端点都被挂起或达到限制,返回 "ALL"
|
|
// 如果所有端点都被挂起或达到限制,返回 "ALL"
|
|
|
if (allEndpointsSuspended || allEndpointsLimited)
|
|
if (allEndpointsSuspended || allEndpointsLimited)
|
|
@@ -212,12 +232,12 @@ public partial class TkEndpointManager
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- public static async Task SuspendAsync(int accountId, string endpoint, SuspendReason reason = SuspendReason.Passive, int? customHoldMinutes = null)
|
|
|
|
|
- {
|
|
|
|
|
- var holdMinutes = await GetEndpointHoldMinutesAsync(accountId);
|
|
|
|
|
- TimeSpan suspendDuration;
|
|
|
|
|
- if (customHoldMinutes.HasValue)
|
|
|
|
|
- {
|
|
|
|
|
|
|
+ public static async Task SuspendAsync(int accountId, string endpoint, SuspendReason reason = SuspendReason.Passive, int? customHoldMinutes = null, bool notifyOtherNodes = true)
|
|
|
|
|
+ {
|
|
|
|
|
+ var holdMinutes = await GetEndpointHoldMinutesAsync(accountId);
|
|
|
|
|
+ TimeSpan suspendDuration;
|
|
|
|
|
+ if (customHoldMinutes.HasValue)
|
|
|
|
|
+ {
|
|
|
suspendDuration = TimeSpan.FromMinutes(customHoldMinutes.Value);
|
|
suspendDuration = TimeSpan.FromMinutes(customHoldMinutes.Value);
|
|
|
}
|
|
}
|
|
|
else
|
|
else
|
|
@@ -226,15 +246,37 @@ public partial class TkEndpointManager
|
|
|
{
|
|
{
|
|
|
SuspendReason.HourlyLimit => CalculateHourlySuspendDuration(),
|
|
SuspendReason.HourlyLimit => CalculateHourlySuspendDuration(),
|
|
|
SuspendReason.DailyLimit => CalculateDailySuspendDuration(),
|
|
SuspendReason.DailyLimit => CalculateDailySuspendDuration(),
|
|
|
- _ => TimeSpan.FromMinutes(holdMinutes.GetValueOrDefault(endpoint, 240))
|
|
|
|
|
- };
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
- RedisHelper.Set($"tk_suspend:{accountId}:{endpoint}", 1, (int)suspendDuration.TotalSeconds);
|
|
|
|
|
-
|
|
|
|
|
- _ = new LoggerLibrary("转链接口风控", accountId.ToString()).Info($"{accountId}_{endpoint}", reason.ToString()).SaveAsync();
|
|
|
|
|
- _ = NotifyCore.NotifyAsync($"【转链接口风控】{accountId}_{endpoint} ({reason})");
|
|
|
|
|
- }
|
|
|
|
|
|
|
+ _ => TimeSpan.FromMinutes(holdMinutes.GetValueOrDefault(endpoint, 240))
|
|
|
|
|
+ };
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ await SuspendForDurationAsync(accountId, endpoint, suspendDuration, reason, notifyOtherNodes);
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ public static async Task SuspendForDurationAsync(int accountId, string endpoint, TimeSpan suspendDuration, SuspendReason reason = SuspendReason.Passive, bool notifyOtherNodes = true)
|
|
|
|
|
+ {
|
|
|
|
|
+ if (string.IsNullOrWhiteSpace(endpoint)) return;
|
|
|
|
|
+
|
|
|
|
|
+ string key = BuildSuspendKey(accountId, endpoint);
|
|
|
|
|
+ CacheSuspendState(key, true, suspendDuration);
|
|
|
|
|
+
|
|
|
|
|
+ try
|
|
|
|
|
+ {
|
|
|
|
|
+ RedisHelper.Set(key, 1, (int)Math.Ceiling(suspendDuration.TotalSeconds));
|
|
|
|
|
+ }
|
|
|
|
|
+ catch (Exception)
|
|
|
|
|
+ {
|
|
|
|
|
+ // 本地状态已写入,Redis 持久化失败时只降级,不阻塞主流程
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ if (notifyOtherNodes)
|
|
|
|
|
+ {
|
|
|
|
|
+ _ = EndPointCore.NotifyChangeSuspend(accountId, endpoint, release: false, durationSeconds: (int)Math.Ceiling(suspendDuration.TotalSeconds));
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ _ = new LoggerLibrary("转链接口风控", accountId.ToString()).Info($"{accountId}_{endpoint}", reason.ToString()).SaveAsync();
|
|
|
|
|
+ _ = NotifyCore.NotifyAsync($"【转链接口风控】{accountId}_{endpoint} ({reason})");
|
|
|
|
|
+ }
|
|
|
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
/// <summary>
|
|
@@ -242,25 +284,52 @@ public partial class TkEndpointManager
|
|
|
/// </summary>
|
|
/// </summary>
|
|
|
/// <param name="accountId">账号ID</param>
|
|
/// <param name="accountId">账号ID</param>
|
|
|
/// <param name="endpoint">要释放的端点名称,如果为null则释放所有端点的挂起状态</param>
|
|
/// <param name="endpoint">要释放的端点名称,如果为null则释放所有端点的挂起状态</param>
|
|
|
- public static void ReleaseSuspend(int accountId, string endpoint = null)
|
|
|
|
|
- {
|
|
|
|
|
- if (endpoint == null)
|
|
|
|
|
- {
|
|
|
|
|
- var pattern = $"tk_suspend:{accountId}:*";
|
|
|
|
|
- var keys = RedisHelper.Keys(pattern);
|
|
|
|
|
- foreach (var key in keys)
|
|
|
|
|
- {
|
|
|
|
|
- RedisHelper.Del(key);
|
|
|
|
|
- }
|
|
|
|
|
- }
|
|
|
|
|
- else
|
|
|
|
|
- {
|
|
|
|
|
- RedisHelper.Del($"tk_suspend:{accountId}:{endpoint}");
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
- var action = endpoint == null ? "释放所有挂起" : $"释放挂起({endpoint})";
|
|
|
|
|
- _ = new LoggerLibrary("转链接口风控", accountId.ToString()).Info($"{accountId}", action).SaveAsync();
|
|
|
|
|
- _ = NotifyCore.NotifyAsync($"【转链接口风控】{accountId} {action}");
|
|
|
|
|
|
|
+ public static void ReleaseSuspend(int accountId, string endpoint = null, bool notifyOtherNodes = true)
|
|
|
|
|
+ {
|
|
|
|
|
+ if (endpoint == null)
|
|
|
|
|
+ {
|
|
|
|
|
+ foreach (var key in suspendStateCache.Keys.Where(key => key.StartsWith($"tk_suspend:{accountId}:", StringComparison.Ordinal)))
|
|
|
|
|
+ {
|
|
|
|
|
+ suspendStateCache.TryRemove(key, out _);
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ try
|
|
|
|
|
+ {
|
|
|
|
|
+ var pattern = $"tk_suspend:{accountId}:*";
|
|
|
|
|
+ var keys = RedisHelper.Keys(pattern);
|
|
|
|
|
+ foreach (var key in keys)
|
|
|
|
|
+ {
|
|
|
|
|
+ RedisHelper.Del(key);
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ catch (Exception)
|
|
|
|
|
+ {
|
|
|
|
|
+ // Redis 不可用时只清理本地状态
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ else
|
|
|
|
|
+ {
|
|
|
|
|
+ string key = BuildSuspendKey(accountId, endpoint);
|
|
|
|
|
+ CacheSuspendState(key, false, SuspendStateCacheDuration);
|
|
|
|
|
+
|
|
|
|
|
+ try
|
|
|
|
|
+ {
|
|
|
|
|
+ RedisHelper.Del(key);
|
|
|
|
|
+ }
|
|
|
|
|
+ catch (Exception)
|
|
|
|
|
+ {
|
|
|
|
|
+ // Redis 不可用时只清理本地状态
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ if (notifyOtherNodes)
|
|
|
|
|
+ {
|
|
|
|
|
+ _ = EndPointCore.NotifyChangeSuspend(accountId, endpoint, release: true);
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ var action = endpoint == null ? "释放所有挂起" : $"释放挂起({endpoint})";
|
|
|
|
|
+ _ = new LoggerLibrary("转链接口风控", accountId.ToString()).Info($"{accountId}", action).SaveAsync();
|
|
|
|
|
+ _ = NotifyCore.NotifyAsync($"【转链接口风控】{accountId} {action}");
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
@@ -301,13 +370,80 @@ public partial class TkEndpointManager
|
|
|
DailyLimit
|
|
DailyLimit
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- private class AccountEndpointState
|
|
|
|
|
- {
|
|
|
|
|
- // 只保留轮询索引
|
|
|
|
|
- private readonly ConcurrentDictionary<string, int> roundRobinIndices = new();
|
|
|
|
|
- public int GetOrAddAccountIndex(string key) => roundRobinIndices.GetOrAdd(key, -1);
|
|
|
|
|
- public void UpdateAccountIndex(string key, int newIndex) => roundRobinIndices[key] = newIndex;
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
-
|
|
|
|
|
-}
|
|
|
|
|
|
|
+ private class AccountEndpointState
|
|
|
|
|
+ {
|
|
|
|
|
+ // 只保留轮询索引
|
|
|
|
|
+ private readonly ConcurrentDictionary<string, int> roundRobinIndices = new();
|
|
|
|
|
+ public int GetOrAddAccountIndex(string key) => roundRobinIndices.GetOrAdd(key, -1);
|
|
|
|
|
+ public void UpdateAccountIndex(string key, int newIndex) => roundRobinIndices[key] = newIndex;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ private static string BuildSuspendKey(int accountId, string endpoint) => $"tk_suspend:{accountId}:{endpoint}";
|
|
|
|
|
+
|
|
|
|
|
+ private static Dictionary<string, bool> GetSuspendStates(int accountId, IEnumerable<string> endpoints)
|
|
|
|
|
+ {
|
|
|
|
|
+ var result = new Dictionary<string, bool>(StringComparer.Ordinal);
|
|
|
|
|
+ foreach (var endpoint in endpoints.Where(e => !string.IsNullOrWhiteSpace(e)).Distinct(StringComparer.Ordinal))
|
|
|
|
|
+ {
|
|
|
|
|
+ result[endpoint] = IsEndpointSuspended(accountId, endpoint);
|
|
|
|
|
+ }
|
|
|
|
|
+ return result;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ private static bool TryGetCachedSuspendState(string key, out bool isSuspended)
|
|
|
|
|
+ {
|
|
|
|
|
+ if (suspendStateCache.TryGetValue(key, out var entry))
|
|
|
|
|
+ {
|
|
|
|
|
+ if (entry.ExpiresAtUtc > DateTime.UtcNow)
|
|
|
|
|
+ {
|
|
|
|
|
+ isSuspended = entry.IsSuspended;
|
|
|
|
|
+ return true;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ suspendStateCache.TryRemove(key, out _);
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ isSuspended = false;
|
|
|
|
|
+ return false;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ private static void CacheSuspendState(string key, bool isSuspended, TimeSpan duration)
|
|
|
|
|
+ {
|
|
|
|
|
+ if (duration <= TimeSpan.Zero)
|
|
|
|
|
+ {
|
|
|
|
|
+ suspendStateCache.TryRemove(key, out _);
|
|
|
|
|
+ return;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ suspendStateCache[key] = new SuspendCacheEntry(isSuspended, DateTime.UtcNow.Add(duration));
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ private static Task RefreshSuspendStateAsync(string key)
|
|
|
|
|
+ {
|
|
|
|
|
+ if (!suspendStateRefreshInFlight.TryAdd(key, 0))
|
|
|
|
|
+ return Task.CompletedTask;
|
|
|
|
|
+
|
|
|
|
|
+ return Task.Run(() =>
|
|
|
|
|
+ {
|
|
|
|
|
+ try
|
|
|
|
|
+ {
|
|
|
|
|
+ bool isSuspended = RedisHelper.Exists(key);
|
|
|
|
|
+ CacheSuspendState(
|
|
|
|
|
+ key,
|
|
|
|
|
+ isSuspended,
|
|
|
|
|
+ isSuspended ? SuspendStateCacheDuration : SuspendStateFailureBackoffDuration);
|
|
|
|
|
+ }
|
|
|
|
|
+ catch (Exception)
|
|
|
|
|
+ {
|
|
|
|
|
+ CacheSuspendState(key, false, SuspendStateFailureBackoffDuration);
|
|
|
|
|
+ }
|
|
|
|
|
+ finally
|
|
|
|
|
+ {
|
|
|
|
|
+ suspendStateRefreshInFlight.TryRemove(key, out _);
|
|
|
|
|
+ }
|
|
|
|
|
+ });
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ private readonly record struct SuspendCacheEntry(bool IsSuspended, DateTime ExpiresAtUtc);
|
|
|
|
|
+
|
|
|
|
|
+}
|