TracksParseMetricCore.cs 37 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831
  1. using System.Collections.Concurrent;
  2. using System.Data;
  3. using System.Threading;
  4. using Dapper;
  5. using dodohold.core;
  6. using YunhuiKit;
  7. namespace molilian.core
  8. {
  9. public partial class TracksCore
  10. {
  11. private const string ParseMetricTypeName = "转链";
  12. private const string ParseMetricRequest = "request";
  13. private const string ParseMetricCall = "call";
  14. private const string ParseMetricSuccess = "success";
  15. private const int ParseMetricFlushIntervalMs = 1000;
  16. private static readonly ConcurrentDictionary<ParseMetricCounterKey, ParseMetricCounter> PendingParseMetricCounters = new();
  17. private static readonly object ParseMetricFlushTimerLock = new();
  18. private static Timer? ParseMetricFlushTimer;
  19. private static int ParseMetricFlushRunning;
  20. private static int ParseMetricLifecycleHooked;
  21. public static Task<bool> RecordParseRequestAsync(UnionParseRequest request)
  22. {
  23. request.ParseRequestRecorded = true;
  24. var dimension = ResolveParseMetricDimension(request.Channel, request.RiskStrategy, request.LaunchScene);
  25. return IncrementParseMetricAsync(ParseMetricRequest, dimension.Platform, dimension.RiskStrategy, dimension.LaunchScene, request.AccountId);
  26. }
  27. /// <summary>
  28. /// 命中隐藏号:回退入口处已计入的请求数,隐藏号请求不出现在任何统计中
  29. /// </summary>
  30. public static Task<bool> RevertParseRequestAsync(UnionParseRequest request)
  31. {
  32. if (!request.ParseRequestRecorded) return Task.FromResult(false);
  33. request.ParseRequestRecorded = false;
  34. var dimension = ResolveParseMetricDimension(request.Channel, request.RiskStrategy, request.LaunchScene);
  35. return IncrementParseMetricAsync(ParseMetricRequest, dimension.Platform, dimension.RiskStrategy, dimension.LaunchScene, request.AccountId, -1);
  36. }
  37. public static Task<bool> RecordParseAccountRequestAsync(string channel, string riskStrategy, int launchScene, int accountId)
  38. {
  39. if (accountId <= 0) return Task.FromResult(false);
  40. var dimension = ResolveParseMetricDimension(channel, riskStrategy, launchScene);
  41. return IncrementParseAccountMetricAsync(ParseMetricRequest, dimension.Platform, dimension.RiskStrategy, dimension.LaunchScene, accountId);
  42. }
  43. public static Task<bool> RecordParseResultAsync(string channel, string riskStrategy, int launchScene, int accountId, bool success)
  44. {
  45. var dimension = ResolveParseMetricDimension(channel, riskStrategy, launchScene);
  46. return IncrementParseResultMetricAsync(dimension.Platform, dimension.RiskStrategy, dimension.LaunchScene, accountId, success);
  47. }
  48. /// <summary>
  49. /// RecordParseResultAsync 写入的 success 计数 key(daily、hour),供历史数据修复使用
  50. /// </summary>
  51. public static string[] BuildParseMetricSuccessKeys(DateTime metricTime, string channel, string riskStrategy, int launchScene)
  52. {
  53. var dimension = ResolveParseMetricDimension(channel, riskStrategy, launchScene);
  54. return
  55. [
  56. BuildParseMetricCountKey(ParseMetricSuccess, "daily", metricTime.ToString("yyyyMMdd"), dimension.Platform, dimension.RiskStrategy, dimension.LaunchScene),
  57. BuildParseMetricCountKey(ParseMetricSuccess, "hour", metricTime.ToString("yyyyMMddHH"), dimension.Platform, dimension.RiskStrategy, dimension.LaunchScene),
  58. ];
  59. }
  60. public static async Task<TrackParseMetricReportResult> GetParseMetricReportAsync(
  61. DateTime start,
  62. DateTime end,
  63. int page,
  64. int size,
  65. bool getTotal,
  66. int accountId = -1,
  67. bool accountBreakdownOnly = false,
  68. string platform = "",
  69. string riskStrategy = "",
  70. int? launchScene = null)
  71. {
  72. start = start.Date;
  73. end = end.Date;
  74. if (end < start) end = start;
  75. page = Math.Max(page, 1);
  76. size = Math.Max(size, 1);
  77. var list = new List<TrackParseMetricReportDTO>();
  78. for (var date = start; date <= end; date = date.AddDays(1))
  79. {
  80. string dateStr = date.ToString("yyyyMMdd");
  81. string indexKey = BuildParseMetricIndexKey("daily", dateStr);
  82. string[] members = await TkLogCore.GetTotalKeysAsync(indexKey) ?? [];
  83. foreach (var member in members)
  84. {
  85. if (!TryParseParseMetricIndexValue(member, out var dimension)) continue;
  86. if (!IsMatchedParseMetricDimension(dimension, accountId, accountBreakdownOnly, platform, riskStrategy, launchScene)) continue;
  87. var requestTask = TkLogCore.GetTotalAsync(BuildParseMetricCountKey(ParseMetricRequest, "daily", dateStr, dimension.Platform, dimension.RiskStrategy, dimension.LaunchScene, dimension.AccountId));
  88. var callTask = TkLogCore.GetTotalAsync(BuildParseMetricCountKey(ParseMetricCall, "daily", dateStr, dimension.Platform, dimension.RiskStrategy, dimension.LaunchScene, dimension.AccountId));
  89. var successTask = TkLogCore.GetTotalAsync(BuildParseMetricCountKey(ParseMetricSuccess, "daily", dateStr, dimension.Platform, dimension.RiskStrategy, dimension.LaunchScene, dimension.AccountId));
  90. await Task.WhenAll(requestTask, callTask, successTask);
  91. int requestCount = requestTask.Result;
  92. int callCount = callTask.Result;
  93. int successCount = successTask.Result;
  94. if (requestCount <= 0 && callCount <= 0 && successCount <= 0) continue;
  95. list.Add(new TrackParseMetricReportDTO
  96. {
  97. report_date = date,
  98. platform = dimension.Platform,
  99. typename = ParseMetricTypeName,
  100. risk_strategy = dimension.RiskStrategy,
  101. launch_scene = dimension.LaunchScene,
  102. account_id = dimension.AccountId,
  103. request_count = requestCount,
  104. call_count = callCount,
  105. success_count = successCount
  106. });
  107. }
  108. }
  109. list = list
  110. .OrderByDescending(item => item.report_date)
  111. .ThenBy(item => item.platform)
  112. .ThenBy(item => item.risk_strategy)
  113. .ThenBy(item => item.launch_scene)
  114. .ThenBy(item => item.account_id)
  115. .ToList();
  116. int count = list.Count;
  117. list = list
  118. .Skip((page - 1) * size)
  119. .Take(size)
  120. .ToList();
  121. return new TrackParseMetricReportResult
  122. {
  123. list = list,
  124. count = getTotal ? count : list.Count
  125. };
  126. }
  127. public static async Task<List<TrackParseMetricHourlyDTO>> GetParseMetricHourlyReportAsync(
  128. DateTime targetDate,
  129. string platform,
  130. string riskStrategy,
  131. int? launchScene = null,
  132. int accountId = 0)
  133. {
  134. targetDate = targetDate.Date;
  135. platform = NormalizeParseMetricPlatform(platform);
  136. riskStrategy = NormalizeReportScene(riskStrategy);
  137. accountId = Math.Max(accountId, 0);
  138. var list = new List<TrackParseMetricHourlyDTO>();
  139. string dateStr = targetDate.ToString("yyyyMMdd");
  140. for (int hour = 0; hour < 24; hour++)
  141. {
  142. string hourStr = $"{dateStr}{hour:00}";
  143. string indexKey = BuildParseMetricIndexKey("hour", hourStr);
  144. string[] members = await TkLogCore.GetTotalKeysAsync(indexKey) ?? [];
  145. long requestCount = 0;
  146. long callCount = 0;
  147. long successCount = 0;
  148. foreach (var member in members)
  149. {
  150. if (!TryParseParseMetricIndexValue(member, out var dimension)) continue;
  151. if (!IsMatchedParseMetricDimension(dimension, accountId, false, platform, riskStrategy, launchScene)) continue;
  152. var requestTask = TkLogCore.GetTotalAsync(BuildParseMetricCountKey(ParseMetricRequest, "hour", hourStr, dimension.Platform, dimension.RiskStrategy, dimension.LaunchScene, dimension.AccountId));
  153. var callTask = TkLogCore.GetTotalAsync(BuildParseMetricCountKey(ParseMetricCall, "hour", hourStr, dimension.Platform, dimension.RiskStrategy, dimension.LaunchScene, dimension.AccountId));
  154. var successTask = TkLogCore.GetTotalAsync(BuildParseMetricCountKey(ParseMetricSuccess, "hour", hourStr, dimension.Platform, dimension.RiskStrategy, dimension.LaunchScene, dimension.AccountId));
  155. await Task.WhenAll(requestTask, callTask, successTask);
  156. requestCount += requestTask.Result;
  157. callCount += callTask.Result;
  158. successCount += successTask.Result;
  159. }
  160. list.Add(new TrackParseMetricHourlyDTO
  161. {
  162. report_date = targetDate,
  163. platform = platform,
  164. typename = ParseMetricTypeName,
  165. risk_strategy = riskStrategy,
  166. launch_scene = launchScene ?? -1,
  167. account_id = accountId,
  168. hour = hour,
  169. request_count = SafeInt(requestCount),
  170. call_count = SafeInt(callCount),
  171. success_count = SafeInt(successCount)
  172. });
  173. }
  174. return list;
  175. }
  176. public static async Task<TrackParseMetricBackfillResult> BackfillParseMetricCountersAsync(
  177. DateTime targetDate,
  178. string platform = "",
  179. string riskStrategy = "",
  180. int? launchScene = null)
  181. {
  182. targetDate = targetDate.Date;
  183. platform = NormalizeParseMetricPlatform(platform);
  184. riskStrategy = NormalizeReportScene(riskStrategy);
  185. bool hasPlatformFilter = !string.IsNullOrWhiteSpace(platform);
  186. bool hasRiskStrategyFilter = !string.IsNullOrWhiteSpace(riskStrategy);
  187. bool hasLaunchSceneFilter = launchScene.HasValue;
  188. bool hasDimensionFilter = hasPlatformFilter || hasRiskStrategyFilter || hasLaunchSceneFilter;
  189. var result = new TrackParseMetricBackfillResult
  190. {
  191. report_date = targetDate.ToString("yyyy-MM-dd")
  192. };
  193. var allSources = new[]
  194. {
  195. new ParseMetricBackfillSource("tb", $"tk_parse_logs_{targetDate:yyyyMMdd}"),
  196. new ParseMetricBackfillSource("jd", $"jd_parse_logs_{targetDate:yyyyMMdd}"),
  197. new ParseMetricBackfillSource("pdd", $"pdd_parse_logs_{targetDate:yyyyMMdd}")
  198. };
  199. var sources = hasDimensionFilter
  200. ? allSources.Where(source => !hasPlatformFilter || string.Equals(source.Platform, platform, StringComparison.OrdinalIgnoreCase)).ToArray()
  201. : allSources;
  202. using var conn = DBContext.GetOpenConnection();
  203. DateTime start = targetDate;
  204. DateTime end = targetDate.AddDays(1);
  205. result.redis_cleared_keys = hasDimensionFilter
  206. ? await ClearParseMetricCountersForDimensionAsync(targetDate, platform, riskStrategy, launchScene)
  207. : await ClearParseMetricCountersForDateAsync(targetDate);
  208. foreach (var source in sources)
  209. {
  210. if (!TableExists(conn, source.TableName))
  211. {
  212. result.details.Add(new { platform = source.Platform, table = source.TableName, exists = false, rows = 0, dimensions = 0 });
  213. continue;
  214. }
  215. const string riskStrategySql = "COALESCE(NULLIF(TRIM(riskStrategy), ''), '')";
  216. const string rawLaunchSceneSql = "COALESCE(launchScene, -1)";
  217. string launchSceneSql = $"CASE WHEN LOWER({riskStrategySql}) IN ('os','tbpush','brwsimilar','icon') AND {rawLaunchSceneSql}=-1 THEN 0 ELSE {rawLaunchSceneSql} END";
  218. var dimensionWhereItems = new List<string>();
  219. if (hasRiskStrategyFilter) dimensionWhereItems.Add($"{riskStrategySql}=@riskStrategy");
  220. if (hasLaunchSceneFilter) dimensionWhereItems.Add($"{launchSceneSql}=@launchScene");
  221. string dimensionWhere = dimensionWhereItems.Count > 0
  222. ? "\n AND " + string.Join("\n AND ", dimensionWhereItems)
  223. : string.Empty;
  224. string sql = $@"
  225. SELECT
  226. {riskStrategySql} AS risk_strategy,
  227. {launchSceneSql} AS launch_scene,
  228. COALESCE(accountId, 0) AS account_id,
  229. DATE_FORMAT(create_time, '%Y%m%d%H') AS hour_key,
  230. CAST(COUNT(1) AS SIGNED) AS call_count,
  231. CAST(SUM(CASE WHEN success=1 THEN 1 ELSE 0 END) AS SIGNED) AS success_count
  232. FROM `{source.TableName}`
  233. WHERE create_time>=@start AND create_time<@end
  234. {dimensionWhere}
  235. GROUP BY
  236. {riskStrategySql},
  237. {launchSceneSql},
  238. COALESCE(accountId, 0),
  239. DATE_FORMAT(create_time, '%Y%m%d%H')";
  240. var args = new DynamicParameters();
  241. args.Add("start", start);
  242. args.Add("end", end);
  243. if (hasRiskStrategyFilter) args.Add("riskStrategy", riskStrategy);
  244. if (hasLaunchSceneFilter) args.Add("launchScene", launchScene!.Value);
  245. var rows = SqlMapper.Query<ParseMetricBackfillRow>(conn, sql, args).ToList();
  246. int mysqlRows = rows.Sum(row => SafeInt(row.call_count));
  247. int dimensions = ApplyParseMetricBackfillRows(source.Platform, targetDate, rows);
  248. result.mysql_rows += mysqlRows;
  249. result.redis_dimensions += dimensions;
  250. result.details.Add(new { platform = source.Platform, table = source.TableName, exists = true, rows = mysqlRows, dimensions });
  251. }
  252. await Task.CompletedTask;
  253. return result;
  254. }
  255. private static Task<bool> IncrementParseMetricAsync(string metric, string platform, string riskStrategy, int launchScene, int accountId = 0, int count = 1)
  256. {
  257. try
  258. {
  259. AddParseMetricCount(metric, DateTime.Now, platform, riskStrategy, launchScene, 0, count);
  260. if (accountId > 0)
  261. {
  262. AddParseMetricCount(metric, DateTime.Now, platform, riskStrategy, launchScene, accountId, count);
  263. }
  264. return Task.FromResult(true);
  265. }
  266. catch (Exception ex)
  267. {
  268. _ = new LoggerLibrary("TracksCore", "RecordParseMetric")
  269. .Info(ex.Message, ex.StackTrace)
  270. .SaveAsync();
  271. return Task.FromResult(false);
  272. }
  273. }
  274. private static Task<bool> IncrementParseAccountMetricAsync(string metric, string platform, string riskStrategy, int launchScene, int accountId)
  275. {
  276. try
  277. {
  278. accountId = Math.Max(accountId, 0);
  279. if (accountId <= 0) return Task.FromResult(false);
  280. AddParseMetricCount(metric, DateTime.Now, platform, riskStrategy, launchScene, accountId, 1);
  281. return Task.FromResult(true);
  282. }
  283. catch (Exception ex)
  284. {
  285. _ = new LoggerLibrary("TracksCore", "RecordParseAccountMetric")
  286. .Info(ex.Message, ex.StackTrace)
  287. .SaveAsync();
  288. return Task.FromResult(false);
  289. }
  290. }
  291. private static Task<bool> IncrementParseResultMetricAsync(string platform, string riskStrategy, int launchScene, int accountId, bool success)
  292. {
  293. try
  294. {
  295. accountId = Math.Max(accountId, 0);
  296. AddParseMetricCount(ParseMetricCall, DateTime.Now, platform, riskStrategy, launchScene, 0, 1);
  297. if (success) AddParseMetricCount(ParseMetricSuccess, DateTime.Now, platform, riskStrategy, launchScene, 0, 1);
  298. if (accountId > 0)
  299. {
  300. AddParseMetricCount(ParseMetricCall, DateTime.Now, platform, riskStrategy, launchScene, accountId, 1);
  301. if (success) AddParseMetricCount(ParseMetricSuccess, DateTime.Now, platform, riskStrategy, launchScene, accountId, 1);
  302. }
  303. return Task.FromResult(true);
  304. }
  305. catch (Exception ex)
  306. {
  307. _ = new LoggerLibrary("TracksCore", "RecordParseResultMetric")
  308. .Info(ex.Message, ex.StackTrace)
  309. .SaveAsync();
  310. return Task.FromResult(false);
  311. }
  312. }
  313. private static int ApplyParseMetricBackfillRows(string platform, DateTime targetDate, List<ParseMetricBackfillRow> rows)
  314. {
  315. string dateStr = targetDate.ToString("yyyyMMdd");
  316. int dimensions = 0;
  317. var dailyGroups = rows.GroupBy(row => new
  318. {
  319. RiskStrategy = NormalizeReportScene(row.risk_strategy),
  320. LaunchScene = NormalizeParseMetricLaunchScene(row.risk_strategy, row.launch_scene)
  321. });
  322. foreach (var group in dailyGroups)
  323. {
  324. int callCount = group.Sum(row => SafeInt(row.call_count));
  325. int successCount = group.Sum(row => SafeInt(row.success_count));
  326. SetParseMetricCount(ParseMetricCall, "daily", dateStr, platform, group.Key.RiskStrategy, group.Key.LaunchScene, 0, callCount);
  327. SetParseMetricCount(ParseMetricSuccess, "daily", dateStr, platform, group.Key.RiskStrategy, group.Key.LaunchScene, 0, successCount);
  328. dimensions++;
  329. }
  330. var dailyAccountGroups = rows
  331. .Where(row => row.account_id > 0)
  332. .GroupBy(row => new
  333. {
  334. RiskStrategy = NormalizeReportScene(row.risk_strategy),
  335. launch_scene = NormalizeParseMetricLaunchScene(row.risk_strategy, row.launch_scene),
  336. row.account_id
  337. });
  338. foreach (var group in dailyAccountGroups)
  339. {
  340. int callCount = group.Sum(row => SafeInt(row.call_count));
  341. int successCount = group.Sum(row => SafeInt(row.success_count));
  342. SetParseMetricCount(ParseMetricRequest, "daily", dateStr, platform, group.Key.RiskStrategy, group.Key.launch_scene, group.Key.account_id, callCount);
  343. SetParseMetricCount(ParseMetricCall, "daily", dateStr, platform, group.Key.RiskStrategy, group.Key.launch_scene, group.Key.account_id, callCount);
  344. SetParseMetricCount(ParseMetricSuccess, "daily", dateStr, platform, group.Key.RiskStrategy, group.Key.launch_scene, group.Key.account_id, successCount);
  345. dimensions++;
  346. }
  347. var hourlyGroups = rows.GroupBy(row => new
  348. {
  349. RiskStrategy = NormalizeReportScene(row.risk_strategy),
  350. launch_scene = NormalizeParseMetricLaunchScene(row.risk_strategy, row.launch_scene),
  351. row.hour_key
  352. });
  353. foreach (var group in hourlyGroups)
  354. {
  355. int callCount = group.Sum(row => SafeInt(row.call_count));
  356. int successCount = group.Sum(row => SafeInt(row.success_count));
  357. SetParseMetricCount(ParseMetricCall, "hour", group.Key.hour_key, platform, group.Key.RiskStrategy, group.Key.launch_scene, 0, callCount);
  358. SetParseMetricCount(ParseMetricSuccess, "hour", group.Key.hour_key, platform, group.Key.RiskStrategy, group.Key.launch_scene, 0, successCount);
  359. }
  360. var hourlyAccountGroups = rows
  361. .Where(row => row.account_id > 0)
  362. .GroupBy(row => new
  363. {
  364. RiskStrategy = NormalizeReportScene(row.risk_strategy),
  365. launch_scene = NormalizeParseMetricLaunchScene(row.risk_strategy, row.launch_scene),
  366. row.account_id,
  367. row.hour_key
  368. });
  369. foreach (var group in hourlyAccountGroups)
  370. {
  371. int callCount = group.Sum(row => SafeInt(row.call_count));
  372. int successCount = group.Sum(row => SafeInt(row.success_count));
  373. SetParseMetricCount(ParseMetricRequest, "hour", group.Key.hour_key, platform, group.Key.RiskStrategy, group.Key.launch_scene, group.Key.account_id, callCount);
  374. SetParseMetricCount(ParseMetricCall, "hour", group.Key.hour_key, platform, group.Key.RiskStrategy, group.Key.launch_scene, group.Key.account_id, callCount);
  375. SetParseMetricCount(ParseMetricSuccess, "hour", group.Key.hour_key, platform, group.Key.RiskStrategy, group.Key.launch_scene, group.Key.account_id, successCount);
  376. }
  377. return dimensions;
  378. }
  379. private static void AddParseMetricCount(string metric, DateTime metricTime, string platform, string riskStrategy, int launchScene, int accountId, int count)
  380. {
  381. // count 为负数时用于回退(隐藏号)
  382. if (count == 0) return;
  383. QueueParseMetricCount(metric, "daily", metricTime.ToString("yyyyMMdd"), platform, riskStrategy, launchScene, accountId, count);
  384. QueueParseMetricCount(metric, "hour", metricTime.ToString("yyyyMMddHH"), platform, riskStrategy, launchScene, accountId, count);
  385. }
  386. private static void SetParseMetricCount(string metric, string bucketType, string bucketValue, string platform, string riskStrategy, int launchScene, int accountId, int count)
  387. {
  388. if (count < 0) return;
  389. SetOrIncrementParseMetricCount(metric, bucketType, bucketValue, platform, riskStrategy, launchScene, accountId, count, increment: false);
  390. }
  391. private static void SetOrIncrementParseMetricCount(string metric, string bucketType, string bucketValue, string platform, string riskStrategy, int launchScene, int accountId, long count, bool increment)
  392. {
  393. platform = Normalize(platform).ToLowerInvariant();
  394. riskStrategy = NormalizeReportScene(riskStrategy);
  395. launchScene = NormalizeParseMetricLaunchScene(riskStrategy, launchScene);
  396. accountId = Math.Max(accountId, 0);
  397. string countKey = BuildParseMetricCountKey(metric, bucketType, bucketValue, platform, riskStrategy, launchScene, accountId);
  398. string indexKey = BuildParseMetricIndexKey(bucketType, bucketValue);
  399. string indexValue = BuildParseMetricIndexValue(platform, riskStrategy, launchScene, accountId);
  400. int expireSeconds = bucketType == "hour" ? HourlyExpireSeconds : DailyExpireSeconds;
  401. if (increment)
  402. {
  403. IncrementParseMetricRedisKey(countKey, count);
  404. }
  405. else
  406. {
  407. RedisHelper.Set(countKey, count, expireSeconds);
  408. }
  409. RedisHelper.Expire(countKey, expireSeconds);
  410. RedisHelper.SAdd(indexKey, indexValue);
  411. RedisHelper.Expire(indexKey, expireSeconds);
  412. }
  413. public static int FlushParseMetricCounters()
  414. {
  415. if (Interlocked.Exchange(ref ParseMetricFlushRunning, 1) == 1) return 0;
  416. try
  417. {
  418. int flushed = 0;
  419. foreach (var item in PendingParseMetricCounters.ToArray())
  420. {
  421. long count = Interlocked.Exchange(ref item.Value.Count, 0);
  422. if (count == 0) continue;
  423. try
  424. {
  425. SetOrIncrementParseMetricCount(
  426. item.Key.Metric,
  427. item.Key.BucketType,
  428. item.Key.BucketValue,
  429. item.Key.Platform,
  430. item.Key.RiskStrategy,
  431. item.Key.LaunchScene,
  432. item.Key.AccountId,
  433. count,
  434. increment: true);
  435. flushed++;
  436. }
  437. catch (Exception ex)
  438. {
  439. Interlocked.Add(ref item.Value.Count, count);
  440. _ = new LoggerLibrary("TracksCore", "FlushParseMetricCounters")
  441. .Info(ex.Message, ex.StackTrace)
  442. .SaveAsync();
  443. }
  444. }
  445. return flushed;
  446. }
  447. finally
  448. {
  449. Interlocked.Exchange(ref ParseMetricFlushRunning, 0);
  450. }
  451. }
  452. private static void QueueParseMetricCount(string metric, string bucketType, string bucketValue, string platform, string riskStrategy, int launchScene, int accountId, int count)
  453. {
  454. if (count == 0) return;
  455. var key = new ParseMetricCounterKey(
  456. metric,
  457. bucketType,
  458. bucketValue,
  459. Normalize(platform).ToLowerInvariant(),
  460. NormalizeReportScene(riskStrategy),
  461. NormalizeParseMetricLaunchScene(riskStrategy, launchScene),
  462. Math.Max(accountId, 0));
  463. var counter = PendingParseMetricCounters.GetOrAdd(key, _ => new ParseMetricCounter());
  464. Interlocked.Add(ref counter.Count, count);
  465. EnsureParseMetricFlushTimer();
  466. }
  467. private static void EnsureParseMetricFlushTimer()
  468. {
  469. if (ParseMetricFlushTimer != null) return;
  470. lock (ParseMetricFlushTimerLock)
  471. {
  472. if (ParseMetricFlushTimer != null) return;
  473. ParseMetricFlushTimer = new Timer(
  474. _ => FlushParseMetricCounters(),
  475. null,
  476. ParseMetricFlushIntervalMs,
  477. ParseMetricFlushIntervalMs);
  478. if (Interlocked.Exchange(ref ParseMetricLifecycleHooked, 1) == 0)
  479. {
  480. AppDomain.CurrentDomain.ProcessExit += (_, _) => FlushParseMetricCounters();
  481. AppDomain.CurrentDomain.UnhandledException += (_, _) => FlushParseMetricCounters();
  482. }
  483. }
  484. }
  485. private static void IncrementParseMetricRedisKey(string key, long count)
  486. {
  487. while (count > 0)
  488. {
  489. int delta = count > int.MaxValue ? int.MaxValue : (int)count;
  490. RedisHelper.IncrBy(key, delta);
  491. count -= delta;
  492. }
  493. while (count < 0)
  494. {
  495. int delta = count < int.MinValue ? int.MinValue : (int)count;
  496. RedisHelper.IncrBy(key, delta);
  497. count -= delta;
  498. }
  499. }
  500. private static async Task<int> ClearParseMetricCountersForDateAsync(DateTime targetDate)
  501. {
  502. string dateStr = targetDate.ToString("yyyyMMdd");
  503. int cleared = await ClearParseMetricBucketAsync("daily", dateStr);
  504. for (int hour = 0; hour < 24; hour++)
  505. {
  506. cleared += await ClearParseMetricBucketAsync("hour", $"{dateStr}{hour:00}");
  507. }
  508. return cleared;
  509. }
  510. private static async Task<int> ClearParseMetricCountersForDimensionAsync(
  511. DateTime targetDate,
  512. string platform,
  513. string riskStrategy,
  514. int? launchScene)
  515. {
  516. string dateStr = targetDate.ToString("yyyyMMdd");
  517. int cleared = await ClearParseMetricBucketAsync(
  518. "daily",
  519. dateStr,
  520. dimension => IsSameParseMetricDimension(dimension, platform, riskStrategy, launchScene));
  521. for (int hour = 0; hour < 24; hour++)
  522. {
  523. cleared += await ClearParseMetricBucketAsync(
  524. "hour",
  525. $"{dateStr}{hour:00}",
  526. dimension => IsSameParseMetricDimension(dimension, platform, riskStrategy, launchScene));
  527. }
  528. return cleared;
  529. }
  530. private static async Task<int> ClearParseMetricBucketAsync(
  531. string bucketType,
  532. string bucketValue,
  533. Func<ParseMetricDimension, bool>? predicate = null)
  534. {
  535. string indexKey = BuildParseMetricIndexKey(bucketType, bucketValue);
  536. string[] members = await TkLogCore.GetTotalKeysAsync(indexKey) ?? [];
  537. int cleared = 0;
  538. foreach (var member in members)
  539. {
  540. if (!TryParseParseMetricIndexValue(member, out var dimension)) continue;
  541. if (predicate != null && !predicate(dimension)) continue;
  542. RedisHelper.Del(BuildParseMetricCountKey(ParseMetricCall, bucketType, bucketValue, dimension.Platform, dimension.RiskStrategy, dimension.LaunchScene, dimension.AccountId));
  543. RedisHelper.Del(BuildParseMetricCountKey(ParseMetricSuccess, bucketType, bucketValue, dimension.Platform, dimension.RiskStrategy, dimension.LaunchScene, dimension.AccountId));
  544. cleared += 2;
  545. if (dimension.AccountId > 0)
  546. {
  547. RedisHelper.Del(BuildParseMetricCountKey(ParseMetricRequest, bucketType, bucketValue, dimension.Platform, dimension.RiskStrategy, dimension.LaunchScene, dimension.AccountId));
  548. cleared++;
  549. }
  550. }
  551. return cleared;
  552. }
  553. private static string BuildParseMetricCountKey(string metric, string bucketType, string bucketValue, string platform, string riskStrategy, int launchScene, int accountId = 0)
  554. {
  555. string key = $"{RedisPrefix}:parse_metric:{bucketType}:{metric}:{bucketValue}:{EncodeIndexPart(platform)}:{EncodeIndexPart(riskStrategy)}:{launchScene}";
  556. if (accountId > 0) key += $":account:{accountId}";
  557. return key;
  558. }
  559. private static string BuildParseMetricIndexKey(string bucketType, string bucketValue)
  560. {
  561. return $"{RedisPrefix}:parse_metric:{bucketType}:index:{bucketValue}";
  562. }
  563. private static string BuildParseMetricIndexValue(string platform, string riskStrategy, int launchScene, int accountId = 0)
  564. {
  565. string value = $"{EncodeIndexPart(platform)}|{EncodeIndexPart(riskStrategy)}|{launchScene}";
  566. if (accountId > 0) value += $"|{accountId}";
  567. return value;
  568. }
  569. private static bool TryParseParseMetricIndexValue(string value, out ParseMetricDimension dimension)
  570. {
  571. dimension = new ParseMetricDimension();
  572. var parts = (value ?? string.Empty).Split('|');
  573. if (parts.Length < 3 || parts.Length > 4) return false;
  574. if (!int.TryParse(parts[2], out int launchScene)) return false;
  575. int accountId = 0;
  576. if (parts.Length == 4 && (!int.TryParse(parts[3], out accountId) || accountId <= 0)) return false;
  577. dimension = new ParseMetricDimension
  578. {
  579. Platform = DecodeIndexPart(parts[0]),
  580. RiskStrategy = DecodeIndexPart(parts[1]),
  581. LaunchScene = launchScene,
  582. AccountId = accountId
  583. };
  584. return true;
  585. }
  586. private static bool IsMatchedParseMetricDimension(
  587. ParseMetricDimension dimension,
  588. int accountId,
  589. bool accountBreakdownOnly,
  590. string platform,
  591. string riskStrategy,
  592. int? launchScene)
  593. {
  594. if (accountBreakdownOnly)
  595. {
  596. if (dimension.AccountId <= 0) return false;
  597. }
  598. else if (accountId > 0)
  599. {
  600. if (dimension.AccountId != accountId) return false;
  601. }
  602. else if (dimension.AccountId > 0)
  603. {
  604. return false;
  605. }
  606. if (!string.IsNullOrWhiteSpace(platform) && !string.Equals(NormalizeParseMetricPlatform(platform), dimension.Platform, StringComparison.OrdinalIgnoreCase)) return false;
  607. if (!string.IsNullOrWhiteSpace(riskStrategy) && !string.Equals(NormalizeReportScene(riskStrategy), dimension.RiskStrategy, StringComparison.OrdinalIgnoreCase)) return false;
  608. if (launchScene.HasValue && NormalizeParseMetricLaunchScene(dimension.RiskStrategy, launchScene.Value) != NormalizeParseMetricLaunchScene(dimension.RiskStrategy, dimension.LaunchScene)) return false;
  609. return true;
  610. }
  611. private static bool IsSameParseMetricDimension(
  612. ParseMetricDimension dimension,
  613. string platform,
  614. string riskStrategy,
  615. int? launchScene)
  616. {
  617. if (!string.IsNullOrWhiteSpace(platform)
  618. && !string.Equals(NormalizeParseMetricPlatform(platform), dimension.Platform, StringComparison.OrdinalIgnoreCase))
  619. {
  620. return false;
  621. }
  622. if (!string.IsNullOrWhiteSpace(riskStrategy)
  623. && !string.Equals(NormalizeReportScene(riskStrategy), dimension.RiskStrategy, StringComparison.OrdinalIgnoreCase))
  624. {
  625. return false;
  626. }
  627. if (launchScene.HasValue
  628. && NormalizeParseMetricLaunchScene(dimension.RiskStrategy, launchScene.Value) != NormalizeParseMetricLaunchScene(dimension.RiskStrategy, dimension.LaunchScene))
  629. {
  630. return false;
  631. }
  632. return true;
  633. }
  634. private static ParseMetricDimension ResolveParseMetricDimension(string channel, string riskStrategy, int launchScene)
  635. {
  636. channel = Normalize(channel).ToLowerInvariant();
  637. riskStrategy = NormalizeReportScene(riskStrategy);
  638. switch (channel)
  639. {
  640. case "tbpush":
  641. case "brwsimilar":
  642. case "icon":
  643. riskStrategy = channel;
  644. launchScene = 0;
  645. channel = "tb";
  646. break;
  647. }
  648. switch (riskStrategy)
  649. {
  650. case "tbpush":
  651. case "brwsimilar":
  652. case "icon":
  653. if (launchScene == -1) launchScene = 0;
  654. break;
  655. }
  656. launchScene = NormalizeParseMetricLaunchScene(riskStrategy, launchScene);
  657. return new ParseMetricDimension
  658. {
  659. Platform = channel,
  660. RiskStrategy = riskStrategy,
  661. LaunchScene = launchScene,
  662. AccountId = 0
  663. };
  664. }
  665. private static int NormalizeParseMetricLaunchScene(string riskStrategy, int launchScene)
  666. {
  667. if (launchScene != -1) return launchScene;
  668. return NormalizeReportScene(riskStrategy).ToLowerInvariant() switch
  669. {
  670. "os" or "tbpush" or "brwsimilar" or "icon" => 0,
  671. _ => launchScene
  672. };
  673. }
  674. private static string NormalizeParseMetricPlatform(string platform)
  675. {
  676. string value = Normalize(platform).ToLowerInvariant();
  677. return value switch
  678. {
  679. "淘宝" or "taobao" or "tb" or "1" => "tb",
  680. "京东" or "jd" or "13" => "jd",
  681. "拼多多" or "pdd" or "9" => "pdd",
  682. _ => value
  683. };
  684. }
  685. private static bool TableExists(IDbConnection conn, string tableName)
  686. {
  687. const string sql = @"
  688. SELECT COUNT(1)
  689. FROM information_schema.TABLES
  690. WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=@tableName";
  691. return conn.ExecuteScalar<int>(sql, new { tableName }) > 0;
  692. }
  693. private static int SafeInt(long value)
  694. {
  695. if (value <= 0) return 0;
  696. return value > int.MaxValue ? int.MaxValue : (int)value;
  697. }
  698. private sealed record ParseMetricCounterKey(
  699. string Metric,
  700. string BucketType,
  701. string BucketValue,
  702. string Platform,
  703. string RiskStrategy,
  704. int LaunchScene,
  705. int AccountId);
  706. private sealed class ParseMetricCounter
  707. {
  708. public long Count = 0;
  709. }
  710. private sealed class ParseMetricDimension
  711. {
  712. public string Platform { get; set; } = string.Empty;
  713. public string RiskStrategy { get; set; } = string.Empty;
  714. public int LaunchScene { get; set; } = -1;
  715. public int AccountId { get; set; } = 0;
  716. }
  717. private sealed record ParseMetricBackfillSource(string Platform, string TableName);
  718. private sealed class ParseMetricBackfillRow
  719. {
  720. public string risk_strategy { get; set; } = string.Empty;
  721. public int launch_scene { get; set; } = -1;
  722. public int account_id { get; set; } = 0;
  723. public string hour_key { get; set; } = string.Empty;
  724. public long call_count { get; set; } = 0;
  725. public long success_count { get; set; } = 0;
  726. }
  727. }
  728. }