TracksParseMetricCore.cs 36 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811
  1. using System.Collections.Concurrent;
  2. using System.Data;
  3. using System.Threading;
  4. using Dapper;
  5. using dodohold.core;
  6. using YunhuiKit;
  7. namespace molilian.core
  8. {
  9. public partial class TracksCore
  10. {
  11. private const string ParseMetricTypeName = "转链";
  12. private const string ParseMetricRequest = "request";
  13. private const string ParseMetricCall = "call";
  14. private const string ParseMetricSuccess = "success";
  15. private const int ParseMetricFlushIntervalMs = 1000;
  16. private static readonly ConcurrentDictionary<ParseMetricCounterKey, ParseMetricCounter> PendingParseMetricCounters = new();
  17. private static readonly object ParseMetricFlushTimerLock = new();
  18. private static Timer? ParseMetricFlushTimer;
  19. private static int ParseMetricFlushRunning;
  20. private static int ParseMetricLifecycleHooked;
  21. public static Task<bool> RecordParseRequestAsync(UnionParseRequest request)
  22. {
  23. var dimension = ResolveParseMetricDimension(request.Channel, request.RiskStrategy, request.LaunchScene);
  24. return IncrementParseMetricAsync(ParseMetricRequest, dimension.Platform, dimension.RiskStrategy, dimension.LaunchScene, request.AccountId);
  25. }
  26. public static Task<bool> RecordParseAccountRequestAsync(string channel, string riskStrategy, int launchScene, int accountId)
  27. {
  28. if (accountId <= 0) return Task.FromResult(false);
  29. var dimension = ResolveParseMetricDimension(channel, riskStrategy, launchScene);
  30. return IncrementParseAccountMetricAsync(ParseMetricRequest, dimension.Platform, dimension.RiskStrategy, dimension.LaunchScene, accountId);
  31. }
  32. public static Task<bool> RecordParseResultAsync(string channel, string riskStrategy, int launchScene, int accountId, bool success)
  33. {
  34. var dimension = ResolveParseMetricDimension(channel, riskStrategy, launchScene);
  35. return IncrementParseResultMetricAsync(dimension.Platform, dimension.RiskStrategy, dimension.LaunchScene, accountId, success);
  36. }
  37. /// <summary>
  38. /// RecordParseResultAsync 写入的 success 计数 key(daily、hour),供历史数据修复使用
  39. /// </summary>
  40. public static string[] BuildParseMetricSuccessKeys(DateTime metricTime, string channel, string riskStrategy, int launchScene)
  41. {
  42. var dimension = ResolveParseMetricDimension(channel, riskStrategy, launchScene);
  43. return
  44. [
  45. BuildParseMetricCountKey(ParseMetricSuccess, "daily", metricTime.ToString("yyyyMMdd"), dimension.Platform, dimension.RiskStrategy, dimension.LaunchScene),
  46. BuildParseMetricCountKey(ParseMetricSuccess, "hour", metricTime.ToString("yyyyMMddHH"), dimension.Platform, dimension.RiskStrategy, dimension.LaunchScene),
  47. ];
  48. }
  49. public static async Task<TrackParseMetricReportResult> GetParseMetricReportAsync(
  50. DateTime start,
  51. DateTime end,
  52. int page,
  53. int size,
  54. bool getTotal,
  55. int accountId = -1,
  56. bool accountBreakdownOnly = false,
  57. string platform = "",
  58. string riskStrategy = "",
  59. int? launchScene = null)
  60. {
  61. start = start.Date;
  62. end = end.Date;
  63. if (end < start) end = start;
  64. page = Math.Max(page, 1);
  65. size = Math.Max(size, 1);
  66. var list = new List<TrackParseMetricReportDTO>();
  67. for (var date = start; date <= end; date = date.AddDays(1))
  68. {
  69. string dateStr = date.ToString("yyyyMMdd");
  70. string indexKey = BuildParseMetricIndexKey("daily", dateStr);
  71. string[] members = await TkLogCore.GetTotalKeysAsync(indexKey) ?? [];
  72. foreach (var member in members)
  73. {
  74. if (!TryParseParseMetricIndexValue(member, out var dimension)) continue;
  75. if (!IsMatchedParseMetricDimension(dimension, accountId, accountBreakdownOnly, platform, riskStrategy, launchScene)) continue;
  76. var requestTask = TkLogCore.GetTotalAsync(BuildParseMetricCountKey(ParseMetricRequest, "daily", dateStr, dimension.Platform, dimension.RiskStrategy, dimension.LaunchScene, dimension.AccountId));
  77. var callTask = TkLogCore.GetTotalAsync(BuildParseMetricCountKey(ParseMetricCall, "daily", dateStr, dimension.Platform, dimension.RiskStrategy, dimension.LaunchScene, dimension.AccountId));
  78. var successTask = TkLogCore.GetTotalAsync(BuildParseMetricCountKey(ParseMetricSuccess, "daily", dateStr, dimension.Platform, dimension.RiskStrategy, dimension.LaunchScene, dimension.AccountId));
  79. await Task.WhenAll(requestTask, callTask, successTask);
  80. int requestCount = requestTask.Result;
  81. int callCount = callTask.Result;
  82. int successCount = successTask.Result;
  83. if (requestCount <= 0 && callCount <= 0 && successCount <= 0) continue;
  84. list.Add(new TrackParseMetricReportDTO
  85. {
  86. report_date = date,
  87. platform = dimension.Platform,
  88. typename = ParseMetricTypeName,
  89. risk_strategy = dimension.RiskStrategy,
  90. launch_scene = dimension.LaunchScene,
  91. account_id = dimension.AccountId,
  92. request_count = requestCount,
  93. call_count = callCount,
  94. success_count = successCount
  95. });
  96. }
  97. }
  98. list = list
  99. .OrderByDescending(item => item.report_date)
  100. .ThenBy(item => item.platform)
  101. .ThenBy(item => item.risk_strategy)
  102. .ThenBy(item => item.launch_scene)
  103. .ThenBy(item => item.account_id)
  104. .ToList();
  105. int count = list.Count;
  106. list = list
  107. .Skip((page - 1) * size)
  108. .Take(size)
  109. .ToList();
  110. return new TrackParseMetricReportResult
  111. {
  112. list = list,
  113. count = getTotal ? count : list.Count
  114. };
  115. }
  116. public static async Task<List<TrackParseMetricHourlyDTO>> GetParseMetricHourlyReportAsync(
  117. DateTime targetDate,
  118. string platform,
  119. string riskStrategy,
  120. int? launchScene = null,
  121. int accountId = 0)
  122. {
  123. targetDate = targetDate.Date;
  124. platform = NormalizeParseMetricPlatform(platform);
  125. riskStrategy = NormalizeReportScene(riskStrategy);
  126. accountId = Math.Max(accountId, 0);
  127. var list = new List<TrackParseMetricHourlyDTO>();
  128. string dateStr = targetDate.ToString("yyyyMMdd");
  129. for (int hour = 0; hour < 24; hour++)
  130. {
  131. string hourStr = $"{dateStr}{hour:00}";
  132. string indexKey = BuildParseMetricIndexKey("hour", hourStr);
  133. string[] members = await TkLogCore.GetTotalKeysAsync(indexKey) ?? [];
  134. long requestCount = 0;
  135. long callCount = 0;
  136. long successCount = 0;
  137. foreach (var member in members)
  138. {
  139. if (!TryParseParseMetricIndexValue(member, out var dimension)) continue;
  140. if (!IsMatchedParseMetricDimension(dimension, accountId, false, platform, riskStrategy, launchScene)) continue;
  141. var requestTask = TkLogCore.GetTotalAsync(BuildParseMetricCountKey(ParseMetricRequest, "hour", hourStr, dimension.Platform, dimension.RiskStrategy, dimension.LaunchScene, dimension.AccountId));
  142. var callTask = TkLogCore.GetTotalAsync(BuildParseMetricCountKey(ParseMetricCall, "hour", hourStr, dimension.Platform, dimension.RiskStrategy, dimension.LaunchScene, dimension.AccountId));
  143. var successTask = TkLogCore.GetTotalAsync(BuildParseMetricCountKey(ParseMetricSuccess, "hour", hourStr, dimension.Platform, dimension.RiskStrategy, dimension.LaunchScene, dimension.AccountId));
  144. await Task.WhenAll(requestTask, callTask, successTask);
  145. requestCount += requestTask.Result;
  146. callCount += callTask.Result;
  147. successCount += successTask.Result;
  148. }
  149. list.Add(new TrackParseMetricHourlyDTO
  150. {
  151. report_date = targetDate,
  152. platform = platform,
  153. typename = ParseMetricTypeName,
  154. risk_strategy = riskStrategy,
  155. launch_scene = launchScene ?? -1,
  156. account_id = accountId,
  157. hour = hour,
  158. request_count = SafeInt(requestCount),
  159. call_count = SafeInt(callCount),
  160. success_count = SafeInt(successCount)
  161. });
  162. }
  163. return list;
  164. }
  165. public static async Task<TrackParseMetricBackfillResult> BackfillParseMetricCountersAsync(
  166. DateTime targetDate,
  167. string platform = "",
  168. string riskStrategy = "",
  169. int? launchScene = null)
  170. {
  171. targetDate = targetDate.Date;
  172. platform = NormalizeParseMetricPlatform(platform);
  173. riskStrategy = NormalizeReportScene(riskStrategy);
  174. bool hasPlatformFilter = !string.IsNullOrWhiteSpace(platform);
  175. bool hasRiskStrategyFilter = !string.IsNullOrWhiteSpace(riskStrategy);
  176. bool hasLaunchSceneFilter = launchScene.HasValue;
  177. bool hasDimensionFilter = hasPlatformFilter || hasRiskStrategyFilter || hasLaunchSceneFilter;
  178. var result = new TrackParseMetricBackfillResult
  179. {
  180. report_date = targetDate.ToString("yyyy-MM-dd")
  181. };
  182. var allSources = new[]
  183. {
  184. new ParseMetricBackfillSource("tb", $"tk_parse_logs_{targetDate:yyyyMMdd}"),
  185. new ParseMetricBackfillSource("jd", $"jd_parse_logs_{targetDate:yyyyMMdd}"),
  186. new ParseMetricBackfillSource("pdd", $"pdd_parse_logs_{targetDate:yyyyMMdd}")
  187. };
  188. var sources = hasDimensionFilter
  189. ? allSources.Where(source => !hasPlatformFilter || string.Equals(source.Platform, platform, StringComparison.OrdinalIgnoreCase)).ToArray()
  190. : allSources;
  191. using var conn = DBContext.GetOpenConnection();
  192. DateTime start = targetDate;
  193. DateTime end = targetDate.AddDays(1);
  194. result.redis_cleared_keys = hasDimensionFilter
  195. ? await ClearParseMetricCountersForDimensionAsync(targetDate, platform, riskStrategy, launchScene)
  196. : await ClearParseMetricCountersForDateAsync(targetDate);
  197. foreach (var source in sources)
  198. {
  199. if (!TableExists(conn, source.TableName))
  200. {
  201. result.details.Add(new { platform = source.Platform, table = source.TableName, exists = false, rows = 0, dimensions = 0 });
  202. continue;
  203. }
  204. const string riskStrategySql = "COALESCE(NULLIF(TRIM(riskStrategy), ''), '')";
  205. const string rawLaunchSceneSql = "COALESCE(launchScene, -1)";
  206. string launchSceneSql = $"CASE WHEN LOWER({riskStrategySql}) IN ('os','tbpush','brwsimilar','icon') AND {rawLaunchSceneSql}=-1 THEN 0 ELSE {rawLaunchSceneSql} END";
  207. var dimensionWhereItems = new List<string>();
  208. if (hasRiskStrategyFilter) dimensionWhereItems.Add($"{riskStrategySql}=@riskStrategy");
  209. if (hasLaunchSceneFilter) dimensionWhereItems.Add($"{launchSceneSql}=@launchScene");
  210. string dimensionWhere = dimensionWhereItems.Count > 0
  211. ? "\n AND " + string.Join("\n AND ", dimensionWhereItems)
  212. : string.Empty;
  213. string sql = $@"
  214. SELECT
  215. {riskStrategySql} AS risk_strategy,
  216. {launchSceneSql} AS launch_scene,
  217. COALESCE(accountId, 0) AS account_id,
  218. DATE_FORMAT(create_time, '%Y%m%d%H') AS hour_key,
  219. CAST(COUNT(1) AS SIGNED) AS call_count,
  220. CAST(SUM(CASE WHEN success=1 THEN 1 ELSE 0 END) AS SIGNED) AS success_count
  221. FROM `{source.TableName}`
  222. WHERE create_time>=@start AND create_time<@end
  223. {dimensionWhere}
  224. GROUP BY
  225. {riskStrategySql},
  226. {launchSceneSql},
  227. COALESCE(accountId, 0),
  228. DATE_FORMAT(create_time, '%Y%m%d%H')";
  229. var args = new DynamicParameters();
  230. args.Add("start", start);
  231. args.Add("end", end);
  232. if (hasRiskStrategyFilter) args.Add("riskStrategy", riskStrategy);
  233. if (hasLaunchSceneFilter) args.Add("launchScene", launchScene!.Value);
  234. var rows = SqlMapper.Query<ParseMetricBackfillRow>(conn, sql, args).ToList();
  235. int mysqlRows = rows.Sum(row => SafeInt(row.call_count));
  236. int dimensions = ApplyParseMetricBackfillRows(source.Platform, targetDate, rows);
  237. result.mysql_rows += mysqlRows;
  238. result.redis_dimensions += dimensions;
  239. result.details.Add(new { platform = source.Platform, table = source.TableName, exists = true, rows = mysqlRows, dimensions });
  240. }
  241. await Task.CompletedTask;
  242. return result;
  243. }
  244. private static Task<bool> IncrementParseMetricAsync(string metric, string platform, string riskStrategy, int launchScene, int accountId = 0)
  245. {
  246. try
  247. {
  248. AddParseMetricCount(metric, DateTime.Now, platform, riskStrategy, launchScene, 0, 1);
  249. if (accountId > 0)
  250. {
  251. AddParseMetricCount(metric, DateTime.Now, platform, riskStrategy, launchScene, accountId, 1);
  252. }
  253. return Task.FromResult(true);
  254. }
  255. catch (Exception ex)
  256. {
  257. _ = new LoggerLibrary("TracksCore", "RecordParseMetric")
  258. .Info(ex.Message, ex.StackTrace)
  259. .SaveAsync();
  260. return Task.FromResult(false);
  261. }
  262. }
  263. private static Task<bool> IncrementParseAccountMetricAsync(string metric, string platform, string riskStrategy, int launchScene, int accountId)
  264. {
  265. try
  266. {
  267. accountId = Math.Max(accountId, 0);
  268. if (accountId <= 0) return Task.FromResult(false);
  269. AddParseMetricCount(metric, DateTime.Now, platform, riskStrategy, launchScene, accountId, 1);
  270. return Task.FromResult(true);
  271. }
  272. catch (Exception ex)
  273. {
  274. _ = new LoggerLibrary("TracksCore", "RecordParseAccountMetric")
  275. .Info(ex.Message, ex.StackTrace)
  276. .SaveAsync();
  277. return Task.FromResult(false);
  278. }
  279. }
  280. private static Task<bool> IncrementParseResultMetricAsync(string platform, string riskStrategy, int launchScene, int accountId, bool success)
  281. {
  282. try
  283. {
  284. accountId = Math.Max(accountId, 0);
  285. AddParseMetricCount(ParseMetricCall, DateTime.Now, platform, riskStrategy, launchScene, 0, 1);
  286. if (success) AddParseMetricCount(ParseMetricSuccess, DateTime.Now, platform, riskStrategy, launchScene, 0, 1);
  287. if (accountId > 0)
  288. {
  289. AddParseMetricCount(ParseMetricCall, DateTime.Now, platform, riskStrategy, launchScene, accountId, 1);
  290. if (success) AddParseMetricCount(ParseMetricSuccess, DateTime.Now, platform, riskStrategy, launchScene, accountId, 1);
  291. }
  292. return Task.FromResult(true);
  293. }
  294. catch (Exception ex)
  295. {
  296. _ = new LoggerLibrary("TracksCore", "RecordParseResultMetric")
  297. .Info(ex.Message, ex.StackTrace)
  298. .SaveAsync();
  299. return Task.FromResult(false);
  300. }
  301. }
  302. private static int ApplyParseMetricBackfillRows(string platform, DateTime targetDate, List<ParseMetricBackfillRow> rows)
  303. {
  304. string dateStr = targetDate.ToString("yyyyMMdd");
  305. int dimensions = 0;
  306. var dailyGroups = rows.GroupBy(row => new
  307. {
  308. RiskStrategy = NormalizeReportScene(row.risk_strategy),
  309. LaunchScene = NormalizeParseMetricLaunchScene(row.risk_strategy, row.launch_scene)
  310. });
  311. foreach (var group in dailyGroups)
  312. {
  313. int callCount = group.Sum(row => SafeInt(row.call_count));
  314. int successCount = group.Sum(row => SafeInt(row.success_count));
  315. SetParseMetricCount(ParseMetricCall, "daily", dateStr, platform, group.Key.RiskStrategy, group.Key.LaunchScene, 0, callCount);
  316. SetParseMetricCount(ParseMetricSuccess, "daily", dateStr, platform, group.Key.RiskStrategy, group.Key.LaunchScene, 0, successCount);
  317. dimensions++;
  318. }
  319. var dailyAccountGroups = rows
  320. .Where(row => row.account_id > 0)
  321. .GroupBy(row => new
  322. {
  323. RiskStrategy = NormalizeReportScene(row.risk_strategy),
  324. launch_scene = NormalizeParseMetricLaunchScene(row.risk_strategy, row.launch_scene),
  325. row.account_id
  326. });
  327. foreach (var group in dailyAccountGroups)
  328. {
  329. int callCount = group.Sum(row => SafeInt(row.call_count));
  330. int successCount = group.Sum(row => SafeInt(row.success_count));
  331. SetParseMetricCount(ParseMetricRequest, "daily", dateStr, platform, group.Key.RiskStrategy, group.Key.launch_scene, group.Key.account_id, callCount);
  332. SetParseMetricCount(ParseMetricCall, "daily", dateStr, platform, group.Key.RiskStrategy, group.Key.launch_scene, group.Key.account_id, callCount);
  333. SetParseMetricCount(ParseMetricSuccess, "daily", dateStr, platform, group.Key.RiskStrategy, group.Key.launch_scene, group.Key.account_id, successCount);
  334. dimensions++;
  335. }
  336. var hourlyGroups = rows.GroupBy(row => new
  337. {
  338. RiskStrategy = NormalizeReportScene(row.risk_strategy),
  339. launch_scene = NormalizeParseMetricLaunchScene(row.risk_strategy, row.launch_scene),
  340. row.hour_key
  341. });
  342. foreach (var group in hourlyGroups)
  343. {
  344. int callCount = group.Sum(row => SafeInt(row.call_count));
  345. int successCount = group.Sum(row => SafeInt(row.success_count));
  346. SetParseMetricCount(ParseMetricCall, "hour", group.Key.hour_key, platform, group.Key.RiskStrategy, group.Key.launch_scene, 0, callCount);
  347. SetParseMetricCount(ParseMetricSuccess, "hour", group.Key.hour_key, platform, group.Key.RiskStrategy, group.Key.launch_scene, 0, successCount);
  348. }
  349. var hourlyAccountGroups = rows
  350. .Where(row => row.account_id > 0)
  351. .GroupBy(row => new
  352. {
  353. RiskStrategy = NormalizeReportScene(row.risk_strategy),
  354. launch_scene = NormalizeParseMetricLaunchScene(row.risk_strategy, row.launch_scene),
  355. row.account_id,
  356. row.hour_key
  357. });
  358. foreach (var group in hourlyAccountGroups)
  359. {
  360. int callCount = group.Sum(row => SafeInt(row.call_count));
  361. int successCount = group.Sum(row => SafeInt(row.success_count));
  362. SetParseMetricCount(ParseMetricRequest, "hour", group.Key.hour_key, platform, group.Key.RiskStrategy, group.Key.launch_scene, group.Key.account_id, callCount);
  363. SetParseMetricCount(ParseMetricCall, "hour", group.Key.hour_key, platform, group.Key.RiskStrategy, group.Key.launch_scene, group.Key.account_id, callCount);
  364. SetParseMetricCount(ParseMetricSuccess, "hour", group.Key.hour_key, platform, group.Key.RiskStrategy, group.Key.launch_scene, group.Key.account_id, successCount);
  365. }
  366. return dimensions;
  367. }
  368. private static void AddParseMetricCount(string metric, DateTime metricTime, string platform, string riskStrategy, int launchScene, int accountId, int count)
  369. {
  370. if (count <= 0) return;
  371. QueueParseMetricCount(metric, "daily", metricTime.ToString("yyyyMMdd"), platform, riskStrategy, launchScene, accountId, count);
  372. QueueParseMetricCount(metric, "hour", metricTime.ToString("yyyyMMddHH"), platform, riskStrategy, launchScene, accountId, count);
  373. }
  374. private static void SetParseMetricCount(string metric, string bucketType, string bucketValue, string platform, string riskStrategy, int launchScene, int accountId, int count)
  375. {
  376. if (count < 0) return;
  377. SetOrIncrementParseMetricCount(metric, bucketType, bucketValue, platform, riskStrategy, launchScene, accountId, count, increment: false);
  378. }
  379. private static void SetOrIncrementParseMetricCount(string metric, string bucketType, string bucketValue, string platform, string riskStrategy, int launchScene, int accountId, long count, bool increment)
  380. {
  381. platform = Normalize(platform).ToLowerInvariant();
  382. riskStrategy = NormalizeReportScene(riskStrategy);
  383. launchScene = NormalizeParseMetricLaunchScene(riskStrategy, launchScene);
  384. accountId = Math.Max(accountId, 0);
  385. string countKey = BuildParseMetricCountKey(metric, bucketType, bucketValue, platform, riskStrategy, launchScene, accountId);
  386. string indexKey = BuildParseMetricIndexKey(bucketType, bucketValue);
  387. string indexValue = BuildParseMetricIndexValue(platform, riskStrategy, launchScene, accountId);
  388. int expireSeconds = bucketType == "hour" ? HourlyExpireSeconds : DailyExpireSeconds;
  389. if (increment)
  390. {
  391. IncrementParseMetricRedisKey(countKey, count);
  392. }
  393. else
  394. {
  395. RedisHelper.Set(countKey, count, expireSeconds);
  396. }
  397. RedisHelper.Expire(countKey, expireSeconds);
  398. RedisHelper.SAdd(indexKey, indexValue);
  399. RedisHelper.Expire(indexKey, expireSeconds);
  400. }
  401. public static int FlushParseMetricCounters()
  402. {
  403. if (Interlocked.Exchange(ref ParseMetricFlushRunning, 1) == 1) return 0;
  404. try
  405. {
  406. int flushed = 0;
  407. foreach (var item in PendingParseMetricCounters.ToArray())
  408. {
  409. long count = Interlocked.Exchange(ref item.Value.Count, 0);
  410. if (count <= 0) continue;
  411. try
  412. {
  413. SetOrIncrementParseMetricCount(
  414. item.Key.Metric,
  415. item.Key.BucketType,
  416. item.Key.BucketValue,
  417. item.Key.Platform,
  418. item.Key.RiskStrategy,
  419. item.Key.LaunchScene,
  420. item.Key.AccountId,
  421. count,
  422. increment: true);
  423. flushed++;
  424. }
  425. catch (Exception ex)
  426. {
  427. Interlocked.Add(ref item.Value.Count, count);
  428. _ = new LoggerLibrary("TracksCore", "FlushParseMetricCounters")
  429. .Info(ex.Message, ex.StackTrace)
  430. .SaveAsync();
  431. }
  432. }
  433. return flushed;
  434. }
  435. finally
  436. {
  437. Interlocked.Exchange(ref ParseMetricFlushRunning, 0);
  438. }
  439. }
  440. private static void QueueParseMetricCount(string metric, string bucketType, string bucketValue, string platform, string riskStrategy, int launchScene, int accountId, int count)
  441. {
  442. if (count <= 0) return;
  443. var key = new ParseMetricCounterKey(
  444. metric,
  445. bucketType,
  446. bucketValue,
  447. Normalize(platform).ToLowerInvariant(),
  448. NormalizeReportScene(riskStrategy),
  449. NormalizeParseMetricLaunchScene(riskStrategy, launchScene),
  450. Math.Max(accountId, 0));
  451. var counter = PendingParseMetricCounters.GetOrAdd(key, _ => new ParseMetricCounter());
  452. Interlocked.Add(ref counter.Count, count);
  453. EnsureParseMetricFlushTimer();
  454. }
  455. private static void EnsureParseMetricFlushTimer()
  456. {
  457. if (ParseMetricFlushTimer != null) return;
  458. lock (ParseMetricFlushTimerLock)
  459. {
  460. if (ParseMetricFlushTimer != null) return;
  461. ParseMetricFlushTimer = new Timer(
  462. _ => FlushParseMetricCounters(),
  463. null,
  464. ParseMetricFlushIntervalMs,
  465. ParseMetricFlushIntervalMs);
  466. if (Interlocked.Exchange(ref ParseMetricLifecycleHooked, 1) == 0)
  467. {
  468. AppDomain.CurrentDomain.ProcessExit += (_, _) => FlushParseMetricCounters();
  469. AppDomain.CurrentDomain.UnhandledException += (_, _) => FlushParseMetricCounters();
  470. }
  471. }
  472. }
  473. private static void IncrementParseMetricRedisKey(string key, long count)
  474. {
  475. while (count > 0)
  476. {
  477. int delta = count > int.MaxValue ? int.MaxValue : (int)count;
  478. RedisHelper.IncrBy(key, delta);
  479. count -= delta;
  480. }
  481. }
  482. private static async Task<int> ClearParseMetricCountersForDateAsync(DateTime targetDate)
  483. {
  484. string dateStr = targetDate.ToString("yyyyMMdd");
  485. int cleared = await ClearParseMetricBucketAsync("daily", dateStr);
  486. for (int hour = 0; hour < 24; hour++)
  487. {
  488. cleared += await ClearParseMetricBucketAsync("hour", $"{dateStr}{hour:00}");
  489. }
  490. return cleared;
  491. }
  492. private static async Task<int> ClearParseMetricCountersForDimensionAsync(
  493. DateTime targetDate,
  494. string platform,
  495. string riskStrategy,
  496. int? launchScene)
  497. {
  498. string dateStr = targetDate.ToString("yyyyMMdd");
  499. int cleared = await ClearParseMetricBucketAsync(
  500. "daily",
  501. dateStr,
  502. dimension => IsSameParseMetricDimension(dimension, platform, riskStrategy, launchScene));
  503. for (int hour = 0; hour < 24; hour++)
  504. {
  505. cleared += await ClearParseMetricBucketAsync(
  506. "hour",
  507. $"{dateStr}{hour:00}",
  508. dimension => IsSameParseMetricDimension(dimension, platform, riskStrategy, launchScene));
  509. }
  510. return cleared;
  511. }
  512. private static async Task<int> ClearParseMetricBucketAsync(
  513. string bucketType,
  514. string bucketValue,
  515. Func<ParseMetricDimension, bool>? predicate = null)
  516. {
  517. string indexKey = BuildParseMetricIndexKey(bucketType, bucketValue);
  518. string[] members = await TkLogCore.GetTotalKeysAsync(indexKey) ?? [];
  519. int cleared = 0;
  520. foreach (var member in members)
  521. {
  522. if (!TryParseParseMetricIndexValue(member, out var dimension)) continue;
  523. if (predicate != null && !predicate(dimension)) continue;
  524. RedisHelper.Del(BuildParseMetricCountKey(ParseMetricCall, bucketType, bucketValue, dimension.Platform, dimension.RiskStrategy, dimension.LaunchScene, dimension.AccountId));
  525. RedisHelper.Del(BuildParseMetricCountKey(ParseMetricSuccess, bucketType, bucketValue, dimension.Platform, dimension.RiskStrategy, dimension.LaunchScene, dimension.AccountId));
  526. cleared += 2;
  527. if (dimension.AccountId > 0)
  528. {
  529. RedisHelper.Del(BuildParseMetricCountKey(ParseMetricRequest, bucketType, bucketValue, dimension.Platform, dimension.RiskStrategy, dimension.LaunchScene, dimension.AccountId));
  530. cleared++;
  531. }
  532. }
  533. return cleared;
  534. }
  535. private static string BuildParseMetricCountKey(string metric, string bucketType, string bucketValue, string platform, string riskStrategy, int launchScene, int accountId = 0)
  536. {
  537. string key = $"{RedisPrefix}:parse_metric:{bucketType}:{metric}:{bucketValue}:{EncodeIndexPart(platform)}:{EncodeIndexPart(riskStrategy)}:{launchScene}";
  538. if (accountId > 0) key += $":account:{accountId}";
  539. return key;
  540. }
  541. private static string BuildParseMetricIndexKey(string bucketType, string bucketValue)
  542. {
  543. return $"{RedisPrefix}:parse_metric:{bucketType}:index:{bucketValue}";
  544. }
  545. private static string BuildParseMetricIndexValue(string platform, string riskStrategy, int launchScene, int accountId = 0)
  546. {
  547. string value = $"{EncodeIndexPart(platform)}|{EncodeIndexPart(riskStrategy)}|{launchScene}";
  548. if (accountId > 0) value += $"|{accountId}";
  549. return value;
  550. }
  551. private static bool TryParseParseMetricIndexValue(string value, out ParseMetricDimension dimension)
  552. {
  553. dimension = new ParseMetricDimension();
  554. var parts = (value ?? string.Empty).Split('|');
  555. if (parts.Length < 3 || parts.Length > 4) return false;
  556. if (!int.TryParse(parts[2], out int launchScene)) return false;
  557. int accountId = 0;
  558. if (parts.Length == 4 && (!int.TryParse(parts[3], out accountId) || accountId <= 0)) return false;
  559. dimension = new ParseMetricDimension
  560. {
  561. Platform = DecodeIndexPart(parts[0]),
  562. RiskStrategy = DecodeIndexPart(parts[1]),
  563. LaunchScene = launchScene,
  564. AccountId = accountId
  565. };
  566. return true;
  567. }
  568. private static bool IsMatchedParseMetricDimension(
  569. ParseMetricDimension dimension,
  570. int accountId,
  571. bool accountBreakdownOnly,
  572. string platform,
  573. string riskStrategy,
  574. int? launchScene)
  575. {
  576. if (accountBreakdownOnly)
  577. {
  578. if (dimension.AccountId <= 0) return false;
  579. }
  580. else if (accountId > 0)
  581. {
  582. if (dimension.AccountId != accountId) return false;
  583. }
  584. else if (dimension.AccountId > 0)
  585. {
  586. return false;
  587. }
  588. if (!string.IsNullOrWhiteSpace(platform) && !string.Equals(NormalizeParseMetricPlatform(platform), dimension.Platform, StringComparison.OrdinalIgnoreCase)) return false;
  589. if (!string.IsNullOrWhiteSpace(riskStrategy) && !string.Equals(NormalizeReportScene(riskStrategy), dimension.RiskStrategy, StringComparison.OrdinalIgnoreCase)) return false;
  590. if (launchScene.HasValue && NormalizeParseMetricLaunchScene(dimension.RiskStrategy, launchScene.Value) != NormalizeParseMetricLaunchScene(dimension.RiskStrategy, dimension.LaunchScene)) return false;
  591. return true;
  592. }
  593. private static bool IsSameParseMetricDimension(
  594. ParseMetricDimension dimension,
  595. string platform,
  596. string riskStrategy,
  597. int? launchScene)
  598. {
  599. if (!string.IsNullOrWhiteSpace(platform)
  600. && !string.Equals(NormalizeParseMetricPlatform(platform), dimension.Platform, StringComparison.OrdinalIgnoreCase))
  601. {
  602. return false;
  603. }
  604. if (!string.IsNullOrWhiteSpace(riskStrategy)
  605. && !string.Equals(NormalizeReportScene(riskStrategy), dimension.RiskStrategy, StringComparison.OrdinalIgnoreCase))
  606. {
  607. return false;
  608. }
  609. if (launchScene.HasValue
  610. && NormalizeParseMetricLaunchScene(dimension.RiskStrategy, launchScene.Value) != NormalizeParseMetricLaunchScene(dimension.RiskStrategy, dimension.LaunchScene))
  611. {
  612. return false;
  613. }
  614. return true;
  615. }
  616. private static ParseMetricDimension ResolveParseMetricDimension(string channel, string riskStrategy, int launchScene)
  617. {
  618. channel = Normalize(channel).ToLowerInvariant();
  619. riskStrategy = NormalizeReportScene(riskStrategy);
  620. switch (channel)
  621. {
  622. case "tbpush":
  623. case "brwsimilar":
  624. case "icon":
  625. riskStrategy = channel;
  626. launchScene = 0;
  627. channel = "tb";
  628. break;
  629. }
  630. switch (riskStrategy)
  631. {
  632. case "tbpush":
  633. case "brwsimilar":
  634. case "icon":
  635. if (launchScene == -1) launchScene = 0;
  636. break;
  637. }
  638. launchScene = NormalizeParseMetricLaunchScene(riskStrategy, launchScene);
  639. return new ParseMetricDimension
  640. {
  641. Platform = channel,
  642. RiskStrategy = riskStrategy,
  643. LaunchScene = launchScene,
  644. AccountId = 0
  645. };
  646. }
  647. private static int NormalizeParseMetricLaunchScene(string riskStrategy, int launchScene)
  648. {
  649. if (launchScene != -1) return launchScene;
  650. return NormalizeReportScene(riskStrategy).ToLowerInvariant() switch
  651. {
  652. "os" or "tbpush" or "brwsimilar" or "icon" => 0,
  653. _ => launchScene
  654. };
  655. }
  656. private static string NormalizeParseMetricPlatform(string platform)
  657. {
  658. string value = Normalize(platform).ToLowerInvariant();
  659. return value switch
  660. {
  661. "淘宝" or "taobao" or "tb" or "1" => "tb",
  662. "京东" or "jd" or "13" => "jd",
  663. "拼多多" or "pdd" or "9" => "pdd",
  664. _ => value
  665. };
  666. }
  667. private static bool TableExists(IDbConnection conn, string tableName)
  668. {
  669. const string sql = @"
  670. SELECT COUNT(1)
  671. FROM information_schema.TABLES
  672. WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=@tableName";
  673. return conn.ExecuteScalar<int>(sql, new { tableName }) > 0;
  674. }
  675. private static int SafeInt(long value)
  676. {
  677. if (value <= 0) return 0;
  678. return value > int.MaxValue ? int.MaxValue : (int)value;
  679. }
  680. private sealed record ParseMetricCounterKey(
  681. string Metric,
  682. string BucketType,
  683. string BucketValue,
  684. string Platform,
  685. string RiskStrategy,
  686. int LaunchScene,
  687. int AccountId);
  688. private sealed class ParseMetricCounter
  689. {
  690. public long Count = 0;
  691. }
  692. private sealed class ParseMetricDimension
  693. {
  694. public string Platform { get; set; } = string.Empty;
  695. public string RiskStrategy { get; set; } = string.Empty;
  696. public int LaunchScene { get; set; } = -1;
  697. public int AccountId { get; set; } = 0;
  698. }
  699. private sealed record ParseMetricBackfillSource(string Platform, string TableName);
  700. private sealed class ParseMetricBackfillRow
  701. {
  702. public string risk_strategy { get; set; } = string.Empty;
  703. public int launch_scene { get; set; } = -1;
  704. public int account_id { get; set; } = 0;
  705. public string hour_key { get; set; } = string.Empty;
  706. public long call_count { get; set; } = 0;
  707. public long success_count { get; set; } = 0;
  708. }
  709. }
  710. }