|
|
@@ -28,6 +28,8 @@ namespace molilian.api.Controllers
|
|
|
public class TestController : ControllerBase
|
|
|
{
|
|
|
private const string JdStressDeeplinkContent = "openapp.jdmobile://virtual?params=%7B%22category%22%3A%22jump%22%2C%22des%22%3A%22productDetail%22%2C%22skuId%22%3A%2210139003162781%22%2C%22sourceType%22%3A%22Item%22%2C%22sourceValue%22%3A%22view-ware%22%7D";
|
|
|
+ private const int DeeplinkReportStatsExpireSeconds = 90 * 86400;
|
|
|
+ private const int DeeplinkReportRepairMaxDays = 366;
|
|
|
|
|
|
protected IHttpContextAccessor _accessor;
|
|
|
public TestController(IHttpContextAccessor accessor)
|
|
|
@@ -552,6 +554,409 @@ namespace molilian.api.Controllers
|
|
|
});
|
|
|
}
|
|
|
|
|
|
+ [HttpGet]
|
|
|
+ public async Task<ActionResult> RepairDeeplinkReportStats(
|
|
|
+ DateTime startDate = default,
|
|
|
+ DateTime endDate = default,
|
|
|
+ bool dryRun = true,
|
|
|
+ bool useLegacyRedis = false,
|
|
|
+ bool allowPartialSources = false,
|
|
|
+ bool allowCurrentDate = false,
|
|
|
+ string fallbackEndpoint = "",
|
|
|
+ int commandTimeoutSeconds = 600)
|
|
|
+ {
|
|
|
+ var yesterday = DateTime.Now.Date.AddDays(-1);
|
|
|
+ if (startDate == default) startDate = yesterday;
|
|
|
+ if (endDate == default) endDate = startDate;
|
|
|
+
|
|
|
+ startDate = startDate.Date;
|
|
|
+ endDate = endDate.Date;
|
|
|
+ commandTimeoutSeconds = Math.Clamp(commandTimeoutSeconds, 30, 3600);
|
|
|
+
|
|
|
+ if (endDate < startDate)
|
|
|
+ {
|
|
|
+ return new APIResult(new { success = false, message = "endDate 不能早于 startDate" });
|
|
|
+ }
|
|
|
+
|
|
|
+ int requestedDays = (endDate - startDate).Days + 1;
|
|
|
+ if (requestedDays > DeeplinkReportRepairMaxDays)
|
|
|
+ {
|
|
|
+ return new APIResult(new
|
|
|
+ {
|
|
|
+ success = false,
|
|
|
+ message = $"单次最多重建 {DeeplinkReportRepairMaxDays} 天数据"
|
|
|
+ });
|
|
|
+ }
|
|
|
+
|
|
|
+ if (!allowCurrentDate && endDate >= DateTime.Now.Date)
|
|
|
+ {
|
|
|
+ return new APIResult(new
|
|
|
+ {
|
|
|
+ success = false,
|
|
|
+ message = "默认禁止重建当天数据，避免覆盖实时计数；如确需执行请传 allowCurrentDate=true"
|
|
|
+ });
|
|
|
+ }
|
|
|
+
|
|
|
+ var endpoints = (EndPointCore.List(true) ?? Enumerable.Empty<EndPointDTO>())
|
|
|
+ .Where(node => node.status && node.is_public_api)
|
|
|
+ .Where(node => !string.IsNullOrWhiteSpace(EndPointCore.GetRedisServer(node)))
|
|
|
+ .GroupBy(node => node.name, StringComparer.OrdinalIgnoreCase)
|
|
|
+ .Select(group => group.First())
|
|
|
+ .OrderBy(node => node.name, StringComparer.OrdinalIgnoreCase)
|
|
|
+ .ToList();
|
|
|
+ if (endpoints.Count == 0)
|
|
|
+ {
|
|
|
+ return new APIResult(new { success = false, message = "没有可用的公共 API Redis 节点" });
|
|
|
+ }
|
|
|
+
|
|
|
+ EndPointDTO fallbackNode;
|
|
|
+ if (!string.IsNullOrWhiteSpace(fallbackEndpoint))
|
|
|
+ {
|
|
|
+ fallbackNode = endpoints.FirstOrDefault(node => node.name.Equals(
|
|
|
+ fallbackEndpoint.Trim(),
|
|
|
+ StringComparison.OrdinalIgnoreCase
|
|
|
+ ))!;
|
|
|
+ if (fallbackNode == null)
|
|
|
+ {
|
|
|
+ return new APIResult(new
|
|
|
+ {
|
|
|
+ success = false,
|
|
|
+ message = $"fallbackEndpoint={fallbackEndpoint} 不在可用公共节点中",
|
|
|
+ availableEndpoints = endpoints.Select(node => node.name).ToList()
|
|
|
+ });
|
|
|
+ }
|
|
|
+ }
|
|
|
+ else
|
|
|
+ {
|
|
|
+ fallbackNode = endpoints.FirstOrDefault(node => node.name.Equals(
|
|
|
+ EndPointCore.CurrentEndPoint,
|
|
|
+ StringComparison.OrdinalIgnoreCase
|
|
|
+ )) ?? endpoints[0];
|
|
|
+ }
|
|
|
+
|
|
|
+ var endpointNames = endpoints.ToDictionary(
|
|
|
+ node => node.name,
|
|
|
+ node => node.name,
|
|
|
+ StringComparer.OrdinalIgnoreCase
|
|
|
+ );
|
|
|
+ var knownChannelNames = GetKnownDeeplinkReportChannelNames();
|
|
|
+ var repairDates = new List<DeeplinkReportRepairDate>();
|
|
|
+ var warnings = new List<object>();
|
|
|
+ var errors = new List<object>();
|
|
|
+
|
|
|
+ if (useLegacyRedis)
|
|
|
+ {
|
|
|
+ return await RepairDeeplinkReportStatsFromLegacyRedisAsync(
|
|
|
+ endpoints,
|
|
|
+ knownChannelNames,
|
|
|
+ startDate,
|
|
|
+ endDate,
|
|
|
+ dryRun
|
|
|
+ );
|
|
|
+ }
|
|
|
+
|
|
|
+ var parseAdminEndpoint = EndPointCore.GetParseAdmin();
|
|
|
+ bool usesConfiguredParseDatabase = parseAdminEndpoint != null &&
|
|
|
+ !string.IsNullOrWhiteSpace(parseAdminEndpoint.db_server);
|
|
|
+ using var connection = usesConfiguredParseDatabase
|
|
|
+ ? EndPointCore.GetDbConnection(parseAdminEndpoint!.db_server)
|
|
|
+ : DBContext.GetOpenConnection();
|
|
|
+ if (connection.State != ConnectionState.Open) connection.Open();
|
|
|
+
|
|
|
+ foreach (var reportDate in EachDay(startDate, endDate))
|
|
|
+ {
|
|
|
+ string dateKey = reportDate.ToString("yyyyMMdd");
|
|
|
+ string toolTableName = $"tool_parse_logs_{dateKey}";
|
|
|
+ string deeplinkTableName = $"deeplink_parse_logs_{dateKey}";
|
|
|
+ bool toolTableExists = TableExists(connection, toolTableName, commandTimeoutSeconds);
|
|
|
+ bool deeplinkTableExists = TableExists(connection, deeplinkTableName, commandTimeoutSeconds);
|
|
|
+
|
|
|
+ if (!toolTableExists && !deeplinkTableExists)
|
|
|
+ {
|
|
|
+ warnings.Add(new
|
|
|
+ {
|
|
|
+ date = reportDate.ToString("yyyy-MM-dd"),
|
|
|
+ message = "两个来源日表都不存在，已跳过",
|
|
|
+ toolTableName,
|
|
|
+ deeplinkTableName
|
|
|
+ });
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (!allowPartialSources && (!toolTableExists || !deeplinkTableExists))
|
|
|
+ {
|
|
|
+ warnings.Add(new
|
|
|
+ {
|
|
|
+ date = reportDate.ToString("yyyy-MM-dd"),
|
|
|
+ message = "来源日表不完整，已跳过；确认缺失表确实无数据后可传 allowPartialSources=true",
|
|
|
+ toolTableName,
|
|
|
+ toolTableExists,
|
|
|
+ deeplinkTableName,
|
|
|
+ deeplinkTableExists
|
|
|
+ });
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+
|
|
|
+ try
|
|
|
+ {
|
|
|
+ var sourceCounts = new List<DeeplinkReportRepairSourceCount>();
|
|
|
+ if (toolTableExists)
|
|
|
+ {
|
|
|
+ var toolCounts = SqlMapper.Query<DeeplinkReportToolSourceRow>(
|
|
|
+ connection,
|
|
|
+ $@"
|
|
|
+SELECT
|
|
|
+ COALESCE(end_point, '') end_point,
|
|
|
+ channel,
|
|
|
+ COUNT(*) total_count,
|
|
|
+ CAST(COALESCE(SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END), 0) AS SIGNED) success_count
|
|
|
+FROM {toolTableName}
|
|
|
+GROUP BY COALESCE(end_point, ''), channel",
|
|
|
+ commandTimeout: commandTimeoutSeconds
|
|
|
+ );
|
|
|
+
|
|
|
+ sourceCounts.AddRange(toolCounts.Select(item => new DeeplinkReportRepairSourceCount
|
|
|
+ {
|
|
|
+ source = toolTableName,
|
|
|
+ source_endpoint = item.end_point?.Trim() ?? string.Empty,
|
|
|
+ channel_name = GetDeeplinkReportChannelName(item.channel),
|
|
|
+ total_count = item.total_count,
|
|
|
+ success_count = item.success_count,
|
|
|
+ fail_count = Math.Max(0, item.total_count - item.success_count)
|
|
|
+ }));
|
|
|
+ }
|
|
|
+
|
|
|
+ if (deeplinkTableExists)
|
|
|
+ {
|
|
|
+ var deeplinkCounts = SqlMapper.Query<DeeplinkReportNamedSourceRow>(
|
|
|
+ connection,
|
|
|
+ $@"
|
|
|
+SELECT
|
|
|
+ COALESCE(end_point, '') end_point,
|
|
|
+ COALESCE(NULLIF(TRIM(channel_name), ''), 'unknown') channel_name,
|
|
|
+ COUNT(*) total_count,
|
|
|
+ CAST(COALESCE(SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END), 0) AS SIGNED) success_count
|
|
|
+FROM {deeplinkTableName}
|
|
|
+GROUP BY COALESCE(end_point, ''), COALESCE(NULLIF(TRIM(channel_name), ''), 'unknown')",
|
|
|
+ commandTimeout: commandTimeoutSeconds
|
|
|
+ );
|
|
|
+
|
|
|
+ sourceCounts.AddRange(deeplinkCounts.Select(item => new DeeplinkReportRepairSourceCount
|
|
|
+ {
|
|
|
+ source = deeplinkTableName,
|
|
|
+ source_endpoint = item.end_point?.Trim() ?? string.Empty,
|
|
|
+ channel_name = NormalizeDeeplinkReportChannelName(item.channel_name),
|
|
|
+ total_count = item.total_count,
|
|
|
+ success_count = item.success_count,
|
|
|
+ fail_count = Math.Max(0, item.total_count - item.success_count)
|
|
|
+ }));
|
|
|
+ }
|
|
|
+
|
|
|
+ var mergedCounts = new Dictionary<string, DeeplinkReportRepairCount>(StringComparer.OrdinalIgnoreCase);
|
|
|
+ var remappedSources = new List<object>();
|
|
|
+ foreach (var sourceCount in sourceCounts)
|
|
|
+ {
|
|
|
+ bool endpointMatched = endpointNames.TryGetValue(sourceCount.source_endpoint, out var endpointName);
|
|
|
+ endpointName ??= fallbackNode.name;
|
|
|
+ if (!endpointMatched)
|
|
|
+ {
|
|
|
+ remappedSources.Add(new
|
|
|
+ {
|
|
|
+ sourceCount.source,
|
|
|
+ sourceEndpoint = sourceCount.source_endpoint,
|
|
|
+ mappedEndpoint = endpointName,
|
|
|
+ sourceCount.channel_name,
|
|
|
+ sourceCount.total_count
|
|
|
+ });
|
|
|
+ }
|
|
|
+
|
|
|
+ string mergedKey = $"{endpointName}\u001f{sourceCount.channel_name}";
|
|
|
+ if (!mergedCounts.TryGetValue(mergedKey, out var mergedCount))
|
|
|
+ {
|
|
|
+ mergedCount = new DeeplinkReportRepairCount
|
|
|
+ {
|
|
|
+ endpoint_name = endpointName,
|
|
|
+ channel_name = sourceCount.channel_name
|
|
|
+ };
|
|
|
+ mergedCounts[mergedKey] = mergedCount;
|
|
|
+ }
|
|
|
+
|
|
|
+ mergedCount.total_count += sourceCount.total_count;
|
|
|
+ mergedCount.success_count += sourceCount.success_count;
|
|
|
+ mergedCount.fail_count += sourceCount.fail_count;
|
|
|
+ knownChannelNames.Add(sourceCount.channel_name);
|
|
|
+ }
|
|
|
+
|
|
|
+ var dateCounts = mergedCounts.Values
|
|
|
+ .OrderBy(item => item.endpoint_name, StringComparer.OrdinalIgnoreCase)
|
|
|
+ .ThenByDescending(item => item.total_count)
|
|
|
+ .ThenBy(item => item.channel_name, StringComparer.OrdinalIgnoreCase)
|
|
|
+ .ToList();
|
|
|
+ var repairDate = new DeeplinkReportRepairDate
|
|
|
+ {
|
|
|
+ report_date = reportDate,
|
|
|
+ tool_table_exists = toolTableExists,
|
|
|
+ deeplink_table_exists = deeplinkTableExists,
|
|
|
+ counts = dateCounts,
|
|
|
+ remapped_sources = remappedSources
|
|
|
+ };
|
|
|
+ repairDates.Add(repairDate);
|
|
|
+ }
|
|
|
+ catch (Exception ex)
|
|
|
+ {
|
|
|
+ errors.Add(new
|
|
|
+ {
|
|
|
+ scope = "mysql_aggregate",
|
|
|
+ date = reportDate.ToString("yyyy-MM-dd"),
|
|
|
+ error = FormatRepairError(ex)
|
|
|
+ });
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ var redisReports = new List<object>();
|
|
|
+ foreach (var endpoint in endpoints)
|
|
|
+ {
|
|
|
+ var endpointReports = new List<object>();
|
|
|
+ string redisServer = EndPointCore.GetRedisServer(endpoint);
|
|
|
+
|
|
|
+ try
|
|
|
+ {
|
|
|
+ await using var scope = await RedisClientFactory.CreateScopeAsync(redisServer);
|
|
|
+ var redis = scope.Client;
|
|
|
+
|
|
|
+ foreach (var repairDate in repairDates)
|
|
|
+ {
|
|
|
+ string dateKey = repairDate.report_date.ToString("yyyyMMdd");
|
|
|
+ var endpointCounts = repairDate.counts
|
|
|
+ .Where(item => item.endpoint_name.Equals(endpoint.name, StringComparison.OrdinalIgnoreCase))
|
|
|
+ .ToList();
|
|
|
+ var expectedParentCount = SumDeeplinkReportRepairCounts(endpointCounts);
|
|
|
+ var beforeParentCount = await ReadDeeplinkReportBucketAsync(redis, "tool", dateKey);
|
|
|
+ string channelIndexKey = $":parse_total:tool:channels:{dateKey}";
|
|
|
+ var indexedChannelNames = await redis.SMembersAsync<string>(channelIndexKey) ?? [];
|
|
|
+ var channelsToClear = new HashSet<string>(knownChannelNames, StringComparer.OrdinalIgnoreCase);
|
|
|
+ channelsToClear.UnionWith(indexedChannelNames);
|
|
|
+ channelsToClear.UnionWith(repairDate.counts
|
|
|
+ .Select(item => item.channel_name)
|
|
|
+ .Where(item => !string.IsNullOrWhiteSpace(item)));
|
|
|
+
|
|
|
+ if (!dryRun)
|
|
|
+ {
|
|
|
+ await redis.DelAsync(channelIndexKey);
|
|
|
+ foreach (var channelToClear in channelsToClear)
|
|
|
+ {
|
|
|
+ await DeleteDeeplinkReportBucketAsync(
|
|
|
+ redis,
|
|
|
+ $"tool:channel:{channelToClear}",
|
|
|
+ dateKey
|
|
|
+ );
|
|
|
+ }
|
|
|
+
|
|
|
+ await WriteDeeplinkReportBucketAsync(redis, "tool", dateKey, expectedParentCount);
|
|
|
+ foreach (var endpointCount in endpointCounts)
|
|
|
+ {
|
|
|
+ await WriteDeeplinkReportBucketAsync(
|
|
|
+ redis,
|
|
|
+ $"tool:channel:{endpointCount.channel_name}",
|
|
|
+ dateKey,
|
|
|
+ endpointCount
|
|
|
+ );
|
|
|
+ }
|
|
|
+
|
|
|
+ var endpointChannelNames = endpointCounts
|
|
|
+ .Where(item => item.total_count > 0)
|
|
|
+ .Select(item => item.channel_name)
|
|
|
+ .Distinct(StringComparer.OrdinalIgnoreCase)
|
|
|
+ .ToArray();
|
|
|
+ if (endpointChannelNames.Length > 0)
|
|
|
+ {
|
|
|
+ await redis.SAddAsync(channelIndexKey, endpointChannelNames);
|
|
|
+ await redis.ExpireAsync(channelIndexKey, DeeplinkReportStatsExpireSeconds);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ var afterParentCount = dryRun
|
|
|
+ ? null
|
|
|
+ : await ReadDeeplinkReportBucketAsync(redis, "tool", dateKey);
|
|
|
+ endpointReports.Add(new
|
|
|
+ {
|
|
|
+ date = repairDate.report_date.ToString("yyyy-MM-dd"),
|
|
|
+ before = beforeParentCount,
|
|
|
+ expected = expectedParentCount,
|
|
|
+ after = afterParentCount,
|
|
|
+ channels = endpointCounts,
|
|
|
+ clearedChannelCount = channelsToClear.Count
|
|
|
+ });
|
|
|
+ }
|
|
|
+ }
|
|
|
+ catch (Exception ex)
|
|
|
+ {
|
|
|
+ errors.Add(new
|
|
|
+ {
|
|
|
+ scope = "redis_write",
|
|
|
+ endpoint = endpoint.name,
|
|
|
+ error = FormatRepairError(ex)
|
|
|
+ });
|
|
|
+ }
|
|
|
+
|
|
|
+ redisReports.Add(new
|
|
|
+ {
|
|
|
+ endpoint = endpoint.name,
|
|
|
+ endpoint.description,
|
|
|
+ reports = endpointReports
|
|
|
+ });
|
|
|
+ }
|
|
|
+
|
|
|
+ var dailyReports = repairDates.Select(item => new
|
|
|
+ {
|
|
|
+ date = item.report_date.ToString("yyyy-MM-dd"),
|
|
|
+ item.tool_table_exists,
|
|
|
+ item.deeplink_table_exists,
|
|
|
+ source = new
|
|
|
+ {
|
|
|
+ total_count = item.counts.Sum(count => count.total_count),
|
|
|
+ success_count = item.counts.Sum(count => count.success_count),
|
|
|
+ fail_count = item.counts.Sum(count => count.fail_count)
|
|
|
+ },
|
|
|
+ channels = item.counts
|
|
|
+ .GroupBy(count => count.channel_name, StringComparer.OrdinalIgnoreCase)
|
|
|
+ .Select(group => new
|
|
|
+ {
|
|
|
+ channel_name = group.Key,
|
|
|
+ total_count = group.Sum(count => count.total_count),
|
|
|
+ success_count = group.Sum(count => count.success_count),
|
|
|
+ fail_count = group.Sum(count => count.fail_count)
|
|
|
+ })
|
|
|
+ .OrderByDescending(count => count.total_count)
|
|
|
+ .ThenBy(count => count.channel_name, StringComparer.OrdinalIgnoreCase)
|
|
|
+ .ToList(),
|
|
|
+ endpoint_channels = item.counts,
|
|
|
+ item.remapped_sources
|
|
|
+ }).ToList();
|
|
|
+
|
|
|
+ return new APIResult(new
|
|
|
+ {
|
|
|
+ success = errors.Count == 0,
|
|
|
+ dryRun,
|
|
|
+ allowPartialSources,
|
|
|
+ allowCurrentDate,
|
|
|
+ startDate = startDate.ToString("yyyy-MM-dd"),
|
|
|
+ endDate = endDate.ToString("yyyy-MM-dd"),
|
|
|
+ fallbackEndpoint = fallbackNode.name,
|
|
|
+ sourceDatabase = new
|
|
|
+ {
|
|
|
+ endpoint = usesConfiguredParseDatabase ? parseAdminEndpoint!.name : EndPointCore.CurrentEndPoint,
|
|
|
+ configured = usesConfiguredParseDatabase
|
|
|
+ },
|
|
|
+ processedDays = repairDates.Count,
|
|
|
+ skippedDays = requestedDays - repairDates.Count,
|
|
|
+ dailyReports,
|
|
|
+ redis = redisReports,
|
|
|
+ warnings,
|
|
|
+ errors
|
|
|
+ });
|
|
|
+ }
|
|
|
+
|
|
|
[HttpGet]
|
|
|
public async Task<ActionResult> RepairTkDailyAccountStats(
|
|
|
DateTime startDate = default,
|
|
|
@@ -817,6 +1222,363 @@ ORDER BY l.accountId, l.end_point",
|
|
|
});
|
|
|
}
|
|
|
|
|
|
+ private static HashSet<string> GetKnownDeeplinkReportChannelNames()
|
|
|
+ {
|
|
|
+ var channelNames = Enum.GetNames(typeof(TkChannelEnum))
|
|
|
+ .ToHashSet(StringComparer.OrdinalIgnoreCase);
|
|
|
+
|
|
|
+ try
|
|
|
+ {
|
|
|
+ var rules = new DBContext.Table("deeplink_parse_rule")
|
|
|
+ .Select<DeeplinkParseRuleDTO>() ?? Enumerable.Empty<DeeplinkParseRuleDTO>();
|
|
|
+ foreach (var rule in rules)
|
|
|
+ {
|
|
|
+ if (!string.IsNullOrWhiteSpace(rule.channel_name))
|
|
|
+ {
|
|
|
+ channelNames.Add(rule.channel_name.Trim());
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ catch
|
|
|
+ {
|
|
|
+ // The source log rows and Redis channel index still provide enough information.
|
|
|
+ }
|
|
|
+
|
|
|
+ return channelNames;
|
|
|
+ }
|
|
|
+
|
|
|
+ private static async Task<ActionResult> RepairDeeplinkReportStatsFromLegacyRedisAsync(
|
|
|
+ List<EndPointDTO> endpoints,
|
|
|
+ HashSet<string> knownChannelNames,
|
|
|
+ DateTime startDate,
|
|
|
+ DateTime endDate,
|
|
|
+ bool dryRun)
|
|
|
+ {
|
|
|
+ var sourceChannelNames = knownChannelNames
|
|
|
+ .Where(IsLegacyDeeplinkReportSourceChannel)
|
|
|
+ .OrderBy(channelName => channelName, StringComparer.OrdinalIgnoreCase)
|
|
|
+ .ToList();
|
|
|
+ var requestedDates = EachDay(startDate, endDate).ToList();
|
|
|
+ var countsByDate = requestedDates.ToDictionary(
|
|
|
+ reportDate => reportDate,
|
|
|
+ _ => new List<DeeplinkReportRepairCount>()
|
|
|
+ );
|
|
|
+ var endpointReports = new List<object>();
|
|
|
+ var warnings = new List<object>();
|
|
|
+ var errors = new List<object>();
|
|
|
+
|
|
|
+ foreach (var endpoint in endpoints)
|
|
|
+ {
|
|
|
+ var dateReports = new List<object>();
|
|
|
+ string redisServer = EndPointCore.GetRedisServer(endpoint);
|
|
|
+
|
|
|
+ try
|
|
|
+ {
|
|
|
+ await using var scope = await RedisClientFactory.CreateScopeAsync(redisServer);
|
|
|
+ var redis = scope.Client;
|
|
|
+
|
|
|
+ foreach (var reportDate in requestedDates)
|
|
|
+ {
|
|
|
+ string dateKey = reportDate.ToString("yyyyMMdd");
|
|
|
+ var readTasks = sourceChannelNames.Select(async channelName =>
|
|
|
+ {
|
|
|
+ var legacyCount = await ReadDeeplinkReportBucketAsync(redis, channelName, dateKey);
|
|
|
+ legacyCount.endpoint_name = endpoint.name;
|
|
|
+ legacyCount.channel_name = channelName;
|
|
|
+ return legacyCount;
|
|
|
+ });
|
|
|
+ var legacyCounts = (await Task.WhenAll(readTasks))
|
|
|
+ .Where(count => count.total_count > 0 || count.success_count > 0 || count.fail_count > 0)
|
|
|
+ .OrderByDescending(count => count.total_count)
|
|
|
+ .ThenBy(count => count.channel_name, StringComparer.OrdinalIgnoreCase)
|
|
|
+ .ToList();
|
|
|
+ var expectedParentCount = SumDeeplinkReportRepairCounts(legacyCounts);
|
|
|
+ var beforeParentCount = await ReadDeeplinkReportBucketAsync(redis, "tool", dateKey);
|
|
|
+
|
|
|
+ if (expectedParentCount.total_count <= 0)
|
|
|
+ {
|
|
|
+ dateReports.Add(new
|
|
|
+ {
|
|
|
+ date = reportDate.ToString("yyyy-MM-dd"),
|
|
|
+ sourceFound = false,
|
|
|
+ before = beforeParentCount,
|
|
|
+ message = "未找到旧渠道统计键，未执行清理或覆盖"
|
|
|
+ });
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+
|
|
|
+ countsByDate[reportDate].AddRange(legacyCounts);
|
|
|
+ string channelIndexKey = $":parse_total:tool:channels:{dateKey}";
|
|
|
+ var indexedChannelNames = await redis.SMembersAsync<string>(channelIndexKey) ?? [];
|
|
|
+ var channelsToClear = new HashSet<string>(sourceChannelNames, StringComparer.OrdinalIgnoreCase);
|
|
|
+ channelsToClear.UnionWith(indexedChannelNames);
|
|
|
+
|
|
|
+ if (!dryRun)
|
|
|
+ {
|
|
|
+ await redis.DelAsync(channelIndexKey);
|
|
|
+ foreach (var channelToClear in channelsToClear)
|
|
|
+ {
|
|
|
+ await DeleteDeeplinkReportBucketAsync(
|
|
|
+ redis,
|
|
|
+ $"tool:channel:{channelToClear}",
|
|
|
+ dateKey
|
|
|
+ );
|
|
|
+ }
|
|
|
+
|
|
|
+ await WriteDeeplinkReportBucketAsync(redis, "tool", dateKey, expectedParentCount);
|
|
|
+ foreach (var legacyCount in legacyCounts)
|
|
|
+ {
|
|
|
+ await WriteDeeplinkReportBucketAsync(
|
|
|
+ redis,
|
|
|
+ $"tool:channel:{legacyCount.channel_name}",
|
|
|
+ dateKey,
|
|
|
+ legacyCount
|
|
|
+ );
|
|
|
+ }
|
|
|
+
|
|
|
+ string[] populatedChannelNames = legacyCounts
|
|
|
+ .Where(count => count.total_count > 0)
|
|
|
+ .Select(count => count.channel_name)
|
|
|
+ .Distinct(StringComparer.OrdinalIgnoreCase)
|
|
|
+ .ToArray();
|
|
|
+ if (populatedChannelNames.Length > 0)
|
|
|
+ {
|
|
|
+ await redis.SAddAsync(channelIndexKey, populatedChannelNames);
|
|
|
+ await redis.ExpireAsync(channelIndexKey, DeeplinkReportStatsExpireSeconds);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ var afterParentCount = dryRun
|
|
|
+ ? null
|
|
|
+ : await ReadDeeplinkReportBucketAsync(redis, "tool", dateKey);
|
|
|
+ dateReports.Add(new
|
|
|
+ {
|
|
|
+ date = reportDate.ToString("yyyy-MM-dd"),
|
|
|
+ sourceFound = true,
|
|
|
+ before = beforeParentCount,
|
|
|
+ expected = expectedParentCount,
|
|
|
+ after = afterParentCount,
|
|
|
+ channels = legacyCounts,
|
|
|
+ sourceKeysPreserved = true
|
|
|
+ });
|
|
|
+ }
|
|
|
+ }
|
|
|
+ catch (Exception ex)
|
|
|
+ {
|
|
|
+ errors.Add(new
|
|
|
+ {
|
|
|
+ scope = "legacy_redis",
|
|
|
+ endpoint = endpoint.name,
|
|
|
+ error = FormatRepairError(ex)
|
|
|
+ });
|
|
|
+ }
|
|
|
+
|
|
|
+ endpointReports.Add(new
|
|
|
+ {
|
|
|
+ endpoint = endpoint.name,
|
|
|
+ endpoint.description,
|
|
|
+ reports = dateReports
|
|
|
+ });
|
|
|
+ }
|
|
|
+
|
|
|
+ foreach (var reportDate in requestedDates.Where(date => countsByDate[date].Count == 0))
|
|
|
+ {
|
|
|
+ warnings.Add(new
|
|
|
+ {
|
|
|
+ date = reportDate.ToString("yyyy-MM-dd"),
|
|
|
+ message = "所有节点均未找到旧渠道统计，不会修改该日数据"
|
|
|
+ });
|
|
|
+ }
|
|
|
+
|
|
|
+ var processedDates = requestedDates
|
|
|
+ .Where(date => countsByDate[date].Count > 0)
|
|
|
+ .ToList();
|
|
|
+ var dailyReports = processedDates.Select(reportDate =>
|
|
|
+ {
|
|
|
+ var endpointChannelCounts = countsByDate[reportDate];
|
|
|
+ var channelCounts = endpointChannelCounts
|
|
|
+ .GroupBy(count => count.channel_name, StringComparer.OrdinalIgnoreCase)
|
|
|
+ .Select(group => new
|
|
|
+ {
|
|
|
+ channel_name = group.Key,
|
|
|
+ total_count = group.Sum(count => count.total_count),
|
|
|
+ success_count = group.Sum(count => count.success_count),
|
|
|
+ fail_count = group.Sum(count => count.fail_count)
|
|
|
+ })
|
|
|
+ .OrderByDescending(count => count.total_count)
|
|
|
+ .ThenBy(count => count.channel_name, StringComparer.OrdinalIgnoreCase)
|
|
|
+ .ToList();
|
|
|
+
|
|
|
+ return new
|
|
|
+ {
|
|
|
+ date = reportDate.ToString("yyyy-MM-dd"),
|
|
|
+ total_count = channelCounts.Sum(count => count.total_count),
|
|
|
+ success_count = channelCounts.Sum(count => count.success_count),
|
|
|
+ fail_count = channelCounts.Sum(count => count.fail_count),
|
|
|
+ channels = channelCounts,
|
|
|
+ endpoint_channels = endpointChannelCounts
|
|
|
+ };
|
|
|
+ }).ToList();
|
|
|
+
|
|
|
+ return new APIResult(new
|
|
|
+ {
|
|
|
+ success = errors.Count == 0,
|
|
|
+ dryRun,
|
|
|
+ useLegacyRedis = true,
|
|
|
+ sourceMode = "legacyRedis",
|
|
|
+ sourceKeysPreserved = true,
|
|
|
+ parentTotalRule = "sum(channel totals)",
|
|
|
+ startDate = startDate.ToString("yyyy-MM-dd"),
|
|
|
+ endDate = endDate.ToString("yyyy-MM-dd"),
|
|
|
+ processedDays = processedDates.Count,
|
|
|
+ skippedDays = requestedDates.Count - processedDates.Count,
|
|
|
+ sourceChannels = sourceChannelNames,
|
|
|
+ dailyReports,
|
|
|
+ redis = endpointReports,
|
|
|
+ warnings,
|
|
|
+ errors
|
|
|
+ });
|
|
|
+ }
|
|
|
+
|
|
|
+ private static bool IsLegacyDeeplinkReportSourceChannel(string channelName)
|
|
|
+ {
|
|
|
+ if (string.IsNullOrWhiteSpace(channelName)) return false;
|
|
|
+
|
|
|
+ return !channelName.Equals("all", StringComparison.OrdinalIgnoreCase) &&
|
|
|
+ !channelName.Equals("tool", StringComparison.OrdinalIgnoreCase) &&
|
|
|
+ !channelName.Equals("unknown", StringComparison.OrdinalIgnoreCase) &&
|
|
|
+ !channelName.Equals("legacy_unclassified", StringComparison.OrdinalIgnoreCase);
|
|
|
+ }
|
|
|
+
|
|
|
+ private static string GetDeeplinkReportChannelName(int channelValue)
|
|
|
+ {
|
|
|
+ if (Enum.IsDefined(typeof(TkChannelEnum), channelValue))
|
|
|
+ {
|
|
|
+ return ((TkChannelEnum)channelValue).ToString();
|
|
|
+ }
|
|
|
+
|
|
|
+ return $"channel_{channelValue}";
|
|
|
+ }
|
|
|
+
|
|
|
+ private static string NormalizeDeeplinkReportChannelName(string channelName)
|
|
|
+ {
|
|
|
+ return string.IsNullOrWhiteSpace(channelName) ? "unknown" : channelName.Trim();
|
|
|
+ }
|
|
|
+
|
|
|
+ private static DeeplinkReportRepairCount SumDeeplinkReportRepairCounts(
|
|
|
+ IEnumerable<DeeplinkReportRepairCount> counts)
|
|
|
+ {
|
|
|
+ return new DeeplinkReportRepairCount
|
|
|
+ {
|
|
|
+ total_count = counts.Sum(item => item.total_count),
|
|
|
+ success_count = counts.Sum(item => item.success_count),
|
|
|
+ fail_count = counts.Sum(item => item.fail_count)
|
|
|
+ };
|
|
|
+ }
|
|
|
+
|
|
|
+ private static async Task<DeeplinkReportRepairCount> ReadDeeplinkReportBucketAsync(
|
|
|
+ YunhuiKit.RedisClient redis,
|
|
|
+ string bucket,
|
|
|
+ string dateKey)
|
|
|
+ {
|
|
|
+ return new DeeplinkReportRepairCount
|
|
|
+ {
|
|
|
+ total_count = await redis.GetAsync<long>($":parse_total:{bucket}:{dateKey}"),
|
|
|
+ success_count = await redis.GetAsync<long>($":parse_total:{bucket}:success:{dateKey}"),
|
|
|
+ fail_count = await redis.GetAsync<long>($":parse_total:{bucket}:fail:{dateKey}")
|
|
|
+ };
|
|
|
+ }
|
|
|
+
|
|
|
+ private static Task<long> DeleteDeeplinkReportBucketAsync(
|
|
|
+ YunhuiKit.RedisClient redis,
|
|
|
+ string bucket,
|
|
|
+ string dateKey)
|
|
|
+ {
|
|
|
+ return redis.DelAsync(
|
|
|
+ $":parse_total:{bucket}:{dateKey}",
|
|
|
+ $":parse_total:{bucket}:success:{dateKey}",
|
|
|
+ $":parse_total:{bucket}:fail:{dateKey}"
|
|
|
+ );
|
|
|
+ }
|
|
|
+
|
|
|
+ private static async Task WriteDeeplinkReportBucketAsync(
|
|
|
+ YunhuiKit.RedisClient redis,
|
|
|
+ string bucket,
|
|
|
+ string dateKey,
|
|
|
+ DeeplinkReportRepairCount counts)
|
|
|
+ {
|
|
|
+ if (counts.total_count <= 0)
|
|
|
+ {
|
|
|
+ await DeleteDeeplinkReportBucketAsync(redis, bucket, dateKey);
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ bool totalWritten = await redis.SetAsync(
|
|
|
+ $":parse_total:{bucket}:{dateKey}",
|
|
|
+ counts.total_count,
|
|
|
+ DeeplinkReportStatsExpireSeconds
|
|
|
+ );
|
|
|
+ bool successWritten = await redis.SetAsync(
|
|
|
+ $":parse_total:{bucket}:success:{dateKey}",
|
|
|
+ counts.success_count,
|
|
|
+ DeeplinkReportStatsExpireSeconds
|
|
|
+ );
|
|
|
+ bool failWritten = await redis.SetAsync(
|
|
|
+ $":parse_total:{bucket}:fail:{dateKey}",
|
|
|
+ counts.fail_count,
|
|
|
+ DeeplinkReportStatsExpireSeconds
|
|
|
+ );
|
|
|
+
|
|
|
+ if (!totalWritten || !successWritten || !failWritten)
|
|
|
+ {
|
|
|
+ throw new InvalidOperationException($"Redis 写入失败：bucket={bucket}, date={dateKey}");
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private sealed class DeeplinkReportToolSourceRow
|
|
|
+ {
|
|
|
+ public string end_point { get; set; } = string.Empty;
|
|
|
+ public int channel { get; set; }
|
|
|
+ public long total_count { get; set; }
|
|
|
+ public long success_count { get; set; }
|
|
|
+ }
|
|
|
+
|
|
|
+ private sealed class DeeplinkReportNamedSourceRow
|
|
|
+ {
|
|
|
+ public string end_point { get; set; } = string.Empty;
|
|
|
+ public string channel_name { get; set; } = string.Empty;
|
|
|
+ public long total_count { get; set; }
|
|
|
+ public long success_count { get; set; }
|
|
|
+ }
|
|
|
+
|
|
|
+ private sealed class DeeplinkReportRepairSourceCount
|
|
|
+ {
|
|
|
+ public string source { get; set; } = string.Empty;
|
|
|
+ public string source_endpoint { get; set; } = string.Empty;
|
|
|
+ public string channel_name { get; set; } = string.Empty;
|
|
|
+ public long total_count { get; set; }
|
|
|
+ public long success_count { get; set; }
|
|
|
+ public long fail_count { get; set; }
|
|
|
+ }
|
|
|
+
|
|
|
+ private sealed class DeeplinkReportRepairCount
|
|
|
+ {
|
|
|
+ public string endpoint_name { get; set; } = string.Empty;
|
|
|
+ public string channel_name { get; set; } = string.Empty;
|
|
|
+ public long total_count { get; set; }
|
|
|
+ public long success_count { get; set; }
|
|
|
+ public long fail_count { get; set; }
|
|
|
+ }
|
|
|
+
|
|
|
+ private sealed class DeeplinkReportRepairDate
|
|
|
+ {
|
|
|
+ public DateTime report_date { get; set; }
|
|
|
+ public bool tool_table_exists { get; set; }
|
|
|
+ public bool deeplink_table_exists { get; set; }
|
|
|
+ public List<DeeplinkReportRepairCount> counts { get; set; } = [];
|
|
|
+ public List<object> remapped_sources { get; set; } = [];
|
|
|
+ }
|
|
|
+
|
|
|
private static IEnumerable<DateTime> EachDay(DateTime startDate, DateTime endDate)
|
|
|
{
|
|
|
for (var date = startDate.Date; date <= endDate.Date; date = date.AddDays(1))
|
dodo hold