|
|
@@ -0,0 +1,467 @@
|
|
|
+using dodohold.core;
|
|
|
+using System.Net;
|
|
|
+using System.Globalization;
|
|
|
+using YunhuiKit;
|
|
|
+
|
|
|
+namespace molilian.core
|
|
|
+{
|
|
|
+ public static class ZlongForwardCore
|
|
|
+ {
|
|
|
+ public const string RedisPrefix = ":union_parse:xhs_zlong_forward";
|
|
|
+ public const string ExceptionRedisPrefix = $"{RedisPrefix}:exception";
|
|
|
+ private const string ExceptionAggregateType = "all";
|
|
|
+ private const string ForwardUrlTemplate = "https://tracker.z-long.cn/ad/tracker/special/liutiyun/{0}";
|
|
|
+ private static readonly string[] SupportedChannels = new[] { "xhs", "ks" };
|
|
|
+ private static readonly HttpClient HttpClient = new()
|
|
|
+ {
|
|
|
+ Timeout = TimeSpan.FromSeconds(10)
|
|
|
+ };
|
|
|
+
|
|
|
+ public static async Task ForwardAsync(string channel, string oaid)
|
|
|
+ {
|
|
|
+ string normalizedChannel = NormalizeChannel(channel);
|
|
|
+ if (string.IsNullOrEmpty(normalizedChannel)) return;
|
|
|
+
|
|
|
+ string requestUrl = string.Empty;
|
|
|
+ try
|
|
|
+ {
|
|
|
+ long unixTimeMilliseconds = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
|
|
|
+ string forwardUrl = string.Format(CultureInfo.InvariantCulture, ForwardUrlTemplate, normalizedChannel);
|
|
|
+ requestUrl = $"{forwardUrl}?oaid={WebUtility.UrlEncode(oaid ?? string.Empty)}&datetime={unixTimeMilliseconds}";
|
|
|
+ }
|
|
|
+ catch (Exception ex)
|
|
|
+ {
|
|
|
+ _ = new LoggerLibrary("zlong_forward")
|
|
|
+ .Info(normalizedChannel, requestUrl)
|
|
|
+ .Info(ex.Message, ex.StackTrace)
|
|
|
+ .SaveAsync();
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ try
|
|
|
+ {
|
|
|
+ SaveCount(normalizedChannel);
|
|
|
+ }
|
|
|
+ catch (Exception ex)
|
|
|
+ {
|
|
|
+ _ = new LoggerLibrary("zlong_forward")
|
|
|
+ .Info(normalizedChannel, requestUrl)
|
|
|
+ .Info(ex.Message, ex.StackTrace)
|
|
|
+ .SaveAsync();
|
|
|
+ }
|
|
|
+
|
|
|
+ try
|
|
|
+ {
|
|
|
+ using var response = await HttpClient.GetAsync(requestUrl, HttpCompletionOption.ResponseHeadersRead);
|
|
|
+ if (response.IsSuccessStatusCode) return;
|
|
|
+
|
|
|
+ SaveExceptionCount(normalizedChannel, $"status_{(int)response.StatusCode}");
|
|
|
+ }
|
|
|
+ catch (TaskCanceledException)
|
|
|
+ {
|
|
|
+ SaveExceptionCount(normalizedChannel, "timeout");
|
|
|
+ }
|
|
|
+ catch (Exception)
|
|
|
+ {
|
|
|
+ SaveExceptionCount(normalizedChannel, "request_error");
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private static void SaveCount(string channel)
|
|
|
+ {
|
|
|
+ DateTime now = DateTime.Now;
|
|
|
+ var metrics = GetBucketMetrics(now);
|
|
|
+
|
|
|
+ foreach (var metric in metrics)
|
|
|
+ {
|
|
|
+ string key = GetCountKey(channel, metric.bucketType, metric.bucketValue);
|
|
|
+ SaveMetricCount(key, GetIndexKey(channel, metric.bucketType), metric.expireSeconds);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ public static async Task<ZlongForwardReportResult> QueryReportAsync(string channels, string bucketType, DateTime? start, DateTime? end, int page, int size)
|
|
|
+ {
|
|
|
+ string[] selectedChannels = NormalizeChannels(channels);
|
|
|
+ string normalizedBucketType = NormalizeBucketType(bucketType);
|
|
|
+ page = Math.Max(page, 1);
|
|
|
+ size = size <= 0 ? 20 : size;
|
|
|
+
|
|
|
+ var list = new List<ZlongForwardReportItem>();
|
|
|
+ DateTime? startTime = start?.Date;
|
|
|
+ DateTime? endTime = end?.Date.AddDays(1).AddTicks(-1);
|
|
|
+
|
|
|
+ foreach (var channel in selectedChannels)
|
|
|
+ {
|
|
|
+ string[] keys = await TkLogCore.GetTotalKeysAsync(GetIndexKey(channel, normalizedBucketType)) ?? Array.Empty<string>();
|
|
|
+ foreach (var key in keys)
|
|
|
+ {
|
|
|
+ if (!TryParseKey(key, out var item)) continue;
|
|
|
+ if (!IsMatched(item, normalizedBucketType, startTime, endTime)) continue;
|
|
|
+
|
|
|
+ list.Add(item);
|
|
|
+ }
|
|
|
+
|
|
|
+ if (!string.Equals(channel, "xhs", StringComparison.OrdinalIgnoreCase)) continue;
|
|
|
+
|
|
|
+ string[] legacyKeys = await TkLogCore.GetTotalKeysAsync(GetLegacyIndexKey(normalizedBucketType)) ?? Array.Empty<string>();
|
|
|
+ foreach (var key in legacyKeys)
|
|
|
+ {
|
|
|
+ if (!TryParseKey(key, out var item)) continue;
|
|
|
+ if (!IsMatched(item, normalizedBucketType, startTime, endTime)) continue;
|
|
|
+
|
|
|
+ list.Add(item);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ var totalTasks = list.Select(async item =>
|
|
|
+ {
|
|
|
+ var totalTask = TkLogCore.GetTotalAsync(item.redis_key);
|
|
|
+ var exceptionTask = TkLogCore.GetTotalAsync(GetExceptionCountKey(item.redis_key, item.channel, item.bucket_type, item.bucket_value));
|
|
|
+ await Task.WhenAll(totalTask, exceptionTask);
|
|
|
+
|
|
|
+ item.total = totalTask.Result;
|
|
|
+ item.exception_count = exceptionTask.Result;
|
|
|
+ return item;
|
|
|
+ });
|
|
|
+
|
|
|
+ list = MergeItems(await Task.WhenAll(totalTasks))
|
|
|
+ .Where(x => x.total > 0 || x.exception_count > 0)
|
|
|
+ .ToList();
|
|
|
+
|
|
|
+ list = list
|
|
|
+ .OrderByDescending(x => x.report_time)
|
|
|
+ .ThenBy(x => GetChannelSortOrder(x.channel))
|
|
|
+ .ThenByDescending(x => x.bucket_value)
|
|
|
+ .ToList();
|
|
|
+
|
|
|
+ int count = list.Count;
|
|
|
+ int skip = (page - 1) * size;
|
|
|
+ int pageCount = count == 0 ? 0 : (int)Math.Ceiling(count / (double)size);
|
|
|
+
|
|
|
+ return new ZlongForwardReportResult
|
|
|
+ {
|
|
|
+ list = list.Skip(skip).Take(size).ToList(),
|
|
|
+ page = page,
|
|
|
+ size = size,
|
|
|
+ count = count,
|
|
|
+ pageCount = pageCount,
|
|
|
+ summary = await GetSummaryAsync(selectedChannels),
|
|
|
+ };
|
|
|
+ }
|
|
|
+
|
|
|
+ public static async Task<ZlongForwardReportSummary> GetSummaryAsync(string channels)
|
|
|
+ {
|
|
|
+ return await GetSummaryAsync(NormalizeChannels(channels));
|
|
|
+ }
|
|
|
+
|
|
|
+ private static async Task<ZlongForwardReportSummary> GetSummaryAsync(string[] channels)
|
|
|
+ {
|
|
|
+ DateTime now = DateTime.Now;
|
|
|
+ var hourTask = GetMetricTotalAsync(channels, "hour", now.ToString("yyyyMMddHH"));
|
|
|
+ var dayTask = GetMetricTotalAsync(channels, "day", now.ToString("yyyyMMdd"));
|
|
|
+ var monthTask = GetMetricTotalAsync(channels, "month", now.ToString("yyyyMM"));
|
|
|
+ var yearTask = GetMetricTotalAsync(channels, "year", now.ToString("yyyy"));
|
|
|
+
|
|
|
+ var hourExceptionTask = GetMetricTotalAsync(channels, "hour", now.ToString("yyyyMMddHH"), true);
|
|
|
+ var dayExceptionTask = GetMetricTotalAsync(channels, "day", now.ToString("yyyyMMdd"), true);
|
|
|
+ var monthExceptionTask = GetMetricTotalAsync(channels, "month", now.ToString("yyyyMM"), true);
|
|
|
+ var yearExceptionTask = GetMetricTotalAsync(channels, "year", now.ToString("yyyy"), true);
|
|
|
+
|
|
|
+ await Task.WhenAll(hourTask, dayTask, monthTask, yearTask, hourExceptionTask, dayExceptionTask, monthExceptionTask, yearExceptionTask);
|
|
|
+
|
|
|
+ return new ZlongForwardReportSummary
|
|
|
+ {
|
|
|
+ hour = hourTask.Result,
|
|
|
+ day = dayTask.Result,
|
|
|
+ month = monthTask.Result,
|
|
|
+ year = yearTask.Result,
|
|
|
+ hour_exception = hourExceptionTask.Result,
|
|
|
+ day_exception = dayExceptionTask.Result,
|
|
|
+ month_exception = monthExceptionTask.Result,
|
|
|
+ year_exception = yearExceptionTask.Result,
|
|
|
+ };
|
|
|
+ }
|
|
|
+
|
|
|
+ private static async Task<int> GetMetricTotalAsync(string[] channels, string bucketType, string bucketValue, bool isException = false)
|
|
|
+ {
|
|
|
+ var tasks = new List<Task<int>>();
|
|
|
+
|
|
|
+ foreach (var channel in channels)
|
|
|
+ {
|
|
|
+ string key = isException
|
|
|
+ ? GetChannelExceptionCountKey(channel, ExceptionAggregateType, bucketType, bucketValue)
|
|
|
+ : GetCountKey(channel, bucketType, bucketValue);
|
|
|
+
|
|
|
+ tasks.Add(TkLogCore.GetTotalAsync(key));
|
|
|
+
|
|
|
+ if (!string.Equals(channel, "xhs", StringComparison.OrdinalIgnoreCase)) continue;
|
|
|
+
|
|
|
+ string legacyKey = isException
|
|
|
+ ? GetLegacyExceptionCountKey(ExceptionAggregateType, bucketType, bucketValue)
|
|
|
+ : GetLegacyCountKey(bucketType, bucketValue);
|
|
|
+
|
|
|
+ tasks.Add(TkLogCore.GetTotalAsync(legacyKey));
|
|
|
+ }
|
|
|
+
|
|
|
+ await Task.WhenAll(tasks);
|
|
|
+ return tasks.Sum(task => task.Result);
|
|
|
+ }
|
|
|
+
|
|
|
+ private static bool IsMatched(ZlongForwardReportItem item, string bucketType, DateTime? startTime, DateTime? endTime)
|
|
|
+ {
|
|
|
+ if (!string.Equals(item.bucket_type, bucketType, StringComparison.OrdinalIgnoreCase)) return false;
|
|
|
+ if (startTime.HasValue && item.report_time < startTime.Value) return false;
|
|
|
+ if (endTime.HasValue && item.report_time > endTime.Value) return false;
|
|
|
+
|
|
|
+ return true;
|
|
|
+ }
|
|
|
+
|
|
|
+ private static List<ZlongForwardReportItem> MergeItems(IEnumerable<ZlongForwardReportItem> items)
|
|
|
+ {
|
|
|
+ return items
|
|
|
+ .GroupBy(item => $"{item.channel}:{item.bucket_type}:{item.bucket_value}", StringComparer.OrdinalIgnoreCase)
|
|
|
+ .Select(group =>
|
|
|
+ {
|
|
|
+ var first = group.First();
|
|
|
+ return new ZlongForwardReportItem
|
|
|
+ {
|
|
|
+ channel = first.channel,
|
|
|
+ bucket_type = first.bucket_type,
|
|
|
+ bucket_value = first.bucket_value,
|
|
|
+ report_time = first.report_time,
|
|
|
+ display_time = first.display_time,
|
|
|
+ total = group.Sum(item => item.total),
|
|
|
+ exception_count = group.Sum(item => item.exception_count),
|
|
|
+ redis_key = $"{first.channel}:{first.bucket_type}:{first.bucket_value}",
|
|
|
+ };
|
|
|
+ })
|
|
|
+ .ToList();
|
|
|
+ }
|
|
|
+
|
|
|
+ private static string NormalizeBucketType(string bucketType)
|
|
|
+ {
|
|
|
+ return bucketType?.Trim().ToLowerInvariant() switch
|
|
|
+ {
|
|
|
+ "hour" => "hour",
|
|
|
+ "month" => "month",
|
|
|
+ "year" => "year",
|
|
|
+ _ => "day",
|
|
|
+ };
|
|
|
+ }
|
|
|
+
|
|
|
+ private static string[] NormalizeChannels(string channels)
|
|
|
+ {
|
|
|
+ if (string.IsNullOrWhiteSpace(channels)) return SupportedChannels;
|
|
|
+
|
|
|
+ var list = new List<string>();
|
|
|
+ foreach (var item in channels.Split(',', StringSplitOptions.RemoveEmptyEntries))
|
|
|
+ {
|
|
|
+ string normalized = item.Trim().ToLowerInvariant();
|
|
|
+ if (string.Equals(normalized, "all", StringComparison.OrdinalIgnoreCase))
|
|
|
+ {
|
|
|
+ return SupportedChannels;
|
|
|
+ }
|
|
|
+
|
|
|
+ normalized = NormalizeChannel(normalized);
|
|
|
+ if (string.IsNullOrEmpty(normalized) || list.Contains(normalized)) continue;
|
|
|
+
|
|
|
+ list.Add(normalized);
|
|
|
+ }
|
|
|
+
|
|
|
+ return list.Count > 0 ? list.ToArray() : SupportedChannels;
|
|
|
+ }
|
|
|
+
|
|
|
+ private static string NormalizeChannel(string channel)
|
|
|
+ {
|
|
|
+ if (string.IsNullOrWhiteSpace(channel)) return string.Empty;
|
|
|
+
|
|
|
+ string normalized = channel.Trim().ToLowerInvariant();
|
|
|
+ return SupportedChannels.Contains(normalized) ? normalized : string.Empty;
|
|
|
+ }
|
|
|
+
|
|
|
+ private static int GetChannelSortOrder(string channel)
|
|
|
+ {
|
|
|
+ int index = Array.IndexOf(SupportedChannels, channel);
|
|
|
+ return index >= 0 ? index : SupportedChannels.Length;
|
|
|
+ }
|
|
|
+
|
|
|
+ private static string GetIndexKey(string channel, string bucketType)
|
|
|
+ {
|
|
|
+ return $"{RedisPrefix}:index:{channel}:{bucketType}";
|
|
|
+ }
|
|
|
+
|
|
|
+ private static string GetLegacyIndexKey(string bucketType)
|
|
|
+ {
|
|
|
+ return $"{RedisPrefix}:index:{bucketType}";
|
|
|
+ }
|
|
|
+
|
|
|
+ private static string GetCountKey(string channel, string bucketType, string bucketValue)
|
|
|
+ {
|
|
|
+ return $"{RedisPrefix}:{channel}:{bucketType}:{bucketValue}";
|
|
|
+ }
|
|
|
+
|
|
|
+ private static string GetLegacyCountKey(string bucketType, string bucketValue)
|
|
|
+ {
|
|
|
+ return $"{RedisPrefix}:{bucketType}:{bucketValue}";
|
|
|
+ }
|
|
|
+
|
|
|
+ private static string GetExceptionCountKey(string redisKey, string channel, string bucketType, string bucketValue)
|
|
|
+ {
|
|
|
+ return IsLegacyCountKey(redisKey)
|
|
|
+ ? GetLegacyExceptionCountKey(ExceptionAggregateType, bucketType, bucketValue)
|
|
|
+ : GetChannelExceptionCountKey(channel, ExceptionAggregateType, bucketType, bucketValue);
|
|
|
+ }
|
|
|
+
|
|
|
+ private static string GetChannelExceptionCountKey(string channel, string exceptionType, string bucketType, string bucketValue)
|
|
|
+ {
|
|
|
+ return $"{ExceptionRedisPrefix}:{channel}:{exceptionType}:{bucketType}:{bucketValue}";
|
|
|
+ }
|
|
|
+
|
|
|
+ private static string GetLegacyExceptionCountKey(string exceptionType, string bucketType, string bucketValue)
|
|
|
+ {
|
|
|
+ return $"{ExceptionRedisPrefix}:{exceptionType}:{bucketType}:{bucketValue}";
|
|
|
+ }
|
|
|
+
|
|
|
+ private static bool IsLegacyCountKey(string key)
|
|
|
+ {
|
|
|
+ var parts = key.Split(':', StringSplitOptions.RemoveEmptyEntries);
|
|
|
+ return parts.Length == 4;
|
|
|
+ }
|
|
|
+
|
|
|
+ private static string GetExceptionIndexKey(string channel, string exceptionType, string bucketType)
|
|
|
+ {
|
|
|
+ return $"{ExceptionRedisPrefix}:index:{channel}:{exceptionType}:{bucketType}";
|
|
|
+ }
|
|
|
+
|
|
|
+ private static (string bucketType, string bucketValue, int expireSeconds)[] GetBucketMetrics(DateTime now)
|
|
|
+ {
|
|
|
+ return
|
|
|
+ [
|
|
|
+ ("hour", now.ToString("yyyyMMddHH"), 86400 * 90),
|
|
|
+ ("day", now.ToString("yyyyMMdd"), 86400 * 366),
|
|
|
+ ("month", now.ToString("yyyyMM"), 86400 * 3660),
|
|
|
+ ("year", now.ToString("yyyy"), 86400 * 3660),
|
|
|
+ ];
|
|
|
+ }
|
|
|
+
|
|
|
+ private static void SaveMetricCount(string key, string indexKey, int expireSeconds)
|
|
|
+ {
|
|
|
+ RedisHelper.IncrBy(key);
|
|
|
+ RedisHelper.Expire(key, expireSeconds);
|
|
|
+ RedisHelper.SAdd(indexKey, key);
|
|
|
+ RedisHelper.Expire(indexKey, expireSeconds);
|
|
|
+ }
|
|
|
+
|
|
|
+ private static void SaveExceptionCount(string channel, string exceptionType)
|
|
|
+ {
|
|
|
+ DateTime now = DateTime.Now;
|
|
|
+ var metrics = GetBucketMetrics(now);
|
|
|
+
|
|
|
+ foreach (var metric in metrics)
|
|
|
+ {
|
|
|
+ string key = GetChannelExceptionCountKey(channel, exceptionType, metric.bucketType, metric.bucketValue);
|
|
|
+ SaveMetricCount(key, GetExceptionIndexKey(channel, exceptionType, metric.bucketType), metric.expireSeconds);
|
|
|
+
|
|
|
+ string aggregateKey = GetChannelExceptionCountKey(channel, ExceptionAggregateType, metric.bucketType, metric.bucketValue);
|
|
|
+ SaveMetricCount(aggregateKey, GetExceptionIndexKey(channel, ExceptionAggregateType, metric.bucketType), metric.expireSeconds);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private static bool TryParseKey(string key, out ZlongForwardReportItem item)
|
|
|
+ {
|
|
|
+ item = new ZlongForwardReportItem();
|
|
|
+ if (string.IsNullOrWhiteSpace(key)) return false;
|
|
|
+
|
|
|
+ var parts = key.Split(':', StringSplitOptions.RemoveEmptyEntries);
|
|
|
+ if (parts.Length < 4) return false;
|
|
|
+
|
|
|
+ string channel = "xhs";
|
|
|
+ string bucketType = parts[^2];
|
|
|
+ string bucketValue = parts[^1];
|
|
|
+ if (parts.Length >= 5)
|
|
|
+ {
|
|
|
+ channel = NormalizeChannel(parts[^3]);
|
|
|
+ if (string.IsNullOrEmpty(channel)) return false;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (!TryParseBucket(bucketType, bucketValue, out var reportTime, out var displayTime)) return false;
|
|
|
+
|
|
|
+ item.channel = channel;
|
|
|
+ item.bucket_type = bucketType;
|
|
|
+ item.bucket_value = bucketValue;
|
|
|
+ item.report_time = reportTime;
|
|
|
+ item.display_time = displayTime;
|
|
|
+ item.redis_key = key;
|
|
|
+ return true;
|
|
|
+ }
|
|
|
+
|
|
|
+ private static bool TryParseBucket(string bucketType, string bucketValue, out DateTime reportTime, out string displayTime)
|
|
|
+ {
|
|
|
+ reportTime = DateTime.MinValue;
|
|
|
+ displayTime = string.Empty;
|
|
|
+
|
|
|
+ string format = bucketType switch
|
|
|
+ {
|
|
|
+ "hour" => "yyyyMMddHH",
|
|
|
+ "day" => "yyyyMMdd",
|
|
|
+ "month" => "yyyyMM",
|
|
|
+ "year" => "yyyy",
|
|
|
+ _ => string.Empty,
|
|
|
+ };
|
|
|
+ if (string.IsNullOrEmpty(format)) return false;
|
|
|
+
|
|
|
+ bool success = DateTime.TryParseExact(
|
|
|
+ bucketValue,
|
|
|
+ format,
|
|
|
+ CultureInfo.InvariantCulture,
|
|
|
+ DateTimeStyles.None,
|
|
|
+ out reportTime);
|
|
|
+
|
|
|
+ if (!success) return false;
|
|
|
+
|
|
|
+ displayTime = bucketType switch
|
|
|
+ {
|
|
|
+ "hour" => reportTime.ToString("yyyy-MM-dd HH:00"),
|
|
|
+ "day" => reportTime.ToString("yyyy-MM-dd"),
|
|
|
+ "month" => reportTime.ToString("yyyy-MM"),
|
|
|
+ "year" => reportTime.ToString("yyyy"),
|
|
|
+ _ => bucketValue,
|
|
|
+ };
|
|
|
+ return true;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ public class ZlongForwardReportSummary
|
|
|
+ {
|
|
|
+ public long hour { get; set; }
|
|
|
+ public long day { get; set; }
|
|
|
+ public long month { get; set; }
|
|
|
+ public long year { get; set; }
|
|
|
+ public long hour_exception { get; set; }
|
|
|
+ public long day_exception { get; set; }
|
|
|
+ public long month_exception { get; set; }
|
|
|
+ public long year_exception { get; set; }
|
|
|
+ }
|
|
|
+
|
|
|
+ public class ZlongForwardReportItem
|
|
|
+ {
|
|
|
+ public string channel { get; set; } = string.Empty;
|
|
|
+ public string bucket_type { get; set; } = string.Empty;
|
|
|
+ public string bucket_value { get; set; } = string.Empty;
|
|
|
+ public DateTime report_time { get; set; } = DateTime.MinValue;
|
|
|
+ public string display_time { get; set; } = string.Empty;
|
|
|
+ public long total { get; set; }
|
|
|
+ public long exception_count { get; set; }
|
|
|
+ public string redis_key { get; set; } = string.Empty;
|
|
|
+ }
|
|
|
+
|
|
|
+ public class ZlongForwardReportResult
|
|
|
+ {
|
|
|
+ public int page { get; set; }
|
|
|
+ public int pageCount { get; set; }
|
|
|
+ public int size { get; set; }
|
|
|
+ public int count { get; set; }
|
|
|
+ public List<ZlongForwardReportItem> list { get; set; } = [];
|
|
|
+ public ZlongForwardReportSummary summary { get; set; } = new();
|
|
|
+ }
|
|
|
+}
|
dodo hold