using dodohold.core; using Microsoft.Extensions.Logging; using System.Diagnostics; using System.Text; namespace molilian.core { public static class SystemMonitor { private static Timer _monitorTimer; private static int _isCollecting; private static readonly AlertState MemoryAlertState = new(); private static readonly int MemoryAlertThresholdMb = GetIntEnv("SystemMonitorMemoryAlertMB", 2048); private static readonly int MemoryAlertRecoveryMb = GetIntEnv("SystemMonitorMemoryRecoveryMB", (int)(MemoryAlertThresholdMb * 0.9)); private static readonly int MemoryAlertConsecutiveHits = GetIntEnv("SystemMonitorMemoryAlertConsecutiveHits", 3); private static readonly int MemoryAlertCooldownMinutes = GetIntEnv("SystemMonitorMemoryAlertCooldownMinutes", 30); public static void StartMonitoring(int intervalSeconds = 30) { _monitorTimer = new Timer( CollectMetrics, null, TimeSpan.Zero, TimeSpan.FromSeconds(intervalSeconds) ); } private static void CollectMetrics(object state) { if (Interlocked.Exchange(ref _isCollecting, 1) == 1) return; try { var metrics = new StringBuilder(); // 线程池信息 ThreadPool.GetMaxThreads(out int maxWorkerThreads, out int maxIoThreads); ThreadPool.GetAvailableThreads(out int availWorkerThreads, out int availIoThreads); ThreadPool.GetMinThreads(out int minWorkerThreads, out int minIoThreads); var usedWorkerThreads = maxWorkerThreads - availWorkerThreads; var usedIoThreads = maxIoThreads - availIoThreads; metrics.AppendLine("=== Thread Pool Stats ==="); metrics.AppendLine($"Worker Threads: {usedWorkerThreads}/{maxWorkerThreads} (used/max)"); metrics.AppendLine($"IO Threads: {usedIoThreads}/{maxIoThreads} (used/max)"); metrics.AppendLine($"Min Worker Threads: {minWorkerThreads}"); metrics.AppendLine($"Min IO Threads: {minIoThreads}"); metrics.AppendLine($"Thread Pool Queue Length: {ThreadPool.PendingWorkItemCount}"); metrics.AppendLine(); // 内存信息 var process = Process.GetCurrentProcess(); var gcInfo = GC.GetGCMemoryInfo(); metrics.AppendLine("=== Memory Stats ==="); metrics.AppendLine($"Working Set: {process.WorkingSet64 / 1024 / 1024} MB"); metrics.AppendLine($"Private Memory: {process.PrivateMemorySize64 / 1024 / 1024} MB"); metrics.AppendLine($"GC Heap Size: {GC.GetTotalMemory(false) / 1024 / 1024} MB"); metrics.AppendLine($"Gen 0 Collections: {GC.CollectionCount(0)}"); metrics.AppendLine($"Gen 1 Collections: {GC.CollectionCount(1)}"); metrics.AppendLine($"Gen 2 Collections: {GC.CollectionCount(2)}"); metrics.AppendLine($"Memory Load: {gcInfo.MemoryLoadBytes / 1024 / 1024} MB"); // 记录日志 _ = new LoggerLibrary("debug", "metrics").Info(metrics.ToString()).SaveAsync(); // 检查是否需要报警 CheckAlertConditions( usedWorkerThreads, maxWorkerThreads, process.WorkingSet64, ThreadPool.PendingWorkItemCount ); } catch (Exception ex) { _ = new LoggerLibrary("debug", "metrics_error") .Info($"Error collecting system metrics: {ex}").SaveAsync(); } finally { Volatile.Write(ref _isCollecting, 0); } } private static void CheckAlertConditions( int usedWorkerThreads, int maxWorkerThreads, long workingSet, long pendingWork) { var alerts = new List(); // 线程池使用率超过80% if ((double)usedWorkerThreads / maxWorkerThreads > 0.8) { alerts.Add($"High thread pool usage: {usedWorkerThreads}/{maxWorkerThreads}"); } HandleMemoryAlert(workingSet); // 线程池队列堆积 if (pendingWork > 100) { alerts.Add($"High thread pool queue length: {pendingWork}"); } if (alerts.Any()) { NotifyCore.Notify($"System alerts:\n{string.Join("\n", alerts)}"); } } public static void StopMonitoring() { _monitorTimer?.Dispose(); } private static void HandleMemoryAlert(long workingSet) { int workingSetMb = (int)(workingSet / 1024 / 1024); bool isTriggered = workingSetMb >= MemoryAlertThresholdMb; bool isRecovered = workingSetMb <= MemoryAlertRecoveryMb; var now = DateTime.UtcNow; lock (MemoryAlertState) { if (isTriggered) { MemoryAlertState.ConsecutiveHits++; bool shouldSend = MemoryAlertState.ConsecutiveHits >= MemoryAlertConsecutiveHits && (!MemoryAlertState.IsActive || now - MemoryAlertState.LastSentAtUtc >= TimeSpan.FromMinutes(MemoryAlertCooldownMinutes)); if (shouldSend) { NotifyCore.Notify( $"System alerts:\nHigh memory usage: {workingSetMb} MB\n" + $"threshold={MemoryAlertThresholdMb} MB, consecutiveHits={MemoryAlertState.ConsecutiveHits}"); MemoryAlertState.IsActive = true; MemoryAlertState.LastSentAtUtc = now; } return; } MemoryAlertState.ConsecutiveHits = 0; if (MemoryAlertState.IsActive && isRecovered) { NotifyCore.Notify( $"System recovery:\nMemory usage recovered to {workingSetMb} MB\n" + $"recoveryThreshold={MemoryAlertRecoveryMb} MB"); MemoryAlertState.IsActive = false; MemoryAlertState.LastSentAtUtc = now; } } } private static int GetIntEnv(string key, int defaultValue) { var raw = Environment.GetEnvironmentVariable(key); return int.TryParse(raw, out var value) && value > 0 ? value : defaultValue; } private sealed class AlertState { public int ConsecutiveHits { get; set; } public bool IsActive { get; set; } public DateTime LastSentAtUtc { get; set; } = DateTime.MinValue; } } }