dotnet/molilian.core/Plus/SystemMonitor.cs

using dodohold.core;
using Microsoft.Extensions.Logging;
using System.Diagnostics;
using System.Text;


namespace molilian.core
{
    public static class SystemMonitor
    {
        private static Timer _monitorTimer;
        private static int _isCollecting;
        private static readonly AlertState MemoryAlertState = new();
        private static readonly int MemoryAlertThresholdMb = GetIntEnv("SystemMonitorMemoryAlertMB", 2048);
        private static readonly int MemoryAlertRecoveryMb = GetIntEnv("SystemMonitorMemoryRecoveryMB", (int)(MemoryAlertThresholdMb * 0.9));
        private static readonly int MemoryAlertConsecutiveHits = GetIntEnv("SystemMonitorMemoryAlertConsecutiveHits", 3);
        private static readonly int MemoryAlertCooldownMinutes = GetIntEnv("SystemMonitorMemoryAlertCooldownMinutes", 30);

        public static void StartMonitoring(int intervalSeconds = 30)
        {
            _monitorTimer = new Timer(
                CollectMetrics,
                null,
                TimeSpan.Zero,
                TimeSpan.FromSeconds(intervalSeconds)
            );
        }

        private static void CollectMetrics(object state)
        {
            if (Interlocked.Exchange(ref _isCollecting, 1) == 1) return;

            try
            {
                var metrics = new StringBuilder();

                // 线程池信息
                ThreadPool.GetMaxThreads(out int maxWorkerThreads, out int maxIoThreads);
                ThreadPool.GetAvailableThreads(out int availWorkerThreads, out int availIoThreads);
                ThreadPool.GetMinThreads(out int minWorkerThreads, out int minIoThreads);

                var usedWorkerThreads = maxWorkerThreads - availWorkerThreads;
                var usedIoThreads = maxIoThreads - availIoThreads;

                metrics.AppendLine("=== Thread Pool Stats ===");
                metrics.AppendLine($"Worker Threads: {usedWorkerThreads}/{maxWorkerThreads} (used/max)");
                metrics.AppendLine($"IO Threads: {usedIoThreads}/{maxIoThreads} (used/max)");
                metrics.AppendLine($"Min Worker Threads: {minWorkerThreads}");
                metrics.AppendLine($"Min IO Threads: {minIoThreads}");
                metrics.AppendLine($"Thread Pool Queue Length: {ThreadPool.PendingWorkItemCount}");
                metrics.AppendLine();

                // 内存信息
                var process = Process.GetCurrentProcess();
                var gcInfo = GC.GetGCMemoryInfo();

                metrics.AppendLine("=== Memory Stats ===");
                metrics.AppendLine($"Working Set: {process.WorkingSet64 / 1024 / 1024} MB");
                metrics.AppendLine($"Private Memory: {process.PrivateMemorySize64 / 1024 / 1024} MB");
                metrics.AppendLine($"GC Heap Size: {GC.GetTotalMemory(false) / 1024 / 1024} MB");
                metrics.AppendLine($"Gen 0 Collections: {GC.CollectionCount(0)}");
                metrics.AppendLine($"Gen 1 Collections: {GC.CollectionCount(1)}");
                metrics.AppendLine($"Gen 2 Collections: {GC.CollectionCount(2)}");
                metrics.AppendLine($"Memory Load: {gcInfo.MemoryLoadBytes / 1024 / 1024} MB");

                // 记录日志
                _ = new LoggerLibrary("debug", "metrics").Info(metrics.ToString()).SaveAsync();

                // 检查是否需要报警
                CheckAlertConditions(
                    usedWorkerThreads, maxWorkerThreads,
                    process.WorkingSet64,
                    ThreadPool.PendingWorkItemCount
                );
            }
            catch (Exception ex)
            {
                _ = new LoggerLibrary("debug", "metrics_error")
                    .Info($"Error collecting system metrics: {ex}").SaveAsync();
            }
            finally
            {
                Volatile.Write(ref _isCollecting, 0);
            }
        }

        private static void CheckAlertConditions(
            int usedWorkerThreads,
            int maxWorkerThreads,
            long workingSet,
            long pendingWork)
        {
            var alerts = new List<string>();

            // 线程池使用率超过80%
            if ((double)usedWorkerThreads / maxWorkerThreads > 0.8)
            {
                alerts.Add($"High thread pool usage: {usedWorkerThreads}/{maxWorkerThreads}");
            }

            HandleMemoryAlert(workingSet);

            // 线程池队列堆积
            if (pendingWork > 100)
            {
                alerts.Add($"High thread pool queue length: {pendingWork}");
            }

            if (alerts.Any())
            {
                NotifyCore.Notify($"System alerts:\n{string.Join("\n", alerts)}");
            }
        }

        public static void StopMonitoring()
        {
            _monitorTimer?.Dispose();
        }

        private static void HandleMemoryAlert(long workingSet)
        {
            int workingSetMb = (int)(workingSet / 1024 / 1024);
            bool isTriggered = workingSetMb >= MemoryAlertThresholdMb;
            bool isRecovered = workingSetMb <= MemoryAlertRecoveryMb;
            var now = DateTime.UtcNow;

            lock (MemoryAlertState)
            {
                if (isTriggered)
                {
                    MemoryAlertState.ConsecutiveHits++;

                    bool shouldSend = MemoryAlertState.ConsecutiveHits >= MemoryAlertConsecutiveHits &&
                        (!MemoryAlertState.IsActive || now - MemoryAlertState.LastSentAtUtc >= TimeSpan.FromMinutes(MemoryAlertCooldownMinutes));

                    if (shouldSend)
                    {
                        NotifyCore.Notify(
                            $"System alerts:\nHigh memory usage: {workingSetMb} MB\n" +
                            $"threshold={MemoryAlertThresholdMb} MB, consecutiveHits={MemoryAlertState.ConsecutiveHits}");
                        MemoryAlertState.IsActive = true;
                        MemoryAlertState.LastSentAtUtc = now;
                    }

                    return;
                }

                MemoryAlertState.ConsecutiveHits = 0;

                if (MemoryAlertState.IsActive && isRecovered)
                {
                    NotifyCore.Notify(
                        $"System recovery:\nMemory usage recovered to {workingSetMb} MB\n" +
                        $"recoveryThreshold={MemoryAlertRecoveryMb} MB");
                    MemoryAlertState.IsActive = false;
                    MemoryAlertState.LastSentAtUtc = now;
                }
            }
        }

        private static int GetIntEnv(string key, int defaultValue)
        {
            var raw = Environment.GetEnvironmentVariable(key);
            return int.TryParse(raw, out var value) && value > 0 ? value : defaultValue;
        }

        private sealed class AlertState
        {
            public int ConsecutiveHits { get; set; }
            public bool IsActive { get; set; }
            public DateTime LastSentAtUtc { get; set; } = DateTime.MinValue;
        }
    }
}