From 82759e7cd34c174f20a585eeb6c64bc5bb15a63b Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 11:51:30 +0800 Subject: [PATCH] fix(agent99): reduce SSH performance probe churn --- agent99-bootstrap.ps1 | 10 + agent99-control-plane.ps1 | 355 +++++++++++++++++- agent99-deploy.ps1 | 60 ++- agent99.config.99.example.json | 10 + agent99.config.example.json | 10 + ...nt99_node_exporter_performance_contract.py | 165 ++++++++ 6 files changed, 594 insertions(+), 16 deletions(-) create mode 100644 scripts/reboot-recovery/tests/test_agent99_node_exporter_performance_contract.py diff --git a/agent99-bootstrap.ps1 b/agent99-bootstrap.ps1 index 1ad66b9e6..be020aeec 100644 --- a/agent99-bootstrap.ps1 +++ b/agent99-bootstrap.ps1 @@ -353,13 +353,23 @@ if (-not (Test-Path $configPath)) { "performance": { "enabled": true, "intervalMinutes": 1, + "nodeExporterTimeoutSeconds": 8, + "nodeExporterUrls": { + "192.168.0.110": "http://192.168.0.110:9100/metrics", + "192.168.0.112": "http://192.168.0.112:9100/metrics", + "192.168.0.188": "http://192.168.0.188:9100/metrics" + }, "loadPerCoreWarning": 0.9, "loadPerCoreCritical": 1.5, "memoryAvailablePercentCritical": 10, + "kernelPercpuMemoryPercentWarning": 10, + "kernelPercpuMemoryPercentCritical": 20, + "kernelSUnreclaimMemoryPercentCritical": 25, "diskUsedPercentWarning": 90, "diskUsedPercentCritical": 95, "captureTopCpuOnWarning": true, "topCpuTimeoutSeconds": 10, + "topCpuDiagnosticCooldownMinutes": 15, "loadShedding": { "enabled": true, "actions": [] } }, "backupHealth": { diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index e16a0a0f7..68bca2b88 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -634,6 +634,9 @@ function Format-AgentTelegramLegacyText { $disk = Get-AgentObjectValue $Data "diskUsedPercent" $null $load = Get-AgentObjectValue $Data "loadPerCore" $null $mem = Get-AgentObjectValue $Data "memAvailablePercent" $null + $kernelPercpu = Get-AgentObjectValue $Data "kernelPercpuMemoryPercent" $null + $kernelSUnreclaim = Get-AgentObjectValue $Data "kernelSUnreclaimMemoryPercent" $null + $metricSource = Get-AgentObjectValue $Data "metricSource" "unknown" $lines += "事件 Event: 主機效能 / host performance" $lines += "主機 Host: $hostName" if ($reasons -contains "perf_readback_failed" -or ($null -eq $disk -and $null -eq $load -and $null -eq $mem)) { @@ -649,6 +652,12 @@ function Format-AgentTelegramLegacyText { } elseif ($reasons -contains "load_per_core_warning" -or $reasons -contains "load_per_core_critical") { $lines += "原因 Reason: CPU load/core=$load。" $lines += "Agent99 動作 Action: 僅使用 allowlisted load reduction；不得任意停止服務。" + } elseif ($reasons -contains "kernel_percpu_memory_critical" -or $reasons -contains "kernel_sunreclaim_memory_critical") { + $lines += "原因 Reason: kernel memory pressure 已達 critical；Percpu=$kernelPercpu percent，SUnreclaim=$kernelSUnreclaim percent。" + $lines += "Agent99 動作 Action: 日常採 Node Exporter 無 SSH 監測；process 診斷限流，並保留受控 kernel remediation 與獨立 verifier。" + } elseif ($reasons -contains "kernel_percpu_memory_warning") { + $lines += "原因 Reason: kernel Percpu memory=$kernelPercpu percent，已達 warning threshold。" + $lines += "Agent99 動作 Action: 持續 Node Exporter readback，SSH 診斷依 cooldown 執行，避免監控本身增加 session churn。" } elseif ($reasons -contains "memory_available_low") { $lines += "原因 Reason: 可用記憶體為 $mem percent。" $lines += "Agent99 動作 Action: 先檢查 memory pressure，再進行 allowlisted remediation。" @@ -660,6 +669,9 @@ function Format-AgentTelegramLegacyText { if ($null -ne $load -and [string]$load -ne "") { $metricParts += "load/core=$load" } if ($null -ne $mem -and [string]$mem -ne "") { $metricParts += "memAvail=$mem percent" } if ($null -ne $disk -and [string]$disk -ne "") { $metricParts += "disk=$disk percent" } + if ($null -ne $kernelPercpu -and [string]$kernelPercpu -ne "") { $metricParts += "kernel Percpu=$kernelPercpu percent" } + if ($null -ne $kernelSUnreclaim -and [string]$kernelSUnreclaim -ne "") { $metricParts += "kernel SUnreclaim=$kernelSUnreclaim percent" } + if ($metricSource -ne "unknown") { $metricParts += "source=$metricSource" } if ($metricParts.Count -gt 0) { $lines += "目前狀態 Current: $($metricParts -join ', ')." } else { @@ -964,22 +976,44 @@ function Get-AgentIncidentCardModel { $load = Get-AgentObjectValue $Data "loadPerCore" $null $memory = Get-AgentObjectValue $Data "memAvailablePercent" $null $disk = Get-AgentObjectValue $Data "diskUsedPercent" $null - $title = if ($reasons -contains "perf_readback_failed") { "$hostName 效能讀回失敗" } else { "$hostName 主機效能異常" } + $kernelPercpu = Get-AgentObjectValue $Data "kernelPercpuMemoryPercent" $null + $kernelSUnreclaim = Get-AgentObjectValue $Data "kernelSUnreclaimMemoryPercent" $null + $metricSource = [string](Get-AgentObjectValue $Data "metricSource" "unknown") + $kernelPressure = [bool](@($reasons | Where-Object { $_ -match '^kernel_' }).Count -gt 0) + $title = if ($reasons -contains "perf_readback_failed") { + "$hostName 效能讀回失敗" + } elseif ($kernelPressure) { + "$hostName kernel memory pressure" + } else { + "$hostName 主機效能異常" + } $target = if ($hostName) { $hostName } else { "managed host" } if ($reasons -contains "perf_readback_failed" -or ($null -eq $load -and $null -eq $memory -and $null -eq $disk)) { $impact = "CPU、記憶體與磁碟數值目前不可信，不能把空白讀回當成主機故障。" $action = "先修復監控 transport/readback，再決定是否需要降載。" $verification = "等待完整三項主機指標重新讀回。" + } elseif ($kernelPressure) { + $impact = "Kernel Percpu 或不可回收記憶體持續偏高，可能壓縮服務可用記憶體；不能誤判為單一 process RSS。" + $action = "日常改用 Node Exporter 無 SSH 監測；process 診斷依 cooldown 限流，修復走受控 kernel remediation。" + $verification = "source=$metricSource；修復後須同時確認 Percpu、SUnreclaim、可用記憶體與 session churn 回落。" } else { $impact = "資源壓力可能降低服務回應速度或可用性。" $action = "只執行 allowlisted 降載或清理，禁止任意停服務或重啟主機。" - $verification = "修復後必須重新讀回 CPU、記憶體與磁碟才可關閉。" + $verification = "source=$metricSource；修復後必須重新讀回 CPU、記憶體與磁碟才可關閉。" + } + $metrics = if ($kernelPressure) { + @( + [pscustomobject]@{ name = "Kernel Percpu"; value = $kernelPercpu; scale = 100.0; suffix = "%" }, + [pscustomobject]@{ name = "Kernel SUnreclaim"; value = $kernelSUnreclaim; scale = 100.0; suffix = "%" }, + [pscustomobject]@{ name = "可用記憶體"; value = $memory; scale = 100.0; suffix = "%" } + ) + } else { + @( + [pscustomobject]@{ name = "CPU load/core"; value = $load; scale = 2.0; suffix = "" }, + [pscustomobject]@{ name = "可用記憶體"; value = $memory; scale = 100.0; suffix = "%" }, + [pscustomobject]@{ name = "磁碟使用"; value = $disk; scale = 100.0; suffix = "%" } + ) } - $metrics = @( - [pscustomobject]@{ name = "CPU load/core"; value = $load; scale = 2.0; suffix = "" }, - [pscustomobject]@{ name = "可用記憶體"; value = $memory; scale = 100.0; suffix = "%" }, - [pscustomobject]@{ name = "磁碟使用"; value = $disk; scale = 100.0; suffix = "%" } - ) $visualKind = "performance" } elseif ($EventType -match "^backup_") { $age = Get-AgentObjectValue $Data "ageMinutes" $null @@ -2730,12 +2764,214 @@ function Convert-AgentDouble { } } +function Get-AgentNodeExporterSource { + param([string]$TargetHost) + + $performance = $Config.performance + if ( + -not $performance -or + -not $performance.PSObject.Properties["nodeExporterUrls"] -or + -not $performance.nodeExporterUrls.PSObject.Properties[$TargetHost] + ) { + return [pscustomobject]@{ + configured = $false + valid = $false + url = $null + timeoutSeconds = 0 + reason = "node_exporter_not_configured" + } + } + + $rawUrl = [string]$performance.nodeExporterUrls.PSObject.Properties[$TargetHost].Value + $uri = $null + $validUri = [Uri]::TryCreate($rawUrl, [UriKind]::Absolute, [ref]$uri) + $valid = [bool]( + $validUri -and + $uri.Scheme -eq "http" -and + $uri.Host -eq $TargetHost -and + $uri.Port -eq 9100 -and + $uri.AbsolutePath -eq "/metrics" -and + -not $uri.UserInfo -and + -not $uri.Query -and + -not $uri.Fragment + ) + $timeoutSeconds = 8 + if ($performance.PSObject.Properties["nodeExporterTimeoutSeconds"]) { + $timeoutSeconds = [math]::Min(15, [math]::Max(2, [int]$performance.nodeExporterTimeoutSeconds)) + } + [pscustomobject]@{ + configured = $true + valid = $valid + url = if ($valid) { $uri.AbsoluteUri } else { $null } + timeoutSeconds = $timeoutSeconds + reason = if ($valid) { "ready" } else { "node_exporter_url_policy_rejected" } + } +} + +function Invoke-AgentNodeExporterPerformanceReadback { + param([string]$TargetHost) + + $source = Get-AgentNodeExporterSource $TargetHost + if (-not $source.configured -or -not $source.valid) { + return [pscustomobject]@{ + configured = [bool]$source.configured + ok = $false + route = "node_exporter_http" + reason = $source.reason + output = "" + elapsedMs = 0 + httpStatus = $null + sourceUrl = $source.url + transportSerialized = $false + transportLockWaitMs = 0 + } + } + + $stopwatch = [Diagnostics.Stopwatch]::StartNew() + try { + $response = Invoke-WebRequest -Method Get -Uri $source.url -UseBasicParsing -TimeoutSec $source.timeoutSeconds + $stopwatch.Stop() + $httpStatus = [int]$response.StatusCode + $content = [string]$response.Content + if ($httpStatus -ne 200 -or -not $content) { + return [pscustomobject]@{ + configured = $true + ok = $false + route = "node_exporter_http" + reason = "node_exporter_http_not_ready" + output = "" + elapsedMs = $stopwatch.ElapsedMilliseconds + httpStatus = $httpStatus + sourceUrl = $source.url + transportSerialized = $false + transportLockWaitMs = 0 + } + } + + $load1 = $null + $memAvailableBytes = $null + $memTotalBytes = $null + $kernelPercpuBytes = $null + $kernelSUnreclaimBytes = $null + $rootAvailableBytes = $null + $rootSizeBytes = $null + $cpuIds = @{} + foreach ($line in @($content -split "`r?`n")) { + if ($null -eq $load1 -and $line -match '^node_load1\s+([^\s]+)\s*$') { + $load1 = Convert-AgentDouble $Matches[1] + continue + } + if ($null -eq $memAvailableBytes -and $line -match '^node_memory_MemAvailable_bytes\s+([^\s]+)\s*$') { + $memAvailableBytes = Convert-AgentDouble $Matches[1] + continue + } + if ($null -eq $memTotalBytes -and $line -match '^node_memory_MemTotal_bytes\s+([^\s]+)\s*$') { + $memTotalBytes = Convert-AgentDouble $Matches[1] + continue + } + if ($null -eq $kernelPercpuBytes -and $line -match '^node_memory_Percpu_bytes\s+([^\s]+)\s*$') { + $kernelPercpuBytes = Convert-AgentDouble $Matches[1] + continue + } + if ($null -eq $kernelSUnreclaimBytes -and $line -match '^node_memory_SUnreclaim_bytes\s+([^\s]+)\s*$') { + $kernelSUnreclaimBytes = Convert-AgentDouble $Matches[1] + continue + } + if ($line -match '^node_cpu_seconds_total\{([^}]*)\}\s+[^\s]+\s*$') { + $labels = $Matches[1] + if ($labels -match '(^|,)mode="idle"(,|$)' -and $labels -match '(^|,)cpu="([^"]+)"(,|$)') { + $cpuIds[$Matches[2]] = $true + } + continue + } + if ($line -match '^node_filesystem_(avail|size)_bytes\{([^}]*)\}\s+([^\s]+)\s*$') { + $metricKind = $Matches[1] + $labels = $Matches[2] + $metricValue = Convert-AgentDouble $Matches[3] + if ($labels -match '(^|,)mountpoint="/"(,|$)') { + if ($metricKind -eq "avail" -and $null -eq $rootAvailableBytes) { $rootAvailableBytes = $metricValue } + if ($metricKind -eq "size" -and $null -eq $rootSizeBytes) { $rootSizeBytes = $metricValue } + } + } + } + + $cores = $cpuIds.Count + if ( + $cores -le 0 -or + $null -eq $load1 -or + $null -eq $memAvailableBytes -or + $null -eq $memTotalBytes -or + $memTotalBytes -le 0 -or + $null -eq $rootAvailableBytes -or + $null -eq $rootSizeBytes -or + $rootSizeBytes -le 0 + ) { + return [pscustomobject]@{ + configured = $true + ok = $false + route = "node_exporter_http" + reason = "node_exporter_required_metrics_missing" + output = "" + elapsedMs = $stopwatch.ElapsedMilliseconds + httpStatus = $httpStatus + sourceUrl = $source.url + transportSerialized = $false + transportLockWaitMs = 0 + } + } + + $memAvailablePercent = [math]::Round(($memAvailableBytes / $memTotalBytes) * 100, 2) + $kernelPercpuMemoryPercent = if ($null -ne $kernelPercpuBytes) { + [math]::Round(($kernelPercpuBytes / $memTotalBytes) * 100, 2) + } else { $null } + $kernelSUnreclaimMemoryPercent = if ($null -ne $kernelSUnreclaimBytes) { + [math]::Round(($kernelSUnreclaimBytes / $memTotalBytes) * 100, 2) + } else { $null } + $diskUsedPercent = [math]::Round((1 - ($rootAvailableBytes / $rootSizeBytes)) * 100, 0) + $diskUsedPercent = [math]::Min(100, [math]::Max(0, $diskUsedPercent)) + $canonical = "perf_schema=agent99_perf_readback_v1 os=linux cores=$cores load1=$load1 mem_available_percent=$memAvailablePercent disk_used_percent=$diskUsedPercent" + return [pscustomobject]@{ + configured = $true + ok = $true + exitCode = 0 + route = "node_exporter_http" + reason = "verified" + output = $canonical + elapsedMs = $stopwatch.ElapsedMilliseconds + httpStatus = $httpStatus + sourceUrl = $source.url + kernelPercpuMemoryPercent = $kernelPercpuMemoryPercent + kernelSUnreclaimMemoryPercent = $kernelSUnreclaimMemoryPercent + kernelMemoryMetricsAvailable = [bool]($null -ne $kernelPercpuMemoryPercent -and $null -ne $kernelSUnreclaimMemoryPercent) + transportSerialized = $false + transportLockWaitMs = 0 + } + } catch { + $stopwatch.Stop() + return [pscustomobject]@{ + configured = $true + ok = $false + route = "node_exporter_http" + reason = "node_exporter_transport_failed" + output = "" + elapsedMs = $stopwatch.ElapsedMilliseconds + httpStatus = $null + sourceUrl = $source.url + transportSerialized = $false + transportLockWaitMs = 0 + } + } +} + function Get-PerfThresholds { $perf = $Config.performance [pscustomobject]@{ loadPerCoreWarning = if ($perf.loadPerCoreWarning) { [double]$perf.loadPerCoreWarning } else { 0.9 } loadPerCoreCritical = if ($perf.loadPerCoreCritical) { [double]$perf.loadPerCoreCritical } else { 1.5 } memoryAvailablePercentCritical = if ($perf.memoryAvailablePercentCritical) { [double]$perf.memoryAvailablePercentCritical } else { 10 } + kernelPercpuMemoryPercentWarning = if ($perf.kernelPercpuMemoryPercentWarning) { [double]$perf.kernelPercpuMemoryPercentWarning } else { 10 } + kernelPercpuMemoryPercentCritical = if ($perf.kernelPercpuMemoryPercentCritical) { [double]$perf.kernelPercpuMemoryPercentCritical } else { 20 } + kernelSUnreclaimMemoryPercentCritical = if ($perf.kernelSUnreclaimMemoryPercentCritical) { [double]$perf.kernelSUnreclaimMemoryPercentCritical } else { 25 } diskUsedPercentWarning = if ($perf.diskUsedPercentWarning) { [double]$perf.diskUsedPercentWarning } else { 90 } diskUsedPercentCritical = if ($perf.diskUsedPercentCritical) { [double]$perf.diskUsedPercentCritical } else { 95 } } @@ -2754,6 +2990,17 @@ function Get-HostPerformance { $readRetries = [int]$Config.performance.readRetries } $results = @() + $previousPerfEvidence = Get-AgentLatestEvidence "agent99-Perf-*.json" + $previousPerfRows = if ( + $previousPerfEvidence.exists -and + -not $previousPerfEvidence.parseError -and + $previousPerfEvidence.data -and + $previousPerfEvidence.data.PSObject.Properties["performance"] + ) { @($previousPerfEvidence.data.performance) } else { @() } + $diagnosticCooldownMinutes = 15 + if ($Config.performance -and $Config.performance.PSObject.Properties["topCpuDiagnosticCooldownMinutes"]) { + $diagnosticCooldownMinutes = [math]::Min(120, [math]::Max(5, [int]$Config.performance.topCpuDiagnosticCooldownMinutes)) + } $command = @' echo __AGENT99_PERF__ echo HOST=$(hostname) @@ -2772,7 +3019,16 @@ df -P / 2>/dev/null } $slowRetryUsed = $false $slowTimeoutSeconds = $null - $readback = Invoke-HostSshText $hostIp $command $hostTimeoutSeconds $readRetries + $nodeExporterReadback = Invoke-AgentNodeExporterPerformanceReadback $hostIp + $metricsFallbackReason = $null + if ($nodeExporterReadback.ok) { + $readback = $nodeExporterReadback + } else { + if ($nodeExporterReadback.configured) { + $metricsFallbackReason = [string]$nodeExporterReadback.reason + } + $readback = Invoke-HostSshText $hostIp $command $hostTimeoutSeconds $readRetries + } if (-not $readback.ok -and $readback.exitCode -eq -2) { $slowTimeoutSeconds = 90 if ($Config.performance -and $Config.performance.PSObject.Properties["slowReadTimeoutSeconds"]) { @@ -2795,6 +3051,8 @@ df -P / 2>/dev/null $load1 = $null $memAvailablePercent = $null $diskUsedPercent = $null + $kernelPercpuMemoryPercent = $null + $kernelSUnreclaimMemoryPercent = $null $canonicalPerfMatch = [regex]::Match( $text, "(?m)^perf_schema=agent99_perf_readback_v1\s+os=(?\S+)\s+cores=(?\d+)\s+load1=(?[0-9]+(?:[\.,][0-9]+)?)\s+mem_available_percent=(?[0-9]+(?:[\.,][0-9]+)?)\s+disk_used_percent=(?[0-9]+(?:[\.,][0-9]+)?)\s*$" @@ -2834,8 +3092,27 @@ df -P / 2>/dev/null if ($diskMatch.Success) { $diskUsedPercent = [double]$diskMatch.Groups[1].Value } } } + if ($nodeExporterReadback.ok) { + if ($nodeExporterReadback.PSObject.Properties["kernelPercpuMemoryPercent"]) { + $kernelPercpuMemoryPercent = $nodeExporterReadback.kernelPercpuMemoryPercent + } + if ($nodeExporterReadback.PSObject.Properties["kernelSUnreclaimMemoryPercent"]) { + $kernelSUnreclaimMemoryPercent = $nodeExporterReadback.kernelSUnreclaimMemoryPercent + } + } $topCpu = @() + $diagnosticStatus = "not_required" + $diagnosticKind = "none" + $diagnosticLastAttemptAt = $null + $diagnosticCapturedAt = $null + $previousPerfRow = $previousPerfRows | Where-Object { [string]$_.host -eq $hostIp } | Select-Object -First 1 + if ($previousPerfRow -and $previousPerfRow.PSObject.Properties["diagnosticLastAttemptAt"]) { + $diagnosticLastAttemptAt = [string]$previousPerfRow.diagnosticLastAttemptAt + } + if ($previousPerfRow -and $previousPerfRow.PSObject.Properties["diagnosticCapturedAt"]) { + $diagnosticCapturedAt = [string]$previousPerfRow.diagnosticCapturedAt + } $loadPerCore = if ($null -ne $load1 -and $cores -gt 0) { [math]::Round($load1 / $cores, 2) } else { $null } $severity = "ok" @@ -2855,6 +3132,17 @@ df -P / 2>/dev/null $severity = "critical" $reasons += "memory_available_low" } + if ($null -ne $kernelPercpuMemoryPercent -and $kernelPercpuMemoryPercent -ge $thresholds.kernelPercpuMemoryPercentCritical) { + $severity = "critical" + $reasons += "kernel_percpu_memory_critical" + } elseif ($null -ne $kernelPercpuMemoryPercent -and $kernelPercpuMemoryPercent -ge $thresholds.kernelPercpuMemoryPercentWarning) { + if ($severity -ne "critical") { $severity = "warning" } + $reasons += "kernel_percpu_memory_warning" + } + if ($null -ne $kernelSUnreclaimMemoryPercent -and $kernelSUnreclaimMemoryPercent -ge $thresholds.kernelSUnreclaimMemoryPercentCritical) { + $severity = "critical" + $reasons += "kernel_sunreclaim_memory_critical" + } if ($null -ne $diskUsedPercent -and $diskUsedPercent -ge $thresholds.diskUsedPercentCritical) { $severity = "critical" $reasons += "disk_used_high" @@ -2871,13 +3159,38 @@ df -P / 2>/dev/null if ($Config.performance -and $Config.performance.PSObject.Properties["topCpuTimeoutSeconds"]) { $topCpuTimeoutSeconds = [int]$Config.performance.topCpuTimeoutSeconds } - $shouldCaptureTopCpu = ($severity -eq "critical") -or ($severity -eq "warning" -and $captureTopCpuOnWarning) + $processDiagnosticReason = [bool](@($reasons | Where-Object { $_ -match '^(load_per_core|memory_available|kernel_)' }).Count -gt 0) + $shouldCaptureTopCpu = $processDiagnosticReason -and (($severity -eq "critical") -or ($severity -eq "warning" -and $captureTopCpuOnWarning)) if ($shouldCaptureTopCpu -and $readback.ok) { - $topReadback = Invoke-HostSshText $hostIp "ps -eo pid,ppid,comm,pcpu,pmem --sort=-pcpu" $topCpuTimeoutSeconds 1 - if ($topReadback.ok) { - $topCpu = @($topReadback.output -split "`n" | Select-Object -First 8) + $diagnosticDue = $true + if ($diagnosticLastAttemptAt) { + try { + $lastAttempt = [DateTimeOffset]::Parse($diagnosticLastAttemptAt) + $diagnosticDue = [bool](([DateTimeOffset]::Now - $lastAttempt).TotalMinutes -ge $diagnosticCooldownMinutes) + } catch { + $diagnosticDue = $true + } + } + if ($diagnosticDue) { + $diagnosticNow = [DateTimeOffset]::Now.ToString("o") + $diagnosticLastAttemptAt = $diagnosticNow + $diagnosticKind = if (@($reasons | Where-Object { $_ -match '^(memory_available|kernel_)' }).Count -gt 0) { "top_rss" } else { "top_cpu" } + $diagnosticCommand = if ($diagnosticKind -eq "top_rss") { + "ps -eo pid,ppid,comm,pcpu,pmem,rss --sort=-rss" + } else { + "ps -eo pid,ppid,comm,pcpu,pmem,rss --sort=-pcpu" + } + $topReadback = Invoke-HostSshText $hostIp $diagnosticCommand $topCpuTimeoutSeconds 1 + if ($topReadback.ok) { + $topCpu = @($topReadback.output -split "`n" | Select-Object -First 8) + $diagnosticCapturedAt = $diagnosticNow + $diagnosticStatus = "captured" + } else { + $diagnosticStatus = "readback_failed_backoff_active" + } } else { - $topCpu = @("top_cpu_readback_failed: $($topReadback.output)") + $diagnosticStatus = "cooldown_suppressed" + $diagnosticKind = if (@($reasons | Where-Object { $_ -match '^(memory_available|kernel_)' }).Count -gt 0) { "top_rss" } else { "top_cpu" } } } @@ -2892,7 +3205,19 @@ df -P / 2>/dev/null loadPerCore = $loadPerCore memAvailablePercent = $memAvailablePercent diskUsedPercent = $diskUsedPercent + kernelPercpuMemoryPercent = $kernelPercpuMemoryPercent + kernelSUnreclaimMemoryPercent = $kernelSUnreclaimMemoryPercent + kernelMemoryMetricsAvailable = [bool]($null -ne $kernelPercpuMemoryPercent -and $null -ne $kernelSUnreclaimMemoryPercent) topCpu = $topCpu + metricSource = if ($nodeExporterReadback.ok) { "node_exporter" } elseif ($nodeExporterReadback.configured) { "ssh_fallback" } else { "ssh" } + metricSourceUrl = if ($nodeExporterReadback.ok) { $nodeExporterReadback.sourceUrl } else { $null } + metricSourceHttpStatus = if ($nodeExporterReadback.ok) { $nodeExporterReadback.httpStatus } else { $null } + metricsFallbackReason = $metricsFallbackReason + diagnosticStatus = $diagnosticStatus + diagnosticKind = $diagnosticKind + diagnosticLastAttemptAt = $diagnosticLastAttemptAt + diagnosticCapturedAt = $diagnosticCapturedAt + diagnosticCooldownMinutes = $diagnosticCooldownMinutes output = $text readTimeoutSeconds = $hostTimeoutSeconds slowReadTimeoutSeconds = $slowTimeoutSeconds @@ -2901,9 +3226,9 @@ df -P / 2>/dev/null transportSerialized = if ($readback.PSObject.Properties["transportSerialized"]) { [bool]$readback.transportSerialized } else { $false } transportLockWaitMs = if ($readback.PSObject.Properties["transportLockWaitMs"]) { [long]$readback.transportLockWaitMs } else { $null } } - Write-AgentLog "perf host=$hostIp severity=$severity loadPerCore=$loadPerCore memAvailPct=$memAvailablePercent diskUsedPct=$diskUsedPercent route=$($readback.route) elapsedMs=$($readback.elapsedMs) transportSerialized=$($result.transportSerialized) transportLockWaitMs=$($result.transportLockWaitMs) timeoutSeconds=$hostTimeoutSeconds slowRetry=$slowRetryUsed" + Write-AgentLog "perf host=$hostIp severity=$severity loadPerCore=$loadPerCore memAvailPct=$memAvailablePercent diskUsedPct=$diskUsedPercent kernelPercpuPct=$kernelPercpuMemoryPercent kernelSUnreclaimPct=$kernelSUnreclaimMemoryPercent route=$($readback.route) metricSource=$($result.metricSource) metricsFallback=$metricsFallbackReason diagnostic=$diagnosticStatus elapsedMs=$($readback.elapsedMs) transportSerialized=$($result.transportSerialized) transportLockWaitMs=$($result.transportLockWaitMs) timeoutSeconds=$hostTimeoutSeconds slowRetry=$slowRetryUsed" if ($severity -in @("warning", "critical") -and -not $SuppressAlerts) { - Record-AgentEvent "performance_$severity" $severity "host=$hostIp loadPerCore=$loadPerCore memAvailPct=$memAvailablePercent diskUsedPct=$diskUsedPercent reasons=$($reasons -join ',')" $result -Alert + Record-AgentEvent "performance_$severity" $severity "host=$hostIp loadPerCore=$loadPerCore memAvailPct=$memAvailablePercent diskUsedPct=$diskUsedPercent kernelPercpuPct=$kernelPercpuMemoryPercent kernelSUnreclaimPct=$kernelSUnreclaimMemoryPercent reasons=$($reasons -join ',')" $result -Alert } $results += $result } diff --git a/agent99-deploy.ps1 b/agent99-deploy.ps1 index 3998fc0f4..5402d50b8 100644 --- a/agent99-deploy.ps1 +++ b/agent99-deploy.ps1 @@ -515,6 +515,44 @@ try { Add-DefaultProperty $config "performance" ([pscustomobject]@{}) Add-DefaultProperty $config.performance "processCpuHostPercentWarning" 25 Add-DefaultProperty $config.performance "processCpuHostPercentCritical" 50 + Add-DefaultProperty $config.performance "nodeExporterTimeoutSeconds" ([int]$policyConfig.performance.nodeExporterTimeoutSeconds) + Add-DefaultProperty $config.performance "nodeExporterUrls" ($policyConfig.performance.nodeExporterUrls | ConvertTo-Json -Depth 5 | ConvertFrom-Json) + Add-DefaultProperty $config.performance "topCpuDiagnosticCooldownMinutes" ([int]$policyConfig.performance.topCpuDiagnosticCooldownMinutes) + Add-DefaultProperty $config.performance "kernelPercpuMemoryPercentWarning" ([int]$policyConfig.performance.kernelPercpuMemoryPercentWarning) + Add-DefaultProperty $config.performance "kernelPercpuMemoryPercentCritical" ([int]$policyConfig.performance.kernelPercpuMemoryPercentCritical) + Add-DefaultProperty $config.performance "kernelSUnreclaimMemoryPercentCritical" ([int]$policyConfig.performance.kernelSUnreclaimMemoryPercentCritical) + foreach ($nodeExporterHost in @("192.168.0.110", "192.168.0.112", "192.168.0.188")) { + $expectedNodeExporterUrl = [string]$policyConfig.performance.nodeExporterUrls.PSObject.Properties[$nodeExporterHost].Value + $currentNodeExporterUrl = if ($config.performance.nodeExporterUrls.PSObject.Properties[$nodeExporterHost]) { + [string]$config.performance.nodeExporterUrls.PSObject.Properties[$nodeExporterHost].Value + } else { "" } + if ($currentNodeExporterUrl -ne $expectedNodeExporterUrl) { + Set-AgentProperty $config.performance.nodeExporterUrls $nodeExporterHost $expectedNodeExporterUrl + $script:ConfigMigrations += [pscustomobject]@{ + name = "node_exporter_performance_route_$($nodeExporterHost.Replace('.', '_'))" + from = $currentNodeExporterUrl + to = $expectedNodeExporterUrl + } + } + } + foreach ($performancePolicyName in @( + "nodeExporterTimeoutSeconds", + "topCpuDiagnosticCooldownMinutes", + "kernelPercpuMemoryPercentWarning", + "kernelPercpuMemoryPercentCritical", + "kernelSUnreclaimMemoryPercentCritical" + )) { + $expectedPerformanceValue = [int]$policyConfig.performance.PSObject.Properties[$performancePolicyName].Value + $currentPerformanceValue = [int]$config.performance.PSObject.Properties[$performancePolicyName].Value + if ($currentPerformanceValue -ne $expectedPerformanceValue) { + Set-AgentProperty $config.performance $performancePolicyName $expectedPerformanceValue + $script:ConfigMigrations += [pscustomobject]@{ + name = "performance_policy_$performancePolicyName" + from = $currentPerformanceValue + to = $expectedPerformanceValue + } + } + } foreach ($sourceAction in @($policyConfig.performance.loadShedding.actions | Where-Object { $_.PSObject.Properties["policyVersion"] })) { $liveAction = $config.performance.loadShedding.actions | Where-Object { $_.name -eq $sourceAction.name } | Select-Object -First 1 @@ -545,6 +583,25 @@ try { $persistedHosts = @($persistedConfig.hosts | ForEach-Object { [string]$_ }) $persistedSelfHealthHosts = @($persistedConfig.selfHealth.requiredHosts | ForEach-Object { [string]$_ }) $persistedPublicUrls = @($persistedConfig.publicUrls | ForEach-Object { [string]$_ }) + $nodeExporterRoutesReady = $true + foreach ($nodeExporterHost in @("192.168.0.110", "192.168.0.112", "192.168.0.188")) { + $expectedNodeExporterUrl = "http://${nodeExporterHost}:9100/metrics" + if ( + -not $persistedConfig.performance.nodeExporterUrls -or + -not $persistedConfig.performance.nodeExporterUrls.PSObject.Properties[$nodeExporterHost] -or + [string]$persistedConfig.performance.nodeExporterUrls.PSObject.Properties[$nodeExporterHost].Value -ne $expectedNodeExporterUrl + ) { + $nodeExporterRoutesReady = $false + } + } + $nodeExporterPolicyReady = [bool]( + $nodeExporterRoutesReady -and + [int]$persistedConfig.performance.nodeExporterTimeoutSeconds -eq 8 -and + [int]$persistedConfig.performance.topCpuDiagnosticCooldownMinutes -eq 15 -and + [int]$persistedConfig.performance.kernelPercpuMemoryPercentWarning -eq 10 -and + [int]$persistedConfig.performance.kernelPercpuMemoryPercentCritical -eq 20 -and + [int]$persistedConfig.performance.kernelSUnreclaimMemoryPercentCritical -eq 25 + ) $persistedEdgeServices = if ($persistedConfig.PSObject.Properties["edgeServiceRecovery"]) { $persistedConfig.edgeServiceRecovery } else { $null } $persistedEdgeServicesReady = [bool]( $persistedEdgeServices -and @@ -571,6 +628,7 @@ try { "192.168.0.111" -notin $persistedSelfHealthHosts -or "https://n8n.wooo.work" -notin $persistedPublicUrls -or "https://ollama.wooo.work" -notin $persistedPublicUrls -or + -not $nodeExporterPolicyReady -or -not $persistedEdgeServicesReady -or -not $host111ExternalReady -or $persistedHost112Vm.Count -ne 1 -or @@ -579,7 +637,7 @@ try { $persistedHost112Vmx -ne $canonicalHost112Vmx -or -not (Test-Path -LiteralPath $persistedHost112Vmx -PathType Leaf) ) { - throw "Canonical Host111 SSH/recovery and Host112 direct/SSD config post-verifier failed. Host188 edge recovery config post-verifier also failed." + throw "Canonical Host111 SSH/recovery and Host112 direct/SSD config post-verifier failed. Node-exporter performance routes or Host188 edge recovery config post-verifier also failed." } $manifestRows = @() diff --git a/agent99.config.99.example.json b/agent99.config.99.example.json index 92980ff42..4533279a3 100644 --- a/agent99.config.99.example.json +++ b/agent99.config.99.example.json @@ -233,13 +233,23 @@ "performance": { "enabled": true, "intervalMinutes": 1, + "nodeExporterTimeoutSeconds": 8, + "nodeExporterUrls": { + "192.168.0.110": "http://192.168.0.110:9100/metrics", + "192.168.0.112": "http://192.168.0.112:9100/metrics", + "192.168.0.188": "http://192.168.0.188:9100/metrics" + }, "loadPerCoreWarning": 0.9, "loadPerCoreCritical": 1.5, "memoryAvailablePercentCritical": 10, + "kernelPercpuMemoryPercentWarning": 10, + "kernelPercpuMemoryPercentCritical": 20, + "kernelSUnreclaimMemoryPercentCritical": 25, "diskUsedPercentWarning": 90, "diskUsedPercentCritical": 95, "captureTopCpuOnWarning": true, "topCpuTimeoutSeconds": 10, + "topCpuDiagnosticCooldownMinutes": 15, "loadShedding": { "enabled": true, "actions": [ diff --git a/agent99.config.example.json b/agent99.config.example.json index 744ab7088..e8e664216 100644 --- a/agent99.config.example.json +++ b/agent99.config.example.json @@ -222,13 +222,23 @@ "performance": { "enabled": true, "intervalMinutes": 1, + "nodeExporterTimeoutSeconds": 8, + "nodeExporterUrls": { + "192.168.0.110": "http://192.168.0.110:9100/metrics", + "192.168.0.112": "http://192.168.0.112:9100/metrics", + "192.168.0.188": "http://192.168.0.188:9100/metrics" + }, "loadPerCoreWarning": 0.9, "loadPerCoreCritical": 1.5, "memoryAvailablePercentCritical": 10, + "kernelPercpuMemoryPercentWarning": 10, + "kernelPercpuMemoryPercentCritical": 20, + "kernelSUnreclaimMemoryPercentCritical": 25, "diskUsedPercentWarning": 90, "diskUsedPercentCritical": 95, "captureTopCpuOnWarning": true, "topCpuTimeoutSeconds": 10, + "topCpuDiagnosticCooldownMinutes": 15, "loadShedding": { "enabled": true, "actions": [] diff --git a/scripts/reboot-recovery/tests/test_agent99_node_exporter_performance_contract.py b/scripts/reboot-recovery/tests/test_agent99_node_exporter_performance_contract.py new file mode 100644 index 000000000..747518b37 --- /dev/null +++ b/scripts/reboot-recovery/tests/test_agent99_node_exporter_performance_contract.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import json +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +CONTROL = ROOT / "agent99-control-plane.ps1" +DEPLOY = ROOT / "agent99-deploy.ps1" +BOOTSTRAP = ROOT / "agent99-bootstrap.ps1" +CONFIGS = ( + ROOT / "agent99.config.example.json", + ROOT / "agent99.config.99.example.json", +) + + +def read(path: Path) -> str: + return path.read_text(encoding="utf-8") + + +def function(source: str, name: str, next_name: str) -> str: + return source[ + source.index(f"function {name} {{") : source.index( + f"function {next_name} {{" + ) + ] + + +def test_node_exporter_routes_are_explicit_and_identical_in_policy_configs() -> None: + expected = { + "192.168.0.110": "http://192.168.0.110:9100/metrics", + "192.168.0.112": "http://192.168.0.112:9100/metrics", + "192.168.0.188": "http://192.168.0.188:9100/metrics", + } + + for path in CONFIGS: + performance = json.loads(read(path))["performance"] + assert performance["nodeExporterUrls"] == expected + assert performance["nodeExporterTimeoutSeconds"] == 8 + assert performance["topCpuDiagnosticCooldownMinutes"] == 15 + assert performance["kernelPercpuMemoryPercentWarning"] == 10 + assert performance["kernelPercpuMemoryPercentCritical"] == 20 + assert performance["kernelSUnreclaimMemoryPercentCritical"] == 25 + + bootstrap = read(BOOTSTRAP) + for host, url in expected.items(): + assert f'"{host}": "{url}"' in bootstrap + + +def test_node_exporter_transport_is_fixed_to_host_port_path_and_no_credentials() -> None: + source = read(CONTROL) + policy = function( + source, + "Get-AgentNodeExporterSource", + "Invoke-AgentNodeExporterPerformanceReadback", + ) + transport = function( + source, + "Invoke-AgentNodeExporterPerformanceReadback", + "Get-PerfThresholds", + ) + + for requirement in ( + '$uri.Scheme -eq "http"', + '$uri.Host -eq $TargetHost', + '$uri.Port -eq 9100', + '$uri.AbsolutePath -eq "/metrics"', + '-not $uri.UserInfo', + '-not $uri.Query', + '-not $uri.Fragment', + 'node_exporter_url_policy_rejected', + ): + assert requirement in policy + assert "Invoke-WebRequest -Method Get" in transport + assert "-TimeoutSec $source.timeoutSeconds" in transport + assert 'reason = "node_exporter_transport_failed"' in transport + assert "password" not in transport.lower() + assert "token" not in transport.lower() + + +def test_node_exporter_parser_requires_complete_load_memory_cpu_and_root_disk() -> None: + source = read(CONTROL) + transport = function( + source, + "Invoke-AgentNodeExporterPerformanceReadback", + "Get-PerfThresholds", + ) + + for metric in ( + "node_load1", + "node_memory_MemAvailable_bytes", + "node_memory_MemTotal_bytes", + "node_memory_Percpu_bytes", + "node_memory_SUnreclaim_bytes", + "node_cpu_seconds_total", + "node_filesystem_(avail|size)_bytes", + ): + assert metric in transport + assert 'mountpoint="/"' in transport + assert 'reason = "node_exporter_required_metrics_missing"' in transport + assert "perf_schema=agent99_perf_readback_v1" in transport + assert 'route = "node_exporter_http"' in transport + assert "kernelPercpuMemoryPercent" in transport + assert "kernelSUnreclaimMemoryPercent" in transport + assert "kernelMemoryMetricsAvailable" in transport + + +def test_perf_uses_metrics_first_and_ssh_only_as_fallback_or_cooldown_diagnostic() -> None: + source = read(CONTROL) + perf = function(source, "Get-HostPerformance", "Record-PerformanceIssuesWithoutRemediation") + + metrics_at = perf.index("Invoke-AgentNodeExporterPerformanceReadback $hostIp") + fallback_at = perf.index("Invoke-HostSshText $hostIp $command") + assert metrics_at < fallback_at + assert "if ($nodeExporterReadback.ok)" in perf + assert "$metricsFallbackReason = [string]$nodeExporterReadback.reason" in perf + assert 'metricSource = if ($nodeExporterReadback.ok) { "node_exporter" }' in perf + assert 'diagnosticStatus = "cooldown_suppressed"' in perf + assert "topCpuDiagnosticCooldownMinutes" in perf + assert "$diagnosticCooldownMinutes = [math]::Min(120, [math]::Max(5" in perf + assert "memory_available_low" in perf + assert "kernel_percpu_memory_warning" in perf + assert "kernel_percpu_memory_critical" in perf + assert "kernel_sunreclaim_memory_critical" in perf + assert "kernelPercpuMemoryPercentWarning" in perf + assert "kernelPercpuMemoryPercentCritical" in perf + assert "kernelSUnreclaimMemoryPercentCritical" in perf + assert '"ps -eo pid,ppid,comm,pcpu,pmem,rss --sort=-rss"' in perf + assert "disk_used_warning" not in perf[ + perf.index("$processDiagnosticReason") : perf.index("$result =") + ] + + +def test_deploy_migrates_and_post_verifies_canonical_metric_routes() -> None: + deploy = read(DEPLOY) + + assert 'Add-DefaultProperty $config.performance "nodeExporterUrls"' in deploy + assert 'Set-AgentProperty $config.performance.nodeExporterUrls' in deploy + assert 'name = "node_exporter_performance_route_' in deploy + assert "$nodeExporterRoutesReady = $true" in deploy + assert "$nodeExporterPolicyReady = [bool](" in deploy + assert "-not $nodeExporterPolicyReady" in deploy + assert "node-exporter performance routes" in deploy.lower() + for policy_name in ( + "kernelPercpuMemoryPercentWarning", + "kernelPercpuMemoryPercentCritical", + "kernelSUnreclaimMemoryPercentCritical", + ): + assert f'Add-DefaultProperty $config.performance "{policy_name}"' in deploy + assert policy_name in deploy[deploy.index("$nodeExporterPolicyReady") :] + + +def test_performance_card_explains_kernel_memory_and_metric_source() -> None: + source = read(CONTROL) + legacy = function(source, "Format-AgentTelegramLegacyText", "Get-AgentSha256Hex") + model = function(source, "Get-AgentIncidentCardModel", "Format-AgentTelegramText") + + for formatter in (legacy, model): + assert "kernel memory pressure" in formatter + assert "Percpu" in formatter + assert "SUnreclaim" in formatter + assert 'Get-AgentObjectValue $Data "metricSource"' in formatter + assert "Node Exporter" in formatter + assert "$metrics = if ($kernelPressure)" in model + assert model.index('name = "Kernel Percpu"') < model.index('name = "CPU load/core"')