fix(agent99): reduce SSH performance probe churn

scripts/reboot-recovery/tests/test_agent99_node_exporter_performance_contract.py

@@ -0,0 +1,165 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[3]
+CONTROL = ROOT / "agent99-control-plane.ps1"
+DEPLOY = ROOT / "agent99-deploy.ps1"
+BOOTSTRAP = ROOT / "agent99-bootstrap.ps1"
+CONFIGS = (
+    ROOT / "agent99.config.example.json",
+    ROOT / "agent99.config.99.example.json",
+)
+
+
+def read(path: Path) -> str:
+    return path.read_text(encoding="utf-8")
+
+
+def function(source: str, name: str, next_name: str) -> str:
+    return source[
+        source.index(f"function {name} {{") : source.index(
+            f"function {next_name} {{"
+        )
+    ]
+
+
+def test_node_exporter_routes_are_explicit_and_identical_in_policy_configs() -> None:
+    expected = {
+        "192.168.0.110": "http://192.168.0.110:9100/metrics",
+        "192.168.0.112": "http://192.168.0.112:9100/metrics",
+        "192.168.0.188": "http://192.168.0.188:9100/metrics",
+    }
+
+    for path in CONFIGS:
+        performance = json.loads(read(path))["performance"]
+        assert performance["nodeExporterUrls"] == expected
+        assert performance["nodeExporterTimeoutSeconds"] == 8
+        assert performance["topCpuDiagnosticCooldownMinutes"] == 15
+        assert performance["kernelPercpuMemoryPercentWarning"] == 10
+        assert performance["kernelPercpuMemoryPercentCritical"] == 20
+        assert performance["kernelSUnreclaimMemoryPercentCritical"] == 25
+
+    bootstrap = read(BOOTSTRAP)
+    for host, url in expected.items():
+        assert f'"{host}": "{url}"' in bootstrap
+
+
+def test_node_exporter_transport_is_fixed_to_host_port_path_and_no_credentials() -> None:
+    source = read(CONTROL)
+    policy = function(
+        source,
+        "Get-AgentNodeExporterSource",
+        "Invoke-AgentNodeExporterPerformanceReadback",
+    )
+    transport = function(
+        source,
+        "Invoke-AgentNodeExporterPerformanceReadback",
+        "Get-PerfThresholds",
+    )
+
+    for requirement in (
+        '$uri.Scheme -eq "http"',
+        '$uri.Host -eq $TargetHost',
+        '$uri.Port -eq 9100',
+        '$uri.AbsolutePath -eq "/metrics"',
+        '-not $uri.UserInfo',
+        '-not $uri.Query',
+        '-not $uri.Fragment',
+        'node_exporter_url_policy_rejected',
+    ):
+        assert requirement in policy
+    assert "Invoke-WebRequest -Method Get" in transport
+    assert "-TimeoutSec $source.timeoutSeconds" in transport
+    assert 'reason = "node_exporter_transport_failed"' in transport
+    assert "password" not in transport.lower()
+    assert "token" not in transport.lower()
+
+
+def test_node_exporter_parser_requires_complete_load_memory_cpu_and_root_disk() -> None:
+    source = read(CONTROL)
+    transport = function(
+        source,
+        "Invoke-AgentNodeExporterPerformanceReadback",
+        "Get-PerfThresholds",
+    )
+
+    for metric in (
+        "node_load1",
+        "node_memory_MemAvailable_bytes",
+        "node_memory_MemTotal_bytes",
+        "node_memory_Percpu_bytes",
+        "node_memory_SUnreclaim_bytes",
+        "node_cpu_seconds_total",
+        "node_filesystem_(avail|size)_bytes",
+    ):
+        assert metric in transport
+    assert 'mountpoint="/"' in transport
+    assert 'reason = "node_exporter_required_metrics_missing"' in transport
+    assert "perf_schema=agent99_perf_readback_v1" in transport
+    assert 'route = "node_exporter_http"' in transport
+    assert "kernelPercpuMemoryPercent" in transport
+    assert "kernelSUnreclaimMemoryPercent" in transport
+    assert "kernelMemoryMetricsAvailable" in transport
+
+
+def test_perf_uses_metrics_first_and_ssh_only_as_fallback_or_cooldown_diagnostic() -> None:
+    source = read(CONTROL)
+    perf = function(source, "Get-HostPerformance", "Record-PerformanceIssuesWithoutRemediation")
+
+    metrics_at = perf.index("Invoke-AgentNodeExporterPerformanceReadback $hostIp")
+    fallback_at = perf.index("Invoke-HostSshText $hostIp $command")
+    assert metrics_at < fallback_at
+    assert "if ($nodeExporterReadback.ok)" in perf
+    assert "$metricsFallbackReason = [string]$nodeExporterReadback.reason" in perf
+    assert 'metricSource = if ($nodeExporterReadback.ok) { "node_exporter" }' in perf
+    assert 'diagnosticStatus = "cooldown_suppressed"' in perf
+    assert "topCpuDiagnosticCooldownMinutes" in perf
+    assert "$diagnosticCooldownMinutes = [math]::Min(120, [math]::Max(5" in perf
+    assert "memory_available_low" in perf
+    assert "kernel_percpu_memory_warning" in perf
+    assert "kernel_percpu_memory_critical" in perf
+    assert "kernel_sunreclaim_memory_critical" in perf
+    assert "kernelPercpuMemoryPercentWarning" in perf
+    assert "kernelPercpuMemoryPercentCritical" in perf
+    assert "kernelSUnreclaimMemoryPercentCritical" in perf
+    assert '"ps -eo pid,ppid,comm,pcpu,pmem,rss --sort=-rss"' in perf
+    assert "disk_used_warning" not in perf[
+        perf.index("$processDiagnosticReason") : perf.index("$result =")
+    ]
+
+
+def test_deploy_migrates_and_post_verifies_canonical_metric_routes() -> None:
+    deploy = read(DEPLOY)
+
+    assert 'Add-DefaultProperty $config.performance "nodeExporterUrls"' in deploy
+    assert 'Set-AgentProperty $config.performance.nodeExporterUrls' in deploy
+    assert 'name = "node_exporter_performance_route_' in deploy
+    assert "$nodeExporterRoutesReady = $true" in deploy
+    assert "$nodeExporterPolicyReady = [bool](" in deploy
+    assert "-not $nodeExporterPolicyReady" in deploy
+    assert "node-exporter performance routes" in deploy.lower()
+    for policy_name in (
+        "kernelPercpuMemoryPercentWarning",
+        "kernelPercpuMemoryPercentCritical",
+        "kernelSUnreclaimMemoryPercentCritical",
+    ):
+        assert f'Add-DefaultProperty $config.performance "{policy_name}"' in deploy
+        assert policy_name in deploy[deploy.index("$nodeExporterPolicyReady") :]
+
+
+def test_performance_card_explains_kernel_memory_and_metric_source() -> None:
+    source = read(CONTROL)
+    legacy = function(source, "Format-AgentTelegramLegacyText", "Get-AgentSha256Hex")
+    model = function(source, "Get-AgentIncidentCardModel", "Format-AgentTelegramText")
+
+    for formatter in (legacy, model):
+        assert "kernel memory pressure" in formatter
+        assert "Percpu" in formatter
+        assert "SUnreclaim" in formatter
+        assert 'Get-AgentObjectValue $Data "metricSource"' in formatter
+        assert "Node Exporter" in formatter
+    assert "$metrics = if ($kernelPressure)" in model
+    assert model.index('name = "Kernel Percpu"') < model.index('name = "CPU load/core"')