From 6da06e35d1ad98ab898ad8683ed5e80837072723 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 10:18:51 +0800 Subject: [PATCH] fix(agent99): verify alert routing matrix --- HOST-REBOOT-AI-AUTOMATION-SOP.md | 28 +++++ agent99-sre-alert-inbox.ps1 | 192 ++++++++++++++++++++++++++++++- 2 files changed, 216 insertions(+), 4 deletions(-) diff --git a/HOST-REBOOT-AI-AUTOMATION-SOP.md b/HOST-REBOOT-AI-AUTOMATION-SOP.md index 069c92438..e527d294e 100644 --- a/HOST-REBOOT-AI-AUTOMATION-SOP.md +++ b/HOST-REBOOT-AI-AUTOMATION-SOP.md @@ -423,6 +423,34 @@ Recovery is complete only when all of the following are captured: - Send user-facing Telegram only for warnings, criticals, unresolved blockers, failed/escalated operator commands, and controlled-remediation actions. - Keep successful automation in evidence, problem counters, KM, and RAG so repeated incidents trend down without flooding the SRE group. +## Agent99 Monitoring Alert Routing Self-Test Receipt + +2026-07-10 10:17 Asia/Taipei: + +- Runtime patch deployed on 99: + - Script: `C:\Wooo\Agent99\bin\agent99-sre-alert-inbox.ps1`. + - New verifier mode: `-SelfTest`. + - Parser readback: `parse_ok`. +- Self-test evidence: + - `C:\Wooo\Agent99\evidence\agent99-SreAlertRoutingSelfTest-20260710-101747.json`. + - `ok=True`. + - `caseCount=9`. + - `passedCount=9`. + - `failedCount=0`. +- Covered alert routes: + - Public route 502 -> `Recover`, `controlledApply=True`. + - Host down / reboot / VM not running -> `Recover`, `controlledApply=True`. + - CPU / memory / disk / performance pressure -> `Perf`, `controlledApply=True`. + - Backup freshness / snapshot / cron -> `BackupCheck`, `controlledApply=False`. + - Harbor / registry -> `HarborRepair`, `controlledApply=True`. + - K3s / pod / deployment / image pull failure -> `AwoooRepair`, `controlledApply=True`. + - Provider freshness / last seen / ingestion / no-false-green -> `ProviderFreshness`, `controlledApply=False`. + - Unknown warning -> `Status`, `controlledApply=False`. + - Info heartbeat noise -> ignored. +- Defect found and fixed during self-test: + - `backup freshness` was incorrectly routed to `ProviderFreshness` because the generic word `freshness` matched too early. + - Fix: route backup/snapshot/cron before provider freshness and restrict provider freshness to provider-specific patterns such as `provider_freshness_signal`, `source_provider_freshness_triage`, `provider window`, `last_seen`, and ingestion. + ## OpenClaw Cold-Start Readback 2026-07-09 17:38 Asia/Taipei: diff --git a/agent99-sre-alert-inbox.ps1 b/agent99-sre-alert-inbox.ps1 index 97128fdac..fff648eb7 100644 --- a/agent99-sre-alert-inbox.ps1 +++ b/agent99-sre-alert-inbox.ps1 @@ -1,6 +1,7 @@ param( [string]$AgentRoot = "C:\Wooo\Agent99", [switch]$RunNow, + [switch]$SelfTest, [int]$MaxAlerts = 10 ) @@ -82,9 +83,6 @@ function Resolve-AgentAlertMode { $text = (Get-AgentAlertText $Alert).ToLowerInvariant() $suggestedMode = [string](Get-AgentField $Alert "suggestedMode" "") - if ($text -match "provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|freshness|last_seen|last seen|ingestion") { - return "ProviderFreshness" - } if ($text -match "502|site_502|public_route_502|route_down|website_down|nginx upstream") { return "Recover" } @@ -97,6 +95,9 @@ function Resolve-AgentAlertMode { if ($text -match "backup|snapshot|restore drill|cron") { return "BackupCheck" } + if ($text -match "provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|last_seen|last seen|provider freshness|provider window|ingestion") { + return "ProviderFreshness" + } if ($text -match "harbor|registry") { return "HarborRepair" } @@ -127,7 +128,7 @@ function New-AgentInstructionFromAlert { $title = [string](Get-AgentField $Alert "title" "") $message = [string](Get-AgentField $Alert "message" "") - if ($kind -eq "provider_freshness_signal" -or $message.ToLowerInvariant() -match "provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|freshness") { + if ($kind -eq "provider_freshness_signal" -or $message.ToLowerInvariant() -match "provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|provider freshness|provider window|last_seen|last seen|ingestion") { return "Provider freshness candidate required; alertId=$AlertId; source=$source; severity=$severity; service=$service; mode=$ModeName; require providerWindow,lastSeen,directReference,verifierReadback; mark missing,expired,timeout,no_false_green; do not switch provider, call paid model, edit env, or reload." } @@ -149,6 +150,189 @@ function Get-AgentControlledApplyForAlert { return [bool]($ModeName -in $remediationModes) } +function Invoke-AgentAlertRoutingSelfTest { + $cases = @( + [pscustomobject]@{ + name = "public route 502" + expectedActionable = $true + expectedMode = "Recover" + expectedControlledApply = $true + alert = [pscustomobject]@{ + id = "selftest-public-502" + source = "agent99-routing-selftest" + severity = "critical" + kind = "public_route_502" + service = "awoooi" + host = "192.168.0.120" + message = "HTTP 502 on public route" + } + }, + [pscustomobject]@{ + name = "host down or reboot" + expectedActionable = $true + expectedMode = "Recover" + expectedControlledApply = $true + alert = [pscustomobject]@{ + id = "selftest-host-down" + source = "agent99-routing-selftest" + severity = "critical" + kind = "host_down" + service = "vmware" + host = "192.168.0.110" + message = "host_reboot detected vm_not_running" + } + }, + [pscustomobject]@{ + name = "performance pressure" + expectedActionable = $true + expectedMode = "Perf" + expectedControlledApply = $true + alert = [pscustomobject]@{ + id = "selftest-performance" + source = "agent99-routing-selftest" + severity = "critical" + kind = "performance_pressure" + service = "reboot-auto-recovery-slo" + host = "110" + message = "load_per_core critical cpu pressure disk_used high" + } + }, + [pscustomobject]@{ + name = "backup freshness" + expectedActionable = $true + expectedMode = "BackupCheck" + expectedControlledApply = $false + alert = [pscustomobject]@{ + id = "selftest-backup" + source = "agent99-routing-selftest" + severity = "warning" + kind = "backup_missing" + service = "backup" + message = "backup_failed snapshot cron freshness missing" + } + }, + [pscustomobject]@{ + name = "harbor registry" + expectedActionable = $true + expectedMode = "HarborRepair" + expectedControlledApply = $true + alert = [pscustomobject]@{ + id = "selftest-harbor" + source = "agent99-routing-selftest" + severity = "critical" + kind = "registry_unhealthy" + service = "harbor" + host = "192.168.0.110" + message = "harbor registry unhealthy" + } + }, + [pscustomobject]@{ + name = "k3s pod failure" + expectedActionable = $true + expectedMode = "AwoooRepair" + expectedControlledApply = $true + alert = [pscustomobject]@{ + id = "selftest-k3s" + source = "agent99-routing-selftest" + severity = "critical" + kind = "pod_unhealthy" + service = "awoooi" + host = "192.168.0.120" + message = "k3s pod crashloopbackoff imagepullbackoff deployment failure" + } + }, + [pscustomobject]@{ + name = "provider freshness" + expectedActionable = $true + expectedMode = "ProviderFreshness" + expectedControlledApply = $false + alert = [pscustomobject]@{ + id = "selftest-provider-freshness" + source = "agent99-routing-selftest" + severity = "warning" + kind = "provider_freshness_signal" + service = "provider_freshness_signal" + message = "provider_freshness_signal source_provider_freshness_triage raw_signal_normalized controlled_runtime_candidate" + } + }, + [pscustomobject]@{ + name = "unknown warning" + expectedActionable = $true + expectedMode = "Status" + expectedControlledApply = $false + alert = [pscustomobject]@{ + id = "selftest-unknown-warning" + source = "agent99-routing-selftest" + severity = "warning" + kind = "unknown" + service = "unknown" + message = "warning with no known remediation pattern" + } + }, + [pscustomobject]@{ + name = "info noise" + expectedActionable = $false + expectedMode = $null + expectedControlledApply = $false + alert = [pscustomobject]@{ + id = "selftest-info-noise" + source = "agent99-routing-selftest" + severity = "info" + kind = "heartbeat" + service = "agent99" + message = "normal heartbeat" + } + } + ) + + $results = @() + foreach ($case in $cases) { + $actionable = Test-AgentAlertActionable $case.alert + $mode = if ($actionable) { Resolve-AgentAlertMode $case.alert } else { $null } + $controlled = if ($actionable -and $mode) { Get-AgentControlledApplyForAlert $case.alert $mode } else { $false } + $instruction = if ($actionable -and $mode) { New-AgentInstructionFromAlert $case.alert $case.alert.id $mode } else { $null } + $ok = [bool]( + $actionable -eq $case.expectedActionable -and + [string]$mode -eq [string]$case.expectedMode -and + $controlled -eq $case.expectedControlledApply + ) + $results += [pscustomobject]@{ + name = $case.name + ok = $ok + alertId = $case.alert.id + actionable = $actionable + expectedActionable = $case.expectedActionable + mode = $mode + expectedMode = $case.expectedMode + controlledApply = $controlled + expectedControlledApply = $case.expectedControlledApply + instruction = $instruction + } + } + + $result = [pscustomobject]@{ + timestamp = (Get-Date -Format o) + ok = [bool](@($results | Where-Object { -not $_.ok }).Count -eq 0) + caseCount = @($results).Count + passedCount = @($results | Where-Object { $_.ok }).Count + failedCount = @($results | Where-Object { -not $_.ok }).Count + allowedModes = $allowedModes + remediationModes = $remediationModes + results = $results + } + $path = Join-Path $EvidenceDir ("agent99-SreAlertRoutingSelfTest-" + (Get-Date -Format "yyyyMMdd-HHmmss") + ".json") + $result | Add-Member -MemberType NoteProperty -Name evidencePath -Value $path -Force + $result | ConvertTo-Json -Depth 10 | Set-Content -Path $path -Encoding UTF8 + $result +} + +if ($SelfTest) { + $selfTestResult = Invoke-AgentAlertRoutingSelfTest + $selfTestResult | ConvertTo-Json -Depth 10 + if (-not $selfTestResult.ok) { exit 1 } + exit 0 +} + $stamp = Get-Date -Format "yyyyMMdd-HHmmss" $evidencePath = Join-Path $EvidenceDir "agent99-SreAlertInbox-$stamp.json" $queued = @()