wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 3 Packages Projects Releases Wiki Activity

fix(agent99): verify alert routing matrix

Browse Source
This commit is contained in:
ogt
2026-07-10 10:18:51 +08:00
parent 8eff3bb9dd
commit 6da06e35d1
2 changed files with 216 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
HOST-REBOOT-AI-AUTOMATION-SOP.md
View File

@@ -423,6 +423,34 @@ Recovery is complete only when all of the following are captured:
- Send user-facing Telegram only for warnings, criticals, unresolved blockers, failed/escalated operator commands, and controlled-remediation actions.
- Keep successful automation in evidence, problem counters, KM, and RAG so repeated incidents trend down without flooding the SRE group.
## Agent99 Monitoring Alert Routing Self-Test Receipt
2026-07-10 10:17 Asia/Taipei:
- Runtime patch deployed on 99:
- Script: `C:\Wooo\Agent99\bin\agent99-sre-alert-inbox.ps1`.
- New verifier mode: `-SelfTest`.
- Parser readback: `parse_ok`.
- Self-test evidence:
- `C:\Wooo\Agent99\evidence\agent99-SreAlertRoutingSelfTest-20260710-101747.json`.
- `ok=True`.
- `caseCount=9`.
- `passedCount=9`.
- `failedCount=0`.
- Covered alert routes:
- Public route 502 -> `Recover`, `controlledApply=True`.
- Host down / reboot / VM not running -> `Recover`, `controlledApply=True`.
- CPU / memory / disk / performance pressure -> `Perf`, `controlledApply=True`.
- Backup freshness / snapshot / cron -> `BackupCheck`, `controlledApply=False`.
- Harbor / registry -> `HarborRepair`, `controlledApply=True`.
- K3s / pod / deployment / image pull failure -> `AwoooRepair`, `controlledApply=True`.
- Provider freshness / last seen / ingestion / no-false-green -> `ProviderFreshness`, `controlledApply=False`.
- Unknown warning -> `Status`, `controlledApply=False`.
- Info heartbeat noise -> ignored.
- Defect found and fixed during self-test:
- `backup freshness` was incorrectly routed to `ProviderFreshness` because the generic word `freshness` matched too early.
- Fix: route backup/snapshot/cron before provider freshness and restrict provider freshness to provider-specific patterns such as `provider_freshness_signal`, `source_provider_freshness_triage`, `provider window`, `last_seen`, and ingestion.
## OpenClaw Cold-Start Readback
2026-07-09 17:38 Asia/Taipei:

192
agent99-sre-alert-inbox.ps1
View File

@@ -1,6 +1,7 @@
param(
[string]$AgentRoot = "C:\Wooo\Agent99",
[switch]$RunNow,
[switch]$SelfTest,
[int]$MaxAlerts = 10
)
@@ -82,9 +83,6 @@ function Resolve-AgentAlertMode {
$text = (Get-AgentAlertText $Alert).ToLowerInvariant()
$suggestedMode = [string](Get-AgentField $Alert "suggestedMode" "")
if ($text -match "provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|freshness|last_seen|last seen|ingestion") {
return "ProviderFreshness"
}
if ($text -match "502|site_502|public_route_502|route_down|website_down|nginx upstream") {
return "Recover"
}
@@ -97,6 +95,9 @@ function Resolve-AgentAlertMode {
if ($text -match "backup|snapshot|restore drill|cron") {
return "BackupCheck"
}
if ($text -match "provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|last_seen|last seen|provider freshness|provider window|ingestion") {
return "ProviderFreshness"
}
if ($text -match "harbor|registry") {
return "HarborRepair"
}
@@ -127,7 +128,7 @@ function New-AgentInstructionFromAlert {
$title = [string](Get-AgentField $Alert "title" "")
$message = [string](Get-AgentField $Alert "message" "")
if ($kind -eq "provider_freshness_signal" -or $message.ToLowerInvariant() -match "provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|freshness") {
if ($kind -eq "provider_freshness_signal" -or $message.ToLowerInvariant() -match "provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|provider freshness|provider window|last_seen|last seen|ingestion") {
return "Provider freshness candidate required; alertId=$AlertId; source=$source; severity=$severity; service=$service; mode=$ModeName; require providerWindow,lastSeen,directReference,verifierReadback; mark missing,expired,timeout,no_false_green; do not switch provider, call paid model, edit env, or reload."
}
@@ -149,6 +150,189 @@ function Get-AgentControlledApplyForAlert {
return [bool]($ModeName -in $remediationModes)
}
function Invoke-AgentAlertRoutingSelfTest {
$cases = @(
[pscustomobject]@{
name = "public route 502"
expectedActionable = $true
expectedMode = "Recover"
expectedControlledApply = $true
alert = [pscustomobject]@{
id = "selftest-public-502"
source = "agent99-routing-selftest"
severity = "critical"
kind = "public_route_502"
service = "awoooi"
host = "192.168.0.120"
message = "HTTP 502 on public route"
}
},
[pscustomobject]@{
name = "host down or reboot"
expectedActionable = $true
expectedMode = "Recover"
expectedControlledApply = $true
alert = [pscustomobject]@{
id = "selftest-host-down"
source = "agent99-routing-selftest"
severity = "critical"
kind = "host_down"
service = "vmware"
host = "192.168.0.110"
message = "host_reboot detected vm_not_running"
}
},
[pscustomobject]@{
name = "performance pressure"
expectedActionable = $true
expectedMode = "Perf"
expectedControlledApply = $true
alert = [pscustomobject]@{
id = "selftest-performance"
source = "agent99-routing-selftest"
severity = "critical"
kind = "performance_pressure"
service = "reboot-auto-recovery-slo"
host = "110"
message = "load_per_core critical cpu pressure disk_used high"
}
},
[pscustomobject]@{
name = "backup freshness"
expectedActionable = $true
expectedMode = "BackupCheck"
expectedControlledApply = $false
alert = [pscustomobject]@{
id = "selftest-backup"
source = "agent99-routing-selftest"
severity = "warning"
kind = "backup_missing"
service = "backup"
message = "backup_failed snapshot cron freshness missing"
}
},
[pscustomobject]@{
name = "harbor registry"
expectedActionable = $true
expectedMode = "HarborRepair"
expectedControlledApply = $true
alert = [pscustomobject]@{
id = "selftest-harbor"
source = "agent99-routing-selftest"
severity = "critical"
kind = "registry_unhealthy"
service = "harbor"
host = "192.168.0.110"
message = "harbor registry unhealthy"
}
},
[pscustomobject]@{
name = "k3s pod failure"
expectedActionable = $true
expectedMode = "AwoooRepair"
expectedControlledApply = $true
alert = [pscustomobject]@{
id = "selftest-k3s"
source = "agent99-routing-selftest"
severity = "critical"
kind = "pod_unhealthy"
service = "awoooi"
host = "192.168.0.120"
message = "k3s pod crashloopbackoff imagepullbackoff deployment failure"
}
},
[pscustomobject]@{
name = "provider freshness"
expectedActionable = $true
expectedMode = "ProviderFreshness"
expectedControlledApply = $false
alert = [pscustomobject]@{
id = "selftest-provider-freshness"
source = "agent99-routing-selftest"
severity = "warning"
kind = "provider_freshness_signal"
service = "provider_freshness_signal"
message = "provider_freshness_signal source_provider_freshness_triage raw_signal_normalized controlled_runtime_candidate"
}
},
[pscustomobject]@{
name = "unknown warning"
expectedActionable = $true
expectedMode = "Status"
expectedControlledApply = $false
alert = [pscustomobject]@{
id = "selftest-unknown-warning"
source = "agent99-routing-selftest"
severity = "warning"
kind = "unknown"
service = "unknown"
message = "warning with no known remediation pattern"
}
},
[pscustomobject]@{
name = "info noise"
expectedActionable = $false
expectedMode = $null
expectedControlledApply = $false
alert = [pscustomobject]@{
id = "selftest-info-noise"
source = "agent99-routing-selftest"
severity = "info"
kind = "heartbeat"
service = "agent99"
message = "normal heartbeat"
}
}
)
$results = @()
foreach ($case in $cases) {
$actionable = Test-AgentAlertActionable $case.alert
$mode = if ($actionable) { Resolve-AgentAlertMode $case.alert } else { $null }
$controlled = if ($actionable -and $mode) { Get-AgentControlledApplyForAlert $case.alert $mode } else { $false }
$instruction = if ($actionable -and $mode) { New-AgentInstructionFromAlert $case.alert $case.alert.id $mode } else { $null }
$ok = [bool](
$actionable -eq $case.expectedActionable -and
[string]$mode -eq [string]$case.expectedMode -and
$controlled -eq $case.expectedControlledApply
)
$results += [pscustomobject]@{
name = $case.name
ok = $ok
alertId = $case.alert.id
actionable = $actionable
expectedActionable = $case.expectedActionable
mode = $mode
expectedMode = $case.expectedMode
controlledApply = $controlled
expectedControlledApply = $case.expectedControlledApply
instruction = $instruction
}
}
$result = [pscustomobject]@{
timestamp = (Get-Date -Format o)
ok = [bool](@($results | Where-Object { -not $_.ok }).Count -eq 0)
caseCount = @($results).Count
passedCount = @($results | Where-Object { $_.ok }).Count
failedCount = @($results | Where-Object { -not $_.ok }).Count
allowedModes = $allowedModes
remediationModes = $remediationModes
results = $results
}
$path = Join-Path $EvidenceDir ("agent99-SreAlertRoutingSelfTest-" + (Get-Date -Format "yyyyMMdd-HHmmss") + ".json")
$result | Add-Member -MemberType NoteProperty -Name evidencePath -Value $path -Force
$result | ConvertTo-Json -Depth 10 | Set-Content -Path $path -Encoding UTF8
$result
}
if ($SelfTest) {
$selfTestResult = Invoke-AgentAlertRoutingSelfTest
$selfTestResult | ConvertTo-Json -Depth 10
if (-not $selfTestResult.ok) { exit 1 }
exit 0
}
$stamp = Get-Date -Format "yyyyMMdd-HHmmss"
$evidencePath = Join-Path $EvidenceDir "agent99-SreAlertInbox-$stamp.json"
$queued = @()