From a8080e13b9d4fb44d77a8810b84164430a02ea02 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 12:45:26 +0800 Subject: [PATCH] feat(agent99): report sre alert completion cards --- HOST-REBOOT-AI-AUTOMATION-SOP.md | 18 +++++ agent99-bootstrap.ps1 | 1 + agent99-control-plane.ps1 | 110 +++++++++++++++++++++++++++---- agent99.config.99.example.json | 1 + agent99.config.example.json | 1 + 5 files changed, 119 insertions(+), 12 deletions(-) diff --git a/HOST-REBOOT-AI-AUTOMATION-SOP.md b/HOST-REBOOT-AI-AUTOMATION-SOP.md index 9cd70852d..a319b24bd 100644 --- a/HOST-REBOOT-AI-AUTOMATION-SOP.md +++ b/HOST-REBOOT-AI-AUTOMATION-SOP.md @@ -463,6 +463,7 @@ Recovery is complete only when all of the following are captured: - P0-12: slow SSH readback after reboot must not be collapsed into "host down". Performance and backup metadata checks must record `elapsedMs`, use configurable timeouts, and run a slow retry on timeout before raising `perf_readback_failed` or backup readback failure. - P0-13: first-run evidence refresh after a noise freeze must use controlled `-SuppressAlerts` for Perf/Backup/SelfCheck so stale evidence is refreshed without Telegram noise; schedules are then re-enabled without suppression. - P0-14: synthetic or `TEST_DO_NOT_PAGE` monitoring tests must still run the full Telegram -> SRE inbox -> ControlTick -> mode evidence path, but `suppressTelegram=true` and `SuppressReason=do_not_page` must propagate end to end so validation never pages the SRE Telegram group. + - P0-15: real SRE alert completion must send a human-readable Traditional Chinese automation result card back to Telegram with action taken, result, evidence, problem count, and KM path. Ordinary successful operator commands remain suppressed unless explicitly configured. - 2026-07-10 10:54 visual card verifier: - Runtime script deployed: `C:\Wooo\Agent99\bin\agent99-control-plane.ps1`. - Backup before deploy: `C:\Wooo\Agent99\bin\agent99-control-plane.ps1.bak-20260710-105439`. @@ -605,6 +606,23 @@ Recovery is complete only when all of the following are captured: - ProviderFreshness evidence: `C:\Wooo\Agent99\evidence\agent99-ProviderFreshness-20260710-123128.json`, `status=candidate_created_waiting_direct_readback`, candidate `provider-freshness-20260710-123129`. - ProviderFreshness Telegram evidence: both `agent_start` and `provider_freshness_triage` have `sent=False`, `suppressed=True`, `reason=do_not_page`. - Problem/KM writeback: `problemKey=agent99-sre-alert-inbox_unknown_provider_freshness_signal`, `occurrences=6`, `resolved=5`, `failed=1`, `lastStatus=resolved`, KM path `C:\Wooo\Agent99\playbooks\agent99-incident-km.md`. + - 2026-07-10 12:42 SRE completion card v4 verifier: + - Runtime script deployed: `C:\Wooo\Agent99\bin\agent99-control-plane.ps1`. + - Backups before deploy: `C:\Wooo\Agent99\bin\agent99-control-plane.ps1.bak-20260710-124012` and `C:\Wooo\Agent99\bin\agent99-control-plane.ps1.bak-20260710-124137`. + - Runtime bootstrap deployed: `C:\Wooo\Agent99\bin\agent99-bootstrap.ps1`, backup `C:\Wooo\Agent99\bin\agent99-bootstrap.ps1.bak-20260710-124425`. + - Runtime config readback: `C:\Wooo\Agent99\config\agent99.config.json` has `telegram.alertSreCommandResults=True`; config backup `C:\Wooo\Agent99\config\agent99.config.json.bak-20260710-124351`. + - New behavior: real `source=agent99-sre-alert-inbox` command results with `alertId` are allowed to send completion cards even when ordinary successful operator commands are suppressed. + - Guard: `suppressTelegram=true` or `TEST_DO_NOT_PAGE` still suppresses the entire path with `reason=do_not_page`. + - Telegram card self-test evidence: `C:\Wooo\Agent99\evidence\agent99-TelegramCardSelfTest-20260710-124202.json`. + - Self-test result: `ok=True`, `messageFormat=incident_card_zh_tw_v4`, `operatorTextOk=True`, `operatorDecisionOk=True`, `operatorVisualExists=True`, `sentTelegram=False`. + - Operator completion card image: `C:\Wooo\Agent99\evidence\telegram-cards\agent99-card-operator_command_result-20260710-124202.png`. + - Do-not-page E2E alert id: `telegram-monitoring-1783687343`. + - Do-not-page E2E evidence: + - Telegram inbox: `C:\Wooo\Agent99\evidence\agent99-TelegramInbox-20260710-124224.json`, `alertedCount=1`, `suppressTelegram=True`. + - SRE inbox: `C:\Wooo\Agent99\evidence\agent99-SreAlertInbox-20260710-124224.json`, `queuedCount=1`, `controlExitCode=0`. + - ControlTick: `C:\Wooo\Agent99\evidence\agent99-ControlTick-20260710-124225.json`, `processedCount=1`, `pendingCount=0`, operator result `sent=False`, `suppressed=True`, `reason=do_not_page`. + - ProviderFreshness: `C:\Wooo\Agent99\evidence\agent99-ProviderFreshness-20260710-124226.json`, `status=direct_readback_ok_no_source_events`, candidate `provider-freshness-20260710-124226`. + - Problem/KM writeback after v4 E2E: `problemKey=agent99-sre-alert-inbox_unknown_provider_freshness_signal`, `occurrences=7`, `resolved=6`, `failed=1`, `lastStatus=resolved`. ## Agent99 Monitoring Alert Routing Self-Test Receipt diff --git a/agent99-bootstrap.ps1 b/agent99-bootstrap.ps1 index 15bfd323d..8537512f3 100644 --- a/agent99-bootstrap.ps1 +++ b/agent99-bootstrap.ps1 @@ -73,6 +73,7 @@ if (-not (Test-Path $configPath)) { "alertAllOperations": false, "alertInfoEvents": false, "alertOperatorCommands": true, + "alertSreCommandResults": true, "dedupeMinutes": 30 }, "selfHealth": { diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index 46719ffce..2342f929c 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -350,22 +350,46 @@ function Format-AgentTelegramText { $instruction = if ($Data -and $Data.PSObject.Properties["instruction"]) { $Data.instruction } else { $null } $exitCode = if ($Data -and $Data.PSObject.Properties["exitCode"]) { $Data.exitCode } else { $null } $evidencePath = if ($Data -and $Data.PSObject.Properties["evidence"]) { $Data.evidence } else { $null } + $resultLabel = if ($exitCode -eq 0 -or [string](Get-AgentObjectValue $Data "ok" "") -match "^(?i:true|1)$") { "已完成 / resolved" } else { "仍需處理 / failed" } + $actionText = switch ($modeName) { + "ProviderFreshness" { "建立 provider freshness candidate，要求 provider window、last seen、direct reference、verifier readback；不切 provider。" } + "BackupCheck" { "讀回備份 freshness、status/log、snapshot metadata、cron contract；不讀備份內容或 secrets。" } + "Perf" { "讀回主機 CPU/記憶體/磁碟；必要時只做 allowlisted 降載。" } + "LoadShed" { "執行 allowlisted 降載並做 post-apply verifier。" } + "Recover" { "依序檢查 VM、SSH、Harbor/K3s/public route，必要時執行受控恢復。" } + "StartVMs" { "檢查並啟動 allowlisted VMware VM。" } + "HarborRepair" { "檢查 Harbor core/redis/registry，必要時執行受控修復。" } + "AwoooRepair" { "檢查 AWOOOI/K3s pod、deployment、image pull 狀態，必要時受控修復。" } + "SecurityTriage" { "建立資安 triage candidate，分類 blast radius、affected host/service、source log reference，不任意 shell、不暴露 secrets。" } + default { "執行 allowlisted Agent99 mode 並寫入 evidence/KM。" } + } $lines += "事件 Event: Agent99 自動化結果 / automation result" $lines += "模式 Mode: $modeName" $lines += "來源 Source: $source" - $lines += "結果 Result: exitCode=$exitCode" + $lines += "結果 Result: $resultLabel, exitCode=$exitCode" + $lines += "Agent99 動作 Action: $actionText" if ($Data -and $Data.PSObject.Properties["problem"]) { $problem = $Data.problem if ($problem -and $problem.PSObject.Properties["occurrences"]) { $lines += "問題計數 Problem count: occurrences=$($problem.occurrences), resolved=$($problem.resolved), failed=$($problem.failed)" + if ($problem.PSObject.Properties["kmPath"]) { + $lines += "KM: $($problem.kmPath)" + } } } if ($instruction -and $exitCode -ne 0) { $lines += "失敗指令 Failure instruction: $instruction" + } elseif ($instruction) { + $lines += "處理摘要 Summary: $(Limit-AgentTextLine $instruction 130)" } if ($evidencePath) { $lines += "證據 Evidence: $evidencePath" } + if ($exitCode -eq 0) { + $lines += "下一步 Next: 若相同問題再發生，依 problem count 追蹤重複率並回寫 PlayBook/KM。" + } else { + $lines += "下一步 Next: 保留 failed queue/evidence，升級到對應 playbook，不做破壞性動作。" + } } elseif ($EventType -eq "agent_control_tick") { $dashboardPath = if ($Data -and $Data.PSObject.Properties["dashboardPath"]) { $Data.dashboardPath } else { $null } $processed = if ($Data -and $Data.PSObject.Properties["processedCount"]) { $Data.processedCount } else { 0 } @@ -428,11 +452,24 @@ function Get-AgentTelegramVisualPath { $null } +function Test-AgentSreOperatorResultAlertEnabled { + param([object]$Data) + if (-not $Data) { return $false } + $source = [string](Get-AgentObjectValue $Data "source" "") + $alertId = [string](Get-AgentObjectValue $Data "alertId" "") + $enabled = $true + if ($Config.telegram -and $Config.telegram.PSObject.Properties["alertSreCommandResults"]) { + $enabled = Convert-AgentBool $Config.telegram.alertSreCommandResults + } + [bool]($enabled -and $source -eq "agent99-sre-alert-inbox" -and $alertId) +} + function New-AgentTelegramCardImage { param( [string]$Severity, [string]$EventType, - [string]$Text + [string]$Text, + [object]$Data = $null ) $visualCardsEnabled = $true @@ -440,7 +477,8 @@ function New-AgentTelegramCardImage { $visualCardsEnabled = [bool]$Config.telegram.visualCardsEnabled } if (-not $visualCardsEnabled) { return $null } - if ($Severity -notin @("warning", "critical")) { return $null } + $forceVisual = [bool]($EventType -match "^operator_command_" -and (Test-AgentSreOperatorResultAlertEnabled $Data)) + if (($Severity -notin @("warning", "critical")) -and -not $forceVisual) { return $null } $cardDir = Join-Path $EvidenceDir "telegram-cards" New-Item -ItemType Directory -Force -Path $cardDir | Out-Null @@ -553,14 +591,51 @@ function Invoke-AgentTelegramCardSelfTest { $text = Format-AgentTelegramText "warning" "performance_warning" "selftest visual card" $sample $imagePath = New-AgentTelegramCardImage "warning" "performance_warning" $text $ok = [bool]($imagePath -and (Test-Path -LiteralPath $imagePath)) + $operatorSample = [pscustomobject]@{ + id = "selftest-sre-result" + mode = "ProviderFreshness" + source = "agent99-sre-alert-inbox" + reason = "sre_alert" + instruction = "Provider freshness candidate required; alertId=selftest-provider; require providerWindow,lastSeen,directReference,verifierReadback; do not switch provider." + alertId = "selftest-provider" + alertKind = "provider_freshness_signal" + alertSeverity = "warning" + alertService = "provider_freshness_signal" + ok = $true + exitCode = 0 + evidence = $jsonPath + problem = [pscustomobject]@{ + key = "selftest-provider-freshness" + occurrences = 2 + resolved = 2 + failed = 0 + kmPath = (Join-Path $Config.agentRoot "playbooks\agent99-incident-km.md") + } + } + $operatorText = Format-AgentTelegramText "info" "operator_command_result" "selftest SRE completion card" $operatorSample + $operatorImagePath = New-AgentTelegramCardImage "info" "operator_command_result" $operatorText $operatorSample + $operatorDecisionOk = Test-AgentSreOperatorResultAlertEnabled $operatorSample + $operatorTextOk = [bool]( + $operatorText -match "已完成 / resolved" -and + $operatorText -match "Agent99 動作 Action" -and + $operatorText -match "問題計數 Problem count" -and + $operatorText -match "KM:" + ) + $operatorVisualOk = [bool]($operatorImagePath -and (Test-Path -LiteralPath $operatorImagePath)) + $ok = [bool]($ok -and $operatorTextOk -and $operatorVisualOk -and $operatorDecisionOk) $result = [pscustomobject]@{ timestamp = (Get-Date -Format o) ok = $ok textPreview = @($text -split "`n" | Select-Object -First 12) visualPath = $imagePath visualExists = $ok + operatorTextPreview = @($operatorText -split "`n" | Select-Object -First 14) + operatorVisualPath = $operatorImagePath + operatorVisualExists = $operatorVisualOk + operatorTextOk = $operatorTextOk + operatorDecisionOk = $operatorDecisionOk sentTelegram = $false - messageFormat = "incident_card_zh_tw_v3" + messageFormat = "incident_card_zh_tw_v4" } $selfTestPath = Join-Path $EvidenceDir "agent99-TelegramCardSelfTest-$stamp.json" $result | ConvertTo-Json -Depth 6 | Set-Content -Path $selfTestPath -Encoding UTF8 @@ -584,7 +659,7 @@ function Invoke-AgentTelegramDeliverySelfTest { ok = $sent testId = $testId sentTelegram = $sent - messageFormat = "incident_card_zh_tw_v3" + messageFormat = "incident_card_zh_tw_v4" attempts = $script:TelegramAttempts } $selfTestPath = Join-Path $EvidenceDir "agent99-TelegramDeliverySelfTest-$stamp.json" @@ -739,12 +814,12 @@ function Send-AgentTelegram { $attempt.error = "telegram_disabled" $script:TelegramAttempts += $attempt return - } - $text = Format-AgentTelegramText $Severity $EventType $Message $Data - $visualPath = Get-AgentTelegramVisualPath $Data - if (-not $visualPath) { - $visualPath = New-AgentTelegramCardImage $Severity $EventType $text - } + } + $text = Format-AgentTelegramText $Severity $EventType $Message $Data + $visualPath = Get-AgentTelegramVisualPath $Data + if (-not $visualPath) { + $visualPath = New-AgentTelegramCardImage $Severity $EventType $text $Data + } $attempt.visualPath = $visualPath if (-not ($token -and ($chatId -or $chatIdOverride))) { $relayResult = Send-AgentTelegramRelay $text $telegram.relay $chatIdOverride $visualPath @@ -859,20 +934,28 @@ function Record-AgentEvent { $operatorOk = $false $operatorMode = "" $operatorId = "" + $operatorSource = "" + $operatorAlertId = "" $operatorOkValue = $null $operatorExitCode = $null if ($Data -is [System.Collections.IDictionary]) { if ($Data.Contains("mode")) { $operatorMode = [string]$Data["mode"] } if ($Data.Contains("id")) { $operatorId = [string]$Data["id"] } + if ($Data.Contains("source")) { $operatorSource = [string]$Data["source"] } + if ($Data.Contains("alertId")) { $operatorAlertId = [string]$Data["alertId"] } if ($Data.Contains("ok")) { $operatorOkValue = $Data["ok"] } if ($Data.Contains("exitCode")) { $operatorExitCode = $Data["exitCode"] } } else { $modeProp = $Data.PSObject.Properties.Match("mode") | Select-Object -First 1 $idProp = $Data.PSObject.Properties.Match("id") | Select-Object -First 1 + $sourceProp = $Data.PSObject.Properties.Match("source") | Select-Object -First 1 + $alertIdProp = $Data.PSObject.Properties.Match("alertId") | Select-Object -First 1 $okProp = $Data.PSObject.Properties.Match("ok") | Select-Object -First 1 $exitCodeProp = $Data.PSObject.Properties.Match("exitCode") | Select-Object -First 1 if ($modeProp) { $operatorMode = [string]$modeProp.Value } if ($idProp) { $operatorId = [string]$idProp.Value } + if ($sourceProp) { $operatorSource = [string]$sourceProp.Value } + if ($alertIdProp) { $operatorAlertId = [string]$alertIdProp.Value } if ($okProp) { $operatorOkValue = $okProp.Value } if ($exitCodeProp) { $operatorExitCode = $exitCodeProp.Value } } @@ -883,9 +966,12 @@ function Record-AgentEvent { if ($Config.telegram -and $Config.telegram.PSObject.Properties["alertSuccessfulOperatorCommands"]) { $alertSuccessfulOperatorCommands = [bool]$Config.telegram.alertSuccessfulOperatorCommands } - if ($operatorOk -and -not $alertSuccessfulOperatorCommands) { + $operatorIsSreAlertResult = Test-AgentSreOperatorResultAlertEnabled $Data + if ($operatorOk -and -not $alertSuccessfulOperatorCommands -and -not $operatorIsSreAlertResult) { Write-AgentLog "telegram_operator_success_suppressed type=$EventType mode=$operatorMode id=$operatorId" $operatorCommandAlert = $false + } elseif ($operatorOk -and $operatorIsSreAlertResult) { + Write-AgentLog "telegram_sre_operator_success_enabled type=$EventType mode=$operatorMode id=$operatorId alertId=$operatorAlertId" } } $shouldAlert = ($Severity -in @("warning", "critical")) -or $operatorCommandAlert -or ($Alert -and $Severity -ne "info") -or ($alertAll -and ($Severity -ne "info" -or $alertInfo)) diff --git a/agent99.config.99.example.json b/agent99.config.99.example.json index f02888aa3..99fb6b33f 100644 --- a/agent99.config.99.example.json +++ b/agent99.config.99.example.json @@ -120,6 +120,7 @@ "alertAllOperations": false, "alertInfoEvents": false, "alertOperatorCommands": true, + "alertSreCommandResults": true, "dedupeMinutes": 30 }, "performance": { diff --git a/agent99.config.example.json b/agent99.config.example.json index fbd64b721..8508d5693 100644 --- a/agent99.config.example.json +++ b/agent99.config.example.json @@ -107,6 +107,7 @@ "alertAllOperations": false, "alertInfoEvents": false, "alertOperatorCommands": true, + "alertSreCommandResults": true, "dedupeMinutes": 30 }, "performance": {