From 0a671bd46aea8ea3dd91b95bf7b81bec4b0a332d Mon Sep 17 00:00:00 2001 From: ogt Date: Sat, 11 Jul 2026 20:19:21 +0800 Subject: [PATCH] fix(agent99): expose verified recovery lifecycle --- agent99-control-plane.ps1 | 152 ++++++++++++++++-- ...enterprise_ai_automation_work_items_api.py | 24 +-- ...nt99_transport_recovery_deploy_contract.py | 24 +++ ...99-enterprise-ai-automation-master-plan.md | 21 ++- .../agent99-live-preflight.ps1 | 11 +- ...t_agent99_recovery_coordinator_contract.py | 4 +- 6 files changed, 203 insertions(+), 33 deletions(-) diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index 002ff49ce..9b974a2e2 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -407,7 +407,43 @@ function Format-AgentTelegramText { $lines += "Agent99 事件卡 / Incident Card" $lines += "嚴重度 Severity: $severityLabel" - if ($EventType -match "^performance_") { + if ($EventType -eq "recovery_slo_result") { + $runId = [string](Get-AgentObjectValue $Data "runId" "unknown") + $scope = [string](Get-AgentObjectValue $Data "scope" "service_recovery") + $elapsedSeconds = [double](Get-AgentObjectValue $Data "elapsedSeconds" 0) + $targetMinutes = [int](Get-AgentObjectValue $Data "targetMinutes" 10) + $completed = Convert-AgentBool (Get-AgentObjectValue $Data "completed" $false) + $withinSlo = Convert-AgentBool (Get-AgentObjectValue $Data "withinSlo" $false) + $coordinatorVerified = Convert-AgentBool (Get-AgentObjectValue $Data "coordinatorVerified" $false) + $rebootSloClaimed = Convert-AgentBool (Get-AgentObjectValue $Data "rebootSloClaimed" $false) + $phases = if ($Data -and $Data.PSObject.Properties["phases"]) { @($Data.phases) } else { @() } + $failedPhases = @($phases | Where-Object { [string]$_.status -ne "ok" }) + $statusLabel = if ($completed -and $withinSlo) { + "已恢復且符合目標 / recovered within target" + } elseif ($completed) { + "已恢復但超過目標 / recovered late" + } else { + "恢復未完成 / recovery incomplete" + } + $scopeLabel = if ($scope -eq "full_host_reboot") { "全主機重啟 / full-host reboot" } else { "服務恢復 / service recovery" } + $phaseSummary = @($phases | ForEach-Object { + "$([string]$_.name)=$([string]$_.status)@$([math]::Round([double]$_.elapsedSeconds, 1))s" + }) -join ", " + $lines += "事件 Event: 主機與服務恢復 / recovery lifecycle" + $lines += "結果 Result: $statusLabel" + $lines += "範圍 Scope: $scopeLabel" + $lines += "耗時 Duration: $elapsedSeconds 秒；目標 Target: $targetMinutes 分鐘" + $lines += "驗證 Verifier: coordinator=$coordinatorVerified, rebootSloClaimed=$rebootSloClaimed" + $lines += "階段 Phases: $(Limit-AgentTextLine $phaseSummary 500)" + $lines += "Run ID: $runId" + if ($failedPhases.Count -gt 0) { + $lines += "Agent99 動作 Action: 保留維護狀態並持續處理失敗階段：$(@($failedPhases | ForEach-Object { $_.name }) -join ', ')。" + } elseif ($scope -eq "service_recovery" -and -not $rebootSloClaimed) { + $lines += "Agent99 動作 Action: 服務 verifier 已通過；本輪不是 fresh reboot drill，不宣稱全主機重啟 SLA。" + } else { + $lines += "Agent99 動作 Action: verifier 已通過，完成事件閉環並保留 recurrence/KM receipt。" + } + } elseif ($EventType -match "^performance_") { $hostName = Get-AgentObjectValue $Data "host" "unknown" $reasons = if ($Data -and $Data.PSObject.Properties["reasons"]) { @($Data.reasons) } else { @() } $disk = Get-AgentObjectValue $Data "diskUsedPercent" $null @@ -650,7 +686,10 @@ function New-AgentTelegramCardImage { $visualCardsEnabled = [bool]$Config.telegram.visualCardsEnabled } if (-not $visualCardsEnabled) { return $null } - $forceVisual = [bool]($EventType -match "^operator_command_" -and (Test-AgentSreOperatorResultAlertEnabled $Data)) + $forceVisual = [bool]( + $EventType -eq "recovery_slo_result" -or + ($EventType -match "^operator_command_" -and (Test-AgentSreOperatorResultAlertEnabled $Data)) + ) if (($Severity -notin @("warning", "critical")) -and -not $forceVisual) { return $null } $cardDir = Join-Path $EvidenceDir "telegram-cards" @@ -660,8 +699,9 @@ function New-AgentTelegramCardImage { try { Add-Type -AssemblyName System.Drawing -ErrorAction Stop - $width = 1040 - $height = 620 + $isRecoveryCard = [bool]($EventType -eq "recovery_slo_result") + $width = if ($isRecoveryCard) { 1200 } else { 1040 } + $height = if ($isRecoveryCard) { 760 } else { 620 } $bitmap = New-Object System.Drawing.Bitmap -ArgumentList $width, $height $graphics = [System.Drawing.Graphics]::FromImage($bitmap) $graphics.SmoothingMode = [System.Drawing.Drawing2D.SmoothingMode]::AntiAlias @@ -688,18 +728,88 @@ function New-AgentTelegramCardImage { $graphics.FillRectangle($bgBrush, 0, 0, $width, $height) $graphics.FillRectangle($panelBrush, 36, 34, $width - 72, $height - 68) $graphics.FillRectangle($accentBrush, 36, 34, 14, $height - 68) - $graphics.DrawString("Agent99 事件卡 / Incident Card", $titleFont, $textBrush, 72, 58) + $cardTitle = if ($isRecoveryCard) { "Agent99 恢復狀態 / Recovery Status" } else { "Agent99 事件卡 / Incident Card" } + $graphics.DrawString($cardTitle, $titleFont, $textBrush, 72, 58) $graphics.DrawString(("嚴重度 Severity: " + $Severity.ToUpperInvariant()), $labelFont, $accentBrush, 72, 106) $graphics.DrawString(("事件 Event: " + (Limit-AgentTextLine $EventType 76)), $labelFont, $mutedBrush, 72, 134) - $bodyLines = @($Text -split "`n" | Where-Object { $_ -and $_ -notmatch "^Agent99 事件卡 / Incident Card$" } | Select-Object -First 12) - $y = 184 - foreach ($line in $bodyLines) { - $graphics.DrawString((Limit-AgentTextLine $line 96), $bodyFont, $textBrush, 72, $y) - $y += 32 - if ($y -gt 545) { break } + if ($isRecoveryCard) { + $completed = Convert-AgentBool (Get-AgentObjectValue $Data "completed" $false) + $withinSlo = Convert-AgentBool (Get-AgentObjectValue $Data "withinSlo" $false) + $scope = [string](Get-AgentObjectValue $Data "scope" "service_recovery") + $elapsedSeconds = [double](Get-AgentObjectValue $Data "elapsedSeconds" 0) + $targetMinutes = [int](Get-AgentObjectValue $Data "targetMinutes" 10) + $runId = [string](Get-AgentObjectValue $Data "runId" "unknown") + $coordinatorVerified = Convert-AgentBool (Get-AgentObjectValue $Data "coordinatorVerified" $false) + $statusText = if ($completed -and $withinSlo) { + "已恢復 / RECOVERED" + } elseif ($completed) { + "已恢復但逾時 / RECOVERED LATE" + } else { + "尚未恢復 / ACTION REQUIRED" + } + $statusColor = if ($completed -and $withinSlo) { + [System.Drawing.Color]::FromArgb(22, 101, 52) + } elseif ($completed) { + [System.Drawing.Color]::FromArgb(180, 83, 9) + } else { + [System.Drawing.Color]::FromArgb(190, 18, 60) + } + $statusBrush = New-Object System.Drawing.SolidBrush -ArgumentList $statusColor + $statusFont = New-Object System.Drawing.Font -ArgumentList $fontFamily, 23, ([System.Drawing.FontStyle]::Bold) + $scopeText = if ($scope -eq "full_host_reboot") { "全主機重啟" } else { "服務恢復" } + $graphics.DrawString($statusText, $statusFont, $statusBrush, 72, 176) + $graphics.DrawString(("範圍 " + $scopeText + " 耗時 " + $elapsedSeconds + " 秒 目標 " + $targetMinutes + " 分鐘"), $bodyFont, $textBrush, 72, 220) + $graphics.DrawString(("Verifier: coordinator=" + $coordinatorVerified + " Run: " + (Limit-AgentTextLine $runId 72)), $smallFont, $mutedBrush, 72, 256) + $graphics.DrawString("恢復階段 / RECOVERY PHASES", $labelFont, $mutedBrush, 72, 296) + + $phaseY = 330 + $phaseIndex = 0 + $phases = if ($Data -and $Data.PSObject.Properties["phases"]) { @($Data.phases) } else { @() } + foreach ($phase in @($phases | Select-Object -First 7)) { + $phaseIndex += 1 + $phaseName = [string]$phase.name + $phaseLabel = switch ($phaseName) { + "vm_start" { "VMware 虛擬主機" } + "host_reachability" { "主機網路與 SSH" } + "host112_guest_readiness" { "Kali 112 Console 與安全服務" } + "harbor_registry" { "Harbor 與 Registry" } + "k3s_workloads" { "K3s 與 AWOOOI workloads" } + "public_routes" { "網站與公開路由" } + "full_sop_scorecard" { "Full-stack SOP scorecard" } + default { $phaseName } + } + $phaseOk = [bool]([string]$phase.status -eq "ok") + $phaseColor = if ($phaseOk) { [System.Drawing.Color]::FromArgb(22, 101, 52) } else { [System.Drawing.Color]::FromArgb(190, 18, 60) } + $phaseBrush = New-Object System.Drawing.SolidBrush -ArgumentList $phaseColor + $rowBrush = New-Object System.Drawing.SolidBrush -ArgumentList ([System.Drawing.Color]::FromArgb(244, 246, 248)) + $graphics.FillRectangle($rowBrush, 72, $phaseY, $width - 144, 38) + $graphics.FillRectangle($phaseBrush, 72, $phaseY, 10, 38) + $graphics.DrawString(("$phaseIndex. " + (Limit-AgentTextLine $phaseLabel 48)), $bodyFont, $textBrush, 98, ($phaseY + 5)) + $phaseState = if ($phaseOk) { "VERIFIED" } else { "FAILED" } + $phaseDuration = [math]::Round([double]$phase.elapsedSeconds, 1) + $graphics.DrawString(("$phaseState ${phaseDuration}s"), $labelFont, $phaseBrush, ($width - 300), ($phaseY + 8)) + $phaseBrush.Dispose() + $rowBrush.Dispose() + $phaseY += 46 + } + if ($phases.Count -eq 0) { + $graphics.DrawString("尚無 phase evidence，事件不可視為完成。", $bodyFont, $accentBrush, 72, 338) + } + $graphics.DrawString("同一 run 保留 Detect -> Apply -> Verify -> Notify receipt", $smallFont, $mutedBrush, 72, 706) + $statusBrush.Dispose() + $statusFont.Dispose() + } else { + $bodyLines = @($Text -split "`n" | Where-Object { $_ -and $_ -notmatch "^Agent99 事件卡 / Incident Card$" } | Select-Object -First 12) + $y = 184 + foreach ($line in $bodyLines) { + $graphics.DrawString((Limit-AgentTextLine $line 96), $bodyFont, $textBrush, 72, $y) + $y += 32 + if ($y -gt 545) { break } + } } - $graphics.DrawString(("產生時間 Generated: " + (Get-Date -Format "yyyy-MM-dd HH:mm:ss zzz")), $smallFont, $mutedBrush, 72, 570) + $footerY = if ($isRecoveryCard) { 724 } else { 570 } + $graphics.DrawString(("產生時間 Generated: " + (Get-Date -Format "yyyy-MM-dd HH:mm:ss zzz")), $smallFont, $mutedBrush, 72, $footerY) $bitmap.Save($imagePath, [System.Drawing.Imaging.ImageFormat]::Png) $graphics.Dispose() @@ -981,7 +1091,7 @@ function Send-AgentTelegram { enabled = [bool]$enabled eventType = $EventType severity = $Severity - messageFormat = "incident_card_zh_tw_v3" + messageFormat = "incident_card_zh_tw_v4" configured = [bool]($token -and $chatId) tokenEnv = if ($tokenSecret) { $tokenSecret.name } else { $null } chatIdEnv = if ($chatSecret) { $chatSecret.name } else { $null } @@ -1158,7 +1268,8 @@ function Record-AgentEvent { Write-AgentLog "telegram_sre_operator_success_enabled type=$EventType mode=$operatorMode id=$operatorId alertId=$operatorAlertId" } } - $shouldAlert = ($Severity -in @("warning", "critical")) -or $operatorCommandAlert -or ($Alert -and $Severity -ne "info") -or ($alertAll -and ($Severity -ne "info" -or $alertInfo)) + $lifecycleInfoAlert = [bool]($Alert -and $Severity -eq "info" -and $EventType -in @("recovery_slo_result")) + $shouldAlert = ($Severity -in @("warning", "critical")) -or $operatorCommandAlert -or $lifecycleInfoAlert -or ($Alert -and $Severity -ne "info") -or ($alertAll -and ($Severity -ne "info" -or $alertInfo)) if ($shouldAlert) { $dedupeMinutes = 30 if ($Config.telegram -and $Config.telegram.PSObject.Properties["dedupeMinutes"]) { @@ -1166,7 +1277,7 @@ function Record-AgentEvent { } $dedupeParts = @($EventType, $Severity) if ($Data) { - foreach ($propName in @("host", "name", "path", "reason", "target", "status", "mode", "alertKind", "source")) { + foreach ($propName in @("host", "name", "path", "reason", "target", "status", "mode", "alertKind", "source", "scope", "completed", "withinSlo", "rebootSloClaimed")) { if ($Data.PSObject.Properties[$propName]) { $dedupeParts += "$propName=$($Data.PSObject.Properties[$propName].Value)" } @@ -1196,6 +1307,17 @@ function Record-AgentEvent { $ageMinutes = ((Get-Date) - (Get-Item $dedupePath).LastWriteTime).TotalMinutes if ($ageMinutes -lt $dedupeMinutes) { Write-AgentLog "telegram_dedupe_skip type=$EventType severity=$Severity ageMinutes=$([math]::Round($ageMinutes, 2)) cooldown=$dedupeMinutes" + $script:TelegramAttempts += [pscustomobject]@{ + timestamp = (Get-Date -Format o) + eventType = $EventType + severity = $Severity + messageFormat = "incident_card_zh_tw_v4" + sent = $false + suppressed = $true + reason = "dedupe_window" + dedupeAgeMinutes = [math]::Round($ageMinutes, 2) + dedupeMinutes = $dedupeMinutes + } return } } diff --git a/apps/api/tests/test_agent99_enterprise_ai_automation_work_items_api.py b/apps/api/tests/test_agent99_enterprise_ai_automation_work_items_api.py index 3c81a8ee1..4d00674bd 100644 --- a/apps/api/tests/test_agent99_enterprise_ai_automation_work_items_api.py +++ b/apps/api/tests/test_agent99_enterprise_ai_automation_work_items_api.py @@ -31,7 +31,7 @@ def test_agent99_enterprise_work_items_loader_returns_complete_scope() -> None: "agent99_enterprise_ai_automation_work_items_v1" ) assert payload["scope_complete"] is False - assert payload["current_p0"]["id"] == "AG99-P0-007" + assert payload["current_p0"]["id"] == "AG99-P0-001" assert payload["summary"] == { "host_count": 7, "product_count": 12, @@ -40,21 +40,27 @@ def test_agent99_enterprise_work_items_loader_returns_complete_scope() -> None: "p0_count": 14, "p1_count": 8, "p2_count": 3, - "in_progress_count": 7, + "in_progress_count": 9, "in_progress_blocked_count": 0, - "planned_count": 16, - "complete_count": 2, + "planned_count": 15, + "complete_count": 1, "current_p0_count": 1, } - assert payload["next_execution_order"][0] == "AG99-P0-007" - assert payload["next_execution_order"][-1] == "AG99-P0-014" + assert payload["next_execution_order"][:5] == [ + "AG99-P0-001", + "AG99-P0-006", + "AG99-P0-003", + "AG99-P0-004", + "AG99-P0-005", + ] + assert payload["next_execution_order"][-1] == "AG99-P0-011" def test_agent99_enterprise_projection_is_bounded() -> None: payload = load_latest_agent99_enterprise_ai_automation_work_items() projection = build_agent99_enterprise_priority_projection(payload) - assert projection["current_p0"]["id"] == "AG99-P0-007" + assert projection["current_p0"]["id"] == "AG99-P0-001" assert projection["coverage"] == { "host_count": 7, "product_count": 12, @@ -73,7 +79,7 @@ def test_priority_readback_projects_agent99_enterprise_order() -> None: payload = load_latest_awoooi_priority_work_order_readback() projection = payload["agent99_enterprise_ai_automation"] - assert projection["current_p0"]["id"] == "AG99-P0-007" + assert projection["current_p0"]["id"] == "AG99-P0-001" assert projection["work_item_counts"]["p0"] == 14 assert ( payload["source_refs"]["agent99_enterprise_ai_automation_work_items_api"] @@ -118,7 +124,7 @@ def test_agent99_enterprise_work_items_endpoint_returns_snapshot() -> None: assert response.status_code == 200 payload = response.json() - assert payload["current_p0"]["id"] == "AG99-P0-007" + assert payload["current_p0"]["id"] == "AG99-P0-001" assert payload["summary"]["host_count"] == 7 assert payload["summary"]["product_count"] == 12 assert payload["summary"]["public_surface_count"] == 23 diff --git a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py index 0e4763a7f..a2ac08670 100644 --- a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py +++ b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py @@ -43,6 +43,30 @@ def test_agent99_auto_recovery_is_single_flight_and_slo_measured() -> None: assert '$host112Recovery -and $host112Recovery.verified' in source +def test_agent99_recovery_completion_has_human_lifecycle_receipt() -> None: + source = CONTROL.read_text(encoding="utf-8") + + assert '$EventType -eq "recovery_slo_result"' in source + assert '主機與服務恢復 / recovery lifecycle' in source + assert '已恢復且符合目標 / recovered within target' in source + assert '$lifecycleInfoAlert' in source + assert '$EventType -in @("recovery_slo_result")' in source + assert 'Agent99 恢復狀態 / Recovery Status' in source + assert '恢復階段 / RECOVERY PHASES' in source + assert 'Detect -> Apply -> Verify -> Notify receipt' in source + + +def test_agent99_records_deduped_telegram_as_an_explicit_attempt() -> None: + source = CONTROL.read_text(encoding="utf-8") + dedupe = source[source.index('if ($dedupeMinutes -gt 0 -and (Test-Path $dedupePath))') :] + dedupe = dedupe[: dedupe.index('Set-Content -Path $dedupePath')] + + assert '$script:TelegramAttempts +=' in dedupe + assert 'suppressed = $true' in dedupe + assert 'reason = "dedupe_window"' in dedupe + assert 'dedupeAgeMinutes' in dedupe + + def test_agent99_host112_recovery_is_allowlisted_and_independently_verified() -> None: source = CONTROL.read_text(encoding="utf-8") function = source[source.index("function Convert-AgentHost112GuestReadback") :] diff --git a/docs/workplans/2026-07-10-agent99-enterprise-ai-automation-master-plan.md b/docs/workplans/2026-07-10-agent99-enterprise-ai-automation-master-plan.md index 32c1e9925..c2f3a69d9 100644 --- a/docs/workplans/2026-07-10-agent99-enterprise-ai-automation-master-plan.md +++ b/docs/workplans/2026-07-10-agent99-enterprise-ai-automation-master-plan.md @@ -1,6 +1,6 @@ # Agent99 全域 AI 自動化主計畫 -> 建立時間：2026-07-10 15:05 CST；最近更新：2026-07-11 17:40 CST +> 建立時間：2026-07-10 15:05 CST；最近更新：2026-07-11 20:14 CST > 狀態：P0 執行中，尚未達成全域 AI 自動化 > 機器可讀總帳：`docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json` > 歷史需求來源：`docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md` @@ -97,13 +97,13 @@ ## 5. 更新後的唯一執行順序 -2026-07-11 事故優先序覆寫後，機器可讀執行順序固定為：`AG99-P0-007 → P0-008 → P0-009 → P0-010 → P0-003 → P0-004 → P0-005 → P0-006 → P0-011 → P0-012 → P0-013 → P0-014`。下表的數字是穩定 work-item ID，不再被誤當目前執行順位；`next_execution_order` 才是 runtime projection 的唯一順位來源，且必須包含每個未完成 P0 恰好一次。 +2026-07-11 20:14 live audit 重新校正後，機器可讀執行順序固定為：`AG99-P0-001 → P0-006 → P0-003 → P0-004 → P0-005 → P0-007 → P0-008 → P0-009 → P0-013 → P0-012 → P0-014 → P0-010 → P0-011`。先關閉 Agent99 假綠與看不見的執行結果，再擴充 ingress、evidence/candidate、verifier 與各 domain 自動化。下表的數字是穩定 work-item ID，不再被誤當目前執行順位；`next_execution_order` 才是 runtime projection 的唯一順位來源，且必須包含每個未完成 P0 恰好一次。 ### Wave 0：先修正 AI 決策真相 | 順序 | ID | 工作 | 狀態 | 驗收 | | --- | --- | --- | --- | --- | -| 1 | AG99-P0-001 | 修正 `exitCode=0 => resolved` 假綠，對齊 99 runtime script 與 Gitea source SHA | **完成** | Production 99 outcome self-test 通過；runtime manifest `7/7` matched、drift `0`；operator replay 為 verified resolved，alert replay 缺 source-event resolved 時維持 verifying | +| 1 | AG99-P0-001 | 修正 `exitCode=0 => resolved`、evidence parse 假綠，對齊 99 runtime script 與 source SHA | **重新開啟；current P0** | Production 99 outcome self-test 通過；required evidence 必須 fresh 且可解析；Recover 必須有 sent 或明確 suppressed receipt；runtime manifest matched、drift `0` | | 2 | AG99-P0-002 | 修正 alert taxonomy、wrong-domain routing、bridge-before-grouping、actionable Telegram card 漏接與重複事件 | **完成；99 runtime `34e32f6e`、14/14 matched** | Alertmanager、AI card、ACTION REQUIRED、CI/CD failure 共用 dispatcher；`suggestedMode`/kind 優先；5 分鐘先 group；同 fingerprint 只建立一個 active incident；保留 acceptance receipt | | 3 | AG99-P0-003 | 強化 ingress：relay token 必填、Telegram chat+sender allowlist、idempotency/single consumer | 待辦 | 未認證 LAN request 拒絕；producer `auto_repair`/policy 被保留；所有 request 可追 trace | | 4 | AG99-P0-004 | 新增 EvidenceCollect / RepairCandidate / MCP lane | 待辦 | 缺 evidence 的事件會建立可查 candidate，不再錯送 Perf/AwoooRepair；`INC-*` 可從收件追到 candidate | @@ -113,8 +113,8 @@ | 順序 | ID | 工作 | 狀態 | 驗收 | | --- | --- | --- | --- | --- | | 5 | AG99-P0-005 | 模式專屬 verifier、真 rollback、cooldown/circuit-breaker | 待辦 | Recover/Perf/LoadShed/Harbor/AWOOOI/backup/provider 各有 post-condition；失敗不寫成功 cooldown | -| 6 | AG99-P0-006 | Agent99 → AWOOOI/AwoooP/IWOOOS completion callback | 待辦 | incident/Run/Work Item/Verifier/recurrence 同一 trace 收斂；Telegram 可讀回報只從結果物件產生 | -| 7 | AG99-P0-007 | 全主機 10 分鐘 reboot state machine | **進行中，current P0；本輪 breach 後已 late recovery，L1/L2 runtime 已整合，L0/600 秒重測待完成** | L0 off-LAN detection/OOB power、L1 Agent99/VMware、L2 110 full-SOP 必須以同一 incident 閉環；首可用/全綠時間與 blocker/ETA 可查 | +| 6 | AG99-P0-006 | Agent99 → AWOOOI/AwoooP/IWOOOS completion callback | **進行中；先關閉 Recover 可見生命週期 receipt** | incident/Run/Work Item/Verifier/recurrence 同一 trace 收斂；Telegram 可讀回報只從結果物件產生 | +| 7 | AG99-P0-007 | 全主機 10 分鐘 reboot state machine | **進行中；本輪 breach 後已 late recovery，L1/L2 runtime 已整合，L0/600 秒重測待完成** | L0 off-LAN detection/OOB power、L1 Agent99/VMware、L2 110 full-SOP 必須以同一 incident 閉環；首可用/全綠時間與 blocker/ETA 可查 | | 8 | AG99-P0-008 | 99 VMware/VMX/autostart/lock 與 Windows Update no-auto-reboot | **本次 boot runtime green；controlled apply/rollback/verifier 完成，待下次 reboot 驗證 startup 時序** | 110/188/112/120/121 VMX、autostart task、guest running 與 no-auto-reboot verifier green；111 由實體 Mac readback 驗證 | ### Wave 2：擴至所有產品與運維面 @@ -191,7 +191,14 @@ - Windows 99 runtime 已部署 `442b319bb80e009291ced7afa4f704cbe3b64886`：manifest `14/14`、mismatch `0`、task `9/9`、SRE relay listening、Startup-Recovery `PT20M`。VMware controlled receipt `windows99-vmware-autostart-apply-20260711-173505.json` 回報 pre/post VMX process `5/5`、五 guests running、process-aware duplicate-start guard true、沒有 reboot/VM power change/secret read。 - 110 artifact `/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-172623` 顯示七主機 reachable、service/data/freshness/backup/188 hygiene/edge/VMware 全綠；只保留本輪時間窗三個 blocker。production-principal `agent99-Recover-20260711-173543.json` 在 `10.3s` 完成 service recovery、coordinator verified，且 `rebootSloClaimed=false`。 - 全部 23 個 public surfaces 已從 110 驗證為預期 2xx/3xx，SignOz bridge 也已受控恢復。這證明「目前已恢復」與「Agent99 已整合 full SOP」，不證明下一次全主機重啟會在 600 秒內完成。 -- L0 external supervisor、可驗證 OOB power 與外部 maintenance origin 仍只到 source contract，未有 production runtime receipt。current P0 因此仍是 `AG99-P0-007`；下一次 drill 只有在 L0/L1/L2 使用同一 incident 且 600 秒內 blocker=0 才能 terminal complete。 +- L0 external supervisor、可驗證 OOB power 與外部 maintenance origin 仍只到 source contract，未有 production runtime receipt。17:40 當時的 current P0 因此是 `AG99-P0-007`；下一次 drill 只有在 L0/L1/L2 使用同一 incident 且 600 秒內 blocker=0 才能 terminal complete。 + +### 2026-07-11 20:14 Agent99 可見閉環 live audit + +- Windows 99 runtime revision `d8c7a81a2a03e72f61116212354657f5e38e00a2` 為 `14/14` matched，九個必要排程皆 healthy、relay listener=1、五台 VMware guests 與五台服務主機皆可達，Harbor/AWOOOI/五個 public checks 與 backup health 皆綠。Agent99 並非完全沒有工作。 +- 最新 production-principal Recover 在 10.9 秒內完成 service recovery，七個 phase 與 full-SOP coordinator verifier 都有 evidence；但 `telegramAttemptCount=0`，所以 operator 看不到「Agent99 做了什麼、是否完成」。 +- 同一 preflight 對最新 SRE/Telegram inbox safe summary 顯示 `RuntimeException`，卻仍因檔案時間新而回傳綠燈。這是明確 false-green regression，不可用更多 dashboard 掩蓋。 +- current P0 因此重新定位為 `AG99-P0-001`，第二順位是 `AG99-P0-006`：先把 required evidence 改成 fresh + parseable、Recover 成功事件形成繁中生命週期卡與 phase timeline、dedupe 也留下 receipt；驗證後才回到 secure ingress、candidate、verifier 與 reboot/各 domain 主線。 ## 7. 工作項強制欄位與治理 @@ -208,4 +215,4 @@ ## 8. 立即下一步 -目前唯一 P0 仍是 `AG99-P0-007`，不另拆支線。L1 Agent99/VMware 與 L2 110 full-SOP 已有 runtime receipts；下一步只收斂剩餘 L0：把 external supervisor 部署到 99/110/188/LAN 之外，接上可驗證 OOB power、外部 maintenance、durable incident 與 Agent99 resume callback，再用同一 incident 做 600 秒 E2E 重測。之後依序前進：`P0-008 → P0-009 → P0-010 → P0-003 → P0-004 → P0-005 → P0-006 → P0-011 → P0-012 → P0-013 → P0-014`。main push 只在唯一 writer slot 進行；沒有 slot 時保留 local commit，不讓推送協調打斷 runtime 主線。 +目前唯一 P0 是 `AG99-P0-001`。立即動作是把 parse-aware preflight、Recover 生命週期卡、phase timeline 與 dedupe receipt 原子部署到 99，再由 production principal 跑一次 Recover；只有 runtime manifest matched、preflight parse-clean、Telegram sent 或 dedupe-suppressed receipt 明確存在才前進。後續固定依序：`P0-006 → P0-003 → P0-004 → P0-005 → P0-007 → P0-008 → P0-009 → P0-013 → P0-012 → P0-014 → P0-010 → P0-011`。main push 只在唯一 writer slot 進行；沒有 slot 時保留 local commit，不讓推送協調打斷 runtime 主線。 diff --git a/scripts/reboot-recovery/agent99-live-preflight.ps1 b/scripts/reboot-recovery/agent99-live-preflight.ps1 index d8c7def96..7f053bea3 100644 --- a/scripts/reboot-recovery/agent99-live-preflight.ps1 +++ b/scripts/reboot-recovery/agent99-live-preflight.ps1 @@ -86,6 +86,7 @@ function Get-EvidenceReadback { $fresh = [bool]($file -and $ageMinutes -le $MaxAgeMinutes) $safeSummary = $null $parseError = "" + $contentHealthy = $true if ($file) { try { $rawEvidence = Get-Content -Path $file.FullName -Raw | ConvertFrom-Json @@ -180,13 +181,17 @@ function Get-EvidenceReadback { } else { $null } } } + if ($Name -in @("sre_alert_inbox", "telegram_inbox")) { + $contentHealthy = [bool]($safeSummary -and $safeSummary.ok) + } } catch { $parseError = $_.Exception.GetType().Name + $contentHealthy = $false } } $parsed = [bool]($file -and -not $parseError) - $usable = [bool]($fresh -and $parsed) + $usable = [bool]($fresh -and $parsed -and $contentHealthy) return [pscustomobject]@{ name = $Name pattern = $Pattern @@ -197,6 +202,7 @@ function Get-EvidenceReadback { maxAgeMinutes = $MaxAgeMinutes fresh = $fresh parsed = $parsed + contentHealthy = $contentHealthy usable = $usable healthy = [bool](-not $Required -or $usable) parseError = $parseError @@ -283,6 +289,7 @@ $taskFailures = @($tasks | Where-Object { -not $_.healthy }) $requiredEvidenceFailures = @($evidence | Where-Object { $_.required -and -not $_.healthy }) $requiredEvidenceStale = @($evidence | Where-Object { $_.required -and -not $_.fresh }) $requiredEvidenceParseFailures = @($evidence | Where-Object { $_.required -and $_.fresh -and -not $_.parsed }) +$requiredEvidenceContentFailures = @($evidence | Where-Object { $_.required -and $_.fresh -and $_.parsed -and -not $_.contentHealthy }) $selfCheckEvidence = $evidence | Where-Object { $_.name -eq "self_check" } | Select-Object -First 1 $selfHealthSeverity = if ($selfCheckEvidence -and $selfCheckEvidence.safeSummary) { [string]$selfCheckEvidence.safeSummary.selfHealthSeverity } else { "unknown" } $alertIncoming = $queues | Where-Object { $_.relativePath -eq "alerts\incoming" } | Select-Object -First 1 @@ -296,6 +303,7 @@ if ($taskFailures.Count -gt 0) { $blockingReasons += "scheduled_task_unhealthy" if ($relayListeners.Count -eq 0) { $blockingReasons += "sre_alert_relay_not_listening" } if ($requiredEvidenceStale.Count -gt 0) { $blockingReasons += "required_evidence_stale" } if ($requiredEvidenceParseFailures.Count -gt 0) { $blockingReasons += "required_evidence_parse_failed" } +if ($requiredEvidenceContentFailures.Count -gt 0) { $blockingReasons += "required_evidence_content_unhealthy" } if ($selfHealthSeverity -ne "ok") { $blockingReasons += "self_health_not_ok" } if ($alertRunning.staleOverFiveMinutes -gt 0) { $blockingReasons += "stale_alert_execution" } if ($alertIncoming.count -gt 0 -or $commandQueue.count -gt 0) { $blockingReasons += "pending_work_before_reboot" } @@ -322,6 +330,7 @@ $result = [pscustomobject]@{ usableRequiredEvidenceCount = [int]@($evidence | Where-Object { $_.required -and $_.usable }).Count requiredEvidenceFailureCount = $requiredEvidenceFailures.Count requiredEvidenceParseFailureCount = $requiredEvidenceParseFailures.Count + requiredEvidenceContentFailureCount = $requiredEvidenceContentFailures.Count selfHealthSeverity = $selfHealthSeverity alertIncomingCount = [int]$alertIncoming.count alertRunningCount = [int]$alertRunning.count diff --git a/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py b/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py index 7ac91cf77..4cbed804a 100644 --- a/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py @@ -122,11 +122,13 @@ def test_live_preflight_rejects_fresh_but_unparseable_required_evidence() -> Non preflight = read("scripts/reboot-recovery/agent99-live-preflight.ps1") assert '$parsed = [bool]($file -and -not $parseError)' in preflight - assert '$usable = [bool]($fresh -and $parsed)' in preflight + assert '$usable = [bool]($fresh -and $parsed -and $contentHealthy)' in preflight assert 'healthy = [bool](-not $Required -or $usable)' in preflight assert '$_.required -and -not $_.healthy' in preflight assert 'required_evidence_parse_failed' in preflight assert 'requiredEvidenceParseFailureCount' in preflight + assert 'required_evidence_content_unhealthy' in preflight + assert 'requiredEvidenceContentFailureCount' in preflight def test_live_preflight_adapts_inbox_evidence_without_raw_message_readback() -> None: