From 244bad7259f0378b949e5ea9ca4c3ac567f3d11b Mon Sep 17 00:00:00 2001 From: ogt Date: Sat, 11 Jul 2026 20:17:24 +0800 Subject: [PATCH] docs(recovery): record host112 agent closure --- docs/LOGBOOK.md | 12 +++ ...ise-ai-automation-work-items.snapshot.json | 60 ++++++++------- ...t112-guest-recovery-20260711.snapshot.json | 35 ++++++++- docs/runbooks/REBOOT-RECOVERY-SOP.md | 6 +- .../agent99-live-preflight.ps1 | 75 +++++++++++++------ ...t_agent99_recovery_coordinator_contract.py | 29 +++++++ 6 files changed, 161 insertions(+), 56 deletions(-) diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index c537fdf2f..4901c91bf 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -54699,3 +54699,15 @@ production browser smoke: **剩餘驗收**： - 112 runtime scope 已恢復並驗證；全主機 program 仍保留歷史 10 分鐘逾時，不以事後綠燈覆蓋。下一次真實全主機 reboot 必須在同一 600 秒 window 重新通過 VMX、112 Console/Wazuh/timer、七主機、服務、資料、備份與 public fallback。 + +## 2026-07-11 - P0-008 Agent99/Prometheus production closure + +**Runtime evidence**： +- Agent99 原子部署 revision `d0e364bfa42947190cf1e88d1d85623e23f84858`，manifest `14/14` matched、mismatch `0`；deploy receipt `C:\Wooo\Agent99\evidence\agent99-deploy-20260711-200939.json`，rollback backup `C:\Wooo\Agent99\deploy\backup-20260711-200939`。 +- production-principal `Recover` receipt `C:\Wooo\Agent99\evidence\agent99-Recover-20260711-200320.json`：七個 phases 全部 `ok`；`host112_guest_readiness` before/after ready，`applyAttempted=false`；transport direct/serialized；coordinator verified；elapsed `44.6s`；`rebootSloClaimed=false`。 +- 最新 110 scorecard `/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-200338` 的 host112 blocker 為 `0`。Prometheus 五項 host112 gauges 均為 `1`，`Host112GuestReadbackMissing` 與 `Host112GuestNotReady` 均 health `ok`、state `inactive`；source/canonical/active/runtime rule SHA 均為 `86fd0e56cb7633da2a5a09960b8bf2dd61ee917c55072b5a16ba68577e8379cc`。 + +**Verification**： +- Agent99 SRE routing self-test `17/17`；Telegram ingest self-test `16/16`，不寫 alert、不送 Telegram。 +- Agent99 tests `49 passed`；reboot-recovery suite `142 passed`；Windows staging contract/synthetic、JSON/YAML parse 通過。 +- 112 runtime scope 已關閉。全主機 program 尚未關閉：只剩 fresh all-host reboot window 的三項歷史 blocker，必須由下一次真實 600 秒 drill 驗證，不能以事後綠燈清除。 diff --git a/docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json b/docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json index b2938466b..2c1b63089 100644 --- a/docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json +++ b/docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json @@ -1,7 +1,7 @@ { "schema_version": "agent99_enterprise_ai_automation_work_items_v1", - "generated_at": "2026-07-11T17:40:00+08:00", - "status": "in_progress_p0_reboot_slo_breached_recovered_late_l1_l2_integrated_l0_required", + "generated_at": "2026-07-11T20:14:33+08:00", + "status": "in_progress_p0_agent99_false_green_and_lifecycle_closure", "scope_complete": false, "north_star": "Make Windows host 192.168.0.99 the policy-controlled on-prem AI execution node for all hosts, VMs, services, products, sites, tools, packages, alerts and recovery workflows, with external monitoring when host 99 is unavailable.", "canonical_sources": { @@ -17,11 +17,11 @@ "runtime_reboot_slo_readback": "https://awoooi.wooo.work/api/v1/agents/reboot-auto-recovery-slo-scorecard" }, "current_p0": { - "id": "AG99-P0-007", - "title": "Agent99-coordinated all-host ten-minute reboot recovery state machine", - "reason": "The approved drill breached T+10 and later recovered. Live Windows boot evidence corrected the observer false negative: host 99 booted before the deadline, but VMware startup was delayed, Agent99 was killed by its PT10M task limit, the reboot event was consumed early and dependent PostgreSQL/K3s services recovered late. Agent99 L1 and 110 L2 are now integrated in production; external L0 observation, out-of-band power and external maintenance remain required before the next 600-second drill can close.", - "implementation_status": "slo_breached_recovered_late_agent99_l1_vmware_and_110_l2_runtime_verified_external_l0_missing", - "next_action": "Deploy the external L0 supervisor outside the local failure domain, bind verified OOB power, maintenance and Agent99 resume callbacks, then repeat one incident-correlated full reboot drill and require zero blockers within 600 seconds." + "id": "AG99-P0-001", + "title": "Agent99 runtime truth, false-green prevention and visible lifecycle receipt", + "reason": "The 2026-07-11 live audit proved that Agent99 runs real recovery work, but the operator cannot reliably see the closure: required inbox evidence can fail safe-summary parsing while preflight remains green, a successful Recover produces zero Telegram attempts because info lifecycle events are filtered out, and dedupe suppression leaves no explicit delivery receipt. Runtime truth and one-run lifecycle visibility must be corrected before adding more automation breadth.", + "implementation_status": "regression_reopened_source_fix_in_progress_runtime_deploy_and_recover_verifier_pending", + "next_action": "Deploy the parse-aware preflight and recovery lifecycle card to Windows 99, run one production-principal Recover, and require one explicit Telegram sent-or-dedupe receipt plus a parse-clean live preflight." }, "current_runtime_truth": { "observed_at": "2026-07-11T17:37:00+08:00", @@ -272,12 +272,12 @@ "id": "AG99-P0-001", "order": 1, "priority": "P0", - "status": "complete", - "title": "Agent outcome truth and source-runtime reconciliation", + "status": "in_progress", + "title": "Agent outcome truth, source-runtime reconciliation and false-green prevention", "owner_lane": "agent99_control_plane", "dependencies": [], "scope_refs": ["host:99", "platform:Agent99", "platform:AWOOOI", "platform:AwoooP"], - "problem": "Command exit success is counted as resolved and deployed Windows scripts can drift from Gitea source.", + "problem": "Command exit success can be counted as resolved, deployed Windows scripts can drift from source, and evidence freshness can be reported green even when the safe summary cannot be parsed.", "trigger": "Any Agent99 command completion or source/runtime manifest mismatch.", "ai_role": "Outcome evaluator and runtime provenance verifier", "action_mode": "controlled_apply", @@ -310,7 +310,9 @@ "C:\\Wooo\\Agent99\\evidence\\agent99-ControlTick-20260710-161919.json" ] }, - "next_action": "Keep runtime-manifest drift and verified-resolution counters under SelfCheck; continue with AG99-P0-002." + "regression_reopened_at": "2026-07-11T20:02:00+08:00", + "regression_evidence": ["Windows 99 live preflight returned green while latest SreAlertInbox and TelegramInbox safe summaries reported RuntimeException", "latest production-principal Recover was verifier-green but telegramAttemptCount=0", "dedupe suppression returned without a Telegram attempt receipt"], + "next_action": "Deploy the parse-aware evidence gate, explicit lifecycle info alert and dedupe receipt; rerun production-principal Recover and keep this item open until runtime readback is parse-clean and notification-visible." }, { "id": "AG99-P0-002", @@ -414,10 +416,10 @@ "id": "AG99-P0-006", "order": 6, "priority": "P0", - "status": "planned", + "status": "in_progress", "title": "Agent99 completion callback and incident closure", "owner_lane": "agent99_awoooi_awooop_integration", - "dependencies": ["AG99-P0-001", "AG99-P0-005"], + "dependencies": ["AG99-P0-001", "AG99-P0-002"], "scope_refs": ["platform:AWOOOI", "platform:AwoooP", "platform:IWOOOS", "surface:Telegram"], "problem": "AWOOOI-to-Agent99 dispatch is not matched by a canonical Agent99-to-AWOOOI completion path.", "trigger": "Any Agent99 run changes state or produces a verifier result.", @@ -430,7 +432,8 @@ "verifier": "Read back identical trace, final state and verifier evidence from AWOOOI and AwoooP.", "callback": "This work item creates the canonical callback contract.", "acceptance": "One trace is visible from alert receipt through final verifier, notification and recurrence counters.", - "next_action": "Define callback schema and idempotent authenticated endpoint." + "implementation_progress": "The first closure slice is being implemented in Agent99 runtime: Recover now has a dedicated Traditional-Chinese lifecycle summary, a phase timeline image, a successful info-event delivery policy and an explicit dedupe receipt. Canonical authenticated AWOOOI/AwoooP/IWOOOS writeback remains open.", + "next_action": "Verify the Windows 99 recovery lifecycle receipt, then define the canonical callback schema and idempotent authenticated AWOOOI/AwoooP/IWOOOS endpoint." }, { "id": "AG99-P0-007", @@ -476,9 +479,9 @@ "verifier": "All required VMX paths exist; guests report running and reachable; host 112 returns systemd running, graphical target, LightDM, Xorg, VMware Tools, Wazuh stack and recovery timer green through a forced-command readback; tasks survive reboot; update policy drift is zero.", "callback": "Update reboot SLO VMware and Windows policy gates.", "acceptance": "99 startup automatically launches VMware guests 110/188/112/120/121 in order; physical host 111 is verified separately; 112 must pass Console-ready and Wazuh service gates rather than only Ping or SSH; no exclusive-lock failure occurs; Windows Update cannot surprise-reboot the host.", - "implementation_progress": "The controlled VMware apply remains green with five running VMX processes. The 112 follow-up restored graphical.target, LightDM and Wazuh Indexer, deployed a bounded guest recovery timer, restricted 110 forced-command readback, exact-command sudo executor and durable rollback installer. The 110 scorecard now reads systemd running plus Console/Wazuh/timer/guest 1/1/1/1, all five host112 Prometheus gauges are 1, host112 blockers are zero, both dedicated alert rules are healthy/inactive, and Windows99 records the SYSTEM session VMX as runtime_running_gui_session_isolated. Reboot-recovery tests pass 142/142.", - "evidence_refs": ["C:\\Wooo\\Agent99\\evidence\\windows99-vmware-autostart-apply-20260711-173505.json", "C:\\Wooo\\Agent99\\deploy\\vmware-autostart-backup-20260711-173505", "/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-194901", "/var/lib/awoooi-host112-recovery/backups/host112-guest-recovery-20260711T195449+0800-43505", "docs/operations/host112-guest-recovery-20260711.snapshot.json", "reboot-recovery suite 142 passed"], - "next_action": "Keep guest power unchanged, route any host112 state change to Agent99 controlled apply and lifecycle callback, then prove the same Console/Wazuh/timer gates inside the next real all-host 600-second drill." + "implementation_progress": "The controlled VMware apply remains green with five running VMX processes. The 112 follow-up restored graphical.target, LightDM and Wazuh Indexer, deployed a bounded guest recovery timer, restricted 110 forced-command readback, exact-command sudo executor and durable rollback installer. The 110 scorecard reads systemd running plus Console/Wazuh/timer/guest 1/1/1/1, all five host112 Prometheus gauges are 1, host112 blockers are zero, both dedicated alert rules are healthy/inactive, and Windows99 records the SYSTEM session VMX as runtime_running_gui_session_isolated. Agent99 revision d0e364bfa42947190cf1e88d1d85623e23f84858 is live with 14/14 manifest parity; production-principal Recover includes host112_guest_readiness, verified ready before and after without unnecessary apply, completed all seven phases in 44.6 seconds, and correctly did not claim a fresh reboot SLO. SRE routing is 17/17 and Telegram ingest is 16/16 in no-send validation. Agent99 tests pass 49/49 and reboot-recovery tests pass 142/142.", + "evidence_refs": ["C:\\Wooo\\Agent99\\evidence\\windows99-vmware-autostart-apply-20260711-173505.json", "C:\\Wooo\\Agent99\\deploy\\vmware-autostart-backup-20260711-173505", "C:\\Wooo\\Agent99\\evidence\\agent99-deploy-20260711-200939.json", "C:\\Wooo\\Agent99\\evidence\\agent99-Recover-20260711-200320.json", "/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-200338", "/var/lib/awoooi-host112-recovery/backups/host112-guest-recovery-20260711T195449+0800-43505", "/home/wooo/monitoring/alerts-unified.canonical.yml.host112-v2.bak.20260711200617", "docs/operations/host112-guest-recovery-20260711.snapshot.json", "Agent99 suite 49 passed", "reboot-recovery suite 142 passed"], + "next_action": "Keep guest power unchanged and retain the closed host112 controlled-recovery route. The remaining program verifier is one fresh all-host 600-second drill; the separate VMware UI must also stop presenting the SYSTEM-owned running VMX as an actionable powered-off state." }, { "id": "AG99-P0-009", @@ -787,24 +790,25 @@ "p0_count": 14, "p1_count": 8, "p2_count": 3, - "in_progress_count": 7, + "in_progress_count": 9, "in_progress_blocked_count": 0, - "planned_count": 16, - "complete_count": 2, + "planned_count": 15, + "complete_count": 1, "current_p0_count": 1 }, "next_execution_order": [ - "AG99-P0-007", - "AG99-P0-008", - "AG99-P0-009", - "AG99-P0-010", + "AG99-P0-001", + "AG99-P0-006", "AG99-P0-003", "AG99-P0-004", "AG99-P0-005", - "AG99-P0-006", - "AG99-P0-011", - "AG99-P0-012", + "AG99-P0-007", + "AG99-P0-008", + "AG99-P0-009", "AG99-P0-013", - "AG99-P0-014" + "AG99-P0-012", + "AG99-P0-014", + "AG99-P0-010", + "AG99-P0-011" ] } diff --git a/docs/operations/host112-guest-recovery-20260711.snapshot.json b/docs/operations/host112-guest-recovery-20260711.snapshot.json index 7cf6034c2..275b13865 100644 --- a/docs/operations/host112-guest-recovery-20260711.snapshot.json +++ b/docs/operations/host112-guest-recovery-20260711.snapshot.json @@ -1,6 +1,6 @@ { "schema_version": "host112_guest_recovery_receipt_v1", - "generated_at": "2026-07-11T19:52:31+08:00", + "generated_at": "2026-07-11T20:10:00+08:00", "work_item_id": "AG99-P0-008", "trace_id": "host112-guest-recovery-20260711T193137+0800-35698", "incident_fingerprint": "host112_vmware_ui_false_powered_off_guest_headless_wazuh_boot_timeout", @@ -48,6 +48,10 @@ "scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh", "scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py", "scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh", + "agent99-control-plane.ps1", + "agent99-sre-alert-inbox.ps1", + "agent99-telegram-inbox.ps1", + "apps/api/src/services/agent99_sre_bridge.py", "ops/monitoring/alerts-unified.yml", "docs/runbooks/REBOOT-RECOVERY-SOP.md" ], @@ -58,7 +62,11 @@ "host_probe": "31dce6f90dc89e2fd38bcde91049aa79711b3e5e11a1ae2d1053e27bbbc1f01f", "scorecard": "835abf7bf6cf587d40467351974f5eba2b84f731c78c2e6c7aaa254f0eaaf33d", "exporter": "85f16a3d190cdca866ad44f03c4c3e4decfbdf0d3af873c8e4af4d66780d5214", - "alert_rules": "2f606acd9d35b3708f57f313fcfe1f75c7b274316de0b27f494636a132e8ba4d" + "agent99_control_plane": "c17ac79cf59b793db885ea85e17663915cab7644e7428762c312d4ac1b7eb002", + "agent99_sre_alert_inbox": "d4ce271c763eae3d52347a71a22e4b76ce66b87ceb33fe928a6aaf1b37ba7088", + "agent99_telegram_inbox": "7607967de29aa0d8c4f22f170068da4ef63a1b889fb6eb927a01dd3d1bdd525c", + "agent99_sre_bridge": "fd7b5c6f77b55d0598f50d757dd8ba3234e16f8833600637ea827d7fcae85160", + "alert_rules": "86fd0e56cb7633da2a5a09960b8bf2dd61ee917c55072b5a16ba68577e8379cc" } }, "decision": { @@ -77,6 +85,8 @@ "staged_source_hashes_matched": true, "control_public_key_fingerprint_verified": true, "focused_contract_tests": "31 passed", + "agent99_tests": "49 passed", + "windows99_staged_contract_and_synthetic_validation": "passed", "json_yaml_parse": "passed" }, "execution_receipt": { @@ -91,6 +101,11 @@ "timer_active": true, "restricted_control_key_installed": true, "sudo_exact_command_installed": true, + "agent99_runtime_revision": "d0e364bfa42947190cf1e88d1d85623e23f84858", + "agent99_runtime_manifest": "14/14 matched, mismatch 0", + "agent99_deploy_evidence": "C:\\Wooo\\Agent99\\evidence\\agent99-deploy-20260711-200939.json", + "agent99_deploy_backup": "C:\\Wooo\\Agent99\\deploy\\backup-20260711-200939", + "prometheus_rule_backup": "/home/wooo/monitoring/alerts-unified.canonical.yml.host112-v2.bak.20260711200617", "host_reboot_performed": false, "vm_power_change_performed": false, "secret_read": false @@ -110,12 +125,24 @@ "host112_filebeat": "active", "forced_command_from_110": "passed", "sudo_n_controlled_apply": "passed_no_action_needed", - "scorecard_artifact": "/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-194901", + "scorecard_artifact": "/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-200338", "host112_active_blocker_count": 0, "prometheus_host112_metrics_green": "5/5", "prometheus_rule_health": "2/2 ok inactive", "prometheus_rule_sha_source_canonical_runtime_matched": true, "prometheus_rule_drift_guard_repaired": false, + "agent99_recover_evidence": "C:\\Wooo\\Agent99\\evidence\\agent99-Recover-20260711-200320.json", + "agent99_host112_phase": "ok", + "agent99_host112_before_ready": true, + "agent99_host112_apply_attempted": false, + "agent99_host112_after_ready": true, + "agent99_host112_transport": "direct_serialized", + "agent99_recovery_completed": true, + "agent99_recovery_coordinator_verified": true, + "agent99_reboot_slo_claimed": false, + "agent99_recovery_elapsed_seconds": 44.6, + "agent99_sre_route_selftest": "17/17 passed", + "agent99_telegram_ingest_selftest": "16/16 passed, no alert write, no Telegram send", "windows99_vmware_verify_ready": true, "windows99_operator_power_truth": "trust_vmx_process_and_guest_readiness_not_interactive_vmware_ui", "reboot_recovery_tests": "142 passed" @@ -133,7 +160,7 @@ "fresh_all_host_reboot_event_missing", "host_boot_observation_older_than_target_window" ], - "telegram_receipt": "not_sent_healthy_state_no_duplicate_notification", + "telegram_receipt": "healthy_state_not_sent; host112 routing and ingest verified by no-send selftests", "km_writeback": "docs/runbooks/REBOOT-RECOVERY-SOP.md", "work_item_writeback": "docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json" } diff --git a/docs/runbooks/REBOOT-RECOVERY-SOP.md b/docs/runbooks/REBOOT-RECOVERY-SOP.md index 76d62d445..51464e779 100644 --- a/docs/runbooks/REBOOT-RECOVERY-SOP.md +++ b/docs/runbooks/REBOOT-RECOVERY-SOP.md @@ -1,6 +1,6 @@ # AWOOOI 重開機恢復 SOP -> **版本**: v5.30 +> **版本**: v5.31 > **最後更新**: 2026-07-11 (台北時間) > **更新者**: Codex > **觸發事件**: 2026-07-11 真實全主機 reboot drill、late recovery 與 Agent99/VMware/runtime 閉環 @@ -65,7 +65,7 @@ L0 source contract 是 `scripts/reboot-recovery/external-l0-recovery-supervisor. 部署前 gate 是：PowerShell parse、Agent99 contract、synthetic fresh/blocked fixture、runtime staging hash；部署後 gate 是：`scripts/reboot-recovery/agent99-live-preflight.ps1` 回 `9/9` tasks、relay listener、queue zero、required evidence fresh、runtime manifest matched，再由 production S4U principal 觸發 `Recover`。Administrator 直接跑 launcher 不能取代 production-principal verifier。 -2026-07-11 17:35 final runtime receipt：Agent99 已原子 promote 為 `442b319bb80e009291ced7afa4f704cbe3b64886`，deploy evidence `C:\Wooo\Agent99\evidence\agent99-deploy-20260711-173023.json`；runtime manifest `14/14`、mismatch `0`、九個 task 存在、SRE relay 監聽、Startup-Recovery limit 為 `PT20M`。`0a4ddb133` 讓 reboot event 保持 `pendingRecovery` 直到 terminal、支援 `slo_breached_recovery_pending/recovered_late/recovered_within_slo`，並修正 observer TCP probe；`442b319bb` 讓 scorecard 接受 principal-aware VM power provenance。production-principal evidence `C:\Wooo\Agent99\evidence\agent99-Recover-20260711-173543.json` 顯示 `scope=service_recovery`、completed/verified true、elapsed `10.3s`、`rebootSloClaimed=false`。 +2026-07-11 20:09 final runtime receipt：Agent99 已原子 promote 為 `d0e364bfa42947190cf1e88d1d85623e23f84858`，deploy evidence `C:\Wooo\Agent99\evidence\agent99-deploy-20260711-200939.json`、rollback backup `C:\Wooo\Agent99\deploy\backup-20260711-200939`；runtime manifest `14/14`、mismatch `0`。production-principal evidence `C:\Wooo\Agent99\evidence\agent99-Recover-20260711-200320.json` 顯示七個 phases 全部 `ok`，新增必要 phase `host112_guest_readiness`，before/after verifier 均為 ready、`applyAttempted=false`，transport 為 direct serialized，recovery completed、coordinator verified，elapsed `44.6s`，且正確維持 `scope=service_recovery`、`rebootSloClaimed=false`。SRE routing self-test `17/17`、Telegram ingest self-test `16/16`，後者不寫 alert、不送 Telegram。這是 service recovery closure，不覆蓋原始 T+10 breach。 Windows 99 VMware task 已經由 `scripts/reboot-recovery/windows99-vmware-autostart-controlled-apply.ps1` 正式受控套用：DryRun、Apply、Verify 同一 receipt；task XML、舊 startup script、Windows Update policy 與 service start modes 都有 rollback。Evidence `C:\Wooo\Agent99\evidence\windows99-vmware-autostart-apply-20260711-173505.json` 回報 pre/post VMX process 都是 `5`、running guest rows `5`、process-aware duplicate-start guard true、沒有 host reboot、沒有 VM power change、沒有 secret read。這才是 Agent99 與重啟 SOP 的 runtime 整合證據；仍不取代下一次 600 秒 E2E drill。 @@ -77,7 +77,7 @@ Windows 99 VMware task 已經由 `scripts/reboot-recovery/windows99-vmware-autos 110 到 112 使用來源限制、`restrict` 與 forced command 的既有公鑰，只能執行 `/usr/local/sbin/awoooi-host112-guest-readiness --check`，不能取得 shell 或 forwarding。Host probe 必須輸出 `console_ready=1`、`services_ready=1`、`recovery_timer_ready=1`、`guest_ready=1`；任何一項缺失都建立具名 `host112_*` blocker，阻止 full-host recovery 與 maintenance withdrawal。Prometheus 固定輸出 `awoooi_host112_guest_readback_present`、`awoooi_host112_console_ready`、`awoooi_host112_services_ready`、`awoooi_host112_recovery_timer_ready`、`awoooi_host112_guest_ready`，Agent99 收到狀態變更後執行受控修復並以同一 incident 回報 verifier 結果。 -2026-07-11 runtime closure：installer runs `host112-guest-recovery-20260711T193137+0800-35698` 與 `host112-guest-recovery-20260711T195449+0800-43505` 已完成備份、受限公鑰、exact-command sudo、timer 與 durable rollback installer，未 reboot、未改 VM power、未讀 secret。110 artifact `/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-194901` 讀回 112 `systemd=running`、Console/Wazuh/timer/guest `1/1/1/1`、五項 Prometheus gauges `5/5`、`host112_* blocker=0`。兩條 112 告警規則 health `2/2 ok` 且 healthy state 為 inactive；source/canonical/container SHA 一致，drift guard `repaired=0`。Windows99 verifier 同輪讀到 `VM_PROCESS alias=112 process_id=22288 session_id=0 owner=NT_AUTHORITY\SYSTEM ui_truth=runtime_running_gui_session_isolated`，因此 operator power truth 固定信任 VMX process + guest readiness，不信任跨 principal 的互動式 VMware UI。完整 receipt 見 `docs/operations/host112-guest-recovery-20260711.snapshot.json`；reboot-recovery tests `142 passed`。下一個未關閉驗收只剩下一次真實全主機 600 秒 drill，不能用這次事後恢復覆蓋歷史逾時。 +2026-07-11 runtime closure：installer runs `host112-guest-recovery-20260711T193137+0800-35698` 與 `host112-guest-recovery-20260711T195449+0800-43505` 已完成備份、受限公鑰、exact-command sudo、timer 與 durable rollback installer，未 reboot、未改 VM power、未讀 secret。110 artifact `/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-200338` 讀回 112 `systemd=running`、Console/Wazuh/timer/guest `1/1/1/1`、五項 Prometheus gauges `5/5`、`host112_* blocker=0`。兩條 112 告警規則 health `2/2 ok` 且 healthy state 為 inactive；source/canonical/active/runtime rule SHA 均為 `86fd0e56cb7633da2a5a09960b8bf2dd61ee917c55072b5a16ba68577e8379cc`。Windows99 verifier 同輪讀到 `VM_PROCESS alias=112 process_id=22288 session_id=0 owner=NT_AUTHORITY\SYSTEM ui_truth=runtime_running_gui_session_isolated`，因此 operator power truth 固定信任 VMX process + guest readiness，不信任跨 principal 的互動式 VMware UI。Agent99 `Recover` 也已把 112 納入必要 phase 並完成 production-principal verifier；完整 receipt 見 `docs/operations/host112-guest-recovery-20260711.snapshot.json`。Agent99 tests `49 passed`、reboot-recovery tests `142 passed`。目前 scorecard 僅保留 `all_required_hosts_not_in_10_minute_reboot_window`、`fresh_all_host_reboot_event_missing`、`host_boot_observation_older_than_target_window`；它們只能由下一次真實全主機 600 秒 drill 關閉。 ### 2026-07-10 P0 Agent99 Transport / Verified Remediation 覆蓋 diff --git a/scripts/reboot-recovery/agent99-live-preflight.ps1 b/scripts/reboot-recovery/agent99-live-preflight.ps1 index a6a4bf66a..d8c7def96 100644 --- a/scripts/reboot-recovery/agent99-live-preflight.ps1 +++ b/scripts/reboot-recovery/agent99-live-preflight.ps1 @@ -89,22 +89,43 @@ function Get-EvidenceReadback { if ($file) { try { $rawEvidence = Get-Content -Path $file.FullName -Raw | ConvertFrom-Json - $hostRows = @($rawEvidence.hosts | ForEach-Object { + $hostRows = if ($rawEvidence.PSObject.Properties["hosts"]) { @($rawEvidence.hosts | ForEach-Object { [pscustomobject]@{ host = [string]$_.host; reachable = [bool]$_.ping } - }) - $vmRows = @($rawEvidence.vmResults | ForEach-Object { + }) } else { @() } + $vmRows = if ($rawEvidence.PSObject.Properties["vmResults"]) { @($rawEvidence.vmResults | ForEach-Object { [pscustomobject]@{ alias = if ($_.PSObject.Properties["alias"]) { [string]$_.alias } else { "" } ok = [bool]$_.ok skipped = [bool]$_.skipped } - }) - $publicRows = @($rawEvidence.public) - $performanceRows = @($rawEvidence.performance) - $telegramRows = @($rawEvidence.telegram) - $safeSummary = [pscustomobject]@{ - mode = [string]$rawEvidence.mode - controlledApply = [bool]$rawEvidence.controlledApply + }) } else { @() } + $publicRows = if ($rawEvidence.PSObject.Properties["public"]) { @($rawEvidence.public) } else { @() } + $performanceRows = if ($rawEvidence.PSObject.Properties["performance"]) { @($rawEvidence.performance) } else { @() } + $telegramRows = if ($rawEvidence.PSObject.Properties["telegram"]) { @($rawEvidence.telegram) } else { @() } + $safeSummary = if ($Name -eq "sre_alert_inbox") { + [pscustomobject]@{ + ok = if ($rawEvidence.PSObject.Properties["ok"]) { [bool]$rawEvidence.ok } else { $false } + queuedCount = if ($rawEvidence.PSObject.Properties["queuedCount"]) { [int]$rawEvidence.queuedCount } else { 0 } + ignoredCount = if ($rawEvidence.PSObject.Properties["ignoredCount"]) { [int]$rawEvidence.ignoredCount } else { 0 } + duplicateSuppressedCount = if ($rawEvidence.PSObject.Properties["duplicateSuppressedCount"]) { [int]$rawEvidence.duplicateSuppressedCount } else { 0 } + failedCount = if ($rawEvidence.PSObject.Properties["failedCount"]) { [int]$rawEvidence.failedCount } else { 0 } + runNow = if ($rawEvidence.PSObject.Properties["runNow"]) { [bool]$rawEvidence.runNow } else { $false } + controlExitCode = if ($rawEvidence.PSObject.Properties["controlExitCode"]) { [int]$rawEvidence.controlExitCode } else { $null } + } + } elseif ($Name -eq "telegram_inbox") { + [pscustomobject]@{ + ok = if ($rawEvidence.PSObject.Properties["ok"]) { [bool]$rawEvidence.ok } else { $false } + reason = if ($rawEvidence.PSObject.Properties["reason"]) { [string]$rawEvidence.reason } else { "" } + synthetic = if ($rawEvidence.PSObject.Properties["synthetic"]) { [bool]$rawEvidence.synthetic } else { $false } + updatesSeen = if ($rawEvidence.PSObject.Properties["updatesSeen"]) { [int]$rawEvidence.updatesSeen } else { 0 } + processedCount = if ($rawEvidence.PSObject.Properties["processedCount"]) { [int]$rawEvidence.processedCount } else { 0 } + alertedCount = if ($rawEvidence.PSObject.Properties["alertedCount"]) { [int]$rawEvidence.alertedCount } else { 0 } + ignoredCount = if ($rawEvidence.PSObject.Properties["ignoredCount"]) { [int]$rawEvidence.ignoredCount } else { 0 } + } + } else { + [pscustomobject]@{ + mode = if ($rawEvidence.PSObject.Properties["mode"]) { [string]$rawEvidence.mode } else { "" } + controlledApply = if ($rawEvidence.PSObject.Properties["controlledApply"]) { [bool]$rawEvidence.controlledApply } else { $false } hostCount = $hostRows.Count hostReachableCount = [int]@($hostRows | Where-Object { $_.reachable }).Count hosts = $hostRows @@ -112,12 +133,14 @@ function Get-EvidenceReadback { vmOkCount = [int]@($vmRows | Where-Object { $_.ok -or $_.skipped }).Count vms = $vmRows harborReady = [bool]( + $rawEvidence.PSObject.Properties["harbor"] -and $rawEvidence.harbor -and $rawEvidence.harbor.coreHealthy -and $rawEvidence.harbor.redisUp -and $rawEvidence.harbor.registryHealthy ) awoooiReady = [bool]( + $rawEvidence.PSObject.Properties["awoooi"] -and $rawEvidence.awoooi -and $rawEvidence.awoooi.deployReady -and -not $rawEvidence.awoooi.imagePullFailure -and @@ -127,24 +150,24 @@ function Get-EvidenceReadback { publicOkCount = [int]@($publicRows | Where-Object { $_.ok }).Count performanceCount = $performanceRows.Count performanceIssueCount = [int]@($performanceRows | Where-Object { $_.severity -in @("warning", "critical") }).Count - selfHealthSeverity = if ($rawEvidence.selfHealth) { [string]$rawEvidence.selfHealth.severity } else { "" } - backupHealthSeverity = if ($rawEvidence.backupHealth) { [string]$rawEvidence.backupHealth.severity } else { "" } - controlProcessedCount = if ($rawEvidence.controlTick) { [int]$rawEvidence.controlTick.processedCount } else { 0 } - controlPendingCount = if ($rawEvidence.controlTick) { [int]$rawEvidence.controlTick.pendingCount } else { 0 } + selfHealthSeverity = if ($rawEvidence.PSObject.Properties["selfHealth"] -and $rawEvidence.selfHealth) { [string]$rawEvidence.selfHealth.severity } else { "" } + backupHealthSeverity = if ($rawEvidence.PSObject.Properties["backupHealth"] -and $rawEvidence.backupHealth) { [string]$rawEvidence.backupHealth.severity } else { "" } + controlProcessedCount = if ($rawEvidence.PSObject.Properties["controlTick"] -and $rawEvidence.controlTick) { [int]$rawEvidence.controlTick.processedCount } else { 0 } + controlPendingCount = if ($rawEvidence.PSObject.Properties["controlTick"] -and $rawEvidence.controlTick) { [int]$rawEvidence.controlTick.pendingCount } else { 0 } telegramAttemptCount = $telegramRows.Count telegramSentCount = [int]@($telegramRows | Where-Object { $_.sent }).Count - recovery = if ($rawEvidence.recoverySlo) { + recovery = if ($rawEvidence.PSObject.Properties["recoverySlo"] -and $rawEvidence.recoverySlo) { [pscustomobject]@{ targetMinutes = [int]$rawEvidence.recoverySlo.targetMinutes elapsedSeconds = [double]$rawEvidence.recoverySlo.elapsedSeconds completed = [bool]$rawEvidence.recoverySlo.completed withinSlo = [bool]$rawEvidence.recoverySlo.withinSlo - phases = @($rawEvidence.recoverySlo.phases | ForEach-Object { + phases = if ($rawEvidence.recoverySlo.PSObject.Properties["phases"]) { @($rawEvidence.recoverySlo.phases | ForEach-Object { [pscustomobject]@{ name = [string]$_.name; status = [string]$_.status; elapsedSeconds = [double]$_.elapsedSeconds } - }) + }) } else { @() } } } else { $null } - recoveryCoordinator = if ($rawEvidence.recoveryCoordinator) { + recoveryCoordinator = if ($rawEvidence.PSObject.Properties["recoveryCoordinator"] -and $rawEvidence.recoveryCoordinator) { [pscustomobject]@{ verified = [bool]$rawEvidence.recoveryCoordinator.verified requireFreshRebootWindow = [bool]$rawEvidence.recoveryCoordinator.requireFreshRebootWindow @@ -155,12 +178,15 @@ function Get-EvidenceReadback { failedChecks = @($rawEvidence.recoveryCoordinator.failedChecks) } } else { $null } + } } } catch { $parseError = $_.Exception.GetType().Name } } + $parsed = [bool]($file -and -not $parseError) + $usable = [bool]($fresh -and $parsed) return [pscustomobject]@{ name = $Name pattern = $Pattern @@ -170,7 +196,9 @@ function Get-EvidenceReadback { ageMinutes = $ageMinutes maxAgeMinutes = $MaxAgeMinutes fresh = $fresh - healthy = [bool](-not $Required -or $fresh) + parsed = $parsed + usable = $usable + healthy = [bool](-not $Required -or $usable) parseError = $parseError safeSummary = $safeSummary } @@ -252,7 +280,9 @@ $evidence = @( $operatingSystem = Get-CimInstance Win32_OperatingSystem $taskFailures = @($tasks | Where-Object { -not $_.healthy }) -$requiredEvidenceFailures = @($evidence | Where-Object { $_.required -and -not $_.fresh }) +$requiredEvidenceFailures = @($evidence | Where-Object { $_.required -and -not $_.healthy }) +$requiredEvidenceStale = @($evidence | Where-Object { $_.required -and -not $_.fresh }) +$requiredEvidenceParseFailures = @($evidence | Where-Object { $_.required -and $_.fresh -and -not $_.parsed }) $selfCheckEvidence = $evidence | Where-Object { $_.name -eq "self_check" } | Select-Object -First 1 $selfHealthSeverity = if ($selfCheckEvidence -and $selfCheckEvidence.safeSummary) { [string]$selfCheckEvidence.safeSummary.selfHealthSeverity } else { "unknown" } $alertIncoming = $queues | Where-Object { $_.relativePath -eq "alerts\incoming" } | Select-Object -First 1 @@ -264,7 +294,8 @@ if (-not $manifest.exists) { $blockingReasons += "runtime_manifest_missing" } if (-not $manifest.runtimeMatched -or $manifest.mismatchCount -ne 0) { $blockingReasons += "runtime_manifest_mismatch" } if ($taskFailures.Count -gt 0) { $blockingReasons += "scheduled_task_unhealthy" } if ($relayListeners.Count -eq 0) { $blockingReasons += "sre_alert_relay_not_listening" } -if ($requiredEvidenceFailures.Count -gt 0) { $blockingReasons += "required_evidence_stale" } +if ($requiredEvidenceStale.Count -gt 0) { $blockingReasons += "required_evidence_stale" } +if ($requiredEvidenceParseFailures.Count -gt 0) { $blockingReasons += "required_evidence_parse_failed" } if ($selfHealthSeverity -ne "ok") { $blockingReasons += "self_health_not_ok" } if ($alertRunning.staleOverFiveMinutes -gt 0) { $blockingReasons += "stale_alert_execution" } if ($alertIncoming.count -gt 0 -or $commandQueue.count -gt 0) { $blockingReasons += "pending_work_before_reboot" } @@ -288,7 +319,9 @@ $result = [pscustomobject]@{ healthyTaskCount = [int]@($tasks | Where-Object { $_.healthy }).Count taskFailureCount = $taskFailures.Count freshRequiredEvidenceCount = [int]@($evidence | Where-Object { $_.required -and $_.fresh }).Count + usableRequiredEvidenceCount = [int]@($evidence | Where-Object { $_.required -and $_.usable }).Count requiredEvidenceFailureCount = $requiredEvidenceFailures.Count + requiredEvidenceParseFailureCount = $requiredEvidenceParseFailures.Count selfHealthSeverity = $selfHealthSeverity alertIncomingCount = [int]$alertIncoming.count alertRunningCount = [int]$alertRunning.count diff --git a/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py b/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py index 8ef748562..7ac91cf77 100644 --- a/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py @@ -116,3 +116,32 @@ def test_live_preflight_is_no_secret_and_no_remote_write() -> None: assert ".env" not in preflight assert "auth.json" not in preflight assert "agent99.config.json" not in preflight + + +def test_live_preflight_rejects_fresh_but_unparseable_required_evidence() -> None: + preflight = read("scripts/reboot-recovery/agent99-live-preflight.ps1") + + assert '$parsed = [bool]($file -and -not $parseError)' in preflight + assert '$usable = [bool]($fresh -and $parsed)' in preflight + assert 'healthy = [bool](-not $Required -or $usable)' in preflight + assert '$_.required -and -not $_.healthy' in preflight + assert 'required_evidence_parse_failed' in preflight + assert 'requiredEvidenceParseFailureCount' in preflight + + +def test_live_preflight_adapts_inbox_evidence_without_raw_message_readback() -> None: + preflight = read("scripts/reboot-recovery/agent99-live-preflight.ps1") + + assert '$Name -eq "sre_alert_inbox"' in preflight + assert '$Name -eq "telegram_inbox"' in preflight + for safe_field in ( + "queuedCount", + "duplicateSuppressedCount", + "failedCount", + "updatesSeen", + "processedCount", + "alertedCount", + ): + assert safe_field in preflight + for forbidden in ("messageText", "rawUpdate", "botToken", "chatId"): + assert forbidden not in preflight