diff --git a/apps/api/src/services/windows99_vmx_source_backup_search_package.py b/apps/api/src/services/windows99_vmx_source_backup_search_package.py index 53306f40c..d4e75b7e7 100644 --- a/apps/api/src/services/windows99_vmx_source_backup_search_package.py +++ b/apps/api/src/services/windows99_vmx_source_backup_search_package.py @@ -162,6 +162,8 @@ def build_windows99_vmx_source_backup_search_package( else _load_backup_search_receipt(operations_dir) ) receipt_applies = _receipt_applies(receipt, target_aliases) + receipt_next_step = str(receipt.get("next_executable_step") or "").strip() + receipt_safe_next_step = str(receipt.get("safe_next_step") or "").strip() host_rows = ( _host_rows_from_receipt(receipt, locator) if receipt_applies @@ -206,10 +208,16 @@ def build_windows99_vmx_source_backup_search_package( ) elif receipt_applies and blocked_rows: status = "blocked_windows99_vmx_backup_source_not_found_some_hosts_unsearched" - next_step = "rerun_backup_search_for_blocked_hosts_or_offline_inventory" + next_step = ( + receipt_next_step + or "rerun_backup_search_for_blocked_hosts_or_offline_inventory" + ) safe_next_step = ( - "rerun_windows99_vmx_source_backup_search_check_mode_for_blocked_hosts_" - "then_update_receipt_no_write" + receipt_safe_next_step + or ( + "rerun_windows99_vmx_source_backup_search_check_mode_for_blocked_hosts_" + "then_update_receipt_no_write" + ) ) elif receipt_applies: status = "blocked_windows99_vmx_backup_source_not_found" diff --git a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py index db5adc95b..ff696d363 100644 --- a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py +++ b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py @@ -895,9 +895,10 @@ def test_awoooi_priority_work_order_readback_routes_closed_backup_to_host_boot_a assert state["windows99_vmx_source_backup_search_target_aliases"] == ["111"] assert state["windows99_vmx_source_backup_search_receipt_readback_present"] is True assert state["windows99_vmx_source_backup_search_host_count"] == 7 - assert state["windows99_vmx_source_backup_search_searched_host_count"] == 4 - assert state["windows99_vmx_source_backup_search_blocked_host_count"] == 3 + assert state["windows99_vmx_source_backup_search_searched_host_count"] == 3 + assert state["windows99_vmx_source_backup_search_blocked_host_count"] == 4 assert state["windows99_vmx_source_backup_search_blocked_host_aliases"] == [ + "99", "110", "188", "112", @@ -1090,7 +1091,7 @@ def test_awoooi_priority_work_order_readback_routes_closed_backup_to_host_boot_a "blocked_windows99_vmx_backup_source_not_found_some_hosts_unsearched" ) assert payload["summary"]["windows99_vmx_source_backup_search_candidate_source_count"] == 0 - assert payload["rollups"]["windows99_vmx_source_backup_search_blocked_host_count"] == 3 + assert payload["rollups"]["windows99_vmx_source_backup_search_blocked_host_count"] == 4 assert "service/data/backup readback is green" in payload[ "next_execution_order" ][0] @@ -1205,7 +1206,7 @@ def test_awoooi_priority_work_order_readback_routes_closed_backup_to_host_boot_a in_progress["evidence"]["windows99_vmx_source_backup_search_candidate_source_count"] == 0 ) - assert in_progress["evidence"]["windows99_vmx_source_backup_search_blocked_host_count"] == 3 + assert in_progress["evidence"]["windows99_vmx_source_backup_search_blocked_host_count"] == 4 assert ( in_progress["evidence"][ "windows99_vmx_source_backup_search_apply_allowed_by_this_package" diff --git a/apps/api/tests/test_windows99_vmx_source_backup_search_package_api.py b/apps/api/tests/test_windows99_vmx_source_backup_search_package_api.py index a0fc3e4a3..8a300785a 100644 --- a/apps/api/tests/test_windows99_vmx_source_backup_search_package_api.py +++ b/apps/api/tests/test_windows99_vmx_source_backup_search_package_api.py @@ -83,12 +83,26 @@ def _backup_search_receipt() -> dict: "collector_readback_present": True, "target_host_alias": "99", "target_vm_aliases": ["111"], + "next_executable_step": ( + "continue_verified_backup_inventory_search_or_authorized_console_source_path" + ), + "safe_next_step": ( + "continue_verified_backup_inventory_search_or_authorized_console_source_path_" + "then_scoped_dry_run_no_write" + ), "host_rows": [ { "host_alias": "99", - "status": "not_matched", + "status": "blocked_host_alias_unresolved", "candidate_source_count": 0, - "candidate_source_count_known": True, + "candidate_source_count_known": False, + "candidate_source_fingerprint_count": 0, + }, + { + "host_alias": "110", + "status": "blocked_ssh_batchmode_unavailable", + "candidate_source_count": 0, + "candidate_source_count_known": False, "candidate_source_fingerprint_count": 0, }, { @@ -112,6 +126,20 @@ def _backup_search_receipt() -> dict: "candidate_source_count_known": False, "candidate_source_fingerprint_count": 0, }, + { + "host_alias": "112", + "status": "blocked_ssh_batchmode_unavailable", + "candidate_source_count": 0, + "candidate_source_count_known": False, + "candidate_source_fingerprint_count": 0, + }, + { + "host_alias": "local", + "status": "not_matched", + "candidate_source_count": 0, + "candidate_source_count_known": True, + "candidate_source_fingerprint_count": 0, + }, ], } @@ -132,14 +160,18 @@ def test_windows99_vmx_source_backup_search_package_builder_is_check_mode_only() assert payload["target_vm_aliases"] == ["111"] assert payload["source_locator_candidate_source_count"] == 0 assert payload["receipt_readback_present"] is True - assert payload["search_host_count"] == 4 + assert payload["search_host_count"] == 7 assert payload["searched_host_count"] == 3 - assert payload["blocked_host_count"] == 1 - assert payload["blocked_host_aliases"] == ["188"] + assert payload["blocked_host_count"] == 4 + assert payload["blocked_host_aliases"] == ["99", "110", "188", "112"] assert payload["candidate_source_count"] == 0 assert payload["candidate_source_found"] is False assert payload["next_executable_step"] == ( - "rerun_backup_search_for_blocked_hosts_or_offline_inventory" + "continue_verified_backup_inventory_search_or_authorized_console_source_path" + ) + assert payload["safe_next_step"] == ( + "continue_verified_backup_inventory_search_or_authorized_console_source_path_" + "then_scoped_dry_run_no_write" ) assert payload["apply_allowed_by_this_package"] is False assert payload["raw_path_output"] is False @@ -183,7 +215,7 @@ def test_windows99_vmx_source_backup_search_endpoint_is_public_safe( assert payload["schema_version"] == "windows99_vmx_source_backup_search_package_v1" assert payload["target_vm_aliases"] == ["111"] assert payload["candidate_source_count"] == 0 - assert payload["blocked_host_aliases"] == ["188"] + assert payload["blocked_host_aliases"] == ["99", "110", "188", "112"] assert payload["public_lan_topology_redacted"] is True assert payload["raw_path_output"] is False assert payload["file_content_read"] is False diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 19c2cf70c..f1f638227 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,9 +1,10 @@ -## 2026-07-03 — 17:57 Windows99 VMX backup/source search package +## 2026-07-03 — 18:19 Windows99 VMX backup/source search package **完成內容**： - P0-4 繼續收斂 111 VMX source blocker；最新 via-110 no-secret locator 已讀回 `candidate_source_count=0`、`source_locator_status=blocked_missing_expected_vmx_source_no_candidate_found`，並取代早先 `candidate_source_count=6` / generic Ubuntu candidate 的歷史 receipt。 - 新增 `scripts/reboot-recovery/collect-windows99-vmx-source-backup-search.sh`：以 SSH public-key / BatchMode 搜尋 Linux backup roots，只輸出 path fingerprints 與 count，不讀 VMX 內容、不輸出 raw path、不寫遠端、不啟停 VM、不重啟服務或主機。 -- 新增 `/api/v1/agents/windows99-vmx-source-backup-search-package`、`windows99_vmx_source_backup_search_package` service 與 priority readback 欄位：目前 receipt 顯示 99 locator / 120 / 121 / local 未找到 verified 111 source，110 transport intermittent，188 / 112 SSH BatchMode unavailable；`candidate_source_count=0`、`blocked_host_count=3`、`apply_allowed_by_this_package=false`。 +- 新增 `/api/v1/agents/windows99-vmx-source-backup-search-package`、`windows99_vmx_source_backup_search_package` service 與 priority readback 欄位：18:19 live check-mode 顯示 120 / 121 / local 未找到 verified 111 source，99 alias 無 Linux backup host mapping，110 / 188 / 112 SSH BatchMode unavailable；`candidate_source_count=0`、`blocked_host_count=4`、`apply_allowed_by_this_package=false`。 +- 修正 collector：`--hosts 99,110,112,188` 這種 bare alias selector 會透過 default host directory 解析；無法解析的 alias 會輸出 `blocked_host_alias_unresolved`，不得被靜默忽略成 searched/blocked 都是 0。 - 更新 `docs/operations/windows99-vmx-source-locator-receipt.snapshot.json` 與新增 `docs/operations/windows99-vmx-source-backup-search-receipt.snapshot.json`，讓 production API 可讀回最新 no-candidate / backup-search truth。 - `FULL-STACK-COLD-START-SOP` 更新至 v1.129，`REBOOT-RECOVERY-SOP` 更新至 v5.20，統帥插入需求台帳更新 P0-4 當前狀態，避免再用舊 generic candidate 當目前 blocker。 @@ -11,13 +12,14 @@ - `DATABASE_URL=postgresql+asyncpg://test:test@localhost/test PYTHONPATH=apps/api python3.11 -m pytest scripts/reboot-recovery/tests/test_windows99_vmx_source_backup_search.py apps/api/tests/test_windows99_vmx_source_backup_search_package_api.py apps/api/tests/test_windows99_vmx_source_locator_readback_api.py -q -p no:cacheprovider`：`7 passed`。 - `DATABASE_URL=postgresql+asyncpg://test:test@localhost/test PYTHONPATH=apps/api python3.11 -m pytest apps/api/tests/test_awoooi_priority_work_order_readback_api.py -q -p no:cacheprovider`：`19 passed`。 - `python3.11 -m py_compile apps/api/src/services/windows99_vmx_source_backup_search_package.py apps/api/src/api/v1/agents.py apps/api/src/services/awoooi_priority_work_order_readback.py`、`bash -n scripts/reboot-recovery/collect-windows99-vmx-source-backup-search.sh`：通過。 +- 18:19 live check-mode：`bash scripts/reboot-recovery/collect-windows99-vmx-source-backup-search.sh --hosts 99,110,112,188 --timeout 45` 回傳 99 `blocked_host_alias_unresolved`，110 / 112 / 188 `blocked_ssh_batchmode_unavailable`，`searched_host_count=0`、`blocked_host_count=4`、`candidate_source_count=0`。 **仍維持**： - 這不代表 111 VMX source 已恢復；目前真 blocker 仍是 verified VMX source / backup path 缺失、guest powered off、Windows99 autostart config not ready / task never ran，以及 primary `reboot_event_required_host_unreachable`。 - 沒有讀 secret / token / `.env` / raw sessions / SQLite / auth；沒有使用 GitHub / gh；沒有啟動或關閉 VM，沒有 relink / restore，沒有重啟 Windows host / VMware service，沒有 Docker / Nginx / K3s / DB / firewall restart / reload，沒有 prune / delete / force push。 **下一步**： -- 跑 full focused verifier / runner pressure guard / `git diff --check` 後 commit / push Gitea main；CD 後讀回 production backup-search endpoint、priority readback 與 SLO scorecard。P0-4 下一個可執行動作是 rerun blocked hosts 的 backup/source search 或離線備份 inventory，找到 verified source path 後才建立 scoped relink dry-run。 +- 跑 full focused verifier / runner pressure guard / `git diff --check` 後 commit / push Gitea main；CD 後讀回 production backup-search endpoint、priority readback 與 SLO scorecard。P0-4 下一個可執行動作是離線備份 inventory 或授權 console source path，找到 verified source path 後才建立 scoped relink dry-run。 ## 2026-07-03 — 17:34 Windows99 no-secret verifier timeout 調整 diff --git a/docs/operations/windows99-vmx-source-backup-search-receipt.snapshot.json b/docs/operations/windows99-vmx-source-backup-search-receipt.snapshot.json index bf7de08f3..1217b5121 100644 --- a/docs/operations/windows99-vmx-source-backup-search-receipt.snapshot.json +++ b/docs/operations/windows99-vmx-source-backup-search-receipt.snapshot.json @@ -1,7 +1,7 @@ { "schema_version": "windows99_vmx_source_backup_search_receipt_v1", - "generated_at": "2026-07-03T17:57:16+08:00", - "receipt_ref": "log://awoooi/p0-006/windows99-vmx-source-backup-search-111-20260703-1757", + "generated_at": "2026-07-03T18:19:39+08:00", + "receipt_ref": "log://awoooi/p0-006/windows99-vmx-source-backup-search-111-20260703-1819", "source": "live_no_secret_filename_search", "collector_readback_present": true, "collector_schema_version": "windows99_vmx_source_backup_search_collector_v1", @@ -12,16 +12,16 @@ { "host_alias": "99", "search_role": "windows_vmware_roots_via_locator", - "status": "not_matched", + "status": "blocked_host_alias_unresolved", "candidate_source_count": 0, - "candidate_source_count_known": true, + "candidate_source_count_known": false, "candidate_source_fingerprint_count": 0, - "collector": "windows99-vmx-source-locator-readback" + "collector": "collect-windows99-vmx-source-backup-search.sh" }, { "host_alias": "110", "search_role": "linux_backup_roots", - "status": "transport_intermittent", + "status": "blocked_ssh_batchmode_unavailable", "candidate_source_count": 0, "candidate_source_count_known": false, "candidate_source_fingerprint_count": 0, @@ -74,12 +74,13 @@ "collector": "bounded_local_filename_search" } ], - "searched_host_count": 4, - "blocked_host_count": 3, + "searched_host_count": 3, + "blocked_host_count": 4, "candidate_source_count": 0, "candidate_source_fingerprint_count": 0, - "search_status": "blocked_no_verified_vmx_source_found_some_hosts_unsearched", - "safe_next_step": "rerun_windows99_vmx_source_backup_search_check_mode_for_blocked_hosts_then_update_receipt_no_write", + "search_status": "blocked_no_verified_vmx_source_found_blocked_hosts_unavailable", + "next_executable_step": "continue_verified_backup_inventory_search_or_authorized_console_source_path", + "safe_next_step": "continue_verified_backup_inventory_search_or_authorized_console_source_path_then_scoped_dry_run_no_write", "public_lan_topology_redacted": true, "redacted_public_display_only": true, "raw_path_output": false, diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index 91ddad352..2909e8863 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -1,6 +1,6 @@ # AWOOOI 全棧冷啟動與主機重啟 SOP -> Version: v1.129 +> Version: v1.130 > Last updated: 2026-07-03 Asia/Taipei > Scope: 99 / 110 / 111 / 112 / 120 / 121 / 188 全棧重啟恢復。112 仍是 Kali / VM guest 訊號，但 2026-06-30 全主機重啟後已納入 10 分鐘 SLO 的必要 boot / power signal；此納入不代表授權任何破壞性 runtime apply。 @@ -10,7 +10,7 @@ 本節是每次接手、開機、關機、重啟後的第一個判定錨點。若日期不是今天，必須先重跑 live check，再更新本節與 `docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md`。 -v1.129 Windows99 VMX backup/source search package rule：2026-07-03 17:57 最新 via-110 no-secret locator 已把目前 truth 從早先 `candidate_source_count=6` 修正為 `candidate_source_count=0`、`source_locator_status=blocked_missing_expected_vmx_source_no_candidate_found`，且 `raw_path_output=false`、`path_fingerprint_only=true`、`secret_value_read=false`、`remote_write_performed=false`、`vm_power_change_performed=false`。因此 P0-4 下一步不得再引用 generic Ubuntu candidate；必須讀 `/api/v1/agents/windows99-vmx-source-backup-search-package` 與 priority readback 的 `windows99_vmx_source_backup_search_*` 欄位。當前 backup/source search receipt 顯示 99/120/121/local 均未找到 verified 111 source，110 為 transport intermittent，188/112 SSH BatchMode 不可用，`candidate_source_count=0`、`blocked_host_count=3`、`apply_allowed_by_this_package=false`。固定下一步是 rerun backup search for blocked hosts 或離線備份 inventory，再取得 verified source path 後建立 scoped relink dry-run；仍不得 power on VM、restore、讀 VMX 內容、讀 secret、重啟 Windows / VMware service / host / Docker / Nginx / K3s / DB / firewall。 +v1.130 Windows99 VMX backup/source search package rule：2026-07-03 18:19 最新 via-110 no-secret locator 已把目前 truth 從早先 `candidate_source_count=6` 修正為 `candidate_source_count=0`、`source_locator_status=blocked_missing_expected_vmx_source_no_candidate_found`，且 `raw_path_output=false`、`path_fingerprint_only=true`、`secret_value_read=false`、`remote_write_performed=false`、`vm_power_change_performed=false`。因此 P0-4 下一步不得再引用 generic Ubuntu candidate；必須讀 `/api/v1/agents/windows99-vmx-source-backup-search-package` 與 priority readback 的 `windows99_vmx_source_backup_search_*` 欄位。當前 backup/source search receipt 顯示 120 / 121 / local 均未找到 verified 111 source，99 alias 無 Linux backup host mapping，110 / 188 / 112 SSH BatchMode 不可用，`candidate_source_count=0`、`blocked_host_count=4`、`apply_allowed_by_this_package=false`。固定下一步是離線備份 inventory 或授權 console source path，再取得 verified source path 後建立 scoped relink dry-run；仍不得 power on VM、restore、讀 VMX 內容、讀 secret、重啟 Windows / VMware service / host / Docker / Nginx / K3s / DB / firewall。 若只是重啟後要快速判斷能不能宣稱恢復，先跑機器可讀摘要：`scripts/reboot-recovery/post-reboot-readiness-summary.sh --no-color`。此腳本會呼叫一頁式總檢查、188 host hygiene checklist 與 Wazuh no-false-green repo gates，並把 delegated logs 和可重放的 `summary.txt` 留在 `/tmp/awoooi-post-reboot-readiness-*`。v1.75 起，同一輪驗收後續步驟必須吃同一個 `$ARTIFACT_DIR/summary.txt`，例如 `scripts/reboot-recovery/post-reboot-declaration-guard.py --summary-file "$ARTIFACT_DIR/summary.txt" --no-color` 與 `scripts/reboot-recovery/post-reboot-next-gate-dispatch.sh --summary-file "$ARTIFACT_DIR/summary.txt" --no-color`；不得在同一輪 evidence chain 反覆重跑 live probes 後混用不同時間點結論。v1.76 起，delegated cold-start 若在 K3s rollout / CD 替換瞬間出現單次 `BLOCKED AWOOOI API not reachable`，但 wrapper 自己的 public `https://awoooi.wooo.work/api/v1/health` route retry 已回 2xx，該 blocker 只列為 route/API warmup evidence warning；public API 仍失敗、其他 non-route blocker、或 retry 後未恢復時，仍維持 hard blocked。宣告 guard 會把 summary 轉成 allowed / forbidden declaration，避免把服務綠誤報成 DR complete、188 host hygiene、Wazuh registry recovered 或 runtime authorized。若 summary 顯示 `SERVICE_GREEN=1` 但 `NEXT_REQUIRED_GATES` 仍非空，再由 dispatch checklist 把尚未完成的 blocker 轉成 owner / evidence / forbidden-action checklist；需要機器可讀 intake 時，再跑 `scripts/reboot-recovery/post-reboot-next-gate-owner-packets.py --summary-file "$ARTIFACT_DIR/summary.txt" --output /tmp/awoooi-post-reboot-owner-packets.json` 產生 `awoooi_post_reboot_next_gate_owner_packets_v1` JSON，立刻跑 `scripts/reboot-recovery/post-reboot-owner-packet-contract-guard.py --packet-file /tmp/awoooi-post-reboot-owner-packets.json`，並用 `scripts/reboot-recovery/post-reboot-owner-response-template.py --owner-packet-file /tmp/awoooi-post-reboot-owner-packets.json --output /tmp/awoooi-post-reboot-owner-response-template.json` 產生只含 active gates 的 placeholder response。dispatch / packet / guard / template 均固定 `DISPATCH_AUTHORIZED=0`、`REQUEST_SENT_COUNT=0`、`OWNER_RESPONSE_ACCEPTED=0`、`HOST_WRITE_AUTHORIZED=0`、`SECRET_VALUE_COLLECTION_ALLOWED=0`、`RUNTIME_GATE=0`；guard 未通過時不得送 owner request、不得寫 escrow marker、不得進維護窗口、不得宣稱 DR / Wazuh registry complete。v1.74 起，任何 owner response JSON 還必須經過 `scripts/reboot-recovery/post-reboot-owner-response-preflight.py --no-color --owner-packet-file --response-file `：空模板、placeholder、secret payload、runtime action request、credential marker write、Wazuh active response / re-enroll / restart、Kali active scan 或缺少 active gate evidence 都必須 fail-closed；preflight 通過也只表示可進入獨立 reviewer acceptance，不是 runtime 授權。需要人工展開時，再跑 `scripts/reboot-recovery/post-start-quick-check.sh --no-color` 並以 `docs/runbooks/REBOOT-POST-START-QUICK-CHECK.md` 作為 fallback。長 SOP 保留完整背景、例外處理與 Plan B；短版 wrapper / checklist 負責每次 T+10 分鐘內的固定判定。 diff --git a/docs/runbooks/REBOOT-RECOVERY-SOP.md b/docs/runbooks/REBOOT-RECOVERY-SOP.md index ae7dce236..57fc34f37 100644 --- a/docs/runbooks/REBOOT-RECOVERY-SOP.md +++ b/docs/runbooks/REBOOT-RECOVERY-SOP.md @@ -1,9 +1,9 @@ # AWOOOI 重開機恢復 SOP -> **版本**: v5.20 +> **版本**: v5.21 > **最後更新**: 2026-07-03 (台北時間) > **更新者**: Codex -> **觸發事件**: 2026-07-03 17:57 Windows99 VMX locator no-candidate truth 與 backup/source search package 上卷 +> **觸發事件**: 2026-07-03 18:19 Windows99 VMX backup/source blocked-host truth 上卷 --- @@ -21,13 +21,13 @@ ## 架構概覽與依賴圖 -### 2026-07-03 17:57 P0-4 VMX Backup / Source Search 覆蓋 +### 2026-07-03 18:19 P0-4 VMX Backup / Source Search 覆蓋 最新 via-110 no-secret locator 已成功收斂為 `candidate_source_count=0`、`source_locator_status=blocked_missing_expected_vmx_source_no_candidate_found`，早先 `candidate_source_count=6` / generic Ubuntu candidate 僅保留為歷史 receipt，不再作為目前 relink 候選。`docs/operations/windows99-vmx-source-locator-receipt.snapshot.json` 已更新為 17:57 no-candidate truth。 -新增 `/api/v1/agents/windows99-vmx-source-backup-search-package` 與 `scripts/reboot-recovery/collect-windows99-vmx-source-backup-search.sh`。此 package / collector 固定只做 filename / path-fingerprint 搜尋，不讀 VMX 內容、不輸出 raw path、不讀 secret、不寫遠端、不改 Windows、不啟停 VM、不重啟任何服務或主機。當前 receipt：99 locator / 120 / 121 / local 均未找到 verified 111 source，110 為 transport intermittent，188 / 112 SSH BatchMode 不可用；`candidate_source_count=0`、`blocked_host_count=3`、`apply_allowed_by_this_package=false`。 +新增 `/api/v1/agents/windows99-vmx-source-backup-search-package` 與 `scripts/reboot-recovery/collect-windows99-vmx-source-backup-search.sh`。此 package / collector 固定只做 filename / path-fingerprint 搜尋，不讀 VMX 內容、不輸出 raw path、不讀 secret、不寫遠端、不改 Windows、不啟停 VM、不重啟任何服務或主機。18:19 receipt：120 / 121 / local 均未找到 verified 111 source，99 alias 無 Linux backup host mapping，110 / 188 / 112 SSH BatchMode 不可用；`candidate_source_count=0`、`blocked_host_count=4`、`apply_allowed_by_this_package=false`。 -P0-4 下一步固定為 rerun blocked hosts 的 backup/source search 或離線備份 inventory，取得 verified source path 後才可建立 scoped relink dry-run + rollback + post-verifier；不得猜 generic Ubuntu candidate、不得直接 relink、不得 restore、不得 power on VM、不得讀 Windows 密碼或 VMX 內容。 +P0-4 下一步固定為離線備份 inventory 或授權 console source path，取得 verified source path 後才可建立 scoped relink dry-run + rollback + post-verifier；不得猜 generic Ubuntu candidate、不得直接 relink、不得 restore、不得 power on VM、不得讀 Windows 密碼或 VMX 內容。 ### 2026-07-03 17:34 P0-4 Verify Timeout 覆蓋 diff --git a/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md b/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md index 6eee6b533..42ff4fb8a 100644 --- a/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md +++ b/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md @@ -57,14 +57,14 @@ | 順序 | ID | 優先序 | 使用者插入要求 | 正規化工作項 | 目前狀態 | 下一個可驗證動作 | | --- | --- | --- | --- | --- | --- | --- | -| 1 | CIR-P0-RBT-001 | P0 | 「主機重啟後 10 分鐘內全部恢復，且要自動判斷所有主機被重啟」 | 建立 99/110/111/112/120/121/188 reboot event detector + 10 分鐘 SLO scorecard + fixed triage order | 2026-07-03 17:57 最新 truth：production/runtime 仍 `can_claim_all_services_recovered_within_target=false`，primary `reboot_event_required_host_unreachable`；Windows99 locator 已從舊 generic candidate 收斂為 `candidate_source_count=0`，backup/source search package 顯示 99/120/121/local 未找到 verified 111 source、110 transport intermittent、188/112 SSH BatchMode unavailable | 優先收斂 111 verified VMX source / backup path、blocked hosts backup search、111 reachability 與 fresh all-host reboot event；不可宣稱 10 分鐘 SLA 已證明，不得用 RDP clipboard 片段或 generic VMX guess 當完成證據 | +| 1 | CIR-P0-RBT-001 | P0 | 「主機重啟後 10 分鐘內全部恢復，且要自動判斷所有主機被重啟」 | 建立 99/110/111/112/120/121/188 reboot event detector + 10 分鐘 SLO scorecard + fixed triage order | 2026-07-03 18:19 最新 truth：production/runtime 仍 `can_claim_all_services_recovered_within_target=false`，primary `reboot_event_required_host_unreachable`；Windows99 locator 已從舊 generic candidate 收斂為 `candidate_source_count=0`，backup/source search package 顯示 120/121/local 未找到 verified 111 source、99 alias 無 Linux backup host mapping、110/188/112 SSH BatchMode unavailable | 優先收斂 111 verified VMX source / backup path、授權 console source path 或離線備份 inventory、111 reachability 與 fresh all-host reboot event；不可宣稱 10 分鐘 SLA 已證明，不得用 RDP clipboard 片段或 generic VMX guess 當完成證據 | | 2 | CIR-P0-RBT-002 | P0 | 「沒有偵測到主機重啟」 | 修正 host reboot/shutdown/up detection：boot_id / uptime / node exporter / Windows exporter / VMware VM power state 都要進同一事件 | Scorecard 已接 collection packet + management probe；08:23 host probe 有 7 列但 99 只有 ping/RDP reachable、uptime unknown，111 unreachable，112/120/121/188 uptime 都已超過 10 分鐘窗口 | 讓 99 verifier / Windows exporter 或等效 no-secret readback 進入 host boot event，並補 111 reachability 證據 | -| 3 | CIR-P0-RBT-003 | P0 | 「192.168.0.99 VMWare 要自動啟動，裡面 111/188/120/121/112 也自動啟動」 | Windows 99 VMware host autostart + guest VM autostart contract；VM host 111/188/120/121/112 開機順序與 readback | Source verifier / parser / API readback / collection packet 已完成；17:57 no-write locator 確認 expected 111 VMX path 不存在、`candidate_source_count=0`、`locate_status=blocked_missing_expected_vmx_source_no_candidate_found`；backup/source package 顯示 `candidate_source_count=0`、`blocked_host_count=3`、`apply_allowed_by_this_package=false`；VMware services / scheduled task / Windows Update policy readback 已有，但 guest power 仍 not ready | 不得直接 relink generic Ubuntu candidate；先 rerun blocked hosts backup/source search 或離線備份 inventory，取得 verified source path 後再做 scoped relink dry-run、rollback 與 post-verifier；不得從 public redacted path 或 scorecard 直接啟動 / 關閉 VM、重啟 Windows service、讀密碼或 host reboot | +| 3 | CIR-P0-RBT-003 | P0 | 「192.168.0.99 VMWare 要自動啟動，裡面 111/188/120/121/112 也自動啟動」 | Windows 99 VMware host autostart + guest VM autostart contract；VM host 111/188/120/121/112 開機順序與 readback | Source verifier / parser / API readback / collection packet 已完成；18:19 no-write locator 確認 expected 111 VMX path 不存在、`candidate_source_count=0`、`locate_status=blocked_missing_expected_vmx_source_no_candidate_found`；backup/source package 顯示 `candidate_source_count=0`、`blocked_host_count=4`、`apply_allowed_by_this_package=false`；VMware services / scheduled task / Windows Update policy readback 已有，但 guest power 仍 not ready | 不得直接 relink generic Ubuntu candidate；先取得授權 console source path 或離線備份 inventory verified source path，再做 scoped relink dry-run、rollback 與 post-verifier；不得從 public redacted path 或 scorecard 直接啟動 / 關閉 VM、重啟 Windows service、讀密碼或 host reboot | | 4 | CIR-P0-RBT-004 | P0 | 「192.168.0.99 不可因 Windows Update 無預警重開」 | Windows Update reboot policy：active hours / no auto-restart / maintenance window / update notification audit | 10:15 source/API patch：若 nested verifier 回 `policy_blockers=["windows_update_policy_readback_missing"]`，API 必須 fail-closed 重新提升 `windows99_update_no_auto_reboot_policy_not_ready`；不得只信舊 Prometheus `windows99_update_no_auto_reboot_ready=true` | 補 no-secret policy readback / drift alert；若 policy 不綠，另走 controlled apply package + rollback + post-verifier，禁止要求或記錄 Windows 密碼，禁止由 scorecard 直接 apply registry | | 5 | CIR-P0-RBT-005 | P0 | 「網站重啟後 502 嚴重影響體驗，要維護頁，外部雲端或專業做法」 | Public maintenance fallback：Nginx / edge / external static maintenance page / status page / fail-open UX，避免 502 直出 | Source + runtime verifier 已實作；Gitea CD `#4459` Success、deploy marker `8d7a6faaf`，production API scorecard 已讀回 `public_maintenance_fallback.ready=true`、raw 5xx=`0`、unreachable without L1=`0`，P0 blockers `11`、readiness `47`；仍不可宣稱整體 10 分鐘 SLO 完成 | 若未來 public route 出現 raw 5xx，先走 L0 Nginx intercept / header verifier；若 edge unreachable，才產生 L1 external static origin / CDN decision record + rollback | | 6 | CIR-P0-RBT-006 | P0 | 「所有主機關機立刻 Telegram 告警，重啟後也要告警，其他告警一併完整思考」 | Down / shutdown suspected / reboot detected / reboot recovered / SLO missed / backup failed / freshness stale / CPU pressure / Gitea queue 告警矩陣 | HostDown / HostRebootEventDetected / RebootAutoRecoverySLOMissed 已存在；per-blocker reboot alerts 與 backup receipt rules 已 deploy/readback。Backup receipt 缺段已從 100 條 stage 噪音收斂為 110 / 188 兩條 host-level pending；仍需完整 shutdown/up E2E receipt | 補 Prometheus / Alertmanager active/resolved 與 outbound receipt；backup alert 先補 `/backup/alert-receipts/*.last_success` 脫敏 marker，不送測試 secret、不重啟主機 | | 7 | CIR-P0-RBT-007 | P0 | 「所有備份包含主機、DB、網站、服務、套件、工具、日誌都沒有監控告警」 | Backup observability coverage：backup job inventory、last success、freshness、offsite、restore drill、Telegram/AwoooP receipt | 已有 backup health exporter / alert rules / Gitea bundle restore dry-run；2026-07-03 runtime 讀回 110 有 88 個 receipt stage requirement、188 有 12 個，`BackupAlertReceiptMetricMissing*` inactive，`BackupAlertReceiptStageMissing` 聚合 pending 110 / 188 各一條 | 補 `/backup/alert-receipts/*.last_success`；再補 Gitea full dump / DB / settings / issues / packages / LFS 與所有工具/log 全量備份監控 | -| 8 | CIR-P0-RBT-008 | P0 | 「每次重啟排查都不一樣，也不知道多久恢復，不符合 SLA」 | 固定化 reboot runbook：fixed triage order、ETA、active blocker、remaining seconds、owner lane、next command | Production scorecard / priority readback 固定目前 P0：primary `reboot_event_required_host_unreachable`；Windows99 locator 最新 `candidate_source_count=0`，backup/source search package 已把 99/120/121/local searched、110/188/112 blocked/unsearched 轉成機器可讀欄位 | 優先收斂 111 VMX source verified artifact、blocked hosts backup search、111 reachability / fresh all-host reboot event、188 startup failed/degraded；不得用不同排查路徑繞過 scorecard | +| 8 | CIR-P0-RBT-008 | P0 | 「每次重啟排查都不一樣，也不知道多久恢復，不符合 SLA」 | 固定化 reboot runbook：fixed triage order、ETA、active blocker、remaining seconds、owner lane、next command | Production scorecard / priority readback 固定目前 P0：primary `reboot_event_required_host_unreachable`；Windows99 locator 最新 `candidate_source_count=0`，backup/source search package 已把 120/121/local searched、99/110/188/112 blocked 轉成機器可讀欄位 | 優先收斂 111 VMX source verified artifact、授權 console source path 或離線備份 inventory、111 reachability / fresh all-host reboot event、188 startup failed/degraded；不得用不同排查路徑繞過 scorecard | | 9 | CIR-P0-RBT-009 | P0 | 「所有產品、網站都要是最新版本；版本和數據是否最新要驗證」 | Product freshness/version matrix：source commit、deploy marker、runtime image、public health、data freshness、latest source availability | AWOOOI Gitea `main=bc8d3af3f chore(cd): deploy dacdd90 [skip ci]`；CD `#4558` log 顯示 API/Web build/push wrapper 全部 `rc=0`、deploy marker `dacdd90`、production deploy readback matched。Production SLO route 已讀回最新 API 行為：`active_blocker_count=10`、`runtime_metric_runtime_readback_added_blockers=["windows99_vmware_autostart_config_not_ready","windows99_update_no_auto_reboot_policy_not_ready"]`、`windows99_update_no_auto_reboot_ready=false`。StockPlatform public freshness / ingestion 仍為 `ok`，latest trading date `2026-07-02`。 | 版本層已補到 AWOOOI production；下一步仍要建立全產品 readback 表，且收斂 Windows99 VMX / service / policy / power 與 host reboot-event blockers | | 10 | CIR-P0-GIT-001 | P0 | 「Gitea 儲存庫都不見了？Gitea 沒完整備份嗎？」 | Gitea repository identity + backup proof + restore drill：不能只看 UI visible，要比對 SSH heads、repo path、bundle backup、restore sample | 2026-07-02 production `/api/v1/agents/gitea-repo-bundle-backup-readback` 已 ready：9 expected repos present/ok、missing=0、failed=0、checksum_missing=0、bundle_fresh=true、all_expected_ok=true、sample_restore_dry_run_ok=true；repo bundle / restore dry-run 層已關閉，不是 repo missing。 | 維持每日 bundle backup + restore dry-run monitoring；另補 Gitea full dump / DB / settings / issues / packages / LFS 備份 readback。禁止刪 repo / 改 visibility / 讀 token / restore 到 production | | 11 | CIR-P0-CPU-001 | P0 | 「110 / 188 CPU 負載持續過高，為什麼沒監控告警、沒主動修復」 | Sustained CPU pressure automation：Alertmanager → controller → evidence → service playbook → verifier → KM writeback | 110 已有 `Host110SustainedModeratePressure`、Gitea playbook、Stock/Postgres evidence；188 仍需同級 controller/alerts readback | 下一步接 `postgres_hot_query_or_backup_export_playbook`；並補 188 equivalent readback，不以單次下降結案 | @@ -134,9 +134,9 @@ | KM / PlayBook / RAG / MCP 整合 | 已被列為 P1，不再遺漏 | 建立 work item schema 與 trust writeback 欄位 | | OpenClaw / Gather-style 持續動畫工作室 | route 已存在，已列為 P1 工作項 | 補 production desktop/mobile smoke、AwoooP 導流與截圖證據 | | AI 專業 UI / 非文字牆 cockpit | 已列為 P2 UX 驗收 | 將長文字區塊收斂成 first-viewport cockpit、cards、flow rows 與 expandable details | -| 10 分鐘 reboot auto-recovery SLA | 2026-07-03 17:57 latest truth：仍 `can_claim_all_services_recovered_within_target=false`；primary `reboot_event_required_host_unreachable`，Windows99 locator `candidate_source_count=0`，backup/source search `candidate_source_count=0` / `blocked_host_count=3`；111 VMX source / guest power / config / policy 仍 blocked | 收斂 111 verified VMX source / backup path、blocked hosts backup search、111 reachability、188 startup failed/degraded，再 rerun host probe + reboot-event detector | +| 10 分鐘 reboot auto-recovery SLA | 2026-07-03 18:19 latest truth：仍 `can_claim_all_services_recovered_within_target=false`；primary `reboot_event_required_host_unreachable`，Windows99 locator `candidate_source_count=0`，backup/source search `candidate_source_count=0` / `blocked_host_count=4`；111 VMX source / guest power / config / policy 仍 blocked | 收斂 111 verified VMX source / backup path、授權 console source path 或離線備份 inventory、111 reachability、188 startup failed/degraded，再 rerun host probe + reboot-event detector | | AWOOOI latest source production deploy readback | Gitea `main=bc8d3af3f`，deploy marker `dacdd90`；public CD `#4558` log job succeeded / production deploy readback matched；production scorecard 已讀回 latest API 行為並把 Windows99 config / policy blockers 上卷 | 下一步轉回 Windows99 VMX / service / policy / power 與 host reboot-event blockers；不得因 deploy/readback 成功宣稱 10 分鐘 SLO 完成 | -| 99 Windows / VMware autostart | Source verifier / parser / API readback / collection packet 已完成；live 99 Verify output 已透過 no-secret collector 收集；VMX locator 最新顯示 expected 111 path missing 且 `candidate_source_count=0`，backup/source package 顯示 verified source 未找到、110/188/112 仍需補 search/transport | 取得 verified VMX source / backup path，產生 scoped relink dry-run，重跑 no-secret collector，確認 VM 111/188/120/121/112 running | +| 99 Windows / VMware autostart | Source verifier / parser / API readback / collection packet 已完成；live 99 Verify output 已透過 no-secret collector 收集；VMX locator 最新顯示 expected 111 path missing 且 `candidate_source_count=0`，backup/source package 顯示 verified source 未找到、99/110/188/112 blocked | 取得 verified VMX source / backup path，產生 scoped relink dry-run，重跑 no-secret collector，確認 VM 111/188/120/121/112 running | | Reboot SLO blocker 收斂 | production scorecard 已固定 `10` blockers / readiness `60%`，source / runtime 已補 per-blocker metric/alert；剩餘主 blocker 是 reboot event / unreachable + Windows99 111 VMX source / guest power / config / policy + 111 reachability / 188 degraded | 依具名 blocker 收斂 99/111/188；不得用 route green 或 RDP 可見宣稱 SLA 完成 | | 全備份監控告警 coverage | exporter/rule 已有 host/DB/site/service/package/tool/log coverage 與 backup alert receipt requirement；runtime rules 已部署，缺段 alert 聚合成 110 / 188 host-level pending；production receipt markers / full Gitea dump / DB/settings/issues/packages/LFS 尚未全綠 | 補 `/backup/alert-receipts/*.last_success`、Gitea full dump 與 restore drill verifier | | Stock/Postgres hot pressure | 110 live 已導向 Stock/Postgres playbook；尚未完成 hot query / backup export playbook closure | 下一步執行 read-only Stock/Postgres evidence 與 source freshness / query attribution | @@ -154,7 +154,7 @@ ## 下一輪固定執行順序 1. 先確認 Gitea `main` / CD / production readback 是否仍與最新 truth 一致。 -2. 接續目前 active reboot P0：用 17:57 locator / backup-search truth 收斂 111 verified VMX source / backup path、blocked hosts backup search、111 reachability、188 startup failed/degraded，再 rerun host probe / reboot-event detector / scorecard。 +2. 接續目前 active reboot P0：用 18:19 locator / backup-search truth 收斂 111 verified VMX source / backup path、授權 console source path 或離線備份 inventory、111 reachability、188 startup failed/degraded，再 rerun host probe / reboot-event detector / scorecard。 3. 補全 reboot auto-recovery P0：99/110/111/112/120/121/188 reboot detection、10 分鐘 SLO scorecard、Telegram down/up/recovered/SLO missed / active-blocker receipts。 4. 補 99 Windows / VMware autostart P0：99 host 自動啟動 VMware，VM 111/188/120/121/112 自動啟動；同時補 Windows Update no-auto-restart verifier。 5. 補 111 reachability / host boot readback，讓 `host_unreachable_after_reboot` 與 `reboot_event_required_host_unreachable` 有單一固定處理路徑。 diff --git a/scripts/reboot-recovery/collect-windows99-vmx-source-backup-search.sh b/scripts/reboot-recovery/collect-windows99-vmx-source-backup-search.sh index 6143e6050..4ecc45201 100755 --- a/scripts/reboot-recovery/collect-windows99-vmx-source-backup-search.sh +++ b/scripts/reboot-recovery/collect-windows99-vmx-source-backup-search.sh @@ -1,7 +1,9 @@ #!/usr/bin/env bash set -euo pipefail -HOSTS="${WINDOWS99_VMX_BACKUP_SEARCH_HOSTS:-110=wooo@192.168.0.110,120=wooo@192.168.0.120,121=wooo@192.168.0.121,188=wooo@192.168.0.188,112=wooo@192.168.0.112}" +DEFAULT_HOSTS="110=wooo@192.168.0.110,120=wooo@192.168.0.120,121=wooo@192.168.0.121,188=wooo@192.168.0.188,112=wooo@192.168.0.112" +HOST_DIRECTORY="${WINDOWS99_VMX_BACKUP_SEARCH_HOST_DIRECTORY:-$DEFAULT_HOSTS}" +HOSTS="${WINDOWS99_VMX_BACKUP_SEARCH_HOSTS:-$DEFAULT_HOSTS}" SEARCH_ROOTS="${WINDOWS99_VMX_BACKUP_SEARCH_ROOTS:-/home/wooo /mnt /backup /backups /var/backups}" MAX_DEPTH="${WINDOWS99_VMX_BACKUP_SEARCH_MAX_DEPTH:-7}" CONNECT_TIMEOUT="${WINDOWS99_VMX_BACKUP_SEARCH_CONNECT_TIMEOUT:-8}" @@ -96,9 +98,30 @@ blocked_hosts=0 IFS=',' read -r -a host_entries <<<"$HOSTS" for entry in "${host_entries[@]}"; do - [[ -n "$entry" && "$entry" == *=* ]] || continue + entry="${entry//[[:space:]]/}" + [[ -n "$entry" ]] || continue alias_name="${entry%%=*}" - ssh_target="${entry#*=}" + ssh_target="" + if [[ "$entry" == *=* ]]; then + ssh_target="${entry#*=}" + else + alias_name="$entry" + IFS=',' read -r -a directory_entries <<<"$HOST_DIRECTORY" + for directory_entry in "${directory_entries[@]}"; do + directory_entry="${directory_entry//[[:space:]]/}" + [[ -n "$directory_entry" && "$directory_entry" == *=* ]] || continue + if [[ "${directory_entry%%=*}" == "$alias_name" ]]; then + ssh_target="${directory_entry#*=}" + break + fi + done + fi + if [[ -z "$ssh_target" || "$ssh_target" == "$alias_name" ]]; then + blocked_hosts=$((blocked_hosts + 1)) + printf 'HOST_SEARCH alias=%s status=blocked_host_alias_unresolved candidate_source_count=0 path_fingerprints=\n' \ + "$alias_name" + continue + fi remote_script=$(cat </dev/null || true; } \\ diff --git a/scripts/reboot-recovery/tests/test_windows99_vmx_source_backup_search.py b/scripts/reboot-recovery/tests/test_windows99_vmx_source_backup_search.py index 0429638d6..33248502f 100644 --- a/scripts/reboot-recovery/tests/test_windows99_vmx_source_backup_search.py +++ b/scripts/reboot-recovery/tests/test_windows99_vmx_source_backup_search.py @@ -116,3 +116,58 @@ def test_backup_search_collector_emits_fingerprints_not_paths(tmp_path: Path) -> assert "file_content_read=false" in result.stdout assert "remote_write_performed=false" in result.stdout assert "vm_power_change_performed=false" in result.stdout + + +def test_backup_search_collector_resolves_bare_aliases_and_blocks_unknown( + tmp_path: Path, +) -> None: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + _write_executable( + fake_bin / "ssh", + """ + #!/usr/bin/env bash + target="${@: -2:1}" + if [[ "$target" == *"blocked"* ]]; then + exit 255 + fi + printf '%s\n' 'fp-alias' + """, + ) + + env = os.environ.copy() + env["PATH"] = f"{fake_bin}:{env['PATH']}" + env["WINDOWS99_VMX_BACKUP_SEARCH_KNOWN_HOSTS_FILE"] = str( + fake_bin / "known_hosts" + ) + env["WINDOWS99_VMX_BACKUP_SEARCH_HOST_DIRECTORY"] = ( + "ok=wooo@ok,blocked=wooo@blocked" + ) + result = subprocess.run( + [ + "bash", + str(SCRIPT), + "--hosts", + "ok,blocked,missing", + "--timeout", + "3", + ], + cwd=ROOT, + env=env, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0 + values = _key_values(result.stdout) + assert values["candidate_source_count"] == "1" + assert values["searched_host_count"] == "1" + assert values["blocked_host_count"] == "2" + assert "HOST_SEARCH alias=ok status=searched candidate_source_count=1" in result.stdout + assert "HOST_SEARCH alias=blocked status=blocked_ssh_batchmode_unavailable" in ( + result.stdout + ) + assert "HOST_SEARCH alias=missing status=blocked_host_alias_unresolved" in ( + result.stdout + )