diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index 6f1241d4b..cb580808f 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -750,6 +750,8 @@ jobs: ;; scripts/reboot-recovery/collect-windows99-vmware-verify.sh) ;; + scripts/reboot-recovery/validate-windows99-console-verify-artifact.py) + ;; scripts/reboot-recovery/windows99-vmware-autostart.ps1) ;; scripts/reboot-recovery/windows99-management-channel-probe.py) @@ -760,6 +762,8 @@ jobs: ;; scripts/reboot-recovery/tests/test_windows99_vmware_verify_collector.py) ;; + scripts/reboot-recovery/tests/test_windows99_console_verify_artifact_validator.py) + ;; scripts/reboot-recovery/tests/test_188_host_hygiene_checklist.py) ;; scripts/reboot-recovery/tests/test_post_start_quick_check_contract.py) @@ -933,6 +937,7 @@ jobs: ../../scripts/reboot-recovery/post-reboot-owner-response-preflight.py \ ../../scripts/reboot-recovery/momo-source-arrival-gate.py \ ../../scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py \ + ../../scripts/reboot-recovery/validate-windows99-console-verify-artifact.py \ ../../scripts/reboot-recovery/windows99-management-channel-probe.py \ ../../scripts/ops/backup-alert-label-contract-check.py \ ../../scripts/ops/backup-health-textfile-exporter.py \ @@ -1055,6 +1060,7 @@ jobs: ../../scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py \ ../../scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py \ ../../scripts/reboot-recovery/tests/test_windows99_vmware_verify_collector.py \ + ../../scripts/reboot-recovery/tests/test_windows99_console_verify_artifact_validator.py \ ../../scripts/reboot-recovery/tests/test_188_host_hygiene_checklist.py \ ../../scripts/reboot-recovery/tests/test_post_start_quick_check_contract.py \ ../../scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py \ diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 3d81071ec..9e4420be0 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,20 @@ +## 2026-07-03 — 04:12 Windows99 console verify artifact 與 CD profile 修復 + +**完成內容**： +- 新增 `scripts/reboot-recovery/validate-windows99-console-verify-artifact.py`，把 99 RDP / Hyper-V console 上執行 `windows99-vmware-autostart.ps1 -Mode Verify` 的 stdout 轉成可餵給 reboot SLO scorecard 的 normalized no-secret artifact。 +- CLI 與 production API scorecard 的 `windows99_verify_collection` packet 新增 `no_secret_console_artifact_validate_command` 與 `validated_windows99_console_verify_artifact_file` safe channel；不再只停在「SSH/WinRM 不可用，等 verifier」。 +- Gitea CD `#4516` 對 `aeb436ba8 fix(reboot): accept windows99 console verify artifacts` 失敗點已定位：主 pytest 已 `3712 passed, 23 skipped`，真正紅點是 B5 step `BLOCKER b5_docker_socket_unavailable`；不是 Windows99 validator / scorecard 測試失敗。 +- 修正 `.gitea/workflows/cd.yaml` controlled-runtime classifier，將 `validate-windows99-console-verify-artifact.py` 與 `test_windows99_console_verify_artifact_validator.py` 納入 allowlist、`py_compile` 與 focused pytest；`ops/runner/test_cd_controlled_runtime_profile.py` 補 regression，避免同類 no-secret verifier source 再誤觸 B5 Docker socket。 + +**驗證**： +- `python3.11 -m pytest ops/runner/test_cd_controlled_runtime_profile.py scripts/reboot-recovery/tests/test_windows99_console_verify_artifact_validator.py scripts/reboot-recovery/tests/test_windows99_vmware_verify_collector.py scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py -q -p no:cacheprovider`：`73 passed`。 +- `DATABASE_URL=postgresql+asyncpg://test:test@localhost/test PYTHONPATH=apps/api python3.11 -m pytest apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py -q -p no:cacheprovider`：`15 passed`。 +- `python3.11 ops/runner/guard-gitea-runner-pressure.py --root .`：通過；`git diff --check`：通過。 + +**仍維持**： +- 此修法建立 console Verify artifact 的機器可讀路徑；尚未宣稱 99 VMware guest power / VMX 已恢復。完成 P0-1/P0-4 仍需實際 no-secret Verify stdout readback 後重跑 SLO scorecard。 +- 未讀 secret / token / `.env` / raw sessions / SQLite / auth；未使用 GitHub / gh；未 workflow_dispatch；未重啟 host / VM / service；未 Docker / Nginx / K3s / DB / firewall restart；未 DROP / TRUNCATE / restore / prune / delete / force push。 + ## 2026-07-03 — 03:55 P0-006 production scorecard 收斂與 SOP 沉澱 **完成內容**： diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index d645ff1cf..df2836674 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -1,6 +1,6 @@ # AWOOOI 全棧冷啟動與主機重啟 SOP -> Version: v1.102 +> Version: v1.103 > Last updated: 2026-07-03 Asia/Taipei > Scope: 99 / 110 / 111 / 112 / 120 / 121 / 188 全棧重啟恢復。112 仍是 Kali / VM guest 訊號，但 2026-06-30 全主機重啟後已納入 10 分鐘 SLO 的必要 boot / power signal；此納入不代表授權任何破壞性 runtime apply。 @@ -38,6 +38,8 @@ v1.101 reboot SLO exporter timeout fallback rule：`scripts/reboot-recovery/rebo v1.102 production scorecard / deploy cutover rule：2026-07-03 03:55 production `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 的目前事實是 `active_blocker_count=8`、`readiness_percent=67`、`can_claim_all_services_recovered_within_target=false`；`SERVICE_GREEN=1`、`PRODUCT_DATA_GREEN=1`、`BACKUP_CORE_GREEN=1`、`HOST_188_SERVICE_GREEN=1`、`WAZUH_DASHBOARD_DEGRADED=false`、Gitea bundle backup `expected_repo_count=12` / `repo_row_count=12` / `all_expected_ok=true`，StockPlatform freshness `status=ok` 且 `latest_trading_date=2026-07-02`。因此 service/data/backup/Wazuh/Gitea bundle blocker 不得因舊 Prometheus label 或 summary timeout 被重開；剩餘 P0 blocker 固定為 host boot detection / all-host fresh reboot event、111 unreachable、99 uptime unknown、Windows99 VMware VMX / guest power verifier。另，deploy marker 切換期間若 public API / SLO route 出現 502 且沒有 maintenance body / fallback header，即使稍後自癒，也必須列入 P0 public UX blocker；下一步固定是 source-controlled deploy drain、maintenance fallback verifier 與 public route watch，不得把短暫 502 當作正常噪音。 +v1.103 Windows99 console verify artifact rule：若 99 可 ping / RDP / Hyper-V VMConnect，但 SSH BatchMode 與 WinRM 都不可用，不能停在「等人工」。必須把 console channel 的 `windows99-vmware-autostart.ps1 -Mode Verify` stdout 存成 no-secret artifact，先跑 `python3.11 scripts/reboot-recovery/validate-windows99-console-verify-artifact.py --input --output `，再用 normalized artifact 重跑 reboot SLO scorecard。validator 必須檢查 `AWOOOI_WINDOWS99_VMWARE_AUTOSTART=1`、`MODE=Verify`、每個 required alias 的 `VMX` / `VM_POWER`、VMware services、autostart task、Windows Update policy 與 ready flags；若看到 secret-like key、欄位缺失或非 Verify marker 必須 fail-closed。此路徑只接受 Verify stdout，不授權 Windows password collection、VM power change、Windows Update policy apply、service restart、host reboot 或 registry edit。 + 2026-07-02 110 control-path / Harbor recovery receipt rule：若 Gitea Harbor repair queue 仍保留 `harbor_110_remote_ssh_publickey_auth_stalled`、remote-control unavailable、jobs stale 或 historical failure，但同一輪本地證據同時證明 `wooo` command path ready、110 local Harbor `/v2/` ready、public/internal registry `/v2/` 回 `401`，則該 Gitea Harbor repair 失敗只能列為 historical queue metadata，不得再當成 current SSH blocker。必須用 `/api/v1/agents/harbor-registry-controlled-recovery-receipt` 或同等 validator 合併 `diagnose-110-ssh-publickey-auth.sh`、`recover-110-control-path-and-harbor-local.sh --check`、public Gitea queue readback 與 registry `/v2/` verifier，並把機器可讀結果寫入 `docs/operations/harbor-110-control-path-recovery-readback-2026-07-02.snapshot.json` 類型的 snapshot。2026-07-02 live receipt 顯示：public/internal registry `/v2/` 均為 `401`、latest visible CD `#4335` 為 `Success`、Gitea Harbor repair failure 已是 `historical_after_latest_cd_success=true`；active blockers 收斂為 110 controlled CD lane config / binary / registration / service guardrail、active action container pressure，以及 Gitea CD jobs head-SHA / stale readback mismatch。若 local-console output 只有 `AWOOOI_110_CONTROLLED_CD_LANE_READY` marker，non110 runner parser 不得從 110 `BLOCKER` 行推導 non110 blocker；non110 只有看到 `AWOOOI_NON110_RUNNER_READY` marker 才能列入 active blocker。 2026-07-02 110 controlled CD lane fail-closed enforcer staging rule：110 runner 壓力事故後，legacy / generic runner 仍必須 fail-closed；但 `awoooi-cd-lane-drain.service` 的非 secret staging artifact 不得再被 enforcer 無差別封回 stub。`scripts/reboot-recovery/enforce-110-runner-failclosed.sh` 只有在 `config.yaml` 符合 `capacity <= 1`、只含 `awoooi-host:host` 與 `awoooi-ubuntu:docker://192.168.0.110:5000/awoooi/ci-runner:act-22.04`、binary 是 executable ELF、systemd unit 具備 `ConditionPathExists=/home/wooo/awoooi-cd-lane-drain/data/.runner`、`CPUAccounting` / `MemoryAccounting` / `TasksAccounting` / `NoNewPrivileges` 等 guardrail，且 service `inactive`、`MainPID=0`、未 enabled / 未 masked 時，才可保留 drain config / binary / unit，並輸出 `CONTROLLED_DRAIN_STAGING_ALLOWED=1` 與 textfile metric。此 staging 規則不得讀 token、不得讀 `.runner` 內容、不得註冊 runner、不得啟動 service；若 registration 缺失，readiness verifier 仍必須只留下 `controlled_cd_lane_registration_missing` / `controlled_cd_lane_service_not_active` 類 blocker。若 `CONTROLLED_DRAIN_STAGING_ALLOWED=0` 且 config / binary 又被搬走，優先修 source enforcer / unit guardrail，不要手工反覆補同一組 artifact。 diff --git a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md index 8e00ccd5c..8cf8572f1 100644 --- a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md +++ b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md @@ -22,7 +22,7 @@ | P0-1 | BLOCKED_HOST_WINDOWS | 全主機 reboot auto-detection / auto-trigger / 10 分鐘恢復 SLO | 2026-07-03 03:55 production scorecard：`active_blocker_count=8`、`readiness_percent=67`、`can_claim_all_services_recovered_within_target=false`；`observed_host_count=7`、`missing_host_count=0`、`unreachable_host_count=1`、111 unreachable、99 uptime unknown。 | 先收斂 99 / Windows99 / VMware 與 111：恢復 no-secret management channel 或 console verify stdout，讀回 VMX / VM power / host uptime，再 rerun host probe + reboot-event detector；不得 reboot、不得 VM power change、不得讀 Windows 密碼。 | | P0-2 | BLOCKED_PUBLIC_UX | Deploy / reboot 期間 public 502 維護頁與外部 fallback | 多次 deploy marker 切換時 SLO/API route 曾短暫 HTTP 502 且沒有 maintenance body / fallback header；目前 03:55 public maintenance runtime readback 為 ready、raw 5xx count `0`，但 cutover-time 502 未被 drain / watch 消除。 | 實作 source-controlled deploy drain / maintenance fallback verifier / public route watch；完成條件是 marker-time probe 不再看到 raw 502，或看到明確 maintenance fallback header/body。 | | P0-3 | PARTIAL_GREEN_SOURCE_RUNTIME | 所有產品 / 網站版本與資料最新性 | Gitea `main=89d4d6112`；Production SLO readback 對齊 deploy marker `89d4d6112 chore(cd): deploy 17ba08c [skip ci]`；Stock freshness `status=ok`、`latest_trading_date=2026-07-02`、blockers `[]`。AWOOOI health HTTP 200 但整體 `degraded`，SignOz / local Ollama 仍需列為 runtime degraded evidence。 | 將 source SHA / deploy marker / runtime endpoint / public route watch 固定進 scorecard；完成條件是每個 public product 都有 source、deploy、runtime、freshness 四層 readback，且 degraded components 有 owner lane。 | -| P0-4 | BLOCKED_WINDOWS99_AUTOSTART | 192.168.0.99 VMware 自動啟動與 VM guest 111 / 188 / 120 / 121 / 112 | Scorecard：`windows99_update_no_auto_reboot_ready=true`、`windows99_vmware_verify_ready=false`、VMX missing alias `111`、powered off aliases `111/112/120/121/188`、WinRM unavailable、SSH BatchMode permission denied、RDP / Hyper-V console channel reachable。 | 只用 no-secret console / management collector 取得 `windows99-vmware-autostart.ps1 -Mode Verify` 輸出；完成條件是 VMX config ready、guest power ready、99 uptime known、all required host reachable。 | +| P0-4 | BLOCKED_WINDOWS99_AUTOSTART | 192.168.0.99 VMware 自動啟動與 VM guest 111 / 188 / 120 / 121 / 112 | Scorecard：`windows99_update_no_auto_reboot_ready=true`、`windows99_vmware_verify_ready=false`、VMX missing alias `111`、powered off aliases `111/112/120/121/188`、WinRM unavailable、SSH BatchMode permission denied、RDP / Hyper-V console channel reachable。 | 只用 no-secret console / management collector 取得 `windows99-vmware-autostart.ps1 -Mode Verify` stdout，先以 `validate-windows99-console-verify-artifact.py` 轉成 normalized artifact，再重跑 SLO scorecard；完成條件是 VMX config ready、guest power ready、99 uptime known、all required host reachable。 | | P0-5 | PARTIAL_GREEN_BACKUP_MONITORING | Gitea / 主機 / DB / 網站 / 服務 / 套件 / 工具 / log 備份監控告警 | Gitea repo bundle readback ready：expected `12`、rows `12`、missing `0`、failed `0`、sample restore dry-run ok；backup core green。這只證明 repo bundle / core backup，不等於 Gitea full dump、DB/settings/issues/packages/LFS、所有工具與 log 全量備份監控完成。 | 補齊 backup health textfile / Prometheus / Telegram receipt matrix：每個 backup scope 必須有 fresh、failed、age、restore-drill、alert-receipt；完成條件是沒有「不知道備份有沒有跑」的盲點。 | | P0-6 | SOURCE_READY_ALERT_RECEIPT_BLOCKED | 主機關機 / 重啟 / SLO miss / backup failure Telegram 告警 | Source 已有 reboot / backup alert rules 與 per-blocker projection；scorecard `telegram_active_blocker_alert_required_count=8`。尚未完成 shutdown / reboot / backup alert 的脫敏 Telegram delivery receipt 全矩陣。 | 補 alert receipt readback：host down、host up、SLO miss、Windows99 blocker、backup stale/failed、deploy 502、freshness stale；完成條件是每類告警都有 sent / received / dedup / escalation evidence。 | | P0-7 | SOURCE_READY_SLA_AUTOMATION | 固定排查順序、ETA / wait reason、自動化判斷與修復 | Scorecard 已固定 `current_phase=host_boot_detection_blocked`、`eta_or_wait_reason=reboot_event_readback_missing_eta_unavailable`、`fixed_triage_order`；但 10 分鐘內自動恢復仍未達標。 | 把每個 blocker 的 next_safe_action、post_verifier、forbidden_actions 接到自動 work item / Telegram / scorecard；完成條件是重啟後自動判斷、主動告警、主動 rerun verifier，不再人工臨場猜流程。 | diff --git a/ops/runner/test_cd_controlled_runtime_profile.py b/ops/runner/test_cd_controlled_runtime_profile.py index 78ec2a209..9d9645d00 100644 --- a/ops/runner/test_cd_controlled_runtime_profile.py +++ b/ops/runner/test_cd_controlled_runtime_profile.py @@ -855,6 +855,7 @@ def test_reboot_auto_recovery_slo_sources_stay_on_controlled_runtime_profile() - "scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh)", "scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py)", "scripts/reboot-recovery/collect-windows99-vmware-verify.sh)", + "scripts/reboot-recovery/validate-windows99-console-verify-artifact.py)", "scripts/reboot-recovery/windows99-vmware-autostart.ps1)", "scripts/reboot-recovery/windows99-management-channel-probe.py)", "scripts/reboot-recovery/full-stack-cold-start-check.sh)", @@ -868,14 +869,17 @@ def test_reboot_auto_recovery_slo_sources_stay_on_controlled_runtime_profile() - "scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py)", "scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py)", "scripts/reboot-recovery/tests/test_windows99_vmware_verify_collector.py)", + "scripts/reboot-recovery/tests/test_windows99_console_verify_artifact_validator.py)", "scripts/reboot-recovery/tests/test_harbor_watchdog_contract.py)", "../../scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py", "../../scripts/reboot-recovery/collect-windows99-vmware-verify.sh", + "../../scripts/reboot-recovery/validate-windows99-console-verify-artifact.py", "../../scripts/reboot-recovery/windows99-management-channel-probe.py", "../../scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py", "../../scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py", "../../scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py", "../../scripts/reboot-recovery/tests/test_windows99_vmware_verify_collector.py", + "../../scripts/reboot-recovery/tests/test_windows99_console_verify_artifact_validator.py", ] for source in expected_sources: assert source in text