diff --git a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py index 280366719..dd7b7e4fa 100644 --- a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py +++ b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py @@ -678,12 +678,25 @@ def _apply_prometheus_windows99_vmware_readback( metric_readback.get("windows99_windows_update_no_auto_reboot_ready") is True ) windows99 = _dict(payload.setdefault("windows99_vmware_autostart", {})) + source_service_blockers = _strings(windows99.get("service_blockers")) + source_policy_blockers = _strings(windows99.get("policy_blockers")) + source_task = _dict(windows99.get("autostart_task")) + source_config_blocked = bool(source_service_blockers) or ( + bool(source_task) and source_task.get("ok") is not True + ) + if source_config_blocked: + windows99_blockers.append("windows99_vmware_autostart_config_not_ready") + if source_policy_blockers: + windows99_blockers.append("windows99_update_no_auto_reboot_policy_not_ready") + update_ready = False + windows99_blockers = _unique_strings(windows99_blockers) + config_ready = not missing_vmx_aliases and not source_config_blocked windows99.update( { "readback_present": True, "verify_ready": verify_ready, "vmrun_present": True, - "config_ready": not missing_vmx_aliases, + "config_ready": config_ready, "power_ready": not powered_off_aliases, "windows_update_no_auto_reboot_ready": update_ready, "missing_vmx_aliases": missing_vmx_aliases, @@ -705,7 +718,7 @@ def _apply_prometheus_windows99_vmware_readback( readback["windows99_vmware_readback_present"] = True readback["windows99_vmware_verify_ready"] = verify_ready readback["windows99_vmrun_present"] = True - readback["windows99_vmware_config_ready"] = not missing_vmx_aliases + readback["windows99_vmware_config_ready"] = config_ready readback["windows99_vmware_power_ready"] = not powered_off_aliases readback["windows99_update_no_auto_reboot_ready"] = update_ready readback["windows99_runtime_metric_overlay_applied"] = True @@ -736,7 +749,7 @@ def _apply_prometheus_windows99_vmware_readback( rollups = _dict(payload.setdefault("rollups", {})) rollups["windows99_vmware_readback_present"] = True rollups["windows99_vmrun_present"] = True - rollups["windows99_vmware_config_ready"] = not missing_vmx_aliases + rollups["windows99_vmware_config_ready"] = config_ready rollups["windows99_vmware_power_ready"] = not powered_off_aliases rollups["windows99_vmware_verify_ready"] = verify_ready rollups["windows99_update_no_auto_reboot_ready"] = update_ready @@ -849,12 +862,19 @@ def _add_runtime_readback_active_blockers_missing_from_metric( ) -> list[str]: service_backup = _dict(payload.get("controlled_service_data_backup_readback")) blocking_fields = _strings(service_backup.get("blocking_fields")) + windows99 = _dict(payload.get("windows99_vmware_autostart")) + windows99_blockers = _strings(windows99.get("blockers")) added = _unique_strings( [ _SERVICE_DATA_BACKUP_BLOCKING_FIELD_BLOCKERS[field] for field in blocking_fields if field in _SERVICE_DATA_BACKUP_BLOCKING_FIELD_BLOCKERS ] + + [ + blocker + for blocker in windows99_blockers + if blocker.startswith("windows99_") + ] ) if not added: return _unique_strings(active_blockers) diff --git a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py index 9142bde7c..f8e3e1fd8 100644 --- a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py +++ b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py @@ -93,6 +93,10 @@ PROMETHEUS_RUNTIME_BLOCKERS = [ PROMETHEUS_SOURCE_RECONCILED_BLOCKER = ( "conversation_event_hot_path_index_migration_source_missing" ) +WINDOWS99_RUNTIME_READBACK_ADDED_BLOCKERS = [ + "windows99_vmware_autostart_config_not_ready", + "windows99_update_no_auto_reboot_policy_not_ready", +] PROMETHEUS_RUNTIME_SOURCE_RECONCILED_BLOCKERS = [ blocker for blocker in PROMETHEUS_RUNTIME_BLOCKERS @@ -399,7 +403,10 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics( payload = load_latest_reboot_auto_recovery_slo_scorecard( prometheus_metric_readback=PROMETHEUS_RUNTIME_READBACK ) - expected_active_blockers = PROMETHEUS_RUNTIME_SOURCE_RECONCILED_BLOCKERS + expected_active_blockers = [ + *PROMETHEUS_RUNTIME_SOURCE_RECONCILED_BLOCKERS, + *WINDOWS99_RUNTIME_READBACK_ADDED_BLOCKERS, + ] assert payload["runtime_scorecard_readback_present"] is False assert payload["runtime_metric_readback_present"] is True @@ -411,7 +418,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics( ) assert payload["runtime_metric_source_control_reconciled_blocker_count"] == 1 assert payload["active_blockers"] == expected_active_blockers - assert payload["active_blocker_count"] == 7 + assert payload["active_blocker_count"] == 9 assert payload["readiness_percent"] == 47 assert payload["primary_blocker"] == "reboot_event_required_host_unreachable" assert payload["next_safe_action"] == ( @@ -420,8 +427,11 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics( assert "backup_core_green_not_1" not in payload["active_blockers"] assert "service_green_not_1" not in payload["active_blockers"] assert PROMETHEUS_SOURCE_RECONCILED_BLOCKER not in payload["active_blockers"] - assert payload["runtime_metric_runtime_readback_added_blockers"] == [] - assert payload["runtime_metric_runtime_readback_added_blocker_count"] == 0 + assert ( + payload["runtime_metric_runtime_readback_added_blockers"] + == WINDOWS99_RUNTIME_READBACK_ADDED_BLOCKERS + ) + assert payload["runtime_metric_runtime_readback_added_blocker_count"] == 2 assert payload["service_green"] is True assert payload["backup_core_green"] is True assert payload["product_data_green"] is True @@ -439,7 +449,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics( is True ) assert payload["controlled_service_data_backup_readback"]["blocking_fields"] == [] - assert payload["active_blocker_action_matrix"]["item_count"] == 7 + assert payload["active_blocker_action_matrix"]["item_count"] == 9 assert payload["windows99_vmware_autostart"]["readback_present"] is True assert payload["windows99_vmware_autostart"]["missing_vmx_aliases"] == ["111"] assert payload["windows99_vmware_autostart"]["powered_off_aliases"] == [ @@ -452,12 +462,13 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics( assert payload["windows99_vmware_autostart"]["blockers"] == [ "windows99_vmware_guest_power_not_ready", "windows99_vmware_vmx_missing", + *WINDOWS99_RUNTIME_READBACK_ADDED_BLOCKERS, ] assert payload["readback"]["windows99_vmware_readback_present"] is True assert payload["readback"]["windows99_vmrun_present"] is True assert payload["readback"]["windows99_vmware_config_ready"] is False assert payload["readback"]["windows99_vmware_power_ready"] is False - assert payload["readback"]["windows99_update_no_auto_reboot_ready"] is True + assert payload["readback"]["windows99_update_no_auto_reboot_ready"] is False assert payload["readback"]["windows99_runtime_metric_overlay_applied"] is True assert ( payload["windows99_verify_collection"]["status"] @@ -472,7 +483,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics( assert payload["rollups"]["windows99_vmrun_present"] is True assert payload["rollups"]["windows99_vmware_config_ready"] is False assert payload["rollups"]["windows99_vmware_power_ready"] is False - assert payload["rollups"]["windows99_update_no_auto_reboot_ready"] is True + assert payload["rollups"]["windows99_update_no_auto_reboot_ready"] is False assert payload["rollups"]["windows99_vmware_missing_vmx_count"] == 1 assert payload["rollups"]["windows99_vmware_powered_off_count"] == 5 assert payload["rollups"]["windows99_missing_vmx_alias_count"] == 1 @@ -517,7 +528,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics( ] == 5 assert payload["active_blocker_action_matrix"]["category_counts"][ "windows99_vmware_autostart" - ] == 2 + ] == 4 assert payload["active_blocker_action_matrix"]["category_counts"].get( "post_reboot_service_readiness", 0, @@ -530,25 +541,25 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics( "security_observability", 0, ) == 0 - assert payload["readback"]["active_blocker_count"] == 7 + assert payload["readback"]["active_blocker_count"] == 9 assert payload["readback"]["runtime_metric_readback_present"] is True assert payload["readback"][ "runtime_metric_source_control_reconciled_blocker_count" ] == 1 assert payload["readback"][ "runtime_metric_runtime_readback_added_blockers" - ] == [] + ] == WINDOWS99_RUNTIME_READBACK_ADDED_BLOCKERS assert payload["readback"][ "runtime_metric_runtime_readback_added_blocker_count" - ] == 0 - assert payload["rollups"]["active_blocker_count"] == 7 + ] == 2 + assert payload["rollups"]["active_blocker_count"] == 9 assert payload["rollups"]["runtime_metric_readback_present"] is True assert payload["rollups"][ "runtime_metric_source_control_reconciled_blocker_count" ] == 1 assert payload["rollups"][ "runtime_metric_runtime_readback_added_blocker_count" - ] == 0 + ] == 2 assert payload["rollups"]["primary_blocker_owner_lane"] == ( "reboot_event_detector_and_host_probe" ) @@ -663,8 +674,11 @@ def test_reboot_auto_recovery_slo_scorecard_keeps_prometheus_source_missing_when prometheus_metric_readback=PROMETHEUS_RUNTIME_READBACK, ) - assert payload["active_blockers"] == PROMETHEUS_RUNTIME_BLOCKERS - assert payload["active_blocker_count"] == 8 + assert payload["active_blockers"] == [ + *PROMETHEUS_RUNTIME_BLOCKERS, + *WINDOWS99_RUNTIME_READBACK_ADDED_BLOCKERS, + ] + assert payload["active_blocker_count"] == 10 assert payload["runtime_metric_source_control_reconciled_blocker_count"] == 0 action_by_blocker = { item["blocker"]: item diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index c8fe08169..1c50b5f85 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,17 @@ +## 2026-07-03 — 10:15 Windows99 nested blocker 上卷到 active blockers + +**完成內容**： +- Production readback 仍不能宣稱 10 分鐘 SLO：`status=blocked_reboot_auto_recovery_slo_not_ready`、`readiness_percent=67`、`primary_blocker=reboot_event_required_host_unreachable`，Windows99 仍有 `missing_vmx_aliases=["111"]`、`powered_off_aliases=["111","112","120","121","188"]`。 +- 發現 runtime evidence 裡 `windows99_vmware_autostart.service_blockers=["VMAuthdService","VMnetDHCP"]` 與 `policy_blockers=["windows_update_policy_readback_missing"]` 只停在 nested payload，未被 Prometheus active blocker label 帶出；這會讓真問題被壓扁成單一 `windows99_vmware_guest_power_not_ready`。 +- 修正 API scorecard overlay：Prometheus runtime metric 套入時，若 Windows99 nested evidence 已有 VMware service / task / policy blocker，必須重新提升 `windows99_vmware_autostart_config_not_ready` 與 `windows99_update_no_auto_reboot_policy_not_ready` 到 `active_blockers`、`active_blocker_action_matrix`、readback、rollups 與 `runtime_metric_runtime_readback_added_blockers`。 +- SOP 升到 v1.110：nested Windows99 service / policy blocker 不得被視為噪音；但 scorecard 仍只授權 verifier / check-mode package，不授權 Windows service restart、registry apply、VM power change 或 host reboot。 + +**已跑驗證**： +- `DATABASE_URL=sqlite:///tmp/awoooi-reboot-scorecard-test.db python3.11 -m pytest apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py -q -p no:cacheprovider`：`33 passed`。 + +**仍維持**： +- 未讀 secret / token / `.env` / raw sessions / SQLite / auth；未使用 GitHub / gh；未 workflow_dispatch；未啟動或關閉 VM；未重啟 host / service；未 Docker / Nginx / K3s / DB / firewall restart；未 DROP / TRUNCATE / restore / prune / delete / force push。 + ## 2026-07-03 — 09:53 Windows99 no-secret collector 全候選覆蓋 **完成內容**： diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index 492c3249e..0b85e01fb 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -1,6 +1,6 @@ # AWOOOI 全棧冷啟動與主機重啟 SOP -> Version: v1.109 +> Version: v1.110 > Last updated: 2026-07-03 Asia/Taipei > Scope: 99 / 110 / 111 / 112 / 120 / 121 / 188 全棧重啟恢復。112 仍是 Kali / VM guest 訊號，但 2026-06-30 全主機重啟後已納入 10 分鐘 SLO 的必要 boot / power signal；此納入不代表授權任何破壞性 runtime apply。 @@ -50,6 +50,8 @@ v1.108 Windows99 console artifact reliability blocker rule：若 RDP / Hyper-V V v1.109 Windows99 no-secret collector candidate coverage rule：`collect-windows99-vmware-verify.sh --check` 預設必須 probe 全部已列 SSH candidate，而不是只 probe 前兩個帳號；`--max-auth-users N` / `WINDOWS99_MAX_AUTH_USERS` 只作為明確縮小 blast radius 的 override。所有 probe 仍必須固定 `BatchMode=yes`、`PreferredAuthentications=publickey`、`PasswordAuthentication=no`、`KbdInteractiveAuthentication=no`、`NumberOfPasswordPrompts=0` 與 bounded timeout；不得用密碼 prompt、sshpass、WinRM credential、遠端寫入、VM power change 或 Windows policy apply 來補 verifier。若全候選仍回 `ssh_batchmode_auth_ready=0`，active blocker 必須保持 `windows99_ssh_publickey_auth_missing` / `windows99_remote_execution_channel_unavailable`，下一步才是恢復授權 no-secret management channel 或取得 validator 可接受的 console Verify stdout artifact。 +v1.110 Windows99 nested runtime blocker promotion rule：production `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 不得只信 Prometheus `active_blocker` label 而忽略 Windows99 verifier 巢狀 evidence。若 `windows99_vmware_autostart.service_blockers`、`autostart_task.ok=false` 或 `policy_blockers` 存在，API overlay 必須把 `windows99_vmware_autostart_config_not_ready` 與 `windows99_update_no_auto_reboot_policy_not_ready` 重新提升到 `active_blockers`、`active_blocker_action_matrix`、readback 與 rollups，並列入 `runtime_metric_runtime_readback_added_blockers`。這些 blocker 只授權 no-secret verifier、source config / check-mode package 與 post-verifier；不得從 scorecard 直接重啟 Windows service、改 registry、套 Windows Update policy、啟動 / 關閉 VM 或重啟 host。 + 2026-07-02 110 control-path / Harbor recovery receipt rule：若 Gitea Harbor repair queue 仍保留 `harbor_110_remote_ssh_publickey_auth_stalled`、remote-control unavailable、jobs stale 或 historical failure，但同一輪本地證據同時證明 `wooo` command path ready、110 local Harbor `/v2/` ready、public/internal registry `/v2/` 回 `401`，則該 Gitea Harbor repair 失敗只能列為 historical queue metadata，不得再當成 current SSH blocker。必須用 `/api/v1/agents/harbor-registry-controlled-recovery-receipt` 或同等 validator 合併 `diagnose-110-ssh-publickey-auth.sh`、`recover-110-control-path-and-harbor-local.sh --check`、public Gitea queue readback 與 registry `/v2/` verifier，並把機器可讀結果寫入 `docs/operations/harbor-110-control-path-recovery-readback-2026-07-02.snapshot.json` 類型的 snapshot。2026-07-02 live receipt 顯示：public/internal registry `/v2/` 均為 `401`、latest visible CD `#4335` 為 `Success`、Gitea Harbor repair failure 已是 `historical_after_latest_cd_success=true`；active blockers 收斂為 110 controlled CD lane config / binary / registration / service guardrail、active action container pressure，以及 Gitea CD jobs head-SHA / stale readback mismatch。若 local-console output 只有 `AWOOOI_110_CONTROLLED_CD_LANE_READY` marker，non110 runner parser 不得從 110 `BLOCKER` 行推導 non110 blocker；non110 只有看到 `AWOOOI_NON110_RUNNER_READY` marker 才能列入 active blocker。 2026-07-02 110 controlled CD lane fail-closed enforcer staging rule：110 runner 壓力事故後，legacy / generic runner 仍必須 fail-closed；但 `awoooi-cd-lane-drain.service` 的非 secret staging artifact 不得再被 enforcer 無差別封回 stub。`scripts/reboot-recovery/enforce-110-runner-failclosed.sh` 只有在 `config.yaml` 符合 `capacity <= 1`、只含 `awoooi-host:host` 與 `awoooi-ubuntu:docker://192.168.0.110:5000/awoooi/ci-runner:act-22.04`、binary 是 executable ELF、systemd unit 具備 `ConditionPathExists=/home/wooo/awoooi-cd-lane-drain/data/.runner`、`CPUAccounting` / `MemoryAccounting` / `TasksAccounting` / `NoNewPrivileges` 等 guardrail，且 service `inactive`、`MainPID=0`、未 enabled / 未 masked 時，才可保留 drain config / binary / unit，並輸出 `CONTROLLED_DRAIN_STAGING_ALLOWED=1` 與 textfile metric。此 staging 規則不得讀 token、不得讀 `.runner` 內容、不得註冊 runner、不得啟動 service；若 registration 缺失，readiness verifier 仍必須只留下 `controlled_cd_lane_registration_missing` / `controlled_cd_lane_service_not_active` 類 blocker。若 `CONTROLLED_DRAIN_STAGING_ALLOWED=0` 且 config / binary 又被搬走，優先修 source enforcer / unit guardrail，不要手工反覆補同一組 artifact。 diff --git a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md index 7c6a061a3..38bce3f12 100644 --- a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md +++ b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md @@ -13,7 +13,7 @@ 本段覆蓋舊的「單次重啟後人工排查」做法。所有後續狀態回報必須依此順序推進；噪音若會遮蔽 P0，就掛回同一列，不另開支線。 -### 2026-07-03 09:53 最新 P0 覆蓋排序 +### 2026-07-03 10:15 最新 P0 覆蓋排序 下表覆蓋 2026-06-30 初始事故列；舊表保留為歷史追蹤。所有新插入需求必須掛在本表，不得再分散成臨時支線。 @@ -22,11 +22,11 @@ | P0-1 | BLOCKED_HOST_WINDOWS | 全主機 reboot auto-detection / auto-trigger / 10 分鐘恢復 SLO | 2026-07-03 08:59 production scorecard：`status=blocked_reboot_auto_recovery_slo_not_ready`、`active_blocker_count=8`、`readiness_percent=67`、`primary_blocker=reboot_event_required_host_unreachable`、`can_claim_all_services_recovered_within_target=false`。no-write artifact `/tmp/awoooi-reboot-continue-20260703-085205`：111 socket probe 仍 timeout / unreachable；99 RDP / VMConnect reachable 但 uptime unknown；188 socket probe reachable，前次 host probe 仍為 `systemd_state=degraded` / `startup_active=failed`。Production readback 同步顯示 `windows99_vmware_readback_present=true`、`windows99_vmware_config_ready=false`、`windows99_vmware_power_ready=false`、`windows99_missing_vmx_aliases=["111"]`、`windows99_powered_off_aliases=["111","112","120","121","188"]`。09:22 source/API contract 已把 unreliable console artifact 額外投影成 `windows99_console_clipboard_unreliable` 類 active blocker。 | 先收斂 99 / Windows99 / VMware 與 111：恢復 no-secret management channel 或取得 validator 可接受的完整 console Verify stdout，讀回 VMX / VM power / host uptime，再 rerun host probe + reboot-event detector；不得 reboot、不得 VM power change、不得讀 Windows 密碼，不得用 RDP clipboard 片段當完成證據。 | | P0-2 | BLOCKED_EDGE_PRIVILEGED_APPLY | Deploy / reboot 期間 public 502 維護頁與外部 fallback | Gitea CD `#4519` 已推 deploy marker `3aca484 -> a94ddd5`，但 marker 後 public probe 仍讀到 `https://awoooi.wooo.work/api/v1/health` raw `502`、fallback header/body 空；live 188 `/etc/nginx/sites-enabled/awoooi.wooo.work.conf` 缺 maintenance fallback，`/var/www/maintenance/maintenance.html` 缺失，`ollama@188` 無 passwordless sudo。 | 先跑 `scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh --check` 留 drift receipt；具備 privileged channel 後執行 `--apply`，要求 backup、`nginx -t`、reload、public route probe 全綠。不得讀密碼、不得用 app restart 掩蓋 edge fallback drift。 | | P0-3 | PARTIAL_GREEN_SOURCE_RUNTIME | 所有產品 / 網站版本與資料最新性 | Gitea `main=89d4d6112`；Production SLO readback 對齊 deploy marker `89d4d6112 chore(cd): deploy 17ba08c [skip ci]`；Stock freshness `status=ok`、`latest_trading_date=2026-07-02`、blockers `[]`。AWOOOI health HTTP 200 但整體 `degraded`，SignOz / local Ollama 仍需列為 runtime degraded evidence。 | 將 source SHA / deploy marker / runtime endpoint / public route watch 固定進 scorecard；完成條件是每個 public product 都有 source、deploy、runtime、freshness 四層 readback，且 degraded components 有 owner lane。 | -| P0-4 | BLOCKED_WINDOWS99_AUTOSTART | 192.168.0.99 VMware 自動啟動與 VM guest 111 / 188 / 120 / 121 / 112 | 09:53 no-secret management probe：99 reachable、TCP `22 / 135 / 2179 / 3389 / 445=open`、RDP console / Hyper-V VMConnect reachable，WinRM `5985 / 5986=timeout`，`administrator` SSH BatchMode `permission_denied`，`can_collect_vmware_verify_without_secret=false`，`console_artifact_status=blocked_clipboard_unreliable`。collector 全候選 check 已改成預設 probe 5 個 candidate，實機 readback 仍為 `ssh_auth_probe_user_limit=5`、`ssh_auth_probed_users=5`、`ssh_batchmode_auth_ready=0`、`verify_collection_status=blocked_ssh_publickey_auth_missing`、`remote_verify_attempted=0`。Production scorecard 仍有 Windows99 runtime readback，但回 `missing_vmx_aliases=["111"]`、guest power not ready；RDP console UI/clipboard 測試顯示 clipboard 會貼入 Windows 端舊內容，不能當可靠自動收集通道。 | 只用可靠 no-secret console / management collector 取得 `windows99-vmware-autostart.ps1 -Mode Verify` stdout，先以 `validate-windows99-console-verify-artifact.py` 轉成 normalized artifact，再重跑 SLO scorecard；完成條件是 VMX config ready、guest power ready、99 uptime known、all required host reachable。 | +| P0-4 | BLOCKED_WINDOWS99_AUTOSTART | 192.168.0.99 VMware 自動啟動與 VM guest 111 / 188 / 120 / 121 / 112 | 10:15 source/API patch：Production Windows99 nested readback 仍為 `missing_vmx_aliases=["111"]`、`powered_off_aliases=["111","112","120","121","188"]`，且 nested `service_blockers=["VMAuthdService","VMnetDHCP"]`、`policy_blockers=["windows_update_policy_readback_missing"]` 不能再被壓扁成單一 guest power blocker；API overlay 已補 `windows99_vmware_autostart_config_not_ready` / `windows99_update_no_auto_reboot_policy_not_ready` runtime-readback-added blocker。09:53 no-secret management probe：99 reachable、RDP / Hyper-V VMConnect reachable，WinRM timeout，Mac vantage collector 仍為 `ssh_batchmode_auth_ready=0` / `blocked_ssh_publickey_auth_missing`；production vantage 的 no-secret collector readback 曾回 collected，但完成條件仍取決於 VMX / service / policy / power 全綠。 | 先修 111 VMX source 與 VMware service / policy readback，使 check-mode package 能解釋 powered-off aliases；不得從 scorecard 直接重啟 Windows service、改 registry、啟動 / 關閉 VM 或 host reboot。完成條件是 VMX config ready、guest power ready、99 uptime known、all required host reachable，且 active blocker matrix 顯示 service / policy / VMX / power 分開收斂。 | | P0-5 | RUNTIME_READY_BACKUP_RECEIPT_GAP | Gitea / 主機 / DB / 網站 / 服務 / 套件 / 工具 / log 備份監控告警 | Gitea repo bundle readback ready：expected `12`、rows `12`、missing `0`、failed `0`、sample restore dry-run ok；backup core green。2026-07-03 source / runtime 已部署 `awoooi_backup_alert_receipt_*` 指標與 Prometheus rules；110 exporter 讀回 88 個 stage requirement、188 讀回 12 個 stage requirement，`BackupAlertReceiptMetricMissing*` inactive，`BackupAlertReceiptStageMissing` 已修成每 `host / receipt_channel` 聚合 pending：110 一條、188 一條。 | 補 `/backup/alert-receipts/*.last_success` 脫敏 marker；下一層仍要補 Gitea full dump、DB/settings/issues/packages/LFS、所有工具與 log 全量備份監控。 | | P0-6 | RUNTIME_READY_ALERT_RECEIPT_GAP | 主機關機 / 重啟 / SLO miss / backup failure Telegram 告警 | Reboot per-blocker alert 與 backup receipt alert rules 已 deploy/readback；backup receipt 缺段不再產生 100 條 stage 噪音，現在聚合成 110 / 188 兩條 host-level pending。scorecard 仍有 8 個 reboot active blockers，尚未完成 shutdown / reboot / backup alert 的 production 脫敏 delivery receipt 全矩陣。 | 補 alert receipt readback：host down、host up、SLO miss、Windows99 blocker、backup stale/failed、deploy 502、freshness stale；完成條件是每類告警都有 sent / received / dedup / escalation evidence。 | | P0-7 | SOURCE_READY_SLA_AUTOMATION | 固定排查順序、ETA / wait reason、自動化判斷與修復 | Scorecard 已固定 `current_phase=host_boot_detection_blocked`、`eta_or_wait_reason=reboot_event_readback_missing_eta_unavailable`、`primary_blocker=reboot_event_required_host_unreachable`、fixed triage order 與 next safe action；08:23 artifact 固定下一步為 no-secret Windows99 verify / host probe rerun。09:22 `active_blocker_action_matrix` 已能把 unreliable console artifact 指向 `windows99_console_or_no_secret_management_channel` 與固定 next safe action。10 分鐘內自動恢復仍未達標。 | 把每個 blocker 的 next_safe_action、post_verifier、forbidden_actions 接到自動 work item / Telegram / scorecard；完成條件是重啟後自動判斷、主動告警、主動 rerun verifier，不再人工臨場猜流程。 | -| P0-8 | PARTIAL_READY_POLICY | Windows99 禁止 Windows Update 無預警重啟 | Scorecard：`windows99_update_no_auto_reboot_ready=true`；但 Windows99 no-secret remote execution channel blocked，policy 證據仍需納入持續監控。 | 保留 readback，補週期性 verifier 與 Telegram drift alert；完成條件是 Windows Update policy drift 會自動告警且不需讀 secret。 | +| P0-8 | PARTIAL_READY_POLICY | Windows99 禁止 Windows Update 無預警重啟 | 10:15 source/API patch：若 Windows99 nested verifier 回 `policy_blockers=["windows_update_policy_readback_missing"]`，production API 必須 fail-closed 重新提升 `windows99_update_no_auto_reboot_policy_not_ready`，不得只信舊 Prometheus `windows99_update_no_auto_reboot_ready=true`。 | 保留 no-secret verifier，補週期性 policy readback 與 Telegram drift alert；完成條件是 Windows Update policy drift / missing readback 會自動告警且不需讀 secret，不得從 scorecard 直接 apply registry。 | | 優先 | 狀態 | 工作項 | 2026-06-30 證據 | 下一步 / 完成條件 | |------|------|--------|------------------|-------------------| diff --git a/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md b/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md index d31b96e35..0b173ff51 100644 --- a/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md +++ b/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md @@ -59,8 +59,8 @@ | --- | --- | --- | --- | --- | --- | --- | | 1 | CIR-P0-RBT-001 | P0 | 「主機重啟後 10 分鐘內全部恢復，且要自動判斷所有主機被重啟」 | 建立 99/110/111/112/120/121/188 reboot event detector + 10 分鐘 SLO scorecard + fixed triage order | 2026-07-03 08:59 production scorecard：readiness `67%`、active blockers `8`、`can_claim_all_services_recovered_within_target=false`；artifact `/tmp/awoooi-reboot-continue-20260703-085205`：111 不可達、99 uptime unknown、188 startup failed/degraded；09:22 已把 unreliable console artifact 投影成 machine-readable blocker | 優先收斂 99 local console Verify output / no-secret management channel 與 111 reachability；不可宣稱 10 分鐘 SLA 已證明，不得用 RDP clipboard 片段當完成證據 | | 2 | CIR-P0-RBT-002 | P0 | 「沒有偵測到主機重啟」 | 修正 host reboot/shutdown/up detection：boot_id / uptime / node exporter / Windows exporter / VMware VM power state 都要進同一事件 | Scorecard 已接 collection packet + management probe；08:23 host probe 有 7 列但 99 只有 ping/RDP reachable、uptime unknown，111 unreachable，112/120/121/188 uptime 都已超過 10 分鐘窗口 | 讓 99 verifier / Windows exporter 或等效 no-secret readback 進入 host boot event，並補 111 reachability 證據 | -| 3 | CIR-P0-RBT-003 | P0 | 「192.168.0.99 VMWare 要自動啟動，裡面 111/188/120/121/112 也自動啟動」 | Windows 99 VMware host autostart + guest VM autostart contract；VM host 111/188/120/121/112 開機順序與 readback | Source verifier / parser / API readback / collection packet 已完成；09:53 management probe 讀回 99 host reachable、RDP / Hyper-V VMConnect reachable，但 SSH BatchMode `permission_denied`、WinRM timeout，`console_artifact_status=blocked_clipboard_unreliable`；collector 已改成預設 probe 全 5 個 candidate，實機仍為 `ssh_batchmode_auth_ready=0`、`verify_collection_status=blocked_ssh_publickey_auth_missing`、`remote_verify_attempted=0` | 收集 validator 可接受的完整 local console Verify output 或恢復 no-secret management channel，再確認 `VMRUN_PRESENT`、scheduled task、VMware services、VM power、VMX present 全綠 | -| 4 | CIR-P0-RBT-004 | P0 | 「192.168.0.99 不可因 Windows Update 無預警重開」 | Windows Update reboot policy：active hours / no auto-restart / maintenance window / update notification audit | Source verifier 已補 `WINDOWS_UPDATE_POLICY` 與 `WINDOWS_UPDATE_NO_AUTO_REBOOT_READY`；collection packet 已列 forbidden actions；99 management channel 尚不能收 policy readback | 取得 Verify output；若 policy 不綠，再走 controlled apply，禁止要求或記錄 Windows 密碼 | +| 3 | CIR-P0-RBT-003 | P0 | 「192.168.0.99 VMWare 要自動啟動，裡面 111/188/120/121/112 也自動啟動」 | Windows 99 VMware host autostart + guest VM autostart contract；VM host 111/188/120/121/112 開機順序與 readback | Source verifier / parser / API readback / collection packet 已完成；10:15 source/API patch 已把 nested `service_blockers=["VMAuthdService","VMnetDHCP"]` 與 VMX / power evidence 提升到 active blocker matrix，不再只顯示 guest power not ready；09:53 management probe 讀回 99 host reachable、RDP / Hyper-V VMConnect reachable，但 SSH BatchMode `permission_denied`、WinRM timeout，`console_artifact_status=blocked_clipboard_unreliable` | 先修 111 VMX source 與 VMware service / autostart task check-mode package，重跑 no-secret collector 與 scorecard；不得從 scorecard 直接啟動 / 關閉 VM、重啟 Windows service、讀密碼或 host reboot | +| 4 | CIR-P0-RBT-004 | P0 | 「192.168.0.99 不可因 Windows Update 無預警重開」 | Windows Update reboot policy：active hours / no auto-restart / maintenance window / update notification audit | 10:15 source/API patch：若 nested verifier 回 `policy_blockers=["windows_update_policy_readback_missing"]`，API 必須 fail-closed 重新提升 `windows99_update_no_auto_reboot_policy_not_ready`；不得只信舊 Prometheus `windows99_update_no_auto_reboot_ready=true` | 補 no-secret policy readback / drift alert；若 policy 不綠，另走 controlled apply package + rollback + post-verifier，禁止要求或記錄 Windows 密碼，禁止由 scorecard 直接 apply registry | | 5 | CIR-P0-RBT-005 | P0 | 「網站重啟後 502 嚴重影響體驗，要維護頁，外部雲端或專業做法」 | Public maintenance fallback：Nginx / edge / external static maintenance page / status page / fail-open UX，避免 502 直出 | Source + runtime verifier 已實作；Gitea CD `#4459` Success、deploy marker `8d7a6faaf`，production API scorecard 已讀回 `public_maintenance_fallback.ready=true`、raw 5xx=`0`、unreachable without L1=`0`，P0 blockers `11`、readiness `47`；仍不可宣稱整體 10 分鐘 SLO 完成 | 若未來 public route 出現 raw 5xx，先走 L0 Nginx intercept / header verifier；若 edge unreachable，才產生 L1 external static origin / CDN decision record + rollback | | 6 | CIR-P0-RBT-006 | P0 | 「所有主機關機立刻 Telegram 告警，重啟後也要告警，其他告警一併完整思考」 | Down / shutdown suspected / reboot detected / reboot recovered / SLO missed / backup failed / freshness stale / CPU pressure / Gitea queue 告警矩陣 | HostDown / HostRebootEventDetected / RebootAutoRecoverySLOMissed 已存在；per-blocker reboot alerts 與 backup receipt rules 已 deploy/readback。Backup receipt 缺段已從 100 條 stage 噪音收斂為 110 / 188 兩條 host-level pending；仍需完整 shutdown/up E2E receipt | 補 Prometheus / Alertmanager active/resolved 與 outbound receipt；backup alert 先補 `/backup/alert-receipts/*.last_success` 脫敏 marker，不送測試 secret、不重啟主機 | | 7 | CIR-P0-RBT-007 | P0 | 「所有備份包含主機、DB、網站、服務、套件、工具、日誌都沒有監控告警」 | Backup observability coverage：backup job inventory、last success、freshness、offsite、restore drill、Telegram/AwoooP receipt | 已有 backup health exporter / alert rules / Gitea bundle restore dry-run；2026-07-03 runtime 讀回 110 有 88 個 receipt stage requirement、188 有 12 個，`BackupAlertReceiptMetricMissing*` inactive，`BackupAlertReceiptStageMissing` 聚合 pending 110 / 188 各一條 | 補 `/backup/alert-receipts/*.last_success`；再補 Gitea full dump / DB / settings / issues / packages / LFS 與所有工具/log 全量備份監控 |