From f22b85faa556f60d79f7dd6f825f234ab70facf7 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 2 Jul 2026 22:04:55 +0800 Subject: [PATCH] fix(reboot): overlay maintenance runtime readback --- .../reboot_auto_recovery_slo_scorecard.py | 145 ++++++++++++++++++ ..._reboot_auto_recovery_slo_scorecard_api.py | 65 ++++++++ docs/LOGBOOK.md | 15 ++ .../PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md | 17 ++ ...r-inserted-requirements-priority-ledger.md | 35 +++-- 5 files changed, 260 insertions(+), 17 deletions(-) diff --git a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py index 6ad6af568..aa991fbea 100644 --- a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py +++ b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py @@ -8,6 +8,7 @@ write StockPlatform data; it only reads the committed JSON scorecard. from __future__ import annotations import json +import os from datetime import datetime from pathlib import Path from typing import Any @@ -19,10 +20,25 @@ _DEFAULT_OPERATIONS_DIR = default_operations_dir(Path(__file__)) _SCORECARD_FILE = "awoooi-reboot-auto-recovery-slo-scorecard.snapshot.json" _SOURCE_SCHEMA_VERSION = "awoooi_reboot_auto_recovery_slo_scorecard_v1" _API_SCHEMA_VERSION = "reboot_auto_recovery_slo_scorecard_readback_v1" +_PUBLIC_MAINTENANCE_RUNTIME_FILE_ENV = ( + "AWOOOI_PUBLIC_MAINTENANCE_FALLBACK_READBACK_FILE" +) +_PUBLIC_MAINTENANCE_RUNTIME_DIR_ENV = "AWOOOI_REBOOT_RECOVERY_LOG_DIR" +_DEFAULT_REBOOT_RECOVERY_LOG_DIR = Path("/home/wooo/reboot-recovery") +_PUBLIC_MAINTENANCE_RUNTIME_PATTERN = ( + "reboot-auto-recovery-slo-*/public-maintenance-fallback.json" +) +_PUBLIC_MAINTENANCE_BLOCKERS = { + "public_maintenance_fallback_runtime_readback_missing", + "public_route_raw_5xx_without_maintenance_fallback", + "public_route_unreachable_without_external_l1_fallback", +} def load_latest_reboot_auto_recovery_slo_scorecard( operations_dir: Path | None = None, + public_maintenance_runtime_path: Path | None = None, + public_maintenance_runtime_dir: Path | None = None, ) -> dict[str, Any]: """Load and validate the committed P0-006 reboot recovery scorecard.""" directory = operations_dir or _DEFAULT_OPERATIONS_DIR @@ -36,10 +52,134 @@ def load_latest_reboot_auto_recovery_slo_scorecard( raise ValueError(f"{path}: unsupported schema_version={scorecard.get('schema_version')!r}") payload = _build_payload(scorecard, path) + public_maintenance_runtime = _load_latest_public_maintenance_runtime_readback( + runtime_path=public_maintenance_runtime_path, + runtime_dir=public_maintenance_runtime_dir, + ) + if public_maintenance_runtime: + apply_public_maintenance_runtime_readback(payload, public_maintenance_runtime) _require_operation_boundaries(payload, str(path)) return payload +def _load_latest_public_maintenance_runtime_readback( + *, + runtime_path: Path | None = None, + runtime_dir: Path | None = None, +) -> dict[str, Any]: + explicit_path = runtime_path or _path_from_env(_PUBLIC_MAINTENANCE_RUNTIME_FILE_ENV) + if explicit_path: + return _read_json_file(explicit_path) + + directory = ( + runtime_dir + or _path_from_env(_PUBLIC_MAINTENANCE_RUNTIME_DIR_ENV) + or _DEFAULT_REBOOT_RECOVERY_LOG_DIR + ) + try: + candidates = [ + path + for path in directory.glob(_PUBLIC_MAINTENANCE_RUNTIME_PATTERN) + if path.is_file() + ] + except OSError: + return {} + if not candidates: + return {} + latest = max(candidates, key=lambda path: path.stat().st_mtime) + return _read_json_file(latest) + + +def _path_from_env(name: str) -> Path | None: + value = os.environ.get(name, "").strip() + return Path(value) if value else None + + +def _read_json_file(path: Path) -> dict[str, Any]: + try: + with path.open(encoding="utf-8") as handle: + payload = json.load(handle) + except (FileNotFoundError, json.JSONDecodeError, OSError): + return {} + return payload if isinstance(payload, dict) else {} + + +def apply_public_maintenance_runtime_readback( + payload: dict[str, Any], + runtime_readback: dict[str, Any], +) -> None: + """Overlay exporter public-route fallback evidence onto the scorecard.""" + public_maintenance = _build_public_maintenance_fallback_readback(runtime_readback) + payload["public_maintenance_fallback"] = public_maintenance + + active_blockers = [ + blocker + for blocker in _strings(payload.get("active_blockers")) + if blocker not in _PUBLIC_MAINTENANCE_BLOCKERS + ] + active_blockers.extend(_strings(public_maintenance.get("blockers"))) + active_blockers = _unique_strings(active_blockers) + payload["active_blockers"] = active_blockers + + required_checks = _dict(payload.setdefault("required_checks", {})) + required_checks["public_maintenance_fallback_runtime_ready"] = ( + public_maintenance.get("ready") is True + ) + completed_check_count = sum(1 for value in required_checks.values() if value) + readiness_percent = _percent( + completed_check_count / max(len(required_checks), 1) * 100 + ) + active_blocker_count = len(active_blockers) + payload["active_blocker_count"] = active_blocker_count + payload["readiness_percent"] = readiness_percent + + progress = _dict(payload.setdefault("reboot_sop_progress", {})) + progress["active_blockers"] = active_blockers + progress["active_blocker_count"] = active_blocker_count + progress["readiness_percent"] = readiness_percent + progress["current_phase"] = _reboot_sop_current_phase( + active_blockers, + payload.get("can_claim_all_services_recovered_within_target") is True, + ) + progress["primary_blocker"] = _reboot_sop_primary_blocker(active_blockers) + payload["reboot_sop_progress"] = progress + payload["current_phase"] = progress["current_phase"] + payload["primary_blocker"] = progress["primary_blocker"] + + readback = _dict(payload.setdefault("readback", {})) + readback["active_blocker_count"] = active_blocker_count + readback["readiness_percent"] = readiness_percent + readback["public_maintenance_runtime_readback_present"] = ( + public_maintenance.get("runtime_readback_present") is True + ) + readback["public_maintenance_fallback_ready"] = ( + public_maintenance.get("ready") is True + ) + readback["public_route_raw_5xx_without_fallback_count"] = _int( + public_maintenance.get("raw_5xx_without_fallback_count") + ) + + rollups = _dict(payload.setdefault("rollups", {})) + rollups["active_blocker_count"] = active_blocker_count + rollups["completed_check_count"] = completed_check_count + rollups["readiness_percent"] = readiness_percent + rollups["public_maintenance_runtime_readback_present"] = ( + public_maintenance.get("runtime_readback_present") is True + ) + rollups["public_maintenance_fallback_ready"] = ( + public_maintenance.get("ready") is True + ) + rollups["public_route_raw_5xx_without_fallback_count"] = _int( + public_maintenance.get("raw_5xx_without_fallback_count") + ) + rollups["public_route_unreachable_without_external_fallback_count"] = _int( + public_maintenance.get("route_unreachable_without_external_fallback_count") + ) + rollups["public_maintenance_fallback_route_count"] = _int( + public_maintenance.get("maintenance_fallback_route_count") + ) + + def apply_stockplatform_runtime_readback( payload: dict[str, Any], runtime_readback: dict[str, Any], @@ -870,6 +1010,8 @@ def _reboot_sop_current_phase(active_blockers: list[str], can_claim_slo: bool) - for blocker in active_blockers ): return "product_data_freshness_blocked" + if any(blocker in _PUBLIC_MAINTENANCE_BLOCKERS for blocker in active_blockers): + return "public_maintenance_fallback_blocked" if any("backup" in blocker for blocker in active_blockers): return "backup_readback_blocked" if "local_disk_free_below_minimum" in active_blockers: @@ -900,6 +1042,9 @@ def _reboot_sop_primary_blocker(active_blockers: list[str]) -> str: "windows99_vmware_autostart_config_not_ready", "windows99_vmware_guest_power_not_ready", "windows99_update_no_auto_reboot_policy_not_ready", + "public_route_raw_5xx_without_maintenance_fallback", + "public_route_unreachable_without_external_l1_fallback", + "public_maintenance_fallback_runtime_readback_missing", "post_start_blocked_not_zero", "service_green_not_1", "host_188_service_green_not_1", diff --git a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py index 11da92234..432caca49 100644 --- a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py +++ b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py @@ -1,5 +1,7 @@ from __future__ import annotations +import json + from fastapi import FastAPI from fastapi.testclient import TestClient @@ -39,6 +41,35 @@ EXPECTED_WINDOWS99_NEXT_STEP = ( "verify_readback_then_rerun_reboot_scorecard_no_reboot" ) +PUBLIC_MAINTENANCE_RUNTIME_READY = { + "schema_version": "awoooi_public_maintenance_fallback_probe_v1", + "generated_at": "2026-07-02T18:08:00+08:00", + "runtime_readback_present": True, + "ready": True, + "target_url_count": 6, + "route_count": 6, + "raw_5xx_without_fallback_count": 0, + "route_unreachable_without_external_fallback_count": 0, + "maintenance_fallback_route_count": 0, + "raw_5xx_without_fallback_urls": [], + "route_unreachable_without_external_fallback_urls": [], + "maintenance_fallback_urls": [], + "blockers": [], + "routes": [ + { + "url": "https://awoooi.wooo.work/api/v1/health", + "http_status": 200, + "status": "normal_origin_serving", + "fallback_header": "", + "maintenance_body_detected": False, + "maintenance_fallback_serving": False, + "raw_5xx_without_fallback": False, + "route_unreachable_without_external_fallback": False, + "error": "", + } + ], +} + def test_reboot_auto_recovery_slo_scorecard_loader_exposes_stockplatform_gate(): payload = load_latest_reboot_auto_recovery_slo_scorecard() @@ -46,6 +77,40 @@ def test_reboot_auto_recovery_slo_scorecard_loader_exposes_stockplatform_gate(): _assert_reboot_slo_payload(payload) +def test_reboot_auto_recovery_slo_scorecard_overlays_public_maintenance_runtime_artifact( + tmp_path, +): + runtime_path = tmp_path / "public-maintenance-fallback.json" + runtime_path.write_text( + json.dumps(PUBLIC_MAINTENANCE_RUNTIME_READY), + encoding="utf-8", + ) + + payload = load_latest_reboot_auto_recovery_slo_scorecard( + public_maintenance_runtime_path=runtime_path + ) + + assert ( + "public_maintenance_fallback_runtime_readback_missing" + not in payload["active_blockers"] + ) + assert payload["active_blocker_count"] == 11 + assert payload["readiness_percent"] == 47 + assert payload["required_checks"][ + "public_maintenance_fallback_runtime_ready" + ] is True + assert payload["public_maintenance_fallback"]["runtime_readback_present"] is True + assert payload["public_maintenance_fallback"]["ready"] is True + assert payload["readback"]["public_maintenance_runtime_readback_present"] is True + assert payload["readback"]["public_maintenance_fallback_ready"] is True + assert payload["rollups"]["public_maintenance_runtime_readback_present"] is True + assert payload["rollups"]["public_maintenance_fallback_ready"] is True + assert payload["reboot_sop_progress"]["current_phase"] == ( + "host_boot_detection_blocked" + ) + assert payload["reboot_sop_progress"]["active_blocker_count"] == 11 + + def test_reboot_auto_recovery_slo_scorecard_endpoint_returns_readback(monkeypatch): monkeypatch.setattr( agents, diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index ef60f1d53..37d1ba0bf 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -14,6 +14,21 @@ **仍維持**： - 未讀 secret / token / `.env` / raw sessions / SQLite / auth；未使用 GitHub / gh；未 SSH、未 Docker / K8s / firewall / Wazuh 寫入、未重啟 host / VM / service、未 workflow_dispatch、未 DROP / TRUNCATE / restore / prune。 +## 2026-07-02 — 22:02 P0-006 public maintenance runtime artifact overlay + +**完成內容**： +- 補上 `apps/api/src/services/reboot_auto_recovery_slo_scorecard.py` runtime artifact overlay：API scorecard 仍以 `docs/operations/awoooi-reboot-auto-recovery-slo-scorecard.snapshot.json` 為基準，但會讀取 `AWOOOI_PUBLIC_MAINTENANCE_FALLBACK_READBACK_FILE` 或 `/home/wooo/reboot-recovery/reboot-auto-recovery-slo-*/public-maintenance-fallback.json` 的 exporter artifact，將 public maintenance fallback runtime readback 套疊進 `active_blockers` / `required_checks` / `readback` / `rollups`。 +- 缺 artifact 時仍維持 fail-closed `public_maintenance_fallback_runtime_readback_missing`；artifact 證明 `ready=true` 時才清除此 blocker，不把 source-only fallback 宣稱為 runtime ready。 +- 補齊 API SOP phase / primary blocker 對 public fallback blocker 的分類，避免只剩 502 fallback 問題時被歸成 generic `slo_blocked`。 + +**驗證**： +- `DATABASE_URL=postgresql+asyncpg://test:test@localhost/test python3.11 -m pytest apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py -q -p no:cacheprovider`：`25 passed`。 +- `python3 ops/runner/guard-gitea-runner-pressure.py --root .`：`GITEA_RUNNER_PRESSURE_GUARD_OK`。 +- 用本輪 live artifact `/tmp/awoooi-public-maintenance-fallback-after-4456.json` overlay 本機 API loader：active blockers `12 -> 11`、readiness `40 -> 47`、`public_maintenance_runtime_readback_present=true`、`public_maintenance_fallback_ready=true`、`public_maintenance_fallback_runtime_readback_missing=false`。 + +**仍維持**： +- Production API 尚未讀回此 overlay；下一步必須推 Gitea CD 並讀回 `/api/v1/agents/reboot-auto-recovery-slo-scorecard`。未使用 GitHub / `gh` / GitHub API；未讀 secret / token / `.env` / raw sessions / SQLite / auth；未 workflow_dispatch；未重啟主機 / VM / Docker / Nginx / K3s / DB / firewall；未切 DNS / CDN / provider。 + ## 2026-07-02 — 21:56 P0-006 L0 fallback CD profile blocker fix **完成內容**： diff --git a/docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md b/docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md index 3a45e97d5..783ea9dfa 100644 --- a/docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md +++ b/docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md @@ -72,3 +72,20 @@ python3 scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py \ - Source verifier：`scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py` 必須確認上述 source 不可退回 `/50x.html`。 - Runtime verifier：在 edge host 用 staging vhost / `nginx -t` / header readback 證明 `X-AWOOOI-Fallback=local-maintenance`；通過前不得宣稱 production runtime 已接管 raw 502。 + +## 2026-07-02 API runtime artifact overlay + +`/api/v1/agents/reboot-auto-recovery-slo-scorecard` 不得只依賴 committed snapshot 判斷 public maintenance fallback；它必須能讀取 `reboot-auto-recovery-slo-exporter.sh` 產出的 runtime artifact。 + +讀取順序： + +1. `AWOOOI_PUBLIC_MAINTENANCE_FALLBACK_READBACK_FILE` 指定的 JSON。 +2. `/home/wooo/reboot-recovery/reboot-auto-recovery-slo-*/public-maintenance-fallback.json` 中最新一份。 +3. 找不到 artifact 時維持 fail-closed `public_maintenance_fallback_runtime_readback_missing`。 + +驗收： + +- Production API 讀回 `readback.public_maintenance_runtime_readback_present=true`。 +- Production API 讀回 `readback.public_maintenance_fallback_ready=true`。 +- `active_blockers` 不包含 `public_maintenance_fallback_runtime_readback_missing`。 +- 若 public routes 都正常，這只代表「沒有裸 5xx 且 runtime artifact 已接上 scorecard」；仍需 staging vhost / `nginx -t` / header readback 才能宣稱 L0 intercept 在 upstream failure 時生效。 diff --git a/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md b/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md index 0228f98a9..79086c7c0 100644 --- a/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md +++ b/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md @@ -61,10 +61,10 @@ | 2 | CIR-P0-RBT-002 | P0 | 「沒有偵測到主機重啟」 | 修正 host reboot/shutdown/up detection：boot_id / uptime / node exporter / Windows exporter / VMware VM power state 都要進同一事件 | Scorecard 已接 collection packet + management probe；99 host reachable 但 uptime unknown，111 unreachable，stale hosts 仍存在 | 讓 99 verifier / Windows exporter 或等效 no-secret readback 進入 host boot event，並補 111 reachability 證據 | | 3 | CIR-P0-RBT-003 | P0 | 「192.168.0.99 VMWare 要自動啟動，裡面 111/188/120/121/112 也自動啟動」 | Windows 99 VMware host autostart + guest VM autostart contract；VM host 111/188/120/121/112 開機順序與 readback | Source verifier / parser / API readback / collection packet 已完成；management probe 讀回 `host_reachable=true`、RDP open、`2179` VMConnect / console channel open、SSH BatchMode `permission_denied`、WinRM timeout；snapshot active blockers=`windows99_remote_execution_channel_unavailable`、`windows99_vmware_autostart_readback_missing` | 收集 local console Verify output 或恢復 no-secret management channel，再確認 `VMRUN_PRESENT`、scheduled task、VMware services、VM power、VMX present 全綠 | | 4 | CIR-P0-RBT-004 | P0 | 「192.168.0.99 不可因 Windows Update 無預警重開」 | Windows Update reboot policy：active hours / no auto-restart / maintenance window / update notification audit | Source verifier 已補 `WINDOWS_UPDATE_POLICY` 與 `WINDOWS_UPDATE_NO_AUTO_REBOOT_READY`；collection packet 已列 forbidden actions；99 management channel 尚不能收 policy readback | 取得 Verify output；若 policy 不綠，再走 controlled apply，禁止要求或記錄 Windows 密碼 | -| 5 | CIR-P0-RBT-005 | P0 | 「網站重啟後 502 嚴重影響體驗，要維護頁，外部雲端或專業做法」 | Public maintenance fallback：Nginx / edge / external static maintenance page / status page / fail-open UX，避免 502 直出 | Source + runtime verifier 已實作；AWOOOI Nginx source conf 已從 legacy `/50x.html` 改成 L0 `proxy_intercept_errors on` + `__awoooi-maintenance.html` + `X-AWOOOI-Fallback=local-maintenance`；production post-rollout probe 目前 6 條 public routes 都是 `200`、raw 5xx=`0`，但 CD rollout 期間仍觀察到 `public_health_argocd_wait_http=502`，所以 runtime 接管尚未驗收 | 下一步只做 edge staging vhost / `nginx -t` / header verifier；通過後才 controlled reload；若 edge unreachable，產生 L1 external static origin / CDN decision record + rollback | +| 5 | CIR-P0-RBT-005 | P0 | 「網站重啟後 502 嚴重影響體驗，要維護頁，外部雲端或專業做法」 | Public maintenance fallback：Nginx / edge / external static maintenance page / status page / fail-open UX，避免 502 直出 | Source + runtime verifier 已實作；AWOOOI Nginx source conf 已從 legacy `/50x.html` 改成 L0 `proxy_intercept_errors on` + `__awoooi-maintenance.html` + `X-AWOOOI-Fallback=local-maintenance`；production post-rollout probe 目前 6 條 public routes 都是 `200`、raw 5xx=`0`；API runtime artifact overlay 已本機驗證可將 scorecard blockers `12 -> 11`、readiness `40 -> 47`，但 production API 尚未 CD 讀回 | 先推 Gitea CD 並讀回 `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 清掉 `public_maintenance_fallback_runtime_readback_missing`；再做 edge staging vhost / `nginx -t` / header verifier；通過後才 controlled reload；若 edge unreachable，產生 L1 external static origin / CDN decision record + rollback | | 6 | CIR-P0-RBT-006 | P0 | 「所有主機關機立刻 Telegram 告警，重啟後也要告警，其他告警一併完整思考」 | Down / shutdown suspected / reboot detected / reboot recovered / SLO missed / backup failed / freshness stale / CPU pressure / Gitea queue 告警矩陣 | 部分已有 Alertmanager rule 與 Telegram receipt 補強；仍缺完整 shutdown/up E2E receipt | 建立 Telegram alert matrix + receipt verifier，逐項讀回 Alertmanager active/resolved 與 outbound receipt，不送測試 secret | | 7 | CIR-P0-RBT-007 | P0 | 「所有備份包含主機、DB、網站、服務、套件、工具、日誌都沒有監控告警」 | Backup observability coverage：backup job inventory、last success、freshness、offsite、restore drill、Telegram receipt | 部分已有 backup health exporter / alert rules；全域 coverage 與 restore drill 未全綠 | 建立 backup coverage matrix：host / DB / website / service config / package list / tool scripts / logs，每列有 metric、alert、last_success、restore_verifier | -| 8 | CIR-P0-RBT-008 | P0 | 「每次重啟排查都不一樣，也不知道多久恢復，不符合 SLA」 | 固定化 reboot runbook：fixed triage order、ETA、active blocker、remaining seconds、owner lane、next command | Production scorecard readback 已固定 `status=blocked_reboot_auto_recovery_slo_not_ready`、readiness `43%`、active blockers `11`、`next_safe_action=restore_windows99_no_secret_management_channel_or_collect_local_console_verify_readback_then_rerun_reboot_scorecard_no_reboot` | 優先收斂 99 no-secret Verify / 111 reachability / backup_core / service_green / Wazuh degraded；不得用不同排查路徑繞過 scorecard | +| 8 | CIR-P0-RBT-008 | P0 | 「每次重啟排查都不一樣，也不知道多久恢復，不符合 SLA」 | 固定化 reboot runbook：fixed triage order、ETA、active blocker、remaining seconds、owner lane、next command | Production scorecard readback 已固定 `status=blocked_reboot_auto_recovery_slo_not_ready`、readiness `40%`、active blockers `12`；本機套疊 live public maintenance artifact 後可降為 readiness `47%`、active blockers `11`；`next_safe_action=restore_windows99_no_secret_management_channel_or_collect_local_console_verify_readback_then_rerun_reboot_scorecard_no_reboot` | 先完成 API overlay CD / production readback，再優先收斂 99 no-secret Verify / 111 reachability / backup_core / service_green / Wazuh degraded；不得用不同排查路徑繞過 scorecard | | 9 | CIR-P0-RBT-009 | P0 | 「所有產品、網站都要是最新版本；版本和數據是否最新要驗證」 | Product freshness/version matrix：source commit、deploy marker、runtime image、public health、data freshness、latest source availability | AWOOOI Gitea main 已到 deploy marker `c68b74686`，production source readback `5d5bc86fed` verified；StockPlatform public freshness / ingestion 讀回 `ok`，latest trading date `2026-07-02`，core price/chips/margin/AI recommendations 都是 `2026-07-02` | 建立全產品 readback 表：product、canonical repo、main SHA、deploy marker、public URL、data freshness、blocked reason | | 10 | CIR-P0-GIT-001 | P0 | 「Gitea 儲存庫都不見了？Gitea 沒完整備份嗎？」 | Gitea repository identity + backup proof + restore drill：不能只看 UI visible，要比對 SSH heads、repo path、bundle backup、restore sample | 已有 9 expected repos OK / backup health missing=0 的 handoff；仍需 restore drill 證明 | 補 Gitea repo bundle backup readback + sample restore dry-run verifier；禁止刪 repo / 改 visibility | | 11 | CIR-P0-CPU-001 | P0 | 「110 / 188 CPU 負載持續過高，為什麼沒監控告警、沒主動修復」 | Sustained CPU pressure automation：Alertmanager → controller → evidence → service playbook → verifier → KM writeback | 110 已有 `Host110SustainedModeratePressure`、Gitea playbook、Stock/Postgres evidence；188 仍需同級 controller/alerts readback | 下一步接 `postgres_hot_query_or_backup_export_playbook`；並補 188 equivalent readback，不以單次下降結案 | @@ -132,7 +132,7 @@ | AI 專業 UI / 非文字牆 cockpit | 已列為 P2 UX 驗收 | 將長文字區塊收斂成 first-viewport cockpit、cards、flow rows 與 expandable details | | 10 分鐘 reboot auto-recovery SLA | scorecard source / snapshot / API 已補固定 phase / ETA / blocker / next action 欄位；仍缺 fresh all-host reboot/drill proof | 補 production API readback，缺事件則明確標等待下一次 reboot 或 approved drill | | 99 Windows / VMware autostart | Source verifier / parser / API readback / collection packet 已完成；live 99 Verify output 尚未收集，scorecard 以 `windows99_vmware_autostart_readback_missing` fail-closed；collection packet 顯示 99 reachable、可收 no-secret Verify，但 uptime unknown | 收集 99 no-secret Verify output，確認 VM 111/188/120/121/112 running、scheduled task / services / Windows Update policy 全綠 | -| 502 maintenance fallback | Source + scorecard verifier 已補；真實 L0/L1 runtime 套用仍待 probe / edge readback，不可宣稱完成 | 先跑 production probe；raw 5xx 則推 L0 Nginx intercept，edge unreachable 則推 L1 external static fallback decision + rollback | +| 502 maintenance fallback | Source + scorecard verifier 已補；API runtime artifact overlay 已本機驗證可清掉 `public_maintenance_fallback_runtime_readback_missing`；真實 L0/L1 runtime 套用仍待 edge readback，不可宣稱完成 | 推 Gitea CD 後讀回 production API；raw 5xx 則推 L0 Nginx intercept，edge unreachable 則推 L1 external static fallback decision + rollback | | 全備份監控告警 coverage | 部分 exporter/rule 已存在，但 host/DB/site/service/package/tool/log coverage 未全列 | 建立 backup coverage matrix 與 restore drill verifier | | Stock/Postgres hot pressure | 110 live 已導向 Stock/Postgres playbook；尚未完成 hot query / backup export playbook closure | 下一步執行 read-only Stock/Postgres evidence 與 source freshness / query attribution | @@ -149,17 +149,18 @@ ## 下一輪固定執行順序 1. 先確認 Gitea `main` / CD / production readback 是否仍與最新 truth 一致。 -2. 先把 AI Loop Agent、AI 自動化節點紀錄、LOG 收集/貼標、Telegram 告警 AI route 與 post-apply verifier、IwoooS/nav IA、UI/UX spotlight 落到 API/UI readback。 -3. 接續目前 active P0：110 Stock/Postgres hot pressure，跑 `postgres_hot_query_or_backup_export_playbook` 的 read-only evidence / source freshness / query attribution。 -4. 補全 reboot auto-recovery P0：99/110/111/112/120/121/188 reboot detection、10 分鐘 SLO scorecard、Telegram down/up/recovered/SLO missed receipts。 -5. 補 99 Windows / VMware autostart P0：99 host 自動啟動 VMware，VM 111/188/120/121/112 自動啟動；同時補 Windows Update no-auto-restart verifier。 -6. 補 public maintenance fallback P0：先用 `public-maintenance-fallback-probe.py` / scorecard 讀回 production；raw 502 走 L0 Nginx intercept verifier，edge unreachable 走 L1 external static fallback decision + rollback。 -7. 補 backup observability P0：host / DB / website / service config / package list / tools / logs backup matrix、last success、offsite、restore drill、Telegram receipt。 -8. 補 product freshness/version P0：所有產品網站 source SHA / deploy marker / runtime image / public URL / data freshness readback。 -9. 繼續補 110 controlled drain lane 的 staging / verifier，不恢復 generic runner。 -10. 把全產品 repo identity / backup health / private visibility / dev-prod CI baseline 接到 product governance matrix。 -11. 把本台帳的 P0/P1 工作項映射進 KM / PlayBook / RAG / MCP / LOG，不讓需求只留在聊天。 -12. 補 OpenClaw Live Ops Space 的 production desktop/mobile animation smoke 與 AwoooP 導流。 -13. 補 AI professional UI / 非文字牆 cockpit 驗收，避免只剩長文字頁。 -14. SignOz degraded 分流到 Observability P2，不阻擋 Gitea/CD 主線。 -15. GitHub appeal 前不做任何 GitHub 操作；appeal 後若使用者明確要求，先做風險評估。 +2. 接續目前 active P0-006：把 public maintenance runtime artifact overlay 推進到 Gitea CD，讀回 production API scorecard，確認 blockers `12 -> 11`、readiness `40 -> 47`。 +3. 補全 reboot auto-recovery P0：99/110/111/112/120/121/188 reboot detection、10 分鐘 SLO scorecard、Telegram down/up/recovered/SLO missed receipts。 +4. 補 99 Windows / VMware autostart P0：99 host 自動啟動 VMware，VM 111/188/120/121/112 自動啟動；同時補 Windows Update no-auto-restart verifier。 +5. 補 111 reachability / host boot readback，讓 `host_unreachable_after_reboot` 與 `reboot_event_required_host_unreachable` 有單一固定處理路徑。 +6. 補 backup observability P0：host / DB / website / service config / package list / tools / logs backup matrix、last success、offsite、restore drill、Telegram receipt。 +7. 補 service_green / Wazuh degraded readback，清楚分開 reboot service blocker 與 P2 observability debt。 +8. 補 public maintenance fallback P0：production API overlay 清掉缺讀回後，再做 edge staging vhost / `nginx -t` / header verifier；raw 502 走 L0，edge unreachable 走 L1 external static fallback decision + rollback。 +9. 補 product freshness/version P0：所有產品網站 source SHA / deploy marker / runtime image / public URL / data freshness readback。 +10. 接續 110 / 188 CPU pressure P0：跑 `postgres_hot_query_or_backup_export_playbook` 的 read-only evidence / source freshness / query attribution，並補 188 equivalent readback。 +11. 繼續補 110 controlled drain lane 的 staging / verifier，不恢復 generic runner。 +12. 把全產品 repo identity / backup health / private visibility / dev-prod CI baseline 接到 product governance matrix。 +13. 把 AI Loop Agent、AI 自動化節點紀錄、LOG 收集/貼標、Telegram 告警 AI route 與 post-apply verifier 接到上述 P0 lanes；不得讓它們取代 reboot/SLA 主線順序。 +14. 補 OpenClaw Live Ops Space 的 production desktop/mobile animation smoke 與 AwoooP 導流。 +15. 補 AI professional UI / 非文字牆 cockpit 驗收，避免只剩長文字頁。 +16. GitHub appeal 前不做任何 GitHub 操作；appeal 後若使用者明確要求，先做風險評估。