From 5b0c77495b7fc433940e9b99ec4aba3b6f79c13b Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 2 Jul 2026 21:39:30 +0800 Subject: [PATCH] fix(reboot): apply l0 maintenance fallback source --- docs/LOGBOOK.md | 12 ++++++++++++ .../PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md | 15 ++++++++++++++- ...nder-inserted-requirements-priority-ledger.md | 6 +++--- k8s/nginx/awoooi-prod.conf | 16 ++++++++++++---- .../nginx-502-maintenance-snippet.conf | 1 + ops/nginx/awoooi.wooo.work.conf | 16 ++++++++++++---- .../tests/test_reboot_p0_operational_contract.py | 16 ++++++++++++++++ 7 files changed, 70 insertions(+), 12 deletions(-) diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index beadf06ca..0ec624215 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,15 @@ +## 2026-07-02 — 21:49 P0-006 L0 public maintenance fallback source apply + +**完成內容**： +- 接續 Gitea CD `#4451` success / deploy marker `c68b74686` / production source `5d5bc86fed` readback；public post-rollout probe 讀回 6 條 public routes 都是 `200`，raw 5xx without fallback=`0`。 +- 同一輪 CD build log 仍觀察到 `public_health_argocd_wait_http=502`，所以不能只停在 runtime probe；將 AWOOOI edge Nginx source 從 legacy `/50x.html` 改成 L0 maintenance fallback。 +- `ops/nginx/awoooi.wooo.work.conf` 與 `k8s/nginx/awoooi-prod.conf` 已加入 `proxy_intercept_errors on`、`error_page 502 503 504 /__awoooi-maintenance.html`、`/var/www/maintenance/maintenance.html`、`Retry-After` 與 `X-AWOOOI-Fallback=local-maintenance`。 +- `scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py` 新增 source contract：AWOOOI Nginx source 不得退回 `/50x.html`，必須保留 L0 fallback header 與 maintenance file path。 +- `PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md` 與 commander priority ledger 已同步：目前是 source apply 完成，runtime 仍需 edge staging vhost / `nginx -t` / header readback，通過前不得宣稱 production fallback fully closed。 + +**仍維持**： +- 未使用 GitHub / `gh` / GitHub API；未讀 secret / token / `.env` / raw sessions / SQLite / auth；未 workflow_dispatch；未重啟主機 / VM / Docker / Nginx / K3s / DB / firewall；未切 DNS / CDN / provider。 + ## 2026-07-02 — 21:38 P0-006 public maintenance fallback test fixture closure **完成內容**： diff --git a/docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md b/docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md index e62f80744..3a45e97d5 100644 --- a/docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md +++ b/docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md @@ -17,7 +17,7 @@ ## L0 受控套用 1. 將 `ops/maintenance/maintenance.html` 部署到 edge host 的 `/var/www/maintenance/maintenance.html`。 -2. 在每個 public `server {}` 內 include `ops/maintenance/nginx-502-maintenance-snippet.conf` 的等價內容。 +2. 在每個 public `server {}` 內 include `ops/maintenance/nginx-502-maintenance-snippet.conf` 的等價內容；AWOOOI source conf `ops/nginx/awoooi.wooo.work.conf` 與 `k8s/nginx/awoooi-prod.conf` 已固定 `proxy_intercept_errors on`、`/__awoooi-maintenance.html` 與 `X-AWOOOI-Fallback: local-maintenance`。 3. `nginx -t` 通過後才 reload Nginx。 4. 用壞 upstream staging vhost 或暫存 upstream 驗證 `X-AWOOOI-Fallback: local-maintenance`，不得用 production DB / app restart 當測試手段。 @@ -59,3 +59,16 @@ python3 scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py \ - `public_route_raw_5xx_without_maintenance_fallback`：L0 Nginx intercept 未生效，下一步只能做 staging vhost / `nginx -t` / header verifier，不得用 production app restart 當測試。 - `public_route_unreachable_without_external_l1_fallback`：edge 或外部路徑整段不可達，下一步是 L1 external static origin / CDN health check decision record 與 rollback，不直接切 DNS。 - `public_maintenance_fallback_runtime_readback_missing`：沒有 runtime probe 證據，不得宣稱 public maintenance fallback 已完成。 + +## 2026-07-02 L0 source apply + +已將 AWOOOI edge Nginx source 從 legacy `/50x.html` 改為 L0 maintenance fallback： + +- `ops/nginx/awoooi.wooo.work.conf` +- `k8s/nginx/awoooi-prod.conf` +- `ops/maintenance/nginx-502-maintenance-snippet.conf` + +驗收仍必須分兩層： + +- Source verifier：`scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py` 必須確認上述 source 不可退回 `/50x.html`。 +- Runtime verifier：在 edge host 用 staging vhost / `nginx -t` / header readback 證明 `X-AWOOOI-Fallback=local-maintenance`；通過前不得宣稱 production runtime 已接管 raw 502。 diff --git a/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md b/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md index 47ff74142..0228f98a9 100644 --- a/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md +++ b/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md @@ -61,11 +61,11 @@ | 2 | CIR-P0-RBT-002 | P0 | 「沒有偵測到主機重啟」 | 修正 host reboot/shutdown/up detection：boot_id / uptime / node exporter / Windows exporter / VMware VM power state 都要進同一事件 | Scorecard 已接 collection packet + management probe；99 host reachable 但 uptime unknown，111 unreachable，stale hosts 仍存在 | 讓 99 verifier / Windows exporter 或等效 no-secret readback 進入 host boot event，並補 111 reachability 證據 | | 3 | CIR-P0-RBT-003 | P0 | 「192.168.0.99 VMWare 要自動啟動，裡面 111/188/120/121/112 也自動啟動」 | Windows 99 VMware host autostart + guest VM autostart contract；VM host 111/188/120/121/112 開機順序與 readback | Source verifier / parser / API readback / collection packet 已完成；management probe 讀回 `host_reachable=true`、RDP open、`2179` VMConnect / console channel open、SSH BatchMode `permission_denied`、WinRM timeout；snapshot active blockers=`windows99_remote_execution_channel_unavailable`、`windows99_vmware_autostart_readback_missing` | 收集 local console Verify output 或恢復 no-secret management channel，再確認 `VMRUN_PRESENT`、scheduled task、VMware services、VM power、VMX present 全綠 | | 4 | CIR-P0-RBT-004 | P0 | 「192.168.0.99 不可因 Windows Update 無預警重開」 | Windows Update reboot policy：active hours / no auto-restart / maintenance window / update notification audit | Source verifier 已補 `WINDOWS_UPDATE_POLICY` 與 `WINDOWS_UPDATE_NO_AUTO_REBOOT_READY`；collection packet 已列 forbidden actions；99 management channel 尚不能收 policy readback | 取得 Verify output；若 policy 不綠，再走 controlled apply，禁止要求或記錄 Windows 密碼 | -| 5 | CIR-P0-RBT-005 | P0 | 「網站重啟後 502 嚴重影響體驗，要維護頁，外部雲端或專業做法」 | Public maintenance fallback：Nginx / edge / external static maintenance page / status page / fail-open UX，避免 502 直出 | Source + runtime verifier 已實作：`public-maintenance-fallback-probe.py` 會抓 raw 5xx / L0 header / L1 unreachable，scorecard 會 fail-closed；L0 Nginx 實際套用與 L1 external/CDN cutover 尚需 runtime readback | 先讀回 production probe；若出現 `public_route_raw_5xx_without_maintenance_fallback`，只做 staging vhost / `nginx -t` / header verifier；若 edge unreachable，產生 L1 external static origin / CDN decision record + rollback | +| 5 | CIR-P0-RBT-005 | P0 | 「網站重啟後 502 嚴重影響體驗，要維護頁，外部雲端或專業做法」 | Public maintenance fallback：Nginx / edge / external static maintenance page / status page / fail-open UX，避免 502 直出 | Source + runtime verifier 已實作；AWOOOI Nginx source conf 已從 legacy `/50x.html` 改成 L0 `proxy_intercept_errors on` + `__awoooi-maintenance.html` + `X-AWOOOI-Fallback=local-maintenance`；production post-rollout probe 目前 6 條 public routes 都是 `200`、raw 5xx=`0`，但 CD rollout 期間仍觀察到 `public_health_argocd_wait_http=502`，所以 runtime 接管尚未驗收 | 下一步只做 edge staging vhost / `nginx -t` / header verifier；通過後才 controlled reload；若 edge unreachable，產生 L1 external static origin / CDN decision record + rollback | | 6 | CIR-P0-RBT-006 | P0 | 「所有主機關機立刻 Telegram 告警，重啟後也要告警，其他告警一併完整思考」 | Down / shutdown suspected / reboot detected / reboot recovered / SLO missed / backup failed / freshness stale / CPU pressure / Gitea queue 告警矩陣 | 部分已有 Alertmanager rule 與 Telegram receipt 補強；仍缺完整 shutdown/up E2E receipt | 建立 Telegram alert matrix + receipt verifier，逐項讀回 Alertmanager active/resolved 與 outbound receipt，不送測試 secret | | 7 | CIR-P0-RBT-007 | P0 | 「所有備份包含主機、DB、網站、服務、套件、工具、日誌都沒有監控告警」 | Backup observability coverage：backup job inventory、last success、freshness、offsite、restore drill、Telegram receipt | 部分已有 backup health exporter / alert rules；全域 coverage 與 restore drill 未全綠 | 建立 backup coverage matrix：host / DB / website / service config / package list / tool scripts / logs，每列有 metric、alert、last_success、restore_verifier | -| 8 | CIR-P0-RBT-008 | P0 | 「每次重啟排查都不一樣，也不知道多久恢復，不符合 SLA」 | 固定化 reboot runbook：fixed triage order、ETA、active blocker、remaining seconds、owner lane、next command | Scorecard/API contract 已固定 `next_safe_action=restore_windows99_no_secret_management_channel_or_collect_local_console_verify_readback_then_rerun_reboot_scorecard_no_reboot`；尚待 deploy readback | 推送後讀回 `/api/v1/agents/reboot-auto-recovery-slo-scorecard`，確認 production 顯示 99 management-channel blocker 與 readiness `43%` | -| 9 | CIR-P0-RBT-009 | P0 | 「所有產品、網站都要是最新版本；版本和數據是否最新要驗證」 | Product freshness/version matrix：source commit、deploy marker、runtime image、public health、data freshness、latest source availability | 本輪 StockPlatform public freshness / ingestion 讀回 `ok`，latest trading date `2026-07-01`；全產品版本矩陣仍未統一 | 建立全產品 readback 表：product、canonical repo、main SHA、deploy marker、public URL、data freshness、blocked reason | +| 8 | CIR-P0-RBT-008 | P0 | 「每次重啟排查都不一樣，也不知道多久恢復，不符合 SLA」 | 固定化 reboot runbook：fixed triage order、ETA、active blocker、remaining seconds、owner lane、next command | Production scorecard readback 已固定 `status=blocked_reboot_auto_recovery_slo_not_ready`、readiness `43%`、active blockers `11`、`next_safe_action=restore_windows99_no_secret_management_channel_or_collect_local_console_verify_readback_then_rerun_reboot_scorecard_no_reboot` | 優先收斂 99 no-secret Verify / 111 reachability / backup_core / service_green / Wazuh degraded；不得用不同排查路徑繞過 scorecard | +| 9 | CIR-P0-RBT-009 | P0 | 「所有產品、網站都要是最新版本；版本和數據是否最新要驗證」 | Product freshness/version matrix：source commit、deploy marker、runtime image、public health、data freshness、latest source availability | AWOOOI Gitea main 已到 deploy marker `c68b74686`，production source readback `5d5bc86fed` verified；StockPlatform public freshness / ingestion 讀回 `ok`，latest trading date `2026-07-02`，core price/chips/margin/AI recommendations 都是 `2026-07-02` | 建立全產品 readback 表：product、canonical repo、main SHA、deploy marker、public URL、data freshness、blocked reason | | 10 | CIR-P0-GIT-001 | P0 | 「Gitea 儲存庫都不見了？Gitea 沒完整備份嗎？」 | Gitea repository identity + backup proof + restore drill：不能只看 UI visible，要比對 SSH heads、repo path、bundle backup、restore sample | 已有 9 expected repos OK / backup health missing=0 的 handoff；仍需 restore drill 證明 | 補 Gitea repo bundle backup readback + sample restore dry-run verifier；禁止刪 repo / 改 visibility | | 11 | CIR-P0-CPU-001 | P0 | 「110 / 188 CPU 負載持續過高，為什麼沒監控告警、沒主動修復」 | Sustained CPU pressure automation：Alertmanager → controller → evidence → service playbook → verifier → KM writeback | 110 已有 `Host110SustainedModeratePressure`、Gitea playbook、Stock/Postgres evidence；188 仍需同級 controller/alerts readback | 下一步接 `postgres_hot_query_or_backup_export_playbook`；並補 188 equivalent readback，不以單次下降結案 | | 12 | CIR-P0-CPU-002 | P0 | 「噪音會影響真問題，要整合一起做」 | Alert noise / real issue correlation：backup aggregate noise、CPU pressure、Gitea queue、Stock freshness 要分清主因與次因 | 部分已在 SOP 註記；仍需統一 correlation scorecard | 建立 incident correlation readback：primary_blocker、secondary_noise、ignored_noise_reason、evidence_ref | diff --git a/k8s/nginx/awoooi-prod.conf b/k8s/nginx/awoooi-prod.conf index da01b557e..5dfef1675 100644 --- a/k8s/nginx/awoooi-prod.conf +++ b/k8s/nginx/awoooi-prod.conf @@ -42,6 +42,10 @@ server { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; + # L0 public maintenance fallback：攔截 upstream 502/503/504， + # 避免主機或 K3s rollout 期間把 Nginx raw 5xx 直接給使用者。 + proxy_intercept_errors on; + # ============================================ # SSE 串流路由 (AI 思考 / Dashboard 即時更新) # ⚠️ 關鍵配置: 禁用緩衝 + 長連線 @@ -124,12 +128,16 @@ server { } # ============================================ - # 錯誤頁面 + # Public maintenance fallback # ============================================ - error_page 502 503 504 /50x.html; - location = /50x.html { - root /usr/share/nginx/html; + error_page 502 503 504 /__awoooi-maintenance.html; + location = /__awoooi-maintenance.html { internal; + root /var/www/maintenance; + try_files /maintenance.html =503; + add_header Cache-Control "no-store, no-cache, must-revalidate, max-age=0" always; + add_header Retry-After "120" always; + add_header X-AWOOOI-Fallback "local-maintenance" always; } # ============================================ diff --git a/ops/maintenance/nginx-502-maintenance-snippet.conf b/ops/maintenance/nginx-502-maintenance-snippet.conf index 5ad8ff4fb..96066d282 100644 --- a/ops/maintenance/nginx-502-maintenance-snippet.conf +++ b/ops/maintenance/nginx-502-maintenance-snippet.conf @@ -12,5 +12,6 @@ location = /__awoooi-maintenance.html { root /var/www/maintenance; try_files /maintenance.html =503; add_header Cache-Control "no-store, no-cache, must-revalidate, max-age=0" always; + add_header Retry-After "120" always; add_header X-AWOOOI-Fallback "local-maintenance" always; } diff --git a/ops/nginx/awoooi.wooo.work.conf b/ops/nginx/awoooi.wooo.work.conf index 00d18cedc..c390401c4 100644 --- a/ops/nginx/awoooi.wooo.work.conf +++ b/ops/nginx/awoooi.wooo.work.conf @@ -89,6 +89,10 @@ server { proxy_set_header X-Forwarded-Host $host; proxy_set_header X-Forwarded-Port $server_port; + # L0 public maintenance fallback：攔截 upstream 502/503/504， + # 避免主機或 K3s rollout 期間把 Nginx raw 5xx 直接給使用者。 + proxy_intercept_errors on; + # ========================================================================= # API 路由 (/api/) # ========================================================================= @@ -176,12 +180,16 @@ server { } # ========================================================================= - # 錯誤頁面 + # Public maintenance fallback # ========================================================================= - error_page 502 503 504 /50x.html; - location = /50x.html { - root /usr/share/nginx/html; + error_page 502 503 504 /__awoooi-maintenance.html; + location = /__awoooi-maintenance.html { internal; + root /var/www/maintenance; + try_files /maintenance.html =503; + add_header Cache-Control "no-store, no-cache, must-revalidate, max-age=0" always; + add_header Retry-After "120" always; + add_header X-AWOOOI-Fallback "local-maintenance" always; } # ========================================================================= diff --git a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py index 107e12128..bd61de45a 100644 --- a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py +++ b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py @@ -63,6 +63,22 @@ def test_reboot_p0_contract_has_maintenance_and_telegram_alerts() -> None: assert "awoooi_public_route_raw_5xx_without_maintenance_fallback" in exporter +def test_reboot_p0_contract_applies_l0_fallback_to_awoooi_nginx_sources() -> None: + for path in [ + "ops/nginx/awoooi.wooo.work.conf", + "k8s/nginx/awoooi-prod.conf", + ]: + conf = read(path) + assert "proxy_intercept_errors on;" in conf + assert "error_page 502 503 504 /__awoooi-maintenance.html;" in conf + assert "location = /__awoooi-maintenance.html" in conf + assert "root /var/www/maintenance;" in conf + assert "try_files /maintenance.html =503;" in conf + assert 'add_header Retry-After "120" always;' in conf + assert 'add_header X-AWOOOI-Fallback "local-maintenance" always;' in conf + assert "/50x.html" not in conf + + def test_reboot_slo_scorecard_fails_closed_on_raw_public_502(tmp_path: Path) -> None: summary = tmp_path / "summary.txt" summary.write_text(