diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index cb580808f..e4e327481 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -260,6 +260,8 @@ jobs: ;; ops/nginx/awoooi.wooo.work.conf) ;; + scripts/reboot-recovery/public-maintenance-fallback-probe.py) + ;; product.awoooi.yaml) ;; docs/LOGBOOK.md) @@ -936,6 +938,7 @@ jobs: ../../scripts/reboot-recovery/dr-escrow-evidence-checklist.py \ ../../scripts/reboot-recovery/post-reboot-owner-response-preflight.py \ ../../scripts/reboot-recovery/momo-source-arrival-gate.py \ + ../../scripts/reboot-recovery/public-maintenance-fallback-probe.py \ ../../scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py \ ../../scripts/reboot-recovery/validate-windows99-console-verify-artifact.py \ ../../scripts/reboot-recovery/windows99-management-channel-probe.py \ diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 9e4420be0..14866e12e 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,17 @@ +## 2026-07-03 — 04:24 Public maintenance fallback location-level hardening + +**完成內容**： +- Gitea CD `#4517` deploy marker 切換期間，production route watch 抓到 AWOOOI API / SLO route 短暫 `502` 且無 `X-AWOOOI-Fallback` / maintenance body；此項維持 P0 public UX blocker，不再視為部署噪音。 +- `ops/nginx/awoooi.wooo.work.conf` 與 `k8s/nginx/awoooi-prod.conf` 的 AWOOOI API、SSE、WebSocket、front-end 與 static proxy location 全部補上 location-level `proxy_intercept_errors on` 與 `error_page 502 503 504 =503 /__awoooi-maintenance.html`。 +- `ops/maintenance/nginx-502-maintenance-snippet.conf` 改成 `=503`，讓維護頁明確回 `503` / `Retry-After`，避免 fallback 本身仍被 probe 或使用者解讀為裸 `502`。 +- `scripts/reboot-recovery/public-maintenance-fallback-probe.py` 改成只要 `502` / `503` / `504` 同時帶 `X-AWOOOI-Fallback=local-maintenance` 與維護頁 body，就分類為 maintenance fallback serving；缺 header/body 才算 raw 5xx。 +- `scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py` 補上 source contract：所有 AWOOOI public proxy location 必須有 L0 fallback，不得只靠 server 層繼承。 +- `.gitea/workflows/cd.yaml` controlled-runtime profile 納入 `public-maintenance-fallback-probe.py`，避免維護頁修復誤落 full/B5 Docker socket lane。 + +**仍維持**： +- 這是 source-level hardening；完成條件仍是 Gitea CD 部署後 marker-time public route watch 不再讀到 raw 502，或讀到明確 maintenance fallback header/body。 +- 未讀 secret / token / `.env` / raw sessions / SQLite / auth；未使用 GitHub / gh；未 workflow_dispatch；未重啟 host / VM / service；未 Docker / Nginx / K3s / DB / firewall restart；未 DROP / TRUNCATE / restore / prune / delete / force push。 + ## 2026-07-03 — 04:12 Windows99 console verify artifact 與 CD profile 修復 **完成內容**： @@ -308,7 +322,7 @@ **完成內容**： - 接續 Gitea CD `#4451` success / deploy marker `c68b74686` / production source `5d5bc86fed` readback；public post-rollout probe 讀回 6 條 public routes 都是 `200`，raw 5xx without fallback=`0`。 - 同一輪 CD build log 仍觀察到 `public_health_argocd_wait_http=502`，所以不能只停在 runtime probe；將 AWOOOI edge Nginx source 從 legacy `/50x.html` 改成 L0 maintenance fallback。 -- `ops/nginx/awoooi.wooo.work.conf` 與 `k8s/nginx/awoooi-prod.conf` 已加入 `proxy_intercept_errors on`、`error_page 502 503 504 /__awoooi-maintenance.html`、`/var/www/maintenance/maintenance.html`、`Retry-After` 與 `X-AWOOOI-Fallback=local-maintenance`。 +- `ops/nginx/awoooi.wooo.work.conf` 與 `k8s/nginx/awoooi-prod.conf` 已加入 `proxy_intercept_errors on`、`error_page 502 503 504 =503 /__awoooi-maintenance.html`、`/var/www/maintenance/maintenance.html`、`Retry-After` 與 `X-AWOOOI-Fallback=local-maintenance`。 - `scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py` 新增 source contract：AWOOOI Nginx source 不得退回 `/50x.html`，必須保留 L0 fallback header 與 maintenance file path。 - `PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md` 與 commander priority ledger 已同步：目前是 source apply 完成，runtime 仍需 edge staging vhost / `nginx -t` / header readback，通過前不得宣稱 production fallback fully closed。 diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index df2836674..0c604be8b 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -1,6 +1,6 @@ # AWOOOI 全棧冷啟動與主機重啟 SOP -> Version: v1.103 +> Version: v1.104 > Last updated: 2026-07-03 Asia/Taipei > Scope: 99 / 110 / 111 / 112 / 120 / 121 / 188 全棧重啟恢復。112 仍是 Kali / VM guest 訊號，但 2026-06-30 全主機重啟後已納入 10 分鐘 SLO 的必要 boot / power signal；此納入不代表授權任何破壞性 runtime apply。 @@ -40,6 +40,8 @@ v1.102 production scorecard / deploy cutover rule：2026-07-03 03:55 production v1.103 Windows99 console verify artifact rule：若 99 可 ping / RDP / Hyper-V VMConnect，但 SSH BatchMode 與 WinRM 都不可用，不能停在「等人工」。必須把 console channel 的 `windows99-vmware-autostart.ps1 -Mode Verify` stdout 存成 no-secret artifact，先跑 `python3.11 scripts/reboot-recovery/validate-windows99-console-verify-artifact.py --input --output `，再用 normalized artifact 重跑 reboot SLO scorecard。validator 必須檢查 `AWOOOI_WINDOWS99_VMWARE_AUTOSTART=1`、`MODE=Verify`、每個 required alias 的 `VMX` / `VM_POWER`、VMware services、autostart task、Windows Update policy 與 ready flags；若看到 secret-like key、欄位缺失或非 Verify marker 必須 fail-closed。此路徑只接受 Verify stdout，不授權 Windows password collection、VM power change、Windows Update policy apply、service restart、host reboot 或 registry edit。 +v1.104 public maintenance location-level intercept rule：deploy marker 切換或主機重啟期間，public API / SLO route 只要回 `502` 且沒有 `X-AWOOOI-Fallback=local-maintenance` 與維護頁 body，即使稍後恢復，也必須列為 P0 public UX blocker。`ops/nginx/awoooi.wooo.work.conf`、`k8s/nginx/awoooi-prod.conf` 與 `ops/maintenance/nginx-502-maintenance-snippet.conf` 必須使用 `error_page 502 503 504 =503 /__awoooi-maintenance.html;`，讓 fallback 明確回 `503` / `Retry-After`；所有 proxy 到 AWOOOI API / Web 的 `location` 必須各自宣告 `proxy_intercept_errors on;` 與同一條 `error_page`，不得只依賴 server 層繼承。`public-maintenance-fallback-probe.py` 需將帶 fallback header/body 的 `502` / `503` / `504` 判定為 maintenance fallback serving；只有缺 header/body 的 5xx 才算 `raw_5xx_without_maintenance_fallback`。完成條件仍是 marker-time public route watch 不再讀到 raw 502；source contract 通過不等於 live Nginx runtime 已套用。 + 2026-07-02 110 control-path / Harbor recovery receipt rule：若 Gitea Harbor repair queue 仍保留 `harbor_110_remote_ssh_publickey_auth_stalled`、remote-control unavailable、jobs stale 或 historical failure，但同一輪本地證據同時證明 `wooo` command path ready、110 local Harbor `/v2/` ready、public/internal registry `/v2/` 回 `401`，則該 Gitea Harbor repair 失敗只能列為 historical queue metadata，不得再當成 current SSH blocker。必須用 `/api/v1/agents/harbor-registry-controlled-recovery-receipt` 或同等 validator 合併 `diagnose-110-ssh-publickey-auth.sh`、`recover-110-control-path-and-harbor-local.sh --check`、public Gitea queue readback 與 registry `/v2/` verifier，並把機器可讀結果寫入 `docs/operations/harbor-110-control-path-recovery-readback-2026-07-02.snapshot.json` 類型的 snapshot。2026-07-02 live receipt 顯示：public/internal registry `/v2/` 均為 `401`、latest visible CD `#4335` 為 `Success`、Gitea Harbor repair failure 已是 `historical_after_latest_cd_success=true`；active blockers 收斂為 110 controlled CD lane config / binary / registration / service guardrail、active action container pressure，以及 Gitea CD jobs head-SHA / stale readback mismatch。若 local-console output 只有 `AWOOOI_110_CONTROLLED_CD_LANE_READY` marker，non110 runner parser 不得從 110 `BLOCKER` 行推導 non110 blocker；non110 只有看到 `AWOOOI_NON110_RUNNER_READY` marker 才能列入 active blocker。 2026-07-02 110 controlled CD lane fail-closed enforcer staging rule：110 runner 壓力事故後，legacy / generic runner 仍必須 fail-closed；但 `awoooi-cd-lane-drain.service` 的非 secret staging artifact 不得再被 enforcer 無差別封回 stub。`scripts/reboot-recovery/enforce-110-runner-failclosed.sh` 只有在 `config.yaml` 符合 `capacity <= 1`、只含 `awoooi-host:host` 與 `awoooi-ubuntu:docker://192.168.0.110:5000/awoooi/ci-runner:act-22.04`、binary 是 executable ELF、systemd unit 具備 `ConditionPathExists=/home/wooo/awoooi-cd-lane-drain/data/.runner`、`CPUAccounting` / `MemoryAccounting` / `TasksAccounting` / `NoNewPrivileges` 等 guardrail，且 service `inactive`、`MainPID=0`、未 enabled / 未 masked 時，才可保留 drain config / binary / unit，並輸出 `CONTROLLED_DRAIN_STAGING_ALLOWED=1` 與 textfile metric。此 staging 規則不得讀 token、不得讀 `.runner` 內容、不得註冊 runner、不得啟動 service；若 registration 缺失，readiness verifier 仍必須只留下 `controlled_cd_lane_registration_missing` / `controlled_cd_lane_service_not_active` 類 blocker。若 `CONTROLLED_DRAIN_STAGING_ALLOWED=0` 且 config / binary 又被搬走，優先修 source enforcer / unit guardrail，不要手工反覆補同一組 artifact。 diff --git a/docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md b/docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md index 783ea9dfa..ea5ce67dc 100644 --- a/docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md +++ b/docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md @@ -17,7 +17,7 @@ ## L0 受控套用 1. 將 `ops/maintenance/maintenance.html` 部署到 edge host 的 `/var/www/maintenance/maintenance.html`。 -2. 在每個 public `server {}` 內 include `ops/maintenance/nginx-502-maintenance-snippet.conf` 的等價內容；AWOOOI source conf `ops/nginx/awoooi.wooo.work.conf` 與 `k8s/nginx/awoooi-prod.conf` 已固定 `proxy_intercept_errors on`、`/__awoooi-maintenance.html` 與 `X-AWOOOI-Fallback: local-maintenance`。 +2. 在每個 public `server {}` 內 include `ops/maintenance/nginx-502-maintenance-snippet.conf` 的等價內容；AWOOOI source conf `ops/nginx/awoooi.wooo.work.conf` 與 `k8s/nginx/awoooi-prod.conf` 已固定 `proxy_intercept_errors on`、`error_page 502 503 504 =503 /__awoooi-maintenance.html` 與 `X-AWOOOI-Fallback: local-maintenance`。所有會 `proxy_pass` 到 AWOOOI API / Web 的 `location` 也必須明確帶同一組 `proxy_intercept_errors` / `error_page`，不得只依賴 server 層繼承。 3. `nginx -t` 通過後才 reload Nginx。 4. 用壞 upstream staging vhost 或暫存 upstream 驗證 `X-AWOOOI-Fallback: local-maintenance`，不得用 production DB / app restart 當測試手段。 @@ -37,7 +37,7 @@ L1 active 條件： ## 驗證 -- L0：對測試 vhost 讀回 `200` 或 `503` 的維護頁 HTML，且 header `X-AWOOOI-Fallback=local-maintenance` 存在。 +- L0：對測試 vhost 讀回 `503` 的維護頁 HTML，且 header `X-AWOOOI-Fallback=local-maintenance` 與 `Retry-After` 存在；probe 若讀到 `502` / `503` / `504` 但同時有 fallback header/body，必須分類為 maintenance fallback，不得算 raw 5xx。 - L1：從外部網路讀回靜態維護頁，且不經過 99 / 110 / 188 / 120 / 121 / 112。 - Recovery：主要 upstream 連續 2 分鐘健康後切回正式 origin；Telegram 發送 recovery 通知，並在 cold-start scorecard 留下 readback。 @@ -73,6 +73,15 @@ python3 scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py \ - Source verifier：`scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py` 必須確認上述 source 不可退回 `/50x.html`。 - Runtime verifier：在 edge host 用 staging vhost / `nginx -t` / header readback 證明 `X-AWOOOI-Fallback=local-maintenance`；通過前不得宣稱 production runtime 已接管 raw 502。 +## 2026-07-03 API / SLO cutover raw 502 hardening + +Gitea CD `#4517` 在 deploy marker 切換期間再次讀到 production API / SLO route 短暫 `502`，且沒有 maintenance body / fallback header。修正口徑： + +- `error_page` 必須使用 `=503`，讓維護頁帶明確 `503` / `Retry-After`，而不是回傳裸 `502`。 +- AWOOOI API、SSE、WebSocket、front-end 與 static proxy location 必須各自明確宣告 `proxy_intercept_errors on` 與同一條 `error_page`。 +- `public-maintenance-fallback-probe.py` 必須把帶 `X-AWOOOI-Fallback=local-maintenance` 與維護頁 body 的 `502` / `503` / `504` 判成 fallback serving；只有缺 header/body 的 5xx 才是 `raw_5xx_without_maintenance_fallback`。 +- 驗收仍以 marker-time public route watch 為準；source verifier 通過不等於 live Nginx runtime 已 reload。 + ## 2026-07-02 API runtime artifact overlay `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 不得只依賴 committed snapshot 判斷 public maintenance fallback；它必須能讀取 `reboot-auto-recovery-slo-exporter.sh` 產出的 runtime artifact。 diff --git a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md index 8cf8572f1..13cee4e70 100644 --- a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md +++ b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md @@ -20,7 +20,7 @@ | 優先 | 狀態 | 工作項 | 最新證據 | 下一步 / 完成條件 | |------|------|--------|----------|-------------------| | P0-1 | BLOCKED_HOST_WINDOWS | 全主機 reboot auto-detection / auto-trigger / 10 分鐘恢復 SLO | 2026-07-03 03:55 production scorecard：`active_blocker_count=8`、`readiness_percent=67`、`can_claim_all_services_recovered_within_target=false`；`observed_host_count=7`、`missing_host_count=0`、`unreachable_host_count=1`、111 unreachable、99 uptime unknown。 | 先收斂 99 / Windows99 / VMware 與 111：恢復 no-secret management channel 或 console verify stdout，讀回 VMX / VM power / host uptime，再 rerun host probe + reboot-event detector；不得 reboot、不得 VM power change、不得讀 Windows 密碼。 | -| P0-2 | BLOCKED_PUBLIC_UX | Deploy / reboot 期間 public 502 維護頁與外部 fallback | 多次 deploy marker 切換時 SLO/API route 曾短暫 HTTP 502 且沒有 maintenance body / fallback header；目前 03:55 public maintenance runtime readback 為 ready、raw 5xx count `0`，但 cutover-time 502 未被 drain / watch 消除。 | 實作 source-controlled deploy drain / maintenance fallback verifier / public route watch；完成條件是 marker-time probe 不再看到 raw 502，或看到明確 maintenance fallback header/body。 | +| P0-2 | IN_PROGRESS_PUBLIC_UX | Deploy / reboot 期間 public 502 維護頁與外部 fallback | Gitea CD `#4517` marker 切換時再次讀到 API / SLO route `502` 且沒有 maintenance body / fallback header；source 已改成 location-level `proxy_intercept_errors`、`error_page 502 503 504 =503 /__awoooi-maintenance.html`，probe 也改成能把帶 fallback header/body 的 5xx 分類為 maintenance fallback。 | 推送 source hardening 後觀察 CD 與 marker-time public route watch；完成條件是 marker-time probe 不再看到 raw 502，或看到明確 maintenance fallback header/body。若 source 已部署但 live 仍裸 502，下一步是受控 edge config readback / staging vhost / `nginx -t`，不得用 app restart 當測試。 | | P0-3 | PARTIAL_GREEN_SOURCE_RUNTIME | 所有產品 / 網站版本與資料最新性 | Gitea `main=89d4d6112`；Production SLO readback 對齊 deploy marker `89d4d6112 chore(cd): deploy 17ba08c [skip ci]`；Stock freshness `status=ok`、`latest_trading_date=2026-07-02`、blockers `[]`。AWOOOI health HTTP 200 但整體 `degraded`，SignOz / local Ollama 仍需列為 runtime degraded evidence。 | 將 source SHA / deploy marker / runtime endpoint / public route watch 固定進 scorecard；完成條件是每個 public product 都有 source、deploy、runtime、freshness 四層 readback，且 degraded components 有 owner lane。 | | P0-4 | BLOCKED_WINDOWS99_AUTOSTART | 192.168.0.99 VMware 自動啟動與 VM guest 111 / 188 / 120 / 121 / 112 | Scorecard：`windows99_update_no_auto_reboot_ready=true`、`windows99_vmware_verify_ready=false`、VMX missing alias `111`、powered off aliases `111/112/120/121/188`、WinRM unavailable、SSH BatchMode permission denied、RDP / Hyper-V console channel reachable。 | 只用 no-secret console / management collector 取得 `windows99-vmware-autostart.ps1 -Mode Verify` stdout，先以 `validate-windows99-console-verify-artifact.py` 轉成 normalized artifact，再重跑 SLO scorecard；完成條件是 VMX config ready、guest power ready、99 uptime known、all required host reachable。 | | P0-5 | PARTIAL_GREEN_BACKUP_MONITORING | Gitea / 主機 / DB / 網站 / 服務 / 套件 / 工具 / log 備份監控告警 | Gitea repo bundle readback ready：expected `12`、rows `12`、missing `0`、failed `0`、sample restore dry-run ok；backup core green。這只證明 repo bundle / core backup，不等於 Gitea full dump、DB/settings/issues/packages/LFS、所有工具與 log 全量備份監控完成。 | 補齊 backup health textfile / Prometheus / Telegram receipt matrix：每個 backup scope 必須有 fresh、failed、age、restore-drill、alert-receipt；完成條件是沒有「不知道備份有沒有跑」的盲點。 | diff --git a/k8s/nginx/awoooi-prod.conf b/k8s/nginx/awoooi-prod.conf index 5dfef1675..32abf3f3b 100644 --- a/k8s/nginx/awoooi-prod.conf +++ b/k8s/nginx/awoooi-prod.conf @@ -52,6 +52,8 @@ server { # ============================================ location ~ ^/api/v1/(agent|dashboard)/stream { proxy_pass http://awoooi_prod_api; + proxy_intercept_errors on; + error_page 502 503 504 =503 /__awoooi-maintenance.html; # 禁用緩衝 (AI 打字機效果零延遲) proxy_buffering off; @@ -77,6 +79,8 @@ server { # ============================================ location /api/sentry-tunnel { proxy_pass http://awoooi_prod_web; + proxy_intercept_errors on; + error_page 502 503 504 =503 /__awoooi-maintenance.html; proxy_http_version 1.1; proxy_set_header Connection "keep-alive"; @@ -89,6 +93,8 @@ server { # ============================================ location /api/ { proxy_pass http://awoooi_prod_api; + proxy_intercept_errors on; + error_page 502 503 504 =503 /__awoooi-maintenance.html; proxy_http_version 1.1; proxy_set_header Connection "keep-alive"; @@ -102,6 +108,8 @@ server { # ============================================ location /api/health { proxy_pass http://awoooi_prod_api/health; + proxy_intercept_errors on; + error_page 502 503 504 =503 /__awoooi-maintenance.html; proxy_read_timeout 5s; # 允許監控系統存取 @@ -114,6 +122,8 @@ server { # ============================================ location / { proxy_pass http://awoooi_prod_web; + proxy_intercept_errors on; + error_page 502 503 504 =503 /__awoooi-maintenance.html; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; @@ -122,6 +132,8 @@ server { # 靜態資源快取 location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2)$ { proxy_pass http://awoooi_prod_web; + proxy_intercept_errors on; + error_page 502 503 504 =503 /__awoooi-maintenance.html; expires 7d; add_header Cache-Control "public, immutable"; } @@ -130,7 +142,7 @@ server { # ============================================ # Public maintenance fallback # ============================================ - error_page 502 503 504 /__awoooi-maintenance.html; + error_page 502 503 504 =503 /__awoooi-maintenance.html; location = /__awoooi-maintenance.html { internal; root /var/www/maintenance; diff --git a/ops/maintenance/nginx-502-maintenance-snippet.conf b/ops/maintenance/nginx-502-maintenance-snippet.conf index 96066d282..78ac476f5 100644 --- a/ops/maintenance/nginx-502-maintenance-snippet.conf +++ b/ops/maintenance/nginx-502-maintenance-snippet.conf @@ -5,7 +5,7 @@ # example /var/www/maintenance/maintenance.html on the edge Nginx host. proxy_intercept_errors on; -error_page 502 503 504 /__awoooi-maintenance.html; +error_page 502 503 504 =503 /__awoooi-maintenance.html; location = /__awoooi-maintenance.html { internal; diff --git a/ops/nginx/awoooi.wooo.work.conf b/ops/nginx/awoooi.wooo.work.conf index c390401c4..ae17aa172 100644 --- a/ops/nginx/awoooi.wooo.work.conf +++ b/ops/nginx/awoooi.wooo.work.conf @@ -99,6 +99,8 @@ server { # 標準 REST API location /api/ { proxy_pass http://awoooi_api; + proxy_intercept_errors on; + error_page 502 503 504 =503 /__awoooi-maintenance.html; proxy_connect_timeout 60s; proxy_send_timeout 60s; proxy_read_timeout 60s; @@ -114,6 +116,8 @@ server { # /api/v1/agent/thinking - AI 思考過程串流 location ~ ^/api/v1/(dashboard/stream|agent/thinking) { proxy_pass http://awoooi_api; + proxy_intercept_errors on; + error_page 502 503 504 =503 /__awoooi-maintenance.html; # SSE 必要設定: 禁用緩衝 proxy_buffering off; @@ -138,6 +142,8 @@ server { # ------------------------------------------------------------------------- location /api/v1/ws { proxy_pass http://awoooi_api; + proxy_intercept_errors on; + error_page 502 503 504 =503 /__awoooi-maintenance.html; # WebSocket 升級 proxy_set_header Upgrade $http_upgrade; @@ -153,6 +159,8 @@ server { # ========================================================================= location / { proxy_pass http://awoooi_web; + proxy_intercept_errors on; + error_page 502 503 504 =503 /__awoooi-maintenance.html; proxy_connect_timeout 60s; proxy_send_timeout 60s; proxy_read_timeout 60s; @@ -166,6 +174,8 @@ server { # ------------------------------------------------------------------------- location /_next/static/ { proxy_pass http://awoooi_web; + proxy_intercept_errors on; + error_page 502 503 504 =503 /__awoooi-maintenance.html; proxy_cache_valid 200 365d; add_header Cache-Control "public, max-age=31536000, immutable"; } @@ -182,7 +192,7 @@ server { # ========================================================================= # Public maintenance fallback # ========================================================================= - error_page 502 503 504 /__awoooi-maintenance.html; + error_page 502 503 504 =503 /__awoooi-maintenance.html; location = /__awoooi-maintenance.html { internal; root /var/www/maintenance; diff --git a/ops/runner/test_cd_controlled_runtime_profile.py b/ops/runner/test_cd_controlled_runtime_profile.py index 9d9645d00..c45c743b8 100644 --- a/ops/runner/test_cd_controlled_runtime_profile.py +++ b/ops/runner/test_cd_controlled_runtime_profile.py @@ -92,6 +92,7 @@ def test_public_maintenance_fallback_sources_stay_on_controlled_runtime_profile( "ops/maintenance/maintenance.html)", "ops/maintenance/nginx-502-maintenance-snippet.conf)", "ops/nginx/awoooi.wooo.work.conf)", + "scripts/reboot-recovery/public-maintenance-fallback-probe.py)", "scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py)", ] for source in expected_sources: diff --git a/scripts/reboot-recovery/public-maintenance-fallback-probe.py b/scripts/reboot-recovery/public-maintenance-fallback-probe.py index 6b3fb7ca4..00427ad9c 100644 --- a/scripts/reboot-recovery/public-maintenance-fallback-probe.py +++ b/scripts/reboot-recovery/public-maintenance-fallback-probe.py @@ -83,7 +83,7 @@ def fetch_url(url: str, *, timeout: float, max_bytes: int) -> dict[str, Any]: fallback_header = headers.get("x-awoooi-fallback", "") maintenance_body_detected = any(marker in text for marker in MAINTENANCE_MARKERS) maintenance_fallback_serving = ( - status in {200, 503} + status in {200, 502, 503, 504} and fallback_header == "local-maintenance" and maintenance_body_detected ) diff --git a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py index 18d31fe23..5693eb69c 100644 --- a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py +++ b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py @@ -1,9 +1,11 @@ from __future__ import annotations +import importlib.util import json import subprocess import sys from pathlib import Path +from types import ModuleType ROOT = Path(__file__).resolve().parents[3] @@ -13,6 +15,61 @@ def read(path: str) -> str: return (ROOT / path).read_text(encoding="utf-8") +def load_public_maintenance_probe() -> ModuleType: + path = ROOT / "scripts/reboot-recovery/public-maintenance-fallback-probe.py" + spec = importlib.util.spec_from_file_location( + "awoooi_public_maintenance_fallback_probe", + path, + ) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def location_block(conf: str, marker: str) -> str: + start = conf.index(marker) + brace = conf.index("{", start) + depth = 0 + for index in range(brace, len(conf)): + char = conf[index] + if char == "{": + depth += 1 + elif char == "}": + depth -= 1 + if depth == 0: + return conf[start : index + 1] + raise AssertionError(f"unterminated nginx location block: {marker}") + + +def assert_proxy_location_has_l0_fallback(conf: str, marker: str) -> None: + block = location_block(conf, marker) + + assert "proxy_pass " in block + assert "proxy_intercept_errors on;" in block + assert "error_page 502 503 504 =503 /__awoooi-maintenance.html;" in block + + +class FakeHttpResponse: + def __init__(self, status: int, headers: dict[str, str], body: bytes) -> None: + self.status = status + self.headers = headers + self.body = body + + def __enter__(self) -> FakeHttpResponse: + return self + + def __exit__(self, *_args: object) -> None: + return None + + def getcode(self) -> int: + return self.status + + def read(self, _max_bytes: int) -> bytes: + return self.body + + def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() -> None: host_probe = read("scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh") windows99 = read("scripts/reboot-recovery/windows99-vmware-autostart.ps1") @@ -56,11 +113,13 @@ def test_reboot_p0_contract_has_maintenance_and_telegram_alerts() -> None: assert alert_name in alerts assert "notification_type: TYPE-3" in alerts assert "proxy_intercept_errors on" in snippet + assert "error_page 502 503 504 =503 /__awoooi-maintenance.html;" in snippet assert "X-AWOOOI-Fallback" in snippet assert "502" in runbook assert "L0" in runbook assert "L1" in runbook assert "raw_5xx_without_fallback" in probe + assert "status in {200, 502, 503, 504}" in probe assert "public-maintenance-fallback.json" in exporter assert "--public-maintenance-file" in exporter assert "awoooi_public_route_raw_5xx_without_maintenance_fallback" in exporter @@ -76,7 +135,7 @@ def test_reboot_p0_contract_applies_l0_fallback_to_awoooi_nginx_sources() -> Non ]: conf = read(path) assert "proxy_intercept_errors on;" in conf - assert "error_page 502 503 504 /__awoooi-maintenance.html;" in conf + assert "error_page 502 503 504 =503 /__awoooi-maintenance.html;" in conf assert "location = /__awoooi-maintenance.html" in conf assert "root /var/www/maintenance;" in conf assert "try_files /maintenance.html =503;" in conf @@ -84,6 +143,53 @@ def test_reboot_p0_contract_applies_l0_fallback_to_awoooi_nginx_sources() -> Non assert 'add_header X-AWOOOI-Fallback "local-maintenance" always;' in conf assert "/50x.html" not in conf + ops_conf = read("ops/nginx/awoooi.wooo.work.conf") + for marker in [ + "location /api/ {", + "location ~ ^/api/v1/(dashboard/stream|agent/thinking) {", + "location /api/v1/ws {", + "location / {\n proxy_pass http://awoooi_web;", + "location /_next/static/ {", + ]: + assert_proxy_location_has_l0_fallback(ops_conf, marker) + + k8s_conf = read("k8s/nginx/awoooi-prod.conf") + for marker in [ + "location ~ ^/api/v1/(agent|dashboard)/stream {", + "location /api/sentry-tunnel {", + "location /api/ {", + "location /api/health {", + "location / {\n proxy_pass http://awoooi_prod_web;", + "location ~* \\.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2)$ {", + ]: + assert_proxy_location_has_l0_fallback(k8s_conf, marker) + + +def test_public_maintenance_probe_does_not_count_fallback_5xx_as_raw( + monkeypatch, +) -> None: + probe = load_public_maintenance_probe() + + def fake_urlopen(_request, timeout: float) -> FakeHttpResponse: + assert timeout == 8.0 + return FakeHttpResponse( + 502, + {"X-AWOOOI-Fallback": "local-maintenance"}, + b"HTTP 502 / 503 / 504 fallback page", + ) + + monkeypatch.setattr(probe.urllib.request, "urlopen", fake_urlopen) + + route = probe.fetch_url( + "https://awoooi.wooo.work/api/v1/health", + timeout=8.0, + max_bytes=65536, + ) + + assert route["status"] == "maintenance_fallback_serving" + assert route["maintenance_fallback_serving"] is True + assert route["raw_5xx_without_fallback"] is False + def test_reboot_slo_scorecard_fails_closed_on_raw_public_502(tmp_path: Path) -> None: summary = tmp_path / "summary.txt"