From 84f6c6d4e306fe690d00bc0ebde5b4cf73112781 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 3 Jul 2026 04:38:49 +0800 Subject: [PATCH] fix(reboot): add edge maintenance fallback apply gate --- .gitea/workflows/cd.yaml | 3 + docs/LOGBOOK.md | 5 + docs/runbooks/FULL-STACK-COLD-START-SOP.md | 4 +- .../PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md | 33 +++ ...oot-cold-start-backup-recovery-workplan.md | 2 +- ops/nginx/awoooi.wooo.work.conf | 10 +- .../test_cd_controlled_runtime_profile.py | 1 + .../public-maintenance-edge-fallback-apply.sh | 222 ++++++++++++++++++ .../test_reboot_p0_operational_contract.py | 20 ++ 9 files changed, 293 insertions(+), 7 deletions(-) create mode 100755 scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index e4e327481..2b3fceb16 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -260,6 +260,8 @@ jobs: ;; ops/nginx/awoooi.wooo.work.conf) ;; + scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh) + ;; scripts/reboot-recovery/public-maintenance-fallback-probe.py) ;; product.awoooi.yaml) @@ -979,6 +981,7 @@ jobs: ../../scripts/reboot-recovery/install-cold-start-monitor-110.sh \ ../../scripts/reboot-recovery/momo-drive-token-source-recovery-preflight.sh \ ../../scripts/reboot-recovery/full-stack-recovery-scorecard.sh \ + ../../scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh \ ../../scripts/reboot-recovery/harbor-watchdog.sh \ ../../scripts/reboot-recovery/awoooi-startup-110.sh \ ../../scripts/reboot-recovery/diagnose-110-ssh-publickey-auth.sh \ diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 14866e12e..545237f87 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -7,6 +7,11 @@ - `scripts/reboot-recovery/public-maintenance-fallback-probe.py` 改成只要 `502` / `503` / `504` 同時帶 `X-AWOOOI-Fallback=local-maintenance` 與維護頁 body，就分類為 maintenance fallback serving；缺 header/body 才算 raw 5xx。 - `scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py` 補上 source contract：所有 AWOOOI public proxy location 必須有 L0 fallback，不得只靠 server 層繼承。 - `.gitea/workflows/cd.yaml` controlled-runtime profile 納入 `public-maintenance-fallback-probe.py`，避免維護頁修復誤落 full/B5 Docker socket lane。 +- Gitea CD `#4519` 後續驗證：deploy marker `3aca484 chore(cd): deploy a94ddd5 [skip ci]` 已推上 main，production deploy readback matched；但 marker 後 public probe `/tmp/awoooi-public-maintenance-fallback-after-4519-20260703-0433.json` 仍讀到 `https://awoooi.wooo.work/api/v1/health` HTTP `502`、fallback header 空、maintenance body false，`raw_5xx_without_fallback_count=1`。 +- live edge readback 證實 188 正在使用 `/etc/nginx/sites-enabled/awoooi.wooo.work.conf`，該 live conf 保留 VIP `192.168.0.125` 但缺 `proxy_intercept_errors` / `error_page` / fallback header，且 `/var/www/maintenance/maintenance.html` 缺失；`ollama@192.168.0.188` 可讀但無 passwordless sudo，`root@192.168.0.188` publickey denied。 +- `ops/nginx/awoooi.wooo.work.conf` 已改回對齊 live VIP `192.168.0.125:32334/32335`，避免用舊 `192.168.0.120` source 覆蓋 edge。 +- 新增 `scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh`：`--check` 做 no-secret live drift readback；`--apply` 必須先通過 passwordless sudo，會備份 live conf、安裝 maintenance file、`nginx -t`、reload、再 readback；`--rollback` 可用備份路徑回復。 +- live `--check` readback：`REMOTE_CONF_READABLE=1`、`LIVE_CONF_HAS_INTERCEPT=0`、`LIVE_CONF_HAS_ERROR_PAGE_503=0`、`LIVE_CONF_HAS_FALLBACK_HEADER=0`、`LIVE_CONF_HAS_INTERNAL_LOCATION=0`、`LIVE_CONF_HAS_API_VIP=1`、`LIVE_CONF_HAS_WEB_VIP=1`、`MAINTENANCE_FILE_PRESENT=0`、`SUDO_N_READY=0`、`EDGE_FALLBACK_READY=0`、`BLOCKER edge_public_maintenance_live_config_drift`。 **仍維持**： - 這是 source-level hardening；完成條件仍是 Gitea CD 部署後 marker-time public route watch 不再讀到 raw 502，或讀到明確 maintenance fallback header/body。 diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index 0c604be8b..b0da7bf39 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -1,6 +1,6 @@ # AWOOOI 全棧冷啟動與主機重啟 SOP -> Version: v1.104 +> Version: v1.105 > Last updated: 2026-07-03 Asia/Taipei > Scope: 99 / 110 / 111 / 112 / 120 / 121 / 188 全棧重啟恢復。112 仍是 Kali / VM guest 訊號，但 2026-06-30 全主機重啟後已納入 10 分鐘 SLO 的必要 boot / power signal；此納入不代表授權任何破壞性 runtime apply。 @@ -42,6 +42,8 @@ v1.103 Windows99 console verify artifact rule：若 99 可 ping / RDP / Hyper-V v1.104 public maintenance location-level intercept rule：deploy marker 切換或主機重啟期間，public API / SLO route 只要回 `502` 且沒有 `X-AWOOOI-Fallback=local-maintenance` 與維護頁 body，即使稍後恢復，也必須列為 P0 public UX blocker。`ops/nginx/awoooi.wooo.work.conf`、`k8s/nginx/awoooi-prod.conf` 與 `ops/maintenance/nginx-502-maintenance-snippet.conf` 必須使用 `error_page 502 503 504 =503 /__awoooi-maintenance.html;`，讓 fallback 明確回 `503` / `Retry-After`；所有 proxy 到 AWOOOI API / Web 的 `location` 必須各自宣告 `proxy_intercept_errors on;` 與同一條 `error_page`，不得只依賴 server 層繼承。`public-maintenance-fallback-probe.py` 需將帶 fallback header/body 的 `502` / `503` / `504` 判定為 maintenance fallback serving；只有缺 header/body 的 5xx 才算 `raw_5xx_without_maintenance_fallback`。完成條件仍是 marker-time public route watch 不再讀到 raw 502；source contract 通過不等於 live Nginx runtime 已套用。 +v1.105 public maintenance edge source-to-live drift rule：Gitea CD `#4519` 已部署 `a94ddd5` 並推 deploy marker `3aca484`，但 marker 後 public probe 仍讀到 `https://awoooi.wooo.work/api/v1/health` raw `502`、無 fallback header/body；稍後 route 自癒為 `200` 不得視為完成。live readback 證實 188 使用 `/etc/nginx/sites-enabled/awoooi.wooo.work.conf`，該檔仍是 VIP `192.168.0.125` 版舊配置但缺 `proxy_intercept_errors` / `error_page` / `X-AWOOOI-Fallback`，且 `/var/www/maintenance/maintenance.html` 缺失。repo `ops/nginx/awoooi.wooo.work.conf` 必須保留 VIP `192.168.0.125:32334/32335`，不得用舊 `192.168.0.120` config 覆蓋 live。`scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh --check` 是 no-secret readback；`--apply` 只能在 passwordless sudo channel 存在時執行，必須備份 live conf、安裝 maintenance file、`nginx -t` 通過、reload、再 rerun public route probe。若 `SUDO_N_READY=0`，active blocker 固定為 `privileged_edge_apply_channel_unavailable`，不得讀密碼、不得重啟 Nginx / Docker / host，也不得宣稱 P0-2 完成。 + 2026-07-02 110 control-path / Harbor recovery receipt rule：若 Gitea Harbor repair queue 仍保留 `harbor_110_remote_ssh_publickey_auth_stalled`、remote-control unavailable、jobs stale 或 historical failure，但同一輪本地證據同時證明 `wooo` command path ready、110 local Harbor `/v2/` ready、public/internal registry `/v2/` 回 `401`，則該 Gitea Harbor repair 失敗只能列為 historical queue metadata，不得再當成 current SSH blocker。必須用 `/api/v1/agents/harbor-registry-controlled-recovery-receipt` 或同等 validator 合併 `diagnose-110-ssh-publickey-auth.sh`、`recover-110-control-path-and-harbor-local.sh --check`、public Gitea queue readback 與 registry `/v2/` verifier，並把機器可讀結果寫入 `docs/operations/harbor-110-control-path-recovery-readback-2026-07-02.snapshot.json` 類型的 snapshot。2026-07-02 live receipt 顯示：public/internal registry `/v2/` 均為 `401`、latest visible CD `#4335` 為 `Success`、Gitea Harbor repair failure 已是 `historical_after_latest_cd_success=true`；active blockers 收斂為 110 controlled CD lane config / binary / registration / service guardrail、active action container pressure，以及 Gitea CD jobs head-SHA / stale readback mismatch。若 local-console output 只有 `AWOOOI_110_CONTROLLED_CD_LANE_READY` marker，non110 runner parser 不得從 110 `BLOCKER` 行推導 non110 blocker；non110 只有看到 `AWOOOI_NON110_RUNNER_READY` marker 才能列入 active blocker。 2026-07-02 110 controlled CD lane fail-closed enforcer staging rule：110 runner 壓力事故後，legacy / generic runner 仍必須 fail-closed；但 `awoooi-cd-lane-drain.service` 的非 secret staging artifact 不得再被 enforcer 無差別封回 stub。`scripts/reboot-recovery/enforce-110-runner-failclosed.sh` 只有在 `config.yaml` 符合 `capacity <= 1`、只含 `awoooi-host:host` 與 `awoooi-ubuntu:docker://192.168.0.110:5000/awoooi/ci-runner:act-22.04`、binary 是 executable ELF、systemd unit 具備 `ConditionPathExists=/home/wooo/awoooi-cd-lane-drain/data/.runner`、`CPUAccounting` / `MemoryAccounting` / `TasksAccounting` / `NoNewPrivileges` 等 guardrail，且 service `inactive`、`MainPID=0`、未 enabled / 未 masked 時，才可保留 drain config / binary / unit，並輸出 `CONTROLLED_DRAIN_STAGING_ALLOWED=1` 與 textfile metric。此 staging 規則不得讀 token、不得讀 `.runner` 內容、不得註冊 runner、不得啟動 service；若 registration 缺失，readiness verifier 仍必須只留下 `controlled_cd_lane_registration_missing` / `controlled_cd_lane_service_not_active` 類 blocker。若 `CONTROLLED_DRAIN_STAGING_ALLOWED=0` 且 config / binary 又被搬走，優先修 source enforcer / unit guardrail，不要手工反覆補同一組 artifact。 diff --git a/docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md b/docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md index ea5ce67dc..b7997a0a1 100644 --- a/docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md +++ b/docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md @@ -82,6 +82,39 @@ Gitea CD `#4517` 在 deploy marker 切換期間再次讀到 production API / SLO - `public-maintenance-fallback-probe.py` 必須把帶 `X-AWOOOI-Fallback=local-maintenance` 與維護頁 body 的 `502` / `503` / `504` 判成 fallback serving；只有缺 header/body 的 5xx 才是 `raw_5xx_without_maintenance_fallback`。 - 驗收仍以 marker-time public route watch 為準；source verifier 通過不等於 live Nginx runtime 已 reload。 +## 2026-07-03 #4519 live edge drift + +`#4519` 部署 source `a94ddd5` 後，deploy marker `3aca484 chore(cd): deploy a94ddd5 [skip ci]` 已推上 Gitea main，build log 也讀到 production deploy readback matched；但 marker 後 public probe `/tmp/awoooi-public-maintenance-fallback-after-4519-20260703-0433.json` 仍回： + +- `https://awoooi.wooo.work/api/v1/health`：HTTP `502` +- `X-AWOOOI-Fallback`：空 +- maintenance body：false +- `raw_5xx_without_fallback_count=1` + +live edge readback： + +- 188 readable config：`/etc/nginx/sites-enabled/awoooi.wooo.work.conf` +- config 仍走 VIP `192.168.0.125:32334/32335` +- config 缺 `proxy_intercept_errors` / `error_page 502 503 504 =503` / fallback header +- `/var/www/maintenance/maintenance.html` 缺失 +- `ollama@192.168.0.188` 無 passwordless sudo，不能由一般 SSH channel apply + +因此 P0-2 目前 blocker 不是 app deploy，而是 live edge source-to-runtime drift。下一步固定： + +```bash +scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh --check +``` + +若 `SUDO_N_READY=1`，才能執行： + +```bash +scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh --apply +python3.11 scripts/reboot-recovery/public-maintenance-fallback-probe.py --timeout 8 \ + --output /tmp/public-maintenance-fallback-after-edge-apply.json +``` + +若 `SUDO_N_READY=0`，不得讀密碼、不得重啟 Nginx、不得用 app restart 當測試；需走 console/root-controlled apply 或維護窗口。 + ## 2026-07-02 API runtime artifact overlay `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 不得只依賴 committed snapshot 判斷 public maintenance fallback；它必須能讀取 `reboot-auto-recovery-slo-exporter.sh` 產出的 runtime artifact。 diff --git a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md index 13cee4e70..d88f11570 100644 --- a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md +++ b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md @@ -20,7 +20,7 @@ | 優先 | 狀態 | 工作項 | 最新證據 | 下一步 / 完成條件 | |------|------|--------|----------|-------------------| | P0-1 | BLOCKED_HOST_WINDOWS | 全主機 reboot auto-detection / auto-trigger / 10 分鐘恢復 SLO | 2026-07-03 03:55 production scorecard：`active_blocker_count=8`、`readiness_percent=67`、`can_claim_all_services_recovered_within_target=false`；`observed_host_count=7`、`missing_host_count=0`、`unreachable_host_count=1`、111 unreachable、99 uptime unknown。 | 先收斂 99 / Windows99 / VMware 與 111：恢復 no-secret management channel 或 console verify stdout，讀回 VMX / VM power / host uptime，再 rerun host probe + reboot-event detector；不得 reboot、不得 VM power change、不得讀 Windows 密碼。 | -| P0-2 | IN_PROGRESS_PUBLIC_UX | Deploy / reboot 期間 public 502 維護頁與外部 fallback | Gitea CD `#4517` marker 切換時再次讀到 API / SLO route `502` 且沒有 maintenance body / fallback header；source 已改成 location-level `proxy_intercept_errors`、`error_page 502 503 504 =503 /__awoooi-maintenance.html`，probe 也改成能把帶 fallback header/body 的 5xx 分類為 maintenance fallback。 | 推送 source hardening 後觀察 CD 與 marker-time public route watch；完成條件是 marker-time probe 不再看到 raw 502，或看到明確 maintenance fallback header/body。若 source 已部署但 live 仍裸 502，下一步是受控 edge config readback / staging vhost / `nginx -t`，不得用 app restart 當測試。 | +| P0-2 | BLOCKED_EDGE_PRIVILEGED_APPLY | Deploy / reboot 期間 public 502 維護頁與外部 fallback | Gitea CD `#4519` 已推 deploy marker `3aca484 -> a94ddd5`，但 marker 後 public probe 仍讀到 `https://awoooi.wooo.work/api/v1/health` raw `502`、fallback header/body 空；live 188 `/etc/nginx/sites-enabled/awoooi.wooo.work.conf` 缺 maintenance fallback，`/var/www/maintenance/maintenance.html` 缺失，`ollama@188` 無 passwordless sudo。 | 先跑 `scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh --check` 留 drift receipt；具備 privileged channel 後執行 `--apply`，要求 backup、`nginx -t`、reload、public route probe 全綠。不得讀密碼、不得用 app restart 掩蓋 edge fallback drift。 | | P0-3 | PARTIAL_GREEN_SOURCE_RUNTIME | 所有產品 / 網站版本與資料最新性 | Gitea `main=89d4d6112`；Production SLO readback 對齊 deploy marker `89d4d6112 chore(cd): deploy 17ba08c [skip ci]`；Stock freshness `status=ok`、`latest_trading_date=2026-07-02`、blockers `[]`。AWOOOI health HTTP 200 但整體 `degraded`，SignOz / local Ollama 仍需列為 runtime degraded evidence。 | 將 source SHA / deploy marker / runtime endpoint / public route watch 固定進 scorecard；完成條件是每個 public product 都有 source、deploy、runtime、freshness 四層 readback，且 degraded components 有 owner lane。 | | P0-4 | BLOCKED_WINDOWS99_AUTOSTART | 192.168.0.99 VMware 自動啟動與 VM guest 111 / 188 / 120 / 121 / 112 | Scorecard：`windows99_update_no_auto_reboot_ready=true`、`windows99_vmware_verify_ready=false`、VMX missing alias `111`、powered off aliases `111/112/120/121/188`、WinRM unavailable、SSH BatchMode permission denied、RDP / Hyper-V console channel reachable。 | 只用 no-secret console / management collector 取得 `windows99-vmware-autostart.ps1 -Mode Verify` stdout，先以 `validate-windows99-console-verify-artifact.py` 轉成 normalized artifact，再重跑 SLO scorecard；完成條件是 VMX config ready、guest power ready、99 uptime known、all required host reachable。 | | P0-5 | PARTIAL_GREEN_BACKUP_MONITORING | Gitea / 主機 / DB / 網站 / 服務 / 套件 / 工具 / log 備份監控告警 | Gitea repo bundle readback ready：expected `12`、rows `12`、missing `0`、failed `0`、sample restore dry-run ok；backup core green。這只證明 repo bundle / core backup，不等於 Gitea full dump、DB/settings/issues/packages/LFS、所有工具與 log 全量備份監控完成。 | 補齊 backup health textfile / Prometheus / Telegram receipt matrix：每個 backup scope 必須有 fresh、failed、age、restore-drill、alert-receipt；完成條件是沒有「不知道備份有沒有跑」的盲點。 | diff --git a/ops/nginx/awoooi.wooo.work.conf b/ops/nginx/awoooi.wooo.work.conf index ae17aa172..4961cbbdc 100644 --- a/ops/nginx/awoooi.wooo.work.conf +++ b/ops/nginx/awoooi.wooo.work.conf @@ -7,22 +7,22 @@ # 日期: 2026-03-21 # # 後端架構: -# - 前端 (Next.js): K3s NodePort 192.168.0.120:32335 -# - API (FastAPI): K3s NodePort 192.168.0.120:32334 +# - 前端 (Next.js): K3s VIP 192.168.0.125:32335 +# - API (FastAPI): K3s VIP 192.168.0.125:32334 # # ⚠️ 警告: 絕對不允許出現 Legacy 系統的 Port (31234/31235) # ============================================================================= # ----------------------------------------------------------------------------- -# Upstream 定義 (K3s NodePort) +# Upstream 定義 (K3s VIP) # ----------------------------------------------------------------------------- upstream awoooi_web { - server 192.168.0.120:32335; + server 192.168.0.125:32335; keepalive 32; } upstream awoooi_api { - server 192.168.0.120:32334; + server 192.168.0.125:32334; keepalive 64; } diff --git a/ops/runner/test_cd_controlled_runtime_profile.py b/ops/runner/test_cd_controlled_runtime_profile.py index c45c743b8..e7c10644b 100644 --- a/ops/runner/test_cd_controlled_runtime_profile.py +++ b/ops/runner/test_cd_controlled_runtime_profile.py @@ -92,6 +92,7 @@ def test_public_maintenance_fallback_sources_stay_on_controlled_runtime_profile( "ops/maintenance/maintenance.html)", "ops/maintenance/nginx-502-maintenance-snippet.conf)", "ops/nginx/awoooi.wooo.work.conf)", + "scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh)", "scripts/reboot-recovery/public-maintenance-fallback-probe.py)", "scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py)", ] diff --git a/scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh b/scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh new file mode 100755 index 000000000..0064ab8f2 --- /dev/null +++ b/scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh @@ -0,0 +1,222 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +MODE="check" +ROLLBACK_CONF="" + +EDGE_HOST="${EDGE_HOST:-192.168.0.188}" +EDGE_USER="${EDGE_USER:-ollama}" +EDGE_TARGET="${EDGE_USER}@${EDGE_HOST}" +REMOTE_CONF="${REMOTE_CONF:-/etc/nginx/sites-enabled/awoooi.wooo.work.conf}" +REMOTE_MAINT_DIR="${REMOTE_MAINT_DIR:-/var/www/maintenance}" +REMOTE_MAINT_FILE="${REMOTE_MAINT_FILE:-${REMOTE_MAINT_DIR}/maintenance.html}" +REMOTE_BACKUP_DIR="${REMOTE_BACKUP_DIR:-/home/ollama/awoooi-edge-maintenance-backups}" +LOCAL_CONF="${LOCAL_CONF:-${ROOT_DIR}/ops/nginx/awoooi.wooo.work.conf}" +LOCAL_MAINT_FILE="${LOCAL_MAINT_FILE:-${ROOT_DIR}/ops/maintenance/maintenance.html}" + +SSH_OPTS=( + -o BatchMode=yes + -o ConnectTimeout="${SSH_CONNECT_TIMEOUT_SECONDS:-8}" + -o StrictHostKeyChecking=accept-new +) + +usage() { + cat <<'USAGE' +Usage: + public-maintenance-edge-fallback-apply.sh --check + public-maintenance-edge-fallback-apply.sh --apply + public-maintenance-edge-fallback-apply.sh --rollback + +No secret collection is allowed. --apply requires an existing passwordless sudo +channel on the edge host; otherwise it exits fail-closed before writing. +USAGE +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --check) + MODE="check" + ;; + --apply) + MODE="apply" + ;; + --rollback) + MODE="rollback" + shift + ROLLBACK_CONF="${1:-}" + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "UNKNOWN_ARG=$1" >&2 + usage >&2 + exit 64 + ;; + esac + shift +done + +if [ ! -f "$LOCAL_CONF" ]; then + echo "BLOCKER local_nginx_conf_missing path=$LOCAL_CONF" + exit 66 +fi +if [ ! -f "$LOCAL_MAINT_FILE" ]; then + echo "BLOCKER local_maintenance_file_missing path=$LOCAL_MAINT_FILE" + exit 66 +fi + +remote_check() { + ssh "${SSH_OPTS[@]}" "$EDGE_TARGET" \ + "REMOTE_CONF='$REMOTE_CONF' REMOTE_MAINT_FILE='$REMOTE_MAINT_FILE' bash -s" <<'REMOTE' +set -eu +echo "AWOOOI_PUBLIC_MAINTENANCE_EDGE_FALLBACK=1" +echo "MODE=check" +echo "HOSTNAME=$(hostname 2>/dev/null || true)" +echo "WHOAMI=$(whoami 2>/dev/null || true)" +echo "REMOTE_CONF=${REMOTE_CONF}" +if [ -r "$REMOTE_CONF" ]; then + echo "REMOTE_CONF_READABLE=1" +else + echo "REMOTE_CONF_READABLE=0" +fi +if [ -r "$REMOTE_CONF" ]; then + grep -q 'proxy_intercept_errors on;' "$REMOTE_CONF" && echo "LIVE_CONF_HAS_INTERCEPT=1" || echo "LIVE_CONF_HAS_INTERCEPT=0" + grep -q 'error_page 502 503 504 =503 /__awoooi-maintenance.html;' "$REMOTE_CONF" && echo "LIVE_CONF_HAS_ERROR_PAGE_503=1" || echo "LIVE_CONF_HAS_ERROR_PAGE_503=0" + grep -q 'X-AWOOOI-Fallback "local-maintenance"' "$REMOTE_CONF" && echo "LIVE_CONF_HAS_FALLBACK_HEADER=1" || echo "LIVE_CONF_HAS_FALLBACK_HEADER=0" + grep -q 'location = /__awoooi-maintenance.html' "$REMOTE_CONF" && echo "LIVE_CONF_HAS_INTERNAL_LOCATION=1" || echo "LIVE_CONF_HAS_INTERNAL_LOCATION=0" + grep -q '192.168.0.125:32334' "$REMOTE_CONF" && echo "LIVE_CONF_HAS_API_VIP=1" || echo "LIVE_CONF_HAS_API_VIP=0" + grep -q '192.168.0.125:32335' "$REMOTE_CONF" && echo "LIVE_CONF_HAS_WEB_VIP=1" || echo "LIVE_CONF_HAS_WEB_VIP=0" +else + echo "LIVE_CONF_HAS_INTERCEPT=0" + echo "LIVE_CONF_HAS_ERROR_PAGE_503=0" + echo "LIVE_CONF_HAS_FALLBACK_HEADER=0" + echo "LIVE_CONF_HAS_INTERNAL_LOCATION=0" + echo "LIVE_CONF_HAS_API_VIP=0" + echo "LIVE_CONF_HAS_WEB_VIP=0" +fi +if [ -r "$REMOTE_MAINT_FILE" ]; then + echo "MAINTENANCE_FILE_PRESENT=1" + grep -q 'AWOOOI 維護模式' "$REMOTE_MAINT_FILE" && echo "MAINTENANCE_FILE_HAS_MARKER=1" || echo "MAINTENANCE_FILE_HAS_MARKER=0" +else + echo "MAINTENANCE_FILE_PRESENT=0" + echo "MAINTENANCE_FILE_HAS_MARKER=0" +fi +if sudo -n true >/dev/null 2>&1; then + echo "SUDO_N_READY=1" +else + echo "SUDO_N_READY=0" +fi +REMOTE_CONF_READABLE=$(if [ -r "$REMOTE_CONF" ]; then echo 1; else echo 0; fi) +LIVE_CONF_HAS_INTERCEPT=$(if [ -r "$REMOTE_CONF" ] && grep -q 'proxy_intercept_errors on;' "$REMOTE_CONF"; then echo 1; else echo 0; fi) +LIVE_CONF_HAS_ERROR_PAGE_503=$(if [ -r "$REMOTE_CONF" ] && grep -q 'error_page 502 503 504 =503 /__awoooi-maintenance.html;' "$REMOTE_CONF"; then echo 1; else echo 0; fi) +LIVE_CONF_HAS_FALLBACK_HEADER=$(if [ -r "$REMOTE_CONF" ] && grep -q 'X-AWOOOI-Fallback "local-maintenance"' "$REMOTE_CONF"; then echo 1; else echo 0; fi) +LIVE_CONF_HAS_INTERNAL_LOCATION=$(if [ -r "$REMOTE_CONF" ] && grep -q 'location = /__awoooi-maintenance.html' "$REMOTE_CONF"; then echo 1; else echo 0; fi) +LIVE_CONF_HAS_API_VIP=$(if [ -r "$REMOTE_CONF" ] && grep -q '192.168.0.125:32334' "$REMOTE_CONF"; then echo 1; else echo 0; fi) +LIVE_CONF_HAS_WEB_VIP=$(if [ -r "$REMOTE_CONF" ] && grep -q '192.168.0.125:32335' "$REMOTE_CONF"; then echo 1; else echo 0; fi) +MAINTENANCE_FILE_PRESENT=$(if [ -r "$REMOTE_MAINT_FILE" ]; then echo 1; else echo 0; fi) +MAINTENANCE_FILE_HAS_MARKER=$(if [ -r "$REMOTE_MAINT_FILE" ] && grep -q 'AWOOOI 維護模式' "$REMOTE_MAINT_FILE"; then echo 1; else echo 0; fi) +if [ "$REMOTE_CONF_READABLE" = 1 ] \ + && [ "$LIVE_CONF_HAS_INTERCEPT" = 1 ] \ + && [ "$LIVE_CONF_HAS_ERROR_PAGE_503" = 1 ] \ + && [ "$LIVE_CONF_HAS_FALLBACK_HEADER" = 1 ] \ + && [ "$LIVE_CONF_HAS_INTERNAL_LOCATION" = 1 ] \ + && [ "$LIVE_CONF_HAS_API_VIP" = 1 ] \ + && [ "$LIVE_CONF_HAS_WEB_VIP" = 1 ] \ + && [ "$MAINTENANCE_FILE_PRESENT" = 1 ] \ + && [ "$MAINTENANCE_FILE_HAS_MARKER" = 1 ]; then + echo "EDGE_FALLBACK_READY=1" +else + echo "EDGE_FALLBACK_READY=0" + echo "BLOCKER edge_public_maintenance_live_config_drift" +fi +REMOTE +} + +remote_apply() { + local tmp_conf="/tmp/awoooi.wooo.work.conf.$$" + local tmp_maint="/tmp/awoooi-maintenance.$$" + scp -q "${SSH_OPTS[@]}" "$LOCAL_CONF" "${EDGE_TARGET}:${tmp_conf}" + scp -q "${SSH_OPTS[@]}" "$LOCAL_MAINT_FILE" "${EDGE_TARGET}:${tmp_maint}" + ssh "${SSH_OPTS[@]}" "$EDGE_TARGET" \ + "REMOTE_CONF='$REMOTE_CONF' REMOTE_MAINT_DIR='$REMOTE_MAINT_DIR' REMOTE_MAINT_FILE='$REMOTE_MAINT_FILE' REMOTE_BACKUP_DIR='$REMOTE_BACKUP_DIR' TMP_CONF='$tmp_conf' TMP_MAINT='$tmp_maint' bash -s" <<'REMOTE' +set -eu +echo "AWOOOI_PUBLIC_MAINTENANCE_EDGE_FALLBACK=1" +echo "MODE=apply" +if ! sudo -n true >/dev/null 2>&1; then + echo "SUDO_N_READY=0" + echo "BLOCKER privileged_edge_apply_channel_unavailable" + rm -f "$TMP_CONF" "$TMP_MAINT" + exit 75 +fi +echo "SUDO_N_READY=1" +mkdir -p "$REMOTE_BACKUP_DIR" +backup="${REMOTE_BACKUP_DIR}/awoooi.wooo.work.conf.before-maintenance.$(date +%Y%m%d%H%M%S)" +if [ -f "$REMOTE_CONF" ]; then + sudo cp "$REMOTE_CONF" "$backup" +else + : >"${backup}.missing" +fi +sudo mkdir -p "$REMOTE_MAINT_DIR" +sudo install -m 0644 "$TMP_CONF" "$REMOTE_CONF" +sudo install -m 0644 "$TMP_MAINT" "$REMOTE_MAINT_FILE" +rm -f "$TMP_CONF" "$TMP_MAINT" +if ! sudo nginx -t; then + echo "BLOCKER nginx_test_failed_restoring_backup" + if [ -f "$backup" ]; then + sudo cp "$backup" "$REMOTE_CONF" + sudo nginx -t || true + fi + exit 76 +fi +sudo systemctl reload nginx +echo "BACKUP_CONF=$backup" +echo "NGINX_RELOAD_DONE=1" +REMOTE + remote_check +} + +remote_rollback() { + if [ -z "$ROLLBACK_CONF" ]; then + echo "BLOCKER rollback_conf_required" + exit 64 + fi + ssh "${SSH_OPTS[@]}" "$EDGE_TARGET" \ + "REMOTE_CONF='$REMOTE_CONF' ROLLBACK_CONF='$ROLLBACK_CONF' bash -s" <<'REMOTE' +set -eu +echo "AWOOOI_PUBLIC_MAINTENANCE_EDGE_FALLBACK=1" +echo "MODE=rollback" +if ! sudo -n true >/dev/null 2>&1; then + echo "SUDO_N_READY=0" + echo "BLOCKER privileged_edge_apply_channel_unavailable" + exit 75 +fi +if [ ! -f "$ROLLBACK_CONF" ]; then + echo "BLOCKER rollback_conf_not_found path=$ROLLBACK_CONF" + exit 66 +fi +sudo cp "$ROLLBACK_CONF" "$REMOTE_CONF" +sudo nginx -t +sudo systemctl reload nginx +echo "ROLLBACK_DONE=1" +REMOTE + remote_check +} + +case "$MODE" in + check) + remote_check + ;; + apply) + remote_apply + ;; + rollback) + remote_rollback + ;; + *) + echo "UNKNOWN_MODE=$MODE" >&2 + exit 64 + ;; +esac diff --git a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py index 5693eb69c..a9bd83b0b 100644 --- a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py +++ b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py @@ -144,6 +144,8 @@ def test_reboot_p0_contract_applies_l0_fallback_to_awoooi_nginx_sources() -> Non assert "/50x.html" not in conf ops_conf = read("ops/nginx/awoooi.wooo.work.conf") + assert "192.168.0.125:32334" in ops_conf + assert "192.168.0.125:32335" in ops_conf for marker in [ "location /api/ {", "location ~ ^/api/v1/(dashboard/stream|agent/thinking) {", @@ -191,6 +193,24 @@ def test_public_maintenance_probe_does_not_count_fallback_5xx_as_raw( assert route["raw_5xx_without_fallback"] is False +def test_public_maintenance_edge_apply_script_is_guarded_and_reversible() -> None: + script = read("scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh") + + assert "MODE=\"check\"" in script + assert "MODE=\"apply\"" in script + assert "MODE=\"rollback\"" in script + assert "sudo -n true" in script + assert "BLOCKER privileged_edge_apply_channel_unavailable" in script + assert "nginx -t" in script + assert "systemctl reload nginx" in script + assert "BACKUP_CONF=" in script + assert "EDGE_FALLBACK_READY=1" in script + assert "EDGE_FALLBACK_READY=0" in script + assert "raw_secret" not in script.lower() + assert "docker restart" not in script + assert "reboot " not in script + + def test_reboot_slo_scorecard_fails_closed_on_raw_public_502(tmp_path: Path) -> None: summary = tmp_path / "summary.txt" summary.write_text(