From 1c3c7279acaf33ed387a2ee876f48f967135f2ff Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 30 Jun 2026 22:37:33 +0800 Subject: [PATCH] fix(recovery): bound 188 startup clawbot rebuild --- docs/LOGBOOK.md | 29 +++++++++++++++ docs/runbooks/FULL-STACK-COLD-START-SOP.md | 27 ++++++++++++++ .../reboot-recovery/awoooi-startup.service | 4 +-- scripts/reboot-recovery/awoooi-startup.sh | 35 ++++++++++++++++--- .../tests/test_188_host_hygiene_checklist.py | 20 +++++++++++ 5 files changed, 108 insertions(+), 7 deletions(-) diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index a11801309..e44e6075f 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -50778,3 +50778,32 @@ production browser smoke: **下一步**： - commit / push 後等待 Gitea CD；若仍卡 Harbor/110，P0 主線下一步是 110 local console / root shell 執行 recovery package check-mode，讀回 `awoooi-host` controlled lane verifier、Harbor queue、registry `/v2/`、deploy marker，並把 post-apply verifier receipt 寫回 AI Loop queue / KM / RAG / MCP / PlayBook。 + +## 2026-06-30 — 22:42 188 startup timeout / ClawBot rebuild SLO 修復 + +**完成內容**： +- 讀回 22:30 live post-reboot readiness：`SERVICE_GREEN=0`、`PRODUCT_DATA_GREEN=0`、`BACKUP_CORE_GREEN=0`、`HOST_188_SERVICE_GREEN=0`、`OVERALL_DECLARATION=SERVICE_BLOCKED`；Stock freshness / ingestion 仍是 `postgres_not_ready`。 +- 產生最新 SLO scorecard `/tmp/awoooi-reboot-slo-live-20260630-2231-scorecard.json`：`status=blocked_reboot_auto_recovery_slo_not_ready`、`can_claim_all_services_recovered_within_target=false`；111 unreachable，99 boot / VMware autostart unknown，110/112/120/121/188 皆已超過 10 分鐘窗口。 +- 讀回 188 host hygiene：`awoooi-startup.service` 因 300 秒 timeout 失敗，卡在 ClawBot `docker compose build --no-cache`；但 live `clawbot` / `clawbot-redis` 容器已 healthy，證明舊 startup path 會把單一 rebuild 誤放大成整機 post-reboot failed。 +- 修正 `scripts/reboot-recovery/awoooi-startup.sh`：ClawBot startup 改成 bounded `docker compose up -d`，rebuild 預設 opt-in (`CLAWBOT_STARTUP_REBUILD_ALLOWED=0`) 且有 timeout；移除預設 `--no-cache` rebuild。 +- 修正 `scripts/reboot-recovery/awoooi-startup.service`：`TimeoutStartSec=600`，對齊 10 分鐘 SLO，但不允許單一服務 rebuild 佔滿整個恢復窗口。 +- 補 `scripts/reboot-recovery/tests/test_188_host_hygiene_checklist.py`，鎖住 bounded / opt-in rebuild 與 600 秒 unit timeout。 + +**本地驗證結果**： +- `bash -n scripts/reboot-recovery/awoooi-startup.sh scripts/reboot-recovery/awoooi-startup.service scripts/reboot-recovery/188-host-hygiene-maintenance-checklist.sh`：通過。 +- `DATABASE_URL=sqlite+aiosqlite:////tmp/awoooi-codex-api-test.db PYTHONPATH=apps/api python3.11 -m pytest scripts/reboot-recovery/tests/test_188_host_hygiene_checklist.py scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py ops/runner/test_cd_controlled_runtime_profile.py -q`：`42 passed`。 +- `python3.11 ops/runner/guard-gitea-runner-pressure.py --root .`：通過，`auto_branch_events_on_110=0`、`generic_runner_labels=0`。 +- `node scripts/ci/check-gitea-step-env-secrets.js .gitea/workflows/cd.yaml .gitea/workflows/harbor-110-local-repair.yaml`：通過。 + +**live truth**： +- `https://registry.wooo.work/v2/`、`http://192.168.0.110:5000/v2/`、`https://harbor.wooo.work/api/v2.0/health` 仍回 502。 +- public Gitea queue 仍是 `blocked_harbor_110_repair_no_matching_runner`；CD `#4100` running，Harbor repair `#4099` Waiting，no matching label `awoooi-host`。 +- Production delivery / SLO API 仍含 2026-06-29 舊資料，不可作為本輪恢復證據；本輪以 live `/tmp/awoooi-reboot-slo-live-20260630-2231-scorecard.json` 為準。 + +**仍維持**： +- 沒有讀 secret / token / `.env` / raw sessions / SQLite / auth；沒有讀 `.runner` 內容。 +- 沒有使用 GitHub / gh / GitHub API / GitHub Actions。 +- 沒有重啟主機，沒有 Docker / Nginx / K3s / DB restart，沒有 workflow_dispatch，沒有 DB write / restore / prune。 + +**下一步**： +- commit / push 後等待 Gitea CD；若 runtime 仍卡 Harbor / 110，P0 主線仍是恢復 110 `awoooi-host` controlled lane 或 110 local recovery package，再重讀 registry `/v2/`、full-stack cold-start、Stock freshness、backup core 與 SLO scorecard。 diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index 7847500c9..7ff0fe459 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -2334,3 +2334,30 @@ All must be true: - `stockplatform-shared-ui-monitor.timer` / service source-of-truth 仍需清理或重建；2026-06-12 只停用 stale timer 以解除 host degraded。 - 111 local Ollama fallback 目前不可達；production provider 由 GCP-A / GCP-B 承接，但 111 恢復應另列 AI provider resilience 工作。 - 本 SOP v1.5 新增內容已用繁體中文補強；舊章節仍有英文段落，後續 runbook hygiene 應分批翻譯，不要在事故 P0 中混入大規模格式重排。 + +### 16.1 188 startup / ClawBot rebuild 不得阻塞 SLO + +2026-06-30 主機重啟後，188 `awoooi-startup.service` 在 `TimeoutStartSec=300` 內被 systemd 砍掉；live journal 顯示卡在 ClawBot `docker compose build --no-cache`，但事後 `clawbot` / `clawbot-redis` 容器已 healthy。此類情況不可再解讀為需要盲目 restart / rebuild；正確做法是讓 startup oneshot 完成，將 ClawBot 健康交給 post-reboot verifier 判斷。 + +新的 source-of-truth： + +- `scripts/reboot-recovery/awoooi-startup.sh` +- `scripts/reboot-recovery/awoooi-startup.service` +- `scripts/reboot-recovery/tests/test_188_host_hygiene_checklist.py` + +規則： + +- ClawBot startup 只允許 bounded `docker compose up -d`，預設確認窗口 `CLAWBOT_UP_TIMEOUT_SECONDS=90`。 +- ClawBot image rebuild 預設停用：`CLAWBOT_STARTUP_REBUILD_ALLOWED=0`。 +- 只有在 maintenance / controlled recovery 明確設定 `CLAWBOT_STARTUP_REBUILD_ALLOWED=1` 時，才允許 bounded rebuild，且 `CLAWBOT_BUILD_TIMEOUT_SECONDS=120`。 +- 不得在 reboot startup path 預設執行 `docker compose build --no-cache`。 +- `awoooi-startup.service` timeout 對齊 10 分鐘 SLO：`TimeoutStartSec=600`。 +- 若 ClawBot 未確認 running，startup script 只留下 warning；`post-reboot-readiness-summary.sh` / `full-stack-cold-start-check.sh` 必須負責把它標成服務 blocker。 + +驗證命令： + +```bash +bash -n scripts/reboot-recovery/awoooi-startup.sh +DATABASE_URL=sqlite+aiosqlite:////tmp/awoooi-codex-api-test.db PYTHONPATH=apps/api \ + python3.11 -m pytest scripts/reboot-recovery/tests/test_188_host_hygiene_checklist.py -q +``` diff --git a/scripts/reboot-recovery/awoooi-startup.service b/scripts/reboot-recovery/awoooi-startup.service index 871cb460e..fb6e84b15 100644 --- a/scripts/reboot-recovery/awoooi-startup.service +++ b/scripts/reboot-recovery/awoooi-startup.service @@ -22,8 +22,8 @@ Wants=redis-server.service ollama.service nginx.service Type=oneshot RemainAfterExit=yes ExecStart=/usr/local/bin/awoooi-startup.sh -# 允許較長的啟動時間（修復流程可能需要時間） -TimeoutStartSec=300 +# 對齊 10 分鐘 post-reboot SLO；單一服務 rebuild 不得阻塞整個 oneshot。 +TimeoutStartSec=600 StandardOutput=journal StandardError=journal diff --git a/scripts/reboot-recovery/awoooi-startup.sh b/scripts/reboot-recovery/awoooi-startup.sh index 8b0c49a1a..03fd9d7d1 100644 --- a/scripts/reboot-recovery/awoooi-startup.sh +++ b/scripts/reboot-recovery/awoooi-startup.sh @@ -5,6 +5,8 @@ # 不在自動啟動腳本內執行 pg_resetwal，避免資料破壞被誤判成恢復。 # 2026-06-26 Codex: 允許受控 recovery container 提供 14/main runtime； # 不再因 systemd postgresql@14-main failed 而誤判活 DB 為不可用。 +# 2026-06-30 Codex: ClawBot rebuild 改為 bounded / opt-in，避免重開機 +# startup oneshot 被單一 image rebuild timeout 拖成 failed。 # 部署位置: /usr/local/bin/awoooi-startup.sh (on 192.168.0.188) # systemd unit: /etc/systemd/system/awoooi-startup.service @@ -14,6 +16,20 @@ exec > >(tee -a "$LOG") 2>&1 log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; } +CLAWBOT_UP_TIMEOUT_SECONDS="${CLAWBOT_UP_TIMEOUT_SECONDS:-90}" +CLAWBOT_BUILD_TIMEOUT_SECONDS="${CLAWBOT_BUILD_TIMEOUT_SECONDS:-120}" +CLAWBOT_STARTUP_REBUILD_ALLOWED="${CLAWBOT_STARTUP_REBUILD_ALLOWED:-0}" + +run_bounded() { + seconds="$1" + shift + if command -v timeout >/dev/null 2>&1; then + timeout "${seconds}s" "$@" + else + "$@" + fi +} + postgres_runtime_ready() { if systemctl is-active postgresql@14-main >/dev/null 2>&1; then log "✅ PostgreSQL systemd unit active" @@ -187,11 +203,20 @@ CLAWBOT_DIR="/home/ollama/clawbot-v5" if [ -f "$CLAWBOT_DIR/docker-compose.yml" ]; then log "啟動 ClawBot..." cd "$CLAWBOT_DIR" - # 若 image snapshot 損壞，先 rebuild - if ! docker compose up -d 2>&1 | tee /tmp/clawbot-up.log | grep -q "Started\|Running\|healthy"; then - log "ClawBot 啟動失敗，嘗試 rebuild..." - docker compose build --no-cache 2>&1 | tail -5 || true - docker compose up -d 2>&1 | tail -3 || log "⚠️ ClawBot rebuild 也失敗，跳過" + if ! run_bounded "$CLAWBOT_UP_TIMEOUT_SECONDS" docker compose up -d 2>&1 | tee /tmp/clawbot-up.log | grep -q "Started\|Running\|healthy"; then + log "⚠️ ClawBot 啟動未在 ${CLAWBOT_UP_TIMEOUT_SECONDS}s 內確認；交由 post-check 判定" + if [ "$CLAWBOT_STARTUP_REBUILD_ALLOWED" = "1" ]; then + log "ClawBot bounded rebuild 已允許，最多等待 ${CLAWBOT_BUILD_TIMEOUT_SECONDS}s..." + run_bounded "$CLAWBOT_BUILD_TIMEOUT_SECONDS" docker compose build 2>&1 | tail -5 || log "⚠️ ClawBot bounded rebuild 未完成" + run_bounded "$CLAWBOT_UP_TIMEOUT_SECONDS" docker compose up -d 2>&1 | tail -3 || log "⚠️ ClawBot rebuild 後啟動仍未確認" + else + log "⚠️ ClawBot rebuild 預設停用，避免拖垮 10 分鐘 post-reboot SLO" + fi + fi + if docker inspect -f '{{.State.Running}}' clawbot 2>/dev/null | grep -q '^true$'; then + log "✅ ClawBot container running" + else + log "⚠️ ClawBot container 未確認 running；post-reboot verifier 會標示服務 blocker" fi log "✅ ClawBot 啟動指令已發送" else diff --git a/scripts/reboot-recovery/tests/test_188_host_hygiene_checklist.py b/scripts/reboot-recovery/tests/test_188_host_hygiene_checklist.py index 61d54ba19..86fff2295 100644 --- a/scripts/reboot-recovery/tests/test_188_host_hygiene_checklist.py +++ b/scripts/reboot-recovery/tests/test_188_host_hygiene_checklist.py @@ -5,6 +5,8 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[3] SCRIPT = ROOT / "scripts" / "reboot-recovery" / "188-host-hygiene-maintenance-checklist.sh" +STARTUP_188 = ROOT / "scripts" / "reboot-recovery" / "awoooi-startup.sh" +STARTUP_188_SERVICE = ROOT / "scripts" / "reboot-recovery" / "awoooi-startup.service" def test_188_and_110_default_to_reachable_runtime_identities() -> None: @@ -29,3 +31,21 @@ def test_110_self_check_can_run_locally_without_ssh_config_drift() -> None: assert '-o NumberOfPasswordPrompts=0' in text assert 'ssh "${ssh_opts[@]}" "$target" \\' in text assert "timeout ${SSH_COMMAND_TIMEOUT_SECONDS}s bash -lc ${quoted_command}" in text + + +def test_188_startup_keeps_clawbot_rebuild_bounded_and_opt_in() -> None: + text = STARTUP_188.read_text(encoding="utf-8") + + assert 'CLAWBOT_UP_TIMEOUT_SECONDS="${CLAWBOT_UP_TIMEOUT_SECONDS:-90}"' in text + assert 'CLAWBOT_BUILD_TIMEOUT_SECONDS="${CLAWBOT_BUILD_TIMEOUT_SECONDS:-120}"' in text + assert 'CLAWBOT_STARTUP_REBUILD_ALLOWED="${CLAWBOT_STARTUP_REBUILD_ALLOWED:-0}"' in text + assert "run_bounded()" in text + assert "docker compose build --no-cache" not in text + assert 'if [ "$CLAWBOT_STARTUP_REBUILD_ALLOWED" = "1" ]; then' in text + assert "post-reboot SLO" in text + + +def test_188_startup_unit_timeout_matches_reboot_slo() -> None: + text = STARTUP_188_SERVICE.read_text(encoding="utf-8") + + assert "TimeoutStartSec=600" in text