fix(recovery): bound 188 startup clawbot rebuild
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / tests (push) Has been cancelled

This commit is contained in:
Your Name
2026-06-30 22:37:33 +08:00
parent 51d87b237d
commit 1c3c7279ac
5 changed files with 108 additions and 7 deletions

View File

@@ -22,8 +22,8 @@ Wants=redis-server.service ollama.service nginx.service
Type=oneshot
RemainAfterExit=yes
ExecStart=/usr/local/bin/awoooi-startup.sh
# 允許較長的啟動時間(修復流程可能需要時間)
TimeoutStartSec=300
# 對齊 10 分鐘 post-reboot SLO單一服務 rebuild 不得阻塞整個 oneshot。
TimeoutStartSec=600
StandardOutput=journal
StandardError=journal

View File

@@ -5,6 +5,8 @@
# 不在自動啟動腳本內執行 pg_resetwal避免資料破壞被誤判成恢復。
# 2026-06-26 Codex: 允許受控 recovery container 提供 14/main runtime
# 不再因 systemd postgresql@14-main failed 而誤判活 DB 為不可用。
# 2026-06-30 Codex: ClawBot rebuild 改為 bounded / opt-in避免重開機
# startup oneshot 被單一 image rebuild timeout 拖成 failed。
# 部署位置: /usr/local/bin/awoooi-startup.sh (on 192.168.0.188)
# systemd unit: /etc/systemd/system/awoooi-startup.service
@@ -14,6 +16,20 @@ exec > >(tee -a "$LOG") 2>&1
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }
CLAWBOT_UP_TIMEOUT_SECONDS="${CLAWBOT_UP_TIMEOUT_SECONDS:-90}"
CLAWBOT_BUILD_TIMEOUT_SECONDS="${CLAWBOT_BUILD_TIMEOUT_SECONDS:-120}"
CLAWBOT_STARTUP_REBUILD_ALLOWED="${CLAWBOT_STARTUP_REBUILD_ALLOWED:-0}"
run_bounded() {
seconds="$1"
shift
if command -v timeout >/dev/null 2>&1; then
timeout "${seconds}s" "$@"
else
"$@"
fi
}
postgres_runtime_ready() {
if systemctl is-active postgresql@14-main >/dev/null 2>&1; then
log "✅ PostgreSQL systemd unit active"
@@ -187,11 +203,20 @@ CLAWBOT_DIR="/home/ollama/clawbot-v5"
if [ -f "$CLAWBOT_DIR/docker-compose.yml" ]; then
log "啟動 ClawBot..."
cd "$CLAWBOT_DIR"
# 若 image snapshot 損壞,先 rebuild
if ! docker compose up -d 2>&1 | tee /tmp/clawbot-up.log | grep -q "Started\|Running\|healthy"; then
log "ClawBot 啟動失敗,嘗試 rebuild..."
docker compose build --no-cache 2>&1 | tail -5 || true
docker compose up -d 2>&1 | tail -3 || log "⚠️ ClawBot rebuild 也失敗,跳過"
if ! run_bounded "$CLAWBOT_UP_TIMEOUT_SECONDS" docker compose up -d 2>&1 | tee /tmp/clawbot-up.log | grep -q "Started\|Running\|healthy"; then
log "⚠️ ClawBot 啟動未在 ${CLAWBOT_UP_TIMEOUT_SECONDS}s 內確認;交由 post-check 判定"
if [ "$CLAWBOT_STARTUP_REBUILD_ALLOWED" = "1" ]; then
log "ClawBot bounded rebuild 已允許,最多等待 ${CLAWBOT_BUILD_TIMEOUT_SECONDS}s..."
run_bounded "$CLAWBOT_BUILD_TIMEOUT_SECONDS" docker compose build 2>&1 | tail -5 || log "⚠️ ClawBot bounded rebuild 未完成"
run_bounded "$CLAWBOT_UP_TIMEOUT_SECONDS" docker compose up -d 2>&1 | tail -3 || log "⚠️ ClawBot rebuild 後啟動仍未確認"
else
log "⚠️ ClawBot rebuild 預設停用,避免拖垮 10 分鐘 post-reboot SLO"
fi
fi
if docker inspect -f '{{.State.Running}}' clawbot 2>/dev/null | grep -q '^true$'; then
log "✅ ClawBot container running"
else
log "⚠️ ClawBot container 未確認 runningpost-reboot verifier 會標示服務 blocker"
fi
log "✅ ClawBot 啟動指令已發送"
else

View File

@@ -5,6 +5,8 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
SCRIPT = ROOT / "scripts" / "reboot-recovery" / "188-host-hygiene-maintenance-checklist.sh"
STARTUP_188 = ROOT / "scripts" / "reboot-recovery" / "awoooi-startup.sh"
STARTUP_188_SERVICE = ROOT / "scripts" / "reboot-recovery" / "awoooi-startup.service"
def test_188_and_110_default_to_reachable_runtime_identities() -> None:
@@ -29,3 +31,21 @@ def test_110_self_check_can_run_locally_without_ssh_config_drift() -> None:
assert '-o NumberOfPasswordPrompts=0' in text
assert 'ssh "${ssh_opts[@]}" "$target" \\' in text
assert "timeout ${SSH_COMMAND_TIMEOUT_SECONDS}s bash -lc ${quoted_command}" in text
def test_188_startup_keeps_clawbot_rebuild_bounded_and_opt_in() -> None:
text = STARTUP_188.read_text(encoding="utf-8")
assert 'CLAWBOT_UP_TIMEOUT_SECONDS="${CLAWBOT_UP_TIMEOUT_SECONDS:-90}"' in text
assert 'CLAWBOT_BUILD_TIMEOUT_SECONDS="${CLAWBOT_BUILD_TIMEOUT_SECONDS:-120}"' in text
assert 'CLAWBOT_STARTUP_REBUILD_ALLOWED="${CLAWBOT_STARTUP_REBUILD_ALLOWED:-0}"' in text
assert "run_bounded()" in text
assert "docker compose build --no-cache" not in text
assert 'if [ "$CLAWBOT_STARTUP_REBUILD_ALLOWED" = "1" ]; then' in text
assert "post-reboot SLO" in text
def test_188_startup_unit_timeout_matches_reboot_slo() -> None:
text = STARTUP_188_SERVICE.read_text(encoding="utf-8")
assert "TimeoutStartSec=600" in text