fix(recovery): bound 188 startup clawbot rebuild
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / tests (push) Has been cancelled
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / tests (push) Has been cancelled
This commit is contained in:
@@ -22,8 +22,8 @@ Wants=redis-server.service ollama.service nginx.service
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/usr/local/bin/awoooi-startup.sh
|
||||
# 允許較長的啟動時間(修復流程可能需要時間)
|
||||
TimeoutStartSec=300
|
||||
# 對齊 10 分鐘 post-reboot SLO;單一服務 rebuild 不得阻塞整個 oneshot。
|
||||
TimeoutStartSec=600
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
# 不在自動啟動腳本內執行 pg_resetwal,避免資料破壞被誤判成恢復。
|
||||
# 2026-06-26 Codex: 允許受控 recovery container 提供 14/main runtime;
|
||||
# 不再因 systemd postgresql@14-main failed 而誤判活 DB 為不可用。
|
||||
# 2026-06-30 Codex: ClawBot rebuild 改為 bounded / opt-in,避免重開機
|
||||
# startup oneshot 被單一 image rebuild timeout 拖成 failed。
|
||||
# 部署位置: /usr/local/bin/awoooi-startup.sh (on 192.168.0.188)
|
||||
# systemd unit: /etc/systemd/system/awoooi-startup.service
|
||||
|
||||
@@ -14,6 +16,20 @@ exec > >(tee -a "$LOG") 2>&1
|
||||
|
||||
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }
|
||||
|
||||
CLAWBOT_UP_TIMEOUT_SECONDS="${CLAWBOT_UP_TIMEOUT_SECONDS:-90}"
|
||||
CLAWBOT_BUILD_TIMEOUT_SECONDS="${CLAWBOT_BUILD_TIMEOUT_SECONDS:-120}"
|
||||
CLAWBOT_STARTUP_REBUILD_ALLOWED="${CLAWBOT_STARTUP_REBUILD_ALLOWED:-0}"
|
||||
|
||||
run_bounded() {
|
||||
seconds="$1"
|
||||
shift
|
||||
if command -v timeout >/dev/null 2>&1; then
|
||||
timeout "${seconds}s" "$@"
|
||||
else
|
||||
"$@"
|
||||
fi
|
||||
}
|
||||
|
||||
postgres_runtime_ready() {
|
||||
if systemctl is-active postgresql@14-main >/dev/null 2>&1; then
|
||||
log "✅ PostgreSQL systemd unit active"
|
||||
@@ -187,11 +203,20 @@ CLAWBOT_DIR="/home/ollama/clawbot-v5"
|
||||
if [ -f "$CLAWBOT_DIR/docker-compose.yml" ]; then
|
||||
log "啟動 ClawBot..."
|
||||
cd "$CLAWBOT_DIR"
|
||||
# 若 image snapshot 損壞,先 rebuild
|
||||
if ! docker compose up -d 2>&1 | tee /tmp/clawbot-up.log | grep -q "Started\|Running\|healthy"; then
|
||||
log "ClawBot 啟動失敗,嘗試 rebuild..."
|
||||
docker compose build --no-cache 2>&1 | tail -5 || true
|
||||
docker compose up -d 2>&1 | tail -3 || log "⚠️ ClawBot rebuild 也失敗,跳過"
|
||||
if ! run_bounded "$CLAWBOT_UP_TIMEOUT_SECONDS" docker compose up -d 2>&1 | tee /tmp/clawbot-up.log | grep -q "Started\|Running\|healthy"; then
|
||||
log "⚠️ ClawBot 啟動未在 ${CLAWBOT_UP_TIMEOUT_SECONDS}s 內確認;交由 post-check 判定"
|
||||
if [ "$CLAWBOT_STARTUP_REBUILD_ALLOWED" = "1" ]; then
|
||||
log "ClawBot bounded rebuild 已允許,最多等待 ${CLAWBOT_BUILD_TIMEOUT_SECONDS}s..."
|
||||
run_bounded "$CLAWBOT_BUILD_TIMEOUT_SECONDS" docker compose build 2>&1 | tail -5 || log "⚠️ ClawBot bounded rebuild 未完成"
|
||||
run_bounded "$CLAWBOT_UP_TIMEOUT_SECONDS" docker compose up -d 2>&1 | tail -3 || log "⚠️ ClawBot rebuild 後啟動仍未確認"
|
||||
else
|
||||
log "⚠️ ClawBot rebuild 預設停用,避免拖垮 10 分鐘 post-reboot SLO"
|
||||
fi
|
||||
fi
|
||||
if docker inspect -f '{{.State.Running}}' clawbot 2>/dev/null | grep -q '^true$'; then
|
||||
log "✅ ClawBot container running"
|
||||
else
|
||||
log "⚠️ ClawBot container 未確認 running;post-reboot verifier 會標示服務 blocker"
|
||||
fi
|
||||
log "✅ ClawBot 啟動指令已發送"
|
||||
else
|
||||
|
||||
@@ -5,6 +5,8 @@ from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
SCRIPT = ROOT / "scripts" / "reboot-recovery" / "188-host-hygiene-maintenance-checklist.sh"
|
||||
STARTUP_188 = ROOT / "scripts" / "reboot-recovery" / "awoooi-startup.sh"
|
||||
STARTUP_188_SERVICE = ROOT / "scripts" / "reboot-recovery" / "awoooi-startup.service"
|
||||
|
||||
|
||||
def test_188_and_110_default_to_reachable_runtime_identities() -> None:
|
||||
@@ -29,3 +31,21 @@ def test_110_self_check_can_run_locally_without_ssh_config_drift() -> None:
|
||||
assert '-o NumberOfPasswordPrompts=0' in text
|
||||
assert 'ssh "${ssh_opts[@]}" "$target" \\' in text
|
||||
assert "timeout ${SSH_COMMAND_TIMEOUT_SECONDS}s bash -lc ${quoted_command}" in text
|
||||
|
||||
|
||||
def test_188_startup_keeps_clawbot_rebuild_bounded_and_opt_in() -> None:
|
||||
text = STARTUP_188.read_text(encoding="utf-8")
|
||||
|
||||
assert 'CLAWBOT_UP_TIMEOUT_SECONDS="${CLAWBOT_UP_TIMEOUT_SECONDS:-90}"' in text
|
||||
assert 'CLAWBOT_BUILD_TIMEOUT_SECONDS="${CLAWBOT_BUILD_TIMEOUT_SECONDS:-120}"' in text
|
||||
assert 'CLAWBOT_STARTUP_REBUILD_ALLOWED="${CLAWBOT_STARTUP_REBUILD_ALLOWED:-0}"' in text
|
||||
assert "run_bounded()" in text
|
||||
assert "docker compose build --no-cache" not in text
|
||||
assert 'if [ "$CLAWBOT_STARTUP_REBUILD_ALLOWED" = "1" ]; then' in text
|
||||
assert "post-reboot SLO" in text
|
||||
|
||||
|
||||
def test_188_startup_unit_timeout_matches_reboot_slo() -> None:
|
||||
text = STARTUP_188_SERVICE.read_text(encoding="utf-8")
|
||||
|
||||
assert "TimeoutStartSec=600" in text
|
||||
|
||||
Reference in New Issue
Block a user