From cb5ab900c4391c4892bd3f8b93e8b88303c52a1c Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 1 May 2026 16:16:27 +0800 Subject: [PATCH] fix(ci): preserve gitea runner jobs on shutdown --- docs/LOGBOOK.md | 14 +++ ops/runner/README.md | 27 ++++++ ops/runner/gitea-act-runner-host.service | 20 ++++ .../install-gitea-host-runner-service.sh | 93 +++++++++++++++++++ scripts/reboot-recovery/awoooi-startup-110.sh | 44 +++++++-- 5 files changed, 191 insertions(+), 7 deletions(-) create mode 100644 ops/runner/gitea-act-runner-host.service create mode 100755 ops/runner/install-gitea-host-runner-service.sh diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 8255eae20..1df28c054 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -6,6 +6,20 @@ --- +## 2026-05-01 | Gitea host runner graceful shutdown guard + +承接 `b0da6da1` production deploy：ArgoCD 已 `Synced Healthy` 到 deploy commit `f72419dd`，API/worker/web 也都跑新 image，但 Gitea commit status 將 `build-and-deploy` 標成 failure、`post-deploy-checks` 卡 pending。Live runner log 顯示 host `act_runner` 在 job 收尾前收到 shutdown，且使用預設 `shutdown_timeout=0s`，造成部署已完成但狀態回寫失真。 + +### 完成 +- 新增 `ops/runner/gitea-act-runner-host.service`，讓 110 host-level `act_runner` 由 systemd 管理，不再依賴裸 `nohup` 程序。 +- 新增 `ops/runner/install-gitea-host-runner-service.sh`，會把 `/home/wooo/act-runner/config.yaml` 正規化為 `shutdown_timeout: 1h`、安裝 systemd service、停用 Docker-wrapped `gitea-runner` restart policy，且在 `GITEA-ACTIONS-TASK-*` 正在跑時拒絕重啟。 +- `scripts/reboot-recovery/awoooi-startup-110.sh` 改為優先啟動 `gitea-act-runner-host.service`，並在 reboot recovery 時補上 `shutdown_timeout: 1h`。 +- `ops/runner/README.md` 補第三層 runner 修復：graceful shutdown service 與 status mismatch 根因。 + +### 驗證 +- Live root cause：`act_runner generate-config` 顯示預設 `runner.shutdown_timeout: 0s`；110 config 當時未覆寫。 +- Live deploy state：ArgoCD `Synced Healthy f72419dd`，`awoooi-api`/`awoooi-worker`/`awoooi-web` 均已使用 `b0da6da1` image。 + ## 2026-05-01 | Agent Loop shadow structured metadata guard 承接 P1 canary 上線後的 production 觀測：`ENABLE_OPENCLAW_AGENT_LOOP_SHADOW=True`、max iteration 2 已在 API pod 生效；`mcp_audit_log` 已有 MCP 呼叫，但尚未看到新的 `openclaw_agent_loop_shadow` production incident log。下一步先讓 shadow 一旦觸發就留下可評估、可治理的結構化結果，而不是直接改主決策。 diff --git a/ops/runner/README.md b/ops/runner/README.md index e7547561a..cd3f3dd9c 100644 --- a/ops/runner/README.md +++ b/ops/runner/README.md @@ -141,6 +141,7 @@ runner: ```yaml runner: capacity: 1 + shutdown_timeout: 1h labels: - "ubuntu-latest:docker://192.168.0.110:5000/awoooi/ci-runner:act-22.04" - "ubuntu-22.04:docker://192.168.0.110:5000/awoooi/ci-runner:act-22.04" @@ -153,6 +154,32 @@ Docker-wrapped `gitea-runner` container 必須停用，避免它用同一份 con `scripts/ops/docker-health-monitor.sh` 預設也必須排除 `gitea-runner`， 否則每 5 分鐘的 Docker 自動修復會把已停用的 runner container 拉起來。 +### 第三層修復: graceful shutdown service + +2026-05-01 發現 build/deploy 已推 GitOps deploy commit，production 也 +`Synced Healthy`，但 Gitea commit status 仍顯示 `build-and-deploy` failure。 +根因是 host-level `act_runner` 收到停止訊號時使用預設 +`runner.shutdown_timeout: 0s`，log 會出現： + +```text +runner: wooo-runner shutdown initiated, waiting 0s for running jobs to complete +``` + +因此 daemon 重啟會直接取消仍在收尾的 job，造成「實際已部署、狀態回寫失敗」。 +110 必須安裝 systemd host runner service，並把 shutdown timeout 固定為 1h： + +```bash +cd /path/to/awoooi +RESTART_NOW=1 bash ops/runner/install-gitea-host-runner-service.sh +``` + +此 script 會： + +- 更新 `/home/wooo/act-runner/config.yaml` 的 `shutdown_timeout: 1h` +- 安裝 `/etc/systemd/system/gitea-act-runner-host.service` +- 停用 Docker-wrapped `gitea-runner` container 的 restart policy +- 拒絕在 `GITEA-ACTIONS-TASK-*` container 正在跑時重啟 runner + --- 版本: v2.0 | 更新: 2026-03-29 | 作者: Claude Code 變更: v1.0→v2.0 序列建構取代 Job Concurrency Groups diff --git a/ops/runner/gitea-act-runner-host.service b/ops/runner/gitea-act-runner-host.service new file mode 100644 index 000000000..8eeab707c --- /dev/null +++ b/ops/runner/gitea-act-runner-host.service @@ -0,0 +1,20 @@ +[Unit] +Description=Gitea Actions Host Runner (AWOOOI) +After=network-online.target docker.service +Wants=network-online.target +Requires=docker.service + +[Service] +Type=simple +User=wooo +WorkingDirectory=/home/wooo/act-runner/data +Environment=HOME=/home/wooo +ExecStart=/home/wooo/act-runner/act_runner daemon --config /home/wooo/act-runner/config.yaml +Restart=always +RestartSec=15 +KillSignal=SIGINT +TimeoutStopSec=3700 +SuccessExitStatus=0 130 143 + +[Install] +WantedBy=multi-user.target diff --git a/ops/runner/install-gitea-host-runner-service.sh b/ops/runner/install-gitea-host-runner-service.sh new file mode 100755 index 000000000..2613bc569 --- /dev/null +++ b/ops/runner/install-gitea-host-runner-service.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +set -euo pipefail + +RUNNER_DIR="${RUNNER_DIR:-/home/wooo/act-runner}" +RUNNER_USER="${RUNNER_USER:-wooo}" +SERVICE_NAME="${SERVICE_NAME:-gitea-act-runner-host.service}" +SHUTDOWN_TIMEOUT="${SHUTDOWN_TIMEOUT:-1h}" +SERVICE_SRC="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/gitea-act-runner-host.service" +CONFIG_FILE="${RUNNER_DIR}/config.yaml" + +if [ ! -x "${RUNNER_DIR}/act_runner" ]; then + echo "act_runner binary not found: ${RUNNER_DIR}/act_runner" >&2 + exit 1 +fi + +if [ ! -f "${CONFIG_FILE}" ]; then + echo "act_runner config not found: ${CONFIG_FILE}" >&2 + exit 1 +fi + +python3 - "${CONFIG_FILE}" "${SHUTDOWN_TIMEOUT}" <<'PY' +import sys +from pathlib import Path + +path = Path(sys.argv[1]) +shutdown_timeout = sys.argv[2] +lines = path.read_text().splitlines() + +in_runner = False +runner_indent = None +insert_at = None +found = False +updated = [] + +for idx, line in enumerate(lines): + stripped = line.strip() + if stripped == "runner:": + in_runner = True + runner_indent = len(line) - len(line.lstrip()) + insert_at = idx + 1 + updated.append(line) + continue + + if in_runner: + indent = len(line) - len(line.lstrip()) + if stripped and indent <= runner_indent: + if not found and insert_at is not None: + updated.insert(insert_at, f" shutdown_timeout: {shutdown_timeout}") + found = True + in_runner = False + elif stripped.startswith("shutdown_timeout:"): + updated.append(f" shutdown_timeout: {shutdown_timeout}") + found = True + continue + elif stripped.startswith("timeout:") or stripped.startswith("capacity:"): + insert_at = len(updated) + 1 + + updated.append(line) + +if in_runner and not found and insert_at is not None: + updated.insert(insert_at, f" shutdown_timeout: {shutdown_timeout}") + found = True + +if not found: + raise SystemExit("runner section not found in config") + +path.write_text("\n".join(updated) + "\n") +PY + +sudo install -o root -g root -m 0644 "${SERVICE_SRC}" "/etc/systemd/system/${SERVICE_NAME}" +sudo systemctl daemon-reload +sudo systemctl enable "${SERVICE_NAME}" >/dev/null + +if docker ps --format '{{.Names}}' | grep -qx 'gitea-runner'; then + echo "Disabling legacy docker-wrapped gitea-runner container" + docker update --restart=no gitea-runner >/dev/null 2>&1 || true + docker stop gitea-runner >/dev/null 2>&1 || true +fi + +if [ "${RESTART_NOW:-0}" = "1" ]; then + if docker ps --format '{{.Names}}' | grep -q '^GITEA-ACTIONS-TASK-'; then + echo "Refusing to restart: Gitea Actions task containers are running" >&2 + exit 1 + fi + sudo systemctl restart "${SERVICE_NAME}" +elif pgrep -u "${RUNNER_USER}" -f "${RUNNER_DIR}/act_runner daemon" >/dev/null; then + echo "Existing act_runner daemon is still running; service will take over after the next safe restart." +else + sudo systemctl start "${SERVICE_NAME}" +fi + +sudo systemctl --no-pager --full status "${SERVICE_NAME}" | sed -n '1,18p' +grep -n 'shutdown_timeout' "${CONFIG_FILE}" diff --git a/scripts/reboot-recovery/awoooi-startup-110.sh b/scripts/reboot-recovery/awoooi-startup-110.sh index 4dbd8878e..fc3e897f4 100644 --- a/scripts/reboot-recovery/awoooi-startup-110.sh +++ b/scripts/reboot-recovery/awoooi-startup-110.sh @@ -188,7 +188,8 @@ fi # ────────────────────────────────────────────── log "[6/6] 啟動 Gitea Act Runner..." RUNNER_DIR="/home/wooo/act-runner" -if [ -f "$RUNNER_DIR/docker-compose.yml" ]; then +RUNNER_SERVICE="gitea-act-runner-host.service" +if [ -x "$RUNNER_DIR/act_runner" ] && [ -f "$RUNNER_DIR/config.yaml" ]; then # 若舊的 .runner 配置指向過期 hostname，先清除讓 runner 重新註冊 RUNNER_FILE="$RUNNER_DIR/data/.runner" if [ -f "$RUNNER_FILE" ]; then @@ -199,18 +200,47 @@ if [ -f "$RUNNER_DIR/docker-compose.yml" ]; then fi fi - cd "$RUNNER_DIR" - docker compose up -d 2>&1 | tail -3 + # act_runner 預設 shutdown_timeout=0s，會在 daemon 重啟時立刻取消 + # 正在執行的 CD job。這會造成部署實際完成但 Gitea status 失真。 + python3 - "$RUNNER_DIR/config.yaml" <<'PY' || true +import sys +from pathlib import Path + +path = Path(sys.argv[1]) +lines = path.read_text().splitlines() +if any(line.strip().startswith("shutdown_timeout:") for line in lines): + lines = [ + " shutdown_timeout: 1h" if line.strip().startswith("shutdown_timeout:") else line + for line in lines + ] +else: + for idx, line in enumerate(lines): + if line.strip().startswith("timeout:") and idx > 0: + lines.insert(idx + 1, " shutdown_timeout: 1h") + break +path.write_text("\n".join(lines) + "\n") +PY + + if systemctl list-unit-files "$RUNNER_SERVICE" >/dev/null 2>&1; then + systemctl enable --now "$RUNNER_SERVICE" >/dev/null 2>&1 || true + elif ! pgrep -f "$RUNNER_DIR/act_runner daemon" >/dev/null; then + nohup "$RUNNER_DIR/run-host-runner.sh" >> "$RUNNER_DIR/host-runner.log" 2>&1 & + fi + + # 已停用 Docker-wrapped runner；避免它搶走 host label job。 + docker update --restart=no gitea-runner >/dev/null 2>&1 || true + docker stop gitea-runner >/dev/null 2>&1 || true + sleep 15 # 驗證 runner 已連線 Gitea - if docker logs gitea-runner --tail 5 2>/dev/null | grep -q "SUCCESS\|Connected\|Listening"; then - log "✅ Gitea Act Runner 已連線" + if pgrep -f "$RUNNER_DIR/act_runner daemon" >/dev/null; then + log "✅ Gitea host act_runner 已啟動" else - log "⚠️ Gitea Act Runner 可能尚未連線，查看: docker logs gitea-runner" + log "⚠️ Gitea host act_runner 可能尚未啟動，查看: $RUNNER_DIR/host-runner.log" fi else - log "⚠️ 找不到 act-runner compose 檔案: $RUNNER_DIR/docker-compose.yml" + log "⚠️ 找不到 act-runner binary/config: $RUNNER_DIR" fi # ──────────────────────────────────────────────