From 83b7aa491ea0bc3690b693f3171bfe4ae9193163 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 1 Jul 2026 09:19:13 +0800 Subject: [PATCH] fix(runner): gate non110 keepalive on host pressure [skip ci] --- docs/LOGBOOK.md | 17 ++++++++++++++++ ...stall-awoooi-non110-runner-user-service.sh | 17 ++++++++++++++++ ...stall_awoooi_non110_runner_user_service.py | 20 +++++++++++++++++++ 3 files changed, 54 insertions(+) diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 36a3129e4..79116899b 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -11,6 +11,23 @@ **邊界**：未讀 token / secret / `.env` / raw sessions / SQLite / auth，未讀 `.runner` 內容，未使用 GitHub / `gh` / GitHub API，未 workflow_dispatch，未重啟主機、未 restart Docker / Nginx / K3s / DB / firewall；本輪 runtime 寫入限於 188 user-level runner service enable / start 與 keepalive timer enable。 +## 2026-07-01 — 09:17 non-110 keepalive pressure guard / 188 CPU loop stop + +**照主線修正的問題**： +- 09:14 重新打開 188 non-110 runner 後，live 188 再次出現 `awoooi-cd-*-api-tests` 與 `k3s-postgres-recovery` CPU 壓力；根因不是 service disable 不生效，而是 keepalive timer / sentinel 會把 runner 重新拉起。 +- 對 188 做受控止血：disable / stop `awoooi-non110-runner-keepalive.timer`、stop keepalive service、移除 `.awoooi-non110-runner-enabled` sentinel、disable / stop runner service 與 autostart path，並停止當下 `awoooi-cd-*-api-tests` / `stockplatform-ubuntu-runner` container；讀回 runner / autostart / keepalive 全部 `inactive`，未讀 `.runner` 內容。 +- `ops/runner/install-awoooi-non110-runner-user-service.sh` 補上 `PRESSURE_GUARD=/usr/local/bin/awoooi-wait-host-web-build-pressure.sh`，runner service `ExecStartPre`、autostart service 與 keepalive service 在啟動前都要通過 host pressure guard。 +- rollback unit / `--rollback` 同步 stop / disable keepalive timer、stop keepalive service、reset failed、移除 enable sentinel，避免下一次只關 runner service 卻被 keepalive 再拉起。 +- 測試補上 service / keepalive / rollback / readback 斷言，確保未來 non-110 runner 恢復時仍維持 pressure guard 與可回滾。 + +**驗證**： +- `python3.11 -m pytest ops/runner/test_install_awoooi_non110_runner_user_service.py -q` 通過（2 passed）。 +- `bash -n ops/runner/install-awoooi-non110-runner-user-service.sh scripts/ci/wait-host-web-build-pressure.sh` 通過。 +- `python3.11 ops/runner/guard-gitea-runner-pressure.py --root .` 通過，`auto_branch_events_on_110=0`、`generic_runner_labels=0`。 +- `git diff --check` 通過。 + +**邊界**：本輪只做 188 user-level non-110 runner 降壓與 source guard 固化；未讀 secret / token / `.env` / raw sessions / SQLite / auth；未讀 `.runner` 內容；未使用 GitHub / `gh` / GitHub API；未 workflow_dispatch；未重啟主機、未 restart Docker / Nginx / K3s / DB / firewall；未恢復 legacy / generic runner。 + ## 2026-07-01 — 09:14 non-110 runner controlled apply / Harbor remote control-channel / B5 Docker socket fail-fast **照主線修正的問題**： diff --git a/ops/runner/install-awoooi-non110-runner-user-service.sh b/ops/runner/install-awoooi-non110-runner-user-service.sh index 06599a796..8083f940d 100755 --- a/ops/runner/install-awoooi-non110-runner-user-service.sh +++ b/ops/runner/install-awoooi-non110-runner-user-service.sh @@ -27,6 +27,9 @@ CPU_QUOTA="${CPU_QUOTA:-200%}" MEMORY_HIGH="${MEMORY_HIGH:-6G}" MEMORY_MAX="${MEMORY_MAX:-8G}" TASKS_MAX="${TASKS_MAX:-512}" +PRESSURE_GUARD="${PRESSURE_GUARD:-/usr/local/bin/awoooi-wait-host-web-build-pressure.sh}" +PRESSURE_GUARD_ATTEMPTS="${PRESSURE_GUARD_ATTEMPTS:-1}" +PRESSURE_GUARD_SLEEP_SECONDS="${PRESSURE_GUARD_SLEEP_SECONDS:-0}" ACTION="check" if [ "$#" -gt 0 ]; then @@ -90,6 +93,8 @@ WorkingDirectory=${RUNNER_DIR} Environment=HOME=${RUNNER_HOME} ExecStartPre=/usr/bin/test -x ${RUNNER_BINARY} ExecStartPre=/usr/bin/test -S /var/run/docker.sock +ExecStartPre=/usr/bin/test -x ${PRESSURE_GUARD} +ExecStartPre=/usr/bin/env HOST_WEB_BUILD_PRESSURE_ATTEMPTS=${PRESSURE_GUARD_ATTEMPTS} HOST_WEB_BUILD_PRESSURE_SLEEP_SECONDS=${PRESSURE_GUARD_SLEEP_SECONDS} ${PRESSURE_GUARD} ExecStart=${RUNNER_BINARY} daemon --config ${RUNNER_CONFIG} Restart=always RestartSec=20 @@ -187,6 +192,10 @@ Type=oneshot ExecStart=-/usr/bin/systemctl --user stop ${SERVICE_NAME} ExecStart=-/usr/bin/systemctl --user disable ${SERVICE_NAME} ExecStart=-/usr/bin/systemctl --user reset-failed ${SERVICE_NAME} +ExecStart=-/usr/bin/systemctl --user stop ${KEEPALIVE_TIMER_NAME} +ExecStart=-/usr/bin/systemctl --user disable ${KEEPALIVE_TIMER_NAME} +ExecStart=-/usr/bin/systemctl --user stop ${KEEPALIVE_SERVICE_NAME} +ExecStart=-/usr/bin/systemctl --user reset-failed ${KEEPALIVE_SERVICE_NAME} ExecStart=-/usr/bin/systemctl --user stop ${AUTOSTART_PATH_NAME} ExecStart=-/usr/bin/systemctl --user disable ${AUTOSTART_PATH_NAME} ExecStart=-/usr/bin/systemctl --user reset-failed ${AUTOSTART_SERVICE_NAME} @@ -209,6 +218,8 @@ Type=oneshot ExecStart=/usr/bin/test -x ${RUNNER_BINARY} ExecStart=/usr/bin/test -s ${RUNNER_CONFIG} ExecStart=/usr/bin/test -s ${RUNNER_REGISTRATION} +ExecStart=/usr/bin/test -x ${PRESSURE_GUARD} +ExecStart=/usr/bin/env HOST_WEB_BUILD_PRESSURE_ATTEMPTS=${PRESSURE_GUARD_ATTEMPTS} HOST_WEB_BUILD_PRESSURE_SLEEP_SECONDS=${PRESSURE_GUARD_SLEEP_SECONDS} ${PRESSURE_GUARD} ExecStart=/usr/bin/touch ${ENABLE_SENTINEL} ExecStart=/usr/bin/systemctl --user reset-failed ${SERVICE_NAME} ExecStart=/usr/bin/systemctl --user enable ${SERVICE_NAME} @@ -245,6 +256,8 @@ Type=oneshot ExecStart=/usr/bin/test -x ${RUNNER_BINARY} ExecStart=/usr/bin/test -s ${RUNNER_CONFIG} ExecStart=/usr/bin/test -s ${RUNNER_REGISTRATION} +ExecStart=/usr/bin/test -x ${PRESSURE_GUARD} +ExecStart=/usr/bin/env HOST_WEB_BUILD_PRESSURE_ATTEMPTS=${PRESSURE_GUARD_ATTEMPTS} HOST_WEB_BUILD_PRESSURE_SLEEP_SECONDS=${PRESSURE_GUARD_SLEEP_SECONDS} ${PRESSURE_GUARD} ExecStart=/usr/bin/systemctl --user daemon-reload ExecStart=-/usr/bin/systemctl --user reset-failed ${SERVICE_NAME} ExecStart=/usr/bin/systemctl --user enable ${SERVICE_NAME} @@ -282,6 +295,9 @@ readback() { printf 'autostart_path_name=%s\n' "$AUTOSTART_PATH_NAME" printf 'keepalive_service_name=%s\n' "$KEEPALIVE_SERVICE_NAME" printf 'keepalive_timer_name=%s\n' "$KEEPALIVE_TIMER_NAME" + printf 'pressure_guard=%s\n' "$PRESSURE_GUARD" + printf 'pressure_guard_attempts=%s\n' "$PRESSURE_GUARD_ATTEMPTS" + printf 'pressure_guard_sleep_seconds=%s\n' "$PRESSURE_GUARD_SLEEP_SECONDS" printf 'secret_values_collected=false\n' printf 'runner_token_read=false\n' printf 'raw_runner_registration_read=false\n' @@ -360,6 +376,7 @@ enable_runner() { rollback_runner() { systemctl --user stop "$KEEPALIVE_TIMER_NAME" >/dev/null 2>&1 || true systemctl --user disable "$KEEPALIVE_TIMER_NAME" >/dev/null 2>&1 || true + systemctl --user stop "$KEEPALIVE_SERVICE_NAME" >/dev/null 2>&1 || true systemctl --user reset-failed "$KEEPALIVE_SERVICE_NAME" >/dev/null 2>&1 || true systemctl --user stop "$SERVICE_NAME" >/dev/null 2>&1 || true systemctl --user disable "$SERVICE_NAME" >/dev/null 2>&1 || true diff --git a/ops/runner/test_install_awoooi_non110_runner_user_service.py b/ops/runner/test_install_awoooi_non110_runner_user_service.py index 5ed555628..1010f946e 100644 --- a/ops/runner/test_install_awoooi_non110_runner_user_service.py +++ b/ops/runner/test_install_awoooi_non110_runner_user_service.py @@ -95,20 +95,35 @@ def test_apply_with_existing_registration_does_not_start_runner(tmp_path: Path) assert "secret-token-like-content" not in result.stdout unit_dir = tmp_path / "home/.config/systemd/user" + service = unit_dir / "awoooi-non110-runner.service" + rollback = unit_dir / "awoooi-non110-runner-rollback.service" autostart = unit_dir / "awoooi-non110-runner-autostart.service" keepalive_service = unit_dir / "awoooi-non110-runner-keepalive.service" keepalive_timer = unit_dir / "awoooi-non110-runner-keepalive.timer" + service_text = service.read_text(encoding="utf-8") + assert "ExecStartPre=/usr/bin/test -x /usr/local/bin/awoooi-wait-host-web-build-pressure.sh" in service_text + assert "HOST_WEB_BUILD_PRESSURE_ATTEMPTS=1" in service_text + assert "HOST_WEB_BUILD_PRESSURE_SLEEP_SECONDS=0" in service_text + rollback_text = rollback.read_text(encoding="utf-8") + assert "stop awoooi-non110-runner-keepalive.timer" in rollback_text + assert "disable awoooi-non110-runner-keepalive.timer" in rollback_text + assert "stop awoooi-non110-runner-keepalive.service" in rollback_text + assert "rm -f" in rollback_text assert "ConditionPathExists=!" in autostart.read_text(encoding="utf-8") assert "enable --now awoooi-non110-runner-keepalive.timer" in autostart.read_text( encoding="utf-8" ) keepalive_service_text = keepalive_service.read_text(encoding="utf-8") + assert "ExecStart=/usr/bin/test -x /usr/local/bin/awoooi-wait-host-web-build-pressure.sh" in keepalive_service_text + assert "HOST_WEB_BUILD_PRESSURE_ATTEMPTS=1" in keepalive_service_text + assert "HOST_WEB_BUILD_PRESSURE_SLEEP_SECONDS=0" in keepalive_service_text assert "ExecStart=/usr/bin/systemctl --user daemon-reload" in keepalive_service_text assert ( "ExecStart=-/usr/bin/systemctl --user reset-failed awoooi-non110-runner.service" in keepalive_service_text ) assert "OnUnitInactiveSec=15s" in keepalive_timer.read_text(encoding="utf-8") + assert "pressure_guard=/usr/local/bin/awoooi-wait-host-web-build-pressure.sh" in result.stdout log = (tmp_path / "systemctl.log").read_text(encoding="utf-8") assert "enable --now awoooi-non110-runner-autostart.path" not in log @@ -127,6 +142,11 @@ def test_enable_starts_runner_and_keepalive_without_printing_registration( runner_dir = tmp_path / "home/act-runner-awoooi" assert (runner_dir / ".awoooi-non110-runner-enabled").exists() + service_text = ( + tmp_path / "home/.config/systemd/user/awoooi-non110-runner.service" + ).read_text(encoding="utf-8") + assert "ExecStartPre=/usr/bin/test -x /usr/local/bin/awoooi-wait-host-web-build-pressure.sh" in service_text + assert "HOST_WEB_BUILD_PRESSURE_ATTEMPTS=1" in service_text log = (tmp_path / "systemctl.log").read_text(encoding="utf-8") assert "enable --now awoooi-non110-runner-keepalive.timer" in log assert "enable awoooi-non110-runner.service" in log