From 723207315cd0114ea57b2b04987a67882673292f Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 28 Jun 2026 14:59:12 +0800 Subject: [PATCH] fix(ci): keep 110 cd lane fail closed [skip ci] --- docs/LOGBOOK.md | 14 ++++ .../awoooi-enforce-runner-failclosed-110.sh | 25 ++++++ scripts/reboot-recovery/awoooi-startup-110.sh | 82 ++++--------------- 3 files changed, 54 insertions(+), 67 deletions(-) diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 2102dace4..c785db61f 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,17 @@ +## 2026-06-28 — 14:57 110 runner/CD fail-close enforcer 與 startup 收斂 + +**完成內容**： +- Source：`awoooi-enforce-runner-failclosed-110.sh` 補齊 regular / drain cd-lane 與 legacy runner entrypoint fail-closed stub；drain lane 非 preserve 時會封 binary 後再次 stop / mask / reset-failed，避免舊 `Restart=always` unit 留在 auto-restart。 +- Source：`awoooi-startup-110.sh` 移除 110 startup 自動開啟 controlled / drain cd-lane 的 sentinel / env 分支；110 runner/CD 壓力事故期間，regular 與 drain lane 一律 fail-closed，恢復需另開搬遷或硬限流變更。 +- 110 live：同步安裝兩支腳本到 `/usr/local/bin` 並加 immutable；執行 enforcer `--apply` 後延遲 20 秒 `--check` 通過，regular / drain cd-lane 均 `masked / inactive / dead`，runner / action runner / job container process 全部 `0`。 + +**驗證結果**： +- `bash -n scripts/reboot-recovery/awoooi-startup-110.sh scripts/reboot-recovery/awoooi-enforce-runner-failclosed-110.sh scripts/reboot-recovery/p3-controlled-release-gate.sh scripts/reboot-recovery/post-start-quick-check.sh scripts/reboot-recovery/full-stack-cold-start-check.sh`：通過。 +- `git diff --check`：通過。 +- P3 release gate：`PASS=38 WARN=3 BLOCKED=0`；`CD_LANE_CONTROLLED mode=failclosed`、`CD_LANE_DRAIN_CONTROLLED mode=failclosed`、`BAD_RUNNER_GUARDRAILS 0`、`NO_ACTIVE_JOB_CONTAINERS`。 + +**邊界**：沒有啟動 legacy runner / controlled drain lane / generic runner；沒有把 host pressure gate 改成 warn-only；沒有讀 runner token / secret / raw session / SQLite；沒有 force push。 + ## 2026-06-28 — 14:20 IwoooS Wazuh manager registry 驗收口徑收斂 **完成內容**： diff --git a/scripts/reboot-recovery/awoooi-enforce-runner-failclosed-110.sh b/scripts/reboot-recovery/awoooi-enforce-runner-failclosed-110.sh index e73a476b2..01cce3d44 100755 --- a/scripts/reboot-recovery/awoooi-enforce-runner-failclosed-110.sh +++ b/scripts/reboot-recovery/awoooi-enforce-runner-failclosed-110.sh @@ -21,10 +21,20 @@ HARD_FAILCLOSED_UNITS=( ) DRAIN_UNIT="awoooi-cd-lane-drain.service" +CD_LANE_DIR="/home/wooo/awoooi-cd-lane" +CD_LANE_BINARY="$CD_LANE_DIR/awoooi_cd_lane" DRAIN_DIR="/home/wooo/awoooi-cd-lane-drain" DRAIN_BINARY="$DRAIN_DIR/awoooi_cd_lane_controlled" DRAIN_CONFIG="$DRAIN_DIR/config.yaml" +FAILCLOSED_ENTRYPOINTS=( + "/home/wooo/act-runner/act_runner" + "/home/wooo/act-runner/act_runner.real-20260628-runner-pressure-guard" + "/home/wooo/act-runner-controlled/act_runner" + "/home/wooo/awoooi-controlled-runner/awoooi_controlled_runner" + "$CD_LANE_BINARY" +) + LEGACY_SENTINELS=( "/run/awoooi-runner-host-enabled" "/run/awoooi-start-controlled-cd-lane" @@ -249,6 +259,15 @@ seal_quarantined_runner_sources() { ) } +seal_failclosed_entrypoints() { + local path parent + for path in "${FAILCLOSED_ENTRYPOINTS[@]}"; do + parent="$(dirname "$path")" + [ -d "$parent" ] || continue + write_failclosed_stub "$path" + done +} + seal_opener_templates() { local path local tmp @@ -503,17 +522,23 @@ apply_failclosed() { stop_and_mask_action_runner_units kill_runner_processes remove_legacy_sentinels + seal_failclosed_entrypoints if [ "$preserve_drain" = "1" ]; then : else stop_and_mask_unit "$DRAIN_UNIT" pkill -KILL -f "^${DRAIN_BINARY}" >/dev/null 2>&1 || true remove_drain_sentinels + if [ -d "$DRAIN_DIR" ]; then + write_failclosed_stub "$DRAIN_BINARY" + fi + stop_and_mask_unit "$DRAIN_UNIT" fi seal_opener_templates seal_root_restore_sources seal_quarantined_runner_sources as_root systemctl daemon-reload >/dev/null 2>&1 || true + as_root systemctl reset-failed "$DRAIN_UNIT" >/dev/null 2>&1 || true } if ! host_is_110 && [ "${AWOOOI_FAILCLOSED_ALLOW_NON_110:-0}" != "1" ]; then diff --git a/scripts/reboot-recovery/awoooi-startup-110.sh b/scripts/reboot-recovery/awoooi-startup-110.sh index 92a994a4d..47fa41579 100644 --- a/scripts/reboot-recovery/awoooi-startup-110.sh +++ b/scripts/reboot-recovery/awoooi-startup-110.sh @@ -206,6 +206,8 @@ START_CONTROLLED_CD_LANE="${AWOOOI_START_CONTROLLED_CD_LANE:-0}" START_GITEA_RUNNER_ALLOWED=0 START_CD_LANE_ALLOWED=0 RUNNER_FAIL_CLOSED_SERVICES=( + "awoooi-cd-lane.service" + "awoooi-cd-lane-drain.service" "awoooi-direct-runner-open.service" "awoooi-direct-runner.service" "gitea-act-runner-host.service" @@ -214,19 +216,18 @@ RUNNER_FAIL_CLOSED_SERVICES=( "gitea-act-runner-awoooi-open.service" ) RUNNER_FAIL_CLOSED_BINARY_PATHS=( + "/home/wooo/awoooi-cd-lane/awoooi_cd_lane" + "/home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled" "/home/wooo/act-runner/act_runner" "/home/wooo/act-runner/act_runner.real-20260628-runner-pressure-guard" "/home/wooo/act-runner-controlled/act_runner" "/home/wooo/awoooi-controlled-runner/awoooi_controlled_runner" ) -# Legacy host runner still needs both keys. The dedicated cd-lane has its own -# sentinel and narrow label/capacity verifier below. +# Host runner still needs both keys. The direct cd-lane stays fail-closed until +# it is migrated or hard-limited outside this production host pressure lane. if [ "$START_GITEA_RUNNER_ON_BOOT" = "1" ] && [ -e "$RUNNER_ENABLE_SENTINEL" ]; then START_GITEA_RUNNER_ALLOWED=1 fi -if [ -e "$CD_LANE_ENABLE_SENTINEL" ] || [ "$START_CONTROLLED_CD_LANE" = "1" ]; then - START_CD_LANE_ALLOWED=1 -fi mask_runner_unit_file() { local unit="$1" @@ -279,29 +280,17 @@ EOF install_cd_lane_fail_closed_unit() { local unit_file="/etc/systemd/system/awoooi-cd-lane.service" - local tmp local quarantine_stamp quarantine_stamp="$(date +%Y%m%d%H%M%S)" + systemctl mask awoooi-cd-lane.service >/dev/null 2>&1 || true if [ -e "$unit_file" ] || [ -L "$unit_file" ]; then chattr -i "$unit_file" >/dev/null 2>&1 || true - if ! grep -q "AWOOOI direct CD lane fail-closed" "$unit_file" 2>/dev/null; then + if ! { [ -L "$unit_file" ] && [ "$(readlink "$unit_file" 2>/dev/null || true)" = "/dev/null" ]; }; then mv "$unit_file" "${unit_file}.quarantined-runner-incident-${quarantine_stamp}" >/dev/null 2>&1 || true fi fi - tmp="$(mktemp)" - cat >"$tmp" <<'EOF' -[Unit] -Description=AWOOOI direct CD lane fail-closed after 2026-06-28 pressure incident -ConditionPathExists=/run/awoooi-cd-lane-enabled - -[Service] -Type=oneshot -ExecStart=/bin/false -EOF - install -o root -g root -m 0444 "$tmp" "$unit_file" >/dev/null 2>&1 || true - rm -f "$tmp" - chattr +i "$unit_file" >/dev/null 2>&1 || true + ln -sfn /dev/null "$unit_file" >/dev/null 2>&1 || true } install_controlled_cd_lane_unit() { @@ -480,36 +469,10 @@ quarantine_cd_lane_root_restore_sources_fail_closed() { apply_cd_lane_fail_closed_guard() { local unit - if cd_lane_drain_is_controlled_available; then - if cd_lane_drain_is_controlled_open; then - log "✅ controlled cd-lane drain verifier passed; preserving drain lane and fail-closing regular lane only" - else - log "✅ controlled cd-lane drain assets verified; restoring drain unit and fail-closing regular lane only" - fi - systemctl kill --signal=SIGKILL "$CD_LANE_SERVICE" >/dev/null 2>&1 || true - systemctl stop "$CD_LANE_SERVICE" >/dev/null 2>&1 || true - systemctl disable "$CD_LANE_SERVICE" >/dev/null 2>&1 || true - install_cd_lane_fail_closed_unit - pkill -KILL -f "^${CD_LANE_BINARY} daemon" >/dev/null 2>&1 || true - install_controlled_cd_lane_drain_unit - quarantine_cd_lane_root_restore_sources_fail_closed - systemctl daemon-reload >/dev/null 2>&1 || true - systemctl enable --now "$CD_LANE_DRAIN_SERVICE" >/dev/null 2>&1 || true - return 0 - fi - if { [ -e "$CD_LANE_ENABLE_SENTINEL" ] || [ -e "/run/awoooi-cd-lane-controlled-open" ] || [ "$START_CONTROLLED_CD_LANE" = "1" ]; } \ - && cd_lane_config_is_controlled \ - && file "$CD_LANE_BINARY" 2>/dev/null | grep -qi "ELF"; then - log "✅ controlled cd-lane verifier passed; keeping dedicated lane open" - install_controlled_cd_lane_unit - quarantine_cd_lane_root_restore_sources_fail_closed - systemctl daemon-reload >/dev/null 2>&1 || true - systemctl enable --now "$CD_LANE_SERVICE" >/dev/null 2>&1 || true - return 0 - fi for unit in awoooi-cd-lane.service awoooi-cd-lane-drain.service; do systemctl kill --signal=SIGKILL "$unit" >/dev/null 2>&1 || true systemctl stop "$unit" >/dev/null 2>&1 || true + systemctl reset-failed "$unit" >/dev/null 2>&1 || true systemctl disable "$unit" >/dev/null 2>&1 || true if [ "$unit" = "awoooi-cd-lane.service" ]; then install_cd_lane_fail_closed_unit @@ -526,6 +489,7 @@ apply_cd_lane_fail_closed_guard() { guard_runner_binary_fail_closed "$CD_LANE_DIR/awoooi_cd_lane" guard_runner_binary_fail_closed "$CD_LANE_DRAIN_DIR/awoooi_cd_lane_controlled" systemctl daemon-reload >/dev/null 2>&1 || true + systemctl reset-failed awoooi-cd-lane.service awoooi-cd-lane-drain.service >/dev/null 2>&1 || true } ensure_cd_lane_fail_closed() { @@ -533,19 +497,8 @@ ensure_cd_lane_fail_closed() { } ensure_controlled_cd_lane_open() { - if ! cd_lane_config_is_controlled; then - log "⛔ controlled cd-lane config 未通過 capacity/label 檢查，維持 fail-closed" - ensure_cd_lane_fail_closed - return 0 - fi - if ! file "$CD_LANE_BINARY" 2>/dev/null | grep -qi "ELF"; then - log "⛔ controlled cd-lane binary 不是可執行 ELF，維持 fail-closed" - ensure_cd_lane_fail_closed - return 0 - fi - install_controlled_cd_lane_unit - systemctl daemon-reload >/dev/null 2>&1 || true - systemctl enable --now "$CD_LANE_SERVICE" >/dev/null 2>&1 || true + log "⛔ 110 runner/CD pressure guard is fail-closed; controlled lane requires a separate migration or hard-limit change" + ensure_cd_lane_fail_closed } ensure_host_runner_fail_closed() { @@ -683,13 +636,8 @@ else log "⚠️ 找不到 act-runner binary/config: $RUNNER_DIR" fi -if [ "$START_CD_LANE_ALLOWED" = "1" ]; then - log "✅ controlled cd-lane 具備 sentinel/env 授權，執行 capacity/label/binary verifier 後受控開啟" - ensure_controlled_cd_lane_open -else - log "⏸️ controlled cd-lane 未要求啟動；保留合格 drain lane，regular lane 維持 fail-closed" - ensure_cd_lane_fail_closed -fi +log "⏸️ direct cd-lane / drain lane 維持 fail-closed；需完成搬遷或硬限流後才可用獨立變更恢復" +ensure_cd_lane_fail_closed # ────────────────────────────────────────────── # STEP 7: Sentry（Error Tracking）