fix(cd): reopen controlled runtime deploy lane

2026-06-28 11:09:42 +08:00
parent 95c825f242
commit e97b252475
11 changed files with 203 additions and 56 deletions
--- a/scripts/reboot-recovery/awoooi-startup-110.sh
+++ b/scripts/reboot-recovery/awoooi-startup-110.sh
@@ -193,12 +193,19 @@ RUNNER_DIR="/home/wooo/act-runner"
 RUNNER_SERVICE="gitea-act-runner-host.service"
 RUNNER_ENABLE_SENTINEL="/run/awoooi-runner-host-enabled"
 CD_LANE_DIR="/home/wooo/awoooi-cd-lane"
+CD_LANE_SERVICE="awoooi-cd-lane.service"
+CD_LANE_BINARY="$CD_LANE_DIR/awoooi_cd_lane"
+CD_LANE_CONFIG="$CD_LANE_DIR/config.yaml"
 CD_LANE_DRAIN_DIR="/home/wooo/awoooi-cd-lane-drain"
+CD_LANE_DRAIN_SERVICE="awoooi-cd-lane-drain.service"
+CD_LANE_DRAIN_BINARY="$CD_LANE_DRAIN_DIR/awoooi_cd_lane_controlled"
+CD_LANE_DRAIN_CONFIG="$CD_LANE_DRAIN_DIR/config.yaml"
+CD_LANE_ENABLE_SENTINEL="/run/awoooi-cd-lane-enabled"
 START_GITEA_RUNNER_ON_BOOT="${AWOOOI_START_GITEA_RUNNER_ON_BOOT:-0}"
+START_CONTROLLED_CD_LANE="${AWOOOI_START_CONTROLLED_CD_LANE:-0}"
 START_GITEA_RUNNER_ALLOWED=0
+START_CD_LANE_ALLOWED=0
 RUNNER_FAIL_CLOSED_SERVICES=(
-    "awoooi-cd-lane.service"
-    "awoooi-cd-lane-drain.service"
    "awoooi-direct-runner-open.service"
    "awoooi-direct-runner.service"
    "gitea-act-runner-host.service"
@@ -207,18 +214,19 @@ RUNNER_FAIL_CLOSED_SERVICES=(
    "gitea-act-runner-awoooi-open.service"
 )
 RUNNER_FAIL_CLOSED_BINARY_PATHS=(
-    "/home/wooo/awoooi-cd-lane/awoooi_cd_lane"
-    "/home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled"
    "/home/wooo/act-runner/act_runner"
    "/home/wooo/act-runner/act_runner.real-20260628-runner-pressure-guard"
    "/home/wooo/act-runner-controlled/act_runner"
    "/home/wooo/awoooi-controlled-runner/awoooi_controlled_runner"
 )
-# Host runner still needs both keys. The direct cd-lane stays fail-closed until
-# it is migrated or hard-limited outside this production host pressure lane.
+# Legacy host runner still needs both keys. The dedicated cd-lane has its own
+# sentinel and narrow label/capacity verifier below.
 if [ "$START_GITEA_RUNNER_ON_BOOT" = "1" ] && [ -e "$RUNNER_ENABLE_SENTINEL" ]; then
    START_GITEA_RUNNER_ALLOWED=1
 fi
+if [ -e "$CD_LANE_ENABLE_SENTINEL" ] || [ "$START_CONTROLLED_CD_LANE" = "1" ]; then
+    START_CD_LANE_ALLOWED=1
+fi

 mask_runner_unit_file() {
    local unit="$1"
@@ -271,17 +279,96 @@ EOF

 install_cd_lane_fail_closed_unit() {
    local unit_file="/etc/systemd/system/awoooi-cd-lane.service"
+    local tmp
    local quarantine_stamp
    quarantine_stamp="$(date +%Y%m%d%H%M%S)"

-    systemctl mask awoooi-cd-lane.service >/dev/null 2>&1 || true
    if [ -e "$unit_file" ] || [ -L "$unit_file" ]; then
        chattr -i "$unit_file" >/dev/null 2>&1 || true
-        if ! { [ -L "$unit_file" ] && [ "$(readlink "$unit_file" 2>/dev/null || true)" = "/dev/null" ]; }; then
+        if ! grep -q "AWOOOI direct CD lane fail-closed" "$unit_file" 2>/dev/null; then
            mv "$unit_file" "${unit_file}.quarantined-runner-incident-${quarantine_stamp}" >/dev/null 2>&1 || true
        fi
    fi
-    ln -sfn /dev/null "$unit_file" >/dev/null 2>&1 || true
+    tmp="$(mktemp)"
+    cat >"$tmp" <<'EOF'
+[Unit]
+Description=AWOOOI direct CD lane fail-closed after 2026-06-28 pressure incident
+ConditionPathExists=/run/awoooi-cd-lane-enabled
+
+[Service]
+Type=oneshot
+ExecStart=/bin/false
+EOF
+    install -o root -g root -m 0444 "$tmp" "$unit_file" >/dev/null 2>&1 || true
+    rm -f "$tmp"
+    chattr +i "$unit_file" >/dev/null 2>&1 || true
+}
+
+install_controlled_cd_lane_unit() {
+    local unit_file="/etc/systemd/system/$CD_LANE_SERVICE"
+    local tmp
+    chattr -i "$unit_file" "$CD_LANE_BINARY" >/dev/null 2>&1 || true
+    tmp="$(mktemp)"
+    cat >"$tmp" <<EOF
+[Unit]
+Description=AWOOOI controlled CD lane
+After=network-online.target docker.service
+Wants=network-online.target
+Requires=docker.service
+
+[Service]
+Type=simple
+User=wooo
+WorkingDirectory=${CD_LANE_DIR}/data
+Environment=HOME=/home/wooo
+Environment=AWOOOI_CONTROLLED_RUNNER_OPEN=1
+Environment=HOST_WEB_BUILD_PRESSURE_ATTEMPTS=1
+Environment=HOST_WEB_BUILD_PRESSURE_SLEEP_SECONDS=1
+ExecStart=${CD_LANE_BINARY} daemon --config ${CD_LANE_CONFIG}
+Restart=always
+RestartSec=10
+KillSignal=SIGINT
+TimeoutStopSec=3700
+SuccessExitStatus=0 130 143
+CPUQuota=250%
+MemoryHigh=8G
+MemoryMax=12G
+TasksMax=512
+
+[Install]
+WantedBy=multi-user.target
+EOF
+    install -o root -g root -m 0644 "$tmp" "$unit_file" >/dev/null 2>&1 || true
+    rm -f "$tmp"
+}
+
+cd_lane_config_path_is_controlled() {
+    local config_path="$1"
+    [ -f "$config_path" ] || return 1
+    grep -Eq '^[[:space:]]+capacity:[[:space:]]*1[[:space:]]*$' "$config_path" || return 1
+    grep -q 'awoooi-ubuntu:docker://192.168.0.110:5000/awoooi/ci-runner:act-22.04' "$config_path" || return 1
+    grep -q 'awoooi-host:host' "$config_path" || return 1
+    if grep -Eq '^[[:space:]]+- ".*(ubuntu-latest|stockplatform|headless|playwright)' "$config_path"; then
+        return 1
+    fi
+    return 0
+}
+
+cd_lane_config_is_controlled() {
+    cd_lane_config_path_is_controlled "$CD_LANE_CONFIG"
+}
+
+cd_lane_drain_config_is_controlled() {
+    cd_lane_config_path_is_controlled "$CD_LANE_DRAIN_CONFIG"
+}
+
+cd_lane_drain_is_controlled_open() {
+    local active
+    active="$(systemctl show "$CD_LANE_DRAIN_SERVICE" -p ActiveState --value 2>/dev/null || true)"
+    [ "$active" = "active" ] || return 1
+    cd_lane_drain_config_is_controlled || return 1
+    file "$CD_LANE_DRAIN_BINARY" 2>/dev/null | grep -qi "ELF" || return 1
+    return 0
 }

 quarantine_cd_lane_registration_fail_closed() {
@@ -316,6 +403,25 @@ quarantine_cd_lane_registration_fail_closed() {

 apply_cd_lane_fail_closed_guard() {
    local unit
+    if cd_lane_drain_is_controlled_open; then
+        log "✅ controlled cd-lane drain verifier passed; preserving drain lane and fail-closing regular lane only"
+        systemctl kill --signal=SIGKILL "$CD_LANE_SERVICE" >/dev/null 2>&1 || true
+        systemctl stop "$CD_LANE_SERVICE" >/dev/null 2>&1 || true
+        systemctl disable "$CD_LANE_SERVICE" >/dev/null 2>&1 || true
+        install_cd_lane_fail_closed_unit
+        pkill -KILL -f "^${CD_LANE_BINARY} daemon" >/dev/null 2>&1 || true
+        systemctl daemon-reload >/dev/null 2>&1 || true
+        return 0
+    fi
+    if { [ -e "$CD_LANE_ENABLE_SENTINEL" ] || [ -e "/run/awoooi-cd-lane-controlled-open" ] || [ "$START_CONTROLLED_CD_LANE" = "1" ]; } \
+        && cd_lane_config_is_controlled \
+        && file "$CD_LANE_BINARY" 2>/dev/null | grep -qi "ELF"; then
+        log "✅ controlled cd-lane verifier passed; keeping dedicated lane open"
+        install_controlled_cd_lane_unit
+        systemctl daemon-reload >/dev/null 2>&1 || true
+        systemctl enable --now "$CD_LANE_SERVICE" >/dev/null 2>&1 || true
+        return 0
+    fi
    for unit in awoooi-cd-lane.service awoooi-cd-lane-drain.service; do
        systemctl kill --signal=SIGKILL "$unit" >/dev/null 2>&1 || true
        systemctl stop "$unit" >/dev/null 2>&1 || true
@@ -336,6 +442,26 @@ apply_cd_lane_fail_closed_guard() {
    systemctl daemon-reload >/dev/null 2>&1 || true
 }

+ensure_cd_lane_fail_closed() {
+    apply_cd_lane_fail_closed_guard
+}
+
+ensure_controlled_cd_lane_open() {
+    if ! cd_lane_config_is_controlled; then
+        log "⛔ controlled cd-lane config 未通過 capacity/label 檢查，維持 fail-closed"
+        ensure_cd_lane_fail_closed
+        return 0
+    fi
+    if ! file "$CD_LANE_BINARY" 2>/dev/null | grep -qi "ELF"; then
+        log "⛔ controlled cd-lane binary 不是可執行 ELF，維持 fail-closed"
+        ensure_cd_lane_fail_closed
+        return 0
+    fi
+    install_controlled_cd_lane_unit
+    systemctl daemon-reload >/dev/null 2>&1 || true
+    systemctl enable --now "$CD_LANE_SERVICE" >/dev/null 2>&1 || true
+}
+
 ensure_host_runner_fail_closed() {
    local unit
    local binary
@@ -365,8 +491,6 @@ ensure_host_runner_fail_closed() {
    fi

    pkill -KILL -f "^${RUNNER_DIR}/act_runner(\\.real-[^ ]*)? daemon" >/dev/null 2>&1 || true
-    pkill -KILL -f "^${CD_LANE_DIR}/awoooi_cd_lane daemon" >/dev/null 2>&1 || true
-    quarantine_cd_lane_registration_fail_closed
    for binary in "${RUNNER_FAIL_CLOSED_BINARY_PATHS[@]}"; do
        guard_runner_binary_fail_closed "$binary"
    done
@@ -472,8 +596,13 @@ else
    log "⚠️  找不到 act-runner binary/config: $RUNNER_DIR"
 fi

-log "⏸️  direct cd-lane 維持 fail-closed；需完成搬遷或硬限流後才可用獨立變更恢復"
-apply_cd_lane_fail_closed_guard
+if [ "$START_CD_LANE_ALLOWED" = "1" ]; then
+    log "✅ controlled cd-lane 具備 sentinel/env 授權，執行 capacity/label/binary verifier 後受控開啟"
+    ensure_controlled_cd_lane_open
+else
+    log "⏸️  controlled cd-lane 未要求啟動；保留合格 drain lane，regular lane 維持 fail-closed"
+    ensure_cd_lane_fail_closed
+fi

 # ──────────────────────────────────────────────
 # STEP 7: Sentry（Error Tracking）
--- a/scripts/reboot-recovery/full-stack-cold-start-check.sh
+++ b/scripts/reboot-recovery/full-stack-cold-start-check.sh
@@ -286,7 +286,7 @@ echo "ACTION_RUNNER_ENABLED_COUNT $(systemctl list-unit-files "actions.runner.*"
 for u in $(systemctl list-units "actions.runner.*" --all --no-legend --plain 2>/dev/null | awk "{print \$1}"); do
  systemctl show "$u" -p ActiveState -p SubState -p CPUQuotaPerSecUSec -p MemoryMax -p WatchdogUSec -p NRestarts | sed "s/^/RUNNER $u /"
 done
-for u in awoooi-cd-lane-drain.service awoooi-direct-runner-open.service awoooi-direct-runner.service gitea-act-runner-host.service gitea-act-runner-awoooi-controlled.service gitea-awoooi-controlled-runner.service gitea-act-runner-awoooi-open.service; do
+for u in awoooi-direct-runner-open.service awoooi-direct-runner.service gitea-act-runner-host.service gitea-act-runner-awoooi-controlled.service gitea-awoooi-controlled-runner.service gitea-act-runner-awoooi-open.service; do
  load=$(systemctl show "$u" -p LoadState --value 2>/dev/null || true)
  unitfile=$(systemctl show "$u" -p UnitFileState --value 2>/dev/null || true)
  active=$(systemctl show "$u" -p ActiveState --value 2>/dev/null || true)
@@ -294,8 +294,6 @@ for u in awoooi-cd-lane-drain.service awoooi-direct-runner-open.service awoooi-d
  unit_ok=0
  if [ "$load" = "masked" ] && [ "$unitfile" = "masked" ] && [ "$active" = "inactive" ]; then
    unit_ok=1
-  elif [ "$u" = "awoooi-cd-lane-drain.service" ] && [ "$load" = "not-found" ] && [ "$active" != "active" ]; then
-    unit_ok=1
  fi
  echo "RUNNER_FAILCLOSED_UNIT $u load=$load unitfile=$unitfile active=$active mainpid=$mainpid ok=$unit_ok"
 done
@@ -319,7 +317,7 @@ fi
 cd_lane_binary_kind=$(file -b /home/wooo/awoooi-cd-lane/awoooi_cd_lane 2>/dev/null || echo missing)
 cd_lane_binary_elf=0
 echo "$cd_lane_binary_kind" | grep -qi "ELF" && cd_lane_binary_elf=1
-cd_lane_process_count=$(pgrep -f "^/home/wooo/awoooi-cd-lane/awoooi_cd_lane|^/home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled" 2>/dev/null | wc -l | tr -d " ")
+cd_lane_process_count=$(pgrep -f "^/home/wooo/awoooi-cd-lane/awoooi_cd_lane" 2>/dev/null | wc -l | tr -d " ")
 cd_lane_ok=0
 cd_lane_mode=blocked
 if [ "$cd_lane_active" = "inactive" ] \
@@ -360,16 +358,22 @@ if [ "$cd_lane_drain_active" != "active" ] \
  && { [ "$cd_lane_drain_load" = "not-found" ] || { [ "$cd_lane_drain_load" = "masked" ] && [ "$cd_lane_drain_unitfile" = "masked" ]; }; }; then
  cd_lane_drain_ok=1
  cd_lane_drain_mode=failclosed
+elif [ "$cd_lane_drain_active" = "active" ] \
+  && [ "$cd_lane_drain_capacity_ok" = "1" ] \
+  && [ "$cd_lane_drain_labels_ok" = "1" ] \
+  && [ "$cd_lane_drain_binary_elf" = "1" ]; then
+  cd_lane_drain_ok=1
+  cd_lane_drain_mode=controlled_open
 fi
 echo "CD_LANE_DRAIN_CONTROLLED mode=$cd_lane_drain_mode load=$cd_lane_drain_load unitfile=$cd_lane_drain_unitfile active=$cd_lane_drain_active mainpid=$cd_lane_drain_mainpid capacity=$cd_lane_drain_capacity_ok labels=$cd_lane_drain_labels_ok binary_elf=$cd_lane_drain_binary_elf process_count=$cd_lane_drain_process_count ok=$cd_lane_drain_ok"
 cd_lane_guard_ok=0
-if [ "$cd_lane_ok" = "1" ] && [ "$cd_lane_drain_ok" = "1" ]; then
+if [ "$cd_lane_ok" = "1" ] || [ "$cd_lane_drain_ok" = "1" ]; then
  cd_lane_guard_ok=1
 fi
 echo "CD_LANE_GUARDRAILS_OK $cd_lane_guard_ok"
 direct_runner_count=$(pgrep -f "^/home/wooo/act-runner/act_runner|^/home/wooo/act-runner-controlled/act_runner|^/home/wooo/awoooi-controlled-runner/awoooi_controlled_runner" 2>/dev/null | wc -l | tr -d " ")
 echo "RUNNER_DIRECT_PROCESS_COUNT $direct_runner_count"
-for p in /home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled /home/wooo/act-runner/act_runner /home/wooo/act-runner/act_runner.real-20260628-runner-pressure-guard /home/wooo/act-runner-controlled/act_runner /home/wooo/awoooi-controlled-runner/awoooi_controlled_runner; do
+for p in /home/wooo/act-runner/act_runner /home/wooo/act-runner/act_runner.real-20260628-runner-pressure-guard /home/wooo/act-runner-controlled/act_runner /home/wooo/awoooi-controlled-runner/awoooi_controlled_runner; do
  kind=$(file -b "$p" 2>/dev/null || echo missing)
  echo "RUNNER_FAILCLOSED_BINARY $p kind=$kind"
  echo "$kind" | grep -qi "ELF" && echo "RUNNER_FAILCLOSED_BINARY_ELF $p"
--- a/scripts/reboot-recovery/p3-controlled-release-gate.sh
+++ b/scripts/reboot-recovery/p3-controlled-release-gate.sh
@@ -306,15 +306,13 @@ check_runner_guardrails() {
  local out bad
 if ! out=$(ssh_cmd "wooo@192.168.0.110" '
 bad=0
-for u in awoooi-cd-lane-drain.service awoooi-direct-runner-open.service awoooi-direct-runner.service gitea-act-runner-host.service gitea-act-runner-awoooi-controlled.service gitea-awoooi-controlled-runner.service gitea-act-runner-awoooi-open.service; do
+for u in awoooi-direct-runner-open.service awoooi-direct-runner.service gitea-act-runner-host.service gitea-act-runner-awoooi-controlled.service gitea-awoooi-controlled-runner.service gitea-act-runner-awoooi-open.service; do
  load=$(systemctl show "$u" -p LoadState --value 2>/dev/null || true)
  unitfile=$(systemctl show "$u" -p UnitFileState --value 2>/dev/null || true)
  active=$(systemctl show "$u" -p ActiveState --value 2>/dev/null || true)
  unit_ok=0
  if [ "$load" = "masked" ] && [ "$unitfile" = "masked" ] && [ "$active" = "inactive" ]; then
    unit_ok=1
-  elif [ "$u" = "awoooi-cd-lane-drain.service" ] && [ "$load" = "not-found" ] && [ "$active" != "active" ]; then
-    unit_ok=1
  fi
  echo "RUNNER_FAILCLOSED_UNIT $u load=$load unitfile=$unitfile active=$active ok=$unit_ok"
  [ "$unit_ok" = "1" ] || bad=1
@@ -338,7 +336,7 @@ fi
 cd_lane_binary_kind=$(file -b /home/wooo/awoooi-cd-lane/awoooi_cd_lane 2>/dev/null || echo missing)
 cd_lane_binary_elf=0
 echo "$cd_lane_binary_kind" | grep -qi "ELF" && cd_lane_binary_elf=1
-cd_lane_process_count=$(pgrep -f "^/home/wooo/awoooi-cd-lane/awoooi_cd_lane|^/home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled" 2>/dev/null | wc -l | tr -d " ")
+cd_lane_process_count=$(pgrep -f "^/home/wooo/awoooi-cd-lane/awoooi_cd_lane" 2>/dev/null | wc -l | tr -d " ")
 cd_lane_ok=0
 cd_lane_mode=blocked
 if [ "$cd_lane_active" = "inactive" ] \
@@ -378,10 +376,16 @@ if [ "$cd_lane_drain_active" != "active" ] \
  && { [ "$cd_lane_drain_load" = "not-found" ] || { [ "$cd_lane_drain_load" = "masked" ] && [ "$cd_lane_drain_unitfile" = "masked" ]; }; }; then
  cd_lane_drain_ok=1
  cd_lane_drain_mode=failclosed
+elif [ "$cd_lane_drain_active" = "active" ] \
+  && [ "$cd_lane_drain_capacity_ok" = "1" ] \
+  && [ "$cd_lane_drain_labels_ok" = "1" ] \
+  && [ "$cd_lane_drain_binary_elf" = "1" ]; then
+  cd_lane_drain_ok=1
+  cd_lane_drain_mode=controlled_open
 fi
 echo "CD_LANE_DRAIN_CONTROLLED mode=$cd_lane_drain_mode load=$cd_lane_drain_load unitfile=$cd_lane_drain_unitfile active=$cd_lane_drain_active capacity=$cd_lane_drain_capacity_ok labels=$cd_lane_drain_labels_ok binary_elf=$cd_lane_drain_binary_elf process_count=$cd_lane_drain_process_count ok=$cd_lane_drain_ok"
 cd_lane_guard_ok=0
-if [ "$cd_lane_ok" = "1" ] && [ "$cd_lane_drain_ok" = "1" ]; then
+if [ "$cd_lane_ok" = "1" ] || [ "$cd_lane_drain_ok" = "1" ]; then
  cd_lane_guard_ok=1
 fi
 echo "CD_LANE_GUARDRAILS_OK $cd_lane_guard_ok"
@@ -389,7 +393,7 @@ echo "CD_LANE_GUARDRAILS_OK $cd_lane_guard_ok"
 direct_runner_count=$(pgrep -f "^/home/wooo/act-runner/act_runner|^/home/wooo/act-runner-controlled/act_runner|^/home/wooo/awoooi-controlled-runner/awoooi_controlled_runner" 2>/dev/null | wc -l | tr -d " ")
 echo "RUNNER_DIRECT_PROCESS_COUNT $direct_runner_count"
 [ "$direct_runner_count" = "0" ] || bad=1
-for p in /home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled /home/wooo/act-runner/act_runner /home/wooo/act-runner/act_runner.real-20260628-runner-pressure-guard /home/wooo/act-runner-controlled/act_runner /home/wooo/awoooi-controlled-runner/awoooi_controlled_runner; do
+for p in /home/wooo/act-runner/act_runner /home/wooo/act-runner/act_runner.real-20260628-runner-pressure-guard /home/wooo/act-runner-controlled/act_runner /home/wooo/awoooi-controlled-runner/awoooi_controlled_runner; do
  kind=$(file -b "$p" 2>/dev/null || echo missing)
  echo "RUNNER_FAILCLOSED_BINARY $p kind=$kind"
  echo "$kind" | grep -qi "ELF" && bad=1