fix(recovery): seal runner failclosed disablers [skip ci]
This commit is contained in:
Your Name
@@ -286,115 +286,61 @@ echo "ACTION_RUNNER_ENABLED_COUNT $(systemctl list-unit-files "actions.runner.*"
|
||||
for u in $(systemctl list-units "actions.runner.*" --all --no-legend --plain 2>/dev/null | awk "{print \$1}"); do
|
||||
systemctl show "$u" -p ActiveState -p SubState -p CPUQuotaPerSecUSec -p MemoryMax -p WatchdogUSec -p NRestarts | sed "s/^/RUNNER $u /"
|
||||
done
|
||||
for u in awoooi-direct-runner-open.service awoooi-direct-runner.service gitea-act-runner-host.service gitea-act-runner-awoooi-controlled.service gitea-awoooi-controlled-runner.service gitea-act-runner-awoooi-open.service; do
|
||||
for u in awoooi-cd-lane.service awoooi-cd-lane-drain.service awoooi-direct-runner-open.service awoooi-direct-runner.service gitea-act-runner-host.service gitea-act-runner-awoooi-controlled.service gitea-awoooi-controlled-runner.service gitea-act-runner-awoooi-open.service; do
|
||||
load=$(systemctl show "$u" -p LoadState --value 2>/dev/null || true)
|
||||
unitfile=$(systemctl show "$u" -p UnitFileState --value 2>/dev/null || true)
|
||||
active=$(systemctl show "$u" -p ActiveState --value 2>/dev/null || true)
|
||||
mainpid=$(systemctl show "$u" -p MainPID --value 2>/dev/null || true)
|
||||
unit_ok=0
|
||||
if [ "$load" = "masked" ] && [ "$unitfile" = "masked" ] && [ "$active" = "inactive" ]; then
|
||||
unit_stub=0
|
||||
if [ "$load" = "masked" ] && [ "$unitfile" = "masked" ] && [ "$active" = "inactive" ] && [ "${mainpid:-0}" = "0" ]; then
|
||||
unit_ok=1
|
||||
elif [ "$active" = "inactive" ] && [ "${mainpid:-0}" = "0" ] \
|
||||
&& systemctl cat "$u" 2>/dev/null | grep -q "ConditionPathExists=/run/awoooi-runner-migrated-or-hard-limited"; then
|
||||
unit_stub=1
|
||||
unit_ok=1
|
||||
fi
|
||||
echo "RUNNER_FAILCLOSED_UNIT $u load=$load unitfile=$unitfile active=$active mainpid=$mainpid ok=$unit_ok"
|
||||
echo "RUNNER_FAILCLOSED_UNIT $u load=$load unitfile=$unitfile active=$active mainpid=$mainpid stub=$unit_stub ok=$unit_ok"
|
||||
done
|
||||
cd_lane_load=$(systemctl show awoooi-cd-lane.service -p LoadState --value 2>/dev/null || true)
|
||||
cd_lane_unitfile=$(systemctl show awoooi-cd-lane.service -p UnitFileState --value 2>/dev/null || true)
|
||||
cd_lane_active=$(systemctl show awoooi-cd-lane.service -p ActiveState --value 2>/dev/null || true)
|
||||
cd_lane_mainpid=$(systemctl show awoooi-cd-lane.service -p MainPID --value 2>/dev/null || true)
|
||||
cd_lane_execstart=$(systemctl show awoooi-cd-lane.service -p ExecStart --value 2>/dev/null || true)
|
||||
cd_lane_sentinel=missing
|
||||
[ -e /run/awoooi-cd-lane-enabled ] && cd_lane_sentinel=present
|
||||
cd_lane_capacity_ok=0
|
||||
cd_lane_labels_ok=0
|
||||
if grep -Eq "^[[:space:]]+capacity:[[:space:]]*1[[:space:]]*$" /home/wooo/awoooi-cd-lane/config.yaml 2>/dev/null; then
|
||||
cd_lane_capacity_ok=1
|
||||
fi
|
||||
if grep -q "awoooi-ubuntu:docker://192.168.0.110:5000/awoooi/ci-runner:act-22.04" /home/wooo/awoooi-cd-lane/config.yaml 2>/dev/null \
|
||||
&& grep -q "awoooi-host:host" /home/wooo/awoooi-cd-lane/config.yaml 2>/dev/null \
|
||||
&& ! grep -Eq "^[[:space:]]+- \".*(ubuntu-latest|stockplatform|headless|playwright)" /home/wooo/awoooi-cd-lane/config.yaml 2>/dev/null; then
|
||||
cd_lane_labels_ok=1
|
||||
fi
|
||||
cd_lane_binary_kind=$(file -b /home/wooo/awoooi-cd-lane/awoooi_cd_lane 2>/dev/null || echo missing)
|
||||
cd_lane_binary_elf=0
|
||||
echo "$cd_lane_binary_kind" | grep -qi "ELF" && cd_lane_binary_elf=1
|
||||
cd_lane_process_count=$(pgrep -f "^/home/wooo/awoooi-cd-lane/awoooi_cd_lane" 2>/dev/null | wc -l | tr -d " ")
|
||||
cd_lane_ok=0
|
||||
cd_lane_mode=blocked
|
||||
if [ "$cd_lane_active" = "inactive" ] \
|
||||
&& [ "$cd_lane_sentinel" = "missing" ] \
|
||||
&& [ "$cd_lane_binary_elf" = "0" ] \
|
||||
&& [ "$cd_lane_process_count" = "0" ] \
|
||||
&& { { [ "$cd_lane_load" = "masked" ] && [ "$cd_lane_unitfile" = "masked" ]; } || echo "$cd_lane_execstart" | grep -q "/bin/false"; }; then
|
||||
cd_lane_ok=1
|
||||
cd_lane_mode=failclosed
|
||||
elif [ "$cd_lane_sentinel" = "present" ] && [ "$cd_lane_active" = "active" ] && [ "$cd_lane_capacity_ok" = "1" ] && [ "$cd_lane_labels_ok" = "1" ] && [ "$cd_lane_binary_elf" = "1" ]; then
|
||||
cd_lane_ok=1
|
||||
cd_lane_mode=controlled_open
|
||||
fi
|
||||
echo "CD_LANE_CONTROLLED mode=$cd_lane_mode load=$cd_lane_load unitfile=$cd_lane_unitfile active=$cd_lane_active mainpid=$cd_lane_mainpid sentinel=$cd_lane_sentinel capacity=$cd_lane_capacity_ok labels=$cd_lane_labels_ok binary_elf=$cd_lane_binary_elf process_count=$cd_lane_process_count ok=$cd_lane_ok"
|
||||
cd_lane_drain_load=$(systemctl show awoooi-cd-lane-drain.service -p LoadState --value 2>/dev/null || true)
|
||||
cd_lane_drain_unitfile=$(systemctl show awoooi-cd-lane-drain.service -p UnitFileState --value 2>/dev/null || true)
|
||||
cd_lane_drain_active=$(systemctl show awoooi-cd-lane-drain.service -p ActiveState --value 2>/dev/null || true)
|
||||
cd_lane_drain_mainpid=$(systemctl show awoooi-cd-lane-drain.service -p MainPID --value 2>/dev/null || true)
|
||||
cd_lane_drain_cpu_accounting=$(systemctl show awoooi-cd-lane-drain.service -p CPUAccounting --value 2>/dev/null || true)
|
||||
cd_lane_drain_cpu_quota=$(systemctl show awoooi-cd-lane-drain.service -p CPUQuotaPerSecUSec --value 2>/dev/null || true)
|
||||
cd_lane_drain_memory_accounting=$(systemctl show awoooi-cd-lane-drain.service -p MemoryAccounting --value 2>/dev/null || true)
|
||||
cd_lane_drain_memory_max=$(systemctl show awoooi-cd-lane-drain.service -p MemoryMax --value 2>/dev/null || true)
|
||||
cd_lane_drain_tasks_accounting=$(systemctl show awoooi-cd-lane-drain.service -p TasksAccounting --value 2>/dev/null || true)
|
||||
cd_lane_drain_tasks_max=$(systemctl show awoooi-cd-lane-drain.service -p TasksMax --value 2>/dev/null || true)
|
||||
cd_lane_drain_limits_ok=0
|
||||
if [ "$cd_lane_drain_cpu_accounting" = "yes" ] \
|
||||
&& [ -n "$cd_lane_drain_cpu_quota" ] && [ "$cd_lane_drain_cpu_quota" != "infinity" ] \
|
||||
&& [ "$cd_lane_drain_memory_accounting" = "yes" ] \
|
||||
&& [ -n "$cd_lane_drain_memory_max" ] && [ "$cd_lane_drain_memory_max" != "infinity" ] \
|
||||
&& [ "$cd_lane_drain_tasks_accounting" = "yes" ] \
|
||||
&& [ -n "$cd_lane_drain_tasks_max" ] && [ "$cd_lane_drain_tasks_max" != "infinity" ]; then
|
||||
cd_lane_drain_limits_ok=1
|
||||
fi
|
||||
cd_lane_drain_capacity_ok=0
|
||||
cd_lane_drain_labels_ok=0
|
||||
if grep -Eq "^[[:space:]]+capacity:[[:space:]]*1[[:space:]]*$" /home/wooo/awoooi-cd-lane-drain/config.yaml 2>/dev/null; then
|
||||
cd_lane_drain_capacity_ok=1
|
||||
fi
|
||||
if grep -q "awoooi-ubuntu:docker://192.168.0.110:5000/awoooi/ci-runner:act-22.04" /home/wooo/awoooi-cd-lane-drain/config.yaml 2>/dev/null \
|
||||
&& grep -q "awoooi-host:host" /home/wooo/awoooi-cd-lane-drain/config.yaml 2>/dev/null \
|
||||
&& ! grep -Eq "^[[:space:]]+- \".*(ubuntu-latest|stockplatform|headless|playwright)" /home/wooo/awoooi-cd-lane-drain/config.yaml 2>/dev/null; then
|
||||
cd_lane_drain_labels_ok=1
|
||||
fi
|
||||
cd_lane_drain_binary_kind=$(file -b /home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled 2>/dev/null || echo missing)
|
||||
cd_lane_drain_binary_elf=0
|
||||
echo "$cd_lane_drain_binary_kind" | grep -qi "ELF" && cd_lane_drain_binary_elf=1
|
||||
cd_lane_drain_process_count=$(pgrep -f "^/home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled" 2>/dev/null | wc -l | tr -d " ")
|
||||
cd_lane_drain_ok=0
|
||||
cd_lane_drain_mode=blocked
|
||||
if [ "$cd_lane_drain_active" != "active" ] \
|
||||
&& [ "$cd_lane_drain_binary_elf" = "0" ] \
|
||||
&& [ "$cd_lane_drain_process_count" = "0" ] \
|
||||
&& { [ "$cd_lane_drain_load" = "not-found" ] || { [ "$cd_lane_drain_load" = "masked" ] && [ "$cd_lane_drain_unitfile" = "masked" ]; }; }; then
|
||||
cd_lane_drain_ok=1
|
||||
cd_lane_drain_mode=failclosed
|
||||
elif [ "$cd_lane_drain_active" = "active" ] \
|
||||
&& [ "$cd_lane_drain_capacity_ok" = "1" ] \
|
||||
&& [ "$cd_lane_drain_labels_ok" = "1" ] \
|
||||
&& [ "$cd_lane_drain_binary_elf" = "1" ] \
|
||||
&& [ "$cd_lane_drain_limits_ok" = "1" ]; then
|
||||
cd_lane_drain_ok=1
|
||||
cd_lane_drain_mode=controlled_open
|
||||
fi
|
||||
echo "CD_LANE_DRAIN_CONTROLLED mode=$cd_lane_drain_mode load=$cd_lane_drain_load unitfile=$cd_lane_drain_unitfile active=$cd_lane_drain_active mainpid=$cd_lane_drain_mainpid capacity=$cd_lane_drain_capacity_ok labels=$cd_lane_drain_labels_ok binary_elf=$cd_lane_drain_binary_elf limits=$cd_lane_drain_limits_ok process_count=$cd_lane_drain_process_count ok=$cd_lane_drain_ok"
|
||||
enforcer_timer_active=$(systemctl is-active awoooi-runner-failclosed-enforcer.timer 2>/dev/null || true)
|
||||
enforcer_timer_enabled=$(systemctl is-enabled awoooi-runner-failclosed-enforcer.timer 2>/dev/null || true)
|
||||
enforcer_service_result=$(systemctl show awoooi-runner-failclosed-enforcer.service -p Result --value 2>/dev/null || true)
|
||||
echo "RUNNER_FAILCLOSED_ENFORCER timer_active=$enforcer_timer_active timer_enabled=$enforcer_timer_enabled service_result=$enforcer_service_result"
|
||||
authority_timer_active=$(systemctl is-active awoooi-runner-failclosed-authority.timer 2>/dev/null || true)
|
||||
authority_timer_enabled=$(systemctl is-enabled awoooi-runner-failclosed-authority.timer 2>/dev/null || true)
|
||||
authority_service_result=$(systemctl show awoooi-runner-failclosed-authority.service -p Result --value 2>/dev/null || true)
|
||||
echo "RUNNER_FAILCLOSED_AUTHORITY timer_active=$authority_timer_active timer_enabled=$authority_timer_enabled service_result=$authority_service_result"
|
||||
cd_lane_process_count=$(pgrep -f "^/home/wooo/awoooi-cd-lane/awoooi_cd_lane|^/home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled" 2>/dev/null | wc -l | tr -d " ")
|
||||
echo "CD_LANE_PROCESS_COUNT $cd_lane_process_count"
|
||||
cd_lane_root_restore_left=unknown
|
||||
if sudo -n true >/dev/null 2>&1; then
|
||||
cd_lane_root_restore_left=$(sudo -n find /root -maxdepth 1 -type d \( -name "awoooi-cd-lane-disabled-*" -o -name "awoooi-cd-lane-drain-disabled-*" \) -print 2>/dev/null | wc -l | tr -d " ")
|
||||
cd_lane_root_restore_left=$(sudo -n find /root -maxdepth 1 -type d \( -name "awoooi-runner-restore-sources-disabled*" -o -name "awoooi-cd-lane-disabled*" -o -name "awoooi-cd-lane-drain-disabled*" \) -print 2>/dev/null | wc -l | tr -d " ")
|
||||
fi
|
||||
echo "CD_LANE_ROOT_RESTORE_SOURCES left=$cd_lane_root_restore_left"
|
||||
sentinel_left=0
|
||||
for s in /run/awoooi-runner-host-enabled /run/awoooi-start-controlled-cd-lane /run/awoooi-start-controlled-cd-lane-drain /run/awoooi-start-cd-lane-allowed /run/awoooi-cd-lane-drain-ok /run/awoooi-cd-lane-ok /run/awoooi-cd-lane-enabled /run/awoooi-cd-lane-controlled-open; do
|
||||
[ -e "$s" ] && sentinel_left=$((sentinel_left + 1))
|
||||
done
|
||||
echo "RUNNER_SENTINELS_LEFT $sentinel_left"
|
||||
active_job_containers=$(docker ps --format "{{.Names}}" 2>/dev/null | grep -Ec "^(GITEA-ACTIONS-|awoooi-cd-)" || true)
|
||||
echo "ACTIVE_JOB_CONTAINERS $active_job_containers"
|
||||
cd_lane_guard_ok=0
|
||||
if { [ "$cd_lane_ok" = "1" ] || [ "$cd_lane_drain_ok" = "1" ]; } && [ "$cd_lane_root_restore_left" = "0" ]; then
|
||||
if [ "$enforcer_timer_active" = "active" ] \
|
||||
&& [ "$enforcer_timer_enabled" = "enabled" ] \
|
||||
&& [ "$enforcer_service_result" = "success" ] \
|
||||
&& [ "$authority_timer_active" = "active" ] \
|
||||
&& [ "$authority_timer_enabled" = "enabled" ] \
|
||||
&& [ "$authority_service_result" = "success" ] \
|
||||
&& [ "$cd_lane_process_count" = "0" ] \
|
||||
&& [ "$cd_lane_root_restore_left" = "0" ] \
|
||||
&& [ "$sentinel_left" = "0" ] \
|
||||
&& [ "$active_job_containers" = "0" ]; then
|
||||
cd_lane_guard_ok=1
|
||||
fi
|
||||
echo "CD_LANE_GUARDRAILS_OK $cd_lane_guard_ok"
|
||||
direct_runner_count=$(pgrep -f "^/home/wooo/act-runner/act_runner|^/home/wooo/act-runner-controlled/act_runner|^/home/wooo/awoooi-controlled-runner/awoooi_controlled_runner" 2>/dev/null | wc -l | tr -d " ")
|
||||
echo "RUNNER_DIRECT_PROCESS_COUNT $direct_runner_count"
|
||||
for p in /home/wooo/act-runner/act_runner /home/wooo/act-runner/act_runner.real-20260628-runner-pressure-guard /home/wooo/act-runner-controlled/act_runner /home/wooo/awoooi-controlled-runner/awoooi_controlled_runner; do
|
||||
for p in /home/wooo/awoooi-cd-lane/awoooi_cd_lane /home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled /home/wooo/act-runner/act_runner /home/wooo/act-runner/act_runner.real-20260628-runner-pressure-guard /home/wooo/act-runner-controlled/act_runner /home/wooo/awoooi-controlled-runner/awoooi_controlled_runner; do
|
||||
kind=$(file -b "$p" 2>/dev/null || echo missing)
|
||||
echo "RUNNER_FAILCLOSED_BINARY $p kind=$kind"
|
||||
echo "$kind" | grep -qi "ELF" && echo "RUNNER_FAILCLOSED_BINARY_ELF $p"
|
||||
@@ -423,12 +369,15 @@ docker ps --format "DOCKER {{.Names}}\t{{.Status}}" | head -120
|
||||
warn "runner watchdog state not confirmed"
|
||||
fi
|
||||
if awk '$1 == "RUNNER_FAILCLOSED_UNIT" && $NF != "ok=1" {bad=1} END {exit bad}' <<<"$out"; then
|
||||
ok "110 legacy direct/Gitea runner units are fail-closed"
|
||||
ok "110 runner/CD lane units are fail-closed"
|
||||
else
|
||||
fail "110 legacy direct/Gitea runner units are not fail-closed"
|
||||
fail "110 runner/CD lane units are not fail-closed"
|
||||
fi
|
||||
grep -q "CD_LANE_GUARDRAILS_OK 1" <<<"$out" && ok "110 controlled cd-lane is safe, drained, or fail-closed" || fail "110 controlled cd-lane is neither safe-open/drained nor fail-closed"
|
||||
grep -q "RUNNER_FAILCLOSED_ENFORCER timer_active=active timer_enabled=enabled service_result=success" <<<"$out" && ok "110 fail-closed enforcer timer active and successful" || fail "110 fail-closed enforcer timer not healthy"
|
||||
grep -q "RUNNER_FAILCLOSED_AUTHORITY timer_active=active timer_enabled=enabled service_result=success" <<<"$out" && ok "110 fail-closed authority timer active and successful" || fail "110 fail-closed authority timer not healthy"
|
||||
grep -q "CD_LANE_GUARDRAILS_OK 1" <<<"$out" && ok "110 cd-lane/drain lane are fail-closed with enforcer" || fail "110 cd-lane/drain lane fail-closed guardrails incomplete"
|
||||
grep -q "RUNNER_DIRECT_PROCESS_COUNT 0" <<<"$out" && ok "110 legacy direct runner process count is zero" || fail "110 legacy direct runner process detected"
|
||||
grep -q "ACTIVE_JOB_CONTAINERS 0" <<<"$out" && ok "110 Gitea/CD job container count is zero" || fail "110 Gitea/CD job container still active"
|
||||
grep -q "RUNNER_FAILCLOSED_BINARY_ELF" <<<"$out" && fail "110 runner fail-closed binary path restored to ELF" || ok "110 runner binary paths are fail-closed stubs or missing"
|
||||
grep -q "sentry-self-hosted-clickhouse-1.*Restarting" <<<"$out" && warn "Sentry ClickHouse restarting" || ok "Sentry ClickHouse not visibly restarting"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user