814 lines
28 KiB
Bash
Executable File
814 lines
28 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# AWOOOI 110 runner/CD lane fail-closed enforcer.
|
|
# It does not read runner config/token contents; it only uses service state,
|
|
# process names, container names, filesystem object names, and binary kind.
|
|
|
|
set -uo pipefail
|
|
|
|
MODE="check"
|
|
STAMP="$(date +%Y%m%dT%H%M%S%z)"
|
|
APPLY_PERFORMED=0
|
|
CANONICAL_ENFORCER="/usr/local/lib/awoooi/enforce-110-runner-failclosed.sh"
|
|
AUTHORITY_ENFORCER="/usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh"
|
|
COMPAT_ENFORCER="/usr/local/bin/awoooi-enforce-runner-failclosed-110.sh"
|
|
|
|
usage() {
|
|
cat <<'USAGE'
|
|
Usage: enforce-110-runner-failclosed.sh [--check|--apply]
|
|
|
|
--check Read-only status check. Exit non-zero if runner/CD lane is open.
|
|
--apply Stop/mask runner/CD lane entrypoints and seal restore sources.
|
|
USAGE
|
|
}
|
|
|
|
while [ "$#" -gt 0 ]; do
|
|
case "$1" in
|
|
--check)
|
|
MODE="check"
|
|
;;
|
|
--apply)
|
|
MODE="apply"
|
|
;;
|
|
-h|--help)
|
|
usage
|
|
exit 0
|
|
;;
|
|
*)
|
|
echo "unknown argument: $1" >&2
|
|
usage >&2
|
|
exit 64
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
RUNNER_UNITS=(
|
|
"awoooi-cd-lane.service"
|
|
"awoooi-cd-lane-drain.service"
|
|
"awoooi-direct-runner-open.service"
|
|
"awoooi-direct-runner.service"
|
|
"gitea-act-runner-host.service"
|
|
"gitea-act-runner-awoooi-controlled.service"
|
|
"gitea-awoooi-controlled-runner.service"
|
|
"gitea-act-runner-awoooi-open.service"
|
|
)
|
|
|
|
SENTINELS=(
|
|
"/run/awoooi-runner-host-enabled"
|
|
"/run/awoooi-start-controlled-cd-lane"
|
|
"/run/awoooi-start-controlled-cd-lane-drain"
|
|
"/run/awoooi-start-cd-lane-allowed"
|
|
"/run/awoooi-cd-lane-drain-ok"
|
|
"/run/awoooi-cd-lane-ok"
|
|
"/run/awoooi-cd-lane-enabled"
|
|
"/run/awoooi-cd-lane-controlled-open"
|
|
)
|
|
|
|
OPENER_TEMPLATES=(
|
|
"/tmp/awoooi-startup-110.sh.codex-drain-available"
|
|
"/tmp/awoooi-startup-110.sh.codex-controlled"
|
|
"/tmp/awoooi-startup-110.sh.codex-controlled-open"
|
|
"/tmp/enforce-110-runner-failclosed.sh"
|
|
"/tmp/awoooi-enforce-runner-failclosed-110.sh"
|
|
"/tmp/awoooi-enforce-runner-failclosed-110.sh.codex"
|
|
)
|
|
|
|
OPENER_UNIT_TEMPLATES=(
|
|
"/tmp/awoooi-cd-lane.service"
|
|
"/tmp/awoooi-cd-lane-drain.service"
|
|
"/tmp/gitea-act-runner-host.service"
|
|
"/tmp/gitea-act-runner-host.user.service"
|
|
"/tmp/gitea-act-runner-awoooi-open.service"
|
|
"/tmp/gitea-act-runner-awoooi-open.warn.service"
|
|
"/tmp/gitea-act-runner-awoooi-controlled.service"
|
|
)
|
|
|
|
STARTUP_OPEN_DROPINS=(
|
|
"/etc/systemd/system/awoooi-startup-110.service.d/10-runner-sentinel-open.conf"
|
|
)
|
|
|
|
LIVE_BINARY_PATHS=(
|
|
"/home/wooo/awoooi-cd-lane/awoooi_cd_lane"
|
|
"/home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled"
|
|
"/home/wooo/act-runner/act_runner"
|
|
"/home/wooo/act-runner/act_runner.real-20260628-runner-pressure-guard"
|
|
"/home/wooo/act-runner-controlled/act_runner"
|
|
"/home/wooo/awoooi-controlled-runner/awoooi_controlled_runner"
|
|
)
|
|
|
|
as_root() {
|
|
if [ "${EUID:-$(id -u)}" -eq 0 ]; then
|
|
"$@"
|
|
else
|
|
sudo -n "$@"
|
|
fi
|
|
}
|
|
|
|
host_is_110() {
|
|
if command -v ip >/dev/null 2>&1; then
|
|
ip -o -4 addr show 2>/dev/null | awk '{print $4}' | grep -q '^192\.168\.0\.110/'
|
|
return $?
|
|
fi
|
|
hostname -I 2>/dev/null | tr ' ' '\n' | grep -qx '192.168.0.110'
|
|
}
|
|
|
|
count_active_job_containers() {
|
|
if ! command -v docker >/dev/null 2>&1; then
|
|
echo 0
|
|
return
|
|
fi
|
|
docker ps --format '{{.Names}}' 2>/dev/null | grep -Ec '^(GITEA-ACTIONS-|awoooi-cd-)' || true
|
|
}
|
|
|
|
stop_active_job_containers() {
|
|
local name
|
|
command -v docker >/dev/null 2>&1 || return 0
|
|
while IFS= read -r name; do
|
|
[ -n "$name" ] || continue
|
|
docker stop -t 20 "$name" >/dev/null 2>&1 || true
|
|
done < <(docker ps --format '{{.Names}}' 2>/dev/null | grep -E '^(GITEA-ACTIONS-|awoooi-cd-)' || true)
|
|
}
|
|
|
|
count_lane_processes() {
|
|
pgrep -f '^/home/wooo/awoooi-cd-lane/awoooi_cd_lane|^/home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled' 2>/dev/null | wc -l | tr -d ' '
|
|
}
|
|
|
|
count_runner_processes() {
|
|
pgrep -f '^/home/wooo/act-runner/act_runner|^/home/wooo/act-runner-controlled/act_runner|^/home/wooo/awoooi-controlled-runner/awoooi_controlled_runner|Runner.Listener|Runner.Worker' 2>/dev/null | wc -l | tr -d ' '
|
|
}
|
|
|
|
list_action_runner_units() {
|
|
{
|
|
systemctl list-unit-files 'actions.runner.*' --no-legend --plain 2>/dev/null | awk '{print $1}'
|
|
systemctl list-units 'actions.runner.*' --all --no-legend --plain 2>/dev/null | awk '{print $1}'
|
|
} | sort -u
|
|
}
|
|
|
|
stop_and_mask_units() {
|
|
local unit
|
|
for unit in "${RUNNER_UNITS[@]}"; do
|
|
as_root systemctl kill --signal=SIGKILL "$unit" >/dev/null 2>&1 || true
|
|
as_root systemctl stop "$unit" >/dev/null 2>&1 || true
|
|
as_root systemctl reset-failed "$unit" >/dev/null 2>&1 || true
|
|
as_root systemctl disable "$unit" >/dev/null 2>&1 || true
|
|
as_root systemctl mask "$unit" >/dev/null 2>&1 || mask_unit_file_to_devnull "$unit"
|
|
mask_unit_file_to_devnull "$unit"
|
|
done
|
|
}
|
|
|
|
stop_and_mask_action_runner_units() {
|
|
local unit
|
|
while IFS= read -r unit; do
|
|
[ -n "$unit" ] || continue
|
|
as_root systemctl kill --signal=SIGKILL "$unit" >/dev/null 2>&1 || true
|
|
as_root systemctl stop "$unit" >/dev/null 2>&1 || true
|
|
as_root systemctl reset-failed "$unit" >/dev/null 2>&1 || true
|
|
as_root systemctl disable "$unit" >/dev/null 2>&1 || true
|
|
as_root systemctl mask "$unit" >/dev/null 2>&1 || mask_unit_file_to_devnull "$unit"
|
|
mask_unit_file_to_devnull "$unit"
|
|
done < <(list_action_runner_units)
|
|
}
|
|
|
|
kill_runner_processes() {
|
|
pkill -KILL -f '^/home/wooo/awoooi-cd-lane/awoooi_cd_lane' >/dev/null 2>&1 || true
|
|
pkill -KILL -f '^/home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled' >/dev/null 2>&1 || true
|
|
pkill -KILL -f '^/home/wooo/act-runner/act_runner' >/dev/null 2>&1 || true
|
|
pkill -KILL -f '^/home/wooo/act-runner-controlled/act_runner' >/dev/null 2>&1 || true
|
|
pkill -KILL -f '^/home/wooo/awoooi-controlled-runner/awoooi_controlled_runner' >/dev/null 2>&1 || true
|
|
pkill -KILL -f 'Runner.Listener|Runner.Worker' >/dev/null 2>&1 || true
|
|
}
|
|
|
|
remove_sentinels() {
|
|
local path
|
|
for path in "${SENTINELS[@]}"; do
|
|
as_root rm -f "$path" >/dev/null 2>&1 || true
|
|
done
|
|
}
|
|
|
|
write_failclosed_stub() {
|
|
local path="$1"
|
|
local tmp
|
|
tmp="$(mktemp)"
|
|
cat >"$tmp" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -eu
|
|
echo "AWOOOI 110 runner/CD lane is fail-closed after the 2026-06-28 pressure incident; migrate or hard-rate-limit before enabling." >&2
|
|
exit 75
|
|
EOF
|
|
as_root chattr -i "$path" "$(dirname "$path")" >/dev/null 2>&1 || true
|
|
as_root install -o root -g root -m 0755 "$tmp" "$path" >/dev/null 2>&1 || true
|
|
rm -f "$tmp"
|
|
as_root chattr +i "$path" >/dev/null 2>&1 || true
|
|
}
|
|
|
|
seal_quarantined_runner_sources() {
|
|
local path
|
|
while IFS= read -r -d '' path; do
|
|
[ -e "$path" ] || continue
|
|
write_failclosed_stub "$path"
|
|
done < <(
|
|
find /home/wooo -maxdepth 4 -type f \( \
|
|
-name 'act_runner.quarantined-*' -o \
|
|
-name 'act_runner.real-*.quarantined-*' \
|
|
\) -print0 2>/dev/null || true
|
|
)
|
|
}
|
|
|
|
seal_lane_binary_restore_sources() {
|
|
local path
|
|
while IFS= read -r -d '' path; do
|
|
[ -e "$path" ] || continue
|
|
write_failclosed_stub "$path"
|
|
done < <(
|
|
{
|
|
find /home/wooo/awoooi-cd-lane /home/wooo/awoooi-cd-lane-drain \
|
|
-maxdepth 4 -type f -name 'awoooi_cd_lane*' -print0 2>/dev/null
|
|
} || true
|
|
)
|
|
}
|
|
|
|
quarantine_lane_registration_sources() {
|
|
local lane_dir
|
|
local path
|
|
local quarantine_dir
|
|
local target
|
|
for lane_dir in "/home/wooo/awoooi-cd-lane" "/home/wooo/awoooi-cd-lane-drain"; do
|
|
[ -d "$lane_dir" ] || continue
|
|
quarantine_dir="$lane_dir/quarantine-failclosed-${STAMP}"
|
|
as_root chattr -i "$lane_dir" "$lane_dir/data" >/dev/null 2>&1 || true
|
|
as_root mkdir -p "$quarantine_dir" >/dev/null 2>&1 || true
|
|
while IFS= read -r -d '' path; do
|
|
[ -e "$path" ] || continue
|
|
as_root chattr -i "$path" >/dev/null 2>&1 || true
|
|
target="$quarantine_dir/$(basename "$path")"
|
|
as_root mv "$path" "$target" >/dev/null 2>&1 || true
|
|
as_root chmod 0400 "$target" >/dev/null 2>&1 || true
|
|
as_root chattr +i "$target" >/dev/null 2>&1 || true
|
|
done < <(
|
|
{
|
|
find "$lane_dir" -maxdepth 1 \( -name 'config.yaml' -o -name 'config.yaml.*' -o -name '.runner' -o -name '.runner.*' \) -print0 2>/dev/null
|
|
find "$lane_dir/data" -maxdepth 1 \( -name '.runner' -o -name '.runner.*' \) -print0 2>/dev/null
|
|
} || true
|
|
)
|
|
as_root chattr +i "$lane_dir" "$lane_dir/data" >/dev/null 2>&1 || true
|
|
done
|
|
}
|
|
|
|
seal_live_binary_paths() {
|
|
local path
|
|
for path in "${LIVE_BINARY_PATHS[@]}"; do
|
|
write_failclosed_stub "$path"
|
|
done
|
|
}
|
|
|
|
seal_opener_templates() {
|
|
local path
|
|
local tmp
|
|
tmp="$(mktemp)"
|
|
cat >"$tmp" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -eu
|
|
if [ -x /usr/local/lib/awoooi/enforce-110-runner-failclosed.sh ]; then
|
|
exec /usr/local/lib/awoooi/enforce-110-runner-failclosed.sh --apply
|
|
fi
|
|
if [ -x /usr/local/bin/awoooi-enforce-runner-failclosed-110.sh ]; then
|
|
exec /usr/local/bin/awoooi-enforce-runner-failclosed-110.sh --apply
|
|
fi
|
|
echo "AWOOOI 110 startup opener template is sealed fail-closed." >&2
|
|
exit 0
|
|
EOF
|
|
for path in "${OPENER_TEMPLATES[@]}"; do
|
|
as_root chattr -i "$path" >/dev/null 2>&1 || true
|
|
as_root install -o root -g root -m 0755 "$tmp" "$path" >/dev/null 2>&1 || true
|
|
as_root chattr +i "$path" >/dev/null 2>&1 || true
|
|
done
|
|
rm -f "$tmp"
|
|
}
|
|
|
|
seal_tmp_enforcer_backups() {
|
|
local path
|
|
local tmp
|
|
tmp="$(mktemp)"
|
|
cat >"$tmp" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -eu
|
|
if [ -x /usr/local/lib/awoooi/enforce-110-runner-failclosed.sh ]; then
|
|
exec /usr/local/lib/awoooi/enforce-110-runner-failclosed.sh --apply
|
|
fi
|
|
exec /usr/local/bin/awoooi-enforce-runner-failclosed-110.sh --apply
|
|
EOF
|
|
while IFS= read -r -d '' path; do
|
|
[ -e "$path" ] || [ -L "$path" ] || continue
|
|
as_root chattr -i "$path" >/dev/null 2>&1 || true
|
|
as_root install -o root -g root -m 0755 "$tmp" "$path" >/dev/null 2>&1 || true
|
|
as_root chattr +i "$path" >/dev/null 2>&1 || true
|
|
done < <(
|
|
find /tmp -maxdepth 1 -type f -name '*enforce-110-runner-failclosed*.sh*' -print0 2>/dev/null || true
|
|
)
|
|
rm -f "$tmp"
|
|
}
|
|
|
|
seal_opener_unit_templates() {
|
|
local path
|
|
local tmp
|
|
tmp="$(mktemp)"
|
|
cat >"$tmp" <<'EOF'
|
|
[Unit]
|
|
Description=AWOOOI 110 runner/CD lane opener sealed fail-closed after pressure incident
|
|
ConditionPathExists=/run/awoooi-runner-migrated-or-hard-limited
|
|
|
|
[Service]
|
|
Type=oneshot
|
|
ExecStart=/bin/false
|
|
EOF
|
|
for path in "${OPENER_UNIT_TEMPLATES[@]}"; do
|
|
as_root chattr -i "$path" >/dev/null 2>&1 || true
|
|
as_root install -o root -g root -m 0644 "$tmp" "$path" >/dev/null 2>&1 || true
|
|
as_root chattr +i "$path" >/dev/null 2>&1 || true
|
|
done
|
|
rm -f "$tmp"
|
|
}
|
|
|
|
remove_unit_wants_links() {
|
|
local unit="$1"
|
|
local path
|
|
while IFS= read -r -d '' path; do
|
|
as_root chattr -i "$path" >/dev/null 2>&1 || true
|
|
as_root rm -f "$path" >/dev/null 2>&1 || true
|
|
done < <(
|
|
as_root find /etc/systemd/system -type l \( \
|
|
-path "*/multi-user.target.wants/$unit" -o \
|
|
-path "*/graphical.target.wants/$unit" -o \
|
|
-path "*/default.target.wants/$unit" \
|
|
\) -print0 2>/dev/null || true
|
|
)
|
|
}
|
|
|
|
repair_enforcer_entrypoints() {
|
|
local current
|
|
local tmp
|
|
current="$(readlink -f "$0" 2>/dev/null || printf '%s' "$0")"
|
|
as_root mkdir -p "$(dirname "$CANONICAL_ENFORCER")" >/dev/null 2>&1 || true
|
|
as_root mkdir -p "$(dirname "$AUTHORITY_ENFORCER")" >/dev/null 2>&1 || true
|
|
if [ -f "$current" ] && [ "$current" != "$CANONICAL_ENFORCER" ]; then
|
|
as_root chattr -i "$CANONICAL_ENFORCER" >/dev/null 2>&1 || true
|
|
as_root install -o root -g root -m 0755 "$current" "$CANONICAL_ENFORCER" >/dev/null 2>&1 || true
|
|
fi
|
|
as_root chattr +i "$CANONICAL_ENFORCER" >/dev/null 2>&1 || true
|
|
if [ -f "$current" ] && [ "$current" != "$AUTHORITY_ENFORCER" ]; then
|
|
as_root chattr -i "$AUTHORITY_ENFORCER" >/dev/null 2>&1 || true
|
|
as_root install -o root -g root -m 0755 "$current" "$AUTHORITY_ENFORCER" >/dev/null 2>&1 || true
|
|
fi
|
|
as_root chattr +i "$AUTHORITY_ENFORCER" >/dev/null 2>&1 || true
|
|
|
|
tmp="$(mktemp)"
|
|
cat >"$tmp" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -eu
|
|
if [ -x /usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh ]; then
|
|
exec /usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh "$@"
|
|
fi
|
|
exec /usr/local/lib/awoooi/enforce-110-runner-failclosed.sh "$@"
|
|
EOF
|
|
as_root chattr -i "$COMPAT_ENFORCER" >/dev/null 2>&1 || true
|
|
as_root install -o root -g root -m 0755 "$tmp" "$COMPAT_ENFORCER" >/dev/null 2>&1 || true
|
|
rm -f "$tmp"
|
|
as_root chattr +i "$COMPAT_ENFORCER" >/dev/null 2>&1 || true
|
|
}
|
|
|
|
repair_enforcer_systemd_units() {
|
|
local service_tmp
|
|
local timer_tmp
|
|
local authority_service_tmp
|
|
local authority_timer_tmp
|
|
local unit_path
|
|
command -v systemctl >/dev/null 2>&1 || return 0
|
|
|
|
service_tmp="$(mktemp)"
|
|
cat >"$service_tmp" <<'EOF'
|
|
[Unit]
|
|
Description=AWOOOI 110 runner/CD lane fail-closed enforcer
|
|
Documentation=file:/usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh
|
|
Wants=network-online.target
|
|
After=network-online.target docker.service
|
|
|
|
[Service]
|
|
Type=oneshot
|
|
ExecStart=/usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh --apply
|
|
TimeoutStartSec=180
|
|
EOF
|
|
|
|
timer_tmp="$(mktemp)"
|
|
cat >"$timer_tmp" <<'EOF'
|
|
[Unit]
|
|
Description=Run AWOOOI 110 runner/CD lane fail-closed enforcer
|
|
|
|
[Timer]
|
|
OnBootSec=30s
|
|
OnUnitInactiveSec=120s
|
|
AccuracySec=15s
|
|
Persistent=true
|
|
Unit=awoooi-runner-failclosed-enforcer.service
|
|
|
|
[Install]
|
|
WantedBy=timers.target
|
|
EOF
|
|
|
|
authority_service_tmp="$(mktemp)"
|
|
cat >"$authority_service_tmp" <<'EOF'
|
|
[Unit]
|
|
Description=AWOOOI 110 runner/CD lane fail-closed authority
|
|
Documentation=file:/usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh
|
|
Wants=network-online.target
|
|
After=network-online.target docker.service
|
|
|
|
[Service]
|
|
Type=oneshot
|
|
ExecStart=/usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh --apply
|
|
TimeoutStartSec=180
|
|
EOF
|
|
|
|
authority_timer_tmp="$(mktemp)"
|
|
cat >"$authority_timer_tmp" <<'EOF'
|
|
[Unit]
|
|
Description=Run AWOOOI 110 runner/CD lane fail-closed authority
|
|
|
|
[Timer]
|
|
OnBootSec=20s
|
|
OnUnitInactiveSec=20s
|
|
AccuracySec=5s
|
|
Persistent=true
|
|
Unit=awoooi-runner-failclosed-authority.service
|
|
|
|
[Install]
|
|
WantedBy=timers.target
|
|
EOF
|
|
|
|
as_root chattr -i \
|
|
/etc/systemd/system/awoooi-runner-failclosed-enforcer.service \
|
|
/etc/systemd/system/awoooi-runner-failclosed-enforcer.timer \
|
|
/etc/systemd/system/awoooi-runner-failclosed-authority.service \
|
|
/etc/systemd/system/awoooi-runner-failclosed-authority.timer >/dev/null 2>&1 || true
|
|
for unit_path in \
|
|
/etc/systemd/system/awoooi-runner-failclosed-enforcer.service \
|
|
/etc/systemd/system/awoooi-runner-failclosed-enforcer.timer \
|
|
/etc/systemd/system/awoooi-runner-failclosed-authority.service \
|
|
/etc/systemd/system/awoooi-runner-failclosed-authority.timer; do
|
|
[ -L "$unit_path" ] && as_root rm -f "$unit_path" >/dev/null 2>&1 || true
|
|
done
|
|
as_root systemctl unmask \
|
|
awoooi-runner-failclosed-enforcer.service \
|
|
awoooi-runner-failclosed-enforcer.timer \
|
|
awoooi-runner-failclosed-authority.service \
|
|
awoooi-runner-failclosed-authority.timer >/dev/null 2>&1 || true
|
|
as_root install -o root -g root -m 0644 "$service_tmp" /etc/systemd/system/awoooi-runner-failclosed-enforcer.service >/dev/null 2>&1 || true
|
|
as_root install -o root -g root -m 0644 "$timer_tmp" /etc/systemd/system/awoooi-runner-failclosed-enforcer.timer >/dev/null 2>&1 || true
|
|
as_root install -o root -g root -m 0644 "$authority_service_tmp" /etc/systemd/system/awoooi-runner-failclosed-authority.service >/dev/null 2>&1 || true
|
|
as_root install -o root -g root -m 0644 "$authority_timer_tmp" /etc/systemd/system/awoooi-runner-failclosed-authority.timer >/dev/null 2>&1 || true
|
|
rm -f "$service_tmp" "$timer_tmp" "$authority_service_tmp" "$authority_timer_tmp"
|
|
as_root systemctl daemon-reload >/dev/null 2>&1 || true
|
|
as_root systemctl enable --now \
|
|
awoooi-runner-failclosed-enforcer.timer \
|
|
awoooi-runner-failclosed-authority.timer >/dev/null 2>&1 || true
|
|
}
|
|
|
|
repair_enforcer_cron_authority() {
|
|
local tmp
|
|
tmp="$(mktemp)"
|
|
cat >"$tmp" <<'EOF'
|
|
SHELL=/bin/bash
|
|
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
|
* * * * * root /usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh --apply >>/var/log/awoooi-runner-failclosed-authority-cron.log 2>&1
|
|
EOF
|
|
as_root chattr -i /etc/cron.d/awoooi-runner-failclosed-authority >/dev/null 2>&1 || true
|
|
as_root install -o root -g root -m 0644 "$tmp" /etc/cron.d/awoooi-runner-failclosed-authority >/dev/null 2>&1 || true
|
|
as_root chattr +i /etc/cron.d/awoooi-runner-failclosed-authority >/dev/null 2>&1 || true
|
|
rm -f "$tmp"
|
|
}
|
|
|
|
seal_enforcer_disabler_artifacts() {
|
|
local path
|
|
local target_root="/root/awoooi-runner-restore-sources-sealed-${STAMP}/enforcer-disablers"
|
|
while IFS= read -r -d '' path; do
|
|
[ -e "$path" ] || [ -L "$path" ] || continue
|
|
as_root mkdir -p "$target_root" >/dev/null 2>&1 || true
|
|
as_root chattr -R -i "$path" >/dev/null 2>&1 || true
|
|
as_root mv "$path" "$target_root/$(basename "$path").sealed" >/dev/null 2>&1 || true
|
|
done < <(
|
|
as_root find /etc/systemd/system -maxdepth 1 -type d \( \
|
|
-name 'awoooi-runner-failclosed-opened-*' -o \
|
|
-name 'awoooi-runner-failclosed-*-opened-*' -o \
|
|
-name 'awoooi-runner-failclosed-quarantine-*' -o \
|
|
-name 'failclosed-final-mask-*' \
|
|
\) -print0 2>/dev/null || true
|
|
)
|
|
}
|
|
|
|
seal_unit_activation_artifacts() {
|
|
local unit
|
|
for unit in "${RUNNER_UNITS[@]}"; do
|
|
remove_unit_wants_links "$unit"
|
|
done
|
|
while IFS= read -r unit; do
|
|
[ -n "$unit" ] || continue
|
|
remove_unit_wants_links "$unit"
|
|
done < <(list_action_runner_units)
|
|
}
|
|
|
|
seal_startup_open_dropins() {
|
|
local path
|
|
local tmp
|
|
local target_root="/root/awoooi-runner-restore-sources-sealed-${STAMP}/systemd-dropins"
|
|
for path in "${STARTUP_OPEN_DROPINS[@]}"; do
|
|
[ -e "$path" ] || [ -L "$path" ] || continue
|
|
as_root mkdir -p "$target_root" >/dev/null 2>&1 || true
|
|
as_root chattr -i "$path" >/dev/null 2>&1 || true
|
|
as_root mv "$path" "$target_root/$(basename "$path").sealed" >/dev/null 2>&1 || true
|
|
done
|
|
|
|
if [ -d /etc/systemd/system/awoooi-startup-110.service.d ]; then
|
|
tmp="$(mktemp)"
|
|
cat >"$tmp" <<'EOF'
|
|
[Service]
|
|
Environment=AWOOOI_START_GITEA_RUNNER_ON_BOOT=0
|
|
EOF
|
|
as_root install -o root -g root -m 0644 "$tmp" /etc/systemd/system/awoooi-startup-110.service.d/99-runner-failclosed.conf >/dev/null 2>&1 || true
|
|
rm -f "$tmp"
|
|
fi
|
|
}
|
|
|
|
seal_startup_backup_openers() {
|
|
local path
|
|
local target_root="/root/awoooi-runner-restore-sources-sealed-${STAMP}/usr-local-startup-openers"
|
|
while IFS= read -r -d '' path; do
|
|
[ -e "$path" ] || [ -L "$path" ] || continue
|
|
as_root mkdir -p "$target_root" >/dev/null 2>&1 || true
|
|
as_root chattr -i "$path" >/dev/null 2>&1 || true
|
|
as_root mv "$path" "$target_root/$(basename "$path").sealed" >/dev/null 2>&1 || true
|
|
done < <(
|
|
as_root find /usr/local/bin -maxdepth 1 -type f \( \
|
|
-name 'awoooi-startup-110.sh.*controlled*' -o \
|
|
-name 'awoooi-startup-110.sh.before-controlled*' -o \
|
|
-name 'awoooi-startup-110.sh.bak-*controlled*' \
|
|
\) -print0 2>/dev/null || true
|
|
)
|
|
}
|
|
|
|
seal_systemd_unit_backups() {
|
|
local path
|
|
local target_root="/root/awoooi-runner-restore-sources-sealed-${STAMP}/systemd-unit-backups"
|
|
while IFS= read -r -d '' path; do
|
|
[ -e "$path" ] || [ -L "$path" ] || continue
|
|
as_root mkdir -p "$target_root" >/dev/null 2>&1 || true
|
|
as_root chattr -i "$path" >/dev/null 2>&1 || true
|
|
as_root mv "$path" "$target_root/$(basename "$path").sealed" >/dev/null 2>&1 || true
|
|
done < <(
|
|
as_root find /etc/systemd/system -maxdepth 1 \( \
|
|
-name 'awoooi-cd-lane.service.*' -o \
|
|
-name 'awoooi-cd-lane-drain.service.*' -o \
|
|
-name 'gitea-act-runner-host.service.*' -o \
|
|
-name 'gitea-act-runner-awoooi-controlled.service.*' -o \
|
|
-name 'gitea-act-runner-awoooi-open.service.*' \
|
|
\) -print0 2>/dev/null || true
|
|
)
|
|
}
|
|
|
|
seal_root_live_artifact_files() {
|
|
local path
|
|
local target_root="/root/awoooi-runner-restore-sources-sealed-${STAMP}/root-live-artifacts"
|
|
while IFS= read -r -d '' path; do
|
|
[ -e "$path" ] || [ -L "$path" ] || continue
|
|
as_root mkdir -p "$target_root" >/dev/null 2>&1 || true
|
|
as_root chattr -i "$path" >/dev/null 2>&1 || true
|
|
as_root mv "$path" "$target_root/$(basename "$path").sealed" >/dev/null 2>&1 || true
|
|
done < <(
|
|
as_root find /root -maxdepth 1 \( \
|
|
-name 'awoooi-runner-live-artifact-disabled-*' -o \
|
|
-name 'awoooi-drain-unit-quarantine-*' \
|
|
\) -print0 2>/dev/null || true
|
|
)
|
|
}
|
|
|
|
seal_root_restore_sources() {
|
|
local path
|
|
local final_root="/root/awoooi-runner-restore-sources-sealed-${STAMP}"
|
|
local target_root="$final_root/root"
|
|
local moved=0
|
|
|
|
while IFS= read -r -d '' path; do
|
|
[ -d "$path" ] || continue
|
|
if [ "$moved" -eq 0 ]; then
|
|
as_root mkdir -p "$target_root" >/dev/null 2>&1 || true
|
|
moved=1
|
|
fi
|
|
as_root chattr -R -i "$path" >/dev/null 2>&1 || true
|
|
as_root mv "$path" "$target_root/" >/dev/null 2>&1 || true
|
|
done < <(
|
|
as_root find /root -maxdepth 1 -type d \( \
|
|
-name 'awoooi-runner-restore-sources-disabled*' -o \
|
|
-name 'awoooi-cd-lane-disabled*' -o \
|
|
-name 'awoooi-cd-lane-drain-disabled*' \
|
|
\) -print0 2>/dev/null || true
|
|
)
|
|
}
|
|
|
|
seal_home_upload_controlled_open_sources() {
|
|
local path
|
|
local target_root="/root/awoooi-runner-restore-sources-sealed-${STAMP}/home-upload-openers"
|
|
local current
|
|
local unit_tmp
|
|
current="$(readlink -f "$0" 2>/dev/null || printf '%s' "$0")"
|
|
|
|
unit_tmp="$(mktemp)"
|
|
cat >"$unit_tmp" <<'EOF'
|
|
[Unit]
|
|
Description=AWOOOI 110 controlled opener upload sealed fail-closed
|
|
ConditionPathExists=/run/awoooi-runner-migrated-or-hard-limited
|
|
|
|
[Service]
|
|
Type=oneshot
|
|
ExecStart=/bin/false
|
|
EOF
|
|
|
|
while IFS= read -r -d '' path; do
|
|
[ -d "$path" ] || continue
|
|
as_root mkdir -p "$target_root" >/dev/null 2>&1 || true
|
|
as_root chattr -R -i "$path" >/dev/null 2>&1 || true
|
|
as_root mv "$path" "$target_root/$(basename "$path").sealed" >/dev/null 2>&1 || true
|
|
as_root mkdir -p "$path" >/dev/null 2>&1 || true
|
|
if [ -f "$current" ]; then
|
|
as_root install -o root -g root -m 0755 "$current" "$path/awoooi-enforce-runner-failclosed-110.sh" >/dev/null 2>&1 || true
|
|
as_root install -o root -g root -m 0755 "$current" "$path/enforce-110-runner-failclosed.sh" >/dev/null 2>&1 || true
|
|
fi
|
|
write_failclosed_stub "$path/awoooi-startup-110.sh"
|
|
write_failclosed_stub "$path/awoooi_cd_lane_controlled"
|
|
as_root install -o root -g root -m 0644 "$unit_tmp" "$path/awoooi-cd-lane-drain.service" >/dev/null 2>&1 || true
|
|
as_root install -o root -g root -m 0644 "$unit_tmp" "$path/awoooi-cd-lane.service" >/dev/null 2>&1 || true
|
|
as_root chattr +i "$path" "$path"/* >/dev/null 2>&1 || true
|
|
done < <(
|
|
as_root find /home/wooo -maxdepth 1 -type d -name 'codex-upload-controlled-open*' -print0 2>/dev/null || true
|
|
)
|
|
rm -f "$unit_tmp"
|
|
}
|
|
|
|
mask_unit_file_to_devnull() {
|
|
local unit="$1"
|
|
local path="/etc/systemd/system/$unit"
|
|
as_root chattr -i "$path" >/dev/null 2>&1 || true
|
|
if [ -e "$path" ] || [ -L "$path" ]; then
|
|
if ! { [ -L "$path" ] && [ "$(readlink "$path" 2>/dev/null || true)" = "/dev/null" ]; }; then
|
|
as_root mv "$path" "${path}.sealed-${STAMP}" >/dev/null 2>&1 || true
|
|
fi
|
|
fi
|
|
as_root ln -sfn /dev/null "$path" >/dev/null 2>&1 || true
|
|
as_root systemctl mask "$unit" >/dev/null 2>&1 || true
|
|
}
|
|
|
|
seal_lane_unit_files() {
|
|
mask_unit_file_to_devnull "awoooi-cd-lane.service"
|
|
mask_unit_file_to_devnull "awoooi-cd-lane-drain.service"
|
|
}
|
|
|
|
root_restore_sources_left() {
|
|
as_root find /root -maxdepth 1 -type d \( \
|
|
-name 'awoooi-runner-restore-sources-disabled*' -o \
|
|
-name 'awoooi-cd-lane-disabled*' -o \
|
|
-name 'awoooi-cd-lane-drain-disabled*' \
|
|
\) -print 2>/dev/null | wc -l | tr -d ' '
|
|
}
|
|
|
|
unit_ok() {
|
|
local unit="$1"
|
|
local load active unitfile mainpid
|
|
load="$(systemctl show "$unit" -p LoadState --value 2>/dev/null || true)"
|
|
active="$(systemctl show "$unit" -p ActiveState --value 2>/dev/null || true)"
|
|
unitfile="$(systemctl show "$unit" -p UnitFileState --value 2>/dev/null || true)"
|
|
mainpid="$(systemctl show "$unit" -p MainPID --value 2>/dev/null || true)"
|
|
{ [ "$active" = "inactive" ] || [ "$active" = "failed" ] || [ "$active" = "unknown" ] || [ -z "$active" ]; } || return 1
|
|
[ "${mainpid:-0}" = "0" ] || return 1
|
|
if [ "$load" = "masked" ] || [ "$unitfile" = "masked" ]; then
|
|
return 0
|
|
fi
|
|
if [ "$active" = "inactive" ] \
|
|
&& systemctl cat "$unit" 2>/dev/null | grep -q 'ConditionPathExists=/run/awoooi-runner-migrated-or-hard-limited'; then
|
|
return 0
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
runner_units_bad_count() {
|
|
local unit bad=0
|
|
for unit in "${RUNNER_UNITS[@]}"; do
|
|
unit_ok "$unit" || bad=$((bad + 1))
|
|
done
|
|
while IFS= read -r unit; do
|
|
[ -n "$unit" ] || continue
|
|
unit_ok "$unit" || bad=$((bad + 1))
|
|
done < <(list_action_runner_units)
|
|
echo "$bad"
|
|
}
|
|
|
|
write_metrics() {
|
|
local dir="$1"
|
|
local tmp
|
|
[ -d "$dir" ] || return 0
|
|
tmp="$(mktemp)"
|
|
cat >"$tmp" <<EOF
|
|
# HELP awoooi_runner_failclosed_enforcer_last_run_timestamp Last successful run timestamp.
|
|
# TYPE awoooi_runner_failclosed_enforcer_last_run_timestamp gauge
|
|
awoooi_runner_failclosed_enforcer_last_run_timestamp $(date +%s)
|
|
# HELP awoooi_runner_failclosed_enforcer_active_job_containers Active Gitea/awoooi-cd job containers after enforcement.
|
|
# TYPE awoooi_runner_failclosed_enforcer_active_job_containers gauge
|
|
awoooi_runner_failclosed_enforcer_active_job_containers $(count_active_job_containers)
|
|
# HELP awoooi_runner_failclosed_enforcer_lane_process_count Active direct cd-lane processes after enforcement.
|
|
# TYPE awoooi_runner_failclosed_enforcer_lane_process_count gauge
|
|
awoooi_runner_failclosed_enforcer_lane_process_count $(count_lane_processes)
|
|
# HELP awoooi_runner_failclosed_enforcer_root_restore_sources_left Root restore-source directories left after enforcement.
|
|
# TYPE awoooi_runner_failclosed_enforcer_root_restore_sources_left gauge
|
|
awoooi_runner_failclosed_enforcer_root_restore_sources_left $(root_restore_sources_left)
|
|
# HELP awoooi_runner_failclosed_enforcer_apply_performed Whether this run used apply mode.
|
|
# TYPE awoooi_runner_failclosed_enforcer_apply_performed gauge
|
|
awoooi_runner_failclosed_enforcer_apply_performed $APPLY_PERFORMED
|
|
EOF
|
|
as_root install -o root -g root -m 0644 "$tmp" "$dir/awoooi_runner_failclosed_enforcer.prom" >/dev/null 2>&1 || true
|
|
rm -f "$tmp"
|
|
}
|
|
|
|
print_readback() {
|
|
local unit
|
|
echo "ENFORCER_MODE=$MODE"
|
|
echo "ENFORCER_HOST_110=1"
|
|
echo "APPLY_PERFORMED=$APPLY_PERFORMED"
|
|
echo "ACTIVE_JOB_CONTAINERS=$(count_active_job_containers)"
|
|
echo "LANE_PROCESS_COUNT=$(count_lane_processes)"
|
|
echo "RUNNER_PROCESS_COUNT=$(count_runner_processes)"
|
|
echo "ROOT_RESTORE_SOURCES_LEFT=$(root_restore_sources_left)"
|
|
echo "RUNNER_UNITS_BAD_COUNT=$(runner_units_bad_count)"
|
|
for unit in "${RUNNER_UNITS[@]}"; do
|
|
load="$(systemctl show "$unit" -p LoadState --value 2>/dev/null || true)"
|
|
active="$(systemctl show "$unit" -p ActiveState --value 2>/dev/null || true)"
|
|
unitfile="$(systemctl show "$unit" -p UnitFileState --value 2>/dev/null || true)"
|
|
echo "RUNNER_UNIT $unit load=${load:-unknown} active=${active:-unknown} unitfile=${unitfile:-unknown}"
|
|
done
|
|
while IFS= read -r unit; do
|
|
[ -n "$unit" ] || continue
|
|
load="$(systemctl show "$unit" -p LoadState --value 2>/dev/null || true)"
|
|
active="$(systemctl show "$unit" -p ActiveState --value 2>/dev/null || true)"
|
|
unitfile="$(systemctl show "$unit" -p UnitFileState --value 2>/dev/null || true)"
|
|
echo "ACTION_RUNNER_UNIT $unit load=${load:-unknown} active=${active:-unknown} unitfile=${unitfile:-unknown}"
|
|
done < <(list_action_runner_units)
|
|
}
|
|
|
|
apply_failclosed() {
|
|
APPLY_PERFORMED=1
|
|
repair_enforcer_entrypoints
|
|
seal_enforcer_disabler_artifacts
|
|
repair_enforcer_systemd_units
|
|
repair_enforcer_cron_authority
|
|
stop_active_job_containers
|
|
stop_and_mask_units
|
|
stop_and_mask_action_runner_units
|
|
kill_runner_processes
|
|
remove_sentinels
|
|
seal_unit_activation_artifacts
|
|
seal_startup_open_dropins
|
|
seal_startup_backup_openers
|
|
seal_systemd_unit_backups
|
|
seal_root_live_artifact_files
|
|
seal_lane_unit_files
|
|
seal_live_binary_paths
|
|
seal_lane_binary_restore_sources
|
|
quarantine_lane_registration_sources
|
|
seal_opener_templates
|
|
seal_tmp_enforcer_backups
|
|
seal_opener_unit_templates
|
|
seal_root_restore_sources
|
|
seal_home_upload_controlled_open_sources
|
|
seal_quarantined_runner_sources
|
|
as_root systemctl daemon-reload >/dev/null 2>&1 || true
|
|
}
|
|
|
|
if ! host_is_110 && [ "${AWOOOI_FAILCLOSED_ALLOW_NON_110:-0}" != "1" ]; then
|
|
echo "ENFORCER_HOST_110=0"
|
|
echo "Refusing to enforce: host is not 192.168.0.110. Set AWOOOI_FAILCLOSED_ALLOW_NON_110=1 only for controlled tests." >&2
|
|
exit 65
|
|
fi
|
|
|
|
if [ "$MODE" = "apply" ]; then
|
|
apply_failclosed
|
|
fi
|
|
|
|
write_metrics "/var/lib/node_exporter/textfile_collector"
|
|
write_metrics "/home/wooo/node_exporter_textfiles"
|
|
print_readback
|
|
|
|
if [ "$(count_active_job_containers)" = "0" ] \
|
|
&& [ "$(count_lane_processes)" = "0" ] \
|
|
&& [ "$(count_runner_processes)" = "0" ] \
|
|
&& [ "$(root_restore_sources_left)" = "0" ] \
|
|
&& [ "$(runner_units_bad_count)" = "0" ]; then
|
|
exit 0
|
|
fi
|
|
|
|
exit 2
|