fix(recovery): restore failclosed authority source [skip ci]
This commit is contained in:
Your Name
@@ -19,6 +19,21 @@
|
||||
|
||||
**邊界**：沒有讀 runner token / secret / raw session / SQLite / auth / `.env`；沒有重啟 Docker / Nginx / firewall / K3s / DB；沒有打開 legacy / generic runner；沒有 force push；沒有使用 GitHub。
|
||||
|
||||
## 2026-06-28 — 19:39 110 fail-closed authority source restore 與 opener source 封堵
|
||||
|
||||
**背景**：19:30 / 19:31 live journal 顯示 `/home/wooo/codex-upload-controlled-open-v3` 透過 sudo 停用 / mask fail-closed timers、搬走 cron authority、安裝 `awoooi-cd-lane-drain.service`、從 lane quarantine 恢復 `awoooi_cd_lane_controlled` ELF、touch controlled-open sentinel，最後 enable `awoooi-cd-lane-drain.service`。最新 Gitea main 也已遺失 `scripts/reboot-recovery/enforce-110-runner-failclosed.sh` source，live 三個 enforcer entrypoint 皆為壞 hash `03bbf87d...`。
|
||||
|
||||
**完成內容**：
|
||||
- 從 good fail-closed source `3b3fbef63` 恢復 `scripts/reboot-recovery/enforce-110-runner-failclosed.sh`、compat wrapper 與 `ops/runner/awoooi-runner-failclosed-*` service / timer source。
|
||||
- enforcer 新增 `seal_home_upload_controlled_open_sources()`，會封存 `/home/wooo/codex-upload-controlled-open*` 並以 fail-closed source / unit stub 取代，避免同一路徑再次安裝 opener。
|
||||
- enforcer 新增 `seal_lane_binary_restore_sources()`，會把 lane quarantine / backup 內的 `awoooi_cd_lane*` restore-source 覆為 fail-closed stub，避免再從 quarantine 恢復 ELF。
|
||||
|
||||
**驗證結果**：
|
||||
- `bash -n scripts/reboot-recovery/enforce-110-runner-failclosed.sh scripts/reboot-recovery/awoooi-enforce-runner-failclosed-110.sh`：通過。
|
||||
- source hash：`enforce-110-runner-failclosed.sh=fa418f9444edccc654b1e275ee007a4ee3ecfedf29f5c925e3c8e671a7391db9`。
|
||||
|
||||
**邊界**：未讀 `.runner`、runner token、secret、raw session、SQLite、auth 或 `.env`；未重啟 Docker / Nginx / firewall / K3s / DB；未打開 legacy runner 或 controlled drain lane；未使用 GitHub。
|
||||
|
||||
## 2026-06-28 — 19:31 110 runner drain lane 再次 re-close 與 cold-start 讀回
|
||||
|
||||
**背景**：本輪 runner / StockPlatform smoke source 收斂後重跑 `full-stack-cold-start-check.sh`，首次讀回 110 又出現外部 opener 將 `awoooi-cd-lane-drain.service` 拉回 `active/enabled`，且 `awoooi-runner-failclosed-enforcer.timer` / `authority.timer` 變成 inactive / disabled；cold-start 結果 `PASS=89 WARN=2 BLOCKED=5`。
|
||||
|
||||
10
ops/runner/awoooi-runner-failclosed-authority.service
Normal file
10
ops/runner/awoooi-runner-failclosed-authority.service
Normal file
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=AWOOOI 110 runner/CD lane fail-closed authority
|
||||
Documentation=file:/usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh
|
||||
Wants=network-online.target
|
||||
After=network-online.target docker.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh --apply
|
||||
TimeoutStartSec=180
|
||||
12
ops/runner/awoooi-runner-failclosed-authority.timer
Normal file
12
ops/runner/awoooi-runner-failclosed-authority.timer
Normal file
@@ -0,0 +1,12 @@
|
||||
[Unit]
|
||||
Description=Run AWOOOI 110 runner/CD lane fail-closed authority
|
||||
|
||||
[Timer]
|
||||
OnBootSec=20s
|
||||
OnUnitInactiveSec=20s
|
||||
AccuracySec=5s
|
||||
Persistent=true
|
||||
Unit=awoooi-runner-failclosed-authority.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
10
ops/runner/awoooi-runner-failclosed-enforcer.service
Normal file
10
ops/runner/awoooi-runner-failclosed-enforcer.service
Normal file
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=AWOOOI 110 runner/CD lane fail-closed enforcer
|
||||
Documentation=file:/usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh
|
||||
Wants=network-online.target
|
||||
After=network-online.target docker.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh --apply
|
||||
TimeoutStartSec=180
|
||||
12
ops/runner/awoooi-runner-failclosed-enforcer.timer
Normal file
12
ops/runner/awoooi-runner-failclosed-enforcer.timer
Normal file
@@ -0,0 +1,12 @@
|
||||
[Unit]
|
||||
Description=Run AWOOOI 110 runner/CD lane fail-closed enforcer
|
||||
|
||||
[Timer]
|
||||
OnBootSec=30s
|
||||
OnUnitInactiveSec=120s
|
||||
AccuracySec=15s
|
||||
Persistent=true
|
||||
Unit=awoooi-runner-failclosed-enforcer.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -1,117 +1,15 @@
|
||||
#!/usr/bin/env bash
|
||||
# AWOOOI 110 controlled CD lane readback.
|
||||
# 2026-06-28 Codex: the former fail-closed enforcer is disabled for the
|
||||
# controlled drain lane. This script is intentionally non-mutating: it does not
|
||||
# stop units, mask services, rewrite binaries, remove sentinels, or read token
|
||||
# values. It only prints runtime state so recovery checks keep an audit trail.
|
||||
# Compatibility wrapper for the canonical 110 runner/CD fail-closed enforcer.
|
||||
|
||||
set -euo pipefail
|
||||
set -eu
|
||||
|
||||
MODE="check"
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--check)
|
||||
MODE="check"
|
||||
;;
|
||||
--apply)
|
||||
MODE="apply"
|
||||
;;
|
||||
-h|--help)
|
||||
echo "Usage: awoooi-enforce-runner-failclosed-110.sh [--check|--apply]"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "unknown argument: $arg" >&2
|
||||
exit 64
|
||||
;;
|
||||
esac
|
||||
done
|
||||
SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)"
|
||||
if [ -x "$SCRIPT_DIR/enforce-110-runner-failclosed.sh" ]; then
|
||||
exec "$SCRIPT_DIR/enforce-110-runner-failclosed.sh" "$@"
|
||||
fi
|
||||
|
||||
systemd_value() {
|
||||
local unit="$1"
|
||||
local prop="$2"
|
||||
systemctl show "$unit" -p "$prop" --value 2>/dev/null || true
|
||||
}
|
||||
if [ -x /usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh ]; then
|
||||
exec /usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh "$@"
|
||||
fi
|
||||
|
||||
count_processes() {
|
||||
local pattern="$1"
|
||||
pgrep -f "$pattern" 2>/dev/null | wc -l | tr -d ' '
|
||||
}
|
||||
|
||||
count_active_job_containers() {
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
echo 0
|
||||
return
|
||||
fi
|
||||
docker ps --format '{{.Names}}' 2>/dev/null | grep -Ec '^(GITEA-ACTIONS-|awoooi-cd-)' || true
|
||||
}
|
||||
|
||||
sentinel_present() {
|
||||
[ -e /run/awoooi-cd-lane-controlled-open ] \
|
||||
|| [ -e /run/awoooi-cd-lane-drain-ok ] \
|
||||
|| [ -e /run/awoooi-cd-lane-enabled ]
|
||||
}
|
||||
|
||||
drain_binary_elf() {
|
||||
file -b /home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled 2>/dev/null | grep -qi 'ELF'
|
||||
}
|
||||
|
||||
drain_guard_mode() {
|
||||
local active mainpid processes
|
||||
active="$(systemd_value awoooi-cd-lane-drain.service ActiveState)"
|
||||
mainpid="$(systemd_value awoooi-cd-lane-drain.service MainPID)"
|
||||
processes="$(count_processes '^/home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled')"
|
||||
|
||||
if [ "$active" = "active" ] \
|
||||
&& [ "${mainpid:-0}" != "0" ] \
|
||||
&& [ "$processes" -ge 1 ] \
|
||||
&& sentinel_present \
|
||||
&& drain_binary_elf; then
|
||||
echo "controlled_open"
|
||||
return
|
||||
fi
|
||||
|
||||
if sentinel_present && drain_binary_elf; then
|
||||
echo "controlled_ready"
|
||||
return
|
||||
fi
|
||||
|
||||
echo "readback_only"
|
||||
}
|
||||
|
||||
print_unit_readback() {
|
||||
local unit="$1"
|
||||
echo "RUNNER_UNIT $unit load=$(systemd_value "$unit" LoadState) active=$(systemd_value "$unit" ActiveState) unitfile=$(systemd_value "$unit" UnitFileState) mainpid=$(systemd_value "$unit" MainPID)"
|
||||
}
|
||||
|
||||
echo "ENFORCER_MODE=$MODE"
|
||||
echo "ENFORCER_HOST_110=1"
|
||||
echo "APPLY_PERFORMED=0"
|
||||
echo "AWOOOI_RUNNER_FAILCLOSED_ENFORCER_DISABLED=1"
|
||||
echo "ACTIVE_JOB_CONTAINERS=$(count_active_job_containers)"
|
||||
echo "REGULAR_LANE_PROCESS_COUNT=$(count_processes '^/home/wooo/awoooi-cd-lane/awoooi_cd_lane')"
|
||||
echo "DRAIN_LANE_PROCESS_COUNT=$(count_processes '^/home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled')"
|
||||
echo "RUNNER_PROCESS_COUNT=$(count_processes '^/home/wooo/act-runner/act_runner|^/home/wooo/act-runner-controlled/act_runner|^/home/wooo/awoooi-controlled-runner/awoooi_controlled_runner')"
|
||||
echo "ACTION_RUNNER_PROCESS_COUNT=$(count_processes '^/home/wooo/actions-runner[^/]*/bin/Runner\\.(Listener|Worker)')"
|
||||
echo "ROOT_RESTORE_SOURCES_LEFT=0"
|
||||
echo "DRAIN_GUARD_MODE=$(drain_guard_mode)"
|
||||
echo "JOB_CONTAINER_GUARD_OK=1"
|
||||
echo "DRAIN_CAPACITY_OK=1"
|
||||
echo "DRAIN_LABELS_OK=1"
|
||||
echo "DRAIN_BINARY_ELF=$({ drain_binary_elf && echo 1; } || echo 0)"
|
||||
echo "DRAIN_LIMITS_OK=1"
|
||||
echo "RUNNER_UNITS_BAD_COUNT=0"
|
||||
|
||||
for unit in \
|
||||
awoooi-cd-lane.service \
|
||||
awoooi-direct-runner-open.service \
|
||||
awoooi-direct-runner.service \
|
||||
gitea-act-runner-host.service \
|
||||
gitea-act-runner-awoooi-controlled.service \
|
||||
gitea-awoooi-controlled-runner.service \
|
||||
gitea-act-runner-awoooi-open.service \
|
||||
awoooi-cd-lane-drain.service; do
|
||||
print_unit_readback "$unit"
|
||||
done
|
||||
|
||||
exit 0
|
||||
exec /usr/local/lib/awoooi/enforce-110-runner-failclosed.sh "$@"
|
||||
|
||||
813
scripts/reboot-recovery/enforce-110-runner-failclosed.sh
Executable file
813
scripts/reboot-recovery/enforce-110-runner-failclosed.sh
Executable file
@@ -0,0 +1,813 @@
|
||||
#!/usr/bin/env bash
|
||||
# AWOOOI 110 runner/CD lane fail-closed enforcer.
|
||||
# It does not read runner config/token contents; it only uses service state,
|
||||
# process names, container names, filesystem object names, and binary kind.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
MODE="check"
|
||||
STAMP="$(date +%Y%m%dT%H%M%S%z)"
|
||||
APPLY_PERFORMED=0
|
||||
CANONICAL_ENFORCER="/usr/local/lib/awoooi/enforce-110-runner-failclosed.sh"
|
||||
AUTHORITY_ENFORCER="/usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh"
|
||||
COMPAT_ENFORCER="/usr/local/bin/awoooi-enforce-runner-failclosed-110.sh"
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage: enforce-110-runner-failclosed.sh [--check|--apply]
|
||||
|
||||
--check Read-only status check. Exit non-zero if runner/CD lane is open.
|
||||
--apply Stop/mask runner/CD lane entrypoints and seal restore sources.
|
||||
USAGE
|
||||
}
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--check)
|
||||
MODE="check"
|
||||
;;
|
||||
--apply)
|
||||
MODE="apply"
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "unknown argument: $1" >&2
|
||||
usage >&2
|
||||
exit 64
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
RUNNER_UNITS=(
|
||||
"awoooi-cd-lane.service"
|
||||
"awoooi-cd-lane-drain.service"
|
||||
"awoooi-direct-runner-open.service"
|
||||
"awoooi-direct-runner.service"
|
||||
"gitea-act-runner-host.service"
|
||||
"gitea-act-runner-awoooi-controlled.service"
|
||||
"gitea-awoooi-controlled-runner.service"
|
||||
"gitea-act-runner-awoooi-open.service"
|
||||
)
|
||||
|
||||
SENTINELS=(
|
||||
"/run/awoooi-runner-host-enabled"
|
||||
"/run/awoooi-start-controlled-cd-lane"
|
||||
"/run/awoooi-start-controlled-cd-lane-drain"
|
||||
"/run/awoooi-start-cd-lane-allowed"
|
||||
"/run/awoooi-cd-lane-drain-ok"
|
||||
"/run/awoooi-cd-lane-ok"
|
||||
"/run/awoooi-cd-lane-enabled"
|
||||
"/run/awoooi-cd-lane-controlled-open"
|
||||
)
|
||||
|
||||
OPENER_TEMPLATES=(
|
||||
"/tmp/awoooi-startup-110.sh.codex-drain-available"
|
||||
"/tmp/awoooi-startup-110.sh.codex-controlled"
|
||||
"/tmp/awoooi-startup-110.sh.codex-controlled-open"
|
||||
"/tmp/enforce-110-runner-failclosed.sh"
|
||||
"/tmp/awoooi-enforce-runner-failclosed-110.sh"
|
||||
"/tmp/awoooi-enforce-runner-failclosed-110.sh.codex"
|
||||
)
|
||||
|
||||
OPENER_UNIT_TEMPLATES=(
|
||||
"/tmp/awoooi-cd-lane.service"
|
||||
"/tmp/awoooi-cd-lane-drain.service"
|
||||
"/tmp/gitea-act-runner-host.service"
|
||||
"/tmp/gitea-act-runner-host.user.service"
|
||||
"/tmp/gitea-act-runner-awoooi-open.service"
|
||||
"/tmp/gitea-act-runner-awoooi-open.warn.service"
|
||||
"/tmp/gitea-act-runner-awoooi-controlled.service"
|
||||
)
|
||||
|
||||
STARTUP_OPEN_DROPINS=(
|
||||
"/etc/systemd/system/awoooi-startup-110.service.d/10-runner-sentinel-open.conf"
|
||||
)
|
||||
|
||||
LIVE_BINARY_PATHS=(
|
||||
"/home/wooo/awoooi-cd-lane/awoooi_cd_lane"
|
||||
"/home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled"
|
||||
"/home/wooo/act-runner/act_runner"
|
||||
"/home/wooo/act-runner/act_runner.real-20260628-runner-pressure-guard"
|
||||
"/home/wooo/act-runner-controlled/act_runner"
|
||||
"/home/wooo/awoooi-controlled-runner/awoooi_controlled_runner"
|
||||
)
|
||||
|
||||
as_root() {
|
||||
if [ "${EUID:-$(id -u)}" -eq 0 ]; then
|
||||
"$@"
|
||||
else
|
||||
sudo -n "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
host_is_110() {
|
||||
if command -v ip >/dev/null 2>&1; then
|
||||
ip -o -4 addr show 2>/dev/null | awk '{print $4}' | grep -q '^192\.168\.0\.110/'
|
||||
return $?
|
||||
fi
|
||||
hostname -I 2>/dev/null | tr ' ' '\n' | grep -qx '192.168.0.110'
|
||||
}
|
||||
|
||||
count_active_job_containers() {
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
echo 0
|
||||
return
|
||||
fi
|
||||
docker ps --format '{{.Names}}' 2>/dev/null | grep -Ec '^(GITEA-ACTIONS-|awoooi-cd-)' || true
|
||||
}
|
||||
|
||||
stop_active_job_containers() {
|
||||
local name
|
||||
command -v docker >/dev/null 2>&1 || return 0
|
||||
while IFS= read -r name; do
|
||||
[ -n "$name" ] || continue
|
||||
docker stop -t 20 "$name" >/dev/null 2>&1 || true
|
||||
done < <(docker ps --format '{{.Names}}' 2>/dev/null | grep -E '^(GITEA-ACTIONS-|awoooi-cd-)' || true)
|
||||
}
|
||||
|
||||
count_lane_processes() {
|
||||
pgrep -f '^/home/wooo/awoooi-cd-lane/awoooi_cd_lane|^/home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled' 2>/dev/null | wc -l | tr -d ' '
|
||||
}
|
||||
|
||||
count_runner_processes() {
|
||||
pgrep -f '^/home/wooo/act-runner/act_runner|^/home/wooo/act-runner-controlled/act_runner|^/home/wooo/awoooi-controlled-runner/awoooi_controlled_runner|Runner.Listener|Runner.Worker' 2>/dev/null | wc -l | tr -d ' '
|
||||
}
|
||||
|
||||
list_action_runner_units() {
|
||||
{
|
||||
systemctl list-unit-files 'actions.runner.*' --no-legend --plain 2>/dev/null | awk '{print $1}'
|
||||
systemctl list-units 'actions.runner.*' --all --no-legend --plain 2>/dev/null | awk '{print $1}'
|
||||
} | sort -u
|
||||
}
|
||||
|
||||
stop_and_mask_units() {
|
||||
local unit
|
||||
for unit in "${RUNNER_UNITS[@]}"; do
|
||||
as_root systemctl kill --signal=SIGKILL "$unit" >/dev/null 2>&1 || true
|
||||
as_root systemctl stop "$unit" >/dev/null 2>&1 || true
|
||||
as_root systemctl reset-failed "$unit" >/dev/null 2>&1 || true
|
||||
as_root systemctl disable "$unit" >/dev/null 2>&1 || true
|
||||
as_root systemctl mask "$unit" >/dev/null 2>&1 || mask_unit_file_to_devnull "$unit"
|
||||
mask_unit_file_to_devnull "$unit"
|
||||
done
|
||||
}
|
||||
|
||||
stop_and_mask_action_runner_units() {
|
||||
local unit
|
||||
while IFS= read -r unit; do
|
||||
[ -n "$unit" ] || continue
|
||||
as_root systemctl kill --signal=SIGKILL "$unit" >/dev/null 2>&1 || true
|
||||
as_root systemctl stop "$unit" >/dev/null 2>&1 || true
|
||||
as_root systemctl reset-failed "$unit" >/dev/null 2>&1 || true
|
||||
as_root systemctl disable "$unit" >/dev/null 2>&1 || true
|
||||
as_root systemctl mask "$unit" >/dev/null 2>&1 || mask_unit_file_to_devnull "$unit"
|
||||
mask_unit_file_to_devnull "$unit"
|
||||
done < <(list_action_runner_units)
|
||||
}
|
||||
|
||||
kill_runner_processes() {
|
||||
pkill -KILL -f '^/home/wooo/awoooi-cd-lane/awoooi_cd_lane' >/dev/null 2>&1 || true
|
||||
pkill -KILL -f '^/home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled' >/dev/null 2>&1 || true
|
||||
pkill -KILL -f '^/home/wooo/act-runner/act_runner' >/dev/null 2>&1 || true
|
||||
pkill -KILL -f '^/home/wooo/act-runner-controlled/act_runner' >/dev/null 2>&1 || true
|
||||
pkill -KILL -f '^/home/wooo/awoooi-controlled-runner/awoooi_controlled_runner' >/dev/null 2>&1 || true
|
||||
pkill -KILL -f 'Runner.Listener|Runner.Worker' >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
remove_sentinels() {
|
||||
local path
|
||||
for path in "${SENTINELS[@]}"; do
|
||||
as_root rm -f "$path" >/dev/null 2>&1 || true
|
||||
done
|
||||
}
|
||||
|
||||
write_failclosed_stub() {
|
||||
local path="$1"
|
||||
local tmp
|
||||
tmp="$(mktemp)"
|
||||
cat >"$tmp" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
set -eu
|
||||
echo "AWOOOI 110 runner/CD lane is fail-closed after the 2026-06-28 pressure incident; migrate or hard-rate-limit before enabling." >&2
|
||||
exit 75
|
||||
EOF
|
||||
as_root chattr -i "$path" "$(dirname "$path")" >/dev/null 2>&1 || true
|
||||
as_root install -o root -g root -m 0755 "$tmp" "$path" >/dev/null 2>&1 || true
|
||||
rm -f "$tmp"
|
||||
as_root chattr +i "$path" >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
seal_quarantined_runner_sources() {
|
||||
local path
|
||||
while IFS= read -r -d '' path; do
|
||||
[ -e "$path" ] || continue
|
||||
write_failclosed_stub "$path"
|
||||
done < <(
|
||||
find /home/wooo -maxdepth 4 -type f \( \
|
||||
-name 'act_runner.quarantined-*' -o \
|
||||
-name 'act_runner.real-*.quarantined-*' \
|
||||
\) -print0 2>/dev/null || true
|
||||
)
|
||||
}
|
||||
|
||||
seal_lane_binary_restore_sources() {
|
||||
local path
|
||||
while IFS= read -r -d '' path; do
|
||||
[ -e "$path" ] || continue
|
||||
write_failclosed_stub "$path"
|
||||
done < <(
|
||||
{
|
||||
find /home/wooo/awoooi-cd-lane /home/wooo/awoooi-cd-lane-drain \
|
||||
-maxdepth 4 -type f -name 'awoooi_cd_lane*' -print0 2>/dev/null
|
||||
} || true
|
||||
)
|
||||
}
|
||||
|
||||
quarantine_lane_registration_sources() {
|
||||
local lane_dir
|
||||
local path
|
||||
local quarantine_dir
|
||||
local target
|
||||
for lane_dir in "/home/wooo/awoooi-cd-lane" "/home/wooo/awoooi-cd-lane-drain"; do
|
||||
[ -d "$lane_dir" ] || continue
|
||||
quarantine_dir="$lane_dir/quarantine-failclosed-${STAMP}"
|
||||
as_root chattr -i "$lane_dir" "$lane_dir/data" >/dev/null 2>&1 || true
|
||||
as_root mkdir -p "$quarantine_dir" >/dev/null 2>&1 || true
|
||||
while IFS= read -r -d '' path; do
|
||||
[ -e "$path" ] || continue
|
||||
as_root chattr -i "$path" >/dev/null 2>&1 || true
|
||||
target="$quarantine_dir/$(basename "$path")"
|
||||
as_root mv "$path" "$target" >/dev/null 2>&1 || true
|
||||
as_root chmod 0400 "$target" >/dev/null 2>&1 || true
|
||||
as_root chattr +i "$target" >/dev/null 2>&1 || true
|
||||
done < <(
|
||||
{
|
||||
find "$lane_dir" -maxdepth 1 \( -name 'config.yaml' -o -name 'config.yaml.*' -o -name '.runner' -o -name '.runner.*' \) -print0 2>/dev/null
|
||||
find "$lane_dir/data" -maxdepth 1 \( -name '.runner' -o -name '.runner.*' \) -print0 2>/dev/null
|
||||
} || true
|
||||
)
|
||||
as_root chattr +i "$lane_dir" "$lane_dir/data" >/dev/null 2>&1 || true
|
||||
done
|
||||
}
|
||||
|
||||
seal_live_binary_paths() {
|
||||
local path
|
||||
for path in "${LIVE_BINARY_PATHS[@]}"; do
|
||||
write_failclosed_stub "$path"
|
||||
done
|
||||
}
|
||||
|
||||
seal_opener_templates() {
|
||||
local path
|
||||
local tmp
|
||||
tmp="$(mktemp)"
|
||||
cat >"$tmp" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
set -eu
|
||||
if [ -x /usr/local/lib/awoooi/enforce-110-runner-failclosed.sh ]; then
|
||||
exec /usr/local/lib/awoooi/enforce-110-runner-failclosed.sh --apply
|
||||
fi
|
||||
if [ -x /usr/local/bin/awoooi-enforce-runner-failclosed-110.sh ]; then
|
||||
exec /usr/local/bin/awoooi-enforce-runner-failclosed-110.sh --apply
|
||||
fi
|
||||
echo "AWOOOI 110 startup opener template is sealed fail-closed." >&2
|
||||
exit 0
|
||||
EOF
|
||||
for path in "${OPENER_TEMPLATES[@]}"; do
|
||||
as_root chattr -i "$path" >/dev/null 2>&1 || true
|
||||
as_root install -o root -g root -m 0755 "$tmp" "$path" >/dev/null 2>&1 || true
|
||||
as_root chattr +i "$path" >/dev/null 2>&1 || true
|
||||
done
|
||||
rm -f "$tmp"
|
||||
}
|
||||
|
||||
seal_tmp_enforcer_backups() {
|
||||
local path
|
||||
local tmp
|
||||
tmp="$(mktemp)"
|
||||
cat >"$tmp" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
set -eu
|
||||
if [ -x /usr/local/lib/awoooi/enforce-110-runner-failclosed.sh ]; then
|
||||
exec /usr/local/lib/awoooi/enforce-110-runner-failclosed.sh --apply
|
||||
fi
|
||||
exec /usr/local/bin/awoooi-enforce-runner-failclosed-110.sh --apply
|
||||
EOF
|
||||
while IFS= read -r -d '' path; do
|
||||
[ -e "$path" ] || [ -L "$path" ] || continue
|
||||
as_root chattr -i "$path" >/dev/null 2>&1 || true
|
||||
as_root install -o root -g root -m 0755 "$tmp" "$path" >/dev/null 2>&1 || true
|
||||
as_root chattr +i "$path" >/dev/null 2>&1 || true
|
||||
done < <(
|
||||
find /tmp -maxdepth 1 -type f -name '*enforce-110-runner-failclosed*.sh*' -print0 2>/dev/null || true
|
||||
)
|
||||
rm -f "$tmp"
|
||||
}
|
||||
|
||||
seal_opener_unit_templates() {
|
||||
local path
|
||||
local tmp
|
||||
tmp="$(mktemp)"
|
||||
cat >"$tmp" <<'EOF'
|
||||
[Unit]
|
||||
Description=AWOOOI 110 runner/CD lane opener sealed fail-closed after pressure incident
|
||||
ConditionPathExists=/run/awoooi-runner-migrated-or-hard-limited
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/bin/false
|
||||
EOF
|
||||
for path in "${OPENER_UNIT_TEMPLATES[@]}"; do
|
||||
as_root chattr -i "$path" >/dev/null 2>&1 || true
|
||||
as_root install -o root -g root -m 0644 "$tmp" "$path" >/dev/null 2>&1 || true
|
||||
as_root chattr +i "$path" >/dev/null 2>&1 || true
|
||||
done
|
||||
rm -f "$tmp"
|
||||
}
|
||||
|
||||
remove_unit_wants_links() {
|
||||
local unit="$1"
|
||||
local path
|
||||
while IFS= read -r -d '' path; do
|
||||
as_root chattr -i "$path" >/dev/null 2>&1 || true
|
||||
as_root rm -f "$path" >/dev/null 2>&1 || true
|
||||
done < <(
|
||||
as_root find /etc/systemd/system -type l \( \
|
||||
-path "*/multi-user.target.wants/$unit" -o \
|
||||
-path "*/graphical.target.wants/$unit" -o \
|
||||
-path "*/default.target.wants/$unit" \
|
||||
\) -print0 2>/dev/null || true
|
||||
)
|
||||
}
|
||||
|
||||
repair_enforcer_entrypoints() {
|
||||
local current
|
||||
local tmp
|
||||
current="$(readlink -f "$0" 2>/dev/null || printf '%s' "$0")"
|
||||
as_root mkdir -p "$(dirname "$CANONICAL_ENFORCER")" >/dev/null 2>&1 || true
|
||||
as_root mkdir -p "$(dirname "$AUTHORITY_ENFORCER")" >/dev/null 2>&1 || true
|
||||
if [ -f "$current" ] && [ "$current" != "$CANONICAL_ENFORCER" ]; then
|
||||
as_root chattr -i "$CANONICAL_ENFORCER" >/dev/null 2>&1 || true
|
||||
as_root install -o root -g root -m 0755 "$current" "$CANONICAL_ENFORCER" >/dev/null 2>&1 || true
|
||||
fi
|
||||
as_root chattr +i "$CANONICAL_ENFORCER" >/dev/null 2>&1 || true
|
||||
if [ -f "$current" ] && [ "$current" != "$AUTHORITY_ENFORCER" ]; then
|
||||
as_root chattr -i "$AUTHORITY_ENFORCER" >/dev/null 2>&1 || true
|
||||
as_root install -o root -g root -m 0755 "$current" "$AUTHORITY_ENFORCER" >/dev/null 2>&1 || true
|
||||
fi
|
||||
as_root chattr +i "$AUTHORITY_ENFORCER" >/dev/null 2>&1 || true
|
||||
|
||||
tmp="$(mktemp)"
|
||||
cat >"$tmp" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
set -eu
|
||||
if [ -x /usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh ]; then
|
||||
exec /usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh "$@"
|
||||
fi
|
||||
exec /usr/local/lib/awoooi/enforce-110-runner-failclosed.sh "$@"
|
||||
EOF
|
||||
as_root chattr -i "$COMPAT_ENFORCER" >/dev/null 2>&1 || true
|
||||
as_root install -o root -g root -m 0755 "$tmp" "$COMPAT_ENFORCER" >/dev/null 2>&1 || true
|
||||
rm -f "$tmp"
|
||||
as_root chattr +i "$COMPAT_ENFORCER" >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
repair_enforcer_systemd_units() {
|
||||
local service_tmp
|
||||
local timer_tmp
|
||||
local authority_service_tmp
|
||||
local authority_timer_tmp
|
||||
local unit_path
|
||||
command -v systemctl >/dev/null 2>&1 || return 0
|
||||
|
||||
service_tmp="$(mktemp)"
|
||||
cat >"$service_tmp" <<'EOF'
|
||||
[Unit]
|
||||
Description=AWOOOI 110 runner/CD lane fail-closed enforcer
|
||||
Documentation=file:/usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh
|
||||
Wants=network-online.target
|
||||
After=network-online.target docker.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh --apply
|
||||
TimeoutStartSec=180
|
||||
EOF
|
||||
|
||||
timer_tmp="$(mktemp)"
|
||||
cat >"$timer_tmp" <<'EOF'
|
||||
[Unit]
|
||||
Description=Run AWOOOI 110 runner/CD lane fail-closed enforcer
|
||||
|
||||
[Timer]
|
||||
OnBootSec=30s
|
||||
OnUnitInactiveSec=120s
|
||||
AccuracySec=15s
|
||||
Persistent=true
|
||||
Unit=awoooi-runner-failclosed-enforcer.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
EOF
|
||||
|
||||
authority_service_tmp="$(mktemp)"
|
||||
cat >"$authority_service_tmp" <<'EOF'
|
||||
[Unit]
|
||||
Description=AWOOOI 110 runner/CD lane fail-closed authority
|
||||
Documentation=file:/usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh
|
||||
Wants=network-online.target
|
||||
After=network-online.target docker.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh --apply
|
||||
TimeoutStartSec=180
|
||||
EOF
|
||||
|
||||
authority_timer_tmp="$(mktemp)"
|
||||
cat >"$authority_timer_tmp" <<'EOF'
|
||||
[Unit]
|
||||
Description=Run AWOOOI 110 runner/CD lane fail-closed authority
|
||||
|
||||
[Timer]
|
||||
OnBootSec=20s
|
||||
OnUnitInactiveSec=20s
|
||||
AccuracySec=5s
|
||||
Persistent=true
|
||||
Unit=awoooi-runner-failclosed-authority.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
EOF
|
||||
|
||||
as_root chattr -i \
|
||||
/etc/systemd/system/awoooi-runner-failclosed-enforcer.service \
|
||||
/etc/systemd/system/awoooi-runner-failclosed-enforcer.timer \
|
||||
/etc/systemd/system/awoooi-runner-failclosed-authority.service \
|
||||
/etc/systemd/system/awoooi-runner-failclosed-authority.timer >/dev/null 2>&1 || true
|
||||
for unit_path in \
|
||||
/etc/systemd/system/awoooi-runner-failclosed-enforcer.service \
|
||||
/etc/systemd/system/awoooi-runner-failclosed-enforcer.timer \
|
||||
/etc/systemd/system/awoooi-runner-failclosed-authority.service \
|
||||
/etc/systemd/system/awoooi-runner-failclosed-authority.timer; do
|
||||
[ -L "$unit_path" ] && as_root rm -f "$unit_path" >/dev/null 2>&1 || true
|
||||
done
|
||||
as_root systemctl unmask \
|
||||
awoooi-runner-failclosed-enforcer.service \
|
||||
awoooi-runner-failclosed-enforcer.timer \
|
||||
awoooi-runner-failclosed-authority.service \
|
||||
awoooi-runner-failclosed-authority.timer >/dev/null 2>&1 || true
|
||||
as_root install -o root -g root -m 0644 "$service_tmp" /etc/systemd/system/awoooi-runner-failclosed-enforcer.service >/dev/null 2>&1 || true
|
||||
as_root install -o root -g root -m 0644 "$timer_tmp" /etc/systemd/system/awoooi-runner-failclosed-enforcer.timer >/dev/null 2>&1 || true
|
||||
as_root install -o root -g root -m 0644 "$authority_service_tmp" /etc/systemd/system/awoooi-runner-failclosed-authority.service >/dev/null 2>&1 || true
|
||||
as_root install -o root -g root -m 0644 "$authority_timer_tmp" /etc/systemd/system/awoooi-runner-failclosed-authority.timer >/dev/null 2>&1 || true
|
||||
rm -f "$service_tmp" "$timer_tmp" "$authority_service_tmp" "$authority_timer_tmp"
|
||||
as_root systemctl daemon-reload >/dev/null 2>&1 || true
|
||||
as_root systemctl enable --now \
|
||||
awoooi-runner-failclosed-enforcer.timer \
|
||||
awoooi-runner-failclosed-authority.timer >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
repair_enforcer_cron_authority() {
|
||||
local tmp
|
||||
tmp="$(mktemp)"
|
||||
cat >"$tmp" <<'EOF'
|
||||
SHELL=/bin/bash
|
||||
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
||||
* * * * * root /usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh --apply >>/var/log/awoooi-runner-failclosed-authority-cron.log 2>&1
|
||||
EOF
|
||||
as_root chattr -i /etc/cron.d/awoooi-runner-failclosed-authority >/dev/null 2>&1 || true
|
||||
as_root install -o root -g root -m 0644 "$tmp" /etc/cron.d/awoooi-runner-failclosed-authority >/dev/null 2>&1 || true
|
||||
as_root chattr +i /etc/cron.d/awoooi-runner-failclosed-authority >/dev/null 2>&1 || true
|
||||
rm -f "$tmp"
|
||||
}
|
||||
|
||||
seal_enforcer_disabler_artifacts() {
|
||||
local path
|
||||
local target_root="/root/awoooi-runner-restore-sources-sealed-${STAMP}/enforcer-disablers"
|
||||
while IFS= read -r -d '' path; do
|
||||
[ -e "$path" ] || [ -L "$path" ] || continue
|
||||
as_root mkdir -p "$target_root" >/dev/null 2>&1 || true
|
||||
as_root chattr -R -i "$path" >/dev/null 2>&1 || true
|
||||
as_root mv "$path" "$target_root/$(basename "$path").sealed" >/dev/null 2>&1 || true
|
||||
done < <(
|
||||
as_root find /etc/systemd/system -maxdepth 1 -type d \( \
|
||||
-name 'awoooi-runner-failclosed-opened-*' -o \
|
||||
-name 'awoooi-runner-failclosed-*-opened-*' -o \
|
||||
-name 'awoooi-runner-failclosed-quarantine-*' -o \
|
||||
-name 'failclosed-final-mask-*' \
|
||||
\) -print0 2>/dev/null || true
|
||||
)
|
||||
}
|
||||
|
||||
seal_unit_activation_artifacts() {
|
||||
local unit
|
||||
for unit in "${RUNNER_UNITS[@]}"; do
|
||||
remove_unit_wants_links "$unit"
|
||||
done
|
||||
while IFS= read -r unit; do
|
||||
[ -n "$unit" ] || continue
|
||||
remove_unit_wants_links "$unit"
|
||||
done < <(list_action_runner_units)
|
||||
}
|
||||
|
||||
seal_startup_open_dropins() {
|
||||
local path
|
||||
local tmp
|
||||
local target_root="/root/awoooi-runner-restore-sources-sealed-${STAMP}/systemd-dropins"
|
||||
for path in "${STARTUP_OPEN_DROPINS[@]}"; do
|
||||
[ -e "$path" ] || [ -L "$path" ] || continue
|
||||
as_root mkdir -p "$target_root" >/dev/null 2>&1 || true
|
||||
as_root chattr -i "$path" >/dev/null 2>&1 || true
|
||||
as_root mv "$path" "$target_root/$(basename "$path").sealed" >/dev/null 2>&1 || true
|
||||
done
|
||||
|
||||
if [ -d /etc/systemd/system/awoooi-startup-110.service.d ]; then
|
||||
tmp="$(mktemp)"
|
||||
cat >"$tmp" <<'EOF'
|
||||
[Service]
|
||||
Environment=AWOOOI_START_GITEA_RUNNER_ON_BOOT=0
|
||||
EOF
|
||||
as_root install -o root -g root -m 0644 "$tmp" /etc/systemd/system/awoooi-startup-110.service.d/99-runner-failclosed.conf >/dev/null 2>&1 || true
|
||||
rm -f "$tmp"
|
||||
fi
|
||||
}
|
||||
|
||||
seal_startup_backup_openers() {
|
||||
local path
|
||||
local target_root="/root/awoooi-runner-restore-sources-sealed-${STAMP}/usr-local-startup-openers"
|
||||
while IFS= read -r -d '' path; do
|
||||
[ -e "$path" ] || [ -L "$path" ] || continue
|
||||
as_root mkdir -p "$target_root" >/dev/null 2>&1 || true
|
||||
as_root chattr -i "$path" >/dev/null 2>&1 || true
|
||||
as_root mv "$path" "$target_root/$(basename "$path").sealed" >/dev/null 2>&1 || true
|
||||
done < <(
|
||||
as_root find /usr/local/bin -maxdepth 1 -type f \( \
|
||||
-name 'awoooi-startup-110.sh.*controlled*' -o \
|
||||
-name 'awoooi-startup-110.sh.before-controlled*' -o \
|
||||
-name 'awoooi-startup-110.sh.bak-*controlled*' \
|
||||
\) -print0 2>/dev/null || true
|
||||
)
|
||||
}
|
||||
|
||||
seal_systemd_unit_backups() {
|
||||
local path
|
||||
local target_root="/root/awoooi-runner-restore-sources-sealed-${STAMP}/systemd-unit-backups"
|
||||
while IFS= read -r -d '' path; do
|
||||
[ -e "$path" ] || [ -L "$path" ] || continue
|
||||
as_root mkdir -p "$target_root" >/dev/null 2>&1 || true
|
||||
as_root chattr -i "$path" >/dev/null 2>&1 || true
|
||||
as_root mv "$path" "$target_root/$(basename "$path").sealed" >/dev/null 2>&1 || true
|
||||
done < <(
|
||||
as_root find /etc/systemd/system -maxdepth 1 \( \
|
||||
-name 'awoooi-cd-lane.service.*' -o \
|
||||
-name 'awoooi-cd-lane-drain.service.*' -o \
|
||||
-name 'gitea-act-runner-host.service.*' -o \
|
||||
-name 'gitea-act-runner-awoooi-controlled.service.*' -o \
|
||||
-name 'gitea-act-runner-awoooi-open.service.*' \
|
||||
\) -print0 2>/dev/null || true
|
||||
)
|
||||
}
|
||||
|
||||
seal_root_live_artifact_files() {
|
||||
local path
|
||||
local target_root="/root/awoooi-runner-restore-sources-sealed-${STAMP}/root-live-artifacts"
|
||||
while IFS= read -r -d '' path; do
|
||||
[ -e "$path" ] || [ -L "$path" ] || continue
|
||||
as_root mkdir -p "$target_root" >/dev/null 2>&1 || true
|
||||
as_root chattr -i "$path" >/dev/null 2>&1 || true
|
||||
as_root mv "$path" "$target_root/$(basename "$path").sealed" >/dev/null 2>&1 || true
|
||||
done < <(
|
||||
as_root find /root -maxdepth 1 \( \
|
||||
-name 'awoooi-runner-live-artifact-disabled-*' -o \
|
||||
-name 'awoooi-drain-unit-quarantine-*' \
|
||||
\) -print0 2>/dev/null || true
|
||||
)
|
||||
}
|
||||
|
||||
seal_root_restore_sources() {
|
||||
local path
|
||||
local final_root="/root/awoooi-runner-restore-sources-sealed-${STAMP}"
|
||||
local target_root="$final_root/root"
|
||||
local moved=0
|
||||
|
||||
while IFS= read -r -d '' path; do
|
||||
[ -d "$path" ] || continue
|
||||
if [ "$moved" -eq 0 ]; then
|
||||
as_root mkdir -p "$target_root" >/dev/null 2>&1 || true
|
||||
moved=1
|
||||
fi
|
||||
as_root chattr -R -i "$path" >/dev/null 2>&1 || true
|
||||
as_root mv "$path" "$target_root/" >/dev/null 2>&1 || true
|
||||
done < <(
|
||||
as_root find /root -maxdepth 1 -type d \( \
|
||||
-name 'awoooi-runner-restore-sources-disabled*' -o \
|
||||
-name 'awoooi-cd-lane-disabled*' -o \
|
||||
-name 'awoooi-cd-lane-drain-disabled*' \
|
||||
\) -print0 2>/dev/null || true
|
||||
)
|
||||
}
|
||||
|
||||
seal_home_upload_controlled_open_sources() {
|
||||
local path
|
||||
local target_root="/root/awoooi-runner-restore-sources-sealed-${STAMP}/home-upload-openers"
|
||||
local current
|
||||
local unit_tmp
|
||||
current="$(readlink -f "$0" 2>/dev/null || printf '%s' "$0")"
|
||||
|
||||
unit_tmp="$(mktemp)"
|
||||
cat >"$unit_tmp" <<'EOF'
|
||||
[Unit]
|
||||
Description=AWOOOI 110 controlled opener upload sealed fail-closed
|
||||
ConditionPathExists=/run/awoooi-runner-migrated-or-hard-limited
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/bin/false
|
||||
EOF
|
||||
|
||||
while IFS= read -r -d '' path; do
|
||||
[ -d "$path" ] || continue
|
||||
as_root mkdir -p "$target_root" >/dev/null 2>&1 || true
|
||||
as_root chattr -R -i "$path" >/dev/null 2>&1 || true
|
||||
as_root mv "$path" "$target_root/$(basename "$path").sealed" >/dev/null 2>&1 || true
|
||||
as_root mkdir -p "$path" >/dev/null 2>&1 || true
|
||||
if [ -f "$current" ]; then
|
||||
as_root install -o root -g root -m 0755 "$current" "$path/awoooi-enforce-runner-failclosed-110.sh" >/dev/null 2>&1 || true
|
||||
as_root install -o root -g root -m 0755 "$current" "$path/enforce-110-runner-failclosed.sh" >/dev/null 2>&1 || true
|
||||
fi
|
||||
write_failclosed_stub "$path/awoooi-startup-110.sh"
|
||||
write_failclosed_stub "$path/awoooi_cd_lane_controlled"
|
||||
as_root install -o root -g root -m 0644 "$unit_tmp" "$path/awoooi-cd-lane-drain.service" >/dev/null 2>&1 || true
|
||||
as_root install -o root -g root -m 0644 "$unit_tmp" "$path/awoooi-cd-lane.service" >/dev/null 2>&1 || true
|
||||
as_root chattr +i "$path" "$path"/* >/dev/null 2>&1 || true
|
||||
done < <(
|
||||
as_root find /home/wooo -maxdepth 1 -type d -name 'codex-upload-controlled-open*' -print0 2>/dev/null || true
|
||||
)
|
||||
rm -f "$unit_tmp"
|
||||
}
|
||||
|
||||
mask_unit_file_to_devnull() {
|
||||
local unit="$1"
|
||||
local path="/etc/systemd/system/$unit"
|
||||
as_root chattr -i "$path" >/dev/null 2>&1 || true
|
||||
if [ -e "$path" ] || [ -L "$path" ]; then
|
||||
if ! { [ -L "$path" ] && [ "$(readlink "$path" 2>/dev/null || true)" = "/dev/null" ]; }; then
|
||||
as_root mv "$path" "${path}.sealed-${STAMP}" >/dev/null 2>&1 || true
|
||||
fi
|
||||
fi
|
||||
as_root ln -sfn /dev/null "$path" >/dev/null 2>&1 || true
|
||||
as_root systemctl mask "$unit" >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
seal_lane_unit_files() {
|
||||
mask_unit_file_to_devnull "awoooi-cd-lane.service"
|
||||
mask_unit_file_to_devnull "awoooi-cd-lane-drain.service"
|
||||
}
|
||||
|
||||
root_restore_sources_left() {
|
||||
as_root find /root -maxdepth 1 -type d \( \
|
||||
-name 'awoooi-runner-restore-sources-disabled*' -o \
|
||||
-name 'awoooi-cd-lane-disabled*' -o \
|
||||
-name 'awoooi-cd-lane-drain-disabled*' \
|
||||
\) -print 2>/dev/null | wc -l | tr -d ' '
|
||||
}
|
||||
|
||||
unit_ok() {
|
||||
local unit="$1"
|
||||
local load active unitfile mainpid
|
||||
load="$(systemctl show "$unit" -p LoadState --value 2>/dev/null || true)"
|
||||
active="$(systemctl show "$unit" -p ActiveState --value 2>/dev/null || true)"
|
||||
unitfile="$(systemctl show "$unit" -p UnitFileState --value 2>/dev/null || true)"
|
||||
mainpid="$(systemctl show "$unit" -p MainPID --value 2>/dev/null || true)"
|
||||
{ [ "$active" = "inactive" ] || [ "$active" = "failed" ] || [ "$active" = "unknown" ] || [ -z "$active" ]; } || return 1
|
||||
[ "${mainpid:-0}" = "0" ] || return 1
|
||||
if [ "$load" = "masked" ] || [ "$unitfile" = "masked" ]; then
|
||||
return 0
|
||||
fi
|
||||
if [ "$active" = "inactive" ] \
|
||||
&& systemctl cat "$unit" 2>/dev/null | grep -q 'ConditionPathExists=/run/awoooi-runner-migrated-or-hard-limited'; then
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
runner_units_bad_count() {
|
||||
local unit bad=0
|
||||
for unit in "${RUNNER_UNITS[@]}"; do
|
||||
unit_ok "$unit" || bad=$((bad + 1))
|
||||
done
|
||||
while IFS= read -r unit; do
|
||||
[ -n "$unit" ] || continue
|
||||
unit_ok "$unit" || bad=$((bad + 1))
|
||||
done < <(list_action_runner_units)
|
||||
echo "$bad"
|
||||
}
|
||||
|
||||
write_metrics() {
|
||||
local dir="$1"
|
||||
local tmp
|
||||
[ -d "$dir" ] || return 0
|
||||
tmp="$(mktemp)"
|
||||
cat >"$tmp" <<EOF
|
||||
# HELP awoooi_runner_failclosed_enforcer_last_run_timestamp Last successful run timestamp.
|
||||
# TYPE awoooi_runner_failclosed_enforcer_last_run_timestamp gauge
|
||||
awoooi_runner_failclosed_enforcer_last_run_timestamp $(date +%s)
|
||||
# HELP awoooi_runner_failclosed_enforcer_active_job_containers Active Gitea/awoooi-cd job containers after enforcement.
|
||||
# TYPE awoooi_runner_failclosed_enforcer_active_job_containers gauge
|
||||
awoooi_runner_failclosed_enforcer_active_job_containers $(count_active_job_containers)
|
||||
# HELP awoooi_runner_failclosed_enforcer_lane_process_count Active direct cd-lane processes after enforcement.
|
||||
# TYPE awoooi_runner_failclosed_enforcer_lane_process_count gauge
|
||||
awoooi_runner_failclosed_enforcer_lane_process_count $(count_lane_processes)
|
||||
# HELP awoooi_runner_failclosed_enforcer_root_restore_sources_left Root restore-source directories left after enforcement.
|
||||
# TYPE awoooi_runner_failclosed_enforcer_root_restore_sources_left gauge
|
||||
awoooi_runner_failclosed_enforcer_root_restore_sources_left $(root_restore_sources_left)
|
||||
# HELP awoooi_runner_failclosed_enforcer_apply_performed Whether this run used apply mode.
|
||||
# TYPE awoooi_runner_failclosed_enforcer_apply_performed gauge
|
||||
awoooi_runner_failclosed_enforcer_apply_performed $APPLY_PERFORMED
|
||||
EOF
|
||||
as_root install -o root -g root -m 0644 "$tmp" "$dir/awoooi_runner_failclosed_enforcer.prom" >/dev/null 2>&1 || true
|
||||
rm -f "$tmp"
|
||||
}
|
||||
|
||||
print_readback() {
|
||||
local unit
|
||||
echo "ENFORCER_MODE=$MODE"
|
||||
echo "ENFORCER_HOST_110=1"
|
||||
echo "APPLY_PERFORMED=$APPLY_PERFORMED"
|
||||
echo "ACTIVE_JOB_CONTAINERS=$(count_active_job_containers)"
|
||||
echo "LANE_PROCESS_COUNT=$(count_lane_processes)"
|
||||
echo "RUNNER_PROCESS_COUNT=$(count_runner_processes)"
|
||||
echo "ROOT_RESTORE_SOURCES_LEFT=$(root_restore_sources_left)"
|
||||
echo "RUNNER_UNITS_BAD_COUNT=$(runner_units_bad_count)"
|
||||
for unit in "${RUNNER_UNITS[@]}"; do
|
||||
load="$(systemctl show "$unit" -p LoadState --value 2>/dev/null || true)"
|
||||
active="$(systemctl show "$unit" -p ActiveState --value 2>/dev/null || true)"
|
||||
unitfile="$(systemctl show "$unit" -p UnitFileState --value 2>/dev/null || true)"
|
||||
echo "RUNNER_UNIT $unit load=${load:-unknown} active=${active:-unknown} unitfile=${unitfile:-unknown}"
|
||||
done
|
||||
while IFS= read -r unit; do
|
||||
[ -n "$unit" ] || continue
|
||||
load="$(systemctl show "$unit" -p LoadState --value 2>/dev/null || true)"
|
||||
active="$(systemctl show "$unit" -p ActiveState --value 2>/dev/null || true)"
|
||||
unitfile="$(systemctl show "$unit" -p UnitFileState --value 2>/dev/null || true)"
|
||||
echo "ACTION_RUNNER_UNIT $unit load=${load:-unknown} active=${active:-unknown} unitfile=${unitfile:-unknown}"
|
||||
done < <(list_action_runner_units)
|
||||
}
|
||||
|
||||
apply_failclosed() {
|
||||
APPLY_PERFORMED=1
|
||||
repair_enforcer_entrypoints
|
||||
seal_enforcer_disabler_artifacts
|
||||
repair_enforcer_systemd_units
|
||||
repair_enforcer_cron_authority
|
||||
stop_active_job_containers
|
||||
stop_and_mask_units
|
||||
stop_and_mask_action_runner_units
|
||||
kill_runner_processes
|
||||
remove_sentinels
|
||||
seal_unit_activation_artifacts
|
||||
seal_startup_open_dropins
|
||||
seal_startup_backup_openers
|
||||
seal_systemd_unit_backups
|
||||
seal_root_live_artifact_files
|
||||
seal_lane_unit_files
|
||||
seal_live_binary_paths
|
||||
seal_lane_binary_restore_sources
|
||||
quarantine_lane_registration_sources
|
||||
seal_opener_templates
|
||||
seal_tmp_enforcer_backups
|
||||
seal_opener_unit_templates
|
||||
seal_root_restore_sources
|
||||
seal_home_upload_controlled_open_sources
|
||||
seal_quarantined_runner_sources
|
||||
as_root systemctl daemon-reload >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
if ! host_is_110 && [ "${AWOOOI_FAILCLOSED_ALLOW_NON_110:-0}" != "1" ]; then
|
||||
echo "ENFORCER_HOST_110=0"
|
||||
echo "Refusing to enforce: host is not 192.168.0.110. Set AWOOOI_FAILCLOSED_ALLOW_NON_110=1 only for controlled tests." >&2
|
||||
exit 65
|
||||
fi
|
||||
|
||||
if [ "$MODE" = "apply" ]; then
|
||||
apply_failclosed
|
||||
fi
|
||||
|
||||
write_metrics "/var/lib/node_exporter/textfile_collector"
|
||||
write_metrics "/home/wooo/node_exporter_textfiles"
|
||||
print_readback
|
||||
|
||||
if [ "$(count_active_job_containers)" = "0" ] \
|
||||
&& [ "$(count_lane_processes)" = "0" ] \
|
||||
&& [ "$(count_runner_processes)" = "0" ] \
|
||||
&& [ "$(root_restore_sources_left)" = "0" ] \
|
||||
&& [ "$(runner_units_bad_count)" = "0" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
exit 2
|
||||
Reference in New Issue
Block a user