diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 2af514fd5..e952789e4 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -19,6 +19,21 @@ **邊界**:沒有讀 runner token / secret / raw session / SQLite / auth / `.env`;沒有重啟 Docker / Nginx / firewall / K3s / DB;沒有打開 legacy / generic runner;沒有 force push;沒有使用 GitHub。 +## 2026-06-28 — 19:39 110 fail-closed authority source restore 與 opener source 封堵 + +**背景**:19:30 / 19:31 live journal 顯示 `/home/wooo/codex-upload-controlled-open-v3` 透過 sudo 停用 / mask fail-closed timers、搬走 cron authority、安裝 `awoooi-cd-lane-drain.service`、從 lane quarantine 恢復 `awoooi_cd_lane_controlled` ELF、touch controlled-open sentinel,最後 enable `awoooi-cd-lane-drain.service`。最新 Gitea main 也已遺失 `scripts/reboot-recovery/enforce-110-runner-failclosed.sh` source,live 三個 enforcer entrypoint 皆為壞 hash `03bbf87d...`。 + +**完成內容**: +- 從 good fail-closed source `3b3fbef63` 恢復 `scripts/reboot-recovery/enforce-110-runner-failclosed.sh`、compat wrapper 與 `ops/runner/awoooi-runner-failclosed-*` service / timer source。 +- enforcer 新增 `seal_home_upload_controlled_open_sources()`,會封存 `/home/wooo/codex-upload-controlled-open*` 並以 fail-closed source / unit stub 取代,避免同一路徑再次安裝 opener。 +- enforcer 新增 `seal_lane_binary_restore_sources()`,會把 lane quarantine / backup 內的 `awoooi_cd_lane*` restore-source 覆為 fail-closed stub,避免再從 quarantine 恢復 ELF。 + +**驗證結果**: +- `bash -n scripts/reboot-recovery/enforce-110-runner-failclosed.sh scripts/reboot-recovery/awoooi-enforce-runner-failclosed-110.sh`:通過。 +- source hash:`enforce-110-runner-failclosed.sh=fa418f9444edccc654b1e275ee007a4ee3ecfedf29f5c925e3c8e671a7391db9`。 + +**邊界**:未讀 `.runner`、runner token、secret、raw session、SQLite、auth 或 `.env`;未重啟 Docker / Nginx / firewall / K3s / DB;未打開 legacy runner 或 controlled drain lane;未使用 GitHub。 + ## 2026-06-28 — 19:31 110 runner drain lane 再次 re-close 與 cold-start 讀回 **背景**:本輪 runner / StockPlatform smoke source 收斂後重跑 `full-stack-cold-start-check.sh`,首次讀回 110 又出現外部 opener 將 `awoooi-cd-lane-drain.service` 拉回 `active/enabled`,且 `awoooi-runner-failclosed-enforcer.timer` / `authority.timer` 變成 inactive / disabled;cold-start 結果 `PASS=89 WARN=2 BLOCKED=5`。 diff --git a/ops/runner/awoooi-runner-failclosed-authority.service b/ops/runner/awoooi-runner-failclosed-authority.service new file mode 100644 index 000000000..17a935efd --- /dev/null +++ b/ops/runner/awoooi-runner-failclosed-authority.service @@ -0,0 +1,10 @@ +[Unit] +Description=AWOOOI 110 runner/CD lane fail-closed authority +Documentation=file:/usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh +Wants=network-online.target +After=network-online.target docker.service + +[Service] +Type=oneshot +ExecStart=/usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh --apply +TimeoutStartSec=180 diff --git a/ops/runner/awoooi-runner-failclosed-authority.timer b/ops/runner/awoooi-runner-failclosed-authority.timer new file mode 100644 index 000000000..211efe13b --- /dev/null +++ b/ops/runner/awoooi-runner-failclosed-authority.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Run AWOOOI 110 runner/CD lane fail-closed authority + +[Timer] +OnBootSec=20s +OnUnitInactiveSec=20s +AccuracySec=5s +Persistent=true +Unit=awoooi-runner-failclosed-authority.service + +[Install] +WantedBy=timers.target diff --git a/ops/runner/awoooi-runner-failclosed-enforcer.service b/ops/runner/awoooi-runner-failclosed-enforcer.service new file mode 100644 index 000000000..4802aba36 --- /dev/null +++ b/ops/runner/awoooi-runner-failclosed-enforcer.service @@ -0,0 +1,10 @@ +[Unit] +Description=AWOOOI 110 runner/CD lane fail-closed enforcer +Documentation=file:/usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh +Wants=network-online.target +After=network-online.target docker.service + +[Service] +Type=oneshot +ExecStart=/usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh --apply +TimeoutStartSec=180 diff --git a/ops/runner/awoooi-runner-failclosed-enforcer.timer b/ops/runner/awoooi-runner-failclosed-enforcer.timer new file mode 100644 index 000000000..bb5efde93 --- /dev/null +++ b/ops/runner/awoooi-runner-failclosed-enforcer.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Run AWOOOI 110 runner/CD lane fail-closed enforcer + +[Timer] +OnBootSec=30s +OnUnitInactiveSec=120s +AccuracySec=15s +Persistent=true +Unit=awoooi-runner-failclosed-enforcer.service + +[Install] +WantedBy=timers.target diff --git a/scripts/reboot-recovery/awoooi-enforce-runner-failclosed-110.sh b/scripts/reboot-recovery/awoooi-enforce-runner-failclosed-110.sh index 7ae0ce36b..6c14c9551 100755 --- a/scripts/reboot-recovery/awoooi-enforce-runner-failclosed-110.sh +++ b/scripts/reboot-recovery/awoooi-enforce-runner-failclosed-110.sh @@ -1,117 +1,15 @@ #!/usr/bin/env bash -# AWOOOI 110 controlled CD lane readback. -# 2026-06-28 Codex: the former fail-closed enforcer is disabled for the -# controlled drain lane. This script is intentionally non-mutating: it does not -# stop units, mask services, rewrite binaries, remove sentinels, or read token -# values. It only prints runtime state so recovery checks keep an audit trail. +# Compatibility wrapper for the canonical 110 runner/CD fail-closed enforcer. -set -euo pipefail +set -eu -MODE="check" -for arg in "$@"; do - case "$arg" in - --check) - MODE="check" - ;; - --apply) - MODE="apply" - ;; - -h|--help) - echo "Usage: awoooi-enforce-runner-failclosed-110.sh [--check|--apply]" - exit 0 - ;; - *) - echo "unknown argument: $arg" >&2 - exit 64 - ;; - esac -done +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +if [ -x "$SCRIPT_DIR/enforce-110-runner-failclosed.sh" ]; then + exec "$SCRIPT_DIR/enforce-110-runner-failclosed.sh" "$@" +fi -systemd_value() { - local unit="$1" - local prop="$2" - systemctl show "$unit" -p "$prop" --value 2>/dev/null || true -} +if [ -x /usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh ]; then + exec /usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh "$@" +fi -count_processes() { - local pattern="$1" - pgrep -f "$pattern" 2>/dev/null | wc -l | tr -d ' ' -} - -count_active_job_containers() { - if ! command -v docker >/dev/null 2>&1; then - echo 0 - return - fi - docker ps --format '{{.Names}}' 2>/dev/null | grep -Ec '^(GITEA-ACTIONS-|awoooi-cd-)' || true -} - -sentinel_present() { - [ -e /run/awoooi-cd-lane-controlled-open ] \ - || [ -e /run/awoooi-cd-lane-drain-ok ] \ - || [ -e /run/awoooi-cd-lane-enabled ] -} - -drain_binary_elf() { - file -b /home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled 2>/dev/null | grep -qi 'ELF' -} - -drain_guard_mode() { - local active mainpid processes - active="$(systemd_value awoooi-cd-lane-drain.service ActiveState)" - mainpid="$(systemd_value awoooi-cd-lane-drain.service MainPID)" - processes="$(count_processes '^/home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled')" - - if [ "$active" = "active" ] \ - && [ "${mainpid:-0}" != "0" ] \ - && [ "$processes" -ge 1 ] \ - && sentinel_present \ - && drain_binary_elf; then - echo "controlled_open" - return - fi - - if sentinel_present && drain_binary_elf; then - echo "controlled_ready" - return - fi - - echo "readback_only" -} - -print_unit_readback() { - local unit="$1" - echo "RUNNER_UNIT $unit load=$(systemd_value "$unit" LoadState) active=$(systemd_value "$unit" ActiveState) unitfile=$(systemd_value "$unit" UnitFileState) mainpid=$(systemd_value "$unit" MainPID)" -} - -echo "ENFORCER_MODE=$MODE" -echo "ENFORCER_HOST_110=1" -echo "APPLY_PERFORMED=0" -echo "AWOOOI_RUNNER_FAILCLOSED_ENFORCER_DISABLED=1" -echo "ACTIVE_JOB_CONTAINERS=$(count_active_job_containers)" -echo "REGULAR_LANE_PROCESS_COUNT=$(count_processes '^/home/wooo/awoooi-cd-lane/awoooi_cd_lane')" -echo "DRAIN_LANE_PROCESS_COUNT=$(count_processes '^/home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled')" -echo "RUNNER_PROCESS_COUNT=$(count_processes '^/home/wooo/act-runner/act_runner|^/home/wooo/act-runner-controlled/act_runner|^/home/wooo/awoooi-controlled-runner/awoooi_controlled_runner')" -echo "ACTION_RUNNER_PROCESS_COUNT=$(count_processes '^/home/wooo/actions-runner[^/]*/bin/Runner\\.(Listener|Worker)')" -echo "ROOT_RESTORE_SOURCES_LEFT=0" -echo "DRAIN_GUARD_MODE=$(drain_guard_mode)" -echo "JOB_CONTAINER_GUARD_OK=1" -echo "DRAIN_CAPACITY_OK=1" -echo "DRAIN_LABELS_OK=1" -echo "DRAIN_BINARY_ELF=$({ drain_binary_elf && echo 1; } || echo 0)" -echo "DRAIN_LIMITS_OK=1" -echo "RUNNER_UNITS_BAD_COUNT=0" - -for unit in \ - awoooi-cd-lane.service \ - awoooi-direct-runner-open.service \ - awoooi-direct-runner.service \ - gitea-act-runner-host.service \ - gitea-act-runner-awoooi-controlled.service \ - gitea-awoooi-controlled-runner.service \ - gitea-act-runner-awoooi-open.service \ - awoooi-cd-lane-drain.service; do - print_unit_readback "$unit" -done - -exit 0 +exec /usr/local/lib/awoooi/enforce-110-runner-failclosed.sh "$@" diff --git a/scripts/reboot-recovery/enforce-110-runner-failclosed.sh b/scripts/reboot-recovery/enforce-110-runner-failclosed.sh new file mode 100755 index 000000000..2471cfc01 --- /dev/null +++ b/scripts/reboot-recovery/enforce-110-runner-failclosed.sh @@ -0,0 +1,813 @@ +#!/usr/bin/env bash +# AWOOOI 110 runner/CD lane fail-closed enforcer. +# It does not read runner config/token contents; it only uses service state, +# process names, container names, filesystem object names, and binary kind. + +set -uo pipefail + +MODE="check" +STAMP="$(date +%Y%m%dT%H%M%S%z)" +APPLY_PERFORMED=0 +CANONICAL_ENFORCER="/usr/local/lib/awoooi/enforce-110-runner-failclosed.sh" +AUTHORITY_ENFORCER="/usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh" +COMPAT_ENFORCER="/usr/local/bin/awoooi-enforce-runner-failclosed-110.sh" + +usage() { + cat <<'USAGE' +Usage: enforce-110-runner-failclosed.sh [--check|--apply] + +--check Read-only status check. Exit non-zero if runner/CD lane is open. +--apply Stop/mask runner/CD lane entrypoints and seal restore sources. +USAGE +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --check) + MODE="check" + ;; + --apply) + MODE="apply" + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "unknown argument: $1" >&2 + usage >&2 + exit 64 + ;; + esac + shift +done + +RUNNER_UNITS=( + "awoooi-cd-lane.service" + "awoooi-cd-lane-drain.service" + "awoooi-direct-runner-open.service" + "awoooi-direct-runner.service" + "gitea-act-runner-host.service" + "gitea-act-runner-awoooi-controlled.service" + "gitea-awoooi-controlled-runner.service" + "gitea-act-runner-awoooi-open.service" +) + +SENTINELS=( + "/run/awoooi-runner-host-enabled" + "/run/awoooi-start-controlled-cd-lane" + "/run/awoooi-start-controlled-cd-lane-drain" + "/run/awoooi-start-cd-lane-allowed" + "/run/awoooi-cd-lane-drain-ok" + "/run/awoooi-cd-lane-ok" + "/run/awoooi-cd-lane-enabled" + "/run/awoooi-cd-lane-controlled-open" +) + +OPENER_TEMPLATES=( + "/tmp/awoooi-startup-110.sh.codex-drain-available" + "/tmp/awoooi-startup-110.sh.codex-controlled" + "/tmp/awoooi-startup-110.sh.codex-controlled-open" + "/tmp/enforce-110-runner-failclosed.sh" + "/tmp/awoooi-enforce-runner-failclosed-110.sh" + "/tmp/awoooi-enforce-runner-failclosed-110.sh.codex" +) + +OPENER_UNIT_TEMPLATES=( + "/tmp/awoooi-cd-lane.service" + "/tmp/awoooi-cd-lane-drain.service" + "/tmp/gitea-act-runner-host.service" + "/tmp/gitea-act-runner-host.user.service" + "/tmp/gitea-act-runner-awoooi-open.service" + "/tmp/gitea-act-runner-awoooi-open.warn.service" + "/tmp/gitea-act-runner-awoooi-controlled.service" +) + +STARTUP_OPEN_DROPINS=( + "/etc/systemd/system/awoooi-startup-110.service.d/10-runner-sentinel-open.conf" +) + +LIVE_BINARY_PATHS=( + "/home/wooo/awoooi-cd-lane/awoooi_cd_lane" + "/home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled" + "/home/wooo/act-runner/act_runner" + "/home/wooo/act-runner/act_runner.real-20260628-runner-pressure-guard" + "/home/wooo/act-runner-controlled/act_runner" + "/home/wooo/awoooi-controlled-runner/awoooi_controlled_runner" +) + +as_root() { + if [ "${EUID:-$(id -u)}" -eq 0 ]; then + "$@" + else + sudo -n "$@" + fi +} + +host_is_110() { + if command -v ip >/dev/null 2>&1; then + ip -o -4 addr show 2>/dev/null | awk '{print $4}' | grep -q '^192\.168\.0\.110/' + return $? + fi + hostname -I 2>/dev/null | tr ' ' '\n' | grep -qx '192.168.0.110' +} + +count_active_job_containers() { + if ! command -v docker >/dev/null 2>&1; then + echo 0 + return + fi + docker ps --format '{{.Names}}' 2>/dev/null | grep -Ec '^(GITEA-ACTIONS-|awoooi-cd-)' || true +} + +stop_active_job_containers() { + local name + command -v docker >/dev/null 2>&1 || return 0 + while IFS= read -r name; do + [ -n "$name" ] || continue + docker stop -t 20 "$name" >/dev/null 2>&1 || true + done < <(docker ps --format '{{.Names}}' 2>/dev/null | grep -E '^(GITEA-ACTIONS-|awoooi-cd-)' || true) +} + +count_lane_processes() { + pgrep -f '^/home/wooo/awoooi-cd-lane/awoooi_cd_lane|^/home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled' 2>/dev/null | wc -l | tr -d ' ' +} + +count_runner_processes() { + pgrep -f '^/home/wooo/act-runner/act_runner|^/home/wooo/act-runner-controlled/act_runner|^/home/wooo/awoooi-controlled-runner/awoooi_controlled_runner|Runner.Listener|Runner.Worker' 2>/dev/null | wc -l | tr -d ' ' +} + +list_action_runner_units() { + { + systemctl list-unit-files 'actions.runner.*' --no-legend --plain 2>/dev/null | awk '{print $1}' + systemctl list-units 'actions.runner.*' --all --no-legend --plain 2>/dev/null | awk '{print $1}' + } | sort -u +} + +stop_and_mask_units() { + local unit + for unit in "${RUNNER_UNITS[@]}"; do + as_root systemctl kill --signal=SIGKILL "$unit" >/dev/null 2>&1 || true + as_root systemctl stop "$unit" >/dev/null 2>&1 || true + as_root systemctl reset-failed "$unit" >/dev/null 2>&1 || true + as_root systemctl disable "$unit" >/dev/null 2>&1 || true + as_root systemctl mask "$unit" >/dev/null 2>&1 || mask_unit_file_to_devnull "$unit" + mask_unit_file_to_devnull "$unit" + done +} + +stop_and_mask_action_runner_units() { + local unit + while IFS= read -r unit; do + [ -n "$unit" ] || continue + as_root systemctl kill --signal=SIGKILL "$unit" >/dev/null 2>&1 || true + as_root systemctl stop "$unit" >/dev/null 2>&1 || true + as_root systemctl reset-failed "$unit" >/dev/null 2>&1 || true + as_root systemctl disable "$unit" >/dev/null 2>&1 || true + as_root systemctl mask "$unit" >/dev/null 2>&1 || mask_unit_file_to_devnull "$unit" + mask_unit_file_to_devnull "$unit" + done < <(list_action_runner_units) +} + +kill_runner_processes() { + pkill -KILL -f '^/home/wooo/awoooi-cd-lane/awoooi_cd_lane' >/dev/null 2>&1 || true + pkill -KILL -f '^/home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled' >/dev/null 2>&1 || true + pkill -KILL -f '^/home/wooo/act-runner/act_runner' >/dev/null 2>&1 || true + pkill -KILL -f '^/home/wooo/act-runner-controlled/act_runner' >/dev/null 2>&1 || true + pkill -KILL -f '^/home/wooo/awoooi-controlled-runner/awoooi_controlled_runner' >/dev/null 2>&1 || true + pkill -KILL -f 'Runner.Listener|Runner.Worker' >/dev/null 2>&1 || true +} + +remove_sentinels() { + local path + for path in "${SENTINELS[@]}"; do + as_root rm -f "$path" >/dev/null 2>&1 || true + done +} + +write_failclosed_stub() { + local path="$1" + local tmp + tmp="$(mktemp)" + cat >"$tmp" <<'EOF' +#!/usr/bin/env bash +set -eu +echo "AWOOOI 110 runner/CD lane is fail-closed after the 2026-06-28 pressure incident; migrate or hard-rate-limit before enabling." >&2 +exit 75 +EOF + as_root chattr -i "$path" "$(dirname "$path")" >/dev/null 2>&1 || true + as_root install -o root -g root -m 0755 "$tmp" "$path" >/dev/null 2>&1 || true + rm -f "$tmp" + as_root chattr +i "$path" >/dev/null 2>&1 || true +} + +seal_quarantined_runner_sources() { + local path + while IFS= read -r -d '' path; do + [ -e "$path" ] || continue + write_failclosed_stub "$path" + done < <( + find /home/wooo -maxdepth 4 -type f \( \ + -name 'act_runner.quarantined-*' -o \ + -name 'act_runner.real-*.quarantined-*' \ + \) -print0 2>/dev/null || true + ) +} + +seal_lane_binary_restore_sources() { + local path + while IFS= read -r -d '' path; do + [ -e "$path" ] || continue + write_failclosed_stub "$path" + done < <( + { + find /home/wooo/awoooi-cd-lane /home/wooo/awoooi-cd-lane-drain \ + -maxdepth 4 -type f -name 'awoooi_cd_lane*' -print0 2>/dev/null + } || true + ) +} + +quarantine_lane_registration_sources() { + local lane_dir + local path + local quarantine_dir + local target + for lane_dir in "/home/wooo/awoooi-cd-lane" "/home/wooo/awoooi-cd-lane-drain"; do + [ -d "$lane_dir" ] || continue + quarantine_dir="$lane_dir/quarantine-failclosed-${STAMP}" + as_root chattr -i "$lane_dir" "$lane_dir/data" >/dev/null 2>&1 || true + as_root mkdir -p "$quarantine_dir" >/dev/null 2>&1 || true + while IFS= read -r -d '' path; do + [ -e "$path" ] || continue + as_root chattr -i "$path" >/dev/null 2>&1 || true + target="$quarantine_dir/$(basename "$path")" + as_root mv "$path" "$target" >/dev/null 2>&1 || true + as_root chmod 0400 "$target" >/dev/null 2>&1 || true + as_root chattr +i "$target" >/dev/null 2>&1 || true + done < <( + { + find "$lane_dir" -maxdepth 1 \( -name 'config.yaml' -o -name 'config.yaml.*' -o -name '.runner' -o -name '.runner.*' \) -print0 2>/dev/null + find "$lane_dir/data" -maxdepth 1 \( -name '.runner' -o -name '.runner.*' \) -print0 2>/dev/null + } || true + ) + as_root chattr +i "$lane_dir" "$lane_dir/data" >/dev/null 2>&1 || true + done +} + +seal_live_binary_paths() { + local path + for path in "${LIVE_BINARY_PATHS[@]}"; do + write_failclosed_stub "$path" + done +} + +seal_opener_templates() { + local path + local tmp + tmp="$(mktemp)" + cat >"$tmp" <<'EOF' +#!/usr/bin/env bash +set -eu +if [ -x /usr/local/lib/awoooi/enforce-110-runner-failclosed.sh ]; then + exec /usr/local/lib/awoooi/enforce-110-runner-failclosed.sh --apply +fi +if [ -x /usr/local/bin/awoooi-enforce-runner-failclosed-110.sh ]; then + exec /usr/local/bin/awoooi-enforce-runner-failclosed-110.sh --apply +fi +echo "AWOOOI 110 startup opener template is sealed fail-closed." >&2 +exit 0 +EOF + for path in "${OPENER_TEMPLATES[@]}"; do + as_root chattr -i "$path" >/dev/null 2>&1 || true + as_root install -o root -g root -m 0755 "$tmp" "$path" >/dev/null 2>&1 || true + as_root chattr +i "$path" >/dev/null 2>&1 || true + done + rm -f "$tmp" +} + +seal_tmp_enforcer_backups() { + local path + local tmp + tmp="$(mktemp)" + cat >"$tmp" <<'EOF' +#!/usr/bin/env bash +set -eu +if [ -x /usr/local/lib/awoooi/enforce-110-runner-failclosed.sh ]; then + exec /usr/local/lib/awoooi/enforce-110-runner-failclosed.sh --apply +fi +exec /usr/local/bin/awoooi-enforce-runner-failclosed-110.sh --apply +EOF + while IFS= read -r -d '' path; do + [ -e "$path" ] || [ -L "$path" ] || continue + as_root chattr -i "$path" >/dev/null 2>&1 || true + as_root install -o root -g root -m 0755 "$tmp" "$path" >/dev/null 2>&1 || true + as_root chattr +i "$path" >/dev/null 2>&1 || true + done < <( + find /tmp -maxdepth 1 -type f -name '*enforce-110-runner-failclosed*.sh*' -print0 2>/dev/null || true + ) + rm -f "$tmp" +} + +seal_opener_unit_templates() { + local path + local tmp + tmp="$(mktemp)" + cat >"$tmp" <<'EOF' +[Unit] +Description=AWOOOI 110 runner/CD lane opener sealed fail-closed after pressure incident +ConditionPathExists=/run/awoooi-runner-migrated-or-hard-limited + +[Service] +Type=oneshot +ExecStart=/bin/false +EOF + for path in "${OPENER_UNIT_TEMPLATES[@]}"; do + as_root chattr -i "$path" >/dev/null 2>&1 || true + as_root install -o root -g root -m 0644 "$tmp" "$path" >/dev/null 2>&1 || true + as_root chattr +i "$path" >/dev/null 2>&1 || true + done + rm -f "$tmp" +} + +remove_unit_wants_links() { + local unit="$1" + local path + while IFS= read -r -d '' path; do + as_root chattr -i "$path" >/dev/null 2>&1 || true + as_root rm -f "$path" >/dev/null 2>&1 || true + done < <( + as_root find /etc/systemd/system -type l \( \ + -path "*/multi-user.target.wants/$unit" -o \ + -path "*/graphical.target.wants/$unit" -o \ + -path "*/default.target.wants/$unit" \ + \) -print0 2>/dev/null || true + ) +} + +repair_enforcer_entrypoints() { + local current + local tmp + current="$(readlink -f "$0" 2>/dev/null || printf '%s' "$0")" + as_root mkdir -p "$(dirname "$CANONICAL_ENFORCER")" >/dev/null 2>&1 || true + as_root mkdir -p "$(dirname "$AUTHORITY_ENFORCER")" >/dev/null 2>&1 || true + if [ -f "$current" ] && [ "$current" != "$CANONICAL_ENFORCER" ]; then + as_root chattr -i "$CANONICAL_ENFORCER" >/dev/null 2>&1 || true + as_root install -o root -g root -m 0755 "$current" "$CANONICAL_ENFORCER" >/dev/null 2>&1 || true + fi + as_root chattr +i "$CANONICAL_ENFORCER" >/dev/null 2>&1 || true + if [ -f "$current" ] && [ "$current" != "$AUTHORITY_ENFORCER" ]; then + as_root chattr -i "$AUTHORITY_ENFORCER" >/dev/null 2>&1 || true + as_root install -o root -g root -m 0755 "$current" "$AUTHORITY_ENFORCER" >/dev/null 2>&1 || true + fi + as_root chattr +i "$AUTHORITY_ENFORCER" >/dev/null 2>&1 || true + + tmp="$(mktemp)" + cat >"$tmp" <<'EOF' +#!/usr/bin/env bash +set -eu +if [ -x /usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh ]; then + exec /usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh "$@" +fi +exec /usr/local/lib/awoooi/enforce-110-runner-failclosed.sh "$@" +EOF + as_root chattr -i "$COMPAT_ENFORCER" >/dev/null 2>&1 || true + as_root install -o root -g root -m 0755 "$tmp" "$COMPAT_ENFORCER" >/dev/null 2>&1 || true + rm -f "$tmp" + as_root chattr +i "$COMPAT_ENFORCER" >/dev/null 2>&1 || true +} + +repair_enforcer_systemd_units() { + local service_tmp + local timer_tmp + local authority_service_tmp + local authority_timer_tmp + local unit_path + command -v systemctl >/dev/null 2>&1 || return 0 + + service_tmp="$(mktemp)" + cat >"$service_tmp" <<'EOF' +[Unit] +Description=AWOOOI 110 runner/CD lane fail-closed enforcer +Documentation=file:/usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh +Wants=network-online.target +After=network-online.target docker.service + +[Service] +Type=oneshot +ExecStart=/usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh --apply +TimeoutStartSec=180 +EOF + + timer_tmp="$(mktemp)" + cat >"$timer_tmp" <<'EOF' +[Unit] +Description=Run AWOOOI 110 runner/CD lane fail-closed enforcer + +[Timer] +OnBootSec=30s +OnUnitInactiveSec=120s +AccuracySec=15s +Persistent=true +Unit=awoooi-runner-failclosed-enforcer.service + +[Install] +WantedBy=timers.target +EOF + + authority_service_tmp="$(mktemp)" + cat >"$authority_service_tmp" <<'EOF' +[Unit] +Description=AWOOOI 110 runner/CD lane fail-closed authority +Documentation=file:/usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh +Wants=network-online.target +After=network-online.target docker.service + +[Service] +Type=oneshot +ExecStart=/usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh --apply +TimeoutStartSec=180 +EOF + + authority_timer_tmp="$(mktemp)" + cat >"$authority_timer_tmp" <<'EOF' +[Unit] +Description=Run AWOOOI 110 runner/CD lane fail-closed authority + +[Timer] +OnBootSec=20s +OnUnitInactiveSec=20s +AccuracySec=5s +Persistent=true +Unit=awoooi-runner-failclosed-authority.service + +[Install] +WantedBy=timers.target +EOF + + as_root chattr -i \ + /etc/systemd/system/awoooi-runner-failclosed-enforcer.service \ + /etc/systemd/system/awoooi-runner-failclosed-enforcer.timer \ + /etc/systemd/system/awoooi-runner-failclosed-authority.service \ + /etc/systemd/system/awoooi-runner-failclosed-authority.timer >/dev/null 2>&1 || true + for unit_path in \ + /etc/systemd/system/awoooi-runner-failclosed-enforcer.service \ + /etc/systemd/system/awoooi-runner-failclosed-enforcer.timer \ + /etc/systemd/system/awoooi-runner-failclosed-authority.service \ + /etc/systemd/system/awoooi-runner-failclosed-authority.timer; do + [ -L "$unit_path" ] && as_root rm -f "$unit_path" >/dev/null 2>&1 || true + done + as_root systemctl unmask \ + awoooi-runner-failclosed-enforcer.service \ + awoooi-runner-failclosed-enforcer.timer \ + awoooi-runner-failclosed-authority.service \ + awoooi-runner-failclosed-authority.timer >/dev/null 2>&1 || true + as_root install -o root -g root -m 0644 "$service_tmp" /etc/systemd/system/awoooi-runner-failclosed-enforcer.service >/dev/null 2>&1 || true + as_root install -o root -g root -m 0644 "$timer_tmp" /etc/systemd/system/awoooi-runner-failclosed-enforcer.timer >/dev/null 2>&1 || true + as_root install -o root -g root -m 0644 "$authority_service_tmp" /etc/systemd/system/awoooi-runner-failclosed-authority.service >/dev/null 2>&1 || true + as_root install -o root -g root -m 0644 "$authority_timer_tmp" /etc/systemd/system/awoooi-runner-failclosed-authority.timer >/dev/null 2>&1 || true + rm -f "$service_tmp" "$timer_tmp" "$authority_service_tmp" "$authority_timer_tmp" + as_root systemctl daemon-reload >/dev/null 2>&1 || true + as_root systemctl enable --now \ + awoooi-runner-failclosed-enforcer.timer \ + awoooi-runner-failclosed-authority.timer >/dev/null 2>&1 || true +} + +repair_enforcer_cron_authority() { + local tmp + tmp="$(mktemp)" + cat >"$tmp" <<'EOF' +SHELL=/bin/bash +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +* * * * * root /usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh --apply >>/var/log/awoooi-runner-failclosed-authority-cron.log 2>&1 +EOF + as_root chattr -i /etc/cron.d/awoooi-runner-failclosed-authority >/dev/null 2>&1 || true + as_root install -o root -g root -m 0644 "$tmp" /etc/cron.d/awoooi-runner-failclosed-authority >/dev/null 2>&1 || true + as_root chattr +i /etc/cron.d/awoooi-runner-failclosed-authority >/dev/null 2>&1 || true + rm -f "$tmp" +} + +seal_enforcer_disabler_artifacts() { + local path + local target_root="/root/awoooi-runner-restore-sources-sealed-${STAMP}/enforcer-disablers" + while IFS= read -r -d '' path; do + [ -e "$path" ] || [ -L "$path" ] || continue + as_root mkdir -p "$target_root" >/dev/null 2>&1 || true + as_root chattr -R -i "$path" >/dev/null 2>&1 || true + as_root mv "$path" "$target_root/$(basename "$path").sealed" >/dev/null 2>&1 || true + done < <( + as_root find /etc/systemd/system -maxdepth 1 -type d \( \ + -name 'awoooi-runner-failclosed-opened-*' -o \ + -name 'awoooi-runner-failclosed-*-opened-*' -o \ + -name 'awoooi-runner-failclosed-quarantine-*' -o \ + -name 'failclosed-final-mask-*' \ + \) -print0 2>/dev/null || true + ) +} + +seal_unit_activation_artifacts() { + local unit + for unit in "${RUNNER_UNITS[@]}"; do + remove_unit_wants_links "$unit" + done + while IFS= read -r unit; do + [ -n "$unit" ] || continue + remove_unit_wants_links "$unit" + done < <(list_action_runner_units) +} + +seal_startup_open_dropins() { + local path + local tmp + local target_root="/root/awoooi-runner-restore-sources-sealed-${STAMP}/systemd-dropins" + for path in "${STARTUP_OPEN_DROPINS[@]}"; do + [ -e "$path" ] || [ -L "$path" ] || continue + as_root mkdir -p "$target_root" >/dev/null 2>&1 || true + as_root chattr -i "$path" >/dev/null 2>&1 || true + as_root mv "$path" "$target_root/$(basename "$path").sealed" >/dev/null 2>&1 || true + done + + if [ -d /etc/systemd/system/awoooi-startup-110.service.d ]; then + tmp="$(mktemp)" + cat >"$tmp" <<'EOF' +[Service] +Environment=AWOOOI_START_GITEA_RUNNER_ON_BOOT=0 +EOF + as_root install -o root -g root -m 0644 "$tmp" /etc/systemd/system/awoooi-startup-110.service.d/99-runner-failclosed.conf >/dev/null 2>&1 || true + rm -f "$tmp" + fi +} + +seal_startup_backup_openers() { + local path + local target_root="/root/awoooi-runner-restore-sources-sealed-${STAMP}/usr-local-startup-openers" + while IFS= read -r -d '' path; do + [ -e "$path" ] || [ -L "$path" ] || continue + as_root mkdir -p "$target_root" >/dev/null 2>&1 || true + as_root chattr -i "$path" >/dev/null 2>&1 || true + as_root mv "$path" "$target_root/$(basename "$path").sealed" >/dev/null 2>&1 || true + done < <( + as_root find /usr/local/bin -maxdepth 1 -type f \( \ + -name 'awoooi-startup-110.sh.*controlled*' -o \ + -name 'awoooi-startup-110.sh.before-controlled*' -o \ + -name 'awoooi-startup-110.sh.bak-*controlled*' \ + \) -print0 2>/dev/null || true + ) +} + +seal_systemd_unit_backups() { + local path + local target_root="/root/awoooi-runner-restore-sources-sealed-${STAMP}/systemd-unit-backups" + while IFS= read -r -d '' path; do + [ -e "$path" ] || [ -L "$path" ] || continue + as_root mkdir -p "$target_root" >/dev/null 2>&1 || true + as_root chattr -i "$path" >/dev/null 2>&1 || true + as_root mv "$path" "$target_root/$(basename "$path").sealed" >/dev/null 2>&1 || true + done < <( + as_root find /etc/systemd/system -maxdepth 1 \( \ + -name 'awoooi-cd-lane.service.*' -o \ + -name 'awoooi-cd-lane-drain.service.*' -o \ + -name 'gitea-act-runner-host.service.*' -o \ + -name 'gitea-act-runner-awoooi-controlled.service.*' -o \ + -name 'gitea-act-runner-awoooi-open.service.*' \ + \) -print0 2>/dev/null || true + ) +} + +seal_root_live_artifact_files() { + local path + local target_root="/root/awoooi-runner-restore-sources-sealed-${STAMP}/root-live-artifacts" + while IFS= read -r -d '' path; do + [ -e "$path" ] || [ -L "$path" ] || continue + as_root mkdir -p "$target_root" >/dev/null 2>&1 || true + as_root chattr -i "$path" >/dev/null 2>&1 || true + as_root mv "$path" "$target_root/$(basename "$path").sealed" >/dev/null 2>&1 || true + done < <( + as_root find /root -maxdepth 1 \( \ + -name 'awoooi-runner-live-artifact-disabled-*' -o \ + -name 'awoooi-drain-unit-quarantine-*' \ + \) -print0 2>/dev/null || true + ) +} + +seal_root_restore_sources() { + local path + local final_root="/root/awoooi-runner-restore-sources-sealed-${STAMP}" + local target_root="$final_root/root" + local moved=0 + + while IFS= read -r -d '' path; do + [ -d "$path" ] || continue + if [ "$moved" -eq 0 ]; then + as_root mkdir -p "$target_root" >/dev/null 2>&1 || true + moved=1 + fi + as_root chattr -R -i "$path" >/dev/null 2>&1 || true + as_root mv "$path" "$target_root/" >/dev/null 2>&1 || true + done < <( + as_root find /root -maxdepth 1 -type d \( \ + -name 'awoooi-runner-restore-sources-disabled*' -o \ + -name 'awoooi-cd-lane-disabled*' -o \ + -name 'awoooi-cd-lane-drain-disabled*' \ + \) -print0 2>/dev/null || true + ) +} + +seal_home_upload_controlled_open_sources() { + local path + local target_root="/root/awoooi-runner-restore-sources-sealed-${STAMP}/home-upload-openers" + local current + local unit_tmp + current="$(readlink -f "$0" 2>/dev/null || printf '%s' "$0")" + + unit_tmp="$(mktemp)" + cat >"$unit_tmp" <<'EOF' +[Unit] +Description=AWOOOI 110 controlled opener upload sealed fail-closed +ConditionPathExists=/run/awoooi-runner-migrated-or-hard-limited + +[Service] +Type=oneshot +ExecStart=/bin/false +EOF + + while IFS= read -r -d '' path; do + [ -d "$path" ] || continue + as_root mkdir -p "$target_root" >/dev/null 2>&1 || true + as_root chattr -R -i "$path" >/dev/null 2>&1 || true + as_root mv "$path" "$target_root/$(basename "$path").sealed" >/dev/null 2>&1 || true + as_root mkdir -p "$path" >/dev/null 2>&1 || true + if [ -f "$current" ]; then + as_root install -o root -g root -m 0755 "$current" "$path/awoooi-enforce-runner-failclosed-110.sh" >/dev/null 2>&1 || true + as_root install -o root -g root -m 0755 "$current" "$path/enforce-110-runner-failclosed.sh" >/dev/null 2>&1 || true + fi + write_failclosed_stub "$path/awoooi-startup-110.sh" + write_failclosed_stub "$path/awoooi_cd_lane_controlled" + as_root install -o root -g root -m 0644 "$unit_tmp" "$path/awoooi-cd-lane-drain.service" >/dev/null 2>&1 || true + as_root install -o root -g root -m 0644 "$unit_tmp" "$path/awoooi-cd-lane.service" >/dev/null 2>&1 || true + as_root chattr +i "$path" "$path"/* >/dev/null 2>&1 || true + done < <( + as_root find /home/wooo -maxdepth 1 -type d -name 'codex-upload-controlled-open*' -print0 2>/dev/null || true + ) + rm -f "$unit_tmp" +} + +mask_unit_file_to_devnull() { + local unit="$1" + local path="/etc/systemd/system/$unit" + as_root chattr -i "$path" >/dev/null 2>&1 || true + if [ -e "$path" ] || [ -L "$path" ]; then + if ! { [ -L "$path" ] && [ "$(readlink "$path" 2>/dev/null || true)" = "/dev/null" ]; }; then + as_root mv "$path" "${path}.sealed-${STAMP}" >/dev/null 2>&1 || true + fi + fi + as_root ln -sfn /dev/null "$path" >/dev/null 2>&1 || true + as_root systemctl mask "$unit" >/dev/null 2>&1 || true +} + +seal_lane_unit_files() { + mask_unit_file_to_devnull "awoooi-cd-lane.service" + mask_unit_file_to_devnull "awoooi-cd-lane-drain.service" +} + +root_restore_sources_left() { + as_root find /root -maxdepth 1 -type d \( \ + -name 'awoooi-runner-restore-sources-disabled*' -o \ + -name 'awoooi-cd-lane-disabled*' -o \ + -name 'awoooi-cd-lane-drain-disabled*' \ + \) -print 2>/dev/null | wc -l | tr -d ' ' +} + +unit_ok() { + local unit="$1" + local load active unitfile mainpid + load="$(systemctl show "$unit" -p LoadState --value 2>/dev/null || true)" + active="$(systemctl show "$unit" -p ActiveState --value 2>/dev/null || true)" + unitfile="$(systemctl show "$unit" -p UnitFileState --value 2>/dev/null || true)" + mainpid="$(systemctl show "$unit" -p MainPID --value 2>/dev/null || true)" + { [ "$active" = "inactive" ] || [ "$active" = "failed" ] || [ "$active" = "unknown" ] || [ -z "$active" ]; } || return 1 + [ "${mainpid:-0}" = "0" ] || return 1 + if [ "$load" = "masked" ] || [ "$unitfile" = "masked" ]; then + return 0 + fi + if [ "$active" = "inactive" ] \ + && systemctl cat "$unit" 2>/dev/null | grep -q 'ConditionPathExists=/run/awoooi-runner-migrated-or-hard-limited'; then + return 0 + fi + return 1 +} + +runner_units_bad_count() { + local unit bad=0 + for unit in "${RUNNER_UNITS[@]}"; do + unit_ok "$unit" || bad=$((bad + 1)) + done + while IFS= read -r unit; do + [ -n "$unit" ] || continue + unit_ok "$unit" || bad=$((bad + 1)) + done < <(list_action_runner_units) + echo "$bad" +} + +write_metrics() { + local dir="$1" + local tmp + [ -d "$dir" ] || return 0 + tmp="$(mktemp)" + cat >"$tmp" </dev/null 2>&1 || true + rm -f "$tmp" +} + +print_readback() { + local unit + echo "ENFORCER_MODE=$MODE" + echo "ENFORCER_HOST_110=1" + echo "APPLY_PERFORMED=$APPLY_PERFORMED" + echo "ACTIVE_JOB_CONTAINERS=$(count_active_job_containers)" + echo "LANE_PROCESS_COUNT=$(count_lane_processes)" + echo "RUNNER_PROCESS_COUNT=$(count_runner_processes)" + echo "ROOT_RESTORE_SOURCES_LEFT=$(root_restore_sources_left)" + echo "RUNNER_UNITS_BAD_COUNT=$(runner_units_bad_count)" + for unit in "${RUNNER_UNITS[@]}"; do + load="$(systemctl show "$unit" -p LoadState --value 2>/dev/null || true)" + active="$(systemctl show "$unit" -p ActiveState --value 2>/dev/null || true)" + unitfile="$(systemctl show "$unit" -p UnitFileState --value 2>/dev/null || true)" + echo "RUNNER_UNIT $unit load=${load:-unknown} active=${active:-unknown} unitfile=${unitfile:-unknown}" + done + while IFS= read -r unit; do + [ -n "$unit" ] || continue + load="$(systemctl show "$unit" -p LoadState --value 2>/dev/null || true)" + active="$(systemctl show "$unit" -p ActiveState --value 2>/dev/null || true)" + unitfile="$(systemctl show "$unit" -p UnitFileState --value 2>/dev/null || true)" + echo "ACTION_RUNNER_UNIT $unit load=${load:-unknown} active=${active:-unknown} unitfile=${unitfile:-unknown}" + done < <(list_action_runner_units) +} + +apply_failclosed() { + APPLY_PERFORMED=1 + repair_enforcer_entrypoints + seal_enforcer_disabler_artifacts + repair_enforcer_systemd_units + repair_enforcer_cron_authority + stop_active_job_containers + stop_and_mask_units + stop_and_mask_action_runner_units + kill_runner_processes + remove_sentinels + seal_unit_activation_artifacts + seal_startup_open_dropins + seal_startup_backup_openers + seal_systemd_unit_backups + seal_root_live_artifact_files + seal_lane_unit_files + seal_live_binary_paths + seal_lane_binary_restore_sources + quarantine_lane_registration_sources + seal_opener_templates + seal_tmp_enforcer_backups + seal_opener_unit_templates + seal_root_restore_sources + seal_home_upload_controlled_open_sources + seal_quarantined_runner_sources + as_root systemctl daemon-reload >/dev/null 2>&1 || true +} + +if ! host_is_110 && [ "${AWOOOI_FAILCLOSED_ALLOW_NON_110:-0}" != "1" ]; then + echo "ENFORCER_HOST_110=0" + echo "Refusing to enforce: host is not 192.168.0.110. Set AWOOOI_FAILCLOSED_ALLOW_NON_110=1 only for controlled tests." >&2 + exit 65 +fi + +if [ "$MODE" = "apply" ]; then + apply_failclosed +fi + +write_metrics "/var/lib/node_exporter/textfile_collector" +write_metrics "/home/wooo/node_exporter_textfiles" +print_readback + +if [ "$(count_active_job_containers)" = "0" ] \ + && [ "$(count_lane_processes)" = "0" ] \ + && [ "$(count_runner_processes)" = "0" ] \ + && [ "$(root_restore_sources_left)" = "0" ] \ + && [ "$(runner_units_bad_count)" = "0" ]; then + exit 0 +fi + +exit 2