62820cabbc
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 56s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped
605 lines
21 KiB
Bash
Executable File
605 lines
21 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# Read-only readiness verifier for moving AWOOOI runner/CD work off 110, or for
|
|
# proving an already-installed non-110 runner is hard-limited before auto events
|
|
# are restored. It prints only metadata: labels, capacity, service limits, task
|
|
# counts, and host pressure. It never prints runner tokens or .runner contents.
|
|
|
|
TARGET_HOST_IP="${TARGET_HOST_IP:-192.168.0.188}"
|
|
FORBIDDEN_HOST_IPS="${FORBIDDEN_HOST_IPS:-192.168.0.110}"
|
|
RUNNER_HOME="${RUNNER_HOME:-${HOME:-/home/wooo}}"
|
|
RUNNER_CONFIG_PATHS="${RUNNER_CONFIG_PATHS:-${RUNNER_HOME}/awoooi-non110-runner/data/config.yaml ${RUNNER_HOME}/awoooi-non110-runner/config.yaml ${RUNNER_HOME}/act-runner-awoooi/config.yaml /home/wooo/act-runner-awoooi/config.yaml /home/wooo/awoooi-act-runner/config.yaml /home/wooo/awoooi-non110-runner/config.yaml /home/wooo/act-runner/config.yaml}"
|
|
RUNNER_BINARY_PATHS="${RUNNER_BINARY_PATHS:-${RUNNER_HOME}/awoooi-non110-runner/act_runner ${RUNNER_HOME}/act-runner-awoooi/act_runner /home/wooo/act-runner-awoooi/act_runner /home/wooo/awoooi-act-runner/act_runner /home/wooo/awoooi-non110-runner/act_runner /home/wooo/act-runner/act_runner}"
|
|
RUNNER_DOCKER_IMAGES="${RUNNER_DOCKER_IMAGES:-gitea/act_runner:latest}"
|
|
RUNNER_REGISTRATION_PATHS="${RUNNER_REGISTRATION_PATHS:-${RUNNER_HOME}/awoooi-non110-runner/data/.runner ${RUNNER_HOME}/awoooi-non110-runner/.runner ${RUNNER_HOME}/act-runner-awoooi/.runner /home/wooo/act-runner-awoooi/.runner /home/wooo/awoooi-act-runner/.runner /home/wooo/awoooi-non110-runner/.runner /home/wooo/act-runner/.runner}"
|
|
RUNNER_SERVICE_NAMES="${RUNNER_SERVICE_NAMES:-awoooi-non110-runner.service gitea-act-runner-awoooi.service gitea-act-runner-host.service}"
|
|
RUNNER_AUTOSTART_PATH_UNIT_NAMES="${RUNNER_AUTOSTART_PATH_UNIT_NAMES:-awoooi-non110-runner-autostart.path}"
|
|
RUNNER_KEEPALIVE_SERVICE_UNIT_NAMES="${RUNNER_KEEPALIVE_SERVICE_UNIT_NAMES:-awoooi-non110-runner-keepalive.service}"
|
|
RUNNER_KEEPALIVE_TIMER_UNIT_NAMES="${RUNNER_KEEPALIVE_TIMER_UNIT_NAMES:-awoooi-non110-runner-keepalive.timer}"
|
|
ALLOWED_RUNNER_CONTAINER_NAMES="${ALLOWED_RUNNER_CONTAINER_NAMES:-awoooi-non110-runner stockplatform-ubuntu-runner}"
|
|
ALLOWED_LABEL_NAMES="${ALLOWED_LABEL_NAMES:-awoooi-non110-host awoooi-non110-ubuntu awoooi-host awoooi-ubuntu}"
|
|
FORBIDDEN_LABEL_RE="${FORBIDDEN_LABEL_RE:-^(ubuntu-latest|ubuntu-[0-9].*|self-hosted|stockplatform.*|stock-platform.*|headless.*|playwright.*)$}"
|
|
MAX_CAPACITY="${MAX_CAPACITY:-1}"
|
|
MAX_LOAD_PER_CORE="${MAX_LOAD_PER_CORE:-1.25}"
|
|
MAX_HEAVY_PROCESS_COUNT="${MAX_HEAVY_PROCESS_COUNT:-0}"
|
|
REQUIRE_ROLLBACK_UNIT="${REQUIRE_ROLLBACK_UNIT:-1}"
|
|
ROLLBACK_UNIT_NAMES="${ROLLBACK_UNIT_NAMES:-awoooi-non110-runner-rollback.service gitea-act-runner-awoooi-rollback.service}"
|
|
|
|
BLOCKERS=()
|
|
WARNINGS=()
|
|
READY_CONFIG_COUNT=0
|
|
READY_BINARY_COUNT=0
|
|
READY_REGISTRATION_COUNT=0
|
|
READY_SERVICE_COUNT=0
|
|
READY_ACTIVE_SERVICE_COUNT=0
|
|
READY_AUTOSTART_PATH_COUNT=0
|
|
READY_KEEPALIVE_SERVICE_COUNT=0
|
|
READY_KEEPALIVE_TIMER_COUNT=0
|
|
|
|
section() {
|
|
printf '\n== %s ==\n' "$1"
|
|
}
|
|
|
|
blocker() {
|
|
BLOCKERS+=("$1")
|
|
printf 'BLOCKER %s\n' "$1"
|
|
}
|
|
|
|
warning() {
|
|
WARNINGS+=("$1")
|
|
printf 'WARNING %s\n' "$1"
|
|
}
|
|
|
|
command_exists() {
|
|
command -v "$1" >/dev/null 2>&1
|
|
}
|
|
|
|
host_ips() {
|
|
if command_exists ip; then
|
|
ip -o -4 addr show 2>/dev/null | awk '{print $4}' | sed 's#/.*##' | sort -u
|
|
return 0
|
|
fi
|
|
hostname -I 2>/dev/null | tr ' ' '\n' | awk 'NF' | sort -u || true
|
|
}
|
|
|
|
host_has_ip() {
|
|
local ip="$1"
|
|
host_ips | grep -qx "$ip"
|
|
}
|
|
|
|
systemd_cat() {
|
|
local unit="$1"
|
|
local load_state
|
|
load_state="$(systemctl show "$unit" -p LoadState --value 2>/dev/null || true)"
|
|
if [ -n "$load_state" ] && [ "$load_state" != "not-found" ] && systemctl cat "$unit" >/dev/null 2>&1; then
|
|
systemctl cat "$unit" 2>/dev/null
|
|
return 0
|
|
fi
|
|
load_state="$(systemctl --user show "$unit" -p LoadState --value 2>/dev/null || true)"
|
|
if [ -n "$load_state" ] && [ "$load_state" != "not-found" ] && systemctl --user cat "$unit" >/dev/null 2>&1; then
|
|
systemctl --user cat "$unit" 2>/dev/null
|
|
return 0
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
systemd_show() {
|
|
local unit="$1"
|
|
local out
|
|
if out="$(systemctl show "$unit" -p LoadState -p ActiveState -p UnitFileState -p MainPID -p Result -p ExecMainStatus --no-pager 2>/dev/null)" \
|
|
&& ! grep -q '^LoadState=not-found$' <<<"$out"; then
|
|
printf '%s\n' "$out"
|
|
return 0
|
|
fi
|
|
if out="$(systemctl --user show "$unit" -p LoadState -p ActiveState -p UnitFileState -p MainPID -p Result -p ExecMainStatus --no-pager 2>/dev/null)" \
|
|
&& ! grep -q '^LoadState=not-found$' <<<"$out"; then
|
|
printf '%s\n' "$out"
|
|
return 0
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
extract_runner_capacity() {
|
|
local config_path="$1"
|
|
awk '
|
|
/^runner:[[:space:]]*$/ {
|
|
in_runner=1
|
|
next
|
|
}
|
|
in_runner && /^[^[:space:]]/ && $0 !~ /^runner:[[:space:]]*$/ {
|
|
in_runner=0
|
|
}
|
|
in_runner && /^[[:space:]]*capacity:[[:space:]]*/ {
|
|
line=$0
|
|
sub(/^[[:space:]]*capacity:[[:space:]]*/, "", line)
|
|
gsub(/["'\'']/, "", line)
|
|
print line
|
|
exit
|
|
}
|
|
' "$config_path"
|
|
}
|
|
|
|
extract_runner_labels() {
|
|
local config_path="$1"
|
|
awk '
|
|
/^[[:space:]]*labels:[[:space:]]*$/ {
|
|
in_labels=1
|
|
next
|
|
}
|
|
in_labels && /^[[:space:]]*-[[:space:]]*/ {
|
|
line=$0
|
|
sub(/^[[:space:]]*-[[:space:]]*"/, "", line)
|
|
sub(/^[[:space:]]*-[[:space:]]*/, "", line)
|
|
sub(/"[[:space:]]*$/, "", line)
|
|
print line
|
|
next
|
|
}
|
|
in_labels && /^[^[:space:]]/ {
|
|
in_labels=0
|
|
}
|
|
' "$config_path"
|
|
}
|
|
|
|
label_name() {
|
|
printf '%s' "${1%%:*}"
|
|
}
|
|
|
|
allowed_label() {
|
|
local name="$1"
|
|
local allowed
|
|
for allowed in $ALLOWED_LABEL_NAMES; do
|
|
[ "$name" = "$allowed" ] && return 0
|
|
done
|
|
return 1
|
|
}
|
|
|
|
active_action_container_count() {
|
|
if ! command_exists docker; then
|
|
echo 0
|
|
return 0
|
|
fi
|
|
docker ps --format '{{.Names}}' 2>/dev/null | grep -Ec '^GITEA-ACTIONS-TASK-' || true
|
|
}
|
|
|
|
allowed_runner_container_name() {
|
|
local name="$1"
|
|
local allowed
|
|
for allowed in $ALLOWED_RUNNER_CONTAINER_NAMES; do
|
|
[ "$name" = "$allowed" ] && return 0
|
|
done
|
|
return 1
|
|
}
|
|
|
|
unmanaged_runner_containers() {
|
|
if ! command_exists docker; then
|
|
return 0
|
|
fi
|
|
docker ps --format '{{.Names}}|{{.Image}}' 2>/dev/null | while IFS='|' read -r name image; do
|
|
[ -n "$name" ] || continue
|
|
case "$image" in
|
|
gitea/act_runner:*|*/gitea/act_runner:*)
|
|
if ! allowed_runner_container_name "$name"; then
|
|
printf '%s\n' "$name"
|
|
fi
|
|
;;
|
|
esac
|
|
done
|
|
}
|
|
|
|
heavy_process_count() {
|
|
local count=0
|
|
count="$(
|
|
{
|
|
pgrep -f '(^|/)(chrome|chromium|chromium-browser)( |$)' 2>/dev/null || true
|
|
pgrep -f 'playwright|stockplatform.*smoke|next build|turbo build|vite build' 2>/dev/null || true
|
|
} | sort -u | wc -l | tr -d ' '
|
|
)"
|
|
echo "${count:-0}"
|
|
}
|
|
|
|
load_per_core_ok() {
|
|
if [ ! -r /proc/loadavg ] || ! command_exists awk; then
|
|
warning "loadavg_unavailable"
|
|
return 0
|
|
fi
|
|
local cores load1 ratio
|
|
cores="$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1)"
|
|
load1="$(awk '{print $1}' /proc/loadavg)"
|
|
ratio="$(awk -v load_value="$load1" -v cores="${cores:-1}" 'BEGIN { if (cores < 1) cores=1; printf "%.6f", load_value / cores }')"
|
|
printf 'LOAD_READBACK load1=%s cores=%s load_per_core=%s max=%s\n' "$load1" "$cores" "$ratio" "$MAX_LOAD_PER_CORE"
|
|
awk -v ratio="$ratio" -v max="$MAX_LOAD_PER_CORE" 'BEGIN { exit !(ratio <= max) }'
|
|
}
|
|
|
|
check_host() {
|
|
section "host selector"
|
|
printf 'target_host_ip=%s\n' "$TARGET_HOST_IP"
|
|
printf 'host_ips=%s\n' "$(host_ips | paste -sd, -)"
|
|
|
|
local forbidden
|
|
for forbidden in $FORBIDDEN_HOST_IPS; do
|
|
if host_has_ip "$forbidden"; then
|
|
blocker "target_is_forbidden_host_${forbidden}"
|
|
fi
|
|
done
|
|
|
|
if [ -n "$TARGET_HOST_IP" ] && ! host_has_ip "$TARGET_HOST_IP"; then
|
|
blocker "target_host_ip_not_present_${TARGET_HOST_IP}"
|
|
fi
|
|
}
|
|
|
|
check_configs() {
|
|
section "runner config metadata"
|
|
local config labels label name capacity config_ok
|
|
for config in $RUNNER_CONFIG_PATHS; do
|
|
if [ ! -r "$config" ]; then
|
|
printf 'RUNNER_CONFIG path=%s readable=0\n' "$config"
|
|
continue
|
|
fi
|
|
|
|
config_ok=1
|
|
capacity="$(extract_runner_capacity "$config" | head -1)"
|
|
printf 'RUNNER_CONFIG path=%s readable=1 capacity=%s max_capacity=%s\n' "$config" "${capacity:-missing}" "$MAX_CAPACITY"
|
|
if ! printf '%s' "${capacity:-}" | grep -Eq '^[0-9]+$'; then
|
|
blocker "runner_capacity_missing:${config}"
|
|
config_ok=0
|
|
elif [ "$capacity" -gt "$MAX_CAPACITY" ]; then
|
|
blocker "runner_capacity_too_high:${config}:capacity=${capacity}"
|
|
config_ok=0
|
|
fi
|
|
|
|
labels="$(extract_runner_labels "$config" || true)"
|
|
if [ -z "$labels" ]; then
|
|
blocker "runner_labels_missing:${config}"
|
|
config_ok=0
|
|
fi
|
|
|
|
while IFS= read -r label; do
|
|
[ -n "$label" ] || continue
|
|
name="$(label_name "$label")"
|
|
printf 'RUNNER_LABEL path=%s label=%s name=%s\n' "$config" "$label" "$name"
|
|
if printf '%s' "$name" | grep -Eq "$FORBIDDEN_LABEL_RE"; then
|
|
blocker "forbidden_runner_label:${name}:${config}"
|
|
config_ok=0
|
|
elif ! allowed_label "$name"; then
|
|
blocker "unexpected_runner_label:${name}:${config}"
|
|
config_ok=0
|
|
fi
|
|
done <<<"$labels"
|
|
|
|
if [ "$config_ok" -eq 1 ]; then
|
|
READY_CONFIG_COUNT=$((READY_CONFIG_COUNT + 1))
|
|
fi
|
|
done
|
|
|
|
if [ "$READY_CONFIG_COUNT" -eq 0 ]; then
|
|
blocker "no_ready_runner_config"
|
|
fi
|
|
}
|
|
|
|
check_binaries() {
|
|
section "runner runtime metadata"
|
|
local binary image kind binary_ok image_ok
|
|
for binary in $RUNNER_BINARY_PATHS; do
|
|
binary_ok=0
|
|
if [ -x "$binary" ] && [ -f "$binary" ]; then
|
|
binary_ok=1
|
|
kind="$(file -b "$binary" 2>/dev/null || echo unknown)"
|
|
READY_BINARY_COUNT=$((READY_BINARY_COUNT + 1))
|
|
else
|
|
kind="missing_or_not_executable"
|
|
fi
|
|
printf 'RUNNER_BINARY path=%s executable=%s kind=%s\n' "$binary" "$binary_ok" "$kind"
|
|
done
|
|
|
|
if command_exists docker; then
|
|
for image in $RUNNER_DOCKER_IMAGES; do
|
|
image_ok=0
|
|
if docker image inspect "$image" >/dev/null 2>&1; then
|
|
image_ok=1
|
|
kind="$(docker image inspect "$image" --format '{{.Id}} {{.Created}}' 2>/dev/null || echo present)"
|
|
else
|
|
kind="missing"
|
|
fi
|
|
printf 'RUNNER_DOCKER_IMAGE image=%s present=%s accepted_as_ready_runtime=0 kind=%s\n' "$image" "$image_ok" "$kind"
|
|
done
|
|
else
|
|
printf 'RUNNER_DOCKER_IMAGE docker_available=0\n'
|
|
fi
|
|
|
|
if [ "$READY_BINARY_COUNT" -eq 0 ]; then
|
|
blocker "runner_runtime_missing"
|
|
fi
|
|
}
|
|
|
|
check_registrations() {
|
|
section "runner registration metadata"
|
|
local registration mode size
|
|
for registration in $RUNNER_REGISTRATION_PATHS; do
|
|
if [ -f "$registration" ] && [ -s "$registration" ]; then
|
|
size="$(stat -c '%s' "$registration" 2>/dev/null || echo unknown)"
|
|
mode="$(stat -c '%a' "$registration" 2>/dev/null || echo unknown)"
|
|
printf 'RUNNER_REGISTRATION path=%s present=1 size_bytes=%s mode=%s content_read=false\n' "$registration" "$size" "$mode"
|
|
READY_REGISTRATION_COUNT=$((READY_REGISTRATION_COUNT + 1))
|
|
continue
|
|
fi
|
|
printf 'RUNNER_REGISTRATION path=%s present=0 content_read=false\n' "$registration"
|
|
done
|
|
|
|
if [ "$READY_REGISTRATION_COUNT" -eq 0 ]; then
|
|
blocker "runner_registration_missing"
|
|
fi
|
|
}
|
|
|
|
unit_has_required_limits() {
|
|
local unit="$1"
|
|
local text="$2"
|
|
grep -Eq '^[[:space:]]*CPUQuota=' <<<"$text" || return 1
|
|
grep -Eq '^[[:space:]]*Memory(Max|High)=' <<<"$text" || return 1
|
|
grep -Eq '^[[:space:]]*TasksMax=' <<<"$text" || return 1
|
|
grep -Eq '^[[:space:]]*NoNewPrivileges=true' <<<"$text" || return 1
|
|
grep -Eq '^[[:space:]]*Restart=' <<<"$text" || return 1
|
|
grep -Eq '^[[:space:]]*ConditionPathExists=.*\.runner[[:space:]]*$' <<<"$text" || return 1
|
|
printf 'RUNNER_SERVICE_LIMITS unit=%s cpu=1 memory=1 tasks=1 no_new_privileges=1 restart=1 registration_condition=1\n' "$unit"
|
|
return 0
|
|
}
|
|
|
|
unit_has_verified_runner_target() {
|
|
local unit="$1"
|
|
local text="$2"
|
|
local config binary has_config=0 has_binary=0 matched_config="" matched_binary=""
|
|
for config in $RUNNER_CONFIG_PATHS; do
|
|
if [ -r "$config" ] && grep -Fq -- "$config" <<<"$text"; then
|
|
has_config=1
|
|
matched_config="$config"
|
|
break
|
|
fi
|
|
done
|
|
for binary in $RUNNER_BINARY_PATHS; do
|
|
if [ -x "$binary" ] && [ -f "$binary" ] && grep -Fq -- "$binary" <<<"$text"; then
|
|
has_binary=1
|
|
matched_binary="$binary"
|
|
break
|
|
fi
|
|
done
|
|
if [ "$has_config" -eq 1 ] && [ "$has_binary" -eq 1 ]; then
|
|
printf 'RUNNER_SERVICE_TARGETS unit=%s binary=%s config=%s verified=1\n' "$unit" "$matched_binary" "$matched_config"
|
|
return 0
|
|
fi
|
|
printf 'RUNNER_SERVICE_TARGETS unit=%s binary_verified=%s config_verified=%s verified=0\n' "$unit" "$has_binary" "$has_config"
|
|
return 1
|
|
}
|
|
|
|
check_services() {
|
|
section "runner service metadata"
|
|
local unit text state
|
|
for unit in $RUNNER_SERVICE_NAMES; do
|
|
if ! text="$(systemd_cat "$unit" 2>/dev/null)"; then
|
|
printf 'RUNNER_SERVICE unit=%s installed=0\n' "$unit"
|
|
continue
|
|
fi
|
|
state="$(systemd_show "$unit" | tr '\n' ' ' || true)"
|
|
if grep -q 'LoadState=not-found' <<<"$state"; then
|
|
printf 'RUNNER_SERVICE unit=%s installed=0\n' "$unit"
|
|
continue
|
|
fi
|
|
printf 'RUNNER_SERVICE unit=%s installed=1 %s\n' "$unit" "$state"
|
|
if grep -q 'ActiveState=active' <<<"$state" && grep -Eq 'MainPID=[1-9][0-9]*' <<<"$state"; then
|
|
READY_ACTIVE_SERVICE_COUNT=$((READY_ACTIVE_SERVICE_COUNT + 1))
|
|
printf 'RUNNER_SERVICE_ACTIVE unit=%s active=1 main_pid=1\n' "$unit"
|
|
else
|
|
blocker "runner_service_not_active:${unit}"
|
|
fi
|
|
local limits_ok=0 target_ok=0
|
|
if unit_has_required_limits "$unit" "$text"; then
|
|
limits_ok=1
|
|
else
|
|
blocker "runner_service_limits_missing:${unit}"
|
|
fi
|
|
if unit_has_verified_runner_target "$unit" "$text"; then
|
|
target_ok=1
|
|
else
|
|
blocker "runner_service_target_mismatch:${unit}"
|
|
fi
|
|
if [ "$limits_ok" -eq 1 ] && [ "$target_ok" -eq 1 ]; then
|
|
READY_SERVICE_COUNT=$((READY_SERVICE_COUNT + 1))
|
|
fi
|
|
done
|
|
|
|
if [ "$READY_SERVICE_COUNT" -eq 0 ]; then
|
|
blocker "no_ready_runner_service"
|
|
fi
|
|
if [ "$READY_ACTIVE_SERVICE_COUNT" -eq 0 ]; then
|
|
blocker "no_active_runner_service"
|
|
fi
|
|
}
|
|
|
|
check_autostart_paths() {
|
|
section "runner autostart metadata"
|
|
local unit text state registration_watch active enabled
|
|
for unit in $RUNNER_AUTOSTART_PATH_UNIT_NAMES; do
|
|
if ! text="$(systemd_cat "$unit" 2>/dev/null)"; then
|
|
printf 'RUNNER_AUTOSTART_PATH unit=%s installed=0\n' "$unit"
|
|
continue
|
|
fi
|
|
state="$(systemd_show "$unit" | tr '\n' ' ' || true)"
|
|
if grep -q 'LoadState=not-found' <<<"$state"; then
|
|
printf 'RUNNER_AUTOSTART_PATH unit=%s installed=0\n' "$unit"
|
|
continue
|
|
fi
|
|
registration_watch=0
|
|
active=0
|
|
enabled=0
|
|
if grep -Eq '^[[:space:]]*PathExists=.*\.runner[[:space:]]*$' <<<"$text"; then
|
|
registration_watch=1
|
|
fi
|
|
if grep -q 'ActiveState=active' <<<"$state"; then
|
|
active=1
|
|
fi
|
|
if grep -q 'UnitFileState=enabled' <<<"$state"; then
|
|
enabled=1
|
|
fi
|
|
printf 'RUNNER_AUTOSTART_PATH unit=%s installed=1 %s\n' "$unit" "$state"
|
|
printf 'RUNNER_AUTOSTART_PATH_WATCH unit=%s registration_watch=%s active=%s enabled=%s\n' "$unit" "$registration_watch" "$active" "$enabled"
|
|
if [ "$registration_watch" -eq 1 ] && [ "$active" -eq 1 ] && [ "$enabled" -eq 1 ]; then
|
|
READY_AUTOSTART_PATH_COUNT=$((READY_AUTOSTART_PATH_COUNT + 1))
|
|
fi
|
|
done
|
|
}
|
|
|
|
check_keepalive_timers() {
|
|
section "runner keepalive service metadata"
|
|
local service_unit service_text service_state
|
|
for service_unit in $RUNNER_KEEPALIVE_SERVICE_UNIT_NAMES; do
|
|
if ! service_text="$(systemd_cat "$service_unit" 2>/dev/null)"; then
|
|
printf 'RUNNER_KEEPALIVE_SERVICE unit=%s installed=0\n' "$service_unit"
|
|
continue
|
|
fi
|
|
service_state="$(systemd_show "$service_unit" | tr '\n' ' ' || true)"
|
|
if grep -q 'LoadState=not-found' <<<"$service_state"; then
|
|
printf 'RUNNER_KEEPALIVE_SERVICE unit=%s installed=0\n' "$service_unit"
|
|
continue
|
|
fi
|
|
printf 'RUNNER_KEEPALIVE_SERVICE unit=%s installed=1 %s\n' "$service_unit" "$service_state"
|
|
if grep -q 'ActiveState=failed' <<<"$service_state" \
|
|
|| grep -Eq 'Result=(exit-code|signal|timeout|core-dump)' <<<"$service_state"; then
|
|
blocker "runner_keepalive_service_failed:${service_unit}"
|
|
continue
|
|
fi
|
|
if grep -Eq '^[[:space:]]*ExecStart=-/usr/bin/systemctl --user reset-failed ' <<<"$service_text" \
|
|
&& grep -Eq '^[[:space:]]*ExecStart=/usr/bin/systemctl --user daemon-reload' <<<"$service_text"; then
|
|
READY_KEEPALIVE_SERVICE_COUNT=$((READY_KEEPALIVE_SERVICE_COUNT + 1))
|
|
else
|
|
blocker "runner_keepalive_service_recovery_steps_missing:${service_unit}"
|
|
fi
|
|
done
|
|
|
|
section "runner keepalive metadata"
|
|
local unit text state active enabled interval
|
|
for unit in $RUNNER_KEEPALIVE_TIMER_UNIT_NAMES; do
|
|
if ! text="$(systemd_cat "$unit" 2>/dev/null)"; then
|
|
printf 'RUNNER_KEEPALIVE_TIMER unit=%s installed=0\n' "$unit"
|
|
continue
|
|
fi
|
|
state="$(systemd_show "$unit" | tr '\n' ' ' || true)"
|
|
if grep -q 'LoadState=not-found' <<<"$state"; then
|
|
printf 'RUNNER_KEEPALIVE_TIMER unit=%s installed=0\n' "$unit"
|
|
continue
|
|
fi
|
|
active=0
|
|
enabled=0
|
|
interval=0
|
|
if grep -q 'ActiveState=active' <<<"$state"; then
|
|
active=1
|
|
fi
|
|
if grep -q 'UnitFileState=enabled' <<<"$state"; then
|
|
enabled=1
|
|
fi
|
|
if grep -Eq '^[[:space:]]*OnUnitInactiveSec=' <<<"$text"; then
|
|
interval=1
|
|
fi
|
|
printf 'RUNNER_KEEPALIVE_TIMER unit=%s installed=1 %s\n' "$unit" "$state"
|
|
printf 'RUNNER_KEEPALIVE_TIMER_WATCH unit=%s active=%s enabled=%s interval=%s\n' "$unit" "$active" "$enabled" "$interval"
|
|
if [ "$active" -eq 1 ] && [ "$enabled" -eq 1 ] && [ "$interval" -eq 1 ]; then
|
|
READY_KEEPALIVE_TIMER_COUNT=$((READY_KEEPALIVE_TIMER_COUNT + 1))
|
|
fi
|
|
done
|
|
|
|
if [ "$READY_REGISTRATION_COUNT" -gt 0 ] \
|
|
&& [ "$READY_SERVICE_COUNT" -gt 0 ] \
|
|
&& [ "$READY_KEEPALIVE_SERVICE_COUNT" -eq 0 ]; then
|
|
blocker "runner_keepalive_service_not_ready"
|
|
fi
|
|
if [ "$READY_REGISTRATION_COUNT" -gt 0 ] \
|
|
&& [ "$READY_SERVICE_COUNT" -gt 0 ] \
|
|
&& [ "$READY_KEEPALIVE_TIMER_COUNT" -eq 0 ]; then
|
|
blocker "runner_keepalive_timer_not_ready"
|
|
fi
|
|
}
|
|
|
|
check_rollback() {
|
|
section "rollback metadata"
|
|
local unit found=0
|
|
for unit in $ROLLBACK_UNIT_NAMES; do
|
|
if systemd_cat "$unit" >/dev/null 2>&1; then
|
|
found=1
|
|
printf 'ROLLBACK_UNIT unit=%s installed=1\n' "$unit"
|
|
else
|
|
printf 'ROLLBACK_UNIT unit=%s installed=0\n' "$unit"
|
|
fi
|
|
done
|
|
|
|
if [ "$REQUIRE_ROLLBACK_UNIT" = "1" ] && [ "$found" -eq 0 ]; then
|
|
blocker "rollback_unit_missing"
|
|
fi
|
|
}
|
|
|
|
check_pressure() {
|
|
section "pressure readback"
|
|
local containers heavy unmanaged
|
|
containers="$(active_action_container_count)"
|
|
heavy="$(heavy_process_count)"
|
|
unmanaged="$(unmanaged_runner_containers | paste -sd, -)"
|
|
printf 'ACTIVE_ACTION_CONTAINERS=%s\n' "$containers"
|
|
printf 'UNMANAGED_RUNNER_CONTAINERS=%s\n' "${unmanaged:-none}"
|
|
printf 'HEAVY_PROCESS_COUNT=%s max=%s\n' "$heavy" "$MAX_HEAVY_PROCESS_COUNT"
|
|
[ "$containers" = "0" ] || blocker "active_action_containers_present:${containers}"
|
|
[ -z "$unmanaged" ] || blocker "unmanaged_runner_containers_present:${unmanaged}"
|
|
[ "$heavy" -le "$MAX_HEAVY_PROCESS_COUNT" ] || blocker "heavy_processes_present:${heavy}"
|
|
load_per_core_ok || blocker "host_load_per_core_too_high"
|
|
}
|
|
|
|
main() {
|
|
section "audit metadata"
|
|
printf 'read_only=true\n'
|
|
printf 'secret_values_collected=false\n'
|
|
printf 'runner_token_read=false\n'
|
|
printf 'raw_runner_registration_read=false\n'
|
|
printf 'timestamp=%s\n' "$(date -Is 2>/dev/null || date)"
|
|
printf 'host=%s\n' "$(hostname 2>/dev/null || echo unknown)"
|
|
printf 'user=%s\n' "$(id -un 2>/dev/null || echo unknown)"
|
|
|
|
check_host
|
|
check_configs
|
|
check_binaries
|
|
check_registrations
|
|
check_services
|
|
check_autostart_paths
|
|
check_keepalive_timers
|
|
check_rollback
|
|
check_pressure
|
|
|
|
section "verdict"
|
|
printf 'READY_CONFIG_COUNT=%s\n' "$READY_CONFIG_COUNT"
|
|
printf 'READY_BINARY_COUNT=%s\n' "$READY_BINARY_COUNT"
|
|
printf 'READY_REGISTRATION_COUNT=%s\n' "$READY_REGISTRATION_COUNT"
|
|
printf 'READY_SERVICE_COUNT=%s\n' "$READY_SERVICE_COUNT"
|
|
printf 'READY_ACTIVE_SERVICE_COUNT=%s\n' "$READY_ACTIVE_SERVICE_COUNT"
|
|
printf 'READY_AUTOSTART_PATH_COUNT=%s\n' "$READY_AUTOSTART_PATH_COUNT"
|
|
printf 'READY_KEEPALIVE_SERVICE_COUNT=%s\n' "$READY_KEEPALIVE_SERVICE_COUNT"
|
|
printf 'READY_KEEPALIVE_TIMER_COUNT=%s\n' "$READY_KEEPALIVE_TIMER_COUNT"
|
|
printf 'WARNING_COUNT=%s\n' "${#WARNINGS[@]}"
|
|
printf 'BLOCKER_COUNT=%s\n' "${#BLOCKERS[@]}"
|
|
if [ "${#BLOCKERS[@]}" -eq 0 ]; then
|
|
printf 'AWOOOI_NON110_RUNNER_READY=1\n'
|
|
printf 'safe_next_step=restore_auto_events_to_non110_labels_after_110_failclosed_gate_passes\n'
|
|
return 0
|
|
fi
|
|
printf 'AWOOOI_NON110_RUNNER_READY=0\n'
|
|
if [ "$READY_CONFIG_COUNT" -gt 0 ] \
|
|
&& [ "$READY_BINARY_COUNT" -gt 0 ] \
|
|
&& [ "$READY_SERVICE_COUNT" -gt 0 ] \
|
|
&& [ "$READY_REGISTRATION_COUNT" -eq 0 ]; then
|
|
if [ "$READY_AUTOSTART_PATH_COUNT" -gt 0 ]; then
|
|
printf 'safe_next_step=run_register_awoooi_non110_runner_script_without_printing_token_then_autostart_path_will_enable_service_and_rerun_this_verifier\n'
|
|
return 1
|
|
fi
|
|
printf 'safe_next_step=run_register_awoooi_non110_runner_script_without_printing_token_then_enable_service_and_rerun_this_verifier\n'
|
|
return 1
|
|
fi
|
|
printf 'safe_next_step=install_or_fix_non110_runner_config_service_rollback_then_rerun_this_verifier\n'
|
|
return 1
|
|
}
|
|
|
|
main "$@"
|