#!/usr/bin/env bash set -uo pipefail # One-entry read-only post-reboot check. This wrapper intentionally delegates # deep checks to the existing recovery scripts and does not restart, patch, # delete, import, reload, or write runtime state. ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" SSH_CONNECT_TIMEOUT="${SSH_CONNECT_TIMEOUT:-6}" ROUTE_RETRY_ATTEMPTS="${ROUTE_RETRY_ATTEMPTS:-3}" ROUTE_RETRY_DELAY_SECONDS="${ROUTE_RETRY_DELAY_SECONDS:-2}" STOCK_FRESHNESS_RETRY_ATTEMPTS="${STOCK_FRESHNESS_RETRY_ATTEMPTS:-6}" STOCK_FRESHNESS_RETRY_DELAY_SECONDS="${STOCK_FRESHNESS_RETRY_DELAY_SECONDS:-5}" RUN_COLD_START=1 RUN_MOMO=1 RUN_STOCK=1 RUN_BACKUP=1 RUN_ROUTES=1 RUN_CPU=1 NO_COLOR_FLAG=0 COLD_START_PENDING_BLOCKERS=0 COLD_START_BLOCKED_SUMMARY="" COLD_START_BLOCKED_LINES="" ROUTE_SMOKE_BLOCKED=0 AWOOOI_API_ROUTE_OK=0 STOCK_EOD_WINDOW_PENDING=0 STOCK_EOD_CLASSIFICATION="not_evaluated" STOCK_EOD_NEXT_ACTION="not_evaluated" STOCK_EOD_FIRST_FULL_WINDOW_END_LOCAL="19:15" PASS_COUNT=0 WARN_COUNT=0 BLOCKED_COUNT=0 SERVICE_WARN_COUNT=0 BOUNDARY_WARN_COUNT=0 EVIDENCE_WARN_COUNT=0 HOSTS=( "192.168.0.110" "192.168.0.120" "192.168.0.121" "192.168.0.188" ) ROUTES=( "https://awoooi.wooo.work/" "https://awoooi.wooo.work/api/v1/health" "https://awoooi.wooo.work/zh-TW/iwooos" "https://vibework.wooo.work/" "https://awooogo.wooo.work/" "https://2026fifa.wooo.work/" "https://agent.wooo.work/" "https://mo.wooo.work/" "https://mo.wooo.work/health" "https://stock.wooo.work/" "https://stock.wooo.work/healthz" "https://stock.wooo.work/api/healthz" "https://bitan.wooo.work/" "https://tsenyang.com/" "https://www.tsenyang.com/" "https://vtuber.wooo.work/" "https://gitea.wooo.work/" "https://harbor.wooo.work/" "https://registry.wooo.work/" "https://sentry.wooo.work/" "https://signoz.wooo.work/" "https://langfuse.wooo.work/" "https://aiops.wooo.work/" ) usage() { cat <<'USAGE' Usage: post-start-quick-check.sh [options] Read-only post-reboot quick check for 110 / 120 / 121 / 188. Options: --skip-cold-start Do not run full-stack-cold-start-check.sh. --skip-momo Do not run momo-drive-token-source-recovery-preflight.sh. --skip-stock Do not query StockPlatform data freshness. --skip-backup Do not run /backup/scripts/backup-status.sh on 110. --skip-routes Do not curl public route smoke targets. --skip-cpu Do not read 110 CPU / process summary. --no-color Disable ANSI color. -h, --help Show this help. Environment: ROUTE_RETRY_ATTEMPTS Public route attempts before blocking. Default: 3. ROUTE_RETRY_DELAY_SECONDS Delay between failed public route attempts. Default: 2. STOCK_FRESHNESS_RETRY_ATTEMPTS Stock freshness attempts before blocking. Default: 6. STOCK_FRESHNESS_RETRY_DELAY_SECONDS Delay between failed Stock freshness attempts. Default: 5. Exit codes: 0 = no service blockers. Boundary / evidence warnings may still be present. 1 = service warnings only. 2 = service blockers observed. This script never reads token content and never writes runtime state. USAGE } while [[ $# -gt 0 ]]; do case "$1" in --skip-cold-start) RUN_COLD_START=0 ;; --skip-momo) RUN_MOMO=0 ;; --skip-stock) RUN_STOCK=0 ;; --skip-backup) RUN_BACKUP=0 ;; --skip-routes) RUN_ROUTES=0 ;; --skip-cpu) RUN_CPU=0 ;; --no-color) NO_COLOR_FLAG=1 ;; -h|--help) usage exit 0 ;; *) printf 'Unknown argument: %s\n' "$1" >&2 usage >&2 exit 2 ;; esac shift done if [[ -n "${NO_COLOR:-}" || "$NO_COLOR_FLAG" -eq 1 ]]; then RED="" GREEN="" YELLOW="" BLUE="" NC="" else RED=$'\033[0;31m' GREEN=$'\033[0;32m' YELLOW=$'\033[1;33m' BLUE=$'\033[0;34m' NC=$'\033[0m' fi section() { printf '\n%s=== %s ===%s\n' "$BLUE" "$1" "$NC" } ok() { PASS_COUNT=$((PASS_COUNT + 1)) printf '%sOK%s %s\n' "$GREEN" "$NC" "$*" } warn() { WARN_COUNT=$((WARN_COUNT + 1)) printf '%sWARN%s %s\n' "$YELLOW" "$NC" "$*" } service_warn() { SERVICE_WARN_COUNT=$((SERVICE_WARN_COUNT + 1)) warn "$@" } boundary_warn() { BOUNDARY_WARN_COUNT=$((BOUNDARY_WARN_COUNT + 1)) warn "$@" } evidence_warn() { EVIDENCE_WARN_COUNT=$((EVIDENCE_WARN_COUNT + 1)) warn "$@" } blocked() { BLOCKED_COUNT=$((BLOCKED_COUNT + 1)) printf '%sBLOCKED%s %s\n' "$RED" "$NC" "$*" } ssh_read() { local user_host="$1" local command="$2" ssh -o BatchMode=yes -o ConnectTimeout="$SSH_CONNECT_TIMEOUT" "$user_host" "$command" } run_and_capture() { local label="$1" shift local tmp tmp="$(mktemp -t post-start-quick-check.XXXXXX)" if "$@" >"$tmp" 2>&1; then ok "$label" cat "$tmp" rm -f "$tmp" return 0 fi local rc=$? cat "$tmp" rm -f "$tmp" return "$rc" } section "主機 / SSH" for host in "${HOSTS[@]}"; do if ping -c 1 -W 1 "$host" >/dev/null 2>&1; then ok "PING_OK $host" else blocked "PING_FAIL $host" fi if nc -z -w 2 "$host" 22 >/dev/null 2>&1; then ok "SSH_PORT_OK $host" else blocked "SSH_PORT_FAIL $host" fi done if [[ "$RUN_COLD_START" -eq 1 ]]; then section "Cold-start scorecard" cold_tmp="$(mktemp -t post-start-cold-start.XXXXXX)" cold_rc=0 if bash "$ROOT_DIR/scripts/reboot-recovery/full-stack-cold-start-check.sh" --monitor-read-only --no-color --watch --interval 1 --max-attempts 1 >"$cold_tmp" 2>&1; then cold_rc=0 else cold_rc=$? fi cat "$cold_tmp" cold_summary="$(grep -E 'PASS=[0-9]+ WARN=[0-9]+ BLOCKED=[0-9]+' "$cold_tmp" | tail -n 1 || true)" if [[ -n "$cold_summary" ]]; then ok "cold-start summary: $cold_summary" cold_warn=0 cold_blocked=0 if [[ "$cold_summary" =~ WARN=([0-9]+) ]]; then cold_warn="${BASH_REMATCH[1]}" fi if [[ "$cold_summary" =~ BLOCKED=([0-9]+) ]]; then cold_blocked="${BASH_REMATCH[1]}" fi if [[ "$cold_blocked" -gt 0 ]]; then COLD_START_PENDING_BLOCKERS="$cold_blocked" COLD_START_BLOCKED_SUMMARY="$cold_summary" COLD_START_BLOCKED_LINES="$(grep -E '^BLOCKED ' "$cold_tmp" || true)" evidence_warn "cold-start blockers pending wrapper retry classification: $cold_summary" elif [[ "$cold_warn" -gt 0 ]]; then service_warn "cold-start is warning-only, not blocked: $cold_summary" elif [[ "$cold_rc" -eq 0 ]]; then ok "cold-start command exited 0" else evidence_warn "cold-start exited $cold_rc but summary has no blockers: $cold_summary" fi else if [[ "$cold_rc" -eq 0 ]]; then service_warn "cold-start summary not found" else blocked "cold-start command returned $cold_rc without summary" fi fi rm -f "$cold_tmp" fi if [[ "$RUN_MOMO" -eq 1 ]]; then section "MOMO freshness" momo_tmp="$(mktemp -t post-start-momo.XXXXXX)" bash "$ROOT_DIR/scripts/reboot-recovery/momo-drive-token-source-recovery-preflight.sh" >"$momo_tmp" 2>&1 momo_rc=$? cat "$momo_tmp" momo_summary="$(grep -E 'MOMO_DRIVE_TOKEN_SOURCE_PREFLIGHT PASS=[0-9]+ WARN=[0-9]+ BLOCKED=[0-9]+' "$momo_tmp" | tail -n 1 || true)" case "$momo_rc" in 0) ok "MOMO preflight clean" ;; 1) if [[ "$momo_summary" =~ BLOCKED=0 ]]; then evidence_warn "MOMO preflight has non-service warnings" else service_warn "MOMO preflight has warnings and no clean summary" fi ;; *) blocked "MOMO preflight has blockers" ;; esac grep -E 'MOMO_DRIVE_TOKEN_SOURCE_PREFLIGHT|MOMO_HEALTH_VERSION|DB_MONTHLY_SYNC|DB_DAILY_FRESHNESS|DB_LATEST_DAILY_IMPORT_JOB' "$momo_tmp" || true source_gate_output="$("$ROOT_DIR/scripts/reboot-recovery/momo-source-arrival-gate.py" --preflight-log "$momo_tmp" 2>&1)" source_gate_rc=$? printf '%s\n' "$source_gate_output" if grep -q 'status=blocked_source_absent_fail_closed' <<<"$source_gate_output"; then evidence_warn "MOMO source-arrival gate confirms source absent fail-closed" elif grep -q 'status=source_arrived_ready_for_safe_import_preflight' <<<"$source_gate_output"; then boundary_warn "MOMO source arrived; safe import preflight only, DB/Drive/runtime writes remain unauthorized" elif grep -q 'status=freshness_already_green_recheck_cold_start' <<<"$source_gate_output"; then ok "MOMO source-arrival gate reports freshness green; rerun post-reboot summary before updating declaration" elif [[ "$source_gate_rc" -ne 0 ]]; then service_warn "MOMO source-arrival gate did not produce a known state rc=$source_gate_rc" else evidence_warn "MOMO source-arrival gate produced a non-terminal state" fi rm -f "$momo_tmp" fi if [[ "$RUN_STOCK" -eq 1 ]]; then section "StockPlatform freshness" stock_tmp="$(mktemp -t post-start-stock.XXXXXX)" stock_code="" stock_attempt=1 while [[ "$stock_attempt" -le "$STOCK_FRESHNESS_RETRY_ATTEMPTS" ]]; do stock_code="$(curl -k -sS -o "$stock_tmp" -w '%{http_code}' --max-time 12 "https://stock.wooo.work/api/v1/system/freshness" 2>/dev/null || true)" if [[ "$stock_code" == 2* ]]; then if [[ "$stock_attempt" -gt 1 ]]; then evidence_warn "StockPlatform freshness recovered after attempt=$stock_attempt" fi break fi if [[ "$stock_attempt" -lt "$STOCK_FRESHNESS_RETRY_ATTEMPTS" ]]; then sleep "$STOCK_FRESHNESS_RETRY_DELAY_SECONDS" fi stock_attempt=$((stock_attempt + 1)) done if [[ "$stock_code" != 2* ]]; then blocked "StockPlatform freshness endpoint returned ${stock_code:-curl_failed} attempts=$STOCK_FRESHNESS_RETRY_ATTEMPTS" cat "$stock_tmp" || true else python3 - "$stock_tmp" <<'PY' import json import sys path = sys.argv[1] with open(path, "r", encoding="utf-8") as fh: payload = json.load(fh) print(f"STOCK_FRESHNESS_STATUS {payload.get('status')}") print(f"STOCK_LATEST_TRADING_DATE {payload.get('latest_trading_date')}") print("STOCK_BLOCKERS " + ",".join(payload.get("blockers") or [])) for source in payload.get("sources") or []: print( "STOCK_SOURCE " f"{source.get('source')}|{source.get('status')}|" f"{source.get('latest_date')}|{source.get('row_count')}" ) PY stock_status="$(python3 - "$stock_tmp" <<'PY' import json import sys with open(sys.argv[1], "r", encoding="utf-8") as fh: print(json.load(fh).get("status") or "") PY )" if [[ "$stock_status" == "ok" ]]; then printf 'STOCK_EOD_WINDOW_PENDING 0\n' printf 'STOCK_EOD_CLASSIFICATION ok\n' printf 'STOCK_EOD_NEXT_ACTION none\n' ok "StockPlatform freshness is ok" else stock_blockers="$(python3 - "$stock_tmp" <<'PY' import json import sys with open(sys.argv[1], "r", encoding="utf-8") as fh: print(",".join(json.load(fh).get("blockers") or [])) PY )" stock_eod_context="$(python3 - "$stock_tmp" <<'PY' import json import sys from datetime import datetime, time, timezone, timedelta try: from zoneinfo import ZoneInfo except Exception: # pragma: no cover - old Python fallback ZoneInfo = None with open(sys.argv[1], "r", encoding="utf-8") as fh: payload = json.load(fh) tz = ZoneInfo("Asia/Taipei") if ZoneInfo else timezone(timedelta(hours=8)) now = datetime.now(tz) today = now.date().isoformat() latest = payload.get("latest_trading_date") or "" status = payload.get("status") or "unknown" blockers = payload.get("blockers") or [] first_full_window_end = time(19, 15) final_retry_window_end = time(23, 35) pending = 0 classification = "blocked_unknown" next_action = "investigate_stockplatform_freshness" if status == "ok": classification = "ok" next_action = "none" elif latest == today and blockers and now.time() < first_full_window_end: pending = 1 classification = "pending_first_eod_window" next_action = "wait_for_18_20_19_10_cron_then_recheck" elif latest == today and blockers and now.time() < final_retry_window_end: classification = "after_first_eod_window_blocked" next_action = "inspect_ingestion_logs_and_wait_retry_windows" elif latest == today and blockers: classification = "after_final_eod_window_blocked" next_action = "open_stockplatform_data_recovery_gate" elif blockers: classification = "blocked_non_current_trading_day" next_action = "inspect_trading_calendar_and_ingestion_logs" print(f"STOCK_EOD_WINDOW_PENDING {pending}") print(f"STOCK_EOD_CLASSIFICATION {classification}") print(f"STOCK_EOD_NEXT_ACTION {next_action}") print(f"STOCK_EOD_FIRST_FULL_WINDOW_END_LOCAL {first_full_window_end.strftime('%H:%M')}") print(f"STOCK_EOD_FINAL_RETRY_WINDOW_END_LOCAL {final_retry_window_end.strftime('%H:%M')}") print(f"STOCK_EOD_OBSERVED_AT_LOCAL {now.isoformat(timespec='seconds')}") PY )" printf '%s\n' "$stock_eod_context" STOCK_EOD_WINDOW_PENDING="$(awk '$1 == "STOCK_EOD_WINDOW_PENDING" {print $2}' <<<"$stock_eod_context" | tail -n 1)" STOCK_EOD_CLASSIFICATION="$(awk '$1 == "STOCK_EOD_CLASSIFICATION" {print $2}' <<<"$stock_eod_context" | tail -n 1)" STOCK_EOD_NEXT_ACTION="$(awk '$1 == "STOCK_EOD_NEXT_ACTION" {print $2}' <<<"$stock_eod_context" | tail -n 1)" if [[ "$STOCK_EOD_WINDOW_PENDING" == "1" ]]; then evidence_warn "StockPlatform freshness pending scheduled EOD window: ${STOCK_EOD_CLASSIFICATION:-unknown}; next=${STOCK_EOD_NEXT_ACTION:-unknown}; blockers=${stock_blockers:-no_blocker_list}" else blocked "StockPlatform freshness is ${stock_status:-unknown}: ${stock_blockers:-no_blocker_list}; classification=${STOCK_EOD_CLASSIFICATION:-unknown}" fi fi fi rm -f "$stock_tmp" fi if [[ "$RUN_BACKUP" -eq 1 ]]; then section "Backup / offsite / escrow" backup_tmp="$(mktemp -t post-start-backup.XXXXXX)" if ssh_read "wooo@192.168.0.110" '/backup/scripts/backup-status.sh --no-notify --no-refresh' >"$backup_tmp" 2>&1; then ok "backup-status readback succeeded" else blocked "backup-status readback failed" fi cat "$backup_tmp" if grep -Eq 'core_blockers=0|CORE_BLOCKERS[ =]0' "$backup_tmp"; then ok "backup core blockers are 0" elif grep -Eq 'core_blockers=[1-9]|CORE_BLOCKERS[ =][1-9]' "$backup_tmp"; then blocked "backup core blockers are non-zero" else service_warn "backup core blocker summary not confirmed" fi if grep -Eq 'escrow_missing=0|ESCROW_MISSING_COUNT[ =]0' "$backup_tmp"; then ok "credential escrow missing is 0" elif grep -Eq 'escrow_missing=[1-9]|ESCROW_MISSING_COUNT[ =][1-9]' "$backup_tmp"; then boundary_warn "credential escrow still missing; DR_COMPLETE is forbidden" escrow_status_tmp="$(mktemp -t post-start-escrow-status.XXXXXX)" if ssh_read "wooo@192.168.0.110" '/backup/scripts/mark-credential-escrow-verified.sh --status' >"$escrow_status_tmp" 2>&1; then evidence_warn "credential escrow missing items follow" else evidence_warn "credential escrow missing item readback failed" fi cat "$escrow_status_tmp" rm -f "$escrow_status_tmp" else evidence_warn "credential escrow count not found" fi rm -f "$backup_tmp" fi if [[ "$RUN_ROUTES" -eq 1 ]]; then section "Public routes" for url in "${ROUTES[@]}"; do code="" attempt=1 while [[ "$attempt" -le "$ROUTE_RETRY_ATTEMPTS" ]]; do code="$(curl -k -sS -o /dev/null -w '%{http_code}' --max-time 12 "$url" 2>/dev/null || true)" case "$code" in 2*|3*) break ;; esac if [[ "$attempt" -lt "$ROUTE_RETRY_ATTEMPTS" ]]; then sleep "$ROUTE_RETRY_DELAY_SECONDS" fi attempt=$((attempt + 1)) done case "$code" in 2*|3*) if [[ "$url" == "https://awoooi.wooo.work/api/v1/health" && "$code" == 2* ]]; then AWOOOI_API_ROUTE_OK=1 fi if [[ "$attempt" -gt 1 ]]; then evidence_warn "$code $url recovered_after_attempt=$attempt" else ok "$code $url" fi ;; *) ROUTE_SMOKE_BLOCKED=$((ROUTE_SMOKE_BLOCKED + 1)) blocked "${code:-curl_failed} $url attempts=$ROUTE_RETRY_ATTEMPTS" ;; esac done fi if [[ "$COLD_START_PENDING_BLOCKERS" -gt 0 ]]; then non_route_cold_blockers="$(printf '%s\n' "$COLD_START_BLOCKED_LINES" | grep -Ev '^BLOCKED public route ' || true)" if [[ "$RUN_ROUTES" -eq 1 && "$ROUTE_SMOKE_BLOCKED" -eq 0 && "$AWOOOI_API_ROUTE_OK" -eq 1 ]]; then non_route_cold_blockers="$( printf '%s\n' "$non_route_cold_blockers" | grep -Ev '^BLOCKED AWOOOI API not reachable$|^BLOCKED AWOOI API not reachable$' || true )" fi if [[ "$RUN_ROUTES" -eq 1 && "$ROUTE_SMOKE_BLOCKED" -eq 0 && -z "$non_route_cold_blockers" ]]; then evidence_warn "cold-start route/API warmup blockers recovered under wrapper route retry: $COLD_START_BLOCKED_SUMMARY" printf '%s\n' "$COLD_START_BLOCKED_LINES" else blocked "cold-start has blockers: $COLD_START_BLOCKED_SUMMARY" printf '%s\n' "$COLD_START_BLOCKED_LINES" fi fi if [[ "$RUN_CPU" -eq 1 ]]; then section "110 CPU / process attribution" cpu_tmp="$(mktemp -t post-start-cpu.XXXXXX)" if ssh_read "wooo@192.168.0.110" 'uptime; vmstat 1 5; ps -eo pid,ppid,pgid,stat,pcpu,pmem,comm,args --sort=-pcpu | head -25' >"$cpu_tmp" 2>&1; then ok "110 CPU/process readback succeeded" else evidence_warn "110 CPU/process readback failed" fi cat "$cpu_tmp" if grep -Eiq 'chrome|chromium|playwright' "$cpu_tmp"; then evidence_warn "browser/smoke process is visible; classify orphan vs active parent before action" fi if grep -Eiq 'gitea|actions|runner|npm|pnpm|pytest|pip-audit' "$cpu_tmp"; then ok "active CI/build/test load is visible" fi rm -f "$cpu_tmp" fi section "110 runner fail-closed guard" runner_tmp="$(mktemp -t post-start-runner.XXXXXX)" if ssh_read "wooo@192.168.0.110" ' for u in awoooi-cd-lane.service awoooi-cd-lane-drain.service awoooi-direct-runner-open.service awoooi-direct-runner.service gitea-act-runner-host.service gitea-act-runner-awoooi-controlled.service gitea-awoooi-controlled-runner.service gitea-act-runner-awoooi-open.service; do load=$(systemctl show "$u" -p LoadState --value 2>/dev/null || true) unitfile=$(systemctl show "$u" -p UnitFileState --value 2>/dev/null || true) active=$(systemctl show "$u" -p ActiveState --value 2>/dev/null || true) mainpid=$(systemctl show "$u" -p MainPID --value 2>/dev/null || true) unit_ok=0 unit_stub=0 if [ "$load" = "masked" ] && [ "$unitfile" = "masked" ] && [ "$active" = "inactive" ] && [ "${mainpid:-0}" = "0" ]; then unit_ok=1 elif [ "$active" = "inactive" ] && [ "${mainpid:-0}" = "0" ] \ && systemctl cat "$u" 2>/dev/null | grep -q "ConditionPathExists=/run/awoooi-runner-migrated-or-hard-limited"; then unit_stub=1 unit_ok=1 fi echo "RUNNER_FAILCLOSED_UNIT $u load=$load unitfile=$unitfile active=$active mainpid=$mainpid stub=$unit_stub ok=$unit_ok" done enforcer_timer_active=$(systemctl is-active awoooi-runner-failclosed-enforcer.timer 2>/dev/null || true) enforcer_timer_enabled=$(systemctl is-enabled awoooi-runner-failclosed-enforcer.timer 2>/dev/null || true) enforcer_service_result=$(systemctl show awoooi-runner-failclosed-enforcer.service -p Result --value 2>/dev/null || true) echo "RUNNER_FAILCLOSED_ENFORCER timer_active=$enforcer_timer_active timer_enabled=$enforcer_timer_enabled service_result=$enforcer_service_result" authority_timer_active=$(systemctl is-active awoooi-runner-failclosed-authority.timer 2>/dev/null || true) authority_timer_enabled=$(systemctl is-enabled awoooi-runner-failclosed-authority.timer 2>/dev/null || true) authority_service_result=$(systemctl show awoooi-runner-failclosed-authority.service -p Result --value 2>/dev/null || true) echo "RUNNER_FAILCLOSED_AUTHORITY timer_active=$authority_timer_active timer_enabled=$authority_timer_enabled service_result=$authority_service_result" cd_lane_process_count=$(pgrep -f "^/home/wooo/awoooi-cd-lane/awoooi_cd_lane|^/home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled" 2>/dev/null | wc -l | tr -d " ") echo "CD_LANE_PROCESS_COUNT $cd_lane_process_count" cd_lane_root_restore_left=unknown if sudo -n true >/dev/null 2>&1; then cd_lane_root_restore_left=$(sudo -n find /root -maxdepth 1 -type d \( -name "awoooi-runner-restore-sources-disabled*" -o -name "awoooi-cd-lane-disabled*" -o -name "awoooi-cd-lane-drain-disabled*" \) -print 2>/dev/null | wc -l | tr -d " ") fi echo "CD_LANE_ROOT_RESTORE_SOURCES left=$cd_lane_root_restore_left" sentinel_left=0 for s in /run/awoooi-runner-host-enabled /run/awoooi-start-controlled-cd-lane /run/awoooi-start-controlled-cd-lane-drain /run/awoooi-start-cd-lane-allowed /run/awoooi-cd-lane-drain-ok /run/awoooi-cd-lane-ok /run/awoooi-cd-lane-enabled /run/awoooi-cd-lane-controlled-open; do [ -e "$s" ] && sentinel_left=$((sentinel_left + 1)) done echo "RUNNER_SENTINELS_LEFT $sentinel_left" active_job_containers=$(docker ps --format "{{.Names}}" 2>/dev/null | grep -Ec "^(GITEA-ACTIONS-|awoooi-cd-)" || true) echo "ACTIVE_JOB_CONTAINERS $active_job_containers" cd_lane_guard_ok=0 if [ "$enforcer_timer_active" = "active" ] \ && [ "$enforcer_timer_enabled" = "enabled" ] \ && [ "$enforcer_service_result" = "success" ] \ && [ "$authority_timer_active" = "active" ] \ && [ "$authority_timer_enabled" = "enabled" ] \ && [ "$authority_service_result" = "success" ] \ && [ "$cd_lane_process_count" = "0" ] \ && [ "$cd_lane_root_restore_left" = "0" ] \ && [ "$sentinel_left" = "0" ] \ && [ "$active_job_containers" = "0" ]; then cd_lane_guard_ok=1 fi echo "CD_LANE_GUARDRAILS_OK $cd_lane_guard_ok" direct_runner_count=$(pgrep -f "^/home/wooo/act-runner/act_runner|^/home/wooo/act-runner-controlled/act_runner|^/home/wooo/awoooi-controlled-runner/awoooi_controlled_runner" 2>/dev/null | wc -l | tr -d " ") echo "RUNNER_DIRECT_PROCESS_COUNT $direct_runner_count" for p in /home/wooo/awoooi-cd-lane/awoooi_cd_lane /home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled /home/wooo/act-runner/act_runner /home/wooo/act-runner/act_runner.real-20260628-runner-pressure-guard /home/wooo/act-runner-controlled/act_runner /home/wooo/awoooi-controlled-runner/awoooi_controlled_runner; do kind=$(file -b "$p" 2>/dev/null || echo missing) echo "RUNNER_FAILCLOSED_BINARY $p kind=$kind" echo "$kind" | grep -qi "ELF" && echo "RUNNER_FAILCLOSED_BINARY_ELF $p" done HOST_WEB_BUILD_PRESSURE_ATTEMPTS=1 HOST_WEB_BUILD_PRESSURE_SLEEP_SECONDS=0 /usr/local/bin/awoooi-wait-host-web-build-pressure.sh echo "RUNNER_PRESSURE_GATE_RC $?" ' >"$runner_tmp" 2>&1; then ok "110 controlled runner readback succeeded" else blocked "110 controlled runner readback failed" fi cat "$runner_tmp" if awk '$1 == "RUNNER_FAILCLOSED_UNIT" && $NF != "ok=1" {bad=1} END {exit bad}' "$runner_tmp"; then ok "110 runner/CD lane units are fail-closed" else blocked "110 runner/CD lane units are not fail-closed" fi grep -q "RUNNER_FAILCLOSED_ENFORCER timer_active=active timer_enabled=enabled service_result=success" "$runner_tmp" && ok "110 fail-closed enforcer timer active and successful" || blocked "110 fail-closed enforcer timer not healthy" grep -q "RUNNER_FAILCLOSED_AUTHORITY timer_active=active timer_enabled=enabled service_result=success" "$runner_tmp" && ok "110 fail-closed authority timer active and successful" || blocked "110 fail-closed authority timer not healthy" grep -q "CD_LANE_GUARDRAILS_OK 1" "$runner_tmp" && ok "110 cd-lane/drain lane are fail-closed with enforcer" || blocked "110 cd-lane/drain lane fail-closed guardrails incomplete" grep -q "RUNNER_DIRECT_PROCESS_COUNT 0" "$runner_tmp" && ok "110 legacy direct runner process count is zero" || blocked "110 legacy direct runner process detected" grep -q "ACTIVE_JOB_CONTAINERS 0" "$runner_tmp" && ok "110 Gitea/CD job container count is zero" || blocked "110 Gitea/CD job container still active" grep -q "RUNNER_FAILCLOSED_BINARY_ELF" "$runner_tmp" && blocked "110 runner fail-closed binary path restored to ELF" || ok "110 runner binary paths are fail-closed stubs or missing" grep -q "RUNNER_PRESSURE_GATE_RC 0" "$runner_tmp" && ok "110 host pressure gate returned 0" || blocked "110 host pressure gate is blocking" rm -f "$runner_tmp" section "總結" printf 'POST_START_QUICK_CHECK PASS=%s WARN=%s BLOCKED=%s\n' "$PASS_COUNT" "$WARN_COUNT" "$BLOCKED_COUNT" printf 'POST_START_QUICK_CHECK_WARNINGS SERVICE=%s BOUNDARY=%s EVIDENCE=%s\n' "$SERVICE_WARN_COUNT" "$BOUNDARY_WARN_COUNT" "$EVIDENCE_WARN_COUNT" if [[ "$BLOCKED_COUNT" -gt 0 ]]; then printf 'RESULT=BLOCKED\n' exit 2 fi if [[ "$SERVICE_WARN_COUNT" -gt 0 ]]; then printf 'RESULT=DEGRADED\n' exit 1 fi if [[ "$STOCK_EOD_WINDOW_PENDING" == "1" ]]; then printf 'RESULT=PRODUCT_DATA_PENDING_EOD_WINDOW\n' exit 0 fi if [[ "$BOUNDARY_WARN_COUNT" -gt 0 ]]; then printf 'RESULT=FULL_STACK_GREEN_DR_ESCROW_BLOCKED\n' exit 0 fi if [[ "$EVIDENCE_WARN_COUNT" -gt 0 ]]; then printf 'RESULT=GREEN_WITH_EVIDENCE_WARNINGS\n' exit 0 fi printf 'RESULT=GREEN\n' exit 0