#!/usr/bin/env bash
# AWOOOI full-stack cold-start readiness check.
# Read-only by design. It never restarts, deletes, repairs, or writes remote state.

set -uo pipefail

SSH_COMMAND_TIMEOUT_SECONDS="${SSH_COMMAND_TIMEOUT_SECONDS:-45}"
SSH_110_CHECK_ATTEMPTS="${SSH_110_CHECK_ATTEMPTS:-4}"
SSH_110_CHECK_SLEEP_SECONDS="${SSH_110_CHECK_SLEEP_SECONDS:-5}"
SSH_STRICT_HOST_KEY_CHECKING="${SSH_STRICT_HOST_KEY_CHECKING:-accept-new}"
REGISTRY_HTTPS_URL="${REGISTRY_HTTPS_URL:-https://192.168.0.110:5000/v2/}"
REGISTRY_HTTP_URL="${REGISTRY_HTTP_URL:-http://192.168.0.110:5000/v2/}"
SSH_OPTS=(
  -o BatchMode=yes
  -o ConnectTimeout=6
  -o ConnectionAttempts=1
  -o ServerAliveInterval=5
  -o ServerAliveCountMax=1
  -o StrictHostKeyChecking="$SSH_STRICT_HOST_KEY_CHECKING"
)
SEND_ALERT_TEST=0
MONITOR_READ_ONLY=0
NO_COLOR_FLAG=0
WATCH_MODE=0
WATCH_INTERVAL=60
WATCH_MAX_ATTEMPTS=30
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
MOMO_SOURCE_PREFLIGHT_SCRIPT="${MOMO_SOURCE_PREFLIGHT_SCRIPT:-$SCRIPT_DIR/momo-drive-token-source-recovery-preflight.sh}"

usage() {
  cat <<'USAGE'
Usage: bash scripts/reboot-recovery/full-stack-cold-start-check.sh [options]

Options:
  --send-alert-test       POST one Alertmanager webhook test after AWOOOI API is ready.
  --monitor-read-only     Skip the webhook POST without warning; intended for cron/textfile monitors.
  --watch                 Repeat checks until all gates are GREEN or max attempts is reached.
  --interval SECONDS      Retry interval for --watch. Default: 60.
  --max-attempts COUNT    Max attempts for --watch. Default: 30. Use 0 for unlimited.
  --no-color              Disable ANSI colors in output.
  -h, --help              Show this help.

Default mode is read-only and does not POST an Alertmanager test event.
Use --send-alert-test for the final release gate after AWOOOI API is expected to be ready.
USAGE
}

while [ "$#" -gt 0 ]; do
  arg="$1"
  case "$arg" in
    --send-alert-test)
      SEND_ALERT_TEST=1
      ;;
    --monitor-read-only)
      MONITOR_READ_ONLY=1
      SEND_ALERT_TEST=0
      ;;
    --no-color)
      NO_COLOR_FLAG=1
      ;;
    --watch)
      WATCH_MODE=1
      ;;
    --interval)
      shift
      if ! [[ "${1:-}" =~ ^[0-9]+$ ]] || [ "${1:-0}" -lt 1 ]; then
        echo "--interval requires a positive integer number of seconds" >&2
        exit 64
      fi
      WATCH_INTERVAL="$1"
      ;;
    --max-attempts)
      shift
      if ! [[ "${1:-}" =~ ^[0-9]+$ ]]; then
        echo "--max-attempts requires a non-negative integer" >&2
        exit 64
      fi
      WATCH_MAX_ATTEMPTS="$1"
      ;;
    -h|--help)
      usage
      exit 0
      ;;
    *)
      echo "Unknown argument: $arg" >&2
      usage >&2
      exit 64
      ;;
  esac
  shift
done

if [ -n "${NO_COLOR:-}" ] || [ "$NO_COLOR_FLAG" -eq 1 ]; then
  RED=""
  GREEN=""
  YELLOW=""
  BLUE=""
  NC=""
else
  RED=$'\033[0;31m'
  GREEN=$'\033[0;32m'
  YELLOW=$'\033[1;33m'
  BLUE=$'\033[0;34m'
  NC=$'\033[0m'
fi

PASS=0
WARN=0
FAIL=0

log_section() {
  printf "\n%s=== %s ===%s\n" "$BLUE" "$1" "$NC"
}

ok() {
  printf "%sOK%s %s\n" "$GREEN" "$NC" "$1"
  PASS=$((PASS + 1))
}

warn() {
  printf "%sWARN%s %s\n" "$YELLOW" "$NC" "$1"
  WARN=$((WARN + 1))
}

fail() {
  printf "%sBLOCKED%s %s\n" "$RED" "$NC" "$1"
  FAIL=$((FAIL + 1))
}

run_local() {
  local label="$1"
  shift
  if "$@" >/tmp/awoooi-cold-start-check.out 2>&1; then
    ok "$label"
    cat /tmp/awoooi-cold-start-check.out
    return 0
  fi
  fail "$label"
  cat /tmp/awoooi-cold-start-check.out
  return 1
}

ssh_cmd() {
  local user_host="$1"
  local cmd="$2"
  local prefix=""
  local quoted_cmd=""
  if [ -n "${REMOTE_SUDO_PASSWORD:-}" ]; then
    printf -v prefix 'REMOTE_SUDO_PASSWORD=%q ' "$REMOTE_SUDO_PASSWORD"
  fi
  printf -v quoted_cmd '%q' "$cmd"
  ssh "${SSH_OPTS[@]}" "$user_host" "${prefix}if command -v timeout >/dev/null 2>&1; then timeout ${SSH_COMMAND_TIMEOUT_SECONDS}s bash -lc ${quoted_cmd}; else bash -lc ${quoted_cmd}; fi"
}

host_has_ip() {
  local expected_ip="$1"
  if command -v ip >/dev/null 2>&1; then
    ip -o -4 addr show 2>/dev/null | awk '{print $4}' | grep -q "^${expected_ip}/" && return 0
  fi
  hostname -I 2>/dev/null | tr ' ' '\n' | grep -qx "$expected_ip"
}

host_cmd() {
  local user_host="$1"
  local cmd="$2"
  case "$user_host" in
    *@192.168.0.110)
      if host_has_ip "192.168.0.110"; then
        bash -lc "$cmd"
        return
      fi
      ;;
    *@192.168.0.120)
      if host_has_ip "192.168.0.120"; then
        bash -lc "$cmd"
        return
      fi
      ;;
    *@192.168.0.121)
      if host_has_ip "192.168.0.121"; then
        bash -lc "$cmd"
        return
      fi
      ;;
    *@192.168.0.188)
      if host_has_ip "192.168.0.188"; then
        bash -lc "$cmd"
        return
      fi
      ;;
  esac
  ssh_cmd "$user_host" "$cmd"
}

host_cmd_retry() {
  local user_host="$1"
  local cmd="$2"
  local attempts="$3"
  local sleep_seconds="$4"
  local attempt rc out
  for attempt in $(seq 1 "$attempts"); do
    out="$(host_cmd "$user_host" "$cmd" 2>&1)"
    rc=$?
    if [ "$rc" -eq 0 ]; then
      [ "$attempt" -gt 1 ] && echo "SSH_RETRY_RECOVERED user_host=$user_host attempt=$attempt/$attempts"
      echo "$out"
      return 0
    fi
    echo "SSH_RETRY user_host=$user_host attempt=$attempt/$attempts rc=$rc"
    echo "$out"
    [ "$attempt" -lt "$attempts" ] && sleep "$sleep_seconds"
  done
  return "$rc"
}

probe_http_code() {
  local url="$1"
  local attempt code
  for attempt in 1 2; do
    code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 12 "$url" 2>/dev/null || true)
    if [[ "$code" =~ ^[0-9]{3}$ ]] && [ "$code" != "000" ]; then
      echo "$code"
      return
    fi
    sleep 1
  done
  echo "${code:-000}"
}

probe_http_code_insecure() {
  local url="$1"
  local attempt code
  for attempt in 1 2; do
    code=$(curl -ks -o /dev/null -w "%{http_code}" --max-time 12 "$url" 2>/dev/null || true)
    if [[ "$code" =~ ^[0-9]{3}$ ]] && [ "$code" != "000" ]; then
      echo "$code"
      return
    fi
    sleep 1
  done
  echo "${code:-000}"
}

registry_code_ready() {
  local code="$1"
  [ "$code" = "200" ] || [ "$code" = "401" ]
}

probe_tcp() {
  local host="$1"
  local port="$2"
  nc -G 3 -z "$host" "$port" >/dev/null 2>&1 || nc -w 3 -z "$host" "$port" >/dev/null 2>&1
}

print_neighbor_rows() {
  if command -v arp >/dev/null 2>&1; then
    arp -an | grep -E '192\.168\.0\.(110|120|121|188)'
    return $?
  fi
  if command -v ip >/dev/null 2>&1; then
    ip neigh show | grep -E '192\.168\.0\.(110|120|121|188)'
    return $?
  fi
  return 1
}

print_header() {
  echo "AWOOOI full-stack cold-start check"
  date '+%Y-%m-%d %H:%M:%S %Z'
  echo "Scope: 110 / 120 / 121 / 188. 112 Kali is intentionally skipped."
  echo "Baseline: ops/reboot-recovery/full-stack-cold-start-baseline.yml"
}

check_network() {
  log_section "P0-NETWORK"
  local host
  for host in 110 120 121 188; do
    if ping -c 1 -W 2 "192.168.0.$host" >/dev/null 2>&1; then
      ok "ping 192.168.0.$host"
    else
      fail "ping 192.168.0.$host"
    fi

    if probe_tcp "192.168.0.$host" 22; then
      ok "ssh port 192.168.0.$host:22"
    else
      fail "ssh port 192.168.0.$host:22"
    fi
  done

  if print_neighbor_rows; then
    ok "neighbor evidence printed"
  elif [ "$MONITOR_READ_ONLY" -eq 1 ]; then
    ok "neighbor evidence unavailable in monitor mode; ping and TCP gates provide primary signal"
  else
    warn "no neighbor rows printed for one or more hosts"
  fi
}

check_188() {
  log_section "P0-188-DATA"
  local out
  if ! out=$(host_cmd "ollama@192.168.0.188" '
echo "HOST $(hostname) $(uptime)"
echo "MEM $(free -h | awk "/Mem:/ {print \$2,\$3,\$7}")"
echo "SYSTEMD $(systemctl is-active containerd docker postgresql@14-main redis-server ollama nginx 2>/dev/null | tr "\n" " ")"
echo "PG $(pg_isready -h localhost -p 5432 2>&1)"
echo "REDIS $(redis-cli -p 6380 ping 2>/dev/null || redis-cli ping 2>/dev/null || true)"
echo "PORT5432 $(nc -z -w 2 127.0.0.1 5432 >/dev/null 2>&1 && echo OPEN || echo CLOSED)"
echo "SIGNOZ_CODE $(curl -s -o /dev/null -w "%{http_code}" --max-time 5 http://192.168.0.110:8080/ || true)"
echo "MOMO_HEALTH_CODE $(curl -s -o /dev/null -w "%{http_code}" --max-time 5 http://127.0.0.1:5003/health || true)"
docker ps --format "DOCKER {{.Names}}\t{{.Status}}" | head -80
' 2>&1); then
    fail "ssh 188 read-only check"
    echo "$out"
    return
  fi
  echo "$out"

  grep -q "PORT5432 OPEN" <<<"$out" && ok "188 PostgreSQL port open" || fail "188 PostgreSQL port closed"
  grep -q "accepting connections" <<<"$out" && ok "188 PostgreSQL accepting connections" || fail "188 PostgreSQL not accepting connections"
  grep -q "REDIS PONG" <<<"$out" && ok "188 Redis PONG" || warn "188 Redis not confirmed"
  grep -q "momo-db.*Restarting" <<<"$out" && warn "188 momo-db restarting" || ok "188 momo-db not in visible restart loop"
  grep -Eq "SIGNOZ_CODE (200|302|307)" <<<"$out" && ok "SignOz UI upstream reachable from 188" || warn "SignOz UI upstream not confirmed from 188"
  grep -q "MOMO_HEALTH_CODE 200" <<<"$out" && ok "188 momo health reachable" || warn "188 momo health not confirmed"
}

check_110() {
  log_section "P0-110-REGISTRY-OBSERVABILITY"
  local out registry_https_code registry_http_code
  registry_https_code="$(probe_http_code_insecure "$REGISTRY_HTTPS_URL")"
  registry_http_code="$(probe_http_code "$REGISTRY_HTTP_URL")"
  echo "REGISTRY_EXTERNAL_HTTPS_CODE $registry_https_code $REGISTRY_HTTPS_URL"
  echo "REGISTRY_EXTERNAL_HTTP_CODE $registry_http_code $REGISTRY_HTTP_URL"
  if registry_code_ready "$registry_https_code" || registry_code_ready "$registry_http_code"; then
    ok "110 registry external /v2 reachable"
  else
    fail "110 registry external /v2 not reachable"
  fi

  if ! out=$(host_cmd_retry "wooo@192.168.0.110" '
sc() {
  if command -v timeout >/dev/null 2>&1; then
    timeout 3 systemctl "$@" 2>/dev/null || true
  else
    systemctl "$@" 2>/dev/null || true
  fi
}
echo "HOST $(hostname) $(uptime)"
echo "MEM $(free -h | awk "/Mem:/ {print \$2,\$3,\$7}")"
echo "DOCKER_SYSTEMD $(sc is-active docker)"
echo "HARBOR_CODE $(curl -s -o /dev/null -w "%{http_code}" --max-time 5 http://127.0.0.1:5000/v2/ || true)"
echo "GITEA_CODE $(curl -s -o /dev/null -w "%{http_code}" --max-time 5 http://127.0.0.1:3001/ || true)"
echo "PROM_CODE $(curl -s -o /dev/null -w "%{http_code}" --max-time 5 http://127.0.0.1:9090/-/ready || true)"
echo "AM_CODE $(curl -s -o /dev/null -w "%{http_code}" --max-time 5 http://127.0.0.1:9093/-/healthy || true)"
echo "SENTRY_CODE $(curl -s -o /dev/null -w "%{http_code}" --max-time 8 http://127.0.0.1:9000/ || true)"
echo "ACTION_RUNNER_UNIT_FILE_COUNT $(sc list-unit-files "actions.runner.*" --no-legend --plain | awk "END {print NR+0}")"
echo "ACTION_RUNNER_ACTIVE_COUNT $(sc list-units "actions.runner.*" --state=active --no-legend --plain | awk "END {print NR+0}")"
echo "ACTION_RUNNER_ENABLED_COUNT $(sc list-unit-files "actions.runner.*" --no-legend --plain | awk "\$2 == \"enabled\" {c++} END {print c+0}")"
for u in $(sc list-units "actions.runner.*" --all --no-legend --plain | awk "{print \$1}"); do
  sc show "$u" -p ActiveState -p SubState -p CPUQuotaPerSecUSec -p MemoryMax -p WatchdogUSec -p NRestarts | sed "s/^/RUNNER $u /"
done
for u in awoooi-direct-runner-open.service awoooi-direct-runner.service gitea-act-runner-host.service gitea-act-runner-awoooi-controlled.service gitea-awoooi-controlled-runner.service gitea-act-runner-awoooi-open.service; do
  load=$(sc show "$u" -p LoadState --value)
  unitfile=$(sc show "$u" -p UnitFileState --value)
  active=$(sc show "$u" -p ActiveState --value)
  mainpid=$(sc show "$u" -p MainPID --value)
  unit_ok=0
  if [ "$load" = "masked" ] && [ "$unitfile" = "masked" ] && [ "$active" = "inactive" ]; then
    unit_ok=1
  fi
  echo "RUNNER_FAILCLOSED_UNIT $u load=$load unitfile=$unitfile active=$active mainpid=$mainpid ok=$unit_ok"
done
cd_lane_load=$(sc show awoooi-cd-lane.service -p LoadState --value)
cd_lane_unitfile=$(sc show awoooi-cd-lane.service -p UnitFileState --value)
cd_lane_active=$(sc show awoooi-cd-lane.service -p ActiveState --value)
cd_lane_mainpid=$(sc show awoooi-cd-lane.service -p MainPID --value)
cd_lane_execstart=$(sc show awoooi-cd-lane.service -p ExecStart --value)
cd_lane_sentinel=missing
[ -e /run/awoooi-cd-lane-enabled ] && cd_lane_sentinel=present
cd_lane_capacity_ok=0
cd_lane_labels_ok=0
if grep -Eq "^[[:space:]]+capacity:[[:space:]]*1[[:space:]]*$" /home/wooo/awoooi-cd-lane/config.yaml 2>/dev/null; then
  cd_lane_capacity_ok=1
fi
if grep -q "awoooi-ubuntu:docker://192.168.0.110:5000/awoooi/ci-runner:act-22.04" /home/wooo/awoooi-cd-lane/config.yaml 2>/dev/null \
  && grep -q "awoooi-host:host" /home/wooo/awoooi-cd-lane/config.yaml 2>/dev/null \
  && ! grep -Eq "^[[:space:]]+- \".*(ubuntu-latest|stockplatform|headless|playwright)" /home/wooo/awoooi-cd-lane/config.yaml 2>/dev/null; then
  cd_lane_labels_ok=1
fi
cd_lane_binary_kind=$(file -b /home/wooo/awoooi-cd-lane/awoooi_cd_lane 2>/dev/null || echo missing)
cd_lane_binary_elf=0
echo "$cd_lane_binary_kind" | grep -qi "ELF" && cd_lane_binary_elf=1
cd_lane_process_count=$(pgrep -f "^/home/wooo/awoooi-cd-lane/awoooi_cd_lane" 2>/dev/null | wc -l | tr -d " ")
cd_lane_ok=0
cd_lane_mode=blocked
if [ "$cd_lane_active" = "inactive" ] \
  && [ "$cd_lane_sentinel" = "missing" ] \
  && [ "$cd_lane_binary_elf" = "0" ] \
  && [ "$cd_lane_process_count" = "0" ] \
  && { { [ "$cd_lane_load" = "masked" ] && [ "$cd_lane_unitfile" = "masked" ]; } || echo "$cd_lane_execstart" | grep -q "/bin/false"; }; then
  cd_lane_ok=1
  cd_lane_mode=failclosed
elif [ "$cd_lane_sentinel" = "present" ] && [ "$cd_lane_active" = "active" ] && [ "$cd_lane_capacity_ok" = "1" ] && [ "$cd_lane_labels_ok" = "1" ] && [ "$cd_lane_binary_elf" = "1" ]; then
  cd_lane_ok=1
  cd_lane_mode=controlled_open
fi
echo "CD_LANE_CONTROLLED mode=$cd_lane_mode load=$cd_lane_load unitfile=$cd_lane_unitfile active=$cd_lane_active mainpid=$cd_lane_mainpid sentinel=$cd_lane_sentinel capacity=$cd_lane_capacity_ok labels=$cd_lane_labels_ok binary_elf=$cd_lane_binary_elf process_count=$cd_lane_process_count ok=$cd_lane_ok"
cd_lane_drain_load=$(sc show awoooi-cd-lane-drain.service -p LoadState --value)
cd_lane_drain_unitfile=$(sc show awoooi-cd-lane-drain.service -p UnitFileState --value)
cd_lane_drain_active=$(sc show awoooi-cd-lane-drain.service -p ActiveState --value)
cd_lane_drain_mainpid=$(sc show awoooi-cd-lane-drain.service -p MainPID --value)
cd_lane_drain_cpu_accounting=$(sc show awoooi-cd-lane-drain.service -p CPUAccounting --value)
cd_lane_drain_cpu_quota=$(sc show awoooi-cd-lane-drain.service -p CPUQuotaPerSecUSec --value)
cd_lane_drain_memory_accounting=$(sc show awoooi-cd-lane-drain.service -p MemoryAccounting --value)
cd_lane_drain_memory_max=$(sc show awoooi-cd-lane-drain.service -p MemoryMax --value)
cd_lane_drain_tasks_accounting=$(sc show awoooi-cd-lane-drain.service -p TasksAccounting --value)
cd_lane_drain_tasks_max=$(sc show awoooi-cd-lane-drain.service -p TasksMax --value)
cd_lane_drain_limits_ok=0
if [ "$cd_lane_drain_cpu_accounting" = "yes" ] \
  && [ -n "$cd_lane_drain_cpu_quota" ] && [ "$cd_lane_drain_cpu_quota" != "infinity" ] \
  && [ "$cd_lane_drain_memory_accounting" = "yes" ] \
  && [ -n "$cd_lane_drain_memory_max" ] && [ "$cd_lane_drain_memory_max" != "infinity" ] \
  && [ "$cd_lane_drain_tasks_accounting" = "yes" ] \
  && [ -n "$cd_lane_drain_tasks_max" ] && [ "$cd_lane_drain_tasks_max" != "infinity" ]; then
  cd_lane_drain_limits_ok=1
fi
cd_lane_drain_capacity_ok=0
cd_lane_drain_labels_ok=0
if grep -Eq "^[[:space:]]+capacity:[[:space:]]*1[[:space:]]*$" /home/wooo/awoooi-cd-lane-drain/config.yaml 2>/dev/null; then
  cd_lane_drain_capacity_ok=1
fi
if grep -q "awoooi-ubuntu:docker://192.168.0.110:5000/awoooi/ci-runner:act-22.04" /home/wooo/awoooi-cd-lane-drain/config.yaml 2>/dev/null \
  && grep -q "awoooi-host:host" /home/wooo/awoooi-cd-lane-drain/config.yaml 2>/dev/null \
  && ! grep -Eq "^[[:space:]]+- \".*(ubuntu-latest|stockplatform|headless|playwright)" /home/wooo/awoooi-cd-lane-drain/config.yaml 2>/dev/null; then
  cd_lane_drain_labels_ok=1
fi
cd_lane_drain_binary_kind=$(file -b /home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled 2>/dev/null || echo missing)
cd_lane_drain_binary_elf=0
echo "$cd_lane_drain_binary_kind" | grep -qi "ELF" && cd_lane_drain_binary_elf=1
cd_lane_drain_process_count=$(pgrep -f "^/home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled" 2>/dev/null | wc -l | tr -d " ")
cd_lane_drain_ok=0
cd_lane_drain_mode=blocked
if [ "$cd_lane_drain_active" != "active" ] \
  && [ "$cd_lane_drain_binary_elf" = "0" ] \
  && [ "$cd_lane_drain_process_count" = "0" ] \
  && { [ "$cd_lane_drain_load" = "not-found" ] || { [ "$cd_lane_drain_load" = "masked" ] && [ "$cd_lane_drain_unitfile" = "masked" ]; }; }; then
  cd_lane_drain_ok=1
  cd_lane_drain_mode=failclosed
elif [ "$cd_lane_drain_active" = "active" ] \
  && [ "$cd_lane_drain_capacity_ok" = "1" ] \
  && [ "$cd_lane_drain_labels_ok" = "1" ] \
  && [ "$cd_lane_drain_binary_elf" = "1" ] \
  && [ "$cd_lane_drain_limits_ok" = "1" ]; then
  cd_lane_drain_ok=1
  cd_lane_drain_mode=controlled_open
fi
echo "CD_LANE_DRAIN_CONTROLLED mode=$cd_lane_drain_mode load=$cd_lane_drain_load unitfile=$cd_lane_drain_unitfile active=$cd_lane_drain_active mainpid=$cd_lane_drain_mainpid capacity=$cd_lane_drain_capacity_ok labels=$cd_lane_drain_labels_ok binary_elf=$cd_lane_drain_binary_elf limits=$cd_lane_drain_limits_ok process_count=$cd_lane_drain_process_count ok=$cd_lane_drain_ok"
cd_lane_root_restore_left=unknown
if sudo -n true >/dev/null 2>&1; then
  cd_lane_root_restore_left=$(sudo -n find /root -maxdepth 1 -type d \( -name "awoooi-cd-lane-disabled-*" -o -name "awoooi-cd-lane-drain-disabled-*" \) -print 2>/dev/null | wc -l | tr -d " ")
fi
echo "CD_LANE_ROOT_RESTORE_SOURCES left=$cd_lane_root_restore_left"
cd_lane_guard_ok=0
if { [ "$cd_lane_ok" = "1" ] || [ "$cd_lane_drain_ok" = "1" ]; } && [ "$cd_lane_root_restore_left" = "0" ]; then
  cd_lane_guard_ok=1
fi
echo "CD_LANE_GUARDRAILS_OK $cd_lane_guard_ok"
direct_runner_count=$(pgrep -f "^/home/wooo/act-runner/act_runner|^/home/wooo/act-runner-controlled/act_runner|^/home/wooo/awoooi-controlled-runner/awoooi_controlled_runner" 2>/dev/null | wc -l | tr -d " ")
echo "RUNNER_DIRECT_PROCESS_COUNT $direct_runner_count"
for p in /home/wooo/act-runner/act_runner /home/wooo/act-runner/act_runner.real-20260628-runner-pressure-guard /home/wooo/act-runner-controlled/act_runner /home/wooo/awoooi-controlled-runner/awoooi_controlled_runner; do
  kind=$(file -b "$p" 2>/dev/null || echo missing)
  echo "RUNNER_FAILCLOSED_BINARY $p kind=$kind"
  echo "$kind" | grep -qi "ELF" && echo "RUNNER_FAILCLOSED_BINARY_ELF $p"
done
docker ps --format "DOCKER {{.Names}}\t{{.Status}}" | head -120
' "$SSH_110_CHECK_ATTEMPTS" "$SSH_110_CHECK_SLEEP_SECONDS" 2>&1); then
    fail "ssh 110 read-only check"
    echo "SSH_110_BLOCKER remote_control_channel_unavailable"
    echo "SSH_110_NEXT_ACTION local_console_run_recover_110_control_path_and_harbor_local_check"
    echo "SSH_110_RECOVERY_PACKAGE_EXPECTED /usr/local/bin/recover-110-control-path-and-harbor-local.sh"
    echo "SSH_110_RECOVERY_PACKAGE_EXPECTED /usr/local/bin/repair-110-ssh-publickey-auth-local.sh"
    echo "SSH_110_RECOVERY_PACKAGE_EXPECTED /usr/local/bin/check-awoooi-110-controlled-cd-lane-readiness.sh"
    echo "SSH_110_RECOVERY_PACKAGE_BLOCKER package_presence_not_verifiable_while_remote_control_channel_unavailable"
    echo "SSH_110_RECOVERY_PACKAGE_NEXT_ACTION verify_or_preinstall_local_recovery_package_from_console_before_harbor_repair_retry"
    echo "$out"
    return
  fi
  echo "$out"

  grep -Eq "HARBOR_CODE (200|401)" <<<"$out" && ok "110 Harbor /v2 healthy code" || fail "110 Harbor not healthy"
  grep -Eq "GITEA_CODE (200|302)" <<<"$out" && ok "110 Gitea reachable" || warn "110 Gitea not confirmed"
  grep -q "PROM_CODE 200" <<<"$out" && ok "110 Prometheus ready" || warn "110 Prometheus not ready"
  grep -q "AM_CODE 200" <<<"$out" && ok "110 Alertmanager healthy" || warn "110 Alertmanager not healthy"
  grep -Eq "SENTRY_CODE (200|302|400)" <<<"$out" && ok "110 Sentry HTTP reachable" || warn "110 Sentry HTTP not confirmed"
  local action_runner_active_count action_runner_enabled_count
  action_runner_active_count="$(awk '$1 == "ACTION_RUNNER_ACTIVE_COUNT" {value=$2} END {print value}' <<<"$out")"
  action_runner_enabled_count="$(awk '$1 == "ACTION_RUNNER_ENABLED_COUNT" {value=$2} END {print value}' <<<"$out")"
  if grep -q "WatchdogUSec=0" <<<"$out"; then
    ok "runner watchdog disabled on at least one unit"
  elif [[ "${action_runner_active_count:-0}" == "0" && "${action_runner_enabled_count:-0}" == "0" ]]; then
    ok "110 GitHub Actions runner units intentionally offline and disabled"
  else
    warn "runner watchdog state not confirmed"
  fi
  if awk '$1 == "RUNNER_FAILCLOSED_UNIT" && $NF != "ok=1" {bad=1} END {exit bad}' <<<"$out"; then
    ok "110 legacy direct/Gitea runner units are fail-closed"
  else
    fail "110 legacy direct/Gitea runner units are not fail-closed"
  fi
  grep -q "CD_LANE_GUARDRAILS_OK 1" <<<"$out" && ok "110 controlled cd-lane is safe, drained, or fail-closed" || fail "110 controlled cd-lane is neither safe-open/drained nor fail-closed"
  grep -q "RUNNER_DIRECT_PROCESS_COUNT 0" <<<"$out" && ok "110 legacy direct runner process count is zero" || fail "110 legacy direct runner process detected"
  grep -q "RUNNER_FAILCLOSED_BINARY_ELF" <<<"$out" && fail "110 runner fail-closed binary path restored to ELF" || ok "110 runner binary paths are fail-closed stubs or missing"
  grep -q "sentry-self-hosted-clickhouse-1.*Restarting" <<<"$out" && warn "Sentry ClickHouse restarting" || ok "Sentry ClickHouse not visibly restarting"
}

check_k3s() {
  log_section "P1-K3S"
  local out local_kubectl_out
  if ! out=$(host_cmd "wooo@192.168.0.120" '
echo "HOST $(hostname) $(uptime)"
echo "PG188_PORT $(nc -z -w 2 192.168.0.188 5432 >/dev/null 2>&1 && echo OPEN || echo CLOSED)"
echo "SYSTEMD $(systemctl is-active k3s k3s-agent keepalived 2>/dev/null | tr "\n" " ")"
kcmd() {
  if [ -n "${REMOTE_SUDO_PASSWORD:-}" ]; then
    printf "%s\n" "$REMOTE_SUDO_PASSWORD" | sudo -S -p "" kubectl "$@"
  else
    sudo -n kubectl "$@" 2>/dev/null || kubectl "$@"
  fi
}
kcmd get nodes -o wide 2>/dev/null || true
kcmd get pods -n awoooi-prod -o wide 2>/dev/null || true
node_condition_summary=$(kcmd get nodes -o json 2>/dev/null | python3 -c "import json,sys
try:
  d=json.load(sys.stdin)
except Exception:
  d={\"items\": []}
not_ready=readonly=disk_pressure=0
for node in d.get(\"items\", []):
  conds={c.get(\"type\"): c.get(\"status\") for c in node.get(\"status\",{}).get(\"conditions\",[]) or []}
  if conds.get(\"Ready\") != \"True\":
    not_ready += 1
  if conds.get(\"ReadonlyFilesystem\") == \"True\":
    readonly += 1
  if conds.get(\"DiskPressure\") == \"True\":
    disk_pressure += 1
print(f\"NODE_NOT_READY {not_ready}\")
print(f\"NODE_READONLY_FILESYSTEM_TRUE {readonly}\")
print(f\"NODE_DISK_PRESSURE_TRUE {disk_pressure}\")" || true)
printf "%s\n" "$node_condition_summary"
node_fs_events=$(kcmd get events -A --field-selector involvedObject.kind=Node --sort-by=.lastTimestamp 2>/dev/null \
  | grep -Eiv "InvalidDiskCapacity|image filesystem" \
  | grep -Eic "fsck|I/O error|read-only file system|Structure needs cleaning|orphan linked list|EXT4-fs.*error|XFS.*(corruption|metadata)|Remounting filesystem read-only" || true)
echo "NODE_FS_ERROR_EVENTS ${node_fs_events:-0}"
image_pull_blocked=$(kcmd get pods -n awoooi-prod -o json 2>/dev/null | python3 -c "import json,sys
try:
  d=json.load(sys.stdin)
except Exception:
  d={\"items\": []}
blocked=0
reasons={}
for pod in d.get(\"items\", []):
  for status in pod.get(\"status\", {}).get(\"containerStatuses\", []) or []:
    waiting=(status.get(\"state\") or {}).get(\"waiting\") or {}
    reason=waiting.get(\"reason\") or \"\"
    if reason in {\"ImagePullBackOff\", \"ErrImagePull\"}:
      blocked += 1
      reasons[reason]=reasons.get(reason, 0) + 1
print(\"IMAGE_PULL_BLOCKED\", blocked)
print(\"IMAGE_PULL_REASONS\", \",\".join(f\"{k}:{v}\" for k,v in sorted(reasons.items())) or \"none\")" || true)
printf "%s\n" "$image_pull_blocked"
registry_pull_refused_events=$(kcmd get events -n awoooi-prod --sort-by=.lastTimestamp 2>/dev/null \
  | grep -Ec "Failed to pull image|ImagePullBackOff|ErrImagePull|192\\.168\\.0\\.110:5000.*connect: connection refused" || true)
echo "REGISTRY_PULL_REFUSED_EVENTS ${registry_pull_refused_events:-0}"
ip addr show | grep 192.168.0.125 || true
' 2>&1); then
    fail "ssh 120 k3s read-only check"
    echo "$out"
    return
  fi
  echo "$out"

  if ! grep -q " Ready " <<<"$out"; then
    local_kubectl_out=$(kubectl get nodes -o wide 2>/dev/null || true)
    if [ -n "$local_kubectl_out" ]; then
      echo "LOCAL_KUBECTL_FALLBACK"
      echo "$local_kubectl_out"
    fi
  else
    local_kubectl_out=""
  fi

  grep -q "PG188_PORT OPEN" <<<"$out" && ok "120 can reach 188 PostgreSQL port" || fail "120 cannot reach 188 PostgreSQL"
  grep -q " Ready " <<<"$out$local_kubectl_out" && ok "K3s has Ready node output" || fail "K3s nodes not Ready or kubectl unavailable"
  grep -q "NODE_NOT_READY 0" <<<"$out" && ok "K3s node Ready condition clean" || fail "K3s node Ready condition not clean"
  if grep -q "NODE_FS_ERROR_EVENTS 0" <<<"$out" \
    && grep -q "NODE_READONLY_FILESYSTEM_TRUE 0" <<<"$out" \
    && grep -q "NODE_DISK_PRESSURE_TRUE 0" <<<"$out"; then
    ok "K3s node storage conditions clean"
  else
    fail "K3s node storage condition or severe filesystem event present"
  fi
  grep -q "IMAGE_PULL_BLOCKED 0" <<<"$out" && ok "K3s AWOOOI image pull is not blocked" || fail "K3s AWOOOI image pull blocked"
  grep -q "REGISTRY_PULL_REFUSED_EVENTS 0" <<<"$out" && ok "K3s registry pull has no refused events" || fail "K3s registry pull refused by 110:5000"
  grep -q "192.168.0.125" <<<"$out" && ok "VIP 192.168.0.125 present on 120" || warn "VIP not confirmed on 120"
}

check_workload_and_alertchain() {
  log_section "P2-WORKLOAD-ALERTCHAIN"
  local api_code web_code alert_code
  local out
  if out=$(host_cmd "wooo@192.168.0.120" '
api_code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 http://192.168.0.125:32334/api/v1/health 2>/dev/null || true)
web_code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 http://192.168.0.125:32335/ 2>/dev/null || true)
echo "API_CODE ${api_code:-000}"
echo "WEB_CODE ${web_code:-000}"
' 2>/dev/null); then
    api_code=$(awk '/^API_CODE / {print $2}' <<<"$out")
    web_code=$(awk '/^WEB_CODE / {print $2}' <<<"$out")
  else
    api_code=$(probe_http_code "http://192.168.0.125:32334/api/v1/health")
    web_code=$(probe_http_code "http://192.168.0.125:32335/")
    out="API_CODE $api_code
WEB_CODE $web_code"
  fi

  echo "$out"

  [[ "$api_code" =~ ^[23] ]] && ok "AWOOOI API reachable" || fail "AWOOOI API not reachable"
  [[ "$web_code" =~ ^[23] ]] && ok "AWOOOI Web reachable" || warn "AWOOOI Web not confirmed"

  if [ "$SEND_ALERT_TEST" -eq 1 ]; then
    alert_code=$(host_cmd "wooo@192.168.0.120" 'curl -s -o /tmp/awoooi-alertchain.out -w "%{http_code}" --max-time 8 \
      -X POST "http://192.168.0.125:32334/api/v1/webhooks/alertmanager" \
      -H '"'"'Content-Type: application/json'"'"' \
      -d '"'"'{"receiver":"cold-start-check","status":"firing","alerts":[{"status":"firing","labels":{"alertname":"ColdStartCheck","severity":"info"},"annotations":{"summary":"Cold start check"},"startsAt":"2026-05-05T11:00:00Z","endsAt":"0001-01-01T00:00:00Z","generatorURL":""}],"groupLabels":{},"commonLabels":{},"commonAnnotations":{},"externalURL":"","version":"4","groupKey":"cold-start-check"}'"'"' 2>/dev/null || echo "000"')
    echo "ALERTCHAIN_CODE $alert_code"
    [[ "$alert_code" =~ ^2 ]] && ok "Alertmanager webhook endpoint accepts POST" || warn "Alertmanager webhook E2E not confirmed"
  elif [ "$MONITOR_READ_ONLY" -eq 1 ]; then
    ok "Alertmanager webhook POST intentionally skipped in read-only monitor mode"
  else
    warn "Alertmanager webhook POST skipped; rerun with --send-alert-test after API is ready"
  fi
}

check_public_routes() {
  log_section "P2-PUBLIC-ROUTES"
  local item name url code tls_code
  local routes=(
    "awoooi_api|https://awoooi.wooo.work/api/v1/health"
    "awoooi_web|https://awoooi.wooo.work/"
    "momo_web|https://mo.wooo.work/"
    "momo_health|https://mo.wooo.work/health"
    "gitea|https://gitea.wooo.work/"
    "harbor|https://harbor.wooo.work/"
    "registry|https://registry.wooo.work/"
    "sentry|https://sentry.wooo.work/"
    "signoz|https://signoz.wooo.work/"
    "stock|https://stock.wooo.work/"
    "langfuse|https://langfuse.wooo.work/"
    "bitan|https://bitan.wooo.work/"
    "aiops|https://aiops.wooo.work/"
  )

  for item in "${routes[@]}"; do
    name="${item%%|*}"
    url="${item#*|}"
    code=$(probe_http_code "$url")
    echo "PUBLIC_ROUTE $name $code $url"
    [[ "$code" =~ ^[23] ]] && ok "public route $name reachable" || warn "public route $name not confirmed"
    tls_code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 8 "$url" 2>/dev/null || true)
    tls_code="${tls_code:-000}"
    echo "PUBLIC_ROUTE_TLS $name $tls_code $url"
    [[ "$tls_code" =~ ^[23] ]] && ok "public route $name TLS certificate verified" || fail "public route $name TLS certificate verification failed"
  done
}

check_schedules() {
  log_section "P2-SCHEDULES"
  local out

  if out=$(host_cmd "ollama@192.168.0.188" '
now=$(date +%s)
echo "CRON_188 $(systemctl is-active cron 2>/dev/null || systemctl is-active crond 2>/dev/null || true)"
for f in /home/ollama/node_exporter_textfiles/backup.prom /home/ollama/node_exporter_textfiles/backup_health.prom /home/ollama/node_exporter_textfiles/docker_restart_count.prom /home/ollama/node_exporter_textfiles/docker_stats.prom /home/ollama/node_exporter_textfiles/storage_health.prom; do
  if [ -f "$f" ]; then
    mt=$(stat -c %Y "$f")
    echo "TEXTFILE_188 $(basename "$f") age=$((now - mt))"
  else
    echo "TEXTFILE_188 $(basename "$f") missing"
  fi
done
if [ -f /home/ollama/node_exporter_textfiles/backup.prom ]; then
  awk -v now="$now" "/^backup_110_last_success_timestamp / {printf \"BACKUP_110_AGE %d\\n\", now - int(\$2)}" /home/ollama/node_exporter_textfiles/backup.prom
fi
if [ -f /home/ollama/node_exporter_textfiles/backup_health.prom ]; then
  awk "/^awoooi_backup_job_fresh/ {total++; if (int(\$2) == 0) stale++} /^awoooi_backup_job_configured/ {if (int(\$2) == 0) missing_cron++} /^awoooi_backup_script_present/ {if (int(\$2) == 0) missing_script++} END {printf \"BACKUP_HEALTH_188 total=%d stale=%d missing_cron=%d missing_script=%d\\n\", total+0, stale+0, missing_cron+0, missing_script+0}" /home/ollama/node_exporter_textfiles/backup_health.prom
fi
if [ -f /home/ollama/node_exporter_textfiles/storage_health.prom ]; then
  awk "/^awoooi_host_storage_root_readonly/ {readonly=int(\$2)} /^awoooi_host_storage_current_boot_error_count/ {current=int(\$2)} END {printf \"STORAGE_HEALTH_188 root_readonly=%d current=%d\\n\", readonly+0, current+0}" /home/ollama/node_exporter_textfiles/storage_health.prom
fi
echo "SCHEDULER_CONTAINER_RUNNING $(docker inspect -f "{{.State.Running}}" momo-scheduler 2>/dev/null || true)"
echo "SCHEDULER_CONTAINER_HEALTH $(docker inspect -f "{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}" momo-scheduler 2>/dev/null || true)"
echo "SCHEDULER_REGISTERED $(docker logs --tail 400 momo-scheduler 2>&1 | grep -Ec "全部排程任務已註冊|排程任務已註冊|Scheduler started|APScheduler" || true)"
echo "SCHEDULER_RECENT_ACTIVITY $(docker logs --since 2h momo-scheduler 2>&1 | grep -Ec "AutoImport|Meta-Analysis|Scheduler|排程|任務|批次 [0-9]+: 取得|\\[Feeder\\]|HITL|候選屬" || true)"
echo "MOMO_SOURCE_EMPTY_EVIDENCE_LINES $(docker logs --since 6h momo-scheduler 2>&1 | grep -Ec "找到 0 個 Excel|沒有找到待匯入" || true)"
token_stat=$(stat -c "%u:%g:%a" /home/ollama/momo-pro/config/google_token.json 2>/dev/null || true)
scheduler_uid=$(docker top momo-scheduler -eo pid,user,uid 2>/dev/null | awk "NR==2 {print \$3}" || true)
echo "MOMO_GDRIVE_TOKEN_STAT ${token_stat:-missing} scheduler_uid=${scheduler_uid:-unknown}"
db_user=$(docker exec momo-pro-system printenv POSTGRES_USER 2>/dev/null || true)
db_name=$(docker exec momo-pro-system printenv POSTGRES_DB 2>/dev/null || true)
if [ -n "$db_user" ] && [ -n "$db_name" ]; then
  psql_no_secret() {
    docker exec -e PGCONNECT_TIMEOUT=5 momo-db psql -h 127.0.0.1 -U "$db_user" -d "$db_name" --no-password -Atc "$1" 2>/dev/null || true
  }
  momo_sync=$(psql_no_secret "WITH scope AS (SELECT min(snapshot_date::date) dmin, max(snapshot_date::date) dmax, count(*) sc FROM daily_sales_snapshot WHERE snapshot_date::date >= make_date(extract(year from current_date)::int, extract(month from current_date)::int, 1)), monthly AS (SELECT count(*) mc, min(\"日期\"::date) mmin, max(\"日期\"::date) mmax FROM realtime_sales_monthly, scope WHERE scope.sc > 0 AND \"日期\"::date BETWEEN scope.dmin AND scope.dmax) SELECT coalesce(scope.sc,0)::text || chr(124) || coalesce(monthly.mc,0)::text || chr(124) || coalesce(scope.dmin::text,chr(45)) || chr(124) || coalesce(scope.dmax::text,chr(45)) || chr(124) || coalesce(monthly.mmin::text,chr(45)) || chr(124) || coalesce(monthly.mmax::text,chr(45)) FROM scope, monthly;")
  momo_freshness=$(psql_no_secret "SELECT coalesce((current_date - max(snapshot_date::date))::text, chr(45)) || chr(124) || coalesce(max(snapshot_date::date)::text, chr(45)) FROM daily_sales_snapshot;")
  momo_import_config=$(psql_no_secret "SELECT config_key || chr(61) || config_value FROM import_config;" | awk -F= "\$1 == \"gdrive_folder_path\" {folder=\$2} \$1 == \"gdrive_file_pattern\" {pattern=\$2} END {if (folder || pattern) print folder \"|\" pattern}" || true)
  momo_latest_import_job=$(psql_no_secret "SELECT coalesce(id::text, chr(45)) || chr(124) || coalesce(job_type, chr(45)) || chr(124) || coalesce(status, chr(45)) || chr(124) || coalesce(drive_file_name, chr(45)) || chr(124) || coalesce(replace(created_at::text, chr(32), chr(84)), chr(45)) || chr(124) || coalesce(replace(completed_at::text, chr(32), chr(84)), chr(45)) || chr(124) || coalesce(total_rows::text, chr(45)) || chr(124) || coalesce(success_rows::text, chr(45)) || chr(124) || coalesce(error_rows::text, chr(45)) FROM import_jobs ORDER BY created_at DESC LIMIT 20;" | awk "BEGIN {FS=sprintf(\"%c\",124)} \$2 == \"daily_sales\" {print \$1 \"|\" \$3 \"|\" \$4 \"|\" \$5 \"|\" \$6 \"|\" \$7 \"|\" \$8 \"|\" \$9; exit}" || true)
  tmp_drive_probe="/tmp/awoooi-momo-drive-source-probe.$$"
  cat > "$tmp_drive_probe" <<PYDRIVE
from services.google_drive_service import drive_service
from services.import_service import import_service

def emit(key, value):
    print(f"{key} {value if value not in (None, '') else '-'}")

folder = import_service.get_config("gdrive_folder_path", "當日業績匯入")
pattern = import_service.get_config("gdrive_file_pattern", "即時業績_當日")
archive = import_service.get_config("gdrive_archive_folder", "當日業績匯入/已匯入")
failed = import_service.get_config("gdrive_failed_folder", "匯入失敗")
intake_files = drive_service.list_files_in_folder(folder, pattern)
archive_files = drive_service.list_files_in_folder(archive, pattern)
failed_files = drive_service.list_files_in_folder(failed, pattern)
emit("MOMO_DRIVE_INTAKE_COUNT", len(intake_files))
emit("MOMO_DRIVE_ARCHIVE_COUNT", len(archive_files))
emit("MOMO_DRIVE_FAILED_COUNT", len(failed_files))
emit("MOMO_DRIVE_ARCHIVE_LATEST_DATE", (archive_files[0].get("modifiedTime", "")[:10] if archive_files else "-"))
if not drive_service.service:
    drive_service.authenticate()
global_date = "-"
if drive_service.service:
    safe_pattern = drive_service._escape_query_value(pattern)
    query = (
        f"name contains '{safe_pattern}' and trashed=false and "
        "(mimeType='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' "
        "or mimeType='application/vnd.ms-excel')"
    )
    results = drive_service.service.files().list(
        q=query,
        spaces="drive",
        fields="files(modifiedTime)",
        orderBy="modifiedTime desc",
        pageSize=1,
    ).execute()
    global_files = results.get("files", [])
    if global_files:
        global_date = global_files[0].get("modifiedTime", "")[:10] or "-"
emit("MOMO_DRIVE_GLOBAL_LATEST_DATE", global_date)
PYDRIVE
  momo_drive_source_probe=$(docker exec -i momo-scheduler python - < "$tmp_drive_probe" 2>/dev/null | awk "/^MOMO_DRIVE_/ {print}" || true)
  rm -f "$tmp_drive_probe"
else
  momo_sync=""
  momo_freshness=""
  momo_import_config=""
  momo_latest_import_job=""
  momo_drive_source_probe=""
fi
echo "MOMO_MONTHLY_SYNC ${momo_sync:-unavailable}"
echo "MOMO_DAILY_FRESHNESS ${momo_freshness:-unavailable}"
echo "MOMO_IMPORT_CONFIG ${momo_import_config:-unavailable}"
echo "MOMO_LATEST_IMPORT_JOB ${momo_latest_import_job:-unavailable}"
printf "%s\n" "$momo_drive_source_probe"
' 2>&1); then
    echo "$out"
    grep -q "CRON_188 active" <<<"$out" && ok "188 cron active" || warn "188 cron not confirmed"
    awk '/TEXTFILE_188 backup.prom age=/ {split($3,a,"="); exit !(a[2] < 90000)}' <<<"$out" && ok "188 backup textfile fresh enough" || warn "188 backup textfile stale or missing"
    awk '/TEXTFILE_188 backup_health.prom age=/ {split($3,a,"="); exit !(a[2] < 900)}' <<<"$out" && ok "188 backup health exporter fresh" || warn "188 backup health exporter stale"
    awk '/TEXTFILE_188 docker_restart_count.prom age=/ {split($3,a,"="); exit !(a[2] < 300)}' <<<"$out" && ok "188 docker restart exporter fresh" || warn "188 docker restart exporter stale"
    awk '/TEXTFILE_188 docker_stats.prom age=/ {split($3,a,"="); exit !(a[2] < 300)}' <<<"$out" && ok "188 docker stats exporter fresh" || warn "188 docker stats exporter stale"
    awk '/TEXTFILE_188 storage_health.prom age=/ {split($3,a,"="); exit !(a[2] < 300)}' <<<"$out" && ok "188 storage health exporter fresh" || warn "188 storage health exporter stale"
    grep -q "STORAGE_HEALTH_188 root_readonly=0 current=0" <<<"$out" && ok "188 current boot storage health clean" || warn "188 storage health not clean"
    awk '/BACKUP_110_AGE / {exit !($2 < 90000)}' <<<"$out" && ok "188 backup-from-110 success within 25h" || warn "188 backup-from-110 success not confirmed"
    grep -q "BACKUP_HEALTH_188 total=" <<<"$out" && awk '/BACKUP_HEALTH_188/ {split($3,a,"="); split($4,b,"="); split($5,c,"="); exit !((a[2]+b[2]+c[2]) == 0)}' <<<"$out" && ok "188 backup health has no stale expected jobs" || warn "188 backup health has stale expected jobs"
    if grep -q "SCHEDULER_CONTAINER_HEALTH healthy" <<<"$out" && awk '/SCHEDULER_RECENT_ACTIVITY / {exit !($2 > 0)}' <<<"$out"; then
      ok "188 momo scheduler healthy with recent task activity"
    elif awk '/SCHEDULER_REGISTERED / {exit !($2 > 0)}' <<<"$out"; then
      ok "188 momo scheduler registered jobs"
    else
      warn "188 momo scheduler registration/activity not confirmed"
    fi
    awk '/MOMO_GDRIVE_TOKEN_STAT / {split($2,a,":"); split($3,b,"="); exit !(a[1] == b[2] && a[3] <= 600)}' <<<"$out" && ok "188 momo Google Drive token ownership matches scheduler userns" || warn "188 momo Google Drive token ownership/writeback not confirmed"
    grep -Fq "MOMO_IMPORT_CONFIG 當日業績匯入|即時業績_當日" <<<"$out" && ok "188 momo Drive import config points to expected daily-sales intake" || fail "188 momo Drive import config drifted from expected daily-sales intake"
    awk '/MOMO_LATEST_IMPORT_JOB / {split($2,a,"|"); exit !(a[1] ~ /^[0-9]+$/ && a[2] == "completed" && a[6] == a[7] && a[8] == 0)}' <<<"$out" && ok "188 momo latest daily import job completed cleanly" || warn "188 momo latest daily import job not confirmed clean"
    momo_latest_import_clean=$(awk '
      $1 == "MOMO_LATEST_IMPORT_JOB" {
        seen=1
        split($2,a,"|")
        if (a[1] ~ /^[0-9]+$/ && a[2] == "completed" && a[6] == a[7] && a[8] == 0) print 1;
        else print 0;
      }
      END { if (!seen) print 0 }
    ' <<<"$out")
    momo_source_stale_only=$(awk '
      $1 == "MOMO_DRIVE_INTAKE_COUNT" {intake=$2+0}
      $1 == "MOMO_DRIVE_FAILED_COUNT" {failed=$2+0}
      $1 == "MOMO_DRIVE_GLOBAL_LATEST_DATE" {global=$2}
      $1 == "MOMO_LATEST_IMPORT_JOB" {split($2,a,"|"); completed=substr(a[5],1,10)}
      END {
        if (intake == 0 && failed == 0 && global ~ /^[0-9]{4}-[0-9]{2}-[0-9]{2}$/ && completed ~ /^[0-9]{4}-[0-9]{2}-[0-9]{2}$/ && global <= completed) print 1;
        else print 0;
      }' <<<"$out")
    momo_source_preflight_output=""
    momo_source_preflight_summary=""
    if [ "$momo_source_stale_only" != "1" ] && [ "$momo_latest_import_clean" = "1" ] && [ -x "$MOMO_SOURCE_PREFLIGHT_SCRIPT" ]; then
      momo_source_preflight_output="$(
        "$MOMO_SOURCE_PREFLIGHT_SCRIPT" \
          --host ollama@192.168.0.188 \
          --freshness-max-days 2 2>/dev/null || true
      )"
      momo_source_preflight_summary="$(awk '/^MOMO_DRIVE_TOKEN_SOURCE_PREFLIGHT / {line=$0} END {print line}' <<<"$momo_source_preflight_output")"
      [ -n "$momo_source_preflight_summary" ] && echo "$momo_source_preflight_summary"
      if awk '$1 == "MOMO_SOURCE_ABSENT_WITHOUT_NEWER_DRIVE" {seen=1; ok=($2 == 1)} END {exit !(seen && ok)}' <<<"$momo_source_preflight_output"; then
        momo_source_stale_only=1
      fi
    fi
    if awk '/MOMO_MONTHLY_SYNC / {seen=1; split($2,a,"|"); ok=(a[1] > 0 && a[1] == a[2] && a[3] == a[5] && a[4] == a[6])} END {exit !(seen && ok)}' <<<"$out"; then
      ok "188 momo current-month snapshot and realtime tables match"
    elif [ "$momo_latest_import_clean" = "1" ] && [ "$momo_source_stale_only" = "1" ] && awk '/MOMO_MONTHLY_SYNC / {seen=1; split($2,a,"|"); ok=(a[1] == 0 && a[2] == 0)} END {exit !(seen && ok)}' <<<"$out"; then
      ok "188 momo current-month sync not applicable; Drive has no newer source than last clean import"
    else
      warn "188 momo current-month snapshot/realtime sync not confirmed"
    fi
    if awk '/MOMO_DAILY_FRESHNESS / {split($2,a,"|"); exit !(a[1] ~ /^[0-9]+$/ && a[1] >= 0 && a[1] <= 2)}' <<<"$out"; then
      ok "188 momo daily sales data fresh enough"
    elif awk '/MOMO_DAILY_FRESHNESS / {split($2,a,"|"); exit !(a[1] ~ /^[0-9]+$/ && a[1] >= 3)}' <<<"$out"; then
      if [ "$momo_source_stale_only" = "1" ]; then
        ok "188 momo daily sales source gate has no newer Drive candidate"
        echo "INFO 188 momo daily sales data remains stale; product data freshness is pending source arrival"
      elif [ -x "$MOMO_SOURCE_PREFLIGHT_SCRIPT" ]; then
        if [ -z "$momo_source_preflight_summary" ]; then
          momo_source_preflight_output="$(
            "$MOMO_SOURCE_PREFLIGHT_SCRIPT" \
              --host ollama@192.168.0.188 \
              --freshness-max-days 2 2>/dev/null || true
          )"
          momo_source_preflight_summary="$(awk '/^MOMO_DRIVE_TOKEN_SOURCE_PREFLIGHT / {line=$0} END {print line}' <<<"$momo_source_preflight_output")"
          [ -n "$momo_source_preflight_summary" ] && echo "$momo_source_preflight_summary"
        fi
        if grep -q "BLOCKED=0" <<<"$momo_source_preflight_summary"; then
          warn "188 momo daily sales stale but source preflight has no hard blocker"
        elif awk '/MOMO_SOURCE_EMPTY_EVIDENCE_LINES / {exit !($2 > 0)}' <<<"$out"; then
          fail "188 momo source file absent while daily sales data stale"
        else
          fail "188 momo daily sales data stale beyond 3 days"
        fi
      elif awk '/MOMO_SOURCE_EMPTY_EVIDENCE_LINES / {exit !($2 > 0)}' <<<"$out"; then
        fail "188 momo source file absent while daily sales data stale"
      else
        fail "188 momo daily sales data stale beyond 3 days"
      fi
    else
      warn "188 momo daily sales freshness not confirmed"
    fi
  else
    warn "188 schedule check unavailable"
    echo "$out"
  fi

  if out=$(host_cmd_retry "wooo@192.168.0.110" '
now=$(date +%s)
echo "CRON_110 $(systemctl is-active cron 2>/dev/null || systemctl is-active crond 2>/dev/null || true)"
echo "FAILED_UNITS_110 $(systemctl --failed --no-legend --plain 2>/dev/null | wc -l)"
echo "MOMO_STARTUP_ENABLED $(systemctl is-enabled momo-startup-complete.service 2>/dev/null || true)"
echo "STAGGERED_STARTUP_ENABLED $(systemctl is-enabled wooo-staggered-startup.service 2>/dev/null || true)"
for f in /home/wooo/node_exporter_textfiles/docker_stats.prom /home/wooo/node_exporter_textfiles/systemd_units.prom /home/wooo/node_exporter_textfiles/storage_health.prom /home/wooo/node_exporter_textfiles/backup_health.prom; do
  if [ -f "$f" ]; then
    mt=$(stat -c %Y "$f")
    echo "TEXTFILE_110 $(basename "$f") age=$((now - mt))"
  else
    echo "TEXTFILE_110 $(basename "$f") missing"
  fi
done
if [ -f /home/wooo/node_exporter_textfiles/storage_health.prom ]; then
  awk "/^awoooi_host_storage_root_readonly/ {readonly=int(\$2)} /^awoooi_host_storage_current_boot_error_count/ {current=int(\$2)} END {printf \"STORAGE_HEALTH_110 root_readonly=%d current=%d\\n\", readonly+0, current+0}" /home/wooo/node_exporter_textfiles/storage_health.prom
fi
if [ -f /home/wooo/node_exporter_textfiles/backup_health.prom ]; then
  awk "/^awoooi_backup_job_fresh/ {total++; if (int(\$2) == 0) stale++} /^awoooi_backup_job_configured/ {if (int(\$2) == 0) missing_cron++} /^awoooi_backup_script_present/ {if (int(\$2) == 0) missing_script++} /^awoooi_backup_last_run_failed_count/ {if (\$0 ~ /(exported_job|job)=\"backup_all\"/) failed=int(\$2)} /^awoooi_backup_config_capture_critical_failed_count/ {config_failed=int(\$2)} /^awoooi_backup_integrity_fresh/ {integrity_total++; if (int(\$2) == 0) integrity_stale++} END {printf \"BACKUP_HEALTH_110 total=%d stale=%d missing_cron=%d missing_script=%d failed_count=%d config_failed=%d integrity_total=%d integrity_stale=%d\\n\", total+0, stale+0, missing_cron+0, missing_script+0, failed+0, config_failed+0, integrity_total+0, integrity_stale+0}" /home/wooo/node_exporter_textfiles/backup_health.prom
fi
' "$SSH_110_CHECK_ATTEMPTS" "$SSH_110_CHECK_SLEEP_SECONDS" 2>&1); then
    echo "$out"
    grep -q "CRON_110 active" <<<"$out" && ok "110 cron active" || warn "110 cron not confirmed"
    grep -q "FAILED_UNITS_110 0" <<<"$out" && ok "110 systemd has no failed units" || warn "110 systemd failed units remain"
    grep -q "MOMO_STARTUP_ENABLED disabled" <<<"$out" && ok "110 stale momo startup unit disabled" || warn "110 stale momo startup unit not disabled"
    grep -q "STAGGERED_STARTUP_ENABLED disabled" <<<"$out" && ok "110 stale staggered startup unit disabled" || warn "110 stale staggered startup unit not disabled"
    awk '/TEXTFILE_110 docker_stats.prom age=/ {split($3,a,"="); exit !(a[2] < 300)}' <<<"$out" && ok "110 docker stats exporter fresh" || warn "110 docker stats exporter stale"
    awk '/TEXTFILE_110 systemd_units.prom age=/ {split($3,a,"="); exit !(a[2] < 300)}' <<<"$out" && ok "110 systemd units exporter fresh" || warn "110 systemd units exporter stale"
    awk '/TEXTFILE_110 storage_health.prom age=/ {split($3,a,"="); exit !(a[2] < 300)}' <<<"$out" && ok "110 storage health exporter fresh" || warn "110 storage health exporter stale"
    awk '/TEXTFILE_110 backup_health.prom age=/ {split($3,a,"="); exit !(a[2] < 900)}' <<<"$out" && ok "110 backup health exporter fresh" || warn "110 backup health exporter stale"
    grep -q "STORAGE_HEALTH_110 root_readonly=0 current=0" <<<"$out" && ok "110 current boot storage health clean" || warn "110 storage health not clean"
    if grep -q "BACKUP_HEALTH_110 total=" <<<"$out" && awk '/BACKUP_HEALTH_110/ {split($3,a,"="); split($4,b,"="); split($5,c,"="); split($7,e,"="); exit !((a[2]+b[2]+c[2]+e[2]) == 0)}' <<<"$out"; then
      ok "110 backup health has no stale expected jobs or critical config gaps"
      awk '/BACKUP_HEALTH_110/ {split($6,a,"="); exit !(a[2] > 0)}' <<<"$out" && echo "INFO 110 latest aggregate backup-all still records failed_count but current component freshness is clean"
    else
      warn "110 backup health has stale expected jobs or critical config gaps"
    fi
    awk '/BACKUP_HEALTH_110/ {split($9,a,"="); exit !(a[2] == 0)}' <<<"$out" && ok "110 backup integrity and restore drill fresh" || warn "110 backup integrity or restore drill stale"
  else
    warn "110 schedule check unavailable"
    echo "$out"
  fi

  if out=$(host_cmd "wooo@192.168.0.120" '
kcmd() {
  if [ -n "${REMOTE_SUDO_PASSWORD:-}" ]; then
    printf "%s\n" "$REMOTE_SUDO_PASSWORD" | sudo -S -p "" kubectl "$@"
  else
    sudo -n kubectl "$@" 2>/dev/null || kubectl "$@"
  fi
}
echo "CRON_120 $(systemctl is-active cron 2>/dev/null || systemctl is-active crond 2>/dev/null || true)"
kcmd get cronjobs -n awoooi-prod -o json | python3 -c "import json,sys; d=json.load(sys.stdin); items=d.get(\"items\", []); print(\"CRONJOB_COUNT\", len(items)); print(\"CRONJOB_SUSPENDED\", sum(1 for i in items if i.get(\"spec\",{}).get(\"suspend\")))"
kcmd get jobs -n awoooi-prod -o json | python3 -c "import json,sys,re; d=json.load(sys.stdin)
def owner(job):
  for ref in job.get(\"metadata\",{}).get(\"ownerReferences\",[]) or []:
    if ref.get(\"kind\") == \"CronJob\" and ref.get(\"name\"):
      return ref.get(\"name\")
  name = job.get(\"metadata\",{}).get(\"name\", \"\")
  return re.sub(r\"-[0-9]+$\", \"\", name)
def has_condition(job, kind):
  return any(c.get(\"type\") == kind and c.get(\"status\") == \"True\" for c in job.get(\"status\",{}).get(\"conditions\",[]) or [])
def job_time(job):
  status = job.get(\"status\",{})
  return status.get(\"completionTime\") or status.get(\"startTime\") or \"\"
latest_success = {}
failed_jobs = []
for job in d.get(\"items\", []):
  own = owner(job)
  ts = job_time(job)
  if has_condition(job, \"Complete\"):
    latest_success[own] = max(latest_success.get(own, \"\"), ts)
  if has_condition(job, \"Failed\"):
    failed_jobs.append((own, job.get(\"metadata\",{}).get(\"name\", \"\"), ts))
active_failed = 0
stale_failed = 0
for own, name, ts in failed_jobs:
  if ts and latest_success.get(own, \"\") > ts:
    stale_failed += 1
  else:
    active_failed += 1
print(\"FAILED_JOBS\", len(failed_jobs))
print(\"STALE_FAILED_JOBS\", stale_failed)
print(\"ACTIVE_FAILED_JOBS\", active_failed)"
kcmd get pods -n awoooi-prod --no-headers 2>/dev/null | awk "\$3 !~ /^(Running|Completed)$/ {bad++} END {print \"BAD_PODS\", bad+0}"
' 2>&1); then
    echo "$out"
    grep -q "CRON_120 active" <<<"$out" && ok "120 cron active" || warn "120 cron not confirmed"
    awk '/CRONJOB_COUNT / {exit !($2 >= 4)}' <<<"$out" && ok "K8s AWOOOI CronJobs present" || warn "K8s AWOOOI CronJobs missing"
    grep -q "CRONJOB_SUSPENDED 0" <<<"$out" && ok "K8s AWOOOI CronJobs unsuspended" || warn "K8s AWOOOI CronJob suspended"
    grep -q "ACTIVE_FAILED_JOBS 0" <<<"$out" && ok "K8s AWOOOI has no active failed Jobs" || warn "K8s AWOOOI active failed Jobs remain"
    grep -q "BAD_PODS 0" <<<"$out" && ok "K8s AWOOOI pods Running/Completed only" || warn "K8s AWOOOI bad pod status remains"
  else
    warn "120 K8s schedule check unavailable"
    echo "$out"
  fi

  if out=$(host_cmd "wooo@192.168.0.121" '
echo "CRON_121 $(systemctl is-active cron 2>/dev/null || systemctl is-active crond 2>/dev/null || true)"
crontab -l 2>/dev/null | grep -q "dr-drill.sh" && echo "DR_DRILL_CRON present" || echo "DR_DRILL_CRON missing"
' 2>&1); then
    echo "$out"
    grep -q "CRON_121 active" <<<"$out" && ok "121 cron active" || warn "121 cron not confirmed"
    grep -q "DR_DRILL_CRON present" <<<"$out" && ok "121 DR drill cron present" || warn "121 DR drill cron missing"
  else
    warn "121 schedule check unavailable"
    echo "$out"
  fi
}

summary() {
  log_section "SUMMARY"
  echo "PASS=$PASS WARN=$WARN BLOCKED=$FAIL"
  if [ "$FAIL" -gt 0 ]; then
    echo "Result: BLOCKED. Fix the first blocked gate before releasing runner/CD/AI auto-remediation."
    exit 2
  fi
  if [ "$WARN" -gt 0 ]; then
    echo "Result: DEGRADED. Core gates passed but warnings remain."
    exit 1
  fi
  echo "Result: GREEN. Full stack is ready for controlled runner/CD release."
}

if [ "$WATCH_MODE" -eq 1 ]; then
  attempt=1
  rc=2
  while true; do
    echo "WATCH_ATTEMPT=$attempt"
    args=()
    [ "$MONITOR_READ_ONLY" -eq 1 ] && args+=(--monitor-read-only)
    [ "$NO_COLOR_FLAG" -eq 1 ] && args+=(--no-color)
    [ "$SEND_ALERT_TEST" -eq 1 ] && args+=(--send-alert-test)
    bash "$0" "${args[@]}"
    rc=$?
    [ "$rc" -eq 0 ] && exit 0
    if [ "$WATCH_MAX_ATTEMPTS" -gt 0 ] && [ "$attempt" -ge "$WATCH_MAX_ATTEMPTS" ]; then
      exit "$rc"
    fi
    attempt=$((attempt + 1))
    sleep "$WATCH_INTERVAL"
  done
fi

print_header
check_network
check_188
check_110
check_k3s
check_workload_and_alertchain
check_public_routes
check_schedules
summary