docs(ops): codify full stack cold start recovery

scripts/reboot-recovery/full-stack-cold-start-check.sh

@@ -6,26 +6,61 @@ set -uo pipefail
 
 SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=6)
 SEND_ALERT_TEST=0
+WATCH_MODE=0
+WATCH_INTERVAL=60
+WATCH_MAX_ATTEMPTS=30
 
-for arg in "$@"; do
-  case "$arg" in
+usage() {
+  cat <<'USAGE'
+Usage: bash scripts/reboot-recovery/full-stack-cold-start-check.sh [options]
+
+Options:
+  --send-alert-test       POST one Alertmanager webhook test after AWOOOI API is ready.
+  --watch                 Repeat checks until all gates are GREEN or max attempts is reached.
+  --interval SECONDS      Retry interval for --watch. Default: 60.
+  --max-attempts COUNT    Max attempts for --watch. Default: 30. Use 0 for unlimited.
+  -h, --help              Show this help.
+
+Default mode is read-only and does not POST an Alertmanager test event.
+Use --send-alert-test for the final release gate after AWOOOI API is expected to be ready.
+USAGE
+}
+
+while [ "$#" -gt 0 ]; do
+  case "$1" in
     --send-alert-test)
       SEND_ALERT_TEST=1
       ;;
+    --watch)
+      WATCH_MODE=1
+      ;;
+    --interval)
+      shift
+      if ! [[ "${1:-}" =~ ^[0-9]+$ ]] || [ "${1:-0}" -lt 1 ]; then
+        echo "--interval requires a positive integer number of seconds" >&2
+        exit 64
+      fi
+      WATCH_INTERVAL="$1"
+      ;;
+    --max-attempts)
+      shift
+      if ! [[ "${1:-}" =~ ^[0-9]+$ ]]; then
+        echo "--max-attempts requires a non-negative integer" >&2
+        exit 64
+      fi
+      WATCH_MAX_ATTEMPTS="$1"
+      ;;
     -h|--help)
-      cat <<'USAGE'
-Usage: bash scripts/reboot-recovery/full-stack-cold-start-check.sh [--send-alert-test]
-
-Default mode is read-only and does not POST an Alertmanager test event.
-Use --send-alert-test only after AWOOOI API is expected to be ready.
-USAGE
+      usage
       exit 0
       ;;
     *)
-      echo "Unknown argument: $arg" >&2
+      echo "Unknown argument: $1" >&2
+      usage >&2
       exit 64
       ;;
   esac
+  shift
 done
 
 RED=$'\033[0;31m'
@@ -38,6 +73,12 @@ PASS=0
 WARN=0
 FAIL=0
 
+reset_counters() {
+  PASS=0
+  WARN=0
+  FAIL=0
+}
+
 log_section() {
   printf "\n%s=== %s ===%s\n" "$BLUE" "$1" "$NC"
 }
@@ -104,6 +145,7 @@ print_header() {
   echo "AWOOOI full-stack cold-start check"
   date '+%Y-%m-%d %H:%M:%S %Z'
   echo "Scope: 110 / 120 / 121 / 188. 112 Kali is intentionally skipped."
+  echo "Baseline: ops/reboot-recovery/full-stack-cold-start-baseline.yml"
 }
 
 check_network() {
@@ -385,21 +427,54 @@ summary() {
   echo "PASS=$PASS WARN=$WARN BLOCKED=$FAIL"
   if [ "$FAIL" -gt 0 ]; then
     echo "Result: BLOCKED. Fix the first blocked gate before releasing runner/CD/AI auto-remediation."
-    exit 2
+    return 2
   fi
   if [ "$WARN" -gt 0 ]; then
     echo "Result: DEGRADED. Core gates passed but warnings remain."
-    exit 1
+    return 1
   fi
   echo "Result: GREEN. Full stack is ready for controlled runner/CD release."
+  return 0
 }
 
-print_header
-check_network
-check_188
-check_110
-check_k3s
-check_workload_and_alertchain
-check_public_routes
-check_schedules
-summary
+run_once() {
+  reset_counters
+  print_header
+  check_network
+  check_188
+  check_110
+  check_k3s
+  check_workload_and_alertchain
+  check_public_routes
+  check_schedules
+  summary
+}
+
+if [ "$WATCH_MODE" -eq 1 ]; then
+  attempt=1
+  while :; do
+    if [ "$WATCH_MAX_ATTEMPTS" -eq 0 ]; then
+      printf "\nWatch attempt %s/unlimited\n" "$attempt"
+    else
+      printf "\nWatch attempt %s/%s\n" "$attempt" "$WATCH_MAX_ATTEMPTS"
+    fi
+
+    run_once
+    rc=$?
+    if [ "$rc" -eq 0 ]; then
+      exit 0
+    fi
+
+    if [ "$WATCH_MAX_ATTEMPTS" -ne 0 ] && [ "$attempt" -ge "$WATCH_MAX_ATTEMPTS" ]; then
+      echo "Watch stopped before GREEN. Last result code: $rc"
+      exit "$rc"
+    fi
+
+    echo "Waiting ${WATCH_INTERVAL}s before the next cold-start gate check..."
+    sleep "$WATCH_INTERVAL"
+    attempt=$((attempt + 1))
+  done
+fi
+
+run_once
+exit $?