fix(reboot): separate windows99 verifier timeout

scripts/reboot-recovery/collect-windows99-vmware-verify.sh

@@ -6,6 +6,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 TARGET_HOST="${WINDOWS99_HOST:-192.168.0.99}"
 CONNECT_TIMEOUT="${WINDOWS99_CONNECT_TIMEOUT:-3}"
 SSH_TIMEOUT="${WINDOWS99_SSH_TIMEOUT:-3}"
+REMOTE_VERIFY_TIMEOUT="${WINDOWS99_REMOTE_VERIFY_TIMEOUT:-45}"
 SSH_PORT="${WINDOWS99_SSH_PORT:-22}"
 MAX_AUTH_USERS="${WINDOWS99_MAX_AUTH_USERS:-2}"
 MAX_AUTH_USERS_EXPLICIT=0
@@ -34,6 +35,9 @@ fi
 if ! is_positive_int "${SSH_TIMEOUT}"; then
   SSH_TIMEOUT=3
 fi
+if ! is_positive_int "${REMOTE_VERIFY_TIMEOUT}"; then
+  REMOTE_VERIFY_TIMEOUT=45
+fi
 if ! is_positive_int "${MAX_AUTH_USERS}"; then
   MAX_AUTH_USERS=2
 fi
@@ -84,6 +88,9 @@ fi
 if ! is_positive_int "${SSH_TIMEOUT}"; then
   SSH_TIMEOUT=3
 fi
+if ! is_positive_int "${REMOTE_VERIFY_TIMEOUT}"; then
+  REMOTE_VERIFY_TIMEOUT=45
+fi
 if ! is_positive_int "${MAX_AUTH_USERS}"; then
   MAX_AUTH_USERS=2
 fi
@@ -177,16 +184,28 @@ SSH_OPTS=(
   -p "${SSH_PORT}"
 )
 
+run_ssh_timed() {
+  local user="$1"
+  local timeout_seconds="$2"
+  local timeout_wrapper_seconds="$((timeout_seconds + 1))"
+  shift 2
+  if ! is_positive_int "${timeout_seconds}"; then
+    timeout_seconds="${SSH_TIMEOUT}"
+    timeout_wrapper_seconds="$((SSH_TIMEOUT + 1))"
+  fi
+  if [[ "${SSH_TIMEOUT_WRAPPER}" == "timeout" ]]; then
+    timeout "${timeout_wrapper_seconds}s" ssh "${SSH_OPTS[@]}" -o ConnectTimeout="${timeout_seconds}" "${user}@${TARGET_HOST}" "$@"
+  elif [[ "${SSH_TIMEOUT_WRAPPER}" == "gtimeout" ]]; then
+    gtimeout "${timeout_wrapper_seconds}s" ssh "${SSH_OPTS[@]}" -o ConnectTimeout="${timeout_seconds}" "${user}@${TARGET_HOST}" "$@"
+  else
+    ssh "${SSH_OPTS[@]}" -o ConnectTimeout="${timeout_seconds}" "${user}@${TARGET_HOST}" "$@"
+  fi
+}
+
 run_ssh() {
   local user="$1"
   shift
-  if [[ "${SSH_TIMEOUT_WRAPPER}" == "timeout" ]]; then
-    timeout "$((SSH_TIMEOUT + 1))s" ssh "${SSH_OPTS[@]}" "${user}@${TARGET_HOST}" "$@"
-  elif [[ "${SSH_TIMEOUT_WRAPPER}" == "gtimeout" ]]; then
-    gtimeout "$((SSH_TIMEOUT + 1))s" ssh "${SSH_OPTS[@]}" "${user}@${TARGET_HOST}" "$@"
-  else
-    ssh "${SSH_OPTS[@]}" "${user}@${TARGET_HOST}" "$@"
-  fi
+  run_ssh_timed "${user}" "${SSH_TIMEOUT}" "$@"
 }
 
 if [[ "${PORT_22_OPEN}" == "1" ]]; then
@@ -245,7 +264,7 @@ elif [[ "${LOCAL_VERIFY_SCRIPT_PRESENT}" != "1" ]]; then
 else
   DRY_RUN="false"
   REMOTE_VERIFY_ATTEMPTED=1
-  if REMOTE_VERIFY_OUTPUT="$(run_ssh "${SSH_AUTHENTICATED_USER}" "${REMOTE_VERIFY_COMMAND}" <"${LOCAL_VERIFY_SCRIPT}" 2>&1)"; then
+  if REMOTE_VERIFY_OUTPUT="$(run_ssh_timed "${SSH_AUTHENTICATED_USER}" "${REMOTE_VERIFY_TIMEOUT}" "${REMOTE_VERIFY_COMMAND}" <"${LOCAL_VERIFY_SCRIPT}" 2>&1)"; then
     REMOTE_VERIFY_EXIT_STATUS=0
     if grep -q '^AWOOOI_WINDOWS99_VMWARE_AUTOSTART=1$' <<<"${REMOTE_VERIFY_OUTPUT}" && grep -q '^MODE=Verify$' <<<"${REMOTE_VERIFY_OUTPUT}"; then
       VERIFY_COLLECTION_STATUS="collected_windows99_vmware_verify_stdout"
@@ -269,6 +288,7 @@ printf '%s\n' "target_host=${TARGET_HOST}"
 printf '%s\n' "target_host_alias=99"
 printf '%s\n' "connect_timeout_seconds=${CONNECT_TIMEOUT}"
 printf '%s\n' "ssh_timeout_seconds=${SSH_TIMEOUT}"
+printf '%s\n' "remote_verify_timeout_seconds=${REMOTE_VERIFY_TIMEOUT}"
 printf '%s\n' "port_timeout_wrapper=${PORT_TIMEOUT_WRAPPER}"
 printf '%s\n' "ssh_auth_probe_user_limit=${MAX_AUTH_USERS}"
 printf '%s\n' "ssh_timeout_wrapper=${SSH_TIMEOUT_WRAPPER}"

scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py

@@ -958,8 +958,8 @@ def build_windows99_verify_collection_packet(
             "VMWARE_AUTOSTART_VERIFY_READY",
         ],
         "no_secret_verify_command": (
-            "powershell -ExecutionPolicy Bypass -File "
-            ".\\windows99-vmware-autostart.ps1 -Mode Verify"
+            "powershell -ExecutionPolicy Bypass -Command "
+            "\"& ([scriptblock]::Create([Console]::In.ReadToEnd())) -Mode Verify\""
         ),
         "no_secret_collector_check_command": (
             "bash scripts/reboot-recovery/collect-windows99-vmware-verify.sh --check"

scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py

@@ -455,9 +455,11 @@ def test_missing_windows99_vmware_readback_fails_closed(tmp_path: Path) -> None:
     assert "VMRUN_PRESENT" in payload["windows99_verify_collection"][
         "expected_no_secret_output_fields"
     ]
-    assert "-Mode Verify" in payload["windows99_verify_collection"][
+    no_secret_command = payload["windows99_verify_collection"][
         "no_secret_verify_command"
     ]
+    assert "-Mode Verify" in no_secret_command
+    assert "[Console]::In.ReadToEnd()" in no_secret_command
     assert payload["windows99_verify_collection"][
         "no_secret_collector_check_command"
     ] == "bash scripts/reboot-recovery/collect-windows99-vmware-verify.sh --check"