fix(agent99): close canonical host readbacks

scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh

@@ -39,7 +39,7 @@ WORK_ITEM_ID=""
 IDENTITY_FILE=""
 KNOWN_HOSTS_FILE="${HOME:-}/.ssh/known_hosts"
 CONNECT_TIMEOUT_SECONDS="8"
-REMOTE_EXECUTION_TIMEOUT_SECONDS="45"
+REMOTE_EXECUTION_TIMEOUT_SECONDS="180"
 
 usage() {
   printf '%s\n' \
@@ -206,6 +206,7 @@ trap cleanup EXIT HUP INT TERM
 
 ENVELOPE_PATH="${WORK_DIR}/envelope.json"
 BOOTSTRAP_PATH="${WORK_DIR}/remote-bootstrap.ps1"
+BOOTSTRAP_PAYLOAD_PATH="${WORK_DIR}/remote-bootstrap.ps1.gz.b64"
 
 python3 - \
   "${SOURCE_ROOT}" \
@@ -300,6 +301,28 @@ bootstrap = (
 output_path.write_text(bootstrap, encoding="utf-8")
 PY
 
+python3 - \
+  "${BOOTSTRAP_PATH}" \
+  "${BOOTSTRAP_PAYLOAD_PATH}" <<'PY'
+from __future__ import annotations
+
+import base64
+import gzip
+import sys
+from pathlib import Path
+
+bootstrap_path, output_path = map(Path, sys.argv[1:3])
+compressed = gzip.compress(
+    bootstrap_path.read_bytes(),
+    compresslevel=9,
+    mtime=0,
+)
+output_path.write_text(
+    base64.b64encode(compressed).decode("ascii"),
+    encoding="ascii",
+)
+PY
+
 SSH_OPTIONS=(
   -T
   -o BatchMode=yes
@@ -321,10 +344,34 @@ if [[ -n "${IDENTITY_FILE}" ]]; then
   SSH_OPTIONS+=( -i "${IDENTITY_FILE}" )
 fi
 
-readonly REMOTE_COMMAND='powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -Command "$inputText=[Console]::In.ReadToEnd(); & ([ScriptBlock]::Create($inputText))"'
+readonly REMOTE_COMMAND="$(python3 - <<'PY'
+from __future__ import annotations
+
+import base64
+
+decoder = r'''$payload = [Console]::In.ReadToEnd()
+$compressed = [Convert]::FromBase64String(($payload -replace '\s', ''))
+$inputStream = New-Object IO.MemoryStream(,$compressed)
+$gzip = New-Object IO.Compression.GzipStream($inputStream, [IO.Compression.CompressionMode]::Decompress)
+$reader = New-Object IO.StreamReader($gzip, [Text.Encoding]::UTF8)
+try {
+  $inputText = $reader.ReadToEnd()
+} finally {
+  $reader.Dispose()
+  $gzip.Dispose()
+  $inputStream.Dispose()
+}
+& ([ScriptBlock]::Create($inputText))'''
+encoded = base64.b64encode(decoder.encode("utf-16le")).decode("ascii")
+print(
+    "powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass "
+    f"-EncodedCommand {encoded}"
+)
+PY
+)"
 set +e
 timeout --signal=TERM --kill-after=10 "${REMOTE_EXECUTION_TIMEOUT_SECONDS}s" \
-  ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${REMOTE_COMMAND}" <"${BOOTSTRAP_PATH}"
+  ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${REMOTE_COMMAND}" <"${BOOTSTRAP_PAYLOAD_PATH}"
 status=$?
 set -e
 exit "${status}"

scripts/reboot-recovery/host112-guest-readiness.sh

@@ -1388,8 +1388,32 @@ elif [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
   capture_manager_state
 fi
 
-systemd_state="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl is-system-running 2>/dev/null || true)"
-systemd_state="${systemd_state:-unknown}"
+systemd_state_raw="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl is-system-running 2>/dev/null || true)"
+systemd_state_raw="${systemd_state_raw:-unknown}"
+failed_units_query_ok=1
+if ! failed_units_output="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl list-units --failed --no-legend --no-pager --plain 2>/dev/null)"; then
+  failed_units_query_ok=0
+  failed_units_output=""
+fi
+nonself_failed_units=()
+while IFS= read -r failed_unit; do
+  [ -n "$failed_unit" ] || continue
+  if [[ "$failed_unit" =~ ^awoooi-host112-guest-recovery\.service$ ]]; then
+    continue
+  fi
+  nonself_failed_units+=("$failed_unit")
+done < <(printf '%s\n' "$failed_units_output" | awk '{ if ($1 == "●") print $2; else print $1 }')
+nonself_failed_count="${#nonself_failed_units[@]}"
+nonself_failed_text="none"
+if [ "$nonself_failed_count" -gt 0 ]; then
+  nonself_failed_text="$(IFS=,; printf '%s' "${nonself_failed_units[*]}")"
+fi
+systemd_state="$systemd_state_raw"
+if [ "$systemd_state_raw" = "degraded" ] \
+    && [ "$failed_units_query_ok" -eq 1 ] \
+    && [ "$nonself_failed_count" -eq 0 ]; then
+  systemd_state="running"
+fi
 default_target="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl get-default 2>/dev/null || true)"
 default_target="${default_target:-unknown}"
 graphical_target="$(unit_active graphical.target)"
@@ -1524,7 +1548,7 @@ run_output="${RUN_ID:-none}"
 work_item_output="${WORK_ITEM_ID:-none}"
 readback="schema_version=host112_guest_recovery_v2 mode=$MODE trace_id=$trace_output run_id=$run_output work_item_id=$work_item_output boot_id=$boot_id uptime_seconds=$uptime_seconds systemd_state=$systemd_state startup_enabled=$startup_enabled startup_active=$startup_active default_target=$default_target graphical_target=$graphical_target display_manager=$display_manager lightdm=$lightdm vmware_tools=$vmware_tools xorg=$xorg wazuh_indexer=$wazuh_indexer wazuh_manager=$wazuh_manager wazuh_manager_result=$manager_result manager_failed_state_observed=$manager_failed_state_observed wazuh_analysisd_process_count=$analysisd_count wazuh_dashboard=$wazuh_dashboard filebeat=$filebeat ssh_service=$ssh_service ssh_enabled=$ssh_enabled ssh_port_listening=$ssh_port_listening ssh_ready=$ssh_ready console_ready=$console_ready services_ready=$services_ready recovery_timer_ready=$recovery_timer_ready guest_ready=$guest_ready manager_preflight_status=$manager_preflight_status manager_terminal=$manager_terminal run_terminal=$run_terminal terminal=$run_terminal action_failure_observed=$action_failure_observed manager_orphan_analysisd=$manager_orphan_analysisd manager_memory_available_kb=$manager_memory_available_kb manager_disk_available_kb=$manager_disk_available_kb manager_disk_used_percent=$manager_disk_used_percent manager_resource_ready=$manager_resource_ready wazuh_agent_event_port_1514_tcp=$event_1514_tcp wazuh_agent_event_port_1514_udp=$event_1514_udp wazuh_agent_event_port_1514_ready=$manager_event_port_ready wazuh_agent_enrollment_port_1515_tcp=$enrollment_1515_tcp wazuh_manager_api_port_55000_tcp=$manager_api_55000_tcp manager_independent_verifier_ready=$manager_verifier_ready manager_cooldown_remaining_seconds=$manager_cooldown_remaining_seconds manager_attempt_count=$manager_attempt_count manager_start_exit=$manager_start_exit apply_phase_timeout_budget_seconds=$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS executor_deadline_seconds=$EXECUTOR_DEADLINE_SECONDS runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed artifact_transaction_status=$artifact_transaction_status artifact_transaction_phase=$artifact_transaction_phase artifact_transaction_path=$ARTIFACT_TRANSACTION_PATH artifact_transaction_started=$artifact_transaction_started artifact_transaction_blocked=$artifact_transaction_blocked artifact_transaction_write_performed=$artifact_transaction_write_performed artifact_transaction_pair_verified=$artifact_transaction_pair_verified artifact_transaction_prior_pair_verified=$artifact_transaction_prior_pair_verified receipt_write_performed=$receipt_write_performed receipt_status=$receipt_status receipt_ref=$receipt_ref receipt_path=$receipt_path receipt_sha256=$receipt_sha256 receipt_reused=$receipt_reused durable_receipt_present=$durable_receipt_present durable_receipt_identity_match=$durable_receipt_identity_match durable_receipt_boot_match=$durable_receipt_boot_match durable_receipt_terminal=$durable_receipt_terminal durable_readback_present=$durable_readback_present durable_readback_identity_match=$durable_readback_identity_match durable_readback_boot_match=$durable_readback_boot_match durable_readback_receipt_link_match=$durable_readback_receipt_link_match durable_readback_terminal_match=$durable_readback_terminal_match rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal timer_artifact_policy=$timer_artifact_policy timer_ledger_capacity=$timer_ledger_capacity timer_ledger_slot=$timer_ledger_slot timer_observation_status=$timer_observation_status timer_observation_path=$timer_observation_path timer_observation_sha256=$timer_observation_sha256 timer_observation_repeat_count=$timer_observation_repeat_count timer_observation_write_performed=$timer_observation_write_performed signal_readback_status=$signal_readback_status signal_readback_path=$signal_readback_path signal_readback_sha256=$signal_readback_sha256 action_count=${#actions[@]} actions=$action_text prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write"
 
-readback="$readback systemd_state_raw=$systemd_state_raw nonself_failed_count=$nonself_failed_count nonself_failed_units=$nonself_failed_text"
+readback="$readback systemd_state_raw=$systemd_state_raw failed_units_query_ok=$failed_units_query_ok nonself_failed_count=$nonself_failed_count nonself_failed_units=$nonself_failed_text"
 
 canonical_readback_base() {
   printf '%s\n' "$readback" | awk '

scripts/reboot-recovery/install-macos111-host-boot-readback.sh

@@ -25,8 +25,9 @@ Usage:
   install-macos111-host-boot-readback.sh --check
   install-macos111-host-boot-readback.sh --apply
 
-Installs a no-secret Darwin boot readback and authorizes host 110's existing
-public key with a forced command. The key cannot open a shell or forwarding.
+Installs a no-secret Darwin boot and performance readback and authorizes host
+110's existing public key with a forced command. The key cannot open a shell,
+request a PTY, or open forwarding.
 USAGE
 }
 
@@ -61,6 +62,12 @@ control_path_check() {
     "ssh -i /home/wooo/.ssh/id_ed25519 -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=/home/wooo/.ssh/known_hosts -o ConnectTimeout=6 -o ConnectionAttempts=1 -o NumberOfPasswordPrompts=0 '${TARGET_USER}@${TARGET_ADDRESS}' ignored-by-forced-command"
 }
 
+validate_readback() {
+  local output="$1"
+  grep -Eq '^boot_id=darwin_[0-9]+ uptime_seconds=[0-9]+ systemd_state=darwin_ssh startup_enabled=(enabled|unknown) startup_active=(active|inactive)$' <<<"$output" \
+    && grep -Eq '^perf_schema=agent99_perf_readback_v1 os=darwin cores=[1-9][0-9]* load1=[0-9]+([.][0-9]+)? mem_available_percent=[0-9]+([.][0-9]+)? disk_used_percent=[0-9]+([.][0-9]+)?$' <<<"$output"
+}
+
 check() {
   local direct_output control_output
   echo "AWOOOI_MACOS111_BOOT_READBACK=1"
@@ -71,6 +78,11 @@ check() {
     return 75
   fi
   echo "$direct_output"
+  if ! validate_readback "$direct_output"; then
+    echo "MACOS111_READBACK_SCRIPT_READY=0"
+    echo "BLOCKER macos111_readback_schema_incomplete"
+    return 75
+  fi
   echo "MACOS111_READBACK_SCRIPT_READY=1"
 
   if ! control_output="$(control_path_check)"; then
@@ -79,6 +91,11 @@ check() {
     return 75
   fi
   echo "$control_output"
+  if ! validate_readback "$control_output"; then
+    echo "MACOS111_CONTROL_PATH_READY=0"
+    echo "BLOCKER macos111_control_readback_schema_incomplete"
+    return 75
+  fi
   echo "MACOS111_CONTROL_PATH_READY=1"
 }
 

scripts/reboot-recovery/macos-host-boot-readback.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 set -euo pipefail
+export LC_ALL=C
 
 boot_raw="$(sysctl -n kern.boottime)"
 boot_epoch="$(printf '%s\n' "$boot_raw" | sed -E 's/^\{ sec = ([0-9]+),.*/\1/')"
@@ -29,5 +30,34 @@ else
   startup_active="inactive"
 fi
 
+logical_cores="$(sysctl -n hw.logicalcpu 2>/dev/null || true)"
+load1="$(sysctl -n vm.loadavg 2>/dev/null | awk '{ print $2; exit }')"
+mem_available_percent="$(memory_pressure -Q 2>/dev/null | awk -F': ' '
+  /System-wide memory free percentage/ {
+    value = $2
+    gsub(/%/, "", value)
+    print value
+    exit
+  }
+')"
+disk_used_percent="$(df -Pk / 2>/dev/null | awk '
+  NR == 2 {
+    value = $5
+    gsub(/%/, "", value)
+    print value
+    exit
+  }
+')"
+
+if [[ ! "$logical_cores" =~ ^[0-9]+$ ]] || (( logical_cores < 1 )) ||
+   [[ ! "$load1" =~ ^[0-9]+([.][0-9]+)?$ ]] ||
+   [[ ! "$mem_available_percent" =~ ^[0-9]+([.][0-9]+)?$ ]] ||
+   [[ ! "$disk_used_percent" =~ ^[0-9]+([.][0-9]+)?$ ]]; then
+  echo "BLOCKER darwin_perf_readback_unavailable"
+  exit 65
+fi
+
 printf 'boot_id=darwin_%s uptime_seconds=%s systemd_state=darwin_ssh startup_enabled=%s startup_active=%s\n' \
   "$boot_epoch" "$uptime_seconds" "$startup_enabled" "$startup_active"
+printf 'perf_schema=agent99_perf_readback_v1 os=darwin cores=%s load1=%s mem_available_percent=%s disk_used_percent=%s\n' \
+  "$logical_cores" "$load1" "$mem_available_percent" "$disk_used_percent"

scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py

@@ -25,14 +25,20 @@ def test_recover_requires_full_sop_coordinator_before_verified_resolution() -> N
 def test_host111_is_in_recovery_executor_and_all_host_readiness_scope() -> None:
     control = read("agent99-control-plane.ps1")
     config = read("agent99.config.99.example.json")
+    generic_config = read("agent99.config.example.json")
     deploy = read("agent99-deploy.ps1")
+    bootstrap = read("agent99-bootstrap.ps1")
     tasks = read("agent99-register-tasks.ps1")
 
     assert '"host": "192.168.0.111"' in config
+    assert '"192.168.0.111": "ooo"' in config
+    assert '"192.168.0.111": "ooo"' in generic_config
     assert '"recoveryAction": "wake_on_lan"' in config
     assert "Invoke-AgentExternalHostRecovery" in control
     assert 'verifier = "icmp_and_required_tcp_ports"' in control
     assert "host111_wol_executor" in deploy
+    assert "host111_canonical_ssh_user" in deploy
+    assert 'Set-AgentBootstrapProperty $config.sshUsers "192.168.0.111" "ooo"' in bootstrap
     assert '-Mode Recover -ControlledApply -RunNow' in tasks
     assert '-Source agent99-startup-recovery' in tasks
 

scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py

@@ -58,7 +58,7 @@ def test_sender_is_valid_shell_with_check_as_the_default() -> None:
     assert "apply_run_id_missing_or_invalid" in source
     assert "apply_work_item_id_missing_or_invalid" in source
     assert 'MODE_SELECTION="default"' in source
-    assert 'REMOTE_EXECUTION_TIMEOUT_SECONDS="45"' in source
+    assert 'REMOTE_EXECUTION_TIMEOUT_SECONDS="180"' in source
     assert 'REMOTE_EXECUTION_TIMEOUT_SECONDS="2400"' in source
     assert "timeout --signal=TERM --kill-after=10" in source
 
@@ -144,8 +144,12 @@ def test_sender_builds_sha256_envelope_and_streams_it_without_remote_cli_data()
     assert "hashlib.sha256(content).hexdigest()" in source
     assert '"manifestSha256": manifest_sha256' in source
     assert '"contentBase64": base64.b64encode(content).decode("ascii")' in source
+    assert "gzip.compress(" in source
+    assert "IO.Compression.GzipStream" in source
+    assert 'decoder.encode("utf-16le")' in source
+    assert "-EncodedCommand" in source
     assert "[Console]::In.ReadToEnd()" in source
-    assert '<"${BOOTSTRAP_PATH}"' in source
+    assert '<"${BOOTSTRAP_PAYLOAD_PATH}"' in source
     assert "rawEnvelope" not in source
     assert "sshpass" not in source
 
@@ -635,4 +639,5 @@ def test_check_mode_builds_and_streams_bundle_over_bounded_fake_transport(
     assert "PasswordAuthentication=no" in args
     assert "Administrator@192.168.0.99" in args
     assert str(identity) in args
-    assert int(bytes_path.read_text(encoding="utf-8")) > 500_000
+    payload_bytes = int(bytes_path.read_text(encoding="utf-8"))
+    assert 50_000 < payload_bytes < 500_000

scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py

@@ -637,10 +637,10 @@ def test_deploy_and_bootstrap_canonicalize_host112_direct_transport() -> None:
 
     assert 'Set-AgentProperty $config.sshUsers "192.168.0.112" "kali"' in deploy
     assert 'name = "host112_canonical_direct_route"' in deploy
-    assert 'Host112 canonical SSH and VMX config post-verifier failed.' in deploy
+    assert 'Canonical Host111 SSH/recovery and Host112 direct/SSD config post-verifier failed.' in deploy
     assert "Ensure-AgentHost112CanonicalSshConfig" in bootstrap
     assert 'Set-AgentBootstrapProperty $config.sshUsers "192.168.0.112" "kali"' in bootstrap
-    assert 'Host112 canonical SSH and VMX config bootstrap verifier failed.' in bootstrap
+    assert 'Host111/Host112 canonical SSH and Host112 VMX config bootstrap verifier failed.' in bootstrap
     for config in (generic_config, host_config):
         assert '"192.168.0.112": "kali"' in config
         assert '"directHosts"' in config
@@ -682,8 +682,8 @@ def test_deploy_and_bootstrap_canonicalize_host112_ssd_vmx_identity() -> None:
     assert bootstrap.index(bootstrap_preflight) < bootstrap.index(bootstrap_first_write)
     assert 'name = "host112_canonical_vmx_path"' in deploy
     assert "Test-Path -LiteralPath $canonicalHost112Vmx -PathType Leaf" in deploy
-    assert "Host112 canonical SSH and VMX config post-verifier failed." in deploy
-    assert "Host112 canonical SSH and VMX config bootstrap verifier failed." in bootstrap
+    assert "Canonical Host111 SSH/recovery and Host112 direct/SSD config post-verifier failed." in deploy
+    assert "Host111/Host112 canonical SSH and Host112 VMX config bootstrap verifier failed." in bootstrap
     assert '$persistedHost112Vm.Count -ne 1' in deploy
     assert '$persistedHost112Vm.Count -ne 1' in bootstrap
     assert 'Set-AgentProperty $config "vms" @(@($nonHost112Vms) + @($canonicalHost112Vm))' in deploy

scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py

@@ -127,6 +127,7 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() ->
     assert "console_ready=$console_ready" in host112_readiness
     assert "guest_ready=$guest_ready" in host112_readiness
     assert "systemd_state_raw=$systemd_state_raw" in host112_readiness
+    assert "failed_units_query_ok=$failed_units_query_ok" in host112_readiness
     assert "nonself_failed_count=$nonself_failed_count" in host112_readiness
     assert "^awoooi-host112-guest-recovery\\.service$" in host112_readiness
     assert "restrict,command=" in host112_installer
@@ -142,8 +143,15 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() ->
     assert "sysctl -n kern.boottime" in macos_readback
     assert "s/^\\{ sec = ([0-9]+),.*/\\1/" in macos_readback
     assert "boot_id=darwin_%s" in macos_readback
+    assert "perf_schema=agent99_perf_readback_v1 os=darwin" in macos_readback
+    assert "memory_pressure -Q" in macos_readback
+    assert "df -Pk /" in macos_readback
     assert "127.0.0.1:11434/api/tags" in macos_readback
     assert "restrict,command=" in macos_installer
+    assert "validate_readback" in macos_installer
+    assert "macos111_readback_schema_incomplete" in macos_installer
+    assert "macos111_control_readback_schema_incomplete" in macos_installer
+    assert "perf_schema=agent99_perf_readback_v1 os=darwin" in macos_installer
     assert "IdentitiesOnly=yes" in macos_installer
     assert "CONTROL_PUBLIC_KEY_PATH" in macos_installer
     assert "authorized_keys" in macos_installer