fix(agent99): stop Host112 verifier churn
This commit is contained in:
@@ -116,7 +116,9 @@ control_path_check() {
|
||||
validate_readback() {
|
||||
local output="$1"
|
||||
grep -Eq '^boot_id=darwin_[0-9]+ uptime_seconds=[0-9]+ systemd_state=darwin_ssh startup_enabled=(enabled|unknown) startup_active=(active|inactive)$' <<<"$output" \
|
||||
&& grep -Eq '^perf_schema=agent99_perf_readback_v1 os=darwin cores=[1-9][0-9]* load1=[0-9]+([.][0-9]+)? mem_available_percent=[0-9]+([.][0-9]+)? disk_used_percent=[0-9]+([.][0-9]+)?$' <<<"$output"
|
||||
&& grep -Eq '^perf_schema=agent99_perf_readback_v1 os=darwin cores=[1-9][0-9]* load1=[0-9]+([.][0-9]+)? mem_available_percent=[0-9]+([.][0-9]+)? disk_used_percent=[0-9]+([.][0-9]+)?$' <<<"$output" \
|
||||
&& grep -Eq '^darwin_pressure_schema=agent99_darwin_pressure_v1 cpu_host_percent=[0-9]+([.][0-9]+)? swap_used_percent=[0-9]+([.][0-9]+)?$' <<<"$output" \
|
||||
&& grep -Eq '^darwin_process_sample_v1 pid=[0-9]+ ppid=[0-9]+ cpu_percent=[0-9]+([.][0-9]+)? memory_percent=[0-9]+([.][0-9]+)? rss_kib=[0-9]+ name=[A-Za-z0-9._-]+$' <<<"$output"
|
||||
}
|
||||
|
||||
check() {
|
||||
|
||||
@@ -40,6 +40,37 @@ mem_available_percent="$(memory_pressure -Q 2>/dev/null | awk -F': ' '
|
||||
exit
|
||||
}
|
||||
')"
|
||||
process_cpu_sum="$(ps -A -o %cpu= 2>/dev/null | awk '{ sum += $1 } END { printf "%.2f", sum + 0 }')"
|
||||
cpu_host_percent="$(awk -v total="$process_cpu_sum" -v cores="$logical_cores" 'BEGIN {
|
||||
if (cores <= 0) { exit 1 }
|
||||
value = total / cores
|
||||
if (value > 100) { value = 100 }
|
||||
printf "%.2f", value
|
||||
}')"
|
||||
swap_total_mib="$(sysctl vm.swapusage 2>/dev/null | awk '{
|
||||
for (i = 1; i <= NF; i += 1) {
|
||||
if ($i == "total") {
|
||||
value = $(i + 2)
|
||||
gsub(/M/, "", value)
|
||||
print value
|
||||
exit
|
||||
}
|
||||
}
|
||||
}')"
|
||||
swap_used_mib="$(sysctl vm.swapusage 2>/dev/null | awk '{
|
||||
for (i = 1; i <= NF; i += 1) {
|
||||
if ($i == "used") {
|
||||
value = $(i + 2)
|
||||
gsub(/M/, "", value)
|
||||
print value
|
||||
exit
|
||||
}
|
||||
}
|
||||
}')"
|
||||
swap_used_percent="$(awk -v used="$swap_used_mib" -v total="$swap_total_mib" 'BEGIN {
|
||||
if (total <= 0) { printf "0.00"; exit }
|
||||
printf "%.2f", (used / total) * 100
|
||||
}')"
|
||||
disk_used_percent="$(df -Pk / 2>/dev/null | awk '
|
||||
NR == 2 {
|
||||
value = $5
|
||||
@@ -52,6 +83,8 @@ disk_used_percent="$(df -Pk / 2>/dev/null | awk '
|
||||
if [[ ! "$logical_cores" =~ ^[0-9]+$ ]] || (( logical_cores < 1 )) ||
|
||||
[[ ! "$load1" =~ ^[0-9]+([.][0-9]+)?$ ]] ||
|
||||
[[ ! "$mem_available_percent" =~ ^[0-9]+([.][0-9]+)?$ ]] ||
|
||||
[[ ! "$cpu_host_percent" =~ ^[0-9]+([.][0-9]+)?$ ]] ||
|
||||
[[ ! "$swap_used_percent" =~ ^[0-9]+([.][0-9]+)?$ ]] ||
|
||||
[[ ! "$disk_used_percent" =~ ^[0-9]+([.][0-9]+)?$ ]]; then
|
||||
echo "BLOCKER darwin_perf_readback_unavailable"
|
||||
exit 65
|
||||
@@ -61,3 +94,25 @@ printf 'boot_id=darwin_%s uptime_seconds=%s systemd_state=darwin_ssh startup_ena
|
||||
"$boot_epoch" "$uptime_seconds" "$startup_enabled" "$startup_active"
|
||||
printf 'perf_schema=agent99_perf_readback_v1 os=darwin cores=%s load1=%s mem_available_percent=%s disk_used_percent=%s\n' \
|
||||
"$logical_cores" "$load1" "$mem_available_percent" "$disk_used_percent"
|
||||
printf 'darwin_pressure_schema=agent99_darwin_pressure_v1 cpu_host_percent=%s swap_used_percent=%s\n' \
|
||||
"$cpu_host_percent" "$swap_used_percent"
|
||||
|
||||
ps -Ao pid=,ppid=,%cpu=,%mem=,rss=,comm= -r 2>/dev/null | awk '
|
||||
NR <= 8 {
|
||||
pid = $1
|
||||
ppid = $2
|
||||
cpu = $3
|
||||
memory = $4
|
||||
rss = $5
|
||||
name = ""
|
||||
for (i = 6; i <= NF; i += 1) {
|
||||
name = name (name == "" ? "" : " ") $i
|
||||
}
|
||||
count = split(name, path, "/")
|
||||
name = path[count]
|
||||
gsub(/[^A-Za-z0-9._-]/, "_", name)
|
||||
if (name == "") { name = "unknown" }
|
||||
printf "darwin_process_sample_v1 pid=%s ppid=%s cpu_percent=%s memory_percent=%s rss_kib=%s name=%s\n", \
|
||||
pid, ppid, cpu, memory, rss, name
|
||||
}
|
||||
'
|
||||
|
||||
@@ -38,6 +38,8 @@ def test_node_exporter_routes_are_explicit_and_identical_in_policy_configs() ->
|
||||
assert performance["nodeExporterUrls"] == expected
|
||||
assert performance["nodeExporterTimeoutSeconds"] == 8
|
||||
assert performance["topCpuDiagnosticCooldownMinutes"] == 15
|
||||
assert performance["processCpuHostPercentWarning"] == 25
|
||||
assert performance["processCpuHostPercentCritical"] == 50
|
||||
assert performance["kernelPercpuMemoryPercentWarning"] == 10
|
||||
assert performance["kernelPercpuMemoryPercentCritical"] == 20
|
||||
assert performance["kernelSUnreclaimMemoryPercentCritical"] == 25
|
||||
@@ -117,6 +119,11 @@ def test_perf_uses_metrics_first_and_ssh_only_as_fallback_or_cooldown_diagnostic
|
||||
assert 'metricSource = if ($nodeExporterReadback.ok) { "node_exporter" }' in perf
|
||||
assert 'diagnosticStatus = "cooldown_suppressed"' in perf
|
||||
assert "topCpuDiagnosticCooldownMinutes" in perf
|
||||
assert 'if ($metricOs -eq "darwin" -and $null -ne $cpuHostPercent)' in perf
|
||||
assert '"cpu_host_percent_critical"' in perf
|
||||
assert '"cpu_host_percent_warning"' in perf
|
||||
assert 'diagnosticStatus = "captured_inline"' in perf
|
||||
assert "darwin_process_sample_v1" in perf
|
||||
assert "$diagnosticCooldownMinutes = [math]::Min(120, [math]::Max(5" in perf
|
||||
assert "memory_available_low" in perf
|
||||
assert "kernel_percpu_memory_warning" in perf
|
||||
@@ -145,6 +152,8 @@ def test_deploy_migrates_and_post_verifies_canonical_metric_routes() -> None:
|
||||
"kernelPercpuMemoryPercentWarning",
|
||||
"kernelPercpuMemoryPercentCritical",
|
||||
"kernelSUnreclaimMemoryPercentCritical",
|
||||
"processCpuHostPercentWarning",
|
||||
"processCpuHostPercentCritical",
|
||||
):
|
||||
assert f'Add-DefaultProperty $config.performance "{policy_name}"' in deploy
|
||||
assert policy_name in deploy[deploy.index("$nodeExporterPolicyReady") :]
|
||||
|
||||
@@ -152,6 +152,10 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() ->
|
||||
assert "boot_id=darwin_%s" in macos_readback
|
||||
assert "perf_schema=agent99_perf_readback_v1 os=darwin" in macos_readback
|
||||
assert "memory_pressure -Q" in macos_readback
|
||||
assert "darwin_pressure_schema=agent99_darwin_pressure_v1" in macos_readback
|
||||
assert "darwin_process_sample_v1" in macos_readback
|
||||
assert "cpu_host_percent" in macos_readback
|
||||
assert "swap_used_percent" in macos_readback
|
||||
assert "df -Pk /" in macos_readback
|
||||
assert "127.0.0.1:11434/api/tags" in macos_readback
|
||||
assert "restrict,command=" in macos_installer
|
||||
@@ -159,6 +163,8 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() ->
|
||||
assert "macos111_readback_schema_incomplete" in macos_installer
|
||||
assert "macos111_control_readback_schema_incomplete" in macos_installer
|
||||
assert "perf_schema=agent99_perf_readback_v1 os=darwin" in macos_installer
|
||||
assert "darwin_pressure_schema=agent99_darwin_pressure_v1" in macos_installer
|
||||
assert "darwin_process_sample_v1" in macos_installer
|
||||
assert "controlled_identity_required" in macos_installer
|
||||
assert "incomplete_control_identity" in macos_installer
|
||||
assert "controlled_run_already_exists" in macos_installer
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
SCRIPT = (
|
||||
ROOT
|
||||
/ "scripts"
|
||||
/ "reboot-recovery"
|
||||
/ "windows99-host112-kernel-memory-soft-reboot.ps1"
|
||||
)
|
||||
|
||||
|
||||
def source() -> str:
|
||||
return SCRIPT.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_host112_soft_reboot_is_exact_target_and_same_run_bounded() -> None:
|
||||
text = source()
|
||||
|
||||
assert '[ValidateSet("Check", "Apply", "Verify")]' in text
|
||||
assert "S:\\VMs\\host112\\kali-linux-2025.4-vmware-amd64.vmx" in text
|
||||
assert 'ExpectedSshTarget = "kali@192.168.0.112"' in text
|
||||
assert "Assert-ControlIdentity" in text
|
||||
assert 'gate.traceId -eq $TraceId' in text
|
||||
assert 'gate.runId -eq $RunId' in text
|
||||
assert 'gate.workItemId -eq $WorkItemId' in text
|
||||
assert 'schemaVersion = "windows99_host112_kernel_memory_soft_reboot_v1"' in text
|
||||
|
||||
|
||||
def test_check_gate_requires_pressure_services_and_quiet_broker_source() -> None:
|
||||
text = source()
|
||||
|
||||
assert "$vmxRuntime.exactSystemOwner" in text
|
||||
assert "$guest.servicesReady" in text
|
||||
assert "$guest.memAvailablePercent -le 5" in text
|
||||
assert "$guest.percpuPercent -ge 20" in text
|
||||
assert "$guest.sunreclaimPercent -ge 25" in text
|
||||
assert "$guest.source121Sessions15m -eq 0" in text
|
||||
assert '"check_failed_no_write"' in text
|
||||
|
||||
|
||||
def test_apply_allows_only_vmware_soft_reset_without_destructive_fallback() -> None:
|
||||
text = source()
|
||||
|
||||
assert "& $VmrunPath -T ws reset $VmxPath soft" in text
|
||||
assert "& $VmrunPath -T ws list" in text
|
||||
assert '"SYSTEM vmrun does not own the exact Host112 VMX"' in text
|
||||
assert "exactTargetVisible = $vmrunSeesTarget" in text
|
||||
assert 'resetMode"] = "soft"' in text
|
||||
assert '"soft_reset_failed_no_hard_fallback"' in text
|
||||
assert 'hardResetPerformed = $false' in text
|
||||
assert 'vmStopPerformed = $false' in text
|
||||
assert "reset $VmxPath hard" not in text
|
||||
assert "stop $VmxPath hard" not in text
|
||||
assert "stop $VmxPath soft" not in text
|
||||
|
||||
|
||||
def test_verify_requires_new_boot_services_memory_and_vmx_owner() -> None:
|
||||
text = source()
|
||||
|
||||
assert '$guest.bootId -ne [string]$apply.beforeBootId' in text
|
||||
assert "$guest.memAvailablePercent -ge 25" in text
|
||||
assert "$guest.percpuPercent -lt 5" in text
|
||||
assert "$guest.sunreclaimPercent -lt 10" in text
|
||||
assert "$guest.source121Sessions15m -eq 0" in text
|
||||
assert "$vmxRuntime.exactSystemOwner" in text
|
||||
assert '"verified_recovered"' in text
|
||||
assert '"post_verifier_failed_no_hard_fallback"' in text
|
||||
|
||||
|
||||
def test_windows_transport_uses_agent99_identity_and_process_timeout() -> None:
|
||||
text = source()
|
||||
|
||||
assert '"-o", "IdentitiesOnly=yes"' in text
|
||||
assert '"-o", "BatchMode=yes"' in text
|
||||
assert "Start-Process" in text
|
||||
assert "WaitForExit(35000)" in text
|
||||
assert "taskkill.exe /PID $process.Id /T /F" in text
|
||||
assert 'rawOutputStored = $false' in text
|
||||
assert 'secretReadPerformed = $false' in text
|
||||
@@ -0,0 +1,464 @@
|
||||
param(
|
||||
[ValidateSet("Check", "Apply", "Verify")]
|
||||
[string]$Mode = "Check",
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$TraceId,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$RunId,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$WorkItemId,
|
||||
[string]$GateReceiptPath = "",
|
||||
[string]$ApplyReceiptPath = "",
|
||||
[string]$EvidenceDir = "C:\Wooo\Agent99\evidence",
|
||||
[string]$VmxPath = "S:\VMs\host112\kali-linux-2025.4-vmware-amd64.vmx",
|
||||
[string]$VmrunPath = "C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe",
|
||||
[string]$SshTarget = "kali@192.168.0.112",
|
||||
[string]$SshIdentityFile = "C:\Wooo\Agent99\keys\agent99_ed25519",
|
||||
[int]$VerifyTimeoutSeconds = 420,
|
||||
[int]$VerifyIntervalSeconds = 10,
|
||||
[int]$GateMaxAgeMinutes = 10
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$IdentityPattern = "^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$"
|
||||
$ExpectedVmxPath = "S:\VMs\host112\kali-linux-2025.4-vmware-amd64.vmx"
|
||||
$ExpectedSshTarget = "kali@192.168.0.112"
|
||||
|
||||
function Assert-ControlIdentity {
|
||||
foreach ($value in @($TraceId, $RunId, $WorkItemId)) {
|
||||
if ($value -notmatch $IdentityPattern) {
|
||||
throw "controlled identity is invalid"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-SafeId {
|
||||
param([string]$Value)
|
||||
return ($Value -replace "[^A-Za-z0-9._-]", "_")
|
||||
}
|
||||
|
||||
function Write-ControlReceipt {
|
||||
param(
|
||||
[string]$Phase,
|
||||
[System.Collections.IDictionary]$Receipt
|
||||
)
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $EvidenceDir | Out-Null
|
||||
$safeRunId = Get-SafeId -Value $RunId
|
||||
$path = Join-Path $EvidenceDir (
|
||||
"windows99-host112-kernel-memory-{0}-{1}.json" -f (
|
||||
$Phase.ToLowerInvariant()
|
||||
), $safeRunId
|
||||
)
|
||||
$Receipt["evidencePath"] = $path
|
||||
$Receipt | ConvertTo-Json -Depth 12 | Set-Content `
|
||||
-LiteralPath $path `
|
||||
-Encoding UTF8
|
||||
return $path
|
||||
}
|
||||
|
||||
function Get-VmxRuntime {
|
||||
$matches = @(Get-CimInstance Win32_Process -ErrorAction Stop |
|
||||
Where-Object {
|
||||
$_.Name -eq "vmware-vmx.exe" -and
|
||||
$_.CommandLine -and
|
||||
$_.CommandLine.IndexOf(
|
||||
$VmxPath,
|
||||
[StringComparison]::OrdinalIgnoreCase
|
||||
) -ge 0
|
||||
})
|
||||
$rows = @($matches | ForEach-Object {
|
||||
$owner = Invoke-CimMethod -InputObject $_ -MethodName GetOwner
|
||||
[pscustomobject]@{
|
||||
processId = [int]$_.ProcessId
|
||||
sessionId = [int]$_.SessionId
|
||||
owner = [string]$owner.User
|
||||
}
|
||||
})
|
||||
return [pscustomobject]@{
|
||||
count = $rows.Count
|
||||
rows = $rows
|
||||
exactSystemOwner = [bool](
|
||||
$rows.Count -eq 1 -and
|
||||
$rows[0].sessionId -eq 0 -and
|
||||
$rows[0].owner -eq "SYSTEM"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
function Convert-KeyValueOutput {
|
||||
param([string[]]$Lines)
|
||||
|
||||
$values = @{}
|
||||
foreach ($line in $Lines) {
|
||||
if ($line -match "^([A-Za-z0-9_]+)=(.*)$") {
|
||||
$values[$Matches[1]] = $Matches[2].Trim()
|
||||
}
|
||||
}
|
||||
return $values
|
||||
}
|
||||
|
||||
function Convert-ToInt64 {
|
||||
param(
|
||||
[object]$Value,
|
||||
[long]$Default = -1
|
||||
)
|
||||
|
||||
$parsed = 0L
|
||||
if ([long]::TryParse([string]$Value, [ref]$parsed)) {
|
||||
return $parsed
|
||||
}
|
||||
return $Default
|
||||
}
|
||||
|
||||
function Invoke-Host112Readback {
|
||||
$remoteScript = @'
|
||||
set -uo pipefail
|
||||
mem_value() {
|
||||
awk -v key="$1" '$1 == key ":" {print $2; exit}' /proc/meminfo
|
||||
}
|
||||
unit_state() {
|
||||
systemctl is-active "$1" 2>/dev/null || true
|
||||
}
|
||||
source_121_count="$(
|
||||
timeout 15 journalctl -b -u ssh --since '15 minutes ago' \
|
||||
--grep='Accepted publickey' --no-pager -o cat 2>/dev/null |
|
||||
awk '{for (i=1; i<=NF; i++) if ($i == "from" && $(i+1) == "192.168.0.121") count++} END {print count+0}'
|
||||
)"
|
||||
printf 'boot_id=%s\n' "$(cat /proc/sys/kernel/random/boot_id)"
|
||||
printf 'uptime_seconds=%s\n' "$(awk '{print int($1)}' /proc/uptime)"
|
||||
printf 'systemd_state=%s\n' "$(systemctl is-system-running 2>/dev/null || true)"
|
||||
printf 'mem_total_kb=%s\n' "$(mem_value MemTotal)"
|
||||
printf 'mem_available_kb=%s\n' "$(mem_value MemAvailable)"
|
||||
printf 'percpu_kb=%s\n' "$(mem_value Percpu)"
|
||||
printf 'sunreclaim_kb=%s\n' "$(mem_value SUnreclaim)"
|
||||
printf 'source_121_sessions_15m=%s\n' "$source_121_count"
|
||||
printf 'ssh=%s\n' "$(unit_state ssh)"
|
||||
printf 'lightdm=%s\n' "$(unit_state lightdm)"
|
||||
printf 'open_vm_tools=%s\n' "$(unit_state open-vm-tools)"
|
||||
printf 'wazuh_indexer=%s\n' "$(unit_state wazuh-indexer)"
|
||||
printf 'wazuh_manager=%s\n' "$(unit_state wazuh-manager)"
|
||||
printf 'wazuh_dashboard=%s\n' "$(unit_state wazuh-dashboard)"
|
||||
printf 'filebeat=%s\n' "$(unit_state filebeat)"
|
||||
printf 'recovery_timer=%s\n' "$(unit_state awoooi-host112-guest-recovery.timer)"
|
||||
'@
|
||||
$encoded = [Convert]::ToBase64String(
|
||||
[Text.Encoding]::UTF8.GetBytes($remoteScript)
|
||||
)
|
||||
$remoteCommand = (
|
||||
"printf '%s' '{0}' | base64 -d | timeout 25 bash" -f $encoded
|
||||
)
|
||||
if (-not (Test-Path -LiteralPath $SshIdentityFile)) {
|
||||
throw "Agent99 SSH identity path is unavailable"
|
||||
}
|
||||
$arguments = @(
|
||||
"-i", $SshIdentityFile,
|
||||
"-o", "IdentitiesOnly=yes",
|
||||
"-o", "BatchMode=yes",
|
||||
"-o", "StrictHostKeyChecking=accept-new",
|
||||
"-o", "NumberOfPasswordPrompts=0",
|
||||
"-o", "ConnectTimeout=8",
|
||||
"-o", "ConnectionAttempts=1",
|
||||
"-o", "ServerAliveInterval=5",
|
||||
"-o", "ServerAliveCountMax=1",
|
||||
$SshTarget,
|
||||
$remoteCommand
|
||||
)
|
||||
$id = [guid]::NewGuid().ToString("N")
|
||||
$stdoutPath = Join-Path $env:TEMP "host112-kmem-$id.out"
|
||||
$stderrPath = Join-Path $env:TEMP "host112-kmem-$id.err"
|
||||
$process = Start-Process `
|
||||
-FilePath "ssh.exe" `
|
||||
-ArgumentList $arguments `
|
||||
-NoNewWindow `
|
||||
-RedirectStandardOutput $stdoutPath `
|
||||
-RedirectStandardError $stderrPath `
|
||||
-PassThru
|
||||
$null = $process.Handle
|
||||
if (-not $process.WaitForExit(35000)) {
|
||||
try {
|
||||
& taskkill.exe /PID $process.Id /T /F 2>&1 | Out-Null
|
||||
} catch {}
|
||||
$process.WaitForExit(2000) | Out-Null
|
||||
Remove-Item $stdoutPath, $stderrPath `
|
||||
-Force `
|
||||
-ErrorAction SilentlyContinue
|
||||
throw "Host112 SSH readback timed out after 35 seconds"
|
||||
}
|
||||
$process.WaitForExit() | Out-Null
|
||||
$process.Refresh()
|
||||
$stdout = if (Test-Path -LiteralPath $stdoutPath) {
|
||||
Get-Content -LiteralPath $stdoutPath -Raw
|
||||
} else { "" }
|
||||
$stderr = if (Test-Path -LiteralPath $stderrPath) {
|
||||
Get-Content -LiteralPath $stderrPath -Raw
|
||||
} else { "" }
|
||||
Remove-Item $stdoutPath, $stderrPath `
|
||||
-Force `
|
||||
-ErrorAction SilentlyContinue
|
||||
$output = @($stdout -split "`r?`n" | Where-Object { $_ })
|
||||
$exitCode = [int]$process.ExitCode
|
||||
if ($stderr) {
|
||||
$output += @($stderr -split "`r?`n" | Where-Object { $_ })
|
||||
}
|
||||
$values = Convert-KeyValueOutput -Lines $output
|
||||
$memTotalKb = Convert-ToInt64 $values["mem_total_kb"]
|
||||
$memAvailableKb = Convert-ToInt64 $values["mem_available_kb"]
|
||||
$percpuKb = Convert-ToInt64 $values["percpu_kb"]
|
||||
$sunreclaimKb = Convert-ToInt64 $values["sunreclaim_kb"]
|
||||
$memAvailablePercent = if ($memTotalKb -gt 0 -and $memAvailableKb -ge 0) {
|
||||
[math]::Round(($memAvailableKb / $memTotalKb) * 100, 2)
|
||||
} else { $null }
|
||||
$percpuPercent = if ($memTotalKb -gt 0 -and $percpuKb -ge 0) {
|
||||
[math]::Round(($percpuKb / $memTotalKb) * 100, 2)
|
||||
} else { $null }
|
||||
$sunreclaimPercent = if ($memTotalKb -gt 0 -and $sunreclaimKb -ge 0) {
|
||||
[math]::Round(($sunreclaimKb / $memTotalKb) * 100, 2)
|
||||
} else { $null }
|
||||
$requiredUnits = @(
|
||||
"ssh",
|
||||
"lightdm",
|
||||
"open_vm_tools",
|
||||
"wazuh_indexer",
|
||||
"wazuh_manager",
|
||||
"wazuh_dashboard",
|
||||
"filebeat",
|
||||
"recovery_timer"
|
||||
)
|
||||
$failedUnits = @($requiredUnits | Where-Object {
|
||||
[string]$values[$_] -ne "active"
|
||||
})
|
||||
return [pscustomobject]@{
|
||||
ok = [bool](
|
||||
$exitCode -eq 0 -and
|
||||
$values["boot_id"] -match "^[0-9a-f-]{36}$" -and
|
||||
$values["systemd_state"] -eq "running"
|
||||
)
|
||||
exitCode = $exitCode
|
||||
bootId = [string]$values["boot_id"]
|
||||
uptimeSeconds = Convert-ToInt64 $values["uptime_seconds"]
|
||||
systemdState = [string]$values["systemd_state"]
|
||||
memTotalKb = $memTotalKb
|
||||
memAvailableKb = $memAvailableKb
|
||||
memAvailablePercent = $memAvailablePercent
|
||||
percpuKb = $percpuKb
|
||||
percpuPercent = $percpuPercent
|
||||
sunreclaimKb = $sunreclaimKb
|
||||
sunreclaimPercent = $sunreclaimPercent
|
||||
source121Sessions15m = Convert-ToInt64 `
|
||||
$values["source_121_sessions_15m"]
|
||||
failedUnits = $failedUnits
|
||||
servicesReady = [bool]($failedUnits.Count -eq 0)
|
||||
rawOutputStored = $false
|
||||
}
|
||||
}
|
||||
|
||||
function New-BaseReceipt {
|
||||
param([string]$Phase)
|
||||
|
||||
return [ordered]@{
|
||||
schemaVersion = "windows99_host112_kernel_memory_soft_reboot_v1"
|
||||
phase = $Phase
|
||||
traceId = $TraceId
|
||||
runId = $RunId
|
||||
workItemId = $WorkItemId
|
||||
generatedAt = Get-Date -Format o
|
||||
targetHost = "192.168.0.112"
|
||||
targetVmx = $ExpectedVmxPath
|
||||
runtimeWritePerformed = $false
|
||||
hardResetPerformed = $false
|
||||
vmStopPerformed = $false
|
||||
secretReadPerformed = $false
|
||||
terminal = "started"
|
||||
}
|
||||
}
|
||||
|
||||
Assert-ControlIdentity
|
||||
if ($VmxPath -ne $ExpectedVmxPath -or $SshTarget -ne $ExpectedSshTarget) {
|
||||
throw "target contract mismatch"
|
||||
}
|
||||
|
||||
if ($Mode -eq "Check") {
|
||||
$receipt = New-BaseReceipt -Phase "check"
|
||||
$vmxRuntime = Get-VmxRuntime
|
||||
$guest = Invoke-Host112Readback
|
||||
$kernelPressureCritical = [bool](
|
||||
($null -ne $guest.percpuPercent -and $guest.percpuPercent -ge 20) -or
|
||||
($null -ne $guest.sunreclaimPercent -and $guest.sunreclaimPercent -ge 25)
|
||||
)
|
||||
$gatePassed = [bool](
|
||||
(Test-Path -LiteralPath $VmxPath) -and
|
||||
(Test-Path -LiteralPath $VmrunPath) -and
|
||||
$vmxRuntime.exactSystemOwner -and
|
||||
$guest.ok -and
|
||||
$guest.servicesReady -and
|
||||
$guest.memAvailablePercent -le 5 -and
|
||||
$kernelPressureCritical -and
|
||||
$guest.source121Sessions15m -eq 0
|
||||
)
|
||||
$receipt["vmxRuntime"] = $vmxRuntime
|
||||
$receipt["guest"] = $guest
|
||||
$receipt["kernelPressureCritical"] = $kernelPressureCritical
|
||||
$receipt["gatePassed"] = $gatePassed
|
||||
$receipt["terminal"] = if ($gatePassed) {
|
||||
"check_passed_apply_allowed"
|
||||
} else {
|
||||
"check_failed_no_write"
|
||||
}
|
||||
$evidencePath = Write-ControlReceipt -Phase "check" -Receipt $receipt
|
||||
$receipt | ConvertTo-Json -Depth 12
|
||||
if (-not $gatePassed) { exit 2 }
|
||||
exit 0
|
||||
}
|
||||
|
||||
if ($Mode -eq "Apply") {
|
||||
$receipt = New-BaseReceipt -Phase "apply"
|
||||
if (-not $GateReceiptPath -or -not (Test-Path -LiteralPath $GateReceiptPath)) {
|
||||
throw "fresh check receipt is required"
|
||||
}
|
||||
$gate = Get-Content -LiteralPath $GateReceiptPath -Raw |
|
||||
ConvertFrom-Json
|
||||
$gateAgeMinutes = ((Get-Date) - [datetime]$gate.generatedAt).TotalMinutes
|
||||
$identityMatches = [bool](
|
||||
$gate.traceId -eq $TraceId -and
|
||||
$gate.runId -eq $RunId -and
|
||||
$gate.workItemId -eq $WorkItemId
|
||||
)
|
||||
if (
|
||||
-not $identityMatches -or
|
||||
-not $gate.gatePassed -or
|
||||
$gate.runtimeWritePerformed -or
|
||||
$gateAgeMinutes -lt 0 -or
|
||||
$gateAgeMinutes -gt $GateMaxAgeMinutes
|
||||
) {
|
||||
throw "check receipt is stale or invalid"
|
||||
}
|
||||
$vmxRuntime = Get-VmxRuntime
|
||||
if (-not $vmxRuntime.exactSystemOwner) {
|
||||
throw "host112 VM is not held by the expected SYSTEM runtime"
|
||||
}
|
||||
$vmrunListOutput = @(& $VmrunPath -T ws list 2>&1 |
|
||||
ForEach-Object { "$_" })
|
||||
$vmrunListExitCode = $LASTEXITCODE
|
||||
$vmrunSeesTarget = [bool](@($vmrunListOutput | Where-Object {
|
||||
$_ -and $_.IndexOf(
|
||||
$VmxPath,
|
||||
[StringComparison]::OrdinalIgnoreCase
|
||||
) -ge 0
|
||||
}).Count -eq 1)
|
||||
if ($vmrunListExitCode -ne 0 -or -not $vmrunSeesTarget) {
|
||||
throw "SYSTEM vmrun does not own the exact Host112 VMX"
|
||||
}
|
||||
$receipt["gateReceiptPath"] = $GateReceiptPath
|
||||
$receipt["gateAgeMinutes"] = [math]::Round($gateAgeMinutes, 2)
|
||||
$receipt["preApplyVmrun"] = [ordered]@{
|
||||
exitCode = $vmrunListExitCode
|
||||
exactTargetVisible = $vmrunSeesTarget
|
||||
rawOutputStored = $false
|
||||
}
|
||||
$receipt["beforeBootId"] = [string]$gate.guest.bootId
|
||||
$receipt["beforeMemory"] = [ordered]@{
|
||||
memAvailablePercent = $gate.guest.memAvailablePercent
|
||||
percpuPercent = $gate.guest.percpuPercent
|
||||
sunreclaimPercent = $gate.guest.sunreclaimPercent
|
||||
}
|
||||
$output = @(& $VmrunPath -T ws reset $VmxPath soft 2>&1 |
|
||||
ForEach-Object { "$_" })
|
||||
$exitCode = $LASTEXITCODE
|
||||
$receipt["vmrunExitCode"] = $exitCode
|
||||
$receipt["vmrunOutputStored"] = $false
|
||||
$receipt["runtimeWritePerformed"] = [bool]($exitCode -eq 0)
|
||||
$receipt["resetMode"] = "soft"
|
||||
$receipt["postVerifierPending"] = [bool]($exitCode -eq 0)
|
||||
$receipt["terminal"] = if ($exitCode -eq 0) {
|
||||
"soft_reset_requested_verifier_pending"
|
||||
} else {
|
||||
"soft_reset_failed_no_hard_fallback"
|
||||
}
|
||||
$evidencePath = Write-ControlReceipt -Phase "apply" -Receipt $receipt
|
||||
$receipt | ConvertTo-Json -Depth 12
|
||||
if ($exitCode -ne 0) { exit 2 }
|
||||
exit 0
|
||||
}
|
||||
|
||||
$receipt = New-BaseReceipt -Phase "verify"
|
||||
if (-not $ApplyReceiptPath -or -not (Test-Path -LiteralPath $ApplyReceiptPath)) {
|
||||
throw "apply receipt is required"
|
||||
}
|
||||
$apply = Get-Content -LiteralPath $ApplyReceiptPath -Raw | ConvertFrom-Json
|
||||
$identityMatches = [bool](
|
||||
$apply.traceId -eq $TraceId -and
|
||||
$apply.runId -eq $RunId -and
|
||||
$apply.workItemId -eq $WorkItemId
|
||||
)
|
||||
if (
|
||||
-not $identityMatches -or
|
||||
-not $apply.runtimeWritePerformed -or
|
||||
$apply.resetMode -ne "soft" -or
|
||||
$apply.vmrunExitCode -ne 0
|
||||
) {
|
||||
throw "apply receipt is invalid"
|
||||
}
|
||||
|
||||
$deadline = (Get-Date).AddSeconds($VerifyTimeoutSeconds)
|
||||
$attempts = 0
|
||||
$guest = $null
|
||||
do {
|
||||
$attempts += 1
|
||||
try {
|
||||
$guest = Invoke-Host112Readback
|
||||
} catch {
|
||||
$guest = $null
|
||||
}
|
||||
$bootChanged = [bool](
|
||||
$guest -and
|
||||
$guest.bootId -and
|
||||
$guest.bootId -ne [string]$apply.beforeBootId
|
||||
)
|
||||
$memoryRecovered = [bool](
|
||||
$guest -and
|
||||
$guest.memAvailablePercent -ge 25 -and
|
||||
$guest.percpuPercent -lt 5 -and
|
||||
$guest.sunreclaimPercent -lt 10
|
||||
)
|
||||
if (
|
||||
$guest -and
|
||||
$guest.ok -and
|
||||
$guest.servicesReady -and
|
||||
$bootChanged -and
|
||||
$memoryRecovered
|
||||
) {
|
||||
break
|
||||
}
|
||||
if ((Get-Date) -lt $deadline) {
|
||||
Start-Sleep -Seconds $VerifyIntervalSeconds
|
||||
}
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
|
||||
$vmxRuntime = Get-VmxRuntime
|
||||
$verified = [bool](
|
||||
$guest -and
|
||||
$guest.ok -and
|
||||
$guest.servicesReady -and
|
||||
$guest.bootId -ne [string]$apply.beforeBootId -and
|
||||
$guest.memAvailablePercent -ge 25 -and
|
||||
$guest.percpuPercent -lt 5 -and
|
||||
$guest.sunreclaimPercent -lt 10 -and
|
||||
$guest.source121Sessions15m -eq 0 -and
|
||||
$vmxRuntime.exactSystemOwner
|
||||
)
|
||||
$receipt["applyReceiptPath"] = $ApplyReceiptPath
|
||||
$receipt["beforeBootId"] = [string]$apply.beforeBootId
|
||||
$receipt["after"] = $guest
|
||||
$receipt["vmxRuntime"] = $vmxRuntime
|
||||
$receipt["attempts"] = $attempts
|
||||
$receipt["verified"] = $verified
|
||||
$receipt["terminal"] = if ($verified) {
|
||||
"verified_recovered"
|
||||
} else {
|
||||
"post_verifier_failed_no_hard_fallback"
|
||||
}
|
||||
$evidencePath = Write-ControlReceipt -Phase "verify" -Receipt $receipt
|
||||
$receipt | ConvertTo-Json -Depth 12
|
||||
if (-not $verified) { exit 2 }
|
||||
exit 0
|
||||
Reference in New Issue
Block a user