465 lines
15 KiB
PowerShell
465 lines
15 KiB
PowerShell
param(
|
|
[ValidateSet("Check", "Apply", "Verify")]
|
|
[string]$Mode = "Check",
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$TraceId,
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$RunId,
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$WorkItemId,
|
|
[string]$GateReceiptPath = "",
|
|
[string]$ApplyReceiptPath = "",
|
|
[string]$EvidenceDir = "C:\Wooo\Agent99\evidence",
|
|
[string]$VmxPath = "S:\VMs\host112\kali-linux-2025.4-vmware-amd64.vmx",
|
|
[string]$VmrunPath = "C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe",
|
|
[string]$SshTarget = "kali@192.168.0.112",
|
|
[string]$SshIdentityFile = "C:\Wooo\Agent99\keys\agent99_ed25519",
|
|
[int]$VerifyTimeoutSeconds = 420,
|
|
[int]$VerifyIntervalSeconds = 10,
|
|
[int]$GateMaxAgeMinutes = 10
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
$IdentityPattern = "^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$"
|
|
$ExpectedVmxPath = "S:\VMs\host112\kali-linux-2025.4-vmware-amd64.vmx"
|
|
$ExpectedSshTarget = "kali@192.168.0.112"
|
|
|
|
function Assert-ControlIdentity {
|
|
foreach ($value in @($TraceId, $RunId, $WorkItemId)) {
|
|
if ($value -notmatch $IdentityPattern) {
|
|
throw "controlled identity is invalid"
|
|
}
|
|
}
|
|
}
|
|
|
|
function Get-SafeId {
|
|
param([string]$Value)
|
|
return ($Value -replace "[^A-Za-z0-9._-]", "_")
|
|
}
|
|
|
|
function Write-ControlReceipt {
|
|
param(
|
|
[string]$Phase,
|
|
[System.Collections.IDictionary]$Receipt
|
|
)
|
|
|
|
New-Item -ItemType Directory -Force -Path $EvidenceDir | Out-Null
|
|
$safeRunId = Get-SafeId -Value $RunId
|
|
$path = Join-Path $EvidenceDir (
|
|
"windows99-host112-kernel-memory-{0}-{1}.json" -f (
|
|
$Phase.ToLowerInvariant()
|
|
), $safeRunId
|
|
)
|
|
$Receipt["evidencePath"] = $path
|
|
$Receipt | ConvertTo-Json -Depth 12 | Set-Content `
|
|
-LiteralPath $path `
|
|
-Encoding UTF8
|
|
return $path
|
|
}
|
|
|
|
function Get-VmxRuntime {
|
|
$matches = @(Get-CimInstance Win32_Process -ErrorAction Stop |
|
|
Where-Object {
|
|
$_.Name -eq "vmware-vmx.exe" -and
|
|
$_.CommandLine -and
|
|
$_.CommandLine.IndexOf(
|
|
$VmxPath,
|
|
[StringComparison]::OrdinalIgnoreCase
|
|
) -ge 0
|
|
})
|
|
$rows = @($matches | ForEach-Object {
|
|
$owner = Invoke-CimMethod -InputObject $_ -MethodName GetOwner
|
|
[pscustomobject]@{
|
|
processId = [int]$_.ProcessId
|
|
sessionId = [int]$_.SessionId
|
|
owner = [string]$owner.User
|
|
}
|
|
})
|
|
return [pscustomobject]@{
|
|
count = $rows.Count
|
|
rows = $rows
|
|
exactSystemOwner = [bool](
|
|
$rows.Count -eq 1 -and
|
|
$rows[0].sessionId -eq 0 -and
|
|
$rows[0].owner -eq "SYSTEM"
|
|
)
|
|
}
|
|
}
|
|
|
|
function Convert-KeyValueOutput {
|
|
param([string[]]$Lines)
|
|
|
|
$values = @{}
|
|
foreach ($line in $Lines) {
|
|
if ($line -match "^([A-Za-z0-9_]+)=(.*)$") {
|
|
$values[$Matches[1]] = $Matches[2].Trim()
|
|
}
|
|
}
|
|
return $values
|
|
}
|
|
|
|
function Convert-ToInt64 {
|
|
param(
|
|
[object]$Value,
|
|
[long]$Default = -1
|
|
)
|
|
|
|
$parsed = 0L
|
|
if ([long]::TryParse([string]$Value, [ref]$parsed)) {
|
|
return $parsed
|
|
}
|
|
return $Default
|
|
}
|
|
|
|
function Invoke-Host112Readback {
|
|
$remoteScript = @'
|
|
set -uo pipefail
|
|
mem_value() {
|
|
awk -v key="$1" '$1 == key ":" {print $2; exit}' /proc/meminfo
|
|
}
|
|
unit_state() {
|
|
systemctl is-active "$1" 2>/dev/null || true
|
|
}
|
|
source_121_count="$(
|
|
timeout 15 journalctl -b -u ssh --since '15 minutes ago' \
|
|
--grep='Accepted publickey' --no-pager -o cat 2>/dev/null |
|
|
awk '{for (i=1; i<=NF; i++) if ($i == "from" && $(i+1) == "192.168.0.121") count++} END {print count+0}'
|
|
)"
|
|
printf 'boot_id=%s\n' "$(cat /proc/sys/kernel/random/boot_id)"
|
|
printf 'uptime_seconds=%s\n' "$(awk '{print int($1)}' /proc/uptime)"
|
|
printf 'systemd_state=%s\n' "$(systemctl is-system-running 2>/dev/null || true)"
|
|
printf 'mem_total_kb=%s\n' "$(mem_value MemTotal)"
|
|
printf 'mem_available_kb=%s\n' "$(mem_value MemAvailable)"
|
|
printf 'percpu_kb=%s\n' "$(mem_value Percpu)"
|
|
printf 'sunreclaim_kb=%s\n' "$(mem_value SUnreclaim)"
|
|
printf 'source_121_sessions_15m=%s\n' "$source_121_count"
|
|
printf 'ssh=%s\n' "$(unit_state ssh)"
|
|
printf 'lightdm=%s\n' "$(unit_state lightdm)"
|
|
printf 'open_vm_tools=%s\n' "$(unit_state open-vm-tools)"
|
|
printf 'wazuh_indexer=%s\n' "$(unit_state wazuh-indexer)"
|
|
printf 'wazuh_manager=%s\n' "$(unit_state wazuh-manager)"
|
|
printf 'wazuh_dashboard=%s\n' "$(unit_state wazuh-dashboard)"
|
|
printf 'filebeat=%s\n' "$(unit_state filebeat)"
|
|
printf 'recovery_timer=%s\n' "$(unit_state awoooi-host112-guest-recovery.timer)"
|
|
'@
|
|
$encoded = [Convert]::ToBase64String(
|
|
[Text.Encoding]::UTF8.GetBytes($remoteScript)
|
|
)
|
|
$remoteCommand = (
|
|
"printf '%s' '{0}' | base64 -d | timeout 25 bash" -f $encoded
|
|
)
|
|
if (-not (Test-Path -LiteralPath $SshIdentityFile)) {
|
|
throw "Agent99 SSH identity path is unavailable"
|
|
}
|
|
$arguments = @(
|
|
"-i", $SshIdentityFile,
|
|
"-o", "IdentitiesOnly=yes",
|
|
"-o", "BatchMode=yes",
|
|
"-o", "StrictHostKeyChecking=accept-new",
|
|
"-o", "NumberOfPasswordPrompts=0",
|
|
"-o", "ConnectTimeout=8",
|
|
"-o", "ConnectionAttempts=1",
|
|
"-o", "ServerAliveInterval=5",
|
|
"-o", "ServerAliveCountMax=1",
|
|
$SshTarget,
|
|
$remoteCommand
|
|
)
|
|
$id = [guid]::NewGuid().ToString("N")
|
|
$stdoutPath = Join-Path $env:TEMP "host112-kmem-$id.out"
|
|
$stderrPath = Join-Path $env:TEMP "host112-kmem-$id.err"
|
|
$process = Start-Process `
|
|
-FilePath "ssh.exe" `
|
|
-ArgumentList $arguments `
|
|
-NoNewWindow `
|
|
-RedirectStandardOutput $stdoutPath `
|
|
-RedirectStandardError $stderrPath `
|
|
-PassThru
|
|
$null = $process.Handle
|
|
if (-not $process.WaitForExit(35000)) {
|
|
try {
|
|
& taskkill.exe /PID $process.Id /T /F 2>&1 | Out-Null
|
|
} catch {}
|
|
$process.WaitForExit(2000) | Out-Null
|
|
Remove-Item $stdoutPath, $stderrPath `
|
|
-Force `
|
|
-ErrorAction SilentlyContinue
|
|
throw "Host112 SSH readback timed out after 35 seconds"
|
|
}
|
|
$process.WaitForExit() | Out-Null
|
|
$process.Refresh()
|
|
$stdout = if (Test-Path -LiteralPath $stdoutPath) {
|
|
Get-Content -LiteralPath $stdoutPath -Raw
|
|
} else { "" }
|
|
$stderr = if (Test-Path -LiteralPath $stderrPath) {
|
|
Get-Content -LiteralPath $stderrPath -Raw
|
|
} else { "" }
|
|
Remove-Item $stdoutPath, $stderrPath `
|
|
-Force `
|
|
-ErrorAction SilentlyContinue
|
|
$output = @($stdout -split "`r?`n" | Where-Object { $_ })
|
|
$exitCode = [int]$process.ExitCode
|
|
if ($stderr) {
|
|
$output += @($stderr -split "`r?`n" | Where-Object { $_ })
|
|
}
|
|
$values = Convert-KeyValueOutput -Lines $output
|
|
$memTotalKb = Convert-ToInt64 $values["mem_total_kb"]
|
|
$memAvailableKb = Convert-ToInt64 $values["mem_available_kb"]
|
|
$percpuKb = Convert-ToInt64 $values["percpu_kb"]
|
|
$sunreclaimKb = Convert-ToInt64 $values["sunreclaim_kb"]
|
|
$memAvailablePercent = if ($memTotalKb -gt 0 -and $memAvailableKb -ge 0) {
|
|
[math]::Round(($memAvailableKb / $memTotalKb) * 100, 2)
|
|
} else { $null }
|
|
$percpuPercent = if ($memTotalKb -gt 0 -and $percpuKb -ge 0) {
|
|
[math]::Round(($percpuKb / $memTotalKb) * 100, 2)
|
|
} else { $null }
|
|
$sunreclaimPercent = if ($memTotalKb -gt 0 -and $sunreclaimKb -ge 0) {
|
|
[math]::Round(($sunreclaimKb / $memTotalKb) * 100, 2)
|
|
} else { $null }
|
|
$requiredUnits = @(
|
|
"ssh",
|
|
"lightdm",
|
|
"open_vm_tools",
|
|
"wazuh_indexer",
|
|
"wazuh_manager",
|
|
"wazuh_dashboard",
|
|
"filebeat",
|
|
"recovery_timer"
|
|
)
|
|
$failedUnits = @($requiredUnits | Where-Object {
|
|
[string]$values[$_] -ne "active"
|
|
})
|
|
return [pscustomobject]@{
|
|
ok = [bool](
|
|
$exitCode -eq 0 -and
|
|
$values["boot_id"] -match "^[0-9a-f-]{36}$" -and
|
|
$values["systemd_state"] -eq "running"
|
|
)
|
|
exitCode = $exitCode
|
|
bootId = [string]$values["boot_id"]
|
|
uptimeSeconds = Convert-ToInt64 $values["uptime_seconds"]
|
|
systemdState = [string]$values["systemd_state"]
|
|
memTotalKb = $memTotalKb
|
|
memAvailableKb = $memAvailableKb
|
|
memAvailablePercent = $memAvailablePercent
|
|
percpuKb = $percpuKb
|
|
percpuPercent = $percpuPercent
|
|
sunreclaimKb = $sunreclaimKb
|
|
sunreclaimPercent = $sunreclaimPercent
|
|
source121Sessions15m = Convert-ToInt64 `
|
|
$values["source_121_sessions_15m"]
|
|
failedUnits = $failedUnits
|
|
servicesReady = [bool]($failedUnits.Count -eq 0)
|
|
rawOutputStored = $false
|
|
}
|
|
}
|
|
|
|
function New-BaseReceipt {
|
|
param([string]$Phase)
|
|
|
|
return [ordered]@{
|
|
schemaVersion = "windows99_host112_kernel_memory_soft_reboot_v1"
|
|
phase = $Phase
|
|
traceId = $TraceId
|
|
runId = $RunId
|
|
workItemId = $WorkItemId
|
|
generatedAt = Get-Date -Format o
|
|
targetHost = "192.168.0.112"
|
|
targetVmx = $ExpectedVmxPath
|
|
runtimeWritePerformed = $false
|
|
hardResetPerformed = $false
|
|
vmStopPerformed = $false
|
|
secretReadPerformed = $false
|
|
terminal = "started"
|
|
}
|
|
}
|
|
|
|
Assert-ControlIdentity
|
|
if ($VmxPath -ne $ExpectedVmxPath -or $SshTarget -ne $ExpectedSshTarget) {
|
|
throw "target contract mismatch"
|
|
}
|
|
|
|
if ($Mode -eq "Check") {
|
|
$receipt = New-BaseReceipt -Phase "check"
|
|
$vmxRuntime = Get-VmxRuntime
|
|
$guest = Invoke-Host112Readback
|
|
$kernelPressureCritical = [bool](
|
|
($null -ne $guest.percpuPercent -and $guest.percpuPercent -ge 20) -or
|
|
($null -ne $guest.sunreclaimPercent -and $guest.sunreclaimPercent -ge 25)
|
|
)
|
|
$gatePassed = [bool](
|
|
(Test-Path -LiteralPath $VmxPath) -and
|
|
(Test-Path -LiteralPath $VmrunPath) -and
|
|
$vmxRuntime.exactSystemOwner -and
|
|
$guest.ok -and
|
|
$guest.servicesReady -and
|
|
$guest.memAvailablePercent -le 5 -and
|
|
$kernelPressureCritical -and
|
|
$guest.source121Sessions15m -eq 0
|
|
)
|
|
$receipt["vmxRuntime"] = $vmxRuntime
|
|
$receipt["guest"] = $guest
|
|
$receipt["kernelPressureCritical"] = $kernelPressureCritical
|
|
$receipt["gatePassed"] = $gatePassed
|
|
$receipt["terminal"] = if ($gatePassed) {
|
|
"check_passed_apply_allowed"
|
|
} else {
|
|
"check_failed_no_write"
|
|
}
|
|
$evidencePath = Write-ControlReceipt -Phase "check" -Receipt $receipt
|
|
$receipt | ConvertTo-Json -Depth 12
|
|
if (-not $gatePassed) { exit 2 }
|
|
exit 0
|
|
}
|
|
|
|
if ($Mode -eq "Apply") {
|
|
$receipt = New-BaseReceipt -Phase "apply"
|
|
if (-not $GateReceiptPath -or -not (Test-Path -LiteralPath $GateReceiptPath)) {
|
|
throw "fresh check receipt is required"
|
|
}
|
|
$gate = Get-Content -LiteralPath $GateReceiptPath -Raw |
|
|
ConvertFrom-Json
|
|
$gateAgeMinutes = ((Get-Date) - [datetime]$gate.generatedAt).TotalMinutes
|
|
$identityMatches = [bool](
|
|
$gate.traceId -eq $TraceId -and
|
|
$gate.runId -eq $RunId -and
|
|
$gate.workItemId -eq $WorkItemId
|
|
)
|
|
if (
|
|
-not $identityMatches -or
|
|
-not $gate.gatePassed -or
|
|
$gate.runtimeWritePerformed -or
|
|
$gateAgeMinutes -lt 0 -or
|
|
$gateAgeMinutes -gt $GateMaxAgeMinutes
|
|
) {
|
|
throw "check receipt is stale or invalid"
|
|
}
|
|
$vmxRuntime = Get-VmxRuntime
|
|
if (-not $vmxRuntime.exactSystemOwner) {
|
|
throw "host112 VM is not held by the expected SYSTEM runtime"
|
|
}
|
|
$vmrunListOutput = @(& $VmrunPath -T ws list 2>&1 |
|
|
ForEach-Object { "$_" })
|
|
$vmrunListExitCode = $LASTEXITCODE
|
|
$vmrunSeesTarget = [bool](@($vmrunListOutput | Where-Object {
|
|
$_ -and $_.IndexOf(
|
|
$VmxPath,
|
|
[StringComparison]::OrdinalIgnoreCase
|
|
) -ge 0
|
|
}).Count -eq 1)
|
|
if ($vmrunListExitCode -ne 0 -or -not $vmrunSeesTarget) {
|
|
throw "SYSTEM vmrun does not own the exact Host112 VMX"
|
|
}
|
|
$receipt["gateReceiptPath"] = $GateReceiptPath
|
|
$receipt["gateAgeMinutes"] = [math]::Round($gateAgeMinutes, 2)
|
|
$receipt["preApplyVmrun"] = [ordered]@{
|
|
exitCode = $vmrunListExitCode
|
|
exactTargetVisible = $vmrunSeesTarget
|
|
rawOutputStored = $false
|
|
}
|
|
$receipt["beforeBootId"] = [string]$gate.guest.bootId
|
|
$receipt["beforeMemory"] = [ordered]@{
|
|
memAvailablePercent = $gate.guest.memAvailablePercent
|
|
percpuPercent = $gate.guest.percpuPercent
|
|
sunreclaimPercent = $gate.guest.sunreclaimPercent
|
|
}
|
|
$output = @(& $VmrunPath -T ws reset $VmxPath soft 2>&1 |
|
|
ForEach-Object { "$_" })
|
|
$exitCode = $LASTEXITCODE
|
|
$receipt["vmrunExitCode"] = $exitCode
|
|
$receipt["vmrunOutputStored"] = $false
|
|
$receipt["runtimeWritePerformed"] = [bool]($exitCode -eq 0)
|
|
$receipt["resetMode"] = "soft"
|
|
$receipt["postVerifierPending"] = [bool]($exitCode -eq 0)
|
|
$receipt["terminal"] = if ($exitCode -eq 0) {
|
|
"soft_reset_requested_verifier_pending"
|
|
} else {
|
|
"soft_reset_failed_no_hard_fallback"
|
|
}
|
|
$evidencePath = Write-ControlReceipt -Phase "apply" -Receipt $receipt
|
|
$receipt | ConvertTo-Json -Depth 12
|
|
if ($exitCode -ne 0) { exit 2 }
|
|
exit 0
|
|
}
|
|
|
|
$receipt = New-BaseReceipt -Phase "verify"
|
|
if (-not $ApplyReceiptPath -or -not (Test-Path -LiteralPath $ApplyReceiptPath)) {
|
|
throw "apply receipt is required"
|
|
}
|
|
$apply = Get-Content -LiteralPath $ApplyReceiptPath -Raw | ConvertFrom-Json
|
|
$identityMatches = [bool](
|
|
$apply.traceId -eq $TraceId -and
|
|
$apply.runId -eq $RunId -and
|
|
$apply.workItemId -eq $WorkItemId
|
|
)
|
|
if (
|
|
-not $identityMatches -or
|
|
-not $apply.runtimeWritePerformed -or
|
|
$apply.resetMode -ne "soft" -or
|
|
$apply.vmrunExitCode -ne 0
|
|
) {
|
|
throw "apply receipt is invalid"
|
|
}
|
|
|
|
$deadline = (Get-Date).AddSeconds($VerifyTimeoutSeconds)
|
|
$attempts = 0
|
|
$guest = $null
|
|
do {
|
|
$attempts += 1
|
|
try {
|
|
$guest = Invoke-Host112Readback
|
|
} catch {
|
|
$guest = $null
|
|
}
|
|
$bootChanged = [bool](
|
|
$guest -and
|
|
$guest.bootId -and
|
|
$guest.bootId -ne [string]$apply.beforeBootId
|
|
)
|
|
$memoryRecovered = [bool](
|
|
$guest -and
|
|
$guest.memAvailablePercent -ge 25 -and
|
|
$guest.percpuPercent -lt 5 -and
|
|
$guest.sunreclaimPercent -lt 10
|
|
)
|
|
if (
|
|
$guest -and
|
|
$guest.ok -and
|
|
$guest.servicesReady -and
|
|
$bootChanged -and
|
|
$memoryRecovered
|
|
) {
|
|
break
|
|
}
|
|
if ((Get-Date) -lt $deadline) {
|
|
Start-Sleep -Seconds $VerifyIntervalSeconds
|
|
}
|
|
} while ((Get-Date) -lt $deadline)
|
|
|
|
$vmxRuntime = Get-VmxRuntime
|
|
$verified = [bool](
|
|
$guest -and
|
|
$guest.ok -and
|
|
$guest.servicesReady -and
|
|
$guest.bootId -ne [string]$apply.beforeBootId -and
|
|
$guest.memAvailablePercent -ge 25 -and
|
|
$guest.percpuPercent -lt 5 -and
|
|
$guest.sunreclaimPercent -lt 10 -and
|
|
$guest.source121Sessions15m -eq 0 -and
|
|
$vmxRuntime.exactSystemOwner
|
|
)
|
|
$receipt["applyReceiptPath"] = $ApplyReceiptPath
|
|
$receipt["beforeBootId"] = [string]$apply.beforeBootId
|
|
$receipt["after"] = $guest
|
|
$receipt["vmxRuntime"] = $vmxRuntime
|
|
$receipt["attempts"] = $attempts
|
|
$receipt["verified"] = $verified
|
|
$receipt["terminal"] = if ($verified) {
|
|
"verified_recovered"
|
|
} else {
|
|
"post_verifier_failed_no_hard_fallback"
|
|
}
|
|
$evidencePath = Write-ControlReceipt -Phase "verify" -Receipt $receipt
|
|
$receipt | ConvertTo-Json -Depth 12
|
|
if (-not $verified) { exit 2 }
|
|
exit 0
|