366 lines
17 KiB
PowerShell
366 lines
17 KiB
PowerShell
[CmdletBinding()]
|
|
param(
|
|
[ValidateSet("Check", "Apply")]
|
|
[string]$Mode = "Check",
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern("^agent99-control-loop-maintenance-2[0-9]{7}-[0-9]{6}-[0-9]{3}-[0-9a-f]{8}\.json$")]
|
|
[string]$MaintenanceReceiptRef,
|
|
[ValidateRange(1, 60)]
|
|
[int]$MaxReceiptAgeMinutes = 30,
|
|
[string]$AgentRoot = "C:\Wooo\Agent99"
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
$taskName = "Wooo-Agent99-Control-Loop"
|
|
$runtimeWritePerformed = $false
|
|
|
|
function Test-RecoveryMutexActive {
|
|
$mutex = $null
|
|
$acquired = $false
|
|
try {
|
|
$mutex = [Threading.Mutex]::new($false, "Global\WoooAgent99RecoveryTransportV1")
|
|
try { $acquired = [bool]$mutex.WaitOne(0) } catch [Threading.AbandonedMutexException] { $acquired = $true }
|
|
[bool](-not $acquired)
|
|
} finally {
|
|
if ($acquired -and $mutex) { try { $mutex.ReleaseMutex() } catch {} }
|
|
if ($mutex) { $mutex.Dispose() }
|
|
}
|
|
}
|
|
|
|
function Get-ControlProcesses {
|
|
@(
|
|
Get-CimInstance Win32_Process -Filter "Name='powershell.exe'" -ErrorAction Stop |
|
|
Where-Object {
|
|
([string]$_.CommandLine) -match '(?i)agent99-run\.ps1"?\s+-Mode\s+ControlTick(\s|$)'
|
|
}
|
|
)
|
|
}
|
|
|
|
function Get-RecoverProcesses {
|
|
@(
|
|
Get-CimInstance Win32_Process -Filter "Name='powershell.exe'" -ErrorAction Stop |
|
|
Where-Object {
|
|
([string]$_.CommandLine) -match '(?i)agent99-run\.ps1"?\s+-Mode\s+Recover\s+-ControlledApply(\s|$)'
|
|
}
|
|
)
|
|
}
|
|
|
|
function Write-DurableReceipt {
|
|
param([object]$Value, [string]$Path)
|
|
$temp = "$Path.tmp-$PID-$([guid]::NewGuid().ToString('N'))"
|
|
$stream = $null
|
|
$writer = $null
|
|
try {
|
|
$encoding = New-Object Text.UTF8Encoding($false)
|
|
$stream = [IO.File]::Open($temp, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::None)
|
|
$writer = New-Object IO.StreamWriter($stream, $encoding)
|
|
$writer.Write(($Value | ConvertTo-Json -Depth 8))
|
|
$writer.Flush()
|
|
$stream.Flush($true)
|
|
$writer.Dispose(); $writer = $null
|
|
$stream.Dispose(); $stream = $null
|
|
$beforeHash = (Get-FileHash -LiteralPath $temp -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
Move-Item -LiteralPath $temp -Destination $Path -Force
|
|
$afterHash = (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
if ($beforeHash -ne $afterHash) { throw "receipt_readback_mismatch" }
|
|
} finally {
|
|
if ($writer) { $writer.Dispose() }
|
|
if ($stream) { $stream.Dispose() }
|
|
Remove-Item -LiteralPath $temp -Force -ErrorAction SilentlyContinue
|
|
}
|
|
}
|
|
|
|
function Get-RefreezeReadback {
|
|
$errors = New-Object System.Collections.Generic.List[string]
|
|
$controlCount = $null
|
|
$recoverCount = $null
|
|
$taskEnabled = $null
|
|
$mutexInactive = $false
|
|
$leaseAbsent = $false
|
|
try { $controlCount = @(Get-ControlProcesses).Count } catch { $errors.Add("control_process_readback_failed") }
|
|
try { $recoverCount = @(Get-RecoverProcesses).Count } catch { $errors.Add("recover_process_readback_failed") }
|
|
try {
|
|
$taskReadback = Get-ScheduledTask -TaskName $taskName -TaskPath "\" -ErrorAction Stop
|
|
$taskEnabled = [bool]$taskReadback.Settings.Enabled
|
|
} catch {
|
|
$errors.Add("control_task_readback_failed")
|
|
}
|
|
try {
|
|
$leaseAbsent = [bool](-not (Test-Path -LiteralPath (Join-Path $AgentRoot "state\recovery-transport-lease.json") -ErrorAction Stop))
|
|
} catch {
|
|
$errors.Add("recovery_lease_readback_failed")
|
|
}
|
|
try { $mutexInactive = [bool](-not (Test-RecoveryMutexActive)) } catch { $errors.Add("recovery_mutex_readback_failed") }
|
|
[pscustomobject]@{
|
|
complete = [bool]($errors.Count -eq 0)
|
|
errors = @($errors)
|
|
taskEnabled = $taskEnabled
|
|
controlCount = $controlCount
|
|
recoverCount = $recoverCount
|
|
leaseAbsent = $leaseAbsent
|
|
mutexInactive = $mutexInactive
|
|
verified = [bool](
|
|
$errors.Count -eq 0 -and
|
|
$taskEnabled -is [bool] -and -not $taskEnabled -and
|
|
$controlCount -eq 0 -and
|
|
$recoverCount -eq 0 -and
|
|
$leaseAbsent -and
|
|
$mutexInactive
|
|
)
|
|
}
|
|
}
|
|
|
|
function Wait-RefreezeReadback {
|
|
param([int]$TimeoutSeconds)
|
|
$stopwatch = [Diagnostics.Stopwatch]::StartNew()
|
|
$readback = Get-RefreezeReadback
|
|
while (-not $readback.verified -and $readback.complete -and $stopwatch.Elapsed.TotalSeconds -lt $TimeoutSeconds) {
|
|
Start-Sleep -Milliseconds 250
|
|
$readback = Get-RefreezeReadback
|
|
}
|
|
$readback
|
|
}
|
|
|
|
$maintenanceMutex = [Threading.Mutex]::new($false, "Global\WoooAgent99ControlLoopMaintenanceV1")
|
|
$maintenanceMutexAcquired = $false
|
|
try {
|
|
try {
|
|
$maintenanceMutexAcquired = [bool]$maintenanceMutex.WaitOne(0)
|
|
} catch [Threading.AbandonedMutexException] {
|
|
$maintenanceMutexAcquired = $true
|
|
}
|
|
if (-not $maintenanceMutexAcquired) {
|
|
[pscustomobject]@{
|
|
schemaVersion = "agent99_control_loop_maintenance_restore_v1"
|
|
timestamp = (Get-Date).ToString("o")
|
|
mode = $Mode
|
|
precheckPassed = $false
|
|
blockers = @("maintenance_writer_active")
|
|
terminal = "blocked_single_writer_active"
|
|
runtimeWritePerformed = $false
|
|
secretValueRead = $false
|
|
} | ConvertTo-Json -Depth 5 -Compress
|
|
exit 75
|
|
}
|
|
|
|
$blockers = New-Object System.Collections.Generic.List[string]
|
|
$evidenceDir = Join-Path $AgentRoot "evidence"
|
|
$maintenancePath = Join-Path $evidenceDir $MaintenanceReceiptRef
|
|
$maintenance = $null
|
|
$maintenanceIntent = $null
|
|
$maintenanceSha256 = ""
|
|
$maintenanceAgeMinutes = $null
|
|
if ($env:COMPUTERNAME -ne "WOOO-SUPER") { $blockers.Add("computer_identity_mismatch") }
|
|
if (-not (Test-Path -LiteralPath $maintenancePath -PathType Leaf)) {
|
|
$blockers.Add("maintenance_receipt_missing")
|
|
} else {
|
|
try {
|
|
$maintenance = Get-Content -LiteralPath $maintenancePath -Raw | ConvertFrom-Json
|
|
$maintenanceSha256 = (Get-FileHash -LiteralPath $maintenancePath -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
$maintenanceAgeMinutes = ((Get-Date) - (Get-Item -LiteralPath $maintenancePath).LastWriteTime).TotalMinutes
|
|
} catch {
|
|
$blockers.Add("maintenance_receipt_invalid")
|
|
}
|
|
}
|
|
if ($maintenance) {
|
|
if (
|
|
[string]$maintenance.schemaVersion -ne "agent99_control_loop_maintenance_v2" -or
|
|
[string]$maintenance.mode -ne "Apply" -or
|
|
[string]$maintenance.taskName -ne $taskName -or
|
|
[string]$maintenance.terminal -ne "control_loop_maintenance_frozen" -or
|
|
-not ($maintenance.precheckPassed -is [bool] -and $maintenance.precheckPassed) -or
|
|
-not ($maintenance.controlStopped -is [bool] -and $maintenance.controlStopped) -or
|
|
-not ($maintenance.recoverStopped -is [bool] -and $maintenance.recoverStopped) -or
|
|
-not ($maintenance.leaseMetadataRemoved -is [bool] -and $maintenance.leaseMetadataRemoved) -or
|
|
-not ($maintenance.runtimeWritePerformed -is [bool] -and $maintenance.runtimeWritePerformed) -or
|
|
-not ($maintenance.taskEnabledBefore -is [bool] -and $maintenance.taskEnabledBefore) -or
|
|
-not ($maintenance.taskEnabledAfter -is [bool] -and -not $maintenance.taskEnabledAfter) -or
|
|
-not ($maintenance.vmPowerChangePerformed -is [bool] -and -not $maintenance.vmPowerChangePerformed) -or
|
|
-not ($maintenance.hostRebootPerformed -is [bool] -and -not $maintenance.hostRebootPerformed) -or
|
|
-not ($maintenance.secretValueRead -is [bool] -and -not $maintenance.secretValueRead)
|
|
) {
|
|
$blockers.Add("maintenance_receipt_contract_mismatch")
|
|
}
|
|
if ($null -eq $maintenanceAgeMinutes -or $maintenanceAgeMinutes -lt 0 -or $maintenanceAgeMinutes -gt $MaxReceiptAgeMinutes) {
|
|
$blockers.Add("maintenance_receipt_stale")
|
|
}
|
|
$intentRef = [string]$maintenance.operationIntentRef
|
|
if ($intentRef -notmatch '^agent99-control-loop-maintenance-intent-[0-9]{8}-[0-9]{6}-[0-9]{3}-[0-9a-f]{8}\.json$') {
|
|
$blockers.Add("maintenance_intent_ref_invalid")
|
|
} else {
|
|
$intentPath = Join-Path $evidenceDir $intentRef
|
|
try {
|
|
$maintenanceIntent = Get-Content -LiteralPath $intentPath -Raw -ErrorAction Stop | ConvertFrom-Json
|
|
} catch {
|
|
$blockers.Add("maintenance_intent_invalid")
|
|
}
|
|
if ($maintenanceIntent -and (
|
|
[string]$maintenanceIntent.schemaVersion -ne "agent99_control_loop_maintenance_intent_v1" -or
|
|
[string]$maintenanceIntent.taskName -ne $taskName -or
|
|
(@($maintenanceIntent.plannedActions) -join ",") -ne "disable_control_task,stop_exact_control_tree,stop_exact_recover_tree,remove_exact_recovery_lease" -or
|
|
-not ($maintenanceIntent.taskEnabledBefore -is [bool] -and $maintenanceIntent.taskEnabledBefore) -or
|
|
-not ($maintenanceIntent.vmPowerChangePerformed -is [bool] -and -not $maintenanceIntent.vmPowerChangePerformed) -or
|
|
-not ($maintenanceIntent.secretValueRead -is [bool] -and -not $maintenanceIntent.secretValueRead)
|
|
)) {
|
|
$blockers.Add("maintenance_intent_contract_mismatch")
|
|
}
|
|
}
|
|
}
|
|
|
|
$task = Get-ScheduledTask -TaskName $taskName -TaskPath "\" -ErrorAction SilentlyContinue
|
|
if (-not $task) {
|
|
$blockers.Add("control_task_missing")
|
|
} else {
|
|
$expectedExecute = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe"
|
|
$expectedLauncher = Join-Path $AgentRoot "agent99-run.ps1"
|
|
$expectedArguments = "-NoProfile -ExecutionPolicy Bypass -File `"$expectedLauncher`" -Mode ControlTick"
|
|
$actions = @($task.Actions)
|
|
if ($actions.Count -ne 1) { $blockers.Add("control_task_action_count_mismatch") }
|
|
if ($actions.Count -eq 1) {
|
|
if (-not [string]::Equals([string]$actions[0].Execute, $expectedExecute, [StringComparison]::OrdinalIgnoreCase)) {
|
|
$blockers.Add("control_task_executable_mismatch")
|
|
}
|
|
if (-not [string]::Equals([string]$actions[0].Arguments, $expectedArguments, [StringComparison]::Ordinal)) {
|
|
$blockers.Add("control_task_arguments_mismatch")
|
|
}
|
|
}
|
|
if ([bool]$task.Settings.Enabled) { $blockers.Add("control_task_not_frozen") }
|
|
}
|
|
|
|
$controlBefore = @(Get-ControlProcesses)
|
|
$recoverBefore = @(Get-RecoverProcesses)
|
|
if ($controlBefore.Count -ne 0) { $blockers.Add("control_process_present") }
|
|
if ($recoverBefore.Count -ne 0) { $blockers.Add("recover_process_present") }
|
|
if (Test-Path -LiteralPath (Join-Path $AgentRoot "state\recovery-transport-lease.json")) {
|
|
$blockers.Add("recovery_lease_present")
|
|
}
|
|
if (Test-RecoveryMutexActive) { $blockers.Add("recovery_mutex_active") }
|
|
|
|
$precheckPassed = [bool]($blockers.Count -eq 0)
|
|
$terminal = if ($precheckPassed) { "maintenance_restore_check_verified" } else { "blocked_restore_precheck" }
|
|
$deviceMutationAttempted = $false
|
|
$operationFailureType = ""
|
|
$taskEnabledAfter = if ($task) { [bool]$task.Settings.Enabled } else { $null }
|
|
$controlProcessObserved = $false
|
|
$taskRunObserved = $false
|
|
$rollbackStatus = "not_required"
|
|
$rollbackReadbackErrors = @()
|
|
$operationIntentRef = ""
|
|
|
|
if ($Mode -eq "Apply" -and $precheckPassed) {
|
|
$intentStamp = Get-Date -Format "yyyyMMdd-HHmmss-fff"
|
|
$operationIntentRef = "agent99-control-loop-maintenance-restore-intent-$intentStamp-$([guid]::NewGuid().ToString('N').Substring(0, 8)).json"
|
|
$operationIntentPath = Join-Path $evidenceDir $operationIntentRef
|
|
$operationIntent = [pscustomobject]@{
|
|
schemaVersion = "agent99_control_loop_maintenance_restore_intent_v1"
|
|
timestamp = (Get-Date).ToString("o")
|
|
taskName = $taskName
|
|
maintenanceReceiptRef = $MaintenanceReceiptRef
|
|
maintenanceReceiptSha256 = $maintenanceSha256
|
|
plannedActions = @("enable_exact_control_task", "start_exact_control_task", "verify_control_task_started")
|
|
rollback = "stop and disable the exact control task if postcheck fails"
|
|
vmPowerChangePerformed = $false
|
|
secretValueRead = $false
|
|
}
|
|
Write-DurableReceipt $operationIntent $operationIntentPath
|
|
$runtimeWritePerformed = $true
|
|
|
|
$startedAt = Get-Date
|
|
try {
|
|
$deviceMutationAttempted = $true
|
|
Enable-ScheduledTask -TaskName $taskName -TaskPath "\" | Out-Null
|
|
Start-ScheduledTask -TaskName $taskName -TaskPath "\" -ErrorAction Stop
|
|
$stopwatch = [Diagnostics.Stopwatch]::StartNew()
|
|
while ($stopwatch.Elapsed.TotalSeconds -lt 20) {
|
|
$taskAfter = Get-ScheduledTask -TaskName $taskName -TaskPath "\" -ErrorAction Stop
|
|
$taskInfo = Get-ScheduledTaskInfo -TaskName $taskName -TaskPath "\" -ErrorAction Stop
|
|
$controlsAfter = @(Get-ControlProcesses)
|
|
if ($controlsAfter.Count -gt 0) { $controlProcessObserved = $true }
|
|
$taskRunObserved = [bool]($taskInfo.LastRunTime -ge $startedAt.AddSeconds(-2))
|
|
if ([bool]$taskAfter.Settings.Enabled -and $taskRunObserved -and ($controlProcessObserved -or [string]$taskAfter.State -eq "Running")) {
|
|
break
|
|
}
|
|
Start-Sleep -Milliseconds 250
|
|
}
|
|
$taskAfter = Get-ScheduledTask -TaskName $taskName -TaskPath "\" -ErrorAction Stop
|
|
$taskInfo = Get-ScheduledTaskInfo -TaskName $taskName -TaskPath "\" -ErrorAction Stop
|
|
$taskEnabledAfter = [bool]$taskAfter.Settings.Enabled
|
|
$taskRunObserved = [bool]($taskInfo.LastRunTime -ge $startedAt.AddSeconds(-2))
|
|
$verified = [bool]($taskEnabledAfter -and $taskRunObserved -and ($controlProcessObserved -or [string]$taskAfter.State -eq "Running"))
|
|
if (-not $verified) { throw "control_task_restore_postcheck_failed" }
|
|
$terminal = "control_loop_maintenance_restored"
|
|
} catch {
|
|
$operationFailureType = $_.Exception.GetType().Name
|
|
try { Stop-ScheduledTask -TaskName $taskName -TaskPath "\" -ErrorAction SilentlyContinue } catch {}
|
|
try { Disable-ScheduledTask -TaskName $taskName -TaskPath "\" -ErrorAction Stop | Out-Null } catch {}
|
|
$rollbackReadback = Wait-RefreezeReadback 10
|
|
$taskEnabledAfter = $rollbackReadback.taskEnabled
|
|
$rollbackReadbackErrors = @($rollbackReadback.errors)
|
|
$rollbackVerified = [bool]$rollbackReadback.verified
|
|
$rollbackStatus = if ($rollbackVerified) { "task_refrozen" } else { "manual_recovery_required" }
|
|
$terminal = if ($rollbackVerified) { "maintenance_restore_failed_refrozen" } else { "maintenance_restore_failed_manual_recovery" }
|
|
}
|
|
}
|
|
|
|
$result = [pscustomobject]@{
|
|
schemaVersion = "agent99_control_loop_maintenance_restore_v1"
|
|
timestamp = (Get-Date).ToString("o")
|
|
mode = $Mode
|
|
taskName = $taskName
|
|
maintenanceReceiptRef = $MaintenanceReceiptRef
|
|
maintenanceReceiptSha256 = $maintenanceSha256
|
|
maintenanceAgeMinutes = if ($null -ne $maintenanceAgeMinutes) { [math]::Round($maintenanceAgeMinutes, 2) } else { $null }
|
|
precheckPassed = $precheckPassed
|
|
blockers = @($blockers)
|
|
operationIntentRef = $operationIntentRef
|
|
deviceMutationAttempted = $deviceMutationAttempted
|
|
operationFailureType = $operationFailureType
|
|
taskEnabledAfter = $taskEnabledAfter
|
|
taskRunObserved = $taskRunObserved
|
|
controlProcessObserved = $controlProcessObserved
|
|
rollbackStatus = $rollbackStatus
|
|
rollbackReadbackErrors = @($rollbackReadbackErrors)
|
|
terminal = $terminal
|
|
runtimeWritePerformed = $runtimeWritePerformed
|
|
vmPowerChangePerformed = $false
|
|
hostRebootPerformed = $false
|
|
secretValueRead = $false
|
|
}
|
|
|
|
if ($Mode -eq "Apply") {
|
|
$path = Join-Path $evidenceDir ("agent99-control-loop-maintenance-restore-" + (Get-Date -Format "yyyyMMdd-HHmmss-fff") + "-" + ([guid]::NewGuid().ToString('N').Substring(0, 8)) + ".json")
|
|
try {
|
|
Write-DurableReceipt $result $path
|
|
$result | Add-Member -NotePropertyName receiptRef -NotePropertyValue (Split-Path -Leaf $path) -Force
|
|
} catch {
|
|
$receiptFailureType = $_.Exception.GetType().Name
|
|
if ($deviceMutationAttempted) {
|
|
try { Stop-ScheduledTask -TaskName $taskName -TaskPath "\" -ErrorAction SilentlyContinue } catch {}
|
|
try { Disable-ScheduledTask -TaskName $taskName -TaskPath "\" -ErrorAction Stop | Out-Null } catch {}
|
|
$receiptRollbackReadback = Wait-RefreezeReadback 10
|
|
$receiptRollbackVerified = [bool]$receiptRollbackReadback.verified
|
|
$result.taskEnabledAfter = $receiptRollbackReadback.taskEnabled
|
|
$result.rollbackReadbackErrors = @($receiptRollbackReadback.errors)
|
|
$result.rollbackStatus = if ($receiptRollbackVerified) { "task_refrozen" } else { "manual_recovery_required" }
|
|
$terminal = if ($receiptRollbackVerified) { "maintenance_restore_receipt_failed_refrozen" } else { "maintenance_restore_receipt_failed_manual_recovery" }
|
|
$result.terminal = $terminal
|
|
} else {
|
|
$terminal = "maintenance_restore_receipt_failed_no_mutation"
|
|
$result.terminal = $terminal
|
|
}
|
|
$result.operationFailureType = $receiptFailureType
|
|
$result | Add-Member -NotePropertyName receiptRef -NotePropertyValue "" -Force
|
|
}
|
|
}
|
|
$result | ConvertTo-Json -Depth 8 -Compress
|
|
if (-not $precheckPassed) { exit 64 }
|
|
if ($Mode -eq "Check") { exit 0 }
|
|
if ($terminal -eq "control_loop_maintenance_restored") { exit 0 }
|
|
exit 70
|
|
} finally {
|
|
if ($maintenanceMutexAcquired -and $maintenanceMutex) {
|
|
try { $maintenanceMutex.ReleaseMutex() } catch {}
|
|
}
|
|
if ($maintenanceMutex) { $maintenanceMutex.Dispose() }
|
|
}
|