awoooi/scripts/reboot-recovery/agent99-stale-queue-quarantine.ps1

[CmdletBinding()]
param(
  [ValidateSet("Check", "Apply")]
  [string]$Mode = "Check",
  [Parameter(Mandatory = $true)]
  [ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$")]
  [string]$TraceId,
  [Parameter(Mandatory = $true)]
  [ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$")]
  [string]$RunId,
  [Parameter(Mandatory = $true)]
  [ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$")]
  [string]$WorkItemId,
  [Parameter(Mandatory = $true)]
  [ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._-]{0,180}\.json$")]
  [string]$QueueFileName,
  [Parameter(Mandatory = $true)]
  [ValidatePattern("^[a-fA-F0-9]{64}$")]
  [string]$ExpectedQueueSha256,
  [Parameter(Mandatory = $true)]
  [ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$")]
  [string]$ExpectedCommandId,
  [Parameter(Mandatory = $true)]
  [ValidatePattern("^agent99-control-loop-maintenance-2[0-9]{7}-[0-9]{6}\.json$")]
  [string]$MaintenanceReceiptName,
  [Parameter(Mandatory = $true)]
  [ValidatePattern("^[a-fA-F0-9]{64}$")]
  [string]$ExpectedMaintenanceSha256,
  [ValidateRange(30, 1440)]
  [int]$MinimumAgeMinutes = 30,
  [string]$AgentRoot = "C:\Wooo\Agent99"
)

$ErrorActionPreference = "Stop"
$ExpectedQueueSha256 = $ExpectedQueueSha256.ToLowerInvariant()
$ExpectedMaintenanceSha256 = $ExpectedMaintenanceSha256.ToLowerInvariant()
$queueRoot = Join-Path $AgentRoot "queue"
$failedRoot = Join-Path $queueRoot "failed"
$evidenceRoot = Join-Path $AgentRoot "evidence"
$sourcePath = Join-Path $queueRoot $QueueFileName
$destinationName = "stale-maintenance-$QueueFileName"
$destinationPath = Join-Path $failedRoot $destinationName
$maintenancePath = Join-Path $evidenceRoot $MaintenanceReceiptName
$runtimeWritePerformed = $false

function Get-AgentSha256 {
  param([string]$Path)
  (Get-FileHash -Algorithm SHA256 -LiteralPath $Path).Hash.ToLowerInvariant()
}

function Write-AgentDurableJson {
  param([object]$Value, [string]$Path)

  $temp = "$Path.tmp-$PID-$([guid]::NewGuid().ToString('N'))"
  $stream = $null
  $writer = $null
  try {
    $encoding = New-Object Text.UTF8Encoding($false)
    $stream = [IO.File]::Open(
      $temp,
      [IO.FileMode]::CreateNew,
      [IO.FileAccess]::Write,
      [IO.FileShare]::None
    )
    $writer = New-Object IO.StreamWriter($stream, $encoding)
    $writer.Write(($Value | ConvertTo-Json -Depth 10))
    $writer.Flush()
    $stream.Flush($true)
    $writer.Dispose(); $writer = $null
    $stream.Dispose(); $stream = $null
    [IO.File]::Move($temp, $Path)
    $hash = Get-AgentSha256 $Path
    if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
      throw "durable_receipt_readback_missing"
    }
    return $hash
  } finally {
    if ($writer) { $writer.Dispose() }
    if ($stream) { $stream.Dispose() }
    Remove-Item -LiteralPath $temp -Force -ErrorAction SilentlyContinue
  }
}

function Test-AgentRecoveryMutexActive {
  $mutex = $null
  $acquired = $false
  try {
    $mutex = [Threading.Mutex]::new(
      $false,
      "Global\WoooAgent99RecoveryTransportV1"
    )
    try {
      $acquired = [bool]$mutex.WaitOne(0)
    } catch [Threading.AbandonedMutexException] {
      $acquired = $true
    }
    return [bool](-not $acquired)
  } finally {
    if ($acquired -and $mutex) { try { $mutex.ReleaseMutex() } catch {} }
    if ($mutex) { $mutex.Dispose() }
  }
}

function Get-AgentActiveControlProcesses {
  @(
    Get-CimInstance Win32_Process -Filter "Name='powershell.exe'" -ErrorAction Stop |
      Where-Object {
        ([string]$_.CommandLine) -match '(?i)agent99-run\.ps1"?\s+-Mode\s+ControlTick(\s|$)' -or
        ([string]$_.CommandLine) -match '(?i)agent99-run\.ps1"?\s+-Mode\s+Recover\s+-ControlledApply(\s|$)'
      } |
      ForEach-Object {
        [pscustomobject]@{
          processId = [int]$_.ProcessId
          creationDate = ([datetime]$_.CreationDate).ToUniversalTime().ToString("o")
        }
      }
  )
}

function Get-AgentQuarantineGate {
  if (-not (Test-Path -LiteralPath $sourcePath -PathType Leaf)) {
    throw "exact_queue_source_missing"
  }
  if (Test-Path -LiteralPath $destinationPath) {
    throw "quarantine_destination_already_exists"
  }
  if (-not (Test-Path -LiteralPath $maintenancePath -PathType Leaf)) {
    throw "maintenance_receipt_missing"
  }
  $queueHash = Get-AgentSha256 $sourcePath
  $maintenanceHash = Get-AgentSha256 $maintenancePath
  if ($queueHash -ne $ExpectedQueueSha256) { throw "queue_hash_mismatch" }
  if ($maintenanceHash -ne $ExpectedMaintenanceSha256) {
    throw "maintenance_hash_mismatch"
  }

  $task = Get-ScheduledTask `
    -TaskName "Wooo-Agent99-Control-Loop" `
    -TaskPath "\" `
    -ErrorAction Stop
  if ($task.Settings.Enabled -or [string]$task.State -ne "Disabled") {
    throw "control_loop_not_frozen"
  }
  $activeProcesses = Get-AgentActiveControlProcesses
  if ($activeProcesses.Count -ne 0) { throw "active_control_or_recover_process" }
  $leasePresent = [bool](Test-Path -LiteralPath (
    Join-Path $AgentRoot "state\recovery-transport-lease.json"
  ) -PathType Leaf)
  if ($leasePresent) { throw "recovery_lease_present" }
  if (Test-AgentRecoveryMutexActive) { throw "recovery_mutex_active" }

  $maintenance = Get-Content -LiteralPath $maintenancePath -Raw | ConvertFrom-Json
  if (
    [string]$maintenance.schemaVersion -ne "agent99_control_loop_maintenance_v2" -or
    [string]$maintenance.mode -ne "Apply" -or
    [string]$maintenance.taskName -ne "Wooo-Agent99-Control-Loop" -or
    [string]$maintenance.terminal -ne "control_loop_maintenance_frozen" -or
    $maintenance.precheckPassed -isnot [bool] -or -not $maintenance.precheckPassed -or
    $maintenance.controlStopped -isnot [bool] -or -not $maintenance.controlStopped -or
    $maintenance.recoverStopped -isnot [bool] -or -not $maintenance.recoverStopped -or
    $maintenance.leaseMetadataRemoved -isnot [bool] -or -not $maintenance.leaseMetadataRemoved -or
    $maintenance.runtimeWritePerformed -isnot [bool] -or -not $maintenance.runtimeWritePerformed -or
    $maintenance.vmPowerChangePerformed -isnot [bool] -or $maintenance.vmPowerChangePerformed -or
    $maintenance.vmwareVmxProcessTerminated -isnot [bool] -or $maintenance.vmwareVmxProcessTerminated -or
    $maintenance.vmxLockDeleted -isnot [bool] -or $maintenance.vmxLockDeleted -or
    $maintenance.hostRebootPerformed -isnot [bool] -or $maintenance.hostRebootPerformed -or
    $maintenance.secretValueRead -isnot [bool] -or $maintenance.secretValueRead
  ) {
    throw "maintenance_receipt_contract_invalid"
  }

  $command = Get-Content -LiteralPath $sourcePath -Raw | ConvertFrom-Json
  if (
    [string]$command.id -ne $ExpectedCommandId -or
    [string]$command.mode -ne "Recover" -or
    [string]$command.source -ne "agent99-recovery-observer" -or
    $command.controlledApply -isnot [bool] -or
    -not $command.controlledApply -or
    [string]$command.traceId -notmatch '^00-[0-9a-f]{32}-[0-9a-f]{16}-0[01]$' -or
    [string]$command.workItemId -notmatch '^agent99-auto:[A-Za-z0-9._:@+-]{1,160}$'
  ) {
    throw "queue_identity_contract_invalid"
  }
  $sourceItem = Get-Item -LiteralPath $sourcePath
  $ageMinutes = [math]::Round(((Get-Date) - $sourceItem.LastWriteTime).TotalMinutes, 1)
  if ($ageMinutes -lt $MinimumAgeMinutes) { throw "queue_item_not_stale" }

  [pscustomobject]@{
    queueHash = $queueHash
    maintenanceHash = $maintenanceHash
    ageMinutes = $ageMinutes
    commandId = [string]$command.id
    commandMode = [string]$command.mode
    commandSource = [string]$command.source
    controlTaskEnabled = [bool]$task.Settings.Enabled
    controlTaskState = [string]$task.State
    activeControlProcessCount = [int]$activeProcesses.Count
    recoveryLeasePresent = $leasePresent
    recoveryMutexActive = $false
  }
}

$gate = Get-AgentQuarantineGate
$base = [ordered]@{
  schemaVersion = "agent99_stale_queue_quarantine_v1"
  timestamp = (Get-Date).ToString("o")
  mode = $Mode
  traceId = $TraceId
  runId = $RunId
  workItemId = $WorkItemId
  commandId = $gate.commandId
  commandMode = $gate.commandMode
  commandSource = $gate.commandSource
  sourceFile = $QueueFileName
  sourceSha256 = $gate.queueHash
  destinationFile = $destinationName
  ageMinutes = $gate.ageMinutes
  minimumAgeMinutes = $MinimumAgeMinutes
  maintenanceReceipt = $MaintenanceReceiptName
  maintenanceReceiptSha256 = $gate.maintenanceHash
  activeControlProcessCount = $gate.activeControlProcessCount
  recoveryLeasePresent = $gate.recoveryLeasePresent
  recoveryMutexActive = $gate.recoveryMutexActive
  commandExecuted = $false
  deletePerformed = $false
  otherQueueMutationPerformed = $false
  rawInstructionStored = $false
  secretValueRead = $false
  runtimeWritePerformed = $false
}

if ($Mode -eq "Check") {
  $base["terminal"] = "check_passed_apply_allowed"
  $base | ConvertTo-Json -Depth 8 -Compress
  exit 0
}

$writerMutex = [Threading.Mutex]::new(
  $false,
  "Global\WoooAgent99StaleQueueMaintenanceV1"
)
$writerAcquired = $false
$intentPath = ""
$intentHash = ""
$resultPath = ""
$moved = $false
$rollbackAttempted = $false
$rollbackVerified = $false
try {
  try {
    $writerAcquired = [bool]$writerMutex.WaitOne(0)
  } catch [Threading.AbandonedMutexException] {
    $writerAcquired = $true
  }
  if (-not $writerAcquired) { throw "queue_maintenance_writer_active" }

  $gate = Get-AgentQuarantineGate
  $safeRunId = $RunId -replace "[^A-Za-z0-9._-]", "_"
  $stamp = Get-Date -Format "yyyyMMdd-HHmmss-fff"
  $intentPath = Join-Path $evidenceRoot (
    "agent99-stale-queue-quarantine-$stamp-$safeRunId-intent.json"
  )
  $resultPath = Join-Path $evidenceRoot (
    "agent99-stale-queue-quarantine-$stamp-$safeRunId-result.json"
  )
  $intent = [ordered]@{} + $base
  $intent["schemaVersion"] = "agent99_stale_queue_quarantine_intent_v1"
  $intent["mode"] = "Apply"
  $intent["plannedAction"] = "atomic_move_exact_queue_item_to_failed"
  $intent["rollback"] = "atomic_move_back_if_post_move_hash_verifier_fails"
  $intent["terminal"] = "intent_durable_apply_pending"
  $intentHash = Write-AgentDurableJson $intent $intentPath
  $runtimeWritePerformed = $true

  $gate = Get-AgentQuarantineGate
  [IO.File]::Move($sourcePath, $destinationPath)
  $moved = $true
  $sourceAbsent = -not (Test-Path -LiteralPath $sourcePath)
  $destinationPresent = Test-Path -LiteralPath $destinationPath -PathType Leaf
  $destinationHash = if ($destinationPresent) {
    Get-AgentSha256 $destinationPath
  } else { "" }
  if (-not ($sourceAbsent -and $destinationPresent -and $destinationHash -eq $ExpectedQueueSha256)) {
    throw "post_move_verifier_failed"
  }

  $result = [ordered]@{} + $base
  $result["schemaVersion"] = "agent99_stale_queue_quarantine_result_v1"
  $result["mode"] = "Apply"
  $result["intentFile"] = Split-Path -Leaf $intentPath
  $result["intentSha256"] = $intentHash
  $result["sourceAbsent"] = $sourceAbsent
  $result["destinationPresent"] = $destinationPresent
  $result["destinationSha256"] = $destinationHash
  $result["runtimeWritePerformed"] = $true
  $result["terminal"] = "verified_quarantined"
  $resultHash = Write-AgentDurableJson $result $resultPath
  $result["resultFile"] = Split-Path -Leaf $resultPath
  $result["resultSha256"] = $resultHash
  $result | ConvertTo-Json -Depth 8 -Compress
  exit 0
} catch {
  $failureType = $_.Exception.GetType().Name
  if (
    $moved -and
    -not (Test-Path -LiteralPath $sourcePath) -and
    (Test-Path -LiteralPath $destinationPath -PathType Leaf)
  ) {
    $rollbackAttempted = $true
    if ((Get-AgentSha256 $destinationPath) -eq $ExpectedQueueSha256) {
      [IO.File]::Move($destinationPath, $sourcePath)
      $rollbackVerified = [bool](
        (Test-Path -LiteralPath $sourcePath -PathType Leaf) -and
        -not (Test-Path -LiteralPath $destinationPath) -and
        (Get-AgentSha256 $sourcePath) -eq $ExpectedQueueSha256
      )
    }
  }
  $failure = [ordered]@{} + $base
  $failure["schemaVersion"] = "agent99_stale_queue_quarantine_result_v1"
  $failure["mode"] = "Apply"
  $failure["intentFile"] = if ($intentPath) { Split-Path -Leaf $intentPath } else { "" }
  $failure["intentSha256"] = $intentHash
  $failure["failureType"] = $failureType
  $failure["rollbackAttempted"] = $rollbackAttempted
  $failure["rollbackVerified"] = $rollbackVerified
  $failure["runtimeWritePerformed"] = $runtimeWritePerformed
  $failure["terminal"] = if ($rollbackAttempted -and $rollbackVerified) {
    "failed_rolled_back"
  } elseif ($rollbackAttempted) {
    "rollback_unverified"
  } else {
    "failed_no_queue_move"
  }
  if ($resultPath -and -not (Test-Path -LiteralPath $resultPath)) {
    try {
      $failureHash = Write-AgentDurableJson $failure $resultPath
      $failure["resultFile"] = Split-Path -Leaf $resultPath
      $failure["resultSha256"] = $failureHash
    } catch {}
  }
  $failure | ConvertTo-Json -Depth 8 -Compress
  exit 2
} finally {
  if ($writerAcquired) { try { $writerMutex.ReleaseMutex() } catch {} }
  $writerMutex.Dispose()
}