fix(agent99): unblock bounded maintenance deployment

scripts/reboot-recovery/agent99-stale-queue-quarantine.ps1

@@ -0,0 +1,347 @@
+[CmdletBinding()]
+param(
+  [ValidateSet("Check", "Apply")]
+  [string]$Mode = "Check",
+  [Parameter(Mandatory = $true)]
+  [ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$")]
+  [string]$TraceId,
+  [Parameter(Mandatory = $true)]
+  [ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$")]
+  [string]$RunId,
+  [Parameter(Mandatory = $true)]
+  [ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$")]
+  [string]$WorkItemId,
+  [Parameter(Mandatory = $true)]
+  [ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._-]{0,180}\.json$")]
+  [string]$QueueFileName,
+  [Parameter(Mandatory = $true)]
+  [ValidatePattern("^[a-fA-F0-9]{64}$")]
+  [string]$ExpectedQueueSha256,
+  [Parameter(Mandatory = $true)]
+  [ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$")]
+  [string]$ExpectedCommandId,
+  [Parameter(Mandatory = $true)]
+  [ValidatePattern("^agent99-control-loop-maintenance-2[0-9]{7}-[0-9]{6}\.json$")]
+  [string]$MaintenanceReceiptName,
+  [Parameter(Mandatory = $true)]
+  [ValidatePattern("^[a-fA-F0-9]{64}$")]
+  [string]$ExpectedMaintenanceSha256,
+  [ValidateRange(30, 1440)]
+  [int]$MinimumAgeMinutes = 30,
+  [string]$AgentRoot = "C:\Wooo\Agent99"
+)
+
+$ErrorActionPreference = "Stop"
+$ExpectedQueueSha256 = $ExpectedQueueSha256.ToLowerInvariant()
+$ExpectedMaintenanceSha256 = $ExpectedMaintenanceSha256.ToLowerInvariant()
+$queueRoot = Join-Path $AgentRoot "queue"
+$failedRoot = Join-Path $queueRoot "failed"
+$evidenceRoot = Join-Path $AgentRoot "evidence"
+$sourcePath = Join-Path $queueRoot $QueueFileName
+$destinationName = "stale-maintenance-$QueueFileName"
+$destinationPath = Join-Path $failedRoot $destinationName
+$maintenancePath = Join-Path $evidenceRoot $MaintenanceReceiptName
+$runtimeWritePerformed = $false
+
+function Get-AgentSha256 {
+  param([string]$Path)
+  (Get-FileHash -Algorithm SHA256 -LiteralPath $Path).Hash.ToLowerInvariant()
+}
+
+function Write-AgentDurableJson {
+  param([object]$Value, [string]$Path)
+
+  $temp = "$Path.tmp-$PID-$([guid]::NewGuid().ToString('N'))"
+  $stream = $null
+  $writer = $null
+  try {
+    $encoding = New-Object Text.UTF8Encoding($false)
+    $stream = [IO.File]::Open(
+      $temp,
+      [IO.FileMode]::CreateNew,
+      [IO.FileAccess]::Write,
+      [IO.FileShare]::None
+    )
+    $writer = New-Object IO.StreamWriter($stream, $encoding)
+    $writer.Write(($Value | ConvertTo-Json -Depth 10))
+    $writer.Flush()
+    $stream.Flush($true)
+    $writer.Dispose(); $writer = $null
+    $stream.Dispose(); $stream = $null
+    [IO.File]::Move($temp, $Path)
+    $hash = Get-AgentSha256 $Path
+    if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
+      throw "durable_receipt_readback_missing"
+    }
+    return $hash
+  } finally {
+    if ($writer) { $writer.Dispose() }
+    if ($stream) { $stream.Dispose() }
+    Remove-Item -LiteralPath $temp -Force -ErrorAction SilentlyContinue
+  }
+}
+
+function Test-AgentRecoveryMutexActive {
+  $mutex = $null
+  $acquired = $false
+  try {
+    $mutex = [Threading.Mutex]::new(
+      $false,
+      "Global\WoooAgent99RecoveryTransportV1"
+    )
+    try {
+      $acquired = [bool]$mutex.WaitOne(0)
+    } catch [Threading.AbandonedMutexException] {
+      $acquired = $true
+    }
+    return [bool](-not $acquired)
+  } finally {
+    if ($acquired -and $mutex) { try { $mutex.ReleaseMutex() } catch {} }
+    if ($mutex) { $mutex.Dispose() }
+  }
+}
+
+function Get-AgentActiveControlProcesses {
+  @(
+    Get-CimInstance Win32_Process -Filter "Name='powershell.exe'" -ErrorAction Stop |
+      Where-Object {
+        ([string]$_.CommandLine) -match '(?i)agent99-run\.ps1"?\s+-Mode\s+ControlTick(\s|$)' -or
+        ([string]$_.CommandLine) -match '(?i)agent99-run\.ps1"?\s+-Mode\s+Recover\s+-ControlledApply(\s|$)'
+      } |
+      ForEach-Object {
+        [pscustomobject]@{
+          processId = [int]$_.ProcessId
+          creationDate = ([datetime]$_.CreationDate).ToUniversalTime().ToString("o")
+        }
+      }
+  )
+}
+
+function Get-AgentQuarantineGate {
+  if (-not (Test-Path -LiteralPath $sourcePath -PathType Leaf)) {
+    throw "exact_queue_source_missing"
+  }
+  if (Test-Path -LiteralPath $destinationPath) {
+    throw "quarantine_destination_already_exists"
+  }
+  if (-not (Test-Path -LiteralPath $maintenancePath -PathType Leaf)) {
+    throw "maintenance_receipt_missing"
+  }
+  $queueHash = Get-AgentSha256 $sourcePath
+  $maintenanceHash = Get-AgentSha256 $maintenancePath
+  if ($queueHash -ne $ExpectedQueueSha256) { throw "queue_hash_mismatch" }
+  if ($maintenanceHash -ne $ExpectedMaintenanceSha256) {
+    throw "maintenance_hash_mismatch"
+  }
+
+  $task = Get-ScheduledTask `
+    -TaskName "Wooo-Agent99-Control-Loop" `
+    -TaskPath "\" `
+    -ErrorAction Stop
+  if ($task.Settings.Enabled -or [string]$task.State -ne "Disabled") {
+    throw "control_loop_not_frozen"
+  }
+  $activeProcesses = Get-AgentActiveControlProcesses
+  if ($activeProcesses.Count -ne 0) { throw "active_control_or_recover_process" }
+  $leasePresent = [bool](Test-Path -LiteralPath (
+    Join-Path $AgentRoot "state\recovery-transport-lease.json"
+  ) -PathType Leaf)
+  if ($leasePresent) { throw "recovery_lease_present" }
+  if (Test-AgentRecoveryMutexActive) { throw "recovery_mutex_active" }
+
+  $maintenance = Get-Content -LiteralPath $maintenancePath -Raw | ConvertFrom-Json
+  if (
+    [string]$maintenance.schemaVersion -ne "agent99_control_loop_maintenance_v2" -or
+    [string]$maintenance.mode -ne "Apply" -or
+    [string]$maintenance.taskName -ne "Wooo-Agent99-Control-Loop" -or
+    [string]$maintenance.terminal -ne "control_loop_maintenance_frozen" -or
+    $maintenance.precheckPassed -isnot [bool] -or -not $maintenance.precheckPassed -or
+    $maintenance.controlStopped -isnot [bool] -or -not $maintenance.controlStopped -or
+    $maintenance.recoverStopped -isnot [bool] -or -not $maintenance.recoverStopped -or
+    $maintenance.leaseMetadataRemoved -isnot [bool] -or -not $maintenance.leaseMetadataRemoved -or
+    $maintenance.runtimeWritePerformed -isnot [bool] -or -not $maintenance.runtimeWritePerformed -or
+    $maintenance.vmPowerChangePerformed -isnot [bool] -or $maintenance.vmPowerChangePerformed -or
+    $maintenance.vmwareVmxProcessTerminated -isnot [bool] -or $maintenance.vmwareVmxProcessTerminated -or
+    $maintenance.vmxLockDeleted -isnot [bool] -or $maintenance.vmxLockDeleted -or
+    $maintenance.hostRebootPerformed -isnot [bool] -or $maintenance.hostRebootPerformed -or
+    $maintenance.secretValueRead -isnot [bool] -or $maintenance.secretValueRead
+  ) {
+    throw "maintenance_receipt_contract_invalid"
+  }
+
+  $command = Get-Content -LiteralPath $sourcePath -Raw | ConvertFrom-Json
+  if (
+    [string]$command.id -ne $ExpectedCommandId -or
+    [string]$command.mode -ne "Recover" -or
+    [string]$command.source -ne "agent99-recovery-observer" -or
+    $command.controlledApply -isnot [bool] -or
+    -not $command.controlledApply -or
+    [string]$command.traceId -notmatch '^00-[0-9a-f]{32}-[0-9a-f]{16}-0[01]$' -or
+    [string]$command.workItemId -notmatch '^agent99-auto:[A-Za-z0-9._:@+-]{1,160}$'
+  ) {
+    throw "queue_identity_contract_invalid"
+  }
+  $sourceItem = Get-Item -LiteralPath $sourcePath
+  $ageMinutes = [math]::Round(((Get-Date) - $sourceItem.LastWriteTime).TotalMinutes, 1)
+  if ($ageMinutes -lt $MinimumAgeMinutes) { throw "queue_item_not_stale" }
+
+  [pscustomobject]@{
+    queueHash = $queueHash
+    maintenanceHash = $maintenanceHash
+    ageMinutes = $ageMinutes
+    commandId = [string]$command.id
+    commandMode = [string]$command.mode
+    commandSource = [string]$command.source
+    controlTaskEnabled = [bool]$task.Settings.Enabled
+    controlTaskState = [string]$task.State
+    activeControlProcessCount = [int]$activeProcesses.Count
+    recoveryLeasePresent = $leasePresent
+    recoveryMutexActive = $false
+  }
+}
+
+$gate = Get-AgentQuarantineGate
+$base = [ordered]@{
+  schemaVersion = "agent99_stale_queue_quarantine_v1"
+  timestamp = (Get-Date).ToString("o")
+  mode = $Mode
+  traceId = $TraceId
+  runId = $RunId
+  workItemId = $WorkItemId
+  commandId = $gate.commandId
+  commandMode = $gate.commandMode
+  commandSource = $gate.commandSource
+  sourceFile = $QueueFileName
+  sourceSha256 = $gate.queueHash
+  destinationFile = $destinationName
+  ageMinutes = $gate.ageMinutes
+  minimumAgeMinutes = $MinimumAgeMinutes
+  maintenanceReceipt = $MaintenanceReceiptName
+  maintenanceReceiptSha256 = $gate.maintenanceHash
+  activeControlProcessCount = $gate.activeControlProcessCount
+  recoveryLeasePresent = $gate.recoveryLeasePresent
+  recoveryMutexActive = $gate.recoveryMutexActive
+  commandExecuted = $false
+  deletePerformed = $false
+  otherQueueMutationPerformed = $false
+  rawInstructionStored = $false
+  secretValueRead = $false
+  runtimeWritePerformed = $false
+}
+
+if ($Mode -eq "Check") {
+  $base["terminal"] = "check_passed_apply_allowed"
+  $base | ConvertTo-Json -Depth 8 -Compress
+  exit 0
+}
+
+$writerMutex = [Threading.Mutex]::new(
+  $false,
+  "Global\WoooAgent99StaleQueueMaintenanceV1"
+)
+$writerAcquired = $false
+$intentPath = ""
+$intentHash = ""
+$resultPath = ""
+$moved = $false
+$rollbackAttempted = $false
+$rollbackVerified = $false
+try {
+  try {
+    $writerAcquired = [bool]$writerMutex.WaitOne(0)
+  } catch [Threading.AbandonedMutexException] {
+    $writerAcquired = $true
+  }
+  if (-not $writerAcquired) { throw "queue_maintenance_writer_active" }
+
+  $gate = Get-AgentQuarantineGate
+  $safeRunId = $RunId -replace "[^A-Za-z0-9._-]", "_"
+  $stamp = Get-Date -Format "yyyyMMdd-HHmmss-fff"
+  $intentPath = Join-Path $evidenceRoot (
+    "agent99-stale-queue-quarantine-$stamp-$safeRunId-intent.json"
+  )
+  $resultPath = Join-Path $evidenceRoot (
+    "agent99-stale-queue-quarantine-$stamp-$safeRunId-result.json"
+  )
+  $intent = [ordered]@{} + $base
+  $intent["schemaVersion"] = "agent99_stale_queue_quarantine_intent_v1"
+  $intent["mode"] = "Apply"
+  $intent["plannedAction"] = "atomic_move_exact_queue_item_to_failed"
+  $intent["rollback"] = "atomic_move_back_if_post_move_hash_verifier_fails"
+  $intent["terminal"] = "intent_durable_apply_pending"
+  $intentHash = Write-AgentDurableJson $intent $intentPath
+  $runtimeWritePerformed = $true
+
+  $gate = Get-AgentQuarantineGate
+  [IO.File]::Move($sourcePath, $destinationPath)
+  $moved = $true
+  $sourceAbsent = -not (Test-Path -LiteralPath $sourcePath)
+  $destinationPresent = Test-Path -LiteralPath $destinationPath -PathType Leaf
+  $destinationHash = if ($destinationPresent) {
+    Get-AgentSha256 $destinationPath
+  } else { "" }
+  if (-not ($sourceAbsent -and $destinationPresent -and $destinationHash -eq $ExpectedQueueSha256)) {
+    throw "post_move_verifier_failed"
+  }
+
+  $result = [ordered]@{} + $base
+  $result["schemaVersion"] = "agent99_stale_queue_quarantine_result_v1"
+  $result["mode"] = "Apply"
+  $result["intentFile"] = Split-Path -Leaf $intentPath
+  $result["intentSha256"] = $intentHash
+  $result["sourceAbsent"] = $sourceAbsent
+  $result["destinationPresent"] = $destinationPresent
+  $result["destinationSha256"] = $destinationHash
+  $result["runtimeWritePerformed"] = $true
+  $result["terminal"] = "verified_quarantined"
+  $resultHash = Write-AgentDurableJson $result $resultPath
+  $result["resultFile"] = Split-Path -Leaf $resultPath
+  $result["resultSha256"] = $resultHash
+  $result | ConvertTo-Json -Depth 8 -Compress
+  exit 0
+} catch {
+  $failureType = $_.Exception.GetType().Name
+  if (
+    $moved -and
+    -not (Test-Path -LiteralPath $sourcePath) -and
+    (Test-Path -LiteralPath $destinationPath -PathType Leaf)
+  ) {
+    $rollbackAttempted = $true
+    if ((Get-AgentSha256 $destinationPath) -eq $ExpectedQueueSha256) {
+      [IO.File]::Move($destinationPath, $sourcePath)
+      $rollbackVerified = [bool](
+        (Test-Path -LiteralPath $sourcePath -PathType Leaf) -and
+        -not (Test-Path -LiteralPath $destinationPath) -and
+        (Get-AgentSha256 $sourcePath) -eq $ExpectedQueueSha256
+      )
+    }
+  }
+  $failure = [ordered]@{} + $base
+  $failure["schemaVersion"] = "agent99_stale_queue_quarantine_result_v1"
+  $failure["mode"] = "Apply"
+  $failure["intentFile"] = if ($intentPath) { Split-Path -Leaf $intentPath } else { "" }
+  $failure["intentSha256"] = $intentHash
+  $failure["failureType"] = $failureType
+  $failure["rollbackAttempted"] = $rollbackAttempted
+  $failure["rollbackVerified"] = $rollbackVerified
+  $failure["runtimeWritePerformed"] = $runtimeWritePerformed
+  $failure["terminal"] = if ($rollbackAttempted -and $rollbackVerified) {
+    "failed_rolled_back"
+  } elseif ($rollbackAttempted) {
+    "rollback_unverified"
+  } else {
+    "failed_no_queue_move"
+  }
+  if ($resultPath -and -not (Test-Path -LiteralPath $resultPath)) {
+    try {
+      $failureHash = Write-AgentDurableJson $failure $resultPath
+      $failure["resultFile"] = Split-Path -Leaf $resultPath
+      $failure["resultSha256"] = $failureHash
+    } catch {}
+  }
+  $failure | ConvertTo-Json -Depth 8 -Compress
+  exit 2
+} finally {
+  if ($writerAcquired) { try { $writerMutex.ReleaseMutex() } catch {} }
+  $writerMutex.Dispose()
+}