fix(agent99): quarantine invalid maintenance queue items

2026-07-22 16:17:42 +08:00
parent 521fb7e3ac
commit c1a2615698
2 changed files with 105 additions and 14 deletions
--- a/scripts/reboot-recovery/agent99-stale-queue-quarantine.ps1
+++ b/scripts/reboot-recovery/agent99-stale-queue-quarantine.ps1
@@ -2,6 +2,8 @@
 param(
  [ValidateSet("Check", "Apply")]
  [string]$Mode = "Check",
+  [ValidateSet("StaleAutoRecover", "InvalidUntypedMaintenance")]
+  [string]$QuarantineClass = "StaleAutoRecover",
  [Parameter(Mandatory = $true)]
  [ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$")]
  [string]$TraceId,
@@ -21,12 +23,12 @@ param(
  [ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$")]
  [string]$ExpectedCommandId,
  [Parameter(Mandatory = $true)]
-  [ValidatePattern("^agent99-control-loop-maintenance-2[0-9]{7}-[0-9]{6}\.json$")]
+  [ValidatePattern("^agent99-control-loop-maintenance-2[0-9]{7}-[0-9]{6}(?:-[0-9]{3}-[0-9a-f]{8})?\.json$")]
  [string]$MaintenanceReceiptName,
  [Parameter(Mandatory = $true)]
  [ValidatePattern("^[a-fA-F0-9]{64}$")]
  [string]$ExpectedMaintenanceSha256,
-  [ValidateRange(30, 1440)]
+  [ValidateRange(0, 1440)]
  [int]$MinimumAgeMinutes = 30,
  [string]$AgentRoot = "C:\Wooo\Agent99"
 )
@@ -38,7 +40,11 @@ $queueRoot = Join-Path $AgentRoot "queue"
 $failedRoot = Join-Path $queueRoot "failed"
 $evidenceRoot = Join-Path $AgentRoot "evidence"
 $sourcePath = Join-Path $queueRoot $QueueFileName
-$destinationName = "stale-maintenance-$QueueFileName"
+$destinationName = if ($QuarantineClass -eq "InvalidUntypedMaintenance") {
+  "invalid-untyped-maintenance-$QueueFileName"
+} else {
+  "stale-maintenance-$QueueFileName"
+}
 $destinationPath = Join-Path $failedRoot $destinationName
 $maintenancePath = Join-Path $evidenceRoot $MaintenanceReceiptName
 $runtimeWritePerformed = $false
@@ -145,7 +151,7 @@ function Get-AgentQuarantineGate {
  if ($activeProcesses.Count -ne 0) { throw "active_control_or_recover_process" }
  $leasePresent = [bool](Test-Path -LiteralPath (
    Join-Path $AgentRoot "state\recovery-transport-lease.json"
-  ) -PathType Leaf)
+  ))
  if ($leasePresent) { throw "recovery_lease_present" }
  if (Test-AgentRecoveryMutexActive) { throw "recovery_mutex_active" }

@@ -160,6 +166,9 @@ function Get-AgentQuarantineGate {
    $maintenance.recoverStopped -isnot [bool] -or -not $maintenance.recoverStopped -or
    $maintenance.leaseMetadataRemoved -isnot [bool] -or -not $maintenance.leaseMetadataRemoved -or
    $maintenance.runtimeWritePerformed -isnot [bool] -or -not $maintenance.runtimeWritePerformed -or
+    ($maintenance.PSObject.Properties["taskEnabledAfter"] -and (
+      $maintenance.taskEnabledAfter -isnot [bool] -or $maintenance.taskEnabledAfter
+    )) -or
    $maintenance.vmPowerChangePerformed -isnot [bool] -or $maintenance.vmPowerChangePerformed -or
    $maintenance.vmwareVmxProcessTerminated -isnot [bool] -or $maintenance.vmwareVmxProcessTerminated -or
    $maintenance.vmxLockDeleted -isnot [bool] -or $maintenance.vmxLockDeleted -or
@@ -169,17 +178,66 @@ function Get-AgentQuarantineGate {
    throw "maintenance_receipt_contract_invalid"
  }

+  $maintenanceTimestamp = [datetimeoffset]::MinValue
+  if (-not [datetimeoffset]::TryParse([string]$maintenance.timestamp, [ref]$maintenanceTimestamp)) {
+    throw "maintenance_timestamp_invalid"
+  }
+
  $command = Get-Content -LiteralPath $sourcePath -Raw | ConvertFrom-Json
-  if (
-    [string]$command.id -ne $ExpectedCommandId -or
-    [string]$command.mode -ne "Recover" -or
-    [string]$command.source -ne "agent99-recovery-observer" -or
-    $command.controlledApply -isnot [bool] -or
-    -not $command.controlledApply -or
-    [string]$command.traceId -notmatch '^00-[0-9a-f]{32}-[0-9a-f]{16}-0[01]$' -or
-    [string]$command.workItemId -notmatch '^agent99-auto:[A-Za-z0-9._:@+-]{1,160}$'
-  ) {
-    throw "queue_identity_contract_invalid"
+  if ($QuarantineClass -eq "InvalidUntypedMaintenance") {
+    $requiredPropertySet = @(
+      "approvalId", "automationRunId", "canonicalDigest", "controlledApply",
+      "correlationKey", "createdAt", "dispatchIdentitySchemaVersion",
+      "dispatchRouteId", "executionGeneration", "externalId", "id",
+      "idempotencyKey", "incidentId", "instruction", "lockOwner", "mode",
+      "projectId", "reason", "replyChatId", "requestedBy", "requestPath",
+      "singleFlightKey", "source", "sourceFingerprint", "traceId", "workItemId"
+    ) | Sort-Object
+    $propertySet = @($command.PSObject.Properties.Name | Sort-Object)
+    $propertySetMatches = [bool](
+      $propertySet.Count -eq $requiredPropertySet.Count -and
+      (@($propertySet) -join ",") -eq (@($requiredPropertySet) -join ",")
+    )
+    $emptyTypedFields = @(
+      "approvalId", "automationRunId", "canonicalDigest", "correlationKey",
+      "dispatchIdentitySchemaVersion", "dispatchRouteId", "executionGeneration",
+      "externalId", "idempotencyKey", "incidentId", "lockOwner", "projectId",
+      "singleFlightKey", "sourceFingerprint", "traceId", "workItemId"
+    )
+    $createdAt = [datetimeoffset]::MinValue
+    $createdAtValid = [datetimeoffset]::TryParse([string]$command.createdAt, [ref]$createdAt)
+    if (
+      -not $propertySetMatches -or
+      [string]$command.id -ne $ExpectedCommandId -or
+      [string]$command.id -ne [IO.Path]::GetFileNameWithoutExtension($QueueFileName) -or
+      [string]$command.id -notmatch '^request-2[0-9]{7}-[0-9]{6}-[0-9a-f]{8}$' -or
+      [string]$command.mode -ne "Recover" -or
+      [string]$command.source -ne "agent99-submit-request" -or
+      [string]$command.reason -ne "operator_instruction" -or
+      [string]$command.requestedBy -ne "Administrator" -or
+      $command.controlledApply -isnot [bool] -or
+      $command.controlledApply -or
+      @($emptyTypedFields | Where-Object { [string]$command.$_ -ne "" }).Count -ne 0 -or
+      [string]::IsNullOrWhiteSpace([string]$command.instruction) -or
+      -not $createdAtValid -or
+      $createdAt -lt $maintenanceTimestamp -or
+      $createdAt -gt [datetimeoffset]::Now.AddMinutes(1)
+    ) {
+      throw "invalid_untyped_maintenance_queue_contract_mismatch"
+    }
+  } else {
+    if ($MinimumAgeMinutes -lt 30) { throw "stale_minimum_age_too_low" }
+    if (
+      [string]$command.id -ne $ExpectedCommandId -or
+      [string]$command.mode -ne "Recover" -or
+      [string]$command.source -ne "agent99-recovery-observer" -or
+      $command.controlledApply -isnot [bool] -or
+      -not $command.controlledApply -or
+      [string]$command.traceId -notmatch '^00-[0-9a-f]{32}-[0-9a-f]{16}-0[01]$' -or
+      [string]$command.workItemId -notmatch '^agent99-auto:[A-Za-z0-9._:@+-]{1,160}$'
+    ) {
+      throw "queue_identity_contract_invalid"
+    }
  }
  $sourceItem = Get-Item -LiteralPath $sourcePath
  $ageMinutes = [math]::Round(((Get-Date) - $sourceItem.LastWriteTime).TotalMinutes, 1)
@@ -208,6 +266,7 @@ $base = [ordered]@{
  traceId = $TraceId
  runId = $RunId
  workItemId = $WorkItemId
+  quarantineClass = $QuarantineClass
  commandId = $gate.commandId
  commandMode = $gate.commandMode
  commandSource = $gate.commandSource
@@ -235,6 +294,11 @@ if ($Mode -eq "Check") {
  exit 0
 }

+$maintenanceMutex = [Threading.Mutex]::new(
+  $false,
+  "Global\WoooAgent99ControlLoopMaintenanceV1"
+)
+$maintenanceAcquired = $false
 $writerMutex = [Threading.Mutex]::new(
  $false,
  "Global\WoooAgent99StaleQueueMaintenanceV1"
@@ -247,6 +311,12 @@ $moved = $false
 $rollbackAttempted = $false
 $rollbackVerified = $false
 try {
+  try {
+    $maintenanceAcquired = [bool]$maintenanceMutex.WaitOne(0)
+  } catch [Threading.AbandonedMutexException] {
+    $maintenanceAcquired = $true
+  }
+  if (-not $maintenanceAcquired) { throw "control_loop_maintenance_writer_active" }
  try {
    $writerAcquired = [bool]$writerMutex.WaitOne(0)
  } catch [Threading.AbandonedMutexException] {
@@ -343,5 +413,7 @@ try {
  exit 2
 } finally {
  if ($writerAcquired) { try { $writerMutex.ReleaseMutex() } catch {} }
+  if ($maintenanceAcquired) { try { $maintenanceMutex.ReleaseMutex() } catch {} }
  $writerMutex.Dispose()
+  $maintenanceMutex.Dispose()
 }
--- a/scripts/reboot-recovery/tests/test_agent99_stale_queue_quarantine_contract.py
+++ b/scripts/reboot-recovery/tests/test_agent99_stale_queue_quarantine_contract.py
@@ -12,6 +12,7 @@ def test_stale_queue_quarantine_is_exact_recoverable_and_receipted() -> None:
    assert "ExpectedQueueSha256" in source
    assert "ExpectedMaintenanceSha256" in source
    assert '"Global\\WoooAgent99StaleQueueMaintenanceV1"' in source
+    assert '"Global\\WoooAgent99ControlLoopMaintenanceV1"' in source
    assert '"Global\\WoooAgent99RecoveryTransportV1"' in source
    assert 'Get-ScheduledTask `' in source
    assert '"Wooo-Agent99-Control-Loop"' in source
@@ -35,6 +36,24 @@ def test_stale_queue_quarantine_does_not_execute_or_delete_command() -> None:

    assert "commandExecuted = $false" in source
    assert "deletePerformed = $false" in source
+
+
+def test_invalid_untyped_maintenance_queue_quarantine_is_fail_closed() -> None:
+    source = SCRIPT.read_text(encoding="utf-8")
+
+    assert 'ValidateSet("StaleAutoRecover", "InvalidUntypedMaintenance")' in source
+    assert '"invalid-untyped-maintenance-$QueueFileName"' in source
+    assert 'source -ne "agent99-submit-request"' in source
+    assert 'reason -ne "operator_instruction"' in source
+    assert 'requestedBy -ne "Administrator"' in source
+    assert "$command.controlledApply" in source
+    assert "invalid_untyped_maintenance_queue_contract_mismatch" in source
+    assert "$createdAt -lt $maintenanceTimestamp" in source
+    assert 'quarantineClass = $QuarantineClass' in source
+    assert source.index("$maintenanceMutex.WaitOne(0)") < source.index(
+        "$writerMutex.WaitOne(0)"
+    )
+    assert 'recovery-transport-lease.json"\n  ))' in source
    assert "otherQueueMutationPerformed = $false" in source
    assert "rawInstructionStored = $false" in source
    assert "secretValueRead = $false" in source