diff --git a/scripts/reboot-recovery/agent99-stale-queue-quarantine.ps1 b/scripts/reboot-recovery/agent99-stale-queue-quarantine.ps1 index 317e257a0..c23caf295 100644 --- a/scripts/reboot-recovery/agent99-stale-queue-quarantine.ps1 +++ b/scripts/reboot-recovery/agent99-stale-queue-quarantine.ps1 @@ -2,6 +2,8 @@ param( [ValidateSet("Check", "Apply")] [string]$Mode = "Check", + [ValidateSet("StaleAutoRecover", "InvalidUntypedMaintenance")] + [string]$QuarantineClass = "StaleAutoRecover", [Parameter(Mandatory = $true)] [ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$")] [string]$TraceId, @@ -21,12 +23,12 @@ param( [ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$")] [string]$ExpectedCommandId, [Parameter(Mandatory = $true)] - [ValidatePattern("^agent99-control-loop-maintenance-2[0-9]{7}-[0-9]{6}\.json$")] + [ValidatePattern("^agent99-control-loop-maintenance-2[0-9]{7}-[0-9]{6}(?:-[0-9]{3}-[0-9a-f]{8})?\.json$")] [string]$MaintenanceReceiptName, [Parameter(Mandatory = $true)] [ValidatePattern("^[a-fA-F0-9]{64}$")] [string]$ExpectedMaintenanceSha256, - [ValidateRange(30, 1440)] + [ValidateRange(0, 1440)] [int]$MinimumAgeMinutes = 30, [string]$AgentRoot = "C:\Wooo\Agent99" ) @@ -38,7 +40,11 @@ $queueRoot = Join-Path $AgentRoot "queue" $failedRoot = Join-Path $queueRoot "failed" $evidenceRoot = Join-Path $AgentRoot "evidence" $sourcePath = Join-Path $queueRoot $QueueFileName -$destinationName = "stale-maintenance-$QueueFileName" +$destinationName = if ($QuarantineClass -eq "InvalidUntypedMaintenance") { + "invalid-untyped-maintenance-$QueueFileName" +} else { + "stale-maintenance-$QueueFileName" +} $destinationPath = Join-Path $failedRoot $destinationName $maintenancePath = Join-Path $evidenceRoot $MaintenanceReceiptName $runtimeWritePerformed = $false @@ -145,7 +151,7 @@ function Get-AgentQuarantineGate { if ($activeProcesses.Count -ne 0) { throw "active_control_or_recover_process" } $leasePresent = [bool](Test-Path -LiteralPath ( Join-Path $AgentRoot "state\recovery-transport-lease.json" - ) -PathType Leaf) + )) if ($leasePresent) { throw "recovery_lease_present" } if (Test-AgentRecoveryMutexActive) { throw "recovery_mutex_active" } @@ -160,6 +166,9 @@ function Get-AgentQuarantineGate { $maintenance.recoverStopped -isnot [bool] -or -not $maintenance.recoverStopped -or $maintenance.leaseMetadataRemoved -isnot [bool] -or -not $maintenance.leaseMetadataRemoved -or $maintenance.runtimeWritePerformed -isnot [bool] -or -not $maintenance.runtimeWritePerformed -or + ($maintenance.PSObject.Properties["taskEnabledAfter"] -and ( + $maintenance.taskEnabledAfter -isnot [bool] -or $maintenance.taskEnabledAfter + )) -or $maintenance.vmPowerChangePerformed -isnot [bool] -or $maintenance.vmPowerChangePerformed -or $maintenance.vmwareVmxProcessTerminated -isnot [bool] -or $maintenance.vmwareVmxProcessTerminated -or $maintenance.vmxLockDeleted -isnot [bool] -or $maintenance.vmxLockDeleted -or @@ -169,17 +178,66 @@ function Get-AgentQuarantineGate { throw "maintenance_receipt_contract_invalid" } + $maintenanceTimestamp = [datetimeoffset]::MinValue + if (-not [datetimeoffset]::TryParse([string]$maintenance.timestamp, [ref]$maintenanceTimestamp)) { + throw "maintenance_timestamp_invalid" + } + $command = Get-Content -LiteralPath $sourcePath -Raw | ConvertFrom-Json - if ( - [string]$command.id -ne $ExpectedCommandId -or - [string]$command.mode -ne "Recover" -or - [string]$command.source -ne "agent99-recovery-observer" -or - $command.controlledApply -isnot [bool] -or - -not $command.controlledApply -or - [string]$command.traceId -notmatch '^00-[0-9a-f]{32}-[0-9a-f]{16}-0[01]$' -or - [string]$command.workItemId -notmatch '^agent99-auto:[A-Za-z0-9._:@+-]{1,160}$' - ) { - throw "queue_identity_contract_invalid" + if ($QuarantineClass -eq "InvalidUntypedMaintenance") { + $requiredPropertySet = @( + "approvalId", "automationRunId", "canonicalDigest", "controlledApply", + "correlationKey", "createdAt", "dispatchIdentitySchemaVersion", + "dispatchRouteId", "executionGeneration", "externalId", "id", + "idempotencyKey", "incidentId", "instruction", "lockOwner", "mode", + "projectId", "reason", "replyChatId", "requestedBy", "requestPath", + "singleFlightKey", "source", "sourceFingerprint", "traceId", "workItemId" + ) | Sort-Object + $propertySet = @($command.PSObject.Properties.Name | Sort-Object) + $propertySetMatches = [bool]( + $propertySet.Count -eq $requiredPropertySet.Count -and + (@($propertySet) -join ",") -eq (@($requiredPropertySet) -join ",") + ) + $emptyTypedFields = @( + "approvalId", "automationRunId", "canonicalDigest", "correlationKey", + "dispatchIdentitySchemaVersion", "dispatchRouteId", "executionGeneration", + "externalId", "idempotencyKey", "incidentId", "lockOwner", "projectId", + "singleFlightKey", "sourceFingerprint", "traceId", "workItemId" + ) + $createdAt = [datetimeoffset]::MinValue + $createdAtValid = [datetimeoffset]::TryParse([string]$command.createdAt, [ref]$createdAt) + if ( + -not $propertySetMatches -or + [string]$command.id -ne $ExpectedCommandId -or + [string]$command.id -ne [IO.Path]::GetFileNameWithoutExtension($QueueFileName) -or + [string]$command.id -notmatch '^request-2[0-9]{7}-[0-9]{6}-[0-9a-f]{8}$' -or + [string]$command.mode -ne "Recover" -or + [string]$command.source -ne "agent99-submit-request" -or + [string]$command.reason -ne "operator_instruction" -or + [string]$command.requestedBy -ne "Administrator" -or + $command.controlledApply -isnot [bool] -or + $command.controlledApply -or + @($emptyTypedFields | Where-Object { [string]$command.$_ -ne "" }).Count -ne 0 -or + [string]::IsNullOrWhiteSpace([string]$command.instruction) -or + -not $createdAtValid -or + $createdAt -lt $maintenanceTimestamp -or + $createdAt -gt [datetimeoffset]::Now.AddMinutes(1) + ) { + throw "invalid_untyped_maintenance_queue_contract_mismatch" + } + } else { + if ($MinimumAgeMinutes -lt 30) { throw "stale_minimum_age_too_low" } + if ( + [string]$command.id -ne $ExpectedCommandId -or + [string]$command.mode -ne "Recover" -or + [string]$command.source -ne "agent99-recovery-observer" -or + $command.controlledApply -isnot [bool] -or + -not $command.controlledApply -or + [string]$command.traceId -notmatch '^00-[0-9a-f]{32}-[0-9a-f]{16}-0[01]$' -or + [string]$command.workItemId -notmatch '^agent99-auto:[A-Za-z0-9._:@+-]{1,160}$' + ) { + throw "queue_identity_contract_invalid" + } } $sourceItem = Get-Item -LiteralPath $sourcePath $ageMinutes = [math]::Round(((Get-Date) - $sourceItem.LastWriteTime).TotalMinutes, 1) @@ -208,6 +266,7 @@ $base = [ordered]@{ traceId = $TraceId runId = $RunId workItemId = $WorkItemId + quarantineClass = $QuarantineClass commandId = $gate.commandId commandMode = $gate.commandMode commandSource = $gate.commandSource @@ -235,6 +294,11 @@ if ($Mode -eq "Check") { exit 0 } +$maintenanceMutex = [Threading.Mutex]::new( + $false, + "Global\WoooAgent99ControlLoopMaintenanceV1" +) +$maintenanceAcquired = $false $writerMutex = [Threading.Mutex]::new( $false, "Global\WoooAgent99StaleQueueMaintenanceV1" @@ -247,6 +311,12 @@ $moved = $false $rollbackAttempted = $false $rollbackVerified = $false try { + try { + $maintenanceAcquired = [bool]$maintenanceMutex.WaitOne(0) + } catch [Threading.AbandonedMutexException] { + $maintenanceAcquired = $true + } + if (-not $maintenanceAcquired) { throw "control_loop_maintenance_writer_active" } try { $writerAcquired = [bool]$writerMutex.WaitOne(0) } catch [Threading.AbandonedMutexException] { @@ -343,5 +413,7 @@ try { exit 2 } finally { if ($writerAcquired) { try { $writerMutex.ReleaseMutex() } catch {} } + if ($maintenanceAcquired) { try { $maintenanceMutex.ReleaseMutex() } catch {} } $writerMutex.Dispose() + $maintenanceMutex.Dispose() } diff --git a/scripts/reboot-recovery/tests/test_agent99_stale_queue_quarantine_contract.py b/scripts/reboot-recovery/tests/test_agent99_stale_queue_quarantine_contract.py index 49eb93bb7..c84087643 100644 --- a/scripts/reboot-recovery/tests/test_agent99_stale_queue_quarantine_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_stale_queue_quarantine_contract.py @@ -12,6 +12,7 @@ def test_stale_queue_quarantine_is_exact_recoverable_and_receipted() -> None: assert "ExpectedQueueSha256" in source assert "ExpectedMaintenanceSha256" in source assert '"Global\\WoooAgent99StaleQueueMaintenanceV1"' in source + assert '"Global\\WoooAgent99ControlLoopMaintenanceV1"' in source assert '"Global\\WoooAgent99RecoveryTransportV1"' in source assert 'Get-ScheduledTask `' in source assert '"Wooo-Agent99-Control-Loop"' in source @@ -35,6 +36,24 @@ def test_stale_queue_quarantine_does_not_execute_or_delete_command() -> None: assert "commandExecuted = $false" in source assert "deletePerformed = $false" in source + + +def test_invalid_untyped_maintenance_queue_quarantine_is_fail_closed() -> None: + source = SCRIPT.read_text(encoding="utf-8") + + assert 'ValidateSet("StaleAutoRecover", "InvalidUntypedMaintenance")' in source + assert '"invalid-untyped-maintenance-$QueueFileName"' in source + assert 'source -ne "agent99-submit-request"' in source + assert 'reason -ne "operator_instruction"' in source + assert 'requestedBy -ne "Administrator"' in source + assert "$command.controlledApply" in source + assert "invalid_untyped_maintenance_queue_contract_mismatch" in source + assert "$createdAt -lt $maintenanceTimestamp" in source + assert 'quarantineClass = $QuarantineClass' in source + assert source.index("$maintenanceMutex.WaitOne(0)") < source.index( + "$writerMutex.WaitOne(0)" + ) + assert 'recovery-transport-lease.json"\n ))' in source assert "otherQueueMutationPerformed = $false" in source assert "rawInstructionStored = $false" in source assert "secretValueRead = $false" in source