diff --git a/agent99-contract-check.ps1 b/agent99-contract-check.ps1 index 60f8dfae8..ff2f49b09 100644 --- a/agent99-contract-check.ps1 +++ b/agent99-contract-check.ps1 @@ -203,7 +203,9 @@ $signozMetadataExecutorContract = [bool]( -not $signozMetadataExecutor.Contains('Get-Content -LiteralPath $CredentialFile') ) Add-Check "backup:signoz_metadata_fixed_executor" $signozMetadataExecutorContract "Windows99 dispatches one fixed host110 metadata export contract, consumes only a root-owned credential reference on target, and requires the independent bundle verifier without raw SQLite or secret output" -Add-Check "recovery:auto_trigger" ($control.Contains("Get-AgentBootState") -and $control.Contains("Start-AgentRecoveryFromObservation") -and $control.Contains("recovery_already_queued")) "boot and host-down recovery use a single-flight queue" +Add-Check "recovery:auto_trigger" ($control.Contains("Get-AgentBootState") -and $control.Contains("Start-AgentRecoveryFromObservation") -and $control.Contains("Enter-AgentRecoveryTriggerReservation") -and $control.Contains('"recovery-trigger-reservation.lock"') -and $control.Contains('[System.IO.FileShare]::None') -and $control.Contains('reason = "recovery_trigger_reservation_busy"') -and $control.Contains("recovery_already_queued")) "boot and host-down recovery reserve one atomic single-flight queue decision" +Add-Check "recovery:auto_trigger_controlled_identity" ($control.Contains("New-AgentObservedRecoveryDispatchIdentity") -and $control.Contains('-RouteId "agent99_local_observer"') -and $control.Contains('schema_version = "agent99_controlled_dispatch_identity_v1"') -and $control.Contains('correlationKey = $dispatchIdentity.idempotencyKey') -and $control.Contains('canonicalDigest = $dispatchIdentity.canonicalDigest') -and $control.Contains("Invoke-AgentControlledDispatchIdentitySelfTest")) "observer-created Recover requests carry a stable canonical controlled dispatch identity before queue processing" +Add-Check "recovery:queue_identity_fail_closed" ($control.Contains('dispatchIdentitySchemaVersion = $dispatchIdentity.schemaVersion') -and $control.Contains('Test-AgentControlledDispatchIdentity $dispatchIdentityEnvelope') -and $control.Contains('"controlled_dispatch_identity_invalid"') -and $control.IndexOf('"controlled_dispatch_identity_invalid"') -lt $control.IndexOf('$process = Start-Process -FilePath "powershell.exe" -ArgumentList $args')) "queue consumer rebuilds and validates canonical identity before any controlled executor starts" Add-Check "recovery:durable_boot_event" ($control.Contains("pendingRecovery") -and $control.Contains("host_reboot_recovery_pending") -and $control.Contains("Update-AgentBootRecoveryState") -and $control.Contains("recovered_late")) "boot event remains pending until verified recovery terminal" Add-Check "recovery:terminal_evidence_budget" ($taskRegistration.Contains('$recoverySettings') -and $taskRegistration.Contains('New-TimeSpan -Minutes 20')) "startup recovery outlives the ten-minute SLO long enough to write terminal evidence" Add-Check "recovery:slo" ($control.Contains("recovery_slo_result") -and $control.Contains("withinSlo")) "10-minute recovery evidence is present" diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index 460902a11..0d4618ade 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -14,6 +14,7 @@ param( [switch]$SelfTestSensorGate, [switch]$SelfTestOutcomeContract, [switch]$SelfTestBackupReadbackContract, + [switch]$SelfTestControlledDispatchIdentity, [switch]$SuppressAlerts, [string]$SuppressReason = "", [string]$ConfigPath = "C:\Wooo\Agent99\config\agent99.config.json", @@ -1012,6 +1013,290 @@ function Get-AgentSha256Hex { } } +function Convert-AgentGuidToNetworkBytes { + param([guid]$Guid) + [byte[]]$bytes = $Guid.ToByteArray() + [byte[]]@( + $bytes[3], $bytes[2], $bytes[1], $bytes[0], + $bytes[5], $bytes[4], $bytes[7], $bytes[6], + $bytes[8], $bytes[9], $bytes[10], $bytes[11], + $bytes[12], $bytes[13], $bytes[14], $bytes[15] + ) +} + +function New-AgentUuidV5 { + param( + [guid]$Namespace, + [string]$Name + ) + [byte[]]$namespaceBytes = Convert-AgentGuidToNetworkBytes $Namespace + [byte[]]$nameBytes = [Text.Encoding]::UTF8.GetBytes($Name) + [byte[]]$inputBytes = $namespaceBytes + $nameBytes + $sha1 = [Security.Cryptography.SHA1]::Create() + try { + [byte[]]$hash = $sha1.ComputeHash($inputBytes) + } finally { + $sha1.Dispose() + } + $hash[6] = [byte](($hash[6] -band 0x0f) -bor 0x50) + $hash[8] = [byte](($hash[8] -band 0x3f) -bor 0x80) + $hex = -join @($hash[0..15] | ForEach-Object { $_.ToString("x2") }) + $uuidText = "{0}-{1}-{2}-{3}-{4}" -f $hex.Substring(0, 8), $hex.Substring(8, 4), $hex.Substring(12, 4), $hex.Substring(16, 4), $hex.Substring(20, 12) + [guid]::Parse($uuidText) +} + +function New-AgentCanonicalControlledDispatchIdentity { + param( + [string]$ProjectId, + [string]$IncidentId, + [string]$SourceFingerprint, + [string]$RouteId, + [string]$ExecutionGeneration = "1", + [string]$ApprovalId = "", + [string]$WorkItemId = "" + ) + + $projectId = ([string]$ProjectId).Trim() + if (-not $projectId) { $projectId = "awoooi" } + $incidentId = ([string]$IncidentId).Trim() + $sourceFingerprint = ([string]$SourceFingerprint).Trim() + $routeId = ([string]$RouteId).Trim() + $executionGeneration = ([string]$ExecutionGeneration).Trim() + if (-not $executionGeneration) { $executionGeneration = "1" } + $approvalId = ([string]$ApprovalId).Trim() + if ($approvalId.Length -gt 128) { $approvalId = $approvalId.Substring(0, 128) } + $workItemId = ([string]$WorkItemId).Trim() + if (-not $workItemId) { $workItemId = "agent99-dispatch:$projectId`:$incidentId`:$routeId" } + if ($workItemId.Length -gt 160) { $workItemId = $workItemId.Substring(0, 160) } + if (-not $incidentId) { throw "agent99_dispatch_incident_id_required" } + if (-not $sourceFingerprint) { throw "agent99_dispatch_source_fingerprint_required" } + if (-not $routeId) { throw "agent99_dispatch_route_id_required" } + + $normalized = [ordered]@{ + approval_id = $approvalId + execution_generation = $executionGeneration + incident_id = $incidentId + project_id = $projectId + route_id = $routeId + source_fingerprint = $sourceFingerprint + work_item_id = $workItemId + } + $canonical = $normalized | ConvertTo-Json -Compress -Depth 4 + $digest = Get-AgentSha256Hex $canonical + $singleFlightCanonical = [ordered]@{ + incident_id = $incidentId + project_id = $projectId + route_id = $routeId + source_fingerprint = $sourceFingerprint + } | ConvertTo-Json -Compress -Depth 4 + $singleFlightDigest = Get-AgentSha256Hex $singleFlightCanonical + $runId = (New-AgentUuidV5 -Namespace ([guid]::Parse("6ba7b811-9dad-11d1-80b4-00c04fd430c8")) -Name "agent99-controlled-dispatch:$canonical").ToString() + $traceId = "00-$($digest.Substring(0, 32))-$($digest.Substring(32, 16))-01" + $idempotencyKey = "agent99-controlled:$digest" + $singleFlightKey = "agent99-controlled-flight:$singleFlightDigest" + $public = [ordered]@{ + approval_id = $approvalId + execution_generation = $executionGeneration + idempotency_key = $idempotencyKey + incident_id = $incidentId + lock_owner = $runId + project_id = $projectId + route_id = $routeId + run_id = $runId + schema_version = "agent99_controlled_dispatch_identity_v1" + single_flight_key = $singleFlightKey + source_fingerprint = $sourceFingerprint + trace_id = $traceId + work_item_id = $workItemId + } + + [pscustomobject]@{ + schemaVersion = [string]$public.schema_version + projectId = $projectId + incidentId = $incidentId + sourceFingerprint = $sourceFingerprint + dispatchRouteId = $routeId + executionGeneration = $executionGeneration + approvalId = $approvalId + automationRunId = $runId + traceId = $traceId + workItemId = $workItemId + idempotencyKey = $idempotencyKey + singleFlightKey = $singleFlightKey + lockOwner = $runId + canonicalDigest = Get-AgentSha256Hex ($public | ConvertTo-Json -Compress -Depth 4) + } +} + +function New-AgentObservedRecoveryDispatchIdentity { + param( + [string]$Reason, + [object]$Observation = $null, + [datetimeoffset]$ObservedAt = [datetimeoffset]::Now, + [int]$CooldownMinutes = 30 + ) + + $hosts = @( + @(Get-AgentObjectValue $Observation "hosts" @()) | + ForEach-Object { ([string]$_).Trim() } | + Where-Object { $_ } | + Sort-Object -Unique + ) + $bootId = [string](Get-AgentObjectValue $Observation "bootId" (Get-AgentObjectValue $Observation "boot_id" "")) + $targetHost = [string](Get-AgentObjectValue $Observation "targetHost" (Get-AgentObjectValue $Observation "host" "")) + $windowSeconds = [math]::Max(300, ([math]::Max(1, $CooldownMinutes) * 60)) + $observationWindow = [math]::Floor($ObservedAt.ToUniversalTime().ToUnixTimeSeconds() / $windowSeconds) + $normalizedObservation = [ordered]@{ + boot_id = $bootId.Trim() + hosts = ($hosts -join ",") + reason = ([string]$Reason).Trim() + target_host = $targetHost.Trim() + window = [string]$observationWindow + } + $sourceFingerprint = Get-AgentSha256Hex ($normalizedObservation | ConvertTo-Json -Compress -Depth 4) + $incidentId = "agent99-auto-$($sourceFingerprint.Substring(0, 20))" + New-AgentCanonicalControlledDispatchIdentity ` + -ProjectId "awoooi" ` + -IncidentId $incidentId ` + -SourceFingerprint $sourceFingerprint ` + -RouteId "agent99_local_observer" ` + -ExecutionGeneration "1" ` + -ApprovalId "" ` + -WorkItemId "agent99-auto:$incidentId`:$Reason" +} + +function Test-AgentControlledDispatchIdentity { + param([object]$Identity) + + $required = @( + "schemaVersion", "projectId", "incidentId", "sourceFingerprint", + "dispatchRouteId", "executionGeneration", "automationRunId", "traceId", + "workItemId", "idempotencyKey", "singleFlightKey", "lockOwner", + "canonicalDigest" + ) + foreach ($field in $required) { + if (-not ([string](Get-AgentObjectValue $Identity $field "")).Trim()) { + return [pscustomobject]@{ ok = $false; reason = "identity_field_missing:$field"; expected = $null } + } + } + if (-not $Identity.PSObject.Properties["approvalId"]) { + return [pscustomobject]@{ ok = $false; reason = "identity_field_missing:approvalId"; expected = $null } + } + if ([string]$Identity.schemaVersion -ne "agent99_controlled_dispatch_identity_v1") { + return [pscustomobject]@{ ok = $false; reason = "identity_schema_invalid"; expected = $null } + } + + try { + $expected = New-AgentCanonicalControlledDispatchIdentity ` + -ProjectId ([string]$Identity.projectId) ` + -IncidentId ([string]$Identity.incidentId) ` + -SourceFingerprint ([string]$Identity.sourceFingerprint) ` + -RouteId ([string]$Identity.dispatchRouteId) ` + -ExecutionGeneration ([string]$Identity.executionGeneration) ` + -ApprovalId ([string]$Identity.approvalId) ` + -WorkItemId ([string]$Identity.workItemId) + } catch { + return [pscustomobject]@{ ok = $false; reason = "identity_rebuild_failed"; expected = $null } + } + + foreach ($field in @( + "schemaVersion", "projectId", "incidentId", "sourceFingerprint", + "dispatchRouteId", "executionGeneration", "approvalId", "automationRunId", + "traceId", "workItemId", "idempotencyKey", "singleFlightKey", "lockOwner", + "canonicalDigest" + )) { + if ([string]$Identity.$field -cne [string]$expected.$field) { + return [pscustomobject]@{ ok = $false; reason = "identity_mismatch:$field"; expected = $expected } + } + } + if ([string]$Identity.executionGeneration -notmatch "^[1-3]$") { + return [pscustomobject]@{ ok = $false; reason = "identity_generation_invalid"; expected = $expected } + } + $controlIdentityPattern = "^[A-Za-z0-9][A-Za-z0-9._:@+\-]{0,127}$" + $workItemIdentityPattern = "^[A-Za-z0-9][A-Za-z0-9._:@+\-]{0,159}$" + if ( + [string]$Identity.automationRunId -notmatch $controlIdentityPattern -or + [string]$Identity.traceId -notmatch $controlIdentityPattern -or + [string]$Identity.workItemId -notmatch $workItemIdentityPattern + ) { + return [pscustomobject]@{ ok = $false; reason = "identity_control_value_unsafe"; expected = $expected } + } + [pscustomobject]@{ ok = $true; reason = "identity_canonical_verified"; expected = $expected } +} + +function Enter-AgentRecoveryTriggerReservation { + param([string]$Path) + + try { + $stream = [System.IO.File]::Open( + $Path, + [System.IO.FileMode]::OpenOrCreate, + [System.IO.FileAccess]::ReadWrite, + [System.IO.FileShare]::None + ) + return [pscustomobject]@{ + acquired = $true + reason = "recovery_trigger_reservation_acquired" + stream = $stream + } + } catch [System.IO.IOException] { + return [pscustomobject]@{ + acquired = $false + reason = "recovery_trigger_reservation_busy" + stream = $null + } + } +} + +function Invoke-AgentControlledDispatchIdentitySelfTest { + $observedAt = [datetimeoffset]::Parse("2026-07-18T07:30:00Z") + $observation = [pscustomobject]@{ hosts = @("192.168.0.111"); count = 1 } + $first = New-AgentObservedRecoveryDispatchIdentity -Reason "host_unreachable_detected" -Observation $observation -ObservedAt $observedAt -CooldownMinutes 30 + $second = New-AgentObservedRecoveryDispatchIdentity -Reason "host_unreachable_detected" -Observation $observation -ObservedAt $observedAt.AddMinutes(1) -CooldownMinutes 30 + $valid = Test-AgentControlledDispatchIdentity $first + $tampered = $first | Select-Object * + $tampered.canonicalDigest = "0" * 64 + $tamperedResult = Test-AgentControlledDispatchIdentity $tampered + $tempRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("agent99-controlled-dispatch-selftest-" + [guid]::NewGuid().ToString("N")) + $firstReservation = $null + $secondReservation = $null + try { + New-Item -ItemType Directory -Force -Path $tempRoot | Out-Null + $reservationPath = Join-Path $tempRoot "recovery-trigger-reservation.lock" + $firstReservation = Enter-AgentRecoveryTriggerReservation $reservationPath + $secondReservation = Enter-AgentRecoveryTriggerReservation $reservationPath + $concurrentTriggerSuppressed = [bool]( + $firstReservation.acquired -and + -not $secondReservation.acquired -and + [string]$secondReservation.reason -eq "recovery_trigger_reservation_busy" + ) + } finally { + if ($secondReservation -and $secondReservation.stream) { $secondReservation.stream.Dispose() } + if ($firstReservation -and $firstReservation.stream) { $firstReservation.stream.Dispose() } + Remove-Item -LiteralPath $tempRoot -Recurse -Force -ErrorAction SilentlyContinue + } + $passed = [bool]( + $valid.ok -and + $first.singleFlightKey -eq $second.singleFlightKey -and + $first.automationRunId -eq $second.automationRunId -and + -not $tamperedResult.ok -and + $concurrentTriggerSuppressed + ) + [pscustomobject]@{ + schemaVersion = "agent99_controlled_dispatch_identity_selftest_v1" + ok = $passed + stableSingleFlight = [bool]($first.singleFlightKey -eq $second.singleFlightKey) + stableRunId = [bool]($first.automationRunId -eq $second.automationRunId) + validIdentityAccepted = [bool]$valid.ok + tamperedIdentityRejected = [bool](-not $tamperedResult.ok) + tamperReason = [string]$tamperedResult.reason + concurrentTriggerSuppressed = $concurrentTriggerSuppressed + runtimeWritePerformed = $false + temporaryTestWritePerformed = $true + secretValueLogged = $false + } +} + function Get-AgentIncidentCardModel { param( [string]$Severity, @@ -2772,6 +3057,19 @@ function Start-AgentRecoveryFromObservation { $stateDir = Join-Path $Config.agentRoot "state" $statePath = Join-Path $stateDir "recovery-trigger-state.json" New-Item -ItemType Directory -Force -Path $queueDir, $stateDir | Out-Null + $reservationPath = Join-Path $stateDir "recovery-trigger-reservation.lock" + $reservation = Enter-AgentRecoveryTriggerReservation $reservationPath + if (-not $reservation.acquired) { + return [pscustomobject]@{ + ok = $true + queued = $false + suppressed = $true + reason = [string]$reservation.reason + trigger = $Reason + } + } + + try { $runningQueueHealth = Get-AgentRunningQueueHealth $queueDir $staleRunningNames = @($runningQueueHealth.claims | Where-Object { $_.stale } | ForEach-Object { [string]$_.fileName }) @@ -2800,10 +3098,16 @@ function Start-AgentRecoveryFromObservation { } } - $id = "auto-recover-" + (Get-Date -Format "yyyyMMdd-HHmmss") + "-" + ([guid]::NewGuid().ToString("N").Substring(0, 8)) + $requestedAt = [datetimeoffset]::Now + $id = "auto-recover-" + $requestedAt.ToString("yyyyMMdd-HHmmss") + "-" + ([guid]::NewGuid().ToString("N").Substring(0, 8)) + $dispatchIdentity = New-AgentObservedRecoveryDispatchIdentity ` + -Reason $Reason ` + -Observation $Observation ` + -ObservedAt $requestedAt ` + -CooldownMinutes $autoRecovery.cooldownMinutes $queuePath = Join-Path $queueDir "$id.json" $temporaryPath = "$queuePath.tmp" - $createdAt = Get-Date -Format o + $createdAt = $requestedAt.ToString("o") $request = [pscustomobject]@{ id = $id mode = "Recover" @@ -2811,24 +3115,56 @@ function Start-AgentRecoveryFromObservation { requestedBy = "Agent99" source = "agent99-recovery-observer" externalId = $Reason + correlationKey = $dispatchIdentity.idempotencyKey reason = $Reason instruction = "Agent99 detected $Reason; run the allowlisted VM, host, Harbor, K3s, public-route, and freshness recovery chain." observation = $Observation + dispatchIdentitySchemaVersion = $dispatchIdentity.schemaVersion + automationRunId = $dispatchIdentity.automationRunId + traceId = $dispatchIdentity.traceId + workItemId = $dispatchIdentity.workItemId + idempotencyKey = $dispatchIdentity.idempotencyKey + executionGeneration = $dispatchIdentity.executionGeneration + incidentId = $dispatchIdentity.incidentId + approvalId = $dispatchIdentity.approvalId + projectId = $dispatchIdentity.projectId + sourceFingerprint = $dispatchIdentity.sourceFingerprint + dispatchRouteId = $dispatchIdentity.dispatchRouteId + singleFlightKey = $dispatchIdentity.singleFlightKey + lockOwner = $dispatchIdentity.lockOwner + canonicalDigest = $dispatchIdentity.canonicalDigest createdAt = $createdAt } $request | ConvertTo-Json -Depth 8 | Set-Content -Path $temporaryPath -Encoding UTF8 Move-Item -Force $temporaryPath $queuePath + $stateTemporaryPath = "$statePath.tmp" [pscustomobject]@{ schemaVersion = "agent99_recovery_trigger_state_v1" lastRequestedAt = $createdAt reason = $Reason queueId = $id queuePath = $queuePath - } | ConvertTo-Json -Depth 6 | Set-Content -Path $statePath -Encoding UTF8 + } | ConvertTo-Json -Depth 6 | Set-Content -Path $stateTemporaryPath -Encoding UTF8 + Move-Item -Force $stateTemporaryPath $statePath - $result = [pscustomobject]@{ ok = $true; queued = $true; suppressed = $false; reason = $Reason; id = $id; queuePath = $queuePath } + $result = [pscustomobject]@{ + ok = $true + queued = $true + suppressed = $false + reason = $Reason + id = $id + queuePath = $queuePath + automationRunId = $dispatchIdentity.automationRunId + traceId = $dispatchIdentity.traceId + workItemId = $dispatchIdentity.workItemId + idempotencyKey = $dispatchIdentity.idempotencyKey + canonicalDigest = $dispatchIdentity.canonicalDigest + } Record-AgentEvent "recovery_auto_triggered" "warning" "偵測到 $Reason；已排入單一受控 Recover 佇列。" $result -Alert $result + } finally { + if ($reservation.stream) { $reservation.stream.Dispose() } + } } function Get-AgentRecoverySloConfig { @@ -6607,6 +6943,7 @@ function Invoke-AgentQueuedCommands { $sourceEventResolutionPolicy = if ($command.PSObject.Properties["sourceEventResolutionPolicy"]) { [string]$command.sourceEventResolutionPolicy } else { "external_source_receipt_required" } $sourceEventResolved = if ($command.PSObject.Properties["sourceEventResolved"]) { [object](Convert-AgentBool $command.sourceEventResolved) } else { $null } $sourceEventEvidence = if ($command.PSObject.Properties["sourceEventEvidence"]) { [string]$command.sourceEventEvidence } else { "" } + $dispatchIdentitySchemaVersion = if ($command.PSObject.Properties["dispatchIdentitySchemaVersion"]) { [string]$command.dispatchIdentitySchemaVersion } else { "" } $automationRunId = if ($command.PSObject.Properties["automationRunId"]) { [string]$command.automationRunId } else { "" } $traceId = if ($command.PSObject.Properties["traceId"]) { [string]$command.traceId } else { "" } $workItemId = if ($command.PSObject.Properties["workItemId"]) { [string]$command.workItemId } else { "" } @@ -6649,33 +6986,37 @@ function Invoke-AgentQueuedCommands { $results += $result continue } - $hasDispatchIdentity = [bool]($automationRunId -or $traceId -or $workItemId -or $idempotencyKey -or $incidentId -or $projectId -or $sourceFingerprint -or $dispatchRouteId -or $singleFlightKey -or $lockOwner -or $canonicalDigest) - $dispatchIdentityIncomplete = [bool](-not $automationRunId -or -not $traceId -or -not $workItemId -or -not $idempotencyKey -or -not $incidentId -or -not $projectId -or -not $sourceFingerprint -or -not $dispatchRouteId -or -not $singleFlightKey -or -not $lockOwner -or -not $canonicalDigest -or -not $approvalIdentityPresent -or -not $executionGeneration) + $hasDispatchIdentity = [bool]($dispatchIdentitySchemaVersion -or $automationRunId -or $traceId -or $workItemId -or $idempotencyKey -or $incidentId -or $projectId -or $sourceFingerprint -or $dispatchRouteId -or $singleFlightKey -or $lockOwner -or $canonicalDigest) + $dispatchIdentityIncomplete = [bool](-not $dispatchIdentitySchemaVersion -or -not $automationRunId -or -not $traceId -or -not $workItemId -or -not $idempotencyKey -or -not $incidentId -or -not $projectId -or -not $sourceFingerprint -or -not $dispatchRouteId -or -not $singleFlightKey -or -not $lockOwner -or -not $canonicalDigest -or -not $approvalIdentityPresent -or -not $executionGeneration) $controlIdentityPattern = "^[A-Za-z0-9][A-Za-z0-9._:@+\-]{0,127}$" + $workItemIdentityPattern = "^[A-Za-z0-9][A-Za-z0-9._:@+\-]{0,159}$" $dispatchControlIdentitySafe = [bool]( $automationRunId -match $controlIdentityPattern -and $traceId -match $controlIdentityPattern -and - $workItemId -match $controlIdentityPattern + $workItemId -match $workItemIdentityPattern ) - $canonicalIdentity = [ordered]@{ - approval_id = $approvalId - execution_generation = $executionGeneration - idempotency_key = $idempotencyKey - incident_id = $incidentId - lock_owner = $lockOwner - project_id = $projectId - route_id = $dispatchRouteId - run_id = $automationRunId - schema_version = "agent99_controlled_dispatch_identity_v1" - single_flight_key = $singleFlightKey - source_fingerprint = $sourceFingerprint - trace_id = $traceId - work_item_id = $workItemId + $dispatchIdentityEnvelope = [pscustomobject]@{ + schemaVersion = $dispatchIdentitySchemaVersion + projectId = $projectId + incidentId = $incidentId + sourceFingerprint = $sourceFingerprint + dispatchRouteId = $dispatchRouteId + executionGeneration = $executionGeneration + approvalId = $approvalId + automationRunId = $automationRunId + traceId = $traceId + workItemId = $workItemId + idempotencyKey = $idempotencyKey + singleFlightKey = $singleFlightKey + lockOwner = $lockOwner + canonicalDigest = $canonicalDigest } - $canonicalDigestMatches = [bool]( - $canonicalDigest -match "^[0-9a-f]{64}$" -and - (Get-AgentSha256Hex ($canonicalIdentity | ConvertTo-Json -Compress -Depth 4)) -eq $canonicalDigest - ) + $dispatchIdentityValidation = if ($hasDispatchIdentity -and -not $dispatchIdentityIncomplete) { + Test-AgentControlledDispatchIdentity $dispatchIdentityEnvelope + } else { + [pscustomobject]@{ ok = $false; reason = "identity_incomplete"; expected = $null } + } + $canonicalDigestMatches = [bool]$dispatchIdentityValidation.ok $expectedReconcileEvidenceId = if ($canonicalDigest -match "^[0-9a-f]{64}$") { "same-run-status-$automationRunId-$($canonicalDigest.Substring(0, 12))" } else { @@ -6741,6 +7082,29 @@ function Invoke-AgentQueuedCommands { $results += $result continue } + if ($hasDispatchIdentity -and -not $dispatchIdentityValidation.ok) { + $target = Join-Path $failedDir ("failed-" + $file.Name) + Move-Item -Force $file.FullName $target + $identityRejectionReason = if ([string]$dispatchIdentityValidation.reason -eq "identity_control_value_unsafe") { + "controlled_dispatch_identity_unsafe" + } else { + "controlled_dispatch_identity_invalid" + } + $result = [pscustomobject]@{ + id = $id + mode = $modeName + source = $commandSource + ok = $false + reason = $identityRejectionReason + identityValidation = [string]$dispatchIdentityValidation.reason + suppressTelegram = $suppressTelegram + runtimeWritePerformed = $false + file = $target + } + Record-AgentEvent "operator_command_rejected" "warning" "id=$id mode=$modeName reason=$identityRejectionReason" $result -Alert + $results += $result + continue + } if ($reconcileOnly -and -not $sameRunReconcileContractExact) { $target = Join-Path $failedDir ("failed-" + $file.Name) Move-Item -Force $file.FullName $target @@ -6757,24 +7121,6 @@ function Invoke-AgentQueuedCommands { $results += $result continue } - if ($hasDispatchIdentity -and -not $dispatchControlIdentitySafe) { - $target = Join-Path $failedDir ("failed-" + $file.Name) - Move-Item -Force $file.FullName $target - $result = [pscustomobject]@{ - id = $id - mode = $modeName - source = $commandSource - ok = $false - reason = "controlled_dispatch_identity_unsafe" - suppressTelegram = $suppressTelegram - runtimeWritePerformed = $false - file = $target - } - Record-AgentEvent "operator_command_rejected" "warning" "id=$id mode=$modeName reason=controlled_dispatch_identity_unsafe" $result -Alert - $results += $result - continue - } - $runningPath = Join-Path $queueDir ("running-" + $file.Name) Move-Item -Force $file.FullName $runningPath $startTime = Get-Date @@ -8692,6 +9038,13 @@ if ($SelfTestBackupReadbackContract) { exit 0 } +if ($SelfTestControlledDispatchIdentity) { + $controlledDispatchIdentitySelfTest = Invoke-AgentControlledDispatchIdentitySelfTest + $controlledDispatchIdentitySelfTest | ConvertTo-Json -Depth 10 + if (-not $controlledDispatchIdentitySelfTest.ok) { exit 1 } + exit 0 +} + $recoveryRunId = if ($Mode -eq "Recover") { if ($AutomationRunId) { $AutomationRunId } else { "agent99-recovery-" + (Get-Date -Format "yyyyMMdd-HHmmss") + "-" + ([guid]::NewGuid().ToString("N").Substring(0, 8)) } } else { diff --git a/agent99-sre-alert-inbox.ps1 b/agent99-sre-alert-inbox.ps1 index 79baa43c1..60fe4a714 100644 --- a/agent99-sre-alert-inbox.ps1 +++ b/agent99-sre-alert-inbox.ps1 @@ -1188,6 +1188,7 @@ foreach ($file in $files) { $awoooiMetadata = Get-AgentField $alert "awoooi" $null $dispatchIdentity = Get-AgentField $awoooiMetadata "agent99DispatchIdentity" $null $typedTargetRoute = Get-AgentField $routing "typedTargetRoute" $null + $dispatchIdentitySchemaVersion = [string](Get-AgentField $dispatchIdentity "schema_version" "") $automationRunId = [string](Get-AgentField $dispatchIdentity "run_id" (Get-AgentField $routing "runId" "")) $traceId = [string](Get-AgentField $dispatchIdentity "trace_id" (Get-AgentField $routing "traceId" "")) $workItemId = [string](Get-AgentField $dispatchIdentity "work_item_id" (Get-AgentField $routing "workItemId" "")) @@ -1209,8 +1210,8 @@ foreach ($file in $files) { verifier = [string](Get-AgentField $typedTargetRoute "verifier" "") routeId = $dispatchRouteId } - $hasDispatchIdentity = [bool]($automationRunId -or $traceId -or $workItemId -or $idempotencyKey -or $incidentId -or $projectId -or $sourceFingerprint -or $dispatchRouteId -or $singleFlightKey) - $dispatchIdentityIncomplete = [bool](-not $automationRunId -or -not $traceId -or -not $workItemId -or -not $idempotencyKey -or -not $incidentId -or -not $projectId -or -not $sourceFingerprint -or -not $dispatchRouteId -or -not $singleFlightKey -or -not $lockOwner -or -not $canonicalDigest -or -not $executionGeneration) + $hasDispatchIdentity = [bool]($dispatchIdentitySchemaVersion -or $automationRunId -or $traceId -or $workItemId -or $idempotencyKey -or $incidentId -or $projectId -or $sourceFingerprint -or $dispatchRouteId -or $singleFlightKey) + $dispatchIdentityIncomplete = [bool](-not $dispatchIdentitySchemaVersion -or -not $automationRunId -or -not $traceId -or -not $workItemId -or -not $idempotencyKey -or -not $incidentId -or -not $projectId -or -not $sourceFingerprint -or -not $dispatchRouteId -or -not $singleFlightKey -or -not $lockOwner -or -not $canonicalDigest -or -not $executionGeneration) if (($controlled -and $modeName -in $remediationModes -and -not $hasDispatchIdentity) -or ($hasDispatchIdentity -and $dispatchIdentityIncomplete)) { $failedPath = Join-Path $FailedDir ("failed-" + (Convert-AgentSafeFileToken $alertId) + "-" + $file.Name) Move-Item -Force $runningPath $failedPath @@ -1252,6 +1253,7 @@ foreach ($file in $files) { suppressTelegram = $suppressTelegram sourceEventResolutionPolicy = $sourceEventResolutionPolicy sourceEventResolutionRequired = $sourceEventResolutionRequired + dispatchIdentitySchemaVersion = $dispatchIdentitySchemaVersion automationRunId = $automationRunId traceId = $traceId workItemId = $workItemId @@ -1294,6 +1296,7 @@ foreach ($file in $files) { suppressTelegram = $suppressTelegram sourceEventResolutionPolicy = $sourceEventResolutionPolicy sourceEventResolutionRequired = $sourceEventResolutionRequired + dispatchIdentitySchemaVersion = $dispatchIdentitySchemaVersion automationRunId = $automationRunId traceId = $traceId workItemId = $workItemId diff --git a/agent99-sre-alert-relay.ps1 b/agent99-sre-alert-relay.ps1 index e0564c87b..d8145c27e 100644 --- a/agent99-sre-alert-relay.ps1 +++ b/agent99-sre-alert-relay.ps1 @@ -946,6 +946,7 @@ function Start-AgentSameRunStatusReconcile { sourceEventResolutionPolicy = "external_source_receipt_required" sourceEventResolved = $true sourceEventEvidence = $sourceEvidence + dispatchIdentitySchemaVersion = [string](Get-AgentField $identity "schema_version" "") automationRunId = $runId traceId = [string](Get-AgentField $identity "trace_id" "") workItemId = [string](Get-AgentField $identity "work_item_id" "") diff --git a/apps/api/tests/test_agent99_alert_dispatch_order_contract.py b/apps/api/tests/test_agent99_alert_dispatch_order_contract.py index 1df3ae716..2be226b41 100644 --- a/apps/api/tests/test_agent99_alert_dispatch_order_contract.py +++ b/apps/api/tests/test_agent99_alert_dispatch_order_contract.py @@ -84,6 +84,7 @@ def test_agent99_runtime_has_local_correlation_dedupe_and_lock() -> None: assert "correlationKey = $correlationKey" in source assert "cold[-_ ]start|reboot[-_ ]auto[-_ ]recovery" in source for field in ( + "dispatchIdentitySchemaVersion", "automationRunId", "traceId", "workItemId", @@ -96,6 +97,8 @@ def test_agent99_runtime_has_local_correlation_dedupe_and_lock() -> None: control_plane = CONTROL_PLANE.read_text(encoding="utf-8") assert "controlled_dispatch_identity_missing" in control_plane + assert "controlled_dispatch_identity_invalid" in control_plane + assert "Test-AgentControlledDispatchIdentity $dispatchIdentityEnvelope" in control_plane assert "-Name automationRunId" in control_plane assert "-Name traceId" in control_plane assert "-Name workItemId" in control_plane @@ -110,6 +113,17 @@ def test_agent99_runtime_has_local_correlation_dedupe_and_lock() -> None: assert evidence_ref in control_plane +def test_agent99_queue_producers_project_canonical_identity_schema() -> None: + inbox = SRE_INBOX.read_text(encoding="utf-8") + relay = (ROOT / "agent99-sre-alert-relay.ps1").read_text(encoding="utf-8") + submit = (ROOT / "agent99-submit-request.ps1").read_text(encoding="utf-8") + + assert '$dispatchIdentitySchemaVersion = [string](Get-AgentField $dispatchIdentity "schema_version" "")' in inbox + assert inbox.count("dispatchIdentitySchemaVersion = $dispatchIdentitySchemaVersion") >= 2 + assert 'dispatchIdentitySchemaVersion = [string](Get-AgentField $identity "schema_version" "")' in relay + assert submit.count("dispatchIdentitySchemaVersion = $dispatchIdentity.schemaVersion") >= 2 + + def test_agent99_mutating_inbox_requires_complete_typed_dispatch_envelope() -> None: source = SRE_INBOX.read_text(encoding="utf-8") guard = source[ diff --git a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py index b4519b930..1cb04b65f 100644 --- a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py +++ b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py @@ -7,6 +7,7 @@ ROOT = Path(__file__).resolve().parents[3] CONTROL = ROOT / "agent99-control-plane.ps1" DEPLOY = ROOT / "agent99-deploy.ps1" BOOTSTRAP = ROOT / "agent99-bootstrap.ps1" +CONTRACT_CHECK = ROOT / "agent99-contract-check.ps1" SRE_INBOX = ROOT / "agent99-sre-alert-inbox.ps1" REMOTE_ATOMIC_RECEIVER = ( ROOT / "scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1" @@ -30,14 +31,94 @@ def test_agent99_uses_dedicated_identity_and_preferred_jump_route() -> None: def test_agent99_auto_recovery_is_single_flight_and_slo_measured() -> None: source = CONTROL.read_text(encoding="utf-8") + contract = CONTRACT_CHECK.read_text(encoding="utf-8") trigger = source[source.index("function Start-AgentRecoveryFromObservation") :] trigger = trigger[: trigger.index("function Get-AgentRecoverySloConfig")] + identity = source[source.index("function New-AgentCanonicalControlledDispatchIdentity") :] + identity = identity[: identity.index("function Get-AgentIncidentCardModel")] + consumer = source[source.index("function Invoke-AgentQueuedCommands") :] + consumer = consumer[: consumer.index("function Invoke-AgentControlTick")] assert 'reason = "recovery_already_queued"' in trigger assert 'reason = "recovery_trigger_cooldown"' in trigger + assert '"recovery-trigger-reservation.lock"' in trigger + assert "Enter-AgentRecoveryTriggerReservation $reservationPath" in trigger + assert 'reason = [string]$reservation.reason' in trigger assert 'source = "agent99-recovery-observer"' in trigger assert 'mode = "Recover"' in trigger assert "Start-Process" not in trigger + assert "New-AgentObservedRecoveryDispatchIdentity" in trigger + assert "-RequestId" not in trigger + assert "-Observation $Observation" in trigger + assert "-CooldownMinutes $autoRecovery.cooldownMinutes" in trigger + for field in ( + "dispatchIdentitySchemaVersion", + "automationRunId", + "traceId", + "workItemId", + "idempotencyKey", + "executionGeneration", + "incidentId", + "approvalId", + "projectId", + "sourceFingerprint", + "dispatchRouteId", + "singleFlightKey", + "lockOwner", + "canonicalDigest", + ): + assert f"{field} = $dispatchIdentity." in trigger + assert 'schema_version = "agent99_controlled_dispatch_identity_v1"' in identity + assert '-RouteId "agent99_local_observer"' in identity + assert 'idempotencyKey = "agent99-controlled:$digest"' in identity + assert 'singleFlightKey = "agent99-controlled-flight:$singleFlightDigest"' in identity + normalized = identity[identity.index("$normalized = [ordered]@{") : identity.index("$canonical =")] + normalized_fields = ( + "approval_id", + "execution_generation", + "incident_id", + "project_id", + "route_id", + "source_fingerprint", + "work_item_id", + ) + assert [normalized.index(f"{field} =") for field in normalized_fields] == sorted( + normalized.index(f"{field} =") for field in normalized_fields + ) + public = identity[identity.index("$public = [ordered]@{") : identity.index("[pscustomobject]@{")] + public_fields = ( + "approval_id", + "execution_generation", + "idempotency_key", + "incident_id", + "lock_owner", + "project_id", + "route_id", + "run_id", + "schema_version", + "single_flight_key", + "source_fingerprint", + "trace_id", + "work_item_id", + ) + assert [public.index(f"{field} =") for field in public_fields] == sorted( + public.index(f"{field} =") for field in public_fields + ) + assert "function Test-AgentControlledDispatchIdentity" in identity + assert "function Enter-AgentRecoveryTriggerReservation" in identity + assert "[System.IO.FileShare]::None" in identity + assert "function Invoke-AgentControlledDispatchIdentitySelfTest" in identity + assert "concurrentTriggerSuppressed" in identity + assert "tamperedIdentityRejected" in identity + assert "Test-AgentControlledDispatchIdentity $dispatchIdentityEnvelope" in consumer + assert '"controlled_dispatch_identity_invalid"' in consumer + assert consumer.index('"controlled_dispatch_identity_invalid"') < consumer.index( + '$process = Start-Process -FilePath "powershell.exe"' + ) + assert 'Add-Check "recovery:auto_trigger_controlled_identity"' in contract + assert 'Add-Check "recovery:queue_identity_fail_closed"' in contract + assert 'correlationKey = $dispatchIdentity.idempotencyKey' in contract + assert 'canonicalDigest = $dispatchIdentity.canonicalDigest' in contract assert 'Record-AgentEvent "recovery_slo_result"' in source assert "withinSlo = $withinSlo" in source assert "function Invoke-AgentHost112GuestRecovery" in source