diff --git a/agent99-bootstrap.ps1 b/agent99-bootstrap.ps1 index dfbe7101f..0b9d205ab 100644 --- a/agent99-bootstrap.ps1 +++ b/agent99-bootstrap.ps1 @@ -292,6 +292,10 @@ param( [ValidateSet("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "SecurityTriage", "ControlTick")] [string]`$Mode = "Status", [switch]`$ControlledApply, + [switch]`$ReconcileOnly, + [string]`$ReconcileEvidenceId = "", + [switch]`$ReconcileQueueOnly, + [string]`$ReconcileQueueId = "", [string]`$AutomationRunId = "", [string]`$TraceId = "", [string]`$WorkItemId = "", @@ -299,7 +303,7 @@ param( [string]`$SuppressReason = "" ) -& "$BinDir\agent99-control-plane.ps1" -Mode `$Mode -ControlledApply:`$ControlledApply -AutomationRunId `$AutomationRunId -TraceId `$TraceId -WorkItemId `$WorkItemId -SuppressAlerts:`$SuppressAlerts -SuppressReason `$SuppressReason -ConfigPath "$configPath" -EvidenceDir "$EvidenceDir" +& "$BinDir\agent99-control-plane.ps1" -Mode `$Mode -ControlledApply:`$ControlledApply -ReconcileOnly:`$ReconcileOnly -ReconcileEvidenceId `$ReconcileEvidenceId -ReconcileQueueOnly:`$ReconcileQueueOnly -ReconcileQueueId `$ReconcileQueueId -AutomationRunId `$AutomationRunId -TraceId `$TraceId -WorkItemId `$WorkItemId -SuppressAlerts:`$SuppressAlerts -SuppressReason `$SuppressReason -ConfigPath "$configPath" -EvidenceDir "$EvidenceDir" "@ | Set-Content -Encoding UTF8 $launcher $submitCmd = Join-Path $AgentRoot "agent99-submit-request.cmd" diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index 797fbe601..e1e138604 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -2,6 +2,10 @@ param( [ValidateSet("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "SecurityTriage", "ControlTick")] [string]$Mode = "Status", [switch]$ControlledApply, + [switch]$ReconcileOnly, + [string]$ReconcileEvidenceId = "", + [switch]$ReconcileQueueOnly, + [string]$ReconcileQueueId = "", [string]$AutomationRunId = "", [string]$TraceId = "", [string]$WorkItemId = "", @@ -23,6 +27,11 @@ $stamp = Get-Date -Format "yyyyMMdd-HHmmss" $logPath = Join-Path $EvidenceDir "agent99-$Mode-$stamp.log" $jsonPath = Join-Path $EvidenceDir "agent99-$Mode-$stamp.json" +if ($ReconcileOnly -and $ReconcileEvidenceId -match "^same-run-status-[0-9a-f-]{36}-[0-9a-f]{12}$") { + $logPath = Join-Path $EvidenceDir "agent99-Status-$ReconcileEvidenceId.log" + $jsonPath = Join-Path $EvidenceDir "agent99-Status-$ReconcileEvidenceId.json" +} + function Write-AgentLog { param([string]$Message) $line = "$(Get-Date -Format o) $Message" @@ -37,6 +46,36 @@ function Load-Config { } $Config = Load-Config +if ( + $ReconcileOnly -and + ( + $Mode -ne "Status" -or + $ControlledApply -or + $ReconcileEvidenceId -notmatch "^same-run-status-[0-9a-f-]{36}-[0-9a-f]{12}$" + ) +) { + Write-AgentLog "same_run_reconcile_rejected mode=$Mode controlledApply=$ControlledApply reason=no_write_status_required" + exit 64 +} +if (-not $ReconcileOnly -and $ReconcileEvidenceId) { + Write-AgentLog "same_run_reconcile_rejected mode=$Mode reason=evidence_id_without_reconcile_only" + exit 64 +} +if ( + $ReconcileQueueOnly -and + ( + $Mode -ne "ControlTick" -or + $ControlledApply -or + $ReconcileQueueId -notmatch "^same-run-status-[A-Za-z0-9_.-]{1,96}$" + ) +) { + Write-AgentLog "same_run_reconcile_queue_rejected mode=$Mode controlledApply=$ControlledApply reason=exact_queue_only_required" + exit 64 +} +if (-not $ReconcileQueueOnly -and $ReconcileQueueId) { + Write-AgentLog "same_run_reconcile_queue_rejected mode=$Mode reason=queue_id_without_queue_only" + exit 64 +} $SshUser = if ($Config.sshUser) { $Config.sshUser } else { "wooo" } $SshIdentityFile = if ($Config.PSObject.Properties["sshIdentityFile"] -and $Config.sshIdentityFile) { [Environment]::ExpandEnvironmentVariables([string]$Config.sshIdentityFile) @@ -45,8 +84,16 @@ $SshIdentityFile = if ($Config.PSObject.Properties["sshIdentityFile"] -and $Conf } $script:Events = @() $script:TelegramAttempts = @() -$script:SuppressAgentAlerts = [bool]$SuppressAlerts -$script:SuppressAgentAlertReason = if ($SuppressReason) { $SuppressReason } elseif ($SuppressAlerts) { "suppress_alerts" } else { "" } +$script:SuppressAgentAlerts = [bool]($SuppressAlerts -or $ReconcileOnly -or $ReconcileQueueOnly) +$script:SuppressAgentAlertReason = if ($SuppressReason) { + $SuppressReason +} elseif ($ReconcileOnly -or $ReconcileQueueOnly) { + "same_run_reconcile_no_telegram" +} elseif ($SuppressAlerts) { + "suppress_alerts" +} else { + "" +} $script:Agent99DispatchClientDeadlineSeconds = 1800 $script:Agent99QueueOuterDeadlineSeconds = 1680 $script:Agent99Host112CheckTimeoutSeconds = 30 @@ -1992,9 +2039,12 @@ function Record-AgentEvent { } function Get-AgentBootState { + param([bool]$Persist = $true) $stateDir = Join-Path $Config.agentRoot "state" $statePath = Join-Path $stateDir "boot-state.json" - New-Item -ItemType Directory -Force -Path $stateDir | Out-Null + if ($Persist) { + New-Item -ItemType Directory -Force -Path $stateDir | Out-Null + } try { $os = Get-CimInstance -ClassName Win32_OperatingSystem -ErrorAction Stop $bootTime = ([datetime]$os.LastBootUpTime).ToString("o") @@ -2039,8 +2089,10 @@ function Get-AgentBootState { lastObservedAt = $observedAt statePath = $statePath } - $result | ConvertTo-Json -Depth 5 | Set-Content -Path $statePath -Encoding UTF8 - if ($detected) { + if ($Persist) { + $result | ConvertTo-Json -Depth 5 | Set-Content -Path $statePath -Encoding UTF8 + } + if ($detected -and $Persist) { Record-AgentEvent "host_reboot_detected" "warning" "偵測到 99 主機重新啟動；Agent99 已進入受控恢復主線。" $result -Alert } $result @@ -3029,6 +3081,43 @@ function Get-AgentLatestEvidence { } } +function Get-AgentEvidenceAtPath { + param([string]$Path) + + $file = Get-Item -LiteralPath $Path -ErrorAction SilentlyContinue | + Where-Object { -not $_.PSIsContainer } | + Select-Object -First 1 + if (-not $file) { + return [pscustomobject]@{ + exists = $false + path = $Path + fileName = Split-Path -Leaf $Path + lastWriteTime = $null + ageMinutes = $null + data = $null + parseError = $null + } + } + + $data = $null + $parseError = $null + try { + $data = Get-Content -LiteralPath $file.FullName -Raw | ConvertFrom-Json + } catch { + $parseError = $_.Exception.Message + } + + [pscustomobject]@{ + exists = $true + path = $file.FullName + fileName = $file.Name + lastWriteTime = $file.LastWriteTime.ToString("o") + ageMinutes = [math]::Round(((Get-Date) - $file.LastWriteTime).TotalMinutes, 2) + data = $data + parseError = $parseError + } +} + function Test-AgentScheduledTaskHealth { param([string[]]$TaskNames) @@ -4081,7 +4170,9 @@ function Get-AgentOutcomeContract { [bool]$SourceEventResolutionRequired, [object]$SourceEventResolved = $null, [string]$SourceEventEvidence = "", - [string]$SourceEventResolutionPolicy = "external_source_receipt_required" + [string]$SourceEventResolutionPolicy = "external_source_receipt_required", + [bool]$RequireReconcileOnly = $false, + [string]$ExpectedReconcileEvidenceId = "" ) $checks = @() @@ -4125,13 +4216,60 @@ function Get-AgentOutcomeContract { $publicOk = [bool]($publicRows.Count -gt 0 -and @($publicRows | Where-Object { -not $_.ok }).Count -eq 0) $aiOk = [bool](@($data.aiServices | Where-Object { -not $_.ok }).Count -eq 0) $host112Ok = [bool]($data.host112Recovery -and $data.host112Recovery.verified) + $evidenceReconcileOnly = [bool]( + $data.PSObject.Properties["reconcileOnly"] -and + $data.reconcileOnly -is [bool] -and + $data.reconcileOnly + ) + $evidenceReceiptBound = [bool]( + $RequireReconcileOnly -and + $ExpectedReconcileEvidenceId -match "^same-run-status-[0-9a-f-]{36}-[0-9a-f]{12}$" -and + $data.PSObject.Properties["reconcileEvidenceId"] -and + $data.reconcileEvidenceId -is [string] -and + [string]$data.reconcileEvidenceId -eq $ExpectedReconcileEvidenceId + ) + $reconcileNoWrite = [bool]( + -not $RequireReconcileOnly -or + ( + $evidenceReconcileOnly -and + $evidenceReceiptBound -and + $data.PSObject.Properties["controlledApply"] -and + $data.controlledApply -is [bool] -and + -not $data.controlledApply -and + ( + -not $data.recoveryTrigger -or + ( + $data.recoveryTrigger.PSObject.Properties["queued"] -and + $data.recoveryTrigger.queued -is [bool] -and + -not $data.recoveryTrigger.queued + ) + ) -and + $data.PSObject.Properties["runtimeWritePerformed"] -and + $data.runtimeWritePerformed -is [bool] -and + -not $data.runtimeWritePerformed -and + $data.host112Recovery -and + $data.host112Recovery.PSObject.Properties["runtimeWritePerformed"] -and + $data.host112Recovery.runtimeWritePerformed -is [bool] -and + -not $data.host112Recovery.runtimeWritePerformed + ) + ) $checks += New-AgentOutcomeCheck "required_hosts_reachable" $hostsOk $true "expected=$expectedHosts actual=$($hostRows.Count)" $checks += New-AgentOutcomeCheck "harbor_healthy" $harborOk $checks += New-AgentOutcomeCheck "awoooi_deployments_ready" $awoooOk $checks += New-AgentOutcomeCheck "public_routes_healthy" $publicOk $checks += New-AgentOutcomeCheck "ai_services_healthy" $aiOk $false $checks += New-AgentOutcomeCheck "host112_guest_ready" $host112Ok $true $([string](@($data.host112Recovery.after.failedChecks) -join ',')) - $modeVerifierPassed = [bool]($hostsOk -and $host112Ok -and $harborOk -and $awoooOk -and $publicOk) + if ($RequireReconcileOnly) { + $checks += New-AgentOutcomeCheck "reconcile_only_evidence_bound" $evidenceReceiptBound $true "expected=$ExpectedReconcileEvidenceId" + $checks += New-AgentOutcomeCheck "reconcile_only_no_runtime_write" $reconcileNoWrite $true + if ($evidenceReceiptBound -and $reconcileNoWrite) { + $verifierName = "agent99-status-same-run-no-write-post-condition-v1" + } + } elseif ($evidenceReconcileOnly) { + $checks += New-AgentOutcomeCheck "unexpected_reconcile_only_evidence" $false $true + $reconcileNoWrite = $false + } + $modeVerifierPassed = [bool]($hostsOk -and $host112Ok -and $harborOk -and $awoooOk -and $publicOk -and $reconcileNoWrite) } "Recover" { $expectedVms = @($Config.vms).Count @@ -5096,16 +5234,23 @@ function Ensure-AgentDesktopShortcuts { } function Invoke-AgentQueuedCommands { + param([string]$OnlyCommandId = "") + $queueDir = Join-Path $Config.agentRoot "queue" $processedDir = Join-Path $queueDir "processed" $failedDir = Join-Path $queueDir "failed" New-Item -ItemType Directory -Force -Path $queueDir, $processedDir, $failedDir | Out-Null $allowedModes = @("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "SecurityTriage") - $commands = @(Get-ChildItem -Path $queueDir -Filter "*.json" -File -ErrorAction SilentlyContinue | - Where-Object { $_.Name -notmatch "^(processed|failed|running)-" } | - Sort-Object LastWriteTime | - Select-Object -First 3) + $commands = if ($OnlyCommandId) { + $onlyPath = Join-Path $queueDir "$OnlyCommandId.json" + @(Get-Item -LiteralPath $onlyPath -ErrorAction SilentlyContinue | Select-Object -First 1) + } else { + @(Get-ChildItem -Path $queueDir -Filter "*.json" -File -ErrorAction SilentlyContinue | + Where-Object { $_.Name -notmatch "^(processed|failed|running)-" } | + Sort-Object LastWriteTime | + Select-Object -First 3) + } $results = @() foreach ($file in $commands) { $command = $null @@ -5152,10 +5297,27 @@ function Invoke-AgentQueuedCommands { $replyChatId = if ($command.PSObject.Properties["replyChatId"]) { [string]$command.replyChatId } else { "" } $replyMessageId = if ($command.PSObject.Properties["replyMessageId"]) { [string]$command.replyMessageId } else { "" } $suppressTelegram = if ($command.PSObject.Properties["suppressTelegram"]) { Convert-AgentBool $command.suppressTelegram } else { $false } + $reconcileOnly = if ($command.PSObject.Properties["reconcileOnly"]) { Convert-AgentBool $command.reconcileOnly } else { $false } + $reconcileEvidenceId = if ($command.PSObject.Properties["reconcileEvidenceId"]) { [string]$command.reconcileEvidenceId } else { "" } + if ($OnlyCommandId -and ($id -ne $OnlyCommandId -or -not $reconcileOnly)) { + $target = Join-Path $failedDir ("failed-" + $file.Name) + Move-Item -Force $file.FullName $target + $results += [pscustomobject]@{ + id = $id + mode = $modeName + source = $commandSource + ok = $false + reason = "same_run_reconcile_exact_queue_contract_required" + reconcileOnly = $reconcileOnly + runtimeWritePerformed = $false + file = $target + } + continue + } if ($modeName -notin $allowedModes) { $target = Join-Path $failedDir ("failed-" + $file.Name) Move-Item -Force $file.FullName $target - $result = [pscustomobject]@{ id = $id; mode = $modeName; source = $commandSource; instruction = $commandInstruction; ok = $false; reason = "mode_not_allowlisted"; file = $target } + $result = [pscustomobject]@{ id = $id; mode = $modeName; source = $commandSource; instruction = $commandInstruction; ok = $false; reason = "mode_not_allowlisted"; suppressTelegram = $suppressTelegram; file = $target } Record-AgentEvent "operator_command_rejected" "warning" "id=$id mode=$modeName reason=mode_not_allowlisted" $result -Alert $results += $result continue @@ -5168,6 +5330,54 @@ function Invoke-AgentQueuedCommands { $traceId -match $controlIdentityPattern -and $workItemId -match $controlIdentityPattern ) + $canonicalIdentity = [ordered]@{ + approval_id = $approvalId + execution_generation = $executionGeneration + idempotency_key = $idempotencyKey + incident_id = $incidentId + lock_owner = $lockOwner + project_id = $projectId + route_id = $dispatchRouteId + run_id = $automationRunId + schema_version = "agent99_controlled_dispatch_identity_v1" + single_flight_key = $singleFlightKey + source_fingerprint = $sourceFingerprint + trace_id = $traceId + work_item_id = $workItemId + } + $canonicalDigestMatches = [bool]( + $canonicalDigest -match "^[0-9a-f]{64}$" -and + (Get-AgentSha256Hex ($canonicalIdentity | ConvertTo-Json -Compress -Depth 4)) -eq $canonicalDigest + ) + $expectedReconcileEvidenceId = if ($canonicalDigest -match "^[0-9a-f]{64}$") { + "same-run-status-$automationRunId-$($canonicalDigest.Substring(0, 12))" + } else { + "" + } + $sameRunReconcileContractExact = [bool]( + $reconcileOnly -and + $modeName -eq "Status" -and + -not $controlled -and + $hasDispatchIdentity -and + -not $dispatchIdentityIncomplete -and + $dispatchControlIdentitySafe -and + $canonicalDigestMatches -and + $projectId -eq "awoooi" -and + $workItemId -eq "agent99-dispatch:awoooi:INC-20260711-11C751:legacy-cold-start" -and + $traceId -match "^00-[0-9a-f]{32}-[0-9a-f]{16}-01$" -and + $incidentId -eq "INC-20260711-11C751" -and + $automationRunId -eq "bf30ca01-1080-5725-b2d1-3e1534dfc811" -and + $dispatchRouteId -eq "agent99:host_recovery:Recover" -and + $executionGeneration -eq "1" -and + $lockOwner -eq $automationRunId -and + $sourceFingerprint.StartsWith("legacy-cold-start:") -and + $sourceEventResolutionRequired -and + $sourceEventResolutionPolicy -eq "external_source_receipt_required" -and + $sourceEventResolved -eq $true -and + $sourceEventEvidence -match "^prometheus:cold-start:[A-Za-z0-9._:@+%-]{1,220}$" -and + $reconcileEvidenceId -eq $expectedReconcileEvidenceId -and + $suppressTelegram + ) if (($controlled -and $modeName -eq "Recover" -and -not $hasDispatchIdentity) -or ($hasDispatchIdentity -and $dispatchIdentityIncomplete)) { $target = Join-Path $failedDir ("failed-" + $file.Name) Move-Item -Force $file.FullName $target @@ -5177,6 +5387,7 @@ function Invoke-AgentQueuedCommands { source = $commandSource ok = $false reason = "controlled_dispatch_identity_missing" + suppressTelegram = $suppressTelegram runtimeWritePerformed = $false file = $target } @@ -5184,6 +5395,22 @@ function Invoke-AgentQueuedCommands { $results += $result continue } + if ($reconcileOnly -and -not $sameRunReconcileContractExact) { + $target = Join-Path $failedDir ("failed-" + $file.Name) + Move-Item -Force $file.FullName $target + $result = [pscustomobject]@{ + id = $id + mode = $modeName + source = $commandSource + ok = $false + reason = "same_run_reconcile_exact_identity_no_write_status_required" + reconcileOnly = $true + runtimeWritePerformed = $false + file = $target + } + $results += $result + continue + } if ($hasDispatchIdentity -and -not $dispatchControlIdentitySafe) { $target = Join-Path $failedDir ("failed-" + $file.Name) Move-Item -Force $file.FullName $target @@ -5193,6 +5420,7 @@ function Invoke-AgentQueuedCommands { source = $commandSource ok = $false reason = "controlled_dispatch_identity_unsafe" + suppressTelegram = $suppressTelegram runtimeWritePerformed = $false file = $target } @@ -5212,6 +5440,9 @@ function Invoke-AgentQueuedCommands { if ($automationRunId) { $args += @("-AutomationRunId", $automationRunId) } if ($traceId) { $args += @("-TraceId", $traceId) } if ($workItemId) { $args += @("-WorkItemId", $workItemId) } + if ($reconcileOnly) { + $args += @("-ReconcileOnly", "-ReconcileEvidenceId", $reconcileEvidenceId) + } if ($suppressTelegram) { $args += @("-SuppressAlerts", "-SuppressReason", "do_not_page") } $process = Start-Process -FilePath "powershell.exe" -ArgumentList $args -NoNewWindow -RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath -PassThru $completed = $process.WaitForExit($script:Agent99QueueOuterDeadlineSeconds * 1000) @@ -5221,7 +5452,12 @@ function Invoke-AgentQueuedCommands { } $process.WaitForExit() | Out-Null $process.Refresh() - $latest = Get-AgentLatestEvidence "agent99-$modeName-*.json" + $latest = if ($reconcileOnly) { + $exactEvidencePath = Join-Path $EvidenceDir "agent99-Status-$reconcileEvidenceId.json" + Get-AgentEvidenceAtPath $exactEvidencePath + } else { + Get-AgentLatestEvidence "agent99-$modeName-*.json" + } $exitCode = if (-not $completed) { -2 } elseif ($null -ne $process.ExitCode) { @@ -5231,7 +5467,7 @@ function Invoke-AgentQueuedCommands { } else { -3 } - $outcome = Get-AgentOutcomeContract -ModeName $modeName -TransportExitCode $exitCode -LatestEvidence $latest -StartedAt $startTime -SourceEventResolutionRequired $sourceEventResolutionRequired -SourceEventResolved $sourceEventResolved -SourceEventEvidence $sourceEventEvidence -SourceEventResolutionPolicy $sourceEventResolutionPolicy + $outcome = Get-AgentOutcomeContract -ModeName $modeName -TransportExitCode $exitCode -LatestEvidence $latest -StartedAt $startTime -SourceEventResolutionRequired $sourceEventResolutionRequired -SourceEventResolved $sourceEventResolved -SourceEventEvidence $sourceEventEvidence -SourceEventResolutionPolicy $sourceEventResolutionPolicy -RequireReconcileOnly $reconcileOnly -ExpectedReconcileEvidenceId $reconcileEvidenceId $outcome | Add-Member -MemberType NoteProperty -Name automationRunId -Value $automationRunId -Force $outcome | Add-Member -MemberType NoteProperty -Name traceId -Value $traceId -Force $outcome | Add-Member -MemberType NoteProperty -Name workItemId -Value $workItemId -Force @@ -5272,6 +5508,14 @@ function Invoke-AgentQueuedCommands { alertService = $alertService alertHost = $alertHost suppressTelegram = $suppressTelegram + reconcileOnly = $reconcileOnly + reconcileEvidenceId = $reconcileEvidenceId + evidenceReceiptId = $reconcileEvidenceId + runtimeWritePerformed = [bool]( + $latest.data -and + $latest.data.PSObject.Properties["runtimeWritePerformed"] -and + $latest.data.runtimeWritePerformed + ) transportOk = [bool]($exitCode -eq 0) ok = [bool]$outcome.resolved outcomeState = $outcome.state @@ -5285,6 +5529,8 @@ function Invoke-AgentQueuedCommands { executionGeneration = $executionGeneration incidentId = $incidentId approvalId = $approvalId + projectId = $projectId + routeId = $dispatchRouteId identity = $dispatchIdentity outcome = $outcome exitCode = $exitCode @@ -5293,7 +5539,14 @@ function Invoke-AgentQueuedCommands { stdout = $stdoutPath stderr = $stderrPath } - $problem = Update-AgentProblemKnowledge $result + $problem = if ($reconcileOnly) { + [pscustomobject]@{ + status = "not_written_same_run_reconcile" + reason = "production_outcome_writeback_is_canonical" + } + } else { + Update-AgentProblemKnowledge $result + } $result | Add-Member -MemberType NoteProperty -Name problem -Value $problem -Force $donePath = Join-Path $processedDir ("processed-$id.json") $result | ConvertTo-Json -Depth 8 | Set-Content -Path $donePath -Encoding UTF8 @@ -5301,7 +5554,14 @@ function Invoke-AgentQueuedCommands { $sourceEvidenceRef = "agent99-source:$(Convert-AgentPublicReceiptLeaf $sourceEvidenceValue)" $postVerifierEvidenceValue = if ($latest.path) { [string]$latest.path } else { $donePath } $postVerifierEvidenceRef = "agent99-verifier:$(Convert-AgentPublicReceiptLeaf $postVerifierEvidenceValue)" - $outcomeWriteback = if ($hasDispatchIdentity) { + $outcomeWriteback = if ($reconcileOnly) { + [pscustomobject]@{ + ok = $false + status = "suppressed_same_run_reconcile_source_recheck_required" + attempts = 0 + runtimeClosureVerified = $false + } + } elseif ($hasDispatchIdentity) { Send-AgentOutcomeWriteback $result $postVerifierEvidenceRef $sourceEvidenceRef } else { [pscustomobject]@{ ok = $false; status = "no_durable_dispatch_identity"; attempts = 0; runtimeClosureVerified = $false } @@ -5317,9 +5577,22 @@ function Invoke-AgentQueuedCommands { $recordData | Add-Member -MemberType NoteProperty -Name replyMessageId -Value $replyMessageId -Force } $outcomeSeverity = if ($outcome.state -eq "resolved") { "info" } elseif ($outcome.state -eq "failed") { "critical" } else { "warning" } - Record-AgentEvent "operator_command_result" $outcomeSeverity "id=$id mode=$modeName outcome=$($outcome.state) transportExitCode=$exitCode verifierPassed=$($outcome.verifierPassed) sourceEventResolved=$($outcome.sourceEventResolved) evidence=$($latest.path)" $recordData -Alert + if ($reconcileOnly) { + Record-AgentEvent "operator_command_result" $outcomeSeverity "id=$id mode=$modeName outcome=$($outcome.state) transportExitCode=$exitCode verifierPassed=$($outcome.verifierPassed) sourceEventResolved=$($outcome.sourceEventResolved) evidence=$($latest.path)" $recordData -NoAlert + } else { + Record-AgentEvent "operator_command_result" $outcomeSeverity "id=$id mode=$modeName outcome=$($outcome.state) transportExitCode=$exitCode verifierPassed=$($outcome.verifierPassed) sourceEventResolved=$($outcome.sourceEventResolved) evidence=$($latest.path)" $recordData -Alert + } $telegramAttempt = @($script:TelegramAttempts | Where-Object { $_.eventType -eq "operator_command_result" } | Select-Object -Last 1) - $completionCallback = Invoke-AgentCompletionCallback -Result $result -TelegramAttempt $(if ($telegramAttempt.Count -gt 0) { $telegramAttempt[0] } else { $null }) + $completionCallback = if ($reconcileOnly) { + [pscustomobject]@{ + ok = $false + status = "suppressed_same_run_reconcile" + attempts = 0 + runtimeClosureVerified = $false + } + } else { + Invoke-AgentCompletionCallback -Result $result -TelegramAttempt $(if ($telegramAttempt.Count -gt 0) { $telegramAttempt[0] } else { $null }) + } $result | Add-Member -MemberType NoteProperty -Name completionCallback -Value $completionCallback -Force $donePath = Join-Path $processedDir ("processed-$id.json") $result | ConvertTo-Json -Depth 10 | Set-Content -Path $donePath -Encoding UTF8 @@ -5329,16 +5602,40 @@ function Invoke-AgentQueuedCommands { } function Invoke-AgentControlTick { - $completionCallbackReplay = Invoke-AgentPendingCompletionCallbacks - $processed = Invoke-AgentQueuedCommands - $summary = Get-AgentDashboardSummary - $dashboard = Write-AgentDashboard $summary - $shortcuts = Ensure-AgentDesktopShortcuts $dashboard.htmlPath - $pendingCount = if ($summary.queue) { $summary.queue.pending } else { 0 } + param( + [switch]$QueueOnly, + [string]$OnlyCommandId = "" + ) + + $completionCallbackReplay = if ($QueueOnly) { + [pscustomobject]@{ + enabled = $false + status = "suppressed_same_run_reconcile" + attemptedCount = 0 + verifiedCount = 0 + pendingCount = 0 + receipts = @() + } + } else { + Invoke-AgentPendingCompletionCallbacks + } + $processed = Invoke-AgentQueuedCommands -OnlyCommandId $(if ($QueueOnly) { $OnlyCommandId } else { "" }) + $summary = if ($QueueOnly) { $null } else { Get-AgentDashboardSummary } + $dashboard = if ($QueueOnly) { $null } else { Write-AgentDashboard $summary } + $shortcuts = if ($QueueOnly) { @() } else { Ensure-AgentDesktopShortcuts $dashboard.htmlPath } + $pendingCount = if ($QueueOnly) { + [int](Test-Path (Join-Path (Join-Path $Config.agentRoot "queue") "$OnlyCommandId.json")) + } elseif ($summary.queue) { + $summary.queue.pending + } else { + 0 + } $result = [pscustomobject]@{ ok = $true - dashboardPath = $dashboard.htmlPath - dashboardJson = $dashboard.jsonPath + reconcileQueueOnly = [bool]$QueueOnly + exactQueueId = $OnlyCommandId + dashboardPath = if ($dashboard) { $dashboard.htmlPath } else { $null } + dashboardJson = if ($dashboard) { $dashboard.jsonPath } else { $null } shortcutCount = @($shortcuts).Count shortcuts = $shortcuts processedCount = @($processed).Count @@ -5346,7 +5643,7 @@ function Invoke-AgentControlTick { pendingCount = $pendingCount completionCallbackReplay = $completionCallbackReplay } - Record-AgentEvent "agent_control_tick" "info" "dashboard=$($dashboard.htmlPath) processed=$(@($processed).Count) pending=$pendingCount" $result -NoAlert + Record-AgentEvent "agent_control_tick" "info" "queueOnly=$QueueOnly exactQueueId=$OnlyCommandId dashboard=$($result.dashboardPath) processed=$(@($processed).Count) pending=$pendingCount" $result -NoAlert $result } @@ -6481,7 +6778,11 @@ if ($SelfTestBackupReadbackContract) { } Write-AgentLog "agent99_start mode=$Mode controlledApply=$ControlledApply config=$ConfigPath" -Record-AgentEvent "agent_start" "info" "mode=$Mode controlledApply=$ControlledApply host=$env:COMPUTERNAME" $null -Alert +if ($ReconcileOnly -or $ReconcileQueueOnly) { + Record-AgentEvent "agent_start" "info" "mode=$Mode controlledApply=$ControlledApply host=$env:COMPUTERNAME" $null -NoAlert +} else { + Record-AgentEvent "agent_start" "info" "mode=$Mode controlledApply=$ControlledApply host=$env:COMPUTERNAME" $null -Alert +} $vmResults = @() $hostResults = @() @@ -6512,10 +6813,11 @@ $recoveryRunId = if ($Mode -eq "Recover") { $recoveryConfig = Get-AgentRecoverySloConfig if ($Mode -in @("Status", "Recover")) { - $bootState = Get-AgentBootState + $bootState = Get-AgentBootState -Persist:(-not $ReconcileOnly) } if ( $Mode -eq "Status" -and + -not $ReconcileOnly -and $bootState -and ($bootState.detected -or ($bootState.PSObject.Properties["pendingRecovery"] -and $bootState.pendingRecovery)) ) { @@ -6538,7 +6840,7 @@ if ($Mode -in @("Status", "Recover")) { $recoveryPhases += New-AgentRecoveryPhase "host_reachability" $(if ($unreachableHosts.Count -eq 0) { "ok" } else { "failed" }) (((Get-Date) - $recoveryStartedAt).TotalSeconds) } $autoRecovery = Get-AgentAutoRecoveryConfig - if ($Mode -eq "Status" -and -not $recoveryTrigger -and $autoRecovery.triggerOnHostUnreachable -and $unreachableHosts.Count -ge $autoRecovery.minimumUnreachableHosts) { + if ($Mode -eq "Status" -and -not $ReconcileOnly -and -not $recoveryTrigger -and $autoRecovery.triggerOnHostUnreachable -and $unreachableHosts.Count -ge $autoRecovery.minimumUnreachableHosts) { $recoveryTrigger = Start-AgentRecoveryFromObservation "host_unreachable_detected" ([pscustomobject]@{ hosts = @($unreachableHosts.host); count = $unreachableHosts.Count }) } } @@ -6551,7 +6853,7 @@ if ($Mode -in @("Status", "Recover")) { -FallbackRunId $recoveryRunId if ($Mode -eq "Recover") { $recoveryPhases += New-AgentRecoveryPhase "host112_guest_readiness" $(if ($host112Recovery.verified) { "ok" } else { "failed" }) (((Get-Date) - $recoveryStartedAt).TotalSeconds) - } elseif (-not $host112Recovery.verified -and -not $recoveryTrigger) { + } elseif (-not $ReconcileOnly -and -not $host112Recovery.verified -and -not $recoveryTrigger) { $recoveryTrigger = Start-AgentRecoveryFromObservation "host112_guest_not_ready" $host112Recovery.after } } @@ -6657,7 +6959,7 @@ if ($Mode -in @("SecurityTriage")) { } if ($Mode -in @("ControlTick")) { - $controlTick = Invoke-AgentControlTick + $controlTick = Invoke-AgentControlTick -QueueOnly:$ReconcileQueueOnly -OnlyCommandId $ReconcileQueueId } $publicFailures = @($public | Where-Object { $_ -and -not $_.ok }).Count @@ -6717,6 +7019,14 @@ $result = [pscustomobject]@{ timestamp = (Get-Date -Format o) mode = $Mode controlledApply = [bool]$ControlledApply + reconcileOnly = [bool]$ReconcileOnly + reconcileEvidenceId = $ReconcileEvidenceId + reconcileQueueOnly = [bool]$ReconcileQueueOnly + reconcileQueueId = $ReconcileQueueId + runtimeWritePerformed = [bool]( + ($recoveryTrigger -and $recoveryTrigger.queued) -or + ($host112Recovery -and $host112Recovery.runtimeWritePerformed) + ) sshProcessGuard = $sshProcessGuard bootState = $bootState recoveryTrigger = $recoveryTrigger diff --git a/agent99-sre-alert-relay.ps1 b/agent99-sre-alert-relay.ps1 index ac2180376..283860c36 100644 --- a/agent99-sre-alert-relay.ps1 +++ b/agent99-sre-alert-relay.ps1 @@ -14,10 +14,12 @@ $AlertRoot = Join-Path $AgentRoot "alerts" $IncomingDir = Join-Path $AlertRoot "incoming" $LogDir = Join-Path $AgentRoot "logs" $BinDir = Join-Path $AgentRoot "bin" -$ProcessedDir = Join-Path (Join-Path $AgentRoot "queue") "processed" +$QueueDir = Join-Path $AgentRoot "queue" +$ProcessedDir = Join-Path $QueueDir "processed" +$SameRunStateDir = Join-Path $AgentRoot "state\same-run-reconcile" $SreAlertInboxScript = Join-Path $BinDir "agent99-sre-alert-inbox.ps1" -New-Item -ItemType Directory -Force -Path $EvidenceDir, $IncomingDir, $LogDir, $ProcessedDir | Out-Null +New-Item -ItemType Directory -Force -Path $EvidenceDir, $IncomingDir, $LogDir, $QueueDir, $ProcessedDir, $SameRunStateDir | Out-Null function Convert-AgentSafeFileToken { param([string]$Value) @@ -114,6 +116,73 @@ function Get-AgentOutcomeReadback { $value = [string](Get-AgentField $outcomeEvidenceRefs $name "") if (Test-AgentPublicReceiptRef $value) { $safeEvidenceRefs[$name] = $value } } + $sourceEventEvidence = [string](Get-AgentField $outcome "sourceEventEvidence" "") + if (-not (Test-AgentPublicReceiptRef $sourceEventEvidence)) { + $sourceEventEvidence = "" + } + $evidenceReceiptId = [string](Get-AgentField $result "evidenceReceiptId" "") + if ([bool](Get-AgentField $result "reconcileOnly" $false)) { + $digest = [string](Get-AgentField $canonicalIdentity "canonical_digest" "") + $expectedEvidenceReceiptId = if ($digest -match "^[0-9a-f]{64}$") { + "same-run-status-$RunId-$($digest.Substring(0, 12))" + } else { + "" + } + if ($evidenceReceiptId -ne $expectedEvidenceReceiptId) { + $evidenceReceiptId = "" + } + } + $topLevelStringFieldsComplete = $true + foreach ($name in @( + "automationRunId", "traceId", "workItemId", "projectId", "incidentId", + "routeId", "executionGeneration", "approvalId", "evidenceReceiptId", "mode" + )) { + if (-not $result.PSObject.Properties[$name] -or (Get-AgentField $result $name $null) -isnot [string]) { + $topLevelStringFieldsComplete = $false + break + } + } + $outcomeStringFieldsComplete = $true + foreach ($name in @("schemaVersion", "state", "verifierName", "sourceEventEvidence")) { + if (-not $outcome -or -not $outcome.PSObject.Properties[$name] -or (Get-AgentField $outcome $name $null) -isnot [string]) { + $outcomeStringFieldsComplete = $false + break + } + } + $sameRunContractFieldsComplete = [bool]( + $topLevelStringFieldsComplete -and + $outcomeStringFieldsComplete -and + $result.PSObject.Properties["automationRunId"] -and + $result.PSObject.Properties["traceId"] -and + $result.PSObject.Properties["workItemId"] -and + $result.PSObject.Properties["projectId"] -and + $result.PSObject.Properties["incidentId"] -and + $result.PSObject.Properties["routeId"] -and + $result.PSObject.Properties["executionGeneration"] -and + $result.PSObject.Properties["approvalId"] -and + $result.PSObject.Properties["evidenceReceiptId"] -and + $result.PSObject.Properties["controlledApply"] -and + (Get-AgentField $result "controlledApply" $null) -is [bool] -and + $result.PSObject.Properties["reconcileOnly"] -and + (Get-AgentField $result "reconcileOnly" $null) -is [bool] -and + $result.PSObject.Properties["runtimeWritePerformed"] -and + (Get-AgentField $result "runtimeWritePerformed" $null) -is [bool] -and + $result.PSObject.Properties["mode"] -and + $outcome -and + $outcome.PSObject.Properties["schemaVersion"] -and + $outcome.PSObject.Properties["state"] -and + $outcome.PSObject.Properties["resolved"] -and + (Get-AgentField $outcome "resolved" $null) -is [bool] -and + $outcome.PSObject.Properties["transportOk"] -and + (Get-AgentField $outcome "transportOk" $null) -is [bool] -and + $outcome.PSObject.Properties["verifierName"] -and + $outcome.PSObject.Properties["verifierPassed"] -and + (Get-AgentField $outcome "verifierPassed" $null) -is [bool] -and + $outcome.PSObject.Properties["sourceEventResolved"] -and + (Get-AgentField $outcome "sourceEventResolved" $null) -is [bool] -and + $outcome.PSObject.Properties["sourceEventEvidence"] -and + $outcome.PSObject.Properties["identity"] + ) return @{ ok = $true found = $true @@ -121,12 +190,18 @@ function Get-AgentOutcomeReadback { automationRunId = $RunId traceId = [string](Get-AgentField $result "traceId" "") workItemId = [string](Get-AgentField $result "workItemId" "") + projectId = [string](Get-AgentField $result "projectId" "") idempotencyKey = [string](Get-AgentField $result "idempotencyKey" "") executionGeneration = [string](Get-AgentField $result "executionGeneration" "1") incidentId = [string](Get-AgentField $result "incidentId" "") approvalId = [string](Get-AgentField $result "approvalId" "") + routeId = [string](Get-AgentField $result "routeId" "") + evidenceReceiptId = $evidenceReceiptId + sameRunContractFieldsComplete = $sameRunContractFieldsComplete identity = $canonicalIdentity controlledApply = [bool](Get-AgentField $result "controlledApply" $false) + reconcileOnly = [bool](Get-AgentField $result "reconcileOnly" $false) + runtimeWritePerformed = [bool](Get-AgentField $result "runtimeWritePerformed" $false) mode = [string](Get-AgentField $result "mode" "") outcome = @{ schemaVersion = [string](Get-AgentField $outcome "schemaVersion" "") @@ -136,6 +211,7 @@ function Get-AgentOutcomeReadback { verifierName = [string](Get-AgentField $outcome "verifierName" "") verifierPassed = [bool](Get-AgentField $outcome "verifierPassed" $false) sourceEventResolved = [bool](Get-AgentField $outcome "sourceEventResolved" $false) + sourceEventEvidence = $sourceEventEvidence identity = $canonicalIdentity failedChecks = @((Get-AgentField $outcome "failedChecks" @()) | Select-Object -First 32) checks = $safeChecks @@ -212,6 +288,393 @@ function Start-AgentSreAlertInbox { } } +function Get-AgentSha256Text { + param([string]$Value) + $sha = [Security.Cryptography.SHA256]::Create() + try { + $bytes = [Text.Encoding]::UTF8.GetBytes($Value) + return ([BitConverter]::ToString($sha.ComputeHash($bytes))).Replace("-", "").ToLowerInvariant() + } finally { + $sha.Dispose() + } +} + +function Test-AgentSameRunDispatchIdentity { + param([object]$Identity) + + $required = @( + "schema_version", "project_id", "incident_id", "source_fingerprint", + "route_id", "execution_generation", "approval_id", "run_id", "trace_id", + "work_item_id", "idempotency_key", "single_flight_key", "lock_owner", + "canonical_digest" + ) + if (-not $Identity) { + return [pscustomobject]@{ ok = $false; reason = "identity_missing" } + } + foreach ($name in $required) { + if (-not $Identity.PSObject.Properties[$name]) { + return [pscustomobject]@{ ok = $false; reason = "identity_field_missing_$name" } + } + if ($name -ne "approval_id" -and -not [string](Get-AgentField $Identity $name "")) { + return [pscustomobject]@{ ok = $false; reason = "identity_field_empty_$name" } + } + } + $runId = [string](Get-AgentField $Identity "run_id" "") + $parsedRunId = [guid]::Empty + if (-not [guid]::TryParse($runId, [ref]$parsedRunId)) { + return [pscustomobject]@{ ok = $false; reason = "identity_run_id_invalid" } + } + if ($runId -ne "bf30ca01-1080-5725-b2d1-3e1534dfc811") { + return [pscustomobject]@{ ok = $false; reason = "identity_run_not_eligible" } + } + if ([string](Get-AgentField $Identity "schema_version" "") -ne "agent99_controlled_dispatch_identity_v1") { + return [pscustomobject]@{ ok = $false; reason = "identity_schema_invalid" } + } + if ([string](Get-AgentField $Identity "project_id" "") -ne "awoooi") { + return [pscustomobject]@{ ok = $false; reason = "identity_project_not_eligible" } + } + if ([string](Get-AgentField $Identity "work_item_id" "") -ne "agent99-dispatch:awoooi:INC-20260711-11C751:legacy-cold-start") { + return [pscustomobject]@{ ok = $false; reason = "identity_work_item_not_eligible" } + } + if ([string](Get-AgentField $Identity "trace_id" "") -notmatch "^00-[0-9a-f]{32}-[0-9a-f]{16}-01$") { + return [pscustomobject]@{ ok = $false; reason = "identity_trace_not_eligible" } + } + if ([string](Get-AgentField $Identity "route_id" "") -ne "agent99:host_recovery:Recover") { + return [pscustomobject]@{ ok = $false; reason = "identity_route_not_eligible" } + } + if ([string](Get-AgentField $Identity "incident_id" "") -ne "INC-20260711-11C751") { + return [pscustomobject]@{ ok = $false; reason = "identity_incident_not_eligible" } + } + if ([string](Get-AgentField $Identity "execution_generation" "") -ne "1") { + return [pscustomobject]@{ ok = $false; reason = "identity_generation_advance_forbidden" } + } + if ([string](Get-AgentField $Identity "lock_owner" "") -ne $runId) { + return [pscustomobject]@{ ok = $false; reason = "identity_lock_owner_mismatch" } + } + if (-not ([string](Get-AgentField $Identity "source_fingerprint" "")).StartsWith("legacy-cold-start:")) { + return [pscustomobject]@{ ok = $false; reason = "identity_source_not_legacy_cold_start" } + } + $canonical = [ordered]@{ + approval_id = [string](Get-AgentField $Identity "approval_id" "") + execution_generation = [string](Get-AgentField $Identity "execution_generation" "") + idempotency_key = [string](Get-AgentField $Identity "idempotency_key" "") + incident_id = [string](Get-AgentField $Identity "incident_id" "") + lock_owner = [string](Get-AgentField $Identity "lock_owner" "") + project_id = [string](Get-AgentField $Identity "project_id" "") + route_id = [string](Get-AgentField $Identity "route_id" "") + run_id = $runId + schema_version = [string](Get-AgentField $Identity "schema_version" "") + single_flight_key = [string](Get-AgentField $Identity "single_flight_key" "") + source_fingerprint = [string](Get-AgentField $Identity "source_fingerprint" "") + trace_id = [string](Get-AgentField $Identity "trace_id" "") + work_item_id = [string](Get-AgentField $Identity "work_item_id" "") + } + $expectedDigest = Get-AgentSha256Text ($canonical | ConvertTo-Json -Compress -Depth 4) + if ($expectedDigest -ne [string](Get-AgentField $Identity "canonical_digest" "")) { + return [pscustomobject]@{ ok = $false; reason = "identity_canonical_digest_mismatch" } + } + return [pscustomobject]@{ ok = $true; reason = "identity_valid" } +} + +function Convert-AgentStoredDispatchIdentity { + param([object]$Stored) + + $identity = Get-AgentField $Stored "identity" $null + if ($identity) { return $identity } + return [pscustomobject]@{ + schema_version = "agent99_controlled_dispatch_identity_v1" + project_id = [string](Get-AgentField $Stored "projectId" "") + incident_id = [string](Get-AgentField $Stored "incidentId" "") + source_fingerprint = [string](Get-AgentField $Stored "sourceFingerprint" "") + route_id = [string](Get-AgentField $Stored "dispatchRouteId" "") + execution_generation = [string](Get-AgentField $Stored "executionGeneration" "") + approval_id = [string](Get-AgentField $Stored "approvalId" "") + run_id = [string](Get-AgentField $Stored "automationRunId" "") + trace_id = [string](Get-AgentField $Stored "traceId" "") + work_item_id = [string](Get-AgentField $Stored "workItemId" "") + idempotency_key = [string](Get-AgentField $Stored "idempotencyKey" "") + single_flight_key = [string](Get-AgentField $Stored "singleFlightKey" "") + lock_owner = [string](Get-AgentField $Stored "lockOwner" "") + canonical_digest = [string](Get-AgentField $Stored "canonicalDigest" "") + } +} + +function Get-AgentExistingRecoverIdentity { + param([string]$RunId) + + $files = @() + $files += @(Get-ChildItem -LiteralPath $ProcessedDir -Filter "processed-*.json" -File -ErrorAction SilentlyContinue) + $files += @(Get-ChildItem -LiteralPath $QueueDir -Filter "running-*.json" -File -ErrorAction SilentlyContinue) + $files += @(Get-ChildItem -LiteralPath $QueueDir -Filter "*.json" -File -ErrorAction SilentlyContinue | + Where-Object { $_.Name -notmatch "^(processed|failed|running|same-run-status)-" }) + foreach ($file in @($files | Sort-Object LastWriteTime -Descending | Select-Object -First 256)) { + try { + $stored = Get-Content -LiteralPath $file.FullName -Raw | ConvertFrom-Json + if ([bool](Get-AgentField $stored "reconcileOnly" $false)) { continue } + if ([string](Get-AgentField $stored "mode" "") -ne "Recover") { continue } + if ([string](Get-AgentField $stored "automationRunId" "") -ne $RunId) { continue } + $identity = Convert-AgentStoredDispatchIdentity $stored + $check = Test-AgentSameRunDispatchIdentity $identity + if ($check.ok) { return $identity } + } catch { + continue + } + } + return $null +} + +function Test-AgentSameRunIdentityMatchesExisting { + param([object]$Requested, [object]$Existing) + + foreach ($name in @( + "schema_version", "project_id", "incident_id", "source_fingerprint", + "route_id", "execution_generation", "approval_id", "run_id", "trace_id", + "work_item_id", "idempotency_key", "single_flight_key", "lock_owner", + "canonical_digest" + )) { + if ([string](Get-AgentField $Requested $name "") -ne [string](Get-AgentField $Existing $name "")) { + return $false + } + } + return $true +} + +function Test-AgentSameRunOutcomeEligible { + param( + [object]$Existing, + [object]$Identity, + [string]$SourceEvidence, + [string]$EvidenceReceiptId + ) + + if (-not $Existing -or -not [bool](Get-AgentField $Existing "found" $false)) { return $false } + $outcome = Get-AgentField $Existing "outcome" $null + $existingIdentity = Get-AgentField $Existing "identity" $null + $nestedIdentity = Get-AgentField $outcome "identity" $null + return [bool]( + [string](Get-AgentField $Existing "schemaVersion" "") -eq "agent99_outcome_readback_v1" -and + [bool](Get-AgentField $Existing "sameRunContractFieldsComplete" $false) -and + [string](Get-AgentField $Existing "automationRunId" "") -eq [string](Get-AgentField $Identity "run_id" "") -and + [string](Get-AgentField $Existing "traceId" "") -eq [string](Get-AgentField $Identity "trace_id" "") -and + [string](Get-AgentField $Existing "workItemId" "") -eq [string](Get-AgentField $Identity "work_item_id" "") -and + [string](Get-AgentField $Existing "projectId" "") -eq [string](Get-AgentField $Identity "project_id" "") -and + [string](Get-AgentField $Existing "incidentId" "") -eq [string](Get-AgentField $Identity "incident_id" "") -and + [string](Get-AgentField $Existing "routeId" "") -eq [string](Get-AgentField $Identity "route_id" "") -and + [string](Get-AgentField $Existing "executionGeneration" "") -eq [string](Get-AgentField $Identity "execution_generation" "") -and + [string](Get-AgentField $Existing "approvalId" "") -eq [string](Get-AgentField $Identity "approval_id" "") -and + [string](Get-AgentField $Existing "evidenceReceiptId" "") -eq $EvidenceReceiptId -and + (Test-AgentSameRunIdentityMatchesExisting $existingIdentity $Identity) -and + (Test-AgentSameRunIdentityMatchesExisting $nestedIdentity $Identity) -and + [string](Get-AgentField $Existing "mode" "") -eq "Status" -and + (Get-AgentField $Existing "controlledApply" $null) -is [bool] -and + -not [bool](Get-AgentField $Existing "controlledApply" $true) -and + (Get-AgentField $Existing "reconcileOnly" $null) -is [bool] -and + [bool](Get-AgentField $Existing "reconcileOnly" $false) -and + (Get-AgentField $Existing "runtimeWritePerformed" $null) -is [bool] -and + -not [bool](Get-AgentField $Existing "runtimeWritePerformed" $true) -and + [string](Get-AgentField $outcome "schemaVersion" "") -eq "agent99_outcome_contract_v1" -and + [string](Get-AgentField $outcome "state" "") -eq "resolved" -and + (Get-AgentField $outcome "resolved" $null) -is [bool] -and + [bool](Get-AgentField $outcome "resolved" $false) -and + (Get-AgentField $outcome "transportOk" $null) -is [bool] -and + [bool](Get-AgentField $outcome "transportOk" $false) -and + [string](Get-AgentField $outcome "verifierName" "") -eq "agent99-status-same-run-no-write-post-condition-v1" -and + (Get-AgentField $outcome "verifierPassed" $null) -is [bool] -and + [bool](Get-AgentField $outcome "verifierPassed" $false) -and + (Get-AgentField $outcome "sourceEventResolved" $null) -is [bool] -and + [bool](Get-AgentField $outcome "sourceEventResolved" $false) -and + [string](Get-AgentField $outcome "sourceEventEvidence" "") -eq $SourceEvidence + ) +} + +function Start-AgentSameRunStatusReconcile { + param([object]$Request) + + $identity = Get-AgentField $Request "identity" $null + $identityCheck = Test-AgentSameRunDispatchIdentity $identity + if (-not $identityCheck.ok) { + return [pscustomobject]@{ ok = $false; status = "rejected"; reason = $identityCheck.reason; httpStatus = 400 } + } + $controlledApply = Get-AgentField $Request "controlledApply" $null + $reconcileOnly = Get-AgentField $Request "reconcileOnly" $null + $sourceEventResolved = Get-AgentField $Request "sourceEventResolved" $null + $suppressTelegram = Get-AgentField $Request "suppressTelegram" $null + if ( + [string](Get-AgentField $Request "action" "") -ne "same_run_status_reconcile" -or + [string](Get-AgentField $Request "mode" "") -ne "Status" -or + $controlledApply -isnot [bool] -or $controlledApply -or + $reconcileOnly -isnot [bool] -or -not $reconcileOnly -or + $sourceEventResolved -isnot [bool] -or -not $sourceEventResolved -or + [string](Get-AgentField $Request "sourceEventResolutionPolicy" "") -ne "external_source_receipt_required" -or + $suppressTelegram -isnot [bool] -or -not $suppressTelegram + ) { + return [pscustomobject]@{ ok = $false; status = "rejected"; reason = "same_run_reconcile_contract_invalid"; httpStatus = 400 } + } + $sourceEvidence = [string](Get-AgentField $Request "sourceEventEvidence" "") + if (-not (Test-AgentPublicReceiptRef $sourceEvidence) -or -not $sourceEvidence.StartsWith("prometheus:cold-start:")) { + return [pscustomobject]@{ ok = $false; status = "rejected"; reason = "source_event_evidence_invalid"; httpStatus = 400 } + } + + $runId = [string](Get-AgentField $identity "run_id" "") + $canonicalDigest = [string](Get-AgentField $identity "canonical_digest" "") + $evidenceReceiptId = [string](Get-AgentField $Request "evidenceReceiptId" "") + $expectedEvidenceReceiptId = if ($canonicalDigest -match "^[0-9a-f]{64}$") { + "same-run-status-$runId-$($canonicalDigest.Substring(0, 12))" + } else { + "" + } + if (-not $expectedEvidenceReceiptId -or $evidenceReceiptId -ne $expectedEvidenceReceiptId) { + return [pscustomobject]@{ ok = $false; status = "rejected"; reason = "same_run_evidence_receipt_id_invalid"; httpStatus = 400 } + } + + $existingRecoverIdentity = Get-AgentExistingRecoverIdentity $runId + if (-not $existingRecoverIdentity) { + return [pscustomobject]@{ ok = $false; status = "rejected"; reason = "existing_dispatch_identity_not_found"; httpStatus = 409 } + } + if (-not (Test-AgentSameRunIdentityMatchesExisting $identity $existingRecoverIdentity)) { + return [pscustomobject]@{ ok = $false; status = "rejected"; reason = "existing_dispatch_identity_mismatch"; httpStatus = 409 } + } + + $existing = Get-AgentOutcomeReadback $runId + if (Test-AgentSameRunOutcomeEligible $existing $identity $sourceEvidence $evidenceReceiptId) { + return [pscustomobject]@{ + ok = $true + status = "same_run_status_outcome_already_available" + httpStatus = 200 + automationRunId = $runId + reconcileOnly = $true + controlledApply = $false + evidenceReceiptId = $evidenceReceiptId + queued = $false + } + } + if ($existing.found -and [bool](Get-AgentField $existing "reconcileOnly" $false)) { + return [pscustomobject]@{ ok = $false; status = "rejected"; reason = "existing_same_run_status_outcome_invalid_fail_closed"; httpStatus = 409 } + } + + $safeRunId = Convert-AgentSafeFileToken $runId + $queueId = "same-run-status-$safeRunId" + $queuePath = Join-Path $QueueDir "$queueId.json" + $runningPath = Join-Path $QueueDir "running-$queueId.json" + $lockPath = Join-Path $SameRunStateDir "$queueId.lock.json" + if ((Test-Path -LiteralPath $queuePath) -or (Test-Path -LiteralPath $runningPath) -or (Test-Path -LiteralPath $lockPath)) { + return [pscustomobject]@{ + ok = $true + status = "same_run_status_reconcile_pending" + httpStatus = 202 + automationRunId = $runId + reconcileOnly = $true + controlledApply = $false + evidenceReceiptId = $evidenceReceiptId + queued = $false + } + } + + try { + $lockStream = [IO.File]::Open($lockPath, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::None) + try { + $lockReceipt = @{ + schemaVersion = "agent99_same_run_single_flight_lock_v1" + automationRunId = $runId + evidenceReceiptId = $evidenceReceiptId + canonicalDigest = $canonicalDigest + createdAt = (Get-Date -Format o) + storesRawEvidence = $false + } | ConvertTo-Json -Compress + $lockBytes = [Text.Encoding]::UTF8.GetBytes($lockReceipt) + $lockStream.Write($lockBytes, 0, $lockBytes.Length) + $lockStream.Flush($true) + } finally { + $lockStream.Dispose() + } + } catch [System.IO.IOException] { + return [pscustomobject]@{ + ok = $true + status = "same_run_status_reconcile_pending" + httpStatus = 202 + automationRunId = $runId + reconcileOnly = $true + controlledApply = $false + evidenceReceiptId = $evidenceReceiptId + queued = $false + } + } + + $command = [ordered]@{ + id = $queueId + mode = "Status" + controlledApply = $false + reconcileOnly = $true + reconcileEvidenceId = $evidenceReceiptId + requestedBy = "awoooi-agent99-controlled-dispatch-reconciler" + source = "agent99-same-run-reconcile" + reason = "source_event_resolved_status_reconcile" + instruction = "Read-only Status verifier for the existing run; do not recover, advance generation, page Telegram, or write runtime state." + correlationKey = ([string](Get-AgentField $identity "idempotency_key" "")) + ":same-run-status" + alertId = "same-run-reconcile:$runId" + alertSource = "awoooi-production-source-verifier" + alertKind = "host_recovery" + alertSeverity = "info" + alertService = "cold-start-gate" + alertHost = "" + suppressTelegram = $true + sourceEventResolutionRequired = $true + sourceEventResolutionPolicy = "external_source_receipt_required" + sourceEventResolved = $true + sourceEventEvidence = $sourceEvidence + automationRunId = $runId + traceId = [string](Get-AgentField $identity "trace_id" "") + workItemId = [string](Get-AgentField $identity "work_item_id" "") + idempotencyKey = [string](Get-AgentField $identity "idempotency_key" "") + executionGeneration = [string](Get-AgentField $identity "execution_generation" "") + incidentId = [string](Get-AgentField $identity "incident_id" "") + approvalId = [string](Get-AgentField $identity "approval_id" "") + projectId = [string](Get-AgentField $identity "project_id" "") + sourceFingerprint = [string](Get-AgentField $identity "source_fingerprint" "") + dispatchRouteId = [string](Get-AgentField $identity "route_id" "") + singleFlightKey = [string](Get-AgentField $identity "single_flight_key" "") + lockOwner = [string](Get-AgentField $identity "lock_owner" "") + canonicalDigest = [string](Get-AgentField $identity "canonical_digest" "") + createdAt = (Get-Date -Format o) + } + $tmpPath = Join-Path $QueueDir ".$queueId-$([guid]::NewGuid().ToString("N")).tmp" + $command | ConvertTo-Json -Depth 8 | Set-Content -Encoding UTF8 $tmpPath + Move-Item -Force $tmpPath $queuePath + + $launcher = Join-Path $AgentRoot "agent99-run.ps1" + if (-not (Test-Path $launcher)) { + Remove-Item -Force $queuePath -ErrorAction SilentlyContinue + Remove-Item -Force $lockPath -ErrorAction SilentlyContinue + return [pscustomobject]@{ ok = $false; status = "rejected"; reason = "agent99_launcher_missing"; httpStatus = 503 } + } + $stamp = Get-Date -Format "yyyyMMdd-HHmmss" + $stdout = Join-Path $LogDir "agent99-same-run-status-$stamp.out" + $stderr = Join-Path $LogDir "agent99-same-run-status-$stamp.err" + try { + $process = Start-Process -FilePath "powershell.exe" -ArgumentList @( + "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", $launcher, + "-Mode", "ControlTick", + "-ReconcileQueueOnly", "-ReconcileQueueId", $queueId, + "-SuppressAlerts", "-SuppressReason", "same_run_reconcile_no_telegram" + ) -WindowStyle Hidden -RedirectStandardOutput $stdout -RedirectStandardError $stderr -PassThru + } catch { + Remove-Item -Force $queuePath -ErrorAction SilentlyContinue + Remove-Item -Force $lockPath -ErrorAction SilentlyContinue + return [pscustomobject]@{ ok = $false; status = "rejected"; reason = "control_tick_start_failed"; httpStatus = 503 } + } + return [pscustomobject]@{ + ok = $true + status = "same_run_status_reconcile_queued" + httpStatus = 202 + automationRunId = $runId + reconcileOnly = $true + controlledApply = $false + evidenceReceiptId = $evidenceReceiptId + queued = $true + processId = $process.Id + } +} + if (-not $Prefix.EndsWith("/")) { $Prefix = "$Prefix/" } @@ -253,18 +716,15 @@ try { } $expectedToken = Get-AgentEnvValue $TokenEnv - if ($expectedToken) { - $actualToken = [string]$context.Request.Headers["X-Agent99-Relay-Token"] - if ($actualToken -ne $expectedToken) { - Send-AgentRelayResponse $context 401 @{ ok = $false; error = "invalid_relay_token" } - Write-AgentRelayEvent @{ event = "relay_rejected"; ok = $false; remote = $remote; reason = "invalid_relay_token" } - continue - } + if (-not $expectedToken) { + Send-AgentRelayResponse $context 503 @{ ok = $false; error = "relay_auth_not_configured" } + Write-AgentRelayEvent @{ event = "relay_rejected"; ok = $false; remote = $remote; reason = "relay_auth_not_configured" } + continue } - - if ($context.Request.HttpMethod -eq "GET" -and -not $expectedToken) { - Send-AgentRelayResponse $context 503 @{ ok = $false; found = $false; error = "outcome_readback_auth_not_configured" } - Write-AgentRelayEvent @{ event = "outcome_readback_rejected"; ok = $false; remote = $remote; reason = "auth_not_configured" } + $actualToken = [string]$context.Request.Headers["X-Agent99-Relay-Token"] + if ($actualToken -ne $expectedToken) { + Send-AgentRelayResponse $context 401 @{ ok = $false; error = "invalid_relay_token" } + Write-AgentRelayEvent @{ event = "relay_rejected"; ok = $false; remote = $remote; reason = "invalid_relay_token" } continue } @@ -304,7 +764,34 @@ try { continue } - $alert = $body | ConvertFrom-Json + $requestPayload = $body | ConvertFrom-Json + if ([string](Get-AgentField $requestPayload "schemaVersion" "") -eq "agent99_same_run_status_reconcile_v1") { + $reconcile = Start-AgentSameRunStatusReconcile $requestPayload + $reconcileBody = @{ + ok = [bool]$reconcile.ok + status = [string]$reconcile.status + reason = [string](Get-AgentField $reconcile "reason" "") + automationRunId = [string](Get-AgentField $reconcile "automationRunId" "") + reconcileOnly = [bool](Get-AgentField $reconcile "reconcileOnly" $true) + controlledApply = $false + evidenceReceiptId = [string](Get-AgentField $reconcile "evidenceReceiptId" "") + queued = [bool](Get-AgentField $reconcile "queued" $false) + } + Send-AgentRelayResponse $context ([int]$reconcile.httpStatus) $reconcileBody + Write-AgentRelayEvent @{ + event = "same_run_status_reconcile" + ok = [bool]$reconcile.ok + status = [string]$reconcile.status + remote = $remote + runId = if ($reconcile.ok) { [string]$reconcile.automationRunId } else { "rejected" } + reconcileOnly = $true + controlledApply = $false + storesRawEvidence = $false + } + continue + } + + $alert = $requestPayload $alertId = [string](Get-AgentField $alert "id" ("relay-" + [guid]::NewGuid().ToString("N"))) $safeId = Convert-AgentSafeFileToken $alertId $fileStamp = Get-Date -Format "yyyyMMdd-HHmmss" diff --git a/apps/api/src/jobs/agent99_controlled_dispatch_reconciler_job.py b/apps/api/src/jobs/agent99_controlled_dispatch_reconciler_job.py index 456ea2cbd..bb43f5faa 100644 --- a/apps/api/src/jobs/agent99_controlled_dispatch_reconciler_job.py +++ b/apps/api/src/jobs/agent99_controlled_dispatch_reconciler_job.py @@ -22,11 +22,19 @@ from src.services.agent99_controlled_dispatch_ledger import ( Agent99DispatchIdentity, get_agent99_dispatch_ledger, record_agent99_learning_writeback, + validate_agent99_outcome_identity, ) from src.services.agent99_public_receipts import ( extract_agent99_outcome_evidence_refs, sanitize_agent99_public_receipt_refs, ) +from src.services.agent99_same_run_reconcile import ( + AGENT99_SAME_RUN_STATUS_VERIFIER, + agent99_same_run_evidence_id, + is_cold_start_same_run_identity, + read_cold_start_source_resolution, + request_agent99_same_run_status_reconcile, +) from src.services.agent99_sre_bridge import ( bridge_alertmanager_to_agent99, read_agent99_sre_outcome, @@ -46,6 +54,51 @@ LegacyIncidentFetcher = Callable[..., Awaitable[list[dict[str, Any]]]] Agent99Dispatcher = Callable[..., Awaitable[dict[str, Any]]] +def _is_bound_no_write_status_outcome( + outcome: dict[str, Any] | None, + *, + identity: Agent99DispatchIdentity, + source_receipt_ref: str, +) -> bool: + """Require the fresh Status outcome to prove its source/no-write fences.""" + + if not isinstance(outcome, dict): + return False + contract = ( + outcome.get("outcome") + if isinstance(outcome.get("outcome"), dict) + else {} + ) + return bool( + validate_agent99_outcome_identity(identity, outcome) + and outcome.get("schemaVersion") == "agent99_outcome_readback_v1" + and outcome.get("sameRunContractFieldsComplete") is True + and outcome.get("automationRunId") == str(identity.run_id) + and outcome.get("traceId") == identity.trace_id + and outcome.get("workItemId") == identity.work_item_id + and outcome.get("projectId") == identity.project_id + and outcome.get("incidentId") == identity.incident_id + and outcome.get("routeId") == identity.route_id + and outcome.get("executionGeneration") + == identity.execution_generation + and outcome.get("approvalId") == identity.approval_id + and outcome.get("evidenceReceiptId") + == agent99_same_run_evidence_id(identity) + and contract.get("schemaVersion") == "agent99_outcome_contract_v1" + and outcome.get("mode") == "Status" + and outcome.get("controlledApply") is False + and outcome.get("reconcileOnly") is True + and outcome.get("runtimeWritePerformed") is False + and contract.get("state") == "resolved" + and contract.get("resolved") is True + and contract.get("transportOk") is True + and contract.get("verifierPassed") is True + and contract.get("verifierName") == AGENT99_SAME_RUN_STATUS_VERIFIER + and contract.get("sourceEventResolved") is True + and contract.get("sourceEventEvidence") == source_receipt_ref + ) + + async def _fetch_legacy_cold_start_failures( *, project_id: str, @@ -535,7 +588,15 @@ async def reconcile_agent99_controlled_dispatches_once( ) -> dict[str, int]: ledger = get_agent99_dispatch_ledger() items = await ledger.list_reconcilable(project_id=project_id, limit=limit) - counters = {"scanned": len(items), "verifier_written": 0, "closed": 0} + counters = { + "scanned": len(items), + "source_not_resolved": 0, + "status_reconcile_requested": 0, + "status_reconcile_pending": 0, + "external_no_write_reconciled": 0, + "verifier_written": 0, + "closed": 0, + } for item in items: identity = item["identity"] receipt = item.get("receipt") if isinstance(item.get("receipt"), dict) else {} @@ -552,6 +613,60 @@ async def reconcile_agent99_controlled_dispatches_once( else {} ) } + if is_cold_start_same_run_identity(identity): + source_receipt = await asyncio.to_thread( + read_cold_start_source_resolution, + ) + source_receipt_ref = str(source_receipt.get("receipt_ref") or "") + if ( + source_receipt.get("resolved") is not True + or not source_receipt_ref + ): + counters["source_not_resolved"] += 1 + continue + outcome = await asyncio.to_thread( + read_agent99_sre_outcome, + str(identity.run_id), + ) + if not _is_bound_no_write_status_outcome( + outcome, + identity=identity, + source_receipt_ref=source_receipt_ref, + ): + request_receipt = await asyncio.to_thread( + request_agent99_same_run_status_reconcile, + identity, + source_receipt, + ) + if request_receipt.get("accepted") is True: + counters["status_reconcile_requested"] += 1 + else: + counters["status_reconcile_pending"] += 1 + continue + evidence_refs = extract_agent99_outcome_evidence_refs( + outcome, + { + "agent99_outcome_receipt_id": ( + f"agent99-relay:outcome:{identity.run_id}" + ), + "post_verifier_evidence_ref": ( + "agent99:Status:no-write:" + f"{agent99_same_run_evidence_id(identity)}" + ), + "source_event_evidence_ref": source_receipt_ref, + }, + ) + terminal = await ledger.record_external_no_write_reconciliation( + identity=identity, + outcome_receipt=outcome, + source_receipt=source_receipt, + evidence_refs=evidence_refs, + ) + if terminal.get("receipt_persisted") is True: + counters["external_no_write_reconciled"] += 1 + # This path is deliberately terminal without automation success, + # incident RESOLVED, Telegram, KM, or PlayBook trust writeback. + continue if receipt.get("post_verifier_passed") is not True: outcome = await asyncio.to_thread( read_agent99_sre_outcome, @@ -573,8 +688,12 @@ async def reconcile_agent99_controlled_dispatches_once( "post_verifier_evidence_ref": ( f"agent99:{mode}:verified:{verified_at}" ), - "source_event_evidence_ref": ( - f"incident:{identity.incident_id}:source-resolved:{verified_at}" + "source_event_evidence_ref": str( + outcome_contract.get("sourceEventEvidence") + or ( + f"incident:{identity.incident_id}:" + f"source-resolved:{verified_at}" + ) ), } evidence_refs = extract_agent99_outcome_evidence_refs( diff --git a/apps/api/src/services/agent99_controlled_dispatch_ledger.py b/apps/api/src/services/agent99_controlled_dispatch_ledger.py index fab3e692f..bb6af49e5 100644 --- a/apps/api/src/services/agent99_controlled_dispatch_ledger.py +++ b/apps/api/src/services/agent99_controlled_dispatch_ledger.py @@ -10,6 +10,7 @@ from __future__ import annotations import hashlib import json +import math from dataclasses import dataclass from datetime import UTC, datetime, timedelta from typing import Any @@ -31,6 +32,7 @@ from src.services.agent99_public_receipts import ( AGENT99_BACKUP_REQUIRED_VERIFIER_EVIDENCE_REFS, AGENT99_LEARNING_RECEIPT_REFS, AGENT99_REQUIRED_VERIFIER_EVIDENCE_REFS, + normalize_agent99_public_receipt_ref, sanitize_agent99_public_receipt_refs, ) @@ -59,6 +61,19 @@ def _sha256(value: str) -> str: return hashlib.sha256(value.encode("utf-8")).hexdigest() +def _agent99_exact_nonnegative_int(value: Any) -> int | None: + """Normalize a JSON number only when it is an exact non-negative int.""" + + if isinstance(value, bool) or not isinstance(value, int | float): + return None + if not math.isfinite(float(value)): + return None + normalized = int(value) + if normalized < 0 or float(value) != float(normalized): + return None + return normalized + + @dataclass(frozen=True) class Agent99DispatchIdentity: project_id: str @@ -1329,6 +1344,366 @@ class PostgresAgent99DispatchLedger: "runtime_closure_verified": False, } + async def record_external_no_write_reconciliation( + self, + *, + identity: Agent99DispatchIdentity, + outcome_receipt: dict[str, Any], + source_receipt: dict[str, Any], + evidence_refs: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """Terminalize an externally recovered run without automation credit. + + This is intentionally separate from ``record_verifier`` and + ``record_learning_writeback``. A no-write Status proves current + posture only; it never proves that the original Recover execution + succeeded and must not resolve the incident or increase PlayBook + trust. + """ + + if not validate_agent99_outcome_identity(identity, outcome_receipt): + return { + "status": "external_recovery_identity_mismatch_fail_closed", + "receipt_persisted": False, + "runtime_closure_verified": False, + } + outcome = ( + outcome_receipt.get("outcome") + if isinstance(outcome_receipt.get("outcome"), dict) + else {} + ) + expected_top_level = { + "automationRunId": str(identity.run_id), + "traceId": identity.trace_id, + "workItemId": identity.work_item_id, + "projectId": identity.project_id, + "incidentId": identity.incident_id, + "routeId": identity.route_id, + "executionGeneration": identity.execution_generation, + "approvalId": identity.approval_id, + } + if any( + str(outcome_receipt.get(key) or "") != expected + for key, expected in expected_top_level.items() + ): + return { + "status": "external_recovery_top_level_identity_fail_closed", + "receipt_persisted": False, + "runtime_closure_verified": False, + } + source_ref = normalize_agent99_public_receipt_ref( + source_receipt.get("receipt_ref") + ) + source_values = ( + source_receipt.get("values") + if isinstance(source_receipt.get("values"), dict) + else {} + ) + source_result = str(source_receipt.get("source_result") or "") + source_counts = { + key: _agent99_exact_nonnegative_int(source_values.get(key)) + for key in ( + "monitor_up", + "pass_gates", + "warn_gates", + "blocked_gates", + "green_result", + "degraded_result", + "blocked_result", + "check_failed_result", + "firing_blocking_alerts", + ) + } + source_age = source_receipt.get("age_seconds") + source_age_exact = bool( + not isinstance(source_age, bool) + and isinstance(source_age, int | float) + and float(source_age) >= -60 + and float(source_age) <= 900 + ) + source_observed_at = _agent99_exact_nonnegative_int( + source_receipt.get("observed_at_epoch") + ) + expected_evidence_id = ( + f"same-run-status-{identity.run_id}-" + f"{identity.public_dict()['canonical_digest'][:12]}" + ) + exact_outcome = bool( + outcome_receipt.get("schemaVersion") + == "agent99_outcome_readback_v1" + and outcome_receipt.get("sameRunContractFieldsComplete") is True + and outcome_receipt.get("mode") == "Status" + and outcome_receipt.get("controlledApply") is False + and outcome_receipt.get("reconcileOnly") is True + and outcome_receipt.get("runtimeWritePerformed") is False + and outcome_receipt.get("evidenceReceiptId") + == expected_evidence_id + and outcome.get("schemaVersion") == "agent99_outcome_contract_v1" + and outcome.get("state") == "resolved" + and outcome.get("resolved") is True + and outcome.get("transportOk") is True + and outcome.get("verifierPassed") is True + and outcome.get("verifierName") + == "agent99-status-same-run-no-write-post-condition-v1" + and outcome.get("sourceEventResolved") is True + and source_ref is not None + and outcome.get("sourceEventEvidence") == source_ref + ) + source_exact = bool( + source_receipt.get("schema_version") + == "agent99_cold_start_source_resolution_v1" + and source_receipt.get("resolved") is True + and source_result in {"green", "degraded"} + and source_receipt.get("degraded_provenance") + is (source_result == "degraded") + and source_age_exact + and source_observed_at is not None + and source_observed_at > 0 + and source_counts["monitor_up"] == 1 + and source_counts["pass_gates"] is not None + and source_counts["pass_gates"] > 0 + and source_counts["warn_gates"] is not None + and source_counts["blocked_gates"] == 0 + and source_counts["blocked_result"] == 0 + and source_counts["check_failed_result"] == 0 + and source_counts["firing_blocking_alerts"] == 0 + and ( + ( + source_result == "degraded" + and source_counts["warn_gates"] > 0 + and source_counts["green_result"] == 0 + and source_counts["degraded_result"] == 1 + ) + or ( + source_result == "green" + and source_counts["warn_gates"] == 0 + and source_counts["green_result"] == 1 + and source_counts["degraded_result"] == 0 + ) + ) + ) + safe_refs = _safe_receipt_refs(evidence_refs) + missing_refs = sorted( + set(AGENT99_REQUIRED_VERIFIER_EVIDENCE_REFS) - set(safe_refs) + ) + if not exact_outcome or not source_exact or missing_refs: + return { + "status": "external_recovery_contract_invalid_fail_closed", + "missing_evidence_refs": missing_refs, + "receipt_persisted": False, + "runtime_closure_verified": False, + } + if safe_refs.get("source_event_evidence_ref") != source_ref: + return { + "status": "external_recovery_source_receipt_mismatch_fail_closed", + "receipt_persisted": False, + "runtime_closure_verified": False, + } + + now = _utc_now_naive() + terminal_receipt = { + "schema_version": ( + "agent99_external_recovery_no_write_reconciliation_v1" + ), + "status": "external_recovery_no_write_reconciled", + **_stage_identity(identity), + "mode": "Status", + "verifier_name": ( + "agent99-status-same-run-no-write-post-condition-v1" + ), + "evidence_receipt_id": expected_evidence_id, + "source": { + "receipt_ref": source_ref, + "result": source_result, + "degraded_provenance": bool( + source_receipt.get("degraded_provenance") + ), + "pass_gates": source_counts["pass_gates"], + "warn_gates": source_counts["warn_gates"], + "blocked_gates": 0, + "age_seconds": source_age, + "observed_at_epoch": source_observed_at, + }, + "transport_ok": True, + "status_verifier_passed": True, + "runtime_write_performed": False, + "recover_execution_success": False, + "incident_resolution_authorized": False, + "learning_writeback_authorized": False, + "telegram_authorized": False, + "evidence_refs": safe_refs, + "stores_raw_evidence": False, + } + + try: + async with get_db_context(identity.project_id) as db: + current_result = await db.execute( + select(AwoooPRunState.state, AwoooPRunState.error_detail) + .where( + AwoooPRunState.run_id == identity.run_id, + AwoooPRunState.project_id == identity.project_id, + ) + .with_for_update() + ) + current = current_result.one_or_none() + envelope = _parse_envelope( + current.error_detail if current is not None else None + ) + if ( + current is not None + and current.state == "cancelled" + and isinstance(envelope, dict) + and envelope.get("external_no_write_reconciliation_terminal") + is True + and _stage_identity_matches( + identity, + envelope.get("identity") + if isinstance(envelope.get("identity"), dict) + else {}, + ) + ): + return { + **envelope, + "status": ( + "external_recovery_no_write_terminal_idempotent" + ), + "receipt_persisted": True, + "runtime_closure_verified": False, + } + if ( + current is None + or current.state + not in {"pending", "running", "waiting_tool"} + or not isinstance(envelope, dict) + or not _stage_identity_matches( + identity, + envelope.get("identity") + if isinstance(envelope.get("identity"), dict) + else {}, + ) + ): + return { + "status": ( + "external_recovery_run_not_ready_fail_closed" + ), + "receipt_persisted": False, + "runtime_closure_verified": False, + } + + prior_runtime_execution_attempted = bool( + envelope.get("runtime_execution_attempted") is True + ) + envelope.update({ + "status": "external_recovery_no_write_reconciled_terminal", + "run_terminal_state": "cancelled", + "controlled_apply_authorized": False, + "runtime_execution_authorized": False, + "runtime_execution_attempted": ( + prior_runtime_execution_attempted + ), + "same_run_status_runtime_execution_attempted": False, + "runtime_write_performed": False, + "automation_execution_success": False, + "post_verifier_passed": False, + "external_status_verifier_passed": True, + "external_recovery_source_resolved": True, + "runtime_closure_verified": False, + "closure_complete": False, + "incident_resolution_authorized": False, + "external_no_write_reconciliation_terminal": True, + "external_recovery_reconciliation": terminal_receipt, + "learning_writeback": { + **_stage_identity(identity), + "step_seq": 3, + "status": ( + "not_applicable_external_recovery_no_write" + ), + "required_receipts": [], + "receipt_refs": {}, + "stores_raw_evidence": False, + }, + }) + envelope_json = _stable_json(envelope) + updated = await db.execute( + update(AwoooPRunState) + .where( + AwoooPRunState.run_id == identity.run_id, + AwoooPRunState.project_id == identity.project_id, + AwoooPRunState.state.in_([ + "pending", + "running", + "waiting_tool", + ]), + ) + .values( + state="cancelled", + output_sha256=_sha256(envelope_json), + error_code=None, + error_detail=envelope_json, + heartbeat_at=now, + completed_at=now, + lease_until=None, + next_attempt_at=None, + worker_id=None, + ) + .returning(AwoooPRunState.run_id) + ) + persisted = updated.scalar_one_or_none() is not None + if persisted: + journal_states = { + 1: ( + "compensated", + True, + "external_recovery_superseded_runtime_execution", + ), + 2: ("success", False, None), + 3: ( + "compensated", + True, + "learning_not_applicable_external_recovery_no_write", + ), + } + for step_seq, ( + result_status, + was_blocked, + block_reason, + ) in journal_states.items(): + await db.execute( + update(AwoooPRunStepJournal) + .where( + AwoooPRunStepJournal.run_id == identity.run_id, + AwoooPRunStepJournal.step_seq == step_seq, + ) + .values( + output_hash=_sha256( + _stable_json(terminal_receipt) + ), + compensation_json=terminal_receipt, + result_status=result_status, + error_code=None, + was_blocked=was_blocked, + block_reason=block_reason, + completed_at=now, + ) + ) + except Exception as exc: + logger.warning( + "agent99_external_no_write_reconciliation_failed", + incident_id=identity.incident_id, + run_id=str(identity.run_id), + error=str(exc), + ) + return { + "status": "external_recovery_persistence_failed", + "receipt_persisted": False, + "runtime_closure_verified": False, + } + return { + **envelope, + "receipt_persisted": persisted, + "runtime_closure_verified": False, + } + async def record_learning_writeback( self, *, diff --git a/apps/api/src/services/agent99_same_run_reconcile.py b/apps/api/src/services/agent99_same_run_reconcile.py new file mode 100644 index 000000000..ed4e10555 --- /dev/null +++ b/apps/api/src/services/agent99_same_run_reconcile.py @@ -0,0 +1,358 @@ +"""Fail-closed, no-write Agent99 same-run Status reconciliation. + +This module deliberately keeps the production source verifier separate from +the Agent99 verifier. A missing historical Recover outcome may be refreshed +only after the live cold-start source reports a fresh, unblocked state with no +firing ColdStart alert. The refresh keeps the existing deterministic +dispatch identity and asks the authenticated relay for ``Status`` in +``reconcileOnly`` mode; it never reserves a new run, advances a generation, +or authorizes controlled apply. +""" + +from __future__ import annotations + +import json +import math +import time +import urllib.error +import urllib.parse +import urllib.request +from typing import Any + +from src.core.config import settings +from src.core.logging import get_logger +from src.services.agent99_controlled_dispatch_ledger import ( + Agent99DispatchIdentity, +) +from src.services.agent99_public_receipts import ( + normalize_agent99_public_receipt_ref, +) + +logger = get_logger("awoooi.agent99_same_run_reconcile") + +AGENT99_SAME_RUN_RECONCILE_SCHEMA = "agent99_same_run_status_reconcile_v1" +AGENT99_COLD_START_ROUTE_ID = "agent99:host_recovery:Recover" +AGENT99_COLD_START_INCIDENT_ID = "INC-20260711-11C751" +AGENT99_COLD_START_RUN_ID = "bf30ca01-1080-5725-b2d1-3e1534dfc811" +AGENT99_SAME_RUN_STATUS_VERIFIER = ( + "agent99-status-same-run-no-write-post-condition-v1" +) +AGENT99_COLD_START_METRIC_HOST = "110" +AGENT99_COLD_START_METRIC_SCOPE = "110_120_121_188" +AGENT99_COLD_START_MAX_AGE_SECONDS = 900 + +_COLD_START_QUERIES = { + "monitor_up": ( + 'awoooi_cold_start_monitor_up{host="110",scope="110_120_121_188",' + 'mode="read_only"}' + ), + "pass_gates": ( + 'awoooi_cold_start_pass_gates{host="110",scope="110_120_121_188"}' + ), + "warn_gates": ( + 'awoooi_cold_start_warn_gates{host="110",scope="110_120_121_188"}' + ), + "blocked_gates": ( + 'awoooi_cold_start_blocked_gates{host="110",scope="110_120_121_188"}' + ), + "last_run_timestamp": ( + 'awoooi_cold_start_last_run_timestamp{host="110",' + 'scope="110_120_121_188"}' + ), + "green_result": ( + 'awoooi_cold_start_last_result{host="110",scope="110_120_121_188",' + 'result="green"}' + ), + "degraded_result": ( + 'awoooi_cold_start_last_result{host="110",scope="110_120_121_188",' + 'result="degraded"}' + ), + "blocked_result": ( + 'awoooi_cold_start_last_result{host="110",scope="110_120_121_188",' + 'result="blocked"}' + ), + "check_failed_result": ( + 'awoooi_cold_start_last_result{host="110",scope="110_120_121_188",' + 'result="check_failed"}' + ), + "firing_blocking_alerts": ( + 'sum(ALERTS{alertname=~"ColdStart.*",alertstate="firing",' + 'severity="critical"}) or vector(0)' + ), +} + + +def is_cold_start_same_run_identity(identity: Agent99DispatchIdentity) -> bool: + """Limit this transport to the existing cold-start Recover identity.""" + + return bool( + identity.route_id == AGENT99_COLD_START_ROUTE_ID + and identity.incident_id == AGENT99_COLD_START_INCIDENT_ID + and str(identity.run_id) == AGENT99_COLD_START_RUN_ID + and identity.source_fingerprint.startswith("legacy-cold-start:") + and identity.execution_generation == "1" + ) + + +def agent99_same_run_evidence_id(identity: Agent99DispatchIdentity) -> str: + """Return the only evidence identifier accepted for this bounded run.""" + + digest = identity.public_dict()["canonical_digest"] + return f"same-run-status-{identity.run_id}-{digest[:12]}" + + +def _query_prometheus_scalar( + query_url: str, + query: str, + *, + timeout_seconds: float, +) -> float | None: + target = f"{query_url}?{urllib.parse.urlencode({'query': query})}" + request = urllib.request.Request( + target, + headers={ + "Accept": "application/json", + "User-Agent": "awoooi-agent99-source-verifier/1", + }, + method="GET", + ) + try: + with urllib.request.urlopen(request, timeout=timeout_seconds) as response: + if int(response.getcode()) != 200: + return None + body = response.read(65536).decode("utf-8", errors="replace") + except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError): + return None + try: + payload = json.loads(body) + data = payload.get("data") if isinstance(payload, dict) else None + result = data.get("result") if isinstance(data, dict) else None + if payload.get("status") != "success" or not isinstance(result, list): + return None + if len(result) != 1 or not isinstance(result[0], dict): + return None + sample = result[0].get("value") + if not isinstance(sample, list) or len(sample) != 2: + return None + value = float(sample[1]) + return value if math.isfinite(value) else None + except (AttributeError, TypeError, ValueError, json.JSONDecodeError): + return None + + +def read_cold_start_source_resolution( + *, + prometheus_url: str | None = None, + timeout_seconds: float = 3.0, + now_epoch: float | None = None, +) -> dict[str, Any]: + """Read a bounded production source receipt; missing evidence fails closed.""" + + base_url = str(prometheus_url or settings.PROMETHEUS_URL).rstrip("/") + query_url = f"{base_url}/api/v1/query" + values: dict[str, float] = {} + missing: list[str] = [] + for name, query in _COLD_START_QUERIES.items(): + value = _query_prometheus_scalar( + query_url, + query, + timeout_seconds=timeout_seconds, + ) + if value is None: + missing.append(name) + else: + values[name] = value + + blockers: list[str] = [] + if missing: + blockers.append("source_metrics_missing:" + ",".join(sorted(missing))) + observed_at = float(now_epoch if now_epoch is not None else time.time()) + last_run = values.get("last_run_timestamp") + age_seconds: float | None = None + if last_run is not None: + age_seconds = observed_at - last_run + if age_seconds < -60: + blockers.append("source_clock_skew_future") + elif age_seconds > AGENT99_COLD_START_MAX_AGE_SECONDS: + blockers.append("source_metrics_stale") + if values.get("monitor_up") != 1: + blockers.append("cold_start_monitor_not_up") + for count_name in ("pass_gates", "warn_gates", "blocked_gates"): + count_value = values.get(count_name) + if count_value is not None and ( + count_value < 0 or not count_value.is_integer() + ): + blockers.append(f"cold_start_{count_name}_invalid") + if values.get("pass_gates", 0) <= 0: + blockers.append("cold_start_pass_gates_missing") + if values.get("blocked_gates") != 0: + blockers.append("cold_start_blocked_gates_present") + if values.get("blocked_result") != 0: + blockers.append("cold_start_result_blocked") + if values.get("check_failed_result") != 0: + blockers.append("cold_start_result_check_failed") + warn_gates = values.get("warn_gates") + expected_result = "degraded" if warn_gates is not None and warn_gates > 0 else "green" + if expected_result == "degraded": + if values.get("degraded_result") != 1 or values.get("green_result") != 0: + blockers.append("cold_start_degraded_result_not_one_hot") + elif values.get("green_result") != 1 or values.get("degraded_result") != 0: + blockers.append("cold_start_green_result_not_one_hot") + if values.get("firing_blocking_alerts") != 0: + blockers.append("cold_start_blocking_alert_still_firing") + + resolved = bool(not blockers and last_run is not None) + receipt_ref: str | None = None + if resolved: + receipt_ref = normalize_agent99_public_receipt_ref( + "prometheus:cold-start:" + f"{int(last_run)}:pass-{int(values['pass_gates'])}:" + f"warn-{int(values['warn_gates'])}:blocked-0:" + f"result-{expected_result}:blocking-alerts-0" + ) + if receipt_ref is None: + blockers.append("source_receipt_ref_invalid") + resolved = False + + return { + "schema_version": "agent99_cold_start_source_resolution_v1", + "status": "source_resolved" if resolved else "source_not_resolved", + "resolved": resolved, + "source_result": expected_result if resolved else "unresolved", + "degraded_provenance": bool(resolved and expected_result == "degraded"), + "receipt_ref": receipt_ref, + "observed_at_epoch": int(observed_at), + "age_seconds": round(age_seconds, 3) if age_seconds is not None else None, + "values": { + key: int(value) if float(value).is_integer() else value + for key, value in sorted(values.items()) + }, + "blockers": blockers, + "stores_raw_evidence": False, + } + + +def request_agent99_same_run_status_reconcile( + identity: Agent99DispatchIdentity, + source_receipt: dict[str, Any], + *, + relay_url: str | None = None, + relay_token: str | None = None, + timeout_seconds: float | None = None, +) -> dict[str, Any]: + """Ask Agent99 for one idempotent, no-write Status on the existing run.""" + + if not is_cold_start_same_run_identity(identity): + return { + "status": "identity_not_eligible_fail_closed", + "accepted": False, + "runtime_write_authorized": False, + } + receipt_ref = normalize_agent99_public_receipt_ref( + source_receipt.get("receipt_ref") + ) + if source_receipt.get("resolved") is not True or not receipt_ref: + return { + "status": "source_not_resolved_fail_closed", + "accepted": False, + "runtime_write_authorized": False, + } + if not receipt_ref.startswith("prometheus:cold-start:"): + return { + "status": "source_receipt_not_cold_start_fail_closed", + "accepted": False, + "runtime_write_authorized": False, + } + + target_url = str( + relay_url + if relay_url is not None + else settings.AGENT99_SRE_ALERT_RELAY_URL + ) + token = str( + relay_token + if relay_token is not None + else settings.AGENT99_SRE_ALERT_RELAY_TOKEN + ) + if not target_url or not token: + return { + "status": "relay_auth_not_configured_fail_closed", + "accepted": False, + "runtime_write_authorized": False, + } + timeout = float( + timeout_seconds + if timeout_seconds is not None + else settings.AGENT99_SRE_ALERT_RELAY_TIMEOUT_SECONDS + ) + payload = { + "schemaVersion": AGENT99_SAME_RUN_RECONCILE_SCHEMA, + "action": "same_run_status_reconcile", + "identity": identity.public_dict(), + "mode": "Status", + "controlledApply": False, + "reconcileOnly": True, + "sourceEventResolved": True, + "sourceEventEvidence": receipt_ref, + "sourceEventResolutionPolicy": "external_source_receipt_required", + "suppressTelegram": True, + "evidenceReceiptId": agent99_same_run_evidence_id(identity), + } + request = urllib.request.Request( + target_url, + data=json.dumps(payload, ensure_ascii=False, sort_keys=True).encode("utf-8"), + headers={ + "Accept": "application/json", + "Content-Type": "application/json; charset=utf-8", + "User-Agent": "awoooi-agent99-same-run-reconciler/1", + "X-Agent99-Relay-Token": token, + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + status_code = int(response.getcode()) + body = response.read(65536).decode("utf-8", errors="replace") + except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError) as exc: + logger.info( + "agent99_same_run_status_reconcile_pending", + run_id=str(identity.run_id), + error_type=type(exc).__name__, + ) + return { + "status": "relay_request_failed_fail_closed", + "accepted": False, + "runtime_write_authorized": False, + } + try: + response_payload = json.loads(body) + except json.JSONDecodeError: + response_payload = {} + accepted = bool( + 200 <= status_code < 300 + and isinstance(response_payload, dict) + and response_payload.get("ok") is True + and response_payload.get("reconcileOnly") is True + and response_payload.get("controlledApply") is False + and str(response_payload.get("automationRunId") or "") + == str(identity.run_id) + and str(response_payload.get("evidenceReceiptId") or "") + == agent99_same_run_evidence_id(identity) + ) + return { + "schema_version": "agent99_same_run_status_reconcile_request_receipt_v1", + "status": ( + str(response_payload.get("status") or "accepted") + if accepted + else "relay_response_rejected_fail_closed" + ), + "accepted": accepted, + "http_status": status_code, + "identity": identity.public_dict(), + "source_event_evidence_ref": receipt_ref, + "evidence_receipt_id": agent99_same_run_evidence_id(identity), + "reconcile_only": True, + "controlled_apply": False, + "runtime_write_authorized": False, + "stores_raw_response": False, + } diff --git a/apps/api/src/services/agent99_sre_bridge.py b/apps/api/src/services/agent99_sre_bridge.py index 8dabf1df3..13cde411d 100644 --- a/apps/api/src/services/agent99_sre_bridge.py +++ b/apps/api/src/services/agent99_sre_bridge.py @@ -21,6 +21,7 @@ from src.services.agent99_controlled_dispatch_ledger import ( ) from src.services.agent99_public_receipts import ( extract_agent99_outcome_evidence_refs, + normalize_agent99_public_receipt_ref, ) from src.utils.timezone import now_taipei @@ -697,17 +698,101 @@ def read_agent99_sre_outcome( ): return None safe_evidence_refs = extract_agent99_outcome_evidence_refs(payload) + source_event_evidence = normalize_agent99_public_receipt_ref( + outcome.get("sourceEventEvidence") + ) + expected_evidence_receipt_id = ( + f"same-run-status-{identity.run_id}-" + f"{identity.public_dict()['canonical_digest'][:12]}" + ) + evidence_receipt_id = str(payload.get("evidenceReceiptId") or "") + if evidence_receipt_id != expected_evidence_receipt_id: + evidence_receipt_id = "" + same_run_contract_fields_complete = bool( + payload.get("sameRunContractFieldsComplete") is True + and all( + key in payload + for key in ( + "automationRunId", + "traceId", + "workItemId", + "projectId", + "incidentId", + "routeId", + "executionGeneration", + "approvalId", + "evidenceReceiptId", + "controlledApply", + "reconcileOnly", + "runtimeWritePerformed", + "mode", + ) + ) + and all( + isinstance(payload.get(key), str) + for key in ( + "automationRunId", + "traceId", + "workItemId", + "projectId", + "incidentId", + "routeId", + "executionGeneration", + "approvalId", + "evidenceReceiptId", + "mode", + ) + ) + and isinstance(payload.get("controlledApply"), bool) + and isinstance(payload.get("reconcileOnly"), bool) + and isinstance(payload.get("runtimeWritePerformed"), bool) + and all( + key in outcome + for key in ( + "schemaVersion", + "state", + "resolved", + "transportOk", + "verifierName", + "verifierPassed", + "sourceEventResolved", + "sourceEventEvidence", + "identity", + ) + ) + and all( + isinstance(outcome.get(key), str) + for key in ( + "schemaVersion", + "state", + "verifierName", + "sourceEventEvidence", + ) + ) + and isinstance(outcome.get("resolved"), bool) + and isinstance(outcome.get("transportOk"), bool) + and isinstance(outcome.get("verifierPassed"), bool) + and isinstance(outcome.get("sourceEventResolved"), bool) + ) return { "schemaVersion": "agent99_outcome_readback_v1", "automationRunId": str(payload.get("automationRunId") or ""), "traceId": str(payload.get("traceId") or ""), "workItemId": str(payload.get("workItemId") or ""), + "projectId": str(payload.get("projectId") or ""), + "routeId": str(payload.get("routeId") or ""), "idempotencyKey": str(payload.get("idempotencyKey") or ""), "executionGeneration": str(payload.get("executionGeneration") or "1"), "incidentId": str(payload.get("incidentId") or ""), "approvalId": str(payload.get("approvalId") or ""), + "evidenceReceiptId": evidence_receipt_id, + "sameRunContractFieldsComplete": same_run_contract_fields_complete, "identity": identity.public_dict(), "controlledApply": bool(payload.get("controlledApply") is True), + "reconcileOnly": bool(payload.get("reconcileOnly") is True), + "runtimeWritePerformed": bool( + payload.get("runtimeWritePerformed") is True + ), "mode": str(payload.get("mode") or ""), "outcome": { "schemaVersion": str(outcome.get("schemaVersion") or ""), @@ -719,6 +804,7 @@ def read_agent99_sre_outcome( "sourceEventResolved": bool( outcome.get("sourceEventResolved") is True ), + "sourceEventEvidence": source_event_evidence or "", "failedChecks": [ str(value)[:120] for value in (outcome.get("failedChecks") or [])[:32] diff --git a/apps/api/tests/test_agent99_alert_dispatch_order_contract.py b/apps/api/tests/test_agent99_alert_dispatch_order_contract.py index f55704fa0..9d9fa1c56 100644 --- a/apps/api/tests/test_agent99_alert_dispatch_order_contract.py +++ b/apps/api/tests/test_agent99_alert_dispatch_order_contract.py @@ -134,7 +134,7 @@ def test_agent99_relay_exposes_authenticated_public_safe_outcome_readback() -> N assert "function Get-AgentOutcomeReadback" in source assert '$context.Request.HttpMethod -eq "GET"' in source - assert "outcome_readback_auth_not_configured" in source + assert "relay_auth_not_configured" in source assert 'Headers["X-Agent99-Relay-Token"]' in source assert "storesRawEvidence = $false" in source diff --git a/apps/api/tests/test_agent99_controlled_dispatch_reconciler_job.py b/apps/api/tests/test_agent99_controlled_dispatch_reconciler_job.py index 4b6a4ab95..b230d7722 100644 --- a/apps/api/tests/test_agent99_controlled_dispatch_reconciler_job.py +++ b/apps/api/tests/test_agent99_controlled_dispatch_reconciler_job.py @@ -95,7 +95,15 @@ async def test_reconciler_consumes_authenticated_outcome_and_closes_same_run( result = await job.reconcile_agent99_controlled_dispatches_once() - assert result == {"scanned": 1, "verifier_written": 1, "closed": 1} + assert result == { + "scanned": 1, + "source_not_resolved": 0, + "status_reconcile_requested": 0, + "status_reconcile_pending": 0, + "external_no_write_reconciled": 0, + "verifier_written": 1, + "closed": 1, + } assert len(ledger.verifier_calls) == 1 refs = ledger.verifier_calls[0]["evidence_refs"] assert refs["agent99_outcome_receipt_id"].endswith(str(IDENTITY.run_id)) @@ -116,7 +124,15 @@ async def test_reconciler_does_not_finalize_without_outcome(monkeypatch) -> None result = await job.reconcile_agent99_controlled_dispatches_once() - assert result == {"scanned": 1, "verifier_written": 0, "closed": 0} + assert result == { + "scanned": 1, + "source_not_resolved": 0, + "status_reconcile_requested": 0, + "status_reconcile_pending": 0, + "external_no_write_reconciled": 0, + "verifier_written": 0, + "closed": 0, + } assert ledger.verifier_calls == [] @@ -125,8 +141,12 @@ def test_windows_relay_outcome_readback_is_authenticated_and_public_safe() -> No text = source.read_text(encoding="utf-8") assert '$context.Request.HttpMethod -eq "GET"' in text - assert "outcome_readback_auth_not_configured" in text + assert "relay_auth_not_configured" in text assert 'Headers["X-Agent99-Relay-Token"]' in text + token_gate = text.index("if (-not $expectedToken)") + get_branch = text.index('if ($context.Request.HttpMethod -eq "GET")') + post_branch = text.index('if ($context.Request.HttpMethod -ne "POST")') + assert token_gate < get_branch < post_branch assert "storesRawEvidence = $false" in text assert "stdout" not in text[text.index("function Get-AgentOutcomeReadback"):text.index("function Write-AgentRelayEvent")] diff --git a/apps/api/tests/test_agent99_same_run_reconcile.py b/apps/api/tests/test_agent99_same_run_reconcile.py new file mode 100644 index 000000000..da80a582f --- /dev/null +++ b/apps/api/tests/test_agent99_same_run_reconcile.py @@ -0,0 +1,820 @@ +from __future__ import annotations + +import json +from copy import deepcopy +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from src.jobs import agent99_controlled_dispatch_reconciler_job as job +from src.services import ( + agent99_controlled_dispatch_ledger as ledger_module, +) +from src.services import agent99_same_run_reconcile as reconcile +from src.services import agent99_sre_bridge as bridge +from src.services.agent99_controlled_dispatch_ledger import ( + PostgresAgent99DispatchLedger, + build_agent99_dispatch_identity, + build_agent99_dispatch_receipt_envelope, +) + +IDENTITY = build_agent99_dispatch_identity( + project_id="awoooi", + incident_id="INC-20260711-11C751", + source_fingerprint="legacy-cold-start:" + "a" * 64, + route_id="agent99:host_recovery:Recover", + approval_id="00000000-0000-0000-0000-000000000751", + work_item_id=( + "agent99-dispatch:awoooi:INC-20260711-11C751:legacy-cold-start" + ), +) + + +@pytest.fixture(autouse=True) +def _bind_test_identity_to_bounded_run(monkeypatch) -> None: + monkeypatch.setattr( + reconcile, + "AGENT99_COLD_START_RUN_ID", + str(IDENTITY.run_id), + ) + + +def _source_receipt() -> dict[str, object]: + return { + "schema_version": "agent99_cold_start_source_resolution_v1", + "status": "source_resolved", + "resolved": True, + "source_result": "degraded", + "degraded_provenance": True, + "receipt_ref": ( + "prometheus:cold-start:1784020000:pass-95:" + "warn-1:blocked-0:result-degraded:blocking-alerts-0" + ), + "observed_at_epoch": 1784020030, + "age_seconds": 30.0, + "values": { + "monitor_up": 1, + "pass_gates": 95, + "warn_gates": 1, + "blocked_gates": 0, + "green_result": 0, + "degraded_result": 1, + "blocked_result": 0, + "check_failed_result": 0, + "firing_blocking_alerts": 0, + }, + "blockers": [], + "stores_raw_evidence": False, + } + + +def _valid_outcome() -> dict[str, object]: + identity = IDENTITY.public_dict() + return { + "schemaVersion": "agent99_outcome_readback_v1", + "sameRunContractFieldsComplete": True, + "automationRunId": str(IDENTITY.run_id), + "traceId": IDENTITY.trace_id, + "workItemId": IDENTITY.work_item_id, + "projectId": IDENTITY.project_id, + "incidentId": IDENTITY.incident_id, + "routeId": IDENTITY.route_id, + "executionGeneration": IDENTITY.execution_generation, + "approvalId": IDENTITY.approval_id, + "evidenceReceiptId": reconcile.agent99_same_run_evidence_id(IDENTITY), + "identity": identity, + "mode": "Status", + "controlledApply": False, + "reconcileOnly": True, + "runtimeWritePerformed": False, + "outcome": { + "schemaVersion": "agent99_outcome_contract_v1", + "state": "resolved", + "resolved": True, + "transportOk": True, + "verifierName": reconcile.AGENT99_SAME_RUN_STATUS_VERIFIER, + "verifierPassed": True, + "sourceEventResolved": True, + "sourceEventEvidence": _source_receipt()["receipt_ref"], + "identity": identity, + "verifiedAt": "2026-07-14T16:30:00+08:00", + }, + } + + +def _evidence_refs() -> dict[str, str]: + return { + "agent99_outcome_receipt_id": ( + f"agent99-relay:outcome:{IDENTITY.run_id}" + ), + "post_verifier_evidence_ref": ( + "agent99:Status:no-write:" + f"{reconcile.agent99_same_run_evidence_id(IDENTITY)}" + ), + "source_event_evidence_ref": str(_source_receipt()["receipt_ref"]), + } + + +def test_cold_start_source_resolution_allows_warning_only_but_requires_fresh_zero( + monkeypatch, +) -> None: + values = { + "monitor_up": 1.0, + "pass_gates": 95.0, + "warn_gates": 1.0, + "blocked_gates": 0.0, + "last_run_timestamp": 1784020000.0, + "green_result": 0.0, + "degraded_result": 1.0, + "blocked_result": 0.0, + "check_failed_result": 0.0, + "firing_blocking_alerts": 0.0, + } + + def fake_query(_url, query, *, timeout_seconds): # type: ignore[no-untyped-def] + del timeout_seconds + for name, expected_query in reconcile._COLD_START_QUERIES.items(): + if query == expected_query: + return values[name] + raise AssertionError(query) + + monkeypatch.setattr(reconcile, "_query_prometheus_scalar", fake_query) + + receipt = reconcile.read_cold_start_source_resolution( + prometheus_url="http://prometheus.invalid", + now_epoch=1784020030, + ) + + assert receipt["resolved"] is True + assert receipt["status"] == "source_resolved" + assert receipt["values"]["warn_gates"] == 1 + assert receipt["values"]["blocked_gates"] == 0 + assert receipt["receipt_ref"] == ( + "prometheus:cold-start:1784020000:pass-95:" + "warn-1:blocked-0:result-degraded:blocking-alerts-0" + ) + assert receipt["stores_raw_evidence"] is False + + +@pytest.mark.parametrize( + ("field", "value", "blocker"), + [ + ("blocked_gates", 1.0, "cold_start_blocked_gates_present"), + ( + "firing_blocking_alerts", + 1.0, + "cold_start_blocking_alert_still_firing", + ), + ("monitor_up", 0.0, "cold_start_monitor_not_up"), + ], +) +def test_cold_start_source_resolution_fails_closed( + monkeypatch, + field: str, + value: float, + blocker: str, +) -> None: + values = { + "monitor_up": 1.0, + "pass_gates": 95.0, + "warn_gates": 0.0, + "blocked_gates": 0.0, + "last_run_timestamp": 1784020000.0, + "green_result": 1.0, + "degraded_result": 0.0, + "blocked_result": 0.0, + "check_failed_result": 0.0, + "firing_blocking_alerts": 0.0, + } + values[field] = value + query_to_name = { + query: name for name, query in reconcile._COLD_START_QUERIES.items() + } + monkeypatch.setattr( + reconcile, + "_query_prometheus_scalar", + lambda _url, query, **_kwargs: values[query_to_name[query]], + ) + + receipt = reconcile.read_cold_start_source_resolution( + prometheus_url="http://prometheus.invalid", + now_epoch=1784020030, + ) + + assert receipt["resolved"] is False + assert blocker in receipt["blockers"] + assert receipt["receipt_ref"] is None + + +def test_same_run_status_request_preserves_identity_and_no_write_fences( + monkeypatch, +) -> None: + seen: dict[str, object] = {} + + class FakeResponse: + def getcode(self) -> int: + return 202 + + def read(self, _limit: int) -> bytes: + return json.dumps({ + "ok": True, + "status": "same_run_status_reconcile_queued", + "automationRunId": str(IDENTITY.run_id), + "reconcileOnly": True, + "controlledApply": False, + "evidenceReceiptId": reconcile.agent99_same_run_evidence_id( + IDENTITY + ), + }).encode() + + def __enter__(self): # type: ignore[no-untyped-def] + return self + + def __exit__(self, *_args): # type: ignore[no-untyped-def] + return None + + def fake_urlopen(request, timeout): # type: ignore[no-untyped-def] + seen["url"] = request.full_url + seen["timeout"] = timeout + seen["payload"] = json.loads(request.data.decode()) + return FakeResponse() + + monkeypatch.setattr(reconcile.urllib.request, "urlopen", fake_urlopen) + + result = reconcile.request_agent99_same_run_status_reconcile( + IDENTITY, + _source_receipt(), + relay_url="http://agent99.invalid/agent99/sre-alert/", + relay_token="configured-not-returned", + timeout_seconds=2, + ) + + assert result["accepted"] is True + assert result["runtime_write_authorized"] is False + payload = seen["payload"] + assert payload["schemaVersion"] == "agent99_same_run_status_reconcile_v1" + assert payload["mode"] == "Status" + assert payload["controlledApply"] is False + assert payload["reconcileOnly"] is True + assert payload["suppressTelegram"] is True + assert payload["evidenceReceiptId"] == reconcile.agent99_same_run_evidence_id( + IDENTITY + ) + assert payload["identity"] == IDENTITY.public_dict() + assert payload["identity"]["execution_generation"] == "1" + assert "configured-not-returned" not in json.dumps(result) + + +def test_same_run_status_request_rejects_unresolved_source_without_transport( + monkeypatch, +) -> None: + monkeypatch.setattr( + reconcile.urllib.request, + "urlopen", + lambda *_args, **_kwargs: pytest.fail("transport must not run"), + ) + + result = reconcile.request_agent99_same_run_status_reconcile( + IDENTITY, + {"resolved": False, "receipt_ref": None}, + relay_url="http://agent99.invalid/agent99/sre-alert/", + relay_token="configured", + ) + + assert result == { + "status": "source_not_resolved_fail_closed", + "accepted": False, + "runtime_write_authorized": False, + } + + +@pytest.mark.parametrize( + ("relay_complete", "remove_runtime_field", "invalid_generation_type"), + [(False, False, False), (True, True, False), (True, False, True)], +) +def test_outcome_readback_rejects_incomplete_relay_contract( + monkeypatch, + relay_complete: bool, + remove_runtime_field: bool, + invalid_generation_type: bool, +) -> None: + payload = deepcopy(_valid_outcome()) + payload["sameRunContractFieldsComplete"] = relay_complete + if remove_runtime_field: + payload.pop("runtimeWritePerformed") + if invalid_generation_type: + payload["executionGeneration"] = 1 + + class FakeResponse: + def getcode(self) -> int: + return 200 + + def read(self, _limit: int) -> bytes: + return json.dumps({"found": True, **payload}).encode() + + def __enter__(self): # type: ignore[no-untyped-def] + return self + + def __exit__(self, *_args): # type: ignore[no-untyped-def] + return None + + monkeypatch.setattr( + bridge.urllib.request, + "urlopen", + lambda *_args, **_kwargs: FakeResponse(), + ) + + result = bridge.read_agent99_sre_outcome( + str(IDENTITY.run_id), + relay_url="http://agent99.invalid/agent99/sre-alert/", + relay_token="configured-not-returned", + timeout_seconds=2, + ) + + assert result is not None + assert result["sameRunContractFieldsComplete"] is False + assert job._is_bound_no_write_status_outcome( + result, + identity=IDENTITY, + source_receipt_ref=str(_source_receipt()["receipt_ref"]), + ) is False + + +class _Ledger: + def __init__(self) -> None: + self.verifier_calls: list[dict[str, object]] = [] + self.external_calls: list[dict[str, object]] = [] + + async def list_reconcilable(self, **_kwargs): # type: ignore[no-untyped-def] + return [{ + "identity": IDENTITY, + "receipt": { + "status": "dispatch_accepted_verifier_pending", + "post_verifier_passed": False, + }, + }] + + async def record_verifier(self, **kwargs): # type: ignore[no-untyped-def] + self.verifier_calls.append(kwargs) + return { + "post_verifier_passed": True, + "receipt_persisted": True, + "verifier": {"evidence_refs": kwargs["evidence_refs"]}, + } + + async def record_external_no_write_reconciliation( + self, + **kwargs, + ): # type: ignore[no-untyped-def] + self.external_calls.append(kwargs) + return { + "status": "external_recovery_no_write_reconciled_terminal", + "receipt_persisted": True, + "runtime_closure_verified": False, + } + + +@pytest.mark.asyncio +async def test_reconciler_requests_status_only_after_production_source_resolves( + monkeypatch, +) -> None: + ledger = _Ledger() + requests: list[tuple[object, object]] = [] + monkeypatch.setattr(job, "get_agent99_dispatch_ledger", lambda: ledger) + monkeypatch.setattr(job, "read_agent99_sre_outcome", lambda _run: None) + monkeypatch.setattr( + job, + "read_cold_start_source_resolution", + lambda: _source_receipt(), + ) + monkeypatch.setattr( + job, + "request_agent99_same_run_status_reconcile", + lambda identity, source: requests.append((identity, source)) + or {"accepted": True}, + ) + + result = await job.reconcile_agent99_controlled_dispatches_once() + + assert result["status_reconcile_requested"] == 1 + assert result["verifier_written"] == 0 + assert requests == [(IDENTITY, _source_receipt())] + assert ledger.verifier_calls == [] + + +@pytest.mark.asyncio +async def test_reconciler_consumes_only_bound_no_write_status_outcome( + monkeypatch, +) -> None: + ledger = _Ledger() + source = _source_receipt() + outcome = _valid_outcome() + monkeypatch.setattr(job, "get_agent99_dispatch_ledger", lambda: ledger) + monkeypatch.setattr(job, "read_agent99_sre_outcome", lambda _run: outcome) + monkeypatch.setattr(job, "read_cold_start_source_resolution", lambda: source) + monkeypatch.setattr( + job, + "request_agent99_same_run_status_reconcile", + lambda *_args: pytest.fail("fresh bound outcome must be consumed"), + ) + finalize = AsyncMock(side_effect=AssertionError("learning must not run")) + telegram = AsyncMock(side_effect=AssertionError("Telegram must not run")) + playbook = AsyncMock(side_effect=AssertionError("PlayBook must not run")) + km = AsyncMock(side_effect=AssertionError("KM must not run")) + monkeypatch.setattr(job, "_finalize_learning", finalize) + monkeypatch.setattr(job, "_ensure_telegram_receipt", telegram) + monkeypatch.setattr(job, "_ensure_playbook_trust", playbook) + monkeypatch.setattr(job, "_ensure_km_writeback", km) + + result = await job.reconcile_agent99_controlled_dispatches_once() + + assert result["external_no_write_reconciled"] == 1 + assert result["verifier_written"] == 0 + assert len(ledger.external_calls) == 1 + assert ( + ledger.external_calls[0]["evidence_refs"]["source_event_evidence_ref"] + == source["receipt_ref"] + ) + assert ledger.verifier_calls == [] + finalize.assert_not_awaited() + telegram.assert_not_awaited() + playbook.assert_not_awaited() + km.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_reconciler_rejects_status_outcome_with_mismatched_identity( + monkeypatch, +) -> None: + ledger = _Ledger() + source = _source_receipt() + other = build_agent99_dispatch_identity( + project_id=IDENTITY.project_id, + incident_id=IDENTITY.incident_id, + source_fingerprint=IDENTITY.source_fingerprint, + route_id=IDENTITY.route_id, + work_item_id="different-work-item", + ) + outcome = { + "identity": other.public_dict(), + "mode": "Status", + "controlledApply": False, + "reconcileOnly": True, + "runtimeWritePerformed": False, + "outcome": { + "schemaVersion": "agent99_outcome_contract_v1", + "sourceEventResolved": True, + "sourceEventEvidence": source["receipt_ref"], + "identity": other.public_dict(), + }, + } + requests: list[tuple[object, object]] = [] + monkeypatch.setattr(job, "get_agent99_dispatch_ledger", lambda: ledger) + monkeypatch.setattr(job, "read_agent99_sre_outcome", lambda _run: outcome) + monkeypatch.setattr(job, "read_cold_start_source_resolution", lambda: source) + monkeypatch.setattr( + job, + "request_agent99_same_run_status_reconcile", + lambda identity, receipt: requests.append((identity, receipt)) + or {"accepted": True}, + ) + + result = await job.reconcile_agent99_controlled_dispatches_once() + + assert result["status_reconcile_requested"] == 1 + assert result["verifier_written"] == 0 + assert requests == [(IDENTITY, source)] + assert ledger.verifier_calls == [] + + +def _mutate_path(payload: dict[str, object], path: str, value: object) -> None: + target: dict[str, object] = payload + parts = path.split(".") + for part in parts[:-1]: + nested = target[part] + assert isinstance(nested, dict) + target = nested + target[parts[-1]] = value + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("path", "value"), + [ + ("outcome.state", "degraded"), + ("outcome.state", "failed"), + ("outcome.resolved", False), + ("outcome.transportOk", False), + ("outcome.verifierPassed", False), + ("outcome.verifierName", "agent99-status-post-condition-v1"), + ("reconcileOnly", False), + ("runtimeWritePerformed", True), + ("sameRunContractFieldsComplete", False), + ("traceId", "00-00000000000000000000000000000000-0000000000000000-01"), + ("workItemId", "different-work-item"), + ("projectId", "other-project"), + ("incidentId", "INC-OTHER"), + ("automationRunId", "00000000-0000-0000-0000-000000000000"), + ("routeId", "agent99:host_recovery:Status"), + ("executionGeneration", "2"), + ("evidenceReceiptId", "same-run-status-wrong"), + ("outcome.sourceEventEvidence", "prometheus:cold-start:stale"), + ], +) +async def test_reconciler_rejects_synthetic_bad_same_run_outcomes( + monkeypatch, + path: str, + value: object, +) -> None: + ledger = _Ledger() + source = _source_receipt() + outcome = deepcopy(_valid_outcome()) + _mutate_path(outcome, path, value) + requests: list[tuple[object, object]] = [] + monkeypatch.setattr(job, "get_agent99_dispatch_ledger", lambda: ledger) + monkeypatch.setattr(job, "read_agent99_sre_outcome", lambda _run: outcome) + monkeypatch.setattr(job, "read_cold_start_source_resolution", lambda: source) + monkeypatch.setattr( + job, + "request_agent99_same_run_status_reconcile", + lambda identity, receipt: requests.append((identity, receipt)) + or {"accepted": True}, + ) + finalize = AsyncMock(side_effect=AssertionError("learning must not run")) + monkeypatch.setattr(job, "_finalize_learning", finalize) + + result = await job.reconcile_agent99_controlled_dispatches_once() + + assert result["status_reconcile_requested"] == 1 + assert result["external_no_write_reconciled"] == 0 + assert result["verifier_written"] == 0 + assert requests == [(IDENTITY, source)] + assert ledger.external_calls == [] + assert ledger.verifier_calls == [] + finalize.assert_not_awaited() + + +class _ScalarResult: + def __init__(self, *, value=None, row=None) -> None: + self.value = value + self.row = row + + def scalar_one_or_none(self): # type: ignore[no-untyped-def] + return self.value + + def one_or_none(self): # type: ignore[no-untyped-def] + return self.row + + +class _Context: + def __init__(self, db) -> None: # type: ignore[no-untyped-def] + self.db = db + + async def __aenter__(self): # type: ignore[no-untyped-def] + return self.db + + async def __aexit__(self, *_args): # type: ignore[no-untyped-def] + return False + + +@pytest.mark.asyncio +async def test_external_no_write_terminal_never_records_recover_success_or_closure( + monkeypatch, +) -> None: + envelope = build_agent99_dispatch_receipt_envelope( + identity=IDENTITY, + dispatch_receipt={ + "accepted": True, + "inbox_triggered": True, + "delivery_certainty": "delivered", + }, + controlled_apply_authorized=True, + ) + envelope["runtime_execution_attempted"] = True + + class ReconcileDB: + def __init__(self) -> None: + self.call = 0 + self.statements: list[str] = [] + + async def execute(self, statement): # type: ignore[no-untyped-def] + self.call += 1 + self.statements.append(str(statement)) + if self.call == 1: + return _ScalarResult( + row=SimpleNamespace( + state="waiting_tool", + error_detail=json.dumps(envelope), + ) + ) + if self.call == 2: + return _ScalarResult(value=IDENTITY.run_id) + return _ScalarResult() + + db = ReconcileDB() + monkeypatch.setattr( + ledger_module, + "get_db_context", + lambda _project_id: _Context(db), + ) + + result = await PostgresAgent99DispatchLedger().record_external_no_write_reconciliation( + identity=IDENTITY, + outcome_receipt=_valid_outcome(), + source_receipt=_source_receipt(), + evidence_refs=_evidence_refs(), + ) + + assert result["receipt_persisted"] is True + assert result["run_terminal_state"] == "cancelled" + assert result["automation_execution_success"] is False + assert result["runtime_execution_attempted"] is True + assert result["same_run_status_runtime_execution_attempted"] is False + assert result["post_verifier_passed"] is False + assert result["external_status_verifier_passed"] is True + assert result["runtime_closure_verified"] is False + assert result["closure_complete"] is False + assert result["incident_resolution_authorized"] is False + assert result["external_recovery_reconciliation"][ + "recover_execution_success" + ] is False + assert result["external_recovery_reconciliation"][ + "telegram_authorized" + ] is False + sql = "\n".join(db.statements).lower() + assert "incident_records" not in sql + assert "alert_operation_logs" not in sql + assert "playbook" not in sql + assert "telegram" not in sql + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("path", "value"), + [ + ("runtimeWritePerformed", True), + ("outcome.verifierPassed", False), + ("outcome.state", "degraded"), + ], +) +async def test_external_no_write_terminal_rejects_invalid_contract_before_db( + monkeypatch, + path: str, + value: object, +) -> None: + outcome = deepcopy(_valid_outcome()) + _mutate_path(outcome, path, value) + monkeypatch.setattr( + ledger_module, + "get_db_context", + lambda _project_id: pytest.fail("invalid contract must not touch DB"), + ) + + result = await PostgresAgent99DispatchLedger().record_external_no_write_reconciliation( + identity=IDENTITY, + outcome_receipt=outcome, + source_receipt=_source_receipt(), + evidence_refs=_evidence_refs(), + ) + + assert result["status"] == "external_recovery_contract_invalid_fail_closed" + assert result["receipt_persisted"] is False + + +@pytest.mark.asyncio +@pytest.mark.parametrize("malformed", ["95", float("nan"), float("inf"), -float("inf")]) +async def test_external_no_write_terminal_rejects_malformed_source_counts( + monkeypatch, + malformed: object, +) -> None: + source = deepcopy(_source_receipt()) + values = source["values"] + assert isinstance(values, dict) + values["pass_gates"] = malformed + monkeypatch.setattr( + ledger_module, + "get_db_context", + lambda _project_id: pytest.fail("malformed source must not touch DB"), + ) + + result = await PostgresAgent99DispatchLedger().record_external_no_write_reconciliation( + identity=IDENTITY, + outcome_receipt=_valid_outcome(), + source_receipt=source, + evidence_refs=_evidence_refs(), + ) + + assert result["status"] == "external_recovery_contract_invalid_fail_closed" + assert result["receipt_persisted"] is False + + +def test_windows_relay_and_control_plane_enforce_same_run_no_write_contract() -> None: + root = Path(__file__).resolve().parents[3] + relay = (root / "agent99-sre-alert-relay.ps1").read_text(encoding="utf-8") + control = (root / "agent99-control-plane.ps1").read_text(encoding="utf-8") + bootstrap = (root / "agent99-bootstrap.ps1").read_text(encoding="utf-8") + reconcile_source = ( + root / "apps/api/src/services/agent99_same_run_reconcile.py" + ).read_text(encoding="utf-8") + + assert ( + 'AGENT99_COLD_START_RUN_ID = "bf30ca01-1080-5725-b2d1-3e1534dfc811"' + in reconcile_source + ) + + assert "agent99_same_run_status_reconcile_v1" in relay + assert "function Test-AgentSameRunDispatchIdentity" in relay + assert 'route_id" "") -ne "agent99:host_recovery:Recover"' in relay + assert 'incident_id" "") -ne "INC-20260711-11C751"' in relay + assert 'runId -ne "bf30ca01-1080-5725-b2d1-3e1534dfc811"' in relay + assert 'execution_generation" "") -ne "1"' in relay + assert 'project_id" "") -ne "awoooi"' in relay + assert 'work_item_id" "") -ne "agent99-dispatch:awoooi:' in relay + assert 'trace_id" "") -notmatch "^00-[0-9a-f]{32}-[0-9a-f]{16}-01$"' in relay + assert "function Get-AgentExistingRecoverIdentity" in relay + assert "existing_dispatch_identity_not_found" in relay + assert "existing_dispatch_identity_mismatch" in relay + assert "function Test-AgentSameRunOutcomeEligible" in relay + assert 'sameRunContractFieldsComplete = $sameRunContractFieldsComplete' in relay + assert 'sameRunContractFieldsComplete" $false' in relay + assert "existing_same_run_status_outcome_invalid_fail_closed" in relay + assert "[IO.FileMode]::CreateNew" in relay + assert "agent99_same_run_single_flight_lock_v1" in relay + assert "relay_auth_not_configured" in relay + assert 'mode = "Status"' in relay + assert 'controlledApply = $false' in relay + assert 'reconcileOnly = $true' in relay + assert 'suppressTelegram = $true' in relay + assert 'sourceEventResolutionPolicy = "external_source_receipt_required"' in relay + assert '"-ReconcileQueueOnly", "-ReconcileQueueId", $queueId' in relay + assert '"-SuppressAlerts", "-SuppressReason"' in relay + + assert "[switch]$ReconcileOnly" in control + assert '[string]$ReconcileEvidenceId = ""' in control + assert "[switch]$ReconcileQueueOnly" in control + assert '[string]$ReconcileQueueId = ""' in control + assert 'param([string]$OnlyCommandId = "")' in control + assert 'Get-Item -LiteralPath $onlyPath' in control + assert 'reason = "same_run_reconcile_exact_queue_contract_required"' in control + assert "$canonicalDigestMatches" in control + assert "$sameRunReconcileContractExact" in control + assert '$incidentId -eq "INC-20260711-11C751"' in control + assert ( + '$automationRunId -eq "bf30ca01-1080-5725-b2d1-3e1534dfc811"' + in control + ) + assert '$dispatchRouteId -eq "agent99:host_recovery:Recover"' in control + assert '$executionGeneration -eq "1"' in control + assert '$lockOwner -eq $automationRunId' in control + assert '$projectId -eq "awoooi"' in control + assert '$workItemId -eq "agent99-dispatch:awoooi:' in control + assert '$traceId -match "^00-[0-9a-f]{32}-[0-9a-f]{16}-01$"' in control + assert '$reconcileEvidenceId -eq $expectedReconcileEvidenceId' in control + assert "function Get-AgentEvidenceAtPath" in control + assert 'Get-AgentEvidenceAtPath $exactEvidencePath' in control + assert 'Get-AgentBootState -Persist:(-not $ReconcileOnly)' in control + assert '$Mode -eq "Status" -and\n -not $ReconcileOnly -and' in control + assert '"reconcile_only_no_runtime_write"' in control + assert '$data.runtimeWritePerformed -is [bool]' in control + assert '$data.host112Recovery.runtimeWritePerformed -is [bool]' in control + assert '$data.reconcileEvidenceId -is [string]' in control + assert '"-ReconcileOnly", "-ReconcileEvidenceId", $reconcileEvidenceId' in control + assert 'status = "not_written_same_run_reconcile"' in control + assert 'status = "suppressed_same_run_reconcile"' in control + assert ( + 'status = "suppressed_same_run_reconcile_source_recheck_required"' + in control + ) + assert 'Invoke-AgentControlTick -QueueOnly:$ReconcileQueueOnly' in control + assert "-ReconcileOnly:`$ReconcileOnly" in bootstrap + assert "-ReconcileEvidenceId `$ReconcileEvidenceId" in bootstrap + assert "-ReconcileQueueOnly:`$ReconcileQueueOnly" in bootstrap + + reconcile_job = ( + root + / "apps/api/src/jobs/agent99_controlled_dispatch_reconciler_job.py" + ).read_text(encoding="utf-8") + cold_branch = reconcile_job[ + reconcile_job.index("if is_cold_start_same_run_identity(identity):") : + reconcile_job.index( + 'if receipt.get("post_verifier_passed") is not True:' + ) + ] + assert "record_external_no_write_reconciliation" in cold_branch + assert "_finalize_learning" not in cold_branch + assert "_ensure_telegram_receipt" not in cold_branch + assert "_ensure_playbook_trust" not in cold_branch + assert "_ensure_km_writeback" not in cold_branch + + queue_function = control.index("function Invoke-AgentQueuedCommands") + launch_args = control.index("$args = @(", queue_function) + queue_branch = control[ + control.index("if ($reconcileOnly) {", launch_args) : + control.index("$exitCode = if", launch_args) + ] + assert '"-ReconcileOnly", "-ReconcileEvidenceId"' in queue_branch + assert "Get-AgentEvidenceAtPath $exactEvidencePath" in queue_branch + assert "Send-AgentTelegram" not in queue_branch + assert "Invoke-AgentCompletionCallback" not in queue_branch